upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-07-29 06:10:52 -05:00
Code Issues Packages Projects Releases Wiki Activity

added an option for normalizing input ID images; cleaned up debugging code

Browse Source
This commit is contained in:
bssrdf
2024-02-23 10:42:45 -05:00
parent e32462d2f1
commit 8b89e0ea6b
6 changed files with 20 additions and 474 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
clip.hpp
View File

@@ -958,8 +958,6 @@ struct CLIPVisionModel {
struct ggml_tensor *cls,
struct ggml_tensor *temp,
struct ggml_tensor *positions) {
// input_ids: [N, n_token]
// GGML_ASSERT(input_ids->ne[0] <= position_ids->ne[0]);
const int image_size = x->ne[0];
int batch_size = x->ne[3];
const int num_patches = ((image_size / patch_size) * (image_size / patch_size));
@@ -967,88 +965,32 @@ struct CLIPVisionModel {
int32_t n_layer = num_hidden_layers;
const int d_head = hidden_size / n_head;
// ggml_set_name(x, "id_pixel");
// print_ggml_tensor(x, true, "id_pixel");
ggml_set_name(temp, "temp_input");
int64_t* ne = patch_embeddings->ne;
// struct ggml_tensor *patch_embeddings_f16 = ggml_reshape_3d(ctx0, patch_embeddings, ne[0], ne[1], ne[2]*ne[3]);
// patch_embeddings_f16 = ggml_cast(ctx0, patch_embeddings_f16, GGML_TYPE_F16);
// patch_embeddings_f16 = ggml_reshape_4d(ctx0, patch_embeddings_f16, ne[0], ne[1], ne[2], ne[3]);
struct ggml_tensor *patch_embeddings_f16 = ggml_cast(ctx0, patch_embeddings, GGML_TYPE_F16);
ggml_set_name(patch_embeddings_f16, "patch_embeddings_f16");
// print_ggml_tensor(patch_embeddings_f16, true, "patch_embeddings_f16");
struct ggml_tensor * inp = ggml_conv_2d(ctx0, patch_embeddings_f16, x, patch_size, patch_size, 0, 0, 1, 1);
ggml_set_name(inp, "inp_conv_2d");
// print_ggml_tensor(inp, true, "inp_conv_2d");
inp = ggml_reshape_3d(ctx0, inp, num_patches, hidden_size, batch_size);
ggml_set_name(inp, "inp_reshape_3d");
// print_ggml_tensor(inp, true, "inp_reshape_3d");
// inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));
// print_ggml_tensor(ggml_permute(ctx0, inp, 2, 0, 1, 3), true, "inp_permute");
inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3));
// print_ggml_tensor(inp, true, "inp_cont_perm+cont");
ggml_set_name(inp, "inp_cont");
// ggml_set_name(class_embedding, "class_embedding");
// concat class_embeddings and patch_embeddings
// struct ggml_tensor * embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size);
// struct ggml_tensor * temp = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, 1, batch_size);
// ggml_tensor *class_embedding_rep = ggml_add(ctx0, cast_f32_class, class_embedding);
// ggml_set_name(class_embedding_rep, "add_class_embedding_to_zero");
// print_ggml_tensor(class_embedding, true, "model.class_embedding");
// print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_bef_repeat");
// print_ggml_tensor(temp, true, "temp");
ggml_tensor *class_embedding_rep = ggml_repeat(ctx0, class_embedding, temp);
ggml_set_name(class_embedding_rep, "class_embedding_rep");
// print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep");
// class_embedding_rep = ggml_cast(ctx0, class_embedding_rep, inp->type);
// print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_aft_casting");
struct ggml_tensor *embeddings = ggml_concat(ctx0, class_embedding_rep, inp);
ggml_set_name(embeddings, "embeddings_after_concat");
// print_ggml_tensor(embeddings, true, "embeddings_after_concat");
// print_ggml_tensor(ggml_permute(ctx0, embeddings, 0, 3, 1, 2), true, "embeddings_after_concat_permute");
embeddings = ggml_cont(ctx0, ggml_permute(ctx0, embeddings, 0, 2, 1, 3));
ggml_set_name(embeddings, "embeddings_after_permute");
// embeddings = ggml_acc(ctx0, embeddings, ggml_repeat(ctx0, class_embedding, temp), embeddings->nb[1],
// embeddings->nb[2], embeddings->nb[3], 0);
// struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_positions);
// print_ggml_tensor(embeddings, true, "embeddings_bef_add");
// print_ggml_tensor(ggml_get_rows(ctx0, position_embeddings, positions), true, "position_embeddings_bef_add");
// print_ggml_tensor(ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings), true, "embeddings_add_src1");
// embeddings = ggml_cont(ctx0, ggml_permute(ctx0, embeddings, 0, 1, 3, 2));
embeddings =
// ggml_add(ctx0, embeddings, ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings));
ggml_add(ctx0, embeddings, ggml_get_rows(ctx0, position_embeddings, positions));
ggml_set_name(embeddings, "embeddings_to_transformer");
// print_ggml_tensor(embeddings, true, "embeddings_to_transformer");
// pre-layernorm
embeddings = ggml_nn_layer_norm(ctx0, embeddings, pre_ln_w, pre_ln_w);
ggml_set_name(embeddings, "embeddings_after_pre-layernorm");
// transformer
for (int i = 0; i < num_hidden_layers; i++) {
embeddings = resblocks[i].forward(ctx0, embeddings); // [N, n_token, hidden_size]
}
ggml_set_name(embeddings, "embeddings_after_transformer");
// print_ggml_tensor(embeddings, true, "embeddings_after_transformer");
// get the output of cls token, e.g., 0th index
embeddings = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, embeddings, hidden_size, num_positions * batch_size), cls);
ggml_set_name(embeddings, "embeddings_pooled_after_encoder");
// print_ggml_tensor(embeddings, true, "embeddings_pooled_after_encoder");
// post-layernorm
embeddings = ggml_nn_layer_norm(ctx0, embeddings, post_ln_w, post_ln_b);
ggml_set_name(embeddings, "embeddings_after_post-layernorm");
struct ggml_tensor * cur = embeddings;
@@ -1094,17 +1036,6 @@ struct CLIPVisionModel {
}
}
// if (ggml_backend_is_cpu(backend)) {
// for (int i = 0; i < max_position_embeddings; i++) {
// ggml_set_i32_1d(position_ids, i, i);
// }
// } else {
// std::vector<int> pos_temp;
// for (int i = 0; i < max_position_embeddings; i++) {
// pos_temp.push_back(i);
// }
// ggml_backend_tensor_set(position_ids, pos_temp.data(), 0, ggml_nbytes(position_ids));
// }
}
};
@@ -1326,12 +1257,6 @@ struct FrozenCLIPEmbedderWithCustomWords : public GGMLModule {
clean_input_ids = clean_input_ids_tmp;
}
tokens_acc += clean_index;
// class_tokens_mask = [True if class_token_index <= i < class_token_index+num_id_images else False \
// for i in range(len(clean_input_ids))]
// printf("[");
// for(int i = 0; i < clean_input_ids.size(); ++i)
// printf("%d, ", clean_input_ids[i]);
// printf("]\n");
tokens.insert(tokens.end(), clean_input_ids.begin(), clean_input_ids.end());
weights.insert(weights.end(), clean_input_ids.size(), curr_weight);
}
@@ -1365,11 +1290,11 @@ struct FrozenCLIPEmbedderWithCustomWords : public GGMLModule {
class_token_mask.push_back(false);
}
printf("[");
for (int i = 0; i < tokens.size(); i++) {
printf("%d, ", class_token_mask[i] ? 1 : 0);
}
printf("]\n");
// printf("[");
// for (int i = 0; i < tokens.size(); i++) {
// printf("%d, ", class_token_mask[i] ? 1 : 0);
// }
// printf("]\n");
// for (int i = 0; i < tokens.size(); i++) {
// std::cout << tokens[i] << ":" << weights[i] << ", ";

10
examples/cli/main.cpp
View File

@@ -90,6 +90,7 @@ struct SDParams {
bool verbose = false;
bool vae_tiling = false;
bool control_net_cpu = false;
bool normalize_input = false;
bool vae_on_cpu = false;
bool canny_preprocess = false;
};
@@ -107,12 +108,13 @@ void print_params(SDParams params) {
printf(" embeddings_path: %s\n", params.embeddings_path.c_str());
printf(" stacked_id_embeddings_path: %s\n", params.stacked_id_embeddings_path.c_str());
printf(" input_id_images_path: %s\n", params.input_id_images_path.c_str());
printf(" style ratio: %.2f\n", params.style_ratio);
printf(" style ratio: %.2f\n", params.style_ratio);
printf(" normzalize input image : %s\n", params.normalize_input ? "true" : "false");
printf(" output_path: %s\n", params.output_path.c_str());
printf(" init_img: %s\n", params.input_path.c_str());
printf(" control_image: %s\n", params.control_image_path.c_str());
printf(" controlnet cpu: %s\n", params.control_net_cpu ? "true" : "false");
printf(" vae decoder cpu: %s\n", params.vae_on_cpu ? "true" : "false");
printf(" vae decoder on cpu:%s\n", params.vae_on_cpu ? "true" : "false");
printf(" strength(control): %.2f\n", params.control_strength);
printf(" prompt: %s\n", params.prompt.c_str());
printf(" negative_prompt: %s\n", params.negative_prompt.c_str());
@@ -145,6 +147,7 @@ void print_usage(int argc, const char* argv[]) {
printf(" --embd-dir [EMBEDDING_PATH] path to embeddings.\n");
printf(" --stacked-id-embd-dir [DIR] path to PHOTOMAKER stacked id embeddings.\n");
printf(" --input-id-images-dir [DIR] path to PHOTOMAKER input id images dir.\n");
printf(" --normalize-input normalize PHOTOMAKER input id images\n");
printf(" --upscale-model [ESRGAN_PATH] path to esrgan model. Upscale images after generate, just RealESRGAN_x4plus_anime_6B supported by now.\n");
printf(" --type [TYPE] weight type (f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0)\n");
printf(" If not specified, the default is the type of the weight file.\n");
@@ -367,6 +370,8 @@ void parse_args(int argc, const char** argv, SDParams& params) {
params.vae_tiling = true;
} else if (arg == "--control-net-cpu") {
params.control_net_cpu = true;
} else if (arg == "--normalize-input") {
params.normalize_input = true;
} else if (arg == "--vae-on-cpu") {
params.vae_on_cpu = true; // will slow down latent decoding but necessary for low MEM GPUs
} else if (arg == "--canny") {
@@ -679,6 +684,7 @@ int main(int argc, const char* argv[]) {
control_image,
params.control_strength,
params.style_ratio,
params.normalize_input,
input_id_images);
} else {
sd_image_t input_image = {(uint32_t)params.width,

224
ggml_extend.hpp
View File

@@ -679,18 +679,6 @@ struct GGMLModule {
ggml_allocr_alloc_graph(compute_allocr, gf);
// struct ggml_init_params params = {};
// params.mem_size += 1024*1024*1024; // 100M
// params.mem_size += 2 * ggml_tensor_overhead();
// params.mem_buffer = NULL;
// params.no_alloc = false;
// struct ggml_context* obs_ctx = ggml_init(params);
// if (!obs_ctx) {
// LOG_ERROR("ggml_init() failed");
// return;
// }
if (ggml_backend_is_cpu(backend)) {
ggml_backend_cpu_set_n_threads(backend, n_threads);
}
@@ -700,227 +688,15 @@ struct GGMLModule {
ggml_backend_metal_set_n_cb(backend, n_threads);
}
#endif
struct ggml_tensor * imb = NULL;
#if 0
for (int i = 0; i < gf->n_leafs; i++) {
struct ggml_tensor * t1 = gf->leafs[i];
// if(strcmp(ggml_get_name(t1), "prompt_embeds_input") == 0) {
if(strcmp(ggml_get_name(t1), "id_pixel_values_input") == 0) {
imb = t1;
int64_t stride = imb->ne[0];
int64_t ne1 = imb->ne[1];
int64_t ne2 = imb->ne[2];
int64_t ne3 = imb->ne[3];
float* out_data = new float[ggml_nelements(imb)];
ggml_backend_tensor_get(imb, out_data, 0, ggml_nbytes(imb));
printf("%s tensor: \n", ggml_get_name(t1));
for(int l = 0; l < ne3; ++l){
printf("l = %d: \n", l);
for(int k = 0; k < ne2; ++k){
// float mi = 100.f, mx= -100.f;
printf("k = %d: \n", k);
for(int i = 0; i < ne1; i++){
printf("%d: [", i);
for(int j = 0; j < stride; j++){
float val = out_data[l*ne2*ne1*stride+k*ne1*stride+i*stride+j];
// if(mi > val) mi = val;
// if(mx < val) mx = val;
printf("%f, ", val);
}
printf("]\n");
}
// printf("B. channel, min, max: %d, %f %f \n", k, mi, mx);
}
}
// for(int l = 0; l < ne3; ++l){
// printf("[");
// for(int k = 0; k < 3; ++k){
// for(int i = 0; i < stride; i++){
// for(int j = 0; j < stride; j++){
// float val = out_data[l*3*stride*stride+ k*stride*stride+i*stride+j];
// printf("%f, ", val);
// }
// }
// // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx);
// }
// printf("]\n");
// printf("%s tensor: \n", ggml_get_name(t1));
// for(int i = 0; i < ne1; i++){
// printf("%d: [", i);
// for(int j = 0; j < stride; j++){
// float val = out_data[i*stride+j];
// // if(mi > val) mi = val;
// // if(mx < val) mx = val;
// printf("%f, ", val);
// }
// printf("]\n");
// }
// for(int k = 0; k < 3; ++k){
// float mi = 100.f, mx= -100.f;
// for(int i = 0; i < stride; i++){
// printf("[");
// for(int j = 0; j < stride; j++){
// float val = out_data[k*stride*stride+i*stride+j];
// if(mi > val) mi = val;
// if(mx < val) mx = val;
// printf("%f, ", val);
// }
// printf("]\n");
// }
// printf("B. channel, min, max: %d, %f %f \n", k, mi, mx);
// }
// printf("[");
// for(int i = 0; i < stride; i++){
// printf("%f, ", out_data[i]);
// }
// printf("]\n");
delete out_data;
}
}
#endif
#if 0
// struct ggml_tensor * imb = NULL;
for (int i = 0; i < gf->n_nodes; i++) {
struct ggml_cgraph g1v = ggml_graph_view(gf, i, i + 1);
ggml_backend_graph_compute(backend, &g1v);
struct ggml_tensor * t1 = gf->nodes[i];
// if(strcmp(ggml_get_name(t1), "embeddings_after_add") == 0) {
// if(strcmp(ggml_get_name(t1), "stacked_id_embeds_fuse_fn") == 0) {
// if(strcmp(ggml_get_name(t1), "embeddings_after_transformer") == 0) {
if(strcmp(ggml_get_name(t1), "embeddings_pooled_after_encoder") == 0) {
// if(strcmp(ggml_get_name(t1), "shared_id_embeds_from_vision") == 0) {
// if(strcmp(ggml_get_name(t1), "embeddings_to_transformer") == 0) {
// if(strcmp(ggml_get_name(t1), "inp_reshape_3d") == 0) {
// if(strcmp(ggml_get_name(t1), "inp_conv_2d") == 0) {
// if(strcmp(ggml_get_name(t1), "inp_conv_2d") == 0) {
// if(strcmp(ggml_get_name(t1), "patch_embeddings_f16") == 0) {
// if(strcmp(ggml_get_name(t1), "im2col_as_input") == 0) {
// if(strcmp(ggml_get_name(t1), "image_token_embeds") == 0 ||
// strcmp(ggml_get_name(t1), "valid_id_embeds") == 0) {
// if(strcmp(ggml_get_name(t1), "id_embeds_proj1") == 0 ||
// strcmp(ggml_get_name(t1), "id_embeds_proj2") == 0) {
imb = t1;
int64_t stride = imb->ne[0];
int64_t ne1 = imb->ne[1];
int64_t ne2 = imb->ne[2];
int64_t ne3 = imb->ne[3];
float* out_data = new float[ggml_nelements(imb)];
// ggml_fp16_t * out_data = new ggml_fp16_t[ggml_nelements(imb)];
ggml_backend_tensor_get(imb, out_data, 0, ggml_nbytes(imb));
printf("%s tensor: \n", ggml_get_name(t1));
for(int l = 0; l < ne3; ++l){
printf("l = %d: \n", l);
for(int k = 0; k < ne2; ++k){
// float mi = 1.e16f, mx= -1e16f;
printf("k = %d: \n", k);
for(int i = 0; i < ne1; i++){
printf("%d: [", i);
// float mi = 1.e16f, mx= -1e16f;
for(int j = 0; j < stride; j++){
float val = out_data[l*ne2*ne1*stride+k*ne1*stride+i*stride+j];
// float val = ggml_fp16_to_fp32(out_data[l*ne2*ne1*stride+k*ne1*stride+i*stride+j]);
// if(mi > val) mi = val;
// if(mx < val) mx = val;
printf("%f, ", val);
}
printf("]\n");
// printf("i = %d: min, max: %f %f \n", i, mi, mx);
}
// printf("k = %d: min, max: %f %f \n", k, mi, mx);
}
}
// for(int k = 0; k < ne2; ++k){
// // float mi = 100.f, mx= -100.f;
// printf("%d: \n", k);
// for(int i = 0; i < ne1; i++){
// printf("%d: [", i);
// for(int j = 0; j < stride; j++){
// float val = out_data[k*ne1*stride+i*stride+j];
// // if(mi > val) mi = val;
// // if(mx < val) mx = val;
// printf("%f, ", val);
// }
// printf("]\n");
// }
// // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx);
// }
// for(int i = 0; i < ne1; i++){
// printf("%d: [", i);
// for(int j = 0; j < stride; j++){
// float val = out_data[i*stride+j];
// // if(mi > val) mi = val;
// // if(mx < val) mx = val;
// printf("%f, ", val);
// }
// printf("]\n");
// }
// printf("B. channel, min, max: %d, %f %f \n", k, mi, mx);
// printf("[");
// for(int i = 0; i < stride; i++){
// printf("%f, ", out_data[i]);
// }
// printf("]\n");
delete out_data;
}
}
#endif
ggml_backend_graph_compute(backend, gf);
#ifdef GGML_PERF
ggml_graph_print(gf);
#endif
// if (output != NULL)
// print_ggml_tensor(output, true, "output_before_get");
if (output != NULL) {
ggml_backend_tensor_get_and_sync(backend, gf->nodes[gf->n_nodes - 1], output->data, 0, ggml_nbytes(output));
}
// struct ggml_tensor * imb = get_tensor_from_graph(gf, "id_pixel");
// struct ggml_tensor * imb = get_tensor_from_graph(gf, "id_pixel_values_input");
// if(imb != NULL){
// int64_t stride = imb->ne[0];
// float* out_data = new float[ggml_nelements(imb)];
// ggml_backend_tensor_get(imb, out_data, 0, ggml_nbytes(imb));
// for(int k = 0; k < 3; ++k){
// float mi = 100.f, mx= -100.f;
// for(int i = 0; i < stride; i++){
// printf("[");
// for(int j = 0; j < stride; j++){
// float val = out_data[k*stride*stride+i*stride+j];
// if(mi > val) mi = val;
// if(mx < val) mx = val;
// printf("%f, ", val);
// }
// printf("]\n");
// }
// printf("B. channel, min, max: %d, %f %f \n", k, mi, mx);
// }
// // printf("[");
// // for(int i = 0; i < stride; i++){
// // printf("%f, ", out_data[i]);
// // }
// // printf("]\n");
// delete out_data;
// }
// if (output != NULL)
// print_ggml_tensor(output, true, "output_get");
}
void free_compute_buffer() {

107
pmid.hpp
View File

@@ -148,14 +148,6 @@ struct FuseModule{
struct ggml_tensor* fuse_fn(struct ggml_context* ctx,
struct ggml_tensor* prompt_embeds,
struct ggml_tensor* id_embeds) {
// x: [N, channels, h, w]
// stacked_id_embeds = torch.cat([prompt_embeds, id_embeds], dim=-1)
// stacked_id_embeds = self.mlp1(stacked_id_embeds) + prompt_embeds
// stacked_id_embeds = self.mlp2(stacked_id_embeds)
// stacked_id_embeds = self.layer_norm(stacked_id_embeds)
// return stacked_id_embeds
// in_layers
auto prompt_embeds0 = ggml_cont(ctx, ggml_permute(ctx, prompt_embeds, 2, 0, 1, 3));
auto id_embeds0 = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 2, 0, 1, 3));
@@ -181,38 +173,11 @@ struct FuseModule{
struct ggml_tensor* right) {
// x: [N, channels, h, w]
// in_layers
// # id_embeds shape: [b, max_num_inputs, 1, 2048]
// id_embeds = id_embeds.to(prompt_embeds.dtype)
// num_inputs = class_tokens_mask.sum().unsqueeze(0) # TODO: check for training case
// batch_size, max_num_inputs = id_embeds.shape[:2]
// # seq_length: 77
// seq_length = prompt_embeds.shape[1]
// # flat_id_embeds shape: [b*max_num_inputs, 1, 2048]
// flat_id_embeds = id_embeds.view(
// -1, id_embeds.shape[-2], id_embeds.shape[-1]
// )
// # valid_id_mask [b*max_num_inputs]
// valid_id_mask = (
// torch.arange(max_num_inputs, device=flat_id_embeds.device)[None, :]
// < num_inputs[:, None]
// )
// valid_id_embeds = flat_id_embeds[valid_id_mask.flatten()]
// prompt_embeds = prompt_embeds.view(-1, prompt_embeds.shape[-1])
// class_tokens_mask = class_tokens_mask.view(-1)
struct ggml_tensor * valid_id_embeds = id_embeds;
// # slice out the image token embeddings
struct ggml_tensor * image_token_embeds = ggml_get_rows(ctx, prompt_embeds, class_tokens_mask_pos);
// print_ggml_tensor(image_token_embeds, true, "image_token_embeds");
ggml_set_name(image_token_embeds, "image_token_embeds");
// print_ggml_tensor(valid_id_embeds, true, "valid_id_embeds");
ggml_set_name(valid_id_embeds, "valid_id_embeds");
struct ggml_tensor *stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds);
// print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_before_concat");
ggml_set_name(stacked_id_embeds, "stacked_id_embeds_fuse_fn");
stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3));
if(left && right){
@@ -223,29 +188,11 @@ struct FuseModule{
}else if(right){
stacked_id_embeds = ggml_concat(ctx, stacked_id_embeds, right);
}
// print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_concat");
stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3));
ggml_set_name(stacked_id_embeds, "stacked_id_embeds_add_zero");
// print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_permute_2");
// print_ggml_tensor(class_tokens_mask, true, "class_tokens_mask");
// print_ggml_tensor(prompt_embeds, true, "prompt_embeds");
// assert class_tokens_mask.sum() == stacked_id_embeds.shape[0], f"{class_tokens_mask.sum()} != {stacked_id_embeds.shape[0]}"
// prompt_embeds.masked_scatter_(class_tokens_mask[:, None], stacked_id_embeds.to(prompt_embeds.dtype))
// struct ggml_tensor *prompt_embeds_perm = ggml_cont(ctx, ggml_permute(ctx, prompt_embeds, 1, 0, 2, 3));
// struct ggml_tensor *prompt_embeds_perm = ggml_cont(ctx, ggml_transpose(ctx, prompt_embeds));
// print_ggml_tensor(prompt_embeds, true, "prompt_embeds_perm");
// class_tokens_mask = ggml_repeat(ctx, class_tokens_mask, prompt_embeds_perm);
class_tokens_mask = ggml_cont(ctx, ggml_transpose(ctx, class_tokens_mask));
class_tokens_mask = ggml_repeat(ctx, class_tokens_mask, prompt_embeds);
// print_ggml_tensor(class_tokens_mask, true, "class_tokens_mask_repeat");
// class_tokens_mask = ggml_cont(ctx, ggml_transpose(ctx, class_tokens_mask));
// print_ggml_tensor(class_tokens_mask, true, "class_tokens_mask_transpose");
prompt_embeds = ggml_mul(ctx, prompt_embeds, class_tokens_mask);
// print_ggml_tensor(prompt_embeds, true, "prompt_embeds_after_mul");
ggml_set_name(prompt_embeds, "prompt_embeds_mul_cls");
struct ggml_tensor * updated_prompt_embeds = ggml_add(ctx, prompt_embeds, stacked_id_embeds);
// print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds");
// updated_prompt_embeds = prompt_embeds.view(batch_size, seq_length, -1)
return updated_prompt_embeds;
}
@@ -265,18 +212,12 @@ struct PhotoMakerIDEncoder : public GGMLModule {
fuse_module(2048),
style_strength(sty){
vision_model = CLIPVisionModel();
// fuse_module = FuseModule(2048);
// wtype = GGML_TYPE_F32;
}
// void init_params(ggml_context* ctx, ggml_backend_t backend, ggml_type wtype, ggml_allocr* alloc) {
void init_params(ggml_type wtype) {
// LOG_INFO(" PMID wtype: %s", ggml_type_name(wtype));
ggml_allocr* alloc = ggml_allocr_new_from_buffer(params_buffer);
vision_model.init_params(params_ctx, backend, wtype, alloc);
fuse_module.init_params(params_ctx, wtype, alloc);
// visual_projection_2 = ggml_new_tensor_2d(params_ctx, wtype, 1280, 1024); // python [1024, 1280]
visual_projection_2 = ggml_new_tensor_2d(params_ctx, wtype, 1024, 1280); // python [1024, 1280]
ggml_allocr_alloc(alloc, visual_projection_2);
ggml_allocr_free(alloc);
@@ -291,7 +232,6 @@ struct PhotoMakerIDEncoder : public GGMLModule {
size_t calculate_mem_size() {
wtype = GGML_TYPE_F32;
// LOG_INFO(" PMID wtype: %s", ggml_type_name(wtype));
size_t mem_size = vision_model.calculate_mem_size(wtype);
mem_size += fuse_module.calculate_mem_size(wtype);
@@ -320,67 +260,27 @@ struct PhotoMakerIDEncoder : public GGMLModule {
struct ggml_tensor* right) {
// x: [N, channels, h, w]
// in_layers
ggml_set_name(id_pixel_values, "id_pixel_values_input");
ggml_set_name(prompt_embeds, "prompt_embeds_input");
ggml_set_name(class_tokens_mask, "class_tokens_mask_input");
ggml_set_name(class_tokens_mask_pos, "class_tokens_mask_pos_input");
ggml_set_name(cls, "cls_input");
ggml_set_name(class_embedding_temp, "class_embedding_temp_input");
ggml_set_name(positions, "positions_input");
ggml_set_name(left, "left_input");
ggml_set_name(right, "right_input");
// print_ggml_tensor(prompt_embeds, true, "prompt_embeds");
// print_ggml_tensor(class_embedding_temp, true, "class_embedding_temp");
struct ggml_tensor *shared_id_embeds = vision_model.forward(ctx,
id_pixel_values,
cls,
class_embedding_temp,
positions
); // [batch_size, seq_length, hidden_size]
// print_ggml_tensor(shared_id_embeds, true, "shared_id_embeds_from_vision");
ggml_set_name(shared_id_embeds, "shared_id_embeds_from_vision");
// if(class_tokens_mask->backend == GGML_BACKEND_GPU){
// int *ctm = (int *)malloc(class_tokens_mask->ne[0]);
// ggml_backend_tensor_get(class_tokens_mask, ctm, 0, ggml_nbytes(class_tokens_mask));
// printf("class_tokens_mask[");
// for(int i = 0; i < class_tokens_mask->ne[0]; i++)
// printf("%d, ", ctm[i]);
// printf("]\n");
// free(ctm);
// }
struct ggml_tensor *id_embeds = vision_model.visual_project(ctx, shared_id_embeds); // [batch_size, seq_length, proj_dim(768)]
struct ggml_tensor *id_embeds_2 = ggml_mul_mat(ctx, visual_projection_2, shared_id_embeds); // [batch_size, seq_length, 1280]
// id_embeds = id_embeds.view(b, num_inputs, 1, -1)
// id_embeds_2 = id_embeds_2.view(b, num_inputs, 1, -1)
// id_embeds = torch.cat((id_embeds, id_embeds_2), dim=-1)
ggml_set_name(id_embeds, "id_embeds_proj1");
ggml_set_name(id_embeds_2, "id_embeds_proj2");
id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 2, 0, 1, 3));
id_embeds_2 = ggml_cont(ctx, ggml_permute(ctx, id_embeds_2, 2, 0, 1, 3));
// print_ggml_tensor(id_embeds, true, "id_embeds_after_perm");
// print_ggml_tensor(id_embeds_2, true, "id_embeds_2_after_perm");
id_embeds = ggml_concat(ctx, id_embeds, id_embeds_2); // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right
// print_ggml_tensor(id_embeds, true, "id_embeds_after_cat");
id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 1, 2, 0, 3));
// print_ggml_tensor(id_embeds, true, "id_embeds_after_cat+perm");
struct ggml_tensor * updated_prompt_embeds = fuse_module.forward(ctx,
prompt_embeds, id_embeds,
class_tokens_mask,
class_tokens_mask_pos,
left, right);
// print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds_returned");
return updated_prompt_embeds;
@@ -429,7 +329,6 @@ struct PhotoMakerIDEncoder : public GGMLModule {
ctm.push_back(0.f); // here use 0.f instead of 1.f to make a scale mask
ctmf16.push_back(ggml_fp32_to_fp16(0.f)); // here use 0.f instead of 1.f to make a scale mask
ctmpos.push_back(i);
// printf("push %d, \n", i);
}else{
ctm.push_back(1.f); // here use 1.f instead of 0.f to make a scale mask
ctmf16.push_back(ggml_fp32_to_fp16(1.f)); // here use 0.f instead of 1.f to make a scale mask
@@ -481,10 +380,6 @@ struct PhotoMakerIDEncoder : public GGMLModule {
ggml_backend_tensor_set(cls, cls_h.data(), 0, ggml_nbytes(cls));
ggml_backend_tensor_set(positions, pos.data(), 0, ggml_nbytes(positions));
ggml_backend_tensor_set(class_tokens_mask_pos, ctmpos.data(), 0, ggml_nbytes(class_tokens_mask_pos));
// std::vector<float> zeros;
// for (int i = 0; i < hidden_size; i++) {
// zeros.push_back(0.f);
// }
if(left){
if(type == GGML_TYPE_F16){
std::vector<ggml_fp16_t> zeros(ggml_nelements(left), ggml_fp32_to_fp16(0.f));
@@ -514,9 +409,7 @@ struct PhotoMakerIDEncoder : public GGMLModule {
positions,
left, right
);
// print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds_returned_forward");
ggml_build_forward_expand(gf, updated_prompt_embeds);
// ggml_graph_dump_dot(gf, NULL, "id_encoder.dot");
ggml_free(ctx0);
return gf;

67
stable-diffusion.cpp
View File

@@ -1472,6 +1472,7 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx,
const sd_image_t* control_cond,
float control_strength,
float style_ratio,
bool normalize_input,
std::vector<sd_image_t*> &input_id_images) {
LOG_DEBUG("txt2img %dx%d", width, height);
if (sd_ctx == NULL) {
@@ -1545,68 +1546,12 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx,
float std[] = {0.26862954, 0.26130258, 0.27577711};
for(int i = 0; i < num_input_images; i++) {
sd_image_t* init_image = input_id_images[i];
// sd_mul_images_to_tensor(init_image->data, init_img, i, mean, std);
sd_mul_images_to_tensor(init_image->data, init_img, i, NULL, NULL);
if(normalize_input)
sd_mul_images_to_tensor(init_image->data, init_img, i, mean, std);
else
sd_mul_images_to_tensor(init_image->data, init_img, i, NULL, NULL);
}
// ModelLoader model_loader;
// std::string img_path("examples/scarlett.safetensors");
// if (!model_loader.init_from_file(img_path, "scarlett.")){
// LOG_ERROR("init model loader from file failed: '%s'", img_path.c_str());
// return NULL;
// }
// ggml_backend_t cpu_backend = ggml_backend_cpu_init();
// std::map<std::string, struct ggml_tensor*> tensors_need_to_load;
// std::set<std::string> ignore_tensors;
// tensors_need_to_load["scarlett.img"] = init_img;
// bool success = model_loader.load_tensors(tensors_need_to_load, cpu_backend, ignore_tensors);
// if (!success) {
// LOG_ERROR("load tensors from model loader failed");
// ggml_free(work_ctx);
// return NULL;
// }
// sd_image_t* cropped_images = (sd_image_t*)calloc(num_input_images, sizeof(sd_image_t));
// if (cropped_images == NULL) {
// ggml_free(work_ctx);
// return NULL;
// }
// for (size_t i = 0; i < num_input_images; i++) {
// cropped_images[i].width = 224;
// cropped_images[i].height = 224;
// cropped_images[i].channel = 3;
// cropped_images[i].data = sd_tensor_to_mul_image(init_img, i);
// }
// for (int i = 0; i < num_input_images; i++) {
// if (cropped_images[i].data == NULL) {
// continue;
// }
// std::string final_image_path = "cropped_" + std::to_string(i + 1) + ".png";
// stbi_write_png(final_image_path.c_str(), cropped_images[i].width, cropped_images[i].height,
// cropped_images[i].channel,
// cropped_images[i].data, 0, "");
// printf("save result image to '%s'\n", final_image_path.c_str());
// free(cropped_images[i].data);
// cropped_images[i].data = NULL;
// }
// float* out_data = (float *)(init_img->data);
// int64_t stride = init_img->ne[0];
// for(int k = 0; k < 3; ++k){
// float mi = 100.f, mx= -100.f;
// for(int i = 0; i < stride; i++){
// printf("[");
// for(int j = 0; j < stride; j++){
// float val = out_data[k*stride*stride+i*stride+j];
// if(mi > val) mi = val;
// if(mx < val) mx = val;
// printf("%f, ", val);
// }
// printf("]\n");
// }
// printf(" channel, min, max: %d, %f %f \n", k, mi, mx);
// }
auto cond_tup = sd_ctx->sd->get_learned_condition_with_trigger(work_ctx, prompt,
clip_skip, width, height, num_input_images );
prompts_embeds = std::get<0>(cond_tup);
@@ -1620,7 +1565,7 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx,
}
// Encode input prompt without the trigger word for delayed conditioning
prompt_text_only = sd_ctx->sd->remove_trigger_from_prompt(work_ctx, prompt);
printf("%s || %s \n", prompt.c_str(), prompt_text_only.c_str());
// printf("%s || %s \n", prompt.c_str(), prompt_text_only.c_str());
prompt = prompt_text_only; //
if(sample_steps < 50){
LOG_INFO("sampling steps increases from %d to 50 for PHOTOMAKER", sample_steps);

1
stable-diffusion.h
View File

@@ -135,6 +135,7 @@ SD_API sd_image_t* txt2img(sd_ctx_t* sd_ctx,
const sd_image_t* control_cond,
float control_strength,
float style_strength,
bool normalize_input,
std::vector<sd_image_t*> &input_id_images);
SD_API sd_image_t* img2img(sd_ctx_t* sd_ctx,