From d38b85d87e691f030e8884467b2783bd4831f90f Mon Sep 17 00:00:00 2001 From: bssrdf Date: Sun, 18 Feb 2024 22:18:53 -0500 Subject: [PATCH] fixed a bug in ggml's conv_2d, and now > 1 input images working --- clip.hpp | 12 ++- ggml_extend.hpp | 169 +++++++++++++++++++++++++++++++++---------- pmid.hpp | 22 ++++-- stable-diffusion.cpp | 16 ++-- 4 files changed, 164 insertions(+), 55 deletions(-) diff --git a/clip.hpp b/clip.hpp index 830a9e96..6d9c9890 100644 --- a/clip.hpp +++ b/clip.hpp @@ -975,6 +975,10 @@ struct CLIPVisionModel { // patch_embeddings_f16 = ggml_cast(ctx0, patch_embeddings_f16, GGML_TYPE_F16); // patch_embeddings_f16 = ggml_reshape_4d(ctx0, patch_embeddings_f16, ne[0], ne[1], ne[2], ne[3]); struct ggml_tensor *patch_embeddings_f16 = ggml_cast(ctx0, patch_embeddings, GGML_TYPE_F16); + ggml_set_name(patch_embeddings_f16, "patch_embeddings_f16"); + print_ggml_tensor(patch_embeddings_f16, true, "patch_embeddings_f16"); + + struct ggml_tensor * inp = ggml_conv_2d(ctx0, patch_embeddings_f16, x, patch_size, patch_size, 0, 0, 1, 1); ggml_set_name(inp, "inp_conv_2d"); @@ -1023,7 +1027,7 @@ struct CLIPVisionModel { embeddings = // ggml_add(ctx0, embeddings, ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings)); ggml_add(ctx0, embeddings, ggml_get_rows(ctx0, position_embeddings, positions)); - ggml_set_name(embeddings, "embeddings_after_add"); + ggml_set_name(embeddings, "embeddings_to_transformer"); print_ggml_tensor(embeddings, true, "embeddings_to_transformer"); // pre-layernorm @@ -1035,12 +1039,12 @@ struct CLIPVisionModel { embeddings = resblocks[i].forward(ctx0, embeddings); // [N, n_token, hidden_size] } ggml_set_name(embeddings, "embeddings_after_transformer"); - // print_ggml_tensor(embeddings, true, "embeddings_after_encoder"); + print_ggml_tensor(embeddings, true, "embeddings_after_transformer"); // get the output of cls token, e.g., 0th index embeddings = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, embeddings, hidden_size, num_positions * batch_size), cls); - ggml_set_name(embeddings, "embeddings_after_cls_token"); - // print_ggml_tensor(embeddings, true, "embeddings_pooled_after_encoder"); + ggml_set_name(embeddings, "embeddings_pooled_after_encoder"); + print_ggml_tensor(embeddings, true, "embeddings_pooled_after_encoder"); // post-layernorm embeddings = ggml_nn_layer_norm(ctx0, embeddings, post_ln_w, post_ln_b); diff --git a/ggml_extend.hpp b/ggml_extend.hpp index 17b30cef..a0fcedef 100644 --- a/ggml_extend.hpp +++ b/ggml_extend.hpp @@ -701,30 +701,78 @@ struct GGMLModule { } #endif -#if 0 struct ggml_tensor * imb = NULL; +#if 0 + for (int i = 0; i < gf->n_leafs; i++) { struct ggml_tensor * t1 = gf->leafs[i]; - if(strcmp(ggml_get_name(t1), "vision.patch_embeddings") == 0) { + if(strcmp(ggml_get_name(t1), "class_tokens_mask_pos_input") == 0) { imb = t1; int64_t stride = imb->ne[0]; - int64_t ne3 = imb->ne[3]; - float* out_data = new float[ggml_nelements(imb)]; + int* out_data = new int[ggml_nelements(imb)]; ggml_backend_tensor_get(imb, out_data, 0, ggml_nbytes(imb)); - for(int l = 0; l < ne3; ++l){ printf("["); - for(int k = 0; k < 3; ++k){ - for(int i = 0; i < stride; i++){ - for(int j = 0; j < stride; j++){ - float val = out_data[l*3*stride*stride+ k*stride*stride+i*stride+j]; - printf("%f, ", val); - } - } - // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx); + for(int i = 0; i < stride; i++){ + printf("%d, ", out_data[i]); } printf("]\n"); + delete out_data; + } + + // if(strcmp(ggml_get_name(t1), "prompt_embeds_input") == 0) { + if(strcmp(ggml_get_name(t1), "id_pixel_values_input") == 0) { + + imb = t1; + int64_t stride = imb->ne[0]; + int64_t ne1 = imb->ne[1]; + int64_t ne2 = imb->ne[2]; + int64_t ne3 = imb->ne[3]; + float* out_data = new float[ggml_nelements(imb)]; + ggml_backend_tensor_get(imb, out_data, 0, ggml_nbytes(imb)); + printf("%s tensor: \n", ggml_get_name(t1)); + for(int l = 0; l < ne3; ++l){ + printf("l = %d: \n", l); + for(int k = 0; k < ne2; ++k){ + // float mi = 100.f, mx= -100.f; + printf("k = %d: \n", k); + for(int i = 0; i < ne1; i++){ + printf("%d: [", i); + for(int j = 0; j < stride; j++){ + float val = out_data[l*ne2*ne1*stride+k*ne1*stride+i*stride+j]; + // if(mi > val) mi = val; + // if(mx < val) mx = val; + printf("%f, ", val); + } + printf("]\n"); + } + // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx); + } } + // for(int l = 0; l < ne3; ++l){ + // printf("["); + // for(int k = 0; k < 3; ++k){ + // for(int i = 0; i < stride; i++){ + // for(int j = 0; j < stride; j++){ + // float val = out_data[l*3*stride*stride+ k*stride*stride+i*stride+j]; + // printf("%f, ", val); + // } + // } + // // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx); + // } + // printf("]\n"); + // printf("%s tensor: \n", ggml_get_name(t1)); + // for(int i = 0; i < ne1; i++){ + // printf("%d: [", i); + // for(int j = 0; j < stride; j++){ + // float val = out_data[i*stride+j]; + // // if(mi > val) mi = val; + // // if(mx < val) mx = val; + // printf("%f, ", val); + // } + // printf("]\n"); + // } + // for(int k = 0; k < 3; ++k){ // float mi = 100.f, mx= -100.f; // for(int i = 0; i < stride; i++){ @@ -748,22 +796,67 @@ struct GGMLModule { delete out_data; } } -// #if 0 +//#if 0 +//#endif + // struct ggml_tensor * imb = NULL; for (int i = 0; i < gf->n_nodes; i++) { + struct ggml_cgraph g1v = ggml_graph_view(gf, i, i + 1); + ggml_backend_graph_compute(backend, &g1v); struct ggml_tensor * t1 = gf->nodes[i]; // if(strcmp(ggml_get_name(t1), "embeddings_after_add") == 0) { - if(strcmp(ggml_get_name(t1), "pooled_output") == 0) { + // if(strcmp(ggml_get_name(t1), "stacked_id_embeds_fuse_fn") == 0) { + // if(strcmp(ggml_get_name(t1), "embeddings_after_transformer") == 0) { + if(strcmp(ggml_get_name(t1), "embeddings_pooled_after_encoder") == 0) { + // if(strcmp(ggml_get_name(t1), "shared_id_embeds_from_vision") == 0) { + // if(strcmp(ggml_get_name(t1), "embeddings_to_transformer") == 0) { + // if(strcmp(ggml_get_name(t1), "inp_reshape_3d") == 0) { + // if(strcmp(ggml_get_name(t1), "inp_conv_2d") == 0) { + // if(strcmp(ggml_get_name(t1), "inp_conv_2d") == 0) { + // if(strcmp(ggml_get_name(t1), "patch_embeddings_f16") == 0) { + // if(strcmp(ggml_get_name(t1), "im2col_as_input") == 0) { + // if(strcmp(ggml_get_name(t1), "image_token_embeds") == 0 || + // strcmp(ggml_get_name(t1), "valid_id_embeds") == 0) { + // if(strcmp(ggml_get_name(t1), "id_embeds_proj1") == 0 || + // strcmp(ggml_get_name(t1), "id_embeds_proj2") == 0) { imb = t1; int64_t stride = imb->ne[0]; + int64_t ne1 = imb->ne[1]; + int64_t ne2 = imb->ne[2]; + int64_t ne3 = imb->ne[3]; float* out_data = new float[ggml_nelements(imb)]; + // ggml_fp16_t * out_data = new ggml_fp16_t[ggml_nelements(imb)]; ggml_backend_tensor_get(imb, out_data, 0, ggml_nbytes(imb)); - - // for(int k = 0; k < 3; ++k){ - // float mi = 100.f, mx= -100.f; - // for(int i = 0; i < stride; i++){ - // printf("["); + printf("%s tensor: \n", ggml_get_name(t1)); + for(int l = 0; l < ne3; ++l){ + printf("l = %d: \n", l); + for(int k = 0; k < ne2; ++k){ + // float mi = 1.e16f, mx= -1e16f; + printf("k = %d: \n", k); + for(int i = 0; i < ne1; i++){ + printf("%d: [", i); + // float mi = 1.e16f, mx= -1e16f; + for(int j = 0; j < stride; j++){ + float val = out_data[l*ne2*ne1*stride+k*ne1*stride+i*stride+j]; + // float val = ggml_fp16_to_fp32(out_data[l*ne2*ne1*stride+k*ne1*stride+i*stride+j]); + // if(mi > val) mi = val; + // if(mx < val) mx = val; + printf("%f, ", val); + } + printf("]\n"); + // printf("i = %d: min, max: %f %f \n", i, mi, mx); + } + // printf("k = %d: min, max: %f %f \n", k, mi, mx); + } + } + + + // for(int k = 0; k < ne2; ++k){ + // // float mi = 100.f, mx= -100.f; + // printf("%d: \n", k); + // for(int i = 0; i < ne1; i++){ + // printf("%d: [", i); // for(int j = 0; j < stride; j++){ - // float val = out_data[k*stride*stride+i*stride+j]; + // float val = out_data[k*ne1*stride+i*stride+j]; // // if(mi > val) mi = val; // // if(mx < val) mx = val; // printf("%f, ", val); @@ -772,27 +865,27 @@ struct GGMLModule { // } // // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx); // } - // for(int i = 0; i < 257; i++){ - // printf("["); - // for(int j = 0; j < stride; j++){ - // float val = out_data[i*stride+j]; - // // if(mi > val) mi = val; - // // if(mx < val) mx = val; - // printf("%f, ", val); - // } - // printf("]\n"); - // } + + // for(int i = 0; i < ne1; i++){ + // printf("%d: [", i); + // for(int j = 0; j < stride; j++){ + // float val = out_data[i*stride+j]; + // // if(mi > val) mi = val; + // // if(mx < val) mx = val; + // printf("%f, ", val); + // } + // printf("]\n"); + // } // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx); - printf("["); - for(int i = 0; i < stride; i++){ - printf("%f, ", out_data[i]); - } - printf("]\n"); + // printf("["); + // for(int i = 0; i < stride; i++){ + // printf("%f, ", out_data[i]); + // } + // printf("]\n"); delete out_data; } - struct ggml_cgraph g1v = ggml_graph_view(gf, i, i + 1); - ggml_backend_graph_compute(backend, &g1v); + } #endif ggml_backend_graph_compute(backend, gf); diff --git a/pmid.hpp b/pmid.hpp index 030d2b1b..31e049de 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -205,10 +205,14 @@ struct FuseModule{ struct ggml_tensor * valid_id_embeds = id_embeds; // # slice out the image token embeddings struct ggml_tensor * image_token_embeds = ggml_get_rows(ctx, prompt_embeds, class_tokens_mask_pos); - // print_ggml_tensor(image_token_embeds, true, "image_token_embeds"); - // print_ggml_tensor(valid_id_embeds, true, "valid_id_embeds"); + print_ggml_tensor(image_token_embeds, true, "image_token_embeds"); + ggml_set_name(image_token_embeds, "image_token_embeds"); + print_ggml_tensor(valid_id_embeds, true, "valid_id_embeds"); + ggml_set_name(valid_id_embeds, "valid_id_embeds"); + struct ggml_tensor *stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds); - // print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_before_concat"); + print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_before_concat"); + ggml_set_name(stacked_id_embeds, "stacked_id_embeds_fuse_fn"); stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_permute"); @@ -224,6 +228,7 @@ struct FuseModule{ } // print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_concat"); stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); + ggml_set_name(stacked_id_embeds, "stacked_id_embeds_add_zero"); // print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_permute_2"); // print_ggml_tensor(class_tokens_mask, true, "class_tokens_mask"); // print_ggml_tensor(prompt_embeds, true, "prompt_embeds"); @@ -240,6 +245,7 @@ struct FuseModule{ // print_ggml_tensor(class_tokens_mask, true, "class_tokens_mask_transpose"); prompt_embeds = ggml_mul(ctx, prompt_embeds, class_tokens_mask); // print_ggml_tensor(prompt_embeds, true, "prompt_embeds_after_mul"); + ggml_set_name(prompt_embeds, "prompt_embeds_mul_cls"); struct ggml_tensor * updated_prompt_embeds = ggml_add(ctx, prompt_embeds, stacked_id_embeds); // print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds"); // updated_prompt_embeds = prompt_embeds.view(batch_size, seq_length, -1) @@ -335,7 +341,8 @@ struct PhotoMakerIDEncoder : public GGMLModule { class_embedding_temp, positions ); // [batch_size, seq_length, hidden_size] - // print_ggml_tensor(shared_id_embeds, true, "shared_id_embeds"); + print_ggml_tensor(shared_id_embeds, true, "shared_id_embeds_from_vision"); + ggml_set_name(shared_id_embeds, "shared_id_embeds_from_vision"); // if(class_tokens_mask->backend == GGML_BACKEND_GPU){ // int *ctm = (int *)malloc(class_tokens_mask->ne[0]); @@ -352,8 +359,11 @@ struct PhotoMakerIDEncoder : public GGMLModule { // id_embeds_2 = id_embeds_2.view(b, num_inputs, 1, -1) // id_embeds = torch.cat((id_embeds, id_embeds_2), dim=-1) - // print_ggml_tensor(id_embeds, true, "id_embeds"); - // print_ggml_tensor(id_embeds_2, true, "id_embeds_2"); + print_ggml_tensor(id_embeds, true, "id_embeds_proj1"); + print_ggml_tensor(id_embeds_2, true, "id_embeds_proj2"); + + ggml_set_name(id_embeds, "id_embeds_proj1"); + ggml_set_name(id_embeds_2, "id_embeds_proj2"); id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 2, 0, 1, 3)); id_embeds_2 = ggml_cont(ctx, ggml_permute(ctx, id_embeds_2, 2, 0, 1, 3)); diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index 538d6b0d..c38d17bf 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -639,13 +639,15 @@ public: pmid_model.alloc_compute_buffer(work_ctx, init_img, prompts_embeds, class_tokens_mask); pmid_model.compute(n_threads, init_img, prompts_embeds, class_tokens_mask, res); pmid_model.free_compute_buffer(); - // printf("["); - // int j = 70; - // for(int i = 0; i < 100; i++){ - // float val = *((float *)(res->data)+j*total_hidden_size+i); - // printf("%f, ", val); - // } - // printf("]\n"); + + // for(int j = 0; j < cond_stage_model.text_model.max_position_embeddings; j++){ + // printf("["); + // for(int i = 0; i < total_hidden_size; i++){ + // float val = *((float *)(res->data)+j*total_hidden_size+i); + // printf("%f, ", val); + // } + // printf("]\n"); + // } return res; }