diff --git a/clip.hpp b/clip.hpp index b2fb0264..6042e4b3 100644 --- a/clip.hpp +++ b/clip.hpp @@ -959,11 +959,19 @@ struct CLIPVisionModel { int32_t n_layer = num_hidden_layers; const int d_head = hidden_size / n_head; + ggml_set_name(x, "id_pixel"); + ggml_set_name(temp, "temp_input"); + struct ggml_tensor * inp = ggml_conv_2d(ctx0, patch_embeddings, x, patch_size, patch_size, 0, 0, 1, 1); + ggml_set_name(inp, "inp_conv_2d"); + inp = ggml_reshape_3d(ctx0, inp, num_patches, hidden_size, batch_size); + ggml_set_name(inp, "inp_reshape_3d"); // inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3)); inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 3, 1)); + ggml_set_name(inp, "inp_cont"); + ggml_set_name(class_embedding, "class_embedding"); // concat class_embeddings and patch_embeddings // struct ggml_tensor * embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size); @@ -971,9 +979,12 @@ struct CLIPVisionModel { // struct ggml_tensor * temp = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, 1, batch_size); ggml_tensor *class_embedding_rep = ggml_repeat(ctx0, class_embedding, temp); + ggml_set_name(class_embedding_rep, "class_embedding_rep"); struct ggml_tensor *embeddings = ggml_concat(ctx0, class_embedding_rep, inp); + ggml_set_name(embeddings, "embeddings_after_concat"); embeddings = ggml_cont(ctx0, ggml_permute(ctx0, embeddings, 0, 3, 1, 2)); + ggml_set_name(embeddings, "embeddings_after_permute"); // embeddings = ggml_acc(ctx0, embeddings, ggml_repeat(ctx0, class_embedding, temp), embeddings->nb[1], // embeddings->nb[2], embeddings->nb[3], 0); @@ -982,20 +993,25 @@ struct CLIPVisionModel { embeddings = ggml_add(ctx0, embeddings, ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings)); + ggml_set_name(embeddings, "embeddings_after_add"); // pre-layernorm embeddings = ggml_nn_layer_norm(ctx0, embeddings, pre_ln_w, pre_ln_w); + ggml_set_name(embeddings, "embeddings_after_pre-layernorm"); // transformer for (int i = 0; i < num_hidden_layers; i++) { embeddings = resblocks[i].forward(ctx0, embeddings); // [N, n_token, hidden_size] } + ggml_set_name(embeddings, "embeddings_after_transformer"); // get the output of cls token, e.g., 0th index embeddings = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, embeddings, hidden_size, num_positions * batch_size), cls); + ggml_set_name(embeddings, "embeddings_after_cls_token"); // post-layernorm embeddings = ggml_nn_layer_norm(ctx0, embeddings, post_ln_w, post_ln_b); + ggml_set_name(embeddings, "embeddings_after_post-layernorm"); struct ggml_tensor * cur = embeddings; diff --git a/ggml_extend.hpp b/ggml_extend.hpp index 3fe5b1cc..7bdf319c 100644 --- a/ggml_extend.hpp +++ b/ggml_extend.hpp @@ -651,16 +651,19 @@ struct GGMLModule { ggml_backend_metal_set_n_cb(backend, n_threads); } #endif - + printf("about to do ggml_backend_graph_compute \n"); ggml_backend_graph_compute(backend, gf); #ifdef GGML_PERF ggml_graph_print(gf); #endif - + if (output != NULL) + print_ggml_tensor(output, true, "output_before_get"); if (output != NULL) { ggml_backend_tensor_get_and_sync(backend, gf->nodes[gf->n_nodes - 1], output->data, 0, ggml_nbytes(output)); } + if (output != NULL) + print_ggml_tensor(output, true, "output_get"); } void free_compute_buffer() { diff --git a/pmid.hpp b/pmid.hpp index f71f632d..8dac2c3a 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -211,6 +211,8 @@ struct FuseModule{ stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_permute"); if(left && right){ + print_ggml_tensor(left, true, "left"); + print_ggml_tensor(right, true, "right"); stacked_id_embeds = ggml_concat(ctx, left, stacked_id_embeds); stacked_id_embeds = ggml_concat(ctx, stacked_id_embeds, right); }else if(left){ @@ -221,10 +223,21 @@ struct FuseModule{ print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_concat"); stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_permute_2"); + print_ggml_tensor(class_tokens_mask, true, "class_tokens_mask"); + print_ggml_tensor(prompt_embeds, true, "prompt_embeds"); // assert class_tokens_mask.sum() == stacked_id_embeds.shape[0], f"{class_tokens_mask.sum()} != {stacked_id_embeds.shape[0]}" // prompt_embeds.masked_scatter_(class_tokens_mask[:, None], stacked_id_embeds.to(prompt_embeds.dtype)) - prompt_embeds = ggml_mul(ctx, prompt_embeds, ggml_repeat(ctx, class_tokens_mask, prompt_embeds)); + // struct ggml_tensor *prompt_embeds_perm = ggml_cont(ctx, ggml_permute(ctx, prompt_embeds, 1, 0, 2, 3)); + struct ggml_tensor *prompt_embeds_perm = ggml_cont(ctx, ggml_transpose(ctx, prompt_embeds)); + print_ggml_tensor(prompt_embeds_perm, true, "prompt_embeds_perm"); + class_tokens_mask = ggml_repeat(ctx, class_tokens_mask, prompt_embeds_perm); + print_ggml_tensor(class_tokens_mask, true, "class_tokens_mask_repeat"); + class_tokens_mask = ggml_cont(ctx, ggml_transpose(ctx, class_tokens_mask)); + print_ggml_tensor(class_tokens_mask, true, "class_tokens_mask_transpose"); + prompt_embeds = ggml_mul(ctx, prompt_embeds, class_tokens_mask); + print_ggml_tensor(prompt_embeds, true, "prompt_embeds_after_mul"); struct ggml_tensor * updated_prompt_embeds = ggml_add(ctx, prompt_embeds, stacked_id_embeds); + print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds"); // updated_prompt_embeds = prompt_embeds.view(batch_size, seq_length, -1) return updated_prompt_embeds; } @@ -296,6 +309,17 @@ struct PhotoMakerIDEncoder : public GGMLModule { // x: [N, channels, h, w] // in_layers + ggml_set_name(id_pixel_values, "id_pixel_values_input"); + ggml_set_name(prompt_embeds, "prompt_embeds_input"); + ggml_set_name(class_tokens_mask, "class_tokens_mask_input"); + ggml_set_name(class_tokens_mask_pos, "class_tokens_mask_pos_input"); + ggml_set_name(cls, "cls_input"); + ggml_set_name(class_embedding_temp, "class_embedding_temp_input"); + ggml_set_name(positions, "positions_input"); + ggml_set_name(left, "left_input"); + ggml_set_name(right, "right_input"); + + print_ggml_tensor(prompt_embeds, true, "prompt_embeds"); print_ggml_tensor(class_embedding_temp, true, "class_embedding_temp"); struct ggml_tensor *shared_id_embeds = vision_model.forward(ctx, @@ -306,15 +330,15 @@ struct PhotoMakerIDEncoder : public GGMLModule { ); // [batch_size, seq_length, hidden_size] print_ggml_tensor(shared_id_embeds, true, "shared_id_embeds"); - if(class_tokens_mask->backend == GGML_BACKEND_GPU){ - int *ctm = (int *)malloc(class_tokens_mask->ne[0]); - ggml_backend_tensor_get(class_tokens_mask, ctm, 0, ggml_nbytes(class_tokens_mask)); - printf("class_tokens_mask["); - for(int i = 0; i < class_tokens_mask->ne[0]; i++) - printf("%d, ", ctm[i]); - printf("]\n"); - free(ctm); - } + // if(class_tokens_mask->backend == GGML_BACKEND_GPU){ + // int *ctm = (int *)malloc(class_tokens_mask->ne[0]); + // ggml_backend_tensor_get(class_tokens_mask, ctm, 0, ggml_nbytes(class_tokens_mask)); + // printf("class_tokens_mask["); + // for(int i = 0; i < class_tokens_mask->ne[0]; i++) + // printf("%d, ", ctm[i]); + // printf("]\n"); + // free(ctm); + // } struct ggml_tensor *id_embeds = vision_model.visual_project(ctx, shared_id_embeds); // [batch_size, seq_length, proj_dim(768)] struct ggml_tensor *id_embeds_2 = ggml_mul_mat(ctx, visual_projection_2, shared_id_embeds); // [batch_size, seq_length, 1280] // id_embeds = id_embeds.view(b, num_inputs, 1, -1) @@ -342,6 +366,7 @@ struct PhotoMakerIDEncoder : public GGMLModule { class_tokens_mask, class_tokens_mask_pos, left, right); + print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds_returned"); return updated_prompt_embeds; @@ -366,37 +391,43 @@ struct PhotoMakerIDEncoder : public GGMLModule { struct ggml_cgraph* gf = ggml_new_graph(ctx0); - struct ggml_tensor* id_pixel_values_d = ggml_dup_tensor(ctx0, id_pixel_values); - ggml_allocr_alloc(allocr, id_pixel_values_d); - struct ggml_tensor* prompt_embeds_d = ggml_dup_tensor(ctx0, prompt_embeds); - ggml_allocr_alloc(allocr, prompt_embeds_d); - struct ggml_tensor* class_tokens_mask_d = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, class_tokens_mask.size()); - ggml_allocr_alloc(allocr, class_tokens_mask_d); - int64_t hidden_size = prompt_embeds->ne[0]; int64_t seq_length = prompt_embeds->ne[1]; ggml_type type = prompt_embeds->type; + struct ggml_tensor* id_pixel_values_d = ggml_dup_tensor(ctx0, id_pixel_values); + ggml_allocr_alloc(allocr, id_pixel_values_d); + struct ggml_tensor* prompt_embeds_d = ggml_dup_tensor(ctx0, prompt_embeds); + ggml_allocr_alloc(allocr, prompt_embeds_d); + struct ggml_tensor* class_tokens_mask_d = ggml_new_tensor_1d(ctx0, type, class_tokens_mask.size()); + ggml_allocr_alloc(allocr, class_tokens_mask_d); + + + std::vector ctm; + std::vector ctmf16; std::vector ctmpos; struct ggml_tensor* left = NULL; struct ggml_tensor* right = NULL; for(int i=0; i < class_tokens_mask.size(); i++){ if(class_tokens_mask[i]){ ctm.push_back(0.f); // here use 0.f instead of 1.f to make a scale mask + ctmf16.push_back(ggml_fp32_to_fp16(0.f)); // here use 0.f instead of 1.f to make a scale mask ctmpos.push_back(i); + // printf("push %d, \n", i); }else{ ctm.push_back(1.f); // here use 1.f instead of 0.f to make a scale mask + ctmf16.push_back(ggml_fp32_to_fp16(1.f)); // here use 0.f instead of 1.f to make a scale mask } } if(ctmpos[0] > 0){ - left = ggml_new_tensor_2d(ctx0, type, hidden_size, ctmpos[0]); + left = ggml_new_tensor_3d(ctx0, type, hidden_size, 1, ctmpos[0]); ggml_allocr_alloc(allocr, left); } if(ctmpos[ctmpos.size()-1] < seq_length - 1){ - right = ggml_new_tensor_2d(ctx0, type, - hidden_size, seq_length-ctmpos[ctmpos.size()-1]-1); + right = ggml_new_tensor_3d(ctx0, type, + hidden_size, 1, seq_length-ctmpos[ctmpos.size()-1]-1); ggml_allocr_alloc(allocr, right); } struct ggml_tensor* class_tokens_mask_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ctmpos.size()); @@ -409,7 +440,7 @@ struct PhotoMakerIDEncoder : public GGMLModule { struct ggml_tensor * cls = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, batch_size); - struct ggml_tensor * class_embedding_temp = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, + struct ggml_tensor * class_embedding_temp = ggml_new_tensor_4d(ctx0, type, vision_model.hidden_size, 1, 1, batch_size); struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_positions); ggml_allocr_alloc(allocr, cls); @@ -421,7 +452,10 @@ struct PhotoMakerIDEncoder : public GGMLModule { if (!ggml_allocr_is_measure(allocr)) { ggml_backend_tensor_set(id_pixel_values_d, id_pixel_values->data, 0, ggml_nbytes(id_pixel_values)); ggml_backend_tensor_set(prompt_embeds_d, prompt_embeds->data, 0, ggml_nbytes(prompt_embeds)); - ggml_backend_tensor_set(class_tokens_mask_d, ctm.data(), 0, ggml_nbytes(class_tokens_mask_d)); + if(type == GGML_TYPE_F16) + ggml_backend_tensor_set(class_tokens_mask_d, ctmf16.data(), 0, ggml_nbytes(class_tokens_mask_d)); + else + ggml_backend_tensor_set(class_tokens_mask_d, ctm.data(), 0, ggml_nbytes(class_tokens_mask_d)); std::vector cls_h; for (int b = 0; b < batch_size; b++) { cls_h.push_back(b * num_positions); @@ -462,8 +496,9 @@ struct PhotoMakerIDEncoder : public GGMLModule { positions, left, right ); - + print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds_returned_forward"); ggml_build_forward_expand(gf, updated_prompt_embeds); + // ggml_graph_dump_dot(gf, NULL, "id_encoder.dot"); ggml_free(ctx0); return gf; diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index 778aa2a8..45875215 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -616,11 +616,11 @@ public: total_hidden_size += cond_stage_model.text_model2.hidden_size; } ggml_tensor *res = ggml_new_tensor_2d(work_ctx, - GGML_TYPE_F32, + prompts_embeds->type, total_hidden_size, cond_stage_model.text_model.max_position_embeddings); pmid_model.alloc_compute_buffer(work_ctx, init_img, prompts_embeds, class_tokens_mask); - pmid_model.compute(n_threads, init_img, prompts_embeds, class_tokens_mask, res); + pmid_model.compute(n_threads, init_img, prompts_embeds, class_tokens_mask, res); pmid_model.free_compute_buffer(); return res; } @@ -1495,7 +1495,7 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx, } // Encode input prompt without the trigger word for delayed conditioning prompt_text_only = sd_ctx->sd->remove_trigger_from_prompt(work_ctx, prompt); - // printf("%s || %s \n", prompt.c_str(), prompt_text_only.c_str()); + printf("%s || %s \n", prompt.c_str(), prompt_text_only.c_str()); prompt = prompt_text_only; // }