diff --git a/clip.hpp b/clip.hpp index a2d2d612..49b21654 100644 --- a/clip.hpp +++ b/clip.hpp @@ -548,16 +548,7 @@ public: auto layer_norm2 = std::dynamic_pointer_cast(blocks["layer_norm2"]); auto mlp = std::dynamic_pointer_cast(blocks["mlp"]); - // struct ggml_tensor* r = x; - - struct ggml_tensor* h = layer_norm1->forward(ctx, x); - - h = self_attn->forward(ctx, h, mask); - - - // x = ggml_add(ctx, x, self_attn->forward(ctx, layer_norm1->forward(ctx, x), mask)); - x = ggml_add(ctx, x, h); - + x = ggml_add(ctx, x, self_attn->forward(ctx, layer_norm1->forward(ctx, x), mask)); x = ggml_add(ctx, x, mlp->forward(ctx, layer_norm2->forward(ctx, x))); return x; } @@ -567,16 +558,10 @@ public: auto layer_norm2 = std::dynamic_pointer_cast(blocks["layer_norm2"]); auto mlp = std::dynamic_pointer_cast(blocks["mlp"]); - // struct ggml_tensor* r = x; - - struct ggml_tensor* h = layer_norm1->forward(ctx, x); - - h = self_attn->forward(ctx, h, mask); - - - // x = ggml_add(ctx, x, self_attn->forward(ctx, layer_norm1->forward(ctx, x), mask)); - x = ggml_add(ctx, x, h); - + // struct ggml_tensor* h = layer_norm1->forward(ctx, x); + // h = self_attn->forward(ctx, h, mask); + x = ggml_add(ctx, x, self_attn->forward(ctx, layer_norm1->forward(ctx, x), mask)); + // x = ggml_add(ctx, x, h); x = ggml_add(ctx, x, mlp->forward(ctx, layer_norm2->forward(ctx, x))); return x; } @@ -614,10 +599,8 @@ public: break; } std::string name = "layers." + std::to_string(i); - // printf(" about to do %s\n", name.c_str()); auto layer = std::dynamic_pointer_cast(blocks[name]); x = layer->forward(ctx, x, mask, atten1); // [N, n_token, d_model] - // print_ggml_tensor(x, true, ("layer "+std::to_string(i)).c_str()); // LOG_DEBUG("layer %d", i); } return x; @@ -924,21 +907,17 @@ public: auto post_layernorm = std::dynamic_pointer_cast(blocks["post_layernorm"]); auto x = embeddings->forward(ctx, pixel_values); // [N, num_positions, embed_dim] - print_ggml_tensor(x, true, "embedding"); x = pre_layernorm->forward(ctx, x); - print_ggml_tensor(x, true, "pre_layernorm"); x = encoder->forward(ctx, x, -1, false, false); - print_ggml_tensor(x, true, "encoder"); x = post_layernorm->forward(ctx, x); // [N, n_token, hidden_size] - print_ggml_tensor(x, true, "post_layernorm"); GGML_ASSERT(x->ne[3] == 1); // int64_t max_token_idx = 0; // ggml_tensor* pooled = ggml_view_1d(ctx, x, x->ne[0], x->nb[1] * max_token_idx); // assert N == 1 - x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); - // ggml_view_2d(ctx, w, w->ne[0], w->ne[1] / 2, w->nb[1], 0); - ggml_tensor* pooled = ggml_view_2d(ctx, x, x->ne[0], x->ne[1], x->nb[1], 0); - print_ggml_tensor(pooled, true, "pooled"); + x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3)); + ggml_tensor* pooled = ggml_view_2d(ctx, x, x->ne[0], x->ne[1], x->nb[0], 0); + pooled = ggml_scale(ctx, ggml_cont(ctx, pooled), 1.f); + // print_ggml_tensor(pooled, true, "pooled"); return pooled; // [N, projection_dim] } diff --git a/ggml_extend.hpp b/ggml_extend.hpp index 2761a70c..b9f1f001 100644 --- a/ggml_extend.hpp +++ b/ggml_extend.hpp @@ -926,11 +926,8 @@ public: alloc_compute_buffer(get_graph); reset_compute_ctx(); struct ggml_cgraph* gf = get_graph(); - GGML_ASSERT(ggml_gallocr_alloc_graph(compute_allocr, gf)); - cpy_data_to_backend_tensor(); - if (ggml_backend_is_cpu(backend)) { ggml_backend_cpu_set_n_threads(backend, n_threads); } @@ -1037,16 +1034,11 @@ public: } for (auto& pair : blocks) { auto& block = pair.second; - // if(starts_with(prefix, "pmid")) - // printf("block pair.first: %s, %s \n", prefix.c_str(), pair.first.c_str()); - block->get_param_tensors(tensors, prefix + pair.first); } for (auto& pair : params) { struct ggml_tensor* param = pair.second; - // if(starts_with(prefix, "pmid")) - // printf("params pair.first: %s, %s \n", prefix.c_str(), pair.first.c_str()); tensors[prefix + pair.first] = pair.second; } } diff --git a/pmid.hpp b/pmid.hpp index 8224d50c..a68ec26b 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -225,7 +225,13 @@ public: struct ggml_tensor * valid_id_embeds = id_embeds; // # slice out the image token embeddings + // print_ggml_tensor(class_tokens_mask_pos, false); + ggml_set_name(class_tokens_mask_pos, "class_tokens_mask_pos"); + ggml_set_name(prompt_embeds, "prompt_embeds"); + // print_ggml_tensor(valid_id_embeds, true, "valid_id_embeds"); + // print_ggml_tensor(class_tokens_mask_pos, true, "class_tokens_mask_pos"); struct ggml_tensor * image_token_embeds = ggml_get_rows(ctx, prompt_embeds, class_tokens_mask_pos); + ggml_set_name(image_token_embeds, "image_token_embeds"); struct ggml_tensor *stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds); stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); @@ -242,6 +248,7 @@ public: class_tokens_mask = ggml_repeat(ctx, class_tokens_mask, prompt_embeds); prompt_embeds = ggml_mul(ctx, prompt_embeds, class_tokens_mask); struct ggml_tensor * updated_prompt_embeds = ggml_add(ctx, prompt_embeds, stacked_id_embeds); + ggml_set_name(updated_prompt_embeds, "updated_prompt_embeds"); return updated_prompt_embeds; } @@ -289,6 +296,16 @@ public: VisualProjection visual_projection_2; float style_strength; + + std::vector ctm; + std::vector ctmf16; + std::vector ctmpos; + + std::vector zeros_left_16; + std::vector zeros_left; + std::vector zeros_right_16; + std::vector zeros_right; + public: PhotoMakerIDEncoder(ggml_backend_t backend, ggml_type wtype, SDVersion version = VERSION_XL, float sty = 20.f) @@ -382,14 +399,14 @@ public: // positions // ); // [batch_size, seq_length, hidden_size] struct ggml_tensor *shared_id_embeds = vision_model.forward(ctx, id_pixel_values); // [batch_size, seq_length, hidden_size] - print_ggml_tensor(shared_id_embeds, true, "shared_id_embeds"); + // print_ggml_tensor(shared_id_embeds, true, "shared_id_embeds"); struct ggml_tensor *id_embeds = vision_model.visual_project(ctx, shared_id_embeds); // [batch_size, seq_length, proj_dim(768)] - print_ggml_tensor(id_embeds, true, "id_embeds"); + // print_ggml_tensor(id_embeds, true, "id_embeds"); // struct ggml_tensor *id_embeds_2 = ggml_mul_mat(ctx, visual_projection_2, shared_id_embeds); // [batch_size, seq_length, 1280] struct ggml_tensor *id_embeds_2 = visual_projection_2.forward(ctx, shared_id_embeds); // [batch_size, seq_length, 1280] - print_ggml_tensor(id_embeds_2, true, "id_embeds_2"); + // print_ggml_tensor(id_embeds_2, true, "id_embeds_2"); @@ -399,14 +416,14 @@ public: id_embeds = ggml_concat(ctx, id_embeds, id_embeds_2); // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 1, 2, 0, 3)); - print_ggml_tensor(id_embeds, true, "id_embeds_after_cont+perm"); + // print_ggml_tensor(id_embeds, true, "id_embeds_after_cont+perm"); struct ggml_tensor * updated_prompt_embeds = fuse_module.forward(ctx, prompt_embeds, id_embeds, class_tokens_mask, class_tokens_mask_pos, left, right); - print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds"); + // print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds"); return updated_prompt_embeds; @@ -431,6 +448,15 @@ public: // struct ggml_cgraph* gf = ggml_new_graph(ctx0); + + ctm.clear(); + ctmf16.clear(); + ctmpos.clear(); + zeros_left.clear(); + zeros_left_16.clear(); + zeros_right.clear(); + zeros_right_16.clear(); + ggml_context *ctx0 = compute_ctx; struct ggml_cgraph* gf = ggml_new_graph(compute_ctx); @@ -450,9 +476,6 @@ public: struct ggml_tensor* id_pixel_values_d = to_backend(id_pixel_values); struct ggml_tensor* prompt_embeds_d = to_backend(prompt_embeds); - std::vector ctm; - std::vector ctmf16; - std::vector ctmpos; struct ggml_tensor* left = NULL; struct ggml_tensor* right = NULL; for(int i=0; i < class_tokens_mask.size(); i++){ @@ -517,24 +540,32 @@ public: set_backend_tensor_data(class_tokens_mask_pos, ctmpos.data()); if(left){ if(type == GGML_TYPE_F16){ - std::vector zeros(ggml_nelements(left), ggml_fp32_to_fp16(0.f)); + // std::vector zeros(ggml_nelements(left), ggml_fp32_to_fp16(0.f)); + for(int i = 0; i < ggml_nelements(left); ++i) + zeros_left_16.push_back(ggml_fp32_to_fp16(0.f)); // ggml_backend_tensor_set(left, zeros.data(), 0, ggml_nbytes(left)); - set_backend_tensor_data(left, zeros.data()); + set_backend_tensor_data(left, zeros_left_16.data()); }else{ - std::vector zeros(ggml_nelements(left), 0.f); + // std::vector zeros(ggml_nelements(left), 0.f); + for(int i = 0; i < ggml_nelements(left); ++i) + zeros_left.push_back(0.f); // ggml_backend_tensor_set(left, zeros.data(), 0, ggml_nbytes(left)); - set_backend_tensor_data(left, zeros.data()); + set_backend_tensor_data(left, zeros_left.data()); } } if(right){ if(type == GGML_TYPE_F16){ - std::vector zeros(ggml_nelements(right), ggml_fp32_to_fp16(0.f)); + // std::vector zeros(ggml_nelements(right), ggml_fp32_to_fp16(0.f)); // ggml_backend_tensor_set(right, zeros.data(), 0, ggml_nbytes(right)); - set_backend_tensor_data(right, zeros.data()); + for(int i = 0; i < ggml_nelements(right); ++i) + zeros_right_16.push_back(ggml_fp32_to_fp16(0.f)); + set_backend_tensor_data(right, zeros_right_16.data()); }else{ - std::vector zeros(ggml_nelements(right), 0.f); + // std::vector zeros(ggml_nelements(right), 0.f); + for(int i = 0; i < ggml_nelements(right); ++i) + zeros_right.push_back(0.f); // ggml_backend_tensor_set(right, zeros.data(), 0, ggml_nbytes(right)); - set_backend_tensor_data(right, zeros.data()); + set_backend_tensor_data(right, zeros_right.data()); } } } @@ -554,18 +585,6 @@ public: return gf; } - void alloc_compute_buffer(ggml_context* work_ctx, - struct ggml_tensor* id_pixel_values, - struct ggml_tensor* prompt_embeds, - std::vector &class_tokens_mask) { - auto get_graph = [&]() -> struct ggml_cgraph* { - - // return build_graph(compute_allocr, id_pixel_values, prompt_embeds, class_tokens_mask); - return build_graph(id_pixel_values, prompt_embeds, class_tokens_mask); - }; - GGMLModule::alloc_compute_buffer(get_graph); - } - void compute(const int n_threads, struct ggml_tensor* id_pixel_values, struct ggml_tensor* prompt_embeds, @@ -614,7 +633,7 @@ public: } std::string get_desc() { - return "pmid lora"; + return "lora_pmid"; } size_t get_params_num() { diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index 05f16b6f..22e33760 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -554,6 +554,10 @@ public: // for(int i = 0; i < tokens.size(); ++i) // printf("%d ", tokens[i]); // printf("\n"); + // printf("clsm: \n"); + // for(int i = 0; i < clsm.size(); ++i) + // printf("%d ", clsm[i]?1:0); + // printf("\n"); int64_t t0 = ggml_time_ms(); struct ggml_tensor* hidden_states = NULL; // [N, n_token, hidden_size] struct ggml_tensor* pooled = NULL; @@ -1691,16 +1695,16 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx, else sd_mul_images_to_tensor(init_image->data, init_img, i, NULL, NULL); } - + t0 = ggml_time_ms(); auto cond_tup = sd_ctx->sd->get_learned_condition_with_trigger(work_ctx, prompt, clip_skip, width, height, num_input_images ); - LOG_INFO("get_learned_condition_with_trigger finished"); prompts_embeds = std::get<0>(cond_tup); pooled_prompts_embeds = std::get<1>(cond_tup); // [adm_in_channels, ] class_tokens_mask = std::get<2>(cond_tup); // prompts_embeds = sd_ctx->sd->id_encoder(work_ctx, init_img, prompts_embeds, class_tokens_mask); - LOG_INFO("id_encoder"); + t1 = ggml_time_ms(); + LOG_INFO("Photomaker ID Stacking, taking %" PRId64 " ms", t1 - t0); if (sd_ctx->sd->free_params_immediately) { sd_ctx->sd->pmid_model->free_params_buffer(); }