From 539a94adc7f7ee56fce53b0ed06d0897f80d0185 Mon Sep 17 00:00:00 2001 From: bssrdf Date: Tue, 13 Feb 2024 21:44:17 -0500 Subject: [PATCH] add a few print tensor; tweak in sample again --- clip.hpp | 18 ++++++++++-------- model.cpp | 4 ++-- pmid.hpp | 5 ++++- stable-diffusion.cpp | 12 ++++++++++-- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/clip.hpp b/clip.hpp index 3f1fe71a..d06355dc 100644 --- a/clip.hpp +++ b/clip.hpp @@ -963,22 +963,22 @@ struct CLIPVisionModel { ggml_set_name(x, "id_pixel"); ggml_set_name(temp, "temp_input"); int64_t* ne = patch_embeddings->ne; - struct ggml_tensor *patch_embeddings_f16 = ggml_reshape_3d(ctx0, patch_embeddings, ne[0], ne[1], ne[2]*ne[3]); - patch_embeddings_f16 = ggml_cast(ctx0, patch_embeddings_f16, GGML_TYPE_F16); - patch_embeddings_f16 = ggml_reshape_4d(ctx0, patch_embeddings_f16, ne[0], ne[1], ne[2], ne[3]); - + // struct ggml_tensor *patch_embeddings_f16 = ggml_reshape_3d(ctx0, patch_embeddings, ne[0], ne[1], ne[2]*ne[3]); + // patch_embeddings_f16 = ggml_cast(ctx0, patch_embeddings_f16, GGML_TYPE_F16); + // patch_embeddings_f16 = ggml_reshape_4d(ctx0, patch_embeddings_f16, ne[0], ne[1], ne[2], ne[3]); + struct ggml_tensor *patch_embeddings_f16 = ggml_cast(ctx0, patch_embeddings, GGML_TYPE_F16); struct ggml_tensor * inp = ggml_conv_2d(ctx0, patch_embeddings_f16, x, patch_size, patch_size, 0, 0, 1, 1); ggml_set_name(inp, "inp_conv_2d"); - // print_ggml_tensor(inp, true, "inp_conv_2d"); + print_ggml_tensor(inp, true, "inp_conv_2d"); inp = ggml_reshape_3d(ctx0, inp, num_patches, hidden_size, batch_size); ggml_set_name(inp, "inp_reshape_3d"); - // print_ggml_tensor(inp, true, "inp_reshape_3d"); + print_ggml_tensor(inp, true, "inp_reshape_3d"); // inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3)); // print_ggml_tensor(ggml_permute(ctx0, inp, 2, 0, 1, 3), true, "inp_permute"); inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3)); - // print_ggml_tensor(inp, true, "inp_cont"); + print_ggml_tensor(inp, true, "inp_cont"); ggml_set_name(inp, "inp_cont"); ggml_set_name(class_embedding, "class_embedding"); @@ -996,7 +996,7 @@ struct CLIPVisionModel { ggml_set_name(class_embedding_rep, "class_embedding_rep"); // print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep"); // class_embedding_rep = ggml_cast(ctx0, class_embedding_rep, inp->type); - // print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_aft_casting"); + print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_aft_casting"); struct ggml_tensor *embeddings = ggml_concat(ctx0, class_embedding_rep, inp); ggml_set_name(embeddings, "embeddings_after_concat"); // print_ggml_tensor(embeddings, true, "embeddings_after_concat"); @@ -1027,10 +1027,12 @@ struct CLIPVisionModel { embeddings = resblocks[i].forward(ctx0, embeddings); // [N, n_token, hidden_size] } ggml_set_name(embeddings, "embeddings_after_transformer"); + print_ggml_tensor(embeddings, true, "embeddings_after_encoder"); // get the output of cls token, e.g., 0th index embeddings = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, embeddings, hidden_size, num_positions * batch_size), cls); ggml_set_name(embeddings, "embeddings_after_cls_token"); + print_ggml_tensor(embeddings, true, "embeddings_pooled_after_encoder"); // post-layernorm embeddings = ggml_nn_layer_norm(ctx0, embeddings, post_ln_w, post_ln_b); diff --git a/model.cpp b/model.cpp index c41dceea..8b3036cf 100644 --- a/model.cpp +++ b/model.cpp @@ -789,8 +789,8 @@ bool ModelLoader::init_from_safetensors_file(const std::string& file_path, const ne[i] = shape[i].get(); } TensorStorage tensor_storage(prefix + name, type, ne, n_dims, file_index, ST_HEADER_SIZE_LEN + header_size_ + begin); - // if(strcmp(prefix.c_str(), "pmid") == 0) - // printf(" %s:%s [%zu, %zu, %zu, %zu]\n", (prefix + name).c_str(), ggml_type_name(type), ne[0], ne[1],ne[2], ne[3]); + // if(starts_with(prefix, "pmi")) + // printf(" %s:%s [%zu, %zu, %zu, %zu]\n", (prefix + name).c_str(), ggml_type_name(type), ne[0], ne[1],ne[2], ne[3]); tensor_storage.reverse_ne(); size_t tensor_data_size = end - begin; diff --git a/pmid.hpp b/pmid.hpp index fc6585df..2f77465f 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -552,7 +552,10 @@ struct PhotoMakerLoraModel : public GGMLModule { ModelLoader model_loader; bool load_failed = false; - PhotoMakerLoraModel(const std::string file_path = "") + PhotoMakerLoraModel(){ + } + + PhotoMakerLoraModel(const std::string file_path) : file_path(file_path) { name = "photomaker lora"; if (!model_loader.init_from_file(file_path, "pmid.")) { diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index 56219dd9..e827b983 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -633,9 +633,17 @@ public: prompts_embeds->type, total_hidden_size, cond_stage_model.text_model.max_position_embeddings); + pmid_model.alloc_compute_buffer(work_ctx, init_img, prompts_embeds, class_tokens_mask); pmid_model.compute(n_threads, init_img, prompts_embeds, class_tokens_mask, res); pmid_model.free_compute_buffer(); + // printf("["); + // int j = 70; + // for(int i = 0; i < 100; i++){ + // float val = *((float *)(res->data)+j*total_hidden_size+i); + // printf("%f, ", val); + // } + // printf("]\n"); return res; } @@ -825,8 +833,8 @@ public: diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c, control_net.controls, control_strength, t_emb, c_vector); } else{ - // diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c_id, control_net.controls, control_strength, t_emb, c_vec_id); - diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c_id, control_net.controls, control_strength, t_emb, c_vector); + diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c_id, control_net.controls, control_strength, t_emb, c_vec_id); + // diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c_id, control_net.controls, control_strength, t_emb, c_vector); } }else{ diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c, control_net.controls, control_strength, t_emb, c_vector);