From 857af489a9c1fbf6be9d2e720804b82be38307f9 Mon Sep 17 00:00:00 2001 From: bssrdf Date: Sun, 11 Feb 2024 22:32:35 -0500 Subject: [PATCH] fixed stat_merge_step --- clip.hpp | 7 ++++--- ggml | 2 +- ggml_extend.hpp | 1 + pmid.hpp | 1 + stable-diffusion.cpp | 17 +++++++++++------ 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/clip.hpp b/clip.hpp index 38fadd8e..ed3481d0 100644 --- a/clip.hpp +++ b/clip.hpp @@ -995,7 +995,7 @@ struct CLIPVisionModel { ggml_set_name(class_embedding_rep, "class_embedding_rep"); // print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep"); - class_embedding_rep = ggml_cast(ctx0, class_embedding_rep, inp->type); + // class_embedding_rep = ggml_cast(ctx0, class_embedding_rep, inp->type); // print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_aft_casting"); struct ggml_tensor *embeddings = ggml_concat(ctx0, class_embedding_rep, inp); ggml_set_name(embeddings, "embeddings_after_concat"); @@ -1009,11 +1009,12 @@ struct CLIPVisionModel { // struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_positions); // print_ggml_tensor(embeddings, true, "embeddings_bef_add"); - // print_ggml_tensor(ggml_get_rows(ctx0, position_embeddings, positions), true, "embeddings_add_src1"); + // print_ggml_tensor(ggml_get_rows(ctx0, position_embeddings, positions), true, "position_embeddings_bef_add"); // print_ggml_tensor(ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings), true, "embeddings_add_src1"); // embeddings = ggml_cont(ctx0, ggml_permute(ctx0, embeddings, 0, 1, 3, 2)); embeddings = - ggml_add(ctx0, embeddings, ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings)); + // ggml_add(ctx0, embeddings, ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings)); + ggml_add(ctx0, embeddings, ggml_get_rows(ctx0, position_embeddings, positions)); ggml_set_name(embeddings, "embeddings_after_add"); // print_ggml_tensor(embeddings, true, "embeddings_after_add"); diff --git a/ggml b/ggml index 2f3b12fb..f826935f 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit 2f3b12fbd6cf4cb41ad4c8fdfd65e937f5c92093 +Subproject commit f826935f6b2e358bfb9945afe25c4c448a68da4f diff --git a/ggml_extend.hpp b/ggml_extend.hpp index ce184a83..334d1289 100644 --- a/ggml_extend.hpp +++ b/ggml_extend.hpp @@ -230,6 +230,7 @@ __STATIC_INLINE__ void sd_image_to_tensor(const uint8_t* image_data, ggml_tensor_set_f32(output, pixel_val, ix, iy, k); } } + } } diff --git a/pmid.hpp b/pmid.hpp index 85c2da91..19bc0cf3 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -205,6 +205,7 @@ struct FuseModule{ // # slice out the image token embeddings struct ggml_tensor * image_token_embeds = ggml_get_rows(ctx, prompt_embeds, class_tokens_mask_pos); // print_ggml_tensor(image_token_embeds, true, "image_token_embeds"); + // print_ggml_tensor(valid_id_embeds, true, "valid_id_embeds"); struct ggml_tensor *stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds); // print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_before_concat"); diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index 8d68c440..340d63ec 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -742,6 +742,7 @@ public: const std::vector& sigmas, int start_merge_step, ggml_tensor* c_id, + ggml_tensor* c_vec_id, float control_strength) { size_t steps = sigmas.size() - 1; // x_t = load_tensor_from_file(work_ctx, "./rand0.bin"); @@ -816,6 +817,7 @@ public: if (step <= start_merge_step) diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c, control_net.controls, control_strength, t_emb, c_vector); else + // diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c_id, control_net.controls, control_strength, t_emb, c_vec_id); diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c_id, control_net.controls, control_strength, t_emb, c_vector); }else{ diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c, control_net.controls, control_strength, t_emb, c_vector); @@ -1475,11 +1477,11 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx, // ggml_tensor* class_tokens_mask = NULL; std::vector class_tokens_mask; if(sd_ctx->sd->stacked_id){ - int32_t width = input_id_images[0]->width; - int32_t height = input_id_images[0]->height; + int32_t w = input_id_images[0]->width; + int32_t h = input_id_images[0]->height; int32_t channels = input_id_images[0]->channel; int32_t num_input_images = input_id_images.size(); - init_img = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, width, height, channels, num_input_images); + init_img = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, w, h, channels, num_input_images); // TODO: move these to somewhere else and be user settable float mean[] = {0.48145466, 0.4578275, 0.40821073}; float std[] = {0.26862954, 0.26130258, 0.27577711}; @@ -1557,12 +1559,14 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx, int start_merge_step = -1; if(sd_ctx->sd->stacked_id){ - int style_strength_ratio = 20; - start_merge_step = int(style_strength_ratio / 100 * sample_steps); + float style_strength_ratio = 20.f; + start_merge_step = int(style_strength_ratio / 100.f * sample_steps); if(start_merge_step > 30) start_merge_step = 30; + LOG_INFO("PHOTOMAKER: start_merge_step: %d", start_merge_step); } + struct ggml_tensor* x_0 = sd_ctx->sd->sample(work_ctx, x_t, NULL, @@ -1576,6 +1580,7 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx, sigmas, start_merge_step, prompts_embeds, + pooled_prompts_embeds, control_strength); // struct ggml_tensor* x_0 = load_tensor_from_file(ctx, "samples_ddim.bin"); // print_ggml_tensor(x_0); @@ -1731,7 +1736,7 @@ sd_image_t* img2img(sd_ctx_t* sd_ctx, LOG_INFO("sampling using %s method", sampling_methods_str[sample_method]); struct ggml_tensor* x_0 = sd_ctx->sd->sample(work_ctx, init_latent, noise, c, c_vector, uc, uc_vector, NULL, cfg_scale, sample_method, sigma_sched, - -1, NULL, + -1, NULL, NULL, 1.0f); // struct ggml_tensor *x_0 = load_tensor_from_file(ctx, "samples_ddim.bin"); // print_ggml_tensor(x_0);