diff --git a/clip.hpp b/clip.hpp index 0da75169..b8f777ac 100644 --- a/clip.hpp +++ b/clip.hpp @@ -958,8 +958,6 @@ struct CLIPVisionModel { struct ggml_tensor *cls, struct ggml_tensor *temp, struct ggml_tensor *positions) { - // input_ids: [N, n_token] - // GGML_ASSERT(input_ids->ne[0] <= position_ids->ne[0]); const int image_size = x->ne[0]; int batch_size = x->ne[3]; const int num_patches = ((image_size / patch_size) * (image_size / patch_size)); @@ -967,88 +965,32 @@ struct CLIPVisionModel { int32_t n_layer = num_hidden_layers; const int d_head = hidden_size / n_head; - // ggml_set_name(x, "id_pixel"); - // print_ggml_tensor(x, true, "id_pixel"); - ggml_set_name(temp, "temp_input"); - int64_t* ne = patch_embeddings->ne; - // struct ggml_tensor *patch_embeddings_f16 = ggml_reshape_3d(ctx0, patch_embeddings, ne[0], ne[1], ne[2]*ne[3]); - // patch_embeddings_f16 = ggml_cast(ctx0, patch_embeddings_f16, GGML_TYPE_F16); - // patch_embeddings_f16 = ggml_reshape_4d(ctx0, patch_embeddings_f16, ne[0], ne[1], ne[2], ne[3]); struct ggml_tensor *patch_embeddings_f16 = ggml_cast(ctx0, patch_embeddings, GGML_TYPE_F16); - ggml_set_name(patch_embeddings_f16, "patch_embeddings_f16"); - // print_ggml_tensor(patch_embeddings_f16, true, "patch_embeddings_f16"); - struct ggml_tensor * inp = ggml_conv_2d(ctx0, patch_embeddings_f16, x, patch_size, patch_size, 0, 0, 1, 1); - ggml_set_name(inp, "inp_conv_2d"); - // print_ggml_tensor(inp, true, "inp_conv_2d"); - inp = ggml_reshape_3d(ctx0, inp, num_patches, hidden_size, batch_size); - ggml_set_name(inp, "inp_reshape_3d"); - // print_ggml_tensor(inp, true, "inp_reshape_3d"); - // inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3)); - // print_ggml_tensor(ggml_permute(ctx0, inp, 2, 0, 1, 3), true, "inp_permute"); inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3)); - // print_ggml_tensor(inp, true, "inp_cont_perm+cont"); - ggml_set_name(inp, "inp_cont"); - // ggml_set_name(class_embedding, "class_embedding"); - - // concat class_embeddings and patch_embeddings - // struct ggml_tensor * embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size); - - // struct ggml_tensor * temp = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, 1, batch_size); - // ggml_tensor *class_embedding_rep = ggml_add(ctx0, cast_f32_class, class_embedding); - // ggml_set_name(class_embedding_rep, "add_class_embedding_to_zero"); - // print_ggml_tensor(class_embedding, true, "model.class_embedding"); - // print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_bef_repeat"); - // print_ggml_tensor(temp, true, "temp"); ggml_tensor *class_embedding_rep = ggml_repeat(ctx0, class_embedding, temp); - ggml_set_name(class_embedding_rep, "class_embedding_rep"); - // print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep"); - // class_embedding_rep = ggml_cast(ctx0, class_embedding_rep, inp->type); - // print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_aft_casting"); struct ggml_tensor *embeddings = ggml_concat(ctx0, class_embedding_rep, inp); - ggml_set_name(embeddings, "embeddings_after_concat"); - // print_ggml_tensor(embeddings, true, "embeddings_after_concat"); - // print_ggml_tensor(ggml_permute(ctx0, embeddings, 0, 3, 1, 2), true, "embeddings_after_concat_permute"); embeddings = ggml_cont(ctx0, ggml_permute(ctx0, embeddings, 0, 2, 1, 3)); - ggml_set_name(embeddings, "embeddings_after_permute"); - - // embeddings = ggml_acc(ctx0, embeddings, ggml_repeat(ctx0, class_embedding, temp), embeddings->nb[1], - // embeddings->nb[2], embeddings->nb[3], 0); - // struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_positions); - // print_ggml_tensor(embeddings, true, "embeddings_bef_add"); - // print_ggml_tensor(ggml_get_rows(ctx0, position_embeddings, positions), true, "position_embeddings_bef_add"); - // print_ggml_tensor(ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings), true, "embeddings_add_src1"); - // embeddings = ggml_cont(ctx0, ggml_permute(ctx0, embeddings, 0, 1, 3, 2)); embeddings = - // ggml_add(ctx0, embeddings, ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings)); ggml_add(ctx0, embeddings, ggml_get_rows(ctx0, position_embeddings, positions)); - ggml_set_name(embeddings, "embeddings_to_transformer"); - // print_ggml_tensor(embeddings, true, "embeddings_to_transformer"); // pre-layernorm embeddings = ggml_nn_layer_norm(ctx0, embeddings, pre_ln_w, pre_ln_w); - ggml_set_name(embeddings, "embeddings_after_pre-layernorm"); // transformer for (int i = 0; i < num_hidden_layers; i++) { embeddings = resblocks[i].forward(ctx0, embeddings); // [N, n_token, hidden_size] } - ggml_set_name(embeddings, "embeddings_after_transformer"); - // print_ggml_tensor(embeddings, true, "embeddings_after_transformer"); - // get the output of cls token, e.g., 0th index embeddings = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, embeddings, hidden_size, num_positions * batch_size), cls); - ggml_set_name(embeddings, "embeddings_pooled_after_encoder"); - // print_ggml_tensor(embeddings, true, "embeddings_pooled_after_encoder"); // post-layernorm embeddings = ggml_nn_layer_norm(ctx0, embeddings, post_ln_w, post_ln_b); - ggml_set_name(embeddings, "embeddings_after_post-layernorm"); struct ggml_tensor * cur = embeddings; @@ -1094,17 +1036,6 @@ struct CLIPVisionModel { } } - // if (ggml_backend_is_cpu(backend)) { - // for (int i = 0; i < max_position_embeddings; i++) { - // ggml_set_i32_1d(position_ids, i, i); - // } - // } else { - // std::vector pos_temp; - // for (int i = 0; i < max_position_embeddings; i++) { - // pos_temp.push_back(i); - // } - // ggml_backend_tensor_set(position_ids, pos_temp.data(), 0, ggml_nbytes(position_ids)); - // } } }; @@ -1326,12 +1257,6 @@ struct FrozenCLIPEmbedderWithCustomWords : public GGMLModule { clean_input_ids = clean_input_ids_tmp; } tokens_acc += clean_index; - // class_tokens_mask = [True if class_token_index <= i < class_token_index+num_id_images else False \ - // for i in range(len(clean_input_ids))] - // printf("["); - // for(int i = 0; i < clean_input_ids.size(); ++i) - // printf("%d, ", clean_input_ids[i]); - // printf("]\n"); tokens.insert(tokens.end(), clean_input_ids.begin(), clean_input_ids.end()); weights.insert(weights.end(), clean_input_ids.size(), curr_weight); } @@ -1365,11 +1290,11 @@ struct FrozenCLIPEmbedderWithCustomWords : public GGMLModule { class_token_mask.push_back(false); } - printf("["); - for (int i = 0; i < tokens.size(); i++) { - printf("%d, ", class_token_mask[i] ? 1 : 0); - } - printf("]\n"); + // printf("["); + // for (int i = 0; i < tokens.size(); i++) { + // printf("%d, ", class_token_mask[i] ? 1 : 0); + // } + // printf("]\n"); // for (int i = 0; i < tokens.size(); i++) { // std::cout << tokens[i] << ":" << weights[i] << ", "; diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index bb970118..f2f25853 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -90,6 +90,7 @@ struct SDParams { bool verbose = false; bool vae_tiling = false; bool control_net_cpu = false; + bool normalize_input = false; bool vae_on_cpu = false; bool canny_preprocess = false; }; @@ -107,12 +108,13 @@ void print_params(SDParams params) { printf(" embeddings_path: %s\n", params.embeddings_path.c_str()); printf(" stacked_id_embeddings_path: %s\n", params.stacked_id_embeddings_path.c_str()); printf(" input_id_images_path: %s\n", params.input_id_images_path.c_str()); - printf(" style ratio: %.2f\n", params.style_ratio); + printf(" style ratio: %.2f\n", params.style_ratio); + printf(" normzalize input image : %s\n", params.normalize_input ? "true" : "false"); printf(" output_path: %s\n", params.output_path.c_str()); printf(" init_img: %s\n", params.input_path.c_str()); printf(" control_image: %s\n", params.control_image_path.c_str()); printf(" controlnet cpu: %s\n", params.control_net_cpu ? "true" : "false"); - printf(" vae decoder cpu: %s\n", params.vae_on_cpu ? "true" : "false"); + printf(" vae decoder on cpu:%s\n", params.vae_on_cpu ? "true" : "false"); printf(" strength(control): %.2f\n", params.control_strength); printf(" prompt: %s\n", params.prompt.c_str()); printf(" negative_prompt: %s\n", params.negative_prompt.c_str()); @@ -145,6 +147,7 @@ void print_usage(int argc, const char* argv[]) { printf(" --embd-dir [EMBEDDING_PATH] path to embeddings.\n"); printf(" --stacked-id-embd-dir [DIR] path to PHOTOMAKER stacked id embeddings.\n"); printf(" --input-id-images-dir [DIR] path to PHOTOMAKER input id images dir.\n"); + printf(" --normalize-input normalize PHOTOMAKER input id images\n"); printf(" --upscale-model [ESRGAN_PATH] path to esrgan model. Upscale images after generate, just RealESRGAN_x4plus_anime_6B supported by now.\n"); printf(" --type [TYPE] weight type (f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0)\n"); printf(" If not specified, the default is the type of the weight file.\n"); @@ -367,6 +370,8 @@ void parse_args(int argc, const char** argv, SDParams& params) { params.vae_tiling = true; } else if (arg == "--control-net-cpu") { params.control_net_cpu = true; + } else if (arg == "--normalize-input") { + params.normalize_input = true; } else if (arg == "--vae-on-cpu") { params.vae_on_cpu = true; // will slow down latent decoding but necessary for low MEM GPUs } else if (arg == "--canny") { @@ -679,6 +684,7 @@ int main(int argc, const char* argv[]) { control_image, params.control_strength, params.style_ratio, + params.normalize_input, input_id_images); } else { sd_image_t input_image = {(uint32_t)params.width, diff --git a/ggml_extend.hpp b/ggml_extend.hpp index 6f7f4504..857fb5c8 100644 --- a/ggml_extend.hpp +++ b/ggml_extend.hpp @@ -679,18 +679,6 @@ struct GGMLModule { ggml_allocr_alloc_graph(compute_allocr, gf); - // struct ggml_init_params params = {}; - // params.mem_size += 1024*1024*1024; // 100M - // params.mem_size += 2 * ggml_tensor_overhead(); - // params.mem_buffer = NULL; - // params.no_alloc = false; - // struct ggml_context* obs_ctx = ggml_init(params); - // if (!obs_ctx) { - // LOG_ERROR("ggml_init() failed"); - // return; - // } - - if (ggml_backend_is_cpu(backend)) { ggml_backend_cpu_set_n_threads(backend, n_threads); } @@ -700,227 +688,15 @@ struct GGMLModule { ggml_backend_metal_set_n_cb(backend, n_threads); } #endif - - struct ggml_tensor * imb = NULL; -#if 0 - - for (int i = 0; i < gf->n_leafs; i++) { - struct ggml_tensor * t1 = gf->leafs[i]; - - // if(strcmp(ggml_get_name(t1), "prompt_embeds_input") == 0) { - if(strcmp(ggml_get_name(t1), "id_pixel_values_input") == 0) { - - imb = t1; - int64_t stride = imb->ne[0]; - int64_t ne1 = imb->ne[1]; - int64_t ne2 = imb->ne[2]; - int64_t ne3 = imb->ne[3]; - float* out_data = new float[ggml_nelements(imb)]; - ggml_backend_tensor_get(imb, out_data, 0, ggml_nbytes(imb)); - printf("%s tensor: \n", ggml_get_name(t1)); - for(int l = 0; l < ne3; ++l){ - printf("l = %d: \n", l); - for(int k = 0; k < ne2; ++k){ - // float mi = 100.f, mx= -100.f; - printf("k = %d: \n", k); - for(int i = 0; i < ne1; i++){ - printf("%d: [", i); - for(int j = 0; j < stride; j++){ - float val = out_data[l*ne2*ne1*stride+k*ne1*stride+i*stride+j]; - // if(mi > val) mi = val; - // if(mx < val) mx = val; - printf("%f, ", val); - } - printf("]\n"); - } - // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx); - } - } - - // for(int l = 0; l < ne3; ++l){ - // printf("["); - // for(int k = 0; k < 3; ++k){ - // for(int i = 0; i < stride; i++){ - // for(int j = 0; j < stride; j++){ - // float val = out_data[l*3*stride*stride+ k*stride*stride+i*stride+j]; - // printf("%f, ", val); - // } - // } - // // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx); - // } - // printf("]\n"); - // printf("%s tensor: \n", ggml_get_name(t1)); - // for(int i = 0; i < ne1; i++){ - // printf("%d: [", i); - // for(int j = 0; j < stride; j++){ - // float val = out_data[i*stride+j]; - // // if(mi > val) mi = val; - // // if(mx < val) mx = val; - // printf("%f, ", val); - // } - // printf("]\n"); - // } - - // for(int k = 0; k < 3; ++k){ - // float mi = 100.f, mx= -100.f; - // for(int i = 0; i < stride; i++){ - // printf("["); - // for(int j = 0; j < stride; j++){ - // float val = out_data[k*stride*stride+i*stride+j]; - // if(mi > val) mi = val; - // if(mx < val) mx = val; - // printf("%f, ", val); - // } - // printf("]\n"); - // } - // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx); - // } - - // printf("["); - // for(int i = 0; i < stride; i++){ - // printf("%f, ", out_data[i]); - // } - // printf("]\n"); - delete out_data; - } - } -#endif -#if 0 - - // struct ggml_tensor * imb = NULL; - for (int i = 0; i < gf->n_nodes; i++) { - struct ggml_cgraph g1v = ggml_graph_view(gf, i, i + 1); - ggml_backend_graph_compute(backend, &g1v); - struct ggml_tensor * t1 = gf->nodes[i]; - // if(strcmp(ggml_get_name(t1), "embeddings_after_add") == 0) { - // if(strcmp(ggml_get_name(t1), "stacked_id_embeds_fuse_fn") == 0) { - // if(strcmp(ggml_get_name(t1), "embeddings_after_transformer") == 0) { - if(strcmp(ggml_get_name(t1), "embeddings_pooled_after_encoder") == 0) { - // if(strcmp(ggml_get_name(t1), "shared_id_embeds_from_vision") == 0) { - // if(strcmp(ggml_get_name(t1), "embeddings_to_transformer") == 0) { - // if(strcmp(ggml_get_name(t1), "inp_reshape_3d") == 0) { - // if(strcmp(ggml_get_name(t1), "inp_conv_2d") == 0) { - // if(strcmp(ggml_get_name(t1), "inp_conv_2d") == 0) { - // if(strcmp(ggml_get_name(t1), "patch_embeddings_f16") == 0) { - // if(strcmp(ggml_get_name(t1), "im2col_as_input") == 0) { - // if(strcmp(ggml_get_name(t1), "image_token_embeds") == 0 || - // strcmp(ggml_get_name(t1), "valid_id_embeds") == 0) { - // if(strcmp(ggml_get_name(t1), "id_embeds_proj1") == 0 || - // strcmp(ggml_get_name(t1), "id_embeds_proj2") == 0) { - imb = t1; - int64_t stride = imb->ne[0]; - int64_t ne1 = imb->ne[1]; - int64_t ne2 = imb->ne[2]; - int64_t ne3 = imb->ne[3]; - float* out_data = new float[ggml_nelements(imb)]; - // ggml_fp16_t * out_data = new ggml_fp16_t[ggml_nelements(imb)]; - ggml_backend_tensor_get(imb, out_data, 0, ggml_nbytes(imb)); - printf("%s tensor: \n", ggml_get_name(t1)); - for(int l = 0; l < ne3; ++l){ - printf("l = %d: \n", l); - for(int k = 0; k < ne2; ++k){ - // float mi = 1.e16f, mx= -1e16f; - printf("k = %d: \n", k); - for(int i = 0; i < ne1; i++){ - printf("%d: [", i); - // float mi = 1.e16f, mx= -1e16f; - for(int j = 0; j < stride; j++){ - float val = out_data[l*ne2*ne1*stride+k*ne1*stride+i*stride+j]; - // float val = ggml_fp16_to_fp32(out_data[l*ne2*ne1*stride+k*ne1*stride+i*stride+j]); - // if(mi > val) mi = val; - // if(mx < val) mx = val; - printf("%f, ", val); - } - printf("]\n"); - // printf("i = %d: min, max: %f %f \n", i, mi, mx); - } - // printf("k = %d: min, max: %f %f \n", k, mi, mx); - } - } - - - // for(int k = 0; k < ne2; ++k){ - // // float mi = 100.f, mx= -100.f; - // printf("%d: \n", k); - // for(int i = 0; i < ne1; i++){ - // printf("%d: [", i); - // for(int j = 0; j < stride; j++){ - // float val = out_data[k*ne1*stride+i*stride+j]; - // // if(mi > val) mi = val; - // // if(mx < val) mx = val; - // printf("%f, ", val); - // } - // printf("]\n"); - // } - // // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx); - // } - - // for(int i = 0; i < ne1; i++){ - // printf("%d: [", i); - // for(int j = 0; j < stride; j++){ - // float val = out_data[i*stride+j]; - // // if(mi > val) mi = val; - // // if(mx < val) mx = val; - // printf("%f, ", val); - // } - // printf("]\n"); - // } - // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx); - - // printf("["); - // for(int i = 0; i < stride; i++){ - // printf("%f, ", out_data[i]); - // } - // printf("]\n"); - delete out_data; - } - - } -#endif ggml_backend_graph_compute(backend, gf); #ifdef GGML_PERF ggml_graph_print(gf); #endif - // if (output != NULL) - // print_ggml_tensor(output, true, "output_before_get"); if (output != NULL) { ggml_backend_tensor_get_and_sync(backend, gf->nodes[gf->n_nodes - 1], output->data, 0, ggml_nbytes(output)); } - - // struct ggml_tensor * imb = get_tensor_from_graph(gf, "id_pixel"); - // struct ggml_tensor * imb = get_tensor_from_graph(gf, "id_pixel_values_input"); - // if(imb != NULL){ - // int64_t stride = imb->ne[0]; - // float* out_data = new float[ggml_nelements(imb)]; - // ggml_backend_tensor_get(imb, out_data, 0, ggml_nbytes(imb)); - - // for(int k = 0; k < 3; ++k){ - // float mi = 100.f, mx= -100.f; - // for(int i = 0; i < stride; i++){ - // printf("["); - // for(int j = 0; j < stride; j++){ - // float val = out_data[k*stride*stride+i*stride+j]; - // if(mi > val) mi = val; - // if(mx < val) mx = val; - // printf("%f, ", val); - // } - // printf("]\n"); - // } - // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx); - // } - - // // printf("["); - // // for(int i = 0; i < stride; i++){ - // // printf("%f, ", out_data[i]); - // // } - // // printf("]\n"); - // delete out_data; - // } - - // if (output != NULL) - // print_ggml_tensor(output, true, "output_get"); } void free_compute_buffer() { diff --git a/pmid.hpp b/pmid.hpp index e59d3fcc..c76f0da6 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -148,14 +148,6 @@ struct FuseModule{ struct ggml_tensor* fuse_fn(struct ggml_context* ctx, struct ggml_tensor* prompt_embeds, struct ggml_tensor* id_embeds) { - // x: [N, channels, h, w] - - // stacked_id_embeds = torch.cat([prompt_embeds, id_embeds], dim=-1) - // stacked_id_embeds = self.mlp1(stacked_id_embeds) + prompt_embeds - // stacked_id_embeds = self.mlp2(stacked_id_embeds) - // stacked_id_embeds = self.layer_norm(stacked_id_embeds) - // return stacked_id_embeds - // in_layers auto prompt_embeds0 = ggml_cont(ctx, ggml_permute(ctx, prompt_embeds, 2, 0, 1, 3)); auto id_embeds0 = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 2, 0, 1, 3)); @@ -181,38 +173,11 @@ struct FuseModule{ struct ggml_tensor* right) { // x: [N, channels, h, w] - // in_layers - - // # id_embeds shape: [b, max_num_inputs, 1, 2048] - // id_embeds = id_embeds.to(prompt_embeds.dtype) - // num_inputs = class_tokens_mask.sum().unsqueeze(0) # TODO: check for training case - // batch_size, max_num_inputs = id_embeds.shape[:2] - // # seq_length: 77 - // seq_length = prompt_embeds.shape[1] - // # flat_id_embeds shape: [b*max_num_inputs, 1, 2048] - // flat_id_embeds = id_embeds.view( - // -1, id_embeds.shape[-2], id_embeds.shape[-1] - // ) - // # valid_id_mask [b*max_num_inputs] - // valid_id_mask = ( - // torch.arange(max_num_inputs, device=flat_id_embeds.device)[None, :] - // < num_inputs[:, None] - // ) - // valid_id_embeds = flat_id_embeds[valid_id_mask.flatten()] - - // prompt_embeds = prompt_embeds.view(-1, prompt_embeds.shape[-1]) - // class_tokens_mask = class_tokens_mask.view(-1) struct ggml_tensor * valid_id_embeds = id_embeds; // # slice out the image token embeddings struct ggml_tensor * image_token_embeds = ggml_get_rows(ctx, prompt_embeds, class_tokens_mask_pos); - // print_ggml_tensor(image_token_embeds, true, "image_token_embeds"); - ggml_set_name(image_token_embeds, "image_token_embeds"); - // print_ggml_tensor(valid_id_embeds, true, "valid_id_embeds"); - ggml_set_name(valid_id_embeds, "valid_id_embeds"); struct ggml_tensor *stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds); - // print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_before_concat"); - ggml_set_name(stacked_id_embeds, "stacked_id_embeds_fuse_fn"); stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); if(left && right){ @@ -223,29 +188,11 @@ struct FuseModule{ }else if(right){ stacked_id_embeds = ggml_concat(ctx, stacked_id_embeds, right); } - // print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_concat"); stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); - ggml_set_name(stacked_id_embeds, "stacked_id_embeds_add_zero"); - // print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_permute_2"); - // print_ggml_tensor(class_tokens_mask, true, "class_tokens_mask"); - // print_ggml_tensor(prompt_embeds, true, "prompt_embeds"); - // assert class_tokens_mask.sum() == stacked_id_embeds.shape[0], f"{class_tokens_mask.sum()} != {stacked_id_embeds.shape[0]}" - // prompt_embeds.masked_scatter_(class_tokens_mask[:, None], stacked_id_embeds.to(prompt_embeds.dtype)) - // struct ggml_tensor *prompt_embeds_perm = ggml_cont(ctx, ggml_permute(ctx, prompt_embeds, 1, 0, 2, 3)); - // struct ggml_tensor *prompt_embeds_perm = ggml_cont(ctx, ggml_transpose(ctx, prompt_embeds)); - // print_ggml_tensor(prompt_embeds, true, "prompt_embeds_perm"); - // class_tokens_mask = ggml_repeat(ctx, class_tokens_mask, prompt_embeds_perm); class_tokens_mask = ggml_cont(ctx, ggml_transpose(ctx, class_tokens_mask)); class_tokens_mask = ggml_repeat(ctx, class_tokens_mask, prompt_embeds); - // print_ggml_tensor(class_tokens_mask, true, "class_tokens_mask_repeat"); - // class_tokens_mask = ggml_cont(ctx, ggml_transpose(ctx, class_tokens_mask)); - // print_ggml_tensor(class_tokens_mask, true, "class_tokens_mask_transpose"); prompt_embeds = ggml_mul(ctx, prompt_embeds, class_tokens_mask); - // print_ggml_tensor(prompt_embeds, true, "prompt_embeds_after_mul"); - ggml_set_name(prompt_embeds, "prompt_embeds_mul_cls"); struct ggml_tensor * updated_prompt_embeds = ggml_add(ctx, prompt_embeds, stacked_id_embeds); - // print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds"); - // updated_prompt_embeds = prompt_embeds.view(batch_size, seq_length, -1) return updated_prompt_embeds; } @@ -265,18 +212,12 @@ struct PhotoMakerIDEncoder : public GGMLModule { fuse_module(2048), style_strength(sty){ vision_model = CLIPVisionModel(); - // fuse_module = FuseModule(2048); - // wtype = GGML_TYPE_F32; - } - // void init_params(ggml_context* ctx, ggml_backend_t backend, ggml_type wtype, ggml_allocr* alloc) { void init_params(ggml_type wtype) { - // LOG_INFO(" PMID wtype: %s", ggml_type_name(wtype)); ggml_allocr* alloc = ggml_allocr_new_from_buffer(params_buffer); vision_model.init_params(params_ctx, backend, wtype, alloc); fuse_module.init_params(params_ctx, wtype, alloc); - // visual_projection_2 = ggml_new_tensor_2d(params_ctx, wtype, 1280, 1024); // python [1024, 1280] visual_projection_2 = ggml_new_tensor_2d(params_ctx, wtype, 1024, 1280); // python [1024, 1280] ggml_allocr_alloc(alloc, visual_projection_2); ggml_allocr_free(alloc); @@ -291,7 +232,6 @@ struct PhotoMakerIDEncoder : public GGMLModule { size_t calculate_mem_size() { wtype = GGML_TYPE_F32; - // LOG_INFO(" PMID wtype: %s", ggml_type_name(wtype)); size_t mem_size = vision_model.calculate_mem_size(wtype); mem_size += fuse_module.calculate_mem_size(wtype); @@ -320,67 +260,27 @@ struct PhotoMakerIDEncoder : public GGMLModule { struct ggml_tensor* right) { // x: [N, channels, h, w] - // in_layers - ggml_set_name(id_pixel_values, "id_pixel_values_input"); - ggml_set_name(prompt_embeds, "prompt_embeds_input"); - ggml_set_name(class_tokens_mask, "class_tokens_mask_input"); - ggml_set_name(class_tokens_mask_pos, "class_tokens_mask_pos_input"); - ggml_set_name(cls, "cls_input"); - ggml_set_name(class_embedding_temp, "class_embedding_temp_input"); - ggml_set_name(positions, "positions_input"); - ggml_set_name(left, "left_input"); - ggml_set_name(right, "right_input"); - - - // print_ggml_tensor(prompt_embeds, true, "prompt_embeds"); - // print_ggml_tensor(class_embedding_temp, true, "class_embedding_temp"); struct ggml_tensor *shared_id_embeds = vision_model.forward(ctx, id_pixel_values, cls, class_embedding_temp, positions ); // [batch_size, seq_length, hidden_size] - // print_ggml_tensor(shared_id_embeds, true, "shared_id_embeds_from_vision"); - ggml_set_name(shared_id_embeds, "shared_id_embeds_from_vision"); - // if(class_tokens_mask->backend == GGML_BACKEND_GPU){ - // int *ctm = (int *)malloc(class_tokens_mask->ne[0]); - // ggml_backend_tensor_get(class_tokens_mask, ctm, 0, ggml_nbytes(class_tokens_mask)); - // printf("class_tokens_mask["); - // for(int i = 0; i < class_tokens_mask->ne[0]; i++) - // printf("%d, ", ctm[i]); - // printf("]\n"); - // free(ctm); - // } struct ggml_tensor *id_embeds = vision_model.visual_project(ctx, shared_id_embeds); // [batch_size, seq_length, proj_dim(768)] struct ggml_tensor *id_embeds_2 = ggml_mul_mat(ctx, visual_projection_2, shared_id_embeds); // [batch_size, seq_length, 1280] - // id_embeds = id_embeds.view(b, num_inputs, 1, -1) - // id_embeds_2 = id_embeds_2.view(b, num_inputs, 1, -1) - - // id_embeds = torch.cat((id_embeds, id_embeds_2), dim=-1) - - ggml_set_name(id_embeds, "id_embeds_proj1"); - ggml_set_name(id_embeds_2, "id_embeds_proj2"); id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 2, 0, 1, 3)); id_embeds_2 = ggml_cont(ctx, ggml_permute(ctx, id_embeds_2, 2, 0, 1, 3)); - // print_ggml_tensor(id_embeds, true, "id_embeds_after_perm"); - // print_ggml_tensor(id_embeds_2, true, "id_embeds_2_after_perm"); id_embeds = ggml_concat(ctx, id_embeds, id_embeds_2); // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right - // print_ggml_tensor(id_embeds, true, "id_embeds_after_cat"); - - id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 1, 2, 0, 3)); - // print_ggml_tensor(id_embeds, true, "id_embeds_after_cat+perm"); - struct ggml_tensor * updated_prompt_embeds = fuse_module.forward(ctx, prompt_embeds, id_embeds, class_tokens_mask, class_tokens_mask_pos, left, right); - // print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds_returned"); return updated_prompt_embeds; @@ -429,7 +329,6 @@ struct PhotoMakerIDEncoder : public GGMLModule { ctm.push_back(0.f); // here use 0.f instead of 1.f to make a scale mask ctmf16.push_back(ggml_fp32_to_fp16(0.f)); // here use 0.f instead of 1.f to make a scale mask ctmpos.push_back(i); - // printf("push %d, \n", i); }else{ ctm.push_back(1.f); // here use 1.f instead of 0.f to make a scale mask ctmf16.push_back(ggml_fp32_to_fp16(1.f)); // here use 0.f instead of 1.f to make a scale mask @@ -481,10 +380,6 @@ struct PhotoMakerIDEncoder : public GGMLModule { ggml_backend_tensor_set(cls, cls_h.data(), 0, ggml_nbytes(cls)); ggml_backend_tensor_set(positions, pos.data(), 0, ggml_nbytes(positions)); ggml_backend_tensor_set(class_tokens_mask_pos, ctmpos.data(), 0, ggml_nbytes(class_tokens_mask_pos)); - // std::vector zeros; - // for (int i = 0; i < hidden_size; i++) { - // zeros.push_back(0.f); - // } if(left){ if(type == GGML_TYPE_F16){ std::vector zeros(ggml_nelements(left), ggml_fp32_to_fp16(0.f)); @@ -514,9 +409,7 @@ struct PhotoMakerIDEncoder : public GGMLModule { positions, left, right ); - // print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds_returned_forward"); ggml_build_forward_expand(gf, updated_prompt_embeds); - // ggml_graph_dump_dot(gf, NULL, "id_encoder.dot"); ggml_free(ctx0); return gf; diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index 1f5a9287..b546119c 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -1472,6 +1472,7 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx, const sd_image_t* control_cond, float control_strength, float style_ratio, + bool normalize_input, std::vector &input_id_images) { LOG_DEBUG("txt2img %dx%d", width, height); if (sd_ctx == NULL) { @@ -1545,68 +1546,12 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx, float std[] = {0.26862954, 0.26130258, 0.27577711}; for(int i = 0; i < num_input_images; i++) { sd_image_t* init_image = input_id_images[i]; - // sd_mul_images_to_tensor(init_image->data, init_img, i, mean, std); - sd_mul_images_to_tensor(init_image->data, init_img, i, NULL, NULL); + if(normalize_input) + sd_mul_images_to_tensor(init_image->data, init_img, i, mean, std); + else + sd_mul_images_to_tensor(init_image->data, init_img, i, NULL, NULL); } - // ModelLoader model_loader; - // std::string img_path("examples/scarlett.safetensors"); - // if (!model_loader.init_from_file(img_path, "scarlett.")){ - // LOG_ERROR("init model loader from file failed: '%s'", img_path.c_str()); - // return NULL; - // } - // ggml_backend_t cpu_backend = ggml_backend_cpu_init(); - - // std::map tensors_need_to_load; - // std::set ignore_tensors; - // tensors_need_to_load["scarlett.img"] = init_img; - // bool success = model_loader.load_tensors(tensors_need_to_load, cpu_backend, ignore_tensors); - // if (!success) { - // LOG_ERROR("load tensors from model loader failed"); - // ggml_free(work_ctx); - // return NULL; - // } - - // sd_image_t* cropped_images = (sd_image_t*)calloc(num_input_images, sizeof(sd_image_t)); - // if (cropped_images == NULL) { - // ggml_free(work_ctx); - // return NULL; - // } - // for (size_t i = 0; i < num_input_images; i++) { - // cropped_images[i].width = 224; - // cropped_images[i].height = 224; - // cropped_images[i].channel = 3; - // cropped_images[i].data = sd_tensor_to_mul_image(init_img, i); - // } - // for (int i = 0; i < num_input_images; i++) { - // if (cropped_images[i].data == NULL) { - // continue; - // } - // std::string final_image_path = "cropped_" + std::to_string(i + 1) + ".png"; - // stbi_write_png(final_image_path.c_str(), cropped_images[i].width, cropped_images[i].height, - // cropped_images[i].channel, - // cropped_images[i].data, 0, ""); - // printf("save result image to '%s'\n", final_image_path.c_str()); - // free(cropped_images[i].data); - // cropped_images[i].data = NULL; - // } - // float* out_data = (float *)(init_img->data); - // int64_t stride = init_img->ne[0]; - // for(int k = 0; k < 3; ++k){ - // float mi = 100.f, mx= -100.f; - // for(int i = 0; i < stride; i++){ - // printf("["); - // for(int j = 0; j < stride; j++){ - // float val = out_data[k*stride*stride+i*stride+j]; - // if(mi > val) mi = val; - // if(mx < val) mx = val; - // printf("%f, ", val); - // } - // printf("]\n"); - // } - // printf(" channel, min, max: %d, %f %f \n", k, mi, mx); - // } - auto cond_tup = sd_ctx->sd->get_learned_condition_with_trigger(work_ctx, prompt, clip_skip, width, height, num_input_images ); prompts_embeds = std::get<0>(cond_tup); @@ -1620,7 +1565,7 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx, } // Encode input prompt without the trigger word for delayed conditioning prompt_text_only = sd_ctx->sd->remove_trigger_from_prompt(work_ctx, prompt); - printf("%s || %s \n", prompt.c_str(), prompt_text_only.c_str()); + // printf("%s || %s \n", prompt.c_str(), prompt_text_only.c_str()); prompt = prompt_text_only; // if(sample_steps < 50){ LOG_INFO("sampling steps increases from %d to 50 for PHOTOMAKER", sample_steps); diff --git a/stable-diffusion.h b/stable-diffusion.h index 8a69a6a0..163c1439 100644 --- a/stable-diffusion.h +++ b/stable-diffusion.h @@ -135,6 +135,7 @@ SD_API sd_image_t* txt2img(sd_ctx_t* sd_ctx, const sd_image_t* control_cond, float control_strength, float style_strength, + bool normalize_input, std::vector &input_id_images); SD_API sd_image_t* img2img(sd_ctx_t* sd_ctx,