diff --git a/clip.hpp b/clip.hpp index 156388dd..318cfb5a 100644 --- a/clip.hpp +++ b/clip.hpp @@ -973,15 +973,15 @@ struct CLIPVisionModel { struct ggml_tensor * inp = ggml_conv_2d(ctx0, patch_embeddings_f16, x, patch_size, patch_size, 0, 0, 1, 1); ggml_set_name(inp, "inp_conv_2d"); - // print_ggml_tensor(inp, true, "inp_conv_2d"); + print_ggml_tensor(inp, true, "inp_conv_2d"); inp = ggml_reshape_3d(ctx0, inp, num_patches, hidden_size, batch_size); ggml_set_name(inp, "inp_reshape_3d"); - // print_ggml_tensor(inp, true, "inp_reshape_3d"); + print_ggml_tensor(inp, true, "inp_reshape_3d"); // inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3)); // print_ggml_tensor(ggml_permute(ctx0, inp, 2, 0, 1, 3), true, "inp_permute"); inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3)); - // print_ggml_tensor(inp, true, "inp_cont"); + print_ggml_tensor(inp, true, "inp_cont_perm+cont"); ggml_set_name(inp, "inp_cont"); // ggml_set_name(class_embedding, "class_embedding"); @@ -991,18 +991,18 @@ struct CLIPVisionModel { // struct ggml_tensor * temp = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, 1, batch_size); // ggml_tensor *class_embedding_rep = ggml_add(ctx0, cast_f32_class, class_embedding); // ggml_set_name(class_embedding_rep, "add_class_embedding_to_zero"); - // print_ggml_tensor(class_embedding, true, "model.class_embedding"); + print_ggml_tensor(class_embedding, true, "model.class_embedding"); // print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_bef_repeat"); - // print_ggml_tensor(temp, true, "temp"); + print_ggml_tensor(temp, true, "temp"); ggml_tensor *class_embedding_rep = ggml_repeat(ctx0, class_embedding, temp); ggml_set_name(class_embedding_rep, "class_embedding_rep"); - // print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep"); + print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep"); // class_embedding_rep = ggml_cast(ctx0, class_embedding_rep, inp->type); // print_ggml_tensor(class_embedding_rep, true, "class_embedding_rep_aft_casting"); struct ggml_tensor *embeddings = ggml_concat(ctx0, class_embedding_rep, inp); ggml_set_name(embeddings, "embeddings_after_concat"); - // print_ggml_tensor(embeddings, true, "embeddings_after_concat"); + print_ggml_tensor(embeddings, true, "embeddings_after_concat"); // print_ggml_tensor(ggml_permute(ctx0, embeddings, 0, 3, 1, 2), true, "embeddings_after_concat_permute"); embeddings = ggml_cont(ctx0, ggml_permute(ctx0, embeddings, 0, 2, 1, 3)); ggml_set_name(embeddings, "embeddings_after_permute"); @@ -1019,7 +1019,7 @@ struct CLIPVisionModel { // ggml_add(ctx0, embeddings, ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings)); ggml_add(ctx0, embeddings, ggml_get_rows(ctx0, position_embeddings, positions)); ggml_set_name(embeddings, "embeddings_after_add"); - // print_ggml_tensor(embeddings, true, "embeddings_after_add"); + print_ggml_tensor(embeddings, true, "embeddings_to_transformer"); // pre-layernorm embeddings = ggml_nn_layer_norm(ctx0, embeddings, pre_ln_w, pre_ln_w); @@ -1342,12 +1342,18 @@ struct FrozenCLIPEmbedderWithCustomWords : public GGMLModule { } for(uint32_t i = 0; i < tokens.size(); i++){ - if(class_token_index[0] <= i && i < class_token_index[0]+num_input_imgs) + if(class_token_index[0]+1 <= i && i < class_token_index[0]+1+num_input_imgs) class_token_mask.push_back(true); else class_token_mask.push_back(false); } + printf("["); + for (int i = 0; i < tokens.size(); i++) { + printf("%d, ", class_token_mask[i] ? 1 : 0); + } + printf("]\n"); + // for (int i = 0; i < tokens.size(); i++) { // std::cout << tokens[i] << ":" << weights[i] << ", "; // } diff --git a/pmid.hpp b/pmid.hpp index 8a056f15..58821462 100644 --- a/pmid.hpp +++ b/pmid.hpp @@ -211,10 +211,10 @@ struct FuseModule{ // print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_before_concat"); stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3)); - // print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_permute"); + print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_permute"); if(left && right){ - // print_ggml_tensor(left, true, "left"); - // print_ggml_tensor(right, true, "right"); + print_ggml_tensor(left, true, "left"); + print_ggml_tensor(right, true, "right"); stacked_id_embeds = ggml_concat(ctx, left, stacked_id_embeds); stacked_id_embeds = ggml_concat(ctx, stacked_id_embeds, right); }else if(left){ @@ -474,10 +474,10 @@ struct PhotoMakerIDEncoder : public GGMLModule { ggml_backend_tensor_set(cls, cls_h.data(), 0, ggml_nbytes(cls)); ggml_backend_tensor_set(positions, pos.data(), 0, ggml_nbytes(positions)); ggml_backend_tensor_set(class_tokens_mask_pos, ctmpos.data(), 0, ggml_nbytes(class_tokens_mask_pos)); - std::vector zeros; - for (int i = 0; i < hidden_size; i++) { - zeros.push_back(0.f); - } + // std::vector zeros; + // for (int i = 0; i < hidden_size; i++) { + // zeros.push_back(0.f); + // } if(left){ if(type == GGML_TYPE_F16){ std::vector zeros(ggml_nelements(left), ggml_fp32_to_fp16(0.f)); @@ -752,15 +752,23 @@ struct PhotoMakerLoraModel : public GGMLModule { continue; } - ggml_tensor* lora_up_orig = lora_up; + // print_ggml_tensor(lora_down, true, lora_down_name.c_str()); + // print_ggml_tensor(lora_up, true, lora_up_name.c_str()); + + // ggml_tensor* lora_up_orig = lora_up; applied_lora_tensors.insert(lora_up_name); applied_lora_tensors.insert(lora_down_name); // ggml_mul_mat requires tensor b transposed + // lora_down = ggml_cont(ctx0, ggml_transpose(ctx0, lora_down)); + // struct ggml_tensor* updown = ggml_mul_mat(ctx0, lora_down, lora_up); + // updown = ggml_cont(ctx0, updown); + // same as in lora.hpp lora_down = ggml_cont(ctx0, ggml_transpose(ctx0, lora_down)); - struct ggml_tensor* updown = ggml_mul_mat(ctx0, lora_down, lora_up); - updown = ggml_cont(ctx0, updown); + struct ggml_tensor* updown = ggml_mul_mat(ctx0, lora_up, lora_down); + updown = ggml_cont(ctx0, ggml_transpose(ctx0, updown)); + updown = ggml_reshape(ctx0, updown, weight); GGML_ASSERT(ggml_nelements(updown) == ggml_nelements(weight)); updown = ggml_scale_inplace(ctx0, updown, multiplier); ggml_tensor* final_weight; diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index c999d0b0..4da9fd85 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -1471,8 +1471,8 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx, LOG_INFO("apply_loras completed, taking %.2fs", (t1 - t0) * 1.0f / 1000); if(sd_ctx->sd->stacked_id){ - // sd_ctx->sd->pmid_lora.apply(sd_ctx->sd->tensors, sd_ctx->sd->n_threads); - // sd_ctx->sd->pmid_lora.free_compute_buffer(); + sd_ctx->sd->pmid_lora.apply(sd_ctx->sd->tensors, sd_ctx->sd->n_threads); + sd_ctx->sd->pmid_lora.free_compute_buffer(); if (sd_ctx->sd->free_params_immediately) { sd_ctx->sd->pmid_lora.free_params_buffer(); }