upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-07-29 22:30:43 -05:00
Code Issues Packages Projects Releases Wiki Activity

photomaker working properly now after merging and adapting to GGMLBlock API

Browse Source
This commit is contained in:
bssrdf
2024-02-26 13:32:19 -05:00
parent 4ac3df145e
commit 77b6a42bc3
4 changed files with 64 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
clip.hpp
View File

@@ -548,16 +548,7 @@ public:
auto layer_norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm2"]);
auto mlp = std::dynamic_pointer_cast<CLIPMLP>(blocks["mlp"]);
// struct ggml_tensor* r = x;
struct ggml_tensor* h = layer_norm1->forward(ctx, x);
h = self_attn->forward(ctx, h, mask);
// x = ggml_add(ctx, x, self_attn->forward(ctx, layer_norm1->forward(ctx, x), mask));
x = ggml_add(ctx, x, h);
x = ggml_add(ctx, x, self_attn->forward(ctx, layer_norm1->forward(ctx, x), mask));
x = ggml_add(ctx, x, mlp->forward(ctx, layer_norm2->forward(ctx, x)));
return x;
}
@@ -567,16 +558,10 @@ public:
auto layer_norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks["layer_norm2"]);
auto mlp = std::dynamic_pointer_cast<CLIPMLP>(blocks["mlp"]);
// struct ggml_tensor* r = x;
struct ggml_tensor* h = layer_norm1->forward(ctx, x);
h = self_attn->forward(ctx, h, mask);
// x = ggml_add(ctx, x, self_attn->forward(ctx, layer_norm1->forward(ctx, x), mask));
x = ggml_add(ctx, x, h);
// struct ggml_tensor* h = layer_norm1->forward(ctx, x);
// h = self_attn->forward(ctx, h, mask);
x = ggml_add(ctx, x, self_attn->forward(ctx, layer_norm1->forward(ctx, x), mask));
// x = ggml_add(ctx, x, h);
x = ggml_add(ctx, x, mlp->forward(ctx, layer_norm2->forward(ctx, x)));
return x;
}
@@ -614,10 +599,8 @@ public:
break;
}
std::string name = "layers." + std::to_string(i);
// printf(" about to do %s\n", name.c_str());
auto layer = std::dynamic_pointer_cast<CLIPLayer>(blocks[name]);
x = layer->forward(ctx, x, mask, atten1); // [N, n_token, d_model]
// print_ggml_tensor(x, true, ("layer "+std::to_string(i)).c_str());
// LOG_DEBUG("layer %d", i);
}
return x;
@@ -924,21 +907,17 @@ public:
auto post_layernorm = std::dynamic_pointer_cast<LayerNorm>(blocks["post_layernorm"]);
auto x = embeddings->forward(ctx, pixel_values); // [N, num_positions, embed_dim]
print_ggml_tensor(x, true, "embedding");
x = pre_layernorm->forward(ctx, x);
print_ggml_tensor(x, true, "pre_layernorm");
x = encoder->forward(ctx, x, -1, false, false);
print_ggml_tensor(x, true, "encoder");
x = post_layernorm->forward(ctx, x); // [N, n_token, hidden_size]
print_ggml_tensor(x, true, "post_layernorm");
GGML_ASSERT(x->ne[3] == 1);
// int64_t max_token_idx = 0;
// ggml_tensor* pooled = ggml_view_1d(ctx, x, x->ne[0], x->nb[1] * max_token_idx); // assert N == 1
x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3));
// ggml_view_2d(ctx, w, w->ne[0], w->ne[1] / 2, w->nb[1], 0);
ggml_tensor* pooled = ggml_view_2d(ctx, x, x->ne[0], x->ne[1], x->nb[1], 0);
print_ggml_tensor(pooled, true, "pooled");
x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3));
ggml_tensor* pooled = ggml_view_2d(ctx, x, x->ne[0], x->ne[1], x->nb[0], 0);
pooled = ggml_scale(ctx, ggml_cont(ctx, pooled), 1.f);
// print_ggml_tensor(pooled, true, "pooled");
return pooled; // [N, projection_dim]
}

8
ggml_extend.hpp
View File

@@ -926,11 +926,8 @@ public:
alloc_compute_buffer(get_graph);
reset_compute_ctx();
struct ggml_cgraph* gf = get_graph();
GGML_ASSERT(ggml_gallocr_alloc_graph(compute_allocr, gf));
cpy_data_to_backend_tensor();
if (ggml_backend_is_cpu(backend)) {
ggml_backend_cpu_set_n_threads(backend, n_threads);
}
@@ -1037,16 +1034,11 @@ public:
}
for (auto& pair : blocks) {
auto& block = pair.second;
// if(starts_with(prefix, "pmid"))
// printf("block pair.first: %s, %s \n", prefix.c_str(), pair.first.c_str());
block->get_param_tensors(tensors, prefix + pair.first);
}
for (auto& pair : params) {
struct ggml_tensor* param = pair.second;
// if(starts_with(prefix, "pmid"))
// printf("params pair.first: %s, %s \n", prefix.c_str(), pair.first.c_str());
tensors[prefix + pair.first] = pair.second;
}
}

77
pmid.hpp
View File

@@ -225,7 +225,13 @@ public:
struct ggml_tensor * valid_id_embeds = id_embeds;
// # slice out the image token embeddings
// print_ggml_tensor(class_tokens_mask_pos, false);
ggml_set_name(class_tokens_mask_pos, "class_tokens_mask_pos");
ggml_set_name(prompt_embeds, "prompt_embeds");
// print_ggml_tensor(valid_id_embeds, true, "valid_id_embeds");
// print_ggml_tensor(class_tokens_mask_pos, true, "class_tokens_mask_pos");
struct ggml_tensor * image_token_embeds = ggml_get_rows(ctx, prompt_embeds, class_tokens_mask_pos);
ggml_set_name(image_token_embeds, "image_token_embeds");
struct ggml_tensor *stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds);
stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3));
@@ -242,6 +248,7 @@ public:
class_tokens_mask = ggml_repeat(ctx, class_tokens_mask, prompt_embeds);
prompt_embeds = ggml_mul(ctx, prompt_embeds, class_tokens_mask);
struct ggml_tensor * updated_prompt_embeds = ggml_add(ctx, prompt_embeds, stacked_id_embeds);
ggml_set_name(updated_prompt_embeds, "updated_prompt_embeds");
return updated_prompt_embeds;
}
@@ -289,6 +296,16 @@ public:
VisualProjection visual_projection_2;
float style_strength;
std::vector<float> ctm;
std::vector<ggml_fp16_t> ctmf16;
std::vector<int> ctmpos;
std::vector<ggml_fp16_t> zeros_left_16;
std::vector<float> zeros_left;
std::vector<ggml_fp16_t> zeros_right_16;
std::vector<float> zeros_right;
public:
PhotoMakerIDEncoder(ggml_backend_t backend, ggml_type wtype,
SDVersion version = VERSION_XL, float sty = 20.f)
@@ -382,14 +399,14 @@ public:
// positions
// ); // [batch_size, seq_length, hidden_size]
struct ggml_tensor *shared_id_embeds = vision_model.forward(ctx, id_pixel_values); // [batch_size, seq_length, hidden_size]
print_ggml_tensor(shared_id_embeds, true, "shared_id_embeds");
// print_ggml_tensor(shared_id_embeds, true, "shared_id_embeds");
struct ggml_tensor *id_embeds = vision_model.visual_project(ctx, shared_id_embeds); // [batch_size, seq_length, proj_dim(768)]
print_ggml_tensor(id_embeds, true, "id_embeds");
// print_ggml_tensor(id_embeds, true, "id_embeds");
// struct ggml_tensor *id_embeds_2 = ggml_mul_mat(ctx, visual_projection_2, shared_id_embeds); // [batch_size, seq_length, 1280]
struct ggml_tensor *id_embeds_2 = visual_projection_2.forward(ctx, shared_id_embeds); // [batch_size, seq_length, 1280]
print_ggml_tensor(id_embeds_2, true, "id_embeds_2");
// print_ggml_tensor(id_embeds_2, true, "id_embeds_2");
@@ -399,14 +416,14 @@ public:
id_embeds = ggml_concat(ctx, id_embeds, id_embeds_2); // [batch_size, seq_length, 1, 2048] check whether concat at dim 2 is right
id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 1, 2, 0, 3));
print_ggml_tensor(id_embeds, true, "id_embeds_after_cont+perm");
// print_ggml_tensor(id_embeds, true, "id_embeds_after_cont+perm");
struct ggml_tensor * updated_prompt_embeds = fuse_module.forward(ctx,
prompt_embeds, id_embeds,
class_tokens_mask,
class_tokens_mask_pos,
left, right);
print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds");
// print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds");
return updated_prompt_embeds;
@@ -431,6 +448,15 @@ public:
// struct ggml_cgraph* gf = ggml_new_graph(ctx0);
ctm.clear();
ctmf16.clear();
ctmpos.clear();
zeros_left.clear();
zeros_left_16.clear();
zeros_right.clear();
zeros_right_16.clear();
ggml_context *ctx0 = compute_ctx;
struct ggml_cgraph* gf = ggml_new_graph(compute_ctx);
@@ -450,9 +476,6 @@ public:
struct ggml_tensor* id_pixel_values_d = to_backend(id_pixel_values);
struct ggml_tensor* prompt_embeds_d = to_backend(prompt_embeds);
std::vector<float> ctm;
std::vector<ggml_fp16_t> ctmf16;
std::vector<int> ctmpos;
struct ggml_tensor* left = NULL;
struct ggml_tensor* right = NULL;
for(int i=0; i < class_tokens_mask.size(); i++){
@@ -517,24 +540,32 @@ public:
set_backend_tensor_data(class_tokens_mask_pos, ctmpos.data());
if(left){
if(type == GGML_TYPE_F16){
std::vector<ggml_fp16_t> zeros(ggml_nelements(left), ggml_fp32_to_fp16(0.f));
// std::vector<ggml_fp16_t> zeros(ggml_nelements(left), ggml_fp32_to_fp16(0.f));
for(int i = 0; i < ggml_nelements(left); ++i)
zeros_left_16.push_back(ggml_fp32_to_fp16(0.f));
// ggml_backend_tensor_set(left, zeros.data(), 0, ggml_nbytes(left));
set_backend_tensor_data(left, zeros.data());
set_backend_tensor_data(left, zeros_left_16.data());
}else{
std::vector<float> zeros(ggml_nelements(left), 0.f);
// std::vector<float> zeros(ggml_nelements(left), 0.f);
for(int i = 0; i < ggml_nelements(left); ++i)
zeros_left.push_back(0.f);
// ggml_backend_tensor_set(left, zeros.data(), 0, ggml_nbytes(left));
set_backend_tensor_data(left, zeros.data());
set_backend_tensor_data(left, zeros_left.data());
}
}
if(right){
if(type == GGML_TYPE_F16){
std::vector<ggml_fp16_t> zeros(ggml_nelements(right), ggml_fp32_to_fp16(0.f));
// std::vector<ggml_fp16_t> zeros(ggml_nelements(right), ggml_fp32_to_fp16(0.f));
// ggml_backend_tensor_set(right, zeros.data(), 0, ggml_nbytes(right));
set_backend_tensor_data(right, zeros.data());
for(int i = 0; i < ggml_nelements(right); ++i)
zeros_right_16.push_back(ggml_fp32_to_fp16(0.f));
set_backend_tensor_data(right, zeros_right_16.data());
}else{
std::vector<float> zeros(ggml_nelements(right), 0.f);
// std::vector<float> zeros(ggml_nelements(right), 0.f);
for(int i = 0; i < ggml_nelements(right); ++i)
zeros_right.push_back(0.f);
// ggml_backend_tensor_set(right, zeros.data(), 0, ggml_nbytes(right));
set_backend_tensor_data(right, zeros.data());
set_backend_tensor_data(right, zeros_right.data());
}
}
}
@@ -554,18 +585,6 @@ public:
return gf;
}
void alloc_compute_buffer(ggml_context* work_ctx,
struct ggml_tensor* id_pixel_values,
struct ggml_tensor* prompt_embeds,
std::vector<bool> &class_tokens_mask) {
auto get_graph = [&]() -> struct ggml_cgraph* {
// return build_graph(compute_allocr, id_pixel_values, prompt_embeds, class_tokens_mask);
return build_graph(id_pixel_values, prompt_embeds, class_tokens_mask);
};
GGMLModule::alloc_compute_buffer(get_graph);
}
void compute(const int n_threads,
struct ggml_tensor* id_pixel_values,
struct ggml_tensor* prompt_embeds,
@@ -614,7 +633,7 @@ public:
}
std::string get_desc() {
return "pmid lora";
return "lora_pmid";
}
size_t get_params_num() {

10
stable-diffusion.cpp
View File

@@ -554,6 +554,10 @@ public:
// for(int i = 0; i < tokens.size(); ++i)
// printf("%d ", tokens[i]);
// printf("\n");
// printf("clsm: \n");
// for(int i = 0; i < clsm.size(); ++i)
// printf("%d ", clsm[i]?1:0);
// printf("\n");
int64_t t0 = ggml_time_ms();
struct ggml_tensor* hidden_states = NULL; // [N, n_token, hidden_size]
struct ggml_tensor* pooled = NULL;
@@ -1691,16 +1695,16 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx,
else
sd_mul_images_to_tensor(init_image->data, init_img, i, NULL, NULL);
}
t0 = ggml_time_ms();
auto cond_tup = sd_ctx->sd->get_learned_condition_with_trigger(work_ctx, prompt,
clip_skip, width, height, num_input_images );
LOG_INFO("get_learned_condition_with_trigger finished");
prompts_embeds = std::get<0>(cond_tup);
pooled_prompts_embeds = std::get<1>(cond_tup); // [adm_in_channels, ]
class_tokens_mask = std::get<2>(cond_tup); //
prompts_embeds = sd_ctx->sd->id_encoder(work_ctx, init_img, prompts_embeds, class_tokens_mask);
LOG_INFO("id_encoder");
t1 = ggml_time_ms();
LOG_INFO("Photomaker ID Stacking, taking %" PRId64 " ms", t1 - t0);
if (sd_ctx->sd->free_params_immediately) {
sd_ctx->sd->pmid_model->free_params_buffer();
}