upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-07-29 14:20:57 -05:00
Code Issues Packages Projects Releases Wiki Activity

fixed a bug in ggml's conv_2d, and now > 1 input images working

Browse Source
This commit is contained in:
bssrdf
2024-02-18 22:18:53 -05:00
parent 6064b0e28b
commit d38b85d87e
4 changed files with 164 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
clip.hpp
View File

@@ -975,6 +975,10 @@ struct CLIPVisionModel {
// patch_embeddings_f16 = ggml_cast(ctx0, patch_embeddings_f16, GGML_TYPE_F16);
// patch_embeddings_f16 = ggml_reshape_4d(ctx0, patch_embeddings_f16, ne[0], ne[1], ne[2], ne[3]);
struct ggml_tensor *patch_embeddings_f16 = ggml_cast(ctx0, patch_embeddings, GGML_TYPE_F16);
ggml_set_name(patch_embeddings_f16, "patch_embeddings_f16");
print_ggml_tensor(patch_embeddings_f16, true, "patch_embeddings_f16");
struct ggml_tensor * inp = ggml_conv_2d(ctx0, patch_embeddings_f16, x, patch_size, patch_size, 0, 0, 1, 1);
ggml_set_name(inp, "inp_conv_2d");
@@ -1023,7 +1027,7 @@ struct CLIPVisionModel {
embeddings =
// ggml_add(ctx0, embeddings, ggml_repeat(ctx0, ggml_get_rows(ctx0, position_embeddings, positions), embeddings));
ggml_add(ctx0, embeddings, ggml_get_rows(ctx0, position_embeddings, positions));
ggml_set_name(embeddings, "embeddings_after_add");
ggml_set_name(embeddings, "embeddings_to_transformer");
print_ggml_tensor(embeddings, true, "embeddings_to_transformer");
// pre-layernorm
@@ -1035,12 +1039,12 @@ struct CLIPVisionModel {
embeddings = resblocks[i].forward(ctx0, embeddings); // [N, n_token, hidden_size]
}
ggml_set_name(embeddings, "embeddings_after_transformer");
// print_ggml_tensor(embeddings, true, "embeddings_after_encoder");
print_ggml_tensor(embeddings, true, "embeddings_after_transformer");
// get the output of cls token, e.g., 0th index
embeddings = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, embeddings, hidden_size, num_positions * batch_size), cls);
ggml_set_name(embeddings, "embeddings_after_cls_token");
// print_ggml_tensor(embeddings, true, "embeddings_pooled_after_encoder");
ggml_set_name(embeddings, "embeddings_pooled_after_encoder");
print_ggml_tensor(embeddings, true, "embeddings_pooled_after_encoder");
// post-layernorm
embeddings = ggml_nn_layer_norm(ctx0, embeddings, post_ln_w, post_ln_b);

169
ggml_extend.hpp
View File

@@ -701,30 +701,78 @@ struct GGMLModule {
}
#endif
#if 0
struct ggml_tensor * imb = NULL;
#if 0
for (int i = 0; i < gf->n_leafs; i++) {
struct ggml_tensor * t1 = gf->leafs[i];
if(strcmp(ggml_get_name(t1), "vision.patch_embeddings") == 0) {
if(strcmp(ggml_get_name(t1), "class_tokens_mask_pos_input") == 0) {
imb = t1;
int64_t stride = imb->ne[0];
int64_t ne3 = imb->ne[3];
float* out_data = new float[ggml_nelements(imb)];
int* out_data = new int[ggml_nelements(imb)];
ggml_backend_tensor_get(imb, out_data, 0, ggml_nbytes(imb));
for(int l = 0; l < ne3; ++l){
printf("[");
for(int k = 0; k < 3; ++k){
for(int i = 0; i < stride; i++){
for(int j = 0; j < stride; j++){
float val = out_data[l*3*stride*stride+ k*stride*stride+i*stride+j];
printf("%f, ", val);
}
}
// printf("B. channel, min, max: %d, %f %f \n", k, mi, mx);
for(int i = 0; i < stride; i++){
printf("%d, ", out_data[i]);
}
printf("]\n");
delete out_data;
}
// if(strcmp(ggml_get_name(t1), "prompt_embeds_input") == 0) {
if(strcmp(ggml_get_name(t1), "id_pixel_values_input") == 0) {
imb = t1;
int64_t stride = imb->ne[0];
int64_t ne1 = imb->ne[1];
int64_t ne2 = imb->ne[2];
int64_t ne3 = imb->ne[3];
float* out_data = new float[ggml_nelements(imb)];
ggml_backend_tensor_get(imb, out_data, 0, ggml_nbytes(imb));
printf("%s tensor: \n", ggml_get_name(t1));
for(int l = 0; l < ne3; ++l){
printf("l = %d: \n", l);
for(int k = 0; k < ne2; ++k){
// float mi = 100.f, mx= -100.f;
printf("k = %d: \n", k);
for(int i = 0; i < ne1; i++){
printf("%d: [", i);
for(int j = 0; j < stride; j++){
float val = out_data[l*ne2*ne1*stride+k*ne1*stride+i*stride+j];
// if(mi > val) mi = val;
// if(mx < val) mx = val;
printf("%f, ", val);
}
printf("]\n");
}
// printf("B. channel, min, max: %d, %f %f \n", k, mi, mx);
}
}
// for(int l = 0; l < ne3; ++l){
// printf("[");
// for(int k = 0; k < 3; ++k){
// for(int i = 0; i < stride; i++){
// for(int j = 0; j < stride; j++){
// float val = out_data[l*3*stride*stride+ k*stride*stride+i*stride+j];
// printf("%f, ", val);
// }
// }
// // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx);
// }
// printf("]\n");
// printf("%s tensor: \n", ggml_get_name(t1));
// for(int i = 0; i < ne1; i++){
// printf("%d: [", i);
// for(int j = 0; j < stride; j++){
// float val = out_data[i*stride+j];
// // if(mi > val) mi = val;
// // if(mx < val) mx = val;
// printf("%f, ", val);
// }
// printf("]\n");
// }
// for(int k = 0; k < 3; ++k){
// float mi = 100.f, mx= -100.f;
// for(int i = 0; i < stride; i++){
@@ -748,22 +796,67 @@ struct GGMLModule {
delete out_data;
}
}
// #if 0
//#if 0
//#endif
// struct ggml_tensor * imb = NULL;
for (int i = 0; i < gf->n_nodes; i++) {
struct ggml_cgraph g1v = ggml_graph_view(gf, i, i + 1);
ggml_backend_graph_compute(backend, &g1v);
struct ggml_tensor * t1 = gf->nodes[i];
// if(strcmp(ggml_get_name(t1), "embeddings_after_add") == 0) {
if(strcmp(ggml_get_name(t1), "pooled_output") == 0) {
// if(strcmp(ggml_get_name(t1), "stacked_id_embeds_fuse_fn") == 0) {
// if(strcmp(ggml_get_name(t1), "embeddings_after_transformer") == 0) {
if(strcmp(ggml_get_name(t1), "embeddings_pooled_after_encoder") == 0) {
// if(strcmp(ggml_get_name(t1), "shared_id_embeds_from_vision") == 0) {
// if(strcmp(ggml_get_name(t1), "embeddings_to_transformer") == 0) {
// if(strcmp(ggml_get_name(t1), "inp_reshape_3d") == 0) {
// if(strcmp(ggml_get_name(t1), "inp_conv_2d") == 0) {
// if(strcmp(ggml_get_name(t1), "inp_conv_2d") == 0) {
// if(strcmp(ggml_get_name(t1), "patch_embeddings_f16") == 0) {
// if(strcmp(ggml_get_name(t1), "im2col_as_input") == 0) {
// if(strcmp(ggml_get_name(t1), "image_token_embeds") == 0 ||
// strcmp(ggml_get_name(t1), "valid_id_embeds") == 0) {
// if(strcmp(ggml_get_name(t1), "id_embeds_proj1") == 0 ||
// strcmp(ggml_get_name(t1), "id_embeds_proj2") == 0) {
imb = t1;
int64_t stride = imb->ne[0];
int64_t ne1 = imb->ne[1];
int64_t ne2 = imb->ne[2];
int64_t ne3 = imb->ne[3];
float* out_data = new float[ggml_nelements(imb)];
// ggml_fp16_t * out_data = new ggml_fp16_t[ggml_nelements(imb)];
ggml_backend_tensor_get(imb, out_data, 0, ggml_nbytes(imb));
// for(int k = 0; k < 3; ++k){
// float mi = 100.f, mx= -100.f;
// for(int i = 0; i < stride; i++){
// printf("[");
printf("%s tensor: \n", ggml_get_name(t1));
for(int l = 0; l < ne3; ++l){
printf("l = %d: \n", l);
for(int k = 0; k < ne2; ++k){
// float mi = 1.e16f, mx= -1e16f;
printf("k = %d: \n", k);
for(int i = 0; i < ne1; i++){
printf("%d: [", i);
// float mi = 1.e16f, mx= -1e16f;
for(int j = 0; j < stride; j++){
float val = out_data[l*ne2*ne1*stride+k*ne1*stride+i*stride+j];
// float val = ggml_fp16_to_fp32(out_data[l*ne2*ne1*stride+k*ne1*stride+i*stride+j]);
// if(mi > val) mi = val;
// if(mx < val) mx = val;
printf("%f, ", val);
}
printf("]\n");
// printf("i = %d: min, max: %f %f \n", i, mi, mx);
}
// printf("k = %d: min, max: %f %f \n", k, mi, mx);
}
}
// for(int k = 0; k < ne2; ++k){
// // float mi = 100.f, mx= -100.f;
// printf("%d: \n", k);
// for(int i = 0; i < ne1; i++){
// printf("%d: [", i);
// for(int j = 0; j < stride; j++){
// float val = out_data[k*stride*stride+i*stride+j];
// float val = out_data[k*ne1*stride+i*stride+j];
// // if(mi > val) mi = val;
// // if(mx < val) mx = val;
// printf("%f, ", val);
@@ -772,27 +865,27 @@ struct GGMLModule {
// }
// // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx);
// }
// for(int i = 0; i < 257; i++){
// printf("[");
// for(int j = 0; j < stride; j++){
// float val = out_data[i*stride+j];
// // if(mi > val) mi = val;
// // if(mx < val) mx = val;
// printf("%f, ", val);
// }
// printf("]\n");
// }
// for(int i = 0; i < ne1; i++){
// printf("%d: [", i);
// for(int j = 0; j < stride; j++){
// float val = out_data[i*stride+j];
// // if(mi > val) mi = val;
// // if(mx < val) mx = val;
// printf("%f, ", val);
// }
// printf("]\n");
// }
// printf("B. channel, min, max: %d, %f %f \n", k, mi, mx);
printf("[");
for(int i = 0; i < stride; i++){
printf("%f, ", out_data[i]);
}
printf("]\n");
// printf("[");
// for(int i = 0; i < stride; i++){
// printf("%f, ", out_data[i]);
// }
// printf("]\n");
delete out_data;
}
struct ggml_cgraph g1v = ggml_graph_view(gf, i, i + 1);
ggml_backend_graph_compute(backend, &g1v);
}
#endif
ggml_backend_graph_compute(backend, gf);

22
pmid.hpp
View File

@@ -205,10 +205,14 @@ struct FuseModule{
struct ggml_tensor * valid_id_embeds = id_embeds;
// # slice out the image token embeddings
struct ggml_tensor * image_token_embeds = ggml_get_rows(ctx, prompt_embeds, class_tokens_mask_pos);
// print_ggml_tensor(image_token_embeds, true, "image_token_embeds");
// print_ggml_tensor(valid_id_embeds, true, "valid_id_embeds");
print_ggml_tensor(image_token_embeds, true, "image_token_embeds");
ggml_set_name(image_token_embeds, "image_token_embeds");
print_ggml_tensor(valid_id_embeds, true, "valid_id_embeds");
ggml_set_name(valid_id_embeds, "valid_id_embeds");
struct ggml_tensor *stacked_id_embeds = fuse_fn(ctx, image_token_embeds, valid_id_embeds);
// print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_before_concat");
print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_before_concat");
ggml_set_name(stacked_id_embeds, "stacked_id_embeds_fuse_fn");
stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3));
print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_permute");
@@ -224,6 +228,7 @@ struct FuseModule{
}
// print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_concat");
stacked_id_embeds = ggml_cont(ctx, ggml_permute(ctx, stacked_id_embeds, 0, 2, 1, 3));
ggml_set_name(stacked_id_embeds, "stacked_id_embeds_add_zero");
// print_ggml_tensor(stacked_id_embeds, true, "stacked_id_embeds_after_permute_2");
// print_ggml_tensor(class_tokens_mask, true, "class_tokens_mask");
// print_ggml_tensor(prompt_embeds, true, "prompt_embeds");
@@ -240,6 +245,7 @@ struct FuseModule{
// print_ggml_tensor(class_tokens_mask, true, "class_tokens_mask_transpose");
prompt_embeds = ggml_mul(ctx, prompt_embeds, class_tokens_mask);
// print_ggml_tensor(prompt_embeds, true, "prompt_embeds_after_mul");
ggml_set_name(prompt_embeds, "prompt_embeds_mul_cls");
struct ggml_tensor * updated_prompt_embeds = ggml_add(ctx, prompt_embeds, stacked_id_embeds);
// print_ggml_tensor(updated_prompt_embeds, true, "updated_prompt_embeds");
// updated_prompt_embeds = prompt_embeds.view(batch_size, seq_length, -1)
@@ -335,7 +341,8 @@ struct PhotoMakerIDEncoder : public GGMLModule {
class_embedding_temp,
positions
); // [batch_size, seq_length, hidden_size]
// print_ggml_tensor(shared_id_embeds, true, "shared_id_embeds");
print_ggml_tensor(shared_id_embeds, true, "shared_id_embeds_from_vision");
ggml_set_name(shared_id_embeds, "shared_id_embeds_from_vision");
// if(class_tokens_mask->backend == GGML_BACKEND_GPU){
// int *ctm = (int *)malloc(class_tokens_mask->ne[0]);
@@ -352,8 +359,11 @@ struct PhotoMakerIDEncoder : public GGMLModule {
// id_embeds_2 = id_embeds_2.view(b, num_inputs, 1, -1)
// id_embeds = torch.cat((id_embeds, id_embeds_2), dim=-1)
// print_ggml_tensor(id_embeds, true, "id_embeds");
// print_ggml_tensor(id_embeds_2, true, "id_embeds_2");
print_ggml_tensor(id_embeds, true, "id_embeds_proj1");
print_ggml_tensor(id_embeds_2, true, "id_embeds_proj2");
ggml_set_name(id_embeds, "id_embeds_proj1");
ggml_set_name(id_embeds_2, "id_embeds_proj2");
id_embeds = ggml_cont(ctx, ggml_permute(ctx, id_embeds, 2, 0, 1, 3));
id_embeds_2 = ggml_cont(ctx, ggml_permute(ctx, id_embeds_2, 2, 0, 1, 3));

16
stable-diffusion.cpp
View File

@@ -639,13 +639,15 @@ public:
pmid_model.alloc_compute_buffer(work_ctx, init_img, prompts_embeds, class_tokens_mask);
pmid_model.compute(n_threads, init_img, prompts_embeds, class_tokens_mask, res);
pmid_model.free_compute_buffer();
// printf("[");
// int j = 70;
// for(int i = 0; i < 100; i++){
// float val = *((float *)(res->data)+j*total_hidden_size+i);
// printf("%f, ", val);
// }
// printf("]\n");
// for(int j = 0; j < cond_stage_model.text_model.max_position_embeddings; j++){
// printf("[");
// for(int i = 0; i < total_hidden_size; i++){
// float val = *((float *)(res->data)+j*total_hidden_size+i);
// printf("%f, ", val);
// }
// printf("]\n");
// }
return res;
}