diff --git a/clip.hpp b/clip.hpp index d06355dc..6de96fcb 100644 --- a/clip.hpp +++ b/clip.hpp @@ -960,7 +960,8 @@ struct CLIPVisionModel { int32_t n_layer = num_hidden_layers; const int d_head = hidden_size / n_head; - ggml_set_name(x, "id_pixel"); + // ggml_set_name(x, "id_pixel"); + print_ggml_tensor(x, true, "id_pixel"); ggml_set_name(temp, "temp_input"); int64_t* ne = patch_embeddings->ne; // struct ggml_tensor *patch_embeddings_f16 = ggml_reshape_3d(ctx0, patch_embeddings, ne[0], ne[1], ne[2]*ne[3]); @@ -980,7 +981,7 @@ struct CLIPVisionModel { inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3)); print_ggml_tensor(inp, true, "inp_cont"); ggml_set_name(inp, "inp_cont"); - ggml_set_name(class_embedding, "class_embedding"); + // ggml_set_name(class_embedding, "class_embedding"); // concat class_embeddings and patch_embeddings // struct ggml_tensor * embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size); diff --git a/ggml_extend.hpp b/ggml_extend.hpp index ab38ba60..1b71882e 100644 --- a/ggml_extend.hpp +++ b/ggml_extend.hpp @@ -77,6 +77,25 @@ __STATIC_INLINE__ ggml_fp16_t ggml_tensor_get_f16(const ggml_tensor* tensor, int return *(ggml_fp16_t*)((char*)(tensor->data) + i * tensor->nb[3] + j * tensor->nb[2] + k * tensor->nb[1] + l * tensor->nb[0]); } +static struct ggml_tensor * get_tensor_from_graph(struct ggml_cgraph * gf, const char *name){ + + struct ggml_tensor * res = NULL; + for(int i = 0; i < gf->n_nodes; i++) { + // printf("%d, %s \n", i, gf->nodes[i]->name); + if(strcmp(ggml_get_name(gf->nodes[i]), name) == 0) { + res = gf->nodes[i]; + break; + } + } + for(int i = 0; i < gf->n_leafs; i++) { + // printf("%d, %s \n", i, gf->leafs[i]->name); + if(strcmp(ggml_get_name(gf->leafs[i]), name) == 0) { + res = gf->leafs[i]; + break; + } + } + return res; +} @@ -669,6 +688,87 @@ struct GGMLModule { ggml_backend_metal_set_n_cb(backend, n_threads); } #endif + +#if 0 + struct ggml_tensor * imb = NULL; + for (int i = 0; i < gf->n_leafs; i++) { + struct ggml_tensor * t1 = gf->leafs[i]; + if(strcmp(ggml_get_name(t1), "id_pixel_values_input") == 0) { + imb = t1; + int64_t stride = imb->ne[0]; + float* out_data = new float[ggml_nelements(imb)]; + ggml_backend_tensor_get(imb, out_data, 0, ggml_nbytes(imb)); + + for(int k = 0; k < 3; ++k){ + float mi = 100.f, mx= -100.f; + for(int i = 0; i < stride; i++){ + printf("["); + for(int j = 0; j < stride; j++){ + float val = out_data[k*stride*stride+i*stride+j]; + if(mi > val) mi = val; + if(mx < val) mx = val; + printf("%f, ", val); + } + printf("]\n"); + } + printf("B. channel, min, max: %d, %f %f \n", k, mi, mx); + } + + // printf("["); + // for(int i = 0; i < stride; i++){ + // printf("%f, ", out_data[i]); + // } + // printf("]\n"); + delete out_data; + } + } + + for (int i = 0; i < gf->n_nodes; i++) { + struct ggml_tensor * t1 = gf->nodes[i]; + // if(strcmp(ggml_get_name(t1), "embeddings_after_add") == 0) { + if(strcmp(ggml_get_name(t1), "pooled_output") == 0) { + imb = t1; + int64_t stride = imb->ne[0]; + float* out_data = new float[ggml_nelements(imb)]; + ggml_backend_tensor_get(imb, out_data, 0, ggml_nbytes(imb)); + + // for(int k = 0; k < 3; ++k){ + // float mi = 100.f, mx= -100.f; + // for(int i = 0; i < stride; i++){ + // printf("["); + // for(int j = 0; j < stride; j++){ + // float val = out_data[k*stride*stride+i*stride+j]; + // // if(mi > val) mi = val; + // // if(mx < val) mx = val; + // printf("%f, ", val); + // } + // printf("]\n"); + // } + // // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx); + // } + // for(int i = 0; i < 257; i++){ + // printf("["); + // for(int j = 0; j < stride; j++){ + // float val = out_data[i*stride+j]; + // // if(mi > val) mi = val; + // // if(mx < val) mx = val; + // printf("%f, ", val); + // } + // printf("]\n"); + // } + // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx); + + printf("["); + for(int i = 0; i < stride; i++){ + printf("%f, ", out_data[i]); + } + printf("]\n"); + delete out_data; + } + struct ggml_cgraph g1v = ggml_graph_view(gf, i, i + 1); + ggml_backend_graph_compute(backend, &g1v); + } +#endif ggml_backend_graph_compute(backend, gf); #ifdef GGML_PERF @@ -679,6 +779,38 @@ struct GGMLModule { if (output != NULL) { ggml_backend_tensor_get_and_sync(backend, gf->nodes[gf->n_nodes - 1], output->data, 0, ggml_nbytes(output)); } + + // struct ggml_tensor * imb = get_tensor_from_graph(gf, "id_pixel"); + // struct ggml_tensor * imb = get_tensor_from_graph(gf, "id_pixel_values_input"); + // if(imb != NULL){ + // int64_t stride = imb->ne[0]; + // float* out_data = new float[ggml_nelements(imb)]; + // ggml_backend_tensor_get(imb, out_data, 0, ggml_nbytes(imb)); + + // for(int k = 0; k < 3; ++k){ + // float mi = 100.f, mx= -100.f; + // for(int i = 0; i < stride; i++){ + // printf("["); + // for(int j = 0; j < stride; j++){ + // float val = out_data[k*stride*stride+i*stride+j]; + // if(mi > val) mi = val; + // if(mx < val) mx = val; + // printf("%f, ", val); + // } + // printf("]\n"); + // } + // printf("B. channel, min, max: %d, %f %f \n", k, mi, mx); + // } + + // // printf("["); + // // for(int i = 0; i < stride; i++){ + // // printf("%f, ", out_data[i]); + // // } + // // printf("]\n"); + // delete out_data; + // } + + // if (output != NULL) // print_ggml_tensor(output, true, "output_get"); } diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index e827b983..2bd797e6 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -526,6 +526,10 @@ public: std::vector& tokens = std::get<0>(tokens_and_weights); std::vector& weights = std::get<1>(tokens_and_weights); std::vector& clsm = std::get<2>(tokens_and_weights); + printf("tokens: \n"); + for(int i = 0; i < tokens.size(); ++i) + printf("%d ", tokens[i]); + printf("\n"); int64_t t0 = ggml_time_ms(); struct ggml_tensor* pooled = NULL; size_t total_hidden_size = cond_stage_model.text_model.hidden_size; @@ -833,8 +837,8 @@ public: diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c, control_net.controls, control_strength, t_emb, c_vector); } else{ - diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c_id, control_net.controls, control_strength, t_emb, c_vec_id); - // diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c_id, control_net.controls, control_strength, t_emb, c_vector); + // diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c_id, control_net.controls, control_strength, t_emb, c_vec_id); + diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c_id, control_net.controls, control_strength, t_emb, c_vector); } }else{ diffusion_model.compute(out_cond, n_threads, noised_input, NULL, c, control_net.controls, control_strength, t_emb, c_vector); @@ -1539,6 +1543,22 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx, // free(cropped_images[i].data); // cropped_images[i].data = NULL; // } + // float* out_data = (float *)(init_img->data); + // int64_t stride = init_img->ne[0]; + // for(int k = 0; k < 3; ++k){ + // float mi = 100.f, mx= -100.f; + // for(int i = 0; i < stride; i++){ + // printf("["); + // for(int j = 0; j < stride; j++){ + // float val = out_data[k*stride*stride+i*stride+j]; + // if(mi > val) mi = val; + // if(mx < val) mx = val; + // printf("%f, ", val); + // } + // printf("]\n"); + // } + // printf(" channel, min, max: %d, %f %f \n", k, mi, mx); + // } auto cond_tup = sd_ctx->sd->get_learned_condition_with_trigger(work_ctx, prompt, clip_skip, width, height, num_input_images ); @@ -1610,7 +1630,7 @@ sd_image_t* txt2img(sd_ctx_t* sd_ctx, int start_merge_step = -1; if(sd_ctx->sd->stacked_id){ - float style_strength_ratio = 20.f; + float style_strength_ratio = 30.f; start_merge_step = int(style_strength_ratio / 100.f * sample_steps); if(start_merge_step > 30) start_merge_step = 30;