upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-08-04 09:10:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: add IP-Adapter support for SD 1.5 and SDXL (#1803)

Browse Source
This commit is contained in:
fszontagh
2026-07-24 15:19:44 +02:00
committed by GitHub
parent 5114672c48
commit 8d377073e3
13 changed files with 366 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/model/diffusion/model.hpp
View File

@@ -46,6 +46,8 @@ struct UNetDiffusionExtra {
int num_video_frames = -1;
const std::vector<sd::Tensor<float>>* controls = nullptr;
float control_strength = 0.f;
const sd::Tensor<float>* ip_context = nullptr;
float ip_scale = 1.f;
};
struct SkipLayerDiffusionExtra {

29
src/model/diffusion/unet.hpp
View File

@@ -775,14 +775,17 @@ struct UNetModelRunner : public DiffusionModelRunner {
const sd::Tensor<float>& y_tensor = {},
int num_video_frames = -1,
const std::vector<sd::Tensor<float>>& controls_tensor = {},
float control_strength = 0.f) {
float control_strength = 0.f,
const sd::Tensor<float>& ip_context_tensor = {},
float ip_scale = 1.f) {
ggml_cgraph* gf = new_graph_custom(UNET_GRAPH_SIZE);
ggml_tensor* x = make_input(x_tensor);
ggml_tensor* timesteps = make_input(timesteps_tensor);
ggml_tensor* context = make_optional_input(context_tensor);
ggml_tensor* c_concat = make_optional_input(c_concat_tensor);
ggml_tensor* y = make_optional_input(y_tensor);
ggml_tensor* x = make_input(x_tensor);
ggml_tensor* timesteps = make_input(timesteps_tensor);
ggml_tensor* context = make_optional_input(context_tensor);
ggml_tensor* c_concat = make_optional_input(c_concat_tensor);
ggml_tensor* y = make_optional_input(y_tensor);
ggml_tensor* ip_context = make_optional_input(ip_context_tensor);
std::vector<ggml_tensor*> controls;
controls.reserve(controls_tensor.size());
for (const auto& control_tensor : controls_tensor) {
@@ -793,7 +796,9 @@ struct UNetModelRunner : public DiffusionModelRunner {
num_video_frames = static_cast<int>(x->ne[3]);
}
auto runner_ctx = get_context();
auto runner_ctx = get_context();
runner_ctx.ip_context = ip_context;
runner_ctx.ip_scale = ip_scale;
ggml_tensor* out = unet.forward(&runner_ctx,
x,
@@ -818,14 +823,16 @@ struct UNetModelRunner : public DiffusionModelRunner {
const sd::Tensor<float>& y = {},
int num_video_frames = -1,
const std::vector<sd::Tensor<float>>& controls = {},
float control_strength = 0.f) {
float control_strength = 0.f,
const sd::Tensor<float>& ip_context = {},
float ip_scale = 1.f) {
// x: [N, in_channels, h, w]
// timesteps: [N, ]
// context: [N, max_position, hidden_size]([N, 77, 768]) or [1, max_position, hidden_size]
// c_concat: [N, in_channels, h, w] or [1, in_channels, h, w]
// y: [N, adm_in_channels] or [1, adm_in_channels]
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(x, timesteps, context, c_concat, y, num_video_frames, controls, control_strength);
return build_graph(x, timesteps, context, c_concat, y, num_video_frames, controls, control_strength, ip_context, ip_scale);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
@@ -845,7 +852,9 @@ struct UNetModelRunner : public DiffusionModelRunner {
tensor_or_empty(diffusion_params.y),
extra->num_video_frames,
extra->controls ? *extra->controls : empty_controls,
extra->control_strength);
extra->control_strength,
extra->ip_context ? *extra->ip_context : sd::Tensor<float>{},
extra->ip_scale);
}
void test() {