upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-08-03 08:40:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: add IP-Adapter Plus (Resampler image projection) support (#1839)

Browse Source
This commit is contained in:
fszontagh
2026-08-02 10:15:28 +02:00
committed by GitHub
parent 8457624101
commit 50062a4bba
4 changed files with 173 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -70,7 +70,7 @@ API and command-line option may change frequently.***
- [HunyuanVideo 1.5](./docs/hunyuan_video.md)
- [LingBot-Video](./docs/lingbot_video.md)
- [PhotoMaker](./docs/photo_maker.md) support.
- [IP-Adapter](./docs/ip_adapter.md) support (SD 1.5 and SDXL)
- [IP-Adapter](./docs/ip_adapter.md) support (SD 1.5 and SDXL, including Plus)
- Control Net support with SD 1.5
- [ADetailer](./docs/adetailer.md)
- LoRA support, same as [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#lora)

32
docs/ip_adapter.md
View File

@@ -11,6 +11,10 @@ through a decoupled cross-attention added to every attn2 layer of the
UNet. It composes with Control Net, so a reference image (appearance) and
an OpenPose hint (pose) can be combined in a single generation.
Both the classic adapters and the higher-fidelity **Plus** adapters are
supported; see [Plus variants](#plus-variants) below. The variant is
detected from the weight file, so the same options work for both.
## Required weights
1. A base SD 1.5 or SDXL model.
@@ -21,6 +25,11 @@ an OpenPose hint (pose) can be combined in a single generation.
[h94/IP-Adapter](https://huggingface.co/h94/IP-Adapter):
- SD 1.5: `models/ip-adapter_sd15.safetensors`
- SDXL: `sdxl_models/ip-adapter_sdxl_vit-h.safetensors`
- SD 1.5 Plus: `models/ip-adapter-plus_sd15.safetensors`
- SDXL Plus: `sdxl_models/ip-adapter-plus_sdxl_vit-h.safetensors`
The Plus files (`ip-adapter-plus_*`) are used exactly like the classic
ones; see [Plus variants](#plus-variants).
## Options
@@ -45,6 +54,29 @@ sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safet
The SDXL VAE decode at 1024x1024 is memory heavy; add `--vae-tiling` (and
`--offload-to-cpu`) on GPUs with limited VRAM.
## Plus variants
The Plus adapters (`ip-adapter-plus_sd15`, `ip-adapter-plus_sdxl_vit-h`)
replace the small linear image projection with a Resampler (a
Perceiver-style module with learned latent queries). Instead of pooling the
CLIP-Vision output into one vector, the Resampler attends over the full grid
of penultimate CLIP-Vision hidden states and emits more image tokens (16
instead of 4). The result transfers finer detail and layout from the
reference, at a small extra cost in the image-projection step.
No extra flags are needed. The variant is detected from the weight file (the
Resampler's `image_proj.latents` tensor), and every Resampler dimension is
read from the tensor shapes, so the same `--ip-adapter`,
`--ip-adapter-image`, and `--ip-adapter-strength` options apply. Plus
composes with Control Net in the same way as the classic adapters.
```
sd-cli -m ..\models\sd_v1.5.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter-plus_sd15.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 7 --steps 30 --sampling-method dpm++2m --scheduler karras -W 512 -H 512
```
The startup log line `IP-Adapter: 16 image tokens` (versus `4` for the
classic adapters) confirms a Plus file was loaded.
## Combining with Control Net
Add the usual Control Net options to keep the reference appearance while

153
src/model/adapter/ip_adapter.hpp
View File

@@ -31,8 +31,92 @@ namespace IPAdapter {
}
};
struct Resampler : public GGMLBlock {
int64_t dim = 1280;
int64_t depth = 4;
int64_t num_queries = 16;
int64_t embed_dim = 1280;
int64_t output_dim = 2048;
int64_t ff_inner = 5120;
int64_t dim_head = 64;
int64_t heads = 20;
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
params["latents"] = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, dim, num_queries, 1);
}
Resampler() {}
Resampler(int64_t dim, int64_t depth, int64_t num_queries, int64_t embed_dim, int64_t output_dim, int64_t ff_inner)
: dim(dim), depth(depth), num_queries(num_queries), embed_dim(embed_dim), output_dim(output_dim), ff_inner(ff_inner) {
heads = dim / dim_head;
blocks["proj_in"] = std::shared_ptr<GGMLBlock>(new Linear(embed_dim, dim, true));
blocks["proj_out"] = std::shared_ptr<GGMLBlock>(new Linear(dim, output_dim, true));
blocks["norm_out"] = std::shared_ptr<GGMLBlock>(new LayerNorm(output_dim));
for (int64_t i = 0; i < depth; i++) {
std::string p = "layers." + std::to_string(i);
blocks[p + ".0.norm1"] = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));
blocks[p + ".0.norm2"] = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));
blocks[p + ".0.to_q"] = std::shared_ptr<GGMLBlock>(new Linear(dim, dim, false));
blocks[p + ".0.to_kv"] = std::shared_ptr<GGMLBlock>(new Linear(dim, dim * 2, false));
blocks[p + ".0.to_out"] = std::shared_ptr<GGMLBlock>(new Linear(dim, dim, false));
blocks[p + ".1.0"] = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));
blocks[p + ".1.1"] = std::shared_ptr<GGMLBlock>(new Linear(dim, ff_inner, false));
blocks[p + ".1.3"] = std::shared_ptr<GGMLBlock>(new Linear(ff_inner, dim, false));
}
}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* image_embeds) {
int64_t N = image_embeds->ne[2];
auto proj_in = std::dynamic_pointer_cast<Linear>(blocks["proj_in"]);
auto proj_out = std::dynamic_pointer_cast<Linear>(blocks["proj_out"]);
auto norm_out = std::dynamic_pointer_cast<LayerNorm>(blocks["norm_out"]);
ggml_tensor* x = proj_in->forward(ctx, image_embeds);
ggml_tensor* latents = params["latents"];
if (N > 1) {
latents = ggml_repeat(ctx->ggml_ctx, latents, ggml_new_tensor_3d(ctx->ggml_ctx, GGML_TYPE_F32, dim, num_queries, N));
}
for (int64_t i = 0; i < depth; i++) {
std::string p = "layers." + std::to_string(i);
auto norm1 = std::dynamic_pointer_cast<LayerNorm>(blocks[p + ".0.norm1"]);
auto norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks[p + ".0.norm2"]);
auto to_q = std::dynamic_pointer_cast<Linear>(blocks[p + ".0.to_q"]);
auto to_kv = std::dynamic_pointer_cast<Linear>(blocks[p + ".0.to_kv"]);
auto to_out = std::dynamic_pointer_cast<Linear>(blocks[p + ".0.to_out"]);
ggml_tensor* xn = norm1->forward(ctx, x);
ggml_tensor* ln = norm2->forward(ctx, latents);
ggml_tensor* q = to_q->forward(ctx, ln);
ggml_tensor* kv_in = ggml_concat(ctx->ggml_ctx, xn, ln, 1);
ggml_tensor* kv = to_kv->forward(ctx, kv_in);
int64_t L = kv->ne[1];
ggml_tensor* k = ggml_cont(ctx->ggml_ctx, ggml_view_3d(ctx->ggml_ctx, kv, dim, L, N, kv->nb[1], kv->nb[2], 0));
ggml_tensor* v = ggml_cont(ctx->ggml_ctx, ggml_view_3d(ctx->ggml_ctx, kv, dim, L, N, kv->nb[1], kv->nb[2], dim * kv->nb[0]));
ggml_tensor* attn = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, heads, nullptr, false, false);
attn = to_out->forward(ctx, attn);
latents = ggml_add(ctx->ggml_ctx, latents, attn);
auto ff_norm = std::dynamic_pointer_cast<LayerNorm>(blocks[p + ".1.0"]);
auto ff_fc1 = std::dynamic_pointer_cast<Linear>(blocks[p + ".1.1"]);
auto ff_fc2 = std::dynamic_pointer_cast<Linear>(blocks[p + ".1.3"]);
ggml_tensor* h = ff_norm->forward(ctx, latents);
h = ff_fc1->forward(ctx, h);
h = ggml_gelu_erf(ctx->ggml_ctx, h);
h = ff_fc2->forward(ctx, h);
latents = ggml_add(ctx->ggml_ctx, latents, h);
}
latents = proj_out->forward(ctx, latents);
latents = norm_out->forward(ctx, latents);
return latents;
}
};
struct IPAdapterRunner : public GGMLRunner {
ImageProjModel image_proj;
Resampler resampler;
bool is_plus = false;
int64_t num_tokens = 4;
std::string prefix;
@@ -41,21 +125,54 @@ namespace IPAdapter {
const std::string prefix,
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
: GGMLRunner(backend, weight_manager), prefix(prefix) {
int64_t ctx_dim = 768;
int64_t clip_dim = 1024;
int64_t out_dim = 3072;
auto norm_iter = tensor_storage_map.find(prefix + ".image_proj.norm.weight");
if (norm_iter != tensor_storage_map.end()) {
ctx_dim = norm_iter->second.ne[0];
is_plus = tensor_storage_map.find(prefix + ".image_proj.latents") != tensor_storage_map.end();
if (is_plus) {
int64_t dim = 1280;
int64_t num_queries = 16;
int64_t embed_dim = 1280;
int64_t output_dim = 2048;
int64_t ff_inner = 5120;
auto latents_iter = tensor_storage_map.find(prefix + ".image_proj.latents");
if (latents_iter != tensor_storage_map.end()) {
dim = latents_iter->second.ne[0];
num_queries = latents_iter->second.ne[1];
}
auto proj_in_iter = tensor_storage_map.find(prefix + ".image_proj.proj_in.weight");
if (proj_in_iter != tensor_storage_map.end()) {
embed_dim = proj_in_iter->second.ne[0];
}
auto proj_out_iter = tensor_storage_map.find(prefix + ".image_proj.proj_out.weight");
if (proj_out_iter != tensor_storage_map.end()) {
output_dim = proj_out_iter->second.ne[1];
}
auto ff_iter = tensor_storage_map.find(prefix + ".image_proj.layers.0.1.1.weight");
if (ff_iter != tensor_storage_map.end()) {
ff_inner = ff_iter->second.ne[1];
}
int64_t depth = 0;
while (tensor_storage_map.find(prefix + ".image_proj.layers." + std::to_string(depth) + ".0.to_q.weight") != tensor_storage_map.end()) {
depth++;
}
num_tokens = num_queries;
resampler = Resampler(dim, depth, num_queries, embed_dim, output_dim, ff_inner);
resampler.init(params_ctx, tensor_storage_map, prefix + ".image_proj");
} else {
int64_t ctx_dim = 768;
int64_t clip_dim = 1024;
int64_t out_dim = 3072;
auto norm_iter = tensor_storage_map.find(prefix + ".image_proj.norm.weight");
if (norm_iter != tensor_storage_map.end()) {
ctx_dim = norm_iter->second.ne[0];
}
auto proj_iter = tensor_storage_map.find(prefix + ".image_proj.proj.weight");
if (proj_iter != tensor_storage_map.end()) {
clip_dim = proj_iter->second.ne[0];
out_dim = proj_iter->second.ne[1];
}
num_tokens = out_dim / ctx_dim;
image_proj = ImageProjModel(num_tokens, ctx_dim, clip_dim);
image_proj.init(params_ctx, tensor_storage_map, prefix + ".image_proj");
}
auto proj_iter = tensor_storage_map.find(prefix + ".image_proj.proj.weight");
if (proj_iter != tensor_storage_map.end()) {
clip_dim = proj_iter->second.ne[0];
out_dim = proj_iter->second.ne[1];
}
num_tokens = out_dim / ctx_dim;
image_proj = ImageProjModel(num_tokens, ctx_dim, clip_dim);
image_proj.init(params_ctx, tensor_storage_map, prefix + ".image_proj");
}
std::string get_desc() override {
@@ -63,14 +180,18 @@ namespace IPAdapter {
}
void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string = "") {
image_proj.get_param_tensors(tensors, prefix + ".image_proj");
if (is_plus) {
resampler.get_param_tensors(tensors, prefix + ".image_proj");
} else {
image_proj.get_param_tensors(tensors, prefix + ".image_proj");
}
}
ggml_cgraph* build_graph(const sd::Tensor<float>& image_embeds_tensor) {
ggml_cgraph* gf = new_graph_custom(1024);
ggml_tensor* embeds = make_input(image_embeds_tensor);
auto runner_ctx = get_context();
ggml_tensor* out = image_proj.forward(&runner_ctx, embeds);
ggml_tensor* out = is_plus ? resampler.forward(&runner_ctx, embeds) : image_proj.forward(&runner_ctx, embeds);
ggml_build_forward_expand(gf, out);
return gf;
}

4
src/stable-diffusion.cpp
View File

@@ -2126,7 +2126,9 @@ public:
return;
}
auto image_tensor = sd_image_to_tensor(image);
auto embed = get_clip_vision_output(image_tensor, true, -1);
auto embed = ip_adapter->is_plus
? get_clip_vision_output(image_tensor, false, 2)
: get_clip_vision_output(image_tensor, true, -1);
if (embed.empty()) {
return;
}