mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-07-23 11:20:53 -05:00
feat(animatediff): support img2video via --init-img (#1789)
This commit is contained in:
BIN
assets/animatediff/img2video_demo.gif
Normal file
BIN
assets/animatediff/img2video_demo.gif
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.7 MiB |
@@ -134,6 +134,25 @@ sd-cli -M vid_gen --model realisticVisionV60B1.safetensors \
|
||||
-p "close up photo of a rabbit ...<lora:v3_sd15_adapter:1.0>" ...
|
||||
```
|
||||
|
||||
## img2video
|
||||
|
||||
Pass a pre-rendered image via `-i / --init-img` to animate FROM it. All N output frames start from the encoded init latent, then per-frame noise is added at `--strength`. Character identity, composition, and quality are anchored by the init image; the motion module adds subtle motion on top.
|
||||
|
||||
Left: init image rendered with `-M img_gen`. Right: 8-frame vid_gen output.
|
||||
|
||||
<img src="../assets/animatediff/img2video_demo.gif" width="512"/>
|
||||
|
||||
```
|
||||
sd-cli -M img_gen ... -o init.png # any high-quality still
|
||||
sd-cli -M vid_gen --motion-module mm_sd15_v3.safetensors \
|
||||
-i init.png --strength 0.75 \
|
||||
--cfg-scale 7.0 --sampling-method euler --scheduler karras \
|
||||
-H 512 -W 512 --video-frames 8 --steps 25 -s 42 \
|
||||
-p "..." -o out.avi
|
||||
```
|
||||
|
||||
`--strength` controls how far the motion module is allowed to deviate from the init image (higher = more motion, lower = more static).
|
||||
|
||||
## Notes
|
||||
|
||||
- The motion module was trained at `video_length=16`. Running with
|
||||
|
||||
@@ -4687,8 +4687,16 @@ static std::optional<ImageGenerationLatents> prepare_image_generation_latents(sd
|
||||
init_latent.dim() >= 4 && init_latent.shape()[3] == 1) {
|
||||
int n_frames = sd_ctx->sd->animatediff_num_frames;
|
||||
std::vector<int64_t> shape(init_latent.shape().begin(), init_latent.shape().end());
|
||||
shape[3] = n_frames;
|
||||
init_latent = sd::Tensor<float>(std::move(shape)); // zero-filled batch of N frames; per-frame noise is generated later via randn_like.
|
||||
shape[3] = n_frames;
|
||||
if (!init_image_tensor.empty()) {
|
||||
sd::Tensor<float> replicated(shape);
|
||||
for (int f = 0; f < n_frames; ++f) {
|
||||
sd::ops::slice_assign(&replicated, 3, f, f + 1, init_latent);
|
||||
}
|
||||
init_latent = std::move(replicated);
|
||||
} else {
|
||||
init_latent = sd::Tensor<float>(std::move(shape));
|
||||
}
|
||||
}
|
||||
|
||||
if (!control_image_tensor.empty()) {
|
||||
@@ -6132,6 +6140,7 @@ static bool generate_animatediff_video(sd_ctx_t* sd_ctx,
|
||||
img_gen_params.height = sd_vid_gen_params->height;
|
||||
img_gen_params.sample_params = sd_vid_gen_params->sample_params;
|
||||
img_gen_params.strength = sd_vid_gen_params->strength;
|
||||
img_gen_params.init_image = sd_vid_gen_params->init_image;
|
||||
img_gen_params.seed = sd_vid_gen_params->seed;
|
||||
img_gen_params.batch_count = 1;
|
||||
img_gen_params.control_strength = 1.0f;
|
||||
|
||||
Reference in New Issue
Block a user