mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-07-23 19:30:54 -05:00
Compare commits
5 Commits
master-615
...
master-620
| Author | SHA1 | Date | |
|---|---|---|---|
|
e7eb92fd84 | ||
|
|
50134e51dd | ||
|
|
e43b24cf48 | ||
|
|
06accf2b39 | ||
|
|
cde20d5ef0 |
BIN
assets/ltx2/flf2v.webm
Normal file
BIN
assets/ltx2/flf2v.webm
Normal file
Binary file not shown.
12
docs/ltx2.md
12
docs/ltx2.md
@@ -38,4 +38,16 @@
|
||||
src="../assets/ltx2/i2v.webm"
|
||||
controls
|
||||
muted
|
||||
style="max-width: 100%; height: auto;"></video>
|
||||
|
||||
### LTX-2.3 dev FLF2V
|
||||
|
||||
```
|
||||
.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\..\ComfyUI\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\..\ComfyUI\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\..\ComfyUI\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\..\ComfyUI\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\..\ComfyUI\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors -p "glass flower blossom" --cfg-scale 6.0 --sampling-method euler -v -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 33 --init-img ..\..\ComfyUI\input\start_image.png --end-img ..\..\ComfyUI\input\end_image.png -o flf2v.webm
|
||||
```
|
||||
|
||||
<video
|
||||
src="../assets/ltx2/flf2v.webm"
|
||||
controls
|
||||
muted
|
||||
style="max-width: 100%; height: auto;"></video>
|
||||
@@ -105,7 +105,7 @@ Generation Options:
|
||||
antialiased), or a model name under --hires-upscalers-dir (default: Latent)
|
||||
--extra-sample-args <string> extra sampler/scheduler args, key=value list. lcm supports noise_clip_std,
|
||||
noise_scale_start, noise_scale_end; ltx2 supports max_shift, base_shift,
|
||||
stretch, terminal
|
||||
stretch, terminal; euler_ge supports gamma
|
||||
-H, --height <int> image height, in pixel space (default: 512)
|
||||
-W, --width <int> image width, in pixel space (default: 512)
|
||||
--steps <int> number of sample steps (default: 20)
|
||||
|
||||
@@ -833,7 +833,7 @@ ArgOptions SDGenerationParams::get_options() {
|
||||
&hires_upscaler},
|
||||
{"",
|
||||
"--extra-sample-args",
|
||||
"extra sampler/scheduler args, key=value list. lcm supports noise_clip_std, noise_scale_start, noise_scale_end; ltx2 supports max_shift, base_shift, stretch, terminal",
|
||||
"extra sampler/scheduler args, key=value list. lcm supports noise_clip_std, noise_scale_start, noise_scale_end; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma",
|
||||
&extra_sample_args},
|
||||
};
|
||||
|
||||
|
||||
@@ -207,7 +207,7 @@ Default Generation Options:
|
||||
antialiased), or a model name under --hires-upscalers-dir (default: Latent)
|
||||
--extra-sample-args <string> extra sampler/scheduler args, key=value list. lcm supports noise_clip_std,
|
||||
noise_scale_start, noise_scale_end; ltx2 supports max_shift, base_shift,
|
||||
stretch, terminal
|
||||
stretch, terminal; euler_ge supports gamma
|
||||
-H, --height <int> image height, in pixel space (default: 512)
|
||||
-W, --width <int> image width, in pixel space (default: 512)
|
||||
--steps <int> number of sample steps (default: 20)
|
||||
|
||||
@@ -53,6 +53,7 @@ enum sample_method_t {
|
||||
ER_SDE_SAMPLE_METHOD,
|
||||
EULER_CFG_PP_SAMPLE_METHOD,
|
||||
EULER_A_CFG_PP_SAMPLE_METHOD,
|
||||
EULER_GE_SAMPLE_METHOD,
|
||||
SAMPLE_METHOD_COUNT
|
||||
};
|
||||
|
||||
|
||||
355
src/denoiser.hpp
355
src/denoiser.hpp
@@ -4,11 +4,13 @@
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cmath>
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
#include "ggml_extend.hpp"
|
||||
#include "gits_noise.inl"
|
||||
#include "guidance.h"
|
||||
#include "tensor.hpp"
|
||||
|
||||
/*================================================= CompVisDenoiser ==================================================*/
|
||||
@@ -894,7 +896,7 @@ struct Flux2FlowDenoiser : public FluxFlowDenoiser {
|
||||
}
|
||||
};
|
||||
|
||||
typedef std::function<sd::Tensor<float>(const sd::Tensor<float>&, float, int, sd::Tensor<float>*)> denoise_cb_t;
|
||||
typedef std::function<sd::guidance::GuiderOutput(const sd::Tensor<float>&, float, int)> denoise_cb_t;
|
||||
|
||||
static std::pair<float, float> get_ancestral_step(float sigma_from,
|
||||
float sigma_to,
|
||||
@@ -972,11 +974,11 @@ static sd::Tensor<float> sample_euler_ancestral(denoise_cb_t model,
|
||||
for (int i = 0; i < steps; i++) {
|
||||
float sigma = sigmas[i];
|
||||
float sigma_to = sigmas[i + 1];
|
||||
auto denoised_opt = model(x, sigma, i + 1, nullptr);
|
||||
if (denoised_opt.empty()) {
|
||||
auto denoised_opt = model(x, sigma, i + 1);
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt);
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
if (sigma_to == 0.f) {
|
||||
x = denoised;
|
||||
} else if (eta == 0.f) {
|
||||
@@ -1003,11 +1005,11 @@ static sd::Tensor<float> sample_euler(denoise_cb_t model,
|
||||
int steps = static_cast<int>(sigmas.size()) - 1;
|
||||
for (int i = 0; i < steps; i++) {
|
||||
float sigma = sigmas[i];
|
||||
auto denoised_opt = model(x, sigma, i + 1, nullptr);
|
||||
if (denoised_opt.empty()) {
|
||||
auto denoised_opt = model(x, sigma, i + 1);
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt);
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
sd::Tensor<float> d = (x - denoised) / sigma;
|
||||
x += d * (sigmas[i + 1] - sigma);
|
||||
}
|
||||
@@ -1019,22 +1021,22 @@ static sd::Tensor<float> sample_heun(denoise_cb_t model,
|
||||
const std::vector<float>& sigmas) {
|
||||
int steps = static_cast<int>(sigmas.size()) - 1;
|
||||
for (int i = 0; i < steps; i++) {
|
||||
auto denoised_opt = model(x, sigmas[i], -(i + 1), nullptr);
|
||||
if (denoised_opt.empty()) {
|
||||
auto denoised_opt = model(x, sigmas[i], -(i + 1));
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt);
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
sd::Tensor<float> d = (x - denoised) / sigmas[i];
|
||||
float dt = sigmas[i + 1] - sigmas[i];
|
||||
if (sigmas[i + 1] == 0) {
|
||||
x += d * dt;
|
||||
} else {
|
||||
sd::Tensor<float> x2 = x + d * dt;
|
||||
auto denoised2_opt = model(x2, sigmas[i + 1], i + 1, nullptr);
|
||||
if (denoised2_opt.empty()) {
|
||||
auto denoised2_opt = model(x2, sigmas[i + 1], i + 1);
|
||||
if (denoised2_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised2 = std::move(denoised2_opt);
|
||||
sd::Tensor<float> denoised2 = std::move(denoised2_opt.pred);
|
||||
d = (d + (x2 - denoised2) / sigmas[i + 1]) / 2.0f;
|
||||
x += d * dt;
|
||||
}
|
||||
@@ -1047,11 +1049,11 @@ static sd::Tensor<float> sample_dpm2(denoise_cb_t model,
|
||||
const std::vector<float>& sigmas) {
|
||||
int steps = static_cast<int>(sigmas.size()) - 1;
|
||||
for (int i = 0; i < steps; i++) {
|
||||
auto denoised_opt = model(x, sigmas[i], -(i + 1), nullptr);
|
||||
if (denoised_opt.empty()) {
|
||||
auto denoised_opt = model(x, sigmas[i], -(i + 1));
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt);
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
sd::Tensor<float> d = (x - denoised) / sigmas[i];
|
||||
if (sigmas[i + 1] == 0) {
|
||||
x += d * (sigmas[i + 1] - sigmas[i]);
|
||||
@@ -1060,11 +1062,11 @@ static sd::Tensor<float> sample_dpm2(denoise_cb_t model,
|
||||
float dt_1 = sigma_mid - sigmas[i];
|
||||
float dt_2 = sigmas[i + 1] - sigmas[i];
|
||||
sd::Tensor<float> x2 = x + d * dt_1;
|
||||
auto denoised2_opt = model(x2, sigma_mid, i + 1, nullptr);
|
||||
if (denoised2_opt.empty()) {
|
||||
auto denoised2_opt = model(x2, sigma_mid, i + 1);
|
||||
if (denoised2_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised2 = std::move(denoised2_opt);
|
||||
sd::Tensor<float> denoised2 = std::move(denoised2_opt.pred);
|
||||
x += ((x2 - denoised2) / sigma_mid) * dt_2;
|
||||
}
|
||||
}
|
||||
@@ -1081,11 +1083,11 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral(denoise_cb_t model,
|
||||
|
||||
int steps = static_cast<int>(sigmas.size()) - 1;
|
||||
for (int i = 0; i < steps; i++) {
|
||||
auto denoised_opt = model(x, sigmas[i], -(i + 1), nullptr);
|
||||
if (denoised_opt.empty()) {
|
||||
auto denoised_opt = model(x, sigmas[i], -(i + 1));
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt);
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
auto [sigma_down, sigma_up] = get_ancestral_step(sigmas[i], sigmas[i + 1], eta);
|
||||
|
||||
if (sigma_down == 0) {
|
||||
@@ -1097,11 +1099,11 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral(denoise_cb_t model,
|
||||
float s = t + 0.5f * h;
|
||||
float sigma_s = sigma_fn(s);
|
||||
sd::Tensor<float> x2 = (sigma_s / sigma_fn(t)) * x - (exp(-h * 0.5f) - 1) * denoised;
|
||||
auto denoised2_opt = model(x2, sigma_s, i + 1, nullptr);
|
||||
if (denoised2_opt.empty()) {
|
||||
auto denoised2_opt = model(x2, sigma_s, i + 1);
|
||||
if (denoised2_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised2 = std::move(denoised2_opt);
|
||||
sd::Tensor<float> denoised2 = std::move(denoised2_opt.pred);
|
||||
x = (sigma_fn(t_next) / sigma_fn(t)) * x - (exp(-h) - 1) * denoised2;
|
||||
}
|
||||
|
||||
@@ -1124,11 +1126,11 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral_flow(denoise_cb_t model,
|
||||
|
||||
bool opt_first_step = (1.0 - sigma < 1e-6);
|
||||
|
||||
auto denoised_opt = model(x, sigma, (opt_first_step ? 1 : -1) * (i + 1), nullptr);
|
||||
if (denoised_opt.empty()) {
|
||||
auto denoised_opt = model(x, sigma, (opt_first_step ? 1 : -1) * (i + 1));
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt);
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
|
||||
if (sigma_to == 0.0f) {
|
||||
// Euler method (final step, no noise)
|
||||
@@ -1153,8 +1155,8 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral_flow(denoise_cb_t model,
|
||||
// so sigma_s = 1 = sigma, and sigma_s_i_ratio = sigma_s / sigma = 1
|
||||
// u = (x*sigma_s_i_ratio)+(denoised*(1.0f-sigma_s_i_ratio))
|
||||
// = (x*1)+(denoised*0) = x
|
||||
// so D_i = model(u, sigma_s, i + 1, nullptr)
|
||||
// = model(x, sigma, i + 1, nullptr)
|
||||
// so D_i = model(u, sigma_s, i + 1)
|
||||
// = model(x, sigma, i + 1)
|
||||
// = denoised
|
||||
D_i = denoised;
|
||||
|
||||
@@ -1187,11 +1189,11 @@ static sd::Tensor<float> sample_dpmpp_2s_ancestral_flow(denoise_cb_t model,
|
||||
float sigma_s_i_ratio = sigma_s / sigma;
|
||||
sd::Tensor<float> u = (x * sigma_s_i_ratio) + (denoised * (1.0f - sigma_s_i_ratio));
|
||||
|
||||
auto denoised2_opt = model(u, sigma_s, i + 1, nullptr);
|
||||
if (denoised2_opt.empty()) {
|
||||
auto denoised2_opt = model(u, sigma_s, i + 1);
|
||||
if (denoised2_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
D_i = std::move(denoised2_opt);
|
||||
D_i = std::move(denoised2_opt.pred);
|
||||
}
|
||||
|
||||
float sigma_down_i_ratio = sigma_down / sigma;
|
||||
@@ -1214,11 +1216,11 @@ static sd::Tensor<float> sample_dpmpp_2m(denoise_cb_t model,
|
||||
|
||||
int steps = static_cast<int>(sigmas.size()) - 1;
|
||||
for (int i = 0; i < steps; i++) {
|
||||
auto denoised_opt = model(x, sigmas[i], i + 1, nullptr);
|
||||
if (denoised_opt.empty()) {
|
||||
auto denoised_opt = model(x, sigmas[i], i + 1);
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt);
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
float t = t_fn(sigmas[i]);
|
||||
float t_next = t_fn(sigmas[i + 1]);
|
||||
float h = t_next - t;
|
||||
@@ -1246,11 +1248,11 @@ static sd::Tensor<float> sample_dpmpp_2m_v2(denoise_cb_t model,
|
||||
|
||||
int steps = static_cast<int>(sigmas.size()) - 1;
|
||||
for (int i = 0; i < steps; i++) {
|
||||
auto denoised_opt = model(x, sigmas[i], i + 1, nullptr);
|
||||
if (denoised_opt.empty()) {
|
||||
auto denoised_opt = model(x, sigmas[i], i + 1);
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt);
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
float t = t_fn(sigmas[i]);
|
||||
float t_next = t_fn(sigmas[i + 1]);
|
||||
float h = t_next - t;
|
||||
@@ -1274,91 +1276,61 @@ static sd::Tensor<float> sample_dpmpp_2m_v2(denoise_cb_t model,
|
||||
return x;
|
||||
}
|
||||
|
||||
using SamplerExtraArgs = std::vector<std::pair<std::string, std::string>>;
|
||||
|
||||
static sd::Tensor<float> sample_lcm(denoise_cb_t model,
|
||||
sd::Tensor<float> x,
|
||||
const std::vector<float>& sigmas,
|
||||
std::shared_ptr<RNG> rng,
|
||||
bool is_flow_denoiser,
|
||||
const char* extra_sample_args = nullptr) {
|
||||
const SamplerExtraArgs& extra_sample_args) {
|
||||
struct LCMSampleArgs {
|
||||
float noise_clip_std = 0.0f;
|
||||
float noise_scale_start = 1.0f;
|
||||
float noise_scale_end = 1.0f;
|
||||
};
|
||||
|
||||
auto trim = [](std::string value) -> std::string {
|
||||
const char* whitespace = " \t\r\n";
|
||||
size_t begin = value.find_first_not_of(whitespace);
|
||||
if (begin == std::string::npos) {
|
||||
return "";
|
||||
}
|
||||
size_t end = value.find_last_not_of(whitespace);
|
||||
return value.substr(begin, end - begin + 1);
|
||||
};
|
||||
|
||||
LCMSampleArgs args;
|
||||
if (extra_sample_args != nullptr && extra_sample_args[0] != '\0') {
|
||||
std::string raw(extra_sample_args);
|
||||
size_t start = 0;
|
||||
bool noise_scale_end_was_set = false;
|
||||
bool noise_scale_start_was_set = false;
|
||||
auto parse_arg = [&](const std::string& item) {
|
||||
std::string token = trim(item);
|
||||
if (token.empty()) {
|
||||
return;
|
||||
}
|
||||
size_t eq = token.find('=');
|
||||
if (eq == std::string::npos) {
|
||||
LOG_WARN("ignoring invalid lcm extra sample arg '%s'", token.c_str());
|
||||
return;
|
||||
}
|
||||
bool noise_scale_end_was_set = false;
|
||||
bool noise_scale_start_was_set = false;
|
||||
|
||||
std::string key = trim(token.substr(0, eq));
|
||||
std::string value = trim(token.substr(eq + 1));
|
||||
float parsed = 0.0f;
|
||||
try {
|
||||
size_t consumed = 0;
|
||||
parsed = std::stof(value, &consumed);
|
||||
if (trim(value.substr(consumed)).size() != 0) {
|
||||
LOG_WARN("ignoring invalid lcm extra sample arg '%s'", token.c_str());
|
||||
return;
|
||||
}
|
||||
} catch (const std::exception&) {
|
||||
LOG_WARN("ignoring invalid lcm extra sample arg '%s'", token.c_str());
|
||||
return;
|
||||
}
|
||||
|
||||
if (key == "noise_clip_std") {
|
||||
args.noise_clip_std = parsed;
|
||||
} else if (key == "noise_scale_start") {
|
||||
args.noise_scale_start = parsed;
|
||||
noise_scale_start_was_set = true;
|
||||
} else if (key == "noise_scale_end") {
|
||||
args.noise_scale_end = parsed;
|
||||
noise_scale_end_was_set = true;
|
||||
} else {
|
||||
LOG_WARN("ignoring unknown lcm extra sample arg '%s'", key.c_str());
|
||||
}
|
||||
};
|
||||
|
||||
for (size_t pos = 0; pos <= raw.size(); ++pos) {
|
||||
if (pos == raw.size() || raw[pos] == ',' || raw[pos] == ';') {
|
||||
parse_arg(raw.substr(start, pos - start));
|
||||
start = pos + 1;
|
||||
for (const auto& [key, value] : extra_sample_args) {
|
||||
float parsed = 0.0f;
|
||||
try {
|
||||
size_t consumed = 0;
|
||||
parsed = std::stof(value, &consumed);
|
||||
if (trim(value.substr(consumed)).size() != 0) {
|
||||
LOG_WARN("ignoring invalid lcm extra sample arg '%s'", key.c_str());
|
||||
continue;
|
||||
}
|
||||
} catch (const std::exception&) {
|
||||
LOG_WARN("ignoring invalid lcm extra sample arg '%s=%s'", key.c_str());
|
||||
continue;
|
||||
}
|
||||
if (noise_scale_start_was_set && !noise_scale_end_was_set) {
|
||||
args.noise_scale_end = args.noise_scale_start;
|
||||
if (key == "noise_clip_std") {
|
||||
args.noise_clip_std = parsed;
|
||||
} else if (key == "noise_scale_start") {
|
||||
args.noise_scale_start = parsed;
|
||||
noise_scale_start_was_set = true;
|
||||
} else if (key == "noise_scale_end") {
|
||||
args.noise_scale_end = parsed;
|
||||
noise_scale_end_was_set = true;
|
||||
} else {
|
||||
LOG_WARN("ignoring unknown lcm extra sample arg '%s'", key.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
if (noise_scale_start_was_set && !noise_scale_end_was_set) {
|
||||
args.noise_scale_end = args.noise_scale_start;
|
||||
}
|
||||
|
||||
int steps = static_cast<int>(sigmas.size()) - 1;
|
||||
for (int i = 0; i < steps; i++) {
|
||||
auto denoised_opt = model(x, sigmas[i], i + 1, nullptr);
|
||||
if (denoised_opt.empty()) {
|
||||
auto denoised_opt = model(x, sigmas[i], i + 1);
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
x = std::move(denoised_opt);
|
||||
x = std::move(denoised_opt.pred);
|
||||
if (sigmas[i + 1] > 0) {
|
||||
if (is_flow_denoiser) {
|
||||
x *= (1 - sigmas[i + 1]);
|
||||
@@ -1400,11 +1372,11 @@ static sd::Tensor<float> sample_ipndm(denoise_cb_t model,
|
||||
float sigma = sigmas[i];
|
||||
float sigma_next = sigmas[i + 1];
|
||||
|
||||
auto denoised_opt = model(x, sigma, i + 1, nullptr);
|
||||
if (denoised_opt.empty()) {
|
||||
auto denoised_opt = model(x, sigma, i + 1);
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt);
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
|
||||
sd::Tensor<float> d_cur = (x - denoised) / sigma;
|
||||
int order = std::min(max_order, i + 1);
|
||||
@@ -1444,11 +1416,11 @@ static sd::Tensor<float> sample_ipndm_v(denoise_cb_t model,
|
||||
float sigma = sigmas[i];
|
||||
float t_next = sigmas[i + 1];
|
||||
|
||||
auto denoised_opt = model(x, sigma, i + 1, nullptr);
|
||||
if (denoised_opt.empty()) {
|
||||
auto denoised_opt = model(x, sigma, i + 1);
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt);
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
|
||||
sd::Tensor<float> d_cur = (x - denoised) / sigma;
|
||||
int order = std::min(max_order, i + 1);
|
||||
@@ -1506,11 +1478,11 @@ static sd::Tensor<float> sample_res_multistep(denoise_cb_t model,
|
||||
|
||||
int steps = static_cast<int>(sigmas.size()) - 1;
|
||||
for (int i = 0; i < steps; i++) {
|
||||
auto denoised_opt = model(x, sigmas[i], i + 1, nullptr);
|
||||
if (denoised_opt.empty()) {
|
||||
auto denoised_opt = model(x, sigmas[i], i + 1);
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt);
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
|
||||
float sigma_from = sigmas[i];
|
||||
float sigma_to = sigmas[i + 1];
|
||||
@@ -1583,11 +1555,11 @@ static sd::Tensor<float> sample_res_2s(denoise_cb_t model,
|
||||
float sigma_from = sigmas[i];
|
||||
float sigma_to = sigmas[i + 1];
|
||||
|
||||
auto denoised_opt = model(x, sigma_from, -(i + 1), nullptr);
|
||||
if (denoised_opt.empty()) {
|
||||
auto denoised_opt = model(x, sigma_from, -(i + 1));
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt);
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
|
||||
auto [sigma_down, sigma_up, alpha_scale] = get_ancestral_step(sigma_from, sigma_to, eta, is_flow_denoiser);
|
||||
|
||||
@@ -1609,11 +1581,11 @@ static sd::Tensor<float> sample_res_2s(denoise_cb_t model,
|
||||
sd::Tensor<float> eps1 = denoised - x0;
|
||||
sd::Tensor<float> x2 = x0 + eps1 * (h * a21);
|
||||
|
||||
auto denoised2_opt = model(x2, sigma_c2, i + 1, nullptr);
|
||||
if (denoised2_opt.empty()) {
|
||||
auto denoised2_opt = model(x2, sigma_c2, i + 1);
|
||||
if (denoised2_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised2 = std::move(denoised2_opt);
|
||||
sd::Tensor<float> denoised2 = std::move(denoised2_opt.pred);
|
||||
sd::Tensor<float> eps2 = denoised2 - x0;
|
||||
x = x0 + h * (b1 * eps1 + b2 * eps2);
|
||||
}
|
||||
@@ -1686,10 +1658,11 @@ static sd::Tensor<float> sample_er_sde(denoise_cb_t model,
|
||||
|
||||
int steps = static_cast<int>(sigmas.size()) - 1;
|
||||
for (int i = 0; i < steps; i++) {
|
||||
sd::Tensor<float> denoised = model(x, sigmas[i], i + 1, nullptr);
|
||||
if (denoised.empty()) {
|
||||
auto denoised_opt = model(x, sigmas[i], i + 1);
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
|
||||
int stage_used = std::min(max_stage, i + 1);
|
||||
|
||||
@@ -1804,11 +1777,11 @@ static sd::Tensor<float> sample_tcd(denoise_cb_t model,
|
||||
int timestep_s = (int)floor((1 - eta) * prev_timestep);
|
||||
float sigma = sigmas[i];
|
||||
|
||||
auto denoised_opt = model(x, sigma, i + 1, nullptr);
|
||||
if (denoised_opt.empty()) {
|
||||
auto denoised_opt = model(x, sigma, i + 1);
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt);
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
sd::Tensor<float> d = (x - denoised) / sigma;
|
||||
|
||||
float alpha_prod_t = 1.0f / (sigma * sigma + 1.0f);
|
||||
@@ -1833,16 +1806,15 @@ static sd::Tensor<float> sample_euler_cfg_pp(denoise_cb_t model,
|
||||
const std::vector<float>& sigmas) {
|
||||
int steps = static_cast<int>(sigmas.size()) - 1;
|
||||
for (int i = 0; i < steps; i++) {
|
||||
float sigma = sigmas[i];
|
||||
sd::Tensor<float> uncond_denoised;
|
||||
|
||||
auto denoised_opt = model(x, sigma, i + 1, &uncond_denoised);
|
||||
if (denoised_opt.empty() || uncond_denoised.empty()) {
|
||||
float sigma = sigmas[i];
|
||||
auto denoised_opt = model(x, sigma, i + 1);
|
||||
if (denoised_opt.pred.empty() || denoised_opt.pred_uncond.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt);
|
||||
sd::Tensor<float> d = (x - uncond_denoised) / sigma;
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
sd::Tensor<float> uncond_denoised = std::move(denoised_opt.pred_uncond);
|
||||
sd::Tensor<float> d = (x - uncond_denoised) / sigma;
|
||||
|
||||
x = denoised + d * sigmas[i + 1];
|
||||
}
|
||||
@@ -1856,16 +1828,15 @@ static sd::Tensor<float> sample_euler_ancestral_cfg_pp(denoise_cb_t model,
|
||||
float eta) {
|
||||
int steps = static_cast<int>(sigmas.size()) - 1;
|
||||
for (int i = 0; i < steps; i++) {
|
||||
float sigma = sigmas[i];
|
||||
sd::Tensor<float> uncond_denoised;
|
||||
|
||||
auto denoised_opt = model(x, sigma, i + 1, &uncond_denoised);
|
||||
if (denoised_opt.empty() || uncond_denoised.empty()) {
|
||||
float sigma = sigmas[i];
|
||||
auto denoised_opt = model(x, sigma, i + 1);
|
||||
if (denoised_opt.pred.empty() || denoised_opt.pred_uncond.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt);
|
||||
sd::Tensor<float> d = (x - uncond_denoised) / sigma;
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
sd::Tensor<float> uncond_denoised = std::move(denoised_opt.pred_uncond);
|
||||
sd::Tensor<float> d = (x - uncond_denoised) / sigma;
|
||||
|
||||
auto [sigma_down, sigma_up] = get_ancestral_step(sigmas[i], sigmas[i + 1], eta);
|
||||
|
||||
@@ -1878,6 +1849,113 @@ static sd::Tensor<float> sample_euler_ancestral_cfg_pp(denoise_cb_t model,
|
||||
return x;
|
||||
}
|
||||
|
||||
// https://github.com/ToyotaResearchInstitute/gradient-estimation-sampler
|
||||
static sd::Tensor<float> sample_gradient_estimation(denoise_cb_t model,
|
||||
sd::Tensor<float> x,
|
||||
const std::vector<float>& sigmas,
|
||||
std::shared_ptr<RNG> rng,
|
||||
bool is_flow_denoiser,
|
||||
float eta,
|
||||
const SamplerExtraArgs& extra_sample_args) {
|
||||
float ge_gamma = 2.0f;
|
||||
|
||||
for (const auto& [key, value] : extra_sample_args) {
|
||||
float parsed = 0.0f;
|
||||
try {
|
||||
size_t consumed = 0;
|
||||
parsed = std::stof(value, &consumed);
|
||||
if (trim(value.substr(consumed)).size() != 0) {
|
||||
LOG_WARN("ignoring invalid euler_ge extra sample arg '%s'", key.c_str());
|
||||
continue;
|
||||
}
|
||||
} catch (const std::exception&) {
|
||||
LOG_WARN("ignoring invalid euler_ge extra sample arg '%s'", key.c_str());
|
||||
continue;
|
||||
}
|
||||
if (key == "gamma") {
|
||||
LOG_DEBUG("setting euler_ge gamma to %.2f", parsed);
|
||||
ge_gamma = parsed;
|
||||
} else {
|
||||
LOG_WARN("ignoring unknown euler_ge extra sample arg '%s'", key.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
int steps = static_cast<int>(sigmas.size()) - 1;
|
||||
sd::Tensor<float> old_d;
|
||||
bool has_old_d = false;
|
||||
for (int i = 0; i < steps; i++) {
|
||||
float sigma = sigmas[i];
|
||||
float sigma_to = sigmas[i + 1];
|
||||
auto denoised_opt = model(x, sigma, i + 1);
|
||||
if (denoised_opt.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
sd::Tensor<float> denoised = std::move(denoised_opt.pred);
|
||||
if (sigma_to == 0.f) {
|
||||
x = denoised;
|
||||
} else {
|
||||
auto [sigma_down, sigma_up, alpha_scale] = get_ancestral_step(sigma, sigma_to, eta, is_flow_denoiser);
|
||||
sd::Tensor<float> d = (x - denoised) / sigma;
|
||||
float dt = sigma_down - sigma;
|
||||
if (has_old_d) {
|
||||
sd::Tensor<float> d_bar = d * ge_gamma + old_d * (1.0f - ge_gamma);
|
||||
x += d_bar * dt;
|
||||
} else {
|
||||
x += d * dt;
|
||||
}
|
||||
old_d = std::move(d);
|
||||
has_old_d = true;
|
||||
if (sigma_up > 0.f) {
|
||||
if (is_flow_denoiser) {
|
||||
x *= alpha_scale;
|
||||
}
|
||||
x += sd::Tensor<float>::randn_like(x, rng) * sigma_up;
|
||||
}
|
||||
}
|
||||
}
|
||||
return x;
|
||||
}
|
||||
|
||||
static SamplerExtraArgs parse_sampler_args(const char* extra_sample_args) {
|
||||
SamplerExtraArgs pairs;
|
||||
|
||||
if (extra_sample_args == nullptr || extra_sample_args[0] == '\0') {
|
||||
return pairs;
|
||||
}
|
||||
|
||||
auto trim = [](std::string value) -> std::string {
|
||||
const char* whitespace = " \t\r\n";
|
||||
size_t begin = value.find_first_not_of(whitespace);
|
||||
if (begin == std::string::npos) {
|
||||
return "";
|
||||
}
|
||||
size_t end = value.find_last_not_of(whitespace);
|
||||
return value.substr(begin, end - begin + 1);
|
||||
};
|
||||
|
||||
std::string raw(extra_sample_args);
|
||||
size_t start = 0;
|
||||
|
||||
for (size_t pos = 0; pos <= raw.size(); ++pos) {
|
||||
if (pos == raw.size() || raw[pos] == ',' || raw[pos] == ';') {
|
||||
std::string item = raw.substr(start, pos - start);
|
||||
std::string token = trim(item);
|
||||
|
||||
if (!token.empty()) {
|
||||
size_t eq = token.find('=');
|
||||
if (eq != std::string::npos) {
|
||||
std::string key = trim(token.substr(0, eq));
|
||||
std::string value = trim(token.substr(eq + 1));
|
||||
pairs.emplace_back(std::move(key), std::move(value));
|
||||
}
|
||||
}
|
||||
start = pos + 1;
|
||||
}
|
||||
}
|
||||
|
||||
return pairs;
|
||||
}
|
||||
|
||||
// k diffusion reverse ODE: dx = (x - D(x;\sigma)) / \sigma dt; \sigma(t) = t
|
||||
static sd::Tensor<float> sample_k_diffusion(sample_method_t method,
|
||||
denoise_cb_t model,
|
||||
@@ -1887,6 +1965,7 @@ static sd::Tensor<float> sample_k_diffusion(sample_method_t method,
|
||||
float eta,
|
||||
bool is_flow_denoiser,
|
||||
const char* extra_sample_args) {
|
||||
SamplerExtraArgs extra_args = parse_sampler_args(extra_sample_args);
|
||||
switch (method) {
|
||||
case EULER_A_SAMPLE_METHOD:
|
||||
return sample_euler_ancestral(model, std::move(x), sigmas, rng, is_flow_denoiser, eta);
|
||||
@@ -1906,7 +1985,7 @@ static sd::Tensor<float> sample_k_diffusion(sample_method_t method,
|
||||
case DPMPP2Mv2_SAMPLE_METHOD:
|
||||
return sample_dpmpp_2m_v2(model, std::move(x), sigmas);
|
||||
case LCM_SAMPLE_METHOD:
|
||||
return sample_lcm(model, std::move(x), sigmas, rng, is_flow_denoiser, extra_sample_args);
|
||||
return sample_lcm(model, std::move(x), sigmas, rng, is_flow_denoiser, extra_args);
|
||||
case IPNDM_SAMPLE_METHOD:
|
||||
return sample_ipndm(model, std::move(x), sigmas);
|
||||
case IPNDM_V_SAMPLE_METHOD:
|
||||
@@ -1926,6 +2005,8 @@ static sd::Tensor<float> sample_k_diffusion(sample_method_t method,
|
||||
return sample_euler_cfg_pp(model, std::move(x), sigmas);
|
||||
case EULER_A_CFG_PP_SAMPLE_METHOD:
|
||||
return sample_euler_ancestral_cfg_pp(model, std::move(x), sigmas, rng, eta);
|
||||
case EULER_GE_SAMPLE_METHOD:
|
||||
return sample_gradient_estimation(model, std::move(x), sigmas, rng, is_flow_denoiser, eta, extra_args);
|
||||
default:
|
||||
return {};
|
||||
}
|
||||
|
||||
@@ -40,6 +40,7 @@ struct DiffusionParams {
|
||||
float vace_strength = 1.f;
|
||||
int audio_length = 0;
|
||||
float frame_rate = 24.f;
|
||||
const sd::Tensor<float>* video_positions = nullptr;
|
||||
const std::vector<int>* skip_layers = nullptr;
|
||||
};
|
||||
|
||||
@@ -766,7 +767,8 @@ struct LTXAVModel : public DiffusionModel {
|
||||
tensor_or_empty(diffusion_params.audio_x),
|
||||
tensor_or_empty(diffusion_params.audio_timesteps),
|
||||
diffusion_params.audio_length,
|
||||
diffusion_params.frame_rate);
|
||||
diffusion_params.frame_rate,
|
||||
tensor_or_empty(diffusion_params.video_positions));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
89
src/guidance.cpp
Normal file
89
src/guidance.cpp
Normal file
@@ -0,0 +1,89 @@
|
||||
#include "guidance.h"
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace sd::guidance {
|
||||
|
||||
static bool has_tensor(const sd::Tensor<float>* tensor) {
|
||||
return tensor != nullptr && !tensor->empty();
|
||||
}
|
||||
|
||||
ClassifierFreeGuidance::ClassifierFreeGuidance(float guidance_scale,
|
||||
float image_guidance_scale)
|
||||
: guidance_scale_(guidance_scale),
|
||||
image_guidance_scale_(image_guidance_scale) {
|
||||
}
|
||||
|
||||
GuiderOutput ClassifierFreeGuidance::forward(const GuidanceInput& input,
|
||||
GuiderOutput previous) const {
|
||||
(void)previous;
|
||||
|
||||
GuiderOutput output;
|
||||
if (!has_tensor(input.pred_cond)) {
|
||||
return output;
|
||||
}
|
||||
|
||||
const sd::Tensor<float>& pred_cond = *input.pred_cond;
|
||||
output.pred = pred_cond;
|
||||
if (has_tensor(input.pred_uncond)) {
|
||||
const sd::Tensor<float>& pred_uncond = *input.pred_uncond;
|
||||
if (has_tensor(input.pred_img_cond)) {
|
||||
const sd::Tensor<float>& pred_img_cond = *input.pred_img_cond;
|
||||
output.pred = pred_uncond +
|
||||
image_guidance_scale_ * (pred_img_cond - pred_uncond) +
|
||||
guidance_scale_ * (pred_cond - pred_img_cond);
|
||||
} else {
|
||||
output.pred = pred_uncond + guidance_scale_ * (pred_cond - pred_uncond);
|
||||
}
|
||||
} else if (has_tensor(input.pred_img_cond)) {
|
||||
const sd::Tensor<float>& pred_img_cond = *input.pred_img_cond;
|
||||
output.pred = pred_img_cond + guidance_scale_ * (pred_cond - pred_img_cond);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
SkipLayerGuidance::SkipLayerGuidance(std::vector<int> layers,
|
||||
float scale,
|
||||
float start,
|
||||
float stop)
|
||||
: layers_(std::move(layers)),
|
||||
scale_(scale),
|
||||
start_(start),
|
||||
stop_(stop) {
|
||||
}
|
||||
|
||||
bool SkipLayerGuidance::is_enabled_for_step(const GuidanceInput& input) const {
|
||||
if (scale_ == 0.0f || layers_.empty() || input.schedule_size == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
int start_step = static_cast<int>(start_ * static_cast<float>(input.schedule_size));
|
||||
int stop_step = static_cast<int>(stop_ * static_cast<float>(input.schedule_size));
|
||||
return input.step > start_step && input.step < stop_step;
|
||||
}
|
||||
|
||||
const std::vector<int>& SkipLayerGuidance::layers() const {
|
||||
return layers_;
|
||||
}
|
||||
|
||||
GuiderOutput SkipLayerGuidance::forward(const GuidanceInput& input,
|
||||
GuiderOutput output) const {
|
||||
if (!is_enabled_for_step(input) || !input.predict_skip_layer) {
|
||||
return output;
|
||||
}
|
||||
|
||||
if (output.pred.empty() || !has_tensor(input.pred_cond)) {
|
||||
return GuiderOutput();
|
||||
}
|
||||
|
||||
output.pred_skip_layer = input.predict_skip_layer();
|
||||
if (output.pred_skip_layer.empty()) {
|
||||
return GuiderOutput();
|
||||
}
|
||||
|
||||
output.pred += (*input.pred_cond - output.pred_skip_layer) * scale_;
|
||||
return output;
|
||||
}
|
||||
|
||||
} // namespace sd::guidance
|
||||
70
src/guidance.h
Normal file
70
src/guidance.h
Normal file
@@ -0,0 +1,70 @@
|
||||
#ifndef __SD_GUIDANCE_H__
|
||||
#define __SD_GUIDANCE_H__
|
||||
|
||||
#include <cstddef>
|
||||
#include <functional>
|
||||
#include <vector>
|
||||
|
||||
#include "tensor.hpp"
|
||||
|
||||
namespace sd::guidance {
|
||||
|
||||
struct GuiderOutput {
|
||||
sd::Tensor<float> pred;
|
||||
sd::Tensor<float> pred_cond;
|
||||
sd::Tensor<float> pred_uncond;
|
||||
sd::Tensor<float> pred_img_cond;
|
||||
sd::Tensor<float> pred_skip_layer;
|
||||
};
|
||||
|
||||
struct GuidanceInput {
|
||||
int step = 0;
|
||||
size_t schedule_size = 0;
|
||||
const sd::Tensor<float>* pred_cond = nullptr;
|
||||
const sd::Tensor<float>* pred_uncond = nullptr;
|
||||
const sd::Tensor<float>* pred_img_cond = nullptr;
|
||||
|
||||
std::function<sd::Tensor<float>()> predict_skip_layer;
|
||||
};
|
||||
|
||||
class BaseGuidance {
|
||||
public:
|
||||
virtual ~BaseGuidance() = default;
|
||||
virtual GuiderOutput forward(const GuidanceInput& input,
|
||||
GuiderOutput previous) const = 0;
|
||||
};
|
||||
|
||||
class ClassifierFreeGuidance : public BaseGuidance {
|
||||
float guidance_scale_ = 1.0f;
|
||||
float image_guidance_scale_ = 1.0f;
|
||||
|
||||
public:
|
||||
ClassifierFreeGuidance(float guidance_scale,
|
||||
float image_guidance_scale);
|
||||
|
||||
GuiderOutput forward(const GuidanceInput& input,
|
||||
GuiderOutput previous) const override;
|
||||
};
|
||||
|
||||
class SkipLayerGuidance : public BaseGuidance {
|
||||
std::vector<int> layers_;
|
||||
float scale_ = 0.0f;
|
||||
float start_ = 0.0f;
|
||||
float stop_ = 1.0f;
|
||||
|
||||
public:
|
||||
SkipLayerGuidance(std::vector<int> layers,
|
||||
float scale,
|
||||
float start,
|
||||
float stop);
|
||||
|
||||
bool is_enabled_for_step(const GuidanceInput& input) const;
|
||||
const std::vector<int>& layers() const;
|
||||
|
||||
GuiderOutput forward(const GuidanceInput& input,
|
||||
GuiderOutput previous) const override;
|
||||
};
|
||||
|
||||
} // namespace sd::guidance
|
||||
|
||||
#endif // __SD_GUIDANCE_H__
|
||||
@@ -4,6 +4,138 @@
|
||||
#include "ggml.h"
|
||||
#include "tensor.hpp"
|
||||
|
||||
const float ltxav_latent_rgb_proj[128][3] = {
|
||||
{-0.0293802f, -0.0362516f, -0.0291386f},
|
||||
{0.0117735f, 0.0223435f, 0.018856f},
|
||||
{0.00922335f, 0.0145666f, 0.0038772f},
|
||||
{0.0227299f, 0.0109122f, 0.0131384f},
|
||||
{0.00192413f, 0.0024648f, 0.00689245f},
|
||||
{-0.0105576f, -0.0135933f, -0.00873841f},
|
||||
{-0.0310222f, -0.0396358f, -0.0408445f},
|
||||
{0.0149737f, 0.0316323f, 0.03415f},
|
||||
{0.0027752f, 0.00814889f, 0.0108575f},
|
||||
{-0.000678017f, -0.00180589f, -0.0161684f},
|
||||
{0.0153964f, 0.0159774f, 0.0186479f},
|
||||
{-0.0222799f, -0.0202068f, -0.0181082f},
|
||||
{0.0128696f, 0.00754416f, -0.00673279f},
|
||||
{0.0142729f, 0.00448099f, -0.00193934f},
|
||||
{-0.014066f, -0.0193755f, -0.0160104f},
|
||||
{-0.0176785f, -0.015903f, -0.0152621f},
|
||||
{0.0307381f, 0.0292082f, 0.0328668f},
|
||||
{0.0332928f, 0.0368629f, 0.0440893f},
|
||||
{0.0186304f, 0.0124069f, 0.0160734f},
|
||||
{0.00477787f, -0.00315658f, -0.000145702f},
|
||||
{0.0183099f, 0.0122593f, 0.00599732f},
|
||||
{-0.0194551f, -0.0183924f, -0.0147465f},
|
||||
{0.0025732f, 0.00442582f, 0.0173176f},
|
||||
{-0.0169423f, -0.0293863f, -0.0225908f},
|
||||
{-0.021228f, -0.0265094f, -0.0253049f},
|
||||
{0.0327111f, 0.0187133f, 0.0266184f},
|
||||
{-0.0226425f, -0.0313781f, -0.0414356f},
|
||||
{-0.0163142f, -0.0146144f, -0.0171793f},
|
||||
{0.0192183f, 0.0108411f, 0.00829186f},
|
||||
{-0.032246f, -0.0274846f, -0.0287434f},
|
||||
{0.00345399f, 0.0115567f, 0.015288f},
|
||||
{0.000972292f, 0.00331303f, 0.0110501f},
|
||||
{0.000939494f, -0.00705084f, -0.00979449f},
|
||||
{0.0405155f, 0.0339534f, 0.0419513f},
|
||||
{0.0198596f, 0.0186626f, 0.0213766f},
|
||||
{-0.00982375f, -0.00880439f, -0.00470429f},
|
||||
{-0.0313707f, -0.0258098f, -0.0211663f},
|
||||
{0.0144159f, 0.0117896f, 0.0141573f},
|
||||
{0.0164571f, 0.0149178f, 0.00921599f},
|
||||
{0.0436184f, 0.0346583f, 0.0360647f},
|
||||
{-0.00289744f, -0.000752502f, 0.000675415f},
|
||||
{-0.00621715f, -0.000558851f, 0.0135814f},
|
||||
{-0.00817579f, -0.0113584f, -0.00556793f},
|
||||
{0.00965067f, 0.0178221f, 0.015821f},
|
||||
{0.0211832f, 0.0180827f, 0.0154707f},
|
||||
{-0.00412858f, -0.00374182f, 0.0029568f},
|
||||
{-0.0175603f, -0.0226242f, -0.0279012f},
|
||||
{-0.00437471f, -0.00668329f, 0.000164887f},
|
||||
{-0.0355983f, -0.0419093f, -0.0383065f},
|
||||
{0.0144314f, 0.0192514f, 0.0175639f},
|
||||
{-0.0130693f, -0.00569884f, -0.00341647f},
|
||||
{-0.00184689f, 0.00189034f, -0.00190561f},
|
||||
{0.019457f, 0.00842282f, 0.0123738f},
|
||||
{-0.00477146f, -0.00206932f, 0.00283336f},
|
||||
{-0.0364544f, -0.0256141f, -0.0322336f},
|
||||
{-0.0295634f, -0.0295048f, -0.021057f},
|
||||
{0.0144484f, 0.0191862f, 0.0112445f},
|
||||
{0.0536406f, 0.0582376f, 0.0570966f},
|
||||
{0.0085178f, 0.00748455f, 0.00995162f},
|
||||
{-0.0136637f, -0.0172914f, -0.0195978f},
|
||||
{-0.0339128f, -0.0392692f, -0.0355216f},
|
||||
{0.00612855f, 0.00568303f, -0.00212333f},
|
||||
{-0.0029225f, 0.00668819f, 0.0122131f},
|
||||
{0.00841843f, 0.000181587f, -0.00650644f},
|
||||
{-0.00514432f, 0.0127043f, 0.0168049f},
|
||||
{-0.00997384f, -0.00602262f, -0.0164031f},
|
||||
{0.0233226f, 0.033254f, 0.0307266f},
|
||||
{-0.0110201f, -0.0164169f, -0.0161829f},
|
||||
{-0.0195952f, -0.0177943f, -0.0115377f},
|
||||
{-0.00523918f, -0.00452043f, 0.00267397f},
|
||||
{0.0313464f, 0.0288241f, 0.0262496f},
|
||||
{0.0324018f, 0.0339792f, 0.0312209f},
|
||||
{-0.0163247f, -0.0230503f, -0.0263239f},
|
||||
{0.000420577f, -0.00535659f, -0.00663426f},
|
||||
{-0.012897f, -0.00203767f, -0.000622678f},
|
||||
{-0.0632956f, -0.0651325f, -0.0584479f},
|
||||
{-0.00426634f, -0.0150098f, -0.00719348f},
|
||||
{0.00476109f, 0.00674315f, 0.00895472f},
|
||||
{0.0129384f, 0.0158352f, 0.00963773f},
|
||||
{-0.0333379f, -0.0410522f, -0.0317462f},
|
||||
{0.00344054f, 0.00275915f, 0.00355732f},
|
||||
{0.0209062f, 0.0273453f, 0.0222967f},
|
||||
{0.00827287f, 0.00223045f, 0.00325844f},
|
||||
{-0.0149132f, -0.0183973f, -0.0199781f},
|
||||
{-0.0100786f, -0.0103681f, -0.00218224f},
|
||||
{-0.00791409f, -0.00405153f, -0.00599893f},
|
||||
{0.0176126f, 0.00618342f, -6.6569e-05f},
|
||||
{0.00942486f, -0.00206494f, -0.00580324f},
|
||||
{0.00678093f, -0.00291742f, -0.000921195f},
|
||||
{-0.0221992f, -0.00483162f, -0.000848514f},
|
||||
{-0.0151587f, -0.0157166f, -0.0107302f},
|
||||
{0.00909646f, 0.0171985f, 0.0169785f},
|
||||
{0.0127224f, 0.0170612f, 0.0303428f},
|
||||
{0.0196562f, 0.00212451f, 0.0127744f},
|
||||
{0.0233013f, 0.0228994f, 0.0108387f},
|
||||
{0.00520761f, 0.00992992f, 0.0066267f},
|
||||
{-3.77736e-05f, 0.00460229f, -0.00475132f},
|
||||
{-0.0311763f, -0.0453566f, -0.0486901f},
|
||||
{0.0195798f, 0.0281246f, 0.0180102f},
|
||||
{-0.0174149f, -0.0240867f, -0.0188785f},
|
||||
{0.000104658f, 0.00659008f, 0.0144594f},
|
||||
{-0.00311086f, -0.0241426f, -0.0244164f},
|
||||
{0.0336462f, 0.0305173f, 0.0331101f},
|
||||
{0.0613625f, 0.066561f, 0.0610198f},
|
||||
{-0.0286757f, -0.0325401f, -0.0338036f},
|
||||
{0.0141534f, 0.0188266f, 0.0253059f},
|
||||
{-0.00548197f, -0.00170198f, 0.00561745f},
|
||||
{-0.0117872f, -0.00763218f, -0.0145037f},
|
||||
{-0.0253304f, -0.0245217f, -0.0144905f},
|
||||
{-0.00393624f, 0.00350048f, 0.00765561f},
|
||||
{0.0113625f, 0.00561576f, -0.0113672f},
|
||||
{-0.0301278f, -0.0261472f, -0.0301903f},
|
||||
{0.016863f, 0.0173781f, 0.0170916f},
|
||||
{-0.00495108f, 0.00686749f, 0.00282767f},
|
||||
{0.00125409f, -0.00378072f, -0.00264117f},
|
||||
{-0.00264001f, -0.00529772f, -0.0113109f},
|
||||
{-0.054888f, -0.0575461f, -0.0509146f},
|
||||
{-0.019442f, -0.0232916f, -0.0258637f},
|
||||
{0.0133362f, 0.0161808f, 0.00917951f},
|
||||
{-0.0349002f, -0.0372642f, -0.0466206f},
|
||||
{-0.00216926f, 0.00208738f, 0.00766492f},
|
||||
{0.0268528f, 0.0301179f, 0.0228579f},
|
||||
{0.0226176f, 0.021536f, 0.023152f},
|
||||
{-0.0110646f, -0.00511349f, -0.0137346f},
|
||||
{-0.0098424f, -0.00218176f, 0.00414545f},
|
||||
{0.00200216f, 0.00441732f, -0.0136515f},
|
||||
{0.00695946f, 0.00313109f, -0.00379435f},
|
||||
{0.0188377f, 0.0144059f, 0.0229724f},
|
||||
};
|
||||
float ltxav_latent_rgb_bias[3] = {0.043849f, 0.0201085f, 0.0150286f};
|
||||
|
||||
const float wan_21_latent_rgb_proj[16][3] = {
|
||||
{0.015123f, -0.148418f, 0.479828f},
|
||||
{0.003652f, -0.010680f, -0.037142f},
|
||||
|
||||
154
src/ltxv.hpp
154
src/ltxv.hpp
@@ -243,6 +243,56 @@ namespace LTXV {
|
||||
return build_rope_matrix_from_frequencies(freqs, dim);
|
||||
}
|
||||
|
||||
__STATIC_INLINE__ std::vector<float> build_video_rope_matrix_from_positions(const sd::Tensor<float>& positions,
|
||||
int dim,
|
||||
int num_heads,
|
||||
float theta,
|
||||
const std::vector<int>& max_pos,
|
||||
bool use_middle_indices_grid) {
|
||||
GGML_ASSERT(max_pos.size() == 3);
|
||||
GGML_ASSERT(dim % num_heads == 0);
|
||||
GGML_ASSERT(positions.dim() == 3 || positions.dim() == 4);
|
||||
GGML_ASSERT(positions.shape()[0] == 2);
|
||||
GGML_ASSERT(positions.shape()[1] == 3);
|
||||
if (positions.dim() == 4) {
|
||||
GGML_ASSERT(positions.shape()[3] == 1);
|
||||
}
|
||||
|
||||
const int64_t tokens = positions.shape()[2];
|
||||
const std::vector<float> indices = generate_freq_grid(theta, 3, dim);
|
||||
const int half_dim = dim / 2;
|
||||
const int pad_size = half_dim - static_cast<int>(indices.size()) * 3;
|
||||
std::vector<std::vector<float>> freqs(static_cast<size_t>(tokens), std::vector<float>(half_dim, 0.f));
|
||||
|
||||
for (int64_t token = 0; token < tokens; token++) {
|
||||
int out_idx = 0;
|
||||
for (int i = 0; i < pad_size; i++) {
|
||||
freqs[token][out_idx++] = 0.f;
|
||||
}
|
||||
|
||||
float coords[3];
|
||||
for (int axis = 0; axis < 3; axis++) {
|
||||
float start = positions.dim() == 4 ? positions.index(0, axis, token, 0)
|
||||
: positions.index(0, axis, token);
|
||||
float end = positions.dim() == 4 ? positions.index(1, axis, token, 0)
|
||||
: positions.index(1, axis, token);
|
||||
float coord = use_middle_indices_grid ? 0.5f * (start + end) : start;
|
||||
coords[axis] = coord / static_cast<float>(max_pos[axis]);
|
||||
}
|
||||
|
||||
for (float index : indices) {
|
||||
for (int axis = 0; axis < 3; axis++) {
|
||||
freqs[token][out_idx++] = index * (coords[axis] * 2.f - 1.f);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (num_heads > 1) {
|
||||
return build_rope_matrix_from_frequencies(split_frequencies_by_heads(freqs, dim, num_heads), dim / num_heads);
|
||||
}
|
||||
return build_rope_matrix_from_frequencies(freqs, dim);
|
||||
}
|
||||
|
||||
__STATIC_INLINE__ std::vector<float> build_1d_rope_matrix(int64_t seq_len,
|
||||
int dim,
|
||||
int num_heads = 1,
|
||||
@@ -848,6 +898,31 @@ namespace LTXV {
|
||||
return build_1d_rope_matrix_from_coords(coords, dim, num_heads, theta, static_cast<float>(max_pos_t));
|
||||
}
|
||||
|
||||
__STATIC_INLINE__ std::vector<float> build_video_temporal_rope_matrix_from_positions(const sd::Tensor<float>& positions,
|
||||
int dim,
|
||||
int num_heads,
|
||||
float theta,
|
||||
int max_pos_t,
|
||||
bool use_middle_indices_grid) {
|
||||
GGML_ASSERT(positions.dim() == 3 || positions.dim() == 4);
|
||||
GGML_ASSERT(positions.shape()[0] == 2);
|
||||
GGML_ASSERT(positions.shape()[1] >= 1);
|
||||
if (positions.dim() == 4) {
|
||||
GGML_ASSERT(positions.shape()[3] == 1);
|
||||
}
|
||||
|
||||
std::vector<float> coords;
|
||||
coords.reserve(static_cast<size_t>(positions.shape()[2]));
|
||||
for (int64_t token = 0; token < positions.shape()[2]; token++) {
|
||||
float start = positions.dim() == 4 ? positions.index(0, 0, token, 0)
|
||||
: positions.index(0, 0, token);
|
||||
float end = positions.dim() == 4 ? positions.index(1, 0, token, 0)
|
||||
: positions.index(1, 0, token);
|
||||
coords.push_back(use_middle_indices_grid ? 0.5f * (start + end) : start);
|
||||
}
|
||||
return build_1d_rope_matrix_from_coords(coords, dim, num_heads, theta, static_cast<float>(max_pos_t));
|
||||
}
|
||||
|
||||
__STATIC_INLINE__ float audio_latent_start_time_sec(int64_t latent_index,
|
||||
int audio_latent_downsample_factor = 4,
|
||||
int hop_length = 160,
|
||||
@@ -1664,7 +1739,8 @@ namespace LTXV {
|
||||
const sd::Tensor<float>& audio_x_tensor = {},
|
||||
const sd::Tensor<float>& audio_timesteps_tensor = {},
|
||||
int audio_length = 0,
|
||||
float frame_rate = 24.f) {
|
||||
float frame_rate = 24.f,
|
||||
const sd::Tensor<float>& video_positions_tensor = {}) {
|
||||
auto split_inputs = split_av_latents(x_tensor, audio_length);
|
||||
vx_input_cache = split_inputs.first;
|
||||
if (!audio_x_tensor.empty()) {
|
||||
@@ -1681,19 +1757,31 @@ namespace LTXV {
|
||||
|
||||
ggml_cgraph* gf = new_graph_custom(LTXAV_GRAPH_SIZE);
|
||||
|
||||
float video_frame_rate = frame_rate > 0.f ? frame_rate : 24.f;
|
||||
video_pe_vec = build_video_rope_matrix(vx->ne[0],
|
||||
vx->ne[1],
|
||||
vx->ne[2],
|
||||
static_cast<int>(params.hidden_size),
|
||||
static_cast<int>(params.num_attention_heads),
|
||||
video_frame_rate,
|
||||
params.positional_embedding_theta,
|
||||
params.positional_embedding_max_pos,
|
||||
params.vae_scale_factors,
|
||||
params.causal_temporal_positioning,
|
||||
params.use_middle_indices_grid);
|
||||
auto video_pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, params.attention_head_dim / 2, vx->ne[0] * vx->ne[1] * vx->ne[2] * params.num_attention_heads);
|
||||
float video_frame_rate = frame_rate > 0.f ? frame_rate : 24.f;
|
||||
int64_t video_token_count = vx->ne[0] * vx->ne[1] * vx->ne[2];
|
||||
bool has_video_positions = !video_positions_tensor.empty();
|
||||
if (has_video_positions) {
|
||||
GGML_ASSERT(video_positions_tensor.shape()[2] == video_token_count);
|
||||
video_pe_vec = build_video_rope_matrix_from_positions(video_positions_tensor,
|
||||
static_cast<int>(params.hidden_size),
|
||||
static_cast<int>(params.num_attention_heads),
|
||||
params.positional_embedding_theta,
|
||||
params.positional_embedding_max_pos,
|
||||
params.use_middle_indices_grid);
|
||||
} else {
|
||||
video_pe_vec = build_video_rope_matrix(vx->ne[0],
|
||||
vx->ne[1],
|
||||
vx->ne[2],
|
||||
static_cast<int>(params.hidden_size),
|
||||
static_cast<int>(params.num_attention_heads),
|
||||
video_frame_rate,
|
||||
params.positional_embedding_theta,
|
||||
params.positional_embedding_max_pos,
|
||||
params.vae_scale_factors,
|
||||
params.causal_temporal_positioning,
|
||||
params.use_middle_indices_grid);
|
||||
}
|
||||
auto video_pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, params.attention_head_dim / 2, video_token_count * params.num_attention_heads);
|
||||
ggml_set_name(video_pe, "ltxav_video_pe");
|
||||
set_backend_tensor_data(video_pe, video_pe_vec.data());
|
||||
|
||||
@@ -1712,18 +1800,27 @@ namespace LTXV {
|
||||
set_backend_tensor_data(audio_pe, audio_pe_vec.data());
|
||||
|
||||
int temporal_max_pos = std::max(params.positional_embedding_max_pos[0], params.audio_positional_embedding_max_pos[0]);
|
||||
video_cross_pe_vec = build_video_temporal_rope_matrix(vx->ne[0],
|
||||
vx->ne[1],
|
||||
vx->ne[2],
|
||||
static_cast<int>(params.audio_cross_attention_dim),
|
||||
static_cast<int>(params.audio_num_attention_heads),
|
||||
video_frame_rate,
|
||||
params.positional_embedding_theta,
|
||||
temporal_max_pos,
|
||||
std::get<0>(params.vae_scale_factors),
|
||||
params.causal_temporal_positioning,
|
||||
true);
|
||||
video_cross_pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, params.audio_attention_head_dim / 2, vx->ne[0] * vx->ne[1] * vx->ne[2] * params.audio_num_attention_heads);
|
||||
if (has_video_positions) {
|
||||
video_cross_pe_vec = build_video_temporal_rope_matrix_from_positions(video_positions_tensor,
|
||||
static_cast<int>(params.audio_cross_attention_dim),
|
||||
static_cast<int>(params.audio_num_attention_heads),
|
||||
params.positional_embedding_theta,
|
||||
temporal_max_pos,
|
||||
true);
|
||||
} else {
|
||||
video_cross_pe_vec = build_video_temporal_rope_matrix(vx->ne[0],
|
||||
vx->ne[1],
|
||||
vx->ne[2],
|
||||
static_cast<int>(params.audio_cross_attention_dim),
|
||||
static_cast<int>(params.audio_num_attention_heads),
|
||||
video_frame_rate,
|
||||
params.positional_embedding_theta,
|
||||
temporal_max_pos,
|
||||
std::get<0>(params.vae_scale_factors),
|
||||
params.causal_temporal_positioning,
|
||||
true);
|
||||
}
|
||||
video_cross_pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, params.audio_attention_head_dim / 2, video_token_count * params.audio_num_attention_heads);
|
||||
ggml_set_name(video_cross_pe, "ltxav_video_cross_pe");
|
||||
set_backend_tensor_data(video_cross_pe, video_cross_pe_vec.data());
|
||||
|
||||
@@ -1806,9 +1903,10 @@ namespace LTXV {
|
||||
const sd::Tensor<float>& audio_x = {},
|
||||
const sd::Tensor<float>& audio_timesteps = {},
|
||||
int audio_length = 0,
|
||||
float frame_rate = 24.f) {
|
||||
float frame_rate = 24.f,
|
||||
const sd::Tensor<float>& video_positions = {}) {
|
||||
auto get_graph = [&]() -> ggml_cgraph* {
|
||||
return build_graph(x, timesteps, context, audio_x, audio_timesteps, audio_length, frame_rate);
|
||||
return build_graph(x, timesteps, context, audio_x, audio_timesteps, audio_length, frame_rate, video_positions);
|
||||
};
|
||||
auto out = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
|
||||
return out;
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
#include "denoiser.hpp"
|
||||
#include "diffusion_model.hpp"
|
||||
#include "esrgan.hpp"
|
||||
#include "guidance.h"
|
||||
#include "lora.hpp"
|
||||
#include "ltx_audio_vae.h"
|
||||
#include "ltx_vae.hpp"
|
||||
@@ -80,6 +81,7 @@ const char* sampling_methods_str[] = {
|
||||
"ER-SDE",
|
||||
"Euler CFG++",
|
||||
"Euler A CFG++",
|
||||
"Euler GE",
|
||||
};
|
||||
|
||||
/*================================================== Helper Functions ================================================*/
|
||||
@@ -1606,17 +1608,32 @@ public:
|
||||
void* step_callback_data,
|
||||
bool is_noisy) {
|
||||
if (preview_mode == PREVIEW_PROJ) {
|
||||
sd::Tensor<float> _latents = latents;
|
||||
int patch_sz = 1;
|
||||
const float(*latent_rgb_proj)[3] = nullptr;
|
||||
float* latent_rgb_bias = nullptr;
|
||||
bool is_video = preview_latent_tensor_is_video(latents);
|
||||
uint32_t dim = is_video ? static_cast<uint32_t>(latents.shape()[3]) : static_cast<uint32_t>(latents.shape()[2]);
|
||||
if (version == VERSION_LTXAV) {
|
||||
if (is_video) {
|
||||
_latents = sd::ops::slice(_latents, 3, 0, 128);
|
||||
} else {
|
||||
_latents = sd::ops::slice(_latents, 2, 0, 128);
|
||||
}
|
||||
dim = 128;
|
||||
}
|
||||
|
||||
if (dim == 128) {
|
||||
if (sd_version_uses_flux2_vae(version)) {
|
||||
latent_rgb_proj = flux2_latent_rgb_proj;
|
||||
latent_rgb_bias = flux2_latent_rgb_bias;
|
||||
patch_sz = 2;
|
||||
} else if (version == VERSION_LTXAV) {
|
||||
latent_rgb_proj = ltxav_latent_rgb_proj;
|
||||
latent_rgb_bias = ltxav_latent_rgb_bias;
|
||||
} else {
|
||||
LOG_WARN("No latent to RGB projection known for this model");
|
||||
return;
|
||||
}
|
||||
} else if (dim == 48) {
|
||||
if (sd_version_is_wan(version)) {
|
||||
@@ -1656,13 +1673,13 @@ public:
|
||||
return;
|
||||
}
|
||||
|
||||
uint32_t frames = is_video ? static_cast<uint32_t>(latents.shape()[2]) : 1;
|
||||
uint32_t img_width = static_cast<uint32_t>(latents.shape()[0]) * patch_sz;
|
||||
uint32_t img_height = static_cast<uint32_t>(latents.shape()[1]) * patch_sz;
|
||||
uint32_t frames = is_video ? static_cast<uint32_t>(_latents.shape()[2]) : 1;
|
||||
uint32_t img_width = static_cast<uint32_t>(_latents.shape()[0]) * patch_sz;
|
||||
uint32_t img_height = static_cast<uint32_t>(_latents.shape()[1]) * patch_sz;
|
||||
|
||||
uint8_t* data = (uint8_t*)malloc(frames * img_width * img_height * 3 * sizeof(uint8_t));
|
||||
GGML_ASSERT(data != nullptr);
|
||||
preview_latent_video(data, latents, latent_rgb_proj, latent_rgb_bias, patch_sz);
|
||||
preview_latent_video(data, _latents, latent_rgb_proj, latent_rgb_bias, patch_sz);
|
||||
sd_image_t* images = (sd_image_t*)malloc(frames * sizeof(sd_image_t));
|
||||
GGML_ASSERT(images != nullptr);
|
||||
for (uint32_t i = 0; i < frames; i++) {
|
||||
@@ -1827,7 +1844,8 @@ public:
|
||||
float vace_strength,
|
||||
int audio_length,
|
||||
float frame_rate,
|
||||
const sd_cache_params_t* cache_params) {
|
||||
const sd_cache_params_t* cache_params,
|
||||
const sd::Tensor<float>& video_positions = {}) {
|
||||
std::vector<int> skip_layers(guidance.slg.layers, guidance.slg.layers + guidance.slg.layer_count);
|
||||
float cfg_scale = guidance.txt_cfg;
|
||||
float img_cfg_scale = guidance.img_cfg;
|
||||
@@ -1838,8 +1856,9 @@ public:
|
||||
denoiser.get(),
|
||||
sigmas);
|
||||
|
||||
bool needs_uncond_denoised = method == EULER_CFG_PP_SAMPLE_METHOD || method == EULER_A_CFG_PP_SAMPLE_METHOD;
|
||||
// Spectrum cache is not supported for CFG++ samplers
|
||||
if (method == EULER_CFG_PP_SAMPLE_METHOD || method == EULER_A_CFG_PP_SAMPLE_METHOD) {
|
||||
if (needs_uncond_denoised) {
|
||||
if (cache_runtime.spectrum_enabled) {
|
||||
LOG_WARN("Spectrum cache requested but not supported for CFG++ samplers");
|
||||
cache_runtime.spectrum_enabled = false;
|
||||
@@ -1852,6 +1871,11 @@ public:
|
||||
has_skiplayer = false;
|
||||
LOG_WARN("SLG is incompatible with this model type");
|
||||
}
|
||||
sd::guidance::ClassifierFreeGuidance classifier_free_guidance(cfg_scale, img_cfg_scale);
|
||||
sd::guidance::SkipLayerGuidance skip_layer_guidance(has_skiplayer ? skip_layers : std::vector<int>(),
|
||||
has_skiplayer ? slg_scale : 0.0f,
|
||||
guidance.slg.layer_start,
|
||||
guidance.slg.layer_end);
|
||||
|
||||
if (version == VERSION_HIDREAM_O1 && !noise.empty()) {
|
||||
noise *= eta;
|
||||
@@ -1864,7 +1888,7 @@ public:
|
||||
sd::Tensor<float> denoised = x_t;
|
||||
SamplePreviewContext preview = prepare_sample_preview_context();
|
||||
|
||||
auto denoise = [&](const sd::Tensor<float>& x, float sigma, int step, sd::Tensor<float>* out_uncond_denoised = nullptr) -> sd::Tensor<float> {
|
||||
auto denoise = [&](const sd::Tensor<float>& x, float sigma, int step) -> sd::guidance::GuiderOutput {
|
||||
if (step == 1 || step == -1) {
|
||||
pretty_progress(0, (int)steps, 0);
|
||||
}
|
||||
@@ -1897,17 +1921,17 @@ public:
|
||||
}
|
||||
|
||||
if (cache_runtime.spectrum_enabled && cache_runtime.spectrum.should_predict()) {
|
||||
if (out_uncond_denoised == nullptr) {
|
||||
cache_runtime.spectrum.predict(&denoised);
|
||||
if (!denoise_mask.empty()) {
|
||||
denoised = denoised * denoise_mask + init_latent * (1.0f - denoise_mask);
|
||||
}
|
||||
if (sd_should_preview_denoised() && preview.callback != nullptr) {
|
||||
preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
|
||||
}
|
||||
report_sample_progress(step, steps, t0);
|
||||
return denoised;
|
||||
cache_runtime.spectrum.predict(&denoised);
|
||||
if (!denoise_mask.empty()) {
|
||||
denoised = denoised * denoise_mask + init_latent * (1.0f - denoise_mask);
|
||||
}
|
||||
if (sd_should_preview_denoised() && preview.callback != nullptr) {
|
||||
preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
|
||||
}
|
||||
report_sample_progress(step, steps, t0);
|
||||
sd::guidance::GuiderOutput output;
|
||||
output.pred = denoised;
|
||||
return output;
|
||||
}
|
||||
|
||||
if (sd_should_preview_noisy() && preview.callback != nullptr) {
|
||||
@@ -1917,7 +1941,6 @@ public:
|
||||
sd::Tensor<float> cond_out;
|
||||
sd::Tensor<float> uncond_out;
|
||||
sd::Tensor<float> img_cond_out;
|
||||
sd::Tensor<float> skip_cond_out;
|
||||
sd_sample::SampleStepCacheDispatcher step_cache(cache_runtime, step, sigma);
|
||||
std::vector<sd::Tensor<float>> controls;
|
||||
DiffusionParams diffusion_params;
|
||||
@@ -1933,6 +1956,7 @@ public:
|
||||
diffusion_params.vace_strength = vace_strength;
|
||||
diffusion_params.audio_length = audio_length;
|
||||
diffusion_params.frame_rate = frame_rate;
|
||||
diffusion_params.video_positions = video_positions.empty() ? nullptr : &video_positions;
|
||||
diffusion_params.skip_layers = nullptr;
|
||||
|
||||
compute_sample_controls(control_image,
|
||||
@@ -2006,42 +2030,40 @@ public:
|
||||
return {};
|
||||
}
|
||||
}
|
||||
bool is_skiplayer_step = has_skiplayer &&
|
||||
step > (int)(guidance.slg.layer_start * static_cast<int>(sigmas.size())) &&
|
||||
step < (int)(guidance.slg.layer_end * static_cast<int>(sigmas.size()));
|
||||
if (is_skiplayer_step) {
|
||||
sd::guidance::GuidanceInput guidance_input;
|
||||
guidance_input.step = step;
|
||||
guidance_input.schedule_size = sigmas.size();
|
||||
guidance_input.pred_cond = &cond_out;
|
||||
guidance_input.pred_uncond = uncond_out.empty() ? nullptr : &uncond_out;
|
||||
guidance_input.pred_img_cond = img_cond_out.empty() ? nullptr : &img_cond_out;
|
||||
|
||||
sd::guidance::GuiderOutput guided = classifier_free_guidance.forward(guidance_input, {});
|
||||
if (guided.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
if (skip_layer_guidance.is_enabled_for_step(guidance_input)) {
|
||||
LOG_DEBUG("Skipping layers at step %d\n", step);
|
||||
if (!step_cache.is_step_skipped()) {
|
||||
skip_cond_out = run_condition(cond,
|
||||
cond.c_concat.empty() ? nullptr : &cond.c_concat,
|
||||
&skip_layers);
|
||||
if (skip_cond_out.empty()) {
|
||||
return {};
|
||||
}
|
||||
guidance_input.predict_skip_layer = [&]() -> sd::Tensor<float> {
|
||||
return run_condition(cond,
|
||||
cond.c_concat.empty() ? nullptr : &cond.c_concat,
|
||||
&skip_layer_guidance.layers());
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
GGML_ASSERT(!cond_out.empty());
|
||||
sd::Tensor<float> latent_result = cond_out;
|
||||
if (!uncond_out.empty()) {
|
||||
if (!img_cond_out.empty()) {
|
||||
latent_result = uncond_out +
|
||||
img_cfg_scale * (img_cond_out - uncond_out) +
|
||||
cfg_scale * (cond_out - img_cond_out);
|
||||
} else {
|
||||
latent_result = uncond_out + cfg_scale * (cond_out - uncond_out);
|
||||
}
|
||||
} else if (!img_cond_out.empty()) {
|
||||
latent_result = img_cond_out + cfg_scale * (cond_out - img_cond_out);
|
||||
guided = skip_layer_guidance.forward(guidance_input, std::move(guided));
|
||||
if (guided.pred.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
if (is_skiplayer_step && !skip_cond_out.empty()) {
|
||||
latent_result += (cond_out - skip_cond_out) * slg_scale;
|
||||
}
|
||||
denoised = latent_result * c_out + x * c_skip;
|
||||
if (out_uncond_denoised != nullptr) {
|
||||
sd::Tensor<float> base_uncond = !uncond_out.empty() ? uncond_out : cond_out;
|
||||
*out_uncond_denoised = base_uncond * c_out + x * c_skip;
|
||||
denoised = guided.pred * c_out + x * c_skip;
|
||||
sd::guidance::GuiderOutput output;
|
||||
output.pred = denoised;
|
||||
if (needs_uncond_denoised) {
|
||||
const sd::Tensor<float>& base_uncond = !uncond_out.empty() ? uncond_out : cond_out;
|
||||
output.pred_uncond = base_uncond * c_out + x * c_skip;
|
||||
}
|
||||
if (cache_runtime.spectrum_enabled) {
|
||||
cache_runtime.spectrum.update(denoised);
|
||||
@@ -2053,7 +2075,8 @@ public:
|
||||
preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
|
||||
}
|
||||
report_sample_progress(step, steps, t0);
|
||||
return denoised;
|
||||
output.pred = denoised;
|
||||
return output;
|
||||
};
|
||||
|
||||
auto x0_opt = sample_k_diffusion(method, denoise, x_t, sigmas, sampler_rng, eta, is_flow_denoiser, extra_sample_args);
|
||||
@@ -2260,6 +2283,7 @@ const char* sample_method_to_str[] = {
|
||||
"er_sde",
|
||||
"euler_cfg_pp",
|
||||
"euler_a_cfg_pp",
|
||||
"euler_ge",
|
||||
};
|
||||
|
||||
const char* sd_sample_method_name(enum sample_method_t sample_method) {
|
||||
@@ -2789,7 +2813,10 @@ static sd_audio_t* waveform_to_sd_audio(const StableDiffusionGGML* sd,
|
||||
free(audio);
|
||||
return nullptr;
|
||||
}
|
||||
std::memcpy(audio->data, waveform.data(), sample_bytes);
|
||||
|
||||
auto wavaform_t = waveform.permute({1, 0, 2, 3});
|
||||
std::memcpy(audio->data, wavaform_t.data(), sample_bytes);
|
||||
|
||||
return audio;
|
||||
}
|
||||
|
||||
@@ -3213,16 +3240,99 @@ struct ImageGenerationLatents {
|
||||
sd::Tensor<float> concat_latent;
|
||||
sd::Tensor<float> uncond_concat_latent;
|
||||
sd::Tensor<float> audio_latent;
|
||||
sd::Tensor<float> video_positions;
|
||||
sd::Tensor<float> control_image;
|
||||
std::vector<sd::Tensor<float>> ref_images;
|
||||
std::vector<sd::Tensor<float>> ref_latents;
|
||||
sd::Tensor<float> denoise_mask;
|
||||
sd::Tensor<float> clip_vision_output;
|
||||
sd::Tensor<float> vace_context;
|
||||
int64_t ref_image_num = 0;
|
||||
int audio_length = 0;
|
||||
int64_t ref_image_num = 0;
|
||||
int64_t video_conditioning_frame_count = 0;
|
||||
int64_t video_target_frame_count = 0;
|
||||
int audio_length = 0;
|
||||
};
|
||||
|
||||
static float ltxv_latent_corner_to_pixel_frame(int64_t corner_index,
|
||||
int temporal_scale,
|
||||
bool causal_temporal_positioning) {
|
||||
float pixel_t = static_cast<float>(corner_index * temporal_scale);
|
||||
if (causal_temporal_positioning) {
|
||||
pixel_t = std::max(0.f, pixel_t + 1.f - static_cast<float>(temporal_scale));
|
||||
}
|
||||
return pixel_t;
|
||||
}
|
||||
|
||||
static void set_ltxv_video_position(sd::Tensor<float>* positions,
|
||||
int64_t token,
|
||||
float t_start,
|
||||
float t_end,
|
||||
float h_start,
|
||||
float h_end,
|
||||
float w_start,
|
||||
float w_end) {
|
||||
positions->index(0, 0, token, 0) = t_start;
|
||||
positions->index(1, 0, token, 0) = t_end;
|
||||
positions->index(0, 1, token, 0) = h_start;
|
||||
positions->index(1, 1, token, 0) = h_end;
|
||||
positions->index(0, 2, token, 0) = w_start;
|
||||
positions->index(1, 2, token, 0) = w_end;
|
||||
}
|
||||
|
||||
static sd::Tensor<float> build_ltxv_video_positions(int64_t width,
|
||||
int64_t height,
|
||||
int64_t target_latent_frames,
|
||||
int64_t keyframe_latent_frames,
|
||||
int keyframe_frame_idx,
|
||||
int keyframe_pixel_frames,
|
||||
int fps,
|
||||
int spatial_scale,
|
||||
int temporal_scale,
|
||||
bool causal_temporal_positioning) {
|
||||
GGML_ASSERT(width > 0 && height > 0 && target_latent_frames > 0);
|
||||
GGML_ASSERT(keyframe_latent_frames > 0);
|
||||
GGML_ASSERT(fps > 0);
|
||||
|
||||
int64_t total_tokens = width * height * (target_latent_frames + keyframe_latent_frames);
|
||||
sd::Tensor<float> positions({2, 3, total_tokens, 1});
|
||||
int64_t token = 0;
|
||||
|
||||
for (int64_t t = 0; t < target_latent_frames; t++) {
|
||||
float t_start = ltxv_latent_corner_to_pixel_frame(t, temporal_scale, causal_temporal_positioning) / static_cast<float>(fps);
|
||||
float t_end = ltxv_latent_corner_to_pixel_frame(t + 1, temporal_scale, causal_temporal_positioning) / static_cast<float>(fps);
|
||||
for (int64_t h = 0; h < height; h++) {
|
||||
float h_start = static_cast<float>(h * spatial_scale);
|
||||
float h_end = static_cast<float>((h + 1) * spatial_scale);
|
||||
for (int64_t w = 0; w < width; w++) {
|
||||
float w_start = static_cast<float>(w * spatial_scale);
|
||||
float w_end = static_cast<float>((w + 1) * spatial_scale);
|
||||
set_ltxv_video_position(&positions, token++, t_start, t_end, h_start, h_end, w_start, w_end);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (int64_t t = 0; t < keyframe_latent_frames; t++) {
|
||||
float t_start = static_cast<float>(keyframe_frame_idx + t * temporal_scale);
|
||||
float t_end = static_cast<float>(keyframe_frame_idx + (t + 1) * temporal_scale);
|
||||
if (keyframe_pixel_frames == 1) {
|
||||
t_end = t_start + 1.f;
|
||||
}
|
||||
t_start /= static_cast<float>(fps);
|
||||
t_end /= static_cast<float>(fps);
|
||||
for (int64_t h = 0; h < height; h++) {
|
||||
float h_start = static_cast<float>(h * spatial_scale);
|
||||
float h_end = static_cast<float>((h + 1) * spatial_scale);
|
||||
for (int64_t w = 0; w < width; w++) {
|
||||
float w_start = static_cast<float>(w * spatial_scale);
|
||||
float w_end = static_cast<float>((w + 1) * spatial_scale);
|
||||
set_ltxv_video_position(&positions, token++, t_start, t_end, h_start, h_end, w_start, w_end);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return positions;
|
||||
}
|
||||
|
||||
static sd::Tensor<float> pack_ltxav_audio_and_video_latents(const sd::Tensor<float>& video_latent,
|
||||
const sd::Tensor<float>& audio_latent) {
|
||||
if (audio_latent.empty()) {
|
||||
@@ -4133,33 +4243,27 @@ static std::optional<ImageGenerationLatents> prepare_video_generation_latents(sd
|
||||
}
|
||||
|
||||
if (sd_version_is_ltxav(sd_ctx->sd->version)) {
|
||||
if (!end_image.empty() || sd_vid_gen_params->control_frames_size > 0) {
|
||||
LOG_ERROR("LTXAV currently supports txt2vid and init_image i2v only; end_image and control_frames are not implemented");
|
||||
if (sd_vid_gen_params->control_frames_size > 0) {
|
||||
LOG_ERROR("LTXAV control_frames are not implemented");
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
if (!start_image.empty()) {
|
||||
if (!start_image.empty() || !end_image.empty()) {
|
||||
if (sd_ctx->sd->vae_decode_only) {
|
||||
LOG_ERROR("LTXAV init_image i2v requires VAE encoder weights; create the context with vae_decode_only=false");
|
||||
LOG_ERROR("LTXAV image conditioning requires VAE encoder weights; create the context with vae_decode_only=false");
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
LOG_INFO("IMG2VID");
|
||||
|
||||
int64_t t1 = ggml_time_ms();
|
||||
auto init_img = start_image.reshape({start_image.shape()[0],
|
||||
start_image.shape()[1],
|
||||
1,
|
||||
start_image.shape()[2],
|
||||
start_image.shape()[3]});
|
||||
auto init_image_latent = sd_ctx->sd->encode_first_stage(init_img);
|
||||
if (init_image_latent.empty()) {
|
||||
LOG_ERROR("failed to encode LTXAV init image");
|
||||
return std::nullopt;
|
||||
if (!start_image.empty() && !end_image.empty()) {
|
||||
LOG_INFO("FLF2V");
|
||||
} else if (!start_image.empty()) {
|
||||
LOG_INFO("IMG2VID");
|
||||
} else {
|
||||
LOG_INFO("END2VID");
|
||||
}
|
||||
|
||||
int64_t t1 = ggml_time_ms();
|
||||
latents.init_latent = sd_ctx->sd->generate_init_latent(request->width, request->height, request->frames, true);
|
||||
sd::ops::slice_assign(&latents.init_latent, 2, 0, init_image_latent.shape()[2], init_image_latent);
|
||||
|
||||
float conditioning_strength = std::clamp(request->strength, 0.f, 1.f);
|
||||
float conditioned_mask = 1.0f - conditioning_strength;
|
||||
@@ -4169,7 +4273,94 @@ static std::optional<ImageGenerationLatents> prepare_video_generation_latents(sd
|
||||
1,
|
||||
1},
|
||||
1.f);
|
||||
sd::ops::fill_slice(&latents.denoise_mask, 2, 0, init_image_latent.shape()[2], conditioned_mask);
|
||||
|
||||
auto encode_ltxav_condition_image = [&](const sd::Tensor<float>& image, const char* name) -> sd::Tensor<float> {
|
||||
auto condition_image = image.reshape({image.shape()[0],
|
||||
image.shape()[1],
|
||||
1,
|
||||
image.shape()[2],
|
||||
image.shape()[3]});
|
||||
auto condition_latent = sd_ctx->sd->encode_first_stage(condition_image);
|
||||
if (condition_latent.empty()) {
|
||||
LOG_ERROR("failed to encode LTXAV %s image", name);
|
||||
}
|
||||
return condition_latent;
|
||||
};
|
||||
|
||||
auto apply_video_condition_by_latent_index = [&](const sd::Tensor<float>& condition_latent,
|
||||
int64_t latent_idx,
|
||||
const char* name) -> bool {
|
||||
int64_t latent_frames = latents.init_latent.shape()[2];
|
||||
int64_t condition_frames = condition_latent.shape()[2];
|
||||
if (latent_idx < 0 || condition_frames <= 0 || latent_idx + condition_frames > latent_frames) {
|
||||
LOG_ERROR("invalid LTXAV %s image latent range: start=%" PRId64 ", length=%" PRId64 ", latent_frames=%" PRId64,
|
||||
name,
|
||||
latent_idx,
|
||||
condition_frames,
|
||||
latent_frames);
|
||||
return false;
|
||||
}
|
||||
|
||||
sd::ops::slice_assign(&latents.init_latent, 2, latent_idx, latent_idx + condition_frames, condition_latent);
|
||||
sd::ops::fill_slice(&latents.denoise_mask, 2, latent_idx, latent_idx + condition_frames, conditioned_mask);
|
||||
return true;
|
||||
};
|
||||
|
||||
auto apply_video_condition_by_keyframe_index = [&](const sd::Tensor<float>& keyframes,
|
||||
int frame_idx,
|
||||
const char* name) -> bool {
|
||||
int64_t keyframe_frames = keyframes.shape()[2];
|
||||
if (keyframe_frames <= 0 || keyframes.shape()[0] != latents.init_latent.shape()[0] ||
|
||||
keyframes.shape()[1] != latents.init_latent.shape()[1] ||
|
||||
keyframes.shape()[3] != latents.init_latent.shape()[3]) {
|
||||
LOG_ERROR("invalid LTXAV %s keyframe latent shape", name);
|
||||
return false;
|
||||
}
|
||||
|
||||
latents.video_target_frame_count = latents.init_latent.shape()[2];
|
||||
latents.video_conditioning_frame_count = keyframe_frames;
|
||||
latents.init_latent = sd::ops::concat(latents.init_latent, keyframes, 2);
|
||||
|
||||
auto keyframe_mask = sd::full<float>({keyframes.shape()[0],
|
||||
keyframes.shape()[1],
|
||||
keyframes.shape()[2],
|
||||
1,
|
||||
1},
|
||||
conditioned_mask);
|
||||
latents.denoise_mask = sd::ops::concat(latents.denoise_mask, keyframe_mask, 2);
|
||||
latents.video_positions = build_ltxv_video_positions(latents.init_latent.shape()[0],
|
||||
latents.init_latent.shape()[1],
|
||||
latents.video_target_frame_count,
|
||||
keyframe_frames,
|
||||
frame_idx,
|
||||
1,
|
||||
request->fps,
|
||||
request->vae_scale_factor,
|
||||
8,
|
||||
true);
|
||||
return true;
|
||||
};
|
||||
|
||||
if (!start_image.empty()) {
|
||||
auto start_image_latent = encode_ltxav_condition_image(start_image, "init");
|
||||
if (start_image_latent.empty() || !apply_video_condition_by_latent_index(start_image_latent, 0, "init")) {
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
if (!end_image.empty()) {
|
||||
auto end_image_latent = encode_ltxav_condition_image(end_image, "end");
|
||||
if (end_image_latent.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
int frame_idx = request->frames - 1;
|
||||
bool ok = frame_idx == 0 ? apply_video_condition_by_latent_index(end_image_latent, 0, "end")
|
||||
: apply_video_condition_by_keyframe_index(end_image_latent, frame_idx, "end");
|
||||
if (!ok) {
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
int64_t t2 = ggml_time_ms();
|
||||
LOG_INFO("encode_first_stage completed, taking %" PRId64 " ms", t2 - t1);
|
||||
@@ -4525,7 +4716,8 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx,
|
||||
request.vace_strength,
|
||||
latents.audio_length,
|
||||
static_cast<float>(request.fps),
|
||||
request.cache_params);
|
||||
request.cache_params,
|
||||
latents.video_positions);
|
||||
int64_t sampling_end = ggml_time_ms();
|
||||
if (x_t_sampled.empty()) {
|
||||
LOG_ERROR("sampling(high noise) failed after %.2fs", (sampling_end - sampling_start) * 1.0f / 1000);
|
||||
@@ -4570,7 +4762,8 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx,
|
||||
request.vace_strength,
|
||||
latents.audio_length,
|
||||
static_cast<float>(request.fps),
|
||||
request.cache_params);
|
||||
request.cache_params,
|
||||
latents.video_positions);
|
||||
|
||||
int64_t sampling_end = ggml_time_ms();
|
||||
if (sd_ctx->sd->free_params_immediately) {
|
||||
@@ -4599,6 +4792,12 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx,
|
||||
}
|
||||
}
|
||||
|
||||
if (latents.video_conditioning_frame_count > 0) {
|
||||
int64_t target_frames = latents.video_target_frame_count > 0 ? latents.video_target_frame_count
|
||||
: final_latent.shape()[2] - latents.video_conditioning_frame_count;
|
||||
final_latent = sd::ops::slice(final_latent, 2, 0, target_frames);
|
||||
}
|
||||
|
||||
if (latents.ref_image_num > 0) {
|
||||
final_latent = sd::ops::slice(final_latent, 2, latents.ref_image_num, final_latent.shape()[2]);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user