upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-08-04 09:10:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: add hunyuan video 1.5 support (#1795)

Browse Source
This commit is contained in:
leejet
2026-07-18 22:30:10 +08:00
committed by GitHub
parent b290693977
commit ea4e566ccf
19 changed files with 1970 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/model/diffusion/flux.hpp
View File

@@ -706,11 +706,13 @@ namespace Flux {
LastLayer(int64_t hidden_size,
int64_t patch_size,
int64_t out_channels,
bool prune_mod = false,
bool bias = true)
bool prune_mod = false,
bool bias = true,
int64_t patch_volume = 0)
: prune_mod(prune_mod) {
blocks["norm_final"] = std::shared_ptr<GGMLBlock>(new LayerNorm(hidden_size, 1e-06f, false));
blocks["linear"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_size, patch_size * patch_size * out_channels, bias));
int64_t out_dim = (patch_volume > 0 ? patch_volume : patch_size * patch_size) * out_channels;
blocks["linear"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_size, out_dim, bias));
if (!prune_mod) {
blocks["adaLN_modulation.1"] = std::shared_ptr<GGMLBlock>(new Linear(hidden_size, 2 * hidden_size, bias));
}

679
src/model/diffusion/hunyuan.hpp Normal file
View File

@@ -0,0 +1,679 @@
#ifndef __SD_MODEL_DIFFUSION_HUNYUAN_HPP__
#define __SD_MODEL_DIFFUSION_HUNYUAN_HPP__
#include <memory>
#include "model/common/block.hpp"
#include "model/diffusion/flux.hpp"
#include "model/diffusion/mmdit.hpp"
#include "model/diffusion/wan.hpp"
#include "model_manager.h"
namespace Hunyuan {
constexpr int HUNYUAN_VIDEO_GRAPH_SIZE = 65536;
// Ref: https://github.com/huggingface/diffusers/pull/12696
struct IndividualTokenRefinerBlock : public GGMLBlock {
protected:
int64_t num_heads;
public:
IndividualTokenRefinerBlock(int64_t num_heads,
int64_t head_dim,
int64_t mlp_ratio = 4,
bool attn_bias = true)
: num_heads(num_heads) {
int64_t hidden_size = num_heads * head_dim;
blocks["self_attn.qkv"] = std::make_shared<Linear>(hidden_size, hidden_size * 3, attn_bias);
blocks["self_attn.proj"] = std::make_shared<Linear>(hidden_size, hidden_size, attn_bias);
blocks["norm1"] = std::make_shared<LayerNorm>(hidden_size, 1e-6f, true);
blocks["norm2"] = std::make_shared<LayerNorm>(hidden_size, 1e-6f, true);
blocks["mlp.0"] = std::make_shared<Linear>(hidden_size, hidden_size * mlp_ratio);
blocks["mlp.2"] = std::make_shared<Linear>(hidden_size * mlp_ratio, hidden_size);
// adaLN_modulation.0 is nn.SiLU()
blocks["adaLN_modulation.1"] = std::make_shared<Linear>(hidden_size, hidden_size * 2);
}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* txt, ggml_tensor* t_emb, ggml_tensor* mask) {
auto norm1 = std::dynamic_pointer_cast<LayerNorm>(blocks["norm1"]);
auto norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks["norm2"]);
auto self_attn_qkv = std::dynamic_pointer_cast<Linear>(blocks["self_attn.qkv"]);
auto self_attn_proj = std::dynamic_pointer_cast<Linear>(blocks["self_attn.proj"]);
auto mlp_fc1 = std::dynamic_pointer_cast<Linear>(blocks["mlp.0"]);
auto mlp_fc2 = std::dynamic_pointer_cast<Linear>(blocks["mlp.2"]);
auto adaLN_modulation_1 = std::dynamic_pointer_cast<Linear>(blocks["adaLN_modulation.1"]);
// self attn
auto qkv = self_attn_qkv->forward(ctx, norm1->forward(ctx, txt));
auto qkv_vec = split_qkv(ctx->ggml_ctx, qkv);
auto q = qkv_vec[0];
auto k = qkv_vec[1];
auto v = qkv_vec[2];
auto attn_out = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, num_heads, mask, false, ctx->flash_attn_enabled);
attn_out = self_attn_proj->forward(ctx, attn_out);
// adaLN_modulation
auto emb = adaLN_modulation_1->forward(ctx, ggml_silu(ctx->ggml_ctx, t_emb));
auto mods = ggml_ext_chunk(ctx->ggml_ctx, emb, 2, 0);
txt = ggml_add(ctx->ggml_ctx, txt, ggml_mul(ctx->ggml_ctx, attn_out, mods[0]));
// mlp
auto mlp_out = mlp_fc1->forward(ctx, norm2->forward(ctx, txt));
mlp_out = ggml_silu_inplace(ctx->ggml_ctx, mlp_out);
mlp_out = mlp_fc2->forward(ctx, mlp_out);
txt = ggml_add(ctx->ggml_ctx, txt, ggml_mul(ctx->ggml_ctx, mlp_out, mods[1]));
return txt;
}
};
struct IndividualTokenRefiner : public GGMLBlock {
protected:
int num_layers;
public:
IndividualTokenRefiner(int64_t num_heads,
int64_t head_dim,
int num_layers,
int64_t mlp_ratio = 4,
bool attn_bias = true)
: num_layers(num_layers) {
for (int i = 0; i < num_layers; i++) {
blocks["blocks." + std::to_string(i)] = std::make_shared<IndividualTokenRefinerBlock>(num_heads, head_dim, mlp_ratio, attn_bias);
}
}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* txt, ggml_tensor* t_emb, ggml_tensor* mask) {
for (int i = 0; i < num_layers; i++) {
auto block = std::dynamic_pointer_cast<IndividualTokenRefinerBlock>(blocks["blocks." + std::to_string(i)]);
txt = block->forward(ctx, txt, t_emb, mask);
}
return txt;
}
};
struct TokenRefiner : public GGMLBlock {
public:
TokenRefiner(int64_t in_channels,
int64_t num_heads,
int64_t head_dim,
int num_layers,
int64_t mlp_ratio = 4,
bool attn_bias = true) {
int64_t hidden_size = num_heads * head_dim;
blocks["input_embedder"] = std::make_shared<Linear>(in_channels, hidden_size);
blocks["t_embedder"] = std::make_shared<Flux::MLPEmbedder>(256, hidden_size);
blocks["c_embedder"] = std::make_shared<Flux::MLPEmbedder>(in_channels, hidden_size);
blocks["individual_token_refiner"] = std::make_shared<IndividualTokenRefiner>(num_heads, head_dim, num_layers, mlp_ratio, attn_bias);
}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* txt, ggml_tensor* timestep, ggml_tensor* mask) {
auto input_embedder = std::dynamic_pointer_cast<Linear>(blocks["input_embedder"]);
auto t_embedder = std::dynamic_pointer_cast<Flux::MLPEmbedder>(blocks["t_embedder"]);
auto c_embedder = std::dynamic_pointer_cast<Flux::MLPEmbedder>(blocks["c_embedder"]);
auto individual_token_refiner = std::dynamic_pointer_cast<IndividualTokenRefiner>(blocks["individual_token_refiner"]);
auto t_emb = t_embedder->forward(ctx, ggml_ext_timestep_embedding(ctx->ggml_ctx, timestep, 256, 10000, 1.f));
auto h = ggml_cont(ctx->ggml_ctx, ggml_ext_torch_permute(ctx->ggml_ctx, txt, 1, 0, 2, 3));
auto pooled_projections = ggml_scale(ctx->ggml_ctx, ggml_sum_rows(ctx->ggml_ctx, h), 1.f / txt->ne[1]);
pooled_projections = ggml_cont(ctx->ggml_ctx, ggml_ext_torch_permute(ctx->ggml_ctx, pooled_projections, 1, 0, 2, 3));
auto c_emb = c_embedder->forward(ctx, pooled_projections);
t_emb = ggml_add(ctx->ggml_ctx, t_emb, c_emb);
txt = input_embedder->forward(ctx, txt);
txt = individual_token_refiner->forward(ctx, txt, t_emb, mask);
return txt;
}
};
struct ByT5Mapper : public UnaryBlock {
ByT5Mapper(int64_t in_dim, int64_t hidden_size) {
blocks["layernorm"] = std::make_shared<LayerNorm>(in_dim);
blocks["fc1"] = std::make_shared<Linear>(in_dim, 2048);
blocks["fc2"] = std::make_shared<Linear>(2048, 2048);
blocks["fc3"] = std::make_shared<Linear>(2048, hidden_size);
}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
auto layernorm = std::dynamic_pointer_cast<LayerNorm>(blocks["layernorm"]);
auto fc1 = std::dynamic_pointer_cast<Linear>(blocks["fc1"]);
auto fc2 = std::dynamic_pointer_cast<Linear>(blocks["fc2"]);
auto fc3 = std::dynamic_pointer_cast<Linear>(blocks["fc3"]);
x = fc1->forward(ctx, layernorm->forward(ctx, x));
x = ggml_ext_gelu(ctx->ggml_ctx, x);
x = fc2->forward(ctx, x);
x = ggml_ext_gelu(ctx->ggml_ctx, x);
return fc3->forward(ctx, x);
}
};
struct HunyuanVideoConfig {
std::tuple<int, int, int> patch_size = {1, 2, 2};
int64_t in_channels = 65;
int64_t out_channels = 32;
int64_t hidden_size = 2048;
int64_t vec_in_dim = 0;
int64_t context_in_dim = 3584;
int64_t vision_in_dim = 0;
float mlp_ratio = 4.0f;
int num_heads = 16;
int depth = 54;
int depth_single_blocks = 0;
bool qkv_bias = true;
bool guidance_embed = false;
bool use_byt5 = false;
bool use_cond_type_embedding = false;
bool use_meanflow = false;
bool use_meanflow_sum = false;
float theta = 256;
std::vector<int> axes_dim = {16, 56, 56};
int axes_dim_sum = 128;
int64_t patch_volume() const {
return static_cast<int64_t>(std::get<0>(patch_size)) * std::get<1>(patch_size) * std::get<2>(patch_size);
}
static HunyuanVideoConfig detect_from_weights(const String2TensorStorage& tensor_storage_map,
const std::string& prefix) {
HunyuanVideoConfig config;
config.depth = 0;
config.depth_single_blocks = 0;
bool inferred = false;
int64_t img_embed_dim = 0;
for (const auto& [name, storage] : tensor_storage_map) {
if (starts_with(name, prefix) && ends_with(name, "img_in.proj.bias")) {
img_embed_dim = storage.ne[0];
break;
}
}
for (const auto& [name, storage] : tensor_storage_map) {
if (!starts_with(name, prefix)) {
continue;
}
auto update_depth = [&](const char* block_prefix, int* depth) {
size_t pos = name.find(block_prefix);
if (pos == std::string::npos) {
return;
}
pos += strlen(block_prefix);
size_t end = name.find('.', pos);
if (end != std::string::npos) {
*depth = std::max(*depth, atoi(name.substr(pos, end - pos).c_str()) + 1);
}
};
update_depth("double_blocks.", &config.depth);
update_depth("single_blocks.", &config.depth_single_blocks);
if (ends_with(name, "img_in.proj.weight") && storage.n_dims == 5) {
config.patch_size = {static_cast<int>(storage.ne[2]),
static_cast<int>(storage.ne[1]),
static_cast<int>(storage.ne[0])};
config.in_channels = storage.ne[3];
config.hidden_size = storage.ne[4];
inferred = true;
} else if (ends_with(name, "img_in.proj.weight") && storage.n_dims == 4) {
config.patch_size = {static_cast<int>(storage.ne[2]),
static_cast<int>(storage.ne[1]),
static_cast<int>(storage.ne[0])};
if (img_embed_dim > 0 && storage.ne[3] % img_embed_dim == 0) {
config.hidden_size = img_embed_dim;
config.in_channels = storage.ne[3] / img_embed_dim;
}
inferred = true;
} else if (ends_with(name, "txt_in.input_embedder.weight")) {
config.context_in_dim = storage.ne[0];
inferred = true;
} else if (ends_with(name, "vector_in.in_layer.weight")) {
config.vec_in_dim = storage.ne[0];
} else if (ends_with(name, "vision_in.proj.0.weight")) {
config.vision_in_dim = storage.ne[0];
} else if (ends_with(name, "double_blocks.0.img_attn.norm.key_norm.scale") ||
ends_with(name, "double_blocks.0.img_attn.norm.key_norm.weight")) {
config.num_heads = static_cast<int>(config.hidden_size / storage.ne[0]);
} else if (ends_with(name, "double_blocks.0.img_mlp.0.weight")) {
config.mlp_ratio = static_cast<float>(storage.ne[1]) / static_cast<float>(storage.ne[0]);
}
config.guidance_embed = config.guidance_embed || name.find("guidance_in.") != std::string::npos;
config.use_byt5 = config.use_byt5 || name.find("byt5_in.") != std::string::npos;
config.use_meanflow = config.use_meanflow || name.find("time_r_in.") != std::string::npos;
}
config.use_cond_type_embedding = tensor_storage_map.find(prefix + ".cond_type_embedding.weight") != tensor_storage_map.end();
config.use_meanflow_sum = config.vision_in_dim > 0;
auto final_iter = tensor_storage_map.find(prefix + ".final_layer.linear.weight");
if (final_iter != tensor_storage_map.end()) {
config.out_channels = final_iter->second.ne[1] / config.patch_volume();
}
config.qkv_bias = tensor_storage_map.find(prefix + ".double_blocks.0.img_attn.qkv.bias") != tensor_storage_map.end();
GGML_ASSERT(config.hidden_size % config.num_heads == 0);
GGML_ASSERT(config.hidden_size / config.num_heads == config.axes_dim_sum);
if (inferred) {
LOG_DEBUG("hunyuan video: depth = %d, single depth = %d, in_channels = %" PRId64 ", out_channels = %" PRId64 ", hidden_size = %" PRId64 ", context_in_dim = %" PRId64 ", patch_size = %dx%dx%d",
config.depth,
config.depth_single_blocks,
config.in_channels,
config.out_channels,
config.hidden_size,
config.context_in_dim,
std::get<0>(config.patch_size),
std::get<1>(config.patch_size),
std::get<2>(config.patch_size));
}
return config;
}
};
class HunyuanVideoModel : public GGMLBlock {
protected:
HunyuanVideoConfig config;
void init_params(struct ggml_context* ctx,
const String2TensorStorage& tensor_storage_map = {},
const std::string prefix = "") override {
if (config.use_cond_type_embedding) {
ggml_type type = get_type(prefix + "cond_type_embedding.weight", tensor_storage_map, GGML_TYPE_F16);
GGMLBlock::params["cond_type_embedding.weight"] = ggml_new_tensor_2d(ctx, type, config.hidden_size, 3);
}
}
public:
HunyuanVideoModel() {}
explicit HunyuanVideoModel(HunyuanVideoConfig config)
: config(std::move(config)) {
int64_t head_dim = this->config.hidden_size / this->config.num_heads;
blocks["txt_in"] = std::make_shared<TokenRefiner>(this->config.context_in_dim, this->config.num_heads, head_dim, 2);
blocks["img_in"] = std::make_shared<PatchEmbed>(static_cast<int64_t>(224) /*Not used*/,
this->config.patch_size,
this->config.in_channels,
this->config.hidden_size);
blocks["time_in"] = std::make_shared<Flux::MLPEmbedder>(256, this->config.hidden_size);
if (this->config.vec_in_dim > 0) {
blocks["vector_in"] = std::make_shared<Flux::MLPEmbedder>(this->config.vec_in_dim, this->config.hidden_size);
}
if (this->config.vision_in_dim > 0) {
blocks["vision_in"] = std::make_shared<WAN::MLPProj>(this->config.vision_in_dim, this->config.hidden_size);
}
if (this->config.guidance_embed) {
blocks["guidance_in"] = std::make_shared<Flux::MLPEmbedder>(256, this->config.hidden_size);
}
if (this->config.use_byt5) {
blocks["byt5_in"] = std::make_shared<ByT5Mapper>(1472, this->config.hidden_size);
}
if (this->config.use_meanflow) {
blocks["time_r_in"] = std::make_shared<Flux::MLPEmbedder>(256, this->config.hidden_size);
}
for (int i = 0; i < this->config.depth; i++) {
blocks["double_blocks." + std::to_string(i)] = std::make_shared<Flux::DoubleStreamBlock>(this->config.hidden_size,
this->config.num_heads,
this->config.mlp_ratio,
i,
this->config.qkv_bias);
}
for (int i = 0; i < this->config.depth_single_blocks; i++) {
blocks["single_blocks." + std::to_string(i)] = std::make_shared<Flux::SingleStreamBlock>(this->config.hidden_size,
this->config.num_heads,
this->config.mlp_ratio,
i,
0.f);
}
blocks["final_layer"] = std::make_shared<Flux::LastLayer>(this->config.hidden_size,
std::get<2>(this->config.patch_size),
this->config.out_channels,
false,
true,
this->config.patch_volume());
}
ggml_tensor* pad_to_patch_size(struct ggml_context* ctx,
ggml_tensor* x) {
int64_t W = x->ne[0];
int64_t H = x->ne[1];
int64_t T = x->ne[2];
int pt = std::get<0>(config.patch_size);
int ph = std::get<1>(config.patch_size);
int pw = std::get<2>(config.patch_size);
int pad_t = (pt - static_cast<int>(T % pt)) % pt;
int pad_h = (ph - static_cast<int>(H % ph)) % ph;
int pad_w = (pw - static_cast<int>(W % pw)) % pw;
x = ggml_pad(ctx, x, pad_w, pad_h, pad_t, 0); // [N*C, T + pad_t, H + pad_h, W + pad_w]
return x;
}
ggml_tensor* unpatchify(struct ggml_context* ctx,
ggml_tensor* x,
int64_t t_len,
int64_t h_len,
int64_t w_len) {
// x: [N, t_len*h_len*w_len, C*pt*ph*pw]
// return: [N*C, t_len*pt, h_len*ph, w_len*pw]
int64_t N = x->ne[3];
int64_t pt = std::get<0>(config.patch_size);
int64_t ph = std::get<1>(config.patch_size);
int64_t pw = std::get<2>(config.patch_size);
int64_t C = x->ne[0] / pt / ph / pw;
GGML_ASSERT(C * pt * ph * pw == x->ne[0]);
x = ggml_reshape_4d(ctx, x, C, pw * ph * pt, w_len * h_len * t_len, N); // [N, t_len*h_len*w_len, pt*ph*pw, C]
x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 1, 2, 0, 3)); // [N, C, t_len*h_len*w_len, pt*ph*pw]
x = ggml_reshape_4d(ctx, x, pw, ph * pt, w_len, h_len * t_len * C * N); // [N*C*t_len*h_len, w_len, pt*ph, pw]
x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3)); // [N*C*t_len*h_len, pt*ph, w_len, pw]
x = ggml_reshape_4d(ctx, x, pw * w_len, ph, pt, h_len * t_len * C * N); // [N*C*t_len*h_len, pt, ph, w_len*pw]
x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3)); // [N*C*t_len*h_len, ph, pt, w_len*pw]
x = ggml_reshape_4d(ctx, x, pw * w_len, pt, ph * h_len, t_len * C * N); // [N*C*t_len, h_len*ph, pt, w_len*pw]
x = ggml_ext_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 1, 3)); // [N*C*t_len, pt, h_len*ph, w_len*pw]
x = ggml_reshape_4d(ctx, x, pw * w_len, ph * h_len, pt * t_len, C * N); // [N*C, t_len*pt, h_len*ph, w_len*pw]
return x;
}
ggml_tensor* add_condition_type(GGMLRunnerContext* ctx, ggml_tensor* x, int type) {
if (!config.use_cond_type_embedding) {
return x;
}
auto weight = GGMLBlock::params["cond_type_embedding.weight"];
auto row = ggml_view_1d(ctx->ggml_ctx,
weight,
weight->ne[0],
static_cast<size_t>(type) * weight->nb[1]);
auto target = ggml_new_tensor_3d(ctx->ggml_ctx, row->type, config.hidden_size, x->ne[1], x->ne[2]);
auto embed = ggml_repeat(ctx->ggml_ctx, row, target);
embed = ggml_cast(ctx->ggml_ctx, embed, x->type);
return ggml_add(ctx->ggml_ctx, x, embed);
}
ggml_tensor* forward_orig(GGMLRunnerContext* ctx,
ggml_tensor* img,
ggml_tensor* txt,
ggml_tensor* timestep,
ggml_tensor* pe,
ggml_tensor* guidance = nullptr,
ggml_tensor* y = nullptr,
ggml_tensor* txt_byt5 = nullptr,
ggml_tensor* clip_fea = nullptr,
ggml_tensor* timestep_r = nullptr,
int64_t N = 1) {
// img: [N*C, T, H, W], C => in_dim
// txt: [N, L, text_dim]
// timestep: [N,] or [T]
// return: [N, t_len*h_len*w_len, out_dim*pt*ph*pw]
GGML_ASSERT(N == 1);
auto img_in = std::dynamic_pointer_cast<PatchEmbed>(blocks["img_in"]);
auto txt_in = std::dynamic_pointer_cast<TokenRefiner>(blocks["txt_in"]);
auto time_in = std::dynamic_pointer_cast<Flux::MLPEmbedder>(blocks["time_in"]);
auto final_layer = std::dynamic_pointer_cast<Flux::LastLayer>(blocks["final_layer"]);
img = img_in->forward(ctx, img); // [N*C, t_len*h_len*w_len, hidden_size]
txt = txt_in->forward(ctx, txt, timestep, nullptr); // [N, n_txt_token, hidden_size]
auto vec = time_in->forward(ctx, ggml_ext_timestep_embedding(ctx->ggml_ctx, timestep, 256, 10000, 1.f));
if (config.use_meanflow && timestep_r != nullptr) {
auto time_r_in = std::dynamic_pointer_cast<Flux::MLPEmbedder>(blocks["time_r_in"]);
auto vec_r = time_r_in->forward(ctx, ggml_ext_timestep_embedding(ctx->ggml_ctx, timestep_r, 256, 10000, 1000.f));
vec = ggml_add(ctx->ggml_ctx, vec, vec_r);
if (!config.use_meanflow_sum) {
vec = ggml_scale(ctx->ggml_ctx, vec, 0.5f);
}
}
if (config.vec_in_dim > 0 && y != nullptr) {
auto vector_in = std::dynamic_pointer_cast<Flux::MLPEmbedder>(blocks["vector_in"]);
vec = ggml_add(ctx->ggml_ctx, vec, vector_in->forward(ctx, y));
}
if (config.guidance_embed && guidance != nullptr) {
auto guidance_in = std::dynamic_pointer_cast<Flux::MLPEmbedder>(blocks["guidance_in"]);
auto guidance_emb = ggml_ext_timestep_embedding(ctx->ggml_ctx, guidance, 256, 10000, 1.f);
vec = ggml_add(ctx->ggml_ctx, vec, guidance_in->forward(ctx, guidance_emb));
}
txt = add_condition_type(ctx, txt, 0);
if (config.use_byt5 && txt_byt5 != nullptr) {
auto byt5_in = std::dynamic_pointer_cast<ByT5Mapper>(blocks["byt5_in"]);
txt_byt5 = add_condition_type(ctx, byt5_in->forward(ctx, txt_byt5), 1);
txt = config.use_cond_type_embedding ? ggml_concat(ctx->ggml_ctx, txt_byt5, txt, 1)
: ggml_concat(ctx->ggml_ctx, txt, txt_byt5, 1);
}
if (config.vision_in_dim > 0 && clip_fea != nullptr) {
auto vision_in = std::dynamic_pointer_cast<WAN::MLPProj>(blocks["vision_in"]);
clip_fea = add_condition_type(ctx, vision_in->forward(ctx, clip_fea), 2);
txt = ggml_concat(ctx->ggml_ctx, clip_fea, txt, 1);
}
for (int i = 0; i < config.depth; i++) {
auto block = std::dynamic_pointer_cast<Flux::DoubleStreamBlock>(blocks["double_blocks." + std::to_string(i)]);
auto img_txt = block->forward(ctx, img, txt, vec, pe, nullptr);
img = img_txt.first; // [N, n_img_token, hidden_size]
txt = img_txt.second; // [N, n_txt_token, hidden_size]
}
if (config.depth_single_blocks > 0) {
auto txt_img = ggml_concat(ctx->ggml_ctx, txt, img, 1); // [N, n_txt_token + n_img_token, hidden_size]
for (int i = 0; i < config.depth_single_blocks; i++) {
auto block = std::dynamic_pointer_cast<Flux::SingleStreamBlock>(blocks["single_blocks." + std::to_string(i)]);
txt_img = block->forward(ctx, txt_img, vec, pe, nullptr);
}
txt_img = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, txt_img, 0, 2, 1, 3));
img = ggml_view_3d(ctx->ggml_ctx,
txt_img,
txt_img->ne[0],
txt_img->ne[1],
img->ne[1],
txt_img->nb[1],
txt_img->nb[2],
txt_img->nb[2] * txt->ne[1]);
img = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, img, 0, 2, 1, 3));
}
img = final_layer->forward(ctx, img, vec); // (N, t_len*h_len*w_len, out_channels * patch_size ** 3)
return img;
}
ggml_tensor* forward(GGMLRunnerContext* ctx,
ggml_tensor* x,
ggml_tensor* timestep,
ggml_tensor* context,
ggml_tensor* pe,
ggml_tensor* guidance = nullptr,
ggml_tensor* y = nullptr,
ggml_tensor* txt_byt5 = nullptr,
ggml_tensor* clip_fea = nullptr,
ggml_tensor* timestep_r = nullptr,
int64_t N = 1) {
// Forward pass of DiT.
// x: [N*C, T, H, W]
// timestep: [N,]
// context: [N, L, D]
// pe: [L, d_head/2, 2, 2]
// return: [N*C, T, H, W]
GGML_ASSERT(N == 1);
int64_t W = x->ne[0];
int64_t H = x->ne[1];
int64_t T = x->ne[2];
x = pad_to_patch_size(ctx->ggml_ctx, x);
int64_t pt = std::get<0>(config.patch_size);
int64_t ph = std::get<1>(config.patch_size);
int64_t pw = std::get<2>(config.patch_size);
int64_t t_len = (T + pt - 1) / pt;
int64_t h_len = (H + ph - 1) / ph;
int64_t w_len = (W + pw - 1) / pw;
auto out = forward_orig(ctx, x, context, timestep, pe, guidance, y, txt_byt5, clip_fea, timestep_r, N);
out = unpatchify(ctx->ggml_ctx, out, t_len, h_len, w_len); // [N*C, (T+pad_t) + (T2+pad_t2), H + pad_h, W + pad_w]
// slice
out = ggml_ext_slice(ctx->ggml_ctx, out, 2, 0, T); // [N*C, T, H + pad_h, W + pad_w]
out = ggml_ext_slice(ctx->ggml_ctx, out, 1, 0, H); // [N*C, T, H, W + pad_w]
out = ggml_ext_slice(ctx->ggml_ctx, out, 0, 0, W); // [N*C, T, H, W]
return out;
}
};
struct HunyuanVideoRunner : public DiffusionModelRunner {
public:
HunyuanVideoConfig config;
HunyuanVideoModel hunyuan_video;
std::vector<float> pe_vec;
SDVersion version;
HunyuanVideoRunner(ggml_backend_t backend,
const String2TensorStorage& tensor_storage_map = {},
const std::string prefix = "",
SDVersion version = VERSION_HUNYUAN_VIDEO,
std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
: DiffusionModelRunner(backend, prefix, weight_manager),
config(HunyuanVideoConfig::detect_from_weights(tensor_storage_map, prefix)),
version(version) {
LOG_INFO("HunyuanVideo blocks: %d double, %d single", config.depth, config.depth_single_blocks);
hunyuan_video = HunyuanVideoModel(config);
hunyuan_video.init(params_ctx, tensor_storage_map, prefix);
}
std::string get_desc() override {
return "hunyuan_video";
}
void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) override {
hunyuan_video.get_param_tensors(tensors, prefix);
}
ggml_cgraph* build_graph(const sd::Tensor<float>& x_tensor,
const sd::Tensor<float>& timesteps_tensor,
const sd::Tensor<float>& context_tensor,
const sd::Tensor<float>& c_concat_tensor = {},
const sd::Tensor<float>& y_tensor = {},
const sd::Tensor<float>& guidance_tensor = {},
const sd::Tensor<float>& byt5_tensor = {},
const sd::Tensor<float>& vision_tensor = {},
const sd::Tensor<float>& timestep_r_tensor = {}) {
ggml_cgraph* gf = new_graph_custom(HUNYUAN_VIDEO_GRAPH_SIZE);
ggml_tensor* x = make_input(x_tensor);
ggml_tensor* timesteps = make_input(timesteps_tensor);
ggml_tensor* context = make_input(context_tensor);
ggml_tensor* c_concat = make_optional_input(c_concat_tensor);
ggml_tensor* y = make_optional_input(y_tensor);
ggml_tensor* guidance = make_optional_input(guidance_tensor);
ggml_tensor* byt5 = make_optional_input(byt5_tensor);
ggml_tensor* vision = make_optional_input(vision_tensor);
ggml_tensor* timestep_r = make_optional_input(timestep_r_tensor);
GGML_ASSERT(x->ne[3] == config.out_channels);
if (c_concat != nullptr) {
x = ggml_concat(compute_ctx, x, c_concat, 3);
}
GGML_ASSERT(x->ne[3] <= config.in_channels);
if (x->ne[3] < config.in_channels) {
x = ggml_pad(compute_ctx, x, 0, 0, 0, static_cast<int>(config.in_channels - x->ne[3]));
}
int text_len = static_cast<int>(context->ne[1]);
if (byt5 != nullptr) {
text_len += static_cast<int>(byt5->ne[1]);
}
if (vision != nullptr) {
text_len += static_cast<int>(vision->ne[1]);
}
pe_vec = Rope::gen_hunyuan_video_pe(static_cast<int>(x->ne[2]),
static_cast<int>(x->ne[1]),
static_cast<int>(x->ne[0]),
std::get<0>(config.patch_size),
std::get<1>(config.patch_size),
std::get<2>(config.patch_size),
1,
text_len,
config.theta,
config.axes_dim);
int64_t pos_len = static_cast<int64_t>(pe_vec.size() / config.axes_dim_sum / 2);
// LOG_DEBUG("pos_len %d", pos_len);
auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
// pe->data = pe_vec.data();
// print_ggml_tensor(pe, true, "pe");
// pe->data = nullptr;
set_backend_tensor_data(pe, pe_vec.data());
auto runner_ctx = get_context();
ggml_tensor* out = hunyuan_video.forward(&runner_ctx,
x,
timesteps,
context,
pe,
guidance,
y,
byt5,
vision,
timestep_r);
ggml_build_forward_expand(gf, out);
return gf;
}
sd::Tensor<float> compute(int n_threads,
const sd::Tensor<float>& x,
const sd::Tensor<float>& timesteps,
const sd::Tensor<float>& context,
const sd::Tensor<float>& c_concat = {},
const sd::Tensor<float>& y = {},
const sd::Tensor<float>& guidance = {},
const sd::Tensor<float>& byt5 = {},
const sd::Tensor<float>& vision = {},
const sd::Tensor<float>& timestep_r = {}) {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(x, timesteps, context, c_concat, y, guidance, byt5, vision, timestep_r);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
const DiffusionParams& diffusion_params) override {
GGML_ASSERT(diffusion_params.x != nullptr);
GGML_ASSERT(diffusion_params.timesteps != nullptr);
GGML_ASSERT(diffusion_params.context != nullptr);
const auto* extra = diffusion_extra_as<HunyuanVideoDiffusionExtra>(diffusion_params);
return compute(n_threads,
*diffusion_params.x,
*diffusion_params.timesteps,
*diffusion_params.context,
tensor_or_empty(diffusion_params.c_concat),
tensor_or_empty(diffusion_params.y),
tensor_or_empty(extra->guidance),
tensor_or_empty(extra->byt5),
tensor_or_empty(extra->vision),
tensor_or_empty(extra->timestep_r));
}
};
} // namespace Hunyuan
#endif // __SD_MODEL_DIFFUSION_HUNYUAN_HPP__

32
src/model/diffusion/ltxv.hpp
View File

@@ -800,7 +800,7 @@ namespace LTXV {
auto gate_mlp = mods[5];
auto x_norm = rms_norm(ctx->ggml_ctx, x);
x_norm = modulate(ctx->ggml_ctx, x_norm, shift_msa, scale_msa);
x_norm = LTXV::modulate(ctx->ggml_ctx, x_norm, shift_msa, scale_msa);
auto msa = attn1->forward(ctx, x_norm, nullptr, self_attention_mask, pe);
x = ggml_add(ctx->ggml_ctx, x, apply_gate(ctx->ggml_ctx, msa, gate_msa));
@@ -810,12 +810,12 @@ namespace LTXV {
auto gate_q = mods[8];
auto q = rms_norm(ctx->ggml_ctx, x);
q = modulate(ctx->ggml_ctx, q, shift_q, scale_q);
q = LTXV::modulate(ctx->ggml_ctx, q, shift_q, scale_q);
auto context_mod = context;
if (prompt_timestep != nullptr) {
auto prompt_mods = get_prompt_scale_shift_values(ctx, prompt_timestep);
context_mod = modulate(ctx->ggml_ctx, context_mod, prompt_mods[0], prompt_mods[1]);
context_mod = LTXV::modulate(ctx->ggml_ctx, context_mod, prompt_mods[0], prompt_mods[1]);
}
auto mca = attn2->forward(ctx, q, context_mod, attention_mask, nullptr, nullptr);
@@ -826,7 +826,7 @@ namespace LTXV {
}
auto y = rms_norm(ctx->ggml_ctx, x);
y = modulate(ctx->ggml_ctx, y, shift_mlp, scale_mlp);
y = LTXV::modulate(ctx->ggml_ctx, y, shift_mlp, scale_mlp);
auto mlp_out = ff->forward(ctx, y);
x = ggml_add(ctx->ggml_ctx, x, apply_gate(ctx->ggml_ctx, mlp_out, gate_mlp));
return x;
@@ -1177,11 +1177,11 @@ namespace LTXV {
if (cross_attention_adaln) {
auto q_mods = get_ada_values(ctx, table, timestep, dim, 9, 6, 3);
auto q = rms_norm(ctx->ggml_ctx, x);
q = modulate(ctx->ggml_ctx, q, q_mods[0], q_mods[1]);
q = LTXV::modulate(ctx->ggml_ctx, q, q_mods[0], q_mods[1]);
auto context_mod = context;
if (prompt_timestep != nullptr && prompt_table != nullptr) {
auto p_mods = get_ada_values(ctx, prompt_table, prompt_timestep, dim, 2);
context_mod = modulate(ctx->ggml_ctx, context_mod, p_mods[0], p_mods[1]);
context_mod = LTXV::modulate(ctx->ggml_ctx, context_mod, p_mods[0], p_mods[1]);
}
auto out = attn->forward(ctx, q, context_mod, attention_mask, nullptr, nullptr);
return apply_gate(ctx->ggml_ctx, out, q_mods[2]);
@@ -1228,7 +1228,7 @@ namespace LTXV {
auto v_mods = get_ada_values(ctx, v_table, v_timestep, v_dim, cross_attention_adaln ? 9 : 6);
auto v_norm = rms_norm(ctx->ggml_ctx, vx);
v_norm = modulate(ctx->ggml_ctx, v_norm, v_mods[0], v_mods[1]);
v_norm = LTXV::modulate(ctx->ggml_ctx, v_norm, v_mods[0], v_mods[1]);
auto v_sa = attn1->forward(ctx, v_norm, nullptr, self_attention_mask, v_pe);
vx = ggml_add(ctx->ggml_ctx, vx, apply_gate(ctx->ggml_ctx, v_sa, v_mods[2]));
auto v_txt = apply_text_cross_attention(ctx,
@@ -1246,7 +1246,7 @@ namespace LTXV {
if (run_ax) {
auto a_mods = get_ada_values(ctx, a_table, a_timestep, a_dim, cross_attention_adaln ? 9 : 6);
auto a_norm = rms_norm(ctx->ggml_ctx, ax);
a_norm = modulate(ctx->ggml_ctx, a_norm, a_mods[0], a_mods[1]);
a_norm = LTXV::modulate(ctx->ggml_ctx, a_norm, a_mods[0], a_mods[1]);
auto a_sa = audio_attn1->forward(ctx, a_norm, nullptr, nullptr, a_pe);
ax = ggml_add(ctx->ggml_ctx, ax, apply_gate(ctx->ggml_ctx, a_sa, a_mods[2]));
auto a_txt = apply_text_cross_attention(ctx,
@@ -1269,8 +1269,8 @@ namespace LTXV {
auto a2v_video_table = ggml_ext_slice(ctx->ggml_ctx, params["scale_shift_table_a2v_ca_video"], 1, 0, 4);
auto a2v_audio = get_ada_values(ctx, a2v_audio_table, a_cross_scale_shift_timestep, a_dim, 4);
auto a2v_video = get_ada_values(ctx, a2v_video_table, v_cross_scale_shift_timestep, v_dim, 4);
auto vx_scaled = modulate(ctx->ggml_ctx, vx_norm3, a2v_video[1], a2v_video[0]);
auto ax_scaled = modulate(ctx->ggml_ctx, ax_norm3, a2v_audio[1], a2v_audio[0]);
auto vx_scaled = LTXV::modulate(ctx->ggml_ctx, vx_norm3, a2v_video[1], a2v_video[0]);
auto ax_scaled = LTXV::modulate(ctx->ggml_ctx, ax_norm3, a2v_audio[1], a2v_audio[0]);
auto a2v_out = audio_to_video_attn->forward(ctx, vx_scaled, ax_scaled, nullptr, v_cross_pe, a_cross_pe);
auto a2v_gate_table = ggml_ext_slice(ctx->ggml_ctx, params["scale_shift_table_a2v_ca_video"], 1, 4, 5);
auto a2v_gate = get_ada_values(ctx, a2v_gate_table, v_cross_gate_timestep, v_dim, 1)[0];
@@ -1282,8 +1282,8 @@ namespace LTXV {
auto v2a_video_table = ggml_ext_slice(ctx->ggml_ctx, params["scale_shift_table_a2v_ca_video"], 1, 0, 4);
auto v2a_audio = get_ada_values(ctx, v2a_audio_table, a_cross_scale_shift_timestep, a_dim, 4);
auto v2a_video = get_ada_values(ctx, v2a_video_table, v_cross_scale_shift_timestep, v_dim, 4);
auto ax_scaled = modulate(ctx->ggml_ctx, ax_norm3, v2a_audio[3], v2a_audio[2]);
auto vx_scaled = modulate(ctx->ggml_ctx, vx_norm3, v2a_video[3], v2a_video[2]);
auto ax_scaled = LTXV::modulate(ctx->ggml_ctx, ax_norm3, v2a_audio[3], v2a_audio[2]);
auto vx_scaled = LTXV::modulate(ctx->ggml_ctx, vx_norm3, v2a_video[3], v2a_video[2]);
auto v2a_out = video_to_audio_attn->forward(ctx, ax_scaled, vx_scaled, nullptr, a_cross_pe, v_cross_pe);
auto v2a_gate_table = ggml_ext_slice(ctx->ggml_ctx, params["scale_shift_table_a2v_ca_audio"], 1, 4, 5);
auto v2a_gate = get_ada_values(ctx, v2a_gate_table, a_cross_gate_timestep, a_dim, 1)[0];
@@ -1291,14 +1291,14 @@ namespace LTXV {
}
auto a_ff_mods = get_ada_values(ctx, a_table, a_timestep, a_dim, cross_attention_adaln ? 9 : 6, 3, 3);
auto ax_scaled = rms_norm(ctx->ggml_ctx, ax);
ax_scaled = modulate(ctx->ggml_ctx, ax_scaled, a_ff_mods[0], a_ff_mods[1]);
ax_scaled = LTXV::modulate(ctx->ggml_ctx, ax_scaled, a_ff_mods[0], a_ff_mods[1]);
auto a_ff_out = audio_ff->forward(ctx, ax_scaled);
ax = ggml_add(ctx->ggml_ctx, ax, apply_gate(ctx->ggml_ctx, a_ff_out, a_ff_mods[2]));
}
auto v_ff_mods = get_ada_values(ctx, v_table, v_timestep, v_dim, cross_attention_adaln ? 9 : 6, 3, 3);
auto vx_scaled = rms_norm(ctx->ggml_ctx, vx);
vx_scaled = modulate(ctx->ggml_ctx, vx_scaled, v_ff_mods[0], v_ff_mods[1]);
vx_scaled = LTXV::modulate(ctx->ggml_ctx, vx_scaled, v_ff_mods[0], v_ff_mods[1]);
auto v_ff_out = ff->forward(ctx, vx_scaled);
vx = ggml_add(ctx->ggml_ctx, vx, apply_gate(ctx->ggml_ctx, v_ff_out, v_ff_mods[2]));
@@ -1657,14 +1657,14 @@ namespace LTXV {
auto v_shift_scale = get_output_scale_shift(ctx, params["scale_shift_table"], v_embedded_time, config.hidden_size);
vx = norm_out->forward(ctx, vx);
vx = modulate(ctx->ggml_ctx, vx, v_shift_scale[0], v_shift_scale[1]);
vx = LTXV::modulate(ctx->ggml_ctx, vx, v_shift_scale[0], v_shift_scale[1]);
vx = proj_out->forward(ctx, vx);
vx = unpatchify_video(ctx, vx, width, height, frames);
if (ax != nullptr && audio_time > 0) {
auto a_shift_scale = get_output_scale_shift(ctx, params["audio_scale_shift_table"], a_embedded_time, config.audio_hidden_size);
ax = audio_norm_out->forward(ctx, ax);
ax = modulate(ctx->ggml_ctx, ax, a_shift_scale[0], a_shift_scale[1]);
ax = LTXV::modulate(ctx->ggml_ctx, ax, a_shift_scale[0], a_shift_scale[1]);
ax = audio_proj_out->forward(ctx, ax);
ax = unpatchify_audio(ctx, ax, audio_time);
}

94
src/model/diffusion/mmdit.hpp
View File

@@ -136,11 +136,15 @@ struct MMDiTConfig {
};
struct PatchEmbed : public GGMLBlock {
// 2D Image to Patch Embedding
// 2D/3D Image to Patch Embedding
protected:
bool is_3d;
bool flatten;
bool dynamic_img_pad;
int patch_size;
int patch_t;
int patch_h;
int patch_w;
int64_t embed_dim;
public:
PatchEmbed(int64_t img_size = 224,
@@ -149,42 +153,90 @@ public:
int64_t embed_dim = 1536,
bool bias = true,
bool flatten = true,
bool dynamic_img_pad = true)
: patch_size(patch_size),
bool dynamic_img_pad = true,
bool is_3d = false)
: patch_t(is_3d ? patch_size : 1),
patch_h(patch_size),
patch_w(patch_size),
embed_dim(embed_dim),
flatten(flatten),
dynamic_img_pad(dynamic_img_pad) {
dynamic_img_pad(dynamic_img_pad),
is_3d(is_3d) {
// img_size is always None
// patch_size is always 2
// in_chans is always 16
// norm_layer is always False
// strict_img_size is always true, but not used
blocks["proj"] = std::shared_ptr<GGMLBlock>(new Conv2d(in_chans,
embed_dim,
{patch_size, patch_size},
{patch_size, patch_size},
{0, 0},
{1, 1},
bias));
if (is_3d) {
blocks["proj"] = std::make_shared<Conv3d>(in_chans,
embed_dim,
std::tuple{patch_size, patch_size, patch_size},
std::tuple{patch_size, patch_size, patch_size},
std::tuple{0, 0, 0},
std::tuple{1, 1, 1},
bias);
} else {
blocks["proj"] = std::make_shared<Conv2d>(in_chans,
embed_dim,
std::pair{patch_size, patch_size},
std::pair{patch_size, patch_size},
std::pair{0, 0},
std::pair{1, 1},
bias);
}
}
PatchEmbed(int64_t img_size,
std::tuple<int, int, int> patch_size,
int64_t in_chans,
int64_t embed_dim,
bool bias = true,
bool flatten = true,
bool dynamic_img_pad = true)
: patch_t(std::get<0>(patch_size)),
patch_h(std::get<1>(patch_size)),
patch_w(std::get<2>(patch_size)),
embed_dim(embed_dim),
flatten(flatten),
dynamic_img_pad(dynamic_img_pad),
is_3d(true) {
SD_UNUSED(img_size);
blocks["proj"] = std::make_shared<Conv3d>(in_chans,
embed_dim,
patch_size,
patch_size,
std::tuple{0, 0, 0},
std::tuple{1, 1, 1},
bias);
}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
// x: [N, C, H, W]
// return: [N, H*W, embed_dim]
auto proj = std::dynamic_pointer_cast<Conv2d>(blocks["proj"]);
// x: [N, C, H, W] or [N*C, T, H, W]
// return: [N, h_len*w_len, embed_dim] or [N, t_len*h_len*w_len, embed_dim]
auto proj = std::dynamic_pointer_cast<UnaryBlock>(blocks["proj"]);
if (dynamic_img_pad) {
int64_t W = x->ne[0];
int64_t H = x->ne[1];
int pad_h = (patch_size - H % patch_size) % patch_size;
int pad_w = (patch_size - W % patch_size) % patch_size;
x = ggml_pad(ctx->ggml_ctx, x, pad_w, pad_h, 0, 0); // TODO: reflect pad mode
int pad_t = 0;
int pad_h = (patch_h - static_cast<int>(H % patch_h)) % patch_h;
int pad_w = (patch_w - static_cast<int>(W % patch_w)) % patch_w;
if (is_3d) {
int64_t T = x->ne[2];
pad_t = (patch_t - static_cast<int>(T % patch_t)) % patch_t;
}
x = ggml_pad(ctx->ggml_ctx, x, pad_w, pad_h, pad_t, 0); // TODO: reflect pad mode
}
x = proj->forward(ctx, x);
x = proj->forward(ctx, x); // [N, C, h_len, w_len] or [N*C, t_len, h_len, w_len]
if (flatten) {
x = ggml_reshape_3d(ctx->ggml_ctx, x, x->ne[0] * x->ne[1], x->ne[2], x->ne[3]);
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
if (is_3d) {
x = ggml_reshape_3d(ctx->ggml_ctx, x, x->ne[0] * x->ne[1] * x->ne[2], embed_dim, x->ne[3] / embed_dim); // [N, C, t_len*h_len*w_len]
} else {
x = ggml_reshape_3d(ctx->ggml_ctx, x, x->ne[0] * x->ne[1], x->ne[2], x->ne[3]); // [N, C, h_len*w_len]
}
x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3)); // [N, h_len*w_len, C]
}
return x;
}

10
src/model/diffusion/model.hpp
View File

@@ -87,6 +87,13 @@ struct MiniT2IDiffusionExtra {
const sd::Tensor<float>* mask = nullptr;
};
struct HunyuanVideoDiffusionExtra {
const sd::Tensor<float>* guidance = nullptr;
const sd::Tensor<float>* byt5 = nullptr;
const sd::Tensor<float>* vision = nullptr;
const sd::Tensor<float>* timestep_r = nullptr;
};
using DiffusionExtraParams = std::variant<std::monostate,
UNetDiffusionExtra,
SkipLayerDiffusionExtra,
@@ -95,7 +102,8 @@ using DiffusionExtraParams = std::variant<std::monostate,
WanDiffusionExtra,
HiDreamO1DiffusionExtra,
LTXAVDiffusionExtra,
MiniT2IDiffusionExtra>;
MiniT2IDiffusionExtra,
HunyuanVideoDiffusionExtra>;
struct DiffusionParams {
const sd::Tensor<float>* x = nullptr;