upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-09-29 17:38:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

refactor: split ggml extensions and move implementations to cpp files (#1945)

Browse Source
This commit is contained in:
leejet
2026-09-07 23:34:45 +08:00
committed by GitHub
parent d8fb10c029
commit 31ab2b2e08
74 changed files with 4269 additions and 3698 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/model/adapter/ip_adapter.hpp
+4 -2
View File
@@ -1,8 +1,10 @@
#ifndef __SD_MODEL_ADAPTER_IP_ADAPTER_HPP__
#define __SD_MODEL_ADAPTER_IP_ADAPTER_HPP__
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_runner.h"
#include "model/common/block.hpp"
#include "model/common/ggml_block.hpp"
#include "model_loader.h"
namespace IPAdapter {
@@ -200,7 +202,7 @@ namespace IPAdapter {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(image_embeds);
};
return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, true));
return take_or_empty(GGMLRunner::compute(get_graph, n_threads, true));
}
};
src/model/adapter/lora.hpp
+8 -2
View File
@@ -2,7 +2,13 @@
#define __SD_MODEL_ADAPTER_LORA_HPP__
#include <mutex>
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_extend_backend.h"
#include "core/ggml_runner.h"
#include "core/ggml_tensor_utils.h"
#include "core/util.h"
#include "model.h"
#include "model/adapter/lora_ops.h"
#include "model_loader.h"
#include "model_manager.h"
@@ -957,7 +963,7 @@ struct LoraModel : public GGMLRunner {
}
return true;
};
auto result = GGMLRunner::compute<float>(get_graph, n_threads, false, true, read_outputs);
auto result = GGMLRunner::compute(get_graph, n_threads, false, true, read_outputs);
if (!result.has_value()) {
LOG_ERROR("LoRA graph execution failed");
}
src/model/adapter/lora_ops.cpp
+207
View File
@@ -0,0 +1,207 @@
#include "model/adapter/lora_ops.h"
#include <cmath>
#include "core/ggml_extend.h"
#include "core/ggml_extend_backend.h"
ggml_tensor* ggml_ext_merge_lora(ggml_context* ctx,
ggml_tensor* lora_down,
ggml_tensor* lora_up,
ggml_tensor* lora_mid) {
ggml_tensor* updown;
// flat lora tensors to multiply it
int64_t lora_up_rows = lora_up->ne[ggml_n_dims(lora_up) - 1];
lora_up = ggml_reshape_2d(ctx, lora_up, ggml_nelements(lora_up) / lora_up_rows, lora_up_rows);
auto lora_down_n_dims = ggml_n_dims(lora_down);
// assume n_dims should always be a multiple of 2 (otherwise rank 1 doesn't work)
lora_down_n_dims = (lora_down_n_dims + lora_down_n_dims % 2);
int64_t lora_down_rows = lora_down->ne[lora_down_n_dims - 1];
lora_down = ggml_reshape_2d(ctx, lora_down, ggml_nelements(lora_down) / lora_down_rows, lora_down_rows);
// ggml_mul_mat requires tensor b transposed
lora_down = ggml_cont(ctx, ggml_transpose(ctx, lora_down));
if (lora_mid == nullptr) {
updown = ggml_mul_mat(ctx, lora_up, lora_down);
updown = ggml_cont(ctx, ggml_transpose(ctx, updown));
} else {
// undoing tucker decomposition for conv layers.
// lora_mid has shape (3, 3, Rank, Rank)
// lora_down has shape (Rank, In, 1, 1)
// lora_up has shape (Rank, Out, 1, 1)
// conv layer shape is (3, 3, Out, In)
updown = ggml_ext_mul_n_mode(ctx, ggml_ext_mul_n_mode(ctx, lora_mid, lora_down, 3), lora_up, 2);
updown = ggml_cont(ctx, updown);
}
return updown;
}
ggml_tensor* ggml_ext_lokr_forward(
ggml_context* ctx,
ggml_backend_t backend,
ggml_tensor* h, // Input: [q, batch] or [W, H, q, batch]
ggml_tensor* w1, // Outer C (Full rank)
ggml_tensor* w1a, // Outer A (Low rank part 1)
ggml_tensor* w1b, // Outer B (Low rank part 2)
ggml_tensor* w2, // Inner BA (Full rank)
ggml_tensor* w2a, // Inner A (Low rank part 1)
ggml_tensor* w2b, // Inner B (Low rank part 2)
bool is_conv,
WeightAdapter::ForwardParams::conv2d_params_t conv_params,
float scale) {
GGML_ASSERT((w1 != nullptr || (w1a != nullptr && w1b != nullptr)));
GGML_ASSERT((w2 != nullptr || (w2a != nullptr && w2b != nullptr)));
int uq = (w1 != nullptr) ? (int)w1->ne[0] : (int)w1a->ne[0];
int up = (w1 != nullptr) ? (int)w1->ne[1] : (int)w1b->ne[1];
int q_actual = is_conv ? (int)h->ne[2] : (int)h->ne[0];
int vq = q_actual / uq;
int vp = (w2 != nullptr) ? (is_conv ? (int)w2->ne[3] : (int)w2->ne[1])
: (int)w2a->ne[1];
GGML_ASSERT(q_actual == (uq * vq) && "Input dimension mismatch for LoKR split");
ggml_tensor* hb;
if (!is_conv) {
int batch = (int)h->ne[1];
int merge_batch_uq = batch;
int merge_batch_vp = batch;
if (sd_backend_is(backend, "Vulkan")) {
if (batch > 1) {
// no access to backend here, worst case is slightly worse perfs for other backends when built alongside Vulkan backend
int max_batch = 65535;
int max_batch_uq = max_batch / uq;
merge_batch_uq = 1;
for (int i = max_batch_uq; i > 0; i--) {
if (batch % i == 0) {
merge_batch_uq = i;
break;
}
}
int max_batch_vp = max_batch / vp;
merge_batch_vp = 1;
for (int i = max_batch_vp; i > 0; i--) {
if (batch % i == 0) {
merge_batch_vp = i;
break;
}
}
}
}
ggml_tensor* h_split = ggml_reshape_3d(ctx, h, vq, uq * merge_batch_uq, batch / merge_batch_uq);
if (w2 != nullptr) {
hb = ggml_mul_mat(ctx, w2, h_split);
} else {
hb = ggml_mul_mat(ctx, w2b, ggml_mul_mat(ctx, w2a, h_split));
}
if (batch > 1) {
hb = ggml_reshape_3d(ctx, hb, vp, uq, batch);
}
ggml_tensor* hb_t = ggml_cont(ctx, ggml_transpose(ctx, hb));
hb_t = ggml_reshape_3d(ctx, hb_t, uq, vp * merge_batch_vp, batch / merge_batch_vp);
ggml_tensor* hc_t;
if (w1 != nullptr) {
hc_t = ggml_mul_mat(ctx, w1, hb_t);
} else {
hc_t = ggml_mul_mat(ctx, w1b, ggml_mul_mat(ctx, w1a, hb_t));
}
if (batch > 1) {
hc_t = ggml_reshape_3d(ctx, hc_t, up, vp, batch);
}
ggml_tensor* hc = ggml_transpose(ctx, hc_t);
ggml_tensor* out = ggml_reshape_2d(ctx, ggml_cont(ctx, hc), up * vp, batch);
return ggml_ext_scale(ctx, out, scale);
} else {
int batch = (int)h->ne[3];
// 1. Reshape input: [W, H, vq*uq, batch] -> [W, H, vq, uq * batch]
ggml_tensor* h_split = ggml_reshape_4d(ctx, h, h->ne[0], h->ne[1], vq, uq * batch);
if (w2 != nullptr) {
hb = ggml_ext_conv_2d(ctx, h_split, w2, nullptr,
conv_params.s0,
conv_params.s1,
conv_params.p0,
conv_params.p1,
conv_params.d0,
conv_params.d1,
conv_params.direct,
conv_params.circular_x,
conv_params.circular_y,
conv_params.scale);
} else {
// swap a and b order for conv lora
ggml_tensor* a = w2b;
ggml_tensor* b = w2a;
// unpack conv2d weights if needed
if (ggml_n_dims(a) < 4) {
int k = (int)sqrt(a->ne[0] / h_split->ne[2]);
GGML_ASSERT(k * k * h_split->ne[2] == a->ne[0]);
a = ggml_reshape_4d(ctx, a, k, k, a->ne[0] / (k * k), a->ne[1]);
} else if (a->ne[2] != h_split->ne[2]) {
int k = (int)sqrt(a->ne[2] / h_split->ne[2]);
GGML_ASSERT(k * k * h_split->ne[2] == a->ne[2]);
a = ggml_reshape_4d(ctx, a, a->ne[0] * k, a->ne[1] * k, a->ne[2] / (k * k), a->ne[3]);
}
ggml_tensor* ha = ggml_ext_conv_2d(ctx, h_split, a, nullptr,
conv_params.s0,
conv_params.s1,
conv_params.p0,
conv_params.p1,
conv_params.d0,
conv_params.d1,
conv_params.direct,
conv_params.circular_x,
conv_params.circular_y,
conv_params.scale);
// not supporting lora_mid here
hb = ggml_ext_conv_2d(ctx,
ha,
b,
nullptr,
1,
1,
0,
0,
1,
1,
conv_params.direct,
conv_params.circular_x,
conv_params.circular_y,
conv_params.scale);
}
// Current hb shape: [W_out, H_out, vp, uq * batch]
int w_out = (int)hb->ne[0];
int h_out = (int)hb->ne[1];
// ggml_tensor* hb_cat = ggml_reshape_4d(ctx, hb, w_out , h_out , vp * uq, batch);
// [W_out, H_out, vp * uq, batch]
// Now left to compute (W1 kr Id) * hb_cat == (W1 kr W2) cv h
// merge the uq groups of size vp*w_out*h_out
ggml_tensor* hb_merged = ggml_reshape_2d(ctx, hb, w_out * h_out * vp, uq * batch);
ggml_tensor* hc_t;
ggml_tensor* hb_merged_t = ggml_cont(ctx, ggml_transpose(ctx, hb_merged));
if (w1 != nullptr) {
// Would be great to be able to transpose w1 instead to avoid transposing both hb and hc
hc_t = ggml_mul_mat(ctx, w1, hb_merged_t);
} else {
hc_t = ggml_mul_mat(ctx, w1b, ggml_mul_mat(ctx, w1a, hb_merged_t));
}
ggml_tensor* hc = ggml_transpose(ctx, hc_t);
// ungroup
ggml_tensor* out = ggml_reshape_4d(ctx, ggml_cont(ctx, hc), w_out, h_out, up * vp, batch);
return ggml_ext_scale(ctx, out, scale);
}
}
src/model/adapter/lora_ops.h
+25
View File
@@ -0,0 +1,25 @@
#ifndef __SD_MODEL_ADAPTER_LORA_OPS_H__
#define __SD_MODEL_ADAPTER_LORA_OPS_H__
#include "core/ggml_runner.h"
ggml_tensor* ggml_ext_merge_lora(ggml_context* ctx,
ggml_tensor* lora_down,
ggml_tensor* lora_up,
ggml_tensor* lora_mid = nullptr);
ggml_tensor* ggml_ext_lokr_forward(
ggml_context* ctx,
ggml_backend_t backend,
ggml_tensor* h, // Input: [q, batch] or [W, H, q, batch]
ggml_tensor* w1, // Outer C (Full rank)
ggml_tensor* w1a, // Outer A (Low rank part 1)
ggml_tensor* w1b, // Outer B (Low rank part 2)
ggml_tensor* w2, // Inner BA (Full rank)
ggml_tensor* w2a, // Inner A (Low rank part 1)
ggml_tensor* w2b, // Inner B (Low rank part 2)
bool is_conv,
WeightAdapter::ForwardParams::conv2d_params_t conv_params,
float scale);
#endif // __SD_MODEL_ADAPTER_LORA_OPS_H__
src/model/adapter/pmid.hpp
+5 -2
View File
@@ -1,7 +1,10 @@
#ifndef __SD_MODEL_ADAPTER_PMID_HPP__
#define __SD_MODEL_ADAPTER_PMID_HPP__
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_runner.h"
#include "core/util.h"
#include "model/common/ggml_block.hpp"
#include "model/adapter/lora.hpp"
#include "model/common/block.hpp"
@@ -558,7 +561,7 @@ public:
return build_graph(id_pixel_values, prompt_embeds, class_tokens_mask, id_embeds);
};
return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, true));
return take_or_empty(GGMLRunner::compute(get_graph, n_threads, true));
}
};
src/model/adapter/pulid.hpp
+3 -1
View File
@@ -1,8 +1,10 @@
#ifndef __PULID_HPP__
#define __PULID_HPP__
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_runner.h"
#include "model/common/block.hpp"
#include "model/common/ggml_block.hpp"
class PuLIDPerceiverAttentionCA : public GGMLBlock {
public:
src/model/common/block.hpp
+3 -1
View File
@@ -1,9 +1,11 @@
#ifndef __SD_MODEL_COMMON_BLOCK_HPP__
#define __SD_MODEL_COMMON_BLOCK_HPP__
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_runner.h"
#include "core/util.h"
#include "ggml-backend.h"
#include "model/common/ggml_block.hpp"
class DownSampleBlock : public GGMLBlock {
protected:
src/model/common/ggml_block.hpp
+868
View File
@@ -0,0 +1,868 @@
#ifndef __SD_MODEL_COMMON_GGML_BLOCK_HPP__
#define __SD_MODEL_COMMON_GGML_BLOCK_HPP__
#include <cstdint>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <tuple>
#include <unordered_map>
#include <utility>
#include <vector>
#include "core/ggml_extend.h"
#include "core/ggml_runner.h"
#include "model.h"
class GGMLBlock {
protected:
typedef std::unordered_map<std::string, ggml_tensor*> ParameterMap;
typedef std::unordered_map<std::string, std::shared_ptr<GGMLBlock>> GGMLBlockMap;
GGMLBlockMap blocks;
ParameterMap params;
ggml_type get_type(const std::string& name, const String2TensorStorage& tensor_storage_map, ggml_type default_type) {
ggml_type wtype = default_type;
auto iter = tensor_storage_map.find(name);
if (iter != tensor_storage_map.end()) {
const TensorStorage& tensor_storage = iter->second;
if (tensor_storage.expected_type != GGML_TYPE_COUNT) {
wtype = tensor_storage.expected_type;
} else {
wtype = tensor_storage.type;
}
}
return wtype;
}
void init_blocks(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") {
for (auto& pair : blocks) {
auto& block = pair.second;
block->init(ctx, tensor_storage_map, prefix + pair.first);
}
}
virtual void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") {}
virtual enum ggml_op param_usage_op(const std::string& name) const {
(void)name;
return GGML_OP_NONE;
}
public:
void init(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, std::string prefix = "") {
if (prefix.size() > 0) {
prefix = prefix + ".";
}
init_params(ctx, tensor_storage_map, prefix);
init_blocks(ctx, tensor_storage_map, prefix);
}
size_t get_params_num() {
size_t num_tensors = params.size();
for (auto& pair : blocks) {
auto& block = pair.second;
num_tensors += block->get_params_num();
}
return num_tensors;
};
size_t get_params_mem_size() {
size_t mem_size = 0;
for (auto& pair : blocks) {
auto& block = pair.second;
mem_size += block->get_params_mem_size();
}
for (auto& pair : params) {
mem_size += ggml_nbytes(pair.second);
}
return mem_size;
}
void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, std::string prefix = "") {
if (prefix.size() > 0) {
prefix = prefix + ".";
}
for (auto& pair : blocks) {
auto& block = pair.second;
block->get_param_tensors(tensors, prefix + pair.first);
}
for (auto& pair : params) {
ggml_tensor* param = pair.second;
tensors[prefix + pair.first] = pair.second;
ggml_set_name(param, (prefix + pair.first).c_str());
}
}
void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) {
for (auto& pair : blocks) {
pair.second->get_param_tensor_ops(tensor_ops);
}
for (auto& pair : params) {
enum ggml_op op = param_usage_op(pair.first);
if (op != GGML_OP_NONE) {
tensor_ops[pair.second] = op;
}
}
}
virtual std::string get_desc() {
return "GGMLBlock";
}
void get_all_blocks(std::vector<GGMLBlock*>& result) {
result.push_back(this);
for (auto& block_iter : blocks) {
if (block_iter.second) {
block_iter.second->get_all_blocks(result);
}
}
}
};
class UnaryBlock : public GGMLBlock {
public:
virtual ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) = 0;
};
class Identity : public UnaryBlock {
public:
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
return x;
}
};
class Linear : public UnaryBlock {
protected:
int64_t in_features;
int64_t out_features;
bool bias;
bool force_f32;
bool force_prec_f32;
bool has_weight_scale = false;
bool int8_convrot = false;
int int8_convrot_group_size = 0;
float scale;
std::string prefix;
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
this->prefix = prefix;
has_weight_scale = false;
int8_convrot = false;
int8_convrot_group_size = 0;
enum ggml_type wtype = get_type(prefix + "weight", tensor_storage_map, GGML_TYPE_F32);
if (in_features % ggml_blck_size(wtype) != 0 || force_f32) {
wtype = GGML_TYPE_F32;
}
params["weight"] = ggml_new_tensor_2d(ctx, wtype, in_features, out_features);
if (bias) {
enum ggml_type wtype = GGML_TYPE_F32;
params["bias"] = ggml_new_tensor_1d(ctx, wtype, out_features);
}
auto weight_storage = tensor_storage_map.find(prefix + "weight");
const bool is_int8_tensorwise = weight_storage != tensor_storage_map.end() && weight_storage->second.is_int8_tensorwise;
auto weight_scale_storage = tensor_storage_map.find(prefix + "weight_scale");
if (weight_scale_storage != tensor_storage_map.end()) {
const int64_t scale_nelements = weight_scale_storage->second.nelements();
GGML_ASSERT(scale_nelements == 1 || scale_nelements == out_features);
params["weight_scale"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, scale_nelements);
has_weight_scale = true;
}
if (is_int8_tensorwise) {
GGML_ASSERT(wtype == GGML_TYPE_I8);
GGML_ASSERT(has_weight_scale);
int8_convrot = weight_storage->second.int8_convrot;
int8_convrot_group_size = weight_storage->second.int8_convrot_group_size;
}
}
public:
Linear(int64_t in_features,
int64_t out_features,
bool bias = true,
bool force_f32 = false,
bool force_prec_f32 = false,
float scale = 1.f)
: in_features(in_features),
out_features(out_features),
bias(bias),
force_f32(force_f32),
force_prec_f32(force_prec_f32),
scale(scale) {}
void set_scale(float scale_) {
scale = scale_;
}
void set_force_prec_f32(bool force_prec_f32_) {
force_prec_f32 = force_prec_f32_;
}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
ggml_tensor* w = params["weight"];
ggml_tensor* weight_scale = has_weight_scale ? params["weight_scale"] : nullptr;
if (w->type == GGML_TYPE_F8_E4M3 || w->type == GGML_TYPE_F8_E5M2) {
bool supports_fp8_matmul = false;
if (ctx->backend != nullptr) {
ggml_tensor* fp8_matmul = ggml_mul_mat(ctx->ggml_ctx, w, x);
if (force_prec_f32) {
ggml_mul_mat_set_prec(fp8_matmul, GGML_PREC_F32);
}
supports_fp8_matmul = ggml_backend_supports_op(ctx->backend, fp8_matmul);
}
if (!supports_fp8_matmul) {
w = ggml_cast(ctx->ggml_ctx, w, GGML_TYPE_BF16);
}
}
ggml_tensor* b = nullptr;
if (bias) {
b = params["bias"];
}
ggml_tensor* linear_bias = has_weight_scale ? nullptr : b;
ggml_tensor* out = nullptr;
if (w->type == GGML_TYPE_I8) {
if (x->type != GGML_TYPE_F32) {
x = ggml_ext_cast_f32(ctx->ggml_ctx, ctx->backend, x);
}
if (!ggml_is_contiguous(x)) {
x = ggml_cont(ctx->ggml_ctx, x);
}
ggml_tensor* lora_input = x;
if (ctx->weight_adapter && b != nullptr) {
b = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, b, prefix + "bias");
}
if (int8_convrot && scale == 1.f) {
const auto cache_key = std::make_pair(x, int8_convrot_group_size);
auto cached = ctx->int8_convrot_cache.find(cache_key);
if (cached == ctx->int8_convrot_cache.end()) {
x = ggml_quantize_i8_convrot(ctx->ggml_ctx, x, int8_convrot_group_size);
ctx->int8_convrot_cache.emplace(cache_key, x);
} else {
x = cached->second;
}
}
out = ggml_ext_linear_i8_tensorwise(ctx->ggml_ctx,
x,
w,
weight_scale,
b,
int8_convrot ? int8_convrot_group_size : 0,
scale);
if (ctx->weight_adapter) {
WeightAdapter::ForwardParams forward_params;
forward_params.op_type = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR;
forward_params.linear.force_prec_f32 = force_prec_f32;
forward_params.linear.scale = scale;
out = ctx->weight_adapter->add_lora_to_output(ctx->ggml_ctx,
ctx->backend,
lora_input,
w,
out,
prefix,
forward_params);
}
return out;
}
if (has_weight_scale) {
out = ggml_ext_linear(ctx->ggml_ctx, x, w, nullptr, force_prec_f32, scale);
out = ggml_mul(ctx->ggml_ctx, out, weight_scale);
if (ctx->weight_adapter) {
WeightAdapter::ForwardParams forward_params;
forward_params.op_type = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR;
forward_params.linear.force_prec_f32 = force_prec_f32;
forward_params.linear.scale = scale;
out = ctx->weight_adapter->add_lora_to_output(ctx->ggml_ctx,
ctx->backend,
x,
w,
out,
prefix,
forward_params);
if (b != nullptr) {
b = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, b, prefix + "bias");
}
}
if (b != nullptr) {
out = ggml_add_inplace(ctx->ggml_ctx, out, b);
}
return out;
}
if (ctx->weight_adapter) {
WeightAdapter::ForwardParams forward_params;
forward_params.op_type = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR;
forward_params.linear.force_prec_f32 = force_prec_f32;
forward_params.linear.scale = scale;
out = ctx->weight_adapter->forward_with_lora(ctx->ggml_ctx, ctx->backend, x, w, linear_bias, prefix, forward_params);
} else {
out = ggml_ext_linear(ctx->ggml_ctx, x, w, linear_bias, force_prec_f32, scale);
}
return out;
}
};
__STATIC_INLINE__ bool support_get_rows(ggml_type wtype) {
std::set<ggml_type> allow_types = {GGML_TYPE_F16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0};
if (allow_types.find(wtype) != allow_types.end()) {
return true;
}
return false;
}
class Embedding : public UnaryBlock {
protected:
int64_t embedding_dim;
int64_t num_embeddings;
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map, const std::string prefix = "") override {
enum ggml_type wtype = get_type(prefix + "weight", tensor_storage_map, GGML_TYPE_F32);
if (!support_get_rows(wtype)) {
wtype = GGML_TYPE_F32;
}
params["weight"] = ggml_new_tensor_2d(ctx, wtype, embedding_dim, num_embeddings);
}
enum ggml_op param_usage_op(const std::string& name) const override {
return name == "weight" ? GGML_OP_GET_ROWS : GGML_OP_NONE;
}
public:
Embedding(int64_t num_embeddings, int64_t embedding_dim)
: embedding_dim(embedding_dim),
num_embeddings(num_embeddings) {
}
ggml_tensor* forward(GGMLRunnerContext* ctx,
ggml_tensor* input_ids) override {
// input_ids: [N, n_token]
auto weight = params["weight"];
// There are issues with ggml batch inference, so we are expanding it here first.
// TODO: fix ggml batch inference
int64_t n = input_ids->ne[1];
input_ids = ggml_reshape_1d(ctx->ggml_ctx, input_ids, input_ids->ne[0] * input_ids->ne[1]);
input_ids = ggml_reshape_3d(ctx->ggml_ctx, input_ids, input_ids->ne[0], 1, input_ids->ne[1]);
auto embedding = ggml_get_rows(ctx->ggml_ctx, weight, input_ids);
embedding = ggml_reshape_3d(ctx->ggml_ctx, embedding, embedding->ne[0], embedding->ne[1] / n, n);
// [N, n_token, embedding_dim]
return embedding;
}
};
class Conv2d : public UnaryBlock {
protected:
int64_t in_channels;
int64_t out_channels;
std::pair<int, int> kernel_size;
std::pair<int, int> stride;
std::pair<int, int> padding;
std::pair<int, int> dilation;
bool bias;
float scale = 1.f;
std::string prefix;
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map, const std::string prefix = "") override {
this->prefix = prefix;
enum ggml_type wtype = GGML_TYPE_F16;
params["weight"] = ggml_new_tensor_4d(ctx, wtype, kernel_size.second, kernel_size.first, in_channels, out_channels);
if (bias) {
enum ggml_type wtype = GGML_TYPE_F32;
params["bias"] = ggml_new_tensor_1d(ctx, wtype, out_channels);
}
}
public:
Conv2d(int64_t in_channels,
int64_t out_channels,
std::pair<int, int> kernel_size,
std::pair<int, int> stride = {1, 1},
std::pair<int, int> padding = {0, 0},
std::pair<int, int> dilation = {1, 1},
bool bias = true)
: in_channels(in_channels),
out_channels(out_channels),
kernel_size(kernel_size),
stride(stride),
padding(padding),
dilation(dilation),
bias(bias) {}
void set_scale(float scale_value) {
scale = scale_value;
}
std::string get_desc() override {
return "Conv2d";
}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
ggml_tensor* w = params["weight"];
ggml_tensor* b = nullptr;
if (bias) {
b = params["bias"];
}
if (ctx->weight_adapter) {
WeightAdapter::ForwardParams forward_params;
forward_params.op_type = WeightAdapter::ForwardParams::op_type_t::OP_CONV2D;
forward_params.conv2d.s0 = stride.second;
forward_params.conv2d.s1 = stride.first;
forward_params.conv2d.p0 = padding.second;
forward_params.conv2d.p1 = padding.first;
forward_params.conv2d.d0 = dilation.second;
forward_params.conv2d.d1 = dilation.first;
forward_params.conv2d.direct = ctx->conv2d_direct_enabled;
forward_params.conv2d.circular_x = ctx->circular_x_enabled;
forward_params.conv2d.circular_y = ctx->circular_y_enabled;
forward_params.conv2d.scale = scale;
return ctx->weight_adapter->forward_with_lora(ctx->ggml_ctx, ctx->backend, x, w, b, prefix, forward_params);
}
return ggml_ext_conv_2d(ctx->ggml_ctx,
x,
w,
b,
stride.second,
stride.first,
padding.second,
padding.first,
dilation.second,
dilation.first,
ctx->conv2d_direct_enabled,
ctx->circular_x_enabled,
ctx->circular_y_enabled,
scale);
}
};
class Conv2d_grouped : public UnaryBlock {
protected:
int64_t in_channels;
int64_t out_channels;
int groups;
std::pair<int, int> kernel_size;
std::pair<int, int> stride;
std::pair<int, int> padding;
std::pair<int, int> dilation;
bool bias;
float scale = 1.f;
std::string prefix;
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map, const std::string prefix = "") override {
this->prefix = prefix;
enum ggml_type wtype = GGML_TYPE_F16;
params["weight"] = ggml_new_tensor_4d(ctx, wtype, kernel_size.second, kernel_size.first, in_channels / groups, out_channels);
if (bias) {
enum ggml_type wtype = GGML_TYPE_F32;
params["bias"] = ggml_new_tensor_1d(ctx, wtype, out_channels);
}
}
public:
Conv2d_grouped(int64_t in_channels,
int64_t out_channels,
int groups,
std::pair<int, int> kernel_size,
std::pair<int, int> stride = {1, 1},
std::pair<int, int> padding = {0, 0},
std::pair<int, int> dilation = {1, 1},
bool bias = true)
: in_channels(in_channels),
out_channels(out_channels),
groups(groups),
kernel_size(kernel_size),
stride(stride),
padding(padding),
dilation(dilation),
bias(bias) {}
void set_scale(float scale_value) {
scale = scale_value;
}
std::string get_desc() override {
return "Conv2d_grouped";
}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
ggml_tensor* w = params["weight"];
ggml_tensor* b = nullptr;
if (bias) {
b = params["bias"];
}
if (groups == 1) {
if (ctx->weight_adapter) {
WeightAdapter::ForwardParams forward_params;
forward_params.op_type = WeightAdapter::ForwardParams::op_type_t::OP_CONV2D;
forward_params.conv2d.s0 = stride.second;
forward_params.conv2d.s1 = stride.first;
forward_params.conv2d.p0 = padding.second;
forward_params.conv2d.p1 = padding.first;
forward_params.conv2d.d0 = dilation.second;
forward_params.conv2d.d1 = dilation.first;
forward_params.conv2d.direct = ctx->conv2d_direct_enabled;
forward_params.conv2d.circular_x = ctx->circular_x_enabled;
forward_params.conv2d.circular_y = ctx->circular_y_enabled;
forward_params.conv2d.scale = scale;
return ctx->weight_adapter->forward_with_lora(ctx->ggml_ctx, ctx->backend, x, w, b, prefix, forward_params);
}
return ggml_ext_conv_2d(ctx->ggml_ctx, x, w, b,
stride.second, stride.first,
padding.second, padding.first,
dilation.second, dilation.first,
ctx->conv2d_direct_enabled,
ctx->circular_x_enabled,
ctx->circular_y_enabled,
scale);
}
if (groups == in_channels && groups == out_channels) {
ggml_tensor* res;
if (ctx->conv2d_direct_enabled) {
res = ggml_conv_2d_dw_direct(ctx->ggml_ctx, w, x,
stride.second, stride.first,
padding.second, padding.first,
dilation.second, dilation.first);
} else {
res = ggml_conv_2d_dw(ctx->ggml_ctx, w, x,
stride.second, stride.first,
padding.second, padding.first,
dilation.second, dilation.first);
}
if (b) {
b = ggml_reshape_4d(ctx->ggml_ctx, b, 1, 1, b->ne[0], 1);
res = ggml_add_inplace(ctx->ggml_ctx, res, b);
}
return res;
}
int64_t ic_g = in_channels / groups;
int64_t oc_g = out_channels / groups;
std::vector<ggml_tensor*> out_slices(groups);
for (int i = 0; i < groups; ++i) {
size_t x_offset = i * ic_g * x->nb[2];
ggml_tensor* x_i = ggml_view_4d(ctx->ggml_ctx, x,
x->ne[0], x->ne[1], ic_g, x->ne[3],
x->nb[1], x->nb[2], x->nb[3],
x_offset);
size_t w_offset = i * oc_g * w->nb[3];
ggml_tensor* w_i = ggml_view_4d(ctx->ggml_ctx, w,
w->ne[0], w->ne[1], w->ne[2], oc_g,
w->nb[1], w->nb[2], w->nb[3],
w_offset);
ggml_tensor* b_i = nullptr;
if (b) {
size_t b_offset = i * oc_g * b->nb[0];
b_i = ggml_view_1d(ctx->ggml_ctx, b, oc_g, b_offset);
}
if (ctx->weight_adapter) {
WeightAdapter::ForwardParams forward_params;
forward_params.op_type = WeightAdapter::ForwardParams::op_type_t::OP_CONV2D;
forward_params.conv2d.s0 = stride.second;
forward_params.conv2d.s1 = stride.first;
forward_params.conv2d.p0 = padding.second;
forward_params.conv2d.p1 = padding.first;
forward_params.conv2d.d0 = dilation.second;
forward_params.conv2d.d1 = dilation.first;
forward_params.conv2d.direct = ctx->conv2d_direct_enabled;
forward_params.conv2d.circular_x = ctx->circular_x_enabled;
forward_params.conv2d.circular_y = ctx->circular_y_enabled;
forward_params.conv2d.scale = scale;
out_slices[i] = ctx->weight_adapter->forward_with_lora(ctx->ggml_ctx, ctx->backend, x_i, w_i, b_i, prefix, forward_params);
} else {
out_slices[i] = ggml_ext_conv_2d(ctx->ggml_ctx, x_i, w_i, b_i,
stride.second, stride.first,
padding.second, padding.first,
dilation.second, dilation.first,
ctx->conv2d_direct_enabled,
ctx->circular_x_enabled,
ctx->circular_y_enabled,
scale);
}
}
ggml_tensor* out = ggml_ext_vec_concat(ctx->ggml_ctx, out_slices, 2);
return out;
}
};
class Conv3d : public UnaryBlock {
protected:
int64_t in_channels;
int64_t out_channels;
std::tuple<int, int, int> kernel_size;
std::tuple<int, int, int> stride;
std::tuple<int, int, int> padding;
std::tuple<int, int, int> dilation;
bool bias;
bool force_prec_f32;
std::string prefix;
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map, const std::string prefix = "") override {
this->prefix = prefix;
enum ggml_type wtype = GGML_TYPE_F16;
params["weight"] = ggml_new_tensor_4d(ctx,
wtype,
std::get<2>(kernel_size),
std::get<1>(kernel_size),
std::get<0>(kernel_size),
in_channels * out_channels);
if (bias) {
params["bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, out_channels);
}
}
public:
Conv3d(int64_t in_channels,
int64_t out_channels,
std::tuple<int, int, int> kernel_size,
std::tuple<int, int, int> stride = {1, 1, 1},
std::tuple<int, int, int> padding = {0, 0, 0},
std::tuple<int, int, int> dilation = {1, 1, 1},
bool bias = true,
bool force_prec_f32 = false)
: in_channels(in_channels),
out_channels(out_channels),
kernel_size(kernel_size),
stride(stride),
padding(padding),
dilation(dilation),
bias(bias),
force_prec_f32(force_prec_f32) {}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
ggml_tensor* w = params["weight"];
ggml_tensor* b = nullptr;
if (ctx->weight_adapter) {
w = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, w, prefix + "weight");
if (w->type != GGML_TYPE_F16) {
w = ggml_cast(ctx->ggml_ctx, w, GGML_TYPE_F16);
}
}
if (bias) {
b = params["bias"];
if (ctx->weight_adapter) {
b = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, b, prefix + "bias");
}
}
return ggml_ext_conv_3d(ctx->ggml_ctx, ctx->backend, x, w, b, in_channels,
std::get<2>(stride), std::get<1>(stride), std::get<0>(stride),
std::get<2>(padding), std::get<1>(padding), std::get<0>(padding),
std::get<2>(dilation), std::get<1>(dilation), std::get<0>(dilation),
force_prec_f32);
}
};
class LayerNorm : public UnaryBlock {
protected:
int64_t normalized_shape;
float eps;
bool elementwise_affine;
bool bias;
std::string prefix;
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
this->prefix = prefix;
if (elementwise_affine) {
enum ggml_type wtype = GGML_TYPE_F32;
params["weight"] = ggml_new_tensor_1d(ctx, wtype, normalized_shape);
if (bias) {
enum ggml_type wtype = GGML_TYPE_F32;
params["bias"] = ggml_new_tensor_1d(ctx, wtype, normalized_shape);
}
}
}
public:
LayerNorm(int64_t normalized_shape,
float eps = 1e-05f,
bool elementwise_affine = true,
bool bias = true)
: normalized_shape(normalized_shape),
eps(eps),
elementwise_affine(elementwise_affine),
bias(bias) {}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
ggml_tensor* w = nullptr;
ggml_tensor* b = nullptr;
if (elementwise_affine) {
w = params["weight"];
if (ctx->weight_adapter) {
w = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, w, prefix + "weight");
}
if (bias) {
b = params["bias"];
if (ctx->weight_adapter) {
b = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, b, prefix + "bias");
}
}
}
return ggml_ext_layer_norm(ctx->ggml_ctx, x, w, b, eps);
}
};
class GroupNorm : public GGMLBlock {
protected:
int num_groups;
int64_t num_channels;
float eps;
bool affine;
std::string prefix;
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
this->prefix = prefix;
if (affine) {
enum ggml_type wtype = GGML_TYPE_F32;
enum ggml_type bias_wtype = GGML_TYPE_F32;
params["weight"] = ggml_new_tensor_1d(ctx, wtype, num_channels);
params["bias"] = ggml_new_tensor_1d(ctx, bias_wtype, num_channels);
}
}
public:
GroupNorm(int num_groups,
int64_t num_channels,
float eps = 1e-05f,
bool affine = true)
: num_groups(num_groups),
num_channels(num_channels),
eps(eps),
affine(affine) {}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
ggml_tensor* w = nullptr;
ggml_tensor* b = nullptr;
if (affine) {
w = params["weight"];
b = params["bias"];
if (ctx->weight_adapter) {
w = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, w, prefix + "weight");
b = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, b, prefix + "bias");
}
}
return ggml_ext_group_norm(ctx->ggml_ctx, x, w, b, num_groups);
}
};
class GroupNorm32 : public GroupNorm {
public:
GroupNorm32(int64_t num_channels)
: GroupNorm(32, num_channels, 1e-06f) {}
};
class RMSNorm : public UnaryBlock {
protected:
int64_t hidden_size;
float eps;
std::string prefix;
void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, std::string prefix = "") override {
this->prefix = prefix;
enum ggml_type wtype = GGML_TYPE_F32;
params["weight"] = ggml_new_tensor_1d(ctx, wtype, hidden_size);
}
public:
RMSNorm(int64_t hidden_size,
float eps = 1e-06f)
: hidden_size(hidden_size),
eps(eps) {}
ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
ggml_tensor* w = params["weight"];
if (ctx->weight_adapter) {
w = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, w, prefix + "weight");
}
x = ggml_rms_norm(ctx->ggml_ctx, x, eps);
x = ggml_mul_inplace(ctx->ggml_ctx, x, w);
return x;
}
};
class MultiheadAttention : public GGMLBlock {
protected:
int64_t embed_dim;
int64_t n_head;
bool proj_in;
std::string q_proj_name;
std::string k_proj_name;
std::string v_proj_name;
std::string in_proj_name;
std::string out_proj_name;
public:
MultiheadAttention(int64_t embed_dim,
int64_t n_head,
bool qkv_proj_bias = true,
bool out_proj_bias = true,
bool proj_in = false,
std::string q_proj_name = "q_proj",
std::string k_proj_name = "k_proj",
std::string v_proj_name = "v_proj",
std::string in_proj_name = "in_proj",
std::string out_proj_name = "out_proj")
: embed_dim(embed_dim),
n_head(n_head),
proj_in(proj_in),
q_proj_name(q_proj_name),
k_proj_name(k_proj_name),
v_proj_name(v_proj_name),
in_proj_name(in_proj_name),
out_proj_name(out_proj_name) {
if (proj_in) {
blocks[in_proj_name] = std::shared_ptr<GGMLBlock>(new Linear(embed_dim, embed_dim * 3, qkv_proj_bias));
} else {
blocks[q_proj_name] = std::shared_ptr<GGMLBlock>(new Linear(embed_dim, embed_dim, qkv_proj_bias));
blocks[k_proj_name] = std::shared_ptr<GGMLBlock>(new Linear(embed_dim, embed_dim, qkv_proj_bias));
blocks[v_proj_name] = std::shared_ptr<GGMLBlock>(new Linear(embed_dim, embed_dim, qkv_proj_bias));
}
blocks[out_proj_name] = std::shared_ptr<GGMLBlock>(new Linear(embed_dim, embed_dim, out_proj_bias));
}
// x: [N, n_token, embed_dim]
ggml_tensor* forward(GGMLRunnerContext* ctx,
ggml_tensor* x,
ggml_tensor* mask = nullptr) {
auto out_proj = std::dynamic_pointer_cast<Linear>(blocks[out_proj_name]);
ggml_tensor* q;
ggml_tensor* k;
ggml_tensor* v;
if (proj_in) {
auto in_proj = std::dynamic_pointer_cast<Linear>(blocks[in_proj_name]);
auto qkv = in_proj->forward(ctx, x);
auto qkv_vec = split_qkv(ctx->ggml_ctx, qkv);
q = qkv_vec[0];
k = qkv_vec[1];
v = qkv_vec[2];
} else {
auto q_proj = std::dynamic_pointer_cast<Linear>(blocks[q_proj_name]);
auto k_proj = std::dynamic_pointer_cast<Linear>(blocks[k_proj_name]);
auto v_proj = std::dynamic_pointer_cast<Linear>(blocks[v_proj_name]);
q = q_proj->forward(ctx, x);
k = k_proj->forward(ctx, x);
v = v_proj->forward(ctx, x);
}
x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, mask, false); // [N, n_token, embed_dim]
x = out_proj->forward(ctx, x); // [N, n_token, embed_dim]
return x;
}
};
#endif // __SD_MODEL_COMMON_GGML_BLOCK_HPP__
src/model/common/rope.hpp
+5 -1
View File
@@ -2,9 +2,13 @@
#define __SD_MODEL_COMMON_ROPE_HPP__
#include <algorithm>
#include <cassert>
#include <cmath>
#include <set>
#include <vector>
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_runner.h"
#include "core/util.h"
namespace Rope {
enum class EmbedNDLayout {
src/model/detector/yolov8.h
+3 -2
View File
@@ -8,8 +8,9 @@
#include <string>
#include <vector>
#include "core/ggml_extend.hpp"
#include "core/ggml_runner.h"
#include "core/util.h"
#include "model/common/ggml_block.hpp"
struct YOLOv8Config {
std::array<int, 23> out_channels{};
@@ -355,7 +356,7 @@ struct YOLOv8Runner : public GGMLRunner {
sd::Tensor<float> compute(int n_threads, const sd::Tensor<float>& input) {
auto get_graph = [&]() { return build_graph(input); };
return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, false));
return take_or_empty(GGMLRunner::compute(get_graph, n_threads, false));
}
};
src/model/diffusion/anima.hpp
+2 -1
View File
@@ -2,6 +2,7 @@
#define __SD_MODEL_DIFFUSION_ANIMA_HPP__
#include <algorithm>
#include <cinttypes>
#include <cmath>
#include <memory>
#include <utility>
@@ -717,7 +718,7 @@ namespace Anima {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(x, timesteps, context, t5_ids, t5_weights, ref_latents);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/animatediff.hpp
+3 -1
View File
@@ -1,8 +1,10 @@
#ifndef __SD_MODEL_DIFFUSION_ANIMATEDIFF_HPP__
#define __SD_MODEL_DIFFUSION_ANIMATEDIFF_HPP__
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_runner.h"
#include "model/common/block.hpp"
#include "model/common/ggml_block.hpp"
// AnimateDiff (https://arxiv.org/abs/2307.04725) SD 1.5 motion modules.
namespace AnimateDiff {
src/model/diffusion/boogu.hpp
+6 -2
View File
@@ -2,11 +2,15 @@
#define __SD_MODEL_DIFFUSION_BOOGU_HPP__
#include <algorithm>
#include <cinttypes>
#include <cmath>
#include <tuple>
#include <vector>
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_runner.h"
#include "core/util.h"
#include "model/common/ggml_block.hpp"
#include "model/common/rope.hpp"
#include "model/diffusion/dit.hpp"
#include "model/diffusion/model.hpp"
@@ -815,7 +819,7 @@ namespace Boogu {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(x, timesteps, context, ref_latents);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/control.hpp
+1 -1
View File
@@ -435,7 +435,7 @@ struct ControlNet : public GGMLRunner {
}
return true;
};
auto compute_result = GGMLRunner::compute<float>(get_graph, n_threads, false, true, read_outputs);
auto compute_result = GGMLRunner::compute(get_graph, n_threads, false, true, read_outputs);
control_outputs_ggml.clear();
guided_hint_output_ggml = nullptr;
if (!compute_result.has_value()) {
src/model/diffusion/dit.hpp
+2 -1
View File
@@ -1,7 +1,8 @@
#ifndef __SD_MODEL_DIFFUSION_DIT_HPP__
#define __SD_MODEL_DIFFUSION_DIT_HPP__
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_runner.h"
namespace DiT {
inline ggml_tensor* patchify(ggml_context* ctx,
src/model/diffusion/ernie_image.hpp
+2 -1
View File
@@ -1,6 +1,7 @@
#ifndef __SD_MODEL_DIFFUSION_ERNIE_IMAGE_HPP__
#define __SD_MODEL_DIFFUSION_ERNIE_IMAGE_HPP__
#include <cinttypes>
#include <memory>
#include <vector>
@@ -440,7 +441,7 @@ namespace ErnieImage {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(x, timesteps, context);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/flux.hpp
+4 -1
View File
@@ -1,8 +1,11 @@
#ifndef __SD_MODEL_DIFFUSION_FLUX_HPP__
#define __SD_MODEL_DIFFUSION_FLUX_HPP__
#include <cinttypes>
#include <memory>
#include <vector>
#include "core/ggml_extend_backend.h"
#include "core/ggml_tensor_utils.h"
#include "core/util.h"
#include "model/adapter/pulid.hpp"
@@ -1626,7 +1629,7 @@ namespace Flux {
return build_graph(x, timesteps, context, c_concat, y, guidance, ref_latents, ref_index_mode, skip_layers, pulid_id, pulid_id_weight);
};
auto result = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
return result;
}
src/model/diffusion/hidream_o1.hpp
+2 -2
View File
@@ -329,7 +329,7 @@ namespace HiDreamO1 {
auto get_graph = [&]() {
return build_graph(image);
};
auto output = GGMLRunner::compute<float>(get_graph, n_threads, auto_runner_end);
auto output = GGMLRunner::compute(get_graph, n_threads, auto_runner_end);
return output.has_value() ? std::move(output.value()) : sd::Tensor<float>();
}
};
@@ -457,7 +457,7 @@ namespace HiDreamO1 {
auto get_graph = [&]() {
return build_graph(x, timestep, input_ids, input_pos, token_types, vinput_mask, image_embeds, ref_images);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/hunyuan.hpp
+2 -1
View File
@@ -1,6 +1,7 @@
#ifndef __SD_MODEL_DIFFUSION_HUNYUAN_HPP__
#define __SD_MODEL_DIFFUSION_HUNYUAN_HPP__
#include <cinttypes>
#include <memory>
#include "model/common/block.hpp"
@@ -654,7 +655,7 @@ namespace Hunyuan {
return build_graph(x, timesteps, context, c_concat, y, guidance, byt5, vision, timestep_r);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/ideogram4.hpp
+6 -2
View File
@@ -2,14 +2,18 @@
#define __SD_MODEL_DIFFUSION_IDEOGRAM4_HPP__
#include <algorithm>
#include <cinttypes>
#include <cmath>
#include <cstdlib>
#include <memory>
#include <string>
#include <vector>
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_graph_cut.h"
#include "core/ggml_runner.h"
#include "core/util.h"
#include "model/common/ggml_block.hpp"
#include "model/common/rope.hpp"
#include "model/diffusion/model.hpp"
@@ -537,7 +541,7 @@ namespace Ideogram4 {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(x, timesteps, context, use_uncond_model);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/krea2.hpp
+5 -2
View File
@@ -12,8 +12,11 @@
#include <utility>
#include <vector>
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_graph_cut.h"
#include "core/ggml_runner.h"
#include "core/util.h"
#include "model/common/ggml_block.hpp"
#include "model/common/rope.hpp"
#include "model/diffusion/dit.hpp"
#include "model/diffusion/flux.hpp"
@@ -775,7 +778,7 @@ namespace Krea2 {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(x, timesteps, context, ref_latents, ref_image_params);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/lens.hpp
+2 -1
View File
@@ -1,6 +1,7 @@
#ifndef __SD_MODEL_DIFFUSION_LENS_HPP__
#define __SD_MODEL_DIFFUSION_LENS_HPP__
#include <cinttypes>
#include <memory>
#include <vector>
@@ -408,7 +409,7 @@ namespace Lens {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(x, timesteps, context);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/lingbot_video.hpp
+1 -1
View File
@@ -674,7 +674,7 @@ namespace LingBotVideo {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(x, timesteps, context);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/ltxv.hpp
+4 -1
View File
@@ -2,12 +2,15 @@
#define __SD_MODEL_DIFFUSION_LTXV_HPP__
#include <algorithm>
#include <cinttypes>
#include <cmath>
#include <memory>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
#include "core/ggml_extend_backend.h"
#include "core/ggml_tensor_utils.h"
#include "model/common/block.hpp"
#include "model/common/rope.hpp"
@@ -1998,7 +2001,7 @@ namespace LTXV {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(x, timesteps, context, audio_x, audio_timesteps, audio_length, frame_rate, video_positions);
};
auto out = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
auto out = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
return out;
}
src/model/diffusion/mage_flow.hpp
+1 -1
View File
@@ -142,7 +142,7 @@ namespace MageFlow {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(x, timesteps, context, ref_latents);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/minimax_h3.hpp
+5 -3
View File
@@ -2,12 +2,14 @@
#define __SD_MODEL_DIFFUSION_MINIMAX_H3_HPP__
#include <algorithm>
#include <cinttypes>
#include <cmath>
#include <set>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
#include "core/ggml_tensor_utils.h"
#include "core/ggml_graph_cut.h"
#include "model/diffusion/dit.hpp"
@@ -1166,9 +1168,9 @@ namespace MiniMaxH3 {
extra->video_sigma_shift,
extra->audio_sigma_shift);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph,
n_threads,
false),
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph,
n_threads,
false),
params.x->dim());
}
};
src/model/diffusion/minit2i.hpp
+6 -2
View File
@@ -2,6 +2,7 @@
#define __SD_MODEL_DIFFUSION_MINIT2I_HPP__
#include <algorithm>
#include <cinttypes>
#include <cmath>
#include <cstdint>
#include <cstdlib>
@@ -9,7 +10,10 @@
#include <string>
#include <vector>
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_runner.h"
#include "core/util.h"
#include "model/common/ggml_block.hpp"
#include "model/common/rope.hpp"
#include "model/diffusion/dit.hpp"
#include "model/diffusion/model.hpp"
@@ -589,7 +593,7 @@ namespace MiniT2I {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(x, timesteps, context, mask);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/mmdit.hpp
+8 -2
View File
@@ -2,12 +2,18 @@
#define __SD_MODEL_DIFFUSION_MMDIT_HPP__
#include <algorithm>
#include <cinttypes>
#include <memory>
#include <string>
#include <vector>
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_extend_backend.h"
#include "core/ggml_runner.h"
#include "core/ggml_tensor_utils.h"
#include "core/util.h"
#include "model/common/block.hpp"
#include "model/common/ggml_block.hpp"
#include "model/diffusion/model.hpp"
#include "model_loader.h"
@@ -987,7 +993,7 @@ struct MMDiTRunner : public DiffusionModelRunner {
return build_graph(x, timesteps, context, y, skip_layers);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/model.hpp
+1 -1
View File
@@ -5,7 +5,7 @@
#include <utility>
#include <variant>
#include "core/ggml_extend.hpp"
#include "core/ggml_runner.h"
#include "core/tensor_ggml.hpp"
#include "model/common/rope.hpp"
#include "model_manager.h"
src/model/diffusion/pid.hpp
+7 -2
View File
@@ -1,13 +1,18 @@
#ifndef __SD_MODEL_DIFFUSION_PID_HPP__
#define __SD_MODEL_DIFFUSION_PID_HPP__
#include <cinttypes>
#include <cmath>
#include <cstdlib>
#include <memory>
#include <string>
#include <vector>
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_runner.h"
#include "core/ggml_tensor_utils.h"
#include "core/util.h"
#include "model/common/ggml_block.hpp"
#include "model/common/rope.hpp"
#include "model/diffusion/dit.hpp"
#include "model/diffusion/mmdit.hpp"
@@ -938,7 +943,7 @@ namespace Pid {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(x, timesteps, context, lq_latent, degrade_sigma);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/qwen_image.hpp
+3 -1
View File
@@ -2,6 +2,8 @@
#define __SD_MODEL_DIFFUSION_QWEN_IMAGE_HPP__
#include <memory>
#include "core/ggml_extend_backend.h"
#include "core/ggml_tensor_utils.h"
#include "core/util.h"
#include "model/common/block.hpp"
@@ -707,7 +709,7 @@ namespace Qwen {
return build_graph(x, timesteps, context, ref_latents, ref_index_mode);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/sefi_image.hpp
+1
View File
@@ -1,6 +1,7 @@
#ifndef __SD_MODEL_DIFFUSION_SEFI_IMAGE_HPP__
#define __SD_MODEL_DIFFUSION_SEFI_IMAGE_HPP__
#include <cinttypes>
#include <memory>
#include "model/common/block.hpp"
src/model/diffusion/unet.hpp
+2 -1
View File
@@ -3,6 +3,7 @@
#include <algorithm>
#include <vector>
#include "core/ggml_tensor_utils.h"
#include "model.h"
#include "model/common/block.hpp"
@@ -835,7 +836,7 @@ struct UNetModelRunner : public DiffusionModelRunner {
return build_graph(x, timesteps, context, c_concat, y, num_video_frames, controls, control_strength, ip_context, ip_scale);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/wan.hpp
+4 -1
View File
@@ -1,9 +1,12 @@
#ifndef __SD_MODEL_DIFFUSION_WAN_HPP__
#define __SD_MODEL_DIFFUSION_WAN_HPP__
#include <cinttypes>
#include <map>
#include <memory>
#include <utility>
#include "core/ggml_extend_backend.h"
#include "core/ggml_tensor_utils.h"
#include "model/common/block.hpp"
#include "model/common/rope.hpp"
@@ -950,7 +953,7 @@ namespace WAN {
return build_graph(x, timesteps, context, clip_fea, c_concat, time_dim_concat, vace_context, vace_strength);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/diffusion/z_image.hpp
+8 -2
View File
@@ -2,8 +2,14 @@
#define __SD_MODEL_DIFFUSION_Z_IMAGE_HPP__
#include <algorithm>
#include <cinttypes>
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_extend_backend.h"
#include "core/ggml_runner.h"
#include "core/ggml_tensor_utils.h"
#include "core/util.h"
#include "model/common/ggml_block.hpp"
#include "model/diffusion/flux.hpp"
#include "model/diffusion/mmdit.hpp"
#include "model/diffusion/model.hpp"
@@ -636,7 +642,7 @@ namespace ZImage {
return build_graph(x, timesteps, context, ref_latents, ref_index_mode);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
}
sd::Tensor<float> compute(int n_threads,
src/model/te/clip.hpp
+5 -2
View File
@@ -1,8 +1,11 @@
#ifndef __SD_MODEL_TE_CLIP_HPP__
#define __SD_MODEL_TE_CLIP_HPP__
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_runner.h"
#include "core/util.h"
#include "model.h"
#include "model/common/ggml_block.hpp"
#include "tokenizers/clip_tokenizer.h"
/*================================================ FrozenCLIPEmbedder ================================================*/
@@ -572,7 +575,7 @@ struct CLIPTextModelRunner : public GGMLRunner {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(input_ids, num_custom_embeddings, custom_embeddings_data, max_token_idx, return_pooled, clip_skip);
};
auto result = GGMLRunner::compute<float>(get_graph, n_threads, auto_runner_end);
auto result = GGMLRunner::compute(get_graph, n_threads, auto_runner_end);
if (return_pooled) {
return take_or_empty(std::move(result));
}
src/model/te/llm.hpp
+11 -5
View File
@@ -3,6 +3,7 @@
#include <algorithm>
#include <array>
#include <cinttypes>
#include <cmath>
#include <fstream>
#include <functional>
@@ -18,8 +19,13 @@
#include <utility>
#include <vector>
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_extend_backend.h"
#include "core/ggml_runner.h"
#include "core/ggml_tensor_utils.h"
#include "core/util.h"
#include "json.hpp"
#include "model/common/ggml_block.hpp"
#include "model/common/rope.hpp"
#include "model_loader.h"
#include "model_manager.h"
@@ -2091,7 +2097,7 @@ namespace LLM {
out_layers,
return_all_hidden_states);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, auto_runner_end),
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, auto_runner_end),
input_ids.dim() + 1);
}
@@ -2175,7 +2181,7 @@ namespace LLM {
auto get_graph = [&]() -> ggml_cgraph* {
return build_encode_image_graph(image);
};
return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, auto_runner_end));
return take_or_empty(GGMLRunner::compute(get_graph, n_threads, auto_runner_end));
}
ggml_cgraph* build_encode_image_outputs_graph(const sd::Tensor<float>& image_tensor) {
@@ -2287,7 +2293,7 @@ namespace LLM {
auto get_graph = [&]() -> ggml_cgraph* {
return build_encode_image_outputs_graph(image);
};
auto combined = take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, auto_runner_end));
auto combined = take_or_empty(GGMLRunner::compute(get_graph, n_threads, auto_runner_end));
if (combined.empty()) {
return {};
}
@@ -2313,7 +2319,7 @@ namespace LLM {
auto get_graph = [&]() -> ggml_cgraph* {
return build_encode_video_block_outputs_graph(pixel_values, grid_h, grid_w);
};
auto combined = take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, auto_runner_end));
auto combined = take_or_empty(GGMLRunner::compute(get_graph, n_threads, auto_runner_end));
if (combined.empty()) {
return {};
}
src/model/te/t5.hpp
+7 -2
View File
@@ -10,7 +10,12 @@
#include <string>
#include <unordered_map>
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_extend_backend.h"
#include "core/ggml_runner.h"
#include "core/ggml_tensor_utils.h"
#include "core/util.h"
#include "model/common/ggml_block.hpp"
#include "model_loader.h"
#include "model_manager.h"
#include "tokenizers/t5_unigram_tokenizer.h"
@@ -455,7 +460,7 @@ struct T5Runner : public GGMLRunner {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(input_ids, attention_mask);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, auto_runner_end), 3);
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, auto_runner_end), 3);
}
static std::vector<int> _relative_position_bucket(const std::vector<int>& relative_position,
src/model/upscaler/esrgan.hpp
+4 -2
View File
@@ -7,8 +7,10 @@
#include <utility>
#include <vector>
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_runner.h"
#include "core/util.h"
#include "model/common/ggml_block.hpp"
/*
=================================== ESRGAN ===================================
@@ -265,7 +267,7 @@ struct ESRGAN : public GGMLRunner {
sd::Tensor<float> compute(const int n_threads,
const sd::Tensor<float>& x) {
auto get_graph = [&]() -> ggml_cgraph* { return build_graph(x); };
auto result = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
return result;
}
};
src/model/upscaler/ltx_latent_upscaler.hpp
+4 -2
View File
@@ -11,9 +11,11 @@
#include <utility>
#include <vector>
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_graph_cut.h"
#include "core/ggml_runner.h"
#include "core/util.h"
#include "model/common/ggml_block.hpp"
#include "model/diffusion/dit.hpp"
#include "model_loader.h"
@@ -499,7 +501,7 @@ namespace LTXVUpsampler {
}
size_t expected_dim = static_cast<size_t>(x.dim());
auto get_graph = [&]() -> ggml_cgraph* { return build_graph(x); };
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), expected_dim);
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), expected_dim);
}
};
src/model/vae/audio_vae.hpp
+2 -1
View File
@@ -1,7 +1,8 @@
#ifndef __SD_MODEL_VAE_AUDIO_VAE_HPP__
#define __SD_MODEL_VAE_AUDIO_VAE_HPP__
#include "core/ggml_extend.hpp"
#include "core/ggml_runner.h"
#include "core/util.h"
struct AudioVAERunner : public GGMLRunner {
AudioVAERunner(ggml_backend_t backend,
src/model/vae/auto_encoder_kl.hpp
+3 -1
View File
@@ -1,6 +1,8 @@
#ifndef __SD_MODEL_VAE_AUTO_ENCODER_KL_HPP__
#define __SD_MODEL_VAE_AUTO_ENCODER_KL_HPP__
#include <cinttypes>
#include "core/ggml_tensor_utils.h"
#include "model/vae/vae.hpp"
/*================================================== AutoEncoderKL ===================================================*/
@@ -744,7 +746,7 @@ struct AutoEncoderKL : public VAE {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(z, decode_graph);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), z.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), z.dim());
}
sd::Tensor<float> gaussian_latent_sample(const sd::Tensor<float>& moments, std::shared_ptr<RNG> rng) {
src/model/vae/hunyuan_vae.hpp
+3 -3
View File
@@ -825,9 +825,9 @@ namespace Hunyuan {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(graph_input, decode_graph);
};
auto output = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph,
n_threads,
false),
auto output = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph,
n_threads,
false),
graph_input.dim());
if (!output.empty() && input.dim() == 4) {
output.squeeze_(2);
src/model/vae/ltx_audio_vae.hpp
+7 -2
View File
@@ -7,7 +7,12 @@
#include <string>
#include <vector>
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_extend_backend.h"
#include "core/ggml_runner.h"
#include "core/ggml_tensor_utils.h"
#include "core/util.h"
#include "model/common/ggml_block.hpp"
#include "model/vae/audio_vae.hpp"
#include "model_loader.h"
#include "model_manager.h"
@@ -1042,7 +1047,7 @@ namespace LTXV {
ggml_build_forward_expand(gf, waveform);
return gf;
};
auto result = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), 4);
auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), 4);
int64_t t1 = ggml_time_ms();
LOG_INFO("ltx audio vae decode completed, taking %.2fs", (t1 - t0) * 1.0f / 1000);
return result;
src/model/vae/ltx_vae.hpp
+5 -3
View File
@@ -8,6 +8,8 @@
#include <tuple>
#include <utility>
#include <vector>
#include "core/ggml_extend_backend.h"
#include "core/ggml_tensor_utils.h"
#include "model/diffusion/ltxv.hpp"
#include "model/vae/vae.hpp"
@@ -1348,7 +1350,7 @@ struct LTXVideoVAE : public VAE {
static_cast<int>(tile.start),
tile.overlap);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false),
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false),
expected_dim);
});
@@ -1405,7 +1407,7 @@ struct LTXVideoVAE : public VAE {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(input, decode_graph);
};
auto result = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), expected_dim);
auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), expected_dim);
if (result.empty()) {
return {};
}
@@ -1418,7 +1420,7 @@ struct LTXVideoVAE : public VAE {
auto get_graph = [&]() -> ggml_cgraph* {
return build_latent_statistics_graph(z, normalize);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false),
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false),
static_cast<size_t>(z.dim()));
}
src/model/vae/mage_vae.hpp
+1 -1
View File
@@ -490,7 +490,7 @@ namespace MageVAE {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(input, decode_graph);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), input.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), input.dim());
}
int get_encoder_output_channels(int input_channels) override {
src/model/vae/minimax_h3_audio_vae.hpp
+2 -2
View File
@@ -480,7 +480,7 @@ namespace MiniMaxH3 {
return graph;
};
auto result = restore_trailing_singleton_dims(
GGMLRunner::compute<float>(get_graph, n_threads, false),
GGMLRunner::compute(get_graph, n_threads, false),
4);
int64_t t1 = ggml_time_ms();
LOG_INFO("MiniMax-H3 audio VAE encode completed, taking %.2fs",
@@ -500,7 +500,7 @@ namespace MiniMaxH3 {
return graph;
};
auto result = restore_trailing_singleton_dims(
GGMLRunner::compute<float>(get_graph, n_threads, false),
GGMLRunner::compute(get_graph, n_threads, false),
4);
int64_t t1 = ggml_time_ms();
LOG_INFO("MiniMax-H3 audio VAE decode completed, taking %.2fs",
src/model/vae/minimax_h3_vae.hpp
+3 -3
View File
@@ -791,9 +791,9 @@ namespace MiniMaxH3VAE {
return graph;
};
return restore_trailing_singleton_dims(
GGMLRunner::compute<float>(get_graph,
n_threads,
false),
GGMLRunner::compute(get_graph,
n_threads,
false),
5);
}
};
src/model/vae/tae.hpp
+7 -3
View File
@@ -1,8 +1,12 @@
#ifndef __SD_MODEL_VAE_TAE_HPP__
#define __SD_MODEL_VAE_TAE_HPP__
#include "core/ggml_extend.hpp"
#include "core/ggml_extend.h"
#include "core/ggml_runner.h"
#include "core/rng.hpp"
#include "core/util.h"
#include "model.h"
#include "model/common/ggml_block.hpp"
/*
=================================== TinyAutoEncoder ===================================
@@ -787,7 +791,7 @@ struct TinyImageAutoEncoder : public VAE {
return build_graph(z_tensor, decode_graph);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), z_tensor.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), z_tensor.dim());
}
};
@@ -872,7 +876,7 @@ struct TinyVideoAutoEncoder : public VAE {
return build_graph(z_tensor, decode_graph);
};
return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), z_tensor.dim());
return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), z_tensor.dim());
}
};
src/model/vae/vae.hpp
+1
View File
@@ -5,6 +5,7 @@
#include "model/common/block.hpp"
#include "model/vae/vae_tiling.hpp"
#include "model_manager.h"
#include "runtime/tiling.h"
struct VAE : public GGMLRunner {
protected:
src/model/vae/wan_vae.hpp
+4 -2
View File
@@ -4,6 +4,8 @@
#include <map>
#include <memory>
#include <utility>
#include "core/ggml_extend_backend.h"
#include "core/ggml_tensor_utils.h"
#include "model/common/block.hpp"
#include "model/vae/vae.hpp"
@@ -1427,7 +1429,7 @@ namespace WAN {
return build_temporal_tile_graph(input_tile, static_cast<int>(tile.start));
};
return restore_trailing_singleton_dims(
GGMLRunner::compute<float>(get_graph, n_threads, false),
GGMLRunner::compute(get_graph, n_threads, false),
static_cast<size_t>(input.dim()));
});
@@ -1446,7 +1448,7 @@ namespace WAN {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(input.empty() ? z : input, decode_graph);
};
auto result = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false),
auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false),
input.empty() ? z.dim() : input.dim());
if (!result.empty() && z.dim() == 4) {
result.squeeze_(2);