mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-09-21 13:38:01 -05:00
refactor: split generation pipeline out of stable-diffusion.cpp (#1957)
This commit is contained in:
leejet
@@ -229,6 +229,8 @@ file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS
|
||||
"src/model/*/*.h"
|
||||
"src/model/*/*.cpp"
|
||||
"src/model/*/*.hpp"
|
||||
"src/pipeline/*.h"
|
||||
"src/pipeline/*.cpp"
|
||||
"src/runtime/*.h"
|
||||
"src/runtime/*.cpp"
|
||||
"src/runtime/*.hpp"
|
||||
|
||||
@@ -11,6 +11,8 @@ $patterns = @(
|
||||
"src/extensions/*.cpp"
|
||||
"src/extensions/*.h"
|
||||
"src/extensions/*.hpp"
|
||||
"src/pipeline/*.cpp"
|
||||
"src/pipeline/*.h"
|
||||
"src/runtime/*.cpp"
|
||||
"src/runtime/*.h"
|
||||
"src/runtime/*.hpp"
|
||||
|
||||
@@ -9,6 +9,7 @@ for f in src/*.cpp src/*.h src/*.hpp \
|
||||
src/conditioning/*.cpp src/conditioning/*.h src/conditioning/*.hpp \
|
||||
src/core/*.cpp src/core/*.h src/core/*.hpp \
|
||||
src/extensions/*.cpp src/extensions/*.h src/extensions/*.hpp \
|
||||
src/pipeline/*.cpp src/pipeline/*.h \
|
||||
src/runtime/*.cpp src/runtime/*.h src/runtime/*.hpp \
|
||||
src/model/*/*.cpp src/model/*/*.h src/model/*/*.hpp \
|
||||
src/tokenizers/*.h src/tokenizers/*.cpp src/tokenizers/vocab/*.h src/tokenizers/vocab/*.cpp \
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,482 @@
|
||||
#ifndef __SD_PIPELINE_DIFFUSION_ENGINE_H__
|
||||
#define __SD_PIPELINE_DIFFUSION_ENGINE_H__
|
||||
|
||||
#include <atomic>
|
||||
#include <cmath>
|
||||
#include <functional>
|
||||
#include <list>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "core/ggml_extend_backend.h"
|
||||
#include "core/ggml_graph_cut.h"
|
||||
#include "core/tensor.hpp"
|
||||
#include "core/util.h"
|
||||
#include "model/adapter/lora.hpp"
|
||||
#include "model_builders.h"
|
||||
#include "model_manager.h"
|
||||
#include "stable-diffusion.h"
|
||||
|
||||
class RNG;
|
||||
struct Denoiser;
|
||||
struct LoraModel;
|
||||
struct ConditionerParams;
|
||||
struct SDCondition;
|
||||
struct RefImageParams;
|
||||
|
||||
extern const char* model_version_to_str[];
|
||||
|
||||
static inline bool sd_version_supports_ref_latent_img_cfg(SDVersion version) {
|
||||
return version == VERSION_FLUX ||
|
||||
sd_version_is_flux2(version) ||
|
||||
sd_version_is_qwen_image(version) ||
|
||||
sd_version_is_mage_flow(version) ||
|
||||
sd_version_is_longcat(version) ||
|
||||
sd_version_is_z_image(version) ||
|
||||
sd_version_is_boogu_image(version);
|
||||
}
|
||||
|
||||
class StableDiffusionGGML {
|
||||
public:
|
||||
SDBackendManager backend_manager;
|
||||
|
||||
SDVersion version;
|
||||
bool external_vae_is_invalid = false;
|
||||
|
||||
bool circular_x = false;
|
||||
bool circular_y = false;
|
||||
|
||||
std::shared_ptr<RNG> rng;
|
||||
std::shared_ptr<RNG> sampler_rng = nullptr;
|
||||
int n_threads = -1;
|
||||
float default_flow_shift = INFINITY;
|
||||
float active_flow_shift = INFINITY;
|
||||
|
||||
std::shared_ptr<Conditioner> cond_stage_model;
|
||||
std::shared_ptr<FrozenCLIPVisionEmbedder> clip_vision; // for svd or wan2.1 i2v
|
||||
std::shared_ptr<DiffusionModelRunner> diffusion_model;
|
||||
std::shared_ptr<DiffusionModelRunner> high_noise_diffusion_model;
|
||||
std::shared_ptr<VAE> first_stage_model;
|
||||
std::shared_ptr<VAE> preview_vae;
|
||||
std::shared_ptr<AudioVAERunner> audio_vae_model;
|
||||
std::shared_ptr<ControlNet> control_net;
|
||||
std::shared_ptr<IPAdapter::IPAdapterRunner> ip_adapter;
|
||||
sd::Tensor<float> ip_adapter_tokens;
|
||||
sd::Tensor<float> ip_adapter_uncond_tokens;
|
||||
float ip_adapter_strength = 1.0f;
|
||||
std::vector<std::shared_ptr<GenerationExtension>> generation_extensions;
|
||||
struct RuntimeLora {
|
||||
ModelManager::LoraSpec spec;
|
||||
SDBackendModule module;
|
||||
std::shared_ptr<LoraModel> model;
|
||||
|
||||
bool matches(const ModelManager::LoraSpec& other) const {
|
||||
return spec.file_id == other.file_id && spec.file_revision == other.file_revision &&
|
||||
spec.tensor_name_prefix_filter == other.tensor_name_prefix_filter;
|
||||
}
|
||||
};
|
||||
std::vector<RuntimeLora> runtime_lora_models;
|
||||
bool apply_lora_immediately = false;
|
||||
int animatediff_num_frames = 0;
|
||||
|
||||
std::string taesd_path;
|
||||
sd_tiling_params_t vae_tiling_params = {false, false, 0, 0, 0.5f, 0, 0, nullptr};
|
||||
bool enable_mmap = false;
|
||||
sd::ggml_graph_cut::MaxVramAssignment max_vram_assignment;
|
||||
bool disable_prefetch = false;
|
||||
bool disable_segmented_compute = false;
|
||||
bool eager_load = false;
|
||||
std::string backend_spec;
|
||||
std::string params_backend_spec;
|
||||
std::string split_mode_spec;
|
||||
bool auto_fit_enabled = false;
|
||||
|
||||
bool diffusion_conv_direct = false;
|
||||
|
||||
bool is_using_v_parameterization = false;
|
||||
bool is_using_edm_v_parameterization = false;
|
||||
|
||||
std::shared_ptr<ModelManager> model_manager;
|
||||
|
||||
enum class RunnerGroup { Core,
|
||||
VAE,
|
||||
ControlNet,
|
||||
Extensions };
|
||||
using RunnerGroups = std::set<RunnerGroup>;
|
||||
|
||||
struct ModelConfig {
|
||||
sd_ctx_params_t params{};
|
||||
std::list<std::string> strings;
|
||||
std::vector<sd_embedding_t> embeddings;
|
||||
ModelLoader::FileId control_net_file = 0;
|
||||
bool use_tae = false;
|
||||
bool use_audio_vae = false;
|
||||
bool photomaker_source_available = false;
|
||||
bool animatediff_loaded = false;
|
||||
|
||||
explicit ModelConfig(const sd_ctx_params_t& initial)
|
||||
: params(initial) {
|
||||
for (auto member : {&sd_ctx_params_t::model_path, &sd_ctx_params_t::clip_l_path,
|
||||
&sd_ctx_params_t::clip_g_path, &sd_ctx_params_t::clip_vision_path,
|
||||
&sd_ctx_params_t::t5xxl_path, &sd_ctx_params_t::llm_path,
|
||||
&sd_ctx_params_t::llm_vision_path, &sd_ctx_params_t::diffusion_model_path,
|
||||
&sd_ctx_params_t::high_noise_diffusion_model_path, &sd_ctx_params_t::uncond_diffusion_model_path,
|
||||
&sd_ctx_params_t::embeddings_connectors_path, &sd_ctx_params_t::vae_path,
|
||||
&sd_ctx_params_t::audio_vae_path, &sd_ctx_params_t::taesd_path,
|
||||
&sd_ctx_params_t::control_net_path, &sd_ctx_params_t::ip_adapter_path,
|
||||
&sd_ctx_params_t::motion_module_path, &sd_ctx_params_t::photo_maker_path,
|
||||
&sd_ctx_params_t::pulid_weights_path, &sd_ctx_params_t::tensor_type_rules,
|
||||
&sd_ctx_params_t::max_vram, &sd_ctx_params_t::backend,
|
||||
&sd_ctx_params_t::params_backend, &sd_ctx_params_t::split_mode,
|
||||
&sd_ctx_params_t::rpc_servers, &sd_ctx_params_t::model_args}) {
|
||||
strings.emplace_back(SAFE_STR(initial.*member));
|
||||
params.*member = strings.back().c_str();
|
||||
}
|
||||
for (uint32_t i = 0; i < initial.embedding_count; ++i) {
|
||||
strings.emplace_back(SAFE_STR(initial.embeddings[i].name));
|
||||
const char* name = strings.back().c_str();
|
||||
strings.emplace_back(SAFE_STR(initial.embeddings[i].path));
|
||||
embeddings.push_back({name, strings.back().c_str()});
|
||||
}
|
||||
params.embeddings = embeddings.data();
|
||||
}
|
||||
|
||||
ModelConfig(const ModelConfig& other)
|
||||
: ModelConfig(other.params) {
|
||||
control_net_file = other.control_net_file;
|
||||
use_tae = other.use_tae;
|
||||
use_audio_vae = other.use_audio_vae;
|
||||
photomaker_source_available = other.photomaker_source_available;
|
||||
animatediff_loaded = other.animatediff_loaded;
|
||||
}
|
||||
ModelConfig& operator=(const ModelConfig&) = delete;
|
||||
|
||||
void set_control_net(ModelLoader::FileId id, const std::string& path) {
|
||||
control_net_file = id;
|
||||
strings.push_back(path);
|
||||
params.control_net_path = strings.back().c_str();
|
||||
}
|
||||
};
|
||||
|
||||
struct RunnerState {
|
||||
bool ready = false;
|
||||
uint64_t catalog_revision = 0;
|
||||
std::map<RunnerGroup, ModelLoader::FileVersions> sources;
|
||||
};
|
||||
|
||||
std::recursive_mutex execution_mutex;
|
||||
std::unique_ptr<ModelConfig> config_;
|
||||
RunnerState runner_state_;
|
||||
bool executing_ = false;
|
||||
|
||||
std::shared_ptr<Denoiser> denoiser;
|
||||
std::vector<float> file_alphas_cumprod;
|
||||
|
||||
StableDiffusionGGML();
|
||||
~StableDiffusionGGML();
|
||||
|
||||
static const std::map<RunnerGroup, std::set<ModelComponent>>& runner_components();
|
||||
|
||||
static RunnerGroups all_runner_groups();
|
||||
|
||||
ModelLoader::FileVersions runner_source_versions(RunnerGroup group, const ModelLoader& loader) const;
|
||||
|
||||
void capture_runner_sources();
|
||||
|
||||
void end_runners();
|
||||
|
||||
bool reset_runners(const RunnerGroups& groups);
|
||||
|
||||
bool refresh_model_sources();
|
||||
|
||||
bool apply_model_update(ModelLoader candidate,
|
||||
std::unique_ptr<ModelConfig> next_config = nullptr,
|
||||
RunnerGroups groups = {});
|
||||
|
||||
struct ContextOperation {
|
||||
StableDiffusionGGML& sd;
|
||||
std::unique_lock<std::recursive_mutex> lock;
|
||||
bool acquired = false;
|
||||
|
||||
explicit ContextOperation(StableDiffusionGGML& sd)
|
||||
: sd(sd), lock(sd.execution_mutex, std::try_to_lock) {
|
||||
if (!lock.owns_lock() || sd.executing_) {
|
||||
// The caller may be a log callback, so rejecting it must not log.
|
||||
return;
|
||||
}
|
||||
sd.executing_ = true;
|
||||
acquired = true;
|
||||
}
|
||||
|
||||
~ContextOperation() {
|
||||
if (acquired) {
|
||||
sd.executing_ = false;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
struct ExecutionScope {
|
||||
ContextOperation operation;
|
||||
bool ready = false;
|
||||
|
||||
explicit ExecutionScope(StableDiffusionGGML& sd)
|
||||
: operation(sd) {
|
||||
ready = operation.acquired && sd.refresh_model_sources();
|
||||
}
|
||||
|
||||
~ExecutionScope() {
|
||||
if (ready) {
|
||||
operation.sd.end_runners();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
ggml_backend_t backend_for(SDBackendModule module);
|
||||
|
||||
ggml_backend_t params_backend_for(SDBackendModule module);
|
||||
|
||||
std::atomic<sd_cancel_mode_t> cancellation_flag = SD_CANCEL_RESET;
|
||||
|
||||
void set_cancel_flag(enum sd_cancel_mode_t flag);
|
||||
|
||||
void reset_cancel_flag();
|
||||
|
||||
enum sd_cancel_mode_t get_cancel_flag();
|
||||
|
||||
size_t max_graph_vram_bytes_for_module(SDBackendModule module);
|
||||
|
||||
std::vector<size_t> layer_split_vram_limits_for_backends(const std::vector<ggml_backend_t>& backends);
|
||||
|
||||
bool ensure_backend_pair(SDBackendModule module);
|
||||
|
||||
template <typename T>
|
||||
bool register_runner_params(ModelComponent component,
|
||||
const std::shared_ptr<T>& model,
|
||||
SDBackendModule module,
|
||||
size_t* params_mem_size = nullptr);
|
||||
|
||||
template <typename T>
|
||||
bool register_row_split_runner_params(ModelComponent component,
|
||||
const std::shared_ptr<T>& model,
|
||||
SDBackendModule module,
|
||||
const std::vector<ggml_backend_t>& module_backends,
|
||||
std::map<std::string, ggml_tensor*> group_tensors,
|
||||
const std::map<ggml_tensor*, enum ggml_op>& tensor_ops,
|
||||
ModelManager::ResidencyMode residency_mode,
|
||||
size_t* params_mem_size);
|
||||
|
||||
// Register graph-cut layer-split tensors on the primary backend first.
|
||||
// The first real graph assigns each param tensor to a runtime backend
|
||||
// before weights are loaded or staged.
|
||||
template <typename T>
|
||||
bool register_layer_split_runner_params(ModelComponent component,
|
||||
const std::shared_ptr<T>& model,
|
||||
SDBackendModule module,
|
||||
const std::vector<ggml_backend_t>& module_backends,
|
||||
std::map<std::string, ggml_tensor*> group_tensors,
|
||||
const std::map<ggml_tensor*, enum ggml_op>& tensor_ops,
|
||||
ModelManager::ResidencyMode residency_mode,
|
||||
size_t* params_mem_size);
|
||||
|
||||
bool unload_control_net();
|
||||
|
||||
bool load_control_net_from_file(const std::string& path);
|
||||
|
||||
void apply_circular_axes(bool circular_x, bool circular_y);
|
||||
|
||||
bool init_backend();
|
||||
|
||||
bool row_split_active();
|
||||
|
||||
bool graph_cut_layer_split_active();
|
||||
|
||||
std::shared_ptr<RNG> get_rng(rng_type_t rng_type);
|
||||
|
||||
void refresh_compvis_denoiser_sigmas();
|
||||
|
||||
void load_alphas_cumprod();
|
||||
|
||||
bool init_model_loader(ModelLoader& model_loader, ModelConfig& configuration);
|
||||
|
||||
bool init(const sd_ctx_params_t* sd_ctx_params);
|
||||
|
||||
bool uses_tae() const;
|
||||
|
||||
bool tae_preview_only() const;
|
||||
|
||||
void configure_weight_loading();
|
||||
|
||||
sd::model_builders::Context model_build_context();
|
||||
|
||||
bool build_core_runners();
|
||||
|
||||
bool build_vae_runners();
|
||||
|
||||
bool build_control_net_runner();
|
||||
|
||||
bool build_extension_runners();
|
||||
|
||||
bool validate_and_load_runners();
|
||||
|
||||
bool build_denoiser();
|
||||
|
||||
bool build_runners(const RunnerGroups& groups);
|
||||
|
||||
bool is_using_v_parameterization_for_sd2(bool is_inpaint = false);
|
||||
|
||||
static std::string lora_log_id(const ModelManager::LoraSpec& lora);
|
||||
|
||||
std::shared_ptr<LoraModel> load_lora_model(const ModelManager::LoraSpec& lora_spec,
|
||||
SDBackendModule module,
|
||||
LoraModel::filter_t module_filter = nullptr);
|
||||
|
||||
void clear_lora_adapters();
|
||||
|
||||
std::vector<std::shared_ptr<LoraModel>> load_runtime_loras_for_module(const std::vector<ModelManager::LoraSpec>& loras,
|
||||
const std::set<std::string>& model_tensor_names,
|
||||
SDBackendModule module,
|
||||
LoraModel::filter_t module_filter,
|
||||
bool& success,
|
||||
std::vector<RuntimeLora>& next_models);
|
||||
|
||||
bool apply_loras_immediately(const std::vector<ModelManager::LoraSpec>& loras);
|
||||
|
||||
bool apply_loras_at_runtime(const std::vector<ModelManager::LoraSpec>& loras);
|
||||
|
||||
void lora_stat();
|
||||
|
||||
bool apply_loras(const sd_lora_t* loras, uint32_t lora_count);
|
||||
|
||||
void reset_generation_extensions();
|
||||
|
||||
void prepare_generation_extensions(const sd_pm_params_t& pm_params,
|
||||
const sd_pulid_params_t& pulid_params,
|
||||
ConditionerParams& condition_params,
|
||||
int total_steps);
|
||||
|
||||
sd::Tensor<float> get_clip_vision_output(const sd::Tensor<float>& image,
|
||||
bool return_pooled = true,
|
||||
int clip_skip = -1,
|
||||
bool zero_out_masked = false);
|
||||
|
||||
void compute_ip_adapter_tokens(const sd_image_t& image, float strength);
|
||||
|
||||
std::vector<float> process_timesteps(const std::vector<float>& timesteps,
|
||||
const sd::Tensor<float>& init_latent,
|
||||
const sd::Tensor<float>& denoise_mask,
|
||||
int step);
|
||||
|
||||
std::vector<float> process_ltxav_video_timesteps(const std::vector<float>& timesteps,
|
||||
const sd::Tensor<float>& init_latent,
|
||||
const sd::Tensor<float>& denoise_mask);
|
||||
|
||||
void preview_image(int step,
|
||||
const sd::Tensor<float>& latents,
|
||||
enum SDVersion version,
|
||||
preview_t preview_mode,
|
||||
std::function<void(int, int, sd_image_t*, bool, void*)> step_callback,
|
||||
void* step_callback_data,
|
||||
bool is_noisy);
|
||||
|
||||
std::vector<float> prepare_sample_timesteps(float sigma,
|
||||
int shifted_timestep);
|
||||
|
||||
void adjust_sample_step_scalings(int shifted_timestep,
|
||||
const std::vector<float>& timesteps_vec,
|
||||
float c_in,
|
||||
float* c_skip,
|
||||
float* c_out);
|
||||
|
||||
struct SamplePreviewContext {
|
||||
sd_preview_cb_t callback = nullptr;
|
||||
void* data = nullptr;
|
||||
preview_t mode = PREVIEW_NONE;
|
||||
};
|
||||
|
||||
SamplePreviewContext prepare_sample_preview_context();
|
||||
|
||||
void report_sample_progress(int step,
|
||||
size_t total_steps,
|
||||
bool terminal_sigma_is_zero,
|
||||
int64_t* last_progress_us);
|
||||
|
||||
void compute_sample_controls(const sd::Tensor<float>& control_image,
|
||||
const sd::Tensor<float>& noised_input,
|
||||
const sd::Tensor<float>& timesteps_tensor,
|
||||
const SDCondition& condition,
|
||||
std::vector<sd::Tensor<float>>* controls);
|
||||
|
||||
sd::Tensor<float> sample(const std::shared_ptr<DiffusionModelRunner>& work_diffusion_model,
|
||||
bool inverse_noise_scaling,
|
||||
const sd::Tensor<float>& init_latent,
|
||||
sd::Tensor<float> noise,
|
||||
const SDCondition& cond,
|
||||
const SDCondition& uncond,
|
||||
const SDCondition& img_uncond,
|
||||
const sd::Tensor<float>& control_image,
|
||||
float control_strength,
|
||||
const sd_guidance_params_t& guidance,
|
||||
float eta,
|
||||
int shifted_timestep,
|
||||
sample_method_t method,
|
||||
bool is_flow_denoiser,
|
||||
const char* extra_sample_args,
|
||||
const std::vector<float>& sigmas,
|
||||
const std::vector<sd::Tensor<float>>& ref_latents,
|
||||
const RefImageParams& ref_image_params,
|
||||
const sd::Tensor<float>& denoise_mask,
|
||||
const sd::Tensor<float>& vace_context,
|
||||
float vace_strength,
|
||||
int audio_length,
|
||||
float frame_rate,
|
||||
const sd_cache_params_t* cache_params,
|
||||
bool preview_final_step,
|
||||
const sd::Tensor<float>& video_positions = {});
|
||||
|
||||
int get_vae_scale_factor();
|
||||
|
||||
int get_diffusion_model_down_factor();
|
||||
|
||||
int get_latent_channel();
|
||||
|
||||
int get_image_channels() const;
|
||||
|
||||
int get_image_seq_len(int h, int w);
|
||||
|
||||
sd::Tensor<float> generate_init_latent(int width,
|
||||
int height,
|
||||
int frames = 1,
|
||||
bool video = false);
|
||||
|
||||
int video_frames_to_latent_frames(int frames);
|
||||
|
||||
int latent_frames_to_video_frames(int latent_frames);
|
||||
|
||||
int align_video_frames(int frames);
|
||||
|
||||
sd::Tensor<float> encode_to_vae_latents(const sd::Tensor<float>& x);
|
||||
|
||||
sd::Tensor<float> encode_first_stage(const sd::Tensor<float>& x);
|
||||
|
||||
sd::Tensor<float> decode_first_stage(const sd::Tensor<float>& x, bool decode_video = false);
|
||||
|
||||
sd::Tensor<float> normalize_ltx_video_latents(const sd::Tensor<float>& x);
|
||||
|
||||
sd::Tensor<float> un_normalize_ltx_video_latents(const sd::Tensor<float>& x);
|
||||
|
||||
sd::Tensor<float> decode_ltx_audio_latent(const sd::Tensor<float>& audio_latent);
|
||||
|
||||
void set_flow_shift(float flow_shift = INFINITY);
|
||||
|
||||
bool is_flow_denoiser();
|
||||
|
||||
std::string get_default_ref_image_preset(SDVersion version) const;
|
||||
|
||||
RefImageParams resolve_ref_image_params(const char* ref_image_args) const;
|
||||
};
|
||||
|
||||
#endif // __SD_PIPELINE_DIFFUSION_ENGINE_H__
|
||||
@@ -0,0 +1,73 @@
|
||||
#ifndef __SD_PIPELINE_GENERATION_H__
|
||||
#define __SD_PIPELINE_GENERATION_H__
|
||||
|
||||
#include "conditioning/conditioner.hpp"
|
||||
#include "stable-diffusion.h"
|
||||
|
||||
class StableDiffusionGGML;
|
||||
|
||||
static inline bool sd_version_supports_animatediff(SDVersion version) {
|
||||
return version == VERSION_SD1 || version == VERSION_SD1_INPAINT || version == VERSION_SD1_PIX2PIX;
|
||||
}
|
||||
|
||||
namespace sd::pipeline {
|
||||
|
||||
struct ImageGenerationLatents {
|
||||
sd::Tensor<float> init_latent;
|
||||
sd::Tensor<float> concat_latent;
|
||||
sd::Tensor<float> img_uncond_concat_latent;
|
||||
sd::Tensor<float> audio_latent;
|
||||
sd::Tensor<float> video_positions;
|
||||
sd::Tensor<float> control_image;
|
||||
std::vector<sd::Tensor<float>> ref_images;
|
||||
std::vector<sd::Tensor<float>> ref_latents;
|
||||
std::vector<sd::Tensor<float>> reference_audio_latents;
|
||||
std::vector<MiniMaxH3ReferenceBlock> minimax_reference_blocks;
|
||||
std::vector<MiniMaxH3PresentationItem> minimax_presentation_refs;
|
||||
std::vector<int32_t> keyframe_indices;
|
||||
sd::Tensor<float> denoise_mask;
|
||||
sd::Tensor<float> clip_vision_output;
|
||||
sd::Tensor<float> vace_context;
|
||||
int64_t ref_image_num = 0;
|
||||
int64_t video_conditioning_frame_count = 0;
|
||||
int64_t video_target_frame_count = 0;
|
||||
int audio_length = 0;
|
||||
};
|
||||
|
||||
struct ImageGenerationEmbeds {
|
||||
SDCondition cond;
|
||||
SDCondition uncond;
|
||||
SDCondition img_uncond;
|
||||
};
|
||||
|
||||
struct ConditionerRunnerEndOnExit {
|
||||
Conditioner* conditioner = nullptr;
|
||||
~ConditionerRunnerEndOnExit() {
|
||||
if (conditioner != nullptr) {
|
||||
conditioner->runner_end();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Callers hold ExecutionScope; AnimateDiff reuses the image path within the same scope.
|
||||
bool generate_image(StableDiffusionGGML* sd,
|
||||
const sd_img_gen_params_t* sd_img_gen_params,
|
||||
sd_image_t** images_out,
|
||||
int* num_images_out);
|
||||
|
||||
bool generate_video(StableDiffusionGGML* sd,
|
||||
const sd_vid_gen_params_t* sd_vid_gen_params,
|
||||
sd_image_t** frames_out,
|
||||
int* num_frames_out,
|
||||
sd_audio_t** audio_out);
|
||||
|
||||
sd::Tensor<float> upscale_ltx_spatial_video_latent(StableDiffusionGGML* sd,
|
||||
const char* model_path,
|
||||
const sd::Tensor<float>& packed_latent,
|
||||
int audio_length);
|
||||
|
||||
sd::Tensor<float> ensure_image_tensor_channels(sd::Tensor<float> image, int channels);
|
||||
|
||||
} // namespace sd::pipeline
|
||||
|
||||
#endif // __SD_PIPELINE_GENERATION_H__
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,5 @@
|
||||
#ifndef __SD_MODEL_BUILDERS_H__
|
||||
#define __SD_MODEL_BUILDERS_H__
|
||||
#ifndef __SD_PIPELINE_MODEL_BUILDERS_H__
|
||||
#define __SD_PIPELINE_MODEL_BUILDERS_H__
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
@@ -60,4 +60,4 @@ namespace sd::model_builders {
|
||||
|
||||
} // namespace sd::model_builders
|
||||
|
||||
#endif // __SD_MODEL_BUILDERS_H__
|
||||
#endif // __SD_PIPELINE_MODEL_BUILDERS_H__
|
||||
@@ -0,0 +1,471 @@
|
||||
#include "request.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstdlib>
|
||||
#include <ctime>
|
||||
|
||||
#include "diffusion_engine.h"
|
||||
#include "runtime/denoiser.hpp"
|
||||
|
||||
namespace sd::pipeline {
|
||||
|
||||
const char* sampling_methods_str[] = {
|
||||
"Euler",
|
||||
"Euler A",
|
||||
"Heun",
|
||||
"DPM2",
|
||||
"DPM++ (2s)",
|
||||
"DPM++ (2M)",
|
||||
"modified DPM++ (2M)",
|
||||
"iPNDM",
|
||||
"iPNDM_v",
|
||||
"LCM",
|
||||
"DDIM \"trailing\"",
|
||||
"TCD",
|
||||
"Res Multistep",
|
||||
"Res 2s",
|
||||
"ER-SDE",
|
||||
"Euler CFG++",
|
||||
"Euler A CFG++",
|
||||
"Euler GE",
|
||||
"DPM++ (2M) SDE",
|
||||
"DPM++ (2M) SDE BT",
|
||||
"LMS",
|
||||
};
|
||||
|
||||
static_assert(SAMPLE_METHOD_COUNT == sizeof(sampling_methods_str) / sizeof(sampling_methods_str[0]),
|
||||
"\nnumber of elements in sampling_methods_str[] != SAMPLE_METHOD_COUNT");
|
||||
|
||||
static bool sd_version_supports_img_cfg(SDVersion version, bool has_ref_images) {
|
||||
return sd_version_is_inpaint_or_unet_edit(version) ||
|
||||
(has_ref_images && sd_version_supports_ref_latent_img_cfg(version));
|
||||
}
|
||||
|
||||
enum sample_method_t default_sample_method(const StableDiffusionGGML* sd) {
|
||||
if (sd != nullptr) {
|
||||
if (sd_version_is_pid(sd->version)) {
|
||||
return LCM_SAMPLE_METHOD;
|
||||
}
|
||||
if (sd_version_is_dit(sd->version)) {
|
||||
return EULER_SAMPLE_METHOD;
|
||||
}
|
||||
}
|
||||
return EULER_A_SAMPLE_METHOD;
|
||||
}
|
||||
|
||||
enum scheduler_t default_scheduler(const StableDiffusionGGML* sd, enum sample_method_t sample_method) {
|
||||
if (sd != nullptr) {
|
||||
auto edm_v_denoiser = std::dynamic_pointer_cast<EDMVDenoiser>(sd->denoiser);
|
||||
if (edm_v_denoiser) {
|
||||
return EXPONENTIAL_SCHEDULER;
|
||||
}
|
||||
}
|
||||
if (sample_method == LCM_SAMPLE_METHOD || sample_method == TCD_SAMPLE_METHOD) {
|
||||
return LCM_SCHEDULER;
|
||||
} else if (sample_method == DDIM_TRAILING_SAMPLE_METHOD) {
|
||||
return SIMPLE_SCHEDULER;
|
||||
} else if (sd != nullptr && sd_version_is_flux(sd->version)) {
|
||||
return FLUX_SCHEDULER;
|
||||
} else if (sd != nullptr && sd_version_is_flux2(sd->version)) {
|
||||
return FLUX2_SCHEDULER;
|
||||
} else if (sd != nullptr && sd_version_is_ltxav(sd->version)) {
|
||||
return LTX2_SCHEDULER;
|
||||
} else if (sd != nullptr && sd_version_is_ideogram4(sd->version)) {
|
||||
return LOGIT_NORMAL_SCHEDULER;
|
||||
}
|
||||
return DISCRETE_SCHEDULER;
|
||||
}
|
||||
|
||||
static int64_t resolve_seed(int64_t seed) {
|
||||
if (seed >= 0) {
|
||||
return seed;
|
||||
}
|
||||
srand((int)time(nullptr));
|
||||
return rand();
|
||||
}
|
||||
|
||||
static enum sample_method_t resolve_sample_method(StableDiffusionGGML* sd, enum sample_method_t sample_method) {
|
||||
if (sample_method == SAMPLE_METHOD_COUNT) {
|
||||
return default_sample_method(sd);
|
||||
}
|
||||
return sample_method;
|
||||
}
|
||||
|
||||
static scheduler_t resolve_scheduler(StableDiffusionGGML* sd,
|
||||
scheduler_t scheduler,
|
||||
enum sample_method_t sample_method) {
|
||||
if (scheduler == SCHEDULER_COUNT) {
|
||||
return default_scheduler(sd, sample_method);
|
||||
}
|
||||
return scheduler;
|
||||
}
|
||||
|
||||
float resolve_eta(StableDiffusionGGML* sd,
|
||||
float eta,
|
||||
enum sample_method_t sample_method) {
|
||||
if (eta == INFINITY) {
|
||||
if (sd->version == VERSION_HIDREAM_O1) {
|
||||
return 8.f;
|
||||
}
|
||||
switch (sample_method) {
|
||||
case DDIM_TRAILING_SAMPLE_METHOD:
|
||||
case TCD_SAMPLE_METHOD:
|
||||
case RES_MULTISTEP_SAMPLE_METHOD:
|
||||
case RES_2S_SAMPLE_METHOD:
|
||||
return 0.0f;
|
||||
case EULER_A_SAMPLE_METHOD:
|
||||
case DPMPP2S_A_SAMPLE_METHOD:
|
||||
case ER_SDE_SAMPLE_METHOD:
|
||||
case EULER_A_CFG_PP_SAMPLE_METHOD:
|
||||
case DPMPP2M_SDE_SAMPLE_METHOD:
|
||||
case DPMPP2M_SDE_BT_SAMPLE_METHOD:
|
||||
return 1.0f;
|
||||
default:;
|
||||
}
|
||||
return 0.0f;
|
||||
}
|
||||
return eta;
|
||||
}
|
||||
|
||||
GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params) {
|
||||
prompt = SAFE_STR(sd_img_gen_params->prompt);
|
||||
negative_prompt = SAFE_STR(sd_img_gen_params->negative_prompt);
|
||||
width = sd_img_gen_params->width;
|
||||
height = sd_img_gen_params->height;
|
||||
vae_scale_factor = sd->get_vae_scale_factor();
|
||||
diffusion_model_down_factor = sd->get_diffusion_model_down_factor();
|
||||
seed = sd_img_gen_params->seed;
|
||||
batch_count = sd_img_gen_params->batch_count;
|
||||
qwen_image_layers = std::max(0, sd_img_gen_params->qwen_image_layers);
|
||||
clip_skip = sd_img_gen_params->clip_skip;
|
||||
shifted_timestep = sd_img_gen_params->sample_params.shifted_timestep;
|
||||
strength = sd_img_gen_params->strength;
|
||||
control_strength = sd_img_gen_params->control_strength;
|
||||
eta = sd_img_gen_params->sample_params.eta;
|
||||
has_ref_images = sd_img_gen_params->ref_images_count > 0;
|
||||
guidance = sd_img_gen_params->sample_params.guidance;
|
||||
pm_params = sd_img_gen_params->pm_params;
|
||||
pulid_params = sd_img_gen_params->pulid_params;
|
||||
hires = sd_img_gen_params->hires;
|
||||
cache_params = &sd_img_gen_params->cache;
|
||||
resolve(sd);
|
||||
}
|
||||
|
||||
GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params) {
|
||||
prompt = SAFE_STR(sd_vid_gen_params->prompt);
|
||||
negative_prompt = SAFE_STR(sd_vid_gen_params->negative_prompt);
|
||||
width = sd_vid_gen_params->width;
|
||||
height = sd_vid_gen_params->height;
|
||||
requested_frames = std::max(1, sd_vid_gen_params->video_frames);
|
||||
frames = sd->align_video_frames(requested_frames);
|
||||
clip_skip = sd_vid_gen_params->clip_skip;
|
||||
fps = std::max(1, sd_vid_gen_params->fps);
|
||||
if (sd_version_is_minimax_h3(sd->version) && fps != 24) {
|
||||
LOG_WARN("MiniMax-H3 uses 24 fps; overriding requested fps %d", fps);
|
||||
fps = 24;
|
||||
}
|
||||
vae_scale_factor = sd->get_vae_scale_factor();
|
||||
diffusion_model_down_factor = sd->get_diffusion_model_down_factor();
|
||||
seed = sd_vid_gen_params->seed;
|
||||
strength = sd_vid_gen_params->strength;
|
||||
cache_params = &sd_vid_gen_params->cache;
|
||||
vace_strength = sd_vid_gen_params->vace_strength;
|
||||
guidance = sd_vid_gen_params->sample_params.guidance;
|
||||
high_noise_guidance = sd_vid_gen_params->high_noise_sample_params.guidance;
|
||||
hires = sd_vid_gen_params->hires;
|
||||
resolve(sd);
|
||||
if (frames != requested_frames) {
|
||||
LOG_WARN("align video frames from %d to %d for %s",
|
||||
requested_frames,
|
||||
frames,
|
||||
model_version_to_str[sd->version]);
|
||||
}
|
||||
}
|
||||
|
||||
void GenerationRequest::align_generation_request_size() {
|
||||
align_image_size(&width, &height, "generation request");
|
||||
}
|
||||
|
||||
void GenerationRequest::align_image_size(int* target_width, int* target_height, const char* label) {
|
||||
int spatial_multiple = vae_scale_factor * diffusion_model_down_factor;
|
||||
int width_offset = align_up_offset(*target_width, spatial_multiple);
|
||||
int height_offset = align_up_offset(*target_height, spatial_multiple);
|
||||
if (width_offset <= 0 && height_offset <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
int original_width = *target_width;
|
||||
int original_height = *target_height;
|
||||
|
||||
*target_width += width_offset;
|
||||
*target_height += height_offset;
|
||||
LOG_WARN("align %s up %dx%d to %dx%d (multiple=%d)",
|
||||
label,
|
||||
original_width,
|
||||
original_height,
|
||||
*target_width,
|
||||
*target_height,
|
||||
spatial_multiple);
|
||||
}
|
||||
|
||||
void GenerationRequest::resolve_hires() {
|
||||
if (!hires.enabled) {
|
||||
return;
|
||||
}
|
||||
if (hires.upscaler == SD_HIRES_UPSCALER_NONE) {
|
||||
hires.enabled = false;
|
||||
return;
|
||||
}
|
||||
if (hires.upscaler < SD_HIRES_UPSCALER_NONE || hires.upscaler >= SD_HIRES_UPSCALER_COUNT) {
|
||||
LOG_WARN("hires upscaler '%d' is invalid, disabling hires", hires.upscaler);
|
||||
hires.enabled = false;
|
||||
return;
|
||||
}
|
||||
if (hires.upscaler == SD_HIRES_UPSCALER_MODEL && strlen(SAFE_STR(hires.model_path)) == 0) {
|
||||
LOG_WARN("hires model upscaler requires a model path, disabling hires");
|
||||
hires.enabled = false;
|
||||
return;
|
||||
}
|
||||
if (hires.scale <= 0.f && hires.target_width <= 0 && hires.target_height <= 0) {
|
||||
LOG_WARN("hires scale must be positive when no target size is set, disabling hires");
|
||||
hires.enabled = false;
|
||||
return;
|
||||
}
|
||||
if (hires.custom_sigmas_count < 0) {
|
||||
LOG_WARN("hires custom sigmas count is negative, ignoring custom sigmas");
|
||||
hires.custom_sigmas = nullptr;
|
||||
hires.custom_sigmas_count = 0;
|
||||
}
|
||||
if (hires.custom_sigmas_count > 0 && hires.custom_sigmas == nullptr) {
|
||||
LOG_WARN("hires custom sigmas count is positive but custom sigmas are null, ignoring custom sigmas");
|
||||
hires.custom_sigmas_count = 0;
|
||||
}
|
||||
if (hires.custom_sigmas_count == 1) {
|
||||
LOG_WARN("hires custom sigmas requires at least two values, ignoring custom sigmas");
|
||||
hires.custom_sigmas = nullptr;
|
||||
hires.custom_sigmas_count = 0;
|
||||
}
|
||||
hires.denoising_strength = std::clamp(hires.denoising_strength, 0.0001f, 1.f);
|
||||
hires.steps = std::max(0, hires.steps);
|
||||
|
||||
if (hires.target_width > 0 && hires.target_height > 0) {
|
||||
// pass
|
||||
} else if (hires.target_width > 0) {
|
||||
hires.target_height = hires.target_width;
|
||||
} else if (hires.target_height > 0) {
|
||||
hires.target_width = hires.target_height;
|
||||
} else {
|
||||
hires.target_width = static_cast<int>(std::round(width * hires.scale));
|
||||
hires.target_height = static_cast<int>(std::round(height * hires.scale));
|
||||
}
|
||||
|
||||
if (hires.target_width <= 0 || hires.target_height <= 0) {
|
||||
LOG_WARN("hires target size is not positive, disabling hires");
|
||||
hires.enabled = false;
|
||||
return;
|
||||
}
|
||||
align_image_size(&hires.target_width, &hires.target_height, "hires target");
|
||||
}
|
||||
|
||||
void GenerationRequest::resolve_guidance(StableDiffusionGGML* sd,
|
||||
sd_guidance_params_t* guidance,
|
||||
bool* use_uncond,
|
||||
bool* use_img_uncond,
|
||||
bool has_ref_images,
|
||||
const char* stage_name) {
|
||||
GGML_ASSERT(guidance != nullptr);
|
||||
GGML_ASSERT(use_uncond != nullptr);
|
||||
GGML_ASSERT(use_img_uncond != nullptr);
|
||||
// out_img_uncond + text_cfg_scale * (out_cond - out_uncond) + image_cfg_scale * (out_uncond - out_img_uncond)
|
||||
// -> text_cfg_scale * out_cond + (image_cfg_scale - text_cfg_scale) * out_uncond + (1 - image_cfg_scale) * out_img_uncond
|
||||
// out_cond : prompt, image latent
|
||||
// out_uncond : negative prompt, image latent
|
||||
// out_img_uncond : negative prompt, zero image latent
|
||||
// image_cfg_scale == 1 reduces 3-cond CFG to 2-cond CFG.
|
||||
bool img_cfg_was_set = std::isfinite(guidance->img_cfg);
|
||||
if (!img_cfg_was_set) {
|
||||
guidance->img_cfg = 1.f;
|
||||
}
|
||||
|
||||
if (!sd_version_supports_img_cfg(sd->version, has_ref_images)) {
|
||||
if (img_cfg_was_set && guidance->img_cfg != 1.f) {
|
||||
LOG_WARN("3-conditioning CFG is not supported with this model, disabling it for better performance");
|
||||
}
|
||||
guidance->img_cfg = 1.f;
|
||||
}
|
||||
|
||||
if (guidance->img_cfg != guidance->txt_cfg) {
|
||||
*use_uncond = true;
|
||||
}
|
||||
|
||||
if (guidance->img_cfg != 1.f) {
|
||||
*use_img_uncond = true;
|
||||
}
|
||||
|
||||
if (guidance->txt_cfg < 1.f) {
|
||||
const char* prefix = stage_name == nullptr ? "" : stage_name;
|
||||
if (guidance->txt_cfg == 0.f) {
|
||||
LOG_WARN("%sunconditioned mode, images won't follow the prompt (use cfg-scale=1 for distilled models)",
|
||||
prefix);
|
||||
} else {
|
||||
LOG_WARN("%scfg value out of expected range may produce unexpected results", prefix);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void GenerationRequest::resolve(StableDiffusionGGML* sd) {
|
||||
align_generation_request_size();
|
||||
resolve_hires();
|
||||
seed = resolve_seed(seed);
|
||||
|
||||
resolve_guidance(sd, &guidance, &use_uncond, &use_img_uncond, has_ref_images);
|
||||
if (sd->high_noise_diffusion_model) {
|
||||
resolve_guidance(sd,
|
||||
&high_noise_guidance,
|
||||
&use_high_noise_uncond,
|
||||
&use_high_noise_img_uncond,
|
||||
has_ref_images,
|
||||
"high noise: ");
|
||||
}
|
||||
|
||||
if (shifted_timestep > 0 && !sd_version_is_sdxl(sd->version)) {
|
||||
LOG_WARN("timestep shifting is only supported for SDXL models!");
|
||||
shifted_timestep = 0;
|
||||
}
|
||||
}
|
||||
|
||||
SamplePlan::SamplePlan(StableDiffusionGGML* sd,
|
||||
const sd_img_gen_params_t* sd_img_gen_params,
|
||||
const GenerationRequest& request) {
|
||||
sample_method = sd_img_gen_params->sample_params.sample_method;
|
||||
extra_sample_args = sd_img_gen_params->sample_params.extra_sample_args;
|
||||
eta = sd_img_gen_params->sample_params.eta;
|
||||
sample_steps = sd_img_gen_params->sample_params.sample_steps;
|
||||
resolve(sd, &request, &sd_img_gen_params->sample_params);
|
||||
}
|
||||
|
||||
SamplePlan::SamplePlan(StableDiffusionGGML* sd,
|
||||
const sd_vid_gen_params_t* sd_vid_gen_params,
|
||||
const GenerationRequest& request) {
|
||||
sample_method = sd_vid_gen_params->sample_params.sample_method;
|
||||
extra_sample_args = sd_vid_gen_params->sample_params.extra_sample_args;
|
||||
eta = sd_vid_gen_params->sample_params.eta;
|
||||
sample_steps = sd_vid_gen_params->sample_params.sample_steps;
|
||||
if (sd->high_noise_diffusion_model) {
|
||||
high_noise_sample_steps = sd_vid_gen_params->high_noise_sample_params.sample_steps;
|
||||
high_noise_sample_method = sd_vid_gen_params->high_noise_sample_params.sample_method;
|
||||
high_noise_extra_sample_args = sd_vid_gen_params->high_noise_sample_params.extra_sample_args;
|
||||
high_noise_eta = sd_vid_gen_params->high_noise_sample_params.eta;
|
||||
}
|
||||
moe_boundary = sd_vid_gen_params->moe_boundary;
|
||||
resolve(sd, &request, &sd_vid_gen_params->sample_params);
|
||||
}
|
||||
|
||||
void SamplePlan::resolve(StableDiffusionGGML* sd,
|
||||
const GenerationRequest* request,
|
||||
const sd_sample_params_t* sample_params) {
|
||||
sample_method = resolve_sample_method(sd, sample_method);
|
||||
|
||||
total_steps = sample_steps + std::max(0, high_noise_sample_steps);
|
||||
|
||||
if (sample_params->custom_sigmas_count > 0) {
|
||||
sigmas = std::vector<float>(sample_params->custom_sigmas,
|
||||
sample_params->custom_sigmas + sample_params->custom_sigmas_count);
|
||||
total_steps = static_cast<int>(sigmas.size()) - 1;
|
||||
LOG_WARN("total_steps != custom_sigmas_count - 1, set total_steps to %d", total_steps);
|
||||
if (sample_steps >= total_steps) {
|
||||
sample_steps = total_steps;
|
||||
LOG_WARN("total_steps != custom_sigmas_count - 1, set sample_steps to %d", sample_steps);
|
||||
}
|
||||
if (high_noise_sample_steps > 0) {
|
||||
high_noise_sample_steps = total_steps - sample_steps;
|
||||
LOG_WARN("total_steps != custom_sigmas_count - 1, set high_noise_sample_steps to %d", high_noise_sample_steps);
|
||||
}
|
||||
} else {
|
||||
scheduler_t scheduler = resolve_scheduler(sd,
|
||||
sample_params->scheduler,
|
||||
sample_method);
|
||||
int sample_seq_len = sd->get_image_seq_len(request->height, request->width);
|
||||
if (sd_version_is_ltxav(sd->version) && request->frames > 0) {
|
||||
int latent_frames = ((request->frames - 1) / 8) + 1;
|
||||
sample_seq_len *= latent_frames;
|
||||
} else if (sd_version_is_minimax_h3(sd->version) && request->frames > 0) {
|
||||
sample_seq_len *= sd->video_frames_to_latent_frames(request->frames);
|
||||
}
|
||||
sigmas = sd->denoiser->get_sigmas(total_steps,
|
||||
sample_seq_len,
|
||||
scheduler,
|
||||
sd->version,
|
||||
sample_params->extra_sample_args);
|
||||
}
|
||||
|
||||
eta = resolve_eta(sd, eta, sample_method);
|
||||
|
||||
if (high_noise_sample_steps < 0) {
|
||||
for (size_t i = 0; i < sigmas.size(); ++i) {
|
||||
if (sigmas[i] < moe_boundary) {
|
||||
high_noise_sample_steps = static_cast<int>(i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
LOG_VERBOSE("switching from high noise model at step %d", high_noise_sample_steps);
|
||||
}
|
||||
|
||||
LOG_INFO("sampling using %s method", sampling_methods_str[sample_method]);
|
||||
if (high_noise_sample_steps > 0) {
|
||||
high_noise_sample_method = resolve_sample_method(sd,
|
||||
high_noise_sample_method);
|
||||
high_noise_eta = resolve_eta(sd, high_noise_eta, high_noise_sample_method);
|
||||
LOG_INFO("sampling(high noise) using %s method", sampling_methods_str[high_noise_sample_method]);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<float> make_hires_sigma_schedule(StableDiffusionGGML* sd,
|
||||
const sd_hires_params_t& hires,
|
||||
const sd_sample_params_t& sample_params,
|
||||
sample_method_t sample_method,
|
||||
int default_steps,
|
||||
int sample_seq_len,
|
||||
int* scheduler_steps_out) {
|
||||
if (scheduler_steps_out != nullptr) {
|
||||
*scheduler_steps_out = 0;
|
||||
}
|
||||
|
||||
if (hires.custom_sigmas_count > 0 && hires.custom_sigmas != nullptr) {
|
||||
std::vector<float> custom_sigmas(hires.custom_sigmas,
|
||||
hires.custom_sigmas + hires.custom_sigmas_count);
|
||||
if (scheduler_steps_out != nullptr) {
|
||||
*scheduler_steps_out = static_cast<int>(custom_sigmas.size()) - 1;
|
||||
}
|
||||
return custom_sigmas;
|
||||
}
|
||||
|
||||
int effective_steps = hires.steps > 0 ? hires.steps : default_steps;
|
||||
effective_steps = std::max(1, effective_steps);
|
||||
|
||||
// sd-webui behavior: scale up total steps so trimming by denoising_strength yields exactly hires_steps effective steps,
|
||||
// unlike img2img which trims from a fixed step count.
|
||||
int scheduler_steps = static_cast<int>(effective_steps / hires.denoising_strength);
|
||||
scheduler_steps = std::max(1, scheduler_steps);
|
||||
|
||||
scheduler_t scheduler = resolve_scheduler(sd,
|
||||
sample_params.scheduler,
|
||||
sample_method);
|
||||
std::vector<float> sigmas = sd->denoiser->get_sigmas(scheduler_steps,
|
||||
sample_seq_len,
|
||||
scheduler,
|
||||
sd->version,
|
||||
sample_params.extra_sample_args);
|
||||
size_t t_enc = static_cast<size_t>(scheduler_steps * hires.denoising_strength);
|
||||
if (t_enc >= static_cast<size_t>(scheduler_steps)) {
|
||||
t_enc = static_cast<size_t>(scheduler_steps) - 1;
|
||||
}
|
||||
if (scheduler_steps_out != nullptr) {
|
||||
*scheduler_steps_out = scheduler_steps;
|
||||
}
|
||||
return std::vector<float>(sigmas.begin() + scheduler_steps - static_cast<int>(t_enc) - 1,
|
||||
sigmas.end());
|
||||
}
|
||||
|
||||
} // namespace sd::pipeline
|
||||
@@ -0,0 +1,110 @@
|
||||
#ifndef __SD_PIPELINE_REQUEST_H__
|
||||
#define __SD_PIPELINE_REQUEST_H__
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "stable-diffusion.h"
|
||||
|
||||
class StableDiffusionGGML;
|
||||
|
||||
namespace sd::pipeline {
|
||||
|
||||
extern const char* sampling_methods_str[];
|
||||
|
||||
enum sample_method_t default_sample_method(const StableDiffusionGGML* sd);
|
||||
|
||||
enum scheduler_t default_scheduler(const StableDiffusionGGML* sd, enum sample_method_t sample_method);
|
||||
|
||||
float resolve_eta(StableDiffusionGGML* sd,
|
||||
float eta,
|
||||
enum sample_method_t sample_method);
|
||||
|
||||
struct GenerationRequest {
|
||||
std::string prompt;
|
||||
std::string negative_prompt;
|
||||
int width = -1;
|
||||
int height = -1;
|
||||
int clip_skip = -1;
|
||||
int vae_scale_factor = -1;
|
||||
int diffusion_model_down_factor = -1;
|
||||
int64_t seed = -1;
|
||||
bool use_uncond = false;
|
||||
bool use_img_uncond = false;
|
||||
bool use_high_noise_uncond = false;
|
||||
bool use_high_noise_img_uncond = false;
|
||||
bool has_ref_images = false;
|
||||
const sd_cache_params_t* cache_params = nullptr;
|
||||
int batch_count = 1;
|
||||
int qwen_image_layers = 3;
|
||||
int shifted_timestep = 0;
|
||||
float strength = 1.f;
|
||||
float control_strength = 0.f;
|
||||
float eta = 0.f;
|
||||
sd_guidance_params_t guidance = {};
|
||||
sd_guidance_params_t high_noise_guidance = {};
|
||||
sd_pm_params_t pm_params = {};
|
||||
sd_pulid_params_t pulid_params = {};
|
||||
sd_hires_params_t hires = {};
|
||||
int frames = -1;
|
||||
int requested_frames = -1;
|
||||
int fps = 16;
|
||||
float vace_strength = 1.f;
|
||||
|
||||
GenerationRequest(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params);
|
||||
|
||||
GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params);
|
||||
|
||||
void align_generation_request_size();
|
||||
|
||||
void align_image_size(int* target_width, int* target_height, const char* label);
|
||||
|
||||
void resolve_hires();
|
||||
|
||||
static void resolve_guidance(StableDiffusionGGML* sd,
|
||||
sd_guidance_params_t* guidance,
|
||||
bool* use_uncond,
|
||||
bool* use_img_uncond,
|
||||
bool has_ref_images,
|
||||
const char* stage_name = nullptr);
|
||||
|
||||
void resolve(StableDiffusionGGML* sd);
|
||||
};
|
||||
|
||||
struct SamplePlan {
|
||||
enum sample_method_t sample_method = SAMPLE_METHOD_COUNT;
|
||||
enum sample_method_t high_noise_sample_method = SAMPLE_METHOD_COUNT;
|
||||
const char* extra_sample_args = nullptr;
|
||||
const char* high_noise_extra_sample_args = nullptr;
|
||||
float eta = 0.f;
|
||||
float high_noise_eta = 0.f;
|
||||
int sample_steps = 0;
|
||||
int high_noise_sample_steps = 0;
|
||||
int total_steps = 0;
|
||||
float moe_boundary = 0.f;
|
||||
std::vector<float> sigmas;
|
||||
|
||||
SamplePlan(StableDiffusionGGML* sd,
|
||||
const sd_img_gen_params_t* sd_img_gen_params,
|
||||
const GenerationRequest& request);
|
||||
|
||||
SamplePlan(StableDiffusionGGML* sd,
|
||||
const sd_vid_gen_params_t* sd_vid_gen_params,
|
||||
const GenerationRequest& request);
|
||||
|
||||
void resolve(StableDiffusionGGML* sd,
|
||||
const GenerationRequest* request,
|
||||
const sd_sample_params_t* sample_params);
|
||||
};
|
||||
|
||||
std::vector<float> make_hires_sigma_schedule(StableDiffusionGGML* sd,
|
||||
const sd_hires_params_t& hires,
|
||||
const sd_sample_params_t& sample_params,
|
||||
sample_method_t sample_method,
|
||||
int default_steps,
|
||||
int sample_seq_len,
|
||||
int* scheduler_steps_out);
|
||||
|
||||
} // namespace sd::pipeline
|
||||
|
||||
#endif // __SD_PIPELINE_REQUEST_H__
|
||||
File diff suppressed because it is too large
Load Diff
+11
-6342
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user