diff --git a/CMakeLists.txt b/CMakeLists.txt index c6ebaded..134719ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,6 +95,8 @@ option(SD_MUSA "sd: musa backend" OFF) option(SD_BUILD_SHARED_LIBS "sd: build shared libs" OFF) option(SD_BUILD_SHARED_GGML_LIB "sd: build ggml as a separate shared lib" OFF) option(SD_USE_SYSTEM_GGML "sd: use system-installed GGML library" OFF) +option(SD_USE_UPSTREAM_GGML "sd: build with upstream GGML instead of the patched GGML extensions" OFF) +set(SD_GGML_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ggml" CACHE PATH "sd: ggml source directory (also supplies private headers for system ggml)") #option(SD_BUILD_SERVER "sd: build server example" ON) set(CMAKE_C_STANDARD 11) @@ -325,18 +327,7 @@ if (NOT SD_USE_SYSTEM_GGML) endif() # deps -# Only add ggml if it hasn't been added yet -if (NOT TARGET ggml) - if (SD_USE_SYSTEM_GGML) - find_package(ggml REQUIRED) - if (NOT ggml_FOUND) - message(FATAL_ERROR "System-installed GGML library not found.") - endif() - add_library(ggml ALIAS ggml::ggml) - else() - add_subdirectory(ggml) - endif() -endif() +include(cmake/ggml.cmake) add_subdirectory(thirdparty) diff --git a/cmake/ggml.cmake b/cmake/ggml.cmake new file mode 100644 index 00000000..eafc7e2b --- /dev/null +++ b/cmake/ggml.cmake @@ -0,0 +1,27 @@ +if(NOT TARGET ggml AND NOT TARGET ggml::ggml) + if(SD_USE_SYSTEM_GGML) + find_package(ggml REQUIRED) + else() + add_subdirectory("${SD_GGML_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/ggml") + endif() +endif() +if(NOT TARGET ggml) + add_library(ggml ALIAS ggml::ggml) +endif() + +get_target_property(sd_ggml_imported ggml IMPORTED) +if(sd_ggml_imported) + set(sd_ggml_private_include "${SD_GGML_SOURCE_DIR}/src") +else() + get_target_property(sd_ggml_private_include ggml SOURCE_DIR) +endif() +if(NOT EXISTS "${sd_ggml_private_include}/ggml-impl.h") + message(FATAL_ERROR "Set SD_GGML_SOURCE_DIR to the source tree matching the selected ggml library (ggml-impl.h is required).") +endif() +target_include_directories(${SD_LIB} PRIVATE "${sd_ggml_private_include}") +set_property(TARGET ${SD_LIB} PROPERTY SD_GGML_PRIVATE_INCLUDE_DIR "${sd_ggml_private_include}") + +if(SD_USE_UPSTREAM_GGML) + target_compile_definitions(${SD_LIB} PUBLIC SD_USE_UPSTREAM_GGML) + message(WARNING "Using upstream GGML: FP8 and INT8 tensorwise/convrot are disabled. Some operators may be unsupported and performance may be lower than with patched GGML.") +endif() diff --git a/docs/build.md b/docs/build.md index d33f9329..a50ec9bf 100644 --- a/docs/build.md +++ b/docs/build.md @@ -16,6 +16,38 @@ git submodule init git submodule update ``` +## Selecting a GGML source tree + +By default, sd.cpp builds the patched GGML submodule in `ggml/`. To build with +an upstream GGML checkout instead, enable `SD_USE_UPSTREAM_GGML` and set +`SD_GGML_SOURCE_DIR`: + +```shell +cmake -S . -B build-upstream -DSD_USE_UPSTREAM_GGML=ON -DSD_GGML_SOURCE_DIR=../ggml-upstream +cmake --build build-upstream --config Release +``` + +The selected source tree supplies both the library and its private headers. +Backend options such as `-DSD_CUDA=ON` apply to the selected tree as usual. + +`SD_USE_UPSTREAM_GGML` defaults to `OFF`, which enables the patched GGML +extensions. Set it to `ON` when using upstream GGML; it selects the compatibility +mode and does not download or replace the GGML source tree. Upstream mode +disables FP8 and INT8 tensorwise/convrot and rejects their model files with an +explicit error. FP8 weight type requests, tensor type rules and conversion +outputs are also rejected; no automatic conversion is performed. + +Upstream GGML may lack some operators and performance optimizations provided by +the patched version. A warning is emitted during CMake configuration and when +creating an inference context. Ordinary floating-point and shared GGML +quantization types remain available, subject to backend operator support. + +`SD_USE_SYSTEM_GGML=ON` instead links an installed GGML CMake package, located +with `ggml_DIR` or `CMAKE_PREFIX_PATH`. In that mode, `SD_GGML_SOURCE_DIR` must +point to the matching source tree for private headers. The installed library +must use the same ABI settings as sd.cpp, including `GGML_MAX_NAME`. +Set `SD_USE_UPSTREAM_GGML=ON` as well if the installed package is upstream GGML. + ## WebP and WebM Support in Examples The example applications (`examples/cli` and `examples/server`) use `libwebp` to support WebP image I/O, and `examples/cli` can also use `libwebm` for `.webm` video output. Both are enabled by default. WebM output currently reuses `libwebp` to encode each frame as VP8 before muxing with `libwebm`. diff --git a/docs/int8_convrot.md b/docs/int8_convrot.md index d41df8bd..3b65113d 100644 --- a/docs/int8_convrot.md +++ b/docs/int8_convrot.md @@ -2,6 +2,9 @@ sd.cpp can load and execute ComfyUI `int8_tensorwise` safetensors with `convrot` metadata directly. The stored INT8 weights are not converted to another weight type at load time. +This requires the INT8 tensorwise/convrot extensions in the patched GGML. +Builds with `SD_USE_UPSTREAM_GGML=ON` reject these files during loading. + ## Checkpoint format Each quantized linear module contains the following tensors: diff --git a/src/convert.cpp b/src/convert.cpp index 8e94a940..c82d2557 100644 --- a/src/convert.cpp +++ b/src/convert.cpp @@ -362,6 +362,9 @@ bool convert_with_components(const char* model_path, const char* tensor_type_rules, bool convert_name, int n_threads) { + if (!validate_tensor_types(output_type, tensor_type_rules)) { + return false; + } ModelLoader model_loader; bool loaded_any = false; diff --git a/src/core/compute_workspace.cpp b/src/core/compute_workspace.cpp index 58e85fa6..54f0538f 100644 --- a/src/core/compute_workspace.cpp +++ b/src/core/compute_workspace.cpp @@ -11,7 +11,7 @@ #include "core/ggml_graph_cut.h" #include "core/util.h" #include "ggml-cpu.h" -#include "ggml/src/ggml-impl.h" +#include "ggml-impl.h" namespace sd { ComputeWorkspace::~ComputeWorkspace() { diff --git a/src/core/ggml_extend.cpp b/src/core/ggml_extend.cpp index ab55a946..fd65f41c 100644 --- a/src/core/ggml_extend.cpp +++ b/src/core/ggml_extend.cpp @@ -1,6 +1,7 @@ #include "core/ggml_extend.h" #include +#include #include #include "core/ggml_extend_backend.h" @@ -247,6 +248,7 @@ ggml_tensor* ggml_ext_linear_i8_tensorwise(ggml_context* ctx, ggml_tensor* b, int convrot_group_size, float scale) { +#ifndef SD_USE_UPSTREAM_GGML GGML_ASSERT(x->type == GGML_TYPE_F32 || (x->type == GGML_TYPE_I8 && scale == 1.f)); if (scale != 1.f) { x = ggml_ext_scale(ctx, x, scale); @@ -270,6 +272,16 @@ ggml_tensor* ggml_ext_linear_i8_tensorwise(ggml_context* ctx, } } return x; +#else + GGML_UNUSED(ctx); + GGML_UNUSED(x); + GGML_UNUSED(w); + GGML_UNUSED(weight_scale); + GGML_UNUSED(b); + GGML_UNUSED(convrot_group_size); + GGML_UNUSED(scale); + throw std::runtime_error("INT8 tensorwise/convrot is not supported by this ggml build"); +#endif } ggml_tensor* ggml_ext_pad_ext(ggml_context* ctx, diff --git a/src/core/ggml_extend_backend.cpp b/src/core/ggml_extend_backend.cpp index e6fd496a..f3049dc2 100644 --- a/src/core/ggml_extend_backend.cpp +++ b/src/core/ggml_extend_backend.cpp @@ -13,7 +13,7 @@ #endif #include "core/util.h" -#include "ggml/src/ggml-impl.h" +#include "ggml-impl.h" #include "stable-diffusion.h" static std::string trim_copy(const std::string& value) { diff --git a/src/core/ggml_graph_cut.cpp b/src/core/ggml_graph_cut.cpp index 78ebb2d5..b10e7a14 100644 --- a/src/core/ggml_graph_cut.cpp +++ b/src/core/ggml_graph_cut.cpp @@ -16,7 +16,7 @@ #include "ggml-alloc.h" #include "ggml-backend.h" -#include "ggml/src/ggml-impl.h" +#include "ggml-impl.h" namespace sd::ggml_graph_cut { diff --git a/src/core/util.cpp b/src/core/util.cpp index 1653f241..cf85ebef 100644 --- a/src/core/util.cpp +++ b/src/core/util.cpp @@ -414,14 +414,43 @@ std::vector split_string(const std::string& str, char delimiter) { } ggml_type sd_type_to_ggml_type(sd_type_t sdtype) { + if (sdtype == SD_TYPE_F8_E4M3 || sdtype == SD_TYPE_F8_E5M2) { +#ifndef SD_USE_UPSTREAM_GGML + return sdtype == SD_TYPE_F8_E4M3 ? GGML_TYPE_F8_E4M3 : GGML_TYPE_F8_E5M2; +#else + return GGML_TYPE_COUNT; +#endif + } const int type_value = static_cast(sdtype); - if (type_value < std::min(SD_TYPE_COUNT, GGML_TYPE_COUNT)) { + if (type_value >= 0 && type_value < std::min(SD_TYPE_COUNT, GGML_TYPE_COUNT)) { return static_cast(type_value); } else { return GGML_TYPE_COUNT; } } +bool validate_tensor_types(sd_type_t type, const char* tensor_type_rules) { + if (type != SD_TYPE_COUNT && sd_type_to_ggml_type(type) == GGML_TYPE_COUNT) { + LOG_ERROR("weight type %s is not supported by this ggml build", sd_type_name(type)); + return false; + } +#ifdef SD_USE_UPSTREAM_GGML + for (const auto& rule : split_string(SAFE_STR(tensor_type_rules), ',')) { + const auto pos = rule.find('='); + if (pos != std::string::npos) { + const auto name = rule.substr(pos + 1); + if (name == "f8_e4m3" || name == "f8_e5m2") { + LOG_ERROR("FP8 is not supported by this ggml build (tensor type rule '%s')", rule.c_str()); + return false; + } + } + } +#else + GGML_UNUSED(tensor_type_rules); +#endif + return true; +} + KeyValueArgs parse_key_value_args(const char* args, const char* context) { KeyValueArgs pairs; diff --git a/src/core/util.h b/src/core/util.h index 390d063d..334224da 100644 --- a/src/core/util.h +++ b/src/core/util.h @@ -90,6 +90,7 @@ void log_printf(sd_log_level_t level, const char* file, int line, const char* fo void sd_ggml_log_callback(ggml_log_level level, const char* text, void*); ggml_type sd_type_to_ggml_type(sd_type_t sdtype); +bool validate_tensor_types(sd_type_t type, const char* tensor_type_rules); std::string trim(const std::string& s); diff --git a/src/model/common/ggml_block.hpp b/src/model/common/ggml_block.hpp index 54fa9d96..a06fe763 100644 --- a/src/model/common/ggml_block.hpp +++ b/src/model/common/ggml_block.hpp @@ -208,6 +208,7 @@ public: ggml_tensor* w = params["weight"]; const float scale = ctx->linear_scale > 0.f ? ctx->linear_scale : this->scale; ggml_tensor* weight_scale = has_weight_scale ? params["weight_scale"] : nullptr; +#ifndef SD_USE_UPSTREAM_GGML if (w->type == GGML_TYPE_F8_E4M3 || w->type == GGML_TYPE_F8_E5M2) { bool supports_fp8_matmul = false; if (ctx->backend != nullptr) { @@ -221,6 +222,7 @@ public: w = ggml_cast(ctx->ggml_ctx, w, GGML_TYPE_BF16); } } +#endif ggml_tensor* b = nullptr; if (bias) { b = params["bias"]; @@ -238,6 +240,7 @@ public: if (ctx->weight_adapter && b != nullptr) { b = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, b, prefix + "bias"); } +#ifndef SD_USE_UPSTREAM_GGML if (int8_convrot && scale == 1.f) { const auto cache_key = std::make_pair(x, int8_convrot_group_size); auto cached = ctx->int8_convrot_cache.find(cache_key); @@ -248,6 +251,7 @@ public: x = cached->second; } } +#endif out = ggml_ext_linear_i8_tensorwise(ctx->ggml_ctx, x, w, diff --git a/src/model_io/gguf_io.cpp b/src/model_io/gguf_io.cpp index cd22312d..081fcf97 100644 --- a/src/model_io/gguf_io.cpp +++ b/src/model_io/gguf_io.cpp @@ -57,6 +57,18 @@ bool read_gguf_file(const std::string& file_path, size_t data_offset = gguf_reader.data_offset(); for (const auto& gguf_tensor_info : gguf_reader.tensors()) { +#ifdef SD_USE_UPSTREAM_GGML + if (static_cast(gguf_tensor_info.type) == SD_TYPE_F8_E4M3 || + static_cast(gguf_tensor_info.type) == SD_TYPE_F8_E5M2) { + set_error(error, "FP8 is not supported by this ggml build (tensor '" + gguf_tensor_info.name + "')"); + return false; + } +#endif + if (static_cast(gguf_tensor_info.type) >= GGML_TYPE_COUNT || + ggml_get_type_traits(gguf_tensor_info.type)->type_size == 0) { + set_error(error, "unsupported GGUF tensor type (tensor '" + gguf_tensor_info.name + "')"); + return false; + } TensorStorage tensor_storage( gguf_tensor_info.name, gguf_tensor_info.type, diff --git a/src/model_io/safetensors_io.cpp b/src/model_io/safetensors_io.cpp index d8b8dc51..28c55f3e 100644 --- a/src/model_io/safetensors_io.cpp +++ b/src/model_io/safetensors_io.cpp @@ -86,10 +86,12 @@ static ggml_type safetensors_dtype_to_ggml_type(const std::string& dtype) { ttype = GGML_TYPE_F32; } else if (dtype == "F64") { ttype = GGML_TYPE_F32; +#ifndef SD_USE_UPSTREAM_GGML } else if (dtype == "F8_E4M3") { ttype = GGML_TYPE_F8_E4M3; } else if (dtype == "F8_E5M2") { ttype = GGML_TYPE_F8_E5M2; +#endif } else if (dtype == "I32") { ttype = GGML_TYPE_I32; } else if (dtype == "I64") { @@ -230,6 +232,12 @@ bool read_safetensors_file(const std::string& file_path, if (!read_comfy_quant_config(file, file_path, name, data_start + begin, end - begin, config, error)) { return false; } +#ifdef SD_USE_UPSTREAM_GGML + if (config.format == "int8_tensorwise") { + set_error(error, "INT8 tensorwise/convrot is not supported by this ggml build (tensor '" + name + "')"); + return false; + } +#endif const std::string module_name = name.substr(0, name.size() - std::string(".comfy_quant").size()); comfy_quant_configs.emplace(module_name, std::move(config)); } @@ -279,6 +287,12 @@ bool read_safetensors_file(const std::string& file_path, continue; } +#ifdef SD_USE_UPSTREAM_GGML + if (dtype == "F8_E4M3" || dtype == "F8_E5M2") { + set_error(error, "FP8 is not supported by this ggml build (tensor '" + name + "')"); + return false; + } +#endif ggml_type type = safetensors_dtype_to_ggml_type(dtype); if (type == GGML_TYPE_COUNT) { set_error(error, "unsupported dtype '" + dtype + "' (tensor '" + name + "')"); diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index dabd795c..e0663c65 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -857,6 +857,14 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf } bool StableDiffusionGGML::init(const sd_ctx_params_t* sd_ctx_params) { +#ifdef SD_USE_UPSTREAM_GGML + LOG_WARN( + "Using upstream GGML: FP8 and INT8 tensorwise/convrot are disabled. " + "Some operators may be unsupported and performance may be lower than with patched GGML."); +#endif + if (!validate_tensor_types(sd_ctx_params->wtype, sd_ctx_params->tensor_type_rules)) { + return false; + } for (float scale : {sd_ctx_params->linear_scale, sd_ctx_params->attn_scale}) { if (!std::isfinite(scale) || scale < 0.f || (scale > 0.f && !std::isfinite(1.f / scale))) { LOG_ERROR("scale overrides must be finite positive values, or 0 to keep model defaults"); diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 5a56f402..8f603209 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -26,13 +26,26 @@ static float get_cache_reuse_threshold(const sd_cache_params_t& params) { } const char* sd_type_name(enum sd_type_t type) { - if ((int)type < std::min(SD_TYPE_COUNT, GGML_TYPE_COUNT)) { - return ggml_type_name((ggml_type)type); + if (type == SD_TYPE_F8_E4M3) { + return "f8_e4m3"; + } + if (type == SD_TYPE_F8_E5M2) { + return "f8_e5m2"; + } + const auto ggml_type = sd_type_to_ggml_type(type); + if (ggml_type != GGML_TYPE_COUNT) { + return ggml_type_name(ggml_type); } return NONE_STR; } enum sd_type_t str_to_sd_type(const char* str) { + if (!strcmp(str, "f8_e4m3")) { + return SD_TYPE_F8_E4M3; + } + if (!strcmp(str, "f8_e5m2")) { + return SD_TYPE_F8_E5M2; + } for (int i = 0; i < std::min(SD_TYPE_COUNT, GGML_TYPE_COUNT); i++) { auto trait = ggml_get_type_traits((ggml_type)i); if (!strcmp(str, trait->type_name)) {