upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-09-21 05:27:50 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: restore legacy fp8 handling when building with upstream ggml (#2001)

Browse Source
This commit is contained in:
Wagner Bruna
2026-09-21 00:48:30 +08:00
committed by GitHub
parent 137f7409bb
commit 008ca5b492
8 changed files with 106 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmake/ggml.cmake
+1 -1
View File
@@ -23,5 +23,5 @@ set_property(TARGET ${SD_LIB} PROPERTY SD_GGML_PRIVATE_INCLUDE_DIR "${sd_ggml_pr
if(SD_USE_UPSTREAM_GGML)
target_compile_definitions(${SD_LIB} PUBLIC SD_USE_UPSTREAM_GGML)
message(WARNING "Using upstream GGML: FP8 and INT8 tensorwise/convrot are disabled. Some operators may be unsupported and performance may be lower than with patched GGML.")
message(WARNING "Using upstream GGML: INT8 tensorwise/convrot is disabled and FP8 weights are converted to F16 at load time. Some operators may be unsupported and performance may be lower than with patched GGML.")
endif()
docs/build.md
+6 -4
View File
@@ -32,10 +32,12 @@ Backend options such as `-DSD_CUDA=ON` apply to the selected tree as usual.
`SD_USE_UPSTREAM_GGML` defaults to `OFF`, which enables the patched GGML
extensions. Set it to `ON` when using upstream GGML; it selects the compatibility
mode and does not download or replace the GGML source tree. Upstream mode
disables FP8 and INT8 tensorwise/convrot and rejects their model files with an
explicit error. FP8 weight type requests, tensor type rules and conversion
outputs are also rejected; no automatic conversion is performed.
mode and does not download or replace the GGML source tree. Upstream mode keeps
the original FP8 safetensors handling: FP8 tensors are converted to F16 at load
time (one byte per element in the file, two in RAM and VRAM). INT8
tensorwise/convrot is disabled and its model files are rejected with an explicit
error. FP8 GGUF files, FP8 weight type requests and tensor type rules are also
rejected; no automatic conversion is performed.
Upstream GGML may lack some operators and performance optimizations provided by
the patched version. A warning is emitted during CMake configuration and when
docs/ideogram4.md
+3
View File
@@ -18,6 +18,9 @@ at one byte per element in RAM and VRAM. Backends that cannot multiply FP8
weights directly cast only the active layer to a temporary BF16 tensor during
execution; the loader does not expand the entire checkpoint to BF16.
With `SD_USE_UPSTREAM_GGML=ON`, FP8 tensors are converted to F16 at load time
instead (two bytes per element in RAM and VRAM).
Use `ideogram4_fp8.safetensors` and `ideogram4_uncond_fp8.safetensors` directly
with `--diffusion-model` and `--uncond-diffusion-model`, respectively.
src/model_io/safetensors_io.cpp
+18 -9
View File
@@ -86,7 +86,12 @@ static ggml_type safetensors_dtype_to_ggml_type(const std::string& dtype) {
ttype = GGML_TYPE_F32;
} else if (dtype == "F64") {
ttype = GGML_TYPE_F32;
#ifndef SD_USE_UPSTREAM_GGML
#ifdef SD_USE_UPSTREAM_GGML
} else if (dtype == "F8_E4M3") {
ttype = GGML_TYPE_F16;
} else if (dtype == "F8_E5M2") {
ttype = GGML_TYPE_F16;
#else
} else if (dtype == "F8_E4M3") {
ttype = GGML_TYPE_F8_E4M3;
} else if (dtype == "F8_E5M2") {
@@ -287,12 +292,6 @@ bool read_safetensors_file(const std::string& file_path,
continue;
}
#ifdef SD_USE_UPSTREAM_GGML
if (dtype == "F8_E4M3" || dtype == "F8_E5M2") {
set_error(error, "FP8 is not supported by this ggml build (tensor '" + name + "')");
return false;
}
#endif
ggml_type type = safetensors_dtype_to_ggml_type(dtype);
if (type == GGML_TYPE_COUNT) {
set_error(error, "unsupported dtype '" + dtype + "' (tensor '" + name + "')");
@@ -376,10 +375,20 @@ bool read_safetensors_file(const std::string& file_path,
bool tensor_size_ok;
if (dtype == "F8_E4M3") {
tensor_storage.is_f8_e4m3 = true;
tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size);
#ifdef SD_USE_UPSTREAM_GGML
// f8 -> f16
tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size * 2);
#else
tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size);
#endif
} else if (dtype == "F8_E5M2") {
tensor_storage.is_f8_e5m2 = true;
tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size);
#ifdef SD_USE_UPSTREAM_GGML
// f8 -> f16
tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size * 2);
#else
tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size);
#endif
} else if (dtype == "F64") {
tensor_storage.is_f64 = true;
// f64 -> f32
src/model_io/tensor_storage.h
+4
View File
@@ -58,6 +58,10 @@ struct TensorStorage {
int64_t nbytes_to_read() const {
if (is_f64 || is_i64) {
return nbytes() * 2;
#ifdef SD_USE_UPSTREAM_GGML
} else if (is_f8_e4m3 || is_f8_e5m2) {
return nbytes() / 2;
#endif
} else {
return nbytes();
}
src/model_loader.cpp
+70
View File
@@ -35,6 +35,66 @@
/*================================================= Preprocess ==================================================*/
#ifdef SD_USE_UPSTREAM_GGML
uint16_t f8_e4m3_to_f16(uint8_t f8) {
const uint32_t exponent_bias = 7;
if (f8 == 0xff) {
return ggml_fp32_to_fp16(-NAN);
} else if (f8 == 0x7f) {
return ggml_fp32_to_fp16(NAN);
}
uint32_t sign = f8 & 0x80;
uint32_t exponent = (f8 & 0x78) >> 3;
uint32_t mantissa = f8 & 0x07;
uint32_t result = sign << 24;
if (exponent == 0) {
if (mantissa > 0) {
exponent = 0x7f - exponent_bias;
// yes, 2 times
if ((mantissa & 0x04) == 0) {
mantissa &= 0x03;
mantissa <<= 1;
exponent -= 1;
}
if ((mantissa & 0x04) == 0) {
mantissa &= 0x03;
mantissa <<= 1;
exponent -= 1;
}
result |= (mantissa & 0x03) << 21;
result |= exponent << 23;
}
} else {
result |= mantissa << 20;
exponent += 0x7f - exponent_bias;
result |= exponent << 23;
}
return ggml_fp32_to_fp16(*reinterpret_cast<const float*>(&result));
}
uint16_t f8_e5m2_to_f16(uint8_t fp8) {
return static_cast<uint16_t>(fp8) << 8;
}
void f8_e4m3_to_f16_vec(uint8_t* src, uint16_t* dst, int64_t n) {
// support inplace op
for (int64_t i = n - 1; i >= 0; i--) {
dst[i] = f8_e4m3_to_f16(src[i]);
}
}
void f8_e5m2_to_f16_vec(uint8_t* src, uint16_t* dst, int64_t n) {
// support inplace op
for (int64_t i = n - 1; i >= 0; i--) {
dst[i] = f8_e5m2_to_f16(src[i]);
}
}
#endif
void f64_to_f32_vec(double* src, float* dst, int64_t n) {
// support inplace op
for (int64_t i = 0; i < n; i++) {
@@ -861,6 +921,10 @@ std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggm
if (tensor_storage.is_f64 ||
tensor_storage.is_i64 ||
#ifdef SD_USE_UPSTREAM_GGML
tensor_storage.is_f8_e4m3 ||
tensor_storage.is_f8_e5m2 ||
#endif
tensor_storage.type != dst_tensor->type) {
continue;
}
@@ -1150,6 +1214,12 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
f64_to_f32_vec((double*)read_buf, (float*)target_buf, tensor_storage.nelements());
} else if (tensor_storage.is_i64) {
i64_to_i32_vec((int64_t*)read_buf, (int32_t*)target_buf, tensor_storage.nelements());
#ifdef SD_USE_UPSTREAM_GGML
} else if (tensor_storage.is_f8_e4m3) {
f8_e4m3_to_f16_vec((uint8_t*)read_buf, (uint16_t*)target_buf, tensor_storage.nelements());
} else if (tensor_storage.is_f8_e5m2) {
f8_e5m2_to_f16_vec((uint8_t*)read_buf, (uint16_t*)target_buf, tensor_storage.nelements());
#endif
}
if (tensor_storage.type != dst_tensor->type) {
if (convert_buf == nullptr) {
src/model_loader_files.cpp
+1 -1
View File
@@ -147,7 +147,7 @@ bool ModelLoader::add_file_impl(const std::string& path, const std::string& pref
}
if (tensor.index_in_zip < 0) {
const auto& stamp = physical_files[tensor.file_index];
if (tensor.offset > stamp.size || elements / block_size * type_size > stamp.size - tensor.offset) {
if (tensor.offset > stamp.size || static_cast<uint64_t>(tensor.nbytes_to_read()) > stamp.size - tensor.offset) { //kcpp int8 fp8
LOG_ERROR("tensor '%s' extends beyond its model file", tensor.name.c_str());
return false;
}
src/pipeline/diffusion_engine.cpp
+3 -2
View File
@@ -860,8 +860,9 @@ bool StableDiffusionGGML::init_model_loader(ModelLoader& model_loader, ModelConf
bool StableDiffusionGGML::init(const sd_ctx_params_t* sd_ctx_params) {
#ifdef SD_USE_UPSTREAM_GGML
LOG_WARN(
"Using upstream GGML: FP8 and INT8 tensorwise/convrot are disabled. "
"Some operators may be unsupported and performance may be lower than with patched GGML.");
"Using upstream GGML: INT8 tensorwise/convrot is disabled and FP8 weights are "
"converted to F16 at load time. Some operators may be unsupported and performance "
"may be lower than with patched GGML.");
#endif
if (!validate_tensor_types(sd_ctx_params->wtype, sd_ctx_params->tensor_type_rules)) {
return false;