fix: match exact weights in LLM config detection (#1923)

docs/ideogram4.md

@@ -11,7 +11,19 @@
 - Download Qwen3-VL-8B-Instruct
     - gguf: https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/tree/main
 
-## Convert weights
+## Use original FP8 weights
+
+The original Ideogram4 FP8 safetensors can be loaded directly. FP8 tensors stay
+at one byte per element in RAM and VRAM. Backends that cannot multiply FP8
+weights directly cast only the active layer to a temporary BF16 tensor during
+execution; the loader does not expand the entire checkpoint to BF16.
+
+Use `ideogram4_fp8.safetensors` and `ideogram4_uncond_fp8.safetensors` directly
+with `--diffusion-model` and `--uncond-diffusion-model`, respectively.
+
+## Optional conversion for quantization
+
+The following conversion is only needed when creating a quantized GGUF model.
 
 fp8 scale -> bf16
 

examples/cli/main.cpp

@@ -36,6 +36,7 @@ struct SDCliParams {
     SDMode mode             = IMG_GEN;
     std::string output_path = "output.png";
     int output_begin_idx    = -1;
+    int compression_quality = 90;
     std::string image_path;
     std::string metadata_format = "text";
 
@@ -99,6 +100,10 @@ struct SDCliParams {
              "--output-begin-idx",
              "starting index for output image sequence, must be non-negative (default 0 if specified %d in output path, 1 otherwise)",
              &output_begin_idx},
+            {"",
+             "--compression-quality",
+             "compression quality of video and JPEG / WebP images (90 by default)",
+             &compression_quality},
         };
 
         options.bool_options = {
@@ -383,11 +388,13 @@ void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy,
                                  image->data,
                                  image->width,
                                  image->height,
-                                 image->channel)) {
+                                 image->channel,
+                                 "",
+                                 cli_params->compression_quality)) {
             LOG_ERROR("save preview image to '%s' failed", cli_params->preview_path.c_str());
         }
     } else {
-        if (create_video_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, cli_params->preview_fps) != 0) {
+        if (create_video_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, cli_params->preview_fps, cli_params->compression_quality) != 0) {
             LOG_ERROR("save preview video to '%s' failed", cli_params->preview_path.c_str());
         }
     }
@@ -486,7 +493,7 @@ bool save_results(const SDCliParams& cli_params,
         std::string params          = gen_params.embed_image_metadata
                                           ? get_image_params(ctx_params, gen_params, metadata_seed, cli_params.mode)
                                           : "";
-        const bool ok               = write_image_to_file(path.string(), img.data, img.width, img.height, img.channel, params, 90);
+        const bool ok               = write_image_to_file(path.string(), img.data, img.width, img.height, img.channel, params, cli_params.compression_quality);
         LOG_INFO("save result image %d to '%s' (%s)", idx, path.string().c_str(), ok ? "success" : "failure");
         return ok;
     };
@@ -532,7 +539,7 @@ bool save_results(const SDCliParams& cli_params,
         std::string final_ext_lower = ext.string();
         std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower);
         const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm");
-        if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps, 90, mux_audio ? generated_audio : nullptr) == 0) {
+        if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr) == 0) {
             LOG_INFO("save result video to '%s'", video_path.string().c_str());
             if (generated_audio != nullptr && !mux_audio) {
                 fs::path wav_path = video_path;

examples/common/media_io.cpp

@@ -835,6 +835,9 @@ std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images
     const uint32_t audio_byte_rate       = has_audio ? static_cast<uint32_t>(audio->sample_rate * audio_block_align) : 0;
     const uint32_t audio_data_size       = has_audio ? static_cast<uint32_t>(audio_pcm.size()) : 0;
 
+    if (mjpg_quality != quality)
+        LOG_DEBUG("create_mjpg_avi...(): compression quality was limited from %i to %i", quality, mjpg_quality);
+
     std::vector<uint8_t> avi_data;
     avi_data.reserve(static_cast<size_t>(num_images) * 1024);
 

include/stable-diffusion.h

@@ -136,10 +136,13 @@ enum sd_type_t {
     // SD_TYPE_IQ4_NL_4_4 = 36,
     // SD_TYPE_IQ4_NL_4_8 = 37,
     // SD_TYPE_IQ4_NL_8_8 = 38,
-    SD_TYPE_MXFP4 = 39,  // MXFP4 (1 block)
-    SD_TYPE_NVFP4 = 40,  // NVFP4 (4 blocks, E4M3 scale)
-    SD_TYPE_Q1_0  = 41,
-    SD_TYPE_COUNT = 42,
+    SD_TYPE_MXFP4   = 39,  // MXFP4 (1 block)
+    SD_TYPE_NVFP4   = 40,  // NVFP4 (4 blocks, E4M3 scale)
+    SD_TYPE_Q1_0    = 41,
+    SD_TYPE_Q2_0    = 42,
+    SD_TYPE_F8_E4M3 = 43,
+    SD_TYPE_F8_E5M2 = 44,
+    SD_TYPE_COUNT   = 45,
 };
 
 enum sd_log_level_t {

src/core/ggml_extend.hpp

@@ -3407,7 +3407,6 @@ protected:
     bool bias;
     bool force_f32;
     bool force_prec_f32;
-    bool allow_weight_scale;
     bool has_weight_scale       = false;
     bool int8_convrot           = false;
     int int8_convrot_group_size = 0;
@@ -3430,8 +3429,11 @@ protected:
         }
         auto weight_storage           = tensor_storage_map.find(prefix + "weight");
         const bool is_int8_tensorwise = weight_storage != tensor_storage_map.end() && weight_storage->second.is_int8_tensorwise;
-        if ((allow_weight_scale || is_int8_tensorwise) && tensor_storage_map.find(prefix + "weight_scale") != tensor_storage_map.end()) {
-            params["weight_scale"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, out_features);
+        auto weight_scale_storage     = tensor_storage_map.find(prefix + "weight_scale");
+        if (weight_scale_storage != tensor_storage_map.end()) {
+            const int64_t scale_nelements = weight_scale_storage->second.nelements();
+            GGML_ASSERT(scale_nelements == 1 || scale_nelements == out_features);
+            params["weight_scale"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, scale_nelements);
             has_weight_scale       = true;
         }
         if (is_int8_tensorwise) {
@@ -3445,17 +3447,15 @@ protected:
 public:
     Linear(int64_t in_features,
            int64_t out_features,
-           bool bias               = true,
-           bool force_f32          = false,
-           bool force_prec_f32     = false,
-           float scale             = 1.f,
-           bool allow_weight_scale = false)
+           bool bias           = true,
+           bool force_f32      = false,
+           bool force_prec_f32 = false,
+           float scale         = 1.f)
         : in_features(in_features),
           out_features(out_features),
           bias(bias),
           force_f32(force_f32),
           force_prec_f32(force_prec_f32),
-          allow_weight_scale(allow_weight_scale),
           scale(scale) {}
 
     void set_scale(float scale_) {
@@ -3467,7 +3467,11 @@ public:
     }
 
     ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
-        ggml_tensor* w = params["weight"];
+        ggml_tensor* w            = params["weight"];
+        ggml_tensor* weight_scale = has_weight_scale ? params["weight_scale"] : nullptr;
+        if (w->type == GGML_TYPE_F8_E4M3 || w->type == GGML_TYPE_F8_E5M2) {
+            w = ggml_cast(ctx->ggml_ctx, w, GGML_TYPE_BF16);
+        }
         ggml_tensor* b = nullptr;
         if (bias) {
             b = params["bias"];
@@ -3498,7 +3502,7 @@ public:
             out = ggml_ext_linear_i8_tensorwise(ctx->ggml_ctx,
                                                 x,
                                                 w,
-                                                params["weight_scale"],
+                                                weight_scale,
                                                 b,
                                                 int8_convrot ? int8_convrot_group_size : 0,
                                                 scale);
@@ -3517,6 +3521,30 @@ public:
             }
             return out;
         }
+        if (has_weight_scale) {
+            out = ggml_ext_linear(ctx->ggml_ctx, x, w, nullptr, force_prec_f32, scale);
+            out = ggml_mul(ctx->ggml_ctx, out, weight_scale);
+            if (ctx->weight_adapter) {
+                WeightAdapter::ForwardParams forward_params;
+                forward_params.op_type               = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR;
+                forward_params.linear.force_prec_f32 = force_prec_f32;
+                forward_params.linear.scale          = scale;
+                out                                  = ctx->weight_adapter->add_lora_to_output(ctx->ggml_ctx,
+                                                                                               ctx->backend,
+                                                                                               x,
+                                                                                               w,
+                                                                                               out,
+                                                                                               prefix,
+                                                                                               forward_params);
+                if (b != nullptr) {
+                    b = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, b, prefix + "bias");
+                }
+            }
+            if (b != nullptr) {
+                out = ggml_add_inplace(ctx->ggml_ctx, out, b);
+            }
+            return out;
+        }
         if (ctx->weight_adapter) {
             WeightAdapter::ForwardParams forward_params;
             forward_params.op_type               = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR;
@@ -3526,12 +3554,6 @@ public:
         } else {
             out = ggml_ext_linear(ctx->ggml_ctx, x, w, linear_bias, force_prec_f32, scale);
         }
-        if (has_weight_scale) {
-            out = ggml_mul(ctx->ggml_ctx, out, params["weight_scale"]);
-            if (b != nullptr) {
-                out = ggml_add_inplace(ctx->ggml_ctx, out, b);
-            }
-        }
         return out;
     }
 };

src/model/diffusion/ideogram4.hpp

@@ -142,7 +142,7 @@ namespace Ideogram4 {
     __STATIC_INLINE__ std::shared_ptr<Linear> make_linear(int64_t in_features,
                                                           int64_t out_features,
                                                           bool bias = true) {
-        return std::make_shared<Linear>(in_features, out_features, bias, false, false, 1.f, true);
+        return std::make_shared<Linear>(in_features, out_features, bias);
     }
 
     __STATIC_INLINE__ std::vector<float> gen_ideogram4_pe(int grid_h,

src/model/te/llm.hpp

@@ -232,12 +232,12 @@ namespace LLM {
                             }
                         }
                     }
-                    if (contains(name, "visual.blocks.0.mlp.linear_fc1.weight") ||
-                        contains(name, "visual.blocks.0.mlp.gate_proj.weight")) {
+                    if (ends_with(name, "visual.blocks.0.mlp.linear_fc1.weight") ||
+                        ends_with(name, "visual.blocks.0.mlp.gate_proj.weight")) {
                         config.vision.intermediate_size = tensor_storage.ne[1];
                     }
-                    if (contains(name, "visual.merger.linear_fc2.weight") ||
-                        contains(name, "visual.merger.mlp.2.weight")) {
+                    if (ends_with(name, "visual.merger.linear_fc2.weight") ||
+                        ends_with(name, "visual.merger.mlp.2.weight")) {
                         config.vision.out_hidden_size = tensor_storage.ne[1];
                     }
                     continue;
@@ -256,22 +256,26 @@ namespace LLM {
                     config.hidden_size = tensor_storage.ne[0];
                     config.vocab_size  = tensor_storage.ne[1];
                 }
-                if (contains(name, "layers.0.mlp.gate_proj.weight")) {
+                if (ends_with(name, "layers.0.mlp.gate_proj.weight")) {
                     config.intermediate_size = tensor_storage.ne[1];
                 }
-                if (contains(name, "layers.0.mlp.experts.gate_up_proj.weight")) {
+                if (ends_with(name, "layers.0.mlp.experts.gate_up_proj.weight")) {
                     config.intermediate_size = tensor_storage.ne[1] / 2;
                 }
-                if (contains(name, "layers.0.mlp.experts.gate_proj.weight")) {
+                if (ends_with(name, "layers.0.mlp.experts.gate_proj.weight")) {
                     config.intermediate_size = tensor_storage.ne[1];
                 }
             }
             if ((arch == LLMArch::QWEN3 || arch == LLMArch::QWEN3_VL) && config.num_layers == 28) {
                 config.num_heads = 16;
             }
-            if (arch == LLMArch::QWEN3_VL && config.num_layers == 50 && config.hidden_size == 5120) {
-                config.num_heads  = 64;
-                config.final_norm = false;
+            if (arch == LLMArch::QWEN3_VL &&
+                (config.num_layers == 50 || config.num_layers == 64) &&
+                config.hidden_size == 5120) {
+                config.num_heads = 64;
+                if (config.num_layers == 50) {
+                    config.final_norm = false;
+                }
             }
             if (detected_vision_layers > 0) {
                 config.vision.num_layers = detected_vision_layers;

src/model_io/safetensors_io.cpp

@@ -87,9 +87,9 @@ static ggml_type safetensors_dtype_to_ggml_type(const std::string& dtype) {
     } else if (dtype == "F64") {
         ttype = GGML_TYPE_F32;
     } else if (dtype == "F8_E4M3") {
-        ttype = GGML_TYPE_F16;
+        ttype = GGML_TYPE_F8_E4M3;
     } else if (dtype == "F8_E5M2") {
-        ttype = GGML_TYPE_F16;
+        ttype = GGML_TYPE_F8_E5M2;
     } else if (dtype == "I32") {
         ttype = GGML_TYPE_I32;
     } else if (dtype == "I64") {
@@ -328,12 +328,10 @@ bool read_safetensors_file(const std::string& file_path,
         bool tensor_size_ok;
         if (dtype == "F8_E4M3") {
             tensor_storage.is_f8_e4m3 = true;
-            // f8 -> f16
-            tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size * 2);
+            tensor_size_ok            = (tensor_storage.nbytes() == tensor_data_size);
         } else if (dtype == "F8_E5M2") {
             tensor_storage.is_f8_e5m2 = true;
-            // f8 -> f16
-            tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size * 2);
+            tensor_size_ok            = (tensor_storage.nbytes() == tensor_data_size);
         } else if (dtype == "F64") {
             tensor_storage.is_f64 = true;
             // f64 -> f32

src/model_io/tensor_storage.h

@@ -54,9 +54,7 @@ struct TensorStorage {
     }
 
     int64_t nbytes_to_read() const {
-        if (is_f8_e4m3 || is_f8_e5m2) {
-            return nbytes() / 2;
-        } else if (is_f64 || is_i64) {
+        if (is_f64 || is_i64) {
             return nbytes() * 2;
         } else {
             return nbytes();

src/model_loader.cpp

@@ -78,66 +78,6 @@ bool is_unused_tensor(const std::string& name) {
     return false;
 }
 
-uint16_t f8_e4m3_to_f16(uint8_t f8) {
-    // do we need to support uz?
-
-    const uint32_t exponent_bias = 7;
-    if (f8 == 0xff) {
-        return ggml_fp32_to_fp16(-NAN);
-    } else if (f8 == 0x7f) {
-        return ggml_fp32_to_fp16(NAN);
-    }
-
-    uint32_t sign     = f8 & 0x80;
-    uint32_t exponent = (f8 & 0x78) >> 3;
-    uint32_t mantissa = f8 & 0x07;
-    uint32_t result   = sign << 24;
-    if (exponent == 0) {
-        if (mantissa > 0) {
-            exponent = 0x7f - exponent_bias;
-
-            // yes, 2 times
-            if ((mantissa & 0x04) == 0) {
-                mantissa &= 0x03;
-                mantissa <<= 1;
-                exponent -= 1;
-            }
-            if ((mantissa & 0x04) == 0) {
-                mantissa &= 0x03;
-                mantissa <<= 1;
-                exponent -= 1;
-            }
-
-            result |= (mantissa & 0x03) << 21;
-            result |= exponent << 23;
-        }
-    } else {
-        result |= mantissa << 20;
-        exponent += 0x7f - exponent_bias;
-        result |= exponent << 23;
-    }
-
-    return ggml_fp32_to_fp16(*reinterpret_cast<const float*>(&result));
-}
-
-uint16_t f8_e5m2_to_f16(uint8_t fp8) {
-    return static_cast<uint16_t>(fp8) << 8;
-}
-
-void f8_e4m3_to_f16_vec(uint8_t* src, uint16_t* dst, int64_t n) {
-    // support inplace op
-    for (int64_t i = n - 1; i >= 0; i--) {
-        dst[i] = f8_e4m3_to_f16(src[i]);
-    }
-}
-
-void f8_e5m2_to_f16_vec(uint8_t* src, uint16_t* dst, int64_t n) {
-    // support inplace op
-    for (int64_t i = n - 1; i >= 0; i--) {
-        dst[i] = f8_e5m2_to_f16(src[i]);
-    }
-}
-
 void f64_to_f32_vec(double* src, float* dst, int64_t n) {
     // support inplace op
     for (int64_t i = 0; i < n; i++) {
@@ -929,9 +869,7 @@ std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggm
             if (dst_tensor == nullptr)
                 continue;
 
-            if (tensor_storage.is_f8_e4m3 ||
-                tensor_storage.is_f8_e5m2 ||
-                tensor_storage.is_f64 ||
+            if (tensor_storage.is_f64 ||
                 tensor_storage.is_i64 ||
                 tensor_storage.type != dst_tensor->type) {
                 continue;
@@ -1215,11 +1153,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
                     read_time_ms.fetch_add(t1 - t0);
 
                     t0 = ggml_time_ms();
-                    if (tensor_storage.is_f8_e4m3) {
-                        f8_e4m3_to_f16_vec((uint8_t*)read_buf, (uint16_t*)target_buf, tensor_storage.nelements());
-                    } else if (tensor_storage.is_f8_e5m2) {
-                        f8_e5m2_to_f16_vec((uint8_t*)read_buf, (uint16_t*)target_buf, tensor_storage.nelements());
-                    } else if (tensor_storage.is_f64) {
+                    if (tensor_storage.is_f64) {
                         f64_to_f32_vec((double*)read_buf, (float*)target_buf, tensor_storage.nelements());
                     } else if (tensor_storage.is_i64) {
                         i64_to_i32_vec((int64_t*)read_buf, (int32_t*)target_buf, tensor_storage.nelements());

src/name_conversion.cpp

@@ -1569,6 +1569,11 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
         }
     }
 
+    static const std::vector<std::pair<std::string, std::string>> generic_name_map = {
+        {".scale_weight", ".weight_scale"},
+    };
+    replace_with_name_map(name, generic_name_map);
+
     if (is_lora) {
         name = "lora." + name;
     }

src/stable-diffusion.cpp

@@ -2670,7 +2670,9 @@ public:
                     denoised = denoised * denoise_mask + init_latent * (1.0f - denoise_mask);
                 }
                 if (sd_should_preview_denoised() && preview.callback != nullptr) {
-                    preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
+                    if (step % sd_get_preview_interval() == 0) {
+                        preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
+                    }
                 }
                 report_sample_progress(step, steps, &last_progress_us);
                 sd::guidance::GuiderOutput output;
@@ -2679,7 +2681,9 @@ public:
             }
 
             if (sd_should_preview_noisy() && preview.callback != nullptr) {
-                preview_image(step, noised_input, version, preview.mode, preview.callback, preview.data, true);
+                if (step % sd_get_preview_interval() == 0) {
+                    preview_image(step, noised_input, version, preview.mode, preview.callback, preview.data, true);
+                }
             }
 
             sd::Tensor<float> cond_out;
@@ -2889,7 +2893,9 @@ public:
                 denoised = denoised * denoise_mask + init_latent * (1.0f - denoise_mask);
             }
             if (sd_should_preview_denoised() && preview.callback != nullptr) {
-                preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
+                if (step % sd_get_preview_interval() == 0) {
+                    preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false);
+                }
             }
             report_sample_progress(step, steps, &last_progress_us);
             output.pred = denoised;