sync: update ggml

Co-authored-by: leejet <leejet714@gmail.com>

.github/workflows/build.yml

@@ -23,6 +23,7 @@ on:
         "**/*.c",
         "**/*.cpp",
         "**/*.cu",
+        "ggml",
         "examples/server/frontend",
         "examples/server/frontend/**",
       ]
@@ -40,6 +41,7 @@ on:
         "**/*.c",
         "**/*.cpp",
         "**/*.cu",
+        "ggml",
         "examples/server/frontend",
         "examples/server/frontend/**",
       ]

CMakeLists.txt

@@ -11,10 +11,11 @@ endif()
 if (MSVC)
     add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
     add_compile_definitions(_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING)
+    # /MP is MSVC-only: icx rejects it outright once offloading is enabled.
     add_compile_options(
-        $<$<COMPILE_LANGUAGE:C>:/MP>
+        $<$<AND:$<COMPILE_LANGUAGE:C>,$<C_COMPILER_ID:MSVC>>:/MP>
         $<$<COMPILE_LANGUAGE:C>:/utf-8>
-        $<$<COMPILE_LANGUAGE:CXX>:/MP>
+        $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:MSVC>>:/MP>
         $<$<COMPILE_LANGUAGE:CXX>:/utf-8>
     )
 endif()

README.md

@@ -15,6 +15,8 @@ API and command-line option may change frequently.***
 
 ## 🔥Important News
 
+* **2026/08/20** 🚀 stable-diffusion.cpp now supports **LTX-2.5**
+* **2026/08/04** 🚀 stable-diffusion.cpp adds **Day-1 support for MiniMax-H3**
 * **2026/06/25** 🚀 stable-diffusion.cpp now supports **Krea2**
 * **2026/06/04** 🚀 stable-diffusion.cpp now supports **Ideogram4**
 * **2026/05/31** 🚀 stable-diffusion.cpp now supports **PiD**
@@ -66,11 +68,12 @@ API and command-line option may change frequently.***
     - [Mage-Flow-Edit](./docs/mage_flow.md#image-editing)
   - Video Models
     - [Wan2.1/Wan2.2](./docs/wan.md)
-    - [LTX-2.3](./docs/ltx2.md)
+    - [MiniMax-H3](./docs/minimax_h3.md)
+    - [LTX-2.3/LTX-2.5](./docs/ltx2.md)
     - [HunyuanVideo 1.5](./docs/hunyuan_video.md)
     - [LingBot-Video](./docs/lingbot_video.md)
   - [PhotoMaker](./docs/photo_maker.md) support.
-  - [IP-Adapter](./docs/ip_adapter.md) support (SD 1.5 and SDXL)
+  - [IP-Adapter](./docs/ip_adapter.md) support (SD 1.5 and SDXL, including Plus)
   - Control Net support with SD 1.5
   - [ADetailer](./docs/adetailer.md)
   - LoRA support, same as [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#lora)
@@ -150,6 +153,7 @@ For runtime and parameter backend placement, see the [backend selection guide](.
 - [LCM/LCM-LoRA](./docs/lcm.md)
 - [Docker](./docs/docker.md)
 - [Quantization and GGUF](./docs/quantization_and_gguf.md)
+- [INT8 convrot safetensors](./docs/int8_convrot.md)
 - [Inference acceleration via caching](./docs/caching.md)
 
 ## Bindings

docs/ideogram4.md

@@ -11,7 +11,19 @@
 - Download Qwen3-VL-8B-Instruct
     - gguf: https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/tree/main
 
-## Convert weights
+## Use original FP8 weights
+
+The original Ideogram4 FP8 safetensors can be loaded directly. FP8 tensors stay
+at one byte per element in RAM and VRAM. Backends that cannot multiply FP8
+weights directly cast only the active layer to a temporary BF16 tensor during
+execution; the loader does not expand the entire checkpoint to BF16.
+
+Use `ideogram4_fp8.safetensors` and `ideogram4_uncond_fp8.safetensors` directly
+with `--diffusion-model` and `--uncond-diffusion-model`, respectively.
+
+## Optional conversion for quantization
+
+The following conversion is only needed when creating a quantized GGUF model.
 
 fp8 scale -> bf16
 

docs/int8_convrot.md

@@ -0,0 +1,102 @@
+# INT8 Convrot Safetensors
+
+sd.cpp can load and execute ComfyUI `int8_tensorwise` safetensors with `convrot` metadata directly. The stored INT8 weights are not converted to another weight type at load time.
+
+## Checkpoint format
+
+Each quantized linear module contains the following tensors:
+
+- `<module>.weight`: an I8 weight matrix.
+- `<module>.weight_scale`: one floating-point scale for each output row. ComfyUI's two-dimensional `[out_features, 1]` representation is normalized to a one-dimensional tensor while loading.
+- `<module>.comfy_quant`: a U8 tensor containing the JSON quantization configuration.
+
+A supported configuration has this form:
+
+```json
+{
+  "format": "int8_tensorwise",
+  "convrot": true,
+  "convrot_groupsize": 256
+}
+```
+
+The convrot group size must be a power of four and must divide the input feature dimension. The commonly used configuration is H256, with `convrot_groupsize` set to `256`.
+
+## How INT8 convrot works
+
+Convrot combines an offline rotation of the weights with the same rotation of the activations at runtime. The rotation uses a normalized regular Hadamard matrix constructed recursively from
+
+```text
+     [ 1  1  1 -1 ]
+     [ 1  1 -1  1 ]
+H4 = [ 1 -1  1  1 ] / 2
+     [-1  1  1  1 ]
+```
+
+For a group size `G = 4^n`, the transform is the normalized Kronecker power of `H4`. It is applied independently to every contiguous group of `G` input features. The resulting block-diagonal rotation matrix `R` is orthogonal and symmetric, so `R R^T = I`.
+
+For an original floating-point linear layer
+
+```text
+Y = X W^T + b
+```
+
+the checkpoint stores a rotated weight matrix `W_rot = W R`, quantized per output row. At runtime sd.cpp computes `X_rot = X R`. Ignoring quantization error,
+
+```text
+X_rot W_rot^T = X R (W R)^T = X R R^T W^T = X W^T
+```
+
+The rotation therefore preserves the linear operation. Its purpose is to spread isolated large values across each feature group, reducing the effect of outliers on tensorwise INT8 quantization.
+
+### Weight quantization
+
+The rotated weights are quantized offline with one scale per output row:
+
+```text
+s_w[o]      = max_i(abs(W_rot[o, i])) / 127
+Q_w[o, i]   = clamp(round(W_rot[o, i] / s_w[o]), -127, 127)
+```
+
+`Q_w` is stored in `<module>.weight`, and `s_w` is stored in `<module>.weight_scale`.
+
+### Runtime activation quantization
+
+For every activation row, sd.cpp applies the group-wise Hadamard rotation and then calculates one dynamic scale across the entire rotated row:
+
+```text
+s_x[r]      = max_i(abs(X_rot[r, i])) / 127
+Q_x[r, i]   = clamp(round(X_rot[r, i] / s_x[r]), -127, 127)
+```
+
+The matrix multiplication accumulates into signed 32-bit integers:
+
+```text
+A[r, o] = sum_i(Q_x[r, i] * Q_w[o, i])
+```
+
+The floating-point output is reconstructed as
+
+```text
+Y[r, o] ~= A[r, o] * s_x[r] * s_w[o] + b[o]
+```
+
+The packed runtime activation tensor contains the I8 activation rows and their floating-point row scales. Linear layers that share the same input and convrot group size reuse this packed tensor, avoiding repeated rotation and activation quantization within the graph.
+
+## Backend support
+
+- CPU provides the portable regular Hadamard, activation quantization, INT8 matrix multiplication, and scale restoration implementations.
+- NVIDIA CUDA devices with compute capability 7.5 or newer use the native accelerated path. For H256, CUDA fuses the rotation, row-wise maximum reduction, and activation quantization. It uses cuBLAS for I8 x I8 to I32 GEMM and a CUDA kernel for scale restoration and bias addition.
+- Vulkan and other GPU backends do not currently have dedicated INT8 convrot kernels. They use the backend scheduler to fall back to CPU, which is expected to be substantially slower than the CUDA path.
+
+LoRA adapters are applied at runtime without modifying the INT8 weights. The INT8 convrot path computes the base linear output, while LoRA, LoHa, LoKr, and raw weight-difference adapters compute their output corrections from the original, unrotated activation and add them to the base output. `--lora-apply-mode auto` selects this path for models containing INT8 tensorwise weights. If `immediately` is requested, sd.cpp falls back to runtime application because merging an adapter would require dequantizing and rotating its weight update, then recalculating the per-row scales and requantizing the result.
+
+The dedicated CUDA convrot activation path currently requires a group size of `256`; other supported group sizes use CPU execution.
+
+## Example
+
+ComfyUI INT8 convrot safetensors can be passed to `--diffusion-model` without conversion:
+
+```powershell
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\krea2_turbo_int8_convrot.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p "a lovely cat holding a sign says 'krea2.cpp'" --steps 8 --cfg-scale 1 --diffusion-fa -v --offload-to-cpu
+```

docs/ip_adapter.md

@@ -11,6 +11,10 @@ through a decoupled cross-attention added to every attn2 layer of the
 UNet. It composes with Control Net, so a reference image (appearance) and
 an OpenPose hint (pose) can be combined in a single generation.
 
+Both the classic adapters and the higher-fidelity **Plus** adapters are
+supported; see [Plus variants](#plus-variants) below. The variant is
+detected from the weight file, so the same options work for both.
+
 ## Required weights
 
 1. A base SD 1.5 or SDXL model.
@@ -21,6 +25,11 @@ an OpenPose hint (pose) can be combined in a single generation.
    [h94/IP-Adapter](https://huggingface.co/h94/IP-Adapter):
    - SD 1.5: `models/ip-adapter_sd15.safetensors`
    - SDXL: `sdxl_models/ip-adapter_sdxl_vit-h.safetensors`
+   - SD 1.5 Plus: `models/ip-adapter-plus_sd15.safetensors`
+   - SDXL Plus: `sdxl_models/ip-adapter-plus_sdxl_vit-h.safetensors`
+
+   The Plus files (`ip-adapter-plus_*`) are used exactly like the classic
+   ones; see [Plus variants](#plus-variants).
 
 ## Options
 
@@ -45,6 +54,29 @@ sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safet
 The SDXL VAE decode at 1024x1024 is memory heavy; add `--vae-tiling` (and
 `--offload-to-cpu`) on GPUs with limited VRAM.
 
+## Plus variants
+
+The Plus adapters (`ip-adapter-plus_sd15`, `ip-adapter-plus_sdxl_vit-h`)
+replace the small linear image projection with a Resampler (a
+Perceiver-style module with learned latent queries). Instead of pooling the
+CLIP-Vision output into one vector, the Resampler attends over the full grid
+of penultimate CLIP-Vision hidden states and emits more image tokens (16
+instead of 4). The result transfers finer detail and layout from the
+reference, at a small extra cost in the image-projection step.
+
+No extra flags are needed. The variant is detected from the weight file (the
+Resampler's `image_proj.latents` tensor), and every Resampler dimension is
+read from the tensor shapes, so the same `--ip-adapter`,
+`--ip-adapter-image`, and `--ip-adapter-strength` options apply. Plus
+composes with Control Net in the same way as the classic adapters.
+
+```
+sd-cli -m ..\models\sd_v1.5.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter-plus_sd15.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 7 --steps 30 --sampling-method dpm++2m --scheduler karras -W 512 -H 512
+```
+
+The startup log line `IP-Adapter: 16 image tokens` (versus `4` for the
+classic adapters) confirms a Plus file was loaded.
+
 ## Combining with Control Net
 
 Add the usual Control Net options to keep the reference appearance while

docs/ltx2.md

@@ -1,7 +1,17 @@
 # How to Use
 
+Both LTX-2.3 and LTX-2.5 are supported. The two share a transformer, video VAE and audio
+VAE architecture; LTX-2.5 drops the video FFN biases, adds a learned keyframe
+absolute-position embedding, and pairs with a Gemma 4 text encoder instead of Gemma 3.
+Everything is detected from the weights, so the command lines differ only in which files
+you pass.
+
+# LTX-2.3
+
 ## Download weights
 
+### LTX-2.3
+
 - Download LTX-2.3
     - safetensors: https://huggingface.co/Kijai/LTX2.3_comfy/tree/main/diffusion_models
     - gguf: https://huggingface.co/unsloth/LTX-2.3-GGUF/tree/main
@@ -16,6 +26,30 @@
 - Download LTX spatial latent upscaler
     - safetensors: https://huggingface.co/Lightricks/LTX-2.3/resolve/main/ltx-2.3-spatial-upscaler-x2-1.1.safetensors
 
+### LTX-2.5
+
+- Download LTX-2.5
+    - safetensors: https://huggingface.co/Lightricks/LTX-2.5/tree/main/diffusion_models
+    - gguf: https://huggingface.co/vantagewithai/LTX-2.5-GGUF/tree/main
+- Download the text encoder. This is a Gemma 4 12B fine-tuned for LTX with the text
+  projection bundled in, so no separate `--embeddings-connectors` file is needed. Google's
+  stock Gemma 4 is not a substitute.
+    - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/text_encoders/gemma4-12b-with-proj-ltx-2.5-bf16.safetensors
+- Download the video vae. Use the **conv** variant: `ltx-2.5-video-vae-conv-bf16.safetensors`.
+  The default `ltx-2.5-video-vae-bf16.safetensors` is a diffusion decoder, which is not
+  implemented here.
+    - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/vae/ltx-2.5-video-vae-conv-bf16.safetensors
+- Download the audio vae
+    - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/vae/ltx-2.5-audio-vae-bf16.safetensors
+- Download the LTX spatial latent upscaler
+    - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/latent_upscale_models/ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors
+
+To run the text encoder quantized, convert it once with sd-cli:
+
+```
+.\bin\Release\sd-cli.exe -M convert -m ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-bf16.safetensors --type q8_0 -o ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf
+```
+
 ## Examples
 
 ### LTX-2.3 dev T2V
@@ -74,4 +108,32 @@ By default, the hires refine pass uses the main sampler and scheduler, then trim
   src="../assets/ltx2/hires_i2v.webm"
   controls
   muted
-  style="max-width: 100%; height: auto;"></video>
+  style="max-width: 100%; height: auto;"></video>
+
+### LTX-2.5 dev T2V
+
+```
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf -p "A wide aerial shot of a red vintage convertible driving along a coastal cliff road at sunset, waves crashing below" --cfg-scale 3.0 --sampling-method euler -v -n "worst quality, low quality, blurry, distorted, artifacts" -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 121 --fps 24 -o t2v.webm
+```
+
+### LTX-2.5 dev I2V
+
+```
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf -p "a lovely cat blinking slowly, gentle camera push in" --cfg-scale 3.0 --sampling-method euler -v -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 121 -i ..\assets\ernie_image\turbo_example.png -o i2v.webm
+```
+
+### LTX-2.5 spatial latent upscale
+
+Works exactly like the LTX-2.3 upscaler described below; put
+`ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors` under `--hires-upscalers-dir` and
+pass its name without path or extension to `--hires-upscaler`.
+
+```
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf --hires-upscalers-dir ..\models\latent_upscale_models --hires-upscaler ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0 --hires --hires-steps 6 -p "a lovely cat" --cfg-scale 3.0 --sampling-method euler -v -W 640 -H 360 --diffusion-fa --offload-to-cpu --video-frames 121 -o hires_t2v.webm
+```
+
+## Not implemented
+
+- The diffusion video decoder (`ltx-2.5-video-vae-bf16.safetensors`). Use the conv VAE.
+- The temporal latent upscaler and the duration head (`--auto-duration`); pass
+  `--video-frames` explicitly.

docs/minimax_h3.md

@@ -0,0 +1,96 @@
+# MiniMax-H3
+
+MiniMax-H3 jointly generates video and stereo audio with a packed diffusion
+transformer. The implementation supports text-to-audio-video (T2VA), optional
+first-frame conditioning (I2VA), first/last-frame conditioning (FL2VA), and
+image/video/audio reference conditioning (Ref2VA).
+
+## Model files
+
+Pass the four MiniMax-H3 components separately:
+
+- `--diffusion-model`: MiniMax-H3 diffusion transformer
+- `--vae`: MiniMax-H3 video VAE
+- `--audio-vae`: MiniMax-H3 audio VAE
+- `--llm`: the MiniMax-H3 Qwen3-VL-32B text encoder checkpoint
+
+The text encoder must be the MiniMax-H3 variant: Qwen3-VL-32B truncated to 50
+language layers and exported without the final language-model normalization.
+Its Qwen3-VL vision tower, including the three DeepStack mergers, must also be
+present. If the vision tower is stored separately, pass it with `--llm_vision`.
+
+Both the original time-embedder DiT and the smaller AdaLN curve-table variant
+are detected from their weights.
+
+### Download weights
+
+- Download minimax_h3_fl2va/minimax_h3_ref2va
+    - safetensors: https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/diffusion_models
+    - gguf: https://huggingface.co/leejet/MiniMax-H3-GGUF/tree/main
+- Download qwen3vl_32b_minimax_h3
+    - safetensors: https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/text_encoders
+    - gguf: https://huggingface.co/leejet/MiniMax-H3-GGUF/tree/main
+- Download vae
+    - safetensors: https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/vae
+- Download audio vae
+    - safetensors: https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/vae
+
+## Text-to-audio-video
+
+```sh
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\minimax_h3_fl2va-Q4_K_M.gguf --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "A cute American Shorthair silver tabby kitten surfs on a tropical ocean wave, riding a white surfboard with the clear text 'sd.cpp' on it. Cinematic tracking shot, realistic water, bright sunlight, smooth motion, and consistent character appearance. Add upbeat tropical surf-rock background music with cheerful drums and guitar, synchronized with the kitten’s energetic surfing." --cfg-scale 1.0 -v -W 864 -H 480 --diffusion-fa --offload-to-cpu --rng cpu --fps 24 --video-frames 56
+```
+
+<video src=../assets/minimax-h3/t2av.mp4 controls="controls" muted="muted" type="video/mp4"></video>
+
+Omitting `--audio-vae` still runs the joint diffusion model but produces video without a
+decoded audio track.
+
+## First/last-frame conditioning
+
+Add `--init-img` for I2VA, or both `--init-img` and `--end-img` for FL2VA:
+
+```sh
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\minimax_h3_fl2va-Q4_K_M.gguf --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "a lovely cat" -i ..\assets\ernie_image\turbo_example.png --cfg-scale 1.0 -v -W 864 -H 480 --diffusion-fa --offload-to-cpu --rng cpu --fps 24 --video-frames 56
+```
+
+<video src=../assets/minimax-h3/i2av.mp4 controls="controls" muted="muted" type="video/mp4"></video>
+
+## Reference-to-audio-video conditioning
+
+Ref2VA accepts any combination of reference images, reference videos, paired
+video soundtracks, and standalone audio references:
+
+```sh
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\minimax_h3_ref2va_pruned-Q4_K_M.gguf --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "Use the cat from <Picture 1> as the main character. Keep the cat’s appearance, fur color, facial features, and identity consistent with the reference image. Create a 2-second cinematic video: start with an extreme close-up shot of the cat’s face, focusing on its cute expression and detailed fur texture. The camera slowly rotates around the cat’s head, creating a dynamic reveal. Then smoothly pull back and zoom out to reveal the full scene: the cat is standing confidently on a surfboard, riding ocean waves. Water splashes around the board, sea breeze gently moves the cat’s fur, and the cat maintains a cute and fearless expression while surfing. Smooth camera movement, cinematic orbit shot, seamless zoom-out transition, low-angle wide shot, realistic ocean environment, golden sunlight, dynamic waves, high-quality realistic style, natural motion, no distortion, keep the cat’s identity unchanged." -r ..\assets\ernie_image\turbo_example.png --cfg-scale 1.0 -v -W 864 -H 480 --diffusion-fa --offload-to-cpu --rng cpu --fps 24 --video-frames 56
+```
+
+<video src=../assets/minimax-h3/r2av.mp4 controls="controls" muted="muted" type="video/mp4"></video>
+
+`--ref-image`, `--ref-video`, and `--ref-audio` can each be repeated. A
+reference video is a directory of image frames sorted lexicographically and is
+treated as 24 fps. Repeated `--ref-video-audio` WAV files are paired by index
+with repeated `--ref-video` inputs. WAV PCM (8/16/24/32-bit) and 32/64-bit
+floating-point samples are accepted; audio is converted to stereo 32 kHz by the
+pipeline.
+
+Reference inputs are presented to Qwen3-VL in image, video, then audio order.
+Videos are sampled at 2 fps for the Qwen presentation while their full 24 fps
+latents condition the diffusion transformer. Paired video and audio references
+share the same timeline. Ref2VA cannot be combined with `--init-img` or
+`--end-img` in one request.
+
+Reference images keep their aspect ratio and are only downscaled when their
+pixel area exceeds the requested generation canvas.
+
+The C API exposes the same inputs through `ref_images`, `ref_videos`, and
+`ref_audios` in `sd_vid_gen_params_t`. Each `sd_ref_video_t` supplies its own
+frame rate and optional soundtrack; non-24-fps inputs are resampled internally.
+
+## Shape and runtime notes
+
+- Width and height are aligned upward to a multiple of 32.
+- Frame count is aligned upward to the `17k + 5` grid, with a minimum of 5.
+- MiniMax-H3 runs at 24 fps; another requested value is overridden.
+- The default video flow shift is 12. The audio stream is mapped internally to
+  its shift of 3, so the regular samplers can operate on the packed AV latent.

examples/cli/main.cpp

@@ -36,6 +36,7 @@ struct SDCliParams {
     SDMode mode             = IMG_GEN;
     std::string output_path = "output.png";
     int output_begin_idx    = -1;
+    int compression_quality = 90;
     std::string image_path;
     std::string metadata_format = "text";
 
@@ -99,6 +100,10 @@ struct SDCliParams {
              "--output-begin-idx",
              "starting index for output image sequence, must be non-negative (default 0 if specified %d in output path, 1 otherwise)",
              &output_begin_idx},
+            {"",
+             "--compression-quality",
+             "compression quality of video and JPEG / WebP images (90 by default)",
+             &compression_quality},
         };
 
         options.bool_options = {
@@ -383,11 +388,13 @@ void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy,
                                  image->data,
                                  image->width,
                                  image->height,
-                                 image->channel)) {
+                                 image->channel,
+                                 "",
+                                 cli_params->compression_quality)) {
             LOG_ERROR("save preview image to '%s' failed", cli_params->preview_path.c_str());
         }
     } else {
-        if (create_video_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, cli_params->preview_fps) != 0) {
+        if (create_video_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, cli_params->preview_fps, cli_params->compression_quality) != 0) {
             LOG_ERROR("save preview video to '%s' failed", cli_params->preview_path.c_str());
         }
     }
@@ -486,7 +493,7 @@ bool save_results(const SDCliParams& cli_params,
         std::string params          = gen_params.embed_image_metadata
                                           ? get_image_params(ctx_params, gen_params, metadata_seed, cli_params.mode)
                                           : "";
-        const bool ok               = write_image_to_file(path.string(), img.data, img.width, img.height, img.channel, params, 90);
+        const bool ok               = write_image_to_file(path.string(), img.data, img.width, img.height, img.channel, params, cli_params.compression_quality);
         LOG_INFO("save result image %d to '%s' (%s)", idx, path.string().c_str(), ok ? "success" : "failure");
         return ok;
     };
@@ -532,7 +539,7 @@ bool save_results(const SDCliParams& cli_params,
         std::string final_ext_lower = ext.string();
         std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower);
         const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm");
-        if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps, 90, mux_audio ? generated_audio : nullptr) == 0) {
+        if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr) == 0) {
             LOG_INFO("save result video to '%s'", video_path.string().c_str());
             if (generated_audio != nullptr && !mux_audio) {
                 fs::path wav_path = video_path;
@@ -754,6 +761,18 @@ int main(int argc, const char* argv[]) {
         return true;
     };
 
+    auto load_audio = [&](const std::string& path, SDAudioOwner& audio) -> bool {
+        std::vector<float> samples;
+        uint32_t sample_rate = 0;
+        uint32_t channels    = 0;
+        if (!load_wav_from_file(path, samples, sample_rate, channels)) {
+            LOG_ERROR("load WAV audio from '%s' failed", path.c_str());
+            return false;
+        }
+        audio.reset(std::move(samples), sample_rate, channels);
+        return true;
+    };
+
     if (gen_params.init_image_path.size() > 0) {
         if (!load_image_and_update_size(gen_params.init_image_path, gen_params.init_image)) {
             return 1;
@@ -777,6 +796,37 @@ int main(int argc, const char* argv[]) {
         }
     }
 
+    if (!gen_params.ref_video_paths.empty()) {
+        gen_params.ref_videos.clear();
+        gen_params.ref_videos.reserve(gen_params.ref_video_paths.size());
+        for (const auto& path : gen_params.ref_video_paths) {
+            std::vector<SDImageOwner> frames;
+            if (!load_images_from_dir(path, frames, 0, 0, 0, cli_params.verbose) || frames.empty()) {
+                LOG_ERROR("load reference video frames from '%s' failed", path.c_str());
+                return 1;
+            }
+            gen_params.ref_videos.push_back(std::move(frames));
+        }
+
+        gen_params.ref_video_audios.clear();
+        gen_params.ref_video_audios.resize(gen_params.ref_videos.size());
+        for (size_t i = 0; i < gen_params.ref_video_audio_paths.size(); ++i) {
+            if (!load_audio(gen_params.ref_video_audio_paths[i], gen_params.ref_video_audios[i])) {
+                return 1;
+            }
+        }
+    }
+
+    if (!gen_params.ref_audio_paths.empty()) {
+        gen_params.ref_audios.clear();
+        gen_params.ref_audios.resize(gen_params.ref_audio_paths.size());
+        for (size_t i = 0; i < gen_params.ref_audio_paths.size(); ++i) {
+            if (!load_audio(gen_params.ref_audio_paths[i], gen_params.ref_audios[i])) {
+                return 1;
+            }
+        }
+    }
+
     if (gen_params.mask_image_path.size() > 0) {
         if (!load_sd_image_from_file(gen_params.mask_image.put(),
                                      gen_params.mask_image_path.c_str(),

examples/common/common.cpp

@@ -1008,7 +1008,7 @@ ArgOptions SDGenerationParams::get_options() {
          &hires_upscaler},
         {"",
          "--extra-sample-args",
-         "extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma;; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware",
+         "extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma; beta scheduler supports alpha, beta; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware; lms supports lms_max_order, lms_shift, lms_divisions",
          (int)',',
          &extra_sample_args},
         {"",
@@ -1404,6 +1404,30 @@ ArgOptions SDGenerationParams::get_options() {
         return 1;
     };
 
+    auto on_ref_video_arg = [&](int argc, const char** argv, int index) {
+        if (++index >= argc) {
+            return -1;
+        }
+        ref_video_paths.push_back(argv[index]);
+        return 1;
+    };
+
+    auto on_ref_video_audio_arg = [&](int argc, const char** argv, int index) {
+        if (++index >= argc) {
+            return -1;
+        }
+        ref_video_audio_paths.push_back(argv[index]);
+        return 1;
+    };
+
+    auto on_ref_audio_arg = [&](int argc, const char** argv, int index) {
+        if (++index >= argc) {
+            return -1;
+        }
+        ref_audio_paths.push_back(argv[index]);
+        return 1;
+    };
+
     auto on_cache_mode_arg = [&](int argc, const char** argv, int index) {
         if (++index >= argc) {
             return -1;
@@ -1531,6 +1555,16 @@ ArgOptions SDGenerationParams::get_options() {
         return 1;
     };
 
+    std::string sample_methods = sample_method_to_str[0];
+    for (int i = 1; i < SAMPLE_METHOD_COUNT; i++) {
+        sample_methods += ", " + std::string(sample_method_to_str[i]);
+    }
+
+    std::string schedulers = scheduler_to_str[0];
+    for (int i = 1; i < SCHEDULER_COUNT; i++) {
+        schedulers += ", " + std::string(scheduler_to_str[i]);
+    }
+
     options.manual_options = {
         {"-s",
          "--seed",
@@ -1538,17 +1572,18 @@ ArgOptions SDGenerationParams::get_options() {
          on_seed_arg},
         {"",
          "--sampling-method",
-         "sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, dpm++2m_sde, dpm++2m_sde_bt, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp]"
-         "(default: euler for Flux/SD3/Wan, euler_a otherwise)",
+         "sampling method, one of [" + sample_methods + "], "
+                                                        "default: euler for Flux/SD3/Wan, euler_a otherwise",
          on_sample_method_arg},
         {"",
          "--high-noise-sampling-method",
-         "(high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, dpm++2m_sde, dpm++2m_sde_bt, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp]"
-         " default: euler for Flux/SD3/Wan, euler_a otherwise",
+         "(high noise) sampling method, one of [" + sample_methods + "], "
+                                                                     "default: euler for Flux/SD3/Wan, euler_a otherwise",
          on_high_noise_sample_method_arg},
         {"",
          "--scheduler",
-         "denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent, ltx2, logit_normal, flux2, flux, beta], alias: normal=discrete, default: model-specific",
+         "denoiser sigma scheduler, one of [" + schedulers + "], "
+                                                             "alias: normal=discrete, default: model-specific",
          on_scheduler_arg},
         {"",
          "--sigmas",
@@ -1568,8 +1603,20 @@ ArgOptions SDGenerationParams::get_options() {
          on_high_noise_skip_layers_arg},
         {"-r",
          "--ref-image",
-         "reference image for Flux Kontext models (can be used multiple times)",
+         "reference image for Flux Kontext or MiniMax-H3 Ref2VA (can be used multiple times)",
          on_ref_image_arg},
+        {"",
+         "--ref-video",
+         "MiniMax-H3 Ref2VA reference video frame directory at 24 fps (can be used multiple times)",
+         on_ref_video_arg},
+        {"",
+         "--ref-video-audio",
+         "WAV soundtrack paired by index with --ref-video (can be used multiple times)",
+         on_ref_video_audio_arg},
+        {"",
+         "--ref-audio",
+         "standalone WAV reference for MiniMax-H3 Ref2VA (can be used multiple times)",
+         on_ref_audio_arg},
         {"",
          "--cache-mode",
          "caching method: 'easycache' (DiT), 'ucache' (UNET), 'dbcache'/'taylorseer'/'cache-dit' (DiT block-level), 'spectrum' (UNET/DiT Chebyshev+Taylor forecasting)",
@@ -1901,6 +1948,7 @@ bool SDGenerationParams::from_json_str(
 
     load_if_exists("strength", strength);
     load_if_exists("control_strength", control_strength);
+    load_if_exists("ip_adapter_strength", ip_adapter_strength);
     load_if_exists("moe_boundary", moe_boundary);
     load_if_exists("vace_strength", vace_strength);
 
@@ -2072,6 +2120,10 @@ bool SDGenerationParams::from_json_str(
         LOG_ERROR("invalid control_image");
         return false;
     }
+    if (!parse_image_json_field(j, "ip_adapter_image", 3, width, height, ip_adapter_image)) {
+        LOG_ERROR("invalid ip_adapter_image");
+        return false;
+    }
 
     return true;
 }
@@ -2361,6 +2413,16 @@ bool SDGenerationParams::validate(SDMode mode) {
         return false;
     }
 
+    if (ref_video_audio_paths.size() > ref_video_paths.size()) {
+        LOG_ERROR("error: each --ref-video-audio needs a corresponding --ref-video");
+        return false;
+    }
+
+    if (mode != VID_GEN && (!ref_video_paths.empty() || !ref_video_audio_paths.empty() || !ref_audio_paths.empty())) {
+        LOG_ERROR("error: reference video and audio inputs require vid_gen mode");
+        return false;
+    }
+
     if (sample_params.shifted_timestep < 0 || sample_params.shifted_timestep > 1000) {
         LOG_ERROR("error: shifted_timestep must be in range [0, 1000]");
         return false;
@@ -2555,6 +2617,35 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() {
         control_frame_views.push_back(frame.get());
     }
 
+    ref_image_views.clear();
+    ref_image_views.reserve(ref_images.size());
+    for (auto& image : ref_images) {
+        ref_image_views.push_back(image.get());
+    }
+
+    ref_video_frame_views.clear();
+    ref_video_frame_views.resize(ref_videos.size());
+    ref_video_views.clear();
+    ref_video_views.reserve(ref_videos.size());
+    for (size_t i = 0; i < ref_videos.size(); ++i) {
+        auto& frame_views = ref_video_frame_views[i];
+        frame_views.reserve(ref_videos[i].size());
+        for (auto& frame : ref_videos[i]) {
+            frame_views.push_back(frame.get());
+        }
+        sd_audio_t audio = i < ref_video_audios.size() ? ref_video_audios[i].get() : sd_audio_t{};
+        ref_video_views.push_back({frame_views.empty() ? nullptr : frame_views.data(),
+                                   static_cast<int>(frame_views.size()),
+                                   24,
+                                   audio});
+    }
+
+    ref_audio_views.clear();
+    ref_audio_views.reserve(ref_audios.size());
+    for (auto& audio : ref_audios) {
+        ref_audio_views.push_back(audio.get());
+    }
+
     sample_params.guidance.slg.layers                 = skip_layers.empty() ? nullptr : skip_layers.data();
     sample_params.guidance.slg.layer_count            = skip_layers.size();
     high_noise_sample_params.guidance.slg.layers      = high_noise_skip_layers.empty() ? nullptr : high_noise_skip_layers.data();
@@ -2573,6 +2664,12 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() {
     params.clip_skip                 = clip_skip;
     params.init_image                = init_image.get();
     params.end_image                 = end_image.get();
+    params.ref_images                = ref_image_views.empty() ? nullptr : ref_image_views.data();
+    params.ref_images_count          = static_cast<int>(ref_image_views.size());
+    params.ref_videos                = ref_video_views.empty() ? nullptr : ref_video_views.data();
+    params.ref_videos_count          = static_cast<int>(ref_video_views.size());
+    params.ref_audios                = ref_audio_views.empty() ? nullptr : ref_audio_views.data();
+    params.ref_audios_count          = static_cast<int>(ref_audio_views.size());
     params.control_frames            = control_frame_views.empty() ? nullptr : control_frame_views.data();
     params.control_frames_size       = static_cast<int>(control_frame_views.size());
     params.width                     = get_resolved_width();
@@ -2652,6 +2749,9 @@ std::string SDGenerationParams::to_string() const {
         << "  mask_image_path: \"" << mask_image_path << "\",\n"
         << "  control_image_path: \"" << control_image_path << "\",\n"
         << "  ref_image_paths: " << vec_str_to_string(ref_image_paths) << ",\n"
+        << "  ref_video_paths: " << vec_str_to_string(ref_video_paths) << ",\n"
+        << "  ref_video_audio_paths: " << vec_str_to_string(ref_video_audio_paths) << ",\n"
+        << "  ref_audio_paths: " << vec_str_to_string(ref_audio_paths) << ",\n"
         << "  control_video_path: \"" << control_video_path << "\",\n"
         << "  auto_resize_ref_image: " << (auto_resize_ref_image ? "true" : "false") << ",\n"
         << "  increase_ref_index: " << (increase_ref_index ? "true" : "false") << ",\n"
@@ -2807,6 +2907,7 @@ std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params,
     root["clip_skip"]             = gen_params.clip_skip;
     root["strength"]              = gen_params.strength;
     root["control_strength"]      = gen_params.control_strength;
+    root["ip_adapter_strength"]   = gen_params.ip_adapter_strength;
     root["auto_resize_ref_image"] = gen_params.auto_resize_ref_image;
     root["increase_ref_index"]    = gen_params.increase_ref_index;
     if (mode == VID_GEN) {

examples/common/common.h

@@ -212,6 +212,9 @@ struct SDGenerationParams {
     std::string control_image_path;
     std::string ip_adapter_image_path;
     std::vector<std::string> ref_image_paths;
+    std::vector<std::string> ref_video_paths;
+    std::vector<std::string> ref_video_audio_paths;
+    std::vector<std::string> ref_audio_paths;
     std::string control_video_path;
 
     sd_sample_params_t sample_params;
@@ -275,6 +278,9 @@ struct SDGenerationParams {
     SDImageOwner init_image;
     SDImageOwner end_image;
     std::vector<SDImageOwner> ref_images;
+    std::vector<std::vector<SDImageOwner>> ref_videos;
+    std::vector<SDAudioOwner> ref_video_audios;
+    std::vector<SDAudioOwner> ref_audios;
     SDImageOwner mask_image;
     SDImageOwner control_image;
     SDImageOwner ip_adapter_image;
@@ -283,6 +289,9 @@ struct SDGenerationParams {
 
     // Backing storage for sd_img_gen_params_t view fields.
     std::vector<sd_image_t> ref_image_views;
+    std::vector<std::vector<sd_image_t>> ref_video_frame_views;
+    std::vector<sd_ref_video_t> ref_video_views;
+    std::vector<sd_audio_t> ref_audio_views;
     std::vector<sd_image_t> pm_id_image_views;
     std::vector<sd_image_t> control_frame_views;
 

examples/common/media_io.cpp

@@ -835,6 +835,9 @@ std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images
     const uint32_t audio_byte_rate       = has_audio ? static_cast<uint32_t>(audio->sample_rate * audio_block_align) : 0;
     const uint32_t audio_data_size       = has_audio ? static_cast<uint32_t>(audio_pcm.size()) : 0;
 
+    if (mjpg_quality != quality)
+        LOG_DEBUG("create_mjpg_avi...(): compression quality was limited from %i to %i", quality, mjpg_quality);
+
     std::vector<uint8_t> avi_data;
     avi_data.reserve(static_cast<size_t>(num_images) * 1024);
 
@@ -1374,3 +1377,132 @@ bool write_wav_to_file(const std::string& path,
     file.write(reinterpret_cast<const char*>(pcm.data()), static_cast<std::streamsize>(pcm.size() * sizeof(int16_t)));
     return file.good();
 }
+
+static uint16_t read_le16(const uint8_t* data) {
+    return static_cast<uint16_t>(data[0]) |
+           (static_cast<uint16_t>(data[1]) << 8);
+}
+
+static uint32_t read_le32(const uint8_t* data) {
+    return static_cast<uint32_t>(data[0]) |
+           (static_cast<uint32_t>(data[1]) << 8) |
+           (static_cast<uint32_t>(data[2]) << 16) |
+           (static_cast<uint32_t>(data[3]) << 24);
+}
+
+bool load_wav_from_file(const std::string& path,
+                        std::vector<float>& interleaved_samples,
+                        uint32_t& sample_rate,
+                        uint32_t& channels) {
+    interleaved_samples.clear();
+    sample_rate = 0;
+    channels    = 0;
+
+    std::ifstream file(path, std::ios::binary);
+    uint8_t riff_header[12];
+    if (!file.read(reinterpret_cast<char*>(riff_header), sizeof(riff_header)) ||
+        std::memcmp(riff_header, "RIFF", 4) != 0 ||
+        std::memcmp(riff_header + 8, "WAVE", 4) != 0) {
+        return false;
+    }
+
+    uint16_t audio_format    = 0;
+    uint16_t bits_per_sample = 0;
+    uint16_t block_align     = 0;
+    std::streampos data_pos  = std::streampos(-1);
+    uint32_t data_size       = 0;
+
+    while (file.good()) {
+        uint8_t chunk_header[8];
+        if (!file.read(reinterpret_cast<char*>(chunk_header), sizeof(chunk_header))) {
+            break;
+        }
+        uint32_t chunk_size           = read_le32(chunk_header + 4);
+        std::streampos chunk_data_pos = file.tellg();
+
+        if (std::memcmp(chunk_header, "fmt ", 4) == 0) {
+            if (chunk_size < 16) {
+                return false;
+            }
+            std::vector<uint8_t> fmt(chunk_size);
+            if (!file.read(reinterpret_cast<char*>(fmt.data()), chunk_size)) {
+                return false;
+            }
+            audio_format    = read_le16(fmt.data());
+            channels        = read_le16(fmt.data() + 2);
+            sample_rate     = read_le32(fmt.data() + 4);
+            block_align     = read_le16(fmt.data() + 12);
+            bits_per_sample = read_le16(fmt.data() + 14);
+            if (audio_format == 0xfffe && chunk_size >= 40) {
+                audio_format = read_le16(fmt.data() + 24);
+            }
+        } else if (std::memcmp(chunk_header, "data", 4) == 0) {
+            data_pos  = chunk_data_pos;
+            data_size = chunk_size;
+            file.seekg(chunk_size, std::ios::cur);
+        } else {
+            file.seekg(chunk_size, std::ios::cur);
+        }
+
+        if (!file.good()) {
+            break;
+        }
+        if ((chunk_size & 1) != 0) {
+            file.seekg(1, std::ios::cur);
+        }
+    }
+
+    const uint32_t bytes_per_sample = (bits_per_sample + 7) / 8;
+    if (data_pos == std::streampos(-1) || data_size == 0 || channels == 0 || sample_rate == 0 ||
+        block_align == 0 || bytes_per_sample == 0 || block_align < channels * bytes_per_sample ||
+        (audio_format != 1 && audio_format != 3)) {
+        return false;
+    }
+
+    const uint64_t frame_count = data_size / block_align;
+    if (frame_count == 0 || frame_count > SIZE_MAX / channels) {
+        return false;
+    }
+    std::vector<uint8_t> pcm(data_size);
+    file.clear();
+    file.seekg(data_pos);
+    if (!file.read(reinterpret_cast<char*>(pcm.data()), data_size)) {
+        return false;
+    }
+
+    interleaved_samples.resize(static_cast<size_t>(frame_count * channels));
+    for (uint64_t frame = 0; frame < frame_count; ++frame) {
+        const uint8_t* frame_data = pcm.data() + frame * block_align;
+        for (uint32_t channel = 0; channel < channels; ++channel) {
+            const uint8_t* sample_data = frame_data + channel * bytes_per_sample;
+            float sample               = 0.0f;
+            if (audio_format == 3 && bits_per_sample == 32) {
+                std::memcpy(&sample, sample_data, sizeof(sample));
+            } else if (audio_format == 3 && bits_per_sample == 64) {
+                double value;
+                std::memcpy(&value, sample_data, sizeof(value));
+                sample = static_cast<float>(value);
+            } else if (audio_format == 1 && bits_per_sample == 8) {
+                sample = (static_cast<int>(sample_data[0]) - 128) / 128.0f;
+            } else if (audio_format == 1 && bits_per_sample == 16) {
+                sample = static_cast<int16_t>(read_le16(sample_data)) / 32768.0f;
+            } else if (audio_format == 1 && bits_per_sample == 24) {
+                int32_t value = static_cast<int32_t>(sample_data[0]) |
+                                (static_cast<int32_t>(sample_data[1]) << 8) |
+                                (static_cast<int32_t>(sample_data[2]) << 16);
+                if ((value & 0x800000) != 0) {
+                    value |= ~0xffffff;
+                }
+                sample = value / 8388608.0f;
+            } else if (audio_format == 1 && bits_per_sample == 32) {
+                int32_t value = static_cast<int32_t>(read_le32(sample_data));
+                sample        = value / 2147483648.0f;
+            } else {
+                interleaved_samples.clear();
+                return false;
+            }
+            interleaved_samples[static_cast<size_t>(frame * channels + channel)] = sample;
+        }
+    }
+    return true;
+}

examples/common/media_io.h

@@ -110,4 +110,9 @@ bool write_wav_to_file(const std::string& path,
                        uint32_t channels,
                        uint32_t sample_rate);
 
+bool load_wav_from_file(const std::string& path,
+                        std::vector<float>& interleaved_samples,
+                        uint32_t& sample_rate,
+                        uint32_t& channels);
+
 #endif  // __MEDIA_IO_H__

examples/common/resource_owners.hpp

@@ -141,6 +141,37 @@ public:
     }
 };
 
+class SDAudioOwner {
+private:
+    uint32_t sample_rate_ = 0;
+    uint32_t channels_    = 0;
+    std::vector<float> samples_;
+
+public:
+    SDAudioOwner() = default;
+
+    void reset(std::vector<float> samples = {}, uint32_t sample_rate = 0, uint32_t channels = 0) {
+        samples_     = std::move(samples);
+        sample_rate_ = sample_rate;
+        channels_    = channels;
+    }
+
+    bool empty() const {
+        return samples_.empty();
+    }
+
+    sd_audio_t get() {
+        return {sample_rate_,
+                channels_,
+                channels_ == 0 ? 0 : static_cast<uint64_t>(samples_.size() / channels_),
+                samples_.empty() ? nullptr : samples_.data()};
+    }
+
+    const std::vector<float>& samples() const {
+        return samples_;
+    }
+};
+
 class SDImageVec {
 private:
     std::vector<sd_image_t> images_;

examples/server/api.md

@@ -528,6 +528,7 @@ Shared default fields used by both `img_gen` and `vid_gen`:
 | `auto_resize_ref_image` | `boolean` |
 | `increase_ref_index` | `boolean` |
 | `control_strength` | `number` |
+| `ip_adapter_strength` | `number` |
 | `hires` | `object` |
 | `hires.enabled` | `boolean` |
 | `hires.upscaler` | `string` |
@@ -567,6 +568,7 @@ Fields returned in `features_by_mode.img_gen`:
 - `init_image`
 - `mask_image`
 - `control_image`
+- `ip_adapter_image`
 - `ref_images`
 - `lora`
 - `vae_tiling`
@@ -653,12 +655,14 @@ Example:
   "auto_resize_ref_image": true,
   "increase_ref_index": false,
   "control_strength": 0.9,
+  "ip_adapter_strength": 1.0,
   "embed_image_metadata": true,
 
   "init_image": null,
   "ref_images": [],
   "mask_image": null,
   "control_image": null,
+  "ip_adapter_image": null,
 
   "sample_params": {
     "scheduler": "discrete",
@@ -733,6 +737,7 @@ Channel expectations:
 - `init_image`: 3 channels
 - `ref_images[]`: 3 channels
 - `control_image`: 3 channels
+- `ip_adapter_image`: 3 channels
 - `mask_image`: 1 channel
 
 If omitted or null:
@@ -757,6 +762,7 @@ Top-level scalar fields:
 | `auto_resize_ref_image` | `boolean` |
 | `increase_ref_index` | `boolean` |
 | `control_strength` | `number` |
+| `ip_adapter_strength` | `number` |
 | `embed_image_metadata` | `boolean` |
 
 Image fields:
@@ -767,6 +773,7 @@ Image fields:
 | `ref_images` | `array<string>` |
 | `mask_image` | `string \| null` |
 | `control_image` | `string \| null` |
+| `ip_adapter_image` | `string \| null` |
 
 LoRA fields:
 
@@ -958,7 +965,7 @@ Response fields:
 Compared with `img_gen`, the `vid_gen` request body:
 
 - `vid_gen` is a single video sequence job, so `batch_count` is not part of the request schema
-- `ref_images`, `mask_image`, `control_image`, `control_strength`, and `embed_image_metadata` are not part of the request schema
+- `ref_images`, `mask_image`, `control_image`, `control_strength`, `ip_adapter_image`, `ip_adapter_strength`, and `embed_image_metadata` are not part of the request schema
 - `vid_gen` adds `end_image`, `control_frames`, `high_noise_sample_params`, `video_frames`, `fps`, `moe_boundary`, and `vace_strength`
 
 Example:

examples/server/routes_sdcpp.cpp

@@ -130,6 +130,7 @@ static json make_img_gen_defaults_json(const SDGenerationParams& defaults, const
         {"auto_resize_ref_image", defaults.auto_resize_ref_image},
         {"increase_ref_index", defaults.increase_ref_index},
         {"control_strength", defaults.control_strength},
+        {"ip_adapter_strength", defaults.ip_adapter_strength},
         {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers)},
         {"hires", make_hires_json(defaults)},
         {"vae_tiling_params", make_vae_tiling_json(defaults.vae_tiling_params)},
@@ -173,6 +174,7 @@ static json make_img_gen_features_json() {
         {"init_image", true},
         {"mask_image", true},
         {"control_image", true},
+        {"ip_adapter_image", true},
         {"ref_images", true},
         {"lora", true},
         {"vae_tiling", true},

include/stable-diffusion.h

@@ -56,9 +56,12 @@ enum sample_method_t {
     EULER_GE_SAMPLE_METHOD,
     DPMPP2M_SDE_SAMPLE_METHOD,
     DPMPP2M_SDE_BT_SAMPLE_METHOD,
+    LMS_SAMPLE_METHOD,
     SAMPLE_METHOD_COUNT
 };
 
+extern SD_API const char* sample_method_to_str[];
+
 enum scheduler_t {
     DISCRETE_SCHEDULER,
     KARRAS_SCHEDULER,
@@ -79,6 +82,8 @@ enum scheduler_t {
     SCHEDULER_COUNT
 };
 
+extern SD_API const char* scheduler_to_str[];
+
 enum prediction_t {
     EPS_PRED,
     V_PRED,
@@ -131,10 +136,13 @@ enum sd_type_t {
     // SD_TYPE_IQ4_NL_4_4 = 36,
     // SD_TYPE_IQ4_NL_4_8 = 37,
     // SD_TYPE_IQ4_NL_8_8 = 38,
-    SD_TYPE_MXFP4 = 39,  // MXFP4 (1 block)
-    SD_TYPE_NVFP4 = 40,  // NVFP4 (4 blocks, E4M3 scale)
-    SD_TYPE_Q1_0  = 41,
-    SD_TYPE_COUNT = 42,
+    SD_TYPE_MXFP4   = 39,  // MXFP4 (1 block)
+    SD_TYPE_NVFP4   = 40,  // NVFP4 (4 blocks, E4M3 scale)
+    SD_TYPE_Q1_0    = 41,
+    SD_TYPE_Q2_0    = 42,
+    SD_TYPE_F8_E4M3 = 43,
+    SD_TYPE_F8_E5M2 = 44,
+    SD_TYPE_COUNT   = 45,
 };
 
 enum sd_log_level_t {
@@ -246,6 +254,13 @@ typedef struct {
     uint8_t* data;
 } sd_image_t;
 
+typedef struct {
+    sd_image_t* frames;
+    int frame_count;
+    int fps;
+    sd_audio_t audio;
+} sd_ref_video_t;
+
 typedef struct {
     int* layers;
     size_t layer_count;
@@ -396,6 +411,12 @@ typedef struct {
     int clip_skip;
     sd_image_t init_image;
     sd_image_t end_image;
+    sd_image_t* ref_images;
+    int ref_images_count;
+    sd_ref_video_t* ref_videos;
+    int ref_videos_count;
+    sd_audio_t* ref_audios;
+    int ref_audios_count;
     sd_image_t* control_frames;
     int control_frames_size;
     int width;

scripts/merge_safetensors.py

@@ -0,0 +1,335 @@
+#!/usr/bin/env python3
+"""Merge selected tensors from multiple safetensors files without loading weights.
+
+Edit ``OUTPUT_PATH`` and ``SOURCE_RULES`` below, then run:
+
+    python scripts/merge_safetensors.py
+
+Each source rule uses regular expressions against complete tensor names.
+``include`` is required and matches when any expression succeeds. ``exclude``
+wins over ``include``. Expressions are evaluated with ``re.search``.
+"""
+
+import json
+import os
+import re
+import struct
+from dataclasses import dataclass
+from pathlib import Path
+from typing import BinaryIO
+
+
+# -----------------------------------------------------------------------------
+# Configuration
+# -----------------------------------------------------------------------------
+
+OUTPUT_PATH = Path(r".minimax_h3_fl2va_pruned_bf16.safetensors")
+
+SOURCE_RULES = [
+    {
+        "path": Path(r".minimax_h3_fl2va_bf16.safetensors"),
+        "include": [r".*"],
+        "exclude": [r".*adaln_proj\.linear.*", r"time_embedder.*"],
+    },
+    {
+        "path": Path(r".minimax_h3_fl2va_pruned_int8_convrot.safetensors"),
+        "include": [r"^.*adaln_proj\.linear.*", "adaln_t_table"],
+        "exclude": [],
+    },
+]
+
+# Safetensors metadata is optional. Set this to a dict[str, str] if needed.
+OUTPUT_METADATA = None
+
+# Refuse to replace an existing output unless explicitly enabled.
+OVERWRITE_OUTPUT = False
+
+# Only tensor headers and this fixed-size buffer are held in memory.
+COPY_BUFFER_SIZE = 8 * 1024 * 1024
+PROGRESS_INTERVAL = 1024 * 1024 * 1024
+MAX_HEADER_SIZE = 256 * 1024 * 1024
+
+
+@dataclass(frozen=True)
+class TensorEntry:
+    name: str
+    source_path: Path
+    source_data_offset: int
+    source_start: int
+    source_end: int
+    dtype: str
+    shape: list[int]
+
+    @property
+    def size(self) -> int:
+        return self.source_end - self.source_start
+
+
+def format_bytes(size: int) -> str:
+    value = float(size)
+    for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
+        if value < 1024.0 or unit == "TiB":
+            return f"{value:.2f} {unit}"
+        value /= 1024.0
+    raise AssertionError("unreachable")
+
+
+def read_exact(file: BinaryIO, size: int, description: str) -> bytes:
+    data = file.read(size)
+    if len(data) != size:
+        raise ValueError(f"truncated {description}: expected {size} bytes, got {len(data)}")
+    return data
+
+
+def read_safetensors_header(path: Path) -> tuple[dict, int, int]:
+    file_size = path.stat().st_size
+    with path.open("rb") as file:
+        header_size = struct.unpack("<Q", read_exact(file, 8, f"header size in {path}"))[0]
+        if header_size == 0 or header_size > MAX_HEADER_SIZE:
+            raise ValueError(
+                f"invalid header size in {path}: {header_size} "
+                f"(limit: {MAX_HEADER_SIZE})"
+            )
+        header_bytes = read_exact(file, header_size, f"header in {path}")
+
+    try:
+        header = json.loads(header_bytes)
+    except (UnicodeDecodeError, json.JSONDecodeError) as error:
+        raise ValueError(f"invalid safetensors JSON header in {path}: {error}") from error
+    if not isinstance(header, dict):
+        raise ValueError(f"safetensors header in {path} is not an object")
+
+    data_offset = 8 + header_size
+    if data_offset > file_size:
+        raise ValueError(f"safetensors data offset is past end of file: {path}")
+    return header, data_offset, file_size
+
+
+def parse_tensor_entry(
+    name: str,
+    info: object,
+    source_path: Path,
+    source_data_offset: int,
+    source_file_size: int,
+) -> TensorEntry:
+    if not isinstance(info, dict):
+        raise ValueError(f"{source_path}: tensor {name!r} has an invalid header entry")
+
+    dtype = info.get("dtype")
+    shape = info.get("shape")
+    offsets = info.get("data_offsets")
+    if not isinstance(dtype, str):
+        raise ValueError(f"{source_path}: tensor {name!r} has an invalid dtype")
+    if not isinstance(shape, list) or not all(
+        isinstance(dimension, int) and dimension >= 0 for dimension in shape
+    ):
+        raise ValueError(f"{source_path}: tensor {name!r} has an invalid shape")
+    if (
+        not isinstance(offsets, list)
+        or len(offsets) != 2
+        or not all(isinstance(offset, int) for offset in offsets)
+    ):
+        raise ValueError(f"{source_path}: tensor {name!r} has invalid data offsets")
+
+    start, end = offsets
+    if start < 0 or end < start or source_data_offset + end > source_file_size:
+        raise ValueError(
+            f"{source_path}: tensor {name!r} byte range [{start}, {end}) "
+            "is outside the file"
+        )
+
+    return TensorEntry(
+        name=name,
+        source_path=source_path,
+        source_data_offset=source_data_offset,
+        source_start=start,
+        source_end=end,
+        dtype=dtype,
+        shape=list(shape),
+    )
+
+
+def compile_patterns(rule_index: int, field: str, values: object) -> list[re.Pattern[str]]:
+    if not isinstance(values, list) or not all(isinstance(value, str) for value in values):
+        raise TypeError(f"SOURCE_RULES[{rule_index}][{field!r}] must be a list of strings")
+    try:
+        return [re.compile(value) for value in values]
+    except re.error as error:
+        raise ValueError(
+            f"invalid regex in SOURCE_RULES[{rule_index}][{field!r}]: {error}"
+        ) from error
+
+
+def collect_entries() -> list[TensorEntry]:
+    if not SOURCE_RULES:
+        raise ValueError("SOURCE_RULES must contain at least one source")
+
+    entries: list[TensorEntry] = []
+    selected_by_name: dict[str, TensorEntry] = {}
+    header_cache: dict[Path, tuple[dict, int, int]] = {}
+
+    for rule_index, rule in enumerate(SOURCE_RULES):
+        if not isinstance(rule, dict) or "path" not in rule or "include" not in rule:
+            raise TypeError(
+                f"SOURCE_RULES[{rule_index}] must contain 'path' and 'include'"
+            )
+
+        source_path = Path(rule["path"])
+        if not source_path.is_file():
+            raise FileNotFoundError(f"source file does not exist: {source_path}")
+        source_path = source_path.resolve()
+
+        include = compile_patterns(rule_index, "include", rule["include"])
+        exclude = compile_patterns(rule_index, "exclude", rule.get("exclude", []))
+        if not include:
+            raise ValueError(f"SOURCE_RULES[{rule_index}]['include'] must not be empty")
+
+        if source_path not in header_cache:
+            header_cache[source_path] = read_safetensors_header(source_path)
+        header, data_offset, file_size = header_cache[source_path]
+
+        matched = 0
+        for name, info in header.items():
+            if name == "__metadata__":
+                continue
+            if not any(pattern.search(name) for pattern in include):
+                continue
+            if any(pattern.search(name) for pattern in exclude):
+                continue
+
+            entry = parse_tensor_entry(name, info, source_path, data_offset, file_size)
+            previous = selected_by_name.get(name)
+            if previous is not None:
+                raise ValueError(
+                    f"tensor {name!r} was selected more than once:\n"
+                    f"  first:  {previous.source_path}\n"
+                    f"  second: {source_path}"
+                )
+            selected_by_name[name] = entry
+            print(f"entry {entry}")
+            entries.append(entry)
+            matched += 1
+
+        print(f"Rule {rule_index}: selected {matched} tensors from {source_path}")
+        if matched == 0:
+            raise ValueError(
+                f"SOURCE_RULES[{rule_index}] did not select any tensors; check its regexes"
+            )
+
+    if not entries:
+        raise ValueError("no tensors were selected")
+    return entries
+
+
+def build_output_header(entries: list[TensorEntry]) -> tuple[bytes, int]:
+    header: dict[str, object] = {}
+    if OUTPUT_METADATA is not None:
+        if not isinstance(OUTPUT_METADATA, dict) or not all(
+            isinstance(key, str) and isinstance(value, str)
+            for key, value in OUTPUT_METADATA.items()
+        ):
+            raise TypeError("OUTPUT_METADATA must be None or a dict[str, str]")
+        header["__metadata__"] = OUTPUT_METADATA
+
+    output_offset = 0
+    for entry in entries:
+        header[entry.name] = {
+            "dtype": entry.dtype,
+            "shape": entry.shape,
+            "data_offsets": [output_offset, output_offset + entry.size],
+        }
+        output_offset += entry.size
+
+    header_bytes = json.dumps(header, separators=(",", ":"), ensure_ascii=False).encode(
+        "utf-8"
+    )
+    header_bytes += b" " * (-len(header_bytes) % 8)
+    return header_bytes, output_offset
+
+
+def copy_tensor(source: BinaryIO, output: BinaryIO, entry: TensorEntry) -> None:
+    source.seek(entry.source_data_offset + entry.source_start)
+    remaining = entry.size
+    while remaining:
+        chunk = source.read(min(COPY_BUFFER_SIZE, remaining))
+        if not chunk:
+            raise OSError(
+                f"unexpected end of file while copying {entry.name!r} "
+                f"from {entry.source_path}"
+            )
+        output.write(chunk)
+        remaining -= len(chunk)
+
+
+def write_output(entries: list[TensorEntry]) -> None:
+    if COPY_BUFFER_SIZE <= 0:
+        raise ValueError("COPY_BUFFER_SIZE must be positive")
+
+    output_path = OUTPUT_PATH.resolve()
+    source_paths = {entry.source_path.resolve() for entry in entries}
+    if output_path in source_paths:
+        raise ValueError("OUTPUT_PATH must not be one of the source files")
+    if output_path.exists() and not OVERWRITE_OUTPUT:
+        raise FileExistsError(
+            f"output already exists: {output_path}; set OVERWRITE_OUTPUT = True to replace it"
+        )
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    partial_path = output_path.with_name(output_path.name + ".partial")
+    if partial_path.exists():
+        raise FileExistsError(
+            f"partial output already exists: {partial_path}; remove it before retrying"
+        )
+
+    header_bytes, tensor_bytes = build_output_header(entries)
+    print(
+        f"Writing {len(entries)} tensors ({format_bytes(tensor_bytes)}) to {output_path}"
+    )
+
+    current_source_path: Path | None = None
+    current_source: BinaryIO | None = None
+    copied = 0
+    next_progress = PROGRESS_INTERVAL
+    try:
+        with partial_path.open("xb") as output:
+            output.write(struct.pack("<Q", len(header_bytes)))
+            output.write(header_bytes)
+
+            try:
+                for entry in entries:
+                    if entry.source_path != current_source_path:
+                        if current_source is not None:
+                            current_source.close()
+                        current_source = entry.source_path.open("rb")
+                        current_source_path = entry.source_path
+
+                    copy_tensor(current_source, output, entry)
+                    copied += entry.size
+                    if PROGRESS_INTERVAL > 0 and copied >= next_progress:
+                        print(
+                            f"  copied {format_bytes(copied)} / "
+                            f"{format_bytes(tensor_bytes)}"
+                        )
+                        while next_progress <= copied:
+                            next_progress += PROGRESS_INTERVAL
+            finally:
+                if current_source is not None:
+                    current_source.close()
+
+        if copied != tensor_bytes:
+            raise OSError(f"copied {copied} tensor bytes, expected {tensor_bytes}")
+        os.replace(partial_path, output_path)
+    except BaseException:
+        partial_path.unlink(missing_ok=True)
+        raise
+
+    print(f"Done: {output_path} ({format_bytes(output_path.stat().st_size)})")
+
+
+def main() -> None:
+    entries = collect_entries()
+    write_output(entries)
+
+
+if __name__ == "__main__":
+    main()

src/conditioning/conditioner.hpp

@@ -2,8 +2,10 @@
 #define __SD_CONDITIONING_CONDITIONER_HPP__
 
 #include <cmath>
+#include <iomanip>
 #include <limits>
 #include <optional>
+#include <sstream>
 
 #include "core/tensor_ggml.hpp"
 #include "core/util.h"
@@ -25,6 +27,8 @@ struct SDCondition {
     sd::Tensor<int32_t> c_vinput_mask;
     std::vector<std::pair<int, sd::Tensor<float>>> c_image_embeds;
     std::vector<sd::Tensor<float>> c_ref_images;
+    std::vector<sd::Tensor<float>> c_ref_audios;
+    std::vector<MiniMaxH3ReferenceBlock> c_reference_blocks;
 
     std::vector<sd::Tensor<float>> extra_c_crossattns;
 
@@ -55,6 +59,12 @@ struct SDCondition {
             }
         }
 
+        for (const auto& tensor : c_ref_audios) {
+            if (!tensor.empty()) {
+                return false;
+            }
+        }
+
         for (const auto& tensor : extra_c_crossattns) {
             if (!tensor.empty()) {
                 return false;
@@ -65,6 +75,18 @@ struct SDCondition {
     }
 };
 
+enum class MiniMaxH3PresentationKind {
+    IMAGE,
+    VIDEO,
+    AUDIO,
+};
+
+struct MiniMaxH3PresentationItem {
+    MiniMaxH3PresentationKind kind = MiniMaxH3PresentationKind::IMAGE;
+    std::vector<sd::Tensor<float>> frames;
+    std::vector<float> timestamps;
+};
+
 static inline sd::Tensor<float> apply_token_weights(sd::Tensor<float> hidden_states,
                                                     const std::vector<float>& weights) {
     if (hidden_states.empty()) {
@@ -102,11 +124,12 @@ static inline sd::Tensor<float> apply_token_weights(sd::Tensor<float> hidden_sta
 
 struct ConditionerParams {
     std::string text;
-    int clip_skip                                    = -1;
-    int width                                        = -1;
-    int height                                       = -1;
-    bool zero_out_masked                             = false;
-    const std::vector<sd::Tensor<float>>* ref_images = nullptr;  // for qwen image edit
+    int clip_skip                                                       = -1;
+    int width                                                           = -1;
+    int height                                                          = -1;
+    bool zero_out_masked                                                = false;
+    const std::vector<sd::Tensor<float>>* ref_images                    = nullptr;  // for qwen image edit
+    const std::vector<MiniMaxH3PresentationItem>* minimax_h3_references = nullptr;
     RefImageParams ref_image_params;
 };
 
@@ -117,6 +140,7 @@ public:
     virtual SDCondition get_learned_condition(int n_threads,
                                               const ConditionerParams& conditioner_params) = 0;
     virtual void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors)           = 0;
+    virtual void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) {}
     virtual void set_max_graph_vram_bytes(size_t max_vram_bytes) {}
     virtual void set_stream_layers_enabled(bool enabled) {}
     virtual void set_runtime_backends(const std::vector<ggml_backend_t>& backends) {}
@@ -1664,6 +1688,10 @@ struct AnimaConditioner : public Conditioner {
         llm->get_param_tensors(tensors, "text_encoders.llm");
     }
 
+    void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) override {
+        llm->get_param_tensor_ops(tensor_ops);
+    }
+
     void set_max_graph_vram_bytes(size_t max_vram_bytes) override {
         llm->set_max_graph_vram_bytes(max_vram_bytes);
     }
@@ -1805,6 +1833,7 @@ struct LLMEmbedder : public Conditioner {
                    sd_version_is_boogu_image(version) ||
                    sd_version_is_sefi_image(version) ||
                    sd_version_is_krea2(version) ||
+                   sd_version_is_minimax_h3(version) ||
                    sd_version_is_mage_flow(version)) {
             arch = LLM::LLMArch::QWEN3_VL;
         } else if (sd_version_is_z_image(version) || version == VERSION_OVIS_IMAGE || version == VERSION_FLUX2_KLEIN) {
@@ -1847,6 +1876,10 @@ struct LLMEmbedder : public Conditioner {
         }
     }
 
+    void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) override {
+        llm->get_param_tensor_ops(tensor_ops);
+    }
+
     void set_max_graph_vram_bytes(size_t max_vram_bytes) override {
         llm->set_max_graph_vram_bytes(max_vram_bytes);
         if (byt5) {
@@ -1983,8 +2016,10 @@ struct LLMEmbedder : public Conditioner {
                                     const std::vector<std::pair<int, sd::Tensor<float>>>& image_embeds,
                                     const std::set<int>& out_layers,
                                     int prompt_template_encode_start_idx,
-                                    bool spell_quotes = false,
-                                    int max_length    = 100000000) {
+                                    bool spell_quotes                                       = false,
+                                    int max_length                                          = 100000000,
+                                    const LLM::DeepStackImageEmbeds& deepstack_image_embeds = {},
+                                    const std::vector<LLM::ImageGrid>& image_grids          = {}) {
         auto tokens_weights_mask = tokenize(prompt, prompt_attn_range, min_length, max_length, spell_quotes);
         auto& tokens             = std::get<0>(tokens_weights_mask);
         auto& weights            = std::get<1>(tokens_weights_mask);
@@ -2017,7 +2052,9 @@ struct LLMEmbedder : public Conditioner {
                                           false,
                                           false,
                                           true,
-                                          true);
+                                          true,
+                                          deepstack_image_embeds,
+                                          image_grids);
         GGML_ASSERT(!hidden_states.empty());
         hidden_states = apply_token_weights(std::move(hidden_states), weights);
         GGML_ASSERT(hidden_states.shape()[1] > prompt_template_encode_start_idx);
@@ -2099,6 +2136,8 @@ struct LLMEmbedder : public Conditioner {
         std::vector<std::string> extra_prompts;
         std::vector<std::pair<int, int>> extra_prompts_attn_range;
         std::vector<std::pair<int, sd::Tensor<float>>> image_embeds;
+        LLM::DeepStackImageEmbeds deepstack_image_embeds;
+        std::vector<LLM::ImageGrid> image_grids;
         int prompt_template_encode_start_idx = 34;
         int min_length                       = 0;  // pad tokens
         int max_length                       = 100000000;
@@ -2109,7 +2148,131 @@ struct LLMEmbedder : public Conditioner {
         int64_t t0                     = ggml_time_ms();
         RefImageResizeMode resize_mode = conditioner_params.ref_image_params.vlm_resize_mode;
 
-        if (sd_version_is_hunyuan_video(version)) {
+        if (sd_version_is_minimax_h3(version)) {
+            prompt_template_encode_start_idx = 0;
+            out_layers                       = {50};
+            prompt_attn_range                = {0, 0};
+
+            if (llm->enable_vision) {
+                const std::string placeholder = "<|image_pad|>";
+                const int patch_size          = llm->config.vision.patch_size;
+                const int factor              = patch_size * llm->config.vision.spatial_merge_size;
+
+                auto resize_for_vision = [&](const sd::Tensor<float>& image) {
+                    int height = static_cast<int>(image.shape()[1]);
+                    int width  = static_cast<int>(image.shape()[0]);
+                    int h_bar  = std::max(factor, static_cast<int>(std::round(static_cast<double>(height) / factor)) * factor);
+                    int w_bar  = std::max(factor, static_cast<int>(std::round(static_cast<double>(width) / factor)) * factor);
+                    resize_image_dims(height,
+                                      width,
+                                      h_bar,
+                                      w_bar,
+                                      factor,
+                                      3136,
+                                      12845056,
+                                      RefImageResizeMode::AREA);
+                    auto resized = sd::ops::interpolate(
+                        image,
+                        std::vector<int64_t>{w_bar, h_bar, image.shape()[2], image.shape()[3]});
+                    for (int64_t i = 0; i < resized.numel(); ++i) {
+                        resized[i] = std::clamp(resized[i], 0.f, 1.f) * 2.f - 1.f;
+                    }
+                    return resized;
+                };
+
+                auto add_vision_outputs = [&](std::vector<sd::Tensor<float>> image_outputs,
+                                              int grid_h,
+                                              int grid_w) {
+                    GGML_ASSERT(image_outputs.size() == 4);
+                    auto image_embed = std::move(image_outputs[0]);
+                    prompt += "<|vision_start|>";
+                    int image_embed_idx = static_cast<int>(tokenizer->encode(prompt, nullptr).size());
+                    image_embeds.emplace_back(image_embed_idx, image_embed);
+                    if (deepstack_image_embeds.empty()) {
+                        deepstack_image_embeds.resize(image_outputs.size() - 1);
+                    }
+                    for (size_t layer = 0; layer < deepstack_image_embeds.size(); ++layer) {
+                        deepstack_image_embeds[layer].emplace_back(image_embed_idx, std::move(image_outputs[layer + 1]));
+                    }
+                    image_grids.push_back({image_embed_idx,
+                                           static_cast<int>(image_embed.shape()[1]),
+                                           grid_h,
+                                           grid_w});
+                    for (int64_t i = 0; i < image_embed.shape()[1]; ++i) {
+                        prompt += placeholder;
+                    }
+                    prompt += "<|vision_end|>";
+                };
+
+                const auto* references = conditioner_params.minimax_h3_references;
+                if (references != nullptr && !references->empty()) {
+                    int picture_index = 0;
+                    int video_index   = 0;
+                    int audio_index   = 0;
+                    for (const auto& item : *references) {
+                        if (item.kind == MiniMaxH3PresentationKind::AUDIO) {
+                            prompt += "<Audio " + std::to_string(++audio_index) + ">: ";
+                            continue;
+                        }
+                        if (item.kind == MiniMaxH3PresentationKind::IMAGE) {
+                            GGML_ASSERT(item.frames.size() == 1);
+                            auto resized = resize_for_vision(item.frames[0]);
+                            prompt += "<Picture " + std::to_string(++picture_index) + ">: ";
+                            add_vision_outputs(llm->encode_image_outputs(n_threads,
+                                                                         resized,
+                                                                         false,
+                                                                         true,
+                                                                         true),
+                                               static_cast<int>(resized.shape()[1]) / patch_size,
+                                               static_cast<int>(resized.shape()[0]) / patch_size);
+                            continue;
+                        }
+
+                        GGML_ASSERT(!item.frames.empty());
+                        prompt += "<Video " + std::to_string(++video_index) + ">: ";
+                        for (size_t frame = 0; frame < item.frames.size(); frame += 2) {
+                            size_t next = std::min(frame + 1, item.frames.size() - 1);
+                            float t0    = frame < item.timestamps.size() ? item.timestamps[frame] : frame / 2.f;
+                            float t1    = next < item.timestamps.size() ? item.timestamps[next] : next / 2.f;
+                            std::ostringstream timestamp;
+                            timestamp << '<' << std::fixed << std::setprecision(1) << (t0 + t1) * 0.5f << " seconds>";
+                            prompt += timestamp.str();
+
+                            auto first  = resize_for_vision(item.frames[frame]);
+                            auto second = resize_for_vision(item.frames[next]);
+                            if (first.shape()[0] != second.shape()[0] || first.shape()[1] != second.shape()[1]) {
+                                second = sd::ops::interpolate(second,
+                                                              std::vector<int64_t>{first.shape()[0],
+                                                                                   first.shape()[1],
+                                                                                   second.shape()[2],
+                                                                                   second.shape()[3]});
+                            }
+                            auto pair = sd::ops::concat(first.unsqueeze(2), second.unsqueeze(2), 2);
+                            add_vision_outputs(llm->encode_video_block_outputs(n_threads,
+                                                                               pair,
+                                                                               false,
+                                                                               true,
+                                                                               true),
+                                               static_cast<int>(first.shape()[1]) / patch_size,
+                                               static_cast<int>(first.shape()[0]) / patch_size);
+                        }
+                    }
+                } else if (conditioner_params.ref_images != nullptr) {
+                    for (size_t i = 0; i < conditioner_params.ref_images->size(); ++i) {
+                        auto resized = resize_for_vision((*conditioner_params.ref_images)[i]);
+                        prompt += "<Picture " + std::to_string(i + 1) + ">: ";
+                        add_vision_outputs(llm->encode_image_outputs(n_threads,
+                                                                     resized,
+                                                                     false,
+                                                                     true,
+                                                                     true),
+                                           static_cast<int>(resized.shape()[1]) / patch_size,
+                                           static_cast<int>(resized.shape()[0]) / patch_size);
+                    }
+                }
+            }
+            prompt += conditioner_params.text;
+        } else if (sd_version_is_hunyuan_video(version)) {
             prompt_template_encode_start_idx = 98;
             out_layers                       = {26};
 
@@ -2657,7 +2820,9 @@ struct LLMEmbedder : public Conditioner {
                                            out_layers,
                                            prompt_template_encode_start_idx,
                                            spell_quotes,
-                                           max_length);
+                                           max_length,
+                                           deepstack_image_embeds,
+                                           image_grids);
         std::vector<sd::Tensor<float>> extra_hidden_states_vec;
         if (sd_version_is_hunyuan_video(version) && byt5) {
             std::vector<std::string> quoted_texts;
@@ -2718,6 +2883,17 @@ struct LLMEmbedder : public Conditioner {
         SDCondition result;
         result.c_crossattn        = std::move(hidden_states);
         result.extra_c_crossattns = std::move(extra_hidden_states_vec);
+        if (sd_version_is_minimax_h3(version)) {
+            std::vector<int32_t> tags(static_cast<size_t>(result.c_crossattn.shape()[1]), 1);
+            for (const auto& [index, image_embed] : image_embeds) {
+                int64_t begin = std::max<int64_t>(0, index - 1);
+                int64_t end   = std::min<int64_t>(static_cast<int64_t>(tags.size()),
+                                                index + image_embed.shape()[1] + 1);
+                std::fill(tags.begin() + begin, tags.begin() + end, 0);
+            }
+            int64_t tag_count    = static_cast<int64_t>(tags.size());
+            result.c_token_types = sd::Tensor<int32_t>({tag_count}, std::move(tags));
+        }
         return result;
     }
 };
@@ -2802,15 +2978,36 @@ struct LTXAVEmbedder : public Conditioner {
     std::shared_ptr<GemmaTokenizer> tokenizer;
     std::shared_ptr<LLM::LLMRunner> llm;
     std::shared_ptr<LTXAVTextProjectionRunner> projector;
+    std::string projector_prefix;
     bool dual_projection = false;
 
+    // Gemma 4 keeps a per-layer output scalar that no Gemma 3 checkpoint has, and widens its
+    // full-attention heads to 512 so their q_proj is twice a sliding layer's.
+    static LLM::LLMArch detect_gemma_arch(const String2TensorStorage& tensor_storage_map,
+                                          const std::string& llm_prefix) {
+        if (tensor_storage_map.find(llm_prefix + ".model.layers.0.layer_scalar") != tensor_storage_map.end()) {
+            return LLM::LLMArch::GEMMA4_12B;
+        }
+        auto global_q  = tensor_storage_map.find(llm_prefix + ".model.layers.5.self_attn.q_proj.weight");
+        auto sliding_q = tensor_storage_map.find(llm_prefix + ".model.layers.0.self_attn.q_proj.weight");
+        if (global_q != tensor_storage_map.end() &&
+            sliding_q != tensor_storage_map.end() &&
+            global_q->second.ne[1] == sliding_q->second.ne[1] * 2) {
+            return LLM::LLMArch::GEMMA4_12B;
+        }
+        return LLM::LLMArch::GEMMA3_12B;
+    }
+
     LTXAVEmbedder(ggml_backend_t backend,
                   const String2TensorStorage& tensor_storage_map      = {},
                   const std::string& llm_prefix                       = "text_encoders.llm",
                   const std::string& projector_prefix                 = "text_embedding_projection",
-                  std::shared_ptr<RunnerWeightManager> weight_manager = nullptr) {
+                  std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+        : projector_prefix(projector_prefix) {
+        LLM::LLMArch arch = detect_gemma_arch(tensor_storage_map, llm_prefix);
+        LOG_INFO("ltxav text encoder: %s", arch == LLM::LLMArch::GEMMA4_12B ? "gemma 4" : "gemma 3");
         tokenizer       = std::make_shared<GemmaTokenizer>();
-        llm             = std::make_shared<LLM::LLMRunner>(LLM::LLMArch::GEMMA3_12B,
+        llm             = std::make_shared<LLM::LLMRunner>(arch,
                                                backend,
                                                tensor_storage_map,
                                                llm_prefix,
@@ -2825,7 +3022,11 @@ struct LTXAVEmbedder : public Conditioner {
 
     void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
         llm->get_param_tensors(tensors, "text_encoders.llm");
-        projector->get_param_tensors(tensors, "text_embedding_projection");
+        projector->get_param_tensors(tensors, projector_prefix);
+    }
+
+    void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) override {
+        llm->get_param_tensor_ops(tensor_ops);
     }
 
     void set_flash_attention_enabled(bool enabled) override {

src/core/ggml_extend.hpp

@@ -1039,6 +1039,38 @@ __STATIC_INLINE__ ggml_tensor* ggml_ext_linear(ggml_context* ctx,
     return x;
 }
 
+__STATIC_INLINE__ ggml_tensor* ggml_ext_linear_i8_tensorwise(ggml_context* ctx,
+                                                             ggml_tensor* x,
+                                                             ggml_tensor* w,
+                                                             ggml_tensor* weight_scale,
+                                                             ggml_tensor* b,
+                                                             int convrot_group_size,
+                                                             float scale = 1.f) {
+    GGML_ASSERT(x->type == GGML_TYPE_F32 || (x->type == GGML_TYPE_I8 && scale == 1.f));
+    if (scale != 1.f) {
+        x = ggml_ext_scale(ctx, x, scale);
+    }
+
+    ggml_tensor* fused_bias = scale == 1.f ? b : nullptr;
+    if (x->ne[2] * x->ne[3] > 1024) {
+        int64_t ne2 = x->ne[2];
+        int64_t ne3 = x->ne[3];
+        x           = ggml_reshape_2d(ctx, x, x->ne[0], x->ne[1] * x->ne[2] * x->ne[3]);
+        x           = ggml_mul_mat_i8_tensorwise(ctx, w, x, weight_scale, fused_bias, convrot_group_size);
+        x           = ggml_reshape_4d(ctx, x, x->ne[0], x->ne[1] / ne2 / ne3, ne2, ne3);
+    } else {
+        x = ggml_mul_mat_i8_tensorwise(ctx, w, x, weight_scale, fused_bias, convrot_group_size);
+    }
+
+    if (scale != 1.f) {
+        x = ggml_ext_scale(ctx, x, 1.f / scale);
+        if (b != nullptr) {
+            x = ggml_add_inplace(ctx, x, b);
+        }
+    }
+    return x;
+}
+
 __STATIC_INLINE__ ggml_tensor* ggml_ext_pad_ext(ggml_context* ctx,
                                                 ggml_backend_t backend,
                                                 ggml_tensor* x,
@@ -1679,6 +1711,13 @@ struct WeightAdapter {
                                            ggml_tensor* b,
                                            const std::string& prefix,
                                            ForwardParams forward_params)                                                              = 0;
+    virtual ggml_tensor* add_lora_to_output(ggml_context* ctx,
+                                            ggml_backend_t backend,
+                                            ggml_tensor* x,
+                                            ggml_tensor* w,
+                                            ggml_tensor* output,
+                                            const std::string& prefix,
+                                            ForwardParams forward_params)                                                             = 0;
     virtual size_t get_extra_graph_size()                                                                                             = 0;
 };
 
@@ -1696,6 +1735,7 @@ struct GGMLRunnerContext {
     std::function<ggml_tensor*(const std::string&)> get_cache_tensor;
     std::function<void(const std::string&, ggml_tensor*)> cache_tensor;
     std::function<void(ggml_tensor*, const void*)> set_backend_tensor_data;
+    std::map<std::pair<ggml_tensor*, int>, ggml_tensor*> int8_convrot_cache;
 
     void capture_tensor(const std::string& name, ggml_tensor* tensor) {
         if (debug_tensors == nullptr || tensor == nullptr) {
@@ -1753,8 +1793,9 @@ protected:
     std::vector<size_t> graph_cut_layer_split_backend_vram_limits_;
 
     std::vector<ggml_backend_t> extra_runtime_backends;  // borrowed (SDBackendManager-owned)
-    ggml_backend_sched_t sched             = nullptr;    // owned, multi-device only
-    ggml_backend_t cpu_fallback_backend    = nullptr;    // owned, sched requires a trailing CPU backend
+    ggml_backend_sched_t sched             = nullptr;    // owned
+    size_t sched_graph_capacity            = 0;
+    ggml_backend_t cpu_fallback_backend    = nullptr;  // owned, sched requires a trailing CPU backend
     bool multi_device_eval_callback_warned = false;
 
     std::shared_ptr<WeightAdapter> weight_adapter = nullptr;
@@ -2040,9 +2081,19 @@ protected:
     // Pass explicit buffer types: synthesized defaults can make CUDA devices
     // report supporting each other's buffers and skip a required copy.
     bool ensure_sched(ggml_cgraph* gf) {
-        if (sched != nullptr) {
+        const size_t required_graph_size = gf != nullptr
+                                               ? std::max<size_t>(1,
+                                                                  (size_t)ggml_graph_n_nodes(gf) +
+                                                                      sd::ggml_graph_cut::leaf_count(gf))
+                                               : 1;
+        if (sched != nullptr && sched_graph_capacity >= required_graph_size) {
             return true;
         }
+        if (sched != nullptr) {
+            ggml_backend_sched_free(sched);
+            sched                = nullptr;
+            sched_graph_capacity = 0;
+        }
         std::vector<ggml_backend_t> backends;
         backends.reserve(extra_runtime_backends.size() + 2);
         backends.push_back(runtime_backend);
@@ -2070,20 +2121,17 @@ protected:
             bufts.push_back(buft);
         }
 
-        size_t graph_size = MAX_GRAPH_SIZE;
-        if (gf != nullptr) {
-            graph_size = std::max<size_t>(graph_size, (size_t)ggml_graph_n_nodes(gf));
-        }
         sched = ggml_backend_sched_new(backends.data(),
                                        bufts.data(),
                                        (int)backends.size(),
-                                       graph_size,
+                                       required_graph_size,
                                        /*parallel=*/false,
                                        /*op_offload=*/false);
         if (sched == nullptr) {
             LOG_ERROR("%s: failed to create backend sched", get_desc().c_str());
             return false;
         }
+        sched_graph_capacity = required_graph_size;
         return true;
     }
 
@@ -2147,8 +2195,22 @@ protected:
         return !extra_runtime_backends.empty();
     }
 
+    bool graph_requires_backend_fallback(ggml_cgraph* gf) const {
+        if (gf == nullptr || sd_backend_is_cpu(runtime_backend)) {
+            return false;
+        }
+        const int n_nodes = ggml_graph_n_nodes(gf);
+        for (int i = 0; i < n_nodes; ++i) {
+            ggml_tensor* node = ggml_graph_node(gf, i);
+            if (node != nullptr && !ggml_backend_supports_op(runtime_backend, node)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     bool alloc_compute_buffer(ggml_cgraph* gf) {
-        if (is_multi_device()) {
+        if (sched != nullptr || is_multi_device() || graph_requires_backend_fallback(gf)) {
             // The sched replaces the gallocr. Do NOT ggml_backend_sched_reserve
             // the graph here: reserve runs split_graph, which rewires the
             // graph's src pointers to sched-internal copy tensors, and the
@@ -2156,6 +2218,10 @@ protected:
             // rewired graph, silently corrupting every cross-backend input. A
             // graph must be split at most once; the alloc in execute_graph
             // performs the real allocation.
+            if (compute_allocr != nullptr) {
+                ggml_gallocr_free(compute_allocr);
+                compute_allocr = nullptr;
+            }
             return ensure_sched(gf);
         }
         if (compute_allocr != nullptr) {
@@ -2390,13 +2456,14 @@ protected:
         GGML_ASSERT(gf != nullptr);
 
         size_t effective_budget = max_graph_vram_bytes;
+        size_t free_clamp       = SIZE_MAX;
         if (stream_layers_enabled && max_graph_vram_bytes > 0 && runtime_backend != nullptr) {
             ggml_backend_dev_t dev = ggml_backend_get_device(runtime_backend);
             if (dev != nullptr && ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_CPU) {
                 size_t free_vram = 0, total_vram = 0;
                 ggml_backend_dev_memory(dev, &free_vram, &total_vram);
                 constexpr size_t safety_margin = 512ull * 1024 * 1024;
-                size_t free_clamp              = (free_vram > safety_margin) ? (free_vram - safety_margin) : 0;
+                free_clamp                     = (free_vram > safety_margin) ? (free_vram - safety_margin) : 0;
                 if (free_clamp < effective_budget) {
                     LOG_DEBUG("%s clamping streaming budget: actual free VRAM %.2f MB < user cap %.2f MB",
                               get_desc().c_str(),
@@ -2413,7 +2480,9 @@ protected:
                 observed_max_effective_budget_ = effective_budget;
                 budget_increased               = true;
             } else {
-                effective_budget = observed_max_effective_budget_;
+                // Keep the plan cache stable, but never plan above what is free now:
+                // another model or process can take VRAM after the first measurement.
+                effective_budget = std::min(observed_max_effective_budget_, free_clamp);
             }
         }
 
@@ -2753,7 +2822,7 @@ protected:
         };
         ComputeBufferGuard compute_buffer_guard(this, free_compute_buffer);
 
-        if (is_multi_device()) {
+        if (sched != nullptr) {
             ggml_backend_sched_reset(sched);
             pin_multi_device_nodes(gf);  // reset clears the pins; re-apply before alloc
             if (!ggml_backend_sched_alloc_graph(sched, gf)) {
@@ -2774,9 +2843,9 @@ protected:
         }
 
         ggml_status status;
-        if (is_multi_device()) {
+        if (sched != nullptr) {
             if (sd_get_backend_eval_callback() != nullptr && !multi_device_eval_callback_warned) {
-                LOG_WARN("%s: eval callback is not supported with multiple runtime backends; ignoring",
+                LOG_WARN("%s: eval callback is not supported with the backend scheduler; ignoring",
                          get_desc().c_str());
                 multi_device_eval_callback_warned = true;
             }
@@ -3012,18 +3081,16 @@ public:
         }
         if (sched != nullptr) {
             ggml_backend_sched_free(sched);
-            sched = nullptr;
+            sched                = nullptr;
+            sched_graph_capacity = 0;
         }
     }
 
     // do copy after alloc graph
     void set_backend_tensor_data(ggml_tensor* tensor, const void* data) {
-        if (is_multi_device()) {
-            // The sched only assigns a backend (and thus a buffer) to tensors
-            // that participate in the graph; flag standalone data tensors as
-            // inputs so they get one.
-            ggml_set_input(tensor);
-        }
+        // The scheduler only allocates standalone data tensors when they are
+        // marked as graph inputs. The flag is harmless for single-backend graphs.
+        ggml_set_input(tensor);
         backend_tensor_data_map[tensor] = data;
     }
 
@@ -3240,6 +3307,11 @@ protected:
 
     virtual void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") {}
 
+    virtual enum ggml_op param_usage_op(const std::string& name) const {
+        (void)name;
+        return GGML_OP_NONE;
+    }
+
 public:
     void init(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, std::string prefix = "") {
         if (prefix.size() > 0) {
@@ -3290,6 +3362,18 @@ public:
         }
     }
 
+    void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) {
+        for (auto& pair : blocks) {
+            pair.second->get_param_tensor_ops(tensor_ops);
+        }
+        for (auto& pair : params) {
+            enum ggml_op op = param_usage_op(pair.first);
+            if (op != GGML_OP_NONE) {
+                tensor_ops[pair.second] = op;
+            }
+        }
+    }
+
     virtual std::string get_desc() {
         return "GGMLBlock";
     }
@@ -3323,15 +3407,18 @@ protected:
     bool bias;
     bool force_f32;
     bool force_prec_f32;
-    bool allow_weight_scale;
-    bool has_weight_scale = false;
+    bool has_weight_scale       = false;
+    bool int8_convrot           = false;
+    int int8_convrot_group_size = 0;
     float scale;
     std::string prefix;
 
     void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
-        this->prefix         = prefix;
-        has_weight_scale     = false;
-        enum ggml_type wtype = get_type(prefix + "weight", tensor_storage_map, GGML_TYPE_F32);
+        this->prefix            = prefix;
+        has_weight_scale        = false;
+        int8_convrot            = false;
+        int8_convrot_group_size = 0;
+        enum ggml_type wtype    = get_type(prefix + "weight", tensor_storage_map, GGML_TYPE_F32);
         if (in_features % ggml_blck_size(wtype) != 0 || force_f32) {
             wtype = GGML_TYPE_F32;
         }
@@ -3340,26 +3427,35 @@ protected:
             enum ggml_type wtype = GGML_TYPE_F32;
             params["bias"]       = ggml_new_tensor_1d(ctx, wtype, out_features);
         }
-        if (allow_weight_scale && tensor_storage_map.find(prefix + "weight_scale") != tensor_storage_map.end()) {
-            params["weight_scale"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, out_features);
+        auto weight_storage           = tensor_storage_map.find(prefix + "weight");
+        const bool is_int8_tensorwise = weight_storage != tensor_storage_map.end() && weight_storage->second.is_int8_tensorwise;
+        auto weight_scale_storage     = tensor_storage_map.find(prefix + "weight_scale");
+        if (weight_scale_storage != tensor_storage_map.end()) {
+            const int64_t scale_nelements = weight_scale_storage->second.nelements();
+            GGML_ASSERT(scale_nelements == 1 || scale_nelements == out_features);
+            params["weight_scale"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, scale_nelements);
             has_weight_scale       = true;
         }
+        if (is_int8_tensorwise) {
+            GGML_ASSERT(wtype == GGML_TYPE_I8);
+            GGML_ASSERT(has_weight_scale);
+            int8_convrot            = weight_storage->second.int8_convrot;
+            int8_convrot_group_size = weight_storage->second.int8_convrot_group_size;
+        }
     }
 
 public:
     Linear(int64_t in_features,
            int64_t out_features,
-           bool bias               = true,
-           bool force_f32          = false,
-           bool force_prec_f32     = false,
-           float scale             = 1.f,
-           bool allow_weight_scale = false)
+           bool bias           = true,
+           bool force_f32      = false,
+           bool force_prec_f32 = false,
+           float scale         = 1.f)
         : in_features(in_features),
           out_features(out_features),
           bias(bias),
           force_f32(force_f32),
           force_prec_f32(force_prec_f32),
-          allow_weight_scale(allow_weight_scale),
           scale(scale) {}
 
     void set_scale(float scale_) {
@@ -3371,13 +3467,94 @@ public:
     }
 
     ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
-        ggml_tensor* w = params["weight"];
+        ggml_tensor* w            = params["weight"];
+        ggml_tensor* weight_scale = has_weight_scale ? params["weight_scale"] : nullptr;
+        if (w->type == GGML_TYPE_F8_E4M3 || w->type == GGML_TYPE_F8_E5M2) {
+            bool supports_fp8_matmul = false;
+            if (ctx->backend != nullptr) {
+                ggml_tensor* fp8_matmul = ggml_mul_mat(ctx->ggml_ctx, w, x);
+                if (force_prec_f32) {
+                    ggml_mul_mat_set_prec(fp8_matmul, GGML_PREC_F32);
+                }
+                supports_fp8_matmul = ggml_backend_supports_op(ctx->backend, fp8_matmul);
+            }
+            if (!supports_fp8_matmul) {
+                w = ggml_cast(ctx->ggml_ctx, w, GGML_TYPE_BF16);
+            }
+        }
         ggml_tensor* b = nullptr;
         if (bias) {
             b = params["bias"];
         }
         ggml_tensor* linear_bias = has_weight_scale ? nullptr : b;
         ggml_tensor* out         = nullptr;
+        if (w->type == GGML_TYPE_I8) {
+            if (x->type != GGML_TYPE_F32) {
+                x = ggml_ext_cast_f32(ctx->ggml_ctx, ctx->backend, x);
+            }
+            if (!ggml_is_contiguous(x)) {
+                x = ggml_cont(ctx->ggml_ctx, x);
+            }
+            ggml_tensor* lora_input = x;
+            if (ctx->weight_adapter && b != nullptr) {
+                b = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, b, prefix + "bias");
+            }
+            if (int8_convrot && scale == 1.f) {
+                const auto cache_key = std::make_pair(x, int8_convrot_group_size);
+                auto cached          = ctx->int8_convrot_cache.find(cache_key);
+                if (cached == ctx->int8_convrot_cache.end()) {
+                    x = ggml_quantize_i8_convrot(ctx->ggml_ctx, x, int8_convrot_group_size);
+                    ctx->int8_convrot_cache.emplace(cache_key, x);
+                } else {
+                    x = cached->second;
+                }
+            }
+            out = ggml_ext_linear_i8_tensorwise(ctx->ggml_ctx,
+                                                x,
+                                                w,
+                                                weight_scale,
+                                                b,
+                                                int8_convrot ? int8_convrot_group_size : 0,
+                                                scale);
+            if (ctx->weight_adapter) {
+                WeightAdapter::ForwardParams forward_params;
+                forward_params.op_type               = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR;
+                forward_params.linear.force_prec_f32 = force_prec_f32;
+                forward_params.linear.scale          = scale;
+                out                                  = ctx->weight_adapter->add_lora_to_output(ctx->ggml_ctx,
+                                                                                               ctx->backend,
+                                                                                               lora_input,
+                                                                                               w,
+                                                                                               out,
+                                                                                               prefix,
+                                                                                               forward_params);
+            }
+            return out;
+        }
+        if (has_weight_scale) {
+            out = ggml_ext_linear(ctx->ggml_ctx, x, w, nullptr, force_prec_f32, scale);
+            out = ggml_mul(ctx->ggml_ctx, out, weight_scale);
+            if (ctx->weight_adapter) {
+                WeightAdapter::ForwardParams forward_params;
+                forward_params.op_type               = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR;
+                forward_params.linear.force_prec_f32 = force_prec_f32;
+                forward_params.linear.scale          = scale;
+                out                                  = ctx->weight_adapter->add_lora_to_output(ctx->ggml_ctx,
+                                                                                               ctx->backend,
+                                                                                               x,
+                                                                                               w,
+                                                                                               out,
+                                                                                               prefix,
+                                                                                               forward_params);
+                if (b != nullptr) {
+                    b = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, b, prefix + "bias");
+                }
+            }
+            if (b != nullptr) {
+                out = ggml_add_inplace(ctx->ggml_ctx, out, b);
+            }
+            return out;
+        }
         if (ctx->weight_adapter) {
             WeightAdapter::ForwardParams forward_params;
             forward_params.op_type               = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR;
@@ -3387,12 +3564,6 @@ public:
         } else {
             out = ggml_ext_linear(ctx->ggml_ctx, x, w, linear_bias, force_prec_f32, scale);
         }
-        if (has_weight_scale) {
-            out = ggml_mul(ctx->ggml_ctx, out, params["weight_scale"]);
-            if (b != nullptr) {
-                out = ggml_add_inplace(ctx->ggml_ctx, out, b);
-            }
-        }
         return out;
     }
 };
@@ -3417,6 +3588,10 @@ protected:
         params["weight"] = ggml_new_tensor_2d(ctx, wtype, embedding_dim, num_embeddings);
     }
 
+    enum ggml_op param_usage_op(const std::string& name) const override {
+        return name == "weight" ? GGML_OP_GET_ROWS : GGML_OP_NONE;
+    }
+
 public:
     Embedding(int64_t num_embeddings, int64_t embedding_dim)
         : embedding_dim(embedding_dim),

src/core/ggml_graph_cut.cpp

@@ -453,11 +453,12 @@ namespace sd::ggml_graph_cut {
         if (tensor == nullptr || tensor->name[0] == '\0') {
             return false;
         }
-        return std::strncmp(tensor->name, GGML_RUNNER_CUT_PREFIX, std::strlen(GGML_RUNNER_CUT_PREFIX)) == 0;
+        return starts_with(tensor->name, GGML_RUNNER_CUT_PREFIX) &&
+               ends_with(tensor->name, GGML_RUNNER_CUT_SUFFIX);
     }
 
     std::string make_graph_cut_name(const std::string& group, const std::string& output) {
-        return std::string(GGML_RUNNER_CUT_PREFIX) + group + "|" + output;
+        return std::string(GGML_RUNNER_CUT_PREFIX) + group + "|" + output + GGML_RUNNER_CUT_SUFFIX;
     }
 
     void mark_graph_cut(ggml_tensor* tensor, const std::string& group, const std::string& output) {
@@ -603,7 +604,43 @@ namespace sd::ggml_graph_cut {
         GGML_ASSERT(gf != nullptr);
         GGML_ASSERT(graph_ctx_out != nullptr);
 
-        const size_t graph_size = segment.internal_node_indices.size() + segment.input_refs.size() + 8;
+        // Collect leaf inputs and internal nodes, then any tensor they
+        // reference that is not already represented, notably the view_src of a
+        // view-typed input leaf. ggml_gallocr sizes its hash set from
+        // n_nodes + n_leafs (plus a 25% margin that rounds down to zero for a
+        // one-node segment), so every distinct tensor it will hash must be
+        // counted here or a tiny segment overflows the hash set and aborts.
+        std::vector<ggml_tensor*> leaves;
+        std::unordered_set<ggml_tensor*> represented;
+        for (const auto& input : segment.input_refs) {
+            ggml_tensor* current_input = input_tensor(gf, input);
+            if (current_input == nullptr) {
+                continue;
+            }
+            if (represented.insert(current_input).second) {
+                leaves.push_back(current_input);
+            }
+        }
+        for (int node_idx : segment.internal_node_indices) {
+            represented.insert(ggml_graph_node(gf, node_idx));
+        }
+        auto add_reference = [&](ggml_tensor* tensor) {
+            if (tensor != nullptr && represented.insert(tensor).second) {
+                leaves.push_back(tensor);
+            }
+        };
+        for (int node_idx : segment.internal_node_indices) {
+            ggml_tensor* node = ggml_graph_node(gf, node_idx);
+            for (int src_idx = 0; src_idx < GGML_MAX_SRC; ++src_idx) {
+                add_reference(node->src[src_idx]);
+            }
+            add_reference(node->view_src);
+        }
+        for (size_t i = 0; i < leaves.size(); ++i) {
+            add_reference(leaves[i]->view_src);
+        }
+
+        const size_t graph_size = segment.internal_node_indices.size() + leaves.size() + 8;
         ggml_init_params params = {
             /*.mem_size   =*/ggml_graph_overhead_custom(graph_size, false) + 1024,
             /*.mem_buffer =*/nullptr,
@@ -614,13 +651,9 @@ namespace sd::ggml_graph_cut {
         ggml_cgraph* segment_graph = ggml_new_graph_custom(graph_ctx, graph_size, false);
         GGML_ASSERT(segment_graph != nullptr);
 
-        for (const auto& input : segment.input_refs) {
-            ggml_tensor* current_input = input_tensor(gf, input);
-            if (current_input == nullptr) {
-                continue;
-            }
+        for (ggml_tensor* leaf : leaves) {
             GGML_ASSERT(segment_graph->n_leafs < segment_graph->size);
-            segment_graph->leafs[segment_graph->n_leafs++] = current_input;
+            segment_graph->leafs[segment_graph->n_leafs++] = leaf;
         }
 
         for (int output_node_index : segment.output_node_indices) {
@@ -751,7 +784,9 @@ namespace sd::ggml_graph_cut {
 
             plan.has_cuts = true;
             std::string full_name(node->name);
-            std::string payload = full_name.substr(std::strlen(GGML_RUNNER_CUT_PREFIX));
+            size_t prefix_len   = std::strlen(GGML_RUNNER_CUT_PREFIX);
+            size_t suffix_len   = std::strlen(GGML_RUNNER_CUT_SUFFIX);
+            std::string payload = full_name.substr(prefix_len, full_name.size() - prefix_len - suffix_len);
             size_t sep          = payload.find('|');
             std::string group   = sep == std::string::npos ? payload : payload.substr(0, sep);
 

src/core/ggml_graph_cut.h

@@ -68,6 +68,7 @@ namespace sd::ggml_graph_cut {
     };
 
     static constexpr const char* GGML_RUNNER_CUT_PREFIX = "ggml_runner_cut:";
+    static constexpr const char* GGML_RUNNER_CUT_SUFFIX = "|";
 
     struct MaxVramAssignment {
         float default_gib = 0.f;

src/extensions/photomaker_extension.cpp

@@ -56,7 +56,7 @@ tokenize_photomaker_trigger(FrozenCLIPEmbedderWithCustomWords& clip_conditioner,
                                           true);
     std::vector<bool> class_token_mask;
     for (int i = 0; i < tokens.size(); i++) {
-        class_token_mask.push_back(class_idx + 1 <= i && i < class_idx + 1 + trigger_token_count);
+        class_token_mask.push_back(class_idx >= 0 && class_idx + 1 <= i && i < class_idx + 1 + trigger_token_count);
     }
 
     return std::make_tuple(tokens, weights, class_token_mask);

src/model.h

@@ -43,6 +43,7 @@ enum SDVersion {
     VERSION_FLUX2,
     VERSION_FLUX2_KLEIN,
     VERSION_LTXAV,
+    VERSION_MINIMAX_H3,
     VERSION_HIDREAM_O1,
     VERSION_Z_IMAGE,
     VERSION_BOOGU_IMAGE,
@@ -123,6 +124,10 @@ static inline bool sd_version_is_ltxav(SDVersion version) {
     return false;
 }
 
+static inline bool sd_version_is_minimax_h3(SDVersion version) {
+    return version == VERSION_MINIMAX_H3;
+}
+
 static inline bool sd_version_is_wan(SDVersion version) {
     if (version == VERSION_WAN2 || version == VERSION_WAN2_2_I2V || version == VERSION_WAN2_2_TI2V) {
         return true;
@@ -272,6 +277,7 @@ static inline bool sd_version_is_dit(SDVersion version) {
     if (sd_version_is_flux(version) ||
         sd_version_is_flux2(version) ||
         sd_version_is_ltxav(version) ||
+        sd_version_is_minimax_h3(version) ||
         sd_version_is_sd3(version) ||
         sd_version_is_wan(version) ||
         sd_version_is_lingbot_video(version) ||

src/model/adapter/ip_adapter.hpp

@@ -31,8 +31,92 @@ namespace IPAdapter {
         }
     };
 
+    struct Resampler : public GGMLBlock {
+        int64_t dim         = 1280;
+        int64_t depth       = 4;
+        int64_t num_queries = 16;
+        int64_t embed_dim   = 1280;
+        int64_t output_dim  = 2048;
+        int64_t ff_inner    = 5120;
+        int64_t dim_head    = 64;
+        int64_t heads       = 20;
+
+        void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
+            params["latents"] = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, dim, num_queries, 1);
+        }
+
+        Resampler() {}
+        Resampler(int64_t dim, int64_t depth, int64_t num_queries, int64_t embed_dim, int64_t output_dim, int64_t ff_inner)
+            : dim(dim), depth(depth), num_queries(num_queries), embed_dim(embed_dim), output_dim(output_dim), ff_inner(ff_inner) {
+            heads              = dim / dim_head;
+            blocks["proj_in"]  = std::shared_ptr<GGMLBlock>(new Linear(embed_dim, dim, true));
+            blocks["proj_out"] = std::shared_ptr<GGMLBlock>(new Linear(dim, output_dim, true));
+            blocks["norm_out"] = std::shared_ptr<GGMLBlock>(new LayerNorm(output_dim));
+            for (int64_t i = 0; i < depth; i++) {
+                std::string p           = "layers." + std::to_string(i);
+                blocks[p + ".0.norm1"]  = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));
+                blocks[p + ".0.norm2"]  = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));
+                blocks[p + ".0.to_q"]   = std::shared_ptr<GGMLBlock>(new Linear(dim, dim, false));
+                blocks[p + ".0.to_kv"]  = std::shared_ptr<GGMLBlock>(new Linear(dim, dim * 2, false));
+                blocks[p + ".0.to_out"] = std::shared_ptr<GGMLBlock>(new Linear(dim, dim, false));
+                blocks[p + ".1.0"]      = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));
+                blocks[p + ".1.1"]      = std::shared_ptr<GGMLBlock>(new Linear(dim, ff_inner, false));
+                blocks[p + ".1.3"]      = std::shared_ptr<GGMLBlock>(new Linear(ff_inner, dim, false));
+            }
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* image_embeds) {
+            int64_t N     = image_embeds->ne[2];
+            auto proj_in  = std::dynamic_pointer_cast<Linear>(blocks["proj_in"]);
+            auto proj_out = std::dynamic_pointer_cast<Linear>(blocks["proj_out"]);
+            auto norm_out = std::dynamic_pointer_cast<LayerNorm>(blocks["norm_out"]);
+
+            ggml_tensor* x       = proj_in->forward(ctx, image_embeds);
+            ggml_tensor* latents = params["latents"];
+            if (N > 1) {
+                latents = ggml_repeat(ctx->ggml_ctx, latents, ggml_new_tensor_3d(ctx->ggml_ctx, GGML_TYPE_F32, dim, num_queries, N));
+            }
+
+            for (int64_t i = 0; i < depth; i++) {
+                std::string p = "layers." + std::to_string(i);
+                auto norm1    = std::dynamic_pointer_cast<LayerNorm>(blocks[p + ".0.norm1"]);
+                auto norm2    = std::dynamic_pointer_cast<LayerNorm>(blocks[p + ".0.norm2"]);
+                auto to_q     = std::dynamic_pointer_cast<Linear>(blocks[p + ".0.to_q"]);
+                auto to_kv    = std::dynamic_pointer_cast<Linear>(blocks[p + ".0.to_kv"]);
+                auto to_out   = std::dynamic_pointer_cast<Linear>(blocks[p + ".0.to_out"]);
+
+                ggml_tensor* xn    = norm1->forward(ctx, x);
+                ggml_tensor* ln    = norm2->forward(ctx, latents);
+                ggml_tensor* q     = to_q->forward(ctx, ln);
+                ggml_tensor* kv_in = ggml_concat(ctx->ggml_ctx, xn, ln, 1);
+                ggml_tensor* kv    = to_kv->forward(ctx, kv_in);
+                int64_t L          = kv->ne[1];
+                ggml_tensor* k     = ggml_cont(ctx->ggml_ctx, ggml_view_3d(ctx->ggml_ctx, kv, dim, L, N, kv->nb[1], kv->nb[2], 0));
+                ggml_tensor* v     = ggml_cont(ctx->ggml_ctx, ggml_view_3d(ctx->ggml_ctx, kv, dim, L, N, kv->nb[1], kv->nb[2], dim * kv->nb[0]));
+                ggml_tensor* attn  = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, heads, nullptr, false, false);
+                attn               = to_out->forward(ctx, attn);
+                latents            = ggml_add(ctx->ggml_ctx, latents, attn);
+
+                auto ff_norm   = std::dynamic_pointer_cast<LayerNorm>(blocks[p + ".1.0"]);
+                auto ff_fc1    = std::dynamic_pointer_cast<Linear>(blocks[p + ".1.1"]);
+                auto ff_fc2    = std::dynamic_pointer_cast<Linear>(blocks[p + ".1.3"]);
+                ggml_tensor* h = ff_norm->forward(ctx, latents);
+                h              = ff_fc1->forward(ctx, h);
+                h              = ggml_gelu_erf(ctx->ggml_ctx, h);
+                h              = ff_fc2->forward(ctx, h);
+                latents        = ggml_add(ctx->ggml_ctx, latents, h);
+            }
+
+            latents = proj_out->forward(ctx, latents);
+            latents = norm_out->forward(ctx, latents);
+            return latents;
+        }
+    };
+
     struct IPAdapterRunner : public GGMLRunner {
         ImageProjModel image_proj;
+        Resampler resampler;
+        bool is_plus       = false;
         int64_t num_tokens = 4;
         std::string prefix;
 
@@ -41,21 +125,54 @@ namespace IPAdapter {
                         const std::string prefix,
                         std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
             : GGMLRunner(backend, weight_manager), prefix(prefix) {
-            int64_t ctx_dim  = 768;
-            int64_t clip_dim = 1024;
-            int64_t out_dim  = 3072;
-            auto norm_iter   = tensor_storage_map.find(prefix + ".image_proj.norm.weight");
-            if (norm_iter != tensor_storage_map.end()) {
-                ctx_dim = norm_iter->second.ne[0];
+            is_plus = tensor_storage_map.find(prefix + ".image_proj.latents") != tensor_storage_map.end();
+            if (is_plus) {
+                int64_t dim         = 1280;
+                int64_t num_queries = 16;
+                int64_t embed_dim   = 1280;
+                int64_t output_dim  = 2048;
+                int64_t ff_inner    = 5120;
+                auto latents_iter   = tensor_storage_map.find(prefix + ".image_proj.latents");
+                if (latents_iter != tensor_storage_map.end()) {
+                    dim         = latents_iter->second.ne[0];
+                    num_queries = latents_iter->second.ne[1];
+                }
+                auto proj_in_iter = tensor_storage_map.find(prefix + ".image_proj.proj_in.weight");
+                if (proj_in_iter != tensor_storage_map.end()) {
+                    embed_dim = proj_in_iter->second.ne[0];
+                }
+                auto proj_out_iter = tensor_storage_map.find(prefix + ".image_proj.proj_out.weight");
+                if (proj_out_iter != tensor_storage_map.end()) {
+                    output_dim = proj_out_iter->second.ne[1];
+                }
+                auto ff_iter = tensor_storage_map.find(prefix + ".image_proj.layers.0.1.1.weight");
+                if (ff_iter != tensor_storage_map.end()) {
+                    ff_inner = ff_iter->second.ne[1];
+                }
+                int64_t depth = 0;
+                while (tensor_storage_map.find(prefix + ".image_proj.layers." + std::to_string(depth) + ".0.to_q.weight") != tensor_storage_map.end()) {
+                    depth++;
+                }
+                num_tokens = num_queries;
+                resampler  = Resampler(dim, depth, num_queries, embed_dim, output_dim, ff_inner);
+                resampler.init(params_ctx, tensor_storage_map, prefix + ".image_proj");
+            } else {
+                int64_t ctx_dim  = 768;
+                int64_t clip_dim = 1024;
+                int64_t out_dim  = 3072;
+                auto norm_iter   = tensor_storage_map.find(prefix + ".image_proj.norm.weight");
+                if (norm_iter != tensor_storage_map.end()) {
+                    ctx_dim = norm_iter->second.ne[0];
+                }
+                auto proj_iter = tensor_storage_map.find(prefix + ".image_proj.proj.weight");
+                if (proj_iter != tensor_storage_map.end()) {
+                    clip_dim = proj_iter->second.ne[0];
+                    out_dim  = proj_iter->second.ne[1];
+                }
+                num_tokens = out_dim / ctx_dim;
+                image_proj = ImageProjModel(num_tokens, ctx_dim, clip_dim);
+                image_proj.init(params_ctx, tensor_storage_map, prefix + ".image_proj");
             }
-            auto proj_iter = tensor_storage_map.find(prefix + ".image_proj.proj.weight");
-            if (proj_iter != tensor_storage_map.end()) {
-                clip_dim = proj_iter->second.ne[0];
-                out_dim  = proj_iter->second.ne[1];
-            }
-            num_tokens = out_dim / ctx_dim;
-            image_proj = ImageProjModel(num_tokens, ctx_dim, clip_dim);
-            image_proj.init(params_ctx, tensor_storage_map, prefix + ".image_proj");
         }
 
         std::string get_desc() override {
@@ -63,14 +180,18 @@ namespace IPAdapter {
         }
 
         void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string = "") {
-            image_proj.get_param_tensors(tensors, prefix + ".image_proj");
+            if (is_plus) {
+                resampler.get_param_tensors(tensors, prefix + ".image_proj");
+            } else {
+                image_proj.get_param_tensors(tensors, prefix + ".image_proj");
+            }
         }
 
         ggml_cgraph* build_graph(const sd::Tensor<float>& image_embeds_tensor) {
             ggml_cgraph* gf     = new_graph_custom(1024);
             ggml_tensor* embeds = make_input(image_embeds_tensor);
             auto runner_ctx     = get_context();
-            ggml_tensor* out    = image_proj.forward(&runner_ctx, embeds);
+            ggml_tensor* out    = is_plus ? resampler.forward(&runner_ctx, embeds) : image_proj.forward(&runner_ctx, embeds);
             ggml_build_forward_expand(gf, out);
             return gf;
         }

src/model/adapter/lora.hpp

@@ -14,6 +14,8 @@ struct LoraModel : public GGMLRunner {
     std::unordered_map<std::string, ggml_tensor*> lora_tensors;
     std::map<ggml_tensor*, ggml_tensor*> original_tensor_to_final_tensor;
     std::set<std::string> applied_lora_tensors;
+    std::set<std::string> skipped_incompatible_lora_tensors;
+    std::set<std::string> warned_incompatible_model_tensors;
     std::string file_path;
     std::shared_ptr<ModelManager> model_manager;
     ggml_backend_t params_backend = nullptr;
@@ -133,6 +135,8 @@ struct LoraModel : public GGMLRunner {
         lora_tensors.clear();
         original_tensor_to_final_tensor.clear();
         applied_lora_tensors.clear();
+        skipped_incompatible_lora_tensors.clear();
+        warned_incompatible_model_tensors.clear();
         applied             = false;
         tensor_preprocessed = false;
     }
@@ -338,7 +342,9 @@ struct LoraModel : public GGMLRunner {
             iter = lora_tensors.find(hada_1_mid_name);
             if (iter != lora_tensors.end()) {
                 hada_1_mid = ggml_ext_cast_f32(ctx, backend, iter->second);
-                hada_1_up  = ggml_cont(ctx, ggml_transpose(ctx, hada_1_up));
+                if (hada_1_up != nullptr) {
+                    hada_1_up = ggml_cont(ctx, ggml_transpose(ctx, hada_1_up));
+                }
             }
 
             iter = lora_tensors.find(hada_2_down_name);
@@ -354,7 +360,9 @@ struct LoraModel : public GGMLRunner {
             iter = lora_tensors.find(hada_2_mid_name);
             if (iter != lora_tensors.end()) {
                 hada_2_mid = ggml_ext_cast_f32(ctx, backend, iter->second);
-                hada_2_up  = ggml_cont(ctx, ggml_transpose(ctx, hada_2_up));
+                if (hada_2_up != nullptr) {
+                    hada_2_up = ggml_cont(ctx, ggml_transpose(ctx, hada_2_up));
+                }
             }
 
             if (hada_1_up == nullptr || hada_1_down == nullptr || hada_2_up == nullptr || hada_2_down == nullptr) {
@@ -546,7 +554,27 @@ struct LoraModel : public GGMLRunner {
                 }
             }
 
-            GGML_ASSERT(ggml_nelements(diff) == ggml_nelements(model_tensor));
+            if (ggml_nelements(diff) != ggml_nelements(model_tensor)) {
+                const std::string lora_tensor_prefix = "lora." + model_tensor_name + ".";
+                for (const auto& tensor_name : applied_lora_tensors) {
+                    if (starts_with(tensor_name, lora_tensor_prefix)) {
+                        skipped_incompatible_lora_tensors.insert(tensor_name);
+                    }
+                }
+                if (warned_incompatible_model_tensors.insert(model_tensor_name).second) {
+                    LOG_WARN("skip incompatible LoRA tensor |%s|: model shape = [%lld, %lld, %lld, %lld], LoRA shape = [%lld, %lld, %lld, %lld]",
+                             model_tensor_name.c_str(),
+                             static_cast<long long>(model_tensor->ne[0]),
+                             static_cast<long long>(model_tensor->ne[1]),
+                             static_cast<long long>(model_tensor->ne[2]),
+                             static_cast<long long>(model_tensor->ne[3]),
+                             static_cast<long long>(diff->ne[0]),
+                             static_cast<long long>(diff->ne[1]),
+                             static_cast<long long>(diff->ne[2]),
+                             static_cast<long long>(diff->ne[3]));
+                }
+                return nullptr;
+            }
             diff = ggml_reshape(ctx, diff, model_tensor);
         }
         return diff;
@@ -555,10 +583,15 @@ struct LoraModel : public GGMLRunner {
     ggml_tensor* get_out_diff(ggml_context* ctx,
                               ggml_backend_t backend,
                               ggml_tensor* x,
+                              ggml_tensor* model_weight,
                               WeightAdapter::ForwardParams forward_params,
                               const std::string& model_tensor_name) {
         ggml_tensor* out_diff = nullptr;
         int index             = 0;
+
+        std::vector<std::string> used_tensors;
+        bool is_conv2d = forward_params.op_type == WeightAdapter::ForwardParams::op_type_t::OP_CONV2D;
+
         while (true) {
             std::string key;
             if (index == 0) {
@@ -566,7 +599,6 @@ struct LoraModel : public GGMLRunner {
             } else {
                 key = model_tensor_name + "." + std::to_string(index);
             }
-            bool is_conv2d = forward_params.op_type == WeightAdapter::ForwardParams::op_type_t::OP_CONV2D;
 
             std::string lokr_w1_name   = "lora." + key + ".lokr_w1";
             std::string lokr_w1_a_name = "lora." + key + ".lokr_w1_a";
@@ -634,7 +666,6 @@ struct LoraModel : public GGMLRunner {
                 if (iter != lora_tensors.end()) {
                     float alpha = ggml_ext_backend_tensor_get_f32(iter->second);
                     scale_value = alpha / rank;
-                    applied_lora_tensors.insert(alpha_name);
                 }
 
                 if (rank == 1) {
@@ -649,19 +680,27 @@ struct LoraModel : public GGMLRunner {
                     out_diff = ggml_concat(ctx, out_diff, curr_out_diff, 0);
                 }
 
-                if (lokr_w1)
-                    applied_lora_tensors.insert(lokr_w1_name);
-                if (lokr_w1_a)
-                    applied_lora_tensors.insert(lokr_w1_a_name);
-                if (lokr_w1_b)
-                    applied_lora_tensors.insert(lokr_w1_b_name);
-                if (lokr_w2)
-                    applied_lora_tensors.insert(lokr_w2_name);
-                if (lokr_w2_a)
-                    applied_lora_tensors.insert(lokr_w2_a_name);
-                if (lokr_w2_b)
-                    applied_lora_tensors.insert(lokr_w2_b_name);
-                applied_lora_tensors.insert(alpha_name);
+                if (lokr_w1) {
+                    used_tensors.push_back(lokr_w1_name);
+                }
+                if (lokr_w1_a) {
+                    used_tensors.push_back(lokr_w1_a_name);
+                }
+                if (lokr_w1_b) {
+                    used_tensors.push_back(lokr_w1_b_name);
+                }
+                if (lokr_w2) {
+                    used_tensors.push_back(lokr_w2_name);
+                }
+                if (lokr_w2_a) {
+                    used_tensors.push_back(lokr_w2_a_name);
+                }
+                if (lokr_w2_b) {
+                    used_tensors.push_back(lokr_w2_b_name);
+                }
+                if (iter != lora_tensors.end()) {
+                    used_tensors.push_back(alpha_name);
+                }
 
                 index++;
                 continue;
@@ -707,27 +746,60 @@ struct LoraModel : public GGMLRunner {
                 break;
             }
 
-            applied_lora_tensors.insert(lora_up_name);
-            applied_lora_tensors.insert(lora_down_name);
+            if (!is_conv2d) {
+                const int64_t down_in  = lora_down->ne[0];
+                const int64_t down_out = lora_down->ne[1];
+                const int64_t up_in    = lora_up->ne[0];
 
-            if (lora_mid) {
-                applied_lora_tensors.insert(lora_mid_name);
+                bool compatible = down_in == model_weight->ne[0];
+                if (lora_mid != nullptr) {
+                    compatible = compatible &&
+                                 lora_mid->ne[0] == down_out &&
+                                 up_in == lora_mid->ne[1];
+                } else {
+                    compatible = compatible && up_in == down_out;
+                }
+
+                if (!compatible) {
+                    skipped_incompatible_lora_tensors.insert(lora_down_name);
+                    skipped_incompatible_lora_tensors.insert(lora_up_name);
+                    if (lora_mid != nullptr) {
+                        skipped_incompatible_lora_tensors.insert(lora_mid_name);
+                    }
+                    if (lora_tensors.find(scale_name) != lora_tensors.end()) {
+                        skipped_incompatible_lora_tensors.insert(scale_name);
+                    } else if (lora_tensors.find(alpha_name) != lora_tensors.end()) {
+                        skipped_incompatible_lora_tensors.insert(alpha_name);
+                    }
+                    if (warned_incompatible_model_tensors.insert(model_tensor_name).second) {
+                        LOG_WARN("skip incompatible LoRA tensor |%s|: model input dim = %lld, down shape = [%lld, %lld], up shape = [%lld, %lld]",
+                                 model_tensor_name.c_str(),
+                                 static_cast<long long>(model_weight->ne[0]),
+                                 static_cast<long long>(down_in),
+                                 static_cast<long long>(down_out),
+                                 static_cast<long long>(up_in),
+                                 static_cast<long long>(lora_up->ne[1]));
+                    }
+                    index++;
+                    continue;
+                }
             }
 
             float scale_value = 1.0f;
+            std::string scale_tensor_name;
 
             int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1];
             iter         = lora_tensors.find(scale_name);
             if (iter != lora_tensors.end()) {
-                scale_value = ggml_ext_backend_tensor_get_f32(iter->second);
-                applied_lora_tensors.insert(scale_name);
+                scale_value       = ggml_ext_backend_tensor_get_f32(iter->second);
+                scale_tensor_name = scale_name;
             } else {
                 iter = lora_tensors.find(alpha_name);
                 if (iter != lora_tensors.end()) {
-                    float alpha = ggml_ext_backend_tensor_get_f32(iter->second);
-                    scale_value = alpha / rank;
+                    float alpha       = ggml_ext_backend_tensor_get_f32(iter->second);
+                    scale_value       = alpha / rank;
+                    scale_tensor_name = alpha_name;
                     // LOG_DEBUG("rank %s %ld %.2f %.2f", alpha_name.c_str(), rank, alpha, scale_value);
-                    applied_lora_tensors.insert(alpha_name);
                 }
             }
             scale_value *= multiplier;
@@ -787,15 +859,45 @@ struct LoraModel : public GGMLRunner {
             }
 
             auto curr_out_diff = ggml_ext_scale(ctx, lx, scale_value, true);
-
             if (out_diff == nullptr) {
                 out_diff = curr_out_diff;
             } else {
-                out_diff = ggml_concat(ctx, out_diff, curr_out_diff, 0);
+                out_diff = ggml_concat(ctx, out_diff, curr_out_diff, is_conv2d ? 2 : 0);
+            }
+
+            used_tensors.push_back(lora_up_name);
+            used_tensors.push_back(lora_down_name);
+            if (lora_mid) {
+                used_tensors.push_back(lora_mid_name);
+            }
+            if (!scale_tensor_name.empty()) {
+                used_tensors.push_back(scale_tensor_name);
             }
 
             index++;
         }
+
+        if (out_diff == nullptr)
+            return nullptr;
+
+        int64_t expected_out_dim = is_conv2d ? model_weight->ne[3] : model_weight->ne[1];
+        int64_t actual_out_dim   = out_diff->ne[is_conv2d ? 2 : 0];
+
+        if (actual_out_dim != expected_out_dim) {
+            for (const auto& name : used_tensors) {
+                skipped_incompatible_lora_tensors.insert(name);
+            }
+            if (warned_incompatible_model_tensors.insert(model_tensor_name).second) {
+                LOG_WARN("skip incompatible LoRA tensors for |%s|: output dim %lld != model dim %lld",
+                         model_tensor_name.c_str(), actual_out_dim, expected_out_dim);
+            }
+            return nullptr;
+        }
+
+        for (const auto& name : used_tensors) {
+            applied_lora_tensors.insert(name);
+        }
+
         return out_diff;
     }
 
@@ -869,10 +971,13 @@ struct LoraModel : public GGMLRunner {
     void stat(bool at_runntime = false) {
         size_t total_lora_tensors_count   = 0;
         size_t applied_lora_tensors_count = 0;
+        size_t skipped_lora_tensors_count = 0;
 
         for (auto& kv : lora_tensors) {
             total_lora_tensors_count++;
-            if (applied_lora_tensors.find(kv.first) == applied_lora_tensors.end()) {
+            if (skipped_incompatible_lora_tensors.find(kv.first) != skipped_incompatible_lora_tensors.end()) {
+                skipped_lora_tensors_count++;
+            } else if (applied_lora_tensors.find(kv.first) == applied_lora_tensors.end()) {
                 if (!at_runntime) {
                     LOG_WARN("unused lora tensor |%s|", kv.first.c_str());
                     print_ggml_tensor(kv.second, true);
@@ -884,12 +989,17 @@ struct LoraModel : public GGMLRunner {
         /* Don't worry if this message shows up twice in the logs per LoRA,
          * this function is called once to calculate the required buffer size
          * and then again to actually generate a graph to be used */
-        if (!at_runntime && applied_lora_tensors_count != total_lora_tensors_count) {
+        size_t compatible_lora_tensors_count = total_lora_tensors_count - skipped_lora_tensors_count;
+        if (!at_runntime && applied_lora_tensors_count != compatible_lora_tensors_count) {
             LOG_WARN("Only (%lu / %lu) LoRA tensors have been applied, lora_file_path = %s",
-                     applied_lora_tensors_count, total_lora_tensors_count, file_path.c_str());
+                     applied_lora_tensors_count, compatible_lora_tensors_count, file_path.c_str());
         } else {
             LOG_INFO("(%lu / %lu) LoRA tensors have been applied, lora_file_path = %s",
-                     applied_lora_tensors_count, total_lora_tensors_count, file_path.c_str());
+                     applied_lora_tensors_count, compatible_lora_tensors_count, file_path.c_str());
+        }
+        if (skipped_lora_tensors_count > 0) {
+            LOG_WARN("(%lu / %lu) incompatible LoRA tensors have been skipped, lora_file_path = %s",
+                     skipped_lora_tensors_count, total_lora_tensors_count, file_path.c_str());
         }
     }
 };
@@ -953,7 +1063,7 @@ public:
                                    forward_params.conv2d.scale);
         }
         for (auto& lora_model : lora_models) {
-            ggml_tensor* out_diff = lora_model->get_out_diff(ctx, backend, x, forward_params, prefix + "weight");
+            ggml_tensor* out_diff = lora_model->get_out_diff(ctx, backend, x, w, forward_params, prefix + "weight");
             if (out_diff == nullptr) {
                 continue;
             }
@@ -962,6 +1072,34 @@ public:
         return out;
     }
 
+    ggml_tensor* add_lora_to_output(ggml_context* ctx,
+                                    ggml_backend_t backend,
+                                    ggml_tensor* x,
+                                    ggml_tensor* w,
+                                    ggml_tensor* output,
+                                    const std::string& prefix,
+                                    WeightAdapter::ForwardParams forward_params) override {
+        for (auto& lora_model : lora_models) {
+            ggml_tensor* weight_diff = lora_model->get_weight_diff(prefix + "weight", backend, ctx, w, false);
+            if (weight_diff != nullptr) {
+                GGML_ASSERT(forward_params.op_type == ForwardParams::op_type_t::OP_LINEAR);
+                ggml_tensor* out_diff = ggml_ext_linear(ctx,
+                                                        x,
+                                                        weight_diff,
+                                                        nullptr,
+                                                        forward_params.linear.force_prec_f32,
+                                                        forward_params.linear.scale);
+                output                = ggml_add_inplace(ctx, output, out_diff);
+            }
+
+            ggml_tensor* out_diff = lora_model->get_out_diff(ctx, backend, x, w, forward_params, prefix + "weight");
+            if (out_diff != nullptr) {
+                output = ggml_add_inplace(ctx, output, out_diff);
+            }
+        }
+        return output;
+    }
+
     size_t get_extra_graph_size() override {
         size_t lora_tensor_num = 0;
         for (auto& lora_model : lora_models) {

src/model/common/block.hpp

@@ -268,10 +268,11 @@ public:
                 int64_t dim_out,
                 int64_t mult          = 4,
                 Activation activation = Activation::GEGLU,
-                bool precision_fix    = false) {
+                bool precision_fix    = false,
+                bool bias             = true) {
         int64_t inner_dim = dim * mult;
         if (activation == Activation::GELU) {
-            blocks["net.0"] = std::shared_ptr<GGMLBlock>(new GELU(dim, inner_dim));
+            blocks["net.0"] = std::shared_ptr<GGMLBlock>(new GELU(dim, inner_dim, bias));
         } else {
             blocks["net.0"] = std::shared_ptr<GGMLBlock>(new GEGLU(dim, inner_dim));
         }
@@ -285,7 +286,7 @@ public:
         // The purpose of the scale here is to prevent NaN issues in certain situations.
         // For example, when using Vulkan without enabling force_prec_f32,
         // or when using CUDA but the weights are k-quants.
-        blocks["net.2"] = std::shared_ptr<GGMLBlock>(new Linear(inner_dim, dim_out, true, false, force_prec_f32, scale));
+        blocks["net.2"] = std::shared_ptr<GGMLBlock>(new Linear(inner_dim, dim_out, bias, false, force_prec_f32, scale));
     }
 
     ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {

src/model/diffusion/ideogram4.hpp

@@ -142,7 +142,7 @@ namespace Ideogram4 {
     __STATIC_INLINE__ std::shared_ptr<Linear> make_linear(int64_t in_features,
                                                           int64_t out_features,
                                                           bool bias = true) {
-        return std::make_shared<Linear>(in_features, out_features, bias, false, false, 1.f, true);
+        return std::make_shared<Linear>(in_features, out_features, bias);
     }
 
     __STATIC_INLINE__ std::vector<float> gen_ideogram4_pe(int grid_h,

src/model/diffusion/krea2.hpp

@@ -180,9 +180,12 @@ namespace Krea2 {
 
         ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
             ggml_tensor* scale = params["scale"];
-            scale              = ggml_add(ctx->ggml_ctx, scale, ggml_ext_ones(ctx->ggml_ctx, scale->ne[0], 1, 1, 1));
-            x                  = ggml_rms_norm(ctx->ggml_ctx, x, eps);
-            x                  = ggml_mul_inplace(ctx->ggml_ctx, x, scale);
+            if (ctx->weight_adapter) {
+                scale = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, scale, prefix + "scale.weight");
+            }
+            scale = ggml_add(ctx->ggml_ctx, scale, ggml_ext_ones(ctx->ggml_ctx, scale->ne[0], 1, 1, 1));
+            x     = ggml_rms_norm(ctx->ggml_ctx, x, eps);
+            x     = ggml_mul_inplace(ctx->ggml_ctx, x, scale);
             return x;
         }
     };
@@ -295,10 +298,11 @@ namespace Krea2 {
     class KreaDoubleSharedModulation : public GGMLBlock {
     protected:
         int64_t dim;
+        std::string prefix;
 
         void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
             GGML_UNUSED(tensor_storage_map);
-            GGML_UNUSED(prefix);
+            this->prefix  = prefix;
             params["lin"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, dim * 6);
         }
 
@@ -307,7 +311,11 @@ namespace Krea2 {
             : dim(dim) {}
 
         std::vector<ggml_tensor*> forward(GGMLRunnerContext* ctx, ggml_tensor* vec) {
-            auto lin = ggml_repeat(ctx->ggml_ctx, params["lin"], vec);
+            auto lin = params["lin"];
+            if (ctx->weight_adapter) {
+                lin = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, lin, prefix + "lin.weight");
+            }
+            lin      = ggml_repeat(ctx->ggml_ctx, lin, vec);
             auto out = ggml_add(ctx->ggml_ctx, vec, lin);
             return ggml_ext_chunk(ctx->ggml_ctx, out, 6, 0);
         }
@@ -316,10 +324,11 @@ namespace Krea2 {
     class KreaFinalModulation : public GGMLBlock {
     protected:
         int64_t dim;
+        std::string prefix;
 
         void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
             GGML_UNUSED(tensor_storage_map);
-            GGML_UNUSED(prefix);
+            this->prefix  = prefix;
             params["lin"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, dim, 2);
         }
 
@@ -328,7 +337,11 @@ namespace Krea2 {
             : dim(dim) {}
 
         std::vector<ggml_tensor*> forward(GGMLRunnerContext* ctx, ggml_tensor* vec) {
-            auto out = ggml_add(ctx->ggml_ctx, params["lin"], vec);
+            auto lin = params["lin"];
+            if (ctx->weight_adapter) {
+                lin = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, lin, prefix + "lin.weight");
+            }
+            auto out = ggml_add(ctx->ggml_ctx, lin, vec);
             return ggml_ext_chunk(ctx->ggml_ctx, out, 2, 1);
         }
     };

src/model/diffusion/ltxv.hpp

@@ -129,6 +129,10 @@ namespace LTXV {
         bool self_attention_gated  = false;
         bool cross_attention_gated = false;
 
+        bool ff_bias                         = true;
+        bool audio_ff_bias                   = true;
+        bool use_keyframes_abs_pos_embedding = false;
+
         static std::pair<int64_t, int64_t> infer_attention_layout(int64_t hidden_size,
                                                                   int64_t preferred_heads = -1) {
             if (preferred_heads > 0 && hidden_size % preferred_heads == 0) {
@@ -207,6 +211,19 @@ namespace LTXV {
                 tensor_storage_map.find(prefix + ".transformer_blocks.0.audio_attn2.to_gate_logits.weight") != tensor_storage_map.end()) {
                 config.cross_attention_gated = true;
             }
+            // LTX 2.5 sets ff_bias=false but leaves audio_ff_bias at its default, so the two
+            // branches must be detected separately; older checkpoints ship both sets of biases.
+            if (tensor_storage_map.find(prefix + ".transformer_blocks.0.ff.net.0.proj.bias") == tensor_storage_map.end() &&
+                tensor_storage_map.find(prefix + ".transformer_blocks.0.ff.net.2.bias") == tensor_storage_map.end()) {
+                config.ff_bias = false;
+            }
+            if (tensor_storage_map.find(prefix + ".transformer_blocks.0.audio_ff.net.0.proj.bias") == tensor_storage_map.end() &&
+                tensor_storage_map.find(prefix + ".transformer_blocks.0.audio_ff.net.2.bias") == tensor_storage_map.end()) {
+                config.audio_ff_bias = false;
+            }
+            if (tensor_storage_map.find(prefix + ".keyframes_abs_pos_embedding") != tensor_storage_map.end()) {
+                config.use_keyframes_abs_pos_embedding = true;
+            }
             if (tensor_storage_map.find(prefix + ".caption_projection.linear_1.weight") == tensor_storage_map.end() &&
                 tensor_storage_map.find(prefix + ".caption_projection.linear_2.weight") == tensor_storage_map.end()) {
                 config.use_caption_projection = false;
@@ -874,8 +891,7 @@ namespace LTXV {
                          const String2TensorStorage& tensor_storage_map = {},
                          const std::string prefix                       = "") override {
             if (num_learnable_registers > 0) {
-                ggml_type wtype               = get_type(prefix + "learnable_registers", tensor_storage_map, GGML_TYPE_F32);
-                params["learnable_registers"] = ggml_new_tensor_2d(ctx, wtype, hidden_size, num_learnable_registers);
+                params["learnable_registers"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_size, num_learnable_registers);
             }
         }
 
@@ -1130,7 +1146,9 @@ namespace LTXV {
                                 int64_t a_context_dim,
                                 bool apply_gated_attention,
                                 bool cross_attention_adaln,
-                                bool video_rope_interleaved)
+                                bool video_rope_interleaved,
+                                bool ff_bias       = true,
+                                bool audio_ff_bias = true)
             : v_dim(v_dim),
               a_dim(a_dim),
               cross_attention_adaln(cross_attention_adaln) {
@@ -1140,8 +1158,8 @@ namespace LTXV {
             blocks["audio_attn2"]         = std::make_shared<CrossAttention>(a_dim, a_context_dim, a_heads, ad_head, apply_gated_attention, false);
             blocks["audio_to_video_attn"] = std::make_shared<CrossAttention>(v_dim, a_dim, a_heads, ad_head, apply_gated_attention, false);
             blocks["video_to_audio_attn"] = std::make_shared<CrossAttention>(a_dim, v_dim, a_heads, ad_head, apply_gated_attention, false);
-            blocks["ff"]                  = std::make_shared<FeedForward>(v_dim, v_dim, 4, FeedForward::Activation::GELU);
-            blocks["audio_ff"]            = std::make_shared<FeedForward>(a_dim, a_dim, 4, FeedForward::Activation::GELU);
+            blocks["ff"]                  = std::make_shared<FeedForward>(v_dim, v_dim, 4, FeedForward::Activation::GELU, false, ff_bias);
+            blocks["audio_ff"]            = std::make_shared<FeedForward>(a_dim, a_dim, 4, FeedForward::Activation::GELU, false, audio_ff_bias);
         }
 
         std::vector<ggml_tensor*> get_ada_values(GGMLRunnerContext* ctx,
@@ -1320,6 +1338,12 @@ namespace LTXV {
                                                                    get_type(prefix + "audio_scale_shift_table", tensor_storage_map, GGML_TYPE_F32),
                                                                    config.audio_hidden_size,
                                                                    2);
+            if (config.use_keyframes_abs_pos_embedding) {
+                params["keyframes_abs_pos_embedding"] = ggml_new_tensor_2d(ctx,
+                                                                           get_type(prefix + "keyframes_abs_pos_embedding", tensor_storage_map, GGML_TYPE_F32),
+                                                                           config.hidden_size,
+                                                                           1);
+            }
         }
 
         LTXAVModelBlock(const LTXAVConfig& config)
@@ -1386,7 +1410,9 @@ namespace LTXV {
                                                                                                               config.audio_cross_attention_dim,
                                                                                                               config.self_attention_gated || config.cross_attention_gated,
                                                                                                               config.cross_attention_adaln,
-                                                                                                              config.video_rope_interleaved);
+                                                                                                              config.video_rope_interleaved,
+                                                                                                              config.ff_bias,
+                                                                                                              config.audio_ff_bias);
             }
 
             blocks["norm_out"]       = std::make_shared<LayerNorm>(config.hidden_size, 1e-6f, false);
@@ -1534,6 +1560,38 @@ namespace LTXV {
             return {v_context, a_context};
         }
 
+        // The video encoder is causal, so the first latent frame covers a single pixel frame while
+        // every later one covers temporal_scale_factor. LTX 2.5 marks that token class with a
+        // learned embedding added right after patchify_proj.
+        ggml_tensor* apply_keyframes_abs_pos_embedding(GGMLRunnerContext* ctx,
+                                                       ggml_tensor* vx,
+                                                       int64_t tokens_per_latent_frame) {
+            if (!config.use_keyframes_abs_pos_embedding || params.count("keyframes_abs_pos_embedding") == 0) {
+                return vx;
+            }
+            int64_t tokens = vx->ne[1];
+            if (tokens_per_latent_frame <= 0 || tokens_per_latent_frame > tokens) {
+                return vx;
+            }
+            auto embedding = params["keyframes_abs_pos_embedding"];
+            auto first     = ggml_cont(ctx->ggml_ctx,
+                                       ggml_view_3d(ctx->ggml_ctx, vx, vx->ne[0], tokens_per_latent_frame, vx->ne[2], vx->nb[1], vx->nb[2], 0));
+            first          = ggml_add(ctx->ggml_ctx, first, embedding);
+            if (tokens_per_latent_frame == tokens) {
+                return first;
+            }
+            auto rest = ggml_cont(ctx->ggml_ctx,
+                                  ggml_view_3d(ctx->ggml_ctx,
+                                               vx,
+                                               vx->ne[0],
+                                               tokens - tokens_per_latent_frame,
+                                               vx->ne[2],
+                                               vx->nb[1],
+                                               vx->nb[2],
+                                               tokens_per_latent_frame * vx->nb[1]));
+            return ggml_concat(ctx->ggml_ctx, first, rest, 1);
+        }
+
         std::vector<ggml_tensor*> get_output_scale_shift(GGMLRunnerContext* ctx,
                                                          ggml_tensor* table,
                                                          ggml_tensor* embedded_timestep,
@@ -1575,6 +1633,7 @@ namespace LTXV {
 
             vx = patchify_video(ctx, vx, n);
             vx = patchify_proj->forward(ctx, vx);
+            vx = apply_keyframes_abs_pos_embedding(ctx, vx, width * height);
             if (ax != nullptr && ggml_nelements(ax) > 0 && audio_time > 0) {
                 ax = patchify_audio(ctx, ax);
                 ax = audio_patchify_proj->forward(ctx, ax);

src/model/diffusion/model.hpp

@@ -87,6 +87,31 @@ struct LTXAVDiffusionExtra {
     const sd::Tensor<float>* video_positions = nullptr;
 };
 
+enum class MiniMaxH3ReferenceKind : int32_t {
+    IMAGE,
+    VIDEO,
+    AUDIO,
+    VIDEO_AUDIO,
+};
+
+struct MiniMaxH3ReferenceBlock {
+    MiniMaxH3ReferenceKind kind = MiniMaxH3ReferenceKind::IMAGE;
+    int32_t video_index         = -1;
+    int32_t audio_index         = -1;
+};
+
+struct MiniMaxH3DiffusionExtra {
+    const sd::Tensor<int32_t>* text_token_tags                    = nullptr;
+    const sd::Tensor<int32_t>* keyframe_indices                   = nullptr;
+    const std::vector<sd::Tensor<float>>* reference_audio_latents = nullptr;
+    const std::vector<MiniMaxH3ReferenceBlock>* reference_blocks  = nullptr;
+    int audio_length                                              = 0;
+    float video_sigma_shift                                       = 12.f;
+    float audio_sigma_shift                                       = 3.f;
+    // Negative when the outer sampler is not a single-evaluation Euler step.
+    float next_video_sigma = -1.f;
+};
+
 struct MiniT2IDiffusionExtra {
     const sd::Tensor<float>* mask = nullptr;
 };
@@ -106,6 +131,7 @@ using DiffusionExtraParams = std::variant<std::monostate,
                                           WanDiffusionExtra,
                                           HiDreamO1DiffusionExtra,
                                           LTXAVDiffusionExtra,
+                                          MiniMaxH3DiffusionExtra,
                                           MiniT2IDiffusionExtra,
                                           HunyuanVideoDiffusionExtra>;
 

src/model/te/llm.hpp

@@ -40,6 +40,7 @@ namespace LLM {
         MINISTRAL_3_3B,
         GEMMA3_12B,
         GEMMA2_2B,
+        GEMMA4_12B,
         GPT_OSS_20B,
         ARCH_COUNT,
     };
@@ -52,6 +53,7 @@ namespace LLM {
         "ministral3.3b",
         "gemma3_12b",
         "gemma2_2b",
+        "gemma4_12b",
         "gpt_oss_20b",
     };
 
@@ -79,9 +81,20 @@ namespace LLM {
         int window_size                     = 112;
         int num_position_embeddings         = 0;
         std::set<int> fullatt_block_indexes = {7, 15, 23, 31};
-        bool split_patch_embed              = false;
+        std::vector<int> deepstack_visual_indexes;
+        bool split_patch_embed = false;
     };
 
+    struct ImageGrid {
+        int index  = 0;
+        int size   = 0;
+        int grid_h = 0;
+        int grid_w = 0;
+    };
+
+    using ImageEmbeds          = std::vector<std::pair<int, sd::Tensor<float>>>;
+    using DeepStackImageEmbeds = std::vector<ImageEmbeds>;
+
     struct LLMConfig {
         LLMArch arch                    = LLMArch::QWEN2_5_VL;
         int64_t num_layers              = 28;
@@ -93,6 +106,7 @@ namespace LLM {
         bool qkv_bias                   = true;
         bool attention_out_bias         = false;
         bool qk_norm                    = false;
+        bool final_norm                 = true;
         bool rms_norm_add               = false;
         bool normalize_input            = false;
         int64_t vocab_size              = 152064;
@@ -108,6 +122,15 @@ namespace LLM {
         bool have_vision_weight = false;
         bool llama_cpp_style    = false;
 
+        // gemma4 config
+        int global_head_dim         = 0;
+        int num_global_kv_heads     = 0;
+        float global_partial_rotary = 1.f;
+        bool global_k_eq_v          = false;
+        bool v_norm                 = false;
+        bool layer_scalar           = false;
+        bool unscaled_attention     = false;
+
         static LLMConfig detect_from_weights(const String2TensorStorage& tensor_storage_map,
                                              const std::string& prefix,
                                              LLMArch arch) {
@@ -145,6 +168,27 @@ namespace LLM {
                 config.rope_thetas             = {1000000.f, 10000.f};
                 config.rope_scales             = {8.f, 1.f};
                 config.sliding_attention       = {1024, 1024, 1024, 1024, 1024, 0};
+            } else if (arch == LLMArch::GEMMA4_12B) {
+                config.head_dim                = 256;
+                config.num_heads               = 16;
+                config.num_kv_heads            = 8;
+                config.global_head_dim         = 512;
+                config.num_global_kv_heads     = 1;
+                config.global_partial_rotary   = 0.25f;
+                config.global_k_eq_v           = true;
+                config.v_norm                  = true;
+                config.layer_scalar            = true;
+                config.unscaled_attention      = true;
+                config.qkv_bias                = false;
+                config.qk_norm                 = true;
+                config.rms_norm_eps            = 1e-6f;
+                config.rms_norm_add            = false;
+                config.normalize_input         = true;
+                config.max_position_embeddings = 262144;
+                config.mlp_activation          = MLPActivation::GELU_TANH;
+                config.rope_thetas             = {1000000.f, 10000.f};
+                config.rope_scales             = {1.f, 1.f};
+                config.sliding_attention       = {1024, 1024, 1024, 1024, 1024, 0};
             } else if (arch == LLMArch::GEMMA2_2B) {
                 config.head_dim                = 256;
                 config.num_heads               = 8;
@@ -220,12 +264,12 @@ namespace LLM {
                             }
                         }
                     }
-                    if (contains(name, "visual.blocks.0.mlp.linear_fc1.weight") ||
-                        contains(name, "visual.blocks.0.mlp.gate_proj.weight")) {
+                    if (ends_with(name, "visual.blocks.0.mlp.linear_fc1.weight") ||
+                        ends_with(name, "visual.blocks.0.mlp.gate_proj.weight")) {
                         config.vision.intermediate_size = tensor_storage.ne[1];
                     }
-                    if (contains(name, "visual.merger.linear_fc2.weight") ||
-                        contains(name, "visual.merger.mlp.2.weight")) {
+                    if (ends_with(name, "visual.merger.linear_fc2.weight") ||
+                        ends_with(name, "visual.merger.mlp.2.weight")) {
                         config.vision.out_hidden_size = tensor_storage.ne[1];
                     }
                     continue;
@@ -244,22 +288,37 @@ namespace LLM {
                     config.hidden_size = tensor_storage.ne[0];
                     config.vocab_size  = tensor_storage.ne[1];
                 }
-                if (contains(name, "layers.0.mlp.gate_proj.weight")) {
+                if (ends_with(name, "layers.0.mlp.gate_proj.weight")) {
                     config.intermediate_size = tensor_storage.ne[1];
                 }
-                if (contains(name, "layers.0.mlp.experts.gate_up_proj.weight")) {
+                if (ends_with(name, "layers.0.mlp.experts.gate_up_proj.weight")) {
                     config.intermediate_size = tensor_storage.ne[1] / 2;
                 }
-                if (contains(name, "layers.0.mlp.experts.gate_proj.weight")) {
+                if (ends_with(name, "layers.0.mlp.experts.gate_proj.weight")) {
                     config.intermediate_size = tensor_storage.ne[1];
                 }
             }
             if ((arch == LLMArch::QWEN3 || arch == LLMArch::QWEN3_VL) && config.num_layers == 28) {
                 config.num_heads = 16;
             }
+            if (arch == LLMArch::QWEN3_VL &&
+                (config.num_layers == 50 || config.num_layers == 64) &&
+                config.hidden_size == 5120) {
+                config.num_heads = 64;
+                if (config.num_layers == 50) {
+                    config.final_norm = false;
+                }
+            }
             if (detected_vision_layers > 0) {
                 config.vision.num_layers = detected_vision_layers;
             }
+            if (arch == LLMArch::QWEN3_VL) {
+                if (config.vision.num_layers == 24) {
+                    config.vision.deepstack_visual_indexes = {5, 11, 17};
+                } else if (config.vision.num_layers == 27) {
+                    config.vision.deepstack_visual_indexes = {8, 16, 24};
+                }
+            }
             LOG_DEBUG("llm: num_layers = %" PRId64 ", vocab_size = %" PRId64 ", hidden_size = %" PRId64 ", intermediate_size = %" PRId64,
                       config.num_layers,
                       config.vocab_size,
@@ -541,6 +600,37 @@ namespace LLM {
         return input_embed;
     }
 
+    static ggml_tensor* add_deepstack_image_embeds(GGMLRunnerContext* ctx,
+                                                   ggml_tensor* x,
+                                                   const std::vector<std::pair<int, ggml_tensor*>>& image_embeds) {
+        if (image_embeds.empty()) {
+            return x;
+        }
+
+        GGML_ASSERT(x->ne[2] == 1);
+        auto raw_x          = ggml_cast(ctx->ggml_ctx, x, image_embeds[0].second->type);
+        int64_t token_start = 0;
+        ggml_tensor* output = nullptr;
+        for (const auto& [index, image_embed] : image_embeds) {
+            GGML_ASSERT(index >= token_start);
+            GGML_ASSERT(index + image_embed->ne[1] <= raw_x->ne[1]);
+            if (index > token_start) {
+                auto text_embed = ggml_ext_slice(ctx->ggml_ctx, raw_x, 1, token_start, index);
+                output          = output == nullptr ? text_embed : ggml_concat(ctx->ggml_ctx, output, text_embed, 1);
+            }
+            auto visual_embed = ggml_ext_slice(ctx->ggml_ctx, raw_x, 1, index, index + image_embed->ne[1]);
+            visual_embed      = ggml_add(ctx->ggml_ctx, visual_embed, image_embed);
+            output            = output == nullptr ? visual_embed : ggml_concat(ctx->ggml_ctx, output, visual_embed, 1);
+            token_start       = index + image_embed->ne[1];
+        }
+        if (token_start < raw_x->ne[1]) {
+            auto text_embed = ggml_ext_slice(ctx->ggml_ctx, raw_x, 1, token_start, raw_x->ne[1]);
+            output          = output == nullptr ? text_embed : ggml_concat(ctx->ggml_ctx, output, text_embed, 1);
+        }
+        GGML_ASSERT(output != nullptr && output->ne[1] == raw_x->ne[1]);
+        return output;
+    }
+
     struct VisionMLP : public GGMLBlock {
     protected:
         LLMVisionArch arch_;
@@ -723,6 +813,33 @@ namespace LLM {
         }
     };
 
+    struct Qwen3VLDeepStackMerger : public GGMLBlock {
+    protected:
+        int64_t merge_dim;
+
+    public:
+        Qwen3VLDeepStackMerger(int64_t dim,
+                               int64_t context_dim,
+                               int64_t spatial_merge_size)
+            : merge_dim(context_dim * spatial_merge_size * spatial_merge_size) {
+            blocks["norm"]       = std::make_shared<LayerNorm>(merge_dim, 1e-6f);
+            blocks["linear_fc1"] = std::make_shared<Linear>(merge_dim, merge_dim, true);
+            blocks["linear_fc2"] = std::make_shared<Linear>(merge_dim, dim, true);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            auto norm       = std::dynamic_pointer_cast<LayerNorm>(blocks["norm"]);
+            auto linear_fc1 = std::dynamic_pointer_cast<Linear>(blocks["linear_fc1"]);
+            auto linear_fc2 = std::dynamic_pointer_cast<Linear>(blocks["linear_fc2"]);
+
+            x = ggml_reshape_2d(ctx->ggml_ctx, x, merge_dim, ggml_nelements(x) / merge_dim);
+            x = norm->forward(ctx, x);
+            x = linear_fc1->forward(ctx, x);
+            x = ggml_gelu_erf(ctx->ggml_ctx, x);
+            return linear_fc2->forward(ctx, x);
+        }
+    };
+
     struct VisionAttention : public GGMLBlock {
     protected:
         bool llama_cpp_style;
@@ -844,6 +961,7 @@ namespace LLM {
         int spatial_merge_size;
         int num_grid_per_side;
         std::set<int> fullatt_block_indexes;
+        std::vector<int> deepstack_visual_indexes;
 
     public:
         VisionModel(bool llama_cpp_style,
@@ -853,7 +971,8 @@ namespace LLM {
               num_layers(vision_params.num_layers),
               spatial_merge_size(vision_params.spatial_merge_size),
               num_grid_per_side(vision_params.num_position_embeddings > 0 ? static_cast<int>(std::sqrt(vision_params.num_position_embeddings)) : 0),
-              fullatt_block_indexes(vision_params.fullatt_block_indexes) {
+              fullatt_block_indexes(vision_params.fullatt_block_indexes),
+              deepstack_visual_indexes(vision_params.deepstack_visual_indexes) {
             blocks["patch_embed"] = std::shared_ptr<GGMLBlock>(new VisionPatchEmbed(vision_params.split_patch_embed,
                                                                                     arch_,
                                                                                     vision_params.patch_size,
@@ -875,6 +994,11 @@ namespace LLM {
                                                                                 vision_params.out_hidden_size,
                                                                                 vision_params.hidden_size,
                                                                                 spatial_merge_size));
+            for (size_t i = 0; i < deepstack_visual_indexes.size(); ++i) {
+                blocks["deepstack_merger_list." + std::to_string(i)] = std::make_shared<Qwen3VLDeepStackMerger>(vision_params.out_hidden_size,
+                                                                                                                vision_params.hidden_size,
+                                                                                                                spatial_merge_size);
+            }
         }
 
         std::shared_ptr<Embedding> pos_embedder() {
@@ -893,13 +1017,13 @@ namespace LLM {
             return spatial_merge_size;
         }
 
-        ggml_tensor* forward(GGMLRunnerContext* ctx,
-                             ggml_tensor* pixel_values,
-                             ggml_tensor* pe,
-                             ggml_tensor* window_index,
-                             ggml_tensor* window_inverse_index,
-                             ggml_tensor* window_mask,
-                             ggml_tensor* pos_embeds = nullptr) {
+        std::vector<ggml_tensor*> forward_outputs(GGMLRunnerContext* ctx,
+                                                  ggml_tensor* pixel_values,
+                                                  ggml_tensor* pe,
+                                                  ggml_tensor* window_index,
+                                                  ggml_tensor* window_inverse_index,
+                                                  ggml_tensor* window_mask,
+                                                  ggml_tensor* pos_embeds = nullptr) {
             // pixel_values: [grid_t*(H/mh/ph)*(W/mw/pw)*mh*mw, C*pt*ph*pw]
             // window_index: [grid_t*(H/mh/ph)*(W/mw/pw)]
             // window_inverse_index: [grid_t*(H/mh/ph)*(W/mw/pw)]
@@ -919,6 +1043,7 @@ namespace LLM {
                 x = ggml_reshape_4d(ctx->ggml_ctx, x, x->ne[0] / spatial_merge_size / spatial_merge_size, x->ne[1] * spatial_merge_size * spatial_merge_size, x->ne[2], x->ne[3]);
             }
 
+            std::vector<ggml_tensor*> deepstack_outputs;
             for (int i = 0; i < num_layers; i++) {
                 auto block = std::dynamic_pointer_cast<VisionBlock>(blocks["blocks." + std::to_string(i)]);
 
@@ -926,8 +1051,12 @@ namespace LLM {
                 if (fullatt_block_indexes.find(i) != fullatt_block_indexes.end()) {
                     mask = nullptr;
                 }
-                x = block->forward(ctx, x, pe, mask);
-                if (i == 0) {
+                x                 = block->forward(ctx, x, pe, mask);
+                auto deepstack_it = std::find(deepstack_visual_indexes.begin(), deepstack_visual_indexes.end(), i);
+                if (deepstack_it != deepstack_visual_indexes.end()) {
+                    size_t deepstack_index = static_cast<size_t>(std::distance(deepstack_visual_indexes.begin(), deepstack_it));
+                    auto deepstack_merger  = std::dynamic_pointer_cast<Qwen3VLDeepStackMerger>(blocks["deepstack_merger_list." + std::to_string(deepstack_index)]);
+                    deepstack_outputs.push_back(deepstack_merger->forward(ctx, x));
                 }
                 sd::ggml_graph_cut::mark_graph_cut(x, "llm.vision.blocks." + std::to_string(i), "x");
             }
@@ -939,7 +1068,19 @@ namespace LLM {
                 x = ggml_get_rows(ctx->ggml_ctx, x, window_inverse_index);
             }
 
-            return x;
+            std::vector<ggml_tensor*> outputs = {x};
+            outputs.insert(outputs.end(), deepstack_outputs.begin(), deepstack_outputs.end());
+            return outputs;
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx,
+                             ggml_tensor* pixel_values,
+                             ggml_tensor* pe,
+                             ggml_tensor* window_index,
+                             ggml_tensor* window_inverse_index,
+                             ggml_tensor* window_mask,
+                             ggml_tensor* pos_embeds = nullptr) {
+            return forward_outputs(ctx, pixel_values, pe, window_index, window_inverse_index, window_mask, pos_embeds)[0];
         }
     };
 
@@ -954,6 +1095,11 @@ namespace LLM {
         std::vector<float> rope_thetas;
         std::vector<float> rope_scales;
         bool has_attention_sinks;
+        bool k_eq_v;
+        bool v_norm;
+        bool unscaled_attention;
+        float rms_norm_eps;
+        int rope_pairs;
 
         void init_params(ggml_context* ctx,
                          const String2TensorStorage& tensor_storage_map = {},
@@ -964,24 +1110,48 @@ namespace LLM {
         }
 
     public:
-        Attention(const LLMConfig& config)
+        Attention(const LLMConfig& config, bool global_layer = false)
             : arch(config.arch),
               num_heads(config.num_heads),
-              num_kv_heads(config.num_kv_heads),
-              head_dim(config.head_dim),
+              num_kv_heads(global_layer && config.num_global_kv_heads > 0 ? config.num_global_kv_heads : config.num_kv_heads),
+              head_dim(global_layer && config.global_head_dim > 0 ? config.global_head_dim : config.head_dim),
               qk_norm(config.qk_norm),
               max_position_embeddings(config.max_position_embeddings),
               rope_thetas(config.rope_thetas),
               rope_scales(config.rope_scales),
-              has_attention_sinks(config.arch == LLMArch::GPT_OSS_20B) {
+              has_attention_sinks(config.arch == LLMArch::GPT_OSS_20B),
+              k_eq_v(global_layer && config.global_k_eq_v),
+              v_norm(config.v_norm),
+              unscaled_attention(config.unscaled_attention),
+              rms_norm_eps(config.rms_norm_eps),
+              rope_pairs(0) {
             blocks["q_proj"] = std::make_shared<Linear>(config.hidden_size, num_heads * head_dim, config.qkv_bias);
             blocks["k_proj"] = std::make_shared<Linear>(config.hidden_size, num_kv_heads * head_dim, config.qkv_bias);
-            blocks["v_proj"] = std::make_shared<Linear>(config.hidden_size, num_kv_heads * head_dim, config.qkv_bias);
+            if (!k_eq_v) {
+                blocks["v_proj"] = std::make_shared<Linear>(config.hidden_size, num_kv_heads * head_dim, config.qkv_bias);
+            }
             blocks["o_proj"] = std::make_shared<Linear>(num_heads * head_dim, config.hidden_size, config.attention_out_bias);
             if (config.qk_norm) {
                 blocks["q_norm"] = std::make_shared<LLMRMSNorm>(head_dim, config.rms_norm_eps, config.rms_norm_add);
                 blocks["k_norm"] = std::make_shared<LLMRMSNorm>(head_dim, config.rms_norm_eps, config.rms_norm_add);
             }
+            // Proportional RoPE rotates only the leading `rope_pairs` dimension pairs of the head;
+            // the rest are left unrotated through freq_factors (see rope_freq_factors()).
+            float partial = global_layer ? config.global_partial_rotary : 1.f;
+            rope_pairs    = static_cast<int>(partial * head_dim / 2.f);
+        }
+
+        // ggml applies theta_i / freq_factors[i], so a huge factor collapses the angle to zero and
+        // leaves that pair unrotated. This reproduces transformers' "proportional" RoPE, whose
+        // inv_freq is zero-padded past `rope_pairs`, without reordering the head.
+        ggml_tensor* rope_freq_factors(ggml_context* ctx) const {
+            int pairs = head_dim / 2;
+            if (rope_pairs >= pairs) {
+                return nullptr;
+            }
+            auto rotated   = ggml_ext_ones(ctx, rope_pairs, 1, 1, 1);
+            auto unrotated = ggml_ext_full(ctx, 1e30f, pairs - rope_pairs, 1, 1, 1);
+            return ggml_concat(ctx, rotated, unrotated, 0);
         }
 
         ggml_tensor* forward(GGMLRunnerContext* ctx,
@@ -994,12 +1164,12 @@ namespace LLM {
             int64_t N       = x->ne[2];
             auto q_proj     = std::dynamic_pointer_cast<Linear>(blocks["q_proj"]);
             auto k_proj     = std::dynamic_pointer_cast<Linear>(blocks["k_proj"]);
-            auto v_proj     = std::dynamic_pointer_cast<Linear>(blocks["v_proj"]);
+            auto v_proj     = k_eq_v ? nullptr : std::dynamic_pointer_cast<Linear>(blocks["v_proj"]);
             auto out_proj   = std::dynamic_pointer_cast<Linear>(blocks["o_proj"]);
 
-            auto q = q_proj->forward(ctx, x);  // [N, n_token, num_heads*head_dim]
-            auto k = k_proj->forward(ctx, x);  // [N, n_token, num_kv_heads*head_dim]
-            auto v = v_proj->forward(ctx, x);  // [N, n_token, num_kv_heads*head_dim]
+            auto q = q_proj->forward(ctx, x);               // [N, n_token, num_heads*head_dim]
+            auto k = k_proj->forward(ctx, x);               // [N, n_token, num_kv_heads*head_dim]
+            auto v = k_eq_v ? k : v_proj->forward(ctx, x);  // [N, n_token, num_kv_heads*head_dim]
 
             q = ggml_reshape_4d(ctx->ggml_ctx, q, head_dim, num_heads, n_token, N);     // [N, n_token, num_heads, head_dim]
             k = ggml_reshape_4d(ctx->ggml_ctx, k, head_dim, num_kv_heads, n_token, N);  // [N, n_token, num_kv_heads, head_dim]
@@ -1012,6 +1182,10 @@ namespace LLM {
                 q = q_norm->forward(ctx, q);
                 k = k_norm->forward(ctx, k);
             }
+            if (v_norm) {
+                // Gemma 4 normalizes V with a weightless RMS norm, and never rotates it.
+                v = ggml_rms_norm(ctx->ggml_ctx, v, rms_norm_eps);
+            }
 
             if (arch == LLMArch::MISTRAL_SMALL_3_2) {
                 q = ggml_rope_ext(ctx->ggml_ctx, q, input_pos, nullptr, 128, GGML_ROPE_TYPE_NORMAL, 8192, 1000000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
@@ -1082,6 +1256,35 @@ namespace LLM {
                                                  1.f,
                                                  32.f,
                                                  1.f);
+            } else if (arch == LLMArch::GEMMA4_12B) {
+                float rope_theta  = (rope_index == 1 ? 10000.0f : 1000000.0f);
+                auto freq_factors = rope_freq_factors(ctx->ggml_ctx);
+                q                 = ggml_rope_ext(ctx->ggml_ctx,
+                                                  q,
+                                                  input_pos,
+                                                  freq_factors,
+                                                  head_dim,
+                                                  GGML_ROPE_TYPE_NEOX,
+                                                  static_cast<int>(max_position_embeddings),
+                                                  rope_theta,
+                                                  1.f,
+                                                  0.f,
+                                                  1.f,
+                                                  32.f,
+                                                  1.f);
+                k                 = ggml_rope_ext(ctx->ggml_ctx,
+                                                  k,
+                                                  input_pos,
+                                                  freq_factors,
+                                                  head_dim,
+                                                  GGML_ROPE_TYPE_NEOX,
+                                                  static_cast<int>(max_position_embeddings),
+                                                  rope_theta,
+                                                  1.f,
+                                                  0.f,
+                                                  1.f,
+                                                  32.f,
+                                                  1.f);
             } else if (arch == LLMArch::GEMMA2_2B) {
                 q = ggml_rope_ext(ctx->ggml_ctx,
                                   q,
@@ -1119,6 +1322,11 @@ namespace LLM {
                 k               = ggml_rope_multi(ctx->ggml_ctx, k, input_pos, nullptr, head_dim, sections, GGML_ROPE_TYPE_MROPE, 128000, 1000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
             }
 
+            if (unscaled_attention) {
+                // Gemma 4 attends with scaling=1.0; undo the helper's own 1/sqrt(head_dim).
+                q = ggml_ext_scale(ctx->ggml_ctx, q, std::sqrt(static_cast<float>(head_dim)));
+            }
+
             q = ggml_cont(ctx->ggml_ctx, ggml_ext_torch_permute(ctx->ggml_ctx, q, 0, 2, 1, 3));  // [N, num_heads, n_token, head_dim]
             q = ggml_reshape_3d(ctx->ggml_ctx, q, q->ne[0], q->ne[1], q->ne[2] * q->ne[3]);      // [N*num_heads, n_token, head_dim]
 
@@ -1157,15 +1365,30 @@ namespace LLM {
     protected:
         LLMArch arch;
         int sliding_attention;
+        bool has_layer_scalar;
         std::string post_attention_norm_name;
         std::string pre_ffw_norm_name;
         std::string post_ffw_norm_name;
 
+        void init_params(ggml_context* ctx,
+                         const String2TensorStorage& tensor_storage_map = {},
+                         std::string prefix                             = "") override {
+            GGMLBlock::init_params(ctx, tensor_storage_map, prefix);
+            if (has_layer_scalar) {
+                params["layer_scalar"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
+            }
+        }
+
     public:
         TransformerBlock(const LLMConfig& config, int layer_index)
             : arch(config.arch),
-              sliding_attention(0) {
-            if (config.arch == LLMArch::GEMMA3_12B) {
+              sliding_attention(0),
+              has_layer_scalar(config.layer_scalar) {
+            if (config.arch == LLMArch::GEMMA4_12B) {
+                post_attention_norm_name = "post_attention_layernorm";
+                pre_ffw_norm_name        = "pre_feedforward_layernorm";
+                post_ffw_norm_name       = "post_feedforward_layernorm";
+            } else if (config.arch == LLMArch::GEMMA3_12B || config.arch == LLMArch::GEMMA4_12B) {
                 post_attention_norm_name = "post_attention_norm";       // attn_post_norm
                 pre_ffw_norm_name        = "post_attention_layernorm";  // ffn_norm
                 post_ffw_norm_name       = "post_ffw_norm";             // ffn_post_norm
@@ -1179,7 +1402,10 @@ namespace LLM {
                 pre_ffw_norm_name = "post_attention_layernorm";  // ffn_norm
             }
 
-            blocks["self_attn"] = std::make_shared<Attention>(config);
+            if (!config.sliding_attention.empty()) {
+                sliding_attention = config.sliding_attention[layer_index % config.sliding_attention.size()];
+            }
+            blocks["self_attn"] = std::make_shared<Attention>(config, sliding_attention == 0);
             if (config.arch == LLMArch::GPT_OSS_20B) {
                 blocks["mlp"] = std::make_shared<GPTOSSMLP>(config);
             } else {
@@ -1196,9 +1422,6 @@ namespace LLM {
             if (!post_ffw_norm_name.empty()) {
                 blocks[post_ffw_norm_name] = std::make_shared<LLMRMSNorm>(config.hidden_size, config.rms_norm_eps, config.rms_norm_add);
             }
-            if (!config.sliding_attention.empty()) {
-                sliding_attention = config.sliding_attention[layer_index % config.sliding_attention.size()];
-            }
         }
 
         ggml_tensor* forward(GGMLRunnerContext* ctx,
@@ -1220,7 +1443,7 @@ namespace LLM {
             }
             ggml_tensor* block_attention_mask = attention_mask;
             int rope_index                    = 0;
-            if ((arch == LLMArch::GEMMA3_12B || arch == LLMArch::GPT_OSS_20B) && sliding_attention > 0) {
+            if ((arch == LLMArch::GEMMA3_12B || arch == LLMArch::GEMMA4_12B || arch == LLMArch::GPT_OSS_20B) && sliding_attention > 0) {
                 block_attention_mask = sliding_attention_mask;
                 rope_index           = 1;
             }
@@ -1247,6 +1470,10 @@ namespace LLM {
             }
             x = ggml_add_inplace(ctx->ggml_ctx, x, residual);
 
+            if (has_layer_scalar) {
+                x = ggml_mul(ctx->ggml_ctx, x, params["layer_scalar"]);
+            }
+
             return x;
         }
     };
@@ -1263,7 +1490,9 @@ namespace LLM {
             for (int i = 0; i < num_layers; i++) {
                 blocks["layers." + std::to_string(i)] = std::shared_ptr<GGMLBlock>(new TransformerBlock(config, i));
             }
-            blocks["norm"] = std::shared_ptr<GGMLBlock>(new LLMRMSNorm(config.hidden_size, config.rms_norm_eps, config.rms_norm_add));
+            if (config.final_norm) {
+                blocks["norm"] = std::shared_ptr<GGMLBlock>(new LLMRMSNorm(config.hidden_size, config.rms_norm_eps, config.rms_norm_add));
+            }
         }
 
         ggml_tensor* embed(GGMLRunnerContext* ctx,
@@ -1278,9 +1507,11 @@ namespace LLM {
                                     ggml_tensor* input_pos,
                                     ggml_tensor* attention_mask,
                                     std::set<int> out_layers,
-                                    ggml_tensor* sliding_attention_mask = nullptr,
-                                    bool return_all_hidden_states       = false) {
-            auto norm = std::dynamic_pointer_cast<LLMRMSNorm>(blocks["norm"]);
+                                    const std::vector<std::vector<std::pair<int, ggml_tensor*>>>& deepstack_image_embeds = {},
+                                    ggml_tensor* sliding_attention_mask                                                  = nullptr,
+                                    bool return_all_hidden_states                                                        = false) {
+            auto norm = config.final_norm ? std::dynamic_pointer_cast<LLMRMSNorm>(blocks["norm"])
+                                          : nullptr;
             std::vector<ggml_tensor*> intermediate_outputs;
 
             if (config.normalize_input) {
@@ -1295,6 +1526,9 @@ namespace LLM {
                 auto block = std::dynamic_pointer_cast<TransformerBlock>(blocks["layers." + std::to_string(i)]);
 
                 x = block->forward(ctx, x, input_pos, attention_mask, sliding_attention_mask);
+                if (i < static_cast<int>(deepstack_image_embeds.size())) {
+                    x = add_deepstack_image_embeds(ctx, x, deepstack_image_embeds[static_cast<size_t>(i)]);
+                }
                 if (return_all_hidden_states || out_layers.size() > 1) {
                     x = ggml_cont(ctx->ggml_ctx, x);
                 }
@@ -1308,7 +1542,7 @@ namespace LLM {
                 }
             }
 
-            auto normed_x = norm->forward(ctx, x);
+            auto normed_x = norm == nullptr ? x : norm->forward(ctx, x);
             if (return_all_hidden_states) {
                 intermediate_outputs.push_back(normed_x);
                 x = intermediate_outputs[0];
@@ -1336,6 +1570,7 @@ namespace LLM {
                              ggml_tensor* attention_mask,
                              ggml_tensor* sliding_attention_mask,
                              std::vector<std::pair<int, ggml_tensor*>> image_embeds,
+                             const std::vector<std::vector<std::pair<int, ggml_tensor*>>>& deepstack_image_embeds,
                              std::set<int> out_layers,
                              bool return_all_hidden_states = false) {
             // input_ids: [N, n_token]
@@ -1347,6 +1582,7 @@ namespace LLM {
                                   input_pos,
                                   attention_mask,
                                   std::move(out_layers),
+                                  deepstack_image_embeds,
                                   sliding_attention_mask,
                                   return_all_hidden_states);
         }
@@ -1372,6 +1608,7 @@ namespace LLM {
                              ggml_tensor* attention_mask,
                              ggml_tensor* sliding_attention_mask,
                              std::vector<std::pair<int, ggml_tensor*>> image_embeds,
+                             const std::vector<std::vector<std::pair<int, ggml_tensor*>>>& deepstack_image_embeds,
                              std::set<int> out_layers,
                              bool return_all_hidden_states = false) {
             // input_ids: [N, n_token]
@@ -1383,6 +1620,7 @@ namespace LLM {
                                     attention_mask,
                                     sliding_attention_mask,
                                     image_embeds,
+                                    deepstack_image_embeds,
                                     out_layers,
                                     return_all_hidden_states);
             return x;
@@ -1524,7 +1762,8 @@ namespace LLM {
                                                 std::vector<float>& window_mask_vec,
                                                 std::vector<float>& pe_vec,
                                                 std::array<std::vector<int32_t>, 4>& pos_embed_idx_data,
-                                                std::array<std::vector<float>, 4>& pos_embed_weight_data) {
+                                                std::array<std::vector<float>, 4>& pos_embed_weight_data,
+                                                std::vector<ggml_tensor*>* output_tensors = nullptr) {
             GGML_ASSERT(image->ne[1] % (vision_params.patch_size * vision_params.spatial_merge_size) == 0);
             GGML_ASSERT(image->ne[0] % (vision_params.patch_size * vision_params.spatial_merge_size) == 0);
 
@@ -1556,7 +1795,11 @@ namespace LLM {
                 int pos_len = static_cast<int>(pe_vec.size() / head_dim / 2);
                 auto pe     = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, head_dim / 2, pos_len);
                 runner->set_backend_tensor_data(pe, pe_vec.data());
-                return vision_model->forward(runner_ctx, pixel_values, pe, nullptr, nullptr, nullptr, pos_embeds);
+                auto outputs = vision_model->forward_outputs(runner_ctx, pixel_values, pe, nullptr, nullptr, nullptr, pos_embeds);
+                if (output_tensors != nullptr) {
+                    *output_tensors = outputs;
+                }
+                return outputs[0];
             }
 
             int llm_grid_h             = grid_h / vision_params.spatial_merge_size;
@@ -1622,7 +1865,11 @@ namespace LLM {
             auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, head_dim / 2, pos_len);
             runner->set_backend_tensor_data(pe, pe_vec.data());
 
-            return vision_model->forward(runner_ctx, pixel_values, pe, window_index, window_inverse_index, window_mask);
+            auto output = vision_model->forward(runner_ctx, pixel_values, pe, window_index, window_inverse_index, window_mask);
+            if (output_tensors != nullptr) {
+                *output_tensors = {output};
+            }
+            return output;
         }
 
     public:
@@ -1657,12 +1904,17 @@ namespace LLM {
             model.get_param_tensors(tensors, prefix);
         }
 
+        void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) {
+            model.get_param_tensor_ops(tensor_ops);
+        }
+
         ggml_tensor* forward(GGMLRunnerContext* ctx,
                              ggml_tensor* input_ids,
                              ggml_tensor* input_pos,
                              ggml_tensor* attention_mask,
                              ggml_tensor* sliding_attention_mask,
                              std::vector<std::pair<int, ggml_tensor*>> image_embeds,
+                             const std::vector<std::vector<std::pair<int, ggml_tensor*>>>& deepstack_image_embeds,
                              std::set<int> out_layers,
                              bool return_all_hidden_states = false) {
             auto hidden_states = model.forward(ctx,
@@ -1671,6 +1923,7 @@ namespace LLM {
                                                attention_mask,
                                                sliding_attention_mask,
                                                image_embeds,
+                                               deepstack_image_embeds,
                                                out_layers,
                                                return_all_hidden_states);  // [N, n_token, hidden_size]
             return hidden_states;
@@ -1689,7 +1942,9 @@ namespace LLM {
 
         ggml_cgraph* build_graph(const sd::Tensor<int32_t>& input_ids_tensor,
                                  const sd::Tensor<float>& attention_mask_tensor,
-                                 const std::vector<std::pair<int, sd::Tensor<float>>>& image_embeds_tensor,
+                                 const ImageEmbeds& image_embeds_tensor,
+                                 const DeepStackImageEmbeds& deepstack_image_embeds_tensor,
+                                 const std::vector<ImageGrid>& image_grids,
                                  std::set<int> out_layers,
                                  bool return_all_hidden_states = false) {
             ggml_cgraph* gf        = new_graph_custom(LLM_GRAPH_SIZE);
@@ -1700,12 +1955,20 @@ namespace LLM {
                 ggml_tensor* embed = make_input(embed_tensor);
                 image_embeds.emplace_back(idx, embed);
             }
+            std::vector<std::vector<std::pair<int, ggml_tensor*>>> deepstack_image_embeds(deepstack_image_embeds_tensor.size());
+            for (size_t layer = 0; layer < deepstack_image_embeds_tensor.size(); ++layer) {
+                deepstack_image_embeds[layer].reserve(deepstack_image_embeds_tensor[layer].size());
+                for (const auto& [idx, embed_tensor] : deepstack_image_embeds_tensor[layer]) {
+                    deepstack_image_embeds[layer].emplace_back(idx, make_input(embed_tensor));
+                }
+            }
 
             int64_t n_tokens = input_ids->ne[0];
             if (config.arch == LLMArch::MISTRAL_SMALL_3_2 ||
                 config.arch == LLMArch::MINISTRAL_3_3B ||
                 config.arch == LLMArch::QWEN3 ||
                 config.arch == LLMArch::GEMMA3_12B ||
+                config.arch == LLMArch::GEMMA4_12B ||
                 config.arch == LLMArch::GEMMA2_2B ||
                 config.arch == LLMArch::GPT_OSS_20B) {
                 input_pos_vec.resize(n_tokens);
@@ -1720,6 +1983,30 @@ namespace LLM {
                     input_pos_vec[2 * n_tokens + i] = i;
                     input_pos_vec[3 * n_tokens + i] = 0;
                 }
+                if (config.arch == LLMArch::QWEN3_VL && !image_grids.empty()) {
+                    int offset = 0;
+                    for (const auto& grid : image_grids) {
+                        int end      = grid.index + grid.size;
+                        int grid_h   = grid.grid_h / config.vision.spatial_merge_size;
+                        int grid_w   = grid.grid_w / config.vision.spatial_merge_size;
+                        int len_max  = std::max(grid_h, grid_w);
+                        int next_pos = grid.index + len_max + offset;
+                        GGML_ASSERT(grid.index >= 0 && end <= n_tokens);
+                        GGML_ASSERT(grid_h > 0 && grid_w > 0 && grid.size == grid_h * grid_w);
+                        for (int token = end; token < n_tokens; ++token) {
+                            int pos                             = next_pos + token - end;
+                            input_pos_vec[token]                = pos;
+                            input_pos_vec[n_tokens + token]     = pos;
+                            input_pos_vec[2 * n_tokens + token] = pos;
+                        }
+                        for (int token = 0; token < grid.size; ++token) {
+                            input_pos_vec[grid.index + token]                = grid.index + offset;
+                            input_pos_vec[n_tokens + grid.index + token]     = grid.index + offset + token / grid_w;
+                            input_pos_vec[2 * n_tokens + grid.index + token] = grid.index + offset + token % grid_w;
+                        }
+                        offset += len_max - grid.size;
+                    }
+                }
             }
 
             auto input_pos = ggml_new_tensor_1d(compute_ctx,
@@ -1746,7 +2033,7 @@ namespace LLM {
                 set_backend_tensor_data(attention_mask, attention_mask_vec.data());
             }
 
-            if (config.arch == LLMArch::GEMMA3_12B || config.arch == LLMArch::GPT_OSS_20B) {
+            if (config.arch == LLMArch::GEMMA3_12B || config.arch == LLMArch::GEMMA4_12B || config.arch == LLMArch::GPT_OSS_20B) {
                 int sliding_window = 0;
                 for (int window : config.sliding_attention) {
                     sliding_window = std::max(sliding_window, window);
@@ -1777,6 +2064,7 @@ namespace LLM {
                                                  attention_mask,
                                                  sliding_attention_mask,
                                                  image_embeds,
+                                                 deepstack_image_embeds,
                                                  out_layers,
                                                  return_all_hidden_states);
 
@@ -1788,16 +2076,20 @@ namespace LLM {
         sd::Tensor<float> compute(const int n_threads,
                                   const sd::Tensor<int32_t>& input_ids,
                                   const sd::Tensor<float>& attention_mask,
-                                  const std::vector<std::pair<int, sd::Tensor<float>>>& image_embeds,
+                                  const ImageEmbeds& image_embeds,
                                   std::set<int> out_layers,
-                                  bool return_all_hidden_states = false,
-                                  bool auto_free                = true,
-                                  bool free_compute_buffer      = true,
-                                  bool free_compute_params      = true) {
+                                  bool return_all_hidden_states                      = false,
+                                  bool auto_free                                     = true,
+                                  bool free_compute_buffer                           = true,
+                                  bool free_compute_params                           = true,
+                                  const DeepStackImageEmbeds& deepstack_image_embeds = {},
+                                  const std::vector<ImageGrid>& image_grids          = {}) {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_graph(input_ids,
                                    attention_mask,
                                    image_embeds,
+                                   deepstack_image_embeds,
+                                   image_grids,
                                    out_layers,
                                    return_all_hidden_states);
             };
@@ -1847,6 +2139,24 @@ namespace LLM {
                                        pos_embed_weight_data_);
         }
 
+        std::vector<ggml_tensor*> encode_image_outputs(GGMLRunnerContext* runner_ctx, ggml_tensor* image) {
+            std::vector<ggml_tensor*> outputs;
+            encode_image_common(this,
+                                compute_ctx,
+                                runner_ctx,
+                                image,
+                                config.vision,
+                                model.vision_model(),
+                                window_index_vec,
+                                window_inverse_index_vec,
+                                window_mask_vec,
+                                pe_vec,
+                                pos_embed_idx_data_,
+                                pos_embed_weight_data_,
+                                &outputs);
+            return outputs;
+        }
+
         ggml_cgraph* build_encode_image_graph(const sd::Tensor<float>& image_tensor) {
             ggml_cgraph* gf    = new_graph_custom(LLM_GRAPH_SIZE);
             ggml_tensor* image = make_input(image_tensor);
@@ -1871,6 +2181,166 @@ namespace LLM {
             };
             return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, auto_free, free_compute_buffer, free_compute_params));
         }
+
+        ggml_cgraph* build_encode_image_outputs_graph(const sd::Tensor<float>& image_tensor) {
+            ggml_cgraph* gf    = new_graph_custom(LLM_GRAPH_SIZE);
+            ggml_tensor* image = make_input(image_tensor);
+
+            auto runner_ctx = get_context();
+            auto outputs    = encode_image_outputs(&runner_ctx, image);
+            GGML_ASSERT(!outputs.empty());
+            auto combined = outputs[0];
+            for (size_t i = 1; i < outputs.size(); ++i) {
+                combined = ggml_concat(compute_ctx, combined, outputs[i], 0);
+            }
+            ggml_build_forward_expand(gf, combined);
+            return gf;
+        }
+
+        static sd::Tensor<float> process_video_block_tensor(const sd::Tensor<float>& frames,
+                                                            const LLMVisionConfig& vision_params) {
+            GGML_ASSERT(frames.dim() == 5);
+            GGML_ASSERT(frames.shape()[2] == vision_params.temporal_patch_size);
+            GGML_ASSERT(frames.shape()[3] == vision_params.in_channels);
+            GGML_ASSERT(frames.shape()[4] == 1);
+
+            int64_t width       = frames.shape()[0];
+            int64_t height      = frames.shape()[1];
+            int64_t temporal    = frames.shape()[2];
+            int64_t channels    = frames.shape()[3];
+            int64_t patch       = vision_params.patch_size;
+            int64_t merge       = vision_params.spatial_merge_size;
+            int64_t grid_w      = width / patch;
+            int64_t grid_h      = height / patch;
+            int64_t feature     = channels * temporal * patch * patch;
+            int64_t token_count = grid_h * grid_w;
+            sd::Tensor<float> output({feature, token_count});
+
+            int64_t token = 0;
+            for (int64_t block_h = 0; block_h < grid_h / merge; ++block_h) {
+                for (int64_t block_w = 0; block_w < grid_w / merge; ++block_w) {
+                    for (int64_t inner_h = 0; inner_h < merge; ++inner_h) {
+                        for (int64_t inner_w = 0; inner_w < merge; ++inner_w) {
+                            int64_t patch_h = block_h * merge + inner_h;
+                            int64_t patch_w = block_w * merge + inner_w;
+                            int64_t offset  = 0;
+                            for (int64_t c = 0; c < channels; ++c) {
+                                for (int64_t t = 0; t < temporal; ++t) {
+                                    for (int64_t y = 0; y < patch; ++y) {
+                                        for (int64_t x = 0; x < patch; ++x) {
+                                            output.index(offset++, token) =
+                                                frames.index(patch_w * patch + x,
+                                                             patch_h * patch + y,
+                                                             t,
+                                                             c,
+                                                             0);
+                                        }
+                                    }
+                                }
+                            }
+                            ++token;
+                        }
+                    }
+                }
+            }
+            return output;
+        }
+
+        ggml_cgraph* build_encode_video_block_outputs_graph(const sd::Tensor<float>& pixel_values_tensor,
+                                                            int grid_h,
+                                                            int grid_w) {
+            ggml_cgraph* gf   = new_graph_custom(LLM_GRAPH_SIZE);
+            auto pixel_values = make_input(pixel_values_tensor);
+            auto runner_ctx   = get_context();
+            auto vision       = model.vision_model();
+            int head_dim      = static_cast<int>(config.vision.hidden_size / config.vision.num_heads);
+            auto pos_embeds   = build_patch_pos_embeds(&runner_ctx, vision, grid_h, grid_w);
+            window_index_vec.resize(static_cast<size_t>((grid_h / config.vision.spatial_merge_size) *
+                                                        (grid_w / config.vision.spatial_merge_size)));
+            for (int i = 0; i < static_cast<int>(window_index_vec.size()); ++i) {
+                window_index_vec[static_cast<size_t>(i)] = i;
+            }
+            pe_vec      = Rope::gen_qwen2vl_pe(grid_h,
+                                               grid_w,
+                                               config.vision.spatial_merge_size,
+                                               window_index_vec,
+                                               10000,
+                                               {head_dim / 2, head_dim / 2});
+            int pos_len = static_cast<int>(pe_vec.size() / head_dim / 2);
+            auto pe     = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, head_dim / 2, pos_len);
+            set_backend_tensor_data(pe, pe_vec.data());
+            auto outputs = vision->forward_outputs(&runner_ctx,
+                                                   pixel_values,
+                                                   pe,
+                                                   nullptr,
+                                                   nullptr,
+                                                   nullptr,
+                                                   pos_embeds);
+            GGML_ASSERT(!outputs.empty());
+            auto combined = outputs[0];
+            for (size_t i = 1; i < outputs.size(); ++i) {
+                combined = ggml_concat(compute_ctx, combined, outputs[i], 0);
+            }
+            ggml_build_forward_expand(gf, combined);
+            return gf;
+        }
+
+        std::vector<sd::Tensor<float>> encode_image_outputs(const int n_threads,
+                                                            const sd::Tensor<float>& image,
+                                                            bool auto_free           = false,
+                                                            bool free_compute_buffer = false,
+                                                            bool free_compute_params = false) {
+            auto get_graph = [&]() -> ggml_cgraph* {
+                return build_encode_image_outputs_graph(image);
+            };
+            auto combined = take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, auto_free, free_compute_buffer, free_compute_params));
+            if (combined.empty()) {
+                return {};
+            }
+            size_t output_count = config.vision.deepstack_visual_indexes.size() + 1;
+            GGML_ASSERT(combined.shape()[0] == config.hidden_size * static_cast<int64_t>(output_count));
+            std::vector<sd::Tensor<float>> outputs;
+            outputs.reserve(output_count);
+            for (size_t i = 0; i < output_count; ++i) {
+                outputs.push_back(sd::ops::slice(combined,
+                                                 0,
+                                                 static_cast<int64_t>(i) * config.hidden_size,
+                                                 static_cast<int64_t>(i + 1) * config.hidden_size));
+            }
+            return outputs;
+        }
+
+        std::vector<sd::Tensor<float>> encode_video_block_outputs(const int n_threads,
+                                                                  const sd::Tensor<float>& frames,
+                                                                  bool auto_free           = false,
+                                                                  bool free_compute_buffer = false,
+                                                                  bool free_compute_params = false) {
+            int grid_h        = static_cast<int>(frames.shape()[1] / config.vision.patch_size);
+            int grid_w        = static_cast<int>(frames.shape()[0] / config.vision.patch_size);
+            auto pixel_values = process_video_block_tensor(frames, config.vision);
+            auto get_graph    = [&]() -> ggml_cgraph* {
+                return build_encode_video_block_outputs_graph(pixel_values, grid_h, grid_w);
+            };
+            auto combined = take_or_empty(GGMLRunner::compute<float>(get_graph,
+                                                                     n_threads,
+                                                                     auto_free,
+                                                                     free_compute_buffer,
+                                                                     free_compute_params));
+            if (combined.empty()) {
+                return {};
+            }
+            size_t output_count = config.vision.deepstack_visual_indexes.size() + 1;
+            GGML_ASSERT(combined.shape()[0] == config.hidden_size * static_cast<int64_t>(output_count));
+            std::vector<sd::Tensor<float>> outputs;
+            outputs.reserve(output_count);
+            for (size_t i = 0; i < output_count; ++i) {
+                outputs.push_back(sd::ops::slice(combined,
+                                                 0,
+                                                 static_cast<int64_t>(i) * config.hidden_size,
+                                                 static_cast<int64_t>(i + 1) * config.hidden_size));
+            }
+            return outputs;
+        }
     };
 
     struct LLMEmbedder {

src/model/vae/audio_vae.hpp

@@ -0,0 +1,28 @@
+#ifndef __SD_MODEL_VAE_AUDIO_VAE_HPP__
+#define __SD_MODEL_VAE_AUDIO_VAE_HPP__
+
+#include "core/ggml_extend.hpp"
+
+struct AudioVAERunner : public GGMLRunner {
+    AudioVAERunner(ggml_backend_t backend,
+                   std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+        : GGMLRunner(backend, weight_manager) {}
+
+    virtual void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) = 0;
+    virtual size_t get_params_mem_size()                                         = 0;
+    virtual std::string get_desc()                                               = 0;
+    virtual sd::Tensor<float> encode(int n_threads,
+                                     const sd::Tensor<float>& waveform) {
+        SD_UNUSED(n_threads);
+        SD_UNUSED(waveform);
+        return {};
+    }
+    virtual sd::Tensor<float> decode(int n_threads,
+                                     const sd::Tensor<float>& latent_tensor) = 0;
+    virtual int input_sample_rate() const {
+        return output_sample_rate();
+    }
+    virtual int output_sample_rate() const = 0;
+};
+
+#endif  // __SD_MODEL_VAE_AUDIO_VAE_HPP__

src/model/vae/ltx_audio_vae.hpp

@@ -8,6 +8,7 @@
 #include <vector>
 
 #include "core/ggml_extend.hpp"
+#include "model/vae/audio_vae.hpp"
 #include "model_loader.h"
 #include "model_manager.h"
 
@@ -996,7 +997,7 @@ namespace LTXV {
         }
     };
 
-    struct LTXAudioVAERunner : public GGMLRunner {
+    struct LTXAudioVAERunner : public AudioVAERunner {
         LTXAudioVAEConfig config;
         LTXAudioVAE model;
         std::string weight_prefix;
@@ -1006,7 +1007,7 @@ namespace LTXV {
                           const String2TensorStorage& tensor_storage_map,
                           const std::string& prefix                           = "",
                           std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
-            : GGMLRunner(backend, weight_manager),
+            : AudioVAERunner(backend, weight_manager),
               weight_prefix(prefix),
               config(LTXAudioVAEConfig::detect_from_weights(tensor_storage_map)),
               model(config) {
@@ -1017,20 +1018,20 @@ namespace LTXV {
             }
         }
 
-        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) {
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
             model.get_param_tensors(tensors, weight_prefix);
         }
 
-        size_t get_params_mem_size() {
+        size_t get_params_mem_size() override {
             return model.get_params_mem_size();
         }
 
-        std::string get_desc() {
+        std::string get_desc() override {
             return "ltx_audio_vae";
         }
 
         sd::Tensor<float> decode(int n_threads,
-                                 const sd::Tensor<float>& latent_tensor) {
+                                 const sd::Tensor<float>& latent_tensor) override {
             int64_t t0     = ggml_time_ms();
             auto get_graph = [&]() -> ggml_cgraph* {
                 auto latent                  = make_input(latent_tensor);
@@ -1047,6 +1048,10 @@ namespace LTXV {
             return result;
         }
 
+        int output_sample_rate() const override {
+            return config.output_sample_rate();
+        }
+
         void test(const std::string& input_path) {
             auto z = sd::load_tensor_from_file_as_tensor<float>(input_path);
             GGML_ASSERT(!z.empty());

src/model/vae/minimax_h3_audio_vae.hpp

@@ -0,0 +1,514 @@
+#ifndef __SD_MODEL_VAE_MINIMAX_H3_AUDIO_VAE_HPP__
+#define __SD_MODEL_VAE_MINIMAX_H3_AUDIO_VAE_HPP__
+
+#include <array>
+#include <string>
+#include <vector>
+
+#include "model/vae/audio_vae.hpp"
+#include "model/vae/ltx_audio_vae.hpp"
+
+namespace MiniMaxH3 {
+
+    struct AudioSnake1D : public UnaryBlock {
+        int64_t channels;
+
+        explicit AudioSnake1D(int64_t channels)
+            : channels(channels) {}
+
+        void init_params(ggml_context* ctx,
+                         const String2TensorStorage& tensor_storage_map = {},
+                         const std::string prefix                       = "") override {
+            SD_UNUSED(tensor_storage_map);
+            SD_UNUSED(prefix);
+            params["alpha"] = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, channels, 1);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
+            auto alpha       = params["alpha"];
+            auto oscillation = ggml_sin(ctx->ggml_ctx, ggml_mul(ctx->ggml_ctx, x, alpha));
+            oscillation      = ggml_mul(ctx->ggml_ctx, oscillation, oscillation);
+            auto eps         = ggml_ext_scale(ctx->ggml_ctx, ggml_ext_ones(ctx->ggml_ctx, 1, 1, 1, 1), 1e-9f);
+            return ggml_add(ctx->ggml_ctx,
+                            x,
+                            ggml_div(ctx->ggml_ctx, oscillation, ggml_add(ctx->ggml_ctx, alpha, eps)));
+        }
+    };
+
+    struct AudioEncoderResidualUnit : public GGMLBlock {
+        int64_t channels;
+
+        AudioEncoderResidualUnit(int64_t channels, int dilation)
+            : channels(channels) {
+            blocks["block.0"] = std::make_shared<AudioSnake1D>(channels);
+            blocks["block.1"] = std::make_shared<LTXV::Conv1D>(channels,
+                                                               channels,
+                                                               7,
+                                                               1,
+                                                               3 * dilation,
+                                                               dilation);
+            blocks["block.2"] = std::make_shared<AudioSnake1D>(channels);
+            blocks["block.3"] = std::make_shared<LTXV::Conv1D>(channels, channels, 1);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            auto act1  = std::dynamic_pointer_cast<AudioSnake1D>(blocks["block.0"]);
+            auto conv1 = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["block.1"]);
+            auto act2  = std::dynamic_pointer_cast<AudioSnake1D>(blocks["block.2"]);
+            auto conv2 = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["block.3"]);
+            auto h     = conv2->forward(ctx, act2->forward(ctx, conv1->forward(ctx, act1->forward(ctx, x))));
+            if (x->ne[0] != h->ne[0]) {
+                int64_t pad = (x->ne[0] - h->ne[0]) / 2;
+                x           = ggml_ext_slice(ctx->ggml_ctx, x, 0, pad, x->ne[0] - pad);
+            }
+            return ggml_add(ctx->ggml_ctx, x, h);
+        }
+    };
+
+    struct AudioEncoderBlock : public GGMLBlock {
+        int64_t out_channels;
+
+        AudioEncoderBlock(int64_t out_channels, int stride)
+            : out_channels(out_channels) {
+            int64_t in_channels = out_channels / 2;
+            blocks["block.0"]   = std::make_shared<AudioEncoderResidualUnit>(in_channels, 1);
+            blocks["block.1"]   = std::make_shared<AudioEncoderResidualUnit>(in_channels, 3);
+            blocks["block.2"]   = std::make_shared<AudioEncoderResidualUnit>(in_channels, 9);
+            blocks["block.3"]   = std::make_shared<AudioSnake1D>(in_channels);
+            blocks["block.4"]   = std::make_shared<LTXV::Conv1D>(in_channels,
+                                                               out_channels,
+                                                               2 * stride,
+                                                               stride,
+                                                               static_cast<int>(std::ceil(stride / 2.f)));
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            for (int i = 0; i < 3; ++i) {
+                auto unit = std::dynamic_pointer_cast<AudioEncoderResidualUnit>(blocks["block." + std::to_string(i)]);
+                x         = unit->forward(ctx, x);
+            }
+            auto act  = std::dynamic_pointer_cast<AudioSnake1D>(blocks["block.3"]);
+            auto conv = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["block.4"]);
+            return conv->forward(ctx, act->forward(ctx, x));
+        }
+    };
+
+    struct AudioEncoder : public GGMLBlock {
+        static constexpr std::array<int, 5> strides = {2, 4, 4, 5, 5};
+
+        AudioEncoder() {
+            int64_t channels  = 64;
+            blocks["block.0"] = std::make_shared<LTXV::Conv1D>(1, channels, 7, 1, 3);
+            for (size_t i = 0; i < strides.size(); ++i) {
+                channels *= 2;
+                blocks["block." + std::to_string(i + 1)] = std::make_shared<AudioEncoderBlock>(channels, strides[i]);
+            }
+            blocks["block.6"] = std::make_shared<AudioSnake1D>(channels);
+            blocks["block.7"] = std::make_shared<LTXV::Conv1D>(channels, 2048, 3, 1, 1);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            auto input = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["block.0"]);
+            x          = input->forward(ctx, x);
+            for (size_t i = 0; i < strides.size(); ++i) {
+                auto block = std::dynamic_pointer_cast<AudioEncoderBlock>(blocks["block." + std::to_string(i + 1)]);
+                x          = block->forward(ctx, x);
+            }
+            auto act = std::dynamic_pointer_cast<AudioSnake1D>(blocks["block.6"]);
+            auto out = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["block.7"]);
+            return out->forward(ctx, act->forward(ctx, x));
+        }
+    };
+
+    struct AudioGeGLUMLP : public GGMLBlock {
+        AudioGeGLUMLP(int64_t hidden_size, int64_t intermediate_size) {
+            blocks["norm"] = std::make_shared<LayerNorm>(hidden_size);
+            blocks["w0"]   = std::make_shared<Linear>(hidden_size, intermediate_size, true);
+            blocks["w1"]   = std::make_shared<Linear>(hidden_size, intermediate_size, true);
+            blocks["w2"]   = std::make_shared<Linear>(intermediate_size, hidden_size, true);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            auto norm = std::dynamic_pointer_cast<LayerNorm>(blocks["norm"]);
+            auto w0   = std::dynamic_pointer_cast<Linear>(blocks["w0"]);
+            auto w1   = std::dynamic_pointer_cast<Linear>(blocks["w1"]);
+            auto w2   = std::dynamic_pointer_cast<Linear>(blocks["w2"]);
+            x         = norm->forward(ctx, x);
+            auto gate = ggml_ext_gelu(ctx->ggml_ctx, w0->forward(ctx, x), true);
+            return w2->forward(ctx, ggml_mul(ctx->ggml_ctx, gate, w1->forward(ctx, x)));
+        }
+    };
+
+    struct AudioCausalAttention : public GGMLBlock {
+        static constexpr int64_t in_channels  = 2048;
+        static constexpr int64_t out_channels = 32;
+        static constexpr int64_t num_head     = 8;
+        static constexpr int64_t head_dim     = in_channels / num_head;
+
+        AudioCausalAttention() {
+            blocks["qkv"]  = std::make_shared<Linear>(in_channels, in_channels * 3, false);
+            blocks["proj"] = std::make_shared<Linear>(out_channels, out_channels, true);
+        }
+
+        void init_params(ggml_context* ctx,
+                         const String2TensorStorage& tensor_storage_map = {},
+                         const std::string prefix                       = "") override {
+            GGMLBlock::init_params(ctx, tensor_storage_map, prefix);
+            params["q_bias"]      = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
+            params["zero_k_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
+            params["v_bias"]      = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            auto qkv_layer  = std::dynamic_pointer_cast<Linear>(blocks["qkv"]);
+            auto proj       = std::dynamic_pointer_cast<Linear>(blocks["proj"]);
+            auto qkv        = ggml_ext_chunk(ctx->ggml_ctx, qkv_layer->forward(ctx, x), 3, 0);
+            auto bias_shape = [&](ggml_tensor* bias) {
+                return ggml_reshape_4d(ctx->ggml_ctx, bias, bias->ne[0], 1, 1, 1);
+            };
+            auto q = ggml_add(ctx->ggml_ctx, qkv[0], bias_shape(params["q_bias"]));
+            auto k = ggml_add(ctx->ggml_ctx, qkv[1], bias_shape(params["zero_k_bias"]));
+            auto v = ggml_add(ctx->ggml_ctx, qkv[2], bias_shape(params["v_bias"]));
+
+            int64_t sequence = x->ne[1];
+            auto mask        = ggml_diag_mask_inf(ctx->ggml_ctx,
+                                                  ggml_ext_zeros(ctx->ggml_ctx, sequence, sequence, 1, 1),
+                                                  0);
+            auto attn_out    = ggml_ext_attention_ext(ctx->ggml_ctx,
+                                                      ctx->backend,
+                                                      q,
+                                                      k,
+                                                      v,
+                                                      num_head,
+                                                      mask,
+                                                      false,
+                                                      ctx->flash_attn_enabled);
+            int64_t batch    = attn_out->ne[2] * attn_out->ne[3];
+            attn_out         = ggml_reshape_4d(ctx->ggml_ctx, attn_out, head_dim, num_head, sequence, batch);
+            attn_out         = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, attn_out, 1, 0, 2, 3));
+            attn_out         = ggml_mean(ctx->ggml_ctx, attn_out);
+            attn_out         = ggml_reshape_3d(ctx->ggml_ctx, attn_out, head_dim, sequence, batch);
+
+            constexpr int64_t pool = head_dim / out_channels;
+            attn_out               = ggml_reshape_4d(ctx->ggml_ctx, attn_out, pool, out_channels, sequence, batch);
+            attn_out               = ggml_mean(ctx->ggml_ctx, attn_out);
+            attn_out               = ggml_reshape_3d(ctx->ggml_ctx, attn_out, out_channels, sequence, batch);
+            return proj->forward(ctx, attn_out);
+        }
+    };
+
+    struct AudioAttentionProjection : public GGMLBlock {
+        AudioAttentionProjection() {
+            blocks["norm1"] = std::make_shared<LayerNorm>(2048);
+            blocks["attn"]  = std::make_shared<AudioCausalAttention>();
+            blocks["proj"]  = std::make_shared<Linear>(2048, 32, true);
+            blocks["norm3"] = std::make_shared<LayerNorm>(2048);
+            blocks["norm2"] = std::make_shared<LayerNorm>(32);
+            blocks["mlp"]   = std::make_shared<AudioGeGLUMLP>(32, 64);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            auto norm1 = std::dynamic_pointer_cast<LayerNorm>(blocks["norm1"]);
+            auto attn  = std::dynamic_pointer_cast<AudioCausalAttention>(blocks["attn"]);
+            auto proj  = std::dynamic_pointer_cast<Linear>(blocks["proj"]);
+            auto norm3 = std::dynamic_pointer_cast<LayerNorm>(blocks["norm3"]);
+            auto norm2 = std::dynamic_pointer_cast<LayerNorm>(blocks["norm2"]);
+            auto mlp   = std::dynamic_pointer_cast<AudioGeGLUMLP>(blocks["mlp"]);
+            x          = ggml_add(ctx->ggml_ctx,
+                                  proj->forward(ctx, norm3->forward(ctx, x)),
+                                  attn->forward(ctx, norm1->forward(ctx, x)));
+            return ggml_add(ctx->ggml_ctx, x, mlp->forward(ctx, norm2->forward(ctx, x)));
+        }
+    };
+
+    struct AudioAMPBlock : public GGMLBlock {
+        int channels;
+
+        AudioAMPBlock(int channels,
+                      int kernel_size,
+                      const std::array<int, 3>& dilations)
+            : channels(channels) {
+            for (int i = 0; i < 3; ++i) {
+                blocks["activations." + std::to_string(i * 2)] =
+                    std::make_shared<LTXV::Activation1D>(channels);
+                blocks["activations." + std::to_string(i * 2 + 1)] =
+                    std::make_shared<LTXV::Activation1D>(channels);
+                blocks["convs1." + std::to_string(i)] =
+                    std::make_shared<LTXV::Conv1D>(channels,
+                                                   channels,
+                                                   kernel_size,
+                                                   1,
+                                                   (kernel_size * dilations[i] - dilations[i]) / 2,
+                                                   dilations[i]);
+                blocks["convs2." + std::to_string(i)] =
+                    std::make_shared<LTXV::Conv1D>(channels,
+                                                   channels,
+                                                   kernel_size,
+                                                   1,
+                                                   kernel_size / 2);
+            }
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            for (int i = 0; i < 3; ++i) {
+                auto act1 = std::dynamic_pointer_cast<LTXV::Activation1D>(
+                    blocks["activations." + std::to_string(i * 2)]);
+                auto act2 = std::dynamic_pointer_cast<LTXV::Activation1D>(
+                    blocks["activations." + std::to_string(i * 2 + 1)]);
+                auto conv1 = std::dynamic_pointer_cast<LTXV::Conv1D>(
+                    blocks["convs1." + std::to_string(i)]);
+                auto conv2 = std::dynamic_pointer_cast<LTXV::Conv1D>(
+                    blocks["convs2." + std::to_string(i)]);
+
+                auto h = conv1->forward(ctx, act1->forward(ctx, x));
+                h      = conv2->forward(ctx, act2->forward(ctx, h));
+                x      = ggml_add(ctx->ggml_ctx, x, h);
+            }
+            return x;
+        }
+    };
+
+    struct BigVGAN : public GGMLBlock {
+        static constexpr int initial_channels                     = 1024;
+        static constexpr int num_kernels                          = 3;
+        static constexpr int num_upsamples                        = 7;
+        static constexpr std::array<int, num_upsamples> rates     = {5, 5, 2, 2, 2, 2, 2};
+        static constexpr std::array<int, num_upsamples> kernels   = {9, 9, 4, 4, 4, 4, 4};
+        static constexpr std::array<int, num_kernels> res_kernels = {3, 7, 11};
+
+        BigVGAN() {
+            blocks["conv_pre"] = std::make_shared<LTXV::Conv1D>(2048,
+                                                                initial_channels,
+                                                                7,
+                                                                1,
+                                                                3);
+            int channels       = initial_channels;
+            for (int i = 0; i < num_upsamples; ++i) {
+                int next_channels = initial_channels / (1 << (i + 1));
+                blocks["ups." + std::to_string(i) + ".0"] =
+                    std::make_shared<LTXV::ConvTranspose1D>(channels,
+                                                            next_channels,
+                                                            kernels[i],
+                                                            rates[i],
+                                                            (kernels[i] - rates[i]) / 2);
+                for (int j = 0; j < num_kernels; ++j) {
+                    blocks["resblocks." + std::to_string(i * num_kernels + j)] =
+                        std::make_shared<AudioAMPBlock>(next_channels,
+                                                        res_kernels[j],
+                                                        std::array<int, 3>{1, 3, 5});
+                }
+                channels = next_channels;
+            }
+            blocks["activation_post"] = std::make_shared<LTXV::Activation1D>(channels);
+            blocks["conv_post"]       = std::make_shared<LTXV::Conv1D>(channels,
+                                                                 1,
+                                                                 7,
+                                                                 1,
+                                                                 3,
+                                                                 1,
+                                                                 false);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            auto conv_pre = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["conv_pre"]);
+            x             = conv_pre->forward(ctx, x);
+            for (int i = 0; i < num_upsamples; ++i) {
+                auto up = std::dynamic_pointer_cast<LTXV::ConvTranspose1D>(
+                    blocks["ups." + std::to_string(i) + ".0"]);
+                x = up->forward(ctx, x);
+
+                ggml_tensor* sum = nullptr;
+                for (int j = 0; j < num_kernels; ++j) {
+                    auto block = std::dynamic_pointer_cast<AudioAMPBlock>(
+                        blocks["resblocks." + std::to_string(i * num_kernels + j)]);
+                    auto value = block->forward(ctx, x);
+                    sum        = sum == nullptr ? value : ggml_add(ctx->ggml_ctx, sum, value);
+                }
+                x = ggml_ext_scale(ctx->ggml_ctx, sum, 1.f / num_kernels);
+            }
+            auto activation = std::dynamic_pointer_cast<LTXV::Activation1D>(blocks["activation_post"]);
+            auto conv_post  = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["conv_post"]);
+            return ggml_clamp(ctx->ggml_ctx,
+                              conv_post->forward(ctx, activation->forward(ctx, x)),
+                              -1.f,
+                              1.f);
+        }
+    };
+
+    struct AudioVAE : public GGMLBlock {
+        static constexpr int kLatentChannels = 32;
+
+        AudioVAE() {
+            blocks["encoder"]     = std::make_shared<AudioEncoder>();
+            blocks["pre_block"]   = std::make_shared<AudioAttentionProjection>();
+            blocks["mean_proj"]   = std::make_shared<LTXV::Conv1D>(kLatentChannels, kLatentChannels, 1);
+            blocks["logs_proj"]   = std::make_shared<LTXV::Conv1D>(kLatentChannels, kLatentChannels, 1);
+            blocks["dec_in_proj"] = std::make_shared<LTXV::Conv1D>(kLatentChannels,
+                                                                   2048,
+                                                                   1);
+            blocks["decoder"]     = std::make_shared<BigVGAN>();
+        }
+
+        void init_params(ggml_context* ctx,
+                         const String2TensorStorage& tensor_storage_map = {},
+                         const std::string prefix                       = "") override {
+            SD_UNUSED(tensor_storage_map);
+            SD_UNUSED(prefix);
+            params["latents_mean"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, kLatentChannels);
+            params["latents_std"]  = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, kLatentChannels);
+        }
+
+        ggml_tensor* encode(GGMLRunnerContext* ctx, ggml_tensor* waveform) {
+            GGML_ASSERT(waveform->ne[1] * waveform->ne[2] * waveform->ne[3] == 2);
+            auto encoder   = std::dynamic_pointer_cast<AudioEncoder>(blocks["encoder"]);
+            auto pre       = std::dynamic_pointer_cast<AudioAttentionProjection>(blocks["pre_block"]);
+            auto mean_proj = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["mean_proj"]);
+
+            // GGML's batched conv1d storage interleaves the stream dimension
+            // with output channels. Subsequent layers then read stereo samples
+            // as adjacent feature channels. Run each mono stream independently,
+            // matching PyTorch's reshape(B*S, 1, samples), and concatenate only
+            // the completed normalized latents.
+            const int64_t streams = waveform->ne[2] * waveform->ne[3];
+            waveform              = ggml_reshape_3d(ctx->ggml_ctx,
+                                                    waveform,
+                                                    waveform->ne[0],
+                                                    1,
+                                                    streams);
+            ggml_tensor* stereo_z = nullptr;
+            for (int64_t stream = 0; stream < streams; ++stream) {
+                auto mono = ggml_ext_slice(ctx->ggml_ctx, waveform, 2, stream, stream + 1);
+                auto x    = encoder->forward(ctx, mono);
+                x         = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
+                x         = pre->forward(ctx, x);
+                x         = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
+                auto z    = mean_proj->forward(ctx, x);
+
+                auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1);
+                auto std  = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1);
+                z         = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std);
+                z         = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 2, 1, 3));
+                stereo_z  = stereo_z == nullptr ? z : ggml_concat(ctx->ggml_ctx, stereo_z, z, 1);
+            }
+            return stereo_z;
+        }
+
+        ggml_tensor* decode(GGMLRunnerContext* ctx, ggml_tensor* latent) {
+            GGML_ASSERT(latent->ne[1] == 2 && latent->ne[2] == kLatentChannels);
+            latent = ggml_cont(ctx->ggml_ctx,
+                               ggml_permute(ctx->ggml_ctx, latent, 0, 2, 1, 3));
+
+            auto mean = ggml_reshape_4d(ctx->ggml_ctx,
+                                        params["latents_mean"],
+                                        1,
+                                        kLatentChannels,
+                                        1,
+                                        1);
+            auto std  = ggml_reshape_4d(ctx->ggml_ctx,
+                                        params["latents_std"],
+                                        1,
+                                        kLatentChannels,
+                                        1,
+                                        1);
+            latent    = ggml_add(ctx->ggml_ctx,
+                                 ggml_mul(ctx->ggml_ctx, latent, std),
+                                 mean);
+
+            auto dec_in           = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["dec_in_proj"]);
+            auto decoder          = std::dynamic_pointer_cast<BigVGAN>(blocks["decoder"]);
+            int64_t streams       = latent->ne[2] * latent->ne[3];
+            latent                = ggml_reshape_3d(ctx->ggml_ctx,
+                                                    latent,
+                                                    latent->ne[0],
+                                                    latent->ne[1],
+                                                    streams);
+            ggml_tensor* waveform = nullptr;
+            for (int64_t stream = 0; stream < streams; ++stream) {
+                auto stream_latent   = ggml_ext_slice(ctx->ggml_ctx, latent, 2, stream, stream + 1);
+                auto stream_waveform = decoder->forward(ctx, dec_in->forward(ctx, stream_latent));
+                waveform             = waveform == nullptr
+                                           ? stream_waveform
+                                           : ggml_concat(ctx->ggml_ctx, waveform, stream_waveform, 2);
+            }
+            return ggml_reshape_4d(ctx->ggml_ctx,
+                                   waveform,
+                                   waveform->ne[0],
+                                   streams,
+                                   1,
+                                   1);
+        }
+    };
+
+    struct AudioVAERunner : public ::AudioVAERunner {
+        AudioVAE model;
+        std::string weight_prefix;
+
+        AudioVAERunner(ggml_backend_t backend,
+                       const String2TensorStorage& tensor_storage_map,
+                       const std::string& prefix                           = "",
+                       std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+            : ::AudioVAERunner(backend, weight_manager),
+              weight_prefix(prefix) {
+            model.init(params_ctx, tensor_storage_map, prefix);
+        }
+
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
+            model.get_param_tensors(tensors, weight_prefix);
+        }
+
+        size_t get_params_mem_size() override {
+            return model.get_params_mem_size();
+        }
+
+        std::string get_desc() override {
+            return "minimax_h3_audio_vae";
+        }
+
+        int output_sample_rate() const override {
+            return 32000;
+        }
+
+        sd::Tensor<float> encode(int n_threads,
+                                 const sd::Tensor<float>& waveform) override {
+            int64_t t0     = ggml_time_ms();
+            auto get_graph = [&]() -> ggml_cgraph* {
+                auto input      = make_input(waveform);
+                auto runner_ctx = get_context();
+                auto latent     = model.encode(&runner_ctx, input);
+                auto graph      = new_graph_custom(655360);
+                ggml_build_forward_expand(graph, latent);
+                return graph;
+            };
+            auto result = restore_trailing_singleton_dims(
+                GGMLRunner::compute<float>(get_graph, n_threads, false, false, false),
+                4);
+            int64_t t1 = ggml_time_ms();
+            LOG_INFO("MiniMax-H3 audio VAE encode completed, taking %.2fs",
+                     (t1 - t0) / 1000.f);
+            return result;
+        }
+
+        sd::Tensor<float> decode(int n_threads,
+                                 const sd::Tensor<float>& latent_tensor) override {
+            int64_t t0     = ggml_time_ms();
+            auto get_graph = [&]() -> ggml_cgraph* {
+                auto latent     = make_input(latent_tensor);
+                auto runner_ctx = get_context();
+                auto waveform   = model.decode(&runner_ctx, latent);
+                auto graph      = new_graph_custom(655360);
+                ggml_build_forward_expand(graph, waveform);
+                return graph;
+            };
+            auto result = restore_trailing_singleton_dims(
+                GGMLRunner::compute<float>(get_graph, n_threads, false, false, false),
+                4);
+            int64_t t1 = ggml_time_ms();
+            LOG_INFO("MiniMax-H3 audio VAE decode completed, taking %.2fs",
+                     (t1 - t0) / 1000.f);
+            return result;
+        }
+    };
+
+}  // namespace MiniMaxH3
+
+#endif  // __SD_MODEL_VAE_MINIMAX_H3_AUDIO_VAE_HPP__

src/model/vae/minimax_h3_vae.hpp

@@ -0,0 +1,805 @@
+#ifndef __SD_MODEL_VAE_MINIMAX_H3_VAE_HPP__
+#define __SD_MODEL_VAE_MINIMAX_H3_VAE_HPP__
+
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <memory>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+#include "model/common/rope.hpp"
+#include "model/diffusion/dit.hpp"
+#include "model/vae/vae.hpp"
+
+namespace MiniMaxH3VAE {
+
+    constexpr int H3_VIDEO_VAE_GRAPH_SIZE = 262144;
+
+    struct CausalConv3d : public Conv3d {
+        std::tuple<int, int, int> temporal_padding;
+
+        CausalConv3d(int64_t in_channels,
+                     int64_t out_channels,
+                     std::tuple<int, int, int> kernel_size,
+                     std::tuple<int, int, int> stride  = {1, 1, 1},
+                     std::tuple<int, int, int> padding = {0, 0, 0})
+            : Conv3d(in_channels,
+                     out_channels,
+                     kernel_size,
+                     stride,
+                     {0, 0, 0}),
+              temporal_padding(padding) {}
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
+            auto reflect_pad = [&](ggml_tensor* value, int dim, int amount) {
+                for (int i = 0; i < amount; ++i) {
+                    GGML_ASSERT(value->ne[dim] > 1);
+                    auto left  = ggml_ext_slice(ctx->ggml_ctx, value, dim, 1, 2);
+                    auto right = ggml_ext_slice(ctx->ggml_ctx,
+                                                value,
+                                                dim,
+                                                value->ne[dim] - 2,
+                                                value->ne[dim] - 1);
+                    value      = ggml_concat(ctx->ggml_ctx, left, value, dim);
+                    value      = ggml_concat(ctx->ggml_ctx, value, right, dim);
+                }
+                return value;
+            };
+
+            x                = reflect_pad(x, 0, std::get<2>(temporal_padding));
+            x                = reflect_pad(x, 1, std::get<1>(temporal_padding));
+            int temporal_pad = std::get<0>(temporal_padding) * 2;
+            if (temporal_pad > 0) {
+                x = ggml_ext_pad_ext(ctx->ggml_ctx,
+                                     ctx->backend,
+                                     x,
+                                     0,
+                                     0,
+                                     0,
+                                     0,
+                                     temporal_pad,
+                                     0,
+                                     0,
+                                     0);
+            }
+            return Conv3d::forward(ctx, x);
+        }
+    };
+
+    struct TemporalGroupNorm : public GroupNorm {
+        explicit TemporalGroupNorm(int64_t channels)
+            : GroupNorm(32, channels, 1e-6f, true) {}
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            ggml_tensor* result = nullptr;
+            for (int64_t t = 0; t < x->ne[2]; ++t) {
+                auto frame = ggml_ext_slice(ctx->ggml_ctx, x, 2, t, t + 1);
+                GGML_ASSERT(frame->ne[3] % num_channels == 0);
+                int64_t batch_size = frame->ne[3] / num_channels;
+                frame              = ggml_cont(ctx->ggml_ctx, frame);
+                frame              = ggml_reshape_4d(ctx->ggml_ctx,
+                                                     frame,
+                                                     frame->ne[0],
+                                                     frame->ne[1],
+                                                     num_channels,
+                                                     batch_size);
+                frame              = GroupNorm::forward(ctx, frame);
+                frame              = ggml_reshape_4d(ctx->ggml_ctx,
+                                                     frame,
+                                                     frame->ne[0],
+                                                     frame->ne[1],
+                                                     1,
+                                                     num_channels * batch_size);
+                result             = result == nullptr ? frame : ggml_concat(ctx->ggml_ctx, result, frame, 2);
+            }
+            return result;
+        }
+    };
+
+    struct Downsample3D : public GGMLBlock {
+        int spatial_stride;
+
+        Downsample3D(int64_t in_channels,
+                     int64_t out_channels,
+                     int temporal_stride,
+                     int spatial_stride)
+            : spatial_stride(spatial_stride) {
+            blocks["conv"] = std::make_shared<CausalConv3d>(in_channels,
+                                                            out_channels,
+                                                            std::tuple{3, 3, 3},
+                                                            std::tuple{temporal_stride, spatial_stride, spatial_stride},
+                                                            std::tuple{1, 0, 0});
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            if (spatial_stride == 2) {
+                GGML_ASSERT(x->ne[0] > 1 && x->ne[1] > 1);
+                auto right  = ggml_ext_slice(ctx->ggml_ctx, x, 0, x->ne[0] - 2, x->ne[0] - 1);
+                x           = ggml_concat(ctx->ggml_ctx, x, right, 0);
+                auto bottom = ggml_ext_slice(ctx->ggml_ctx, x, 1, x->ne[1] - 2, x->ne[1] - 1);
+                x           = ggml_concat(ctx->ggml_ctx, x, bottom, 1);
+            }
+            return std::dynamic_pointer_cast<CausalConv3d>(blocks["conv"])->forward(ctx, x);
+        }
+    };
+
+    struct ResnetBlock3D : public GGMLBlock {
+        int64_t in_channels;
+        int64_t out_channels;
+
+        ResnetBlock3D(int64_t in_channels,
+                      int64_t out_channels)
+            : in_channels(in_channels), out_channels(out_channels) {
+            blocks["norm1"] = std::make_shared<TemporalGroupNorm>(in_channels);
+            blocks["norm2"] = std::make_shared<TemporalGroupNorm>(out_channels);
+            blocks["conv1"] = std::make_shared<CausalConv3d>(in_channels,
+                                                             out_channels,
+                                                             std::tuple{3, 3, 3},
+                                                             std::tuple{1, 1, 1},
+                                                             std::tuple{1, 1, 1});
+            blocks["conv2"] = std::make_shared<CausalConv3d>(out_channels,
+                                                             out_channels,
+                                                             std::tuple{3, 3, 3},
+                                                             std::tuple{1, 1, 1},
+                                                             std::tuple{1, 1, 1});
+            if (in_channels != out_channels) {
+                blocks["nin_shortcut"] = std::make_shared<CausalConv3d>(in_channels,
+                                                                        out_channels,
+                                                                        std::tuple{1, 1, 1});
+            }
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            auto norm1 = std::dynamic_pointer_cast<TemporalGroupNorm>(blocks["norm1"]);
+            auto norm2 = std::dynamic_pointer_cast<TemporalGroupNorm>(blocks["norm2"]);
+            auto conv1 = std::dynamic_pointer_cast<CausalConv3d>(blocks["conv1"]);
+            auto conv2 = std::dynamic_pointer_cast<CausalConv3d>(blocks["conv2"]);
+            auto h     = conv1->forward(ctx, ggml_silu(ctx->ggml_ctx, norm1->forward(ctx, x)));
+            h          = conv2->forward(ctx, ggml_silu(ctx->ggml_ctx, norm2->forward(ctx, h)));
+            if (in_channels != out_channels) {
+                x = std::dynamic_pointer_cast<CausalConv3d>(blocks["nin_shortcut"])->forward(ctx, x);
+            }
+            return ggml_add(ctx->ggml_ctx, x, h);
+        }
+    };
+
+    struct Encoder : public GGMLBlock {
+        static constexpr int levels                            = 6;
+        static constexpr std::array<int, levels> multipliers   = {1, 2, 2, 4, 4, 8};
+        static constexpr std::array<int, levels> spatial_down  = {2, 2, 2, 2, 1, 1};
+        static constexpr std::array<int, levels> temporal_down = {1, 2, 2, 1, 1, 1};
+
+        Encoder() {
+            constexpr int ch  = 128;
+            blocks["conv_in"] = std::make_shared<CausalConv3d>(3,
+                                                               ch,
+                                                               std::tuple{3, 3, 3},
+                                                               std::tuple{1, 1, 1},
+                                                               std::tuple{1, 1, 1});
+            int64_t previous  = ch;
+            for (int level = 0; level < levels; ++level) {
+                int64_t current = ch * multipliers[level];
+                for (int block = 0; block < 2; ++block) {
+                    blocks["down." + std::to_string(level) + ".block." + std::to_string(block)] =
+                        std::make_shared<ResnetBlock3D>(block == 0 ? previous : current,
+                                                        current);
+                }
+                if (spatial_down[level] * temporal_down[level] > 1) {
+                    blocks["down." + std::to_string(level) + ".downsample"] =
+                        std::make_shared<Downsample3D>(current,
+                                                       current,
+                                                       temporal_down[level],
+                                                       spatial_down[level]);
+                }
+                previous = current;
+            }
+            blocks["norm_out"] = std::make_shared<TemporalGroupNorm>(previous);
+            blocks["conv_out"] = std::make_shared<CausalConv3d>(previous,
+                                                                48,
+                                                                std::tuple{3, 3, 3},
+                                                                std::tuple{1, 1, 1},
+                                                                std::tuple{1, 1, 1});
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            x = std::dynamic_pointer_cast<CausalConv3d>(blocks["conv_in"])->forward(ctx, x);
+            for (int level = 0; level < levels; ++level) {
+                for (int block = 0; block < 2; ++block) {
+                    x = std::dynamic_pointer_cast<ResnetBlock3D>(
+                            blocks["down." + std::to_string(level) + ".block." + std::to_string(block)])
+                            ->forward(ctx, x);
+                }
+                auto downsample = blocks.find("down." + std::to_string(level) + ".downsample");
+                if (downsample != blocks.end()) {
+                    x = std::dynamic_pointer_cast<Downsample3D>(downsample->second)->forward(ctx, x);
+                }
+            }
+            auto norm = std::dynamic_pointer_cast<TemporalGroupNorm>(blocks["norm_out"]);
+            auto conv = std::dynamic_pointer_cast<CausalConv3d>(blocks["conv_out"]);
+            return conv->forward(ctx, ggml_silu(ctx->ggml_ctx, norm->forward(ctx, x)));
+        }
+    };
+
+    static ggml_tensor* attention_layout(ggml_context* ctx, ggml_tensor* x) {
+        x = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3));
+        return ggml_reshape_3d(ctx, x, x->ne[0], x->ne[1], x->ne[2] * x->ne[3]);
+    }
+
+    static ggml_tensor* apply_partial_rope(ggml_context* ctx,
+                                           ggml_tensor* x,
+                                           ggml_tensor* pe) {
+        int64_t rot_dim = pe->ne[2] * 2;
+        auto rotated    = Rope::apply_rope(ctx,
+                                           ggml_ext_slice(ctx, x, 0, 0, rot_dim),
+                                           pe,
+                                           false);
+        if (rot_dim == x->ne[0]) {
+            return rotated;
+        }
+        auto tail = attention_layout(ctx,
+                                     ggml_ext_slice(ctx, x, 0, rot_dim, x->ne[0]));
+        return ggml_concat(ctx, rotated, tail, 0);
+    }
+
+    struct DecoderAttention : public GGMLBlock {
+        static constexpr int num_head = 32;
+        static constexpr int head_dim = 64;
+        static constexpr int dim      = num_head * head_dim;
+
+        DecoderAttention() {
+            blocks["to_qkv"] = std::make_shared<Linear>(dim, dim * 3, true);
+            blocks["to_out"] = std::make_shared<Linear>(dim, dim, true);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx,
+                             ggml_tensor* x,
+                             ggml_tensor* pe) {
+            auto to_qkv         = std::dynamic_pointer_cast<Linear>(blocks["to_qkv"]);
+            auto to_out         = std::dynamic_pointer_cast<Linear>(blocks["to_out"]);
+            auto qkv_projection = to_qkv->forward(ctx, x);
+            int64_t sequence    = x->ne[1];
+            int64_t batch_size  = x->ne[2] * x->ne[3];
+            qkv_projection      = ggml_reshape_4d(ctx->ggml_ctx,
+                                                  qkv_projection,
+                                                  3 * head_dim,
+                                                  num_head,
+                                                  sequence,
+                                                  batch_size);
+            auto qkv            = ggml_ext_chunk(ctx->ggml_ctx, qkv_projection, 3, 0);
+            auto q              = ggml_reshape_4d(ctx->ggml_ctx,
+                                                  qkv[0],
+                                                  head_dim,
+                                                  num_head,
+                                                  sequence,
+                                                  batch_size);
+            auto k              = ggml_reshape_4d(ctx->ggml_ctx,
+                                                  qkv[1],
+                                                  head_dim,
+                                                  num_head,
+                                                  sequence,
+                                                  batch_size);
+            auto v              = ggml_reshape_4d(ctx->ggml_ctx,
+                                                  qkv[2],
+                                                  head_dim,
+                                                  num_head,
+                                                  sequence,
+                                                  batch_size);
+            q                   = ggml_rms_norm(ctx->ggml_ctx, q, 1e-5f);
+            k                   = ggml_rms_norm(ctx->ggml_ctx, k, 1e-5f);
+            q                   = apply_partial_rope(ctx->ggml_ctx, q, pe);
+            k                   = apply_partial_rope(ctx->ggml_ctx, k, pe);
+            auto out            = ggml_ext_attention_ext(ctx->ggml_ctx,
+                                                         ctx->backend,
+                                                         q,
+                                                         k,
+                                                         v,
+                                                         num_head,
+                                                         nullptr,
+                                                         true,
+                                                         ctx->flash_attn_enabled);
+            return to_out->forward(ctx, out);
+        }
+    };
+
+    struct DecoderFeedForward : public GGMLBlock {
+        static constexpr int dim       = 2048;
+        static constexpr int kInnerDim = dim * 4;
+
+        DecoderFeedForward() {
+            blocks["w1"] = std::make_shared<Linear>(dim, kInnerDim * 2, true);
+            blocks["w2"] = std::make_shared<Linear>(kInnerDim, dim, true);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            auto w1   = std::dynamic_pointer_cast<Linear>(blocks["w1"]);
+            auto w2   = std::dynamic_pointer_cast<Linear>(blocks["w2"]);
+            auto gate = ggml_ext_chunk(ctx->ggml_ctx, w1->forward(ctx, x), 2, 0);
+            return w2->forward(ctx,
+                               ggml_mul(ctx->ggml_ctx,
+                                        ggml_silu(ctx->ggml_ctx, gate[0]),
+                                        gate[1]));
+        }
+    };
+
+    struct DecoderBlock : public GGMLBlock {
+        static constexpr int dim = 2048;
+
+        DecoderBlock() {
+            blocks["norm1"] = std::make_shared<RMSNorm>(dim, 1e-5f);
+            blocks["attn"]  = std::make_shared<DecoderAttention>();
+            blocks["norm2"] = std::make_shared<RMSNorm>(dim, 1e-5f);
+            blocks["ff"]    = std::make_shared<DecoderFeedForward>();
+        }
+
+        void init_params(ggml_context* ctx,
+                         const String2TensorStorage& tensor_storage_map = {},
+                         const std::string prefix                       = "") override {
+            SD_UNUSED(tensor_storage_map);
+            SD_UNUSED(prefix);
+            params["scale1"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, dim);
+            params["scale2"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, dim);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx,
+                             ggml_tensor* x,
+                             ggml_tensor* pe) {
+            auto norm1 = std::dynamic_pointer_cast<RMSNorm>(blocks["norm1"]);
+            auto attn  = std::dynamic_pointer_cast<DecoderAttention>(blocks["attn"]);
+            auto norm2 = std::dynamic_pointer_cast<RMSNorm>(blocks["norm2"]);
+            auto ff    = std::dynamic_pointer_cast<DecoderFeedForward>(blocks["ff"]);
+            x          = ggml_add(ctx->ggml_ctx,
+                                  x,
+                                  ggml_mul(ctx->ggml_ctx,
+                                           attn->forward(ctx, norm1->forward(ctx, x), pe),
+                                           params["scale1"]));
+            return ggml_add(ctx->ggml_ctx,
+                            x,
+                            ggml_mul(ctx->ggml_ctx,
+                                     ff->forward(ctx, norm2->forward(ctx, x)),
+                                     params["scale2"]));
+        }
+    };
+
+    struct Decoder : public GGMLBlock {
+        static constexpr int dim                 = 2048;
+        static constexpr int num_layers          = 36;
+        static constexpr int num_register_tokens = 4;
+        static constexpr int patch_size          = 16;
+        static constexpr int patch_size_t        = 4;
+
+        Decoder() {
+            blocks["x_embedder"] = std::make_shared<Linear>(24, dim, true);
+            for (int i = 0; i < num_layers; ++i) {
+                blocks["transformer_blocks." + std::to_string(i)] =
+                    std::make_shared<DecoderBlock>();
+            }
+            blocks["norm_out"] = std::make_shared<LayerNorm>(dim, 1e-5f, true, true);
+            blocks["proj_out"] = std::make_shared<Linear>(dim,
+                                                          3 * patch_size_t * patch_size * patch_size,
+                                                          true,
+                                                          true);
+        }
+
+        void init_params(ggml_context* ctx,
+                         const String2TensorStorage& tensor_storage_map = {},
+                         const std::string prefix                       = "") override {
+            SD_UNUSED(tensor_storage_map);
+            SD_UNUSED(prefix);
+            params["register_tokens"] = ggml_new_tensor_2d(ctx,
+                                                           GGML_TYPE_F32,
+                                                           dim,
+                                                           num_register_tokens);
+            params["mask_token"]      = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, dim);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx,
+                             ggml_tensor* z,
+                             ggml_tensor* pe) {
+            int64_t width      = z->ne[0];
+            int64_t height     = z->ne[1];
+            int64_t num_frames = z->ne[2];
+            int64_t batch_size = z->ne[3] / 24;
+            GGML_ASSERT(batch_size == 1);
+
+            z                   = ggml_cont(ctx->ggml_ctx,
+                                            ggml_ext_torch_permute(ctx->ggml_ctx, z, 3, 0, 1, 2));
+            z                   = ggml_reshape_3d(ctx->ggml_ctx,
+                                                  z,
+                                                  24,
+                                                  width * height * num_frames,
+                                                  batch_size);
+            auto x_embedder     = std::dynamic_pointer_cast<Linear>(blocks["x_embedder"]);
+            auto h              = x_embedder->forward(ctx, z);
+            int64_t num_patches = h->ne[1];
+            h                   = ggml_concat(ctx->ggml_ctx, h, params["register_tokens"], 1);
+            auto zero           = ggml_ext_scale(ctx->ggml_ctx,
+                                                 ggml_ext_slice(ctx->ggml_ctx, h, 1, 0, 1),
+                                                 0.f);
+            h                   = ggml_concat(ctx->ggml_ctx, h, zero, 1);
+
+            for (int i = 0; i < num_layers; ++i) {
+                auto block = std::dynamic_pointer_cast<DecoderBlock>(
+                    blocks["transformer_blocks." + std::to_string(i)]);
+                h = block->forward(ctx, h, pe);
+                sd::ggml_graph_cut::mark_graph_cut(h,
+                                                   "minimax_h3_vae.decoder.blocks." + std::to_string(i),
+                                                   "hidden_states");
+            }
+
+            auto norm_out = std::dynamic_pointer_cast<LayerNorm>(blocks["norm_out"]);
+            auto proj_out = std::dynamic_pointer_cast<Linear>(blocks["proj_out"]);
+            h             = proj_out->forward(ctx, norm_out->forward(ctx, h));
+            h             = ggml_ext_slice(ctx->ggml_ctx, h, 1, 0, num_patches);
+            return DiT::unpatchify_3d(ctx->ggml_ctx,
+                                      h,
+                                      num_frames,
+                                      height,
+                                      width,
+                                      patch_size_t,
+                                      patch_size,
+                                      patch_size,
+                                      true);
+        }
+    };
+
+    struct MiniMaxH3VideoVAE : public GGMLBlock {
+        MiniMaxH3VideoVAE() {
+            blocks["encoder"]         = std::make_shared<Encoder>();
+            blocks["quant_conv"]      = std::make_shared<Conv3d>(48,
+                                                            48,
+                                                            std::tuple{1, 1, 1});
+            blocks["post_quant_conv"] = std::make_shared<Conv3d>(24,
+                                                                 24,
+                                                                 std::tuple{1, 1, 1});
+            blocks["decoder"]         = std::make_shared<Decoder>();
+        }
+
+        ggml_tensor* encode(GGMLRunnerContext* ctx,
+                            ggml_tensor* pixels,
+                            ggml_tensor* pixel_mean,
+                            ggml_tensor* pixel_std) {
+            pixels       = ggml_div(ctx->ggml_ctx,
+                                    ggml_sub(ctx->ggml_ctx, pixels, pixel_mean),
+                                    pixel_std);
+            auto encoder = std::dynamic_pointer_cast<Encoder>(blocks["encoder"]);
+            auto quant   = std::dynamic_pointer_cast<Conv3d>(blocks["quant_conv"]);
+            auto moments = quant->forward(ctx, encoder->forward(ctx, pixels));
+            return ggml_ext_slice(ctx->ggml_ctx, moments, 3, 0, 24);
+        }
+
+        ggml_tensor* decode(GGMLRunnerContext* ctx,
+                            ggml_tensor* latent,
+                            ggml_tensor* pe,
+                            ggml_tensor* pixel_mean,
+                            ggml_tensor* pixel_std) {
+            auto post_quant = std::dynamic_pointer_cast<Conv3d>(blocks["post_quant_conv"]);
+            auto decoder    = std::dynamic_pointer_cast<Decoder>(blocks["decoder"]);
+            auto pixels     = decoder->forward(ctx, post_quant->forward(ctx, latent), pe);
+            pixels          = ggml_add(ctx->ggml_ctx,
+                                       ggml_mul(ctx->ggml_ctx, pixels, pixel_std),
+                                       pixel_mean);
+            return ggml_clamp(ctx->ggml_ctx, pixels, 0.f, 1.f);
+        }
+    };
+
+    struct MiniMaxH3VideoVAERunner : public VAE {
+        MiniMaxH3VideoVAE model;
+        sd::Tensor<float> pixel_mean;
+        sd::Tensor<float> pixel_std;
+        sd::Tensor<float> latents_mean;
+        sd::Tensor<float> latents_std;
+        sd::Tensor<float> rope_cache;
+
+        MiniMaxH3VideoVAERunner(ggml_backend_t backend,
+                                const String2TensorStorage& tensor_storage_map,
+                                const std::string& prefix                           = "first_stage_model",
+                                std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+            : VAE(VERSION_MINIMAX_H3, backend, prefix, weight_manager),
+              pixel_mean({1, 1, 1, 3}, {0.485f, 0.456f, 0.406f}),
+              pixel_std({1, 1, 1, 3}, {0.229f, 0.224f, 0.225f}),
+              latents_mean({1, 1, 1, 24},
+                           {0.858090341091156f, -0.960659146308899f, 1.066164016723633f, -0.509032547473907f,
+                            -0.272758185863495f, -1.367541432380676f, -0.255325496196747f, -0.269075542688370f,
+                            -0.537684082984924f, -0.046409729868174f, 0.665737032890320f, 0.196901276707649f,
+                            -0.546060800552368f, -0.403534203767776f, -0.236830249428749f, 0.259284526109695f,
+                            -0.301339447498322f, 0.211341992020607f, -1.120684862136841f, 0.358193337917328f,
+                            -0.042251437902451f, 0.260482996702194f, 0.228640928864479f, 0.705603182315826f}),
+              latents_std({1, 1, 1, 24},
+                          {1.222377419471741f, 1.276726365089417f, 1.683177471160889f, 1.754945516586304f,
+                           1.563621640205383f, 2.194143533706665f, 0.965313792228699f, 1.056988596916199f,
+                           0.841948926448822f, 0.772995293140411f, 1.895593762397766f, 0.946841835975647f,
+                           0.799680948257446f, 0.449889004230499f, 0.719739973545075f, 0.693629324436188f,
+                           2.961095094680786f, 2.769419908523560f, 3.049618482589722f, 2.108805418014527f,
+                           3.276226282119751f, 3.162735700607300f, 2.281681299209595f, 2.612784385681153f}) {
+            scale_input = false;
+            model.init(params_ctx, tensor_storage_map, prefix);
+        }
+
+        std::string get_desc() override {
+            return "minimax_h3_video_vae";
+        }
+
+        int get_encoder_output_channels(int input_channels) override {
+            SD_UNUSED(input_channels);
+            return 24;
+        }
+
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
+            model.get_param_tensors(tensors, weight_prefix);
+        }
+
+        sd::Tensor<float> vae_output_to_latents(const sd::Tensor<float>& vae_output,
+                                                std::shared_ptr<RNG> rng) override {
+            SD_UNUSED(rng);
+            return vae_output;
+        }
+
+        sd::Tensor<float> diffusion_to_vae_latents(const sd::Tensor<float>& latents) override {
+            return latents * latents_std + latents_mean;
+        }
+
+        sd::Tensor<float> vae_to_diffusion_latents(const sd::Tensor<float>& latents) override {
+            return (latents - latents_mean) / latents_std;
+        }
+
+        static sd::Tensor<float> ensure_video_shape(const sd::Tensor<float>& tensor) {
+            if (tensor.dim() == 5) {
+                return tensor;
+            }
+            GGML_ASSERT(tensor.dim() == 4);
+            return tensor.reshape({tensor.shape()[0],
+                                   tensor.shape()[1],
+                                   1,
+                                   tensor.shape()[2],
+                                   tensor.shape()[3]});
+        }
+
+        static sd_tiling_params_t h3_tiling(sd_tiling_params_t params) {
+            params.enabled        = true;
+            params.tile_size_x    = 16;
+            params.tile_size_y    = 16;
+            params.target_overlap = 0.25f;
+            return params;
+        }
+
+        static sd::Tensor<float> repeat_last_frame(const sd::Tensor<float>& input,
+                                                   int64_t count) {
+            auto result = input;
+            auto last   = sd::ops::slice(input, 2, input.shape()[2] - 1, input.shape()[2]);
+            for (int64_t i = 0; i < count; ++i) {
+                result = sd::ops::concat(result, last, 2);
+            }
+            return result;
+        }
+
+        static sd::Tensor<float> blend_temporal(const sd::Tensor<float>& previous,
+                                                const sd::Tensor<float>& current,
+                                                int64_t extent) {
+            auto output            = current;
+            extent                 = std::min({extent, previous.shape()[2], current.shape()[2]});
+            int64_t previous_start = previous.shape()[2] - extent;
+            for (int64_t b = 0; b < current.shape()[4]; ++b) {
+                for (int64_t c = 0; c < current.shape()[3]; ++c) {
+                    for (int64_t t = 0; t < extent; ++t) {
+                        float wb = static_cast<float>(t) / extent;
+                        float wa = 1.f - wb;
+                        for (int64_t h = 0; h < current.shape()[1]; ++h) {
+                            for (int64_t w = 0; w < current.shape()[0]; ++w) {
+                                output.index(w, h, t, c, b) =
+                                    previous.index(w, h, previous_start + t, c, b) * wa +
+                                    current.index(w, h, t, c, b) * wb;
+                            }
+                        }
+                    }
+                }
+            }
+            return output;
+        }
+
+        sd::Tensor<float> encode(int n_threads,
+                                 const sd::Tensor<float>& x,
+                                 sd_tiling_params_t tiling_params,
+                                 bool circular_x = false,
+                                 bool circular_y = false) override {
+            auto input  = ensure_video_shape(x);
+            auto tiling = h3_tiling(tiling_params);
+            if (input.shape()[2] == 1) {
+                auto encoded = VAE::encode(n_threads, input, tiling, circular_x, circular_y);
+                if (!encoded.empty() && encoded.shape()[2] > 1) {
+                    encoded = sd::ops::slice(encoded,
+                                             2,
+                                             encoded.shape()[2] - 1,
+                                             encoded.shape()[2]);
+                }
+                return encoded;
+            }
+
+            int64_t pad = (-input.shape()[2]) % 17;
+            if (pad < 0) {
+                pad += 17;
+            }
+            if (pad > 0) {
+                input = repeat_last_frame(input, pad);
+            }
+            sd::Tensor<float> result;
+            for (int64_t start = 0; start < input.shape()[2]; start += 17) {
+                auto chunk   = sd::ops::slice(input, 2, start, start + 17);
+                auto encoded = VAE::encode(n_threads, chunk, tiling, circular_x, circular_y);
+                if (encoded.empty()) {
+                    return {};
+                }
+                result = result.empty() ? std::move(encoded)
+                                        : sd::ops::concat(result, encoded, 2);
+            }
+            if (result.shape()[2] > 3) {
+                result = sd::ops::slice(result, 2, 0, result.shape()[2] - 3);
+            }
+            return result;
+        }
+
+        sd::Tensor<float> decode(int n_threads,
+                                 const sd::Tensor<float>& x,
+                                 sd_tiling_params_t tiling_params,
+                                 bool decode_video = false,
+                                 bool circular_x   = false,
+                                 bool circular_y   = false,
+                                 bool silent       = false) override {
+            auto input  = ensure_video_shape(x);
+            auto tiling = h3_tiling(tiling_params);
+            if (input.shape()[2] == 1) {
+                auto decoded = VAE::decode(n_threads,
+                                           input,
+                                           tiling,
+                                           decode_video,
+                                           circular_x,
+                                           circular_y,
+                                           silent);
+                if (!decoded.empty() && decoded.shape()[2] > 1) {
+                    decoded = sd::ops::slice(decoded,
+                                             2,
+                                             decoded.shape()[2] - 1,
+                                             decoded.shape()[2]);
+                }
+                return decoded;
+            }
+
+            constexpr int64_t tokens_per_chunk  = 5;
+            constexpr int64_t token_drop        = 3;
+            constexpr int64_t token_overlap     = 2;
+            constexpr int64_t frames_per_chunk  = 20;
+            constexpr int64_t frame_pre_padding = 3;
+            constexpr int64_t frame_overlap     = 5;
+
+            int64_t pseudo_tokens = input.shape()[2] + token_drop;
+            int64_t pad_tokens    = (tokens_per_chunk - pseudo_tokens % tokens_per_chunk) % tokens_per_chunk;
+            pseudo_tokens += pad_tokens;
+            int64_t num_chunks = pseudo_tokens / tokens_per_chunk - 1;
+            if (num_chunks < 1) {
+                pad_tokens += tokens_per_chunk;
+                num_chunks += 1;
+            }
+            if (pad_tokens > 0) {
+                input = repeat_last_frame(input, pad_tokens);
+            }
+
+            sd::Tensor<float> result;
+            sd::Tensor<float> overlap;
+            for (int64_t i = 0; i < num_chunks; ++i) {
+                int64_t start = i * tokens_per_chunk;
+                int64_t end   = std::min(start + tokens_per_chunk + token_overlap,
+                                         input.shape()[2]);
+                auto chunk    = sd::ops::slice(input, 2, start, end);
+                auto decoded  = VAE::decode(n_threads,
+                                            chunk,
+                                            tiling,
+                                            true,
+                                            circular_x,
+                                            circular_y,
+                                            silent);
+                if (decoded.empty()) {
+                    return {};
+                }
+
+                int64_t first_end = std::min<int64_t>(frames_per_chunk, decoded.shape()[2]);
+                auto first        = sd::ops::slice(decoded,
+                                                   2,
+                                                   std::min<int64_t>(frame_pre_padding, first_end),
+                                                   first_end);
+                if (!overlap.empty()) {
+                    first   = blend_temporal(overlap, first, frame_overlap);
+                    overlap = {};
+                }
+                result = result.empty() ? std::move(first)
+                                        : sd::ops::concat(result, first, 2);
+
+                if (decoded.shape()[2] > frames_per_chunk + frame_pre_padding) {
+                    overlap = sd::ops::slice(decoded,
+                                             2,
+                                             frames_per_chunk + frame_pre_padding,
+                                             decoded.shape()[2]);
+                }
+                if (i == num_chunks - 1 && !overlap.empty()) {
+                    result  = sd::ops::concat(result, overlap, 2);
+                    overlap = {};
+                }
+            }
+
+            int64_t expected_frames = input.shape()[2] <= 1 ? 1 : ((x.shape()[2] - 2) / 5) * 17 + 5;
+            expected_frames         = std::max<int64_t>(1, expected_frames);
+            if (result.shape()[2] > expected_frames) {
+                result = sd::ops::slice(result, 2, 0, expected_frames);
+            }
+            return result;
+        }
+
+        sd::Tensor<float> build_rope(int64_t width,
+                                     int64_t height,
+                                     int64_t num_frames) {
+            std::vector<std::vector<float>> ids;
+            ids.reserve(static_cast<size_t>(width * height * num_frames + 5));
+            constexpr float two_pi = 6.28318530717958647692f;
+            for (int64_t t = 0; t < num_frames; ++t) {
+                float pt = (2.f * ((t + 0.5f) / num_frames) - 1.f) * two_pi;
+                for (int64_t h = 0; h < height; ++h) {
+                    float ph = (2.f * ((h + 0.5f) / height) - 1.f) * two_pi;
+                    for (int64_t w = 0; w < width; ++w) {
+                        float pw = (2.f * ((w + 0.5f) / width) - 1.f) * two_pi;
+                        ids.push_back({pt, ph, pw});
+                    }
+                }
+            }
+            for (int i = 0; i < 5; ++i) {
+                ids.push_back({0.f, 0.f, 0.f});
+            }
+            auto values = Rope::embed_nd(ids,
+                                         1,
+                                         100.f,
+                                         std::vector<int>{16, 16, 16});
+            return sd::Tensor<float>({2,
+                                      2,
+                                      24,
+                                      static_cast<int64_t>(ids.size())},
+                                     std::move(values));
+        }
+
+        sd::Tensor<float> _compute(const int n_threads,
+                                   const sd::Tensor<float>& z,
+                                   bool decode_graph) override {
+            auto input = ensure_video_shape(z);
+            if (decode_graph) {
+                rope_cache = build_rope(input.shape()[0],
+                                        input.shape()[1],
+                                        input.shape()[2]);
+            }
+            auto get_graph = [&]() -> ggml_cgraph* {
+                auto value       = make_input(input);
+                auto mean        = make_input(pixel_mean);
+                auto std         = make_input(pixel_std);
+                auto runner_ctx  = get_context();
+                ggml_tensor* out = nullptr;
+                if (decode_graph) {
+                    auto pe = make_input(rope_cache);
+                    out     = model.decode(&runner_ctx, value, pe, mean, std);
+                } else {
+                    out = model.encode(&runner_ctx, value, mean, std);
+                }
+                auto graph = new_graph_custom(H3_VIDEO_VAE_GRAPH_SIZE);
+                ggml_build_forward_expand(graph, out);
+                return graph;
+            };
+            return restore_trailing_singleton_dims(
+                GGMLRunner::compute<float>(get_graph,
+                                           n_threads,
+                                           false,
+                                           false,
+                                           false),
+                5);
+        }
+    };
+
+}  // namespace MiniMaxH3VAE
+
+#endif  // __SD_MODEL_VAE_MINIMAX_H3_VAE_HPP__

src/model/vae/tae.hpp

@@ -426,10 +426,10 @@ class TinyVideoDecoder : public UnaryBlock {
     static const int num_layers  = 3;
     int channels[num_layers + 1] = {256, 128, 64, 64};
     int patch_size               = 1;
-    int t_upscale                = 1;
     bool is_wide                 = false;
 
 public:
+    int t_upscale = 1;
     TinyVideoDecoder(int z_channels = 4, int patch_size = 1, std::vector<bool> time_upscale = {false, true, true}, bool is_wide = false)
         : z_channels(z_channels), patch_size(patch_size), is_wide(is_wide) {
         t_upscale = 1;
@@ -536,6 +536,10 @@ public:
             patch          = 4;
             time_downscale = {true, true, true};
             time_upscale   = {true, true, true};
+        } else if (sd_version_is_minimax_h3(version)) {
+            z_channels     = 24;
+            patch          = 2;
+            time_downscale = {true, true, false};
         }
         blocks["decoder"] = std::shared_ptr<GGMLBlock>(new TinyVideoDecoder(z_channels, patch, time_upscale, is_wide));
         if (!decode_only) {
@@ -545,24 +549,123 @@ public:
 
     ggml_tensor* decode(GGMLRunnerContext* ctx, ggml_tensor* z) {
         auto decoder = std::dynamic_pointer_cast<TinyVideoDecoder>(blocks["decoder"]);
-        if (sd_version_is_wan(version) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version)) {
+        if (sd_version_is_wan(version) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version) || sd_version_is_minimax_h3(version)) {
             // (W, H, C, T) -> (W, H, T, C)
             z = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 1, 3, 2));
         }
         auto result = decoder->forward(ctx, z);
-        if (sd_version_is_wan(version) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version)) {
+
+        if (sd_version_is_minimax_h3(version)) {
+            int64_t num_frames   = result->ne[3];
+            int64_t chunk_frames = 5 * decoder->t_upscale;
+            int64_t pad          = (chunk_frames - (num_frames % chunk_frames)) % chunk_frames;
+
+            result = ggml_ext_pad_ext(ctx->ggml_ctx, ctx->backend, result, 0, 0, 0, 0, 0, 0, 0, pad, false, false);
+
+            int64_t num_chunks                  = (num_frames + pad) / chunk_frames;
+            auto to_trim                        = decoder->t_upscale - 1;
+            std::vector<ggml_tensor*> to_concat = {};
+            for (int i = 0; i < num_chunks; i++) {
+                auto chunk = ggml_view_4d(ctx->ggml_ctx, result,
+                                          result->ne[0], result->ne[1], result->ne[2], chunk_frames - to_trim,
+                                          result->nb[1], result->nb[2], result->nb[3],
+                                          i * chunk_frames * result->nb[3]);
+                to_concat.push_back(chunk);
+            }
+            result = ggml_ext_vec_concat(ctx->ggml_ctx, to_concat, 3);
+            result = ggml_view_4d(ctx->ggml_ctx, result,
+                                  result->ne[0], result->ne[1], result->ne[2],
+                                  result->ne[3] - decoder->t_upscale * 3,
+                                  result->nb[1], result->nb[2], result->nb[3], 0);
+        }
+
+        if (sd_version_is_wan(version) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version) || sd_version_is_minimax_h3(version)) {
             // (W, H, T, C) -> (W, H, C, T)
             result = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, result, 0, 1, 3, 2));
         }
         return result;
     }
 
-    ggml_tensor* encode(GGMLRunnerContext* ctx, ggml_tensor* x) {
+    ggml_tensor* encode_h3(GGMLRunnerContext* ctx, ggml_tensor* x) {
         auto encoder = std::dynamic_pointer_cast<TinyVideoEncoder>(blocks["encoder"]);
-        if (sd_version_is_wan(version) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version)) {
+
+        int64_t num_frames = x->ne[3];
+        int64_t pad        = (17 - (num_frames % 17)) % 17;
+
+        if (pad > 0) {
+            auto last_frame = ggml_view_4d(ctx->ggml_ctx, x,
+                                           x->ne[0], x->ne[1], x->ne[2], 1,
+                                           x->nb[1], x->nb[2], x->nb[3],
+                                           (num_frames - 1) * x->nb[3]);
+            for (int i = 0; i < pad; i++) {
+                x = ggml_concat(ctx->ggml_ctx, x, last_frame, 3);
+            }
+        }
+
+        int64_t T_padded   = x->ne[3];
+        int64_t num_chunks = T_padded / 17;
+
+        auto zero_frame = ggml_view_4d(ctx->ggml_ctx, x,
+                                       x->ne[0], x->ne[1], x->ne[2], 1,
+                                       x->nb[1], x->nb[2], x->nb[3], 0);
+        auto zeros_1    = ggml_scale(ctx->ggml_ctx, ggml_cont(ctx->ggml_ctx, zero_frame), 0.0f);
+        auto zeros_3    = zeros_1;
+        for (int i = 1; i < 3; i++) {
+            zeros_3 = ggml_concat(ctx->ggml_ctx, zeros_3, zeros_1, 3);
+        }
+        ggml_tensor* out = nullptr;
+        if (false) {
+            std::vector<ggml_tensor*> to_concat = {};
+            for (int i = 0; i < num_chunks; i++) {
+                auto chunk = ggml_view_4d(ctx->ggml_ctx, x,
+                                          x->ne[0], x->ne[1], x->ne[2], 17,
+                                          x->nb[1], x->nb[2], x->nb[3],
+                                          i * 17 * x->nb[3]);
+
+                auto chunk_padded = ggml_concat(ctx->ggml_ctx, zeros_3, chunk, 3);
+
+                to_concat.push_back(chunk_padded);
+            }
+            ggml_tensor* x_in = ggml_ext_vec_concat(ctx->ggml_ctx, to_concat, 3);
+            out               = encoder->forward(ctx, x_in);
+        } else {
+            std::vector<ggml_tensor*> to_concat = {};
+            for (int i = 0; i < num_chunks; i++) {
+                auto chunk = ggml_view_4d(ctx->ggml_ctx, x,
+                                          x->ne[0], x->ne[1], x->ne[2], 17,
+                                          x->nb[1], x->nb[2], x->nb[3],
+                                          i * 17 * x->nb[3]);
+
+                auto chunk_padded = ggml_concat(ctx->ggml_ctx, zeros_3, chunk, 3);
+
+                auto chunk_out = encoder->forward(ctx, chunk_padded);
+                // auto chunk_out = encoder->forward_seq(ctx, chunk_padded); // ~same vram usage, and straight-up slower. it's already sequential enough
+
+                to_concat.push_back(chunk_out);
+            }
+            out = ggml_ext_vec_concat(ctx->ggml_ctx, to_concat, 3);
+        }
+
+        // Return x[:, :-3] - drop the last 3 elements in the T dimension
+        int64_t out_T = out->ne[3];
+        out           = ggml_view_4d(ctx->ggml_ctx, out,
+                                     out->ne[0], out->ne[1], out->ne[2], out_T - 3,
+                                     out->nb[1], out->nb[2], out->nb[3], 0);
+
+        return ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, out, 0, 1, 3, 2));
+    }
+
+    ggml_tensor* encode(GGMLRunnerContext* ctx, ggml_tensor* x) {
+        if (sd_version_is_wan(version) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version) || (sd_version_is_minimax_h3(version) && x->ne[3] > 1)) {
             // (W, H, T, C) -> (W, H, C, T)
             x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 0, 1, 3, 2));
         }
+        if (sd_version_is_minimax_h3(version)) {
+            return encode_h3(ctx, x);
+        }
+
+        auto encoder = std::dynamic_pointer_cast<TinyVideoEncoder>(blocks["encoder"]);
+
         int64_t num_frames = x->ne[3];
         if (num_frames % encoder->t_downscale) {
             // pad to multiple of encoder->t_downscale at the end

src/model/vae/vae.hpp

@@ -74,7 +74,7 @@ public:
         int scale_factor = 8;
         if (version == VERSION_LTXAV) {
             scale_factor = 32;
-        } else if (version == VERSION_WAN2_2_TI2V || sd_version_is_hunyuan_video(version) || sd_version_is_mage_flow(version)) {
+        } else if (version == VERSION_WAN2_2_TI2V || sd_version_is_hunyuan_video(version) || sd_version_is_mage_flow(version) || sd_version_is_minimax_h3(version)) {
             scale_factor = 16;
         } else if (sd_version_uses_flux2_vae(version)) {
             scale_factor = 16;
@@ -115,11 +115,11 @@ public:
         tile_size_y = get_tile_size(params.tile_size_y, params.rel_size_y, latent_y);
     }
 
-    sd::Tensor<float> encode(int n_threads,
-                             const sd::Tensor<float>& x,
-                             sd_tiling_params_t tiling_params,
-                             bool circular_x = false,
-                             bool circular_y = false) {
+    virtual sd::Tensor<float> encode(int n_threads,
+                                     const sd::Tensor<float>& x,
+                                     sd_tiling_params_t tiling_params,
+                                     bool circular_x = false,
+                                     bool circular_y = false) {
         int64_t t0              = ggml_time_ms();
         sd::Tensor<float> input = x;
         sd::Tensor<float> output;
@@ -136,7 +136,8 @@ public:
             // Image VAE encode is more sensitive to tile boundary context than decode.
             // Keep the smaller legacy factor for video VAEs, but default image encode
             // tiles to 64 latent pixels so a 512px SD image is encoded as one tile.
-            const float encode_tile_factor = (sd_version_is_wan(version) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version)) ? 1.30539f : 2.0f;
+            const float encode_tile_factor = sd_version_is_minimax_h3(version) ? 1.f : (sd_version_is_wan(version) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version)) ? 1.30539f
+                                                                                                                                                                                            : 2.0f;
             get_tile_sizes(tile_size_x, tile_size_y, tile_overlap, tiling_params, W, H, encode_tile_factor);
             LOG_DEBUG("VAE Tile size: %dx%d", tile_size_x, tile_size_y);
             output = tiled_compute(input,
@@ -155,7 +156,7 @@ public:
             output = _compute(n_threads, input, false);
         }
 
-        free_compute_buffer();
+        runner_done();
 
         if (output.empty()) {
             LOG_ERROR("vae encode compute failed");
@@ -166,13 +167,13 @@ public:
         return std::move(output);
     }
 
-    sd::Tensor<float> decode(int n_threads,
-                             const sd::Tensor<float>& x,
-                             sd_tiling_params_t tiling_params,
-                             bool decode_video = false,
-                             bool circular_x   = false,
-                             bool circular_y   = false,
-                             bool silent       = false) {
+    virtual sd::Tensor<float> decode(int n_threads,
+                                     const sd::Tensor<float>& x,
+                                     sd_tiling_params_t tiling_params,
+                                     bool decode_video = false,
+                                     bool circular_x   = false,
+                                     bool circular_y   = false,
+                                     bool silent       = false) {
         int64_t t0              = ggml_time_ms();
         sd::Tensor<float> input = x;
         sd::Tensor<float> output;
@@ -206,7 +207,7 @@ public:
             output = _compute(n_threads, input, true);
         }
 
-        free_compute_buffer();
+        runner_done();
 
         if (output.empty()) {
             LOG_ERROR("vae decode compute failed");

src/model_io/pickle_io.cpp

@@ -2,6 +2,7 @@
 
 #include <cstdlib>
 #include <cstring>
+#include <limits>
 #include <string>
 #include <unordered_map>
 #include <utility>
@@ -512,8 +513,51 @@ static bool parse_storage_type(const std::string& global_name, PickleStorageInfo
     return false;
 }
 
-static bool tensor_is_contiguous(const PickleTensorInfo& tensor) {
-    if (tensor.tensor_storage.nelements() == 0) {
+static bool checked_pickle_byte_count(int64_t element_count,
+                                      uint64_t element_nbytes,
+                                      uint64_t* byte_count) {
+    if (element_count < 0 || element_nbytes == 0) {
+        return false;
+    }
+
+    uint64_t count = static_cast<uint64_t>(element_count);
+    if (count > std::numeric_limits<uint64_t>::max() / element_nbytes) {
+        return false;
+    }
+
+    *byte_count = count * element_nbytes;
+    return true;
+}
+
+static bool tensor_layout_is_valid(const PickleTensorInfo& tensor, uint64_t raw_element_nbytes) {
+    if (raw_element_nbytes == 0) {
+        return false;
+    }
+
+    bool has_zero_dimension = false;
+    uint64_t element_count  = 1;
+    for (int i = 0; i < tensor.tensor_storage.n_dims; ++i) {
+        int64_t dimension = tensor.tensor_storage.ne[i];
+        if (dimension < 0) {
+            return false;
+        }
+        if (dimension == 0) {
+            has_zero_dimension = true;
+            continue;
+        }
+
+        uint64_t size = static_cast<uint64_t>(dimension);
+        if (element_count > static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) / size) {
+            return false;
+        }
+        element_count *= size;
+    }
+
+    if (!has_zero_dimension &&
+        element_count > static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) / raw_element_nbytes) {
+        return false;
+    }
+    if (has_zero_dimension) {
         return true;
     }
     if (tensor.stride_n_dims != tensor.tensor_storage.n_dims) {
@@ -932,7 +976,12 @@ bool parse_torch_state_dict_pickle(const uint8_t* buffer,
                 if (storage.key.empty() || !parse_storage_type(pid.items[1].str_value, &storage)) {
                     return false;
                 }
-                storage.nbytes              = (uint64_t)pid.items[4].int_value * storage.raw_element_nbytes;
+                if (!checked_pickle_byte_count(pid.items[4].int_value,
+                                               storage.raw_element_nbytes,
+                                               &storage.nbytes)) {
+                    set_error(error, "invalid storage size in torch pickle");
+                    return false;
+                }
                 storage_nbytes[storage.key] = storage.nbytes;
                 stack.push_back(make_storage_value(storage));
             } break;
@@ -963,7 +1012,12 @@ bool parse_torch_state_dict_pickle(const uint8_t* buffer,
                     tensor.tensor_storage.is_f64      = args.items[0].storage.is_f64;
                     tensor.tensor_storage.is_i64      = args.items[0].storage.is_i64;
                     tensor.tensor_storage.storage_key = args.items[0].storage.key;
-                    tensor.tensor_storage.offset      = (uint64_t)args.items[1].int_value * args.items[0].storage.raw_element_nbytes;
+                    if (!checked_pickle_byte_count(args.items[1].int_value,
+                                                   args.items[0].storage.raw_element_nbytes,
+                                                   &tensor.tensor_storage.offset)) {
+                        set_error(error, "invalid tensor storage offset in torch pickle");
+                        return false;
+                    }
 
                     for (const auto& item : args.items[2].items) {
                         if (item.kind != PickleValue::INT || tensor.tensor_storage.n_dims >= SD_MAX_DIMS) {
@@ -979,7 +1033,8 @@ bool parse_torch_state_dict_pickle(const uint8_t* buffer,
                         tensor.stride[tensor.stride_n_dims++] = item.int_value;
                     }
 
-                    if (!tensor_is_contiguous(tensor)) {
+                    if (!tensor_layout_is_valid(tensor, args.items[0].storage.raw_element_nbytes)) {
+                        set_error(error, "invalid tensor shape or stride in torch pickle");
                         return false;
                     }
                     stack.push_back(make_tensor_value(tensor));

src/model_io/safetensors_io.cpp

@@ -7,6 +7,7 @@
 #include <fstream>
 #include <ostream>
 #include <string>
+#include <unordered_map>
 #include <unordered_set>
 #include <vector>
 
@@ -86,17 +87,59 @@ static ggml_type safetensors_dtype_to_ggml_type(const std::string& dtype) {
     } else if (dtype == "F64") {
         ttype = GGML_TYPE_F32;
     } else if (dtype == "F8_E4M3") {
-        ttype = GGML_TYPE_F16;
+        ttype = GGML_TYPE_F8_E4M3;
     } else if (dtype == "F8_E5M2") {
-        ttype = GGML_TYPE_F16;
+        ttype = GGML_TYPE_F8_E5M2;
     } else if (dtype == "I32") {
         ttype = GGML_TYPE_I32;
     } else if (dtype == "I64") {
         ttype = GGML_TYPE_I32;
+    } else if (dtype == "I8") {
+        ttype = GGML_TYPE_I8;
     }
     return ttype;
 }
 
+struct ComfyQuantConfig {
+    std::string format;
+    bool convrot   = false;
+    int group_size = 0;
+};
+
+static bool read_comfy_quant_config(std::ifstream& file,
+                                    const std::string& file_path,
+                                    const std::string& tensor_name,
+                                    size_t offset,
+                                    size_t size,
+                                    ComfyQuantConfig& config,
+                                    std::string* error) {
+    static constexpr size_t MAX_COMFY_QUANT_CONFIG_SIZE = 64 * 1024;
+    if (size == 0 || size > MAX_COMFY_QUANT_CONFIG_SIZE) {
+        set_error(error, "invalid ComfyUI quantization metadata tensor '" + tensor_name + "' in '" + file_path + "'");
+        return false;
+    }
+
+    std::vector<char> data(size + 1, '\0');
+    file.clear();
+    file.seekg((std::streamoff)offset, std::ios::beg);
+    file.read(data.data(), (std::streamsize)size);
+    if (!file) {
+        set_error(error, "read ComfyUI quantization metadata tensor failed: '" + tensor_name + "'");
+        return false;
+    }
+
+    try {
+        const nlohmann::json json = nlohmann::json::parse(data.data(), data.data() + size);
+        config.format             = json.value("format", "");
+        config.convrot            = json.value("convrot", false);
+        config.group_size         = json.value("convrot_groupsize", 0);
+    } catch (const std::exception&) {
+        set_error(error, "parsing ComfyUI quantization metadata tensor failed: '" + tensor_name + "'");
+        return false;
+    }
+    return true;
+}
+
 // https://huggingface.co/docs/safetensors/index
 bool read_safetensors_file(const std::string& file_path,
                            std::vector<TensorStorage>& tensor_storages,
@@ -163,6 +206,33 @@ bool read_safetensors_file(const std::string& file_path,
         }
     }
 
+    std::unordered_map<std::string, ComfyQuantConfig> comfy_quant_configs;
+    for (const auto& item : header_.items()) {
+        const std::string& name = item.key();
+        if (name == "__metadata__" || !ends_with(name, ".comfy_quant")) {
+            continue;
+        }
+
+        const nlohmann::json& tensor_info = item.value();
+        if (tensor_info.value("dtype", "") != "U8") {
+            set_error(error, "invalid dtype for ComfyUI quantization metadata tensor '" + name + "'");
+            return false;
+        }
+        const size_t begin = tensor_info["data_offsets"][0].get<size_t>();
+        const size_t end   = tensor_info["data_offsets"][1].get<size_t>();
+        if (begin > end || end > file_size_ - data_start) {
+            set_error(error, "data offsets out of bounds for tensor '" + name + "'");
+            return false;
+        }
+
+        ComfyQuantConfig config;
+        if (!read_comfy_quant_config(file, file_path, name, data_start + begin, end - begin, config, error)) {
+            return false;
+        }
+        const std::string module_name = name.substr(0, name.size() - std::string(".comfy_quant").size());
+        comfy_quant_configs.emplace(module_name, std::move(config));
+    }
+
     tensor_storages.clear();
     for (auto& item : header_.items()) {
         std::string name           = item.key();
@@ -220,17 +290,48 @@ bool read_safetensors_file(const std::string& file_path,
         TensorStorage tensor_storage(name, type, ne, n_dims, 0, data_start + begin);
         tensor_storage.reverse_ne();
 
+        if (ends_with(name, ".weight")) {
+            const std::string module_name = name.substr(0, name.size() - std::string(".weight").size());
+            auto config                   = comfy_quant_configs.find(module_name);
+            if (config != comfy_quant_configs.end() && config->second.format == "int8_tensorwise") {
+                if (type != GGML_TYPE_I8) {
+                    set_error(error, "ComfyUI int8_tensorwise weight is not I8: '" + name + "'");
+                    return false;
+                }
+                if (config->second.convrot) {
+                    int group_size_remainder = config->second.group_size;
+                    while (group_size_remainder > 1 && group_size_remainder % 4 == 0) {
+                        group_size_remainder /= 4;
+                    }
+                    if (group_size_remainder != 1 || tensor_storage.ne[0] % config->second.group_size != 0) {
+                        set_error(error, "invalid ComfyUI convrot group size for tensor '" + name + "'");
+                        return false;
+                    }
+                }
+                tensor_storage.is_int8_tensorwise      = true;
+                tensor_storage.int8_convrot            = config->second.convrot;
+                tensor_storage.int8_convrot_group_size = config->second.group_size;
+            }
+        } else if (ends_with(name, ".weight_scale")) {
+            const std::string module_name = name.substr(0, name.size() - std::string(".weight_scale").size());
+            auto config                   = comfy_quant_configs.find(module_name);
+            if (config != comfy_quant_configs.end() && config->second.format == "int8_tensorwise" &&
+                tensor_storage.n_dims == 2 && tensor_storage.ne[0] == 1) {
+                tensor_storage.ne[0]  = tensor_storage.ne[1];
+                tensor_storage.ne[1]  = 1;
+                tensor_storage.n_dims = 1;
+            }
+        }
+
         size_t tensor_data_size = end - begin;
 
         bool tensor_size_ok;
         if (dtype == "F8_E4M3") {
             tensor_storage.is_f8_e4m3 = true;
-            // f8 -> f16
-            tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size * 2);
+            tensor_size_ok            = (tensor_storage.nbytes() == tensor_data_size);
         } else if (dtype == "F8_E5M2") {
             tensor_storage.is_f8_e5m2 = true;
-            // f8 -> f16
-            tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size * 2);
+            tensor_size_ok            = (tensor_storage.nbytes() == tensor_data_size);
         } else if (dtype == "F64") {
             tensor_storage.is_f64 = true;
             // f64 -> f32

src/model_io/tensor_storage.h

@@ -15,14 +15,17 @@
 
 struct TensorStorage {
     std::string name;
-    ggml_type type          = GGML_TYPE_F32;
-    ggml_type expected_type = GGML_TYPE_COUNT;
-    bool is_f8_e4m3         = false;
-    bool is_f8_e5m2         = false;
-    bool is_f64             = false;
-    bool is_i64             = false;
-    int64_t ne[SD_MAX_DIMS] = {1, 1, 1, 1, 1};
-    int n_dims              = 0;
+    ggml_type type              = GGML_TYPE_F32;
+    ggml_type expected_type     = GGML_TYPE_COUNT;
+    bool is_f8_e4m3             = false;
+    bool is_f8_e5m2             = false;
+    bool is_f64                 = false;
+    bool is_i64                 = false;
+    bool is_int8_tensorwise     = false;
+    bool int8_convrot           = false;
+    int int8_convrot_group_size = 0;
+    int64_t ne[SD_MAX_DIMS]     = {1, 1, 1, 1, 1};
+    int n_dims                  = 0;
 
     std::string storage_key;
     size_t file_index = 0;
@@ -51,9 +54,7 @@ struct TensorStorage {
     }
 
     int64_t nbytes_to_read() const {
-        if (is_f8_e4m3 || is_f8_e5m2) {
-            return nbytes() / 2;
-        } else if (is_f64 || is_i64) {
+        if (is_f64 || is_i64) {
             return nbytes() * 2;
         } else {
             return nbytes();

src/model_io/torch_legacy_io.cpp

@@ -139,11 +139,16 @@ bool read_torch_legacy_file(const std::string& file_path,
             if (it == legacy_storage_map.end()) {
                 return false;
             }
-            if (current_offset + LEGACY_STORAGE_HEADER_SIZE + it->second > file_size) {
+            if (current_offset > file_size ||
+                LEGACY_STORAGE_HEADER_SIZE > file_size - current_offset) {
                 return false;
             }
-            storage_offsets[storage_key] = current_offset + LEGACY_STORAGE_HEADER_SIZE;
-            current_offset += LEGACY_STORAGE_HEADER_SIZE + it->second;
+            uint64_t storage_offset = current_offset + LEGACY_STORAGE_HEADER_SIZE;
+            if (it->second > file_size - storage_offset) {
+                return false;
+            }
+            storage_offsets[storage_key] = storage_offset;
+            current_offset               = storage_offset + it->second;
         }
 
         for (auto& tensor_storage : tensor_storages) {
@@ -159,8 +164,10 @@ bool read_torch_legacy_file(const std::string& file_path,
 
             uint64_t base_offset    = it_offset->second;
             uint64_t storage_nbytes = it_size->second;
-            uint64_t tensor_nbytes  = tensor_storage.nbytes_to_read();
-            if (tensor_storage.offset + tensor_nbytes > storage_nbytes) {
+            int64_t tensor_nbytes   = tensor_storage.nbytes_to_read();
+            if (tensor_nbytes < 0 ||
+                tensor_storage.offset > storage_nbytes ||
+                static_cast<uint64_t>(tensor_nbytes) > storage_nbytes - tensor_storage.offset) {
                 return false;
             }
 

src/model_io/torch_zip_io.cpp

@@ -76,8 +76,10 @@ static bool parse_zip_data_pkl(const uint8_t* buffer,
             return false;
         }
 
-        uint64_t tensor_nbytes = tensor_storage.nbytes_to_read();
-        if (tensor_storage.offset + tensor_nbytes > entry_size) {
+        int64_t tensor_nbytes = tensor_storage.nbytes_to_read();
+        if (tensor_nbytes < 0 ||
+            tensor_storage.offset > entry_size ||
+            static_cast<uint64_t>(tensor_nbytes) > entry_size - tensor_storage.offset) {
             set_error(error, "tensor '" + tensor_storage.name + "' exceeds storage entry '" + entry_name + "'");
             return false;
         }

src/model_loader.cpp

@@ -78,66 +78,6 @@ bool is_unused_tensor(const std::string& name) {
     return false;
 }
 
-uint16_t f8_e4m3_to_f16(uint8_t f8) {
-    // do we need to support uz?
-
-    const uint32_t exponent_bias = 7;
-    if (f8 == 0xff) {
-        return ggml_fp32_to_fp16(-NAN);
-    } else if (f8 == 0x7f) {
-        return ggml_fp32_to_fp16(NAN);
-    }
-
-    uint32_t sign     = f8 & 0x80;
-    uint32_t exponent = (f8 & 0x78) >> 3;
-    uint32_t mantissa = f8 & 0x07;
-    uint32_t result   = sign << 24;
-    if (exponent == 0) {
-        if (mantissa > 0) {
-            exponent = 0x7f - exponent_bias;
-
-            // yes, 2 times
-            if ((mantissa & 0x04) == 0) {
-                mantissa &= 0x03;
-                mantissa <<= 1;
-                exponent -= 1;
-            }
-            if ((mantissa & 0x04) == 0) {
-                mantissa &= 0x03;
-                mantissa <<= 1;
-                exponent -= 1;
-            }
-
-            result |= (mantissa & 0x03) << 21;
-            result |= exponent << 23;
-        }
-    } else {
-        result |= mantissa << 20;
-        exponent += 0x7f - exponent_bias;
-        result |= exponent << 23;
-    }
-
-    return ggml_fp32_to_fp16(*reinterpret_cast<const float*>(&result));
-}
-
-uint16_t f8_e5m2_to_f16(uint8_t fp8) {
-    return static_cast<uint16_t>(fp8) << 8;
-}
-
-void f8_e4m3_to_f16_vec(uint8_t* src, uint16_t* dst, int64_t n) {
-    // support inplace op
-    for (int64_t i = n - 1; i >= 0; i--) {
-        dst[i] = f8_e4m3_to_f16(src[i]);
-    }
-}
-
-void f8_e5m2_to_f16_vec(uint8_t* src, uint16_t* dst, int64_t n) {
-    // support inplace op
-    for (int64_t i = n - 1; i >= 0; i--) {
-        dst[i] = f8_e5m2_to_f16(src[i]);
-    }
-}
-
 void f64_to_f32_vec(double* src, float* dst, int64_t n) {
     // support inplace op
     for (int64_t i = 0; i < n; i++) {
@@ -537,6 +477,10 @@ SDVersion ModelLoader::get_sd_version() {
         if (tensor_storage.name.find("model.diffusion_model.adaln_single.emb.timestep_embedder.linear_1.bias") != std::string::npos) {
             return VERSION_LTXAV;
         }
+        if (tensor_storage.name.find("model.diffusion_model.video_patch_proj.weight") != std::string::npos &&
+            tensor_storage_map.find("model.diffusion_model.audio_patch_proj.weight") != tensor_storage_map.end()) {
+            return VERSION_MINIMAX_H3;
+        }
         if (tensor_storage.name.find("model.diffusion_model.blocks.0.cross_attn.norm_k.weight") != std::string::npos) {
             is_wan = true;
         }
@@ -925,9 +869,7 @@ std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggm
             if (dst_tensor == nullptr)
                 continue;
 
-            if (tensor_storage.is_f8_e4m3 ||
-                tensor_storage.is_f8_e5m2 ||
-                tensor_storage.is_f64 ||
+            if (tensor_storage.is_f64 ||
                 tensor_storage.is_i64 ||
                 tensor_storage.type != dst_tensor->type) {
                 continue;
@@ -1053,7 +995,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
         std::atomic<size_t> tensor_idx(0);
         std::atomic<bool> failed(false);
         std::vector<std::thread> workers;
-        std::mutex rpc_backend_mutex;
+        std::mutex backend_tensor_set_mutex;
 
         for (int i = 0; i < n_threads; ++i) {
             workers.emplace_back([&, file_path, is_zip]() {
@@ -1077,6 +1019,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
 
                 std::vector<uint8_t> read_buffer;
                 std::vector<uint8_t> convert_buffer;
+                std::vector<uint8_t> zip_entry_buffer;
 
                 while (true) {
                     int64_t t0, t1;
@@ -1115,34 +1058,60 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
 
                     size_t nbytes_to_read = tensor_storage.nbytes_to_read();
 
-                    auto read_data = [&](char* buf, size_t n) {
+                    auto read_data = [&](char* buf, size_t n) -> bool {
                         if (zip != nullptr) {
-                            zip_entry_openbyindex(zip, tensor_storage.index_in_zip);
+                            if (zip_entry_openbyindex(zip, tensor_storage.index_in_zip) != 0) {
+                                LOG_ERROR("failed to open zip entry for tensor '%s'", tensor_storage.name.c_str());
+                                return false;
+                            }
                             size_t entry_size = zip_entry_size(zip);
+                            if (tensor_storage.offset > entry_size) {
+                                LOG_ERROR("tensor '%s' exceeds its zip storage entry", tensor_storage.name.c_str());
+                                zip_entry_close(zip);
+                                return false;
+                            }
+                            size_t tensor_offset = static_cast<size_t>(tensor_storage.offset);
+                            if (n > entry_size - tensor_offset) {
+                                LOG_ERROR("tensor '%s' exceeds its zip storage entry", tensor_storage.name.c_str());
+                                zip_entry_close(zip);
+                                return false;
+                            }
+
                             if (entry_size != n) {
                                 int64_t t_memcpy_start;
-                                read_buffer.resize(entry_size);
-                                zip_entry_noallocread(zip, (void*)read_buffer.data(), entry_size);
+                                zip_entry_buffer.resize(entry_size);
+                                auto bytes_read = zip_entry_noallocread(zip, (void*)zip_entry_buffer.data(), entry_size);
+                                if (bytes_read < 0 || static_cast<size_t>(bytes_read) != entry_size) {
+                                    LOG_ERROR("failed to read zip entry for tensor '%s'", tensor_storage.name.c_str());
+                                    zip_entry_close(zip);
+                                    return false;
+                                }
                                 t_memcpy_start = ggml_time_ms();
-                                memcpy((void*)buf, (void*)(read_buffer.data() + tensor_storage.offset), n);
+                                memcpy((void*)buf, (void*)(zip_entry_buffer.data() + tensor_offset), n);
                                 memcpy_time_ms.fetch_add(ggml_time_ms() - t_memcpy_start);
                             } else {
-                                zip_entry_noallocread(zip, (void*)buf, n);
+                                auto bytes_read = zip_entry_noallocread(zip, (void*)buf, n);
+                                if (bytes_read < 0 || static_cast<size_t>(bytes_read) != n) {
+                                    LOG_ERROR("failed to read zip entry for tensor '%s'", tensor_storage.name.c_str());
+                                    zip_entry_close(zip);
+                                    return false;
+                                }
                             }
                             zip_entry_close(zip);
                         } else if (mmapped) {
                             if (!mmapped->copy_data(buf, n, tensor_storage.offset)) {
                                 LOG_ERROR("read tensor data failed: '%s'", file_path.c_str());
-                                failed = true;
+                                return false;
                             }
                         } else {
                             file.seekg(tensor_storage.offset);
                             file.read(buf, n);
                             if (!file) {
                                 LOG_ERROR("read tensor data failed: '%s'", file_path.c_str());
-                                failed = true;
+                                return false;
                             }
                         }
+                        return true;
                     };
 
                     char* read_buf    = nullptr;
@@ -1176,16 +1145,15 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
                     }
 
                     t0 = ggml_time_ms();
-                    read_data(read_buf, nbytes_to_read);
+                    if (!read_data(read_buf, nbytes_to_read)) {
+                        failed = true;
+                        break;
+                    }
                     t1 = ggml_time_ms();
                     read_time_ms.fetch_add(t1 - t0);
 
                     t0 = ggml_time_ms();
-                    if (tensor_storage.is_f8_e4m3) {
-                        f8_e4m3_to_f16_vec((uint8_t*)read_buf, (uint16_t*)target_buf, tensor_storage.nelements());
-                    } else if (tensor_storage.is_f8_e5m2) {
-                        f8_e5m2_to_f16_vec((uint8_t*)read_buf, (uint16_t*)target_buf, tensor_storage.nelements());
-                    } else if (tensor_storage.is_f64) {
+                    if (tensor_storage.is_f64) {
                         f64_to_f32_vec((double*)read_buf, (float*)target_buf, tensor_storage.nelements());
                     } else if (tensor_storage.is_i64) {
                         i64_to_i32_vec((int64_t*)read_buf, (int32_t*)target_buf, tensor_storage.nelements());
@@ -1214,17 +1182,8 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
                     if (dst_tensor->buffer != nullptr && !ggml_backend_buffer_is_host(dst_tensor->buffer)) {
                         t0 = ggml_time_ms();
 
-                        // RPC backends require serialized access to prevent concurrency issues
-                        const char* buffer_type_name = ggml_backend_buft_name(ggml_backend_buffer_get_type(dst_tensor->buffer));
-                        bool is_rpc_buffer           = buffer_type_name != nullptr &&
-                                             std::string(buffer_type_name).find("RPC") != std::string::npos;
-
-                        if (is_rpc_buffer) {
-                            std::lock_guard<std::mutex> lock(rpc_backend_mutex);
-                            ggml_backend_tensor_set(dst_tensor, convert_buf, 0, ggml_nbytes(dst_tensor));
-                        } else {
-                            ggml_backend_tensor_set(dst_tensor, convert_buf, 0, ggml_nbytes(dst_tensor));
-                        }
+                        std::lock_guard<std::mutex> lock(backend_tensor_set_mutex);
+                        ggml_backend_tensor_set(dst_tensor, convert_buf, 0, ggml_nbytes(dst_tensor));
 
                         t1 = ggml_time_ms();
                         copy_to_backend_time_ms.fetch_add(t1 - t0);
@@ -1483,6 +1442,9 @@ bool ModelLoader::load_tensors(std::map<std::string, ggml_tensor*>& tensors,
 
 bool ModelLoader::tensor_should_be_converted(const TensorStorage& tensor_storage, ggml_type type) {
     const std::string& name = tensor_storage.name;
+    if (tensor_storage.is_int8_tensorwise) {
+        return false;
+    }
     if (type != GGML_TYPE_COUNT) {
         if (ggml_is_quantized(type) && tensor_storage.ne[0] % ggml_blck_size(type) != 0) {
             // Pass, do not convert

src/model_manager.cpp

@@ -53,6 +53,48 @@ static bool backend_supports_host_buffer(ggml_backend_t backend) {
     return props.caps.buffer_from_host_ptr;
 }
 
+static bool device_supports_param_op(ggml_backend_dev_t device,
+                                     ggml_tensor* weight,
+                                     enum ggml_op op,
+                                     ggml_backend_buffer_type_t buft) {
+    if (op == GGML_OP_NONE) {
+        return true;
+    }
+    if (device == nullptr || weight == nullptr || buft == nullptr || weight->buffer != nullptr) {
+        return false;
+    }
+
+    ggml_init_params params;
+    params.mem_size   = ggml_tensor_overhead() * 2;
+    params.mem_buffer = nullptr;
+    params.no_alloc   = true;
+    ggml_context* ctx = ggml_init(params);
+    if (ctx == nullptr) {
+        return false;
+    }
+
+    ggml_tensor* op_tensor = nullptr;
+    if (op == GGML_OP_GET_ROWS) {
+        ggml_tensor* indices = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1);
+        op_tensor            = ggml_get_rows(ctx, weight, indices);
+    }
+    if (op_tensor == nullptr) {
+        ggml_free(ctx);
+        return false;
+    }
+
+    weight->buffer = ggml_backend_buft_alloc_buffer(buft, 0);
+    if (weight->buffer == nullptr) {
+        ggml_free(ctx);
+        return false;
+    }
+    bool supported = ggml_backend_dev_supports_op(device, op_tensor);
+    ggml_backend_buffer_free(weight->buffer);
+    weight->buffer = nullptr;
+    ggml_free(ctx);
+    return supported;
+}
+
 ModelManager::~ModelManager() {
     release_all();
 }
@@ -135,7 +177,8 @@ bool ModelManager::register_param_tensors(const std::string& desc,
                                           ggml_backend_t params_backend,
                                           size_t* registered_tensor_size,
                                           bool allow_split_buffer,
-                                          bool params_follow_compute_backend) {
+                                          bool params_follow_compute_backend,
+                                          const std::map<ggml_tensor*, enum ggml_op>* tensor_ops) {
     if (desc.empty()) {
         LOG_ERROR("model manager tensor desc is empty");
         return false;
@@ -168,6 +211,12 @@ bool ModelManager::register_param_tensors(const std::string& desc,
         state->params_backend                = params_backend;
         state->allow_split_buffer            = allow_split_buffer;
         state->params_follow_compute_backend = params_follow_compute_backend;
+        if (tensor_ops != nullptr) {
+            auto op_it = tensor_ops->find(tensor);
+            if (op_it != tensor_ops->end()) {
+                state->usage_op = op_it->second;
+            }
+        }
         new_states.push_back(std::move(state));
     }
 
@@ -844,6 +893,22 @@ ggml_backend_buffer_type_t ModelManager::params_buffer_type_for(const TensorStat
     if (params_buft == nullptr) {
         params_buft = ggml_backend_get_default_buffer_type(state.params_backend);
     }
+    if (state.usage_op != GGML_OP_NONE &&
+        state.compute_backend != nullptr) {
+        ggml_backend_dev_t compute_dev = ggml_backend_get_device(state.compute_backend);
+        if (device_supports_param_op(compute_dev, state.tensor, state.usage_op, params_buft)) {
+            return params_buft;
+        }
+
+        ggml_backend_dev_t cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
+        params_buft                = cpu_dev != nullptr ? ggml_backend_dev_buffer_type(cpu_dev) : nullptr;
+        if (!device_supports_param_op(cpu_dev, state.tensor, state.usage_op, params_buft)) {
+            LOG_ERROR("model manager has no compatible buffer for tensor '%s' used by %s",
+                      state.name.c_str(),
+                      ggml_op_name(state.usage_op));
+            return nullptr;
+        }
+    }
     return params_buft;
 }
 

src/model_manager.h

@@ -39,6 +39,7 @@ private:
         bool allow_split_buffer            = false;
         bool params_follow_compute_backend = false;
         bool metadata_validated            = false;
+        enum ggml_op usage_op              = GGML_OP_NONE;
 
         int active_prepare_count = 0;
 
@@ -130,9 +131,10 @@ public:
                                 ResidencyMode residency_mode,
                                 ggml_backend_t compute_backend,
                                 ggml_backend_t params_backend,
-                                size_t* registered_tensor_size     = nullptr,
-                                bool allow_split_buffer            = false,
-                                bool params_follow_compute_backend = false);
+                                size_t* registered_tensor_size                         = nullptr,
+                                bool allow_split_buffer                                = false,
+                                bool params_follow_compute_backend                     = false,
+                                const std::map<ggml_tensor*, enum ggml_op>* tensor_ops = nullptr);
 
     bool unregister_param_tensors(const std::string& desc,
                                   size_t* registered_tensor_size = nullptr);

src/name_conversion.cpp

@@ -149,6 +149,7 @@ std::string convert_cond_stage_model_name(std::string name, std::string prefix)
         {"ffn_up.", "mlp.up_proj."},
         {"ffn_post_norm.", "post_ffw_norm."},
         {"ffn_norm.", "post_attention_layernorm."},
+        {"layer_output_scale.weight", "layer_scalar"},
         {"output_norm.", "model.norm."},
     };
 
@@ -185,6 +186,20 @@ std::string convert_cond_stage_model_name(std::string name, std::string prefix)
 }
 
 std::string convert_qwen3_vl_vision_name(std::string name) {
+    static const std::vector<std::pair<std::string, std::string>> qwen3_vl_deepstack_name_map{
+        {"v.deepstack_merger_list.", "deepstack_merger_list."},
+        {"v.deepstack.5.", "deepstack_merger_list.0."},
+        {"v.deepstack.8.", "deepstack_merger_list.0."},
+        {"v.deepstack.11.", "deepstack_merger_list.1."},
+        {"v.deepstack.16.", "deepstack_merger_list.1."},
+        {"v.deepstack.17.", "deepstack_merger_list.2."},
+        {"v.deepstack.24.", "deepstack_merger_list.2."},
+        {"fc1.", "linear_fc1."},
+        {"fc2.", "linear_fc2."},
+        {"ffn_up.", "linear_fc1."},
+        {"ffn_down.", "linear_fc2."},
+        {"ffn_norm.", "norm."},
+    };
     static const std::vector<std::pair<std::string, std::string>> qwen3_vl_vision_name_map{
         {"mm.0.", "merger.linear_fc1."},
         {"mm.2.", "merger.linear_fc2."},
@@ -201,6 +216,10 @@ std::string convert_qwen3_vl_vision_name(std::string name) {
         {"ln1.", "norm1."},
         {"ln2.", "norm2."},
     };
+    if (contains(name, "v.deepstack_merger_list.") || contains(name, "v.deepstack.")) {
+        replace_with_name_map(name, qwen3_vl_deepstack_name_map);
+        return name;
+    }
     replace_with_name_map(name, qwen3_vl_vision_name_map);
     return name;
 }
@@ -1048,7 +1067,7 @@ std::string convert_diffusers_to_original_wan_vae(std::string name) {
 }
 
 std::string convert_first_stage_model_name(std::string name, std::string prefix, SDVersion version) {
-    if (sd_version_is_hunyuan_video(version) || sd_version_is_mage_flow(version)) {
+    if (sd_version_is_hunyuan_video(version) || sd_version_is_mage_flow(version) || sd_version_is_minimax_h3(version)) {
         return name;
     }
     if (sd_version_uses_wan_vae(version)) {
@@ -1181,6 +1200,7 @@ std::string convert_sep_to_dot(std::string name) {
         "x_embedder",
         "cross_attn",
         "output_proj",
+        "token_refiner",
     };
 
     // record the positions of underscores that should NOT be replaced
@@ -1384,6 +1404,8 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
             {".lora_B.weight", ".weight.lora_up"},
             {".lora_A.default.weight", ".weight.lora_down"},
             {".lora_B.default.weight", ".weight.lora_up"},
+            {".lora_A", ".weight.lora_down"},
+            {".lora_B", ".weight.lora_up"},
             {".lora_linear", ".weight.alpha"},
             {".alpha", ".weight.alpha"},
             {".scale", ".weight.scale"},
@@ -1438,6 +1460,7 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
         {"unet.", "model.diffusion_model."},
         {"transformer.", "model.diffusion_model."},  // dit
         {"vae.", "first_stage_model."},
+        {"text_encoders.llm.text_embedding_projection.", "text_embedding_projection."},
         {"text_encoder.", "cond_stage_model.transformer."},
         {"te.", "cond_stage_model.transformer."},
         {"text_encoder.2.", "cond_stage_model.1.transformer."},
@@ -1449,15 +1472,24 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
         {"te2.", "cond_stage_model.1.transformer."},
         {"te1.", "cond_stage_model.transformer."},
         {"te3.", "text_encoders.t5xxl.transformer."},
+        {"clip_vision.", "cond_stage_model.transformer."},
     };
 
     if (sd_version_is_flux(version)) {
         prefix_map["te1."] = "text_encoders.clip_l.transformer.";
     }
 
+    if (sd_version_is_unet(version)) {
+        prefix_map["clip_l."] = "cond_stage_model.transformer.";
+        prefix_map["clip_g."] = "cond_stage_model.1.transformer.";
+    } else {
+        prefix_map["clip_l."] = "text_encoders.clip_l.transformer.";
+        prefix_map["clip_g."] = "text_encoders.clip_g.transformer.";
+    }
+
     replace_with_prefix_map(name, prefix_map);
 
-    if (sd_version_is_boogu_image(version) || sd_version_is_krea2(version) || sd_version_is_mage_flow(version)) {
+    if (sd_version_is_boogu_image(version) || sd_version_is_krea2(version) || sd_version_is_mage_flow(version) || sd_version_is_minimax_h3(version)) {
         const std::string hf_vision_prefix = "text_encoders.llm.model.visual.";
         if (starts_with(name, hf_vision_prefix)) {
             name = "text_encoders.llm.visual." + name.substr(hf_vision_prefix.size());
@@ -1539,6 +1571,11 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
         }
     }
 
+    static const std::vector<std::pair<std::string, std::string>> generic_name_map = {
+        {".scale_weight", ".weight_scale"},
+    };
+    replace_with_name_map(name, generic_name_map);
+
     if (is_lora) {
         name = "lora." + name;
     }

src/runtime/denoiser.hpp

@@ -306,8 +306,33 @@ struct KarrasScheduler : SigmaScheduler {
 };
 
 struct BetaScheduler : SigmaScheduler {
-    static constexpr double alpha = 0.6;
-    static constexpr double beta  = 0.6;
+    double alpha = 0.6;
+    double beta  = 0.6;
+
+    explicit BetaScheduler(const char* extra_sample_args = nullptr) {
+        parse_extra_sample_args(extra_sample_args);
+        LOG_DEBUG("Beta scheduler: alpha=%.4f, beta=%.4f", alpha, beta);
+    }
+
+    void parse_extra_sample_args(const char* extra_sample_args) {
+        for (const auto& [key, value] : parse_key_value_args(extra_sample_args, "beta scheduler arg")) {
+            if (key == "alpha") {
+                float parsed;
+                if (!parse_strict_float(value, parsed) || parsed <= 0.0) {
+                    LOG_WARN("ignoring invalid beta scheduler arg '%s=%s'", key.c_str(), value.c_str());
+                } else {
+                    alpha = static_cast<double>(parsed);
+                }
+            } else if (key == "beta") {
+                float parsed;
+                if (!parse_strict_float(value, parsed) || parsed <= 0.0) {
+                    LOG_WARN("ignoring invalid beta scheduler arg '%s=%s'", key.c_str(), value.c_str());
+                } else {
+                    beta = static_cast<double>(parsed);
+                }
+            }
+        }
+    }
 
     static double log_beta(double a, double b) {
         return std::lgamma(a) + std::lgamma(b) - std::lgamma(a + b);
@@ -1032,7 +1057,7 @@ struct Denoiser {
                 break;
             case BETA_SCHEDULER:
                 LOG_INFO("get_sigmas with Beta scheduler");
-                scheduler = std::make_shared<BetaScheduler>();
+                scheduler = std::make_shared<BetaScheduler>(extra_sample_args);
                 break;
             case EXPONENTIAL_SCHEDULER:
                 LOG_INFO("get_sigmas exponential scheduler");
@@ -2553,6 +2578,112 @@ static sd::Tensor<float> sample_tcd(denoise_cb_t model,
     return x;
 }
 
+static sd::Tensor<float> sample_lms(denoise_cb_t model,
+                                    sd::Tensor<float> x,
+                                    const std::vector<float>& sigmas,
+                                    const SamplerExtraArgs& extra_sample_args) {
+    // Linear Multi-Step from https://github.com/crowsonkb/k-diffusion,
+    // modified with "history shift" value, which seemingly needs less steps
+    int divisions = 1000;
+    int max_order = 4;
+    int shift     = 1;  // 4, 0 - original; 4, 1 - PR #1843; 3, 1 - smoother image
+    for (const auto& [key, value] : extra_sample_args) {
+        int parsed = 0;
+        if (key == "lms_max_order") {
+            if (!parse_strict_int(value, parsed)) {
+                LOG_WARN("ignoring invalid lms extra sample arg '%s=%s'", key.c_str(), value.c_str());
+                continue;
+            }
+            max_order = std::max(1, parsed);
+            // smaller values make the result softer, closer to Euler
+            // higher values need more steps
+            // values above 12 can produce NaNs, depending on steps and scheduler
+        }
+        if (key == "lms_shift") {
+            if (!parse_strict_int(value, parsed)) {
+                LOG_WARN("ignoring invalid lms extra sample arg '%s=%s'", key.c_str(), value.c_str());
+                continue;
+            }
+            shift = std::max(0, parsed);
+            // for a low number of steps, the value 1 works best
+        }
+        if (key == "lms_divisions") {
+            if (!parse_strict_int(value, parsed)) {
+                LOG_WARN("ignoring invalid lms extra sample arg '%s=%s'", key.c_str(), value.c_str());
+                continue;
+            }
+            divisions = parsed;  // std::max(1, parsed);
+            // values < 1 always produce noise
+            // values above 30M require double precision in the integrator
+            // (they are needless and just slow the integration down, but
+            //  with single precision they softly produce noise
+            //  near the 35M, it can be used for distorted generations)
+        }
+    }
+
+    auto linear_multistep_coeff = [=](const int order, const int m, const int j) -> float {
+        if (!divisions)
+            return sigmas[m + 1] - sigmas[m];  // delta / 0 * 0
+#define LMS_PRECISION float                    // when divisions > 30 millions, the double precision fixes noise
+        const LMS_PRECISION a = sigmas[m], dx = (sigmas[m + 1] - a) / divisions, s = sigmas[m - j];
+        const LMS_PRECISION b0 = a + 0.5f * dx;  // using Riemann middle integral
+        LMS_PRECISION sum      = 0.0f;
+        for (int h = 0; h < divisions; h++) {
+            const LMS_PRECISION b = h * dx + b0;
+            LMS_PRECISION prod    = 1.0f;
+            for (int k = 0; k < j; k++) {
+                const LMS_PRECISION t = sigmas[m - k];
+                prod *= (b - t) / (s - t);
+            }
+            for (int k = j + 1; k < order; k++) {
+                const LMS_PRECISION t = sigmas[m - k];
+                prod *= (b - t) / (s - t);
+            }
+            sum += prod;
+        }
+        return sum * dx;
+    };
+
+    int steps = static_cast<int>(sigmas.size()) - 1;
+    max_order = std::min(max_order, steps);  // history can not be larger than steps
+    LOG_DEBUG("linear multi-step sampler: lms_max_order = %i, lms_shift = %i, lms_divisions = %i", max_order, shift, divisions);
+    std::vector<float> lms_coeff(max_order);
+    std::vector<sd::Tensor<float>> hist = {};
+
+    for (int i = 0; i < steps; i++) {
+        const float sigma = sigmas[i];
+
+        auto denoised_opt = model(x, sigma, i + 1);
+        if (denoised_opt.pred.empty()) {
+            return {};
+        }
+        sd::Tensor<float> denoised = std::move(denoised_opt.pred);
+
+        const int order = std::min(max_order, i + 1);
+
+        for (int c = 0; c < order; c++)  // computing coefficients
+            lms_coeff[c] = linear_multistep_coeff(order, i, c);
+
+        sd::Tensor<float> d_cur = (x - denoised) / sigma;
+        x += d_cur * lms_coeff[0];
+        if (max_order > 1) {  // if max_order == 1, the history is not used (order always < 2)
+            int hist_size_p1 = hist.size() + 1;
+            if (i) {  // history does not exist at 1st step
+                int hist_max = hist.size() - 1;
+                for (int c = 2; c <= order; c++)
+                    x += hist[std::min(hist_max, hist_size_p1 - c + shift)] * lms_coeff[c - 1];
+                // max_order == 4  =>  hist[] index = 2, 1, 0
+                // shift == 1      =>  hist[] index = 2, 2, 1
+            }
+            if (hist_size_p1 == max_order) {
+                hist.erase(hist.begin());
+            }
+            hist.push_back(std::move(d_cur));
+        }
+    }
+    return x;
+}
+
 static sd::Tensor<float> sample_euler_cfg_pp(denoise_cb_t model,
                                              sd::Tensor<float> x,
                                              const std::vector<float>& sigmas) {
@@ -2714,6 +2845,8 @@ static sd::Tensor<float> sample_k_diffusion(sample_method_t method,
             return sample_euler_ancestral(model, std::move(x), sigmas, rng, is_flow_denoiser, eta);
         case TCD_SAMPLE_METHOD:
             return sample_tcd(model, std::move(x), sigmas, rng, eta);
+        case LMS_SAMPLE_METHOD:
+            return sample_lms(model, std::move(x), sigmas, extra_args);
         case EULER_CFG_PP_SAMPLE_METHOD:
             return sample_euler_cfg_pp(model, std::move(x), sigmas);
         case EULER_A_CFG_PP_SAMPLE_METHOD:

src/runtime/latent-preview.h

@@ -4,6 +4,33 @@
 #include "core/tensor.hpp"
 #include "ggml.h"
 
+const float minimax_latent_rgb_proj[24][3] = {
+    {0.19819857f, 0.11584999f, 0.07929777f},
+    {-0.16047224f, -0.10601170f, -0.15996324f},
+    {0.47391951f, 0.37602475f, 0.20267826f},
+    {-0.09857441f, -0.27435449f, -0.51681751f},
+    {-0.18930605f, -0.10512278f, -0.28571478f},
+    {-0.15639569f, -0.18000929f, -0.25432852f},
+    {-0.07176921f, -0.10901598f, -0.06654253f},
+    {-0.05014077f, -0.05839826f, -0.05516087f},
+    {-0.05201424f, -0.04351913f, -0.01507579f},
+    {0.24750438f, 0.13307422f, 0.17684120f},
+    {0.07377446f, 0.10235858f, 0.11707827f},
+    {0.02908304f, 0.06587022f, 0.10643690f},
+    {-0.00670531f, -0.03857879f, 0.01750151f},
+    {-0.07119107f, -0.03083323f, -0.01995450f},
+    {-0.08612627f, -0.07253841f, -0.01442890f},
+    {0.08793202f, 0.08681750f, 0.02994647f},
+    {0.00876893f, 0.02721868f, 0.00091178f},
+    {-0.03484412f, -0.02711262f, -0.00110101f},
+    {-0.00679772f, -0.01844275f, -0.01683359f},
+    {0.04287028f, 0.01601068f, 0.04037397f},
+    {-0.00493432f, -0.00230528f, 0.00353911f},
+    {0.01495088f, 0.00292306f, 0.00416671f},
+    {0.00495307f, 0.05066542f, 0.05210543f},
+    {-0.02154842f, -0.01518524f, 0.00442402f}};
+float minimax_latent_rgb_bias[3] = {0.07776964f, -0.01580954f, -0.06561434f};
+
 const float ltxav_latent_rgb_proj[128][3] = {
     {-0.0293802f, -0.0362516f, -0.0291386f},
     {0.0117735f, 0.0223435f, 0.018856f},

src/tokenizers/bpe_tokenizer.cpp

@@ -205,7 +205,7 @@ std::vector<int> BPETokenizer::encode(const std::string& text, on_new_token_cb_t
         ss << "\"" << token << "\", ";
     }
     ss << "]";
-    LOG_DEBUG("split prompt \"%s\" to tokens %s", text.c_str(), ss.str().c_str());
+    LOG_DEBUG("split prompt \"%s\" to %zu tokens %s", text.c_str(), bpe_tokens.size(), ss.str().c_str());
     return bpe_tokens;
 }