fix: reuse graph cut plans across CFG passes (#1943)

Co-authored-by: leejet <leejet714@gmail.com>

.gitignore

@@ -1,6 +1,7 @@
 build*/
 cmake-build-*/
 test/
+tests/
 .vscode/
 .idea/
 .cache/

CMakeLists.txt

@@ -290,6 +290,10 @@ else()
     add_library(${SD_LIB} STATIC ${SD_LIB_SOURCES})
 endif()
 
+if(MSVC)
+    target_compile_options(${SD_LIB} PRIVATE $<$<COMPILE_LANGUAGE:CXX>:/bigobj>)
+endif()
+
 if(APPLE)
     sd_set_macos_rpaths(${SD_LIB})
 endif()

README.md

@@ -15,6 +15,7 @@ API and command-line option may change frequently.***
 
 ## 🔥Important News
 
+* **2026/08/20** 🚀 stable-diffusion.cpp now supports **LTX-2.5**
 * **2026/08/04** 🚀 stable-diffusion.cpp adds **Day-1 support for MiniMax-H3**
 * **2026/06/25** 🚀 stable-diffusion.cpp now supports **Krea2**
 * **2026/06/04** 🚀 stable-diffusion.cpp now supports **Ideogram4**
@@ -68,7 +69,7 @@ API and command-line option may change frequently.***
   - Video Models
     - [Wan2.1/Wan2.2](./docs/wan.md)
     - [MiniMax-H3](./docs/minimax_h3.md)
-    - [LTX-2.3](./docs/ltx2.md)
+    - [LTX-2.3/LTX-2.5](./docs/ltx2.md)
     - [HunyuanVideo 1.5](./docs/hunyuan_video.md)
     - [LingBot-Video](./docs/lingbot_video.md)
   - [PhotoMaker](./docs/photo_maker.md) support.

docs/animatediff.md

@@ -79,7 +79,7 @@ Low-VRAM streaming (verified with a 2 GiB cap on RTX 3060):
 .\bin\Release\sd-cli.exe -M vid_gen \
     --model ..\models\checkpoints\realisticVisionV60B1.safetensors \
     --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \
-    --max-vram 2.0 --stream-layers --diffusion-fa \
+    --max-vram 2.0 --diffusion-fa \
     -p "photo of coastline, rocks, storm weather, wind, waves, lightning" \
     --cfg-scale 8.0 --sampling-method euler --scheduler discrete \
     -H 384 -W 384 --video-frames 8 --fps 8 --steps 20 -s 42 \

docs/backend.md

@@ -41,7 +41,11 @@ sd-cli -m model.safetensors -p "a cat" --backend cuda0 --params-backend disk
 sd-cli -m model.safetensors -p "a cat" --backend diffusion=cuda0,vae=vulkan0 --max-vram cuda0=6,vulkan0=2
 ```
 
-The budget applies to every module running on that backend.
+The value is a shared per-device budget for managed weights and registered
+runner compute/cache buffers. Live free memory can lower the effective limit
+for each graph run. Driver contexts and allocations made outside the managed
+model runners are not part of this accounting, so it is not a hard physical
+VRAM cap.
 
 Module names are case-insensitive. Hyphens and underscores in module names are ignored, so `clip_vision`, `clip-vision`, and `clipvision` are equivalent.
 
@@ -79,9 +83,10 @@ with `--params-backend diffusion=disk`, released directly from) its own device;
 an explicit assignment such as `te=cpu` keeps the parameters on that backend
 and stages each range to its device on demand.
 
-Layer split cannot be combined with `--max-vram` graph-cut segmentation or
-`--stream-layers` for the split module; those are single-device mechanisms and
-are disabled for it.
+Layer split uses the fixed graph-cut plan to assign blocks across devices, but
+single-device segmented execution and next-segment prefetch are disabled for
+the split module. `--max-vram` can still provide the per-device limits used by
+layer split and auto-fit.
 
 Use `--list-devices` to see the device names available on the system.
 
@@ -104,43 +109,87 @@ Compared to a layer split this uses all GPUs within every layer (instead of
 sequentially device by device) at the cost of a cross-device reduction per
 matmul - usually the faster option when the devices have fast interconnect.
 
-Row split requires backend support for split buffers and is currently
-available on CUDA only; on other backends (or when the listed devices belong
-to different backend registries) the module falls back to a layer split.
+Row split requires a compatible split-buffer export from the linked GGML
+backend. If it is unavailable (or the listed devices belong to different backend
+registries), the module falls back to a layer split.
 Embeddings, normalization weights, biases and other non-block tensors stay in
 regular buffers on the main device.
 
+Row-split execution can use graph segments, but split weights are loaded
+synchronously instead of using the normal single-device prefetch path. Because
+GGML does not expose exact shard allocation sizes, the managed budget currently
+counts a split buffer's full size on each participating device. This is a
+conservative bound and can reject otherwise feasible layouts.
+
 Direct ("immediately") LoRA application cannot patch row-split tensors; with
 `--split-mode row` the automatic LoRA mode selects runtime application, and an
 explicit `--lora-apply-mode immediately` skips the split tensors with a
 warning.
 
-## Automatic placement (`--auto-fit`)
+## Automatic placement (`--auto-fit on|off`)
 
-`--auto-fit` derives the `diffusion` / `te` / `vae` placements from the model
-metadata and the per-device memory budgets, then feeds them into the same
-backend assignment mechanism described above (the chosen specs are printed).
-`--backend` and `--params-backend` are ignored while auto-fit is enabled.
+`--auto-fit` requires `on` or `off` and defaults to `on` when omitted.
+Explicit `--backend` or `--params-backend` assignments disable auto-fit,
+regardless of argument order, even with `--auto-fit on`.
+
+When enabled, auto-fit uses one GPU for `diffusion` / `te` / `vae` computation. It chooses
+the GPU with the largest available memory budget (the first device on a tie),
+then derives parameter placements from the model metadata and the remaining
+memory budgets. The chosen backend specifications are printed.
 
 ```shell
-sd-cli -m model.safetensors -p "a cat" --auto-fit
-sd-cli -m model.safetensors -p "a cat" --auto-fit --max-vram cuda0=8,cuda1=14
-sd-cli -m model.safetensors -p "a cat" --auto-fit --split-mode row
+sd-cli -m model.safetensors -p "a cat" --auto-fit on
+sd-cli -m model.safetensors -p "a cat" --auto-fit on --max-vram cuda0=8,cuda1=14
+sd-cli -m model.safetensors -p "a cat" --auto-fit off
 ```
 
 Budgets reuse `--max-vram`: a positive per-device value caps what auto-fit
 plans with on that device, a negative value means "free memory minus that many
 GiB", and with no budget set each device's free memory minus a 512 MiB margin
-is used. (The same values still drive graph-cut segmented execution for
-modules that end up on a single device.)
+is used. These resolved GPU budgets, including the safety margin, also drive
+the runner's graph-cut capacity checks.
 
-When everything fits resident, components are simply spread across the
-available GPUs. When it does not, auto-fit switches to time-share mode: the
-heavy components get `disk` params residency (loaded for their phase, freed
-after), and a component too large for any single device is split across all
-GPUs with the layer/row split mechanism (`--split-mode` selects which, layer
-by default). Components that fit nowhere fall back to the CPU. If a VAE decode
-still runs out of memory, tiling is enabled and the decode retried once.
+Components are considered in `diffusion`, `te`, `vae` order so that repeatedly
+used diffusion weights have priority. Each component's weights use the first
+storage location with enough remaining budget:
+
+1. The main GPU, leaving estimated space for computation and weight staging.
+2. CPU RAM, reserving the larger of 2 GiB or 10% of available RAM for other work.
+3. Another GPU, choosing the one with the largest remaining budget that fits.
+4. Disk, reloading weights on demand.
+
+GPU cache space follows the same component priority. Before a lower-priority
+component can become permanently resident, the planner leaves room for the full
+weights and estimated compute space of higher-priority offloaded components.
+If offloaded diffusion already needs the entire main GPU budget, TE and VAE also
+use offloaded parameters. Their GPU copies can then be released after their
+phases, leaving more room to reuse diffusion weights across sampling steps.
+CPU parameter residency allows GPU weight caching; it does not force every
+weight to be copied again at every step.
+
+RAM and GPU budgets are shared across components. Each component uses a single
+parameter backend; several other GPUs' capacities are not combined to store
+one component. If available RAM cannot be queried, RAM residency is skipped.
+Other GPUs store weights only: weights are copied to the main GPU for execution.
+Auto-fit does not select multi-GPU layer/row computation, so `--split-mode` does
+not change its placements. Use explicit backend assignments for multi-GPU
+computation.
+
+For example, a diffusion model whose full weights exceed the main GPU's budget
+can use `--backend diffusion=cuda0 --params-backend diffusion=cpu` when RAM is
+sufficient. Automatic graph segmentation can then load the required weights
+for each segment and reclaim idle GPU copies. `--disable-segmented-compute`
+still disables segmentation.
+
+Initial compute reserves are estimates (2 GiB for diffusion and text encoders,
+1 GiB for VAE); higher-priority placements also leave staging space for the
+largest weight tensor of each lower-priority offloaded component. Actual segment
+weights, compute buffers and caches must
+still fit the runner's capacity checks. Offloading weights does not guarantee
+that every resolution or frame count will fit, and auto-fit does not change a
+component to CPU computation solely because its full weights exceed VRAM.
+If a VAE decode fails, auto-fit retries with spatial tiling; supported video
+decoders try temporal tiling first and can then add spatial tiling.
 
 ## Modules
 
@@ -192,7 +241,7 @@ sd-cli -m model.safetensors -p "a cat" --backend cuda0 --params-backend disk
 
 This runs all modules on `cuda0`, reloads parameters from the model file as needed, and releases those parameter buffers after use.
 
-`disk` is never selected implicitly. If `--params-backend` is not set, parameters use the runtime backend.
+Outside `--auto-fit`, `disk` is never selected implicitly. If `--params-backend` is not set, parameters use the runtime backend.
 
 Per-module assignments can be mixed:
 
@@ -241,4 +290,7 @@ The example CLI/server still accepts these older CPU placement flags as compatib
 
 Because this default is inserted first, later explicit `--params-backend` entries can still override it, for example `--offload-to-cpu --params-backend te=disk` keeps non-TE parameters on CPU and reloads TE parameters from disk.
 
-Library callers should set `backend` and `params_backend` directly. The old CPU/offload fields are no longer part of the C API. Explicit `--backend` and `--params-backend` assignments are preferred for new commands.
+Library callers should set `backend` and `params_backend` directly. `sd_ctx_params_init()`
+enables `auto_fit` by default; nonempty `backend` or `params_backend` assignments disable it.
+The old CPU/offload fields are no longer part of the C API. Explicit `--backend` and
+`--params-backend` assignments are preferred for new commands.

docs/ideogram4.md

@@ -11,7 +11,19 @@
 - Download Qwen3-VL-8B-Instruct
     - gguf: https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/tree/main
 
-## Convert weights
+## Use original FP8 weights
+
+The original Ideogram4 FP8 safetensors can be loaded directly. FP8 tensors stay
+at one byte per element in RAM and VRAM. Backends that cannot multiply FP8
+weights directly cast only the active layer to a temporary BF16 tensor during
+execution; the loader does not expand the entire checkpoint to BF16.
+
+Use `ideogram4_fp8.safetensors` and `ideogram4_uncond_fp8.safetensors` directly
+with `--diffusion-model` and `--uncond-diffusion-model`, respectively.
+
+## Optional conversion for quantization
+
+The following conversion is only needed when creating a quantized GGUF model.
 
 fp8 scale -> bf16
 

docs/ltx2.md

@@ -1,7 +1,17 @@
 # How to Use
 
+Both LTX-2.3 and LTX-2.5 are supported. The two share a transformer, video VAE and audio
+VAE architecture; LTX-2.5 drops the video FFN biases, adds a learned keyframe
+absolute-position embedding, and pairs with a Gemma 4 text encoder instead of Gemma 3.
+Everything is detected from the weights, so the command lines differ only in which files
+you pass.
+
+# LTX-2.3
+
 ## Download weights
 
+### LTX-2.3
+
 - Download LTX-2.3
     - safetensors: https://huggingface.co/Kijai/LTX2.3_comfy/tree/main/diffusion_models
     - gguf: https://huggingface.co/unsloth/LTX-2.3-GGUF/tree/main
@@ -16,6 +26,30 @@
 - Download LTX spatial latent upscaler
     - safetensors: https://huggingface.co/Lightricks/LTX-2.3/resolve/main/ltx-2.3-spatial-upscaler-x2-1.1.safetensors
 
+### LTX-2.5
+
+- Download LTX-2.5
+    - safetensors: https://huggingface.co/Lightricks/LTX-2.5/tree/main/diffusion_models
+    - gguf: https://huggingface.co/vantagewithai/LTX-2.5-GGUF/tree/main
+- Download the text encoder. This is a Gemma 4 12B fine-tuned for LTX with the text
+  projection bundled in, so no separate `--embeddings-connectors` file is needed. Google's
+  stock Gemma 4 is not a substitute.
+    - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/text_encoders/gemma4-12b-with-proj-ltx-2.5-bf16.safetensors
+- Download the video vae. Use the **conv** variant: `ltx-2.5-video-vae-conv-bf16.safetensors`.
+  The default `ltx-2.5-video-vae-bf16.safetensors` is a diffusion decoder, which is not
+  implemented here.
+    - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/vae/ltx-2.5-video-vae-conv-bf16.safetensors
+- Download the audio vae
+    - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/vae/ltx-2.5-audio-vae-bf16.safetensors
+- Download the LTX spatial latent upscaler
+    - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/latent_upscale_models/ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors
+
+To run the text encoder quantized, convert it once with sd-cli:
+
+```
+.\bin\Release\sd-cli.exe -M convert -m ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-bf16.safetensors --type q8_0 -o ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf
+```
+
 ## Examples
 
 ### LTX-2.3 dev T2V
@@ -74,4 +108,32 @@ By default, the hires refine pass uses the main sampler and scheduler, then trim
   src="../assets/ltx2/hires_i2v.webm"
   controls
   muted
-  style="max-width: 100%; height: auto;"></video>
+  style="max-width: 100%; height: auto;"></video>
+
+### LTX-2.5 dev T2V
+
+```
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf -p "A wide aerial shot of a red vintage convertible driving along a coastal cliff road at sunset, waves crashing below" --cfg-scale 3.0 --sampling-method euler -v -n "worst quality, low quality, blurry, distorted, artifacts" -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 121 --fps 24 -o t2v.webm
+```
+
+### LTX-2.5 dev I2V
+
+```
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf -p "a lovely cat blinking slowly, gentle camera push in" --cfg-scale 3.0 --sampling-method euler -v -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 121 -i ..\assets\ernie_image\turbo_example.png -o i2v.webm
+```
+
+### LTX-2.5 spatial latent upscale
+
+Works exactly like the LTX-2.3 upscaler described below; put
+`ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors` under `--hires-upscalers-dir` and
+pass its name without path or extension to `--hires-upscaler`.
+
+```
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf --hires-upscalers-dir ..\models\latent_upscale_models --hires-upscaler ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0 --hires --hires-steps 6 -p "a lovely cat" --cfg-scale 3.0 --sampling-method euler -v -W 640 -H 360 --diffusion-fa --offload-to-cpu --video-frames 121 -o hires_t2v.webm
+```
+
+## Not implemented
+
+- The diffusion video decoder (`ltx-2.5-video-vae-bf16.safetensors`). Use the conv VAE.
+- The temporal latent upscaler and the duration head (`--auto-duration`); pass
+  `--video-frames` explicitly.

docs/model_config.md

@@ -67,21 +67,21 @@ Detection should respect `prefix`. For nested weights, construct full names from
 
 Do not add persistent config fields such as `inferred_from_weights` only to
 record whether detection happened. If the function needs to decide whether to
-print a debug line, keep that as local control flow inside `detect_from_weights`.
+print a verbose line, keep that as local control flow inside `detect_from_weights`.
 
 ## Logging
 
-When config values are inferred from weights, print one `LOG_DEBUG` line at the
+When config values are inferred from weights, print one `LOG_VERBOSE` line at the
 end of `detect_from_weights`.
 
 Example:
 
 ```cpp
-LOG_DEBUG("llm: num_layers = %" PRId64 ", vocab_size = %" PRId64 ", hidden_size = %" PRId64 ", intermediate_size = %" PRId64,
-          config.num_layers,
-          config.vocab_size,
-          config.hidden_size,
-          config.intermediate_size);
+LOG_VERBOSE("llm: num_layers = %" PRId64 ", vocab_size = %" PRId64 ", hidden_size = %" PRId64 ", intermediate_size = %" PRId64,
+            config.num_layers,
+            config.vocab_size,
+            config.hidden_size,
+            config.intermediate_size);
 ```
 
 Only print the config detection log when the function actually inferred values

docs/performance.md

@@ -14,8 +14,12 @@ Run by adding `--diffusion-fa` to the arguments and watch for:
 ```
 and the compute buffer shrink in the debug log:
 ```
-[DEBUG] ggml_extend.hpp:1004 - flux compute buffer size: 650.00 MB(VRAM)
+[DEBUG] ggml_runner.cpp:280 - flux compute buffer size: 650.00 MB(VRAM) on CUDA0 (peak across 1 segment)
 ```
+This reports the actual peak compute workspace capacity per backend, including
+CPU fallback. It excludes weights and cache buffers. Within a runner lifecycle,
+the summary is printed only on the first graph or when backend capacities or the
+segment count change.
 
 ## Offload weights to the CPU to save VRAM without reducing generation speed.
 
@@ -43,7 +47,7 @@ Use disk params to reduce both VRAM and RAM usage:
 --backend cuda0 --params-backend disk
 ```
 
-This reloads parameters from the model file on demand and releases them after use. It has the lowest memory residency, but can be slower because weights must be read again. `disk` is never selected implicitly; set it explicitly when RAM usage matters more than reload cost.
+This reloads parameters from the model file on demand, retains unpinned compute copies while space permits, and releases them under pressure or at module-run completion. It has the lowest source-memory residency, but can be slower because evicted weights must be read again. `disk` is never selected implicitly; set it explicitly when RAM usage matters more than reload cost.
 
 Per-module assignments can target only the largest modules:
 
@@ -53,25 +57,37 @@ Per-module assignments can target only the largest modules:
 
 See [backend selection](./backend.md) for full syntax.
 
-## Run models that don't fit in VRAM (CPU streaming).
+## Run models that don't fit in VRAM (automatic segmented execution).
 
-`--offload-to-cpu` alone keeps every parameter in system RAM and stages it to the runtime backend on first use, then leaves it resident there. If the diffusion model is larger than the runtime backend's free memory (e.g. Flux dev at bf16 on an 8 GiB GPU), that residency stops fitting during the sampling loop and generation fails. Two additional flags make it fit by trading a small amount of speed for room:
+`--offload-to-cpu` keeps the source parameters in system RAM and creates compute-side GPU replicas on demand. Unpinned replicas remain resident for reuse, but automatic graph-cut execution evicts them from the last segment backward when the next weight or compute allocation needs space. Disk-backed parameters follow the same policy without retaining a RAM source copy.
 
-- `--max-vram <GiB>` sets a VRAM budget the graph-cut segmenter respects. It cuts each forward pass into segments sized to fit the budget, running them in sequence and freeing intermediate activations between them. Negative values auto-detect free VRAM and spare the given amount (`--max-vram -1` uses most of the free VRAM and keeps ~1 GiB headroom), a positive value caps the budget, `0` disables segmentation.
-- `--stream-layers` streams the diffusion model's transformer blocks one at a time. Each block's parameters are copied from the CPU to the runtime backend just before it runs and evicted when the residency budget is reached. Prefetching hides most of the copy latency behind compute. This flag only takes effect when the diffusion params backend is CPU, so it must be combined with `--offload-to-cpu` (or an explicit `--params-backend diffusion=cpu`); a warning is logged and the flag is ignored otherwise.
+When a graph has cut markers and its missing weights plus incremental compute workspace exceed the available device headroom, it runs its fixed segment list in order. A reusable monolithic compute buffer is not counted as a new allocation. An explicit `--max-vram` budget deducts already-resident managed weights and compute/cache buffers registered by every runner sharing the device, so later graph runs remain segmented when the full graph exceeds the budget. The current segment's weights are pinned during compute, and the next parameter-bearing segment is prefetched when the device supports asynchronous transfer. No opt-in streaming flag is required.
 
-The three flags stack. The recommended shape for "biggest model my card can host":
+- `--max-vram <GiB>` optionally lowers the live-memory limit. A positive value is a managed per-device budget, `0` uses the device's current free memory without an explicit budget, and a negative value snapshots free memory at startup while reserving that many GiB (`--max-vram -1` reserves about 1 GiB). Driver contexts and unrelated external allocations remain outside the managed budget.
+- `--disable-prefetch` disables asynchronous next-segment prefetch while retaining synchronous loading, eviction, and segmented execution.
+- `--disable-segmented-compute` forces monolithic graph execution for diagnostics or compatibility, even when the automatic memory check would select segments.
+
+Single-device monolithic execution also reclaims unpinned weight replicas before
+loading weights or allocating compute workspace, including graphs without cut
+markers and runs with `--disable-segmented-compute`. It still respects the managed
+device budget and fails if the graph cannot fit after reclamation.
+
+Segment completion releases active workspace use while retaining the runner's
+allocator/scheduler capacity. Compatible gallocr reservations are reused across
+graphs; idle workspaces can be reclaimed under pressure and are freed at runner
+completion. Cross-graph caches survive individual graphs, but cut buffers do not.
+
+The recommended shape for "biggest model my card can host" is:
 
 ```shell
 sd-cli --diffusion-model flux1-dev.safetensors ... \
-       --offload-to-cpu --max-vram -1 --stream-layers
+       --offload-to-cpu --max-vram -1
 ```
 
 - `--offload-to-cpu`: params in RAM, staged as needed.
-- `--max-vram -1`: use most of the free VRAM as the compute budget, spare 1 GiB headroom, let the graph-cut segmenter split each forward pass to fit.
-- `--stream-layers`: on top of the segmenter, stream individual transformer blocks so their weights don't all need to be resident at once.
+- `--max-vram -1`: reserve about 1 GiB from the startup free-memory snapshot; live free memory can still lower the effective limit for every graph.
 
-Ordered from fastest to smallest-VRAM: no flags → `--offload-to-cpu` → `--offload-to-cpu --max-vram <N>` → `--offload-to-cpu --max-vram <N> --stream-layers`. Each step down costs a few percent of throughput to buy more room; combined they can run models roughly 3-4x larger than the raw VRAM would allow.
+Use `--params-backend diffusion=disk` instead when reducing system RAM residency is more important than avoiding repeated model-file reads.
 
 ## Use quantization to reduce memory usage.
 

docs/sd.md

@@ -3,7 +3,7 @@
 - download original weights(.ckpt or .safetensors). For example
     - Stable Diffusion v1.4 from https://huggingface.co/CompVis/stable-diffusion-v-1-4-original
     - Stable Diffusion v1.5 from https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5
-    - Stable Diffuison v2.1 from https://huggingface.co/Manojb/stable-diffusion-2-1-base
+    - Stable Diffusion v2.1 from https://huggingface.co/Manojb/stable-diffusion-2-1-base
     - Stable Diffusion 3 2B from https://huggingface.co/stabilityai/stable-diffusion-3-medium
 
 ### txt2img example

docs/sefi_image.md

@@ -44,7 +44,7 @@ The dispatcher picks `alpha` from the filename (`turbo` substring => 1.0, otherw
 ### 5B (needs streaming on 12 GiB VRAM)
 
 ```
-./build/bin/sd-cli --diffusion-model /path/to/sefi_5b_turbo.safetensors --vae /path/to/flux2_ae.safetensors --llm /path/to/qwen3_vl_4b.safetensors -p "a photograph of an orange tabby cat sitting on a couch" --cfg-scale 1.0 --steps 4 -W 1024 -H 1024 -s 42 --diffusion-fa --max-vram 8 --stream-layers --offload-to-cpu -o out.png
+./build/bin/sd-cli --diffusion-model /path/to/sefi_5b_turbo.safetensors --vae /path/to/flux2_ae.safetensors --llm /path/to/qwen3_vl_4b.safetensors -p "a photograph of an orange tabby cat sitting on a couch" --cfg-scale 1.0 --steps 4 -W 1024 -H 1024 -s 42 --diffusion-fa --max-vram 8 --offload-to-cpu -o out.png
 ```
 
 <img alt="SeFi-Image 5B turbo example" src="../assets/sefi_image/example.png" />

examples/cli/README.md

@@ -6,6 +6,11 @@ For detailed command-line arguments, run:
 ./bin/sd-cli -h
 ```
 
+Logging defaults to `info`. Use `--log-level <level>` to select `debug`, `verbose`,
+`info`, `warn`, or `error` (from most to least detailed). Each level includes
+messages at that level and all less detailed levels. `-v` and `--verbose` are
+equivalent to `--log-level verbose`. If repeated, the last logging option wins.
+
 For direct image repair or automatic post-generation YOLOv8 detection followed by cropped inpainting, see
 [ADetailer](../../docs/adetailer.md).
 

examples/cli/main.cpp

@@ -36,12 +36,13 @@ struct SDCliParams {
     SDMode mode             = IMG_GEN;
     std::string output_path = "output.png";
     int output_begin_idx    = -1;
+    int compression_quality = 90;
     std::string image_path;
     std::string metadata_format = "text";
 
-    bool verbose          = false;
-    bool canny_preprocess = false;
-    bool convert_name     = false;
+    sd_log_level_t log_level = SD_LOG_INFO;
+    bool canny_preprocess    = false;
+    bool convert_name        = false;
 
     preview_t preview_method = PREVIEW_NONE;
     int preview_interval     = 1;
@@ -80,7 +81,7 @@ struct SDCliParams {
              &metadata_format},
             {"",
              "--preview-path",
-             "path to write preview image to (default: ./preview.png). Multi-frame previews support .avi, .webm, and animated .webp",
+             "path to write preview image to (default: ./preview.png). For image generation, the filename can have %03d placeholder for sequential numbering. Multi-frame previews support .avi, .webm, and animated .webp",
              0,
              &preview_path},
             {"",
@@ -93,12 +94,16 @@ struct SDCliParams {
         options.int_options = {
             {"",
              "--preview-interval",
-             "interval in denoising steps between consecutive updates of the image preview file (default is 1, meaning updating at every step)",
+             "preview interval: in each sampling pass, positive N updates every Nth denoiser step and -N previews only completed logical step N; 0 previews the final completed step of the first pass (base-resolution or high-noise). Default: 1",
              &preview_interval},
             {"",
              "--output-begin-idx",
              "starting index for output image sequence, must be non-negative (default 0 if specified %d in output path, 1 otherwise)",
              &output_begin_idx},
+            {"",
+             "--compression-quality",
+             "compression quality of video and JPEG / WebP images (90 by default)",
+             &compression_quality},
         };
 
         options.bool_options = {
@@ -110,10 +115,6 @@ struct SDCliParams {
              "--convert-name",
              "convert tensor name (for convert mode)",
              true, &convert_name},
-            {"-v",
-             "--verbose",
-             "print extra info",
-             true, &verbose},
             {"",
              "--color",
              "colors the logging tags according to level",
@@ -215,6 +216,7 @@ struct SDCliParams {
              on_imatrix_in_arg},
         };
 
+        add_log_options(options, log_level);
         return options;
     };
 
@@ -264,7 +266,7 @@ struct SDCliParams {
             << "  output_path: \"" << output_path << "\",\n"
             << "  image_path: \"" << image_path << "\",\n"
             << "  metadata_format: \"" << metadata_format << "\",\n"
-            << "  verbose: " << (verbose ? "true" : "false") << ",\n"
+            << "  log_level: " << log_level_name(log_level) << ",\n"
             << "  color: " << (color ? "true" : "false") << ",\n"
             << "  canny_preprocess: " << (canny_preprocess ? "true" : "false") << ",\n"
             << "  convert_name: " << (convert_name ? "true" : "false") << ",\n"
@@ -302,6 +304,9 @@ void parse_args(int argc, const char** argv, SDCliParams& cli_params, SDContextP
         exit(cli_params.normal_exit ? 0 : 1);
     }
 
+    log_level = cli_params.log_level;
+    log_color = cli_params.color;
+
     bool valid = cli_params.resolve_and_validate();
     if (valid && cli_params.mode != METADATA) {
         valid = ctx_params.resolve_and_validate(cli_params.mode) &&
@@ -318,15 +323,14 @@ void parse_args(int argc, const char** argv, SDCliParams& cli_params, SDContextP
 
 void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) {
     SDCliParams* cli_params = (SDCliParams*)data;
-    log_print(level, log, cli_params->verbose, cli_params->color);
+    log_print(level, log, cli_params->log_level, cli_params->color);
 }
 
 bool load_images_from_dir(const std::string dir,
                           std::vector<SDImageOwner>& images,
                           int expected_width  = 0,
                           int expected_height = 0,
-                          int max_image_num   = 0,
-                          bool verbose        = false) {
+                          int max_image_num   = 0) {
     if (!fs::exists(dir) || !fs::is_directory(dir)) {
         LOG_ERROR("'%s' is not a valid directory\n", dir.c_str());
         return false;
@@ -350,7 +354,7 @@ bool load_images_from_dir(const std::string dir,
         std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
 
         if (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".bmp" || ext == ".webp") {
-            LOG_DEBUG("load image %zu from '%s'", images.size(), path.c_str());
+            LOG_VERBOSE("load image %zu from '%s'", images.size(), path.c_str());
             int width             = 0;
             int height            = 0;
             uint8_t* image_buffer = load_image_from_file(path.c_str(), width, height, expected_width, expected_height);
@@ -372,27 +376,6 @@ bool load_images_from_dir(const std::string dir,
     return true;
 }
 
-void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy, void* data) {
-    (void)step;
-    (void)is_noisy;
-    SDCliParams* cli_params = (SDCliParams*)data;
-    // is_noisy is set to true if the preview corresponds to noisy latents, false if it's denoised latents
-    // unused in this app, it will either be always noisy or always denoised here
-    if (frame_count == 1) {
-        if (!write_image_to_file(cli_params->preview_path,
-                                 image->data,
-                                 image->width,
-                                 image->height,
-                                 image->channel)) {
-            LOG_ERROR("save preview image to '%s' failed", cli_params->preview_path.c_str());
-        }
-    } else {
-        if (create_video_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, cli_params->preview_fps) != 0) {
-            LOG_ERROR("save preview video to '%s' failed", cli_params->preview_path.c_str());
-        }
-    }
-}
-
 std::string format_frame_idx(std::string pattern, int frame_idx) {
     std::smatch match;
     std::string result = pattern;
@@ -412,6 +395,36 @@ std::string format_frame_idx(std::string pattern, int frame_idx) {
     return result;
 }
 
+int continuous_preview_counter = 0;
+
+void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy, void* data) {
+    (void)step;
+    (void)is_noisy;
+    SDCliParams* cli_params = (SDCliParams*)data;
+    // is_noisy is set to true if the preview corresponds to noisy latents, false if it's denoised latents
+    // unused in this app, it will either be always noisy or always denoised here
+    if (frame_count == 1) {
+        fs::path path = cli_params->preview_path;
+        if (encoded_image_format_from_path(path.string()) == EncodedImageFormat::UNKNOWN)
+            path += ".png";
+        if (std::regex_search(path.string(), format_specifier_regex))
+            path = fs::path(format_frame_idx(path.string(), continuous_preview_counter++));
+        if (!write_image_to_file(path.string(),
+                                 image->data,
+                                 image->width,
+                                 image->height,
+                                 image->channel,
+                                 "",
+                                 cli_params->compression_quality)) {
+            LOG_ERROR("save preview image to '%s' failed", path.string().c_str());
+        }
+    } else {
+        if (create_video_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, cli_params->preview_fps, cli_params->compression_quality) != 0) {
+            LOG_ERROR("save preview video to '%s' failed", cli_params->preview_path.c_str());
+        }
+    }
+}
+
 static fs::path get_video_audio_sidecar_path(const SDCliParams& cli_params) {
     fs::path out_path     = cli_params.output_path;
     fs::path base_path    = out_path;
@@ -486,7 +499,7 @@ bool save_results(const SDCliParams& cli_params,
         std::string params          = gen_params.embed_image_metadata
                                           ? get_image_params(ctx_params, gen_params, metadata_seed, cli_params.mode)
                                           : "";
-        const bool ok               = write_image_to_file(path.string(), img.data, img.width, img.height, img.channel, params, 90);
+        const bool ok               = write_image_to_file(path.string(), img.data, img.width, img.height, img.channel, params, cli_params.compression_quality);
         LOG_INFO("save result image %d to '%s' (%s)", idx, path.string().c_str(), ok ? "success" : "failure");
         return ok;
     };
@@ -532,7 +545,7 @@ bool save_results(const SDCliParams& cli_params,
         std::string final_ext_lower = ext.string();
         std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower);
         const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm");
-        if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps, 90, mux_audio ? generated_audio : nullptr) == 0) {
+        if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr) == 0) {
             LOG_INFO("save result video to '%s'", video_path.string().c_str());
             if (generated_audio != nullptr && !mux_audio) {
                 fs::path wav_path = video_path;
@@ -637,8 +650,6 @@ int main(int argc, const char* argv[]) {
 
     parse_args(argc, argv, cli_params, ctx_params, gen_params);
     sd_set_log_callback(sd_log_cb, (void*)&cli_params);
-    log_verbose = cli_params.verbose;
-    log_color   = cli_params.color;
 
     if (cli_params.mode == METADATA) {
         MetadataReadOptions options;
@@ -686,11 +697,11 @@ int main(int argc, const char* argv[]) {
                             cli_params.preview_noisy,
                             (void*)&cli_params);
 
-    LOG_DEBUG("version: %s", version_string().c_str());
-    LOG_DEBUG("%s", sd_get_system_info());
-    LOG_DEBUG("%s", cli_params.to_string().c_str());
-    LOG_DEBUG("%s", ctx_params.to_string().c_str());
-    LOG_DEBUG("%s", gen_params.to_string().c_str());
+    LOG_VERBOSE("version: %s", version_string().c_str());
+    LOG_VERBOSE("%s", sd_get_system_info());
+    LOG_VERBOSE("%s", cli_params.to_string().c_str());
+    LOG_VERBOSE("%s", ctx_params.to_string().c_str());
+    LOG_VERBOSE("%s", gen_params.to_string().c_str());
 
     if (!cli_params.imatrix_out.empty()) {
         if (fs::exists(cli_params.imatrix_out) &&
@@ -794,7 +805,7 @@ int main(int argc, const char* argv[]) {
         gen_params.ref_videos.reserve(gen_params.ref_video_paths.size());
         for (const auto& path : gen_params.ref_video_paths) {
             std::vector<SDImageOwner> frames;
-            if (!load_images_from_dir(path, frames, 0, 0, 0, cli_params.verbose) || frames.empty()) {
+            if (!load_images_from_dir(path, frames) || frames.empty()) {
                 LOG_ERROR("load reference video frames from '%s' failed", path.c_str());
                 return 1;
             }
@@ -876,8 +887,7 @@ int main(int argc, const char* argv[]) {
                                   gen_params.control_frames,
                                   gen_params.get_resolved_width(),
                                   gen_params.get_resolved_height(),
-                                  gen_params.video_frames,
-                                  cli_params.verbose)) {
+                                  gen_params.video_frames)) {
             return 1;
         }
     }
@@ -888,8 +898,7 @@ int main(int argc, const char* argv[]) {
                                   gen_params.pm_id_images,
                                   0,
                                   0,
-                                  0,
-                                  cli_params.verbose)) {
+                                  0)) {
             return 1;
         }
     }

examples/common/common.cpp

@@ -239,6 +239,26 @@ void ArgOptions::print() const {
     }
 }
 
+void add_log_options(ArgOptions& options, sd_log_level_t& level) {
+    options.manual_options.push_back({"", "--log-level",
+                                      "minimum log level, one of [debug, verbose, info, warn, error] (default: info)",
+                                      [&level](int argc, const char** argv, int index) {
+                                          if (++index >= argc) {
+                                              return -1;
+                                          }
+                                          if (!parse_log_level(argv[index], level)) {
+                                              LOG_ERROR("invalid log level %s, must be one of [debug, verbose, info, warn, error]", argv[index]);
+                                              return -1;
+                                          }
+                                          return 1;
+                                      }});
+    options.manual_options.push_back({"-v", "--verbose", "equivalent to --log-level verbose",
+                                      [&level](int, const char**, int) {
+                                          level = SD_LOG_VERBOSE;
+                                          return 0;
+                                      }});
+}
+
 bool parse_options(int argc, const char** argv, const std::vector<ArgOptions>& options_list) {
     bool invalid_arg = false;
     std::string arg;
@@ -502,7 +522,7 @@ ArgOptions SDContextParams::get_options() {
          &rpc_servers},
         {"",
          "--max-vram",
-         "maximum VRAM budget in GiB for graph-cut segmented execution. Accepts a single value or assignments by backend/device, e.g. 6 or cuda0=6,vulkan0=4. 0 disables graph splitting; a negative value auto-detects free VRAM, sparing the specified value",
+         "optional per-device budget in GiB for managed weights and runner buffers during automatic graph-cut execution. Accepts a single value or assignments by backend/device, e.g. 6 or cuda0=6,vulkan0=4. 0 uses live free VRAM without an explicit budget; a negative value reserves that much free VRAM",
          0,
          &max_vram},
     };
@@ -517,19 +537,17 @@ ArgOptions SDContextParams::get_options() {
 
     options.bool_options = {
         {"",
-         "--stream-layers",
-         "enable residency+prefetch streaming on top of --max-vram (no effect without --max-vram; defaults to false)",
-         true, &stream_layers},
+         "--disable-prefetch",
+         "disable asynchronous next-segment weight prefetch (defaults to false)",
+         true, &disable_prefetch},
+        {"",
+         "--disable-segmented-compute",
+         "force monolithic graph execution even when automatic graph cutting is needed (defaults to false)",
+         true, &disable_segmented_compute},
         {"",
          "--eager-load",
          "load all params into the params backend at model-load time instead of lazily on first use (defaults to false)",
          true, &eager_load},
-        {"",
-         "--auto-fit",
-         "pick the diffusion/te/vae device placements automatically from the model size and the per-device "
-         "memory budgets (--max-vram; defaults to free memory minus a small margin). Overrides --backend and "
-         "--params-backend; may split modules across GPUs (--split-mode still selects layer or row)",
-         true, &auto_fit},
         {"",
          "--force-sdxl-vae-conv-scale",
          "force use of conv scale on sdxl vae",
@@ -572,6 +590,23 @@ ArgOptions SDContextParams::get_options() {
          true, &vae_conv_direct},
     };
 
+    auto on_auto_fit_arg = [&](int argc, const char** argv, int index) {
+        if (++index >= argc) {
+            LOG_ERROR("--auto-fit requires 'on' or 'off'");
+            return -1;
+        }
+        const std::string arg = argv[index];
+        if (arg == "on") {
+            auto_fit = true;
+        } else if (arg == "off") {
+            auto_fit = false;
+        } else {
+            LOG_ERROR("invalid --auto-fit value '%s'; expected 'on' or 'off'", argv[index]);
+            return -1;
+        }
+        return 1;
+    };
+
     auto on_type_arg = [&](int argc, const char** argv, int index) {
         if (++index >= argc) {
             return -1;
@@ -643,6 +678,12 @@ ArgOptions SDContextParams::get_options() {
     };
 
     options.manual_options = {
+        {"",
+         "--auto-fit",
+         "on|off (default: on). Use one GPU for diffusion/te/vae computation and place weights on that GPU, "
+         "RAM, another GPU, or disk in that order, according to available memory (--max-vram limits GPU budgets). "
+         "Disabled by explicit --backend or --params-backend; uses automatic graph segmentation when needed",
+         on_auto_fit_arg},
         {"",
          "--type",
          "weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K, q4_K). "
@@ -831,7 +872,8 @@ std::string SDContextParams::to_string() const {
         << "  sampler_rng_type: " << sd_rng_type_name(sampler_rng_type) << ",\n"
         << "  offload_params_to_cpu: " << (offload_params_to_cpu ? "true" : "false") << ",\n"
         << "  max_vram: \"" << max_vram << "\",\n"
-        << "  stream_layers: " << (stream_layers ? "true" : "false") << ",\n"
+        << "  disable_prefetch: " << (disable_prefetch ? "true" : "false") << ",\n"
+        << "  disable_segmented_compute: " << (disable_segmented_compute ? "true" : "false") << ",\n"
         << "  eager_load: " << (eager_load ? "true" : "false") << ",\n"
         << "  backend: \"" << backend << "\",\n"
         << "  params_backend: \"" << params_backend << "\",\n"
@@ -903,7 +945,8 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool taesd_preview) {
     sd_ctx_params.force_sdxl_vae_conv_scale       = force_sdxl_vae_conv_scale;
     sd_ctx_params.vae_format                      = str_to_vae_format(vae_format);
     sd_ctx_params.max_vram                        = max_vram.c_str();
-    sd_ctx_params.stream_layers                   = stream_layers;
+    sd_ctx_params.disable_prefetch                = disable_prefetch;
+    sd_ctx_params.disable_segmented_compute       = disable_segmented_compute;
     sd_ctx_params.eager_load                      = eager_load;
     sd_ctx_params.backend                         = effective_backend.c_str();
     sd_ctx_params.params_backend                  = effective_params_backend.c_str();
@@ -1013,7 +1056,7 @@ ArgOptions SDGenerationParams::get_options() {
          &extra_sample_args},
         {"",
          "--extra-tiling-args",
-         "extra VAE tiling args, key=value list. LTX video VAE supports temporal_tile_frames (default: 4), temporal_tile_overlap (default: 1)",
+         "extra VAE tiling args, key=value list. Supported video VAEs accept temporal_tile_frames/temporal_tile_size (default: 4), temporal_tile_overlap (default: 1)",
          (int)',',
          &extra_tiling_args},
         {"",
@@ -1230,7 +1273,7 @@ ArgOptions SDGenerationParams::get_options() {
          &vae_tiling_params.enabled},
         {"",
          "--temporal-tiling",
-         "enable temporal tiling for LTX video VAE decode",
+         "enable temporal tiling for supported video VAE decode",
          true,
          &vae_tiling_params.temporal_tiling},
         {"",

examples/common/common.h

@@ -107,6 +107,7 @@ struct ArgOptions {
     void print() const;
 };
 
+void add_log_options(ArgOptions& options, sd_log_level_t& level);
 bool parse_options(int argc, const char** argv, const std::vector<ArgOptions>& options_list);
 bool decode_base64_image(const std::string& encoded_input,
                          int target_channels,
@@ -146,17 +147,18 @@ struct SDContextParams {
     std::map<std::string, std::string> embedding_map;
     std::vector<sd_embedding_t> embedding_vec;
 
-    rng_type_t rng_type         = CUDA_RNG;
-    rng_type_t sampler_rng_type = RNG_TYPE_COUNT;
-    bool offload_params_to_cpu  = false;
-    std::string max_vram        = "0";
-    bool stream_layers          = false;
-    bool eager_load             = false;
+    rng_type_t rng_type            = CUDA_RNG;
+    rng_type_t sampler_rng_type    = RNG_TYPE_COUNT;
+    bool offload_params_to_cpu     = false;
+    std::string max_vram           = "0";
+    bool disable_prefetch          = false;
+    bool disable_segmented_compute = false;
+    bool eager_load                = false;
     std::string backend;
     std::string params_backend;
     std::string split_mode;
     std::string model_args;
-    bool auto_fit = false;
+    bool auto_fit = true;
     std::string rpc_servers;
     std::string effective_backend;
     std::string effective_params_backend;

examples/common/log.cpp

@@ -2,8 +2,8 @@
 
 #include <vector>
 
-bool log_verbose = false;
-bool log_color   = false;
+sd_log_level_t log_level = SD_LOG_INFO;
+bool log_color           = false;
 
 std::string sd_basename(const std::string& path) {
     size_t pos = path.find_last_of('/');
@@ -51,12 +51,40 @@ void print_utf8(FILE* stream, const char* utf8) {
 #endif
 }
 
-void log_print(enum sd_log_level_t level, const char* log, bool verbose, bool color) {
+const char* log_level_name(sd_log_level_t level) {
+    switch (level) {
+        case SD_LOG_DEBUG:
+            return "debug";
+        case SD_LOG_VERBOSE:
+            return "verbose";
+        case SD_LOG_INFO:
+            return "info";
+        case SD_LOG_WARN:
+            return "warn";
+        case SD_LOG_ERROR:
+            return "error";
+        default:
+            return "unknown";
+    }
+}
+
+bool parse_log_level(const std::string& name, sd_log_level_t& level) {
+    const sd_log_level_t levels[] = {SD_LOG_DEBUG, SD_LOG_VERBOSE, SD_LOG_INFO, SD_LOG_WARN, SD_LOG_ERROR};
+    for (sd_log_level_t candidate : levels) {
+        if (name == log_level_name(candidate)) {
+            level = candidate;
+            return true;
+        }
+    }
+    return false;
+}
+
+void log_print(enum sd_log_level_t level, const char* log, sd_log_level_t min_level, bool color) {
     int tag_color;
     const char* level_str;
     FILE* out_stream = (level == SD_LOG_ERROR) ? stderr : stdout;
 
-    if (!log || (!verbose && level <= SD_LOG_DEBUG)) {
+    if (!log || level < min_level) {
         return;
     }
 
@@ -65,6 +93,10 @@ void log_print(enum sd_log_level_t level, const char* log, bool verbose, bool co
             tag_color = 37;
             level_str = "DEBUG";
             break;
+        case SD_LOG_VERBOSE:
+            tag_color = 37;
+            level_str = "VERBOSE";
+            break;
         case SD_LOG_INFO:
             tag_color = 34;
             level_str = "INFO";
@@ -84,10 +116,11 @@ void log_print(enum sd_log_level_t level, const char* log, bool verbose, bool co
     }
 
     if (color) {
-        fprintf(out_stream, "\033[%d;1m[%-5s]\033[0m ", tag_color, level_str);
+        fprintf(out_stream, "\033[%d;1m[%-7s]\033[0m ", tag_color, level_str);
     } else {
-        fprintf(out_stream, "[%-5s] ", level_str);
+        fprintf(out_stream, "[%-7s] ", level_str);
     }
+    fflush(out_stream);
     print_utf8(out_stream, log);
     fflush(out_stream);
 }
@@ -109,7 +142,7 @@ void example_log_printf(sd_log_level_t level, const char* file, int line, const
         strncat(log_buffer, "\n", LOG_BUFFER_SIZE - len);
     }
 
-    log_print(level, log_buffer, log_verbose, log_color);
+    log_print(level, log_buffer, log_level, log_color);
 
     va_end(args);
 }

examples/common/log.h

@@ -16,15 +16,18 @@
 
 #include "stable-diffusion.h"
 
-extern bool log_verbose;
+extern sd_log_level_t log_level;
 extern bool log_color;
 
 std::string sd_basename(const std::string& path);
 void print_utf8(FILE* stream, const char* utf8);
-void log_print(sd_log_level_t level, const char* log, bool verbose, bool color);
+const char* log_level_name(sd_log_level_t level);
+bool parse_log_level(const std::string& name, sd_log_level_t& level);
+void log_print(sd_log_level_t level, const char* log, sd_log_level_t min_level, bool color);
 void example_log_printf(sd_log_level_t level, const char* file, int line, const char* format, ...);
 
 #define LOG_DEBUG(format, ...) example_log_printf(SD_LOG_DEBUG, __FILE__, __LINE__, format, ##__VA_ARGS__)
+#define LOG_VERBOSE(format, ...) example_log_printf(SD_LOG_VERBOSE, __FILE__, __LINE__, format, ##__VA_ARGS__)
 #define LOG_INFO(format, ...) example_log_printf(SD_LOG_INFO, __FILE__, __LINE__, format, ##__VA_ARGS__)
 #define LOG_WARN(format, ...) example_log_printf(SD_LOG_WARN, __FILE__, __LINE__, format, ##__VA_ARGS__)
 #define LOG_ERROR(format, ...) example_log_printf(SD_LOG_ERROR, __FILE__, __LINE__, format, ##__VA_ARGS__)

examples/common/media_io.cpp

@@ -835,6 +835,9 @@ std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images
     const uint32_t audio_byte_rate       = has_audio ? static_cast<uint32_t>(audio->sample_rate * audio_block_align) : 0;
     const uint32_t audio_data_size       = has_audio ? static_cast<uint32_t>(audio_pcm.size()) : 0;
 
+    if (mjpg_quality != quality)
+        LOG_VERBOSE("create_mjpg_avi...(): compression quality was limited from %i to %i", quality, mjpg_quality);
+
     std::vector<uint8_t> avi_data;
     avi_data.reserve(static_cast<size_t>(num_images) * 1024);
 

examples/server/README.md

@@ -13,9 +13,14 @@ What this example does:
 * `--llm` selects the text encoder / language model used by this pipeline
 * `--diffusion-fa` enables flash attention in the diffusion model
 * `--offload-to-cpu` reduces VRAM pressure by keeping weights in RAM when possible
-* `-v` enables verbose logging
+* `-v` enables verbose logging (equivalent to `--log-level verbose`)
 * `--cfg-scale 1.0` sets the default CFG scale for generation
 
+Logging defaults to `info`. Use `--log-level <level>` to select `debug`, `verbose`,
+`info`, `warn`, or `error` (from most to least detailed). Each level includes
+messages at that level and all less detailed levels. `-v` and `--verbose` are
+equivalent to `--log-level verbose`. If repeated, the last logging option wins.
+
 After the server starts successfully:
 
 * the web UI is available at `http://127.0.0.1:1234/`

examples/server/api.md

@@ -518,7 +518,8 @@ Shared default fields used by both `img_gen` and `vid_gen`:
 | `output_format` | `string` |
 | `output_compression` | `integer` |
 
-`vae_tiling_params.extra_tiling_args` accepts a key=value list. For LTX video VAE temporal tiling, `temporal_tile_frames` defaults to `4` and `temporal_tile_overlap` defaults to `1`.
+`vae_tiling_params.extra_tiling_args` accepts a key=value list. Supported video VAEs accept `temporal_tile_frames` (alias `temporal_tile_size`, default `4`) and `temporal_tile_overlap` (default `1`).
+LTX and Wan preserve causal state between temporal tiles. Hunyuan Video and TAEHV use overlap blending. MiniMax H3 keeps its model-specific fixed temporal windows because its latent-to-frame mapping is non-linear.
 
 `img_gen`-specific default fields:
 

examples/server/main.cpp

@@ -44,6 +44,9 @@ static void parse_args(int argc,
         exit(svr_params.normal_exit ? 0 : 1);
     }
 
+    log_level = svr_params.log_level;
+    log_color = svr_params.color;
+
     const bool random_seed_requested = default_gen_params.seed < 0;
 
     if (!svr_params.resolve_and_validate() ||
@@ -62,7 +65,7 @@ static void parse_args(int argc,
 
 void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) {
     SDSvrParams* svr_params = (SDSvrParams*)data;
-    log_print(level, log, svr_params->verbose, svr_params->color);
+    log_print(level, log, svr_params->log_level, svr_params->color);
 }
 
 int main(int argc, const char** argv) {
@@ -76,14 +79,12 @@ int main(int argc, const char** argv) {
     parse_args(argc, argv, svr_params, ctx_params, default_gen_params);
 
     sd_set_log_callback(sd_log_cb, (void*)&svr_params);
-    log_verbose = svr_params.verbose;
-    log_color   = svr_params.color;
 
-    LOG_DEBUG("version: %s", version_string().c_str());
-    LOG_DEBUG("%s", sd_get_system_info());
-    LOG_DEBUG("%s", svr_params.to_string().c_str());
-    LOG_DEBUG("%s", ctx_params.to_string().c_str());
-    LOG_DEBUG("%s", default_gen_params.to_string().c_str());
+    LOG_VERBOSE("version: %s", version_string().c_str());
+    LOG_VERBOSE("%s", sd_get_system_info());
+    LOG_VERBOSE("%s", svr_params.to_string().c_str());
+    LOG_VERBOSE("%s", ctx_params.to_string().c_str());
+    LOG_VERBOSE("%s", default_gen_params.to_string().c_str());
 
     sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t(false);
     SDCtxPtr sd_ctx(new_sd_ctx(&sd_ctx_params));

examples/server/routes_openai.cpp

@@ -270,7 +270,7 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
                 return;
             }
 
-            LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
+            LOG_VERBOSE("%s\n", request.gen_params.to_string().c_str());
 
             SDImageVec results;
             if (!execute_sync_img_gen_request(*runtime, request, results, error_message)) {
@@ -344,7 +344,7 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
                 return;
             }
 
-            LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
+            LOG_VERBOSE("%s\n", request.gen_params.to_string().c_str());
 
             SDImageVec results;
             if (!execute_sync_img_gen_request(*runtime, request, results, error_message)) {

examples/server/routes_sdapi.cpp

@@ -330,7 +330,7 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
                 return;
             }
 
-            LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
+            LOG_VERBOSE("%s\n", request.gen_params.to_string().c_str());
 
             sd_img_gen_params_t img_gen_params = request.to_sd_img_gen_params_t();
             SDImageVec results;

examples/server/runtime.cpp

@@ -199,7 +199,6 @@ ArgOptions SDSvrParams::get_options() {
     };
 
     options.bool_options = {
-        {"-v", "--verbose", "print extra info", true, &verbose},
         {"", "--color", "colors the logging tags according to level", true, &color},
     };
 
@@ -212,6 +211,7 @@ ArgOptions SDSvrParams::get_options() {
     options.manual_options = {
         {"-h", "--help", "show this help message and exit", on_help_arg},
     };
+    add_log_options(options, log_level);
     return options;
 }
 
@@ -243,6 +243,7 @@ bool SDSvrParams::resolve_and_validate() {
 std::string SDSvrParams::to_string() const {
     std::ostringstream oss;
     oss << "SDSvrParams {\n"
+        << "  log_level: " << log_level_name(log_level) << ",\n"
         << "  listen_ip: " << listen_ip << ",\n"
         << "  listen_port: \"" << listen_port << "\",\n"
         << "  serve_html_path: \"" << serve_html_path << "\",\n"

examples/server/runtime.h

@@ -22,7 +22,7 @@ struct SDSvrParams {
     int listen_port       = 1234;
     std::string serve_html_path;
     bool normal_exit = false;
-    bool verbose     = false;
+    sd_log_level_t log_level = SD_LOG_INFO;
     bool color       = false;
 
     ArgOptions get_options();

include/stable-diffusion.h

@@ -136,14 +136,18 @@ enum sd_type_t {
     // SD_TYPE_IQ4_NL_4_4 = 36,
     // SD_TYPE_IQ4_NL_4_8 = 37,
     // SD_TYPE_IQ4_NL_8_8 = 38,
-    SD_TYPE_MXFP4 = 39,  // MXFP4 (1 block)
-    SD_TYPE_NVFP4 = 40,  // NVFP4 (4 blocks, E4M3 scale)
-    SD_TYPE_Q1_0  = 41,
-    SD_TYPE_COUNT = 42,
+    SD_TYPE_MXFP4   = 39,  // MXFP4 (1 block)
+    SD_TYPE_NVFP4   = 40,  // NVFP4 (4 blocks, E4M3 scale)
+    SD_TYPE_Q1_0    = 41,
+    SD_TYPE_Q2_0    = 42,
+    SD_TYPE_F8_E4M3 = 43,
+    SD_TYPE_F8_E5M2 = 44,
+    SD_TYPE_COUNT   = 45,
 };
 
 enum sd_log_level_t {
     SD_LOG_DEBUG,
+    SD_LOG_VERBOSE,
     SD_LOG_INFO,
     SD_LOG_WARN,
     SD_LOG_ERROR
@@ -226,8 +230,8 @@ typedef struct {
     bool vae_conv_direct;
     bool force_sdxl_vae_conv_scale;
     enum sd_vae_format_t vae_format;
-    const char* max_vram;  // GiB budget or backend assignment spec for graph-cut segmented param offload (0 = disabled, -1 = auto)
-    bool stream_layers;  // Enable residency+prefetch streaming on top of --max-vram (no effect without --max-vram)
+    const char* max_vram;  // Optional per-device GiB budget for managed weights and runner buffers; 0 uses live free VRAM without an explicit budget
+    bool disable_prefetch;  // Disable asynchronous next-segment weight prefetch
     bool eager_load;  // Load all params into the params backend at model-load time instead of lazily on first use
     const char* backend;
     const char* params_backend;
@@ -235,6 +239,7 @@ typedef struct {
     bool auto_fit;
     const char* rpc_servers;
     const char* model_args;
+    bool disable_segmented_compute;  // Force monolithic graph execution even when automatic graph cutting would fit memory better
 } sd_ctx_params_t;
 
 typedef struct {
@@ -443,6 +448,9 @@ typedef bool (*sd_graph_eval_callback_t)(struct ggml_tensor* t, bool ask, void*
 
 SD_API void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data);
 SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data);
+// In each sampling pass, a positive interval previews every Nth denoiser step, while a
+// negative interval previews only completed logical step -interval. Zero previews the final
+// completed step of the first sampling pass (base-resolution or high-noise).
 SD_API void sd_set_preview_callback(sd_preview_cb_t cb, enum preview_t mode, int interval, bool denoised, bool noisy, void* data);
 SD_API void sd_set_backend_eval_callback(sd_graph_eval_callback_t cb, void* data);
 SD_API int32_t sd_get_num_physical_cores();

src/conditioning/conditioner.hpp

@@ -142,14 +142,13 @@ public:
     virtual void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors)           = 0;
     virtual void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) {}
     virtual void set_max_graph_vram_bytes(size_t max_vram_bytes) {}
-    virtual void set_stream_layers_enabled(bool enabled) {}
     virtual void set_runtime_backends(const std::vector<ggml_backend_t>& backends) {}
     virtual void set_graph_cut_layer_split_enabled(bool enabled) {}
     virtual void set_graph_cut_layer_split_backend_vram_limits(const std::vector<size_t>& limits) {}
     virtual void get_layer_split_param_tensors(std::map<std::string, ggml_tensor*>& tensors) {}
     virtual void set_flash_attention_enabled(bool enabled) = 0;
     virtual void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) {}
-    virtual void runner_done() {}
+    virtual void runner_end() {}
 };
 
 // ldm.modules.encoders.modules.FrozenCLIPEmbedder
@@ -202,13 +201,6 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
         }
     }
 
-    void set_stream_layers_enabled(bool enabled) override {
-        text_model->set_stream_layers_enabled(enabled);
-        if (sd_version_is_sdxl(version)) {
-            text_model2->set_stream_layers_enabled(enabled);
-        }
-    }
-
     void set_runtime_backends(const std::vector<ggml_backend_t>& backends) override {
         text_model->set_runtime_backends(backends);
         if (sd_version_is_sdxl(version)) {
@@ -244,10 +236,10 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
         }
     }
 
-    void runner_done() override {
-        text_model->runner_done();
+    void runner_end() override {
+        text_model->runner_end();
         if (sd_version_is_sdxl(version)) {
-            text_model2->runner_done();
+            text_model2->runner_end();
         }
     }
 
@@ -259,7 +251,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
         }
         auto iter = embedding_pos_map.find(embd_name);
         if (iter != embedding_pos_map.end()) {
-            LOG_DEBUG("embedding already read in: %s", embd_name.c_str());
+            LOG_VERBOSE("embedding already read in: %s", embd_name.c_str());
             for (int i = iter->second.first; i < iter->second.second; i++) {
                 bpe_tokens.push_back(text_model->model.vocab_size + i);
             }
@@ -279,11 +271,11 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
                         embd2       = ggml_new_tensor_2d(embd_ctx, tensor_storage.type, text_model2->model.hidden_size, tensor_storage.n_dims > 1 ? tensor_storage.ne[1] : 1);
                         *dst_tensor = embd2;
                     } else {
-                        LOG_DEBUG("embedding wrong hidden size, got %i, expected %i or %i", tensor_storage.ne[0], text_model->model.hidden_size, text_model2->model.hidden_size);
+                        LOG_VERBOSE("embedding wrong hidden size, got %i, expected %i or %i", tensor_storage.ne[0], text_model->model.hidden_size, text_model2->model.hidden_size);
                         return false;
                     }
                 } else {
-                    LOG_DEBUG("embedding wrong hidden size, got %i, expected %i", tensor_storage.ne[0], text_model->model.hidden_size);
+                    LOG_VERBOSE("embedding wrong hidden size, got %i, expected %i", tensor_storage.ne[0], text_model->model.hidden_size);
                     return false;
                 }
             } else {
@@ -303,10 +295,10 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
                    ggml_nbytes(embd));
             for (int i = 0; i < embd->ne[1]; i++) {
                 bpe_tokens.push_back(text_model->model.vocab_size + num_custom_embeddings);
-                // LOG_DEBUG("new custom token: %i", text_model.vocab_size + num_custom_embeddings);
+                // LOG_VERBOSE("new custom token: %i", text_model.vocab_size + num_custom_embeddings);
                 num_custom_embeddings++;
             }
-            LOG_DEBUG("embedding '%s' applied, custom embeddings: %i", embd_name.c_str(), num_custom_embeddings);
+            LOG_VERBOSE("embedding '%s' applied, custom embeddings: %i", embd_name.c_str(), num_custom_embeddings);
         }
         if (embd2) {
             int64_t hidden_size = text_model2->model.hidden_size;
@@ -316,10 +308,10 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
                    ggml_nbytes(embd2));
             for (int i = 0; i < embd2->ne[1]; i++) {
                 bpe_tokens.push_back(text_model2->model.vocab_size + num_custom_embeddings_2);
-                // LOG_DEBUG("new custom token: %i", text_model.vocab_size + num_custom_embeddings);
+                // LOG_VERBOSE("new custom token: %i", text_model.vocab_size + num_custom_embeddings);
                 num_custom_embeddings_2++;
             }
-            LOG_DEBUG("embedding '%s' applied, custom embeddings: %i (text model 2)", embd_name.c_str(), num_custom_embeddings_2);
+            LOG_VERBOSE("embedding '%s' applied, custom embeddings: %i (text model 2)", embd_name.c_str(), num_custom_embeddings_2);
         }
         int pos_end = num_custom_embeddings;
         if (pos_end == pos_start) {
@@ -368,7 +360,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
                 ss << "['" << item.first << "', " << item.second << "], ";
             }
             ss << "]";
-            LOG_DEBUG("parse '%s' to %s", text.c_str(), ss.str().c_str());
+            LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str());
         }
 
         auto on_new_token_cb = [&](std::string& str, std::vector<int32_t>& bpe_tokens) -> bool {
@@ -389,7 +381,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
                 size_t padding_size = (75 - (current_size % 75)) % 75;  // Ensure no negative padding
 
                 if (padding_size > 0) {
-                    LOG_DEBUG("BREAK token encountered, padding current chunk by %zu tokens.", padding_size);
+                    LOG_VERBOSE("BREAK token encountered, padding current chunk by %zu tokens.", padding_size);
                     tokens.insert(tokens.end(), padding_size, tokenizer.EOS_TOKEN_ID);
                     weights.insert(weights.end(), padding_size, 1.0f);
                 }
@@ -461,9 +453,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
                                                                max_token_idx,
                                                                false,
                                                                clip_skip,
-                                                               false,
-                                                               true,
-                                                               true);
+                                                               false);
                 GGML_ASSERT(!chunk_hidden_states.empty());
                 if (sd_version_is_sdxl(version)) {
                     auto chunk_hidden_states2 = text_model2->compute(n_threads,
@@ -473,9 +463,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
                                                                      max_token_idx,
                                                                      false,
                                                                      clip_skip,
-                                                                     false,
-                                                                     true,
-                                                                     true);
+                                                                     false);
                     GGML_ASSERT(!chunk_hidden_states2.empty());
                     chunk_hidden_states = sd::ops::concat(chunk_hidden_states, chunk_hidden_states2, 0);
 
@@ -487,14 +475,12 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
                                                       max_token_idx,
                                                       true,
                                                       clip_skip,
-                                                      false,
-                                                      true,
-                                                      true);
+                                                      false);
                         GGML_ASSERT(!pooled.empty());
                     }
                 }
                 int64_t t1 = ggml_time_ms();
-                LOG_DEBUG("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
+                LOG_VERBOSE("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
 
                 chunk_hidden_states = apply_token_weights(std::move(chunk_hidden_states), chunk_weights);
 
@@ -608,7 +594,7 @@ struct FrozenCLIPVisionEmbedder : public GGMLRunner {
         auto get_graph = [&]() -> ggml_cgraph* {
             return build_graph(pixel_values, return_pooled, clip_skip);
         };
-        return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, true, true, true));
+        return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, true));
     }
 };
 
@@ -675,18 +661,6 @@ struct SD3CLIPEmbedder : public Conditioner {
         }
     }
 
-    void set_stream_layers_enabled(bool enabled) override {
-        if (clip_l) {
-            clip_l->set_stream_layers_enabled(enabled);
-        }
-        if (clip_g) {
-            clip_g->set_stream_layers_enabled(enabled);
-        }
-        if (t5) {
-            t5->set_stream_layers_enabled(enabled);
-        }
-    }
-
     void set_runtime_backends(const std::vector<ggml_backend_t>& backends) override {
         if (clip_l) {
             clip_l->set_runtime_backends(backends);
@@ -753,15 +727,15 @@ struct SD3CLIPEmbedder : public Conditioner {
         }
     }
 
-    void runner_done() override {
+    void runner_end() override {
         if (clip_l) {
-            clip_l->runner_done();
+            clip_l->runner_end();
         }
         if (clip_g) {
-            clip_g->runner_done();
+            clip_g->runner_end();
         }
         if (t5) {
-            t5->runner_done();
+            t5->runner_end();
         }
     }
 
@@ -778,7 +752,7 @@ struct SD3CLIPEmbedder : public Conditioner {
                 ss << "['" << item.first << "', " << item.second << "], ";
             }
             ss << "]";
-            LOG_DEBUG("parse '%s' to %s", text.c_str(), ss.str().c_str());
+            LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str());
         }
 
         auto on_new_token_cb = [&](std::string& str, std::vector<int32_t>& bpe_tokens) -> bool {
@@ -881,9 +855,7 @@ struct SD3CLIPEmbedder : public Conditioner {
                                                         max_token_idx,
                                                         false,
                                                         clip_skip,
-                                                        false,
-                                                        true,
-                                                        true);
+                                                        false);
                 GGML_ASSERT(!chunk_hidden_states_l.empty());
                 chunk_hidden_states_l = ::apply_token_weights(std::move(chunk_hidden_states_l), chunk_weights);
 
@@ -897,9 +869,7 @@ struct SD3CLIPEmbedder : public Conditioner {
                                                     max_token_idx,
                                                     true,
                                                     clip_skip,
-                                                    false,
-                                                    true,
-                                                    true);
+                                                    false);
                     GGML_ASSERT(!pooled_l.empty());
                 }
             } else {
@@ -928,9 +898,7 @@ struct SD3CLIPEmbedder : public Conditioner {
                                                         max_token_idx,
                                                         false,
                                                         clip_skip,
-                                                        false,
-                                                        true,
-                                                        true);
+                                                        false);
                 GGML_ASSERT(!chunk_hidden_states_g.empty());
                 chunk_hidden_states_g = ::apply_token_weights(std::move(chunk_hidden_states_g), chunk_weights);
 
@@ -944,9 +912,7 @@ struct SD3CLIPEmbedder : public Conditioner {
                                                     max_token_idx,
                                                     true,
                                                     clip_skip,
-                                                    false,
-                                                    true,
-                                                    true);
+                                                    false);
                     GGML_ASSERT(!pooled_g.empty());
                 }
             } else {
@@ -969,9 +935,7 @@ struct SD3CLIPEmbedder : public Conditioner {
                 chunk_hidden_states_t5 = t5->compute(n_threads,
                                                      input_ids,
                                                      sd::Tensor<float>(),
-                                                     false,
-                                                     true,
-                                                     true);
+                                                     false);
                 GGML_ASSERT(!chunk_hidden_states_t5.empty());
                 chunk_hidden_states_t5 = ::apply_token_weights(std::move(chunk_hidden_states_t5), chunk_weights);
             } else {
@@ -996,7 +960,7 @@ struct SD3CLIPEmbedder : public Conditioner {
             }
 
             int64_t t1 = ggml_time_ms();
-            LOG_DEBUG("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
+            LOG_VERBOSE("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
             if (zero_out_masked) {
                 chunk_hidden_states.fill_(0.0f);
             }
@@ -1079,15 +1043,6 @@ struct FluxCLIPEmbedder : public Conditioner {
         }
     }
 
-    void set_stream_layers_enabled(bool enabled) override {
-        if (clip_l) {
-            clip_l->set_stream_layers_enabled(enabled);
-        }
-        if (t5) {
-            t5->set_stream_layers_enabled(enabled);
-        }
-    }
-
     void set_runtime_backends(const std::vector<ggml_backend_t>& backends) override {
         if (clip_l) {
             clip_l->set_runtime_backends(backends);
@@ -1139,12 +1094,12 @@ struct FluxCLIPEmbedder : public Conditioner {
         }
     }
 
-    void runner_done() override {
+    void runner_end() override {
         if (clip_l) {
-            clip_l->runner_done();
+            clip_l->runner_end();
         }
         if (t5) {
-            t5->runner_done();
+            t5->runner_end();
         }
     }
 
@@ -1160,7 +1115,7 @@ struct FluxCLIPEmbedder : public Conditioner {
                 ss << "['" << item.first << "', " << item.second << "], ";
             }
             ss << "]";
-            LOG_DEBUG("parse '%s' to %s", text.c_str(), ss.str().c_str());
+            LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str());
         }
 
         auto on_new_token_cb = [&](std::string& str, std::vector<int32_t>& bpe_tokens) -> bool {
@@ -1247,9 +1202,7 @@ struct FluxCLIPEmbedder : public Conditioner {
                                              max_token_idx,
                                              true,
                                              clip_skip,
-                                             false,
-                                             true,
-                                             true);
+                                             false);
                     GGML_ASSERT(!pooled.empty());
                 } else {
                     pooled = sd::Tensor<float>::zeros({768});
@@ -1268,9 +1221,7 @@ struct FluxCLIPEmbedder : public Conditioner {
                 chunk_hidden_states = t5->compute(n_threads,
                                                   input_ids,
                                                   sd::Tensor<float>(),
-                                                  false,
-                                                  true,
-                                                  true);
+                                                  false);
                 GGML_ASSERT(!chunk_hidden_states.empty());
                 chunk_hidden_states = ::apply_token_weights(std::move(chunk_hidden_states), chunk_weights);
                 if (zero_out_masked) {
@@ -1281,7 +1232,7 @@ struct FluxCLIPEmbedder : public Conditioner {
             }
 
             int64_t t1 = ggml_time_ms();
-            LOG_DEBUG("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
+            LOG_VERBOSE("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
             if (!hidden_states.empty()) {
                 hidden_states = sd::ops::concat(hidden_states, chunk_hidden_states, 1);
             } else {
@@ -1366,12 +1317,6 @@ struct T5CLIPEmbedder : public Conditioner {
         }
     }
 
-    void set_stream_layers_enabled(bool enabled) override {
-        if (t5) {
-            t5->set_stream_layers_enabled(enabled);
-        }
-    }
-
     void set_runtime_backends(const std::vector<ggml_backend_t>& backends) override {
         if (t5) {
             t5->set_runtime_backends(backends);
@@ -1408,9 +1353,9 @@ struct T5CLIPEmbedder : public Conditioner {
         }
     }
 
-    void runner_done() override {
+    void runner_end() override {
         if (t5) {
-            t5->runner_done();
+            t5->runner_end();
         }
     }
 
@@ -1426,7 +1371,7 @@ struct T5CLIPEmbedder : public Conditioner {
                 ss << "['" << item.first << "', " << item.second << "], ";
             }
             ss << "]";
-            LOG_DEBUG("parse '%s' to %s", text.c_str(), ss.str().c_str());
+            LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str());
         }
 
         auto on_new_token_cb = [&](std::string& str, std::vector<int32_t>& bpe_tokens) -> bool {
@@ -1467,7 +1412,7 @@ struct T5CLIPEmbedder : public Conditioner {
                 ++num_pad;
             }
         }
-        // LOG_DEBUG("PAD: %d", num_pad);
+        // LOG_VERBOSE("PAD: %d", num_pad);
     }
 
     SDCondition get_learned_condition_common(int n_threads,
@@ -1508,9 +1453,7 @@ struct T5CLIPEmbedder : public Conditioner {
             auto chunk_hidden_states = t5->compute(n_threads,
                                                    input_ids,
                                                    t5_attn_mask_chunk,
-                                                   false,
-                                                   true,
-                                                   true);
+                                                   false);
             GGML_ASSERT(!chunk_hidden_states.empty());
             chunk_hidden_states = apply_token_weights(std::move(chunk_hidden_states), chunk_weights);
 
@@ -1521,7 +1464,7 @@ struct T5CLIPEmbedder : public Conditioner {
             }
 
             int64_t t1 = ggml_time_ms();
-            LOG_DEBUG("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
+            LOG_VERBOSE("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
 
             if (!hidden_states.empty()) {
                 hidden_states = sd::ops::concat(hidden_states, chunk_hidden_states, 1);
@@ -1582,12 +1525,6 @@ struct MiniT2IConditioner : public Conditioner {
         }
     }
 
-    void set_stream_layers_enabled(bool enabled) override {
-        if (t5) {
-            t5->set_stream_layers_enabled(enabled);
-        }
-    }
-
     void set_runtime_backends(const std::vector<ggml_backend_t>& backends) override {
         if (t5) {
             t5->set_runtime_backends(backends);
@@ -1624,9 +1561,9 @@ struct MiniT2IConditioner : public Conditioner {
         }
     }
 
-    void runner_done() override {
+    void runner_end() override {
         if (t5) {
-            t5->runner_done();
+            t5->runner_end();
         }
     }
 
@@ -1657,9 +1594,7 @@ struct MiniT2IConditioner : public Conditioner {
         sd::Tensor<float> hidden_states = t5->compute(n_threads,
                                                       input_ids,
                                                       sd::Tensor<float>::from_vector(t5_mask),
-                                                      false,
-                                                      true,
-                                                      true);
+                                                      false);
         GGML_ASSERT(!hidden_states.empty());
         result.c_crossattn = std::move(hidden_states);
         result.c_vector    = sd::Tensor<float>::from_vector(mask);
@@ -1696,10 +1631,6 @@ struct AnimaConditioner : public Conditioner {
         llm->set_max_graph_vram_bytes(max_vram_bytes);
     }
 
-    void set_stream_layers_enabled(bool enabled) override {
-        llm->set_stream_layers_enabled(enabled);
-    }
-
     void set_runtime_backends(const std::vector<ggml_backend_t>& backends) override {
         llm->set_runtime_backends(backends);
     }
@@ -1724,8 +1655,8 @@ struct AnimaConditioner : public Conditioner {
         llm->set_weight_adapter(adapter);
     }
 
-    void runner_done() override {
-        llm->runner_done();
+    void runner_end() override {
+        llm->runner_end();
     }
 
     std::tuple<std::vector<int>, std::vector<float>, std::vector<int>, std::vector<float>> tokenize(std::string text) {
@@ -1738,7 +1669,7 @@ struct AnimaConditioner : public Conditioner {
                 ss << "['" << item.first << "', " << item.second << "], ";
             }
             ss << "]";
-            LOG_DEBUG("parse '%s' to %s", text.c_str(), ss.str().c_str());
+            LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str());
         }
 
         std::vector<int> qwen_tokens;
@@ -1787,16 +1718,14 @@ struct AnimaConditioner : public Conditioner {
                                           {},
                                           {},
                                           false,
-                                          false,
-                                          true,
-                                          true);
+                                          false);
         GGML_ASSERT(!hidden_states.empty());
         hidden_states         = apply_token_weights(std::move(hidden_states), qwen_weights);
         auto t5_ids_tensor    = sd::Tensor<int32_t>::from_vector(t5_tokens);
         auto t5_weight_tensor = sd::Tensor<float>::from_vector(t5_weights);
 
         int64_t t1 = ggml_time_ms();
-        LOG_DEBUG("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
+        LOG_VERBOSE("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
 
         SDCondition result;
         result.c_crossattn  = std::move(hidden_states);
@@ -1887,13 +1816,6 @@ struct LLMEmbedder : public Conditioner {
         }
     }
 
-    void set_stream_layers_enabled(bool enabled) override {
-        llm->set_stream_layers_enabled(enabled);
-        if (byt5) {
-            byt5->set_stream_layers_enabled(enabled);
-        }
-    }
-
     void set_runtime_backends(const std::vector<ggml_backend_t>& backends) override {
         llm->set_runtime_backends(backends);
         if (byt5) {
@@ -1942,12 +1864,12 @@ struct LLMEmbedder : public Conditioner {
         }
     }
 
-    void runner_done() override {
+    void runner_end() override {
         if (llm) {
-            llm->runner_done();
+            llm->runner_end();
         }
         if (byt5) {
-            byt5->runner_done();
+            byt5->runner_end();
         }
     }
 
@@ -1984,7 +1906,7 @@ struct LLMEmbedder : public Conditioner {
                 ss << "['" << item.first << "', " << item.second << "], ";
             }
             ss << "]";
-            LOG_DEBUG("parse '%s' to %s", text.c_str(), ss.str().c_str());
+            LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str());
         }
 
         std::vector<int> tokens;
@@ -2051,8 +1973,6 @@ struct LLMEmbedder : public Conditioner {
                                           out_layers,
                                           false,
                                           false,
-                                          true,
-                                          true,
                                           deepstack_image_embeds,
                                           image_grids);
         GGML_ASSERT(!hidden_states.empty());
@@ -2220,9 +2140,7 @@ struct LLMEmbedder : public Conditioner {
                             prompt += "<Picture " + std::to_string(++picture_index) + ">: ";
                             add_vision_outputs(llm->encode_image_outputs(n_threads,
                                                                          resized,
-                                                                         false,
-                                                                         true,
-                                                                         true),
+                                                                         false),
                                                static_cast<int>(resized.shape()[1]) / patch_size,
                                                static_cast<int>(resized.shape()[0]) / patch_size);
                             continue;
@@ -2250,9 +2168,7 @@ struct LLMEmbedder : public Conditioner {
                             auto pair = sd::ops::concat(first.unsqueeze(2), second.unsqueeze(2), 2);
                             add_vision_outputs(llm->encode_video_block_outputs(n_threads,
                                                                                pair,
-                                                                               false,
-                                                                               true,
-                                                                               true),
+                                                                               false),
                                                static_cast<int>(first.shape()[1]) / patch_size,
                                                static_cast<int>(first.shape()[0]) / patch_size);
                         }
@@ -2263,9 +2179,7 @@ struct LLMEmbedder : public Conditioner {
                         prompt += "<Picture " + std::to_string(i + 1) + ">: ";
                         add_vision_outputs(llm->encode_image_outputs(n_threads,
                                                                      resized,
-                                                                     false,
-                                                                     true,
-                                                                     true),
+                                                                     false),
                                            static_cast<int>(resized.shape()[1]) / patch_size,
                                            static_cast<int>(resized.shape()[0]) / patch_size);
                     }
@@ -2310,7 +2224,7 @@ struct LLMEmbedder : public Conditioner {
                     prompt_template_encode_start_idx++;
                 }
             }
-            LOG_DEBUG("prompt_template_encode_start_idx %d", prompt_template_encode_start_idx);
+            LOG_VERBOSE("prompt_template_encode_start_idx %d", prompt_template_encode_start_idx);
 
             prompt = prompt_prefix;
             if (llm->enable_vision && conditioner_params.ref_images != nullptr && !conditioner_params.ref_images->empty()) {
@@ -2350,9 +2264,9 @@ struct LLMEmbedder : public Conditioner {
 
                     resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode);
 
-                    LOG_DEBUG("resize LingBotVideo ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar);
+                    LOG_VERBOSE("resize LingBotVideo ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar);
                     auto resized_image = clip_preprocess(image, w_bar, h_bar);
-                    auto image_embed   = llm->encode_image(n_threads, resized_image, false, true, true);
+                    auto image_embed   = llm->encode_image(n_threads, resized_image, false);
                     GGML_ASSERT(!image_embed.empty());
 
                     std::string image_prefix = prompt + img_prompt + "<|vision_start|>";
@@ -2407,11 +2321,11 @@ struct LLMEmbedder : public Conditioner {
 
                     resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode);
 
-                    LOG_DEBUG("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar);
+                    LOG_VERBOSE("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar);
 
                     auto resized_image = clip_preprocess(image, w_bar, h_bar);
 
-                    auto image_embed = llm->encode_image(n_threads, resized_image, false, true, true);
+                    auto image_embed = llm->encode_image(n_threads, resized_image, false);
                     GGML_ASSERT(!image_embed.empty());
                     image_embeds.emplace_back(image_embed_idx, image_embed);
                     image_embed_idx += 1 + static_cast<int>(image_embed.shape()[1]) + 6;
@@ -2491,10 +2405,10 @@ struct LLMEmbedder : public Conditioner {
 
                     resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode);
 
-                    LOG_DEBUG("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar);
+                    LOG_VERBOSE("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar);
 
                     auto resized_image = clip_preprocess(image, w_bar, h_bar);
-                    auto image_embed   = llm->encode_image(n_threads, resized_image, false, true, true);
+                    auto image_embed   = llm->encode_image(n_threads, resized_image, false);
                     GGML_ASSERT(!image_embed.empty());
 
                     std::string image_prefix = prompt_prefix + img_prompt + "<|vision_start|>";
@@ -2559,10 +2473,10 @@ struct LLMEmbedder : public Conditioner {
 
                     resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode);
 
-                    LOG_DEBUG("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar);
+                    LOG_VERBOSE("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar);
 
                     auto resized_image = clip_preprocess(image, w_bar, h_bar);
-                    auto image_embed   = llm->encode_image(n_threads, resized_image, false, true, true);
+                    auto image_embed   = llm->encode_image(n_threads, resized_image, false);
                     GGML_ASSERT(!image_embed.empty());
 
                     std::string image_prefix = prompt + img_prompt + "Picture " + std::to_string(i + 1) + ": <|vision_start|>";
@@ -2622,10 +2536,10 @@ struct LLMEmbedder : public Conditioner {
 
                     resize_image_dims(height, width, h_bar, w_bar, factor, min_pixels, max_pixels, resize_mode);
 
-                    LOG_DEBUG("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar);
+                    LOG_VERBOSE("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar);
 
                     auto resized_image = clip_preprocess(image, w_bar, h_bar);
-                    auto image_embed   = llm->encode_image(n_threads, resized_image, false, true, true);
+                    auto image_embed   = llm->encode_image(n_threads, resized_image, false);
                     GGML_ASSERT(!image_embed.empty());
                     image_embeds.emplace_back(image_embed_idx, image_embed);
                     image_embed_idx += 1 + static_cast<int>(image_embed.shape()[1]) + 6;
@@ -2802,7 +2716,7 @@ struct LLMEmbedder : public Conditioner {
             }
 
             int64_t t1 = ggml_time_ms();
-            LOG_DEBUG("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
+            LOG_VERBOSE("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
 
             SDCondition result;
             result.c_crossattn = std::move(hidden_states);
@@ -2857,9 +2771,7 @@ struct LLMEmbedder : public Conditioner {
                 auto byt5_hidden_states = byt5->compute(n_threads,
                                                         input_ids,
                                                         sd::Tensor<float>(),
-                                                        false,
-                                                        true,
-                                                        true);
+                                                        false);
                 GGML_ASSERT(!byt5_hidden_states.empty());
                 extra_hidden_states_vec.push_back(std::move(byt5_hidden_states));
             }
@@ -2879,7 +2791,7 @@ struct LLMEmbedder : public Conditioner {
         }
 
         int64_t t1 = ggml_time_ms();
-        LOG_DEBUG("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
+        LOG_VERBOSE("computing condition graph completed, taking %" PRId64 " ms", t1 - t0);
         SDCondition result;
         result.c_crossattn        = std::move(hidden_states);
         result.extra_c_crossattns = std::move(extra_hidden_states_vec);
@@ -2960,13 +2872,11 @@ struct LTXAVTextProjectionRunner : public GGMLRunner {
 
     sd::Tensor<float> compute(int n_threads,
                               const sd::Tensor<float>& x,
-                              bool auto_free           = true,
-                              bool free_compute_buffer = true,
-                              bool free_compute_params = true) {
+                              bool auto_runner_end = true) {
         auto get_graph = [&]() -> ggml_cgraph* {
             return build_graph(x);
         };
-        return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, auto_free, free_compute_buffer, free_compute_params));
+        return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, auto_runner_end));
     }
 };
 
@@ -2978,15 +2888,36 @@ struct LTXAVEmbedder : public Conditioner {
     std::shared_ptr<GemmaTokenizer> tokenizer;
     std::shared_ptr<LLM::LLMRunner> llm;
     std::shared_ptr<LTXAVTextProjectionRunner> projector;
+    std::string projector_prefix;
     bool dual_projection = false;
 
+    // Gemma 4 keeps a per-layer output scalar that no Gemma 3 checkpoint has, and widens its
+    // full-attention heads to 512 so their q_proj is twice a sliding layer's.
+    static LLM::LLMArch detect_gemma_arch(const String2TensorStorage& tensor_storage_map,
+                                          const std::string& llm_prefix) {
+        if (tensor_storage_map.find(llm_prefix + ".model.layers.0.layer_scalar") != tensor_storage_map.end()) {
+            return LLM::LLMArch::GEMMA4_12B;
+        }
+        auto global_q  = tensor_storage_map.find(llm_prefix + ".model.layers.5.self_attn.q_proj.weight");
+        auto sliding_q = tensor_storage_map.find(llm_prefix + ".model.layers.0.self_attn.q_proj.weight");
+        if (global_q != tensor_storage_map.end() &&
+            sliding_q != tensor_storage_map.end() &&
+            global_q->second.ne[1] == sliding_q->second.ne[1] * 2) {
+            return LLM::LLMArch::GEMMA4_12B;
+        }
+        return LLM::LLMArch::GEMMA3_12B;
+    }
+
     LTXAVEmbedder(ggml_backend_t backend,
                   const String2TensorStorage& tensor_storage_map      = {},
                   const std::string& llm_prefix                       = "text_encoders.llm",
                   const std::string& projector_prefix                 = "text_embedding_projection",
-                  std::shared_ptr<RunnerWeightManager> weight_manager = nullptr) {
+                  std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+        : projector_prefix(projector_prefix) {
+        LLM::LLMArch arch = detect_gemma_arch(tensor_storage_map, llm_prefix);
+        LOG_INFO("ltxav text encoder: %s", arch == LLM::LLMArch::GEMMA4_12B ? "gemma 4" : "gemma 3");
         tokenizer       = std::make_shared<GemmaTokenizer>();
-        llm             = std::make_shared<LLM::LLMRunner>(LLM::LLMArch::GEMMA3_12B,
+        llm             = std::make_shared<LLM::LLMRunner>(arch,
                                                backend,
                                                tensor_storage_map,
                                                llm_prefix,
@@ -3001,7 +2932,7 @@ struct LTXAVEmbedder : public Conditioner {
 
     void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
         llm->get_param_tensors(tensors, "text_encoders.llm");
-        projector->get_param_tensors(tensors, "text_embedding_projection");
+        projector->get_param_tensors(tensors, projector_prefix);
     }
 
     void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& tensor_ops) override {
@@ -3039,9 +2970,9 @@ struct LTXAVEmbedder : public Conditioner {
         projector->set_weight_adapter(adapter);
     }
 
-    void runner_done() override {
-        llm->runner_done();
-        projector->runner_done();
+    void runner_end() override {
+        llm->runner_end();
+        projector->runner_end();
     }
 
     std::tuple<std::vector<int>, std::vector<float>, std::vector<float>> tokenize(std::string text,
@@ -3108,9 +3039,7 @@ struct LTXAVEmbedder : public Conditioner {
                                           {},
                                           {},
                                           true,
-                                          false,
-                                          true,
-                                          true);
+                                          false);
         GGML_ASSERT(!hidden_states.empty());
         hidden_states = apply_token_weights(std::move(hidden_states), weights);
 
@@ -3169,7 +3098,7 @@ struct LTXAVEmbedder : public Conditioner {
         }
 
         hidden_states.reshape_({kNumStates * kHiddenSize, valid_tokens});
-        return projector->compute(n_threads, hidden_states, false, true, true);
+        return projector->compute(n_threads, hidden_states, false);
     }
 
     SDCondition get_learned_condition(int n_threads,
@@ -3186,7 +3115,7 @@ struct LTXAVEmbedder : public Conditioner {
         GGML_ASSERT(!hidden_states.empty());
 
         int64_t t1 = ggml_time_ms();
-        LOG_DEBUG("computing LTXAV condition graph completed, taking %" PRId64 " ms", t1 - t0);
+        LOG_VERBOSE("computing LTXAV condition graph completed, taking %" PRId64 " ms", t1 - t0);
 
         SDCondition result;
         result.c_crossattn = std::move(hidden_states);

src/core/backend_fit.cpp

@@ -2,377 +2,393 @@
 
 #include <algorithm>
 #include <cctype>
+#include <cstddef>
 #include <cstdint>
+#include <fstream>
 #include <utility>
 #include <vector>
 
+#if defined(_WIN32)
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#elif defined(__APPLE__)
+#include <mach/mach.h>
+#endif
+
 #include "core/ggml_extend_backend.h"
 #include "core/util.h"
 #include "ggml-backend.h"
 
 namespace sd::backend_fit {
-    namespace {
 
-        constexpr int64_t MiB = 1024ll * 1024;
+    static constexpr int64_t MiB = 1024ll * 1024;
 
-        enum class ComponentKind {
-            DIT         = 0,
-            VAE         = 1,
-            CONDITIONER = 2,
-        };
+    enum class ComponentKind {
+        DIT,
+        CONDITIONER,
+        VAE,
+    };
 
-        struct Component {
+    struct Component {
+        ComponentKind kind;
+        const char* name;
+        int64_t params_bytes  = 0;
+        int64_t reserve_bytes = 0;
+        int64_t staging_bytes = 0;
+    };
+
+    struct Device {
+        std::string name;
+        std::string description;
+        int64_t free_bytes   = 0;
+        int64_t budget_bytes = 0;
+    };
+
+    enum class ParamsLocation {
+        MAIN_GPU,
+        CPU,
+        OTHER_GPU,
+        DISK,
+    };
+
+    struct Decision {
+        ParamsLocation params_location = ParamsLocation::DISK;
+        size_t params_device           = SIZE_MAX;
+    };
+
+    struct Plan {
+        bool valid         = false;
+        size_t main_device = SIZE_MAX;
+        std::vector<Decision> decisions;
+    };
+
+    static bool classify_tensor(const std::string& name, ComponentKind& out) {
+        auto contains = [&](const char* s) { return name.find(s) != std::string::npos; };
+
+        if (contains("model.diffusion_model.") || contains("unet.")) {
+            out = ComponentKind::DIT;
+            return true;
+        }
+        if (contains("first_stage_model.") ||
+            name.rfind("vae.", 0) == 0 ||
+            name.rfind("tae.", 0) == 0) {
+            out = ComponentKind::VAE;
+            return true;
+        }
+        if (contains("text_encoders") ||
+            contains("cond_stage_model") ||
+            contains("te.text_model.") ||
+            contains("conditioner") ||
+            name.rfind("text_encoder.", 0) == 0 ||
+            name.rfind("text_embedding_projection.", 0) == 0 ||
+            contains(".aggregate_embed.")) {
+            out = ComponentKind::CONDITIONER;
+            return true;
+        }
+        return false;
+    }
+
+    static std::vector<Component> estimate_components(ModelLoader& loader, ggml_type override_wtype) {
+        int64_t bytes[3]          = {0, 0, 0};
+        int64_t largest_tensor[3] = {0, 0, 0};
+        for (const auto& [name, stored_tensor] : loader.get_tensor_storage_map()) {
+            TensorStorage ts = stored_tensor;
             ComponentKind kind;
-            const char* name;
-            int64_t params_bytes  = 0;
-            int64_t reserve_bytes = 0;
-            bool splittable       = false;
-        };
-
-        struct Device {
-            ggml_backend_dev_t dev = nullptr;
-            std::string name;
-            std::string description;
-            int64_t free_bytes   = 0;
-            int64_t total_bytes  = 0;
-            int64_t budget_bytes = 0;
-        };
-
-        struct Decision {
-            ComponentKind kind;
-            bool on_cpu = false;
-            std::vector<size_t> device_idxs;
-        };
-
-        struct Plan {
-            bool valid      = false;
-            bool time_share = false;
-            std::vector<Decision> decisions;
-        };
-
-        bool classify_tensor(const std::string& name, ComponentKind& out) {
-            auto contains = [&](const char* s) { return name.find(s) != std::string::npos; };
-
-            if (contains("model.diffusion_model.") || contains("unet.")) {
-                out = ComponentKind::DIT;
-                return true;
+            if (is_unused_tensor(ts.name) || !classify_tensor(ts.name, kind)) {
+                continue;
             }
-            if (contains("first_stage_model.") ||
-                name.rfind("vae.", 0) == 0 ||
-                name.rfind("tae.", 0) == 0) {
-                out = ComponentKind::VAE;
-                return true;
+            if (ts.expected_type != GGML_TYPE_COUNT) {
+                ts.type = ts.expected_type;
+            } else if (override_wtype != GGML_TYPE_COUNT && loader.tensor_should_be_converted(ts, override_wtype)) {
+                ts.type = override_wtype;
             }
-            if (contains("text_encoders") ||
-                contains("cond_stage_model") ||
-                contains("te.text_model.") ||
-                contains("conditioner") ||
-                name.rfind("text_encoder.", 0) == 0 ||
-                name.rfind("text_embedding_projection.", 0) == 0 ||
-                contains(".aggregate_embed.")) {
-                out = ComponentKind::CONDITIONER;
-                return true;
-            }
-            return false;
+            const int64_t tensor_bytes = (int64_t)ts.nbytes() + 64;
+            bytes[int(kind)] += tensor_bytes;
+            largest_tensor[int(kind)] = std::max(largest_tensor[int(kind)], tensor_bytes);
         }
 
-        std::vector<Component> estimate_components(ModelLoader& loader, ggml_type override_wtype) {
-            const auto& storage = loader.get_tensor_storage_map();
+        return {
+            {ComponentKind::DIT, "DiT", bytes[int(ComponentKind::DIT)], 2048 * MiB, largest_tensor[int(ComponentKind::DIT)]},
+            {ComponentKind::CONDITIONER, "Conditioner", bytes[int(ComponentKind::CONDITIONER)], 2048 * MiB, largest_tensor[int(ComponentKind::CONDITIONER)]},
+            {ComponentKind::VAE, "VAE", bytes[int(ComponentKind::VAE)], 1024 * MiB, largest_tensor[int(ComponentKind::VAE)]},
+        };
+    }
 
-            int64_t bytes[3] = {0, 0, 0};
-            for (const auto& [name, ts_const] : storage) {
-                TensorStorage ts = ts_const;
-                if (is_unused_tensor(ts.name)) {
-                    continue;
-                }
-                ComponentKind kind;
-                if (!classify_tensor(ts.name, kind)) {
-                    continue;
-                }
-                if (override_wtype != GGML_TYPE_COUNT &&
-                    loader.tensor_should_be_converted(ts, override_wtype)) {
-                    ts.type = override_wtype;
-                } else if (ts.expected_type != GGML_TYPE_COUNT && ts.expected_type != ts.type) {
-                    ts.type = ts.expected_type;
-                }
-                bytes[int(kind)] += (int64_t)ts.nbytes() + 64;
+    static std::string budget_key(std::string name) {
+        std::transform(name.begin(), name.end(), name.begin(), [](unsigned char c) { return (char)std::tolower(c); });
+        return name;
+    }
+
+    static std::vector<Device> enumerate_gpu_devices(const sd::ggml_graph_cut::MaxVramAssignment& budgets) {
+        std::vector<Device> out;
+        for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
+            ggml_backend_dev_t dev = ggml_backend_dev_get(i);
+            if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU) {
+                continue;
             }
+            Device device;
+            device.name        = ggml_backend_dev_name(dev);
+            device.description = ggml_backend_dev_description(dev);
+            size_t free_bytes = 0, total_bytes = 0;
+            ggml_backend_dev_memory(dev, &free_bytes, &total_bytes);
+            device.free_bytes = (int64_t)free_bytes;
 
-            std::vector<Component> out;
-            out.push_back({ComponentKind::DIT, "DiT", bytes[int(ComponentKind::DIT)], 2048 * MiB, true});
-            out.push_back({ComponentKind::VAE, "VAE", bytes[int(ComponentKind::VAE)], 1024 * MiB, false});
-            out.push_back({ComponentKind::CONDITIONER, "Conditioner", bytes[int(ComponentKind::CONDITIONER)], 2048 * MiB, true});
-            return out;
+            float gib = budgets.default_gib;
+            auto it   = budgets.backend_gib.find(budget_key(device.name));
+            if (it != budgets.backend_gib.end()) {
+                gib = it->second;
+            }
+            if (gib > 0.f) {
+                device.budget_bytes = (int64_t)std::min(gib * 1024.0 * MiB, (double)device.free_bytes);
+            } else if (gib < 0.f) {
+                device.budget_bytes = (int64_t)std::max<double>(device.free_bytes + gib * 1024.0 * MiB, 0);
+            } else {
+                device.budget_bytes = std::max<int64_t>(device.free_bytes - 512 * MiB, 0);
+            }
+            out.push_back(std::move(device));
         }
+        return out;
+    }
 
-        std::vector<Device> enumerate_gpu_devices(const sd::ggml_graph_cut::MaxVramAssignment& budgets) {
-            std::vector<Device> out;
-            for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
-                ggml_backend_dev_t dev = ggml_backend_dev_get(i);
-                if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU) {
-                    continue;
-                }
-                Device d;
-                d.dev             = dev;
-                d.name            = ggml_backend_dev_name(dev);
-                d.description     = ggml_backend_dev_description(dev);
-                size_t free_bytes = 0, total_bytes = 0;
-                ggml_backend_dev_memory(dev, &free_bytes, &total_bytes);
-                d.free_bytes  = (int64_t)free_bytes;
-                d.total_bytes = (int64_t)total_bytes;
-
-                std::string budget_key = d.name;
-                std::transform(budget_key.begin(), budget_key.end(), budget_key.begin(),
-                               [](unsigned char c) { return (char)std::tolower(c); });
-                float gib = budgets.default_gib;
-                auto it   = budgets.backend_gib.find(budget_key);
-                if (it != budgets.backend_gib.end()) {
-                    gib = it->second;
-                }
-                if (gib > 0.f) {
-                    d.budget_bytes = std::min<int64_t>((int64_t)(gib * 1024.0 * 1024.0 * 1024.0), d.free_bytes);
-                } else if (gib < 0.f) {
-                    d.budget_bytes = d.free_bytes + (int64_t)(gib * 1024.0 * 1024.0 * 1024.0);
-                } else {
-                    d.budget_bytes = d.free_bytes - 512 * MiB;
-                }
-                d.budget_bytes = std::max<int64_t>(d.budget_bytes, 0);
-                out.push_back(d);
-            }
-            return out;
+    static int64_t available_ram_bytes() {
+#if defined(_WIN32)
+        MEMORYSTATUSEX status{};
+        status.dwLength = sizeof(status);
+        if (GlobalMemoryStatusEx(&status)) {
+            return (int64_t)status.ullAvailPhys;
         }
-
-        Plan compute_plan(const std::vector<Component>& components, const std::vector<Device>& devices) {
-            Plan plan;
-            if (devices.empty()) {
-                return plan;
+#elif defined(__linux__)
+        std::ifstream meminfo("/proc/meminfo");
+        std::string key, unit;
+        int64_t kib = 0;
+        while (meminfo >> key >> kib >> unit) {
+            if (key == "MemAvailable:" && unit == "kB" && kib >= 0) {
+                return kib * 1024;
             }
+        }
+#elif defined(__APPLE__)
+        const mach_port_t host = mach_host_self();
+        vm_size_t page_size    = 0;
+        vm_statistics64_data_t stats{};
+        mach_msg_type_number_t count = HOST_VM_INFO64_COUNT;
+        const bool ok                = host_page_size(host, &page_size) == KERN_SUCCESS &&
+                        host_statistics64(host, HOST_VM_INFO64, (host_info64_t)&stats, &count) == KERN_SUCCESS;
+        mach_port_deallocate(mach_task_self(), host);
+        if (ok) {
+            return ((int64_t)stats.free_count + stats.inactive_count) * page_size;
+        }
+#endif
+        return -1;
+    }
 
-            std::vector<size_t> order(components.size());
-            for (size_t i = 0; i < order.size(); i++) {
-                order[i] = i;
+    static Plan compute_plan(const std::vector<Component>& components,
+                             const std::vector<Device>& devices,
+                             int64_t ram_budget_bytes) {
+        Plan plan;
+        for (size_t di = 0; di < devices.size(); ++di) {
+            if (devices[di].budget_bytes > 0 &&
+                (plan.main_device == SIZE_MAX || devices[di].budget_bytes > devices[plan.main_device].budget_bytes)) {
+                plan.main_device = di;
             }
-            std::sort(order.begin(), order.end(), [&](size_t a, size_t b) {
-                return components[a].params_bytes > components[b].params_bytes;
-            });
-
-            {
-                std::vector<int64_t> params_sum(devices.size(), 0);
-                std::vector<int64_t> max_reserve(devices.size(), 0);
-                std::vector<Decision> decisions(components.size());
-                bool ok = true;
-                for (size_t ci : order) {
-                    const Component& comp = components[ci];
-                    decisions[ci].kind    = comp.kind;
-                    if (comp.params_bytes == 0) {
-                        continue;
-                    }
-                    int best = -1;
-                    for (size_t di = 0; di < devices.size(); di++) {
-                        int64_t need = params_sum[di] + comp.params_bytes + std::max(max_reserve[di], comp.reserve_bytes);
-                        if (need <= devices[di].budget_bytes &&
-                            (best < 0 || devices[di].budget_bytes - params_sum[di] > devices[best].budget_bytes - params_sum[best])) {
-                            best = (int)di;
-                        }
-                    }
-                    if (best < 0) {
-                        ok = false;
-                        break;
-                    }
-                    params_sum[best] += comp.params_bytes;
-                    max_reserve[best] = std::max(max_reserve[best], comp.reserve_bytes);
-                    decisions[ci].device_idxs.push_back((size_t)best);
-                }
-                if (ok) {
-                    plan.valid      = true;
-                    plan.time_share = false;
-                    plan.decisions  = std::move(decisions);
-                    return plan;
-                }
-            }
-
-            plan.decisions.assign(components.size(), {});
-            for (size_t ci : order) {
-                const Component& comp = components[ci];
-                Decision& decision    = plan.decisions[ci];
-                decision.kind         = comp.kind;
-                if (comp.params_bytes == 0) {
-                    continue;
-                }
-                int best = -1;
-                for (size_t di = 0; di < devices.size(); di++) {
-                    if (comp.params_bytes + comp.reserve_bytes <= devices[di].budget_bytes &&
-                        (best < 0 || devices[di].budget_bytes > devices[best].budget_bytes)) {
-                        best = (int)di;
-                    }
-                }
-                if (best >= 0) {
-                    decision.device_idxs.push_back((size_t)best);
-                    continue;
-                }
-                if (comp.splittable && devices.size() > 1) {
-                    int64_t capacity = 0;
-                    for (const Device& d : devices) {
-                        capacity += std::max<int64_t>(d.budget_bytes - comp.reserve_bytes, 0);
-                    }
-                    if (comp.params_bytes <= capacity) {
-                        std::vector<size_t> idxs(devices.size());
-                        for (size_t i = 0; i < idxs.size(); i++) {
-                            idxs[i] = i;
-                        }
-                        std::sort(idxs.begin(), idxs.end(), [&](size_t a, size_t b) {
-                            return devices[a].budget_bytes > devices[b].budget_bytes;
-                        });
-                        decision.device_idxs = std::move(idxs);
-                        continue;
-                    }
-                }
-                decision.on_cpu = true;
-            }
-            plan.valid      = true;
-            plan.time_share = true;
+        }
+        if (plan.main_device == SIZE_MAX) {
             return plan;
         }
 
-        void print_plan(const Plan& plan,
-                        const std::vector<Component>& components,
-                        const std::vector<Device>& devices) {
-            LOG_INFO("auto-fit plan%s:", plan.time_share ? " (time-share: params load per phase and free after)" : "");
-            LOG_INFO("  devices:");
-            for (const Device& d : devices) {
-                LOG_INFO("    %-12s %-32s free %6lld MiB, budget %6lld MiB",
-                         d.name.c_str(), d.description.c_str(),
-                         (long long)(d.free_bytes / MiB), (long long)(d.budget_bytes / MiB));
-            }
-            LOG_INFO("  components:");
-            for (size_t ci = 0; ci < components.size(); ci++) {
-                const Component& comp    = components[ci];
-                const Decision& decision = plan.decisions[ci];
-                std::string target;
-                if (comp.params_bytes == 0) {
-                    target = "(not present)";
-                } else if (decision.on_cpu) {
-                    target = "CPU";
-                } else {
-                    for (size_t k = 0; k < decision.device_idxs.size(); k++) {
-                        if (k > 0) {
-                            target += " & ";
-                        }
-                        target += devices[decision.device_idxs[k]].name;
-                    }
-                    if (decision.device_idxs.size() > 1) {
-                        target += " (split)";
-                    }
-                }
-                LOG_INFO("    %-12s params %6lld MiB, compute reserve %5lld MiB -> %s",
-                         comp.name,
-                         (long long)(comp.params_bytes / MiB),
-                         (long long)(comp.reserve_bytes / MiB),
-                         target.c_str());
-            }
+        std::vector<size_t> order(components.size());
+        for (size_t ci = 0; ci < components.size(); ++ci) {
+            order[ci] = ci;
         }
+        std::stable_sort(order.begin(), order.end(), [&](size_t a, size_t b) {
+            return components[a].kind < components[b].kind;
+        });
 
-        void append_assignment(std::string& spec, const char* key, const std::string& value) {
-            if (!spec.empty()) {
-                spec += ",";
-            }
-            spec += key;
-            spec += "=";
-            spec += value;
+        std::vector<int64_t> remaining;
+        for (const Device& device : devices) {
+            remaining.push_back(std::max<int64_t>(device.budget_bytes, 0));
         }
+        ram_budget_bytes = std::max<int64_t>(ram_budget_bytes, 0);
+        plan.decisions.resize(components.size());
 
-        void append_component_decision(const std::vector<Component>& components,
-                                       const std::vector<Device>& devices,
-                                       const Plan& plan,
-                                       ComponentKind kind,
-                                       const char* module_key,
-                                       std::string& runtime_spec,
-                                       std::string& params_spec) {
-            for (size_t ci = 0; ci < components.size(); ci++) {
-                if (components[ci].kind != kind || components[ci].params_bytes == 0) {
+        for (size_t ci : order) {
+            const Component& comp = components[ci];
+            Decision& decision    = plan.decisions[ci];
+            if (comp.params_bytes == 0) {
+                continue;
+            }
+
+            // Higher-priority offloaded weights need GPU cache space across graph runs.
+            int64_t headroom = 0;
+            for (size_t other = 0; other < components.size(); ++other) {
+                if (components[other].params_bytes == 0) {
                     continue;
                 }
-                const Decision& decision = plan.decisions[ci];
-                if (decision.on_cpu) {
-                    append_assignment(runtime_spec, module_key, "cpu");
-                    return;
+                const bool resident          = other == ci || plan.decisions[other].params_location == ParamsLocation::MAIN_GPU;
+                const int64_t cached_weights = components[other].kind < comp.kind
+                                                   ? components[other].params_bytes
+                                                   : components[other].staging_bytes;
+                headroom                     = std::max(headroom, components[other].reserve_bytes +
+                                                                      (resident ? 0 : cached_weights));
+            }
+            int64_t& main_remaining = remaining[plan.main_device];
+            if (headroom <= main_remaining && comp.params_bytes <= main_remaining - headroom) {
+                decision.params_location = ParamsLocation::MAIN_GPU;
+                decision.params_device   = plan.main_device;
+                main_remaining -= comp.params_bytes;
+                continue;
+            }
+            if (comp.params_bytes <= ram_budget_bytes) {
+                decision.params_location = ParamsLocation::CPU;
+                ram_budget_bytes -= comp.params_bytes;
+                continue;
+            }
+
+            size_t best = SIZE_MAX;
+            for (size_t di = 0; di < devices.size(); ++di) {
+                if (di != plan.main_device && comp.params_bytes <= remaining[di] &&
+                    (best == SIZE_MAX || remaining[di] > remaining[best])) {
+                    best = di;
                 }
-                if (decision.device_idxs.empty()) {
-                    return;
-                }
-                std::string device_list;
-                for (size_t k = 0; k < decision.device_idxs.size(); k++) {
-                    if (k > 0) {
-                        device_list += "&";
-                    }
-                    device_list += devices[decision.device_idxs[k]].name;
-                }
-                append_assignment(runtime_spec, module_key, device_list);
-                if (plan.time_share) {
-                    append_assignment(params_spec, module_key, "disk");
-                }
-                return;
+            }
+            if (best != SIZE_MAX) {
+                decision.params_location = ParamsLocation::OTHER_GPU;
+                decision.params_device   = best;
+                remaining[best] -= comp.params_bytes;
             }
         }
+        plan.valid = true;
+        return plan;
+    }
 
-    }  // namespace
+    static std::string params_backend_name(const Decision& decision, const std::vector<Device>& devices) {
+        switch (decision.params_location) {
+            case ParamsLocation::MAIN_GPU:
+            case ParamsLocation::OTHER_GPU:
+                return devices[decision.params_device].name;
+            case ParamsLocation::CPU:
+                return "cpu";
+            case ParamsLocation::DISK:
+                return "disk";
+        }
+        return "disk";
+    }
+
+    static void print_plan(const Plan& plan,
+                           const std::vector<Component>& components,
+                           const std::vector<Device>& devices,
+                           int64_t free_ram,
+                           int64_t ram_budget) {
+        LOG_INFO("auto-fit plan (single-GPU compute on %s):", devices[plan.main_device].name.c_str());
+        LOG_INFO("  devices:");
+        for (const Device& device : devices) {
+            LOG_INFO("    %-12s %-32s free %6lld MiB, budget %6lld MiB",
+                     device.name.c_str(), device.description.c_str(),
+                     (long long)(device.free_bytes / MiB), (long long)(device.budget_bytes / MiB));
+        }
+        if (free_ram < 0) {
+            LOG_WARN("auto-fit: available RAM is unknown; skipping CPU parameter residency");
+        } else {
+            LOG_INFO("    RAM          free %6lld MiB, params budget %6lld MiB",
+                     (long long)(free_ram / MiB), (long long)(ram_budget / MiB));
+        }
+        LOG_INFO("  main-GPU weight cache priority: diffusion > te > vae");
+        LOG_INFO("  components (params: main GPU -> RAM -> other GPU -> disk):");
+        for (size_t ci = 0; ci < components.size(); ++ci) {
+            const Component& comp = components[ci];
+            if (comp.params_bytes == 0) {
+                continue;
+            }
+            const std::string params = params_backend_name(plan.decisions[ci], devices);
+            LOG_INFO("    %-12s params %6lld MiB, compute reserve %5lld MiB -> compute %s, params %s",
+                     comp.name, (long long)(comp.params_bytes / MiB), (long long)(comp.reserve_bytes / MiB),
+                     devices[plan.main_device].name.c_str(), params.c_str());
+        }
+    }
+
+    static void append_assignment(std::string& spec, const char* key, const std::string& value) {
+        if (!spec.empty()) {
+            spec += ",";
+        }
+        spec += key;
+        spec += "=";
+        spec += value;
+    }
+
+    static const char* module_key(ComponentKind kind) {
+        switch (kind) {
+            case ComponentKind::DIT:
+                return "diffusion";
+            case ComponentKind::CONDITIONER:
+                return "te";
+            case ComponentKind::VAE:
+                return "vae";
+        }
+        return "";
+    }
 
     bool derive_backend_specs(ModelLoader& loader,
                               ggml_type override_wtype,
                               sd::ggml_graph_cut::MaxVramAssignment& budgets,
                               std::string& runtime_spec,
                               std::string& params_spec) {
-        if (!runtime_spec.empty() || !params_spec.empty()) {
-            LOG_WARN("--auto-fit is enabled; ignoring --backend / --params-backend");
+        std::string error;
+        if (!budgets.canonicalize_backend_keys(&error)) {
+            LOG_ERROR("%s", error.c_str());
+            return false;
         }
 
-        {
-            std::string error;
-            if (!budgets.canonicalize_backend_keys(&error)) {
-                LOG_ERROR("%s", error.c_str());
-                return false;
-            }
-        }
-
-        auto components = estimate_components(loader, override_wtype);
-        auto devices    = enumerate_gpu_devices(budgets);
-        auto plan       = compute_plan(components, devices);
+        const auto components    = estimate_components(loader, override_wtype);
+        const auto devices       = enumerate_gpu_devices(budgets);
+        const int64_t free_ram   = available_ram_bytes();
+        const int64_t ram_budget = std::max<int64_t>(free_ram - std::max<int64_t>(2048 * MiB, free_ram / 10), 0);
+        const auto plan          = compute_plan(components, devices, ram_budget);
+        runtime_spec.clear();
+        params_spec.clear();
         if (!plan.valid) {
-            LOG_WARN("auto-fit: no usable GPU devices; using the default backend");
-            runtime_spec.clear();
-            params_spec.clear();
+            if (devices.empty()) {
+                LOG_WARN("auto-fit: no GPU devices; using the default backend");
+            } else {
+                LOG_WARN("auto-fit: no GPU memory budget available; using CPU");
+                runtime_spec = "cpu";
+            }
             return true;
         }
 
-        print_plan(plan, components, devices);
+        print_plan(plan, components, devices, free_ram, ram_budget);
+        for (size_t ci = 0; ci < components.size(); ++ci) {
+            if (components[ci].params_bytes == 0) {
+                continue;
+            }
+            const char* key = module_key(components[ci].kind);
+            append_assignment(runtime_spec, key, devices[plan.main_device].name);
+            if (plan.decisions[ci].params_location != ParamsLocation::MAIN_GPU) {
+                append_assignment(params_spec, key, params_backend_name(plan.decisions[ci], devices));
+            }
+        }
 
-        std::string derived_runtime_spec;
-        std::string derived_params_spec;
-        append_component_decision(components, devices, plan, ComponentKind::DIT, "diffusion", derived_runtime_spec, derived_params_spec);
-        append_component_decision(components, devices, plan, ComponentKind::CONDITIONER, "te", derived_runtime_spec, derived_params_spec);
-        append_component_decision(components, devices, plan, ComponentKind::VAE, "vae", derived_runtime_spec, derived_params_spec);
-
-        runtime_spec = std::move(derived_runtime_spec);
-        params_spec  = std::move(derived_params_spec);
+        // Keep the planner's safety margin when the runner resolves its device limits.
+        for (const Device& device : devices) {
+            if (device.budget_bytes > 0) {
+                budgets.backend_gib[budget_key(device.name)] = (float)(device.budget_bytes / (1024.0 * MiB));
+            }
+        }
+        budgets.resolved_backend_bytes.clear();
 
         LOG_INFO("auto-fit: --backend \"%s\"%s%s%s",
                  runtime_spec.empty() ? "(default)" : runtime_spec.c_str(),
                  params_spec.empty() ? "" : " --params-backend \"",
-                 params_spec.c_str(),
-                 params_spec.empty() ? "" : "\"");
+                 params_spec.c_str(), params_spec.empty() ? "" : "\"");
         return true;
     }
 
     bool prepare_vae_decode_retry_tiling(sd_tiling_params_t& tiling_params, bool prefer_temporal_tiling) {
-        if (prefer_temporal_tiling) {
-            if (tiling_params.temporal_tiling) {
-                return false;
-            }
+        const char* retry_mode = nullptr;
+        if (prefer_temporal_tiling && !tiling_params.temporal_tiling) {
             tiling_params.temporal_tiling = true;
-        } else {
-            if (tiling_params.enabled) {
-                return false;
-            }
+            retry_mode                    = tiling_params.enabled ? "spatial+temporal" : "temporal";
+        } else if (!tiling_params.enabled) {
             tiling_params.enabled = true;
             if (tiling_params.tile_size_x <= 0) {
                 tiling_params.tile_size_x = 256;
@@ -380,10 +396,13 @@ namespace sd::backend_fit {
             if (tiling_params.tile_size_y <= 0) {
                 tiling_params.tile_size_y = 256;
             }
+            retry_mode = tiling_params.temporal_tiling ? "spatial+temporal" : "spatial";
+        } else {
+            return false;
         }
 
         LOG_WARN("auto-fit: VAE decode failed (likely out of memory); retrying with %s tiling",
-                 tiling_params.temporal_tiling ? "temporal" : "spatial");
+                 retry_mode);
         return true;
     }
 

src/core/compute_workspace.cpp

@@ -0,0 +1,257 @@
+#include "core/compute_workspace.h"
+
+#include <algorithm>
+#include <cstring>
+#include <map>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "core/ggml_extend_backend.h"
+#include "core/ggml_graph_cut.h"
+#include "ggml-cpu.h"
+#include "ggml/src/ggml-impl.h"
+
+namespace sd {
+    ComputeWorkspace::~ComputeWorkspace() {
+        segment_end();
+        release();
+        ggml_backend_free(cpu_backend_);
+    }
+
+    void ComputeWorkspace::set_extra_backends(const std::vector<ggml_backend_t>& backends) {
+        if (extra_backends_ != backends) {
+            GGML_ASSERT(!active_);
+            release();
+            extra_backends_ = backends;
+        }
+    }
+
+    bool ComputeWorkspace::needs_scheduler(ggml_cgraph* graph) const {
+        if (!extra_backends_.empty()) {
+            return true;
+        }
+        for (int i = 0; i < ggml_graph_n_nodes(graph); ++i) {
+            if (!ggml_backend_supports_op(backend_, ggml_graph_node(graph, i))) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    ggml_backend_sched_t ComputeWorkspace::make_scheduler(size_t graph_size) {
+        std::vector<ggml_backend_t> backends{backend_};
+        backends.insert(backends.end(), extra_backends_.begin(), extra_backends_.end());
+        if (!sd_backend_is_cpu(backend_)) {
+            if (cpu_backend_ == nullptr) {
+                cpu_backend_ = sd_backend_cpu_init();
+            }
+            if (cpu_backend_ == nullptr) {
+                return nullptr;
+            }
+            backends.push_back(cpu_backend_);
+        }
+        std::vector<ggml_backend_buffer_type_t> bufts;
+        for (auto backend : backends) {
+            auto buft = backend == cpu_backend_
+                            ? ggml_backend_dev_host_buffer_type(ggml_backend_get_device(backend_))
+                            : nullptr;
+            bufts.push_back(buft != nullptr ? buft : ggml_backend_get_default_buffer_type(backend));
+        }
+        return ggml_backend_sched_new(backends.data(), bufts.data(), static_cast<int>(backends.size()),
+                                      graph_size, false, false);
+    }
+
+    bool ComputeWorkspace::measurement_matches(ggml_cgraph* graph, const Measurement& measurement) const {
+        return measurement.scheduler == needs_scheduler(graph);
+    }
+
+    bool ComputeWorkspace::prepare(const Measurement& measurement) {
+        GGML_ASSERT(!active_);
+        if (measurement.buffers.empty()) {
+            return false;
+        }
+        const bool grows = std::any_of(measurement.buffers.begin(), measurement.buffers.end(),
+                                       [&](const BackendBufferSize& size) { return size.bytes > bytes(size.backend); });
+        if (measurement.scheduler != (scheduler_ != nullptr) || grows) {
+            release();
+        }
+        return true;
+    }
+
+    bool ComputeWorkspace::release_excess(const Measurement& measurement) {
+        return std::any_of(measurement.buffers.begin(), measurement.buffers.end(),
+                           [&](const BackendBufferSize& size) { return bytes(size.backend) > size.bytes; }) &&
+               release();
+    }
+
+    bool ComputeWorkspace::allocate(ggml_cgraph* graph, const AssignNodes& assign_nodes) {
+        GGML_ASSERT(!active_);
+        const bool use_scheduler = needs_scheduler(graph);
+        if (use_scheduler) {
+            if (allocator_ != nullptr) {
+                release();
+            }
+            const size_t capacity = static_cast<size_t>(graph->n_nodes + graph->n_leafs) + 8;
+            if (scheduler_ == nullptr || capacity > scheduler_capacity_) {
+                release();
+                scheduler_          = make_scheduler(capacity);
+                scheduler_capacity_ = capacity;
+            }
+            if (scheduler_ == nullptr) {
+                return false;
+            }
+            ggml_backend_sched_reset(scheduler_);
+            assign_nodes(scheduler_, graph);
+            // Scheduler allocation rewrites sources. Split the execution graph only once.
+            if (!ggml_backend_sched_alloc_graph(scheduler_, graph)) {
+                release();
+                return false;
+            }
+        } else {
+            if (scheduler_ != nullptr) {
+                release();
+            }
+            if (allocator_ == nullptr) {
+                allocator_ = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend_));
+            }
+            auto signature = ggml_graph_cut::graph_layout(graph, true);
+            if (signature != reservation_) {
+                if (!ggml_gallocr_reserve(allocator_, graph)) {
+                    release();
+                    return false;
+                }
+                reservation_ = std::move(signature);
+                ++reservations_;
+            }
+            if (!ggml_gallocr_alloc_graph(allocator_, graph)) {
+                release();
+                return false;
+            }
+        }
+        active_ = true;
+        return true;
+    }
+
+    ComputeWorkspace::Measurement ComputeWorkspace::measure(
+        ggml_cgraph* graph,
+        size_t direct_bytes,
+        const std::function<ggml_backend_t(const ggml_tensor*)>& external_backend,
+        const AssignNodes& assign_nodes) {
+        if (!needs_scheduler(graph)) {
+            return {{{backend_, direct_bytes}}, false};
+        }
+        std::vector<const ggml_tensor*> tensors;
+        std::unordered_set<const ggml_tensor*> seen;
+        auto visit = [&](const ggml_tensor* tensor) {
+            if (tensor != nullptr && seen.insert(tensor).second) {
+                tensors.push_back(tensor);
+            }
+        };
+        for (int i = 0; i < graph->n_nodes; ++i) {
+            visit(graph->nodes[i]);
+        }
+        for (int i = 0; i < graph->n_leafs; ++i) {
+            visit(graph->leafs[i]);
+        }
+        for (size_t i = 0; i < tensors.size(); ++i) {
+            visit(tensors[i]->view_src);
+            for (auto source : tensors[i]->src) {
+                visit(source);
+            }
+        }
+        const size_t graph_size = tensors.size() + 8;
+        auto context            = ggml_init({tensors.size() * ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false), nullptr, true});
+        if (context == nullptr) {
+            return {};
+        }
+        std::unordered_map<const ggml_tensor*, ggml_tensor*> copies;
+        std::map<ggml_backend_t, ggml_backend_buffer_t> external_buffers;
+        for (auto tensor : tensors) {
+            auto copy      = ggml_dup_tensor(context, tensor);
+            *copy          = *tensor;
+            copies[tensor] = copy;
+        }
+        for (const auto& entry : copies) {
+            auto source    = entry.first;
+            auto copy      = entry.second;
+            copy->view_src = source->view_src == nullptr ? nullptr : copies.at(source->view_src);
+            for (int i = 0; i < GGML_MAX_SRC; ++i) {
+                copy->src[i] = source->src[i] == nullptr ? nullptr : copies.at(source->src[i]);
+            }
+            auto external = external_backend(source);
+            if (external != nullptr && source->view_src == nullptr) {
+                auto& buffer = external_buffers[external];
+                if (buffer == nullptr) {
+                    buffer = ggml_backend_alloc_buffer(external, 0);
+                    GGML_ASSERT(buffer != nullptr);
+                    ggml_backend_buffer_set_usage(buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
+                }
+                copy->buffer = buffer;
+                copy->data   = reinterpret_cast<void*>(static_cast<uintptr_t>(1));
+                copy->extra  = nullptr;
+            }
+        }
+        auto copy_graph     = ggml_new_graph_custom(context, graph_size, false);
+        copy_graph->n_nodes = graph->n_nodes;
+        copy_graph->n_leafs = graph->n_leafs;
+        for (int i = 0; i < graph->n_nodes; ++i) {
+            copy_graph->nodes[i] = copies.at(graph->nodes[i]);
+        }
+        for (int i = 0; i < graph->n_leafs; ++i) {
+            copy_graph->leafs[i] = copies.at(graph->leafs[i]);
+        }
+        Measurement result;
+        result.scheduler = true;
+        auto scheduler   = make_scheduler(graph_size);
+        if (scheduler != nullptr) {
+            assign_nodes(scheduler, copy_graph);
+            std::vector<size_t> sizes(extra_backends_.size() + 2);
+            ggml_backend_sched_reserve_size(scheduler, copy_graph, sizes.data());
+            result.buffers.push_back({backend_, sizes[0]});
+            for (size_t i = 0; i < extra_backends_.size(); ++i) {
+                result.buffers.push_back({extra_backends_[i], sizes[i + 1]});
+            }
+            ggml_backend_sched_free(scheduler);
+        }
+        for (const auto& entry : external_buffers) {
+            ggml_backend_buffer_free(entry.second);
+        }
+        ggml_free(context);
+        return result;
+    }
+
+    void ComputeWorkspace::synchronize() const {
+        if (scheduler_ != nullptr) {
+            ggml_backend_sched_synchronize(scheduler_);
+        } else {
+            ggml_backend_synchronize(backend_);
+        }
+    }
+
+    void ComputeWorkspace::segment_end() {
+        if (active_) {
+            synchronize();
+            active_ = false;
+        }
+    }
+
+    bool ComputeWorkspace::release() {
+        if (active_) {
+            return false;
+        }
+        ggml_gallocr_free(allocator_);
+        allocator_ = nullptr;
+        ggml_backend_sched_free(scheduler_);
+        scheduler_          = nullptr;
+        scheduler_capacity_ = 0;
+        reservation_.clear();
+        return true;
+    }
+
+    size_t ComputeWorkspace::bytes(ggml_backend_t backend) const {
+        if (scheduler_ != nullptr) {
+            return ggml_backend_sched_get_buffer_size(scheduler_, backend);
+        }
+        return allocator_ != nullptr && backend == backend_ ? ggml_gallocr_get_buffer_size(allocator_, 0) : 0;
+    }
+}

src/core/compute_workspace.h

@@ -0,0 +1,64 @@
+#ifndef __SD_CORE_COMPUTE_WORKSPACE_H__
+#define __SD_CORE_COMPUTE_WORKSPACE_H__
+
+#include <functional>
+#include <vector>
+
+#include "ggml-alloc.h"
+#include "ggml-backend.h"
+
+namespace sd {
+    struct BackendBufferSize {
+        ggml_backend_t backend = nullptr;
+        size_t bytes           = 0;
+    };
+
+    class ComputeWorkspace {
+        ggml_backend_t backend_;
+        std::vector<ggml_backend_t> extra_backends_;
+        ggml_backend_t cpu_backend_     = nullptr;
+        ggml_gallocr_t allocator_       = nullptr;
+        ggml_backend_sched_t scheduler_ = nullptr;
+        size_t scheduler_capacity_      = 0;
+        std::vector<uint64_t> reservation_;
+        bool active_         = false;
+        size_t reservations_ = 0;
+
+        ggml_backend_sched_t make_scheduler(size_t graph_size);
+        bool needs_scheduler(ggml_cgraph* graph) const;
+
+    public:
+        struct Measurement {
+            std::vector<BackendBufferSize> buffers;
+            bool scheduler = false;
+        };
+        using AssignNodes = std::function<void(ggml_backend_sched_t, ggml_cgraph*)>;
+
+        explicit ComputeWorkspace(ggml_backend_t backend)
+            : backend_(backend) {}
+        ~ComputeWorkspace();
+        ComputeWorkspace(const ComputeWorkspace&)            = delete;
+        ComputeWorkspace& operator=(const ComputeWorkspace&) = delete;
+
+        void set_extra_backends(const std::vector<ggml_backend_t>& backends);
+        bool measurement_matches(ggml_cgraph* graph, const Measurement& measurement) const;
+        bool prepare(const Measurement& measurement);
+        bool release_excess(const Measurement& measurement);
+        bool allocate(ggml_cgraph* graph, const AssignNodes& assign_nodes);
+        Measurement measure(
+            ggml_cgraph* graph,
+            size_t direct_bytes,
+            const std::function<ggml_backend_t(const ggml_tensor*)>& external_backend,
+            const AssignNodes& assign_nodes);
+        void synchronize() const;
+        void segment_end();
+        bool release();
+        bool active() const { return active_; }
+        ggml_backend_sched_t scheduler() const { return scheduler_; }
+        ggml_backend_t cpu_backend() const { return cpu_backend_; }
+        size_t bytes(ggml_backend_t backend) const;
+        size_t reservation_count() const { return reservations_; }
+    };
+}
+
+#endif  // __SD_CORE_COMPUTE_WORKSPACE_H__

src/core/ggml_extend_backend.cpp

@@ -392,7 +392,7 @@ static bool backend_name_exists(const std::string& name) {
 
 static ggml_backend_t init_named_backend(const std::string& name) {
     ggml_backend_load_all_once();
-    LOG_DEBUG("Initializing backend: %s", name.c_str());
+    LOG_VERBOSE("Initializing backend: %s", name.c_str());
     if (trim_copy(name).empty()) {
         return ggml_backend_init_best();
     }
@@ -542,10 +542,10 @@ static ggml_backend_t sd_get_default_backend() {
         if (dev_count == 0) {
             LOG_ERROR("No devices found!");
         } else {
-            LOG_DEBUG("Found %zu backend devices:", dev_count);
+            LOG_VERBOSE("Found %zu backend devices:", dev_count);
             for (size_t i = 0; i < dev_count; ++i) {
                 auto dev = ggml_backend_dev_get(i);
-                LOG_DEBUG("#%zu: %s", i, ggml_backend_dev_name(dev));
+                LOG_VERBOSE("#%zu: %s", i, ggml_backend_dev_name(dev));
             }
         }
     });
@@ -587,7 +587,7 @@ static ggml_backend_t sd_get_default_backend() {
     }
 
     if (sd_backend_is_cpu(backend)) {
-        LOG_DEBUG("Using CPU backend");
+        LOG_VERBOSE("Using CPU backend");
     }
 
     return backend;

src/core/ggml_graph_cut.cpp

@@ -2,6 +2,7 @@
 
 #include <algorithm>
 #include <cctype>
+#include <climits>
 #include <cmath>
 #include <cstring>
 #include <map>
@@ -67,25 +68,6 @@ namespace sd::ggml_graph_cut {
         return -1;
     }
 
-    static Plan::InputShape input_shape(const ggml_tensor* tensor) {
-        Plan::InputShape shape;
-        if (tensor == nullptr) {
-            return shape;
-        }
-        shape.type = tensor->type;
-        for (int i = 0; i < GGML_MAX_DIMS; ++i) {
-            shape.ne[static_cast<size_t>(i)] = tensor->ne[i];
-        }
-        return shape;
-    }
-
-    static size_t graph_cut_segment_vram_bytes(const Segment& segment) {
-        return segment.compute_buffer_size +
-               segment.input_param_bytes +
-               segment.input_previous_cut_bytes +
-               segment.output_bytes;
-    }
-
     static std::string lower_ascii_copy(std::string value) {
         std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) {
             return static_cast<char>(std::tolower(c));
@@ -291,55 +273,6 @@ namespace sd::ggml_graph_cut {
         return max_vram_bytes_to_gib(resolve_auto_max_vram_bytes(-max_vram, backend));
     }
 
-    static bool is_segment_output_needed_after(const Plan& plan,
-                                               size_t end_segment_index,
-                                               int output_node_index) {
-        if (end_segment_index + 1 >= plan.segments.size()) {
-            return false;
-        }
-        for (size_t seg_idx = end_segment_index + 1; seg_idx < plan.segments.size(); ++seg_idx) {
-            const auto& segment = plan.segments[seg_idx];
-            for (const auto& input_ref : segment.input_refs) {
-                if (input_ref.type == Segment::INPUT_PREVIOUS_CUT &&
-                    input_ref.node_index == output_node_index) {
-                    return true;
-                }
-            }
-        }
-        return false;
-    }
-
-    static Segment make_segment_seed(const Plan& plan,
-                                     size_t start_segment_index,
-                                     size_t end_segment_index) {
-        GGML_ASSERT(start_segment_index < plan.segments.size());
-        GGML_ASSERT(end_segment_index < plan.segments.size());
-        GGML_ASSERT(start_segment_index <= end_segment_index);
-
-        Segment seed;
-        const auto& start_segment  = plan.segments[start_segment_index];
-        const auto& target_segment = plan.segments[end_segment_index];
-        std::unordered_set<int> seen_output_node_indices;
-        for (size_t seg_idx = start_segment_index; seg_idx <= end_segment_index; ++seg_idx) {
-            const bool is_boundary_segment = seg_idx == end_segment_index;
-            for (int output_node_index : plan.segments[seg_idx].output_node_indices) {
-                if ((is_boundary_segment ||
-                     is_segment_output_needed_after(plan, end_segment_index, output_node_index)) &&
-                    seen_output_node_indices.insert(output_node_index).second) {
-                    seed.output_node_indices.push_back(output_node_index);
-                }
-            }
-        }
-        if (start_segment_index == end_segment_index) {
-            seed.group_name = target_segment.group_name;
-        } else {
-            seed.group_name = sd_format("%s..%s",
-                                        start_segment.group_name.c_str(),
-                                        target_segment.group_name.c_str());
-        }
-        return seed;
-    }
-
     static void build_segment(ggml_cgraph* gf,
                               Plan& plan,
                               Segment& segment,
@@ -416,31 +349,7 @@ namespace sd::ggml_graph_cut {
                       }
                       return a.display_name < b.display_name;
                   });
-        segment.input_refs = input_refs;
-        for (const auto& input : input_refs) {
-            ggml_tensor* current_input = input_tensor(gf, input);
-            size_t tensor_bytes        = current_input == nullptr
-                                             ? 0
-                                             : (input.type == Segment::INPUT_PREVIOUS_CUT
-                                                    ? cache_tensor_bytes(current_input)
-                                                    : ggml_nbytes(current_input));
-            switch (input.type) {
-                case Segment::INPUT_PREVIOUS_CUT:
-                    segment.input_previous_cut_bytes += tensor_bytes;
-                    break;
-                case Segment::INPUT_PARAM:
-                    segment.input_param_bytes += tensor_bytes;
-                    break;
-                case Segment::INPUT_EXTERNAL:
-                default:
-                    segment.input_external_bytes += tensor_bytes;
-                    break;
-            }
-        }
-        for (int output_node_index : segment.output_node_indices) {
-            ggml_tensor* output = ggml_graph_node(gf, output_node_index);
-            segment.output_bytes += cache_tensor_bytes(output);
-        }
+        segment.input_refs          = input_refs;
         segment.compute_buffer_size = measure_segment_compute_buffer(backend, gf, segment, log_desc);
 
         for (int output_node_index : segment.output_node_indices) {
@@ -449,6 +358,70 @@ namespace sd::ggml_graph_cut {
         plan.segments.push_back(std::move(segment));
     }
 
+    static bool validate_plan(ggml_cgraph* gf,
+                              const Plan& plan,
+                              std::string* validation_error) {
+        auto fail = [&](const std::string& reason) {
+            if (validation_error != nullptr) {
+                *validation_error = reason;
+            }
+            return false;
+        };
+        if (!plan.has_cuts) {
+            return true;
+        }
+        if (plan.segments.size() <= 1) {
+            return fail("fewer than two segments");
+        }
+        const int n_nodes = ggml_graph_n_nodes(gf);
+        std::unordered_set<int> completed_outputs;
+        for (size_t segment_index = 0; segment_index < plan.segments.size(); ++segment_index) {
+            const Segment& segment          = plan.segments[segment_index];
+            const std::string segment_label = "segment " + std::to_string(segment_index) +
+                                              " ('" + segment.group_name + "')";
+            if (segment.internal_node_indices.empty() || segment.output_node_indices.empty()) {
+                return fail(segment_label + " has no internal nodes or outputs");
+            }
+            for (const Segment::InputRef& input : segment.input_refs) {
+                if (input.type == Segment::INPUT_PREVIOUS_CUT) {
+                    if (input.node_index < 0 || input.node_index >= n_nodes ||
+                        completed_outputs.find(input.node_index) == completed_outputs.end()) {
+                        return fail(segment_label + " references an unavailable cut node " +
+                                    std::to_string(input.node_index));
+                    }
+                } else if (input.leaf_index < 0 || input.leaf_index >= gf->n_leafs) {
+                    return fail(segment_label + " references an invalid leaf " +
+                                std::to_string(input.leaf_index));
+                }
+            }
+            std::unordered_set<int> segment_nodes;
+            segment_nodes.reserve(segment.internal_node_indices.size());
+            for (int node_index : segment.internal_node_indices) {
+                if (node_index < 0 || node_index >= n_nodes) {
+                    return fail(segment_label + " contains an invalid node " +
+                                std::to_string(node_index));
+                }
+                if (!segment_nodes.insert(node_index).second) {
+                    return fail(segment_label + " contains duplicate node " +
+                                std::to_string(node_index));
+                }
+            }
+            for (int output_index : segment.output_node_indices) {
+                if (output_index < 0 || output_index >= n_nodes ||
+                    segment_nodes.find(output_index) == segment_nodes.end()) {
+                    return fail(segment_label + " has an output outside its node set: " +
+                                std::to_string(output_index));
+                }
+                if (completed_outputs.find(output_index) != completed_outputs.end()) {
+                    return fail(segment_label + " repeats output node " +
+                                std::to_string(output_index));
+                }
+                completed_outputs.insert(output_index);
+            }
+        }
+        return true;
+    }
+
     bool is_graph_cut_tensor(const ggml_tensor* tensor) {
         if (tensor == nullptr || tensor->name[0] == '\0') {
             return false;
@@ -509,26 +482,87 @@ namespace sd::ggml_graph_cut {
         return ggml_nbytes(cache_src);
     }
 
-    bool plan_matches_graph(ggml_cgraph* gf, const Plan& plan) {
-        GGML_ASSERT(gf != nullptr);
-        if (ggml_graph_n_nodes(gf) != plan.n_nodes || gf->n_leafs != plan.n_leafs) {
-            return false;
-        }
-        for (const auto& input_shape_ref : plan.input_shapes) {
-            if (input_shape_ref.leaf_index < 0 || input_shape_ref.leaf_index >= gf->n_leafs) {
-                return false;
+    std::vector<uint64_t> graph_layout(ggml_cgraph* graph, bool include_bindings) {
+        std::vector<const ggml_tensor*> tensors;
+        std::unordered_map<const ggml_tensor*, size_t> indices;
+        auto add = [&](const ggml_tensor* tensor) {
+            if (tensor != nullptr && indices.emplace(tensor, tensors.size() + 1).second) {
+                tensors.push_back(tensor);
             }
-            ggml_tensor* leaf = gf->leafs[input_shape_ref.leaf_index];
-            if (leaf == nullptr || input_shape_ref.type != leaf->type) {
-                return false;
+        };
+        for (int i = 0; i < graph->n_leafs; ++i) {
+            add(graph->leafs[i]);
+        }
+        for (int i = 0; i < graph->n_nodes; ++i) {
+            add(graph->nodes[i]);
+        }
+        for (size_t i = 0; i < tensors.size(); ++i) {
+            add(tensors[i]->view_src);
+            for (auto source : tensors[i]->src) {
+                add(source);
+            }
+        }
+        std::vector<uint64_t> signature;
+        signature.reserve(tensors.size() * 24);
+        signature.push_back(graph->n_nodes);
+        signature.push_back(graph->n_leafs);
+        for (int i = 0; i < graph->n_leafs; ++i) {
+            signature.push_back(indices.at(graph->leafs[i]));
+        }
+        for (int i = 0; i < graph->n_nodes; ++i) {
+            signature.push_back(indices.at(graph->nodes[i]));
+        }
+        for (auto tensor : tensors) {
+            signature.push_back(tensor->op);
+            signature.push_back(tensor->type);
+            signature.push_back(tensor->flags);
+            signature.push_back(tensor->view_offs);
+            if (include_bindings) {
+                signature.push_back(tensor->data != nullptr);
+                auto buffer = tensor_buffer(tensor);
+                signature.push_back(reinterpret_cast<uintptr_t>(buffer == nullptr ? nullptr : ggml_backend_buffer_get_type(buffer)));
             }
             for (int d = 0; d < GGML_MAX_DIMS; ++d) {
-                if (input_shape_ref.ne[static_cast<size_t>(d)] != leaf->ne[d]) {
-                    return false;
-                }
+                signature.push_back(tensor->ne[d]);
+                signature.push_back(tensor->nb[d]);
+            }
+            signature.push_back(tensor->view_src == nullptr ? 0 : indices.at(tensor->view_src));
+            for (auto source : tensor->src) {
+                signature.push_back(source == nullptr ? 0 : indices.at(source));
+            }
+            for (int value : tensor->op_params) {
+                signature.push_back(static_cast<uint32_t>(value));
             }
         }
-        return true;
+        return signature;
+    }
+
+    static bool plan_matches_graph(ggml_cgraph* gf,
+                                   const Plan& plan,
+                                   const std::vector<uint64_t>& layout) {
+        GGML_ASSERT(gf != nullptr);
+        if (plan.leaf_names.size() != static_cast<size_t>(gf->n_leafs) ||
+            plan.layout != layout) {
+            return false;
+        }
+        for (int i = 0; i < gf->n_leafs; ++i) {
+            if (plan.leaf_names[i] != gf->leafs[i]->name) {
+                return false;
+            }
+        }
+        std::vector<std::pair<int, std::string>> cut_markers;
+        for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) {
+            auto node = ggml_graph_node(gf, i);
+            if (is_graph_cut_tensor(node)) {
+                cut_markers.emplace_back(i, node->name);
+            }
+        }
+        return cut_markers == plan.cut_markers;
+    }
+
+    bool plan_matches_graph(ggml_cgraph* gf, const Plan& plan) {
+        GGML_ASSERT(gf != nullptr);
+        return plan_matches_graph(gf, plan, graph_layout(gf, false));
     }
 
     ggml_tensor* output_tensor(ggml_cgraph* gf, const Segment& segment, size_t output_index) {
@@ -578,26 +612,6 @@ namespace sd::ggml_graph_cut {
         return tensors;
     }
 
-    std::unordered_set<std::string> collect_future_input_names(ggml_cgraph* gf,
-                                                               const Plan& plan,
-                                                               size_t current_segment_index) {
-        GGML_ASSERT(gf != nullptr);
-        std::unordered_set<std::string> future_input_names;
-        for (size_t seg_idx = current_segment_index + 1; seg_idx < plan.segments.size(); ++seg_idx) {
-            const auto& segment = plan.segments[seg_idx];
-            for (const auto& input_ref : segment.input_refs) {
-                if (input_ref.type != Segment::INPUT_PREVIOUS_CUT) {
-                    continue;
-                }
-                ggml_tensor* current_input = input_tensor(gf, input_ref);
-                if (current_input != nullptr && current_input->name[0] != '\0') {
-                    future_input_names.insert(current_input->name);
-                }
-            }
-        }
-        return future_input_names;
-    }
-
     ggml_cgraph* build_segment_graph(ggml_cgraph* gf,
                                      const Segment& segment,
                                      ggml_context** graph_ctx_out) {
@@ -662,6 +676,10 @@ namespace sd::ggml_graph_cut {
                 continue;
             }
             ggml_set_output(output);
+            if (output->view_src != nullptr) {
+                // A consumed output view does not keep its storage alive in gallocr.
+                ggml_set_output(output->view_src);
+            }
         }
         for (int node_idx : segment.internal_node_indices) {
             ggml_graph_add_node(segment_graph, ggml_graph_node(gf, node_idx));
@@ -716,6 +734,10 @@ namespace sd::ggml_graph_cut {
             if (output != nullptr && saved_output_flags.find(output) == saved_output_flags.end()) {
                 saved_output_flags[output] = output->flags;
             }
+            if (output != nullptr && output->view_src != nullptr &&
+                saved_output_flags.find(output->view_src) == saved_output_flags.end()) {
+                saved_output_flags[output->view_src] = output->view_src->flags;
+            }
         }
 
         ggml_context* graph_ctx    = nullptr;
@@ -744,6 +766,46 @@ namespace sd::ggml_graph_cut {
         return buffer_size;
     }
 
+    static size_t measure_graph_compute_buffer(
+        ggml_backend_t backend,
+        ggml_cgraph* gf,
+        const std::unordered_set<const ggml_tensor*>& params_tensor_set) {
+        struct TensorRuntimeBinding {
+            ggml_backend_buffer_t buffer = nullptr;
+            void* data                   = nullptr;
+            void* extra                  = nullptr;
+        };
+        std::unordered_map<ggml_tensor*, TensorRuntimeBinding> saved_bindings;
+        auto mark_external = [&](ggml_tensor* tensor) {
+            if (tensor == nullptr || saved_bindings.find(tensor) != saved_bindings.end()) {
+                return;
+            }
+            saved_bindings[tensor] = {tensor->buffer, tensor->data, tensor->extra};
+            tensor->data           = reinterpret_cast<void*>(static_cast<uintptr_t>(1));
+        };
+        for (int i = 0; i < leaf_count(gf); ++i) {
+            ggml_tensor* leaf = leaf_tensor(gf, i);
+            if (!is_params_tensor(params_tensor_set, leaf)) {
+                continue;
+            }
+            mark_external(leaf);
+            mark_external(leaf->view_src);
+        }
+
+        ggml_gallocr_t allocr = ggml_gallocr_new(
+            ggml_backend_get_default_buffer_type(backend));
+        size_t sizes[1] = {0};
+        ggml_gallocr_reserve_n_size(allocr, gf, nullptr, nullptr, sizes);
+        ggml_gallocr_free(allocr);
+
+        for (const auto& kv : saved_bindings) {
+            kv.first->buffer = kv.second.buffer;
+            kv.first->data   = kv.second.data;
+            kv.first->extra  = kv.second.extra;
+        }
+        return sizes[0];
+    }
+
     Plan build_plan(ggml_backend_t backend,
                     ggml_cgraph* gf,
                     const std::unordered_set<const ggml_tensor*>& params_tensor_set,
@@ -756,24 +818,22 @@ namespace sd::ggml_graph_cut {
         if (n_nodes <= 0) {
             return plan;
         }
-        plan.n_nodes = n_nodes;
-        plan.n_leafs = gf->n_leafs;
+        plan.layout = graph_layout(gf, false);
         for (int i = 0; i < gf->n_leafs; ++i) {
-            ggml_tensor* leaf = gf->leafs[i];
-            if (is_params_tensor(params_tensor_set, leaf)) {
-                continue;
-            }
-            auto shape       = input_shape(leaf);
-            shape.leaf_index = i;
-            plan.input_shapes.push_back(shape);
+            plan.leaf_names.emplace_back(gf->leafs[i]->name);
         }
+        plan.compute_buffer_size =
+            measure_graph_compute_buffer(backend, gf, params_tensor_set);
 
         std::unordered_map<const ggml_tensor*, int> producer_index;
         producer_index.reserve(static_cast<size_t>(n_nodes));
         for (int i = 0; i < n_nodes; ++i) {
-            producer_index[ggml_graph_node(gf, i)] = i;
+            ggml_tensor* node    = ggml_graph_node(gf, i);
+            producer_index[node] = i;
+            if (is_graph_cut_tensor(node)) {
+                plan.cut_markers.push_back({i, node->name});
+            }
         }
-
         std::vector<Segment> grouped_segments;
         std::unordered_map<std::string, size_t> group_to_segment;
         for (int i = 0; i < n_nodes; ++i) {
@@ -824,11 +884,24 @@ namespace sd::ggml_graph_cut {
         if (final_output_index < 0) {
             final_output_index = n_nodes - 1;
         }
-        ggml_tensor* final_output = final_output_index >= 0 ? ggml_graph_node(gf, final_output_index) : nullptr;
-        if (final_output != nullptr && available_cut_output_node_indices.find(final_output_index) == available_cut_output_node_indices.end()) {
-            Segment final_segment;
-            final_segment.group_name = "ggml_runner.final";
+        Segment final_segment;
+        final_segment.group_name = "ggml_runner.final";
+        if (final_output_index >= 0 &&
+            available_cut_output_node_indices.find(final_output_index) ==
+                available_cut_output_node_indices.end()) {
             final_segment.output_node_indices.push_back(final_output_index);
+        }
+        for (int i = 0; i < n_nodes; ++i) {
+            ggml_tensor* node = ggml_graph_node(gf, i);
+            if (i == final_output_index || node == nullptr ||
+                (node->flags & GGML_TENSOR_FLAG_OUTPUT) == 0 ||
+                available_cut_output_node_indices.find(i) !=
+                    available_cut_output_node_indices.end()) {
+                continue;
+            }
+            final_segment.output_node_indices.push_back(i);
+        }
+        if (!final_segment.output_node_indices.empty()) {
             build_segment(gf,
                           plan,
                           final_segment,
@@ -839,210 +912,59 @@ namespace sd::ggml_graph_cut {
                           log_desc);
         }
 
-        return plan;
-    }
-
-    Plan apply_max_vram_budget(ggml_cgraph* gf,
-                               const Plan& base_plan,
-                               size_t max_graph_vram_bytes,
-                               ggml_backend_t backend,
-                               const std::unordered_set<const ggml_tensor*>& params_tensor_set,
-                               const char* log_desc) {
-        GGML_ASSERT(backend != nullptr);
-        GGML_ASSERT(gf != nullptr);
-        int64_t t_budget_begin = ggml_time_ms();
-        if (max_graph_vram_bytes == 0 || !base_plan.has_cuts || base_plan.segments.size() <= 1) {
-            return base_plan;
-        }
-
-        const int n_nodes = ggml_graph_n_nodes(gf);
-        std::unordered_map<const ggml_tensor*, int> producer_index;
-        producer_index.reserve(static_cast<size_t>(n_nodes));
-        for (int i = 0; i < n_nodes; ++i) {
-            producer_index[ggml_graph_node(gf, i)] = i;
-        }
-
-        Plan merged_plan;
-        merged_plan.available = true;
-        merged_plan.has_cuts  = base_plan.has_cuts;
-        merged_plan.valid     = base_plan.valid;
-        merged_plan.n_nodes   = base_plan.n_nodes;
-        merged_plan.n_leafs   = base_plan.n_leafs;
-
-        std::unordered_set<int> available_cut_output_node_indices;
-        available_cut_output_node_indices.reserve(static_cast<size_t>(n_nodes));
-
-        size_t start_segment_index = 0;
-        while (start_segment_index < base_plan.segments.size()) {
-            Plan single_plan;
-            auto single_available_cut_output_node_indices = available_cut_output_node_indices;
-            auto single_seed                              = make_segment_seed(base_plan,
-                                                                              start_segment_index,
-                                                                              start_segment_index);
-            build_segment(gf,
-                          single_plan,
-                          single_seed,
-                          producer_index,
-                          single_available_cut_output_node_indices,
-                          backend,
-                          params_tensor_set,
-                          log_desc);
-            GGML_ASSERT(!single_plan.segments.empty());
-
-            size_t best_end_segment_index = start_segment_index;
-            bool can_merge_next_segment   = graph_cut_segment_vram_bytes(single_plan.segments.back()) <= max_graph_vram_bytes;
-
-            while (can_merge_next_segment && best_end_segment_index + 1 < base_plan.segments.size()) {
-                const size_t next_end_segment_index = best_end_segment_index + 1;
-                Plan candidate_plan;
-                auto candidate_available_cut_output_node_indices = available_cut_output_node_indices;
-                auto candidate_seed                              = make_segment_seed(base_plan,
-                                                                                     start_segment_index,
-                                                                                     next_end_segment_index);
-                build_segment(gf,
-                              candidate_plan,
-                              candidate_seed,
-                              producer_index,
-                              candidate_available_cut_output_node_indices,
-                              backend,
-                              params_tensor_set,
-                              log_desc);
-                GGML_ASSERT(!candidate_plan.segments.empty());
-
-                const auto& candidate_segment = candidate_plan.segments.back();
-                const size_t candidate_bytes  = graph_cut_segment_vram_bytes(candidate_segment);
-                if (candidate_bytes > max_graph_vram_bytes) {
-                    break;
+        std::unordered_set<std::string> future_cut_names;
+        for (auto segment = plan.segments.rbegin(); segment != plan.segments.rend(); ++segment) {
+            segment->future_cut_names = future_cut_names;
+            segment->live_cut_names   = future_cut_names;
+            for (const auto& input : segment->input_refs) {
+                if (input.type != Segment::INPUT_PREVIOUS_CUT) {
+                    continue;
                 }
-
-                best_end_segment_index = next_end_segment_index;
+                segment->live_cut_names.insert(input.display_name);
+                future_cut_names.insert(input.display_name);
             }
-
-            auto best_seed = make_segment_seed(base_plan,
-                                               start_segment_index,
-                                               best_end_segment_index);
-            build_segment(gf,
-                          merged_plan,
-                          best_seed,
-                          producer_index,
-                          available_cut_output_node_indices,
-                          backend,
-                          params_tensor_set,
-                          log_desc);
-            start_segment_index = best_end_segment_index + 1;
         }
 
-        if (log_desc != nullptr && merged_plan.segments.size() != base_plan.segments.size()) {
-            LOG_INFO("%s graph cut max_vram=%.2f MB merged %zu segments -> %zu segments",
+        std::string plan_validation_error;
+        plan.valid = validate_plan(gf, plan, &plan_validation_error);
+        if (!plan.valid && log_desc != nullptr) {
+            LOG_WARN("%s graph cut plan validation failed (%s); using monolithic execution",
                      log_desc,
-                     max_graph_vram_bytes / 1024.0 / 1024.0,
-                     base_plan.segments.size(),
-                     merged_plan.segments.size());
+                     plan_validation_error.c_str());
         }
 
-        if (log_desc != nullptr) {
-            LOG_DEBUG("%s graph cut max_vram budget merge took %lld ms",
-                      log_desc,
-                      ggml_time_ms() - t_budget_begin);
-        }
-
-        return merged_plan;
+        return plan;
     }
 
     Plan resolve_plan(ggml_backend_t backend,
                       ggml_cgraph* gf,
                       PlanCache* cache,
-                      size_t max_graph_vram_bytes,
                       const std::unordered_set<const ggml_tensor*>& params_tensor_set,
                       const char* log_desc) {
         GGML_ASSERT(backend != nullptr);
         GGML_ASSERT(gf != nullptr);
         GGML_ASSERT(cache != nullptr);
 
-        int64_t t_prepare_begin = ggml_time_ms();
-        Plan base_plan;
+        const auto layout = graph_layout(gf, false);
+        auto& plans       = cache->graph_cut_plans;
+        for (auto it = plans.begin(); it != plans.end(); ++it) {
+            if (it->available && plan_matches_graph(gf, *it, layout)) {
+                plans.splice(plans.begin(), plans, it);
+                return plans.front();
+            }
+        }
+
         int64_t t_plan_begin = ggml_time_ms();
-        if (cache->graph_cut_plan.available && plan_matches_graph(gf, cache->graph_cut_plan)) {
-            base_plan = cache->graph_cut_plan;
-        } else {
-            base_plan                                = build_plan(backend, gf, params_tensor_set, log_desc);
-            cache->graph_cut_plan                    = base_plan;
-            cache->graph_cut_plan.available          = true;
-            cache->budgeted_graph_cut_plan.available = false;
-            if (log_desc != nullptr) {
-                LOG_INFO("%s build cached graph cut plan done (taking %lld ms)", log_desc, ggml_time_ms() - t_plan_begin);
-            }
+        plans.push_front(build_plan(backend, gf, params_tensor_set, log_desc));
+        if (plans.size() > PlanCache::MAX_PLANS) {
+            plans.pop_back();
         }
-
-        Plan resolved_plan = base_plan;
-        if (max_graph_vram_bytes > 0 && base_plan.has_cuts) {
-            if (cache->budgeted_graph_cut_plan.available &&
-                cache->budgeted_graph_cut_plan_max_vram_bytes == max_graph_vram_bytes &&
-                plan_matches_graph(gf, cache->budgeted_graph_cut_plan)) {
-                resolved_plan = cache->budgeted_graph_cut_plan;
-            } else {
-                resolved_plan                                 = apply_max_vram_budget(gf,
-                                                                                      base_plan,
-                                                                                      max_graph_vram_bytes,
-                                                                                      backend,
-                                                                                      params_tensor_set,
-                                                                                      log_desc);
-                cache->budgeted_graph_cut_plan                = resolved_plan;
-                cache->budgeted_graph_cut_plan.available      = true;
-                cache->budgeted_graph_cut_plan_max_vram_bytes = max_graph_vram_bytes;
-            }
-        }
-        return resolved_plan;
-    }
-
-    void annotate_residency(Plan& plan, size_t max_graph_vram_bytes) {
-        // Cached plans may be reused with a smaller live budget.
-        for (auto& seg : plan.segments) {
-            seg.residency = SegmentResidency::STREAMED;
-        }
-        if (max_graph_vram_bytes == 0 || plan.segments.size() < 2) {
-            return;
-        }
-
-        bool any_param_bearing = false;
-        for (const auto& seg : plan.segments) {
-            if (seg.input_param_bytes > 0) {
-                any_param_bearing = true;
-                break;
-            }
-        }
-        if (!any_param_bearing) {
-            return;
-        }
-
-        // Leave room for the largest active streamed segment.
-        size_t worst_streamed_footprint = 0;
-        for (const auto& seg : plan.segments) {
-            const size_t seg_footprint = seg.input_param_bytes +
-                                         seg.compute_buffer_size +
-                                         seg.output_bytes +
-                                         seg.input_previous_cut_bytes +
-                                         seg.input_external_bytes;
-            if (seg_footprint > worst_streamed_footprint) {
-                worst_streamed_footprint = seg_footprint;
-            }
-        }
-        constexpr size_t safety = 512ull * 1024 * 1024;
-        const size_t reserved   = safety + worst_streamed_footprint;
-
-        if (max_graph_vram_bytes <= reserved) {
-            return;
-        }
-        const size_t available = max_graph_vram_bytes - reserved;
-
-        size_t cumulative = 0;
-        for (auto& seg : plan.segments) {
-            if (cumulative + seg.input_param_bytes > available) {
-                break;
-            }
-            seg.residency = SegmentResidency::RESIDENT;
-            cumulative += seg.input_param_bytes;
+        if (log_desc != nullptr) {
+            LOG_INFO("%s build cached graph cut plan done (taking %lld ms)",
+                     log_desc,
+                     ggml_time_ms() - t_plan_begin);
         }
+        return plans.front();
     }
 
 }  // namespace sd::ggml_graph_cut

src/core/ggml_graph_cut.h

@@ -3,22 +3,17 @@
 
 #include <array>
 #include <cstdint>
+#include <list>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
+#include <utility>
 #include <vector>
 
 #include "ggml-backend.h"
 #include "ggml.h"
 
 namespace sd::ggml_graph_cut {
-
-    // Streaming residency for a segment's params.
-    enum class SegmentResidency : uint8_t {
-        STREAMED = 0,
-        RESIDENT = 1,
-    };
-
     struct Segment {
         enum InputType {
             INPUT_EXTERNAL = 0,
@@ -33,38 +28,29 @@ namespace sd::ggml_graph_cut {
             int node_index = -1;
         };
 
-        size_t compute_buffer_size      = 0;
-        size_t output_bytes             = 0;
-        size_t input_external_bytes     = 0;
-        size_t input_previous_cut_bytes = 0;
-        size_t input_param_bytes        = 0;
+        size_t compute_buffer_size = 0;
         std::string group_name;
         std::vector<int> internal_node_indices;
         std::vector<int> output_node_indices;
         std::vector<InputRef> input_refs;
-        SegmentResidency residency = SegmentResidency::STREAMED;
+        std::unordered_set<std::string> future_cut_names;
+        std::unordered_set<std::string> live_cut_names;
     };
 
     struct Plan {
-        struct InputShape {
-            int leaf_index                        = -1;
-            ggml_type type                        = GGML_TYPE_COUNT;
-            std::array<int64_t, GGML_MAX_DIMS> ne = {0, 0, 0, 0};
-        };
-
-        bool available = false;
-        bool has_cuts  = false;
-        bool valid     = true;
-        int n_nodes    = 0;
-        int n_leafs    = 0;
-        std::vector<InputShape> input_shapes;
+        bool available             = false;
+        bool has_cuts              = false;
+        bool valid                 = true;
+        size_t compute_buffer_size = 0;
+        std::vector<uint64_t> layout;
+        std::vector<std::string> leaf_names;
+        std::vector<std::pair<int, std::string>> cut_markers;
         std::vector<Segment> segments;
     };
 
     struct PlanCache {
-        Plan graph_cut_plan;
-        Plan budgeted_graph_cut_plan;
-        size_t budgeted_graph_cut_plan_max_vram_bytes = 0;
+        static constexpr size_t MAX_PLANS = 4;
+        std::list<Plan> graph_cut_plans;
     };
 
     static constexpr const char* GGML_RUNNER_CUT_PREFIX = "ggml_runner_cut:";
@@ -89,13 +75,12 @@ namespace sd::ggml_graph_cut {
     ggml_backend_buffer_t tensor_buffer(const ggml_tensor* tensor);
     ggml_tensor* cache_source_tensor(ggml_tensor* tensor);
     size_t cache_tensor_bytes(const ggml_tensor* tensor);
+    // Plans ignore runtime bindings; allocator reservations must include them.
+    std::vector<uint64_t> graph_layout(ggml_cgraph* graph, bool include_bindings);
     bool plan_matches_graph(ggml_cgraph* gf, const Plan& plan);
     ggml_tensor* output_tensor(ggml_cgraph* gf, const Segment& segment, size_t output_index);
     ggml_tensor* input_tensor(ggml_cgraph* gf, const Segment::InputRef& input_ref);
     std::vector<ggml_tensor*> param_tensors(ggml_cgraph* gf, const Segment& segment);
-    std::unordered_set<std::string> collect_future_input_names(ggml_cgraph* gf,
-                                                               const Plan& plan,
-                                                               size_t current_segment_index);
     ggml_cgraph* build_segment_graph(ggml_cgraph* gf,
                                      const Segment& segment,
                                      ggml_context** graph_ctx_out);
@@ -109,21 +94,12 @@ namespace sd::ggml_graph_cut {
                     ggml_cgraph* gf,
                     const std::unordered_set<const ggml_tensor*>& params_tensor_set,
                     const char* log_desc);
-    Plan apply_max_vram_budget(ggml_cgraph* gf,
-                               const Plan& base_plan,
-                               size_t max_graph_vram_bytes,
-                               ggml_backend_t backend,
-                               const std::unordered_set<const ggml_tensor*>& params_tensor_set,
-                               const char* log_desc);
     Plan resolve_plan(ggml_backend_t backend,
                       ggml_cgraph* gf,
                       PlanCache* cache,
-                      size_t max_graph_vram_bytes,
                       const std::unordered_set<const ggml_tensor*>& params_tensor_set,
                       const char* log_desc);
 
-    // Mark leading segments resident when they fit after streamed-segment headroom.
-    void annotate_residency(Plan& plan, size_t max_graph_vram_bytes);
 }  // namespace sd::ggml_graph_cut
 
 #endif  // __SD_CORE_GGML_GRAPH_CUT_H__

src/core/ggml_runner.cpp

@@ -0,0 +1,298 @@
+#include <algorithm>
+#include <map>
+#include <utility>
+
+#include "core/ggml_extend.hpp"
+#include "core/segment_graph_bindings.h"
+#include "core/segment_weight_pipeline.h"
+
+using namespace sd;
+
+static size_t add_bytes(size_t a, size_t b) {
+    return b > SIZE_MAX - a ? SIZE_MAX : a + b;
+}
+
+ComputeWorkspace::Measurement GGMLRunner::measure(ggml_cgraph* graph, size_t direct_bytes) {
+    auto external_backend = [&](const ggml_tensor* tensor) -> ggml_backend_t {
+        if (!params_tensor_set_.count(tensor)) {
+            return nullptr;
+        }
+        auto placement = graph_cut_layer_split_assignments_.find(tensor);
+        return placement == graph_cut_layer_split_assignments_.end() ? runtime_backend : placement->second;
+    };
+    auto assign_nodes = [&](ggml_backend_sched_t scheduler, ggml_cgraph* copy) {
+        pin_multi_device_nodes(scheduler, copy, graph);
+    };
+    return workspace_.measure(graph, direct_bytes, external_backend, assign_nodes);
+}
+
+std::vector<DeviceMemoryRequest> GGMLRunner::memory_requests(
+    const std::vector<BackendBufferSize>& sizes,
+    size_t pending_cache_bytes) const {
+    std::vector<DeviceMemoryRequest> requests;
+    for (const auto& size : sizes) {
+        const size_t retained    = retained_runtime_buffer_bytes(size.backend);
+        const size_t reusable    = workspace_.bytes(size.backend);
+        const size_t cache_bytes = size.backend == runtime_backend ? pending_cache_bytes : 0;
+        const size_t pending     = add_bytes(size.bytes > reusable ? size.bytes - reusable : 0, cache_bytes);
+        size_t limit             = max_graph_vram_bytes;
+        if (is_multi_device()) {
+            size_t index = 0;
+            if (size.backend != runtime_backend) {
+                auto position = std::find(extra_runtime_backends.begin(), extra_runtime_backends.end(), size.backend);
+                index         = static_cast<size_t>(position - extra_runtime_backends.begin()) + 1;
+            }
+            if (index < graph_cut_layer_split_backend_vram_limits_.size()) {
+                limit = graph_cut_layer_split_backend_vram_limits_[index];
+            }
+        }
+        requests.push_back({size.backend, reinterpret_cast<uintptr_t>(this), pending,
+                            retained, limit});
+    }
+    return requests;
+}
+
+bool GGMLRunner::fits(const std::vector<DeviceMemoryRequest>& requests,
+                      const std::vector<ggml_tensor*>& params) const {
+    auto manager = residency_manager.lock();
+    if (manager == nullptr) {
+        return params.empty();
+    }
+    for (const auto& request : requests) {
+        if (!manager->fits_compute_backend_capacity(request, params)) {
+            return false;
+        }
+    }
+    return true;
+}
+
+bool GGMLRunner::execute_segment(ggml_cgraph* graph, int n_threads) {
+    if (sd_backend_is_cpu(runtime_backend)) {
+        sd_backend_cpu_set_n_threads(runtime_backend, n_threads);
+    }
+    if (workspace_.cpu_backend() != nullptr) {
+        sd_backend_cpu_set_n_threads(workspace_.cpu_backend(), n_threads);
+    }
+    auto scheduler = workspace_.scheduler();
+    ggml_status status;
+    if (scheduler != nullptr) {
+        if (sd_get_backend_eval_callback() != nullptr && !multi_device_eval_callback_warned) {
+            LOG_WARN("%s: eval callback is not supported with the backend scheduler; ignoring", get_desc().c_str());
+            multi_device_eval_callback_warned = true;
+        }
+        status = ggml_backend_sched_graph_compute(scheduler, graph);
+    } else {
+        status = sd_backend_graph_compute_with_eval_callback(runtime_backend, graph,
+                                                             sd_get_backend_eval_callback(),
+                                                             sd_get_backend_eval_callback_data());
+    }
+    workspace_.synchronize();
+    if (status != GGML_STATUS_SUCCESS) {
+        LOG_ERROR("%s compute failed: %s", get_desc().c_str(), ggml_status_to_string(status));
+        return false;
+    }
+    const std::string description = get_desc();
+    if (!debug_tensors.empty()) {
+        std::unordered_set<const ggml_tensor*> graph_tensors;
+        const int leaf_count = ggml_graph_cut::leaf_count(graph);
+        const int node_count = ggml_graph_n_nodes(graph);
+        graph_tensors.reserve(static_cast<size_t>(leaf_count + node_count));
+        for (int index = 0; index < leaf_count; ++index) {
+            graph_tensors.insert(ggml_graph_cut::leaf_tensor(graph, index));
+        }
+        for (int index = 0; index < node_count; ++index) {
+            graph_tensors.insert(ggml_graph_node(graph, index));
+        }
+
+        for (const auto& entry : debug_tensors) {
+            ggml_tensor* tensor = entry.first;
+            if (tensor == nullptr || graph_tensors.find(tensor) == graph_tensors.end()) {
+                continue;
+            }
+            ggml_backend_buffer_t buffer =
+                tensor->view_src != nullptr ? tensor->view_src->buffer : tensor->buffer;
+            if (buffer == nullptr) {
+                LOG_WARN("%s skip debug tensor '%s': tensor buffer not set",
+                         description.c_str(),
+                         entry.second.c_str());
+                continue;
+            }
+            if (tensor->type != GGML_TYPE_F32) {
+                LOG_WARN("%s skip debug tensor '%s': only GGML_TYPE_F32 is supported, got %s",
+                         description.c_str(),
+                         entry.second.c_str(),
+                         ggml_type_name(tensor->type));
+                continue;
+            }
+            auto debug_tensor = make_sd_tensor_from_ggml<float>(tensor);
+            print_sd_tensor(debug_tensor, false, entry.second.c_str());
+        }
+    }
+
+    return true;
+}
+
+std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n_threads, bool no_return, const std::function<bool()>& read_outputs) {
+    if (!assign_graph_cut_layer_split_backends(graph)) {
+        return std::nullopt;
+    }
+    const auto params = collect_used_param_tensors(graph);
+    ggml_graph_cut::Plan plan;
+    if (!resolve_graph_cut_plan(graph, &plan)) {
+        return std::nullopt;
+    }
+    const auto full_measurement = measure(graph, plan.compute_buffer_size);
+    if (full_measurement.buffers.empty()) {
+        return std::nullopt;
+    }
+    auto manager         = residency_manager.lock();
+    const bool segmented = !is_multi_device() && !sd_backend_is_cpu(runtime_backend) &&
+                           manager != nullptr && manager->segmented_compute_enabled() &&
+                           plan.valid && plan.has_cuts && plan.segments.size() > 1 &&
+                           !fits(memory_requests(full_measurement.buffers, cache_.pending_bytes(graph)), params);
+    if (!segmented) {
+        ggml_graph_cut::Segment segment;
+        segment.group_name          = "graph";
+        segment.compute_buffer_size = plan.compute_buffer_size;
+        for (int i = 0; i < ggml_graph_n_nodes(graph); ++i) {
+            segment.internal_node_indices.push_back(i);
+        }
+        for (int i = 0; i < ggml_graph_cut::leaf_count(graph); ++i) {
+            auto tensor = ggml_graph_cut::leaf_tensor(graph, i);
+            ggml_graph_cut::Segment::InputRef input;
+            input.leaf_index = i;
+            input.type       = canonical_param_tensor(tensor) != nullptr
+                                   ? ggml_graph_cut::Segment::INPUT_PARAM
+                                   : ggml_graph_cut::Segment::INPUT_EXTERNAL;
+            segment.input_refs.push_back(input);
+        }
+        plan.segments = {std::move(segment)};
+    }
+    const bool segments_changed = plan.segments.size() != logged_segment_count_;
+    if (segments_changed && (segmented || logged_segment_count_ > 1)) {
+        LOG_VERBOSE("%s using %zu segment%s", get_desc().c_str(),
+                    plan.segments.size(), plan.segments.size() == 1 ? "" : "s");
+    }
+    SegmentGraphBindings bindings(cut_cache_, plan, graph);
+    SegmentWeightPipeline weights(manager, runtime_backend, reinterpret_cast<uintptr_t>(this),
+                                  graph, plan, params_tensor_set_,
+                                  segmented && manager != nullptr && manager->prefetch_enabled());
+
+    std::map<ggml_backend_t, size_t> peak_compute_bytes;
+    auto track_compute_buffer = [&](ggml_backend_t backend) {
+        if (backend != nullptr) {
+            auto& peak = peak_compute_bytes[backend];
+            peak       = std::max(peak, workspace_.bytes(backend));
+        }
+    };
+    std::optional<Tensor<float>> output = Tensor<float>();
+    for (size_t index = 0; index < plan.segments.size(); ++index) {
+        const auto& segment = plan.segments[index];
+        const bool last     = index + 1 == plan.segments.size();
+        auto fail_segment   = [&](const char* phase) {
+            LOG_ERROR("%s segment %zu/%zu (%s) failed during %s", get_desc().c_str(),
+                        index + 1, plan.segments.size(), segment.group_name.c_str(), phase);
+            return std::nullopt;
+        };
+        cut_cache_.prune(segment.live_cut_names);
+        bindings.reset(segment);
+        if (!bindings.bind_cached_inputs(segment, get_desc().c_str())) {
+            return fail_segment("input binding");
+        }
+        ggml_context* segment_context = nullptr;
+        auto segment_graph            = segmented
+                                            ? ggml_graph_cut::build_segment_graph(graph, segment, &segment_context)
+                                            : graph;
+        struct SegmentCleanup {
+            GGMLRunner& runner;
+            SegmentWeightPipeline& weights;
+            SegmentGraphBindings& bindings;
+            ggml_context* context;
+            ~SegmentCleanup() {
+                runner.workspace_.segment_end();
+                bindings.restore();
+                weights.segment_end();
+                ggml_free(context);
+                runner.sync_runtime_residency();
+            }
+        } segment_cleanup{*this, weights, bindings, segment_context};
+
+        auto measurement = segmented ? measure(segment_graph, segment.compute_buffer_size) : full_measurement;
+        if (!workspace_.prepare(measurement)) {
+            return fail_segment("workspace preparation");
+        }
+        const size_t cut_bytes       = last ? 0 : cut_cache_.estimate_output_bytes(graph, segment);
+        const size_t new_cache_bytes = add_bytes(cut_bytes, cache_.pending_bytes(segment_graph));
+        auto ensure_capacity         = [&]() {
+            sync_runtime_residency();
+            auto requests = memory_requests(measurement.buffers, new_cache_bytes);
+            if (!fits(requests, weights.params(index)) && workspace_.release_excess(measurement)) {
+                sync_runtime_residency();
+                requests = memory_requests(measurement.buffers, new_cache_bytes);
+            }
+            return weights.ensure_segment_capacity(index, requests);
+        };
+        if (!weights.segment_start(index, ensure_capacity)) {
+            return fail_segment("weight preparation");
+        }
+        // Preparing weights can execute LoRA graphs and reclaim an idle workspace.
+        if (!workspace_.measurement_matches(segment_graph, measurement)) {
+            measurement = measure(segment_graph, segment.compute_buffer_size);
+        }
+        if (!workspace_.prepare(measurement) || !ensure_capacity()) {
+            return fail_segment("workspace capacity check");
+        }
+        if (!workspace_.allocate(segment_graph, [&](ggml_backend_sched_t scheduler, ggml_cgraph* current) {
+                pin_multi_device_nodes(scheduler, current);
+            })) {
+            return fail_segment("workspace allocation");
+        }
+        for (const auto& size : measurement.buffers) {
+            track_compute_buffer(size.backend);
+        }
+        if (workspace_.scheduler() != nullptr) {
+            track_compute_buffer(workspace_.cpu_backend());
+        }
+        if (!ensure_capacity()) {
+            return fail_segment("allocated capacity check");
+        }
+        copy_data_to_backend_tensor(segment_graph, false);
+        auto prefetch_requests = memory_requests(measurement.buffers, new_cache_bytes);
+        if (!prefetch_requests.empty()) {
+            weights.enqueue_next(index, prefetch_requests.front());
+        }
+        LOG_DEBUG("%s executing segment %zu/%zu: %s", get_desc().c_str(),
+                  index + 1, plan.segments.size(), segment.group_name.c_str());
+        if (!execute_segment(segment_graph, n_threads) ||
+            !cache_.capture(segment_graph) ||
+            !cut_cache_.capture(graph, segment, get_desc().c_str())) {
+            return fail_segment("execution or output caching");
+        }
+        sync_runtime_residency();
+        if (last) {
+            if (read_outputs && !read_outputs()) {
+                return fail_segment("output finalization");
+            }
+            if (!no_return) {
+                auto result = ggml_get_tensor(compute_ctx, final_result_name.c_str());
+                output      = read_graph_tensor<float>(result, "output");
+                if (!output.has_value()) {
+                    return fail_segment("output readback");
+                }
+            }
+        }
+        // Final outputs and their callbacks may still be views of consumed cuts.
+        cut_cache_.prune(segment.future_cut_names);
+    }
+    if (segments_changed || peak_compute_bytes != logged_compute_bytes_) {
+        for (const auto& entry : peak_compute_bytes) {
+            LOG_VERBOSE("%s compute buffer size: %.2f MB(%s) on %s (peak across %zu segment%s)",
+                        get_desc().c_str(), entry.second / (1024.0 * 1024.0),
+                        sd_backend_is_cpu(entry.first) ? "RAM" : "VRAM", ggml_backend_name(entry.first),
+                        plan.segments.size(), plan.segments.size() == 1 ? "" : "s");
+        }
+        logged_compute_bytes_ = std::move(peak_compute_bytes);
+        logged_segment_count_ = plan.segments.size();
+    }
+    return output;
+}

src/core/layer_split_partition.cpp

@@ -145,19 +145,24 @@ namespace sd {
         std::vector<int64_t> backend_capacities = graph_cut_layer_split_backend_capacities(split_backends,
                                                                                            backend_vram_limits,
                                                                                            primary_backend_vram_limit);
+        // Existing placements may already occupy the reported free VRAM. Reuse
+        // them; execution checks missing weights and reclaims memory as needed.
+        const bool reuse_assignments = std::all_of(seen_params.begin(), seen_params.end(), [&](ggml_tensor* param) {
+            return param_assignments.count(param) != 0;
+        });
 
         std::vector<ggml_backend_t> backend_by_segment(plan.segments.size(), split_backends[0]);
         size_t current_backend = 0;
         int64_t current_used   = 0;
         for (size_t seg_idx = 0; seg_idx < plan.segments.size(); seg_idx++) {
             int64_t bytes = segment_param_bytes[seg_idx];
-            while (current_backend + 1 < split_backends.size() &&
+            while (!reuse_assignments && current_backend + 1 < split_backends.size() &&
                    bytes > 0 &&
                    current_used + bytes > backend_capacities[current_backend]) {
                 current_backend++;
                 current_used = 0;
             }
-            if (bytes > 0 && current_used + bytes > backend_capacities[current_backend]) {
+            if (!reuse_assignments && bytes > 0 && current_used + bytes > backend_capacities[current_backend]) {
                 LOG_ERROR("%s graph-cut layer split: segment %zu needs %.1f MB on %s, but only %.1f MB is available under current VRAM limits",
                           desc,
                           seg_idx,
@@ -167,7 +172,6 @@ namespace sd {
                 return false;
             }
             current_used += bytes;
-            backend_by_segment[seg_idx] = split_backends[current_backend];
 
             for (ggml_tensor* param : segment_params[seg_idx]) {
                 ggml_backend_t target_backend = split_backends[current_backend];
@@ -186,12 +190,16 @@ namespace sd {
                               ggml_get_name(param));
                     return false;
                 }
-                size_t backend_idx                               = (size_t)std::distance(split_backends.begin(), backend_it);
+                size_t backend_idx = (size_t)std::distance(split_backends.begin(), backend_it);
+                if (reuse_assignments) {
+                    current_backend = backend_idx;
+                }
                 assignment.first_segment_by_backend[backend_idx] = std::min(assignment.first_segment_by_backend[backend_idx], seg_idx);
                 assignment.last_segment_by_backend[backend_idx]  = std::max(assignment.last_segment_by_backend[backend_idx], seg_idx + 1);
                 assignment.tensors_by_backend[backend_idx].push_back(param);
                 assignment.bytes_by_backend[backend_idx] += (int64_t)ggml_nbytes(param);
             }
+            backend_by_segment[seg_idx] = split_backends[current_backend];
         }
 
         const int n_nodes = ggml_graph_n_nodes(gf);
@@ -243,13 +251,13 @@ namespace sd {
                          assignment.tensors_by_backend[i].size(),
                          assignment.bytes_by_backend[i] / (1024.0 * 1024.0));
             } else {
-                LOG_DEBUG("%s graph-cut layer split: %s <- segments [%zu, %zu), %zu tensors, %.1f MB",
-                          desc,
-                          layer_split_backend_device_display_name(split_backends[i]).c_str(),
-                          first_segment,
-                          last_segment,
-                          assignment.tensors_by_backend[i].size(),
-                          assignment.bytes_by_backend[i] / (1024.0 * 1024.0));
+                LOG_VERBOSE("%s graph-cut layer split: %s <- segments [%zu, %zu), %zu tensors, %.1f MB",
+                            desc,
+                            layer_split_backend_device_display_name(split_backends[i]).c_str(),
+                            first_segment,
+                            last_segment,
+                            assignment.tensors_by_backend[i].size(),
+                            assignment.bytes_by_backend[i] / (1024.0 * 1024.0));
             }
         }
     }

src/core/runner_cache.cpp

@@ -0,0 +1,211 @@
+#include "core/runner_cache.h"
+
+#include <algorithm>
+#include <iterator>
+#include <unordered_set>
+
+#include "core/ggml_graph_cut.h"
+#include "core/util.h"
+
+namespace sd {
+    static std::unordered_set<const ggml_tensor*> cache_graph_tensors(ggml_cgraph* graph) {
+        std::unordered_set<const ggml_tensor*> tensors;
+        for (int i = 0; i < ggml_graph_n_nodes(graph); ++i) {
+            tensors.insert(ggml_graph_node(graph, i));
+        }
+        for (int i = 0; i < ggml_graph_cut::leaf_count(graph); ++i) {
+            tensors.insert(ggml_graph_cut::leaf_tensor(graph, i));
+        }
+        return tensors;
+    }
+
+    CachedTensor::~CachedTensor() {
+        ggml_backend_buffer_free(buffer);
+        ggml_free(context);
+    }
+
+    std::unique_ptr<CachedTensor> CachedTensor::copy(ggml_backend_t backend,
+                                                     const std::string& name,
+                                                     ggml_tensor* source) {
+        if (ggml_graph_cut::tensor_buffer(source) == nullptr) {
+            return nullptr;
+        }
+        auto entry     = std::make_unique<CachedTensor>();
+        entry->context = ggml_init({2 * ggml_tensor_overhead(), nullptr, true});
+        if (entry->context == nullptr) {
+            return nullptr;
+        }
+        entry->tensor = ggml_dup_tensor(entry->context, source);
+        // Cut views are rebound with their original strides and offsets.
+        std::copy(std::begin(source->nb), std::end(source->nb), std::begin(entry->tensor->nb));
+        ggml_set_name(entry->tensor, name.c_str());
+        entry->buffer = ggml_backend_alloc_ctx_tensors(entry->context, backend);
+        if (entry->buffer == nullptr) {
+            return nullptr;
+        }
+        if (source->view_src != nullptr || !ggml_is_contiguous(source) || source->buffer == nullptr) {
+            std::vector<uint8_t> data(ggml_nbytes(source));
+            ggml_backend_tensor_get(source, data.data(), 0, data.size());
+            ggml_backend_tensor_set(entry->tensor, data.data(), 0, data.size());
+        } else {
+            ggml_backend_tensor_copy(source, entry->tensor);
+        }
+        return entry;
+    }
+
+    static ggml_tensor* cached_tensor(const CachedTensors& tensors, const std::string& name) {
+        auto entry = tensors.find(name);
+        return entry == tensors.end() ? nullptr : entry->second->tensor;
+    }
+
+    static size_t resident_bytes(const CachedTensors& tensors, ggml_backend_dev_t device) {
+        size_t bytes = 0;
+        for (const auto& entry : tensors) {
+            auto buffer = entry.second->buffer;
+            if (!ggml_backend_buffer_is_host(buffer) &&
+                ggml_backend_buft_get_device(ggml_backend_buffer_get_type(buffer)) == device) {
+                const size_t size = ggml_backend_buffer_get_size(buffer);
+                bytes             = size > SIZE_MAX - bytes ? SIZE_MAX : bytes + size;
+            }
+        }
+        return bytes;
+    }
+
+    ggml_tensor* RunnerCache::get(const std::string& name) const {
+        return cached_tensor(committed_, name);
+    }
+
+    void RunnerCache::stage(const std::string& name, ggml_tensor* tensor) {
+        if (tensor != nullptr) {
+            ggml_set_output(tensor);
+            outputs_[name] = tensor;
+        }
+    }
+
+    size_t RunnerCache::pending_bytes(ggml_cgraph* graph) const {
+        if (outputs_.empty()) {
+            return 0;
+        }
+        auto tensors = cache_graph_tensors(graph);
+        auto buft    = ggml_backend_get_default_buffer_type(backend_);
+        size_t bytes = 0;
+        for (const auto& output : outputs_) {
+            if (pending_.count(output.first) || !tensors.count(output.second)) {
+                continue;
+            }
+            const size_t size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, output.second),
+                                         ggml_backend_buft_get_alignment(buft));
+            bytes             = size > SIZE_MAX - bytes ? SIZE_MAX : bytes + size;
+        }
+        return bytes;
+    }
+
+    size_t RunnerCache::resident_bytes(ggml_backend_dev_t device) const {
+        const size_t committed = sd::resident_bytes(committed_, device);
+        const size_t pending   = sd::resident_bytes(pending_, device);
+        return pending > SIZE_MAX - committed ? SIZE_MAX : committed + pending;
+    }
+
+    bool RunnerCache::capture(ggml_cgraph* graph) {
+        if (outputs_.empty()) {
+            return true;
+        }
+        const auto tensors = cache_graph_tensors(graph);
+        for (const auto& output : outputs_) {
+            if (pending_.count(output.first) || !tensors.count(output.second)) {
+                continue;
+            }
+            GGML_ASSERT(ggml_is_contiguous(output.second));
+            auto entry = CachedTensor::copy(backend_, output.first, output.second);
+            if (entry == nullptr) {
+                return false;
+            }
+            pending_[output.first] = std::move(entry);
+        }
+        ggml_backend_synchronize(backend_);
+        return true;
+    }
+
+    void RunnerCache::graph_end(bool success) {
+        // Graph inputs can still reference the previous generation until graph end.
+        if (success) {
+            for (auto& entry : pending_) {
+                committed_[entry.first] = std::move(entry.second);
+            }
+        }
+        pending_.clear();
+        outputs_.clear();
+    }
+
+    void RunnerCache::clear() {
+        graph_end(false);
+        committed_.clear();
+    }
+
+    ggml_tensor* GraphCutTensorCache::get(const std::string& name) const {
+        return cached_tensor(tensors_, name);
+    }
+
+    size_t GraphCutTensorCache::resident_bytes(ggml_backend_dev_t device) const {
+        return sd::resident_bytes(tensors_, device);
+    }
+
+    size_t GraphCutTensorCache::estimate_output_bytes(
+        ggml_cgraph* graph,
+        const ggml_graph_cut::Segment& segment) const {
+        ggml_backend_buffer_type_t buffer_type =
+            ggml_backend_get_default_buffer_type(backend_);
+        if (buffer_type == nullptr) {
+            return SIZE_MAX;
+        }
+        const size_t alignment = ggml_backend_buft_get_alignment(buffer_type);
+        size_t total_size      = 0;
+        for (size_t output_idx = 0; output_idx < segment.output_node_indices.size(); ++output_idx) {
+            ggml_tensor* output = ggml_graph_cut::output_tensor(graph, segment, output_idx);
+            if (output == nullptr || !ggml_graph_cut::is_graph_cut_tensor(output) ||
+                !segment.future_cut_names.count(output->name)) {
+                continue;
+            }
+            ggml_tensor* source      = ggml_graph_cut::cache_source_tensor(output);
+            const size_t tensor_size = GGML_PAD(
+                ggml_backend_buft_get_alloc_size(buffer_type, source), alignment);
+            total_size = tensor_size > SIZE_MAX - total_size ? SIZE_MAX : total_size + tensor_size;
+        }
+        return total_size;
+    }
+
+    void GraphCutTensorCache::prune(const std::unordered_set<std::string>& keep_names) {
+        for (auto it = tensors_.begin(); it != tensors_.end();) {
+            it = keep_names.count(it->first) ? std::next(it) : tensors_.erase(it);
+        }
+    }
+
+    bool GraphCutTensorCache::capture(ggml_cgraph* graph,
+                                      const ggml_graph_cut::Segment& segment,
+                                      const char* log_desc) {
+        size_t copied_bytes = 0;
+        size_t copied_count = 0;
+        for (int index : segment.output_node_indices) {
+            auto output = ggml_graph_node(graph, index);
+            if (!ggml_graph_cut::is_graph_cut_tensor(output) ||
+                !segment.future_cut_names.count(output->name)) {
+                continue;
+            }
+            auto entry = CachedTensor::copy(backend_, output->name, ggml_graph_cut::cache_source_tensor(output));
+            if (entry == nullptr) {
+                LOG_ERROR("%s failed to capture graph cut tensor: %s", log_desc, output->name);
+                return false;
+            }
+            const size_t size = ggml_backend_buffer_get_size(entry->buffer);
+            copied_bytes      = size > SIZE_MAX - copied_bytes ? SIZE_MAX : copied_bytes + size;
+            ++copied_count;
+            tensors_[output->name] = std::move(entry);
+        }
+        ggml_backend_synchronize(backend_);
+        if (copied_count > 0) {
+            LOG_DEBUG("%s graph cut cache added %6.2f MB (%zu tensors)",
+                      log_desc, copied_bytes / (1024.f * 1024.f), copied_count);
+        }
+        return true;
+    }
+}

src/core/runner_cache.h

@@ -0,0 +1,66 @@
+#ifndef __SD_CORE_RUNNER_CACHE_H__
+#define __SD_CORE_RUNNER_CACHE_H__
+
+#include <map>
+#include <memory>
+#include <string>
+#include <unordered_set>
+
+#include "ggml-backend.h"
+
+namespace sd::ggml_graph_cut {
+    struct Segment;
+}
+
+namespace sd {
+    struct CachedTensor {
+        ggml_context* context        = nullptr;
+        ggml_backend_buffer_t buffer = nullptr;
+        ggml_tensor* tensor          = nullptr;
+        ~CachedTensor();
+        static std::unique_ptr<CachedTensor> copy(ggml_backend_t backend,
+                                                  const std::string& name,
+                                                  ggml_tensor* source);
+    };
+    using CachedTensors = std::map<std::string, std::unique_ptr<CachedTensor>>;
+
+    class RunnerCache {
+        ggml_backend_t backend_;
+        CachedTensors committed_;
+        CachedTensors pending_;
+        std::map<std::string, ggml_tensor*> outputs_;
+
+    public:
+        explicit RunnerCache(ggml_backend_t backend)
+            : backend_(backend) {}
+        RunnerCache(const RunnerCache&)            = delete;
+        RunnerCache& operator=(const RunnerCache&) = delete;
+
+        ggml_tensor* get(const std::string& name) const;
+        void stage(const std::string& name, ggml_tensor* tensor);
+        const std::map<std::string, ggml_tensor*>& outputs() const { return outputs_; }
+        size_t pending_bytes(ggml_cgraph* graph) const;
+        size_t resident_bytes(ggml_backend_dev_t device) const;
+        bool capture(ggml_cgraph* graph);
+        void graph_end(bool success);
+        void clear();
+    };
+
+    class GraphCutTensorCache {
+        ggml_backend_t backend_;
+        CachedTensors tensors_;
+
+    public:
+        explicit GraphCutTensorCache(ggml_backend_t backend)
+            : backend_(backend) {}
+        ggml_tensor* get(const std::string& name) const;
+        size_t resident_bytes(ggml_backend_dev_t device) const;
+        size_t estimate_output_bytes(ggml_cgraph* graph,
+                                     const ggml_graph_cut::Segment& segment) const;
+        bool capture(ggml_cgraph* graph, const ggml_graph_cut::Segment& segment, const char* log_desc);
+        void prune(const std::unordered_set<std::string>& keep_names);
+        void clear() { tensors_.clear(); }
+    };
+}
+
+#endif  // __SD_CORE_RUNNER_CACHE_H__

src/core/segment_graph_bindings.cpp

@@ -0,0 +1,137 @@
+#include "core/segment_graph_bindings.h"
+
+#include <algorithm>
+#include <iterator>
+
+#include "core/ggml_graph_cut.h"
+#include "core/runner_cache.h"
+#include "core/util.h"
+#include "ggml.h"
+
+namespace sd {
+    SegmentGraphBindings::SegmentGraphBindings(GraphCutTensorCache& tensor_cache,
+                                               const ggml_graph_cut::Plan& plan,
+                                               ggml_cgraph* graph)
+        : tensor_cache_(tensor_cache),
+          graph_(graph) {
+        GGML_ASSERT(graph_ != nullptr);
+        for (int i = 0; i < ggml_graph_n_nodes(graph_); ++i) {
+            ggml_tensor* tensor = ggml_graph_node(graph_, i);
+            Topology topology{tensor->op, {}, tensor->view_src, tensor->flags};
+            std::copy(std::begin(tensor->src), std::end(tensor->src), topology.sources.begin());
+            topology_[tensor] = topology;
+        }
+        for (const auto& segment : plan.segments) {
+            for (const auto& input : segment.input_refs) {
+                if (input.type != ggml_graph_cut::Segment::INPUT_EXTERNAL) {
+                    continue;
+                }
+                ggml_tensor* tensor = ggml_graph_cut::input_tensor(graph_, input);
+                if (tensor == nullptr || tensor->buffer == nullptr) {
+                    continue;
+                }
+                external_bindings_[tensor] = {tensor->buffer, tensor->data, tensor->extra};
+            }
+        }
+    }
+
+    void SegmentGraphBindings::reset(const ggml_graph_cut::Segment& segment) {
+        restore();
+        for (const auto& input : segment.input_refs) {
+            ggml_tensor* tensor = ggml_graph_cut::input_tensor(graph_, input);
+            if (tensor == nullptr) {
+                continue;
+            }
+            switch (input.type) {
+                case ggml_graph_cut::Segment::INPUT_PREVIOUS_CUT:
+                    tensor->buffer = nullptr;
+                    tensor->data   = nullptr;
+                    tensor->extra  = nullptr;
+                    break;
+                case ggml_graph_cut::Segment::INPUT_EXTERNAL: {
+                    auto binding = external_bindings_.find(tensor);
+                    if (binding != external_bindings_.end()) {
+                        tensor->buffer = binding->second.buffer;
+                        tensor->data   = binding->second.data;
+                        tensor->extra  = binding->second.extra;
+                    } else {
+                        tensor->buffer = nullptr;
+                        tensor->data   = nullptr;
+                        tensor->extra  = nullptr;
+                    }
+                    break;
+                }
+                case ggml_graph_cut::Segment::INPUT_PARAM:
+                    break;
+            }
+        }
+
+        for (int node_index : segment.internal_node_indices) {
+            ggml_tensor* node = ggml_graph_node(graph_, node_index);
+            if (node == nullptr) {
+                continue;
+            }
+            node->buffer = nullptr;
+            node->data   = nullptr;
+            node->extra  = nullptr;
+        }
+    }
+
+    void SegmentGraphBindings::restore() {
+        for (const auto& entry : topology_) {
+            entry.first->op       = entry.second.op;
+            entry.first->view_src = entry.second.view_source;
+            entry.first->flags    = entry.second.flags;
+            std::copy(entry.second.sources.begin(), entry.second.sources.end(), std::begin(entry.first->src));
+        }
+    }
+
+    bool SegmentGraphBindings::bind_cached_inputs(
+        const ggml_graph_cut::Segment& segment,
+        const char* log_desc) {
+        std::unordered_map<ggml_tensor*, ggml_tensor*> cached_view_sources;
+        for (const auto& input : segment.input_refs) {
+            if (input.type != ggml_graph_cut::Segment::INPUT_PREVIOUS_CUT) {
+                continue;
+            }
+            ggml_tensor* input_tensor = ggml_graph_cut::input_tensor(graph_, input);
+            if (input_tensor == nullptr) {
+                continue;
+            }
+            ggml_tensor* cached_tensor = tensor_cache_.get(input.display_name);
+            if (cached_tensor == nullptr) {
+                LOG_ERROR("%s missing graph cut cache tensor: %s",
+                          log_desc,
+                          input.display_name.c_str());
+                return false;
+            }
+            if (input_tensor->view_src != nullptr) {
+                cached_view_sources[topology_.at(input_tensor).view_source] = cached_tensor;
+                input_tensor->view_src                                      = cached_tensor;
+                input_tensor->buffer                                        = nullptr;
+                input_tensor->data                                          = cached_tensor->data == nullptr
+                                                                                  ? nullptr
+                                                                                  : static_cast<void*>(static_cast<char*>(cached_tensor->data) +
+                                                              input_tensor->view_offs);
+                input_tensor->extra                                         = cached_tensor->extra;
+            } else {
+                input_tensor->buffer = cached_tensor->buffer;
+                input_tensor->data   = cached_tensor->data;
+                input_tensor->extra  = cached_tensor->extra;
+            }
+            for (int source_index = 0; source_index < GGML_MAX_SRC; ++source_index) {
+                input_tensor->src[source_index] = nullptr;
+            }
+            input_tensor->op = GGML_OP_NONE;
+        }
+        // ggml flattens view chains, so descendants also need the cached root.
+        for (int node_index : segment.internal_node_indices) {
+            ggml_tensor* node  = ggml_graph_node(graph_, node_index);
+            auto cached_source = cached_view_sources.find(topology_.at(node).view_source);
+            if (cached_source != cached_view_sources.end()) {
+                node->view_src = cached_source->second;
+            }
+        }
+        return true;
+    }
+}

src/core/segment_graph_bindings.h

@@ -0,0 +1,52 @@
+#ifndef __SD_CORE_SEGMENT_GRAPH_BINDINGS_H__
+#define __SD_CORE_SEGMENT_GRAPH_BINDINGS_H__
+
+#include <array>
+#include <unordered_map>
+#include "ggml.h"
+
+struct ggml_backend_buffer;
+struct ggml_cgraph;
+struct ggml_tensor;
+
+namespace sd {
+    class GraphCutTensorCache;
+
+    namespace ggml_graph_cut {
+        struct Plan;
+        struct Segment;
+    }
+
+    class SegmentGraphBindings {
+    public:
+        SegmentGraphBindings(GraphCutTensorCache& tensor_cache,
+                             const ggml_graph_cut::Plan& plan,
+                             ggml_cgraph* graph);
+
+        void reset(const ggml_graph_cut::Segment& segment);
+        void restore();
+        ~SegmentGraphBindings() { restore(); }
+        bool bind_cached_inputs(const ggml_graph_cut::Segment& segment,
+                                const char* log_desc);
+
+    private:
+        struct ExternalBinding {
+            ggml_backend_buffer* buffer = nullptr;
+            void* data                  = nullptr;
+            void* extra                 = nullptr;
+        };
+
+        GraphCutTensorCache& tensor_cache_;
+        ggml_cgraph* graph_ = nullptr;
+        std::unordered_map<ggml_tensor*, ExternalBinding> external_bindings_;
+        struct Topology {
+            ggml_op op;
+            std::array<ggml_tensor*, GGML_MAX_SRC> sources;
+            ggml_tensor* view_source;
+            int flags;
+        };
+        std::unordered_map<ggml_tensor*, Topology> topology_;
+    };
+}
+
+#endif  // __SD_CORE_SEGMENT_GRAPH_BINDINGS_H__

src/core/segment_weight_pipeline.cpp

@@ -0,0 +1,205 @@
+#include "core/segment_weight_pipeline.h"
+
+#include <utility>
+
+#include "core/ggml_graph_cut.h"
+#include "device_residency_manager.h"
+
+namespace sd {
+    static ggml_tensor* canonical_param(
+        ggml_tensor* tensor,
+        const std::unordered_set<const ggml_tensor*>& params) {
+        for (ggml_tensor* current = tensor; current != nullptr; current = current->view_src) {
+            if (params.find(current) != params.end()) {
+                return current;
+            }
+        }
+        return nullptr;
+    }
+
+    SegmentWeightPipeline::SegmentWeightPipeline(
+        const std::shared_ptr<DeviceResidencyManager>& residency_manager,
+        ggml_backend_t compute_backend,
+        uintptr_t owner_id,
+        ggml_cgraph* graph,
+        const ggml_graph_cut::Plan& plan,
+        const std::unordered_set<const ggml_tensor*>& params,
+        bool enabled)
+        : residency_manager_(residency_manager),
+          compute_backend_(compute_backend),
+          owner_id_(owner_id),
+          enabled_(enabled && residency_manager != nullptr) {
+        segment_params_.resize(plan.segments.size());
+        for (size_t segment_index = 0; segment_index < plan.segments.size(); ++segment_index) {
+            std::unordered_set<ggml_tensor*> seen;
+            for (ggml_tensor* tensor :
+                 ggml_graph_cut::param_tensors(graph, plan.segments[segment_index])) {
+                ggml_tensor* param = canonical_param(tensor, params);
+                if (param != nullptr && seen.insert(param).second) {
+                    segment_params_[segment_index].push_back(param);
+                }
+            }
+        }
+    }
+
+    SegmentWeightPipeline::~SegmentWeightPipeline() {
+        segment_end();
+        clear();
+    }
+
+    size_t SegmentWeightPipeline::next_parameter_segment(size_t segment_index) const {
+        for (size_t next = segment_index + 1; next < segment_params_.size(); ++next) {
+            if (!segment_params_[next].empty()) {
+                return next;
+            }
+        }
+        return SIZE_MAX;
+    }
+
+    std::vector<std::vector<ggml_tensor*>> SegmentWeightPipeline::preferred_eviction_order() const {
+        return {segment_params_.rbegin(), segment_params_.rend()};
+    }
+
+    void SegmentWeightPipeline::disable() {
+        clear();
+        enabled_ = false;
+    }
+
+    void SegmentWeightPipeline::activate(size_t segment_index) {
+        if (!enabled_ || queued_segment_ == SIZE_MAX || queued_segment_ != segment_index) {
+            return;
+        }
+
+        auto manager = residency_manager_.lock();
+        if (manager == nullptr ||
+            !manager->activate_prefetched_params(owner_id_, queued_params_)) {
+            disable();
+            return;
+        }
+        queued_params_.clear();
+        queued_segment_ = SIZE_MAX;
+    }
+
+    bool SegmentWeightPipeline::ensure_segment_capacity(
+        size_t segment_index,
+        const std::vector<DeviceMemoryRequest>& requests) {
+        if (segment_index >= segment_params_.size()) {
+            return false;
+        }
+        auto manager = residency_manager_.lock();
+        if (manager == nullptr) {
+            return segment_params_[segment_index].empty();
+        }
+        std::vector<ggml_tensor*> protected_params = segment_params_[segment_index];
+        protected_params.insert(protected_params.end(), queued_params_.begin(), queued_params_.end());
+        for (const auto& request : requests) {
+            if (!manager->ensure_compute_backend_capacity(request, segment_params_[segment_index],
+                                                          preferred_eviction_order(), protected_params)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    bool SegmentWeightPipeline::segment_start(size_t segment_index, const std::function<bool()>& ensure_capacity) {
+        GGML_ASSERT(pinned_params_.empty());
+        activate(segment_index);
+        if (!ensure_capacity()) {
+            return false;
+        }
+        auto manager = residency_manager_.lock();
+        if (manager == nullptr) {
+            return segment_params_[segment_index].empty();
+        }
+        if (!manager->prepare_params(segment_params_[segment_index])) {
+            return false;
+        }
+        pinned_params_ = segment_params_[segment_index];
+        return true;
+    }
+
+    void SegmentWeightPipeline::segment_end() {
+        if (auto manager = residency_manager_.lock()) {
+            manager->release_compute_backend_params(pinned_params_);
+        }
+        pinned_params_.clear();
+    }
+
+    void SegmentWeightPipeline::enqueue_next(
+        size_t segment_index,
+        const DeviceMemoryRequest& request) {
+        if (!enabled_ || queued_segment_ != SIZE_MAX) {
+            return;
+        }
+
+        const size_t next_segment = next_parameter_segment(segment_index);
+        if (next_segment == SIZE_MAX) {
+            return;
+        }
+
+        std::unordered_set<ggml_tensor*> active_params(
+            segment_params_[segment_index].begin(),
+            segment_params_[segment_index].end());
+        std::vector<ggml_tensor*> params;
+        params.reserve(segment_params_[next_segment].size());
+        for (ggml_tensor* param : segment_params_[next_segment]) {
+            if (active_params.find(param) == active_params.end()) {
+                params.push_back(param);
+            }
+        }
+        if (params.empty()) {
+            return;
+        }
+
+        auto manager = residency_manager_.lock();
+        if (manager == nullptr) {
+            disable();
+            return;
+        }
+        const WeightResidencyInfo residency =
+            manager->inspect_compute_backend_params(params);
+        if (residency.missing_bytes == 0) {
+            return;
+        }
+        if (!residency.async_prefetch_supported) {
+            disable();
+            return;
+        }
+
+        DeviceMemoryRequest backend_request        = request;
+        backend_request.compute_backend            = compute_backend_;
+        backend_request.owner_id                   = owner_id_;
+        std::vector<ggml_tensor*> protected_params = segment_params_[segment_index];
+        protected_params.insert(protected_params.end(), params.begin(), params.end());
+        if (!manager->ensure_compute_backend_capacity(backend_request,
+                                                      params,
+                                                      preferred_eviction_order(),
+                                                      protected_params)) {
+            disable();
+            return;
+        }
+        switch (manager->prefetch_params(owner_id_, params)) {
+            case WeightPrefetchResult::Scheduled:
+                queued_params_  = std::move(params);
+                queued_segment_ = next_segment;
+                return;
+            case WeightPrefetchResult::AlreadyResident:
+                return;
+            case WeightPrefetchResult::Unsupported:
+                disable();
+                return;
+            case WeightPrefetchResult::Failed:
+                disable();
+                return;
+        }
+        disable();
+    }
+
+    void SegmentWeightPipeline::clear() {
+        if (auto manager = residency_manager_.lock()) {
+            manager->clear_prefetched_params(owner_id_);
+        }
+        queued_params_.clear();
+        queued_segment_ = SIZE_MAX;
+    }
+}

src/core/segment_weight_pipeline.h

@@ -0,0 +1,62 @@
+#ifndef __SD_CORE_SEGMENT_WEIGHT_PIPELINE_H__
+#define __SD_CORE_SEGMENT_WEIGHT_PIPELINE_H__
+
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <unordered_set>
+#include <vector>
+
+#include "ggml-backend.h"
+
+struct DeviceMemoryRequest;
+struct DeviceResidencyManager;
+struct ggml_cgraph;
+struct ggml_tensor;
+
+namespace sd::ggml_graph_cut {
+    struct Plan;
+}
+
+namespace sd {
+    class SegmentWeightPipeline {
+    private:
+        std::weak_ptr<DeviceResidencyManager> residency_manager_;
+        ggml_backend_t compute_backend_ = nullptr;
+        uintptr_t owner_id_             = 0;
+        std::vector<std::vector<ggml_tensor*>> segment_params_;
+        std::vector<ggml_tensor*> queued_params_;
+        std::vector<ggml_tensor*> pinned_params_;
+        size_t queued_segment_ = SIZE_MAX;
+        bool enabled_          = true;
+
+        size_t next_parameter_segment(size_t segment_index) const;
+        std::vector<std::vector<ggml_tensor*>> preferred_eviction_order() const;
+        void disable();
+        void activate(size_t segment_index);
+        void clear();
+
+    public:
+        SegmentWeightPipeline(
+            const std::shared_ptr<DeviceResidencyManager>& residency_manager,
+            ggml_backend_t compute_backend,
+            uintptr_t owner_id,
+            ggml_cgraph* graph,
+            const ggml_graph_cut::Plan& plan,
+            const std::unordered_set<const ggml_tensor*>& params,
+            bool enabled = true);
+        ~SegmentWeightPipeline();
+
+        const std::vector<ggml_tensor*>& params(size_t index) const { return segment_params_[index]; }
+        bool ensure_segment_capacity(size_t segment_index,
+                                     const std::vector<DeviceMemoryRequest>& requests);
+        bool segment_start(size_t segment_index, const std::function<bool()>& ensure_capacity);
+        void segment_end();
+        // Prefetch is best effort; segment_start falls back to synchronous loading.
+        void enqueue_next(size_t segment_index,
+                          const DeviceMemoryRequest& request);
+    };
+}
+
+#endif  // __SD_CORE_SEGMENT_WEIGHT_PIPELINE_H__

src/core/tensor_ggml.hpp

@@ -9,6 +9,7 @@
 #include <type_traits>
 
 #include "core/tensor.hpp"
+#include "ggml-backend.h"
 #include "ggml.h"
 
 namespace sd {
@@ -54,10 +55,28 @@ namespace sd {
             GGML_ABORT("ggml tensor type does not match sd::Tensor type");
         }
         Tensor<T> result(shape_from_ggml(tensor));
-        if (tensor->buffer != nullptr) {
-            ggml_backend_tensor_get(tensor, result.data(), 0, ggml_nbytes(tensor));
+        std::vector<uint8_t> strided_data;
+        void* destination = result.data();
+        if (!ggml_is_contiguous(tensor)) {
+            strided_data.resize(ggml_nbytes(tensor));
+            destination = strided_data.data();
+        }
+        auto buffer = tensor->view_src != nullptr ? tensor->view_src->buffer : tensor->buffer;
+        if (buffer != nullptr) {
+            ggml_backend_tensor_get(tensor, destination, 0, ggml_nbytes(tensor));
         } else {
-            std::memcpy(result.data(), tensor->data, ggml_nbytes(tensor));
+            std::memcpy(destination, tensor->data, ggml_nbytes(tensor));
+        }
+        if (!strided_data.empty()) {
+            for (int64_t i = 0; i < result.numel(); ++i) {
+                int64_t index = i;
+                size_t offset = 0;
+                for (int d = 0; d < GGML_MAX_DIMS; ++d) {
+                    offset += static_cast<size_t>(index % tensor->ne[d]) * tensor->nb[d];
+                    index /= tensor->ne[d];
+                }
+                std::memcpy(result.data() + i, strided_data.data() + offset, sizeof(T));
+            }
         }
         return result;
     }

src/core/util.h

@@ -105,6 +105,7 @@ void* sd_get_backend_eval_callback_data();
 bool sd_backend_is(ggml_backend_t backend, const std::string& name);
 
 #define LOG_DEBUG(format, ...) log_printf(SD_LOG_DEBUG, __FILE__, __LINE__, format, ##__VA_ARGS__)
+#define LOG_VERBOSE(format, ...) log_printf(SD_LOG_VERBOSE, __FILE__, __LINE__, format, ##__VA_ARGS__)
 #define LOG_INFO(format, ...) log_printf(SD_LOG_INFO, __FILE__, __LINE__, format, ##__VA_ARGS__)
 #define LOG_WARN(format, ...) log_printf(SD_LOG_WARN, __FILE__, __LINE__, format, ##__VA_ARGS__)
 #define LOG_ERROR(format, ...) log_printf(SD_LOG_ERROR, __FILE__, __LINE__, format, ##__VA_ARGS__)

src/detailer.cpp

@@ -715,7 +715,7 @@ std::vector<ADetailerDetection> ADetailerGGML::predict(sd_image_t image,
     LetterboxInput input  = make_letterbox_input(image, params.input_size);
     int64_t start         = ggml_time_ms();
     sd::Tensor<float> raw = detector->compute(n_threads, input.tensor);
-    detector->free_compute_buffer();
+    detector->runner_end();
     if (raw.empty()) {
         LOG_ERROR("YOLOv8 detector inference failed");
         return {};

src/device_residency_manager.h

@@ -0,0 +1,76 @@
+#ifndef __DEVICE_RESIDENCY_MANAGER_H__
+#define __DEVICE_RESIDENCY_MANAGER_H__
+
+#include <cstdint>
+#include <functional>
+#include <vector>
+
+#include "ggml-backend.h"
+
+struct ggml_tensor;
+
+enum class WeightPrefetchResult {
+    Scheduled,
+    AlreadyResident,
+    Unsupported,
+    Failed,
+};
+
+struct WeightResidencyInfo {
+    bool async_prefetch_supported = false;
+    size_t missing_bytes          = 0;
+};
+
+struct DeviceMemoryRequest {
+    ggml_backend_t compute_backend  = nullptr;
+    uintptr_t owner_id              = 0;
+    size_t pending_allocation_bytes = 0;
+    size_t runtime_resident_bytes   = 0;
+    size_t max_backend_bytes        = 0;
+
+    // Runtime buffers only; the manager accounts for weights separately.
+    size_t runtime_peak_bytes() const {
+        return pending_allocation_bytes > SIZE_MAX - runtime_resident_bytes
+                   ? SIZE_MAX
+                   : runtime_resident_bytes + pending_allocation_bytes;
+    }
+};
+
+struct DeviceResidencyManager {
+    virtual ~DeviceResidencyManager() = default;
+
+    virtual bool segmented_compute_enabled() const                                          = 0;
+    virtual bool prefetch_enabled() const                                                   = 0;
+    virtual void set_workspace_reclaimer(uintptr_t owner_id, std::function<bool()> reclaim) = 0;
+    virtual void remove_runtime_owner(uintptr_t owner_id)                                   = 0;
+    // Capacity requests select their backend's weights; protection spans all backends.
+    virtual bool fits_compute_backend_capacity(const DeviceMemoryRequest& request,
+                                               const std::vector<ggml_tensor*>& required_params) const = 0;
+    virtual bool assign_compute_backend(const std::vector<ggml_tensor*>& tensors,
+                                        ggml_backend_t compute_backend)                                = 0;
+    virtual bool prepare_params(const std::vector<ggml_tensor*>& tensors)                              = 0;
+    virtual void release_compute_backend_params(const std::vector<ggml_tensor*>& tensors)              = 0;
+    virtual void evict_compute_backend_params(const std::vector<ggml_tensor*>& tensors)                = 0;
+    virtual WeightResidencyInfo inspect_compute_backend_params(
+        const std::vector<ggml_tensor*>& tensors) const          = 0;
+    virtual void update_runtime_residency(uintptr_t owner_id,
+                                          ggml_backend_t compute_backend,
+                                          size_t resident_bytes) = 0;
+    virtual bool ensure_compute_backend_capacity(
+        const DeviceMemoryRequest& request,
+        const std::vector<ggml_tensor*>& required_params,
+        const std::vector<std::vector<ggml_tensor*>>& preferred_eviction_order,
+        const std::vector<ggml_tensor*>& protected_params) = 0;
+    virtual WeightPrefetchResult prefetch_params(
+        uintptr_t owner_id,
+        const std::vector<ggml_tensor*>& tensors)                                     = 0;
+    virtual bool activate_prefetched_params(uintptr_t owner_id,
+                                            const std::vector<ggml_tensor*>& tensors) = 0;
+    virtual void clear_prefetched_params(uintptr_t owner_id)                          = 0;
+};
+
+// Transitional alias for model constructors that have not yet adopted the
+// residency-oriented name. It does not introduce a second implementation.
+using RunnerWeightManager = DeviceResidencyManager;
+
+#endif  // __DEVICE_RESIDENCY_MANAGER_H__

src/extensions/generation_extension.h

@@ -49,7 +49,7 @@ struct GenerationExtension {
     virtual void get_param_tensors(std::map<std::string, ggml_tensor*>&) {}
     virtual void collect_loras(std::vector<ModelManager::LoraSpec>&) {}
     virtual void add_ignore_tensors(std::set<std::string>&) const {}
-    virtual void runner_done() {}
+    virtual void runner_end() {}
     virtual void reset_runtime_condition() {}
     virtual bool prepare_condition(GenerationExtensionConditionContext&) {
         return false;

src/extensions/photomaker_extension.cpp

@@ -175,9 +175,9 @@ struct PhotoMakerExtension : public GenerationExtension {
         ignore_tensors.insert("pmid.unet.");
     }
 
-    void runner_done() override {
+    void runner_end() override {
         if (pmid_model != nullptr) {
-            pmid_model->runner_done();
+            pmid_model->runner_end();
         }
     }
 

src/model/adapter/ip_adapter.hpp

@@ -200,7 +200,7 @@ namespace IPAdapter {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_graph(image_embeds);
             };
-            return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, true, true, true));
+            return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, true));
         }
     };
 

src/model/adapter/lora.hpp

@@ -120,18 +120,17 @@ struct LoraModel : public GGMLRunner {
             return false;
         }
 
-        LOG_DEBUG("finished loaded lora");
+        LOG_VERBOSE("finished loaded lora");
         return true;
     }
 
     void release_loaded_tensors() {
-        runner_done();
-        free_compute_buffer();
+        runner_end();
         model_manager.reset();
         free_params_ctx();
         alloc_params_ctx();
-        model_manager  = std::make_shared<ModelManager>();
-        weight_manager = model_manager;
+        model_manager     = std::make_shared<ModelManager>();
+        residency_manager = model_manager;
         lora_tensors.clear();
         original_tensor_to_final_tensor.clear();
         applied_lora_tensors.clear();
@@ -243,7 +242,7 @@ struct LoraModel : public GGMLRunner {
                 if (iter != lora_tensors.end()) {
                     float alpha = ggml_ext_backend_tensor_get_f32(iter->second);
                     scale_value = alpha / rank;
-                    // LOG_DEBUG("rank %s %ld %.2f %.2f", alpha_name.c_str(), rank, alpha, scale_value);
+                    // LOG_VERBOSE("rank %s %ld %.2f %.2f", alpha_name.c_str(), rank, alpha, scale_value);
                     applied_lora_tensors.insert(alpha_name);
                 }
             }
@@ -799,7 +798,7 @@ struct LoraModel : public GGMLRunner {
                     float alpha       = ggml_ext_backend_tensor_get_f32(iter->second);
                     scale_value       = alpha / rank;
                     scale_tensor_name = alpha_name;
-                    // LOG_DEBUG("rank %s %ld %.2f %.2f", alpha_name.c_str(), rank, alpha, scale_value);
+                    // LOG_VERBOSE("rank %s %ld %.2f %.2f", alpha_name.c_str(), rank, alpha, scale_value);
                 }
             }
             scale_value *= multiplier;
@@ -952,16 +951,19 @@ struct LoraModel : public GGMLRunner {
         auto get_graph = [&]() -> ggml_cgraph* {
             return build_lora_graph(model_tensors, model_tensor_names, version);
         };
-        GGMLRunner::compute<float>(get_graph, n_threads, false, false, false, true);
-        stat(!warn_unused);
-        for (auto item : original_tensor_to_final_tensor) {
-            ggml_tensor* original_tensor = item.first;
-            ggml_tensor* final_tensor    = item.second;
-
-            ggml_backend_tensor_copy(final_tensor, original_tensor);
+        auto read_outputs = [&]() {
+            for (const auto& item : original_tensor_to_final_tensor) {
+                ggml_backend_tensor_copy(item.second, item.first);
+            }
+            return true;
+        };
+        auto result = GGMLRunner::compute<float>(get_graph, n_threads, false, true, read_outputs);
+        if (!result.has_value()) {
+            LOG_ERROR("LoRA graph execution failed");
         }
+        stat(!warn_unused);
         original_tensor_to_final_tensor.clear();
-        GGMLRunner::free_compute_buffer();
+        runner_end();
     }
 
     void apply(std::map<std::string, ggml_tensor*> model_tensors, SDVersion version, int n_threads, bool warn_unused = true) {

src/model/adapter/pmid.hpp

@@ -558,7 +558,7 @@ public:
             return build_graph(id_pixel_values, prompt_embeds, class_tokens_mask, id_embeds);
         };
 
-        return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, true, true, true));
+        return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, true));
     }
 };
 
@@ -639,7 +639,7 @@ struct PhotoMakerIDEmbed : public GGMLRunner {
             return false;
         }
 
-        LOG_DEBUG("finished loading PhotoMaker ID Embeds ");
+        LOG_VERBOSE("finished loading PhotoMaker ID Embeds ");
         return true;
     }
 

src/model/common/block.hpp

@@ -268,10 +268,11 @@ public:
                 int64_t dim_out,
                 int64_t mult          = 4,
                 Activation activation = Activation::GEGLU,
-                bool precision_fix    = false) {
+                bool precision_fix    = false,
+                bool bias             = true) {
         int64_t inner_dim = dim * mult;
         if (activation == Activation::GELU) {
-            blocks["net.0"] = std::shared_ptr<GGMLBlock>(new GELU(dim, inner_dim));
+            blocks["net.0"] = std::shared_ptr<GGMLBlock>(new GELU(dim, inner_dim, bias));
         } else {
             blocks["net.0"] = std::shared_ptr<GGMLBlock>(new GEGLU(dim, inner_dim));
         }
@@ -285,7 +286,7 @@ public:
         // The purpose of the scale here is to prevent NaN issues in certain situations.
         // For example, when using Vulkan without enabling force_prec_f32,
         // or when using CUDA but the weights are k-quants.
-        blocks["net.2"] = std::shared_ptr<GGMLBlock>(new Linear(inner_dim, dim_out, true, false, force_prec_f32, scale));
+        blocks["net.2"] = std::shared_ptr<GGMLBlock>(new Linear(inner_dim, dim_out, bias, false, force_prec_f32, scale));
     }
 
     ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
@@ -339,7 +340,7 @@ public:
           enable_ip(enable_ip) {
         int64_t inner_dim = d_head * n_head;
         if (context_dim == 320 && d_head == 320) {
-            // LOG_DEBUG("CrossAttention: temp set dim to 1024 for sdxs_09");
+            // LOG_VERBOSE("CrossAttention: temp set dim to 1024 for sdxs_09");
             xtra_dim    = true;
             context_dim = 1024;
         }
@@ -369,7 +370,7 @@ public:
 
         auto q = to_q->forward(ctx, x);  // [N, n_token, inner_dim]
         if (xtra_dim) {
-            // LOG_DEBUG("CrossAttention: temp set dim to 1024 for sdxs_09");
+            // LOG_VERBOSE("CrossAttention: temp set dim to 1024 for sdxs_09");
             context->ne[0] = 1024;  // patch dim
         }
         auto k = to_k->forward(ctx, context);  // [N, n_context, inner_dim]

src/model/detector/yolov8.h

@@ -68,12 +68,12 @@ struct YOLOv8Config {
         }
 
         if (config.valid) {
-            LOG_DEBUG("yolov8: classes=%d, reg_max=%d, p3=%d, p4=%d, p5=%d",
-                      config.num_classes,
-                      config.reg_max,
-                      config.out_channels[15],
-                      config.out_channels[18],
-                      config.out_channels[21]);
+            LOG_VERBOSE("yolov8: classes=%d, reg_max=%d, p3=%d, p4=%d, p5=%d",
+                        config.num_classes,
+                        config.reg_max,
+                        config.out_channels[15],
+                        config.out_channels[18],
+                        config.out_channels[21]);
         }
         return config;
     }
@@ -355,7 +355,7 @@ struct YOLOv8Runner : public GGMLRunner {
 
     sd::Tensor<float> compute(int n_threads, const sd::Tensor<float>& input) {
         auto get_graph = [&]() { return build_graph(input); };
-        return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false));
+        return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, false));
     }
 };
 

src/model/diffusion/anima.hpp

@@ -46,11 +46,11 @@ namespace Anima {
             }
             if (detected_layers > 0) {
                 config.num_layers = detected_layers;
-                LOG_DEBUG("anima: num_layers = %" PRId64 ", hidden_size = %" PRId64 ", num_heads = %" PRId64 ", head_dim = %" PRId64,
-                          config.num_layers,
-                          config.hidden_size,
-                          config.num_heads,
-                          config.head_dim);
+                LOG_VERBOSE("anima: num_layers = %" PRId64 ", hidden_size = %" PRId64 ", num_heads = %" PRId64 ", head_dim = %" PRId64,
+                            config.num_layers,
+                            config.hidden_size,
+                            config.num_heads,
+                            config.head_dim);
             }
             return config;
         }
@@ -717,7 +717,7 @@ namespace Anima {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_graph(x, timesteps, context, t5_ids, t5_weights, ref_latents);
             };
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
 
         sd::Tensor<float> compute(int n_threads,

src/model/diffusion/boogu.hpp

@@ -109,16 +109,16 @@ namespace Boogu {
             }
             config.timestep_embed_dim = std::min<int64_t>(config.hidden_size, 1024);
 
-            LOG_DEBUG("boogu_image: layers=%" PRId64 ", double_stream_layers=%" PRId64 ", refiner_layers=%" PRId64 ", hidden=%" PRId64 ", heads=%" PRId64 ", kv_heads=%" PRId64 ", head_dim=%" PRId64 ", in_channels=%" PRId64 ", out_channels=%" PRId64,
-                      config.num_layers,
-                      config.num_double_stream_layers,
-                      config.num_refiner_layers,
-                      config.hidden_size,
-                      config.num_attention_heads,
-                      config.num_kv_heads,
-                      config.head_dim,
-                      config.in_channels,
-                      config.out_channels);
+            LOG_VERBOSE("boogu_image: layers=%" PRId64 ", double_stream_layers=%" PRId64 ", refiner_layers=%" PRId64 ", hidden=%" PRId64 ", heads=%" PRId64 ", kv_heads=%" PRId64 ", head_dim=%" PRId64 ", in_channels=%" PRId64 ", out_channels=%" PRId64,
+                        config.num_layers,
+                        config.num_double_stream_layers,
+                        config.num_refiner_layers,
+                        config.hidden_size,
+                        config.num_attention_heads,
+                        config.num_kv_heads,
+                        config.head_dim,
+                        config.in_channels,
+                        config.out_channels);
             return config;
         }
     };
@@ -815,7 +815,7 @@ namespace Boogu {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_graph(x, timesteps, context, ref_latents);
             };
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
 
         sd::Tensor<float> compute(int n_threads,

src/model/diffusion/control.hpp

@@ -423,19 +423,26 @@ struct ControlNet : public GGMLRunner {
             return build_graph(x, hint, timesteps, context, y);
         };
 
-        auto compute_result = GGMLRunner::compute<float>(get_graph, n_threads, false, false, false, true);
+        auto read_outputs = [&]() {
+            controls.clear();
+            controls.reserve(control_outputs_ggml.size());
+            for (ggml_tensor* control : control_outputs_ggml) {
+                auto control_host = restore_trailing_singleton_dims(sd::make_sd_tensor_from_ggml<float>(control), 4);
+                if (control_host.empty()) {
+                    return false;
+                }
+                controls.push_back(std::move(control_host));
+            }
+            return true;
+        };
+        auto compute_result = GGMLRunner::compute<float>(get_graph, n_threads, false, true, read_outputs);
+        control_outputs_ggml.clear();
+        guided_hint_output_ggml = nullptr;
         if (!compute_result.has_value()) {
+            controls.clear();
             return std::nullopt;
         }
-
         guided_hint_cached = get_cache_tensor_by_name(guided_hint_cache_name()) != nullptr;
-        controls.clear();
-        controls.reserve(control_outputs_ggml.size());
-        for (ggml_tensor* control : control_outputs_ggml) {
-            auto control_host = restore_trailing_singleton_dims(sd::make_sd_tensor_from_ggml<float>(control), 4);
-            GGML_ASSERT(!control_host.empty());
-            controls.push_back(std::move(control_host));
-        }
         return controls;
     }
 
@@ -444,10 +451,10 @@ struct ControlNet : public GGMLRunner {
         std::map<std::string, ggml_tensor*> tensors;
         control_net.get_param_tensors(tensors);
 
-        auto manager = std::dynamic_pointer_cast<ModelManager>(weight_manager.lock());
+        auto manager = std::dynamic_pointer_cast<ModelManager>(residency_manager.lock());
         if (manager == nullptr) {
             owned_model_manager = std::make_shared<ModelManager>();
-            weight_manager      = owned_model_manager;
+            residency_manager   = owned_model_manager;
             manager             = owned_model_manager;
         }
 

src/model/diffusion/ernie_image.hpp

@@ -72,13 +72,13 @@ namespace ErnieImage {
             for (int axis_dim : config.axes_dim) {
                 config.axes_dim_sum += axis_dim;
             }
-            LOG_DEBUG("ernie_image: num_layers = %" PRId64 ", hidden_size = %" PRId64 ", num_heads = %" PRId64 ", ffn_hidden_size = %" PRId64 ", in_channels = %" PRId64 ", out_channels = %" PRId64,
-                      config.num_layers,
-                      config.hidden_size,
-                      config.num_heads,
-                      config.ffn_hidden_size,
-                      config.in_channels,
-                      config.out_channels);
+            LOG_VERBOSE("ernie_image: num_layers = %" PRId64 ", hidden_size = %" PRId64 ", num_heads = %" PRId64 ", ffn_hidden_size = %" PRId64 ", in_channels = %" PRId64 ", out_channels = %" PRId64,
+                        config.num_layers,
+                        config.hidden_size,
+                        config.num_heads,
+                        config.ffn_hidden_size,
+                        config.in_channels,
+                        config.out_channels);
             return config;
         }
     };
@@ -440,7 +440,7 @@ namespace ErnieImage {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_graph(x, timesteps, context);
             };
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
 
         sd::Tensor<float> compute(int n_threads,

src/model/diffusion/flux.hpp

@@ -123,16 +123,16 @@ namespace Flux {
                     config.guidance_embed = true;
                 }
                 if (name.find("__x0__") != std::string::npos) {
-                    LOG_DEBUG("using x0 prediction");
+                    LOG_VERBOSE("using x0 prediction");
                     config.chroma_radiance_params.use_x0 = true;
                 }
                 if (name.find("__32x32__") != std::string::npos) {
-                    LOG_DEBUG("using patch size 32");
+                    LOG_VERBOSE("using patch size 32");
                     config.patch_size = 32;
                 }
                 if (name.find("img_in_patch.weight") != std::string::npos) {
                     actual_radiance_patch_size = tensor_storage.ne[0];
-                    LOG_DEBUG("actual radiance patch size: %" PRId64, actual_radiance_patch_size);
+                    LOG_VERBOSE("actual radiance patch size: %" PRId64, actual_radiance_patch_size);
                 }
                 if (name.find("distilled_guidance_layer.in_proj.weight") != std::string::npos) {
                     config.is_chroma = true;
@@ -169,7 +169,7 @@ namespace Flux {
             }
             if (actual_radiance_patch_size > 0 && actual_radiance_patch_size != config.patch_size) {
                 GGML_ASSERT(config.patch_size == 2 * actual_radiance_patch_size);
-                LOG_DEBUG("using fake x2 patch size");
+                LOG_VERBOSE("using fake x2 patch size");
                 config.chroma_radiance_params.fake_patch_size_x2 = true;
             }
             if (head_dim > 0) {
@@ -179,13 +179,13 @@ namespace Flux {
             for (int axis_dim : config.axes_dim) {
                 config.axes_dim_sum += axis_dim;
             }
-            LOG_DEBUG("flux: depth = %d, depth_single_blocks = %d, guidance_embed = %s, context_in_dim = %" PRId64 ", hidden_size = %" PRId64 ", num_heads = %d",
-                      config.depth,
-                      config.depth_single_blocks,
-                      config.guidance_embed ? "true" : "false",
-                      config.context_in_dim,
-                      config.hidden_size,
-                      config.num_heads);
+            LOG_VERBOSE("flux: depth = %d, depth_single_blocks = %d, guidance_embed = %s, context_in_dim = %" PRId64 ", hidden_size = %" PRId64 ", num_heads = %d",
+                        config.depth,
+                        config.depth_single_blocks,
+                        config.guidance_embed ? "true" : "false",
+                        config.context_in_dim,
+                        config.hidden_size,
+                        config.num_heads);
             return config;
         }
     };
@@ -1560,7 +1560,7 @@ namespace Flux {
                                             config.axes_dim,
                                             sd_version_is_longcat(version));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
-            // LOG_DEBUG("pos_len %d", pos_len);
+            // LOG_VERBOSE("pos_len %d", pos_len);
             auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             // pe->data = pe_vec.data();
             // print_ggml_tensor(pe);
@@ -1626,7 +1626,7 @@ namespace Flux {
                 return build_graph(x, timesteps, context, c_concat, y, guidance, ref_latents, ref_index_mode, skip_layers, pulid_id, pulid_id_weight);
             };
 
-            auto result = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            auto result = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
             return result;
         }
 
@@ -1702,7 +1702,7 @@ namespace Flux {
                 GGML_ASSERT(!out_opt.empty());
                 out = std::move(out_opt);
                 print_sd_tensor(out);
-                LOG_DEBUG("flux test done in %lldms", t1 - t0);
+                LOG_VERBOSE("flux test done in %lldms", t1 - t0);
             }
         }
 

src/model/diffusion/hidream_o1.hpp

@@ -325,13 +325,11 @@ namespace HiDreamO1 {
 
         sd::Tensor<float> compute(int n_threads,
                                   const sd::Tensor<float>& image,
-                                  bool auto_free           = true,
-                                  bool free_compute_buffer = true,
-                                  bool free_compute_params = true) {
+                                  bool auto_runner_end = true) {
             auto get_graph = [&]() {
                 return build_graph(image);
             };
-            auto output = GGMLRunner::compute<float>(get_graph, n_threads, auto_free, free_compute_buffer, free_compute_params);
+            auto output = GGMLRunner::compute<float>(get_graph, n_threads, auto_runner_end);
             return output.has_value() ? std::move(output.value()) : sd::Tensor<float>();
         }
     };
@@ -459,7 +457,7 @@ namespace HiDreamO1 {
             auto get_graph = [&]() {
                 return build_graph(x, timestep, input_ids, input_pos, token_types, vinput_mask, image_embeds, ref_images);
             };
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
 
         sd::Tensor<float> compute(int n_threads,
@@ -510,8 +508,8 @@ namespace HiDreamO1 {
             vision_runner->set_weight_adapter(adapter);
         }
 
-        void runner_done() override {
-            vision_runner->runner_done();
+        void runner_end() override {
+            vision_runner->runner_end();
         }
 
         SDCondition get_learned_condition(int n_threads,
@@ -659,7 +657,7 @@ namespace HiDreamO1 {
             result.c_vinput_mask  = sd::Tensor<int32_t>(vinput_mask_shape, std::move(vinput_mask));
             result.c_image_embeds.reserve(vlm_images.size());
             for (const auto& vlm_image : vlm_images) {
-                auto image_embed = vision_runner->compute(n_threads, vlm_image.second, false, true, true);
+                auto image_embed = vision_runner->compute(n_threads, vlm_image.second, false);
                 if (image_embed.empty()) {
                     LOG_ERROR("hidream_o1 conditioner: encode VLM image failed");
                     return SDCondition();

src/model/diffusion/hunyuan.hpp

@@ -266,16 +266,16 @@ namespace Hunyuan {
             GGML_ASSERT(config.hidden_size / config.num_heads == config.axes_dim_sum);
 
             if (inferred) {
-                LOG_DEBUG("hunyuan video: depth = %d, single depth = %d, in_channels = %" PRId64 ", out_channels = %" PRId64 ", hidden_size = %" PRId64 ", context_in_dim = %" PRId64 ", patch_size = %dx%dx%d",
-                          config.depth,
-                          config.depth_single_blocks,
-                          config.in_channels,
-                          config.out_channels,
-                          config.hidden_size,
-                          config.context_in_dim,
-                          std::get<0>(config.patch_size),
-                          std::get<1>(config.patch_size),
-                          std::get<2>(config.patch_size));
+                LOG_VERBOSE("hunyuan video: depth = %d, single depth = %d, in_channels = %" PRId64 ", out_channels = %" PRId64 ", hidden_size = %" PRId64 ", context_in_dim = %" PRId64 ", patch_size = %dx%dx%d",
+                            config.depth,
+                            config.depth_single_blocks,
+                            config.in_channels,
+                            config.out_channels,
+                            config.hidden_size,
+                            config.context_in_dim,
+                            std::get<0>(config.patch_size),
+                            std::get<1>(config.patch_size),
+                            std::get<2>(config.patch_size));
             }
             return config;
         }
@@ -615,7 +615,7 @@ namespace Hunyuan {
                                                          config.theta,
                                                          config.axes_dim);
             int64_t pos_len = static_cast<int64_t>(pe_vec.size() / config.axes_dim_sum / 2);
-            // LOG_DEBUG("pos_len %d", pos_len);
+            // LOG_VERBOSE("pos_len %d", pos_len);
             auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             // pe->data = pe_vec.data();
             // print_ggml_tensor(pe, true, "pe");
@@ -654,7 +654,7 @@ namespace Hunyuan {
                 return build_graph(x, timesteps, context, c_concat, y, guidance, byt5, vision, timestep_r);
             };
 
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
 
         sd::Tensor<float> compute(int n_threads,

src/model/diffusion/ideogram4.hpp

@@ -58,11 +58,11 @@ namespace Ideogram4 {
             }
             if (detected_layers > 0) {
                 config.num_layers = detected_layers;
-                LOG_DEBUG("ideogram4: num_layers = %" PRId64 ", emb_dim = %" PRId64 ", num_heads = %" PRId64 ", intermediate_size = %" PRId64,
-                          config.num_layers,
-                          config.emb_dim,
-                          config.num_heads,
-                          config.intermediate_size);
+                LOG_VERBOSE("ideogram4: num_layers = %" PRId64 ", emb_dim = %" PRId64 ", num_heads = %" PRId64 ", intermediate_size = %" PRId64,
+                            config.num_layers,
+                            config.emb_dim,
+                            config.num_heads,
+                            config.intermediate_size);
             }
             return config;
         }
@@ -142,7 +142,7 @@ namespace Ideogram4 {
     __STATIC_INLINE__ std::shared_ptr<Linear> make_linear(int64_t in_features,
                                                           int64_t out_features,
                                                           bool bias = true) {
-        return std::make_shared<Linear>(in_features, out_features, bias, false, false, 1.f, true);
+        return std::make_shared<Linear>(in_features, out_features, bias);
     }
 
     __STATIC_INLINE__ std::vector<float> gen_ideogram4_pe(int grid_h,
@@ -465,7 +465,7 @@ namespace Ideogram4 {
                 }
             }
             if (has_uncond_model) {
-                LOG_DEBUG("using uncond model");
+                LOG_VERBOSE("using uncond model");
                 uncond_model = Ideogram4Transformer(config);
                 uncond_model.init(params_ctx, tensor_storage_map, uncond_prefix);
             }
@@ -537,7 +537,7 @@ namespace Ideogram4 {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_graph(x, timesteps, context, use_uncond_model);
             };
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
 
         sd::Tensor<float> compute(int n_threads,

src/model/diffusion/krea2.hpp

@@ -143,16 +143,16 @@ namespace Krea2 {
             }
             config.update_axes_dim();
 
-            LOG_DEBUG("krea2: layers=%" PRId64 ", features=%" PRId64 ", heads=%" PRId64 ", kv_heads=%" PRId64 ", text_dim=%" PRId64 ", text_layers=%" PRId64 ", text_heads=%" PRId64 ", text_kv_heads=%" PRId64 ", channels=%" PRId64,
-                      config.layers,
-                      config.features,
-                      config.heads,
-                      config.kv_heads,
-                      config.text_dim,
-                      config.text_layers,
-                      config.text_heads,
-                      config.text_kv_heads,
-                      config.in_channels);
+            LOG_VERBOSE("krea2: layers=%" PRId64 ", features=%" PRId64 ", heads=%" PRId64 ", kv_heads=%" PRId64 ", text_dim=%" PRId64 ", text_layers=%" PRId64 ", text_heads=%" PRId64 ", text_kv_heads=%" PRId64 ", channels=%" PRId64,
+                        config.layers,
+                        config.features,
+                        config.heads,
+                        config.kv_heads,
+                        config.text_dim,
+                        config.text_layers,
+                        config.text_heads,
+                        config.text_kv_heads,
+                        config.in_channels);
             return config;
         }
     };
@@ -775,7 +775,7 @@ namespace Krea2 {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_graph(x, timesteps, context, ref_latents, ref_image_params);
             };
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
 
         sd::Tensor<float> compute(int n_threads,

src/model/diffusion/lens.hpp

@@ -66,14 +66,14 @@ namespace Lens {
             for (int axis_dim : config.axes_dim) {
                 config.axes_dim_sum += axis_dim;
             }
-            LOG_DEBUG("lens: num_layers = %d, selected_layer_count = %d, hidden_size = %" PRId64 ", num_attention_heads = %" PRId64 ", attention_head_dim = %" PRId64 ", in_channels = %" PRId64 ", out_channels = %" PRId64,
-                      config.num_layers,
-                      config.selected_layer_count,
-                      config.num_attention_heads * config.attention_head_dim,
-                      config.num_attention_heads,
-                      config.attention_head_dim,
-                      config.in_channels,
-                      config.out_channels);
+            LOG_VERBOSE("lens: num_layers = %d, selected_layer_count = %d, hidden_size = %" PRId64 ", num_attention_heads = %" PRId64 ", attention_head_dim = %" PRId64 ", in_channels = %" PRId64 ", out_channels = %" PRId64,
+                        config.num_layers,
+                        config.selected_layer_count,
+                        config.num_attention_heads * config.attention_head_dim,
+                        config.num_attention_heads,
+                        config.attention_head_dim,
+                        config.in_channels,
+                        config.out_channels);
             return config;
         }
     };
@@ -408,7 +408,7 @@ namespace Lens {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_graph(x, timesteps, context);
             };
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
 
         sd::Tensor<float> compute(int n_threads,

src/model/diffusion/lingbot_video.hpp

@@ -127,17 +127,17 @@ namespace LingBotVideo {
                 config.topk_group            = 2;
                 config.routed_scaling_factor = 2.5f;
             }
-            LOG_DEBUG("lingbot_video: depth = %" PRId64 ", hidden_size = %" PRId64 ", heads = %" PRId64 ", text_dim = %" PRId64 ", experts = %" PRId64 ", experts_per_tok = %" PRId64 ", n_group = %" PRId64 ", topk_group = %" PRId64 ", route_scale = %.2f, sparse_layers = %zu",
-                      config.depth,
-                      config.hidden_size,
-                      config.num_attention_heads,
-                      config.text_dim,
-                      config.num_experts,
-                      config.num_experts_per_tok,
-                      config.n_group,
-                      config.topk_group,
-                      config.routed_scaling_factor,
-                      config.sparse_layers.size());
+            LOG_VERBOSE("lingbot_video: depth = %" PRId64 ", hidden_size = %" PRId64 ", heads = %" PRId64 ", text_dim = %" PRId64 ", experts = %" PRId64 ", experts_per_tok = %" PRId64 ", n_group = %" PRId64 ", topk_group = %" PRId64 ", route_scale = %.2f, sparse_layers = %zu",
+                        config.depth,
+                        config.hidden_size,
+                        config.num_attention_heads,
+                        config.text_dim,
+                        config.num_experts,
+                        config.num_experts_per_tok,
+                        config.n_group,
+                        config.topk_group,
+                        config.routed_scaling_factor,
+                        config.sparse_layers.size());
             return config;
         }
     };
@@ -674,7 +674,7 @@ namespace LingBotVideo {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_graph(x, timesteps, context);
             };
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
 
         sd::Tensor<float> compute(int n_threads,

src/model/diffusion/ltxv.hpp

@@ -129,6 +129,10 @@ namespace LTXV {
         bool self_attention_gated  = false;
         bool cross_attention_gated = false;
 
+        bool ff_bias                         = true;
+        bool audio_ff_bias                   = true;
+        bool use_keyframes_abs_pos_embedding = false;
+
         static std::pair<int64_t, int64_t> infer_attention_layout(int64_t hidden_size,
                                                                   int64_t preferred_heads = -1) {
             if (preferred_heads > 0 && hidden_size % preferred_heads == 0) {
@@ -207,6 +211,19 @@ namespace LTXV {
                 tensor_storage_map.find(prefix + ".transformer_blocks.0.audio_attn2.to_gate_logits.weight") != tensor_storage_map.end()) {
                 config.cross_attention_gated = true;
             }
+            // LTX 2.5 sets ff_bias=false but leaves audio_ff_bias at its default, so the two
+            // branches must be detected separately; older checkpoints ship both sets of biases.
+            if (tensor_storage_map.find(prefix + ".transformer_blocks.0.ff.net.0.proj.bias") == tensor_storage_map.end() &&
+                tensor_storage_map.find(prefix + ".transformer_blocks.0.ff.net.2.bias") == tensor_storage_map.end()) {
+                config.ff_bias = false;
+            }
+            if (tensor_storage_map.find(prefix + ".transformer_blocks.0.audio_ff.net.0.proj.bias") == tensor_storage_map.end() &&
+                tensor_storage_map.find(prefix + ".transformer_blocks.0.audio_ff.net.2.bias") == tensor_storage_map.end()) {
+                config.audio_ff_bias = false;
+            }
+            if (tensor_storage_map.find(prefix + ".keyframes_abs_pos_embedding") != tensor_storage_map.end()) {
+                config.use_keyframes_abs_pos_embedding = true;
+            }
             if (tensor_storage_map.find(prefix + ".caption_projection.linear_1.weight") == tensor_storage_map.end() &&
                 tensor_storage_map.find(prefix + ".caption_projection.linear_2.weight") == tensor_storage_map.end()) {
                 config.use_caption_projection = false;
@@ -257,12 +274,12 @@ namespace LTXV {
                     config.audio_connector_apply_gated_attention = true;
                 }
             }
-            LOG_DEBUG("ltxav: num_layers = %" PRId64 ", hidden_size = %" PRId64 ", num_attention_heads = %" PRId64 ", audio_hidden_size = %" PRId64 ", audio_num_attention_heads = %" PRId64,
-                      config.num_layers,
-                      config.hidden_size,
-                      config.num_attention_heads,
-                      config.audio_hidden_size,
-                      config.audio_num_attention_heads);
+            LOG_VERBOSE("ltxav: num_layers = %" PRId64 ", hidden_size = %" PRId64 ", num_attention_heads = %" PRId64 ", audio_hidden_size = %" PRId64 ", audio_num_attention_heads = %" PRId64,
+                        config.num_layers,
+                        config.hidden_size,
+                        config.num_attention_heads,
+                        config.audio_hidden_size,
+                        config.audio_num_attention_heads);
             return config;
         }
     };
@@ -874,8 +891,7 @@ namespace LTXV {
                          const String2TensorStorage& tensor_storage_map = {},
                          const std::string prefix                       = "") override {
             if (num_learnable_registers > 0) {
-                ggml_type wtype               = get_type(prefix + "learnable_registers", tensor_storage_map, GGML_TYPE_F32);
-                params["learnable_registers"] = ggml_new_tensor_2d(ctx, wtype, hidden_size, num_learnable_registers);
+                params["learnable_registers"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_size, num_learnable_registers);
             }
         }
 
@@ -1130,7 +1146,9 @@ namespace LTXV {
                                 int64_t a_context_dim,
                                 bool apply_gated_attention,
                                 bool cross_attention_adaln,
-                                bool video_rope_interleaved)
+                                bool video_rope_interleaved,
+                                bool ff_bias       = true,
+                                bool audio_ff_bias = true)
             : v_dim(v_dim),
               a_dim(a_dim),
               cross_attention_adaln(cross_attention_adaln) {
@@ -1140,8 +1158,8 @@ namespace LTXV {
             blocks["audio_attn2"]         = std::make_shared<CrossAttention>(a_dim, a_context_dim, a_heads, ad_head, apply_gated_attention, false);
             blocks["audio_to_video_attn"] = std::make_shared<CrossAttention>(v_dim, a_dim, a_heads, ad_head, apply_gated_attention, false);
             blocks["video_to_audio_attn"] = std::make_shared<CrossAttention>(a_dim, v_dim, a_heads, ad_head, apply_gated_attention, false);
-            blocks["ff"]                  = std::make_shared<FeedForward>(v_dim, v_dim, 4, FeedForward::Activation::GELU);
-            blocks["audio_ff"]            = std::make_shared<FeedForward>(a_dim, a_dim, 4, FeedForward::Activation::GELU);
+            blocks["ff"]                  = std::make_shared<FeedForward>(v_dim, v_dim, 4, FeedForward::Activation::GELU, false, ff_bias);
+            blocks["audio_ff"]            = std::make_shared<FeedForward>(a_dim, a_dim, 4, FeedForward::Activation::GELU, false, audio_ff_bias);
         }
 
         std::vector<ggml_tensor*> get_ada_values(GGMLRunnerContext* ctx,
@@ -1320,6 +1338,12 @@ namespace LTXV {
                                                                    get_type(prefix + "audio_scale_shift_table", tensor_storage_map, GGML_TYPE_F32),
                                                                    config.audio_hidden_size,
                                                                    2);
+            if (config.use_keyframes_abs_pos_embedding) {
+                params["keyframes_abs_pos_embedding"] = ggml_new_tensor_2d(ctx,
+                                                                           get_type(prefix + "keyframes_abs_pos_embedding", tensor_storage_map, GGML_TYPE_F32),
+                                                                           config.hidden_size,
+                                                                           1);
+            }
         }
 
         LTXAVModelBlock(const LTXAVConfig& config)
@@ -1386,7 +1410,9 @@ namespace LTXV {
                                                                                                               config.audio_cross_attention_dim,
                                                                                                               config.self_attention_gated || config.cross_attention_gated,
                                                                                                               config.cross_attention_adaln,
-                                                                                                              config.video_rope_interleaved);
+                                                                                                              config.video_rope_interleaved,
+                                                                                                              config.ff_bias,
+                                                                                                              config.audio_ff_bias);
             }
 
             blocks["norm_out"]       = std::make_shared<LayerNorm>(config.hidden_size, 1e-6f, false);
@@ -1534,6 +1560,38 @@ namespace LTXV {
             return {v_context, a_context};
         }
 
+        // The video encoder is causal, so the first latent frame covers a single pixel frame while
+        // every later one covers temporal_scale_factor. LTX 2.5 marks that token class with a
+        // learned embedding added right after patchify_proj.
+        ggml_tensor* apply_keyframes_abs_pos_embedding(GGMLRunnerContext* ctx,
+                                                       ggml_tensor* vx,
+                                                       int64_t tokens_per_latent_frame) {
+            if (!config.use_keyframes_abs_pos_embedding || params.count("keyframes_abs_pos_embedding") == 0) {
+                return vx;
+            }
+            int64_t tokens = vx->ne[1];
+            if (tokens_per_latent_frame <= 0 || tokens_per_latent_frame > tokens) {
+                return vx;
+            }
+            auto embedding = params["keyframes_abs_pos_embedding"];
+            auto first     = ggml_cont(ctx->ggml_ctx,
+                                       ggml_view_3d(ctx->ggml_ctx, vx, vx->ne[0], tokens_per_latent_frame, vx->ne[2], vx->nb[1], vx->nb[2], 0));
+            first          = ggml_add(ctx->ggml_ctx, first, embedding);
+            if (tokens_per_latent_frame == tokens) {
+                return first;
+            }
+            auto rest = ggml_cont(ctx->ggml_ctx,
+                                  ggml_view_3d(ctx->ggml_ctx,
+                                               vx,
+                                               vx->ne[0],
+                                               tokens - tokens_per_latent_frame,
+                                               vx->ne[2],
+                                               vx->nb[1],
+                                               vx->nb[2],
+                                               tokens_per_latent_frame * vx->nb[1]));
+            return ggml_concat(ctx->ggml_ctx, first, rest, 1);
+        }
+
         std::vector<ggml_tensor*> get_output_scale_shift(GGMLRunnerContext* ctx,
                                                          ggml_tensor* table,
                                                          ggml_tensor* embedded_timestep,
@@ -1575,6 +1633,7 @@ namespace LTXV {
 
             vx = patchify_video(ctx, vx, n);
             vx = patchify_proj->forward(ctx, vx);
+            vx = apply_keyframes_abs_pos_embedding(ctx, vx, width * height);
             if (ax != nullptr && ggml_nelements(ax) > 0 && audio_time > 0) {
                 ax = patchify_audio(ctx, ax);
                 ax = audio_patchify_proj->forward(ctx, ax);
@@ -1939,7 +1998,7 @@ namespace LTXV {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_graph(x, timesteps, context, audio_x, audio_timesteps, audio_length, frame_rate, video_positions);
             };
-            auto out = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            auto out = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
             return out;
         }
 
@@ -2011,7 +2070,7 @@ namespace LTXV {
 
             GGML_ASSERT(!out_opt.empty());
             print_sd_tensor(out_opt, false, "ltxav_out");
-            LOG_DEBUG("ltxav test done in %lldms", t1 - t0);
+            LOG_VERBOSE("ltxav test done in %lldms", t1 - t0);
         }
 
         static void load_from_file_and_test(const std::string& model_path,

src/model/diffusion/mage_flow.hpp

@@ -142,7 +142,7 @@ namespace MageFlow {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_graph(x, timesteps, context, ref_latents);
             };
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
 
         sd::Tensor<float> compute(int n_threads,

src/model/diffusion/minimax_h3.hpp

@@ -106,14 +106,14 @@ namespace MiniMaxH3 {
                 config.rope_inv_freq_len = inv_freq->ne[0];
             }
 
-            LOG_DEBUG("minimax_h3: layers=%" PRId64 ", hidden=%" PRId64 ", heads=%" PRId64
-                      ", head_dim=%" PRId64 ", ffn=%" PRId64 ", adaln_curve=%" PRId64,
-                      config.num_layers,
-                      config.hidden_size,
-                      config.num_attention_heads,
-                      config.attention_head_dim,
-                      config.ffn_hidden_size,
-                      config.adaln_curve_grid);
+            LOG_VERBOSE("minimax_h3: layers=%" PRId64 ", hidden=%" PRId64 ", heads=%" PRId64
+                        ", head_dim=%" PRId64 ", ffn=%" PRId64 ", adaln_curve=%" PRId64,
+                        config.num_layers,
+                        config.hidden_size,
+                        config.num_attention_heads,
+                        config.attention_head_dim,
+                        config.ffn_hidden_size,
+                        config.adaln_curve_grid);
             return config;
         }
     };
@@ -123,13 +123,6 @@ namespace MiniMaxH3 {
         return to_shift * base / (1.f + (to_shift - 1.f) * base);
     }
 
-    static float time_shift_slope(float sigma, float from_shift, float to_shift) {
-        float base = sigma / (from_shift + sigma * (1.f - from_shift));
-        float a    = 1.f + (from_shift - 1.f) * base;
-        float b    = 1.f + (to_shift - 1.f) * base;
-        return to_shift * a * a / (from_shift * b * b);
-    }
-
     struct TimeEmbedder : public GGMLBlock {
         TimeEmbedder(int64_t input_dim, int64_t hidden_dim, int64_t output_dim) {
             blocks["proj_in"]  = std::make_shared<Linear>(input_dim, hidden_dim, true, true);
@@ -594,8 +587,7 @@ namespace MiniMaxH3 {
                                                       const std::vector<TokenModulationSpan>& segments,
                                                       const std::vector<SequenceSegment>& sequence_segments,
                                                       const TokenModulationSpan& video_segment,
-                                                      const TokenModulationSpan& audio_segment,
-                                                      float audio_slope) {
+                                                      const TokenModulationSpan& audio_segment) {
             auto video_proj = std::dynamic_pointer_cast<Linear>(blocks["video_patch_proj"]);
             auto audio_proj = std::dynamic_pointer_cast<Linear>(blocks["audio_patch_proj"]);
 
@@ -715,7 +707,7 @@ namespace MiniMaxH3 {
                                                audio->ne[2]);
             audio_out        = ggml_cont(ctx->ggml_ctx, ggml_ext_torch_permute(ctx->ggml_ctx, audio_out, 1, 2, 0, 3));
             video_out        = ggml_ext_scale(ctx->ggml_ctx, video_out, -1.f);
-            audio_out        = ggml_ext_scale(ctx->ggml_ctx, audio_out, -audio_slope);
+            audio_out        = ggml_ext_scale(ctx->ggml_ctx, audio_out, -1.f);
             return {video_out, audio_out};
         }
     };
@@ -1040,9 +1032,9 @@ namespace MiniMaxH3 {
             GGML_ASSERT(!audio_input_cache.empty());
             GGML_ASSERT(!context_tensor.empty());
 
-            auto video   = make_input(video_input_cache);
-            auto audio   = make_input(audio_input_cache);
-            auto context = make_input(context_tensor);
+            auto video         = make_input(video_input_cache);
+            auto audio_carrier = make_input(audio_input_cache);
+            auto context       = make_input(context_tensor);
             std::vector<ggml_tensor*> condition_inputs;
             condition_inputs.reserve(condition_videos.size());
             for (const auto& condition : condition_videos) {
@@ -1054,21 +1046,26 @@ namespace MiniMaxH3 {
                 audio_condition_inputs.push_back(make_input(condition));
             }
 
-            float sigma_v = std::clamp(timestep[0] / 1000.f, 1e-6f, 1.f);
-            float t_v     = 1.f - sigma_v;
-            float t_a     = 1.f - time_shift_sigma(sigma_v, video_shift, audio_shift);
-            auto layout   = build_layout(context_tensor.shape()[1],
-                                         video_input_cache.shape()[2],
-                                         video_input_cache.shape()[1],
-                                         video_input_cache.shape()[0],
-                                         audio_length,
-                                         condition_videos,
-                                         condition_audios,
-                                         keyframe_indices,
-                                         reference_blocks,
-                                         text_tags,
-                                         t_v,
-                                         t_a);
+            float sigma_v     = std::clamp(timestep[0] / 1000.f, 1e-6f, 1.f);
+            float sigma_a     = time_shift_sigma(sigma_v, video_shift, audio_shift);
+            float audio_scale = video_shift / audio_shift;
+            float t_v         = 1.f - sigma_v;
+            float t_a         = 1.f - sigma_a;
+            // The sampler carries c_a = (sigma_v / sigma_a) * x_a so the packed
+            // latent follows one sigma schedule. Restore x_a for the H3 network.
+            auto audio  = ggml_ext_scale(compute_ctx, audio_carrier, sigma_a / sigma_v);
+            auto layout = build_layout(context_tensor.shape()[1],
+                                       video_input_cache.shape()[2],
+                                       video_input_cache.shape()[1],
+                                       video_input_cache.shape()[0],
+                                       audio_length,
+                                       condition_videos,
+                                       condition_audios,
+                                       keyframe_indices,
+                                       reference_blocks,
+                                       text_tags,
+                                       t_v,
+                                       t_a);
 
             position_input_cache = sd::Tensor<float>(
                 {3, static_cast<int64_t>(layout.positions.size() / 3)},
@@ -1129,10 +1126,15 @@ namespace MiniMaxH3 {
                                             layout.segments,
                                             layout.sequence_segments,
                                             layout.video_segment,
-                                            layout.audio_segment,
-                                            time_shift_slope(sigma_v, video_shift, audio_shift));
-            auto merged     = merge_av_latents(compute_ctx, output.first, output.second);
-            auto graph      = new_graph_custom(H3_GRAPH_SIZE);
+                                            layout.audio_segment);
+            // Convert the model's audio velocity to d(c_a) / d(sigma_v).
+            output.second = ggml_add(compute_ctx,
+                                     ggml_ext_scale(compute_ctx, audio, 1.f - audio_scale),
+                                     ggml_ext_scale(compute_ctx,
+                                                    output.second,
+                                                    1.f + (audio_scale - 1.f) * sigma_a));
+            auto merged   = merge_av_latents(compute_ctx, output.first, output.second);
+            auto graph    = new_graph_custom(H3_GRAPH_SIZE);
             ggml_build_forward_expand(graph, merged);
             return graph;
         }
@@ -1166,8 +1168,6 @@ namespace MiniMaxH3 {
             };
             return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph,
                                                                               n_threads,
-                                                                              false,
-                                                                              false,
                                                                               false),
                                                    params.x->dim());
         }

src/model/diffusion/minit2i.hpp

@@ -108,15 +108,15 @@ namespace MiniT2I {
                 config.head_dim  = config.hidden_size == 1248 ? 52 : 64;
                 config.num_heads = config.hidden_size / config.head_dim;
             }
-            LOG_DEBUG("minit2i: hidden_size=%" PRId64 ", txt_hidden_size=%" PRId64 ", heads=%" PRId64 ", head_dim=%" PRId64 ", double_blocks=%" PRId64 ", txt_blocks=%" PRId64 ", patch=%" PRId64 ", in_channels=%" PRId64,
-                      config.hidden_size,
-                      config.txt_hidden_size,
-                      config.num_heads,
-                      config.head_dim,
-                      config.depth_double,
-                      config.txt_preamble_depth,
-                      config.patch_size,
-                      config.in_channels);
+            LOG_VERBOSE("minit2i: hidden_size=%" PRId64 ", txt_hidden_size=%" PRId64 ", heads=%" PRId64 ", head_dim=%" PRId64 ", double_blocks=%" PRId64 ", txt_blocks=%" PRId64 ", patch=%" PRId64 ", in_channels=%" PRId64,
+                        config.hidden_size,
+                        config.txt_hidden_size,
+                        config.num_heads,
+                        config.head_dim,
+                        config.depth_double,
+                        config.txt_preamble_depth,
+                        config.patch_size,
+                        config.in_channels);
             return config;
         }
     };
@@ -589,7 +589,7 @@ namespace MiniT2I {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_graph(x, timesteps, context, mask);
             };
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
 
         sd::Tensor<float> compute(int n_threads,

src/model/diffusion/mmdit.hpp

@@ -120,16 +120,16 @@ struct MMDiTConfig {
         }
 
         if (has_weight_config) {
-            LOG_DEBUG("mmdit: num_layers = %" PRId64 ", num_mmdit_x_layers = %" PRId64 ", hidden_size = %" PRId64 ", patch_size = %d, in_channels = %" PRId64 ", out_channels = %" PRId64 ", context_size = %" PRId64 ", adm_in_channels = %" PRId64 ", qk_norm = %s",
-                      config.depth,
-                      config.d_self + 1,
-                      config.hidden_size,
-                      config.patch_size,
-                      config.in_channels,
-                      config.out_channels,
-                      config.context_size,
-                      config.adm_in_channels,
-                      config.qk_norm.empty() ? "none" : config.qk_norm.c_str());
+            LOG_VERBOSE("mmdit: num_layers = %" PRId64 ", num_mmdit_x_layers = %" PRId64 ", hidden_size = %" PRId64 ", patch_size = %d, in_channels = %" PRId64 ", out_channels = %" PRId64 ", context_size = %" PRId64 ", adm_in_channels = %" PRId64 ", qk_norm = %s",
+                        config.depth,
+                        config.d_self + 1,
+                        config.hidden_size,
+                        config.patch_size,
+                        config.in_channels,
+                        config.out_channels,
+                        config.context_size,
+                        config.adm_in_channels,
+                        config.qk_norm.empty() ? "none" : config.qk_norm.c_str());
         }
         return config;
     }
@@ -987,7 +987,7 @@ struct MMDiTRunner : public DiffusionModelRunner {
             return build_graph(x, timesteps, context, y, skip_layers);
         };
 
-        return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+        return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
     }
 
     sd::Tensor<float> compute(int n_threads,
@@ -1045,7 +1045,7 @@ struct MMDiTRunner : public DiffusionModelRunner {
             GGML_ASSERT(!out_opt.empty());
             out = std::move(out_opt);
             print_sd_tensor(out);
-            LOG_DEBUG("mmdit test done in %lldms", t1 - t0);
+            LOG_VERBOSE("mmdit test done in %lldms", t1 - t0);
         }
     }
 

src/model/diffusion/pid.hpp

@@ -109,16 +109,16 @@ namespace Pid {
                 config.lq_latent_channels    = latent_proj_in_channels;
                 config.lq_latent_down_factor = latent_proj_in_channels >= 64 ? 16 : 8;
             }
-            LOG_DEBUG("pid: version = %s, patch_depth = %" PRId64 ", pixel_depth = %" PRId64 ", patch_mlp_hidden_dim = %" PRId64 ", lq_latent_channels = %" PRId64 ", lq_hidden_dim = %" PRId64 ", lq_latent_down_factor = %" PRId64 ", lq_latent_unpatchify_factor = %" PRId64 ", lq_interval = %" PRId64,
-                      config.pit_lq_inject ? "1.5" : "1",
-                      config.patch_depth,
-                      config.pixel_depth,
-                      config.patch_mlp_hidden_dim,
-                      config.lq_latent_channels,
-                      config.lq_hidden_dim,
-                      config.lq_latent_down_factor,
-                      config.lq_latent_unpatchify_factor,
-                      config.lq_interval);
+            LOG_VERBOSE("pid: version = %s, patch_depth = %" PRId64 ", pixel_depth = %" PRId64 ", patch_mlp_hidden_dim = %" PRId64 ", lq_latent_channels = %" PRId64 ", lq_hidden_dim = %" PRId64 ", lq_latent_down_factor = %" PRId64 ", lq_latent_unpatchify_factor = %" PRId64 ", lq_interval = %" PRId64,
+                        config.pit_lq_inject ? "1.5" : "1",
+                        config.patch_depth,
+                        config.pixel_depth,
+                        config.patch_mlp_hidden_dim,
+                        config.lq_latent_channels,
+                        config.lq_hidden_dim,
+                        config.lq_latent_down_factor,
+                        config.lq_latent_unpatchify_factor,
+                        config.lq_interval);
             return config;
         }
     };
@@ -938,7 +938,7 @@ namespace Pid {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_graph(x, timesteps, context, lq_latent, degrade_sigma);
             };
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
 
         sd::Tensor<float> compute(int n_threads,

src/model/diffusion/qwen_image.hpp

@@ -49,9 +49,9 @@ namespace Qwen {
                     }
                 }
             }
-            LOG_DEBUG("qwen_image: num_layers = %d, zero_cond_t = %s",
-                      config.num_layers,
-                      config.zero_cond_t ? "true" : "false");
+            LOG_VERBOSE("qwen_image: num_layers = %d, zero_cond_t = %s",
+                        config.num_layers,
+                        config.zero_cond_t ? "true" : "false");
             return config;
         }
     };
@@ -646,7 +646,7 @@ namespace Qwen {
                                                   circular_x_enabled,
                                                   config.axes_dim);
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
-            // LOG_DEBUG("pos_len %d", pos_len);
+            // LOG_VERBOSE("pos_len %d", pos_len);
             auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             // pe->data = pe_vec.data();
             // print_ggml_tensor(pe, true, "pe");
@@ -707,7 +707,7 @@ namespace Qwen {
                 return build_graph(x, timesteps, context, ref_latents, ref_index_mode);
             };
 
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
 
         sd::Tensor<float> compute(int n_threads,
@@ -760,7 +760,7 @@ namespace Qwen {
                 GGML_ASSERT(!out_opt.empty());
                 out = std::move(out_opt);
                 print_sd_tensor(out);
-                LOG_DEBUG("qwen_image test done in %lldms", t1 - t0);
+                LOG_VERBOSE("qwen_image test done in %lldms", t1 - t0);
             }
         }
 

src/model/diffusion/sefi_image.hpp

@@ -34,10 +34,10 @@ namespace SefiImage {
                     config.hidden_size              = tensor_storage.ne[1] * 2;
                 }
             }
-            LOG_DEBUG("sefi_image: semantic_channels = %" PRId64 ", texture_latent_channels = %" PRId64 ", hidden_size = %" PRId64,
-                      config.semantic_channels,
-                      config.texture_latent_channels,
-                      config.hidden_size);
+            LOG_VERBOSE("sefi_image: semantic_channels = %" PRId64 ", texture_latent_channels = %" PRId64 ", hidden_size = %" PRId64,
+                        config.semantic_channels,
+                        config.texture_latent_channels,
+                        config.hidden_size);
             return config;
         }
     };

src/model/diffusion/unet.hpp

@@ -128,15 +128,15 @@ struct UNetConfig {
             }
         }
 
-        LOG_DEBUG("unet: in_channels = %d, out_channels = %d, model_channels = %d, time_embed_dim = %d, context_dim = %d, adm_in_channels = %d, num_res_blocks = %d, tiny_unet = %s",
-                  config.in_channels,
-                  config.out_channels,
-                  config.model_channels,
-                  config.time_embed_dim,
-                  config.context_dim,
-                  config.adm_in_channels,
-                  config.num_res_blocks,
-                  config.tiny_unet ? "true" : "false");
+        LOG_VERBOSE("unet: in_channels = %d, out_channels = %d, model_channels = %d, time_embed_dim = %d, context_dim = %d, adm_in_channels = %d, num_res_blocks = %d, tiny_unet = %s",
+                    config.in_channels,
+                    config.out_channels,
+                    config.model_channels,
+                    config.time_embed_dim,
+                    config.context_dim,
+                    config.adm_in_channels,
+                    config.num_res_blocks,
+                    config.tiny_unet ? "true" : "false");
         return config;
     }
 };
@@ -835,7 +835,7 @@ struct UNetModelRunner : public DiffusionModelRunner {
             return build_graph(x, timesteps, context, c_concat, y, num_video_frames, controls, control_strength, ip_context, ip_scale);
         };
 
-        return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+        return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
     }
 
     sd::Tensor<float> compute(int n_threads,
@@ -904,7 +904,7 @@ struct UNetModelRunner : public DiffusionModelRunner {
             GGML_ASSERT(!out_opt.empty());
             out = std::move(out_opt);
             print_sd_tensor(out);
-            LOG_DEBUG("unet test done in %lldms", t1 - t0);
+            LOG_VERBOSE("unet test done in %lldms", t1 - t0);
         }
     }
 };

src/model/diffusion/wan.hpp

@@ -75,13 +75,13 @@ namespace WAN {
                     config.flf_pos_embed_token_number = 514;
                 }
             }
-            LOG_DEBUG("wan: model_type = %s, num_layers = %d, vace_layers = %d, dim = %" PRId64 ", ffn_dim = %" PRId64 ", num_heads = %" PRId64,
-                      config.model_type.c_str(),
-                      config.num_layers,
-                      config.vace_layers,
-                      config.dim,
-                      config.ffn_dim,
-                      config.num_heads);
+            LOG_VERBOSE("wan: model_type = %s, num_layers = %d, vace_layers = %d, dim = %" PRId64 ", ffn_dim = %" PRId64 ", num_heads = %" PRId64,
+                        config.model_type.c_str(),
+                        config.num_layers,
+                        config.vace_layers,
+                        config.dim,
+                        config.ffn_dim,
+                        config.num_heads);
             return config;
         }
     };
@@ -909,7 +909,7 @@ namespace WAN {
                                            config.theta,
                                            config.axes_dim);
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
-            // LOG_DEBUG("pos_len %d", pos_len);
+            // LOG_VERBOSE("pos_len %d", pos_len);
             auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             // pe->data = pe_vec.data();
             // print_ggml_tensor(pe);
@@ -950,7 +950,7 @@ namespace WAN {
                 return build_graph(x, timesteps, context, clip_fea, c_concat, time_dim_concat, vace_context, vace_strength);
             };
 
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
 
         sd::Tensor<float> compute(int n_threads,
@@ -1007,7 +1007,7 @@ namespace WAN {
                 GGML_ASSERT(!out_opt.empty());
                 out = std::move(out_opt);
                 print_sd_tensor(out);
-                LOG_DEBUG("wan test done in %lldms", t1 - t0);
+                LOG_VERBOSE("wan test done in %lldms", t1 - t0);
             }
         }
 

src/model/diffusion/z_image.hpp

@@ -107,14 +107,14 @@ namespace ZImage {
                     config.num_kv_heads = std::max<int64_t>(1, (qkv_heads - config.num_heads) / 2);
                 }
             }
-            LOG_DEBUG("z_image: num_layers = %" PRId64 ", num_refiner_layers = %" PRId64 ", hidden_size = %" PRId64 ", num_heads = %" PRId64 ", num_kv_heads = %" PRId64 ", in_channels = %" PRId64 ", out_channels = %" PRId64,
-                      config.num_layers,
-                      config.num_refiner_layers,
-                      config.hidden_size,
-                      config.num_heads,
-                      config.num_kv_heads,
-                      config.in_channels,
-                      config.out_channels);
+            LOG_VERBOSE("z_image: num_layers = %" PRId64 ", num_refiner_layers = %" PRId64 ", hidden_size = %" PRId64 ", num_heads = %" PRId64 ", num_kv_heads = %" PRId64 ", in_channels = %" PRId64 ", out_channels = %" PRId64,
+                        config.num_layers,
+                        config.num_refiner_layers,
+                        config.hidden_size,
+                        config.num_heads,
+                        config.num_kv_heads,
+                        config.in_channels,
+                        config.out_channels);
             return config;
         }
     };
@@ -603,7 +603,7 @@ namespace ZImage {
                                                circular_x_enabled,
                                                config.axes_dim);
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
-            // LOG_DEBUG("pos_len %d", pos_len);
+            // LOG_VERBOSE("pos_len %d", pos_len);
             auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             // pe->data = pe_vec.data();
             // print_ggml_tensor(pe, true, "pe");
@@ -636,7 +636,7 @@ namespace ZImage {
                 return build_graph(x, timesteps, context, ref_latents, ref_index_mode);
             };
 
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         }
 
         sd::Tensor<float> compute(int n_threads,
@@ -689,7 +689,7 @@ namespace ZImage {
                 GGML_ASSERT(!out_opt.empty());
                 out = std::move(out_opt);
                 print_sd_tensor(out);
-                LOG_DEBUG("z_image test done in %lldms", t1 - t0);
+                LOG_VERBOSE("z_image test done in %lldms", t1 - t0);
             }
         }
 

src/model/te/clip.hpp

@@ -100,13 +100,13 @@ public:
                          const std::string& graph_cut_prefix = "") {
         // x: [N, n_token, d_model]
         int layer_idx = n_layer - 1;
-        // LOG_DEBUG("clip_skip %d", clip_skip);
+        // LOG_VERBOSE("clip_skip %d", clip_skip);
         if (clip_skip > 0) {
             layer_idx = n_layer - clip_skip;
         }
 
         for (int i = 0; i < n_layer; i++) {
-            // LOG_DEBUG("layer %d", i);
+            // LOG_VERBOSE("layer %d", i);
             if (i == layer_idx + 1) {
                 break;
             }
@@ -116,7 +116,7 @@ public:
             if (!graph_cut_prefix.empty()) {
                 sd::ggml_graph_cut::mark_graph_cut(x, graph_cut_prefix + ".layers." + std::to_string(i), "x");
             }
-            // LOG_DEBUG("layer %d", i);
+            // LOG_VERBOSE("layer %d", i);
         }
         return x;
     }
@@ -320,7 +320,7 @@ public:
             if (text_projection != nullptr) {
                 pooled = ggml_ext_linear(ctx->ggml_ctx, pooled, text_projection, nullptr);
             } else {
-                LOG_DEBUG("identity projection");
+                LOG_VERBOSE("identity projection");
             }
             return pooled;  // [hidden_size, 1, 1]
         }
@@ -568,13 +568,11 @@ struct CLIPTextModelRunner : public GGMLRunner {
                               size_t max_token_idx,
                               bool return_pooled,
                               int clip_skip,
-                              bool auto_free           = true,
-                              bool free_compute_buffer = true,
-                              bool free_compute_params = true) {
+                              bool auto_runner_end = true) {
         auto get_graph = [&]() -> ggml_cgraph* {
             return build_graph(input_ids, num_custom_embeddings, custom_embeddings_data, max_token_idx, return_pooled, clip_skip);
         };
-        auto result = GGMLRunner::compute<float>(get_graph, n_threads, auto_free, free_compute_buffer, free_compute_params);
+        auto result = GGMLRunner::compute<float>(get_graph, n_threads, auto_runner_end);
         if (return_pooled) {
             return take_or_empty(std::move(result));
         }

src/model/te/llm.hpp

@@ -40,6 +40,7 @@ namespace LLM {
         MINISTRAL_3_3B,
         GEMMA3_12B,
         GEMMA2_2B,
+        GEMMA4_12B,
         GPT_OSS_20B,
         ARCH_COUNT,
     };
@@ -52,6 +53,7 @@ namespace LLM {
         "ministral3.3b",
         "gemma3_12b",
         "gemma2_2b",
+        "gemma4_12b",
         "gpt_oss_20b",
     };
 
@@ -120,6 +122,15 @@ namespace LLM {
         bool have_vision_weight = false;
         bool llama_cpp_style    = false;
 
+        // gemma4 config
+        int global_head_dim         = 0;
+        int num_global_kv_heads     = 0;
+        float global_partial_rotary = 1.f;
+        bool global_k_eq_v          = false;
+        bool v_norm                 = false;
+        bool layer_scalar           = false;
+        bool unscaled_attention     = false;
+
         static LLMConfig detect_from_weights(const String2TensorStorage& tensor_storage_map,
                                              const std::string& prefix,
                                              LLMArch arch) {
@@ -157,6 +168,27 @@ namespace LLM {
                 config.rope_thetas             = {1000000.f, 10000.f};
                 config.rope_scales             = {8.f, 1.f};
                 config.sliding_attention       = {1024, 1024, 1024, 1024, 1024, 0};
+            } else if (arch == LLMArch::GEMMA4_12B) {
+                config.head_dim                = 256;
+                config.num_heads               = 16;
+                config.num_kv_heads            = 8;
+                config.global_head_dim         = 512;
+                config.num_global_kv_heads     = 1;
+                config.global_partial_rotary   = 0.25f;
+                config.global_k_eq_v           = true;
+                config.v_norm                  = true;
+                config.layer_scalar            = true;
+                config.unscaled_attention      = true;
+                config.qkv_bias                = false;
+                config.qk_norm                 = true;
+                config.rms_norm_eps            = 1e-6f;
+                config.rms_norm_add            = false;
+                config.normalize_input         = true;
+                config.max_position_embeddings = 262144;
+                config.mlp_activation          = MLPActivation::GELU_TANH;
+                config.rope_thetas             = {1000000.f, 10000.f};
+                config.rope_scales             = {1.f, 1.f};
+                config.sliding_attention       = {1024, 1024, 1024, 1024, 1024, 0};
             } else if (arch == LLMArch::GEMMA2_2B) {
                 config.head_dim                = 256;
                 config.num_heads               = 8;
@@ -232,12 +264,12 @@ namespace LLM {
                             }
                         }
                     }
-                    if (contains(name, "visual.blocks.0.mlp.linear_fc1.weight") ||
-                        contains(name, "visual.blocks.0.mlp.gate_proj.weight")) {
+                    if (ends_with(name, "visual.blocks.0.mlp.linear_fc1.weight") ||
+                        ends_with(name, "visual.blocks.0.mlp.gate_proj.weight")) {
                         config.vision.intermediate_size = tensor_storage.ne[1];
                     }
-                    if (contains(name, "visual.merger.linear_fc2.weight") ||
-                        contains(name, "visual.merger.mlp.2.weight")) {
+                    if (ends_with(name, "visual.merger.linear_fc2.weight") ||
+                        ends_with(name, "visual.merger.mlp.2.weight")) {
                         config.vision.out_hidden_size = tensor_storage.ne[1];
                     }
                     continue;
@@ -256,22 +288,26 @@ namespace LLM {
                     config.hidden_size = tensor_storage.ne[0];
                     config.vocab_size  = tensor_storage.ne[1];
                 }
-                if (contains(name, "layers.0.mlp.gate_proj.weight")) {
+                if (ends_with(name, "layers.0.mlp.gate_proj.weight")) {
                     config.intermediate_size = tensor_storage.ne[1];
                 }
-                if (contains(name, "layers.0.mlp.experts.gate_up_proj.weight")) {
+                if (ends_with(name, "layers.0.mlp.experts.gate_up_proj.weight")) {
                     config.intermediate_size = tensor_storage.ne[1] / 2;
                 }
-                if (contains(name, "layers.0.mlp.experts.gate_proj.weight")) {
+                if (ends_with(name, "layers.0.mlp.experts.gate_proj.weight")) {
                     config.intermediate_size = tensor_storage.ne[1];
                 }
             }
             if ((arch == LLMArch::QWEN3 || arch == LLMArch::QWEN3_VL) && config.num_layers == 28) {
                 config.num_heads = 16;
             }
-            if (arch == LLMArch::QWEN3_VL && config.num_layers == 50 && config.hidden_size == 5120) {
-                config.num_heads  = 64;
-                config.final_norm = false;
+            if (arch == LLMArch::QWEN3_VL &&
+                (config.num_layers == 50 || config.num_layers == 64) &&
+                config.hidden_size == 5120) {
+                config.num_heads = 64;
+                if (config.num_layers == 50) {
+                    config.final_norm = false;
+                }
             }
             if (detected_vision_layers > 0) {
                 config.vision.num_layers = detected_vision_layers;
@@ -283,11 +319,11 @@ namespace LLM {
                     config.vision.deepstack_visual_indexes = {8, 16, 24};
                 }
             }
-            LOG_DEBUG("llm: num_layers = %" PRId64 ", vocab_size = %" PRId64 ", hidden_size = %" PRId64 ", intermediate_size = %" PRId64,
-                      config.num_layers,
-                      config.vocab_size,
-                      config.hidden_size,
-                      config.intermediate_size);
+            LOG_VERBOSE("llm: num_layers = %" PRId64 ", vocab_size = %" PRId64 ", hidden_size = %" PRId64 ", intermediate_size = %" PRId64,
+                        config.num_layers,
+                        config.vocab_size,
+                        config.hidden_size,
+                        config.intermediate_size);
             return config;
         }
     };
@@ -1059,6 +1095,11 @@ namespace LLM {
         std::vector<float> rope_thetas;
         std::vector<float> rope_scales;
         bool has_attention_sinks;
+        bool k_eq_v;
+        bool v_norm;
+        bool unscaled_attention;
+        float rms_norm_eps;
+        int rope_pairs;
 
         void init_params(ggml_context* ctx,
                          const String2TensorStorage& tensor_storage_map = {},
@@ -1069,24 +1110,48 @@ namespace LLM {
         }
 
     public:
-        Attention(const LLMConfig& config)
+        Attention(const LLMConfig& config, bool global_layer = false)
             : arch(config.arch),
               num_heads(config.num_heads),
-              num_kv_heads(config.num_kv_heads),
-              head_dim(config.head_dim),
+              num_kv_heads(global_layer && config.num_global_kv_heads > 0 ? config.num_global_kv_heads : config.num_kv_heads),
+              head_dim(global_layer && config.global_head_dim > 0 ? config.global_head_dim : config.head_dim),
               qk_norm(config.qk_norm),
               max_position_embeddings(config.max_position_embeddings),
               rope_thetas(config.rope_thetas),
               rope_scales(config.rope_scales),
-              has_attention_sinks(config.arch == LLMArch::GPT_OSS_20B) {
+              has_attention_sinks(config.arch == LLMArch::GPT_OSS_20B),
+              k_eq_v(global_layer && config.global_k_eq_v),
+              v_norm(config.v_norm),
+              unscaled_attention(config.unscaled_attention),
+              rms_norm_eps(config.rms_norm_eps),
+              rope_pairs(0) {
             blocks["q_proj"] = std::make_shared<Linear>(config.hidden_size, num_heads * head_dim, config.qkv_bias);
             blocks["k_proj"] = std::make_shared<Linear>(config.hidden_size, num_kv_heads * head_dim, config.qkv_bias);
-            blocks["v_proj"] = std::make_shared<Linear>(config.hidden_size, num_kv_heads * head_dim, config.qkv_bias);
+            if (!k_eq_v) {
+                blocks["v_proj"] = std::make_shared<Linear>(config.hidden_size, num_kv_heads * head_dim, config.qkv_bias);
+            }
             blocks["o_proj"] = std::make_shared<Linear>(num_heads * head_dim, config.hidden_size, config.attention_out_bias);
             if (config.qk_norm) {
                 blocks["q_norm"] = std::make_shared<LLMRMSNorm>(head_dim, config.rms_norm_eps, config.rms_norm_add);
                 blocks["k_norm"] = std::make_shared<LLMRMSNorm>(head_dim, config.rms_norm_eps, config.rms_norm_add);
             }
+            // Proportional RoPE rotates only the leading `rope_pairs` dimension pairs of the head;
+            // the rest are left unrotated through freq_factors (see rope_freq_factors()).
+            float partial = global_layer ? config.global_partial_rotary : 1.f;
+            rope_pairs    = static_cast<int>(partial * head_dim / 2.f);
+        }
+
+        // ggml applies theta_i / freq_factors[i], so a huge factor collapses the angle to zero and
+        // leaves that pair unrotated. This reproduces transformers' "proportional" RoPE, whose
+        // inv_freq is zero-padded past `rope_pairs`, without reordering the head.
+        ggml_tensor* rope_freq_factors(ggml_context* ctx) const {
+            int pairs = head_dim / 2;
+            if (rope_pairs >= pairs) {
+                return nullptr;
+            }
+            auto rotated   = ggml_ext_ones(ctx, rope_pairs, 1, 1, 1);
+            auto unrotated = ggml_ext_full(ctx, 1e30f, pairs - rope_pairs, 1, 1, 1);
+            return ggml_concat(ctx, rotated, unrotated, 0);
         }
 
         ggml_tensor* forward(GGMLRunnerContext* ctx,
@@ -1099,12 +1164,12 @@ namespace LLM {
             int64_t N       = x->ne[2];
             auto q_proj     = std::dynamic_pointer_cast<Linear>(blocks["q_proj"]);
             auto k_proj     = std::dynamic_pointer_cast<Linear>(blocks["k_proj"]);
-            auto v_proj     = std::dynamic_pointer_cast<Linear>(blocks["v_proj"]);
+            auto v_proj     = k_eq_v ? nullptr : std::dynamic_pointer_cast<Linear>(blocks["v_proj"]);
             auto out_proj   = std::dynamic_pointer_cast<Linear>(blocks["o_proj"]);
 
-            auto q = q_proj->forward(ctx, x);  // [N, n_token, num_heads*head_dim]
-            auto k = k_proj->forward(ctx, x);  // [N, n_token, num_kv_heads*head_dim]
-            auto v = v_proj->forward(ctx, x);  // [N, n_token, num_kv_heads*head_dim]
+            auto q = q_proj->forward(ctx, x);               // [N, n_token, num_heads*head_dim]
+            auto k = k_proj->forward(ctx, x);               // [N, n_token, num_kv_heads*head_dim]
+            auto v = k_eq_v ? k : v_proj->forward(ctx, x);  // [N, n_token, num_kv_heads*head_dim]
 
             q = ggml_reshape_4d(ctx->ggml_ctx, q, head_dim, num_heads, n_token, N);     // [N, n_token, num_heads, head_dim]
             k = ggml_reshape_4d(ctx->ggml_ctx, k, head_dim, num_kv_heads, n_token, N);  // [N, n_token, num_kv_heads, head_dim]
@@ -1117,6 +1182,10 @@ namespace LLM {
                 q = q_norm->forward(ctx, q);
                 k = k_norm->forward(ctx, k);
             }
+            if (v_norm) {
+                // Gemma 4 normalizes V with a weightless RMS norm, and never rotates it.
+                v = ggml_rms_norm(ctx->ggml_ctx, v, rms_norm_eps);
+            }
 
             if (arch == LLMArch::MISTRAL_SMALL_3_2) {
                 q = ggml_rope_ext(ctx->ggml_ctx, q, input_pos, nullptr, 128, GGML_ROPE_TYPE_NORMAL, 8192, 1000000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
@@ -1187,6 +1256,35 @@ namespace LLM {
                                                  1.f,
                                                  32.f,
                                                  1.f);
+            } else if (arch == LLMArch::GEMMA4_12B) {
+                float rope_theta  = (rope_index == 1 ? 10000.0f : 1000000.0f);
+                auto freq_factors = rope_freq_factors(ctx->ggml_ctx);
+                q                 = ggml_rope_ext(ctx->ggml_ctx,
+                                                  q,
+                                                  input_pos,
+                                                  freq_factors,
+                                                  head_dim,
+                                                  GGML_ROPE_TYPE_NEOX,
+                                                  static_cast<int>(max_position_embeddings),
+                                                  rope_theta,
+                                                  1.f,
+                                                  0.f,
+                                                  1.f,
+                                                  32.f,
+                                                  1.f);
+                k                 = ggml_rope_ext(ctx->ggml_ctx,
+                                                  k,
+                                                  input_pos,
+                                                  freq_factors,
+                                                  head_dim,
+                                                  GGML_ROPE_TYPE_NEOX,
+                                                  static_cast<int>(max_position_embeddings),
+                                                  rope_theta,
+                                                  1.f,
+                                                  0.f,
+                                                  1.f,
+                                                  32.f,
+                                                  1.f);
             } else if (arch == LLMArch::GEMMA2_2B) {
                 q = ggml_rope_ext(ctx->ggml_ctx,
                                   q,
@@ -1224,6 +1322,11 @@ namespace LLM {
                 k               = ggml_rope_multi(ctx->ggml_ctx, k, input_pos, nullptr, head_dim, sections, GGML_ROPE_TYPE_MROPE, 128000, 1000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
             }
 
+            if (unscaled_attention) {
+                // Gemma 4 attends with scaling=1.0; undo the helper's own 1/sqrt(head_dim).
+                q = ggml_ext_scale(ctx->ggml_ctx, q, std::sqrt(static_cast<float>(head_dim)));
+            }
+
             q = ggml_cont(ctx->ggml_ctx, ggml_ext_torch_permute(ctx->ggml_ctx, q, 0, 2, 1, 3));  // [N, num_heads, n_token, head_dim]
             q = ggml_reshape_3d(ctx->ggml_ctx, q, q->ne[0], q->ne[1], q->ne[2] * q->ne[3]);      // [N*num_heads, n_token, head_dim]
 
@@ -1262,15 +1365,30 @@ namespace LLM {
     protected:
         LLMArch arch;
         int sliding_attention;
+        bool has_layer_scalar;
         std::string post_attention_norm_name;
         std::string pre_ffw_norm_name;
         std::string post_ffw_norm_name;
 
+        void init_params(ggml_context* ctx,
+                         const String2TensorStorage& tensor_storage_map = {},
+                         std::string prefix                             = "") override {
+            GGMLBlock::init_params(ctx, tensor_storage_map, prefix);
+            if (has_layer_scalar) {
+                params["layer_scalar"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
+            }
+        }
+
     public:
         TransformerBlock(const LLMConfig& config, int layer_index)
             : arch(config.arch),
-              sliding_attention(0) {
-            if (config.arch == LLMArch::GEMMA3_12B) {
+              sliding_attention(0),
+              has_layer_scalar(config.layer_scalar) {
+            if (config.arch == LLMArch::GEMMA4_12B) {
+                post_attention_norm_name = "post_attention_layernorm";
+                pre_ffw_norm_name        = "pre_feedforward_layernorm";
+                post_ffw_norm_name       = "post_feedforward_layernorm";
+            } else if (config.arch == LLMArch::GEMMA3_12B || config.arch == LLMArch::GEMMA4_12B) {
                 post_attention_norm_name = "post_attention_norm";       // attn_post_norm
                 pre_ffw_norm_name        = "post_attention_layernorm";  // ffn_norm
                 post_ffw_norm_name       = "post_ffw_norm";             // ffn_post_norm
@@ -1284,7 +1402,10 @@ namespace LLM {
                 pre_ffw_norm_name = "post_attention_layernorm";  // ffn_norm
             }
 
-            blocks["self_attn"] = std::make_shared<Attention>(config);
+            if (!config.sliding_attention.empty()) {
+                sliding_attention = config.sliding_attention[layer_index % config.sliding_attention.size()];
+            }
+            blocks["self_attn"] = std::make_shared<Attention>(config, sliding_attention == 0);
             if (config.arch == LLMArch::GPT_OSS_20B) {
                 blocks["mlp"] = std::make_shared<GPTOSSMLP>(config);
             } else {
@@ -1301,9 +1422,6 @@ namespace LLM {
             if (!post_ffw_norm_name.empty()) {
                 blocks[post_ffw_norm_name] = std::make_shared<LLMRMSNorm>(config.hidden_size, config.rms_norm_eps, config.rms_norm_add);
             }
-            if (!config.sliding_attention.empty()) {
-                sliding_attention = config.sliding_attention[layer_index % config.sliding_attention.size()];
-            }
         }
 
         ggml_tensor* forward(GGMLRunnerContext* ctx,
@@ -1325,7 +1443,7 @@ namespace LLM {
             }
             ggml_tensor* block_attention_mask = attention_mask;
             int rope_index                    = 0;
-            if ((arch == LLMArch::GEMMA3_12B || arch == LLMArch::GPT_OSS_20B) && sliding_attention > 0) {
+            if ((arch == LLMArch::GEMMA3_12B || arch == LLMArch::GEMMA4_12B || arch == LLMArch::GPT_OSS_20B) && sliding_attention > 0) {
                 block_attention_mask = sliding_attention_mask;
                 rope_index           = 1;
             }
@@ -1352,6 +1470,10 @@ namespace LLM {
             }
             x = ggml_add_inplace(ctx->ggml_ctx, x, residual);
 
+            if (has_layer_scalar) {
+                x = ggml_mul(ctx->ggml_ctx, x, params["layer_scalar"]);
+            }
+
             return x;
         }
     };
@@ -1765,9 +1887,9 @@ namespace LLM {
                 enable_vision = false;
             }
             if (enable_vision) {
-                LOG_DEBUG("enable llm vision");
+                LOG_VERBOSE("enable llm vision");
                 if (config.llama_cpp_style) {
-                    LOG_DEBUG("llama.cpp style vision weight");
+                    LOG_VERBOSE("llama.cpp style vision weight");
                 }
             }
             model = LLM(config, enable_vision, config.llama_cpp_style);
@@ -1846,6 +1968,7 @@ namespace LLM {
                 config.arch == LLMArch::MINISTRAL_3_3B ||
                 config.arch == LLMArch::QWEN3 ||
                 config.arch == LLMArch::GEMMA3_12B ||
+                config.arch == LLMArch::GEMMA4_12B ||
                 config.arch == LLMArch::GEMMA2_2B ||
                 config.arch == LLMArch::GPT_OSS_20B) {
                 input_pos_vec.resize(n_tokens);
@@ -1910,7 +2033,7 @@ namespace LLM {
                 set_backend_tensor_data(attention_mask, attention_mask_vec.data());
             }
 
-            if (config.arch == LLMArch::GEMMA3_12B || config.arch == LLMArch::GPT_OSS_20B) {
+            if (config.arch == LLMArch::GEMMA3_12B || config.arch == LLMArch::GEMMA4_12B || config.arch == LLMArch::GPT_OSS_20B) {
                 int sliding_window = 0;
                 for (int window : config.sliding_attention) {
                     sliding_window = std::max(sliding_window, window);
@@ -1956,9 +2079,7 @@ namespace LLM {
                                   const ImageEmbeds& image_embeds,
                                   std::set<int> out_layers,
                                   bool return_all_hidden_states                      = false,
-                                  bool auto_free                                     = true,
-                                  bool free_compute_buffer                           = true,
-                                  bool free_compute_params                           = true,
+                                  bool auto_runner_end                               = true,
                                   const DeepStackImageEmbeds& deepstack_image_embeds = {},
                                   const std::vector<ImageGrid>& image_grids          = {}) {
             auto get_graph = [&]() -> ggml_cgraph* {
@@ -1970,7 +2091,7 @@ namespace LLM {
                                    out_layers,
                                    return_all_hidden_states);
             };
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, auto_free, free_compute_buffer, free_compute_params),
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, auto_runner_end),
                                                    input_ids.dim() + 1);
         }
 
@@ -2050,13 +2171,11 @@ namespace LLM {
 
         sd::Tensor<float> encode_image(const int n_threads,
                                        const sd::Tensor<float>& image,
-                                       bool auto_free           = false,
-                                       bool free_compute_buffer = false,
-                                       bool free_compute_params = false) {
+                                       bool auto_runner_end = false) {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_encode_image_graph(image);
             };
-            return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, auto_free, free_compute_buffer, free_compute_params));
+            return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, auto_runner_end));
         }
 
         ggml_cgraph* build_encode_image_outputs_graph(const sd::Tensor<float>& image_tensor) {
@@ -2164,13 +2283,11 @@ namespace LLM {
 
         std::vector<sd::Tensor<float>> encode_image_outputs(const int n_threads,
                                                             const sd::Tensor<float>& image,
-                                                            bool auto_free           = false,
-                                                            bool free_compute_buffer = false,
-                                                            bool free_compute_params = false) {
+                                                            bool auto_runner_end = false) {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_encode_image_outputs_graph(image);
             };
-            auto combined = take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, auto_free, free_compute_buffer, free_compute_params));
+            auto combined = take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, auto_runner_end));
             if (combined.empty()) {
                 return {};
             }
@@ -2189,20 +2306,14 @@ namespace LLM {
 
         std::vector<sd::Tensor<float>> encode_video_block_outputs(const int n_threads,
                                                                   const sd::Tensor<float>& frames,
-                                                                  bool auto_free           = false,
-                                                                  bool free_compute_buffer = false,
-                                                                  bool free_compute_params = false) {
+                                                                  bool auto_runner_end = false) {
             int grid_h        = static_cast<int>(frames.shape()[1] / config.vision.patch_size);
             int grid_w        = static_cast<int>(frames.shape()[0] / config.vision.patch_size);
             auto pixel_values = process_video_block_tensor(frames, config.vision);
             auto get_graph    = [&]() -> ggml_cgraph* {
                 return build_encode_video_block_outputs_graph(pixel_values, grid_h, grid_w);
             };
-            auto combined = take_or_empty(GGMLRunner::compute<float>(get_graph,
-                                                                     n_threads,
-                                                                     auto_free,
-                                                                     free_compute_buffer,
-                                                                     free_compute_params));
+            auto combined = take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, auto_runner_end));
             if (combined.empty()) {
                 return {};
             }
@@ -2264,7 +2375,7 @@ namespace LLM {
                     ss << "['" << item.first << "', " << item.second << "], ";
                 }
                 ss << "]";
-                LOG_DEBUG("parse '%s' to %s", text.c_str(), ss.str().c_str());
+                LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str());
             }
 
             std::vector<int> tokens;
@@ -2315,7 +2426,7 @@ namespace LLM {
                     out = std::move(out_opt);
                     print_sd_tensor(out, false, "image_embed");
                     image_embed = out;
-                    LOG_DEBUG("llm encode_image test done in %lldms", t1 - t0);
+                    LOG_VERBOSE("llm encode_image test done in %lldms", t1 - t0);
                 }
 
                 std::string placeholder  = "<|image_pad|>";
@@ -2355,7 +2466,7 @@ namespace LLM {
                 GGML_ASSERT(!out_opt.empty());
                 out = std::move(out_opt);
                 print_sd_tensor(out);
-                LOG_DEBUG("llm test done in %lldms", t1 - t0);
+                LOG_VERBOSE("llm test done in %lldms", t1 - t0);
             } else if (test_vit) {
                 // auto image = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 280, 280, 3);
                 // ggml_set_f32(image, 0.f);
@@ -2374,7 +2485,7 @@ namespace LLM {
                 // auto ref_out = load_tensor_from_file(ctx, "qwen2vl.bin");
                 // ggml_ext_tensor_diff(ref_out, out, 0.01f);
 
-                LOG_DEBUG("llm test done in %lldms", t1 - t0);
+                LOG_VERBOSE("llm test done in %lldms", t1 - t0);
             } else if (test_mistral) {
                 std::pair<int, int> prompt_attn_range;
                 std::string text        = "[SYSTEM_PROMPT]You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object\nattribution and actions without speculation.[/SYSTEM_PROMPT][INST]";
@@ -2399,7 +2510,7 @@ namespace LLM {
                 GGML_ASSERT(!out_opt.empty());
                 out = std::move(out_opt);
                 print_sd_tensor(out);
-                LOG_DEBUG("llm test done in %lldms", t1 - t0);
+                LOG_VERBOSE("llm test done in %lldms", t1 - t0);
             } else if (test_qwen3) {
                 std::pair<int, int> prompt_attn_range;
                 std::string text        = "<|im_start|>user\n";
@@ -2424,7 +2535,7 @@ namespace LLM {
                 GGML_ASSERT(!out_opt.empty());
                 out = std::move(out_opt);
                 print_sd_tensor(out);
-                LOG_DEBUG("llm test done in %lldms", t1 - t0);
+                LOG_VERBOSE("llm test done in %lldms", t1 - t0);
             } else {
                 std::pair<int, int> prompt_attn_range;
                 std::string text        = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n";
@@ -2449,7 +2560,7 @@ namespace LLM {
                 GGML_ASSERT(!out_opt.empty());
                 out = std::move(out_opt);
                 print_sd_tensor(out);
-                LOG_DEBUG("llm test done in %lldms", t1 - t0);
+                LOG_VERBOSE("llm test done in %lldms", t1 - t0);
             }
         }
 

src/model/te/t5.hpp

@@ -451,13 +451,11 @@ struct T5Runner : public GGMLRunner {
     sd::Tensor<float> compute(const int n_threads,
                               const sd::Tensor<int32_t>& input_ids,
                               const sd::Tensor<float>& attention_mask,
-                              bool auto_free           = true,
-                              bool free_compute_buffer = true,
-                              bool free_compute_params = true) {
+                              bool auto_runner_end = true) {
         auto get_graph = [&]() -> ggml_cgraph* {
             return build_graph(input_ids, attention_mask);
         };
-        return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, auto_free, free_compute_buffer, free_compute_params), 3);
+        return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, auto_runner_end), 3);
     }
 
     static std::vector<int> _relative_position_bucket(const std::vector<int>& relative_position,
@@ -556,7 +554,7 @@ struct T5Embedder {
                 ss << "['" << item.first << "', " << item.second << "], ";
             }
             ss << "]";
-            LOG_DEBUG("parse '%s' to %s", text.c_str(), ss.str().c_str());
+            LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str());
         }
 
         std::vector<int> tokens;
@@ -614,7 +612,7 @@ struct T5Embedder {
             GGML_ASSERT(!out_opt.empty());
             out = std::move(out_opt);
             print_sd_tensor(out);
-            LOG_DEBUG("t5 test done in %lldms", t1 - t0);
+            LOG_VERBOSE("t5 test done in %lldms", t1 - t0);
         }
     }
 

src/model/upscaler/esrgan.hpp

@@ -74,13 +74,13 @@ struct ESRGANConfig {
         }
 
         if (has_model_tensor || has_conv_up1 || has_conv_up2) {
-            LOG_DEBUG("esrgan: scale = %d, num_block = %d, num_in_ch = %d, num_out_ch = %d, num_feat = %d, num_grow_ch = %d",
-                      config.scale,
-                      config.num_block,
-                      config.num_in_ch,
-                      config.num_out_ch,
-                      config.num_feat,
-                      config.num_grow_ch);
+            LOG_VERBOSE("esrgan: scale = %d, num_block = %d, num_in_ch = %d, num_out_ch = %d, num_feat = %d, num_grow_ch = %d",
+                        config.scale,
+                        config.num_block,
+                        config.num_in_ch,
+                        config.num_out_ch,
+                        config.num_feat,
+                        config.num_grow_ch);
         }
         return config;
     }
@@ -265,7 +265,7 @@ struct ESRGAN : public GGMLRunner {
     sd::Tensor<float> compute(const int n_threads,
                               const sd::Tensor<float>& x) {
         auto get_graph = [&]() -> ggml_cgraph* { return build_graph(x); };
-        auto result    = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), x.dim());
+        auto result    = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
         return result;
     }
 };

src/model/upscaler/ltx_latent_upscaler.hpp

@@ -115,13 +115,13 @@ namespace LTXVUpsampler {
             }
 
             if (inferred) {
-                LOG_DEBUG("ltx latent upsampler: in_channels = %" PRId64 ", mid_channels = %" PRId64 ", num_blocks_per_stage = %d, spatial_scale = %.3f, temporal_up_factor = %d, rational_resampler = %d",
-                          config.in_channels,
-                          config.mid_channels,
-                          config.num_blocks_per_stage,
-                          config.spatial_scale,
-                          config.temporal_up_factor,
-                          config.rational_resampler);
+                LOG_VERBOSE("ltx latent upsampler: in_channels = %" PRId64 ", mid_channels = %" PRId64 ", num_blocks_per_stage = %d, spatial_scale = %.3f, temporal_up_factor = %d, rational_resampler = %d",
+                            config.in_channels,
+                            config.mid_channels,
+                            config.num_blocks_per_stage,
+                            config.spatial_scale,
+                            config.temporal_up_factor,
+                            config.rational_resampler);
             }
             return config;
         }
@@ -499,7 +499,7 @@ namespace LTXVUpsampler {
             }
             size_t expected_dim = static_cast<size_t>(x.dim());
             auto get_graph      = [&]() -> ggml_cgraph* { return build_graph(x); };
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), expected_dim);
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), expected_dim);
         }
     };
 

src/model/vae/auto_encoder_kl.hpp

@@ -744,7 +744,7 @@ struct AutoEncoderKL : public VAE {
         auto get_graph = [&]() -> ggml_cgraph* {
             return build_graph(z, decode_graph);
         };
-        return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), z.dim());
+        return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), z.dim());
     }
 
     sd::Tensor<float> gaussian_latent_sample(const sd::Tensor<float>& moments, std::shared_ptr<RNG> rng) {
@@ -864,7 +864,7 @@ struct AutoEncoderKL : public VAE {
             GGML_ASSERT(!out_opt.empty());
             out = std::move(out_opt);
             print_sd_tensor(out);
-            LOG_DEBUG("encode test done in %lldms", t1 - t0);
+            LOG_VERBOSE("encode test done in %lldms", t1 - t0);
         }
 
         if (false) {
@@ -884,7 +884,7 @@ struct AutoEncoderKL : public VAE {
             GGML_ASSERT(!out_opt.empty());
             out = std::move(out_opt);
             print_sd_tensor(out);
-            LOG_DEBUG("decode test done in %lldms", t1 - t0);
+            LOG_VERBOSE("decode test done in %lldms", t1 - t0);
         }
     };
 };

src/model/vae/hunyuan_vae.hpp

@@ -758,6 +758,15 @@ namespace Hunyuan {
             return "hunyuan_video_vae";
         }
 
+        bool supports_temporal_tiling(VAETemporalDirection direction) const override {
+            return direction == VAETemporalDirection::DECODE;
+        }
+
+        int get_temporal_tile_output_scale(VAETemporalDirection direction) const override {
+            SD_UNUSED(direction);
+            return 4;
+        }
+
         void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
             if (!decode_only) {
                 encoder.get_param_tensors(tensors, weight_prefix + ".encoder");
@@ -818,9 +827,7 @@ namespace Hunyuan {
             };
             auto output = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph,
                                                                                      n_threads,
-                                                                                     true,
-                                                                                     true,
-                                                                                     true),
+                                                                                     false),
                                                           graph_input.dim());
             if (!output.empty() && input.dim() == 4) {
                 output.squeeze_(2);

src/model/vae/ltx_audio_vae.hpp

@@ -172,12 +172,12 @@ namespace LTXV {
             if (config.audio_channels != 2 || config.latent_channels != 8 || config.mel_bins != 64) {
                 return config;
             }
-            LOG_DEBUG("ltx_audio_vae: sample_rate = %d, mel_bins = %d, latent_channels = %d, latent_frequency_bins = %d, has_bwe = %s",
-                      config.sample_rate,
-                      config.mel_bins,
-                      config.latent_channels,
-                      config.latent_frequency_bins,
-                      config.has_bwe ? "true" : "false");
+            LOG_VERBOSE("ltx_audio_vae: sample_rate = %d, mel_bins = %d, latent_channels = %d, latent_frequency_bins = %d, has_bwe = %s",
+                        config.sample_rate,
+                        config.mel_bins,
+                        config.latent_channels,
+                        config.latent_frequency_bins,
+                        config.has_bwe ? "true" : "false");
             return config;
         }
     };
@@ -1042,7 +1042,7 @@ namespace LTXV {
                 ggml_build_forward_expand(gf, waveform);
                 return gf;
             };
-            auto result = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), 4);
+            auto result = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), 4);
             int64_t t1  = ggml_time_ms();
             LOG_INFO("ltx audio vae decode completed, taking %.2fs", (t1 - t0) * 1.0f / 1000);
             return result;
@@ -1063,7 +1063,7 @@ namespace LTXV {
 
             GGML_ASSERT(!out.empty());
             print_sd_tensor(out, false, "ltx_audio_vae_out");
-            LOG_DEBUG("ltx audio vae test done in %lldms", t1 - t0);
+            LOG_VERBOSE("ltx audio vae test done in %lldms", t1 - t0);
         }
 
         static void load_from_file_and_test(const std::string& model_path,

src/model/vae/ltx_vae.hpp

@@ -1126,11 +1126,11 @@ namespace LTXVAE {
                          overlap, window);
                 overlap = window - 1;
             }
-            LOG_DEBUG("Using temporal tiling: temporal_tile_frames = %d, temporal_tile_overlap = %d, total frames = %d, resulting in %d tiles",
-                      window,
-                      overlap,
-                      (int)T,
-                      (T + window - overlap - 1) / (window - overlap));
+            LOG_VERBOSE("Using temporal tiling: temporal_tile_frames = %d, temporal_tile_overlap = %d, total frames = %d, resulting in %d tiles",
+                        window,
+                        overlap,
+                        (int)T,
+                        (T + window - overlap - 1) / (window - overlap));
             ggml_tensor* out = nullptr;
             for (int i = 0; i < (int)T - overlap; i += (window - overlap)) {
                 int feat_idx = 0;
@@ -1213,9 +1213,6 @@ struct LTXVideoVAE : public VAE {
     static constexpr int DEFAULT_TEMPORAL_TILE_OVERLAP = 1;
 
     bool decode_only;
-    bool temporal_tiling_enabled = false;
-    int temporal_tile_frames     = DEFAULT_TEMPORAL_TILE_FRAMES;
-    int temporal_tile_overlap    = DEFAULT_TEMPORAL_TILE_OVERLAP;
     int ltx_vae_version;
     bool timestep_conditioning;
     int patch_size;
@@ -1248,64 +1245,24 @@ struct LTXVideoVAE : public VAE {
         return "ltx_video_vae";
     }
 
-    void set_temporal_tiling_enabled(bool enabled) override {
-        temporal_tiling_enabled = enabled;
+    bool supports_temporal_tiling(VAETemporalDirection direction) const override {
+        return direction == VAETemporalDirection::DECODE;
     }
 
-    void set_tiling_params(const sd_tiling_params_t& params) override {
-        temporal_tiling_enabled = params.temporal_tiling;
-        temporal_tile_frames    = DEFAULT_TEMPORAL_TILE_FRAMES;
-        temporal_tile_overlap   = DEFAULT_TEMPORAL_TILE_OVERLAP;
+    int get_default_temporal_tile_frames(VAETemporalDirection direction) const override {
+        SD_UNUSED(direction);
+        return DEFAULT_TEMPORAL_TILE_FRAMES;
+    }
 
-        for (const auto& [key, value] : parse_key_value_args(params.extra_tiling_args, "LTX VAE extra tiling arg")) {
-            int parsed = 0;
-            if (!parse_strict_int(value, parsed)) {
-                LOG_WARN("ignoring invalid LTX VAE extra tiling arg '%s=%s'", key.c_str(), value.c_str());
-            } else if (key == "temporal_tile_frames") {
-                temporal_tile_frames = std::max(1, parsed);
-            } else if (key == "temporal_tile_overlap") {
-                temporal_tile_overlap = std::max(0, parsed);
-            } else {
-                LOG_WARN("ignoring unknown LTX VAE extra tiling arg '%s'", key.c_str());
-            }
-        }
+    int get_default_temporal_tile_overlap(VAETemporalDirection direction) const override {
+        SD_UNUSED(direction);
+        return DEFAULT_TEMPORAL_TILE_OVERLAP;
     }
 
     void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
         vae.get_param_tensors(tensors, weight_prefix);
     }
 
-    struct TemporalTilePlan {
-        int frames    = 1;
-        int overlap   = 0;
-        int stride    = 1;
-        int num_tiles = 1;
-    };
-
-    TemporalTilePlan resolve_temporal_tile_plan(int64_t total_frames) const {
-        TemporalTilePlan plan;
-        plan.frames  = std::max(1, temporal_tile_frames);
-        plan.overlap = std::max(0, temporal_tile_overlap);
-
-        if (plan.overlap >= plan.frames) {
-            LOG_WARN("temporal_tile_overlap (%d) is greater than or equal to temporal_tile_frames (%d), adjusting values to avoid empty decode windows",
-                     plan.overlap,
-                     plan.frames);
-            plan.overlap = plan.frames - 1;
-        }
-        if (total_frames > 1 && plan.overlap >= total_frames) {
-            LOG_WARN("temporal_tile_overlap (%d) is greater than or equal to total latent frames (%lld), adjusting values to decode at least one tile",
-                     plan.overlap,
-                     (long long)total_frames);
-            plan.overlap = static_cast<int>(total_frames - 1);
-        }
-
-        plan.stride          = std::max(1, plan.frames - plan.overlap);
-        int64_t tiled_frames = std::max<int64_t>(1, total_frames - plan.overlap);
-        plan.num_tiles       = total_frames > 0 ? static_cast<int>((tiled_frames + plan.stride - 1) / plan.stride) : 0;
-        return plan;
-    }
-
     std::string temporal_feat_cache_name(size_t feat_idx) const {
         return "ltx_vae_temporal_feat:" + std::to_string(feat_idx);
     }
@@ -1365,52 +1322,51 @@ struct LTXVideoVAE : public VAE {
 
     sd::Tensor<float> decode_temporal_tiled_streaming(const int n_threads,
                                                       const sd::Tensor<float>& input,
-                                                      size_t expected_dim) {
+                                                      size_t expected_dim,
+                                                      const VAETemporalTilingConfig& config) {
         const int64_t total_frames = input.shape()[2];
-        TemporalTilePlan plan      = resolve_temporal_tile_plan(total_frames);
+        auto plan                  = make_vae_temporal_tile_plan(total_frames, config);
 
-        LOG_DEBUG("Using streaming temporal tiling: temporal_tile_frames=%d, temporal_tile_overlap=%d, total latent frames=%lld, resulting in %d tiles",
-                  plan.frames,
-                  plan.overlap,
-                  (long long)total_frames,
-                  plan.num_tiles);
+        LOG_VERBOSE("Using streaming temporal tiling: temporal_tile_frames=%d, temporal_tile_overlap=%d, total latent frames=%lld, resulting in %d tiles",
+                    plan.tile_frames,
+                    plan.overlap,
+                    (long long)total_frames,
+                    (int)plan.tiles.size());
 
         free_cache_ctx_and_buffer();
-        cache_tensor_map.clear();
 
-        sd::Tensor<float> output;
-        for (int64_t start = 0; start < total_frames - plan.overlap; start += plan.stride) {
-            const int64_t end       = std::min<int64_t>(total_frames, start + plan.frames);
-            const int chunk_overlap = end < total_frames ? plan.overlap : 0;
-            auto z_chunk            = sd::ops::slice(input, 2, start, end);
-
-            LOG_DEBUG("LTX VAE temporal tile %lld/%d: latent frames [%lld, %lld), overlap=%d",
-                      (long long)(start / plan.stride + 1),
-                      plan.num_tiles,
-                      (long long)start,
-                      (long long)end,
-                      chunk_overlap);
+        auto output = process_vae_temporal_tiles(input, plan, [&](const sd::Tensor<float>& z_chunk, const VAETemporalTile& tile) {
+            LOG_VERBOSE("LTX VAE temporal tile %lld/%d: latent frames [%lld, %lld), overlap=%d",
+                        (long long)tile.index + 1,
+                        (int)plan.tiles.size(),
+                        (long long)tile.start,
+                        (long long)tile.end,
+                        tile.overlap);
 
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_temporal_tile_graph(z_chunk,
-                                                 static_cast<int>(start),
-                                                 chunk_overlap);
+                                                 static_cast<int>(tile.start),
+                                                 tile.overlap);
             };
-            auto chunk = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, true, true, true),
-                                                         expected_dim);
-            if (chunk.empty()) {
-                free_cache_ctx_and_buffer();
-                cache_tensor_map.clear();
-                return {};
-            }
-            output = output.empty() ? std::move(chunk) : sd::ops::concat(output, chunk, 2);
-        }
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false),
+                                                   expected_dim);
+        });
 
         free_cache_ctx_and_buffer();
-        cache_tensor_map.clear();
         return output;
     }
 
+    sd::Tensor<float> _compute_temporal_tiled(const int n_threads,
+                                              const sd::Tensor<float>& input,
+                                              VAETemporalDirection direction,
+                                              const VAETemporalTilingConfig& config) override {
+        GGML_ASSERT(direction == VAETemporalDirection::DECODE);
+        return decode_temporal_tiled_streaming(n_threads,
+                                               input,
+                                               static_cast<size_t>(input.dim()),
+                                               config);
+    }
+
     ggml_cgraph* build_latent_statistics_graph(const sd::Tensor<float>& z_tensor, bool normalize) {
         ggml_cgraph* gf = new_graph_custom(1024);
         ggml_tensor* z  = make_input(z_tensor);
@@ -1446,13 +1402,10 @@ struct LTXVideoVAE : public VAE {
                 input = sd::ops::slice(input, 2, 0, cropped_t);
             }
         }
-        if (decode_graph && temporal_tiling_enabled && input.dim() == 5 && input.shape()[2] > 1) {
-            return decode_temporal_tiled_streaming(n_threads, input, expected_dim);
-        }
         auto get_graph = [&]() -> ggml_cgraph* {
             return build_graph(input, decode_graph);
         };
-        auto result = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), expected_dim);
+        auto result = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), expected_dim);
         if (result.empty()) {
             return {};
         }
@@ -1465,7 +1418,7 @@ struct LTXVideoVAE : public VAE {
         auto get_graph = [&]() -> ggml_cgraph* {
             return build_latent_statistics_graph(z, normalize);
         };
-        return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false),
+        return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false),
                                                static_cast<size_t>(z.dim()));
     }
 
@@ -1512,7 +1465,7 @@ struct LTXVideoVAE : public VAE {
 
         GGML_ASSERT(!out.empty());
         print_sd_tensor(out, false, "ltx_vae_out");
-        LOG_DEBUG("ltx vae test done in %lldms", t1 - t0);
+        LOG_VERBOSE("ltx vae test done in %lldms", t1 - t0);
     }
 
     static void load_from_file_and_test(const std::string& model_path,

src/model/vae/mage_vae.hpp

@@ -490,7 +490,7 @@ namespace MageVAE {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_graph(input, decode_graph);
             };
-            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), input.dim());
+            return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), input.dim());
         }
 
         int get_encoder_output_channels(int input_channels) override {

src/model/vae/minimax_h3_audio_vae.hpp

@@ -154,8 +154,9 @@ namespace MiniMaxH3 {
                          const String2TensorStorage& tensor_storage_map = {},
                          const std::string prefix                       = "") override {
             GGMLBlock::init_params(ctx, tensor_storage_map, prefix);
-            params["q_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
-            params["v_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
+            params["q_bias"]      = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
+            params["zero_k_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
+            params["v_bias"]      = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, in_channels);
         }
 
         ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
@@ -166,7 +167,7 @@ namespace MiniMaxH3 {
                 return ggml_reshape_4d(ctx->ggml_ctx, bias, bias->ne[0], 1, 1, 1);
             };
             auto q = ggml_add(ctx->ggml_ctx, qkv[0], bias_shape(params["q_bias"]));
-            auto k = qkv[1];
+            auto k = ggml_add(ctx->ggml_ctx, qkv[1], bias_shape(params["zero_k_bias"]));
             auto v = ggml_add(ctx->ggml_ctx, qkv[2], bias_shape(params["v_bias"]));
 
             int64_t sequence = x->ne[1];
@@ -358,22 +359,38 @@ namespace MiniMaxH3 {
         }
 
         ggml_tensor* encode(GGMLRunnerContext* ctx, ggml_tensor* waveform) {
-            GGML_ASSERT(waveform->ne[1] == 2);
+            GGML_ASSERT(waveform->ne[1] * waveform->ne[2] * waveform->ne[3] == 2);
             auto encoder   = std::dynamic_pointer_cast<AudioEncoder>(blocks["encoder"]);
             auto pre       = std::dynamic_pointer_cast<AudioAttentionProjection>(blocks["pre_block"]);
             auto mean_proj = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["mean_proj"]);
 
-            waveform = ggml_reshape_3d(ctx->ggml_ctx, waveform, waveform->ne[0], 1, waveform->ne[1]);
-            auto x   = encoder->forward(ctx, waveform);  // [B*S, 2048, T]
-            x        = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
-            x        = pre->forward(ctx, x);
-            x        = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
-            auto z   = mean_proj->forward(ctx, x);
+            // GGML's batched conv1d storage interleaves the stream dimension
+            // with output channels. Subsequent layers then read stereo samples
+            // as adjacent feature channels. Run each mono stream independently,
+            // matching PyTorch's reshape(B*S, 1, samples), and concatenate only
+            // the completed normalized latents.
+            const int64_t streams = waveform->ne[2] * waveform->ne[3];
+            waveform              = ggml_reshape_3d(ctx->ggml_ctx,
+                                                    waveform,
+                                                    waveform->ne[0],
+                                                    1,
+                                                    streams);
+            ggml_tensor* stereo_z = nullptr;
+            for (int64_t stream = 0; stream < streams; ++stream) {
+                auto mono = ggml_ext_slice(ctx->ggml_ctx, waveform, 2, stream, stream + 1);
+                auto x    = encoder->forward(ctx, mono);
+                x         = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
+                x         = pre->forward(ctx, x);
+                x         = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
+                auto z    = mean_proj->forward(ctx, x);
 
-            auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1);
-            auto std  = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1);
-            z         = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std);
-            return ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 2, 1, 3));
+                auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1);
+                auto std  = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1);
+                z         = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std);
+                z         = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 2, 1, 3));
+                stereo_z  = stereo_z == nullptr ? z : ggml_concat(ctx->ggml_ctx, stereo_z, z, 1);
+            }
+            return stereo_z;
         }
 
         ggml_tensor* decode(GGMLRunnerContext* ctx, ggml_tensor* latent) {
@@ -463,7 +480,7 @@ namespace MiniMaxH3 {
                 return graph;
             };
             auto result = restore_trailing_singleton_dims(
-                GGMLRunner::compute<float>(get_graph, n_threads, false, false, false),
+                GGMLRunner::compute<float>(get_graph, n_threads, false),
                 4);
             int64_t t1 = ggml_time_ms();
             LOG_INFO("MiniMax-H3 audio VAE encode completed, taking %.2fs",
@@ -483,7 +500,7 @@ namespace MiniMaxH3 {
                 return graph;
             };
             auto result = restore_trailing_singleton_dims(
-                GGMLRunner::compute<float>(get_graph, n_threads, false, false, false),
+                GGMLRunner::compute<float>(get_graph, n_threads, false),
                 4);
             int64_t t1 = ggml_time_ms();
             LOG_INFO("MiniMax-H3 audio VAE decode completed, taking %.2fs",

src/model/vae/minimax_h3_vae.hpp

@@ -558,10 +558,11 @@ namespace MiniMaxH3VAE {
         }
 
         static sd_tiling_params_t h3_tiling(sd_tiling_params_t params) {
-            params.enabled        = true;
-            params.tile_size_x    = 16;
-            params.tile_size_y    = 16;
-            params.target_overlap = 0.25f;
+            params.enabled         = true;
+            params.temporal_tiling = false;
+            params.tile_size_x     = 16;
+            params.tile_size_y     = 16;
+            params.target_overlap  = 0.25f;
             return params;
         }
 
@@ -624,15 +625,13 @@ namespace MiniMaxH3VAE {
             if (pad > 0) {
                 input = repeat_last_frame(input, pad);
             }
-            sd::Tensor<float> result;
-            for (int64_t start = 0; start < input.shape()[2]; start += 17) {
-                auto chunk   = sd::ops::slice(input, 2, start, start + 17);
-                auto encoded = VAE::encode(n_threads, chunk, tiling, circular_x, circular_y);
-                if (encoded.empty()) {
-                    return {};
-                }
-                result = result.empty() ? std::move(encoded)
-                                        : sd::ops::concat(result, encoded, 2);
+            auto plan   = make_vae_temporal_tile_plan(input.shape()[2], {17, 0});
+            auto result = process_vae_temporal_tiles(input, plan, [&](const sd::Tensor<float>& chunk, const VAETemporalTile& tile) {
+                SD_UNUSED(tile);
+                return VAE::encode(n_threads, chunk, tiling, circular_x, circular_y);
+            });
+            if (result.empty()) {
+                return {};
             }
             if (result.shape()[2] > 3) {
                 result = sd::ops::slice(result, 2, 0, result.shape()[2] - 3);
@@ -685,22 +684,21 @@ namespace MiniMaxH3VAE {
                 input = repeat_last_frame(input, pad_tokens);
             }
 
-            sd::Tensor<float> result;
             sd::Tensor<float> overlap;
-            for (int64_t i = 0; i < num_chunks; ++i) {
-                int64_t start = i * tokens_per_chunk;
-                int64_t end   = std::min(start + tokens_per_chunk + token_overlap,
-                                         input.shape()[2]);
-                auto chunk    = sd::ops::slice(input, 2, start, end);
-                auto decoded  = VAE::decode(n_threads,
-                                            chunk,
-                                            tiling,
-                                            true,
-                                            circular_x,
-                                            circular_y,
-                                            silent);
+            auto plan = make_vae_temporal_tile_plan(
+                input.shape()[2],
+                {static_cast<int>(tokens_per_chunk + token_overlap), static_cast<int>(token_overlap)});
+            GGML_ASSERT(plan.tiles.size() == static_cast<size_t>(num_chunks));
+            auto result = process_vae_temporal_tiles(input, plan, [&](const sd::Tensor<float>& chunk, const VAETemporalTile& tile) {
+                auto decoded = VAE::decode(n_threads,
+                                           chunk,
+                                           tiling,
+                                           true,
+                                           circular_x,
+                                           circular_y,
+                                           silent);
                 if (decoded.empty()) {
-                    return {};
+                    return sd::Tensor<float>();
                 }
 
                 int64_t first_end = std::min<int64_t>(frames_per_chunk, decoded.shape()[2]);
@@ -712,8 +710,6 @@ namespace MiniMaxH3VAE {
                     first   = blend_temporal(overlap, first, frame_overlap);
                     overlap = {};
                 }
-                result = result.empty() ? std::move(first)
-                                        : sd::ops::concat(result, first, 2);
 
                 if (decoded.shape()[2] > frames_per_chunk + frame_pre_padding) {
                     overlap = sd::ops::slice(decoded,
@@ -721,10 +717,14 @@ namespace MiniMaxH3VAE {
                                              frames_per_chunk + frame_pre_padding,
                                              decoded.shape()[2]);
                 }
-                if (i == num_chunks - 1 && !overlap.empty()) {
-                    result  = sd::ops::concat(result, overlap, 2);
+                if (tile.last && !overlap.empty()) {
+                    first   = sd::ops::concat(first, overlap, 2);
                     overlap = {};
                 }
+                return first;
+            });
+            if (result.empty()) {
+                return {};
             }
 
             int64_t expected_frames = input.shape()[2] <= 1 ? 1 : ((x.shape()[2] - 2) / 5) * 17 + 5;
@@ -793,8 +793,6 @@ namespace MiniMaxH3VAE {
             return restore_trailing_singleton_dims(
                 GGMLRunner::compute<float>(get_graph,
                                            n_threads,
-                                           false,
-                                           false,
                                            false),
                 5);
         }

src/model/vae/tae.hpp

@@ -65,7 +65,7 @@ public:
 
         if (n_in != n_out) {
             auto skip = std::dynamic_pointer_cast<Conv2d>(blocks["skip"]);
-            LOG_DEBUG("skip");
+            LOG_VERBOSE("skip");
             x = skip->forward(ctx, x);
         }
 
@@ -787,7 +787,7 @@ struct TinyImageAutoEncoder : public VAE {
             return build_graph(z_tensor, decode_graph);
         };
 
-        return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), z_tensor.dim());
+        return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), z_tensor.dim());
     }
 };
 
@@ -819,6 +819,21 @@ struct TinyVideoAutoEncoder : public VAE {
         return "taehv";
     }
 
+    bool supports_temporal_tiling(VAETemporalDirection direction) const override {
+        return direction == VAETemporalDirection::DECODE && !sd_version_is_minimax_h3(version);
+    }
+
+    int get_temporal_tile_output_scale(VAETemporalDirection direction) const override {
+        SD_UNUSED(direction);
+        int scale = 1;
+        for (bool upscale : taehv.time_upscale) {
+            if (upscale) {
+                scale *= 2;
+            }
+        }
+        return scale;
+    }
+
     void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
         taehv.get_param_tensors(tensors, weight_prefix);
     }
@@ -857,7 +872,7 @@ struct TinyVideoAutoEncoder : public VAE {
             return build_graph(z_tensor, decode_graph);
         };
 
-        return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), z_tensor.dim());
+        return restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), z_tensor.dim());
     }
 };
 

src/model/vae/vae.hpp

@@ -3,6 +3,7 @@
 
 #include "core/tensor_ggml.hpp"
 #include "model/common/block.hpp"
+#include "model/vae/vae_tiling.hpp"
 #include "model_manager.h"
 
 struct VAE : public GGMLRunner {
@@ -14,6 +15,87 @@ protected:
                                        const sd::Tensor<float>& z,
                                        bool decode_graph) = 0;
 
+    virtual bool supports_temporal_tiling(VAETemporalDirection direction) const {
+        SD_UNUSED(direction);
+        return false;
+    }
+
+    virtual int get_default_temporal_tile_frames(VAETemporalDirection direction) const {
+        SD_UNUSED(direction);
+        return 4;
+    }
+
+    virtual int get_default_temporal_tile_overlap(VAETemporalDirection direction) const {
+        SD_UNUSED(direction);
+        return 1;
+    }
+
+    virtual int get_temporal_tile_output_scale(VAETemporalDirection direction) const {
+        SD_UNUSED(direction);
+        return 1;
+    }
+
+    virtual sd::Tensor<float> _compute_temporal_tiled(const int n_threads,
+                                                      const sd::Tensor<float>& input,
+                                                      VAETemporalDirection direction,
+                                                      const VAETemporalTilingConfig& config) {
+        if (direction != VAETemporalDirection::DECODE) {
+            return _compute(n_threads, input, false);
+        }
+
+        VAETemporalTilingConfig resolved_config = config;
+        const int output_scale                  = get_temporal_tile_output_scale(direction);
+        if (output_scale > 1 &&
+            resolved_config.overlap == 0 &&
+            input.shape()[2] > resolved_config.tile_frames) {
+            LOG_WARN("%s temporal decode requires at least one overlapping latent frame; using overlap=1",
+                     get_desc().c_str());
+            resolved_config.overlap = 1;
+        }
+
+        auto plan = make_vae_temporal_tile_plan(input.shape()[2], resolved_config);
+        LOG_VERBOSE("%s temporal tiling: tile_frames=%d, overlap=%d, total_frames=%lld, tiles=%d",
+                    get_desc().c_str(),
+                    plan.tile_frames,
+                    plan.overlap,
+                    (long long)input.shape()[2],
+                    (int)plan.tiles.size());
+        return process_vae_temporal_tiles_blended(
+            input,
+            plan,
+            output_scale,
+            [&](const sd::Tensor<float>& input_tile, const VAETemporalTile& tile) {
+                LOG_VERBOSE("%s temporal tile %d/%d: input frames [%lld, %lld)",
+                            get_desc().c_str(),
+                            tile.index + 1,
+                            (int)plan.tiles.size(),
+                            (long long)tile.start,
+                            (long long)tile.end);
+                return _compute(n_threads, input_tile, true);
+            });
+    }
+
+    sd::Tensor<float> compute_with_temporal_tiling(const int n_threads,
+                                                   const sd::Tensor<float>& input,
+                                                   VAETemporalDirection direction,
+                                                   const sd_tiling_params_t& tiling_params) {
+        if (!tiling_params.temporal_tiling || input.dim() != 5 || input.shape()[2] <= 1) {
+            return _compute(n_threads, input, direction == VAETemporalDirection::DECODE);
+        }
+        if (!supports_temporal_tiling(direction)) {
+            LOG_WARN("%s does not support temporal tiling for %s; processing the full temporal dimension",
+                     get_desc().c_str(),
+                     direction == VAETemporalDirection::DECODE ? "decode" : "encode");
+            return _compute(n_threads, input, direction == VAETemporalDirection::DECODE);
+        }
+
+        auto config = resolve_vae_temporal_tiling_config(
+            tiling_params,
+            get_default_temporal_tile_frames(direction),
+            get_default_temporal_tile_overlap(direction));
+        return _compute_temporal_tiled(n_threads, input, direction, config);
+    }
+
     static inline void scale_tensor_to_minus1_1(sd::Tensor<float>* tensor) {
         GGML_ASSERT(tensor != nullptr);
         for (int64_t i = 0; i < tensor->numel(); ++i) {
@@ -40,10 +122,15 @@ protected:
                                     bool circular_x,
                                     bool circular_y,
                                     bool decode_graph,
+                                    const sd_tiling_params_t& tiling_params,
                                     const char* error_message,
                                     bool silent = false) {
         auto on_processing = [&](const sd::Tensor<float>& input_tile) {
-            auto output_tile = _compute(n_threads, input_tile, decode_graph);
+            auto output_tile = compute_with_temporal_tiling(
+                n_threads,
+                input_tile,
+                decode_graph ? VAETemporalDirection::DECODE : VAETemporalDirection::ENCODE,
+                tiling_params);
             if (output_tile.empty()) {
                 LOG_ERROR("%s", error_message);
                 return sd::Tensor<float>();
@@ -86,6 +173,10 @@ public:
 
     virtual int get_encoder_output_channels(int input_channels) = 0;
 
+    bool can_temporal_tile_decode() const {
+        return supports_temporal_tiling(VAETemporalDirection::DECODE);
+    }
+
     void get_tile_sizes(int& tile_size_x,
                         int& tile_size_y,
                         float& tile_overlap,
@@ -139,7 +230,7 @@ public:
             const float encode_tile_factor = sd_version_is_minimax_h3(version) ? 1.f : (sd_version_is_wan(version) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version)) ? 1.30539f
                                                                                                                                                                                             : 2.0f;
             get_tile_sizes(tile_size_x, tile_size_y, tile_overlap, tiling_params, W, H, encode_tile_factor);
-            LOG_DEBUG("VAE Tile size: %dx%d", tile_size_x, tile_size_y);
+            LOG_VERBOSE("VAE Tile size: %dx%d", tile_size_x, tile_size_y);
             output = tiled_compute(input,
                                    n_threads,
                                    static_cast<int>(W),
@@ -151,19 +242,23 @@ public:
                                    circular_x,
                                    circular_y,
                                    false,
+                                   tiling_params,
                                    "vae encode compute failed while processing a tile");
         } else {
-            output = _compute(n_threads, input, false);
+            output = compute_with_temporal_tiling(n_threads,
+                                                  input,
+                                                  VAETemporalDirection::ENCODE,
+                                                  tiling_params);
         }
 
-        runner_done();
+        runner_end();
 
         if (output.empty()) {
             LOG_ERROR("vae encode compute failed");
             return {};
         }
         int64_t t1 = ggml_time_ms();
-        LOG_DEBUG("computing vae encode graph completed, taking %.2fs", (t1 - t0) * 1.0f / 1000);
+        LOG_VERBOSE("computing vae encode graph completed, taking %.2fs", (t1 - t0) * 1.0f / 1000);
         return std::move(output);
     }
 
@@ -177,7 +272,6 @@ public:
         int64_t t0              = ggml_time_ms();
         sd::Tensor<float> input = x;
         sd::Tensor<float> output;
-        set_tiling_params(tiling_params);
 
         if (tiling_params.enabled) {
             const int scale_factor = get_scale_factor();
@@ -187,7 +281,7 @@ public:
             int tile_size_x, tile_size_y;
             get_tile_sizes(tile_size_x, tile_size_y, tile_overlap, tiling_params, input.shape()[0], input.shape()[1]);
             if (!silent) {
-                LOG_DEBUG("VAE Tile size: %dx%d", tile_size_x, tile_size_y);
+                LOG_VERBOSE("VAE Tile size: %dx%d", tile_size_x, tile_size_y);
             }
             output = tiled_compute(
                 input,
@@ -201,13 +295,17 @@ public:
                 circular_x,
                 circular_y,
                 true,
+                tiling_params,
                 "vae decode compute failed while processing a tile",
                 silent);
         } else {
-            output = _compute(n_threads, input, true);
+            output = compute_with_temporal_tiling(n_threads,
+                                                  input,
+                                                  VAETemporalDirection::DECODE,
+                                                  tiling_params);
         }
 
-        runner_done();
+        runner_end();
 
         if (output.empty()) {
             LOG_ERROR("vae decode compute failed");
@@ -217,7 +315,7 @@ public:
             scale_tensor_to_0_1(&output);
         }
         int64_t t1 = ggml_time_ms();
-        LOG_DEBUG("computing vae decode graph completed, taking %.2fs", (t1 - t0) * 1.0f / 1000);
+        LOG_VERBOSE("computing vae decode graph completed, taking %.2fs", (t1 - t0) * 1.0f / 1000);
         return std::move(output);
     }
 
@@ -226,10 +324,6 @@ public:
     virtual sd::Tensor<float> vae_to_diffusion_latents(const sd::Tensor<float>& latents)                           = 0;
     virtual void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors)                                   = 0;
     virtual void set_conv2d_scale(float scale) { SD_UNUSED(scale); };
-    virtual void set_temporal_tiling_enabled(bool enabled) { SD_UNUSED(enabled); };
-    virtual void set_tiling_params(const sd_tiling_params_t& params) {
-        set_temporal_tiling_enabled(params.temporal_tiling);
-    };
 };
 
 struct FakeVAE : public VAE {

src/model/vae/vae_tiling.hpp

@@ -0,0 +1,213 @@
+#ifndef __SD_MODEL_VAE_VAE_TILING_HPP__
+#define __SD_MODEL_VAE_VAE_TILING_HPP__
+
+#include <algorithm>
+#include <cstdint>
+#include <utility>
+#include <vector>
+
+#include "core/tensor.hpp"
+#include "core/util.h"
+
+enum class VAETemporalDirection {
+    ENCODE,
+    DECODE,
+};
+
+struct VAETemporalTilingConfig {
+    int tile_frames = 1;
+    int overlap     = 0;
+};
+
+struct VAETemporalTile {
+    int index     = 0;
+    int64_t start = 0;
+    int64_t end   = 0;
+    int overlap   = 0;
+    bool first    = false;
+    bool last     = false;
+};
+
+struct VAETemporalTilePlan {
+    int tile_frames = 1;
+    int overlap     = 0;
+    int stride      = 1;
+    std::vector<VAETemporalTile> tiles;
+};
+
+inline VAETemporalTilingConfig resolve_vae_temporal_tiling_config(const sd_tiling_params_t& params,
+                                                                  int default_tile_frames,
+                                                                  int default_overlap) {
+    VAETemporalTilingConfig config;
+    config.tile_frames = std::max(1, default_tile_frames);
+    config.overlap     = std::max(0, default_overlap);
+
+    for (const auto& [key, value] : parse_key_value_args(params.extra_tiling_args, "VAE extra tiling arg")) {
+        if (key != "temporal_tile_frames" && key != "temporal_tile_size" && key != "temporal_tile_overlap") {
+            continue;
+        }
+
+        int parsed = 0;
+        if (!parse_strict_int(value, parsed)) {
+            LOG_WARN("ignoring invalid VAE extra tiling arg '%s=%s'", key.c_str(), value.c_str());
+        } else if (key == "temporal_tile_overlap") {
+            config.overlap = std::max(0, parsed);
+        } else {
+            config.tile_frames = std::max(1, parsed);
+        }
+    }
+    return config;
+}
+
+inline VAETemporalTilePlan make_vae_temporal_tile_plan(int64_t total_frames,
+                                                       const VAETemporalTilingConfig& config) {
+    VAETemporalTilePlan plan;
+    plan.tile_frames = std::max(1, config.tile_frames);
+    plan.overlap     = std::max(0, config.overlap);
+    if (total_frames <= 1) {
+        plan.overlap = 0;
+    }
+
+    if (plan.overlap >= plan.tile_frames) {
+        LOG_WARN("temporal_tile_overlap (%d) is greater than or equal to temporal_tile_frames (%d), adjusting values to avoid empty decode windows",
+                 plan.overlap,
+                 plan.tile_frames);
+        plan.overlap = plan.tile_frames - 1;
+    }
+    if (total_frames > 1 && plan.overlap >= total_frames) {
+        LOG_WARN("temporal_tile_overlap (%d) is greater than or equal to total frames (%lld), adjusting values to process at least one tile",
+                 plan.overlap,
+                 (long long)total_frames);
+        plan.overlap = static_cast<int>(total_frames - 1);
+    }
+
+    plan.stride = std::max(1, plan.tile_frames - plan.overlap);
+    for (int64_t start = 0; start < total_frames - plan.overlap; start += plan.stride) {
+        VAETemporalTile tile;
+        tile.index   = static_cast<int>(plan.tiles.size());
+        tile.start   = start;
+        tile.end     = std::min<int64_t>(total_frames, start + plan.tile_frames);
+        tile.overlap = tile.end < total_frames ? plan.overlap : 0;
+        tile.first   = start == 0;
+        tile.last    = tile.end == total_frames;
+        plan.tiles.push_back(tile);
+    }
+    return plan;
+}
+
+template <typename Fn>
+inline sd::Tensor<float> process_vae_temporal_tiles(const sd::Tensor<float>& input,
+                                                    const VAETemporalTilePlan& plan,
+                                                    Fn&& on_processing) {
+    sd::Tensor<float> output;
+    for (const auto& tile : plan.tiles) {
+        auto input_tile  = sd::ops::slice(input, 2, tile.start, tile.end);
+        auto output_tile = on_processing(input_tile, tile);
+        if (output_tile.empty()) {
+            return {};
+        }
+        output = output.empty() ? std::move(output_tile)
+                                : sd::ops::concat(output, output_tile, 2);
+    }
+    return output;
+}
+
+template <typename Fn>
+inline sd::Tensor<float> process_vae_temporal_tiles_blended(const sd::Tensor<float>& input,
+                                                            const VAETemporalTilePlan& plan,
+                                                            int output_scale,
+                                                            Fn&& on_processing) {
+    GGML_ASSERT(output_scale >= 1);
+    const int64_t output_frames = 1 + (input.shape()[2] - 1) * output_scale;
+    const int overlap_frames    = plan.overlap > 0 ? 1 + (plan.overlap - 1) * output_scale : 0;
+    std::vector<float> weights(static_cast<size_t>(output_frames), 0.f);
+    sd::Tensor<float> output;
+
+    auto smootherstep = [](float value) {
+        return value * value * value * (value * (value * 6.f - 15.f) + 10.f);
+    };
+
+    for (const auto& tile : plan.tiles) {
+        auto input_tile  = sd::ops::slice(input, 2, tile.start, tile.end);
+        auto output_tile = on_processing(input_tile, tile);
+        if (output_tile.empty()) {
+            return {};
+        }
+
+        const int64_t expected_tile_frames = 1 + (input_tile.shape()[2] - 1) * output_scale;
+        if (output_tile.dim() < 3 || output_tile.shape()[2] != expected_tile_frames) {
+            LOG_ERROR("unexpected temporal tile output shape: expected %lld frames, got %lld",
+                      (long long)expected_tile_frames,
+                      output_tile.dim() < 3 ? -1LL : (long long)output_tile.shape()[2]);
+            return {};
+        }
+
+        if (output.empty()) {
+            auto output_shape = output_tile.shape();
+            output_shape[2]   = output_frames;
+            output            = sd::Tensor<float>::zeros(std::move(output_shape));
+        } else {
+            if (output.dim() != output_tile.dim()) {
+                LOG_ERROR("temporal tile output rank mismatch: expected %lld, got %lld",
+                          (long long)output.dim(),
+                          (long long)output_tile.dim());
+                return {};
+            }
+            for (size_t dim = 0; dim < static_cast<size_t>(output.dim()); ++dim) {
+                if (dim != 2 && output.shape()[dim] != output_tile.shape()[dim]) {
+                    LOG_ERROR("temporal tile output shape mismatch at dimension %zu", dim);
+                    return {};
+                }
+            }
+        }
+
+        const int64_t output_start = tile.start * output_scale;
+        const int64_t inner        = output.shape()[0] * output.shape()[1];
+        const int64_t outer        = output.numel() / (inner * output.shape()[2]);
+        const int64_t tile_frames  = output_tile.shape()[2];
+        for (int64_t frame = 0; frame < tile_frames; ++frame) {
+            float weight = 1.f;
+            if (!tile.first && overlap_frames > 0 && frame < overlap_frames) {
+                weight *= smootherstep(static_cast<float>(frame + 1) /
+                                       static_cast<float>(overlap_frames + 1));
+            }
+            if (!tile.last && overlap_frames > 0 && frame >= tile_frames - overlap_frames) {
+                weight *= smootherstep(static_cast<float>(tile_frames - frame) /
+                                       static_cast<float>(overlap_frames + 1));
+            }
+
+            const int64_t output_frame = output_start + frame;
+            GGML_ASSERT(output_frame >= 0 && output_frame < output_frames);
+            weights[static_cast<size_t>(output_frame)] += weight;
+            for (int64_t outer_index = 0; outer_index < outer; ++outer_index) {
+                const int64_t src_offset = (outer_index * tile_frames + frame) * inner;
+                const int64_t dst_offset = (outer_index * output_frames + output_frame) * inner;
+                for (int64_t inner_index = 0; inner_index < inner; ++inner_index) {
+                    output[dst_offset + inner_index] += output_tile[src_offset + inner_index] * weight;
+                }
+            }
+        }
+    }
+
+    if (output.empty()) {
+        return {};
+    }
+    const int64_t inner = output.shape()[0] * output.shape()[1];
+    const int64_t outer = output.numel() / (inner * output.shape()[2]);
+    for (int64_t frame = 0; frame < output_frames; ++frame) {
+        const float weight = weights[static_cast<size_t>(frame)];
+        if (weight <= 0.f) {
+            LOG_ERROR("temporal tiling left output frame %lld uncovered", (long long)frame);
+            return {};
+        }
+        for (int64_t outer_index = 0; outer_index < outer; ++outer_index) {
+            const int64_t offset = (outer_index * output_frames + frame) * inner;
+            for (int64_t inner_index = 0; inner_index < inner; ++inner_index) {
+                output[offset + inner_index] /= weight;
+            }
+        }
+    }
+    return output;
+}
+
+#endif  // __SD_MODEL_VAE_VAE_TILING_HPP__

src/model/vae/wan_vae.hpp

@@ -1219,24 +1219,40 @@ namespace WAN {
             return out;
         }
 
-        ggml_tensor* decode_partial(GGMLRunnerContext* ctx,
-                                    ggml_tensor* z,
-                                    int i,
-                                    int64_t b = 1) {
+        ggml_tensor* decode_tiled_chunk(GGMLRunnerContext* ctx,
+                                        ggml_tensor* z,
+                                        int chunk_idx,
+                                        int64_t b = 1) {
             // z: [b*c, t, h, w]
             GGML_ASSERT(b == 1);
 
             auto decoder = std::dynamic_pointer_cast<Decoder3d>(blocks["decoder"]);
             auto conv2   = std::dynamic_pointer_cast<CausalConv3d>(blocks["conv2"]);
 
-            auto x = conv2->forward(ctx, z);
-            // sd::ggml_graph_cut::mark_graph_cut(x, "wan_vae.decode_partial.prelude", "x");
-            auto in   = ggml_ext_slice(ctx->ggml_ctx, x, 2, i, i + 1);  // [b*c, 1, h, w]
-            _conv_idx = 0;
-            auto out  = decoder->forward(ctx, in, b, _feat_map, _conv_idx, i);
-            out       = unpatchify(ctx->ggml_ctx, out, patch_size, b);
-            // sd::ggml_graph_cut::mark_graph_cut(out, "wan_vae.decode_partial.final", "out");
-            return out;
+            ggml_tensor* x;
+            if (is_2D) {
+                auto conv2_2d = std::dynamic_pointer_cast<Conv2dBut3d>(blocks["conv2"]);
+                x             = conv2_2d->forward(ctx, z);
+            } else {
+                x = conv2->forward(ctx, z);
+            }
+
+            ggml_tensor* out = nullptr;
+            for (int64_t frame = 0; frame < x->ne[2]; ++frame) {
+                const int global_frame = chunk_idx + static_cast<int>(frame);
+                auto in                = ggml_ext_slice(ctx->ggml_ctx, x, 2, frame, frame + 1);
+                _conv_idx              = 0;
+                auto out_frame         = decoder->forward(ctx, in, b, _feat_map, _conv_idx, global_frame);
+                if (is_2D && global_frame > 0) {
+                    auto repeated = out_frame;
+                    for (int repeat = 1; repeat < 4; ++repeat) {
+                        repeated = ggml_concat(ctx->ggml_ctx, repeated, out_frame, 2);
+                    }
+                    out_frame = repeated;
+                }
+                out = out == nullptr ? out_frame : ggml_concat(ctx->ggml_ctx, out, out_frame, 2);
+            }
+            return unpatchify(ctx->ggml_ctx, out, patch_size, b);
         }
     };
 
@@ -1262,7 +1278,7 @@ namespace WAN {
                 }
             }
             if (is_2D) {
-                LOG_DEBUG("USING 2D VAE");
+                LOG_VERBOSE("USING 2D VAE");
             }
             ae = WanVAE(decode_only, version, is_2D);
             ae.init(params_ctx, tensor_storage_map, prefix);
@@ -1272,6 +1288,15 @@ namespace WAN {
             return "wan_vae";
         }
 
+        bool supports_temporal_tiling(VAETemporalDirection direction) const override {
+            return direction == VAETemporalDirection::DECODE;
+        }
+
+        int get_temporal_tile_output_scale(VAETemporalDirection direction) const override {
+            SD_UNUSED(direction);
+            return 4;
+        }
+
         void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
             ae.get_param_tensors(tensors, weight_prefix);
         }
@@ -1346,8 +1371,8 @@ namespace WAN {
             return gf;
         }
 
-        ggml_cgraph* build_graph_partial(const sd::Tensor<float>& z_tensor, bool decode_graph, int i) {
-            ggml_cgraph* gf = new_graph_custom(20480);
+        ggml_cgraph* build_temporal_tile_graph(const sd::Tensor<float>& z_tensor, int chunk_idx) {
+            ggml_cgraph* gf = new_graph_custom(std::max<size_t>(20480, 10240 * z_tensor.shape()[2]));
 
             ae.clear_cache();
 
@@ -1360,7 +1385,7 @@ namespace WAN {
 
             auto runner_ctx = get_context();
 
-            ggml_tensor* out = decode_graph ? ae.decode_partial(&runner_ctx, z, i) : ae.encode(&runner_ctx, z);
+            ggml_tensor* out = ae.decode_tiled_chunk(&runner_ctx, z, chunk_idx);
 
             for (size_t feat_idx = 0; feat_idx < ae._feat_map.size(); feat_idx++) {
                 ggml_tensor* feat_cache = ae._feat_map[feat_idx];
@@ -1375,58 +1400,58 @@ namespace WAN {
             return gf;
         }
 
+        sd::Tensor<float> _compute_temporal_tiled(const int n_threads,
+                                                  const sd::Tensor<float>& input,
+                                                  VAETemporalDirection direction,
+                                                  const VAETemporalTilingConfig& config) override {
+            GGML_ASSERT(direction == VAETemporalDirection::DECODE);
+            VAETemporalTilingConfig stateful_config = config;
+            stateful_config.overlap                 = 0;
+            auto plan                               = make_vae_temporal_tile_plan(input.shape()[2], stateful_config);
+
+            LOG_VERBOSE("Wan VAE stateful temporal tiling: tile_frames=%d, total latent frames=%lld, tiles=%d",
+                        plan.tile_frames,
+                        (long long)input.shape()[2],
+                        (int)plan.tiles.size());
+
+            free_cache_ctx_and_buffer();
+            ae.clear_cache();
+
+            auto output = process_vae_temporal_tiles(input, plan, [&](const sd::Tensor<float>& input_tile, const VAETemporalTile& tile) {
+                LOG_VERBOSE("Wan VAE temporal tile %d/%d: latent frames [%lld, %lld)",
+                            tile.index + 1,
+                            (int)plan.tiles.size(),
+                            (long long)tile.start,
+                            (long long)tile.end);
+                auto get_graph = [&]() -> ggml_cgraph* {
+                    return build_temporal_tile_graph(input_tile, static_cast<int>(tile.start));
+                };
+                return restore_trailing_singleton_dims(
+                    GGMLRunner::compute<float>(get_graph, n_threads, false),
+                    static_cast<size_t>(input.dim()));
+            });
+
+            free_cache_ctx_and_buffer();
+            ae.clear_cache();
+            return output;
+        }
+
         sd::Tensor<float> _compute(const int n_threads,
                                    const sd::Tensor<float>& z,
                                    bool decode_graph) override {
-            if (true) {
-                sd::Tensor<float> input;
-                if (z.dim() == 4) {
-                    input = z.unsqueeze(2);
-                }
-                auto get_graph = [&]() -> ggml_cgraph* {
-                    if (input.empty()) {
-                        return build_graph(z, decode_graph);
-                    } else {
-                        return build_graph(input, decode_graph);
-                    }
-                };
-                auto result = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, true, true, true),
-                                                              input.empty() ? z.dim() : input.dim());
-                if (!result.empty() && z.dim() == 4) {
-                    result.squeeze_(2);
-                }
-                return result;
-            } else {  // chunk 1 result is weird
-                ae.clear_cache();
-                int64_t t      = z.shape()[2];
-                int i          = 0;
-                auto get_graph = [&]() -> ggml_cgraph* {
-                    return build_graph_partial(z, decode_graph, i);
-                };
-                auto out_opt = GGMLRunner::compute<float>(get_graph, n_threads, true, true, true);
-                if (!out_opt.has_value()) {
-                    return {};
-                }
-                sd::Tensor<float> out = std::move(*out_opt);
-                ae.clear_cache();
-                if (t == 1) {
-                    return out;
-                }
-
-                sd::Tensor<float> output = std::move(out);
-
-                for (i = 1; i < t; i++) {
-                    auto chunk_opt = GGMLRunner::compute<float>(get_graph, n_threads, true, true, true);
-                    if (!chunk_opt.has_value()) {
-                        return {};
-                    }
-                    out = std::move(*chunk_opt);
-                    ae.clear_cache();
-                    output = sd::ops::concat(output, out, 2);
-                }
-                free_cache_ctx_and_buffer();
-                return output;
+            sd::Tensor<float> input;
+            if (z.dim() == 4) {
+                input = z.unsqueeze(2);
             }
+            auto get_graph = [&]() -> ggml_cgraph* {
+                return build_graph(input.empty() ? z : input, decode_graph);
+            };
+            auto result = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false),
+                                                          input.empty() ? z.dim() : input.dim());
+            if (!result.empty() && z.dim() == 4) {
+                result.squeeze_(2);
+            }
+            return result;
         }
 
         void test() {
@@ -1454,7 +1479,7 @@ namespace WAN {
                 GGML_ASSERT(!out_opt.empty());
                 out = std::move(out_opt);
                 print_sd_tensor(out);
-                LOG_DEBUG("decode test done in %ldms", t1 - t0);
+                LOG_VERBOSE("decode test done in %ldms", t1 - t0);
             }
         };
 

src/model_io/gguf_reader_ext.h

@@ -77,7 +77,7 @@ private:
 
             if (align_val != 0 && (align_val & (align_val - 1)) == 0) {
                 alignment_ = align_val;
-                LOG_DEBUG("Found alignment: %zu", alignment_);
+                LOG_VERBOSE("Found alignment: %zu", alignment_);
             } else {
                 LOG_ERROR("Invalid alignment value %u, fallback to default %zu", align_val, alignment_);
             }
@@ -197,8 +197,8 @@ public:
         if (!safe_read(fin, metadata_kv_count))
             return false;
 
-        LOG_DEBUG("GGUF v%u, tensor_count=%llu, metadata_kv_count=%llu",
-                  version, (unsigned long long)tensor_count, (unsigned long long)metadata_kv_count);
+        LOG_VERBOSE("GGUF v%u, tensor_count=%llu, metadata_kv_count=%llu",
+                    version, (unsigned long long)tensor_count, (unsigned long long)metadata_kv_count);
 
         // --- Read Metadata ---
         for (uint64_t i = 0; i < metadata_kv_count; i++) {

src/model_io/safetensors_io.cpp

@@ -87,9 +87,9 @@ static ggml_type safetensors_dtype_to_ggml_type(const std::string& dtype) {
     } else if (dtype == "F64") {
         ttype = GGML_TYPE_F32;
     } else if (dtype == "F8_E4M3") {
-        ttype = GGML_TYPE_F16;
+        ttype = GGML_TYPE_F8_E4M3;
     } else if (dtype == "F8_E5M2") {
-        ttype = GGML_TYPE_F16;
+        ttype = GGML_TYPE_F8_E5M2;
     } else if (dtype == "I32") {
         ttype = GGML_TYPE_I32;
     } else if (dtype == "I64") {
@@ -237,7 +237,7 @@ bool read_safetensors_file(const std::string& file_path,
     for (auto& item : header_.items()) {
         std::string name           = item.key();
         nlohmann::json tensor_info = item.value();
-        // LOG_DEBUG("%s %s\n", name.c_str(), tensor_info.dump().c_str());
+        // LOG_VERBOSE("%s %s\n", name.c_str(), tensor_info.dump().c_str());
 
         if (name == "__metadata__") {
             continue;
@@ -328,12 +328,10 @@ bool read_safetensors_file(const std::string& file_path,
         bool tensor_size_ok;
         if (dtype == "F8_E4M3") {
             tensor_storage.is_f8_e4m3 = true;
-            // f8 -> f16
-            tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size * 2);
+            tensor_size_ok            = (tensor_storage.nbytes() == tensor_data_size);
         } else if (dtype == "F8_E5M2") {
             tensor_storage.is_f8_e5m2 = true;
-            // f8 -> f16
-            tensor_size_ok = (tensor_storage.nbytes() == tensor_data_size * 2);
+            tensor_size_ok            = (tensor_storage.nbytes() == tensor_data_size);
         } else if (dtype == "F64") {
             tensor_storage.is_f64 = true;
             // f64 -> f32
@@ -352,7 +350,7 @@ bool read_safetensors_file(const std::string& file_path,
 
         tensor_storages.push_back(tensor_storage);
 
-        // LOG_DEBUG("%s %s", tensor_storage.to_string().c_str(), dtype.c_str());
+        // LOG_VERBOSE("%s %s", tensor_storage.to_string().c_str(), dtype.c_str());
     }
 
     return true;

src/model_io/tensor_storage.h

@@ -54,9 +54,7 @@ struct TensorStorage {
     }
 
     int64_t nbytes_to_read() const {
-        if (is_f8_e4m3 || is_f8_e5m2) {
-            return nbytes() / 2;
-        } else if (is_f64 || is_i64) {
+        if (is_f64 || is_i64) {
             return nbytes() * 2;
         } else {
             return nbytes();

src/model_loader.cpp

@@ -78,66 +78,6 @@ bool is_unused_tensor(const std::string& name) {
     return false;
 }
 
-uint16_t f8_e4m3_to_f16(uint8_t f8) {
-    // do we need to support uz?
-
-    const uint32_t exponent_bias = 7;
-    if (f8 == 0xff) {
-        return ggml_fp32_to_fp16(-NAN);
-    } else if (f8 == 0x7f) {
-        return ggml_fp32_to_fp16(NAN);
-    }
-
-    uint32_t sign     = f8 & 0x80;
-    uint32_t exponent = (f8 & 0x78) >> 3;
-    uint32_t mantissa = f8 & 0x07;
-    uint32_t result   = sign << 24;
-    if (exponent == 0) {
-        if (mantissa > 0) {
-            exponent = 0x7f - exponent_bias;
-
-            // yes, 2 times
-            if ((mantissa & 0x04) == 0) {
-                mantissa &= 0x03;
-                mantissa <<= 1;
-                exponent -= 1;
-            }
-            if ((mantissa & 0x04) == 0) {
-                mantissa &= 0x03;
-                mantissa <<= 1;
-                exponent -= 1;
-            }
-
-            result |= (mantissa & 0x03) << 21;
-            result |= exponent << 23;
-        }
-    } else {
-        result |= mantissa << 20;
-        exponent += 0x7f - exponent_bias;
-        result |= exponent << 23;
-    }
-
-    return ggml_fp32_to_fp16(*reinterpret_cast<const float*>(&result));
-}
-
-uint16_t f8_e5m2_to_f16(uint8_t fp8) {
-    return static_cast<uint16_t>(fp8) << 8;
-}
-
-void f8_e4m3_to_f16_vec(uint8_t* src, uint16_t* dst, int64_t n) {
-    // support inplace op
-    for (int64_t i = n - 1; i >= 0; i--) {
-        dst[i] = f8_e4m3_to_f16(src[i]);
-    }
-}
-
-void f8_e5m2_to_f16_vec(uint8_t* src, uint16_t* dst, int64_t n) {
-    // support inplace op
-    for (int64_t i = n - 1; i >= 0; i--) {
-        dst[i] = f8_e5m2_to_f16(src[i]);
-    }
-}
-
 void f64_to_f32_vec(double* src, float* dst, int64_t n) {
     // support inplace op
     for (int64_t i = 0; i < n; i++) {
@@ -225,7 +165,7 @@ void ModelLoader::add_tensor_storage(const TensorStorage& tensor_storage) {
 
 void ModelLoader::set_n_threads(int n_threads) {
     n_threads_ = n_threads > 0 ? n_threads : sd_get_num_physical_cores();
-    LOG_DEBUG("using %d threads for model loading", n_threads_);
+    LOG_VERBOSE("using %d threads for model loading", n_threads_);
 }
 
 bool ModelLoader::init_from_file(const std::string& file_path, const std::string& prefix) {
@@ -263,7 +203,7 @@ void ModelLoader::convert_tensors_name() {
 
     for (auto& [_, tensor_storage] : tensor_storage_map) {
         auto new_name = convert_tensor_name(tensor_storage.name, version);
-        // LOG_DEBUG("%s -> %s", tensor_storage.name.c_str(), new_name.c_str());
+        // LOG_VERBOSE("%s -> %s", tensor_storage.name.c_str(), new_name.c_str());
         tensor_storage.name = new_name;
         new_map[new_name]   = std::move(tensor_storage);
     }
@@ -285,7 +225,7 @@ bool ModelLoader::init_from_file_and_convert_name(const std::string& file_path,
 /*================================================= GGUFModelLoader ==================================================*/
 
 bool ModelLoader::init_from_gguf_file(const std::string& file_path, const std::string& prefix) {
-    LOG_DEBUG("init from '%s'", file_path.c_str());
+    LOG_VERBOSE("init from '%s'", file_path.c_str());
 
     std::vector<TensorStorage> tensor_storages;
     std::string error;
@@ -297,7 +237,7 @@ bool ModelLoader::init_from_gguf_file(const std::string& file_path, const std::s
     size_t file_index = add_file_path(file_path);
 
     for (auto& tensor_storage : tensor_storages) {
-        // LOG_DEBUG("%s", tensor_storage.name.c_str());
+        // LOG_VERBOSE("%s", tensor_storage.name.c_str());
 
         if (!starts_with(tensor_storage.name, prefix)) {
             tensor_storage.name = prefix + tensor_storage.name;
@@ -313,7 +253,7 @@ bool ModelLoader::init_from_gguf_file(const std::string& file_path, const std::s
 /*================================================= SafeTensorsModelLoader ==================================================*/
 
 bool ModelLoader::init_from_safetensors_file(const std::string& file_path, const std::string& prefix) {
-    LOG_DEBUG("init from '%s', prefix = '%s'", file_path.c_str(), prefix.c_str());
+    LOG_VERBOSE("init from '%s', prefix = '%s'", file_path.c_str(), prefix.c_str());
 
     std::vector<TensorStorage> tensor_storages;
     std::string error;
@@ -336,14 +276,14 @@ bool ModelLoader::init_from_safetensors_file(const std::string& file_path, const
 
         add_tensor_storage(tensor_storage);
 
-        // LOG_DEBUG("%s", tensor_storage.to_string().c_str());
+        // LOG_VERBOSE("%s", tensor_storage.to_string().c_str());
     }
 
     return true;
 }
 
 bool ModelLoader::init_from_safetensors_index_file(const std::string& file_path, const std::string& prefix) {
-    LOG_DEBUG("init from safetensors index '%s', prefix = '%s'", file_path.c_str(), prefix.c_str());
+    LOG_VERBOSE("init from safetensors index '%s', prefix = '%s'", file_path.c_str(), prefix.c_str());
 
     std::vector<std::string> shard_paths;
     std::string error;
@@ -364,7 +304,7 @@ bool ModelLoader::init_from_safetensors_index_file(const std::string& file_path,
 /*================================================= TorchLegacyModelLoader ==================================================*/
 
 bool ModelLoader::init_from_torch_legacy_file(const std::string& file_path, const std::string& prefix) {
-    LOG_DEBUG("init from torch legacy '%s'", file_path.c_str());
+    LOG_VERBOSE("init from torch legacy '%s'", file_path.c_str());
 
     std::vector<TensorStorage> tensor_storages;
     std::string error;
@@ -396,7 +336,7 @@ bool ModelLoader::init_from_torch_legacy_file(const std::string& file_path, cons
 /*================================================= TorchZipModelLoader ==================================================*/
 
 bool ModelLoader::init_from_torch_zip_file(const std::string& file_path, const std::string& prefix) {
-    LOG_DEBUG("init from '%s'", file_path.c_str());
+    LOG_VERBOSE("init from '%s'", file_path.c_str());
 
     std::vector<TensorStorage> tensor_storages;
     std::string error;
@@ -415,7 +355,7 @@ bool ModelLoader::init_from_torch_zip_file(const std::string& file_path, const s
 
         add_tensor_storage(tensor_storage);
 
-        // LOG_DEBUG("%s", tensor_storage.to_string().c_str());
+        // LOG_VERBOSE("%s", tensor_storage.to_string().c_str());
     }
 
     return true;
@@ -442,7 +382,7 @@ bool ModelLoader::init_from_diffusers_file(const std::string& file_path, const s
         // return false;
     }
     if (!init_from_safetensors_file(clip_g_path, "te.1.")) {
-        LOG_DEBUG("Couldn't find working second text encoder in %s", file_path.c_str());
+        LOG_VERBOSE("Couldn't find working second text encoder in %s", file_path.c_str());
     }
     return true;
 }
@@ -606,7 +546,7 @@ SDVersion ModelLoader::get_sd_version() {
         }
     }
     if (is_wan) {
-        LOG_DEBUG("patch_embedding_channels %d", patch_embedding_channels);
+        LOG_VERBOSE("patch_embedding_channels %d", patch_embedding_channels);
         if (patch_embedding_channels == 184320 && !has_img_emb) {
             return VERSION_WAN2_2_I2V;
         }
@@ -863,7 +803,7 @@ void ModelLoader::process_model_files(bool enable_mmap, bool writable_mmap) {
         fdata.tensors       = std::move(file_tensors);
 
         if (enable_mmap && !is_zip) {
-            LOG_DEBUG("using mmap for I/O");
+            LOG_VERBOSE("using mmap for I/O");
             std::unique_ptr<MmapWrapper> mmapped = MmapWrapper::create(file_path, writable_mmap);
             if (mmapped) {
                 uint8_t* mmap_data             = static_cast<uint8_t*>(mmapped->writable_data());
@@ -895,7 +835,7 @@ std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggm
     uint64_t mapped_bytes = 0;
     size_t mapped_tensors = 0;
 
-    LOG_DEBUG("memory-mapping tensors...");
+    LOG_VERBOSE("memory-mapping tensors...");
 
     int64_t t_start = ggml_time_ms();
 
@@ -929,9 +869,7 @@ std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggm
             if (dst_tensor == nullptr)
                 continue;
 
-            if (tensor_storage.is_f8_e4m3 ||
-                tensor_storage.is_f8_e5m2 ||
-                tensor_storage.is_f64 ||
+            if (tensor_storage.is_f64 ||
                 tensor_storage.is_i64 ||
                 tensor_storage.type != dst_tensor->type) {
                 continue;
@@ -1039,10 +977,10 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
         if (tensors_to_process.empty()) {
             continue;
         }
-        LOG_DEBUG("loading %zu/%zu tensors from %s",
-                  tensors_to_process.size(),
-                  file_tensors.size(),
-                  file_path.c_str());
+        LOG_VERBOSE("loading %zu/%zu tensors from %s",
+                    tensors_to_process.size(),
+                    file_tensors.size(),
+                    file_path.c_str());
 
         bool is_zip = fdata.is_zip;
 
@@ -1215,11 +1153,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
                     read_time_ms.fetch_add(t1 - t0);
 
                     t0 = ggml_time_ms();
-                    if (tensor_storage.is_f8_e4m3) {
-                        f8_e4m3_to_f16_vec((uint8_t*)read_buf, (uint16_t*)target_buf, tensor_storage.nelements());
-                    } else if (tensor_storage.is_f8_e5m2) {
-                        f8_e5m2_to_f16_vec((uint8_t*)read_buf, (uint16_t*)target_buf, tensor_storage.nelements());
-                    } else if (tensor_storage.is_f64) {
+                    if (tensor_storage.is_f64) {
                         f64_to_f32_vec((double*)read_buf, (float*)target_buf, tensor_storage.nelements());
                     } else if (tensor_storage.is_i64) {
                         i64_to_i32_vec((int64_t*)read_buf, (int32_t*)target_buf, tensor_storage.nelements());
@@ -1439,7 +1373,7 @@ bool ModelLoader::load_tensors(std::map<std::string, ggml_tensor*>& tensors,
     std::mutex tensor_names_mutex;
     auto on_new_tensor_cb = [&](const TensorStorage& tensor_storage, ggml_tensor** dst_tensor) -> bool {
         const std::string& name = tensor_storage.name;
-        // LOG_DEBUG("%s", tensor_storage.to_string().c_str());
+        // LOG_VERBOSE("%s", tensor_storage.to_string().c_str());
         {
             std::lock_guard<std::mutex> lock(tensor_names_mutex);
             tensor_names_in_file.insert(name);

src/model_manager.cpp

@@ -97,6 +97,7 @@ static bool device_supports_param_op(ggml_backend_dev_t device,
 
 ModelManager::~ModelManager() {
     release_all();
+    release_prefetch();
 }
 
 void ModelManager::set_common_ignore_tensors(std::set<std::string> ignore_tensors) {
@@ -142,7 +143,7 @@ size_t estimate_tensors_size(const std::map<std::string, ggml_tensor*>& tensors)
     return size;
 }
 
-void ModelManager::set_split_buffer_type(ggml_backend_t compute_backend, ggml_backend_buffer_type_t split_buft) {
+void ModelManager::set_split_buffer_type(ggml_backend_t compute_backend, ggml_backend_buffer_type_t split_buft, const std::vector<std::pair<ggml_backend_t, size_t>>& device_limits) {
     if (compute_backend == nullptr) {
         return;
     }
@@ -151,6 +152,7 @@ void ModelManager::set_split_buffer_type(ggml_backend_t compute_backend, ggml_ba
         return;
     }
     split_buffer_types_[compute_backend] = split_buft;
+    split_buffer_devices_[split_buft]    = device_limits;
 }
 
 bool ModelManager::tensor_shape_supports_split_buffer(const ggml_tensor* tensor) {
@@ -163,11 +165,10 @@ bool ModelManager::tensor_shape_supports_split_buffer(const ggml_tensor* tensor)
 }
 
 ggml_backend_buffer_type_t ModelManager::split_buffer_type_for(const TensorState& state) const {
-    if (!state.allow_split_buffer || !tensor_shape_supports_split_buffer(state.tensor)) {
+    if (!tensor_shape_supports_split_buffer(state.tensor)) {
         return nullptr;
     }
-    auto it = split_buffer_types_.find(state.compute_backend);
-    return it != split_buffer_types_.end() ? it->second : nullptr;
+    return state.split_buffer_type;
 }
 
 bool ModelManager::register_param_tensors(const std::string& desc,
@@ -202,14 +203,17 @@ bool ModelManager::register_param_tensors(const std::string& desc,
         }
         ggml_set_name(tensor, name.c_str());
 
-        auto state                           = std::make_unique<TensorState>();
-        state->name                          = name;
-        state->tensor                        = tensor;
-        state->desc                          = desc;
-        state->residency_mode                = residency_mode;
-        state->compute_backend               = compute_backend;
-        state->params_backend                = params_backend;
-        state->allow_split_buffer            = allow_split_buffer;
+        auto state             = std::make_unique<TensorState>();
+        state->name            = name;
+        state->tensor          = tensor;
+        state->desc            = desc;
+        state->residency_mode  = residency_mode;
+        state->compute_backend = compute_backend;
+        state->params_backend  = params_backend;
+        auto split_buffer      = split_buffer_types_.find(compute_backend);
+        if (allow_split_buffer && split_buffer != split_buffer_types_.end()) {
+            state->split_buffer_type = split_buffer->second;
+        }
         state->params_follow_compute_backend = params_follow_compute_backend;
         if (tensor_ops != nullptr) {
             auto op_it = tensor_ops->find(tensor);
@@ -239,7 +243,7 @@ bool ModelManager::unregister_param_tensors(const std::string& desc, size_t* reg
         if (state == nullptr || state->desc != desc) {
             continue;
         }
-        if (state->active_prepare_count > 0) {
+        if (state->pin_count > 0) {
             LOG_ERROR("model manager cannot unregister active %s tensor '%s'",
                       desc.c_str(),
                       state->name.c_str());
@@ -255,6 +259,7 @@ bool ModelManager::unregister_param_tensors(const std::string& desc, size_t* reg
         return true;
     }
 
+    clear_all_prefetched_params();
     release_compute_staging_blocks(false);
 
     std::vector<ParamsStorageBlock*> storage_blocks_to_release;
@@ -285,7 +290,7 @@ bool ModelManager::unregister_param_tensors(const std::string& desc, size_t* reg
         if (state == nullptr) {
             continue;
         }
-        if (state->active_prepare_count > 0 || state->staged_to_compute_backend) {
+        if (state->pin_count > 0 || state->staged_to_compute_backend) {
             LOG_ERROR("model manager cannot unregister %s while tensor '%s' is active",
                       desc.c_str(),
                       state->name.c_str());
@@ -401,14 +406,27 @@ bool ModelManager::load_tensors_to_params_backend(const std::vector<TensorState*
         }
         return false;
     }
+    struct PrepareStats {
+        size_t bytes   = 0;
+        size_t tensors = 0;
+        size_t blocks  = 0;
+    };
+    std::map<ggml_backend_buffer_type_t, PrepareStats> prepared;
     for (ParamsStorageBlock* block : created_storage_blocks) {
         if (block != nullptr && block->buffer != nullptr) {
-            LOG_DEBUG("model manager prepared params backend buffer (%6.2f MB, %zu tensors, %s)",
-                      ggml_backend_buffer_get_size(block->buffer) / (1024.f * 1024.f),
-                      block->states.size(),
-                      ggml_backend_buffer_is_host(block->buffer) ? "RAM" : "VRAM");
+            auto& stats = prepared[ggml_backend_buffer_get_type(block->buffer)];
+            stats.bytes += ggml_backend_buffer_get_size(block->buffer);
+            stats.tensors += block->states.size();
+            ++stats.blocks;
         }
     }
+    for (const auto& entry : prepared) {
+        LOG_VERBOSE("model manager prepared params backend buffers (%6.2f MB, %zu tensors, %zu blocks, %s) on %s",
+                    entry.second.bytes / (1024.f * 1024.f),
+                    entry.second.tensors, entry.second.blocks,
+                    ggml_backend_buft_is_host(entry.first) ? "RAM" : "VRAM",
+                    ggml_backend_buft_name(entry.first));
+    }
 
     return true;
 }
@@ -442,68 +460,99 @@ bool ModelManager::stage_tensors_to_compute_backend(const std::vector<TensorStat
     }
 
     for (const auto& pair : states_by_staging_target) {
-        ggml_backend_t compute_backend          = pair.first.first;
-        ggml_backend_buffer_type_t staging_buft = pair.first.second;
-        const std::vector<TensorState*>& states = pair.second;
-        if (states.empty()) {
+        ggml_backend_t compute_backend                 = pair.first.first;
+        ggml_backend_buffer_type_t staging_buft        = pair.first.second;
+        const std::vector<TensorState*>& target_states = pair.second;
+        if (target_states.empty()) {
             continue;
         }
 
-        int64_t t0 = ggml_time_ms();
-
-        ggml_init_params init_params;
-        init_params.mem_size   = std::max<size_t>(1, states.size()) * ggml_tensor_overhead();
-        init_params.mem_buffer = nullptr;
-        init_params.no_alloc   = true;
-
-        ggml_context* staging_ctx = ggml_init(init_params);
-        GGML_ASSERT(staging_ctx != nullptr);
-
-        std::vector<std::pair<TensorState*, ggml_tensor*>> staged_tensors;
-        staged_tensors.reserve(states.size());
-        for (TensorState* state : states) {
-            ggml_tensor* staging_tensor = ggml_dup_tensor(staging_ctx, state->tensor);
-            ggml_set_name(staging_tensor, state->tensor->name);
-            staged_tensors.push_back({state, staging_tensor});
+        const size_t alignment = ggml_backend_buft_get_alignment(staging_buft);
+        size_t backend_limit   = ggml_backend_buft_get_max_size(staging_buft);
+        if (!ggml_backend_buft_is_host(staging_buft) &&
+            (backend_limit == 0 || backend_limit > MAX_RESIDENCY_BLOCK_BYTES)) {
+            backend_limit = MAX_RESIDENCY_BLOCK_BYTES;
         }
 
-        ggml_backend_buffer_t compute_buffer = ggml_backend_alloc_ctx_tensors_from_buft(staging_ctx, staging_buft);
-        if (compute_buffer == nullptr) {
-            LOG_ERROR("model manager alloc compute params backend buffer failed, num_tensors = %zu",
-                      staged_tensors.size());
-            ggml_free(staging_ctx);
+        const int64_t t0     = ggml_time_ms();
+        size_t staged_bytes  = 0;
+        size_t staged_blocks = 0;
+        auto stage_chunk     = [&](const std::vector<TensorState*>& chunk) -> bool {
+            if (chunk.empty()) {
+                return true;
+            }
+            ggml_init_params init_params;
+            init_params.mem_size   = std::max<size_t>(1, chunk.size()) * ggml_tensor_overhead();
+            init_params.mem_buffer = nullptr;
+            init_params.no_alloc   = true;
+
+            ggml_context* staging_ctx = ggml_init(init_params);
+            GGML_ASSERT(staging_ctx != nullptr);
+            std::vector<std::pair<TensorState*, ggml_tensor*>> staged_tensors;
+            staged_tensors.reserve(chunk.size());
+            for (TensorState* state : chunk) {
+                ggml_tensor* staging_tensor = ggml_dup_tensor(staging_ctx, state->tensor);
+                ggml_set_name(staging_tensor, state->tensor->name);
+                staged_tensors.push_back({state, staging_tensor});
+            }
+
+            ggml_backend_buffer_t compute_buffer =
+                ggml_backend_alloc_ctx_tensors_from_buft(staging_ctx, staging_buft);
+            if (compute_buffer == nullptr) {
+                LOG_ERROR("model manager alloc compute params backend buffer failed, num_tensors = %zu",
+                              staged_tensors.size());
+                ggml_free(staging_ctx);
+                return false;
+            }
+            ggml_backend_buffer_set_usage(compute_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
+            for (auto& staged_tensor : staged_tensors) {
+                TensorState* state          = staged_tensor.first;
+                ggml_tensor* managed_tensor = state->tensor;
+                ggml_tensor* staging_tensor = staged_tensor.second;
+                ggml_backend_tensor_copy(managed_tensor, staging_tensor);
+                std::swap(managed_tensor->buffer, staging_tensor->buffer);
+                std::swap(managed_tensor->data, staging_tensor->data);
+                std::swap(managed_tensor->extra, staging_tensor->extra);
+                state->staged_to_compute_backend = true;
+            }
+            ggml_backend_synchronize(compute_backend);
+
+            auto block             = std::make_unique<ComputeStagingBlock>();
+            block->compute_backend = compute_backend;
+            block->buffer          = compute_buffer;
+            block->staging_ctx     = staging_ctx;
+            block->staged_tensors  = std::move(staged_tensors);
+            staged_bytes += ggml_backend_buffer_get_size(compute_buffer);
+            ++staged_blocks;
+            compute_staging_blocks_.push_back(std::move(block));
+            return true;
+        };
+
+        std::vector<TensorState*> chunk;
+        size_t chunk_size = 0;
+        for (TensorState* state : target_states) {
+            const size_t tensor_size = GGML_PAD(
+                ggml_backend_buft_get_alloc_size(staging_buft, state->tensor), alignment);
+            if (!chunk.empty() && backend_limit > 0 &&
+                tensor_size > backend_limit - std::min(chunk_size, backend_limit)) {
+                if (!stage_chunk(chunk)) {
+                    return false;
+                }
+                chunk.clear();
+                chunk_size = 0;
+            }
+            chunk.push_back(state);
+            chunk_size = tensor_size > SIZE_MAX - chunk_size ? SIZE_MAX : chunk_size + tensor_size;
+        }
+        if (!stage_chunk(chunk)) {
             return false;
         }
-        ggml_backend_buffer_set_usage(compute_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
-
-        for (auto& staged_tensor : staged_tensors) {
-            TensorState* state          = staged_tensor.first;
-            ggml_tensor* managed_tensor = state->tensor;
-            ggml_tensor* staging_tensor = staged_tensor.second;
-            ggml_backend_tensor_copy(managed_tensor, staging_tensor);
-            std::swap(managed_tensor->buffer, staging_tensor->buffer);
-            std::swap(managed_tensor->data, staging_tensor->data);
-            std::swap(managed_tensor->extra, staging_tensor->extra);
-        }
-        ggml_backend_synchronize(compute_backend);
-
-        auto block             = std::make_unique<ComputeStagingBlock>();
-        block->compute_backend = compute_backend;
-        block->buffer          = compute_buffer;
-        block->staging_ctx     = staging_ctx;
-        block->staged_tensors  = std::move(staged_tensors);
-        for (auto& staged_tensor : block->staged_tensors) {
-            TensorState* state               = staged_tensor.first;
-            state->staged_to_compute_backend = true;
-        }
-        compute_staging_blocks_.push_back(std::move(block));
-
-        int64_t t1 = ggml_time_ms();
-        LOG_DEBUG("model manager staged compute params (%6.2f MB, %zu tensors) to %s, taking %.2fs",
-                  ggml_backend_buffer_get_size(compute_buffer) / (1024.f * 1024.f),
-                  states.size(),
-                  ggml_backend_name(compute_backend),
-                  (t1 - t0) * 1.0f / 1000);
+        LOG_VERBOSE("model manager staged compute params (%6.2f MB, %zu tensors, %zu blocks) to %s, taking %.2fs",
+                    staged_bytes / (1024.f * 1024.f),
+                    target_states.size(),
+                    staged_blocks,
+                    ggml_backend_name(compute_backend),
+                    (ggml_time_ms() - t0) / 1000.f);
     }
 
     return true;
@@ -608,6 +657,7 @@ bool ModelManager::apply_loras_to_params(const std::vector<TensorState*>& states
 }
 
 void ModelManager::reset_lora_applied_params() {
+    clear_all_prefetched_params();
     release_compute_staging_blocks(true);
     release_params_storage_blocks(true);
     for (auto& state : tensor_states_) {
@@ -726,6 +776,10 @@ bool ModelManager::alloc_params_buffers(const std::vector<TensorState*>& states,
         const std::vector<TensorState*>& states = pair.second;
         size_t alignment                        = ggml_backend_buft_get_alignment(params_buft);
         size_t max_size                         = ggml_backend_buft_get_max_size(params_buft);
+        if (!ggml_backend_buft_is_host(params_buft) &&
+            (max_size == 0 || max_size > MAX_RESIDENCY_BLOCK_BYTES)) {
+            max_size = MAX_RESIDENCY_BLOCK_BYTES;
+        }
 
         auto alloc_chunk = [&](const std::vector<TensorState*>& chunk, size_t chunk_size) -> bool {
             if (chunk.empty() || chunk_size == 0) {
@@ -755,10 +809,10 @@ bool ModelManager::alloc_params_buffers(const std::vector<TensorState*>& states,
                         initialized->data   = nullptr;
                         initialized->extra  = nullptr;
                     }
-                    LOG_DEBUG("model manager releasing params backend buffer (%6.2f MB, %zu tensors, %s)",
-                              ggml_backend_buffer_get_size(buffer) / (1024.f * 1024.f),
-                              initialized_tensors.size(),
-                              ggml_backend_buffer_is_host(buffer) ? "RAM" : "VRAM");
+                    LOG_VERBOSE("model manager releasing params backend buffer (%6.2f MB, %zu tensors, %s)",
+                                ggml_backend_buffer_get_size(buffer) / (1024.f * 1024.f),
+                                initialized_tensors.size(),
+                                ggml_backend_buffer_is_host(buffer) ? "RAM" : "VRAM");
                     ggml_backend_buffer_free(buffer);
                     return false;
                 }
@@ -932,10 +986,6 @@ void ModelManager::free_compute_staging_block(ComputeStagingBlock& block) {
     }
 
     if (block.buffer != nullptr) {
-        LOG_DEBUG("model manager releasing compute params (%6.2f MB, %zu tensors) from %s",
-                  ggml_backend_buffer_get_size(block.buffer) / (1024.f * 1024.f),
-                  block.staged_tensors.size(),
-                  block.compute_backend != nullptr ? ggml_backend_name(block.compute_backend) : "unknown");
         ggml_backend_buffer_free(block.buffer);
         block.buffer = nullptr;
     }
@@ -948,6 +998,12 @@ void ModelManager::free_compute_staging_block(ComputeStagingBlock& block) {
 
 void ModelManager::release_compute_staging_blocks(bool force,
                                                   const std::unordered_set<TensorState*>* target_states) {
+    struct ReleaseStats {
+        size_t bytes   = 0;
+        size_t tensors = 0;
+        size_t blocks  = 0;
+    };
+    std::map<ggml_backend_t, ReleaseStats> released;
     for (auto it = compute_staging_blocks_.begin(); it != compute_staging_blocks_.end();) {
         ComputeStagingBlock* block = it->get();
         bool can_release           = force;
@@ -963,25 +1019,33 @@ void ModelManager::release_compute_staging_blocks(bool force,
                                               target_states->find(state) == target_states->end()) {
                                               return false;
                                           }
-                                          return state->active_prepare_count == 0;
+                                          return state->pin_count == 0;
                                       });
         }
 
         if (can_release) {
+            if (block->buffer != nullptr) {
+                auto& stats = released[block->compute_backend];
+                stats.bytes += ggml_backend_buffer_get_size(block->buffer);
+                stats.tensors += block->staged_tensors.size();
+                ++stats.blocks;
+            }
             free_compute_staging_block(*block);
             it = compute_staging_blocks_.erase(it);
         } else {
             ++it;
         }
     }
+    for (const auto& entry : released) {
+        LOG_DEBUG("model manager released compute params (%6.2f MB, %zu tensors, %zu blocks) from %s",
+                  entry.second.bytes / (1024.f * 1024.f),
+                  entry.second.tensors, entry.second.blocks,
+                  entry.first != nullptr ? ggml_backend_name(entry.first) : "unknown");
+    }
 }
 
 void ModelManager::free_params_storage_block(ParamsStorageBlock& block) {
     if (block.buffer != nullptr) {
-        LOG_DEBUG("model manager releasing params backend buffer (%6.2f MB, %zu tensors, %s)",
-                  ggml_backend_buffer_get_size(block.buffer) / (1024.f * 1024.f),
-                  block.states.size(),
-                  ggml_backend_buffer_is_host(block.buffer) ? "RAM" : "VRAM");
         ggml_backend_buffer_free(block.buffer);
         block.buffer = nullptr;
     }
@@ -1003,6 +1067,12 @@ void ModelManager::free_params_storage_block(ParamsStorageBlock& block) {
 
 void ModelManager::release_params_storage_blocks(bool force,
                                                  const std::unordered_set<TensorState*>* target_states) {
+    struct ReleaseStats {
+        size_t bytes   = 0;
+        size_t tensors = 0;
+        size_t blocks  = 0;
+    };
+    std::map<ggml_backend_buffer_type_t, ReleaseStats> released;
     for (auto it = params_storage_blocks_.begin(); it != params_storage_blocks_.end();) {
         ParamsStorageBlock* block = it->get();
         bool can_release          = force;
@@ -1017,19 +1087,32 @@ void ModelManager::release_params_storage_blocks(bool force,
                                               target_states->find(state) == target_states->end()) {
                                               return false;
                                           }
-                                          return state->active_prepare_count == 0 &&
+                                          return state->pin_count == 0 &&
                                                  !state->staged_to_compute_backend &&
                                                  state->residency_mode == ResidencyMode::Disk;
                                       });
         }
 
         if (can_release) {
+            if (block->buffer != nullptr) {
+                auto& stats = released[ggml_backend_buffer_get_type(block->buffer)];
+                stats.bytes += ggml_backend_buffer_get_size(block->buffer);
+                stats.tensors += block->states.size();
+                ++stats.blocks;
+            }
             free_params_storage_block(*block);
             it = params_storage_blocks_.erase(it);
         } else {
             ++it;
         }
     }
+    for (const auto& entry : released) {
+        LOG_VERBOSE("model manager released params backend buffers (%6.2f MB, %zu tensors, %zu blocks, %s) from %s",
+                    entry.second.bytes / (1024.f * 1024.f),
+                    entry.second.tensors, entry.second.blocks,
+                    ggml_backend_buft_is_host(entry.first) ? "RAM" : "VRAM",
+                    ggml_backend_buft_name(entry.first));
+    }
 }
 
 void ModelManager::erase_params_storage_block(ParamsStorageBlock* block) {
@@ -1044,16 +1127,20 @@ void ModelManager::erase_params_storage_block(ParamsStorageBlock* block) {
 }
 
 void ModelManager::release_all() {
+    clear_all_prefetched_params();
+    runtime_residencies_.clear();
+    workspace_reclaimers_.clear();
     for (auto& state : tensor_states_) {
-        state->active_prepare_count = 0;
-        state->applied_lora_epoch   = UINT64_MAX;
+        state->pin_count          = 0;
+        state->applied_lora_epoch = UINT64_MAX;
     }
     release_compute_staging_blocks(true);
     release_params_storage_blocks(true);
 }
 
 bool ModelManager::resolve_required_tensor_states(const std::vector<ggml_tensor*>& tensors,
-                                                  std::vector<TensorState*>& required_states) const {
+                                                  std::vector<TensorState*>& required_states,
+                                                  ggml_backend_t compute_backend) const {
     required_states.clear();
     std::unordered_set<TensorState*> seen;
     for (ggml_tensor* tensor : tensors) {
@@ -1075,7 +1162,9 @@ bool ModelManager::resolve_required_tensor_states(const std::vector<ggml_tensor*
             LOG_ERROR("model manager tensor '%s' has no tensor state", raw_name);
             return false;
         }
-        if (seen.insert(state).second) {
+        if ((compute_backend == nullptr || state->compute_backend == nullptr ||
+             state->compute_backend == compute_backend) &&
+            seen.insert(state).second) {
             required_states.push_back(state);
         }
     }
@@ -1097,6 +1186,7 @@ bool ModelManager::assign_compute_backend(const std::vector<ggml_tensor*>& tenso
         return false;
     }
 
+    clear_all_prefetched_params();
     for (TensorState* state : required_states) {
         if (state == nullptr || state->tensor == nullptr) {
             continue;
@@ -1110,7 +1200,7 @@ bool ModelManager::assign_compute_backend(const std::vector<ggml_tensor*>& tenso
             continue;
         }
 
-        if (state->active_prepare_count > 0 || state->staged_to_compute_backend) {
+        if (state->pin_count > 0 || state->staged_to_compute_backend) {
             LOG_ERROR("model manager cannot move active tensor '%s' to another compute backend",
                       state->name.c_str());
             return false;
@@ -1130,6 +1220,131 @@ bool ModelManager::assign_compute_backend(const std::vector<ggml_tensor*>& tenso
     return true;
 }
 
+size_t ModelManager::compute_backend_alloc_size(const std::vector<TensorState*>& states,
+                                                bool missing_only) const {
+    size_t total_size = 0;
+    std::unordered_set<TensorState*> seen;
+    for (TensorState* state : states) {
+        if (state == nullptr || state->tensor == nullptr || !seen.insert(state).second ||
+            should_ignore(*state) || is_optional_missing_tensor(state->name)) {
+            continue;
+        }
+        const bool compute_resident =
+            state->compute_backend == state->params_backend
+                ? state->loaded_to_params_backend
+                : state->staged_to_compute_backend;
+        if (missing_only && compute_resident) {
+            continue;
+        }
+
+        ggml_backend_buffer_type_t buffer_type = nullptr;
+        if (state->compute_backend == state->params_backend) {
+            buffer_type = params_buffer_type_for(*state);
+        } else {
+            buffer_type = split_buffer_type_for(*state);
+            if (buffer_type == nullptr && state->compute_backend != nullptr) {
+                buffer_type = ggml_backend_get_default_buffer_type(state->compute_backend);
+            }
+        }
+        if (buffer_type == nullptr) {
+            continue;
+        }
+        const size_t alignment   = ggml_backend_buft_get_alignment(buffer_type);
+        const size_t tensor_size = ggml_backend_buft_get_alloc_size(buffer_type, state->tensor);
+        const size_t alloc_size  = GGML_PAD(tensor_size, alignment);
+        if (alloc_size > SIZE_MAX - total_size) {
+            return SIZE_MAX;
+        }
+        total_size += alloc_size;
+    }
+    return total_size;
+}
+
+size_t ModelManager::compute_backend_resident_bytes(ggml_backend_t compute_backend) const {
+    if (compute_backend == nullptr) {
+        return 0;
+    }
+    ggml_backend_dev_t compute_device = ggml_backend_get_device(compute_backend);
+    if (compute_device == nullptr) {
+        return 0;
+    }
+
+    size_t total_size = 0;
+    auto add_buffer   = [&](ggml_backend_buffer_t buffer) {
+        if (buffer == nullptr || ggml_backend_buffer_is_host(buffer)) {
+            return;
+        }
+        ggml_backend_buffer_type_t buffer_type = ggml_backend_buffer_get_type(buffer);
+        auto split_devices                     = split_buffer_devices_.find(buffer_type);
+        const bool on_device                   = split_devices == split_buffer_devices_.end()
+                                                       ? buffer_type != nullptr && ggml_backend_buft_get_device(buffer_type) == compute_device
+                                                       : std::any_of(split_devices->second.begin(), split_devices->second.end(), [&](const auto& entry) {
+                                         return ggml_backend_get_device(entry.first) == compute_device;
+                                     });
+        if (!on_device) {
+            return;
+        }
+        const size_t buffer_size = ggml_backend_buffer_get_size(buffer);
+        total_size               = buffer_size > SIZE_MAX - total_size ? SIZE_MAX : total_size + buffer_size;
+    };
+
+    for (const auto& block : params_storage_blocks_) {
+        if (block != nullptr) {
+            add_buffer(block->buffer);
+        }
+    }
+    for (const auto& block : compute_staging_blocks_) {
+        if (block != nullptr) {
+            add_buffer(block->buffer);
+        }
+    }
+    for (const auto& entry : prefetch_blocks_) {
+        if (entry.second != nullptr) {
+            for (const auto& block : entry.second->staging_blocks) {
+                if (block != nullptr) {
+                    add_buffer(block->buffer);
+                }
+            }
+        }
+    }
+    return total_size;
+}
+
+void ModelManager::update_runtime_residency(uintptr_t owner_id,
+                                            ggml_backend_t compute_backend,
+                                            size_t resident_bytes) {
+    if (owner_id == 0) {
+        return;
+    }
+    if (compute_backend == nullptr || resident_bytes == 0) {
+        runtime_residencies_.erase({owner_id, compute_backend});
+        return;
+    }
+    runtime_residencies_[{owner_id, compute_backend}] = {compute_backend, resident_bytes};
+}
+
+size_t ModelManager::other_runtime_resident_bytes(uintptr_t owner_id,
+                                                  ggml_backend_t compute_backend) const {
+    if (compute_backend == nullptr) {
+        return 0;
+    }
+    ggml_backend_dev_t compute_device = ggml_backend_get_device(compute_backend);
+    if (compute_device == nullptr) {
+        return 0;
+    }
+    size_t total_size = 0;
+    for (const auto& entry : runtime_residencies_) {
+        if (entry.first.first == owner_id || entry.second.compute_backend == nullptr ||
+            ggml_backend_get_device(entry.second.compute_backend) != compute_device) {
+            continue;
+        }
+        total_size = entry.second.resident_bytes > SIZE_MAX - total_size
+                         ? SIZE_MAX
+                         : total_size + entry.second.resident_bytes;
+    }
+    return total_size;
+}
+
 bool ModelManager::prepare_params(const std::vector<ggml_tensor*>& tensors) {
     if (tensors.empty()) {
         return true;
@@ -1150,18 +1365,20 @@ bool ModelManager::prepare_params(const std::vector<ggml_tensor*>& tensors) {
         return false;
     }
 
+    // LoRA execution may reclaim other residency blocks while these weights are in use.
+    const uint64_t use_epoch = ++residency_epoch_;
+    for (TensorState* state : required_states) {
+        if (state != nullptr) {
+            state->pin_count++;
+            state->last_use_epoch = use_epoch;
+        }
+    }
     if (!apply_loras_to_params(required_states)) {
+        finish_compute_backend_usage(required_states);
         release_compute_staging_blocks(false);
         release_params_storage_blocks(false);
         return false;
     }
-
-    for (TensorState* state : required_states) {
-        if (state == nullptr) {
-            continue;
-        }
-        state->active_prepare_count++;
-    }
     return true;
 }
 
@@ -1175,11 +1392,10 @@ void ModelManager::finish_compute_backend_usage(const std::vector<TensorState*>&
         if (state == nullptr || !target_states.insert(state).second) {
             continue;
         }
-        if (state->active_prepare_count > 0) {
-            state->active_prepare_count--;
+        if (state->pin_count > 0) {
+            state->pin_count--;
         }
     }
-    release_compute_staging_blocks(false, &target_states);
 }
 
 void ModelManager::release_compute_backend_params(const std::vector<ggml_tensor*>& tensors) {
@@ -1193,7 +1409,7 @@ void ModelManager::release_compute_backend_params(const std::vector<ggml_tensor*
     finish_compute_backend_usage(required_states);
 }
 
-void ModelManager::release_params_backend_params(const std::vector<ggml_tensor*>& tensors) {
+void ModelManager::evict_compute_backend_params(const std::vector<ggml_tensor*>& tensors) {
     if (tensors.empty()) {
         return;
     }
@@ -1201,9 +1417,285 @@ void ModelManager::release_params_backend_params(const std::vector<ggml_tensor*>
     if (!resolve_required_tensor_states(tensors, required_states)) {
         return;
     }
-    if (required_states.empty()) {
-        return;
-    }
     std::unordered_set<TensorState*> target_states(required_states.begin(), required_states.end());
+
+    for (const auto& block : compute_staging_blocks_) {
+        const bool intersects = std::any_of(
+            block->staged_tensors.begin(),
+            block->staged_tensors.end(),
+            [&](const std::pair<TensorState*, ggml_tensor*>& pair) {
+                return pair.first != nullptr && target_states.count(pair.first) > 0;
+            });
+        const bool fully_evictable = std::all_of(
+            block->staged_tensors.begin(),
+            block->staged_tensors.end(),
+            [](const std::pair<TensorState*, ggml_tensor*>& pair) {
+                return pair.first == nullptr || pair.first->pin_count == 0;
+            });
+        if (intersects && fully_evictable) {
+            for (const auto& pair : block->staged_tensors) {
+                if (pair.first != nullptr) {
+                    target_states.insert(pair.first);
+                }
+            }
+        }
+    }
+    release_compute_staging_blocks(false, &target_states);
+
+    for (const auto& block : params_storage_blocks_) {
+        const bool intersects = std::any_of(
+            block->states.begin(),
+            block->states.end(),
+            [&](TensorState* state) {
+                return state != nullptr && target_states.count(state) > 0;
+            });
+        const bool fully_evictable = std::all_of(
+            block->states.begin(),
+            block->states.end(),
+            [](TensorState* state) {
+                return state == nullptr ||
+                       (state->pin_count == 0 && !state->staged_to_compute_backend &&
+                        state->residency_mode == ResidencyMode::Disk);
+            });
+        if (intersects && fully_evictable) {
+            target_states.insert(block->states.begin(), block->states.end());
+        }
+    }
     release_params_storage_blocks(false, &target_states);
 }
+WeightResidencyInfo ModelManager::inspect_compute_backend_params(
+    const std::vector<ggml_tensor*>& tensors) const {
+    WeightResidencyInfo info;
+    std::vector<TensorState*> states;
+    if (!resolve_required_tensor_states(tensors, states)) {
+        return info;
+    }
+
+    ggml_backend_t prefetch_compute_backend = nullptr;
+    bool has_missing_params                 = false;
+    bool prefetch_candidate                 = true;
+    for (TensorState* state : states) {
+        if (state == nullptr || should_ignore(*state) ||
+            is_optional_missing_tensor(state->name)) {
+            continue;
+        }
+        const bool compute_resident =
+            state->compute_backend == state->params_backend
+                ? state->loaded_to_params_backend
+                : state->staged_to_compute_backend;
+        if (compute_resident) {
+            continue;
+        }
+        has_missing_params = true;
+        if (split_buffer_type_for(*state) != nullptr) {
+            prefetch_candidate = false;
+        }
+        if (state->compute_backend == state->params_backend ||
+            state->compute_backend == nullptr || sd_backend_is_cpu(state->compute_backend)) {
+            prefetch_candidate = false;
+            continue;
+        }
+        if (prefetch_compute_backend == nullptr) {
+            prefetch_compute_backend = state->compute_backend;
+        } else if (prefetch_compute_backend != state->compute_backend) {
+            prefetch_candidate = false;
+        }
+    }
+    info.missing_bytes = compute_backend_alloc_size(states, true);
+    if (has_missing_params && prefetch_candidate && prefetch_compute_backend != nullptr) {
+        ggml_backend_dev_t device = ggml_backend_get_device(prefetch_compute_backend);
+        if (device != nullptr) {
+            ggml_backend_dev_props props{};
+            ggml_backend_dev_get_props(device, &props);
+            info.async_prefetch_supported = props.caps.async;
+        }
+    }
+    return info;
+}
+
+void ModelManager::set_workspace_reclaimer(uintptr_t owner_id, std::function<bool()> reclaim) {
+    workspace_reclaimers_[owner_id] = std::move(reclaim);
+}
+
+void ModelManager::remove_runtime_owner(uintptr_t owner_id) {
+    workspace_reclaimers_.erase(owner_id);
+    for (auto it = runtime_residencies_.begin(); it != runtime_residencies_.end();) {
+        if (it->first.first == owner_id) {
+            it = runtime_residencies_.erase(it);
+        } else {
+            ++it;
+        }
+    }
+}
+
+ModelManager::CapacityCheck ModelManager::check_capacity(
+    const DeviceMemoryRequest& request,
+    const std::vector<TensorState*>& states) const {
+    CapacityCheck result;
+    if (request.compute_backend == nullptr || sd_backend_is_cpu(request.compute_backend)) {
+        return result;
+    }
+    auto add                     = [](size_t a, size_t b) { return b > SIZE_MAX - a ? SIZE_MAX : a + b; };
+    const size_t missing         = compute_backend_alloc_size(states, true);
+    result.required_device_bytes = add(request.pending_allocation_bytes, missing);
+    result.required_budget_bytes = add(request.runtime_peak_bytes(), missing);
+    auto device                  = ggml_backend_get_device(request.compute_backend);
+    if (device != nullptr) {
+        size_t free_bytes = 0, total_bytes = 0;
+        ggml_backend_dev_memory(device, &free_bytes, &total_bytes);
+        if (free_bytes != 0 || total_bytes != 0) {
+            result.available_device_bytes = free_bytes;
+        }
+    }
+    if (request.max_backend_bytes > 0) {
+        const size_t resident         = add(compute_backend_resident_bytes(request.compute_backend),
+                                            other_runtime_resident_bytes(request.owner_id, request.compute_backend));
+        result.available_budget_bytes = resident < request.max_backend_bytes
+                                            ? request.max_backend_bytes - resident
+                                            : 0;
+    }
+    std::map<ggml_backend_t, size_t> split_devices;
+    for (auto state : states) {
+        auto placement = split_buffer_devices_.find(split_buffer_type_for(*state));
+        if (placement != split_buffer_devices_.end()) {
+            for (const auto& entry : placement->second) {
+                auto inserted = split_devices.emplace(entry);
+                if (!inserted.second && entry.second > 0) {
+                    auto& limit = inserted.first->second;
+                    limit       = limit == 0 ? entry.second : std::min(limit, entry.second);
+                }
+            }
+        }
+    }
+    // GGML exposes only a split buffer's total size, not per-device allocations.
+    // Charge that upper bound on every participant instead of undercounting a shard.
+    for (const auto& entry : split_devices) {
+        size_t free_bytes = 0, total_bytes = 0;
+        ggml_backend_dev_memory(ggml_backend_get_device(entry.first), &free_bytes, &total_bytes);
+        if (free_bytes != 0 || total_bytes != 0) {
+            result.available_device_bytes = std::min(result.available_device_bytes, free_bytes);
+        }
+        if (entry.second > 0) {
+            const size_t resident         = add(compute_backend_resident_bytes(entry.first),
+                                                other_runtime_resident_bytes(request.owner_id, entry.first));
+            const size_t available        = resident < entry.second ? entry.second - resident : 0;
+            result.available_budget_bytes = std::min(result.available_budget_bytes, available);
+        }
+    }
+    return result;
+}
+
+bool ModelManager::fits_compute_backend_capacity(
+    const DeviceMemoryRequest& request,
+    const std::vector<ggml_tensor*>& required_params) const {
+    std::vector<TensorState*> states;
+    return resolve_required_tensor_states(required_params, states, request.compute_backend) &&
+           check_capacity(request, states).fits();
+}
+
+bool ModelManager::ensure_compute_backend_capacity(
+    const DeviceMemoryRequest& request,
+    const std::vector<ggml_tensor*>& required_params,
+    const std::vector<std::vector<ggml_tensor*>>& preferred_eviction_order,
+    const std::vector<ggml_tensor*>& protected_params) {
+    std::vector<TensorState*> required_states;
+    if (!resolve_required_tensor_states(required_params, required_states, request.compute_backend)) {
+        return false;
+    }
+
+    ggml_backend_t compute_backend = request.compute_backend;
+    if (compute_backend == nullptr) {
+        LOG_ERROR("model manager cannot reclaim memory for a null compute backend");
+        return false;
+    }
+    if (sd_backend_is_cpu(compute_backend)) {
+        return true;
+    }
+
+    auto fits = [&]() { return check_capacity(request, required_states).fits(); };
+    if (fits()) {
+        return true;
+    }
+    for (const auto& entry : workspace_reclaimers_) {
+        if (entry.first != request.owner_id) {
+            entry.second();
+            if (fits()) {
+                return true;
+            }
+        }
+    }
+
+    std::unordered_set<TensorState*> protected_states;
+    std::vector<TensorState*> resolved_protected;
+    if (!resolve_required_tensor_states(protected_params, resolved_protected)) {
+        return false;
+    }
+    protected_states.insert(resolved_protected.begin(), resolved_protected.end());
+    for (const auto& entry : prefetch_blocks_) {
+        if (entry.second != nullptr) {
+            protected_states.insert(entry.second->states.begin(), entry.second->states.end());
+        }
+    }
+
+    std::unordered_set<TensorState*> eviction_states;
+    auto add_evictable_state = [&](TensorState* state) {
+        if (state == nullptr || state->compute_backend != compute_backend ||
+            state->pin_count > 0 || protected_states.find(state) != protected_states.end()) {
+            return;
+        }
+        const bool reloadable = state->residency_mode == ResidencyMode::Disk ||
+                                state->compute_backend != state->params_backend;
+        const bool resident = state->compute_backend == state->params_backend
+                                  ? state->loaded_to_params_backend
+                                  : state->staged_to_compute_backend;
+        if (reloadable && resident) {
+            eviction_states.insert(state);
+        }
+    };
+    auto release_eviction_states = [&]() {
+        release_compute_staging_blocks(false, &eviction_states);
+        release_params_storage_blocks(false, &eviction_states);
+        return fits();
+    };
+
+    for (const auto& candidate_params : preferred_eviction_order) {
+        std::vector<TensorState*> candidate_states;
+        if (!resolve_required_tensor_states(candidate_params, candidate_states)) {
+            return false;
+        }
+        for (TensorState* state : candidate_states) {
+            add_evictable_state(state);
+        }
+        if (release_eviction_states()) {
+            return true;
+        }
+    }
+
+    std::vector<TensorState*> global_candidates;
+    global_candidates.reserve(tensor_states_.size());
+    for (const auto& state : tensor_states_) {
+        if (state != nullptr && eviction_states.find(state.get()) == eviction_states.end()) {
+            global_candidates.push_back(state.get());
+        }
+    }
+    std::stable_sort(global_candidates.begin(),
+                     global_candidates.end(),
+                     [](const TensorState* lhs, const TensorState* rhs) {
+                         return lhs->last_use_epoch < rhs->last_use_epoch;
+                     });
+    for (TensorState* state : global_candidates) {
+        add_evictable_state(state);
+        if (release_eviction_states()) {
+            return true;
+        }
+    }
+
+    const auto capacity = check_capacity(request, required_states);
+    LOG_WARN("model manager cannot make enough memory available on %s: need %.2f MB device / %.2f MB budget, available %.2f MB device / %.2f MB budget",
+             ggml_backend_name(compute_backend),
+             capacity.required_device_bytes / (1024.0 * 1024.0),
+             capacity.required_budget_bytes / (1024.0 * 1024.0),
+             capacity.available_device_bytes / (1024.0 * 1024.0),
+             capacity.available_budget_bytes / (1024.0 * 1024.0));
+    return false;
+}

src/model_manager.h

@@ -9,10 +9,10 @@
 #include <unordered_set>
 #include <vector>
 
+#include "device_residency_manager.h"
 #include "model_loader.h"
-#include "weight_manager.h"
 
-class ModelManager : public RunnerWeightManager {
+class ModelManager : public DeviceResidencyManager {
 public:
     enum class ResidencyMode {
         Disk,
@@ -28,24 +28,27 @@ public:
     };
 
 private:
+    static constexpr size_t MAX_RESIDENCY_BLOCK_BYTES = 64ULL * 1024ULL * 1024ULL;
+
     struct TensorState {
         std::string name;
         ggml_tensor* tensor = nullptr;
         std::string desc;
 
-        ResidencyMode residency_mode       = ResidencyMode::ParamBackend;
-        ggml_backend_t compute_backend     = nullptr;
-        ggml_backend_t params_backend      = nullptr;
-        bool allow_split_buffer            = false;
-        bool params_follow_compute_backend = false;
-        bool metadata_validated            = false;
-        enum ggml_op usage_op              = GGML_OP_NONE;
+        ResidencyMode residency_mode                 = ResidencyMode::ParamBackend;
+        ggml_backend_t compute_backend               = nullptr;
+        ggml_backend_t params_backend                = nullptr;
+        ggml_backend_buffer_type_t split_buffer_type = nullptr;
+        bool params_follow_compute_backend           = false;
+        bool metadata_validated                      = false;
+        enum ggml_op usage_op                        = GGML_OP_NONE;
 
-        int active_prepare_count = 0;
+        int pin_count = 0;
 
         bool loaded_to_params_backend  = false;
         bool staged_to_compute_backend = false;
         uint64_t applied_lora_epoch    = UINT64_MAX;
+        uint64_t last_use_epoch        = 0;
     };
 
     struct ParamsStorageBlock {
@@ -61,26 +64,55 @@ private:
         std::vector<std::pair<TensorState*, ggml_tensor*>> staged_tensors;
     };
 
+    struct PrefetchBlock {
+        std::vector<TensorState*> states;
+        ggml_backend_t compute_backend  = nullptr;
+        ggml_backend_t transfer_backend = nullptr;
+        ggml_backend_event_t event      = nullptr;
+        std::vector<std::unique_ptr<ComputeStagingBlock>> staging_blocks;
+    };
+
+    struct RuntimeResidency {
+        ggml_backend_t compute_backend = nullptr;
+        size_t resident_bytes          = 0;
+    };
+
     ModelLoader model_loader_;
     std::vector<std::unique_ptr<TensorState>> tensor_states_;
     std::map<std::string, TensorState*> tensor_states_by_name_;
     std::vector<std::unique_ptr<ParamsStorageBlock>> params_storage_blocks_;
     std::vector<std::unique_ptr<ComputeStagingBlock>> compute_staging_blocks_;
     std::map<ggml_backend_t, ggml_backend_buffer_type_t> split_buffer_types_;
+    std::map<ggml_backend_buffer_type_t, std::vector<std::pair<ggml_backend_t, size_t>>> split_buffer_devices_;
+    std::map<uintptr_t, std::unique_ptr<PrefetchBlock>> prefetch_blocks_;
+    std::map<ggml_backend_t, ggml_backend_t> prefetch_backends_;
+    std::map<std::pair<uintptr_t, ggml_backend_t>, RuntimeResidency> runtime_residencies_;
+    std::map<uintptr_t, std::function<bool()>> workspace_reclaimers_;
     bool warned_split_lora_skip_ = false;
     std::set<std::string> common_ignore_tensors_;
     std::vector<LoraSpec> loras_;
-    SDVersion lora_version_      = VERSION_COUNT;
-    uint64_t current_lora_epoch_ = 0;
-    int n_threads_               = 0;
-    bool enable_mmap_            = false;
-    bool writable_mmap_          = false;
+    SDVersion lora_version_          = VERSION_COUNT;
+    uint64_t current_lora_epoch_     = 0;
+    uint64_t residency_epoch_        = 0;
+    int n_threads_                   = 0;
+    bool enable_mmap_                = false;
+    bool writable_mmap_              = false;
+    bool segmented_compute_disabled_ = false;
+    bool prefetch_disabled_          = false;
 
     void finish_compute_backend_usage(const std::vector<TensorState*>& states);
     void release_all();
 
+    ggml_backend_t prefetch_backend_for(ggml_backend_t compute_backend);
+    bool populate_prefetch_block(PrefetchBlock& block);
+    void synchronize_prefetch_block(PrefetchBlock& block);
+    void free_prefetch_block(PrefetchBlock& block);
+    void clear_all_prefetched_params();
+    void release_prefetch();
+
     bool resolve_required_tensor_states(const std::vector<ggml_tensor*>& tensors,
-                                        std::vector<TensorState*>& required_states) const;
+                                        std::vector<TensorState*>& required_states,
+                                        ggml_backend_t compute_backend = nullptr) const;
     bool should_ignore(const TensorState& state) const;
     bool is_optional_missing_tensor(const std::string& name) const;
     bool validate_tensor(const TensorState& state) const;
@@ -94,6 +126,21 @@ private:
                               std::vector<ParamsStorageBlock*>& created_storage_blocks);
     bool load_tensors(const std::vector<TensorState*>& states);
     bool stage_tensors_to_compute_backend(const std::vector<TensorState*>& states);
+    size_t compute_backend_alloc_size(const std::vector<TensorState*>& states,
+                                      bool missing_only) const;
+    size_t compute_backend_resident_bytes(ggml_backend_t compute_backend) const;
+    struct CapacityCheck {
+        size_t required_device_bytes  = 0;
+        size_t required_budget_bytes  = 0;
+        size_t available_device_bytes = SIZE_MAX;
+        size_t available_budget_bytes = SIZE_MAX;
+        bool fits() const {
+            return required_device_bytes <= available_device_bytes &&
+                   required_budget_bytes <= available_budget_bytes;
+        }
+    };
+    CapacityCheck check_capacity(const DeviceMemoryRequest& request,
+                                 const std::vector<TensorState*>& states) const;
 
     ggml_backend_buffer_type_t params_buffer_type_for(const TensorState& state) const;
     ggml_backend_buffer_type_t split_buffer_type_for(const TensorState& state) const;
@@ -105,6 +152,8 @@ private:
     void free_params_storage_block(ParamsStorageBlock& block);
     void erase_params_storage_block(ParamsStorageBlock* block);
     void reset_lora_applied_params();
+    size_t other_runtime_resident_bytes(uintptr_t owner_id,
+                                        ggml_backend_t compute_backend) const;
 
 public:
     ~ModelManager() override;
@@ -116,11 +165,15 @@ public:
         n_threads_ = n_threads;
         model_loader_.set_n_threads(n_threads);
     }
+    void set_segmented_compute_disabled(bool disabled) {
+        segmented_compute_disabled_ = disabled;
+    }
+    void set_prefetch_disabled(bool disabled) { prefetch_disabled_ = disabled; }
     void set_enable_mmap(bool enable_mmap) { enable_mmap_ = enable_mmap; }
     void set_writable_mmap(bool writable_mmap) { writable_mmap_ = writable_mmap; }
     void set_common_ignore_tensors(std::set<std::string> ignore_tensors);
     void set_loras(std::vector<LoraSpec> loras, SDVersion version);
-    void set_split_buffer_type(ggml_backend_t compute_backend, ggml_backend_buffer_type_t split_buft);
+    void set_split_buffer_type(ggml_backend_t compute_backend, ggml_backend_buffer_type_t split_buft, const std::vector<std::pair<ggml_backend_t, size_t>>& device_limits);
 
     static bool tensor_shape_supports_split_buffer(const ggml_tensor* tensor);
 
@@ -180,8 +233,30 @@ public:
     bool assign_compute_backend(const std::vector<ggml_tensor*>& tensors,
                                 ggml_backend_t compute_backend) override;
     bool prepare_params(const std::vector<ggml_tensor*>& tensors) override;
+    void set_workspace_reclaimer(uintptr_t owner_id, std::function<bool()> reclaim) override;
+    void remove_runtime_owner(uintptr_t owner_id) override;
+    bool fits_compute_backend_capacity(const DeviceMemoryRequest& request,
+                                       const std::vector<ggml_tensor*>& required_params) const override;
+    bool segmented_compute_enabled() const override { return !segmented_compute_disabled_; }
+    bool prefetch_enabled() const override { return !prefetch_disabled_; }
     void release_compute_backend_params(const std::vector<ggml_tensor*>& tensors) override;
-    void release_params_backend_params(const std::vector<ggml_tensor*>& tensors) override;
+    void evict_compute_backend_params(const std::vector<ggml_tensor*>& tensors) override;
+    WeightResidencyInfo inspect_compute_backend_params(
+        const std::vector<ggml_tensor*>& tensors) const override;
+    void update_runtime_residency(uintptr_t owner_id,
+                                  ggml_backend_t compute_backend,
+                                  size_t resident_bytes) override;
+    bool ensure_compute_backend_capacity(
+        const DeviceMemoryRequest& request,
+        const std::vector<ggml_tensor*>& required_params,
+        const std::vector<std::vector<ggml_tensor*>>& preferred_eviction_order,
+        const std::vector<ggml_tensor*>& protected_params) override;
+    WeightPrefetchResult prefetch_params(
+        uintptr_t owner_id,
+        const std::vector<ggml_tensor*>& tensors) override;
+    bool activate_prefetched_params(uintptr_t owner_id,
+                                    const std::vector<ggml_tensor*>& tensors) override;
+    void clear_prefetched_params(uintptr_t owner_id) override;
 };
 
 #endif  // __MODEL_MANAGER_H__

src/model_manager_prefetch.cpp

@@ -0,0 +1,353 @@
+#include "model_manager.h"
+
+#include <algorithm>
+#include <utility>
+
+#include "core/ggml_extend_backend.h"
+#include "core/util.h"
+
+ggml_backend_t ModelManager::prefetch_backend_for(ggml_backend_t compute_backend) {
+    auto existing = prefetch_backends_.find(compute_backend);
+    if (existing != prefetch_backends_.end()) {
+        return existing->second;
+    }
+    if (compute_backend == nullptr) {
+        return nullptr;
+    }
+
+    ggml_backend_dev_t device = ggml_backend_get_device(compute_backend);
+    if (device == nullptr || ggml_backend_dev_type(device) == GGML_BACKEND_DEVICE_TYPE_CPU) {
+        return nullptr;
+    }
+    ggml_backend_t transfer_backend = ggml_backend_dev_init(device, nullptr);
+    if (transfer_backend == nullptr) {
+        LOG_WARN("model manager failed to create a prefetch backend for %s",
+                 ggml_backend_name(compute_backend));
+    }
+    prefetch_backends_[compute_backend] = transfer_backend;
+    return transfer_backend;
+}
+
+void ModelManager::synchronize_prefetch_block(PrefetchBlock& block) {
+    if (block.event != nullptr) {
+        ggml_backend_event_synchronize(block.event);
+        ggml_backend_event_free(block.event);
+        block.event = nullptr;
+    } else if (block.transfer_backend != nullptr) {
+        ggml_backend_synchronize(block.transfer_backend);
+    }
+    block.transfer_backend = nullptr;
+}
+
+void ModelManager::free_prefetch_block(PrefetchBlock& block) {
+    synchronize_prefetch_block(block);
+    for (auto& staging_block : block.staging_blocks) {
+        if (staging_block == nullptr) {
+            continue;
+        }
+        staging_block->staged_tensors.clear();
+        if (staging_block->buffer != nullptr) {
+            ggml_backend_buffer_free(staging_block->buffer);
+            staging_block->buffer = nullptr;
+        }
+        if (staging_block->staging_ctx != nullptr) {
+            ggml_free(staging_block->staging_ctx);
+            staging_block->staging_ctx = nullptr;
+        }
+    }
+    block.staging_blocks.clear();
+}
+
+bool ModelManager::populate_prefetch_block(PrefetchBlock& block) {
+    if (block.states.empty() || block.compute_backend == nullptr) {
+        return false;
+    }
+
+    block.transfer_backend = prefetch_backend_for(block.compute_backend);
+    if (block.transfer_backend == nullptr) {
+        return false;
+    }
+
+    for (TensorState* state : block.states) {
+        if (state == nullptr || state->tensor == nullptr ||
+            state->tensor->buffer == nullptr || state->tensor->data == nullptr ||
+            state->params_backend == nullptr || state->staged_to_compute_backend ||
+            state->pin_count > 0) {
+            return false;
+        }
+    }
+
+    ggml_backend_buffer_type_t buffer_type =
+        ggml_backend_get_default_buffer_type(block.compute_backend);
+    if (buffer_type == nullptr) {
+        return false;
+    }
+    const size_t alignment = ggml_backend_buft_get_alignment(buffer_type);
+    size_t backend_limit   = ggml_backend_buft_get_max_size(buffer_type);
+    if (!ggml_backend_buft_is_host(buffer_type) &&
+        (backend_limit == 0 || backend_limit > MAX_RESIDENCY_BLOCK_BYTES)) {
+        backend_limit = MAX_RESIDENCY_BLOCK_BYTES;
+    }
+
+    auto enqueue_chunk = [&](const std::vector<TensorState*>& chunk) -> bool {
+        if (chunk.empty()) {
+            return true;
+        }
+        ggml_init_params init_params;
+        init_params.mem_size      = chunk.size() * ggml_tensor_overhead();
+        init_params.mem_buffer    = nullptr;
+        init_params.no_alloc      = true;
+        ggml_context* staging_ctx = ggml_init(init_params);
+        if (staging_ctx == nullptr) {
+            return false;
+        }
+
+        auto staging_block             = std::make_unique<ComputeStagingBlock>();
+        staging_block->compute_backend = block.compute_backend;
+        staging_block->staging_ctx     = staging_ctx;
+        staging_block->staged_tensors.reserve(chunk.size());
+        for (TensorState* state : chunk) {
+            ggml_tensor* staging_tensor = ggml_dup_tensor(staging_ctx, state->tensor);
+            ggml_set_name(staging_tensor, state->tensor->name);
+            if (ggml_backend_buffer_is_host(state->tensor->buffer) &&
+                (!ggml_is_contiguous(state->tensor) || !ggml_is_contiguous(staging_tensor) ||
+                 ggml_nbytes(state->tensor) != ggml_nbytes(staging_tensor))) {
+                ggml_free(staging_ctx);
+                staging_block->staging_ctx = nullptr;
+                return false;
+            }
+            staging_block->staged_tensors.push_back({state, staging_tensor});
+        }
+        staging_block->buffer =
+            ggml_backend_alloc_ctx_tensors_from_buft(staging_ctx, buffer_type);
+        if (staging_block->buffer == nullptr) {
+            ggml_free(staging_ctx);
+            staging_block->staging_ctx = nullptr;
+            return false;
+        }
+        ggml_backend_buffer_set_usage(staging_block->buffer,
+                                      GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
+
+        for (const auto& pair : staging_block->staged_tensors) {
+            TensorState* state          = pair.first;
+            ggml_tensor* staging_tensor = pair.second;
+            const bool host_source      = ggml_backend_buffer_is_host(state->tensor->buffer);
+            if (host_source) {
+                ggml_backend_tensor_set_async(block.transfer_backend,
+                                              staging_tensor,
+                                              state->tensor->data,
+                                              0,
+                                              ggml_nbytes(state->tensor));
+            } else {
+                ggml_backend_tensor_copy_async(state->params_backend,
+                                               block.transfer_backend,
+                                               state->tensor,
+                                               staging_tensor);
+            }
+        }
+        block.staging_blocks.push_back(std::move(staging_block));
+        return true;
+    };
+
+    std::vector<TensorState*> chunk;
+    size_t chunk_size = 0;
+    for (TensorState* state : block.states) {
+        const size_t tensor_size = GGML_PAD(
+            ggml_backend_buft_get_alloc_size(buffer_type, state->tensor), alignment);
+        if (!chunk.empty() && backend_limit > 0 &&
+            tensor_size > backend_limit - std::min(chunk_size, backend_limit)) {
+            if (!enqueue_chunk(chunk)) {
+                return false;
+            }
+            chunk.clear();
+            chunk_size = 0;
+        }
+        chunk.push_back(state);
+        chunk_size = tensor_size > SIZE_MAX - chunk_size ? SIZE_MAX : chunk_size + tensor_size;
+    }
+    if (!enqueue_chunk(chunk)) {
+        return false;
+    }
+
+    ggml_backend_dev_t device = ggml_backend_get_device(block.transfer_backend);
+    block.event               = ggml_backend_event_new(device);
+    if (block.event != nullptr) {
+        ggml_backend_event_record(block.event, block.transfer_backend);
+    }
+
+    size_t total_size = 0;
+    for (const auto& staging_block : block.staging_blocks) {
+        if (staging_block != nullptr && staging_block->buffer != nullptr) {
+            const size_t buffer_size = ggml_backend_buffer_get_size(staging_block->buffer);
+            total_size               = buffer_size > SIZE_MAX - total_size ? SIZE_MAX : total_size + buffer_size;
+        }
+    }
+    LOG_DEBUG("model manager queued segment prefetch (%6.2f MB, %zu tensors) to %s",
+              total_size / (1024.f * 1024.f),
+              block.states.size(),
+              ggml_backend_name(block.compute_backend));
+    return true;
+}
+
+WeightPrefetchResult ModelManager::prefetch_params(
+    uintptr_t owner_id,
+    const std::vector<ggml_tensor*>& tensors) {
+    if (tensors.empty()) {
+        return WeightPrefetchResult::AlreadyResident;
+    }
+
+    std::vector<TensorState*> required_states;
+    if (!resolve_required_tensor_states(tensors, required_states)) {
+        return WeightPrefetchResult::Failed;
+    }
+
+    std::vector<TensorState*> states;
+    states.reserve(required_states.size());
+    ggml_backend_t compute_backend = nullptr;
+    bool needs_synchronous_load    = false;
+    for (TensorState* state : required_states) {
+        if (state == nullptr || should_ignore(*state) ||
+            is_optional_missing_tensor(state->name)) {
+            continue;
+        }
+        if (state->compute_backend == state->params_backend) {
+            needs_synchronous_load = needs_synchronous_load ||
+                                     !state->loaded_to_params_backend;
+            continue;
+        }
+        if (state->staged_to_compute_backend || state->pin_count > 0) {
+            continue;
+        }
+        // Split buffers cannot use the primary device's asynchronous upload path.
+        if (split_buffer_type_for(*state) != nullptr) {
+            return WeightPrefetchResult::Unsupported;
+        }
+        if (compute_backend == nullptr) {
+            compute_backend = state->compute_backend;
+        } else if (compute_backend != state->compute_backend) {
+            return WeightPrefetchResult::Failed;
+        }
+        states.push_back(state);
+    }
+    if (states.empty()) {
+        return needs_synchronous_load ? WeightPrefetchResult::Unsupported
+                                      : WeightPrefetchResult::AlreadyResident;
+    }
+    if (compute_backend == nullptr || sd_backend_is_cpu(compute_backend)) {
+        return WeightPrefetchResult::Unsupported;
+    }
+    ggml_backend_dev_t compute_device = ggml_backend_get_device(compute_backend);
+    ggml_backend_dev_props compute_props{};
+    if (compute_device == nullptr) {
+        return WeightPrefetchResult::Unsupported;
+    }
+    ggml_backend_dev_get_props(compute_device, &compute_props);
+    if (!compute_props.caps.async) {
+        return WeightPrefetchResult::Unsupported;
+    }
+    clear_prefetched_params(owner_id);
+    if (!load_tensors_to_params_backend(states)) {
+        return WeightPrefetchResult::Failed;
+    }
+
+    auto block             = std::make_unique<PrefetchBlock>();
+    block->states          = std::move(states);
+    block->compute_backend = compute_backend;
+    if (!populate_prefetch_block(*block)) {
+        free_prefetch_block(*block);
+        return WeightPrefetchResult::Failed;
+    }
+    prefetch_blocks_[owner_id] = std::move(block);
+    return WeightPrefetchResult::Scheduled;
+}
+
+bool ModelManager::activate_prefetched_params(
+    uintptr_t owner_id,
+    const std::vector<ggml_tensor*>& tensors) {
+    std::vector<TensorState*> required_states;
+    if (!resolve_required_tensor_states(tensors, required_states)) {
+        return false;
+    }
+
+    const bool already_staged = std::all_of(
+        required_states.begin(),
+        required_states.end(),
+        [&](TensorState* state) {
+            return state == nullptr || should_ignore(*state) ||
+                   is_optional_missing_tensor(state->name) ||
+                   state->compute_backend == state->params_backend ||
+                   state->staged_to_compute_backend;
+        });
+    if (already_staged) {
+        clear_prefetched_params(owner_id);
+        return true;
+    }
+
+    auto existing = prefetch_blocks_.find(owner_id);
+    if (existing == prefetch_blocks_.end()) {
+        return false;
+    }
+    std::unique_ptr<PrefetchBlock> block = std::move(existing->second);
+    prefetch_blocks_.erase(existing);
+    synchronize_prefetch_block(*block);
+
+    for (const auto& staging_block : block->staging_blocks) {
+        if (staging_block == nullptr) {
+            continue;
+        }
+        for (const auto& pair : staging_block->staged_tensors) {
+            TensorState* state          = pair.first;
+            ggml_tensor* staging_tensor = pair.second;
+            if (state == nullptr || state->tensor == nullptr || staging_tensor == nullptr ||
+                state->staged_to_compute_backend || state->pin_count > 0) {
+                free_prefetch_block(*block);
+                return false;
+            }
+        }
+    }
+    const uint64_t use_epoch = ++residency_epoch_;
+    for (auto& staging_block : block->staging_blocks) {
+        if (staging_block == nullptr) {
+            continue;
+        }
+        for (auto& pair : staging_block->staged_tensors) {
+            TensorState* state          = pair.first;
+            ggml_tensor* staging_tensor = pair.second;
+            std::swap(state->tensor->buffer, staging_tensor->buffer);
+            std::swap(state->tensor->data, staging_tensor->data);
+            std::swap(state->tensor->extra, staging_tensor->extra);
+            state->staged_to_compute_backend = true;
+            state->last_use_epoch            = use_epoch;
+        }
+        compute_staging_blocks_.push_back(std::move(staging_block));
+    }
+    block->staging_blocks.clear();
+    return true;
+}
+
+void ModelManager::clear_prefetched_params(uintptr_t owner_id) {
+    auto existing = prefetch_blocks_.find(owner_id);
+    if (existing == prefetch_blocks_.end()) {
+        return;
+    }
+    std::unique_ptr<PrefetchBlock> block = std::move(existing->second);
+    prefetch_blocks_.erase(existing);
+    free_prefetch_block(*block);
+}
+
+void ModelManager::clear_all_prefetched_params() {
+    for (auto& entry : prefetch_blocks_) {
+        free_prefetch_block(*entry.second);
+    }
+    prefetch_blocks_.clear();
+}
+
+void ModelManager::release_prefetch() {
+    clear_all_prefetched_params();
+    for (auto& entry : prefetch_backends_) {
+        if (entry.second != nullptr) {
+            ggml_backend_free(entry.second);
+        }
+    }
+    prefetch_backends_.clear();
+}

src/name_conversion.cpp

@@ -149,6 +149,7 @@ std::string convert_cond_stage_model_name(std::string name, std::string prefix)
         {"ffn_up.", "mlp.up_proj."},
         {"ffn_post_norm.", "post_ffw_norm."},
         {"ffn_norm.", "post_attention_layernorm."},
+        {"layer_output_scale.weight", "layer_scalar"},
         {"output_norm.", "model.norm."},
     };
 
@@ -1447,7 +1448,7 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
             }
         }
 
-        // LOG_DEBUG("name %s %d", name.c_str(), version);
+        // LOG_VERBOSE("name %s %d", name.c_str(), version);
 
         if (sd_version_is_unet(version) || is_underline || is_lycoris_underline) {
             name = convert_sep_to_dot(name);
@@ -1459,6 +1460,7 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
         {"unet.", "model.diffusion_model."},
         {"transformer.", "model.diffusion_model."},  // dit
         {"vae.", "first_stage_model."},
+        {"text_encoders.llm.text_embedding_projection.", "text_embedding_projection."},
         {"text_encoder.", "cond_stage_model.transformer."},
         {"te.", "cond_stage_model.transformer."},
         {"text_encoder.2.", "cond_stage_model.1.transformer."},
@@ -1569,6 +1571,11 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
         }
     }
 
+    static const std::vector<std::pair<std::string, std::string>> generic_name_map = {
+        {".scale_weight", ".weight_scale"},
+    };
+    replace_with_name_map(name, generic_name_map);
+
     if (is_lora) {
         name = "lora." + name;
     }