mtmd : add MiniCPM-OCR 1.0 support (#30979)

Signed-off-by: tc-mb <tianchi_cai@icloud.com>

ci/run.sh

@@ -447,10 +447,14 @@ function gg_run_qwen3_0_6b {
 
     (time ./bin/llama-imatrix --model ${model_f16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log
 
-    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
-    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa on  --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
-    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa off                ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
-    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa on                 ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+    # TODO: these tests are extremely slow on WebGPU - re-enable when fixed
+    #       https://github.com/ggml-org/llama.cpp/actions/runs/39639985732
+    if [ -z ${GG_BUILD_WEBGPU} ]; then
+        (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+        (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa on  --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+        (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa off                ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+        (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa on                 ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+    fi
 
     function check_ppl {
         qnt="$1"

common/jinja/value.cpp

@@ -425,7 +425,12 @@ const func_builtins & global_builtins() {
             throw raised_exception("Jinja Exception: " + msg);
         }},
         {"dict", toobject},
-        {"namespace", toobject},
+        {"namespace", [](const func_args & args) -> value {
+            // unlike a dict, a namespace has no methods: ns.items is the attribute, as in jinja2
+            value out = toobject(args);
+            cast_val<value_object>(out)->has_builtins = false;
+            return out;
+        }},
         {"strftime_now", [](const func_args & args) -> value {
             args.ensure_vals<value_string>();
             std::string format = args.get_pos(0)->as_string().str();

conversion/__init__.py

@@ -20,6 +20,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "NEOChatModel": "sensenova_u1",
     "AfmoeForCausalLM": "afmoe",
     "LagunaForCausalLM": "laguna",
+    "FlintForCausalLM": "flint",
     "ApertusForCausalLM": "llama",
     "ArceeForCausalLM": "llama",
     "ArcticForCausalLM": "arctic",
@@ -195,6 +196,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "MiniCPMForCausalLM": "minicpm",
     "MiniCPMV4_6ForConditionalGeneration": "minicpm",
     "MiniCPMV4_7ForConditionalGeneration": "minicpm",
+    "MiniCPMOcrForConditionalGeneration": "minicpm",
     "MiniMaxText01ForCausalLM": "minimax",
     "MiniMaxM1ForCausalLM": "minimax",
     "MiniMaxM2ForCausalLM": "minimax",
@@ -360,6 +362,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "MiniMaxM3SparseForConditionalGeneration": "minimax",
     "MiniCPMV4_6ForConditionalGeneration": "minicpm",
     "MiniCPMV4_7ForConditionalGeneration": "minicpm",
+    "MiniCPMOcrForConditionalGeneration": "minicpm",
     "Mistral3ForConditionalGeneration": "llava",
     "Mistral3Model": "llava",
     "NemotronH_Omni_Reasoning_V3": "nemotron",

conversion/base.py

@@ -1701,7 +1701,7 @@ class TextModel(ModelBase):
             self.gguf_writer.add_expert_group_used_count(n_group_used)
             logger.info(f"gguf: expert groups used count = {n_group_used}")
 
-        if (score_func := self.find_hparam(["score_function", "scoring_func", "score_func", "moe_router_activation", "moe_router_activation_func", "expert_selection_fn", "router_score_func"], optional=True)) is not None:
+        if (score_func := self.find_hparam(["score_function", "scoring_func", "score_func", "moe_router_activation", "moe_router_activation_func", "expert_selection_fn", "router_score_func", "router_score_function"], optional=True)) is not None:
             if score_func == "sigmoid":
                 self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
             elif score_func == "softmax":
@@ -1837,6 +1837,9 @@ class TextModel(ModelBase):
         if chkhsh == "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35":
             # ref: https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0
             res = "minerva-7b"
+        if chkhsh == "6e4d72c6e5260b90606e26a0931fa15e4b42ec6e97719d12919f2b1f93478262":
+            # ref: https://huggingface.co/domyn/Domyn-Small-v1.0
+            res = "domyn-small"
         if chkhsh == "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664":
             # ref: https://huggingface.co/tencent/Hunyuan-A13B-Instruct
             res = "hunyuan"
@@ -1939,6 +1942,9 @@ class TextModel(ModelBase):
         if chkhsh == "52df12b4c8d4176e7481aab4b6e8454d1fd0a210a04a574f6d4e067d10e23c3e":
             # ref: https://huggingface.co/CohereLabs/North-Mini-Code-1.0
             res = "cohere2moe"
+        if chkhsh == "d5420e601e979078c670bc503d9ac411c311fa76b76a76267c31e0d6ba816637":
+            # ref: https://huggingface.co/RecursiveSI/flint
+            res = "flint"
         if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea":
             # ref: https://huggingface.co/Qwen/Qwen1.5-7B
             res = "qwen2"

conversion/flint.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from .base import ModelBase, gguf
+from .qwen import Qwen2MoeModel
+
+
+@ModelBase.register("FlintForCausalLM")
+@ModelBase.example("RecursiveSI/flint")
+class FlintModel(Qwen2MoeModel):
+    model_arch = gguf.MODEL_ARCH.FLINT
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        dense = self.hparams["mlp_only_layers"]
+        if dense != list(range(len(dense))):
+            raise ValueError("Flint dense layers must precede the MoE layers")
+        self.gguf_writer.add_leading_dense_block_count(len(dense))
+        self.gguf_writer.add_expert_shared_count(
+            self.hparams["shared_expert_intermediate_size"] // self.hparams["moe_intermediate_size"]
+        )
+        self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])
+        self.gguf_writer.add_expert_weights_scale(1.0)
+        self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
+        self.gguf_writer.add_sliding_window_pattern([
+            kind == "sliding_attention" for kind in self.hparams["layer_types"]
+        ])
+        self.gguf_writer.add_rope_pattern([
+            i not in self.hparams["no_rope_layer_indices"] for i in range(self.block_count)
+        ])
+
+    def modify_tensors(self, data_torch, name, bid):
+        if name.endswith(".softmax_offset"):
+            name += ".weight"
+        if name.endswith(".gate.expert_bias"):
+            if not self.hparams["router_expert_bias"]:
+                return
+            name += ".bias"
+        yield from super().modify_tensors(data_torch, name, bid)

conversion/minicpm.py

@@ -272,3 +272,18 @@ class MiniCPMV4_7VisionModel(MiniCPMV4_6VisionModel):
             yield self.format_tensor_name(gguf.MODEL_TENSOR.V_TOK_EMBD_SEP, suffix=""), data_torch[ids]
             return
         yield from super().modify_tensors(data_torch, name, bid)
+
+
+# MiniCPM-OCR 1.0: the 4.6 stack with model_type minicpm_ocr.
+
+@ModelBase.register("MiniCPMOcrForConditionalGeneration")
+@ModelBase.example("openbmb/MiniCPM-OCR-1.0")
+class MiniCPMOCR1_0TextModel(MiniCPMV4_6TextModel):
+    model_arch = gguf.MODEL_ARCH.QWEN35
+    no_mtp = True
+
+
+@ModelBase.register("MiniCPMOcrForConditionalGeneration")
+@ModelBase.example("openbmb/MiniCPM-OCR-1.0")
+class MiniCPMOCR1_0VisionModel(MiniCPMV4_6VisionModel):
+    projector_type = gguf.VisionProjectorType.MINICPM_OCR_1_0

conversion/nemotron.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import re
 from typing import Any, Callable, Iterable, Sequence, TYPE_CHECKING
 
 import numpy as np
@@ -367,9 +368,21 @@ class NemotronModel(TextModel):
     model_arch = gguf.MODEL_ARCH.NEMOTRON
 
     def set_vocab(self):
-        self._set_vocab_sentencepiece()
-        self.gguf_writer.add_pad_token_id(0)
-        self.gguf_writer.add_unk_token_id(1)
+        if (self.dir_model / "tokenizer.model").exists():
+            self._set_vocab_sentencepiece()
+            self.gguf_writer.add_pad_token_id(0)
+            self.gguf_writer.add_unk_token_id(1)
+        else:
+            # SPM-style BPE (Metaspace + byte fallback): mark <0xXX> tokens as BYTE so they decode to raw bytes
+            tokens, toktypes, tokpre = self.get_vocab_base()
+            toktypes = [gguf.TokenType.BYTE if re.fullmatch(r"<0x[0-9A-F]{2}>", tok) else typ for tok, typ in zip(tokens, toktypes)]
+            self.gguf_writer.add_tokenizer_model("gpt2")
+            self.gguf_writer.add_tokenizer_pre(tokpre)
+            self.gguf_writer.add_token_list(tokens)
+            self.gguf_writer.add_token_types(toktypes)
+
+            special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
+            special_vocab.add_to_gguf(self.gguf_writer)
 
     def set_gguf_parameters(self):
         super().set_gguf_parameters()
@@ -383,7 +396,10 @@ class NemotronModel(TextModel):
         rot_pct = self.rope_parameters["partial_rotary_factor"]
         n_embd = self.find_hparam(["hidden_size", "n_embd"])
         n_head = self.find_hparam(["num_attention_heads", "n_head"])
-        self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head)
+        # Use explicit head_dim when available; fallback keeps standard Nemotron working.
+        # Without this, models where n_head * head_dim != n_embd (e.g. 48 heads, head_dim=128) produce the wrong rope dimension count.
+        head_dim = self.hparams.get("head_dim") or n_embd // n_head
+        self.gguf_writer.add_rope_dimension_count(int(rot_pct * head_dim))
 
         # * RopeScaling for Nemotron
         factor = self.hparams.get("factor") or self.rope_parameters.get("factor")

convert_hf_to_gguf_update.py

@@ -101,6 +101,7 @@ models = [
     {"name": "command-r",        "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", },
     {"name": "tiny_aya",         "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/tiny-aya-base", },
     {"name": "cohere2moe",       "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/North-Mini-Code-1.0", },
+    {"name": "flint",            "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/RecursiveSI/flint", },
     {"name": "qwen2",            "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", },
     {"name": "olmo",             "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", },
     {"name": "dbrx",             "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/databricks/dbrx-base", },
@@ -177,6 +178,7 @@ pre_computed_hashes = [
     {"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/zai-org/GLM-4.5-Air", "chkhsh": "9ca2dd618e8afaf09731a7cf6e2105b373ba6a1821559f258b272fe83e6eb902"},
     {"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/zai-org/GLM-4.7-Flash", "chkhsh": "cdf5f35325780597efd76153d4d1c16778f766173908894c04afc20108536267"},
     {"name": "minerva-7b", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0", "chkhsh": "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35"},
+    {"name": "domyn-small", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/domyn/Domyn-Small-v1.0", "chkhsh": "6e4d72c6e5260b90606e26a0931fa15e4b42ec6e97719d12919f2b1f93478262"},
     {"name": "hunyuan", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-A13B-Instruct", "chkhsh": "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664"},
     {"name": "hunyuan-dense", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-4B-Instruct", "chkhsh": "bba3b3366b646dbdded5dbc42d59598b849371afc42f7beafa914afaa5b70aa6"},
     {"name": "hy_v4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hy4-preview", "chkhsh": "e6ddf9c6686791c12d698d34c31ab9be1fea9af5a3d9a6909783ab382198ae1c"},

docs/backend/SYCL.md

@@ -819,6 +819,9 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
 | GGML_SYCL_MKL_FA_DIAG | 0 (default) or 1 | Enable output fingerprinting for MKL flash attention. Dumps the first 64 float output values for the first 6 FA calls with n_kv ≥ 1024, labeled with kernel type (MKL/TILE/VEC) for cross-kernel comparison. |
 | GGML_SYCL_ENABLE_FUSION | 0 or 1 (default) | Enable fused-kernel dispatch in graph compute. Unsupported types and layouts fall back to the standalone op kernels. See `ggml_sycl_can_fuse()`. |
 | GGML_SYCL_ENABLE_ESIMD | 0 or 1 (default)| Enable ESIMD kernels when available. |
+| GGML_SYCL_ENABLE_XMX_ESIMD | 0 or 1 (default) | Use the XMX (DPAS) multi-column mat-vec kernels for reordered Q2_K to Q6_K and Q8_0 weights on Xe2 GPUs. Speeds up speculative decoding and short prompt batches. |
+| GGML_SYCL_XMX_MAX_COLS | non-negative integer, 80 (default) | Most activation columns (batch size) the `GGML_SYCL_ENABLE_XMX_ESIMD` kernels take; wider batches use the other mat-mul kernels. A larger value keeps XMX for bigger batches (as repeated launches), `0` never uses it. |
+| GGML_SYCL_XMX_GLU_MAX_COLS | 0 to 16, 16 (default) | Most columns the fused gate and up XMX kernel takes; wider batches run gate and up as separate mat-vecs. 16 is the most the kernel is built for, larger values are clamped to 16 and `0` turns the fused kernel off. |
 | GGML_SYCL_XMX_GATHER_TYPES | decimal bitmask, all bits set (default) | Weight formats that may use the XMX dequant-GEMM paths, which dequantize weights straight into the XMX tiles. This speeds up prompt processing of MoE models on GPUs with XMX units (Arc A- and B-series, Arc Pro, Data Center GPU Max), for example pp512 of Qwen3-30B-A3B UD-IQ3_XXS by about 50% on an Arc Pro B60. Bits:<br>* 1: IQ4_NL, 2: IQ3_S, 4: IQ4_XS, 8: IQ3_XXS, 16: IQ2_XXS, 32: IQ2_XS, 64: IQ2_S, 128: IQ1_S, 256: IQ1_M<br>* 512: Q8_0, 1024: Q4_K, 2048: Q5_K, 4096: Q6_K (MoE `MUL_MAT_ID` only)<br>Add values to combine them, for example `3` for IQ4_NL and IQ3_S; `0` disables the paths. A set bit does not force the path: batches of more than 64 tokens per expert or row lengths that are not a multiple of 256 (32 for IQ4_NL and Q8_0) use the library GEMM. |
 | GGML_SYCL_XMX_GATHER_SHAPES | decimal bitmask, 255 (default) | XMX `joint_matrix` combinations the paths of `GGML_SYCL_XMX_GATHER_TYPES` may use; the operand type comes from `GGML_SYCL_DYNAMIC_PRECISION` and the best supported combination is picked automatically (logged as `fg_pick_combo`). Bits:<br>* Xe2, Xe3, Xe-HPC: 1: f16 8x16x16, 2: f16 16x16x16, 4: f16 32x64x16, 8: f16 32x64x32, 32: tf32 8x16x8, 64: bf16 8x16x16<br>* Xe-HPG (Arc A770, ARL-H): 16: f16 8x8x16, 128: bf16 8x8x16<br>Clear a bit to exclude a combination, or set a single bit to force one for testing. |
 | GGML_SYCL_DYNAMIC_PRECISION | `F16` (default with `GGML_SYCL_F16=ON`), `BF16`, `TF32` or `F32` (default otherwise) | Operand type of the XMX dequant-GEMM paths (`GGML_SYCL_XMX_GATHER_TYPES`); accumulation is always f32. `F16` is the fastest, but activations above 65504 overflow. `BF16` keeps the f32 range at a 7-bit mantissa, `TF32` keeps the range and the f16 mantissa but is about 30% slower and needs Xe2, Xe3 or Xe-HPC, and `F32` turns the XMX paths off. Ops that request a higher src1 precision ([TAG_GGML_PREC]) get it regardless of this setting. |

docs/ops.md

@@ -27,7 +27,7 @@ Legend:
 |                        COL2IM_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                           CONCAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
 |                             CONT | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
-|                       CONV_1D_DW | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
+|                       CONV_1D_DW | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ |
 |                          CONV_2D | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
 |                       CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          CONV_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
@@ -76,10 +76,10 @@ Legend:
 |                          MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 |
 |                 MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | 🟡 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 |
 |                       MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ❌ |
-|                  MUL_MAT_ID_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
-|                  MUL_MAT_ID_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
-|                     MUL_MAT_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
-|                     MUL_MAT_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
+|                  MUL_MAT_ID_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                  MUL_MAT_ID_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                     MUL_MAT_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                     MUL_MAT_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                              NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
 |                   OPT_STEP_ADAMW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |

ggml/src/ggml-cpu/CMakeLists.txt

@@ -650,9 +650,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
 
         # Fetch KleidiAI sources:
         include(FetchContent)
-        set(KLEIDIAI_COMMIT_TAG "v1.31.0")
+        set(KLEIDIAI_COMMIT_TAG "v1.32.0")
         set(KLEIDIAI_DOWNLOAD_URL "https://github.com/ARM-software/kleidiai/releases/download/${KLEIDIAI_COMMIT_TAG}/kleidiai-${KLEIDIAI_COMMIT_TAG}-src.tar.gz")
-        set(KLEIDIAI_RELEASE_ARCHIVE_MD5  "930495f3aeb16b25dc18a2bde45924f4")
+        set(KLEIDIAI_RELEASE_ARCHIVE_MD5  "05f1953428ffe8e8c7f585dc6d508887")
 
         set(KLEIDIAI_FETCH_ARGS
             URL ${KLEIDIAI_DOWNLOAD_URL}

ggml/src/ggml-cpu/kleidiai/kernels.cpp

@@ -201,85 +201,117 @@ static struct kai_matmul_uker_config q4_k_matmul_config() {
     return config;
 }
 
-template<struct kai_matmul_uker_api (*GetApi)(void)>
-static size_t q4_k_get_m_step() {
-    const struct kai_matmul_uker_api api = GetApi();
-    const struct kai_matmul_uker_config config = q4_k_matmul_config();
-    return api.get_step(&config).m;
+static struct kai_matmul_uker_config default_matmul_config() {
+    return {};
 }
 
-template<struct kai_matmul_uker_api (*GetApi)(void)>
-static size_t q4_k_get_n_step() {
-    const struct kai_matmul_uker_api api = GetApi();
-    const struct kai_matmul_uker_config config = q4_k_matmul_config();
-    return api.get_step(&config).n;
-}
+template<
+        struct kai_matmul_uker_api (*GetApi)(void),
+        struct kai_matmul_uker_config (*GetConfig)(void),
+        size_t MR, size_t NR, size_t KR, size_t SR>
+struct matmul_uker_adapter {
+    static size_t get_m_step() {
+        const struct kai_matmul_uker_api api = GetApi();
+        const struct kai_matmul_uker_config config = GetConfig();
+        return api.get_step(&config).m;
+    }
 
-template<size_t Value>
-static size_t q4_k_get_constant() {
-    return Value;
-}
+    static size_t get_n_step() {
+        const struct kai_matmul_uker_api api = GetApi();
+        const struct kai_matmul_uker_config config = GetConfig();
+        return api.get_step(&config).n;
+    }
 
-template<struct kai_matmul_uker_api (*GetApi)(void)>
-static size_t q4_k_get_dst_offset(size_t m_idx, size_t n_idx, size_t dst_stride) {
-    const struct kai_matmul_uker_api api = GetApi();
-    const struct kai_matmul_uker_config config = q4_k_matmul_config();
-    const struct kai_matmul_uker_dst_dim_args index = { m_idx, n_idx };
-    const struct kai_matmul_uker_dst_stride_args stride = { dst_stride };
-    return api.get_dst_offset(&config, &index, &stride);
-}
+    static size_t get_mr() {
+        if constexpr (MR == 0) {
+            return get_m_step();
+        }
+        return MR;
+    }
 
-template<struct kai_matmul_uker_api (*GetApi)(void)>
-static size_t q4_k_get_lhs_packed_offset(size_t m_idx, size_t k, size_t bl) {
-    GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
+    static size_t get_nr() {
+        if constexpr (NR == 0) {
+            return get_n_step();
+        }
+        return NR;
+    }
 
-    const struct kai_matmul_uker_api api = GetApi();
-    const struct kai_matmul_uker_config config = q4_k_matmul_config();
-    const struct kai_matmul_uker_lhs_dim_args shape = { m_idx, k };
-    const struct kai_matmul_uker_lhs_dim_args index = { m_idx, 0 };
-    const struct kai_matmul_uker_lhs_stride_args stride = api.get_lhs_stride(&config, &shape);
-    return api.get_lhs_offset(&config, &index, &stride);
-}
+    static size_t get_kr() {
+        return KR;
+    }
 
-template<struct kai_matmul_uker_api (*GetApi)(void)>
-static size_t q4_k_get_rhs_packed_offset(size_t n_idx, size_t k, size_t bl) {
-    GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
+    static size_t get_sr() {
+        return SR;
+    }
 
-    const struct kai_matmul_uker_api api = GetApi();
-    const struct kai_matmul_uker_config config = q4_k_matmul_config();
-    const struct kai_matmul_uker_rhs_dim_args shape = { n_idx, k };
-    const struct kai_matmul_uker_rhs_dim_args index = { n_idx, 0 };
-    const struct kai_matmul_uker_rhs_stride_args stride = api.get_rhs_stride(&config, &shape);
-    return api.get_rhs_offset(&config, &index, &stride);
-}
+    static size_t get_dst_offset(size_t m_idx, size_t n_idx, size_t dst_stride) {
+        const struct kai_matmul_uker_api api = GetApi();
+        const struct kai_matmul_uker_config config = GetConfig();
+        const struct kai_matmul_uker_dst_dim_args index = { m_idx, n_idx };
+        const struct kai_matmul_uker_dst_stride_args stride = { dst_stride };
+        return api.get_dst_offset(&config, &index, &stride);
+    }
 
-template<struct kai_matmul_uker_api (*GetApi)(void)>
-static void q4_k_run_matmul(
-        size_t m, size_t n, size_t k, size_t bl,
-        const void * lhs_packed, const void * rhs_packed,
-        void * dst, size_t dst_stride_row, size_t dst_stride_col,
-        float clamp_min, float clamp_max) {
-    GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
-    GGML_ASSERT(dst_stride_col == sizeof(float));
+    static size_t get_lhs_packed_offset(size_t m_idx, size_t k, size_t bl) {
+        const struct kai_matmul_uker_api api = GetApi();
+        const struct kai_matmul_uker_config config = GetConfig();
+        GGML_ASSERT(bl == config.format.bl);
+        const struct kai_matmul_uker_lhs_dim_args shape = { m_idx, k };
+        const struct kai_matmul_uker_lhs_dim_args index = { m_idx, 0 };
+        const struct kai_matmul_uker_lhs_stride_args stride = api.get_lhs_stride(&config, &shape);
+        return api.get_lhs_offset(&config, &index, &stride);
+    }
 
-    const struct kai_matmul_uker_api api = GetApi();
-    const struct kai_matmul_uker_config config = q4_k_matmul_config();
-    const struct kai_matmul_uker_lhs_dim_args lhs_shape = { m, k };
-    const struct kai_matmul_uker_rhs_dim_args rhs_shape = { n, k };
+    static size_t get_rhs_packed_offset(size_t n_idx, size_t k, size_t bl) {
+        const struct kai_matmul_uker_api api = GetApi();
+        const struct kai_matmul_uker_config config = GetConfig();
+        GGML_ASSERT(bl == config.format.bl);
+        const struct kai_matmul_uker_rhs_dim_args shape = { n_idx, k };
+        const struct kai_matmul_uker_rhs_dim_args index = { n_idx, 0 };
+        const struct kai_matmul_uker_rhs_stride_args stride = api.get_rhs_stride(&config, &shape);
+        return api.get_rhs_offset(&config, &index, &stride);
+    }
 
-    struct kai_matmul_uker_args args = {};
-    args.flags = KAI_MATMUL_UKER_FLAGS_ARGS_CLAMP;
-    args.shape = { m, n, k };
-    args.operand.lhs.ptr = lhs_packed;
-    args.operand.lhs.stride = api.get_lhs_stride(&config, &lhs_shape);
-    args.operand.rhs.ptr = rhs_packed;
-    args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape);
-    args.operand.dst.ptr = dst;
-    args.operand.dst.stride.m = dst_stride_row;
-    args.activation.clamp.min_ptr = &clamp_min;
-    args.activation.clamp.max_ptr = &clamp_max;
-    api.run(&config, &args);
-}
+    static void run(
+            size_t m, size_t n, size_t k, size_t bl,
+            const void * lhs_packed, const void * rhs_packed,
+            void * dst, size_t dst_stride_row, size_t dst_stride_col,
+            float clamp_min, float clamp_max) {
+        const struct kai_matmul_uker_api api = GetApi();
+        const struct kai_matmul_uker_config config = GetConfig();
+        GGML_ASSERT(bl == config.format.bl);
+        GGML_ASSERT(dst_stride_col == sizeof(float));
+        const struct kai_matmul_uker_lhs_dim_args lhs_shape = { m, k };
+        const struct kai_matmul_uker_rhs_dim_args rhs_shape = { n, k };
+
+        struct kai_matmul_uker_args args = {};
+        args.flags = KAI_MATMUL_UKER_FLAGS_ARGS_CLAMP;
+        args.shape = { m, n, k };
+        args.operand.lhs.ptr = lhs_packed;
+        args.operand.lhs.stride = api.get_lhs_stride(&config, &lhs_shape);
+        args.operand.rhs.ptr = rhs_packed;
+        args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape);
+        args.operand.dst.ptr = dst;
+        args.operand.dst.stride.m = dst_stride_row;
+        args.activation.clamp.min_ptr = &clamp_min;
+        args.activation.clamp.max_ptr = &clamp_max;
+        api.run(&config, &args);
+    }
+};
+
+using q4_k_gemm_adapter = matmul_uker_adapter<
+    kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa,
+    q4_k_matmul_config, 0, 0, 4, 2>;
+using q4_k_gemv_adapter = matmul_uker_adapter<
+    kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot,
+    q4_k_matmul_config, 0, 0, 4, 2>;
+// GEMM and GEMV share the RHS layout only at SVE256 (NR=8, KR=8, SR=1).
+using sve_q8_gemm_adapter = matmul_uker_adapter<
+    kai_matmul_clamp_f32_qai8dxp4x8sf32_qsi8cxp4vsx8sf32bf32_16x4vs_sve_i8mm,
+    default_matmul_config, 4, 0, 8, 1>;
+using sve_q8_gemv_adapter = matmul_uker_adapter<
+    kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp8x8_1x8_sve_dot,
+    default_matmul_config, 1, 8, 8, 1>;
 
 static struct kai_matmul_pack_lhs_uker_config q4_k_lhs_pack_config() {
     struct kai_matmul_pack_lhs_uker_config config = {};
@@ -628,7 +660,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
             /* .packed_size_ex        = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
             /* .pack_func_ex          = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
         },
-        /* .required_cpu       = */ CPU_FEATURE_SVE | CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,
+        /* .required_cpu       = */ CPU_FEATURE_SVE | CPU_FEATURE_SVEI8MM | CPU_FEATURE_DOTPROD,
         /* .lhs_type           = */ GGML_TYPE_F32,
         /* .rhs_type           = */ GGML_TYPE_Q4_0,
         /* .op_type            = */ GGML_TYPE_F32,
@@ -737,16 +769,16 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q4_k[] = {
     {
         /* SME2 GEMM */
         {
-            /* .get_m_step            = */ &q4_k_get_m_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
-            /* .get_n_step            = */ &q4_k_get_n_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
-            /* .get_mr                = */ &q4_k_get_m_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
-            /* .get_nr                = */ &q4_k_get_n_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
-            /* .get_kr                = */ &q4_k_get_constant<4>,
-            /* .get_sr                = */ &q4_k_get_constant<2>,
-            /* .get_dst_offset        = */ &q4_k_get_dst_offset<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
-            /* .get_lhs_offset_ex     = */ &q4_k_get_lhs_packed_offset<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
-            /* .get_rhs_packed_offset_ex = */ &q4_k_get_rhs_packed_offset<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
-            /* .run_kernel_ex         = */ &q4_k_run_matmul<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
+            /* .get_m_step            = */ q4_k_gemm_adapter::get_m_step,
+            /* .get_n_step            = */ q4_k_gemm_adapter::get_n_step,
+            /* .get_mr                = */ q4_k_gemm_adapter::get_mr,
+            /* .get_nr                = */ q4_k_gemm_adapter::get_nr,
+            /* .get_kr                = */ q4_k_gemm_adapter::get_kr,
+            /* .get_sr                = */ q4_k_gemm_adapter::get_sr,
+            /* .get_dst_offset        = */ q4_k_gemm_adapter::get_dst_offset,
+            /* .get_lhs_offset_ex     = */ q4_k_gemm_adapter::get_lhs_packed_offset,
+            /* .get_rhs_packed_offset_ex = */ q4_k_gemm_adapter::get_rhs_packed_offset,
+            /* .run_kernel_ex         = */ q4_k_gemm_adapter::run,
         },
         {
             /* .get_offset            = */ kai_get_lhs_offset_lhs_pack_f16pmrx2_f32_neon,
@@ -756,16 +788,16 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q4_k[] = {
         },
         /* SME2 GEMV */
         {
-            /* .get_m_step            = */ &q4_k_get_m_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
-            /* .get_n_step            = */ &q4_k_get_n_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
-            /* .get_mr                = */ &q4_k_get_m_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
-            /* .get_nr                = */ &q4_k_get_n_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
-            /* .get_kr                = */ &q4_k_get_constant<4>,
-            /* .get_sr                = */ &q4_k_get_constant<2>,
-            /* .get_dst_offset        = */ &q4_k_get_dst_offset<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
-            /* .get_lhs_offset_ex     = */ &q4_k_get_lhs_packed_offset<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
-            /* .get_rhs_packed_offset_ex = */ &q4_k_get_rhs_packed_offset<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
-            /* .run_kernel_ex         = */ &q4_k_run_matmul<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
+            /* .get_m_step            = */ q4_k_gemv_adapter::get_m_step,
+            /* .get_n_step            = */ q4_k_gemv_adapter::get_n_step,
+            /* .get_mr                = */ q4_k_gemv_adapter::get_mr,
+            /* .get_nr                = */ q4_k_gemv_adapter::get_nr,
+            /* .get_kr                = */ q4_k_gemv_adapter::get_kr,
+            /* .get_sr                = */ q4_k_gemv_adapter::get_sr,
+            /* .get_dst_offset        = */ q4_k_gemv_adapter::get_dst_offset,
+            /* .get_lhs_offset_ex     = */ q4_k_gemv_adapter::get_lhs_packed_offset,
+            /* .get_rhs_packed_offset_ex = */ q4_k_gemv_adapter::get_rhs_packed_offset,
+            /* .run_kernel_ex         = */ q4_k_gemv_adapter::run,
         },
         {
             /* .get_offset            = */ q4_k_lhs_get_offset,
@@ -883,6 +915,54 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
         /* .rhs_type           = */ GGML_TYPE_Q8_0,
         /* .op_type            = */ GGML_TYPE_F32,
     },
+    {
+        /* SVE I8MM GEMM */
+        {
+            /* .get_m_step            = */ sve_q8_gemm_adapter::get_m_step,
+            /* .get_n_step            = */ sve_q8_gemm_adapter::get_n_step,
+            /* .get_mr                = */ sve_q8_gemm_adapter::get_mr,
+            /* .get_nr                = */ sve_q8_gemm_adapter::get_nr,
+            /* .get_kr                = */ sve_q8_gemm_adapter::get_kr,
+            /* .get_sr                = */ sve_q8_gemm_adapter::get_sr,
+            /* .get_dst_offset        = */ sve_q8_gemm_adapter::get_dst_offset,
+            /* .get_lhs_offset_ex     = */ sve_q8_gemm_adapter::get_lhs_packed_offset,
+            /* .get_rhs_packed_offset_ex = */ sve_q8_gemm_adapter::get_rhs_packed_offset,
+            /* .run_kernel_ex         = */ sve_q8_gemm_adapter::run,
+        },
+        /* .gemm_lhs_info = */ {
+            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,
+            /* .get_packed_offset_ex  = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_quant_pack_qai8dxp_f32>,
+            /* .packed_size_ex        = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f32>,
+            /* .pack_func_ex          = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,
+        },
+        /* SVE DOT GEMV */
+        {
+            /* .get_m_step            = */ sve_q8_gemv_adapter::get_m_step,
+            /* .get_n_step            = */ sve_q8_gemv_adapter::get_n_step,
+            /* .get_mr                = */ sve_q8_gemv_adapter::get_mr,
+            /* .get_nr                = */ sve_q8_gemv_adapter::get_nr,
+            /* .get_kr                = */ sve_q8_gemv_adapter::get_kr,
+            /* .get_sr                = */ sve_q8_gemv_adapter::get_sr,
+            /* .get_dst_offset        = */ sve_q8_gemv_adapter::get_dst_offset,
+            /* .get_lhs_offset_ex     = */ sve_q8_gemv_adapter::get_lhs_packed_offset,
+            /* .get_rhs_packed_offset_ex = */ sve_q8_gemv_adapter::get_rhs_packed_offset,
+            /* .run_kernel_ex         = */ sve_q8_gemv_adapter::run,
+        },
+        /* .gemv_lhs_info = */ {
+            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,
+            /* .get_packed_offset_ex  = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_quant_pack_qai8dxp_f32>,
+            /* .packed_size_ex        = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f32>,
+            /* .pack_func_ex          = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,
+        },
+        /* .rhs_info = */ {
+            /* .packed_size_ex        = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
+            /* .pack_func_ex          = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
+        },
+        /* .required_cpu       = */ CPU_FEATURE_SVE | CPU_FEATURE_SVEI8MM | CPU_FEATURE_DOTPROD,
+        /* .lhs_type           = */ GGML_TYPE_F32,
+        /* .rhs_type           = */ GGML_TYPE_Q8_0,
+        /* .op_type            = */ GGML_TYPE_F32,
+    },
     {
         /* I8MM GEMM */
         {

ggml/src/ggml-cpu/kleidiai/kernels.h

@@ -13,7 +13,8 @@ enum cpu_feature {
     CPU_FEATURE_SVE     = 4,
     CPU_FEATURE_SME     = 8,
     CPU_FEATURE_SME2    = 16,
-    CPU_FEATURE_FP16    = 32
+    CPU_FEATURE_FP16    = 32,
+    CPU_FEATURE_SVEI8MM = 64
 };
 
 inline cpu_feature& operator|=(cpu_feature& lhs, cpu_feature rhs) {

ggml/src/ggml-cpu/kleidiai/kleidiai.cpp

@@ -319,8 +319,10 @@ static void init_kleidiai_context(void) {
 
         size_t detected_smcus = 0;
 
+        // The fixed-width SVE kernels require 32-byte vectors, including the Q8 shared RHS layout.
         ctx.features  = (runtime_feat.has_dotprod  ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) |
                         (runtime_feat.has_i8mm     ? CPU_FEATURE_I8MM    : CPU_FEATURE_NONE) |
+                        (runtime_feat.has_svei8mm  ? CPU_FEATURE_SVEI8MM : CPU_FEATURE_NONE) |
                         (runtime_feat.has_fp16     ? CPU_FEATURE_FP16    : CPU_FEATURE_NONE) |
                         (runtime_feat.sve_cnt == QK8_0 ? CPU_FEATURE_SVE : CPU_FEATURE_NONE);
 
@@ -1616,6 +1618,11 @@ public:
             const size_t kr = kernel->get_kr();
             const size_t sr = kernel->get_sr();
             const ggml_type rhs_type = kernels->rhs_type;
+            if (rhs_type == GGML_TYPE_Q8_0 && (kernels->required_cpu & CPU_FEATURE_SVE)) {
+                GGML_ASSERT(nr == kernels->gemv.get_nr());
+                GGML_ASSERT(kr == kernels->gemv.get_kr());
+                GGML_ASSERT(sr == kernels->gemv.get_sr());
+            }
             const size_t block_len = rhs_type == GGML_TYPE_Q8_0 ? QK8_0 :
                                      rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :
                                      rhs_type == GGML_TYPE_Q4_K ? GGML_KLEIDIAI_Q4_K_BL :

ggml/src/ggml-feats.h

@@ -22,6 +22,10 @@
 #define HWCAP2_I8MM (1ULL << 13)
 #endif
 
+#if !defined(HWCAP2_SVEI8MM)
+#define HWCAP2_SVEI8MM (1ULL << 9)
+#endif
+
 #if !defined(HWCAP_ASIMDDP)
 #define HWCAP_ASIMDDP (1 << 20)
 #endif
@@ -63,6 +67,10 @@
 #define PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE 47
 #endif
 
+#if !defined(PF_ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE 57
+#endif
+
 #if !defined(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE)
 #define PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE 66
 #endif
@@ -87,6 +95,7 @@ typedef struct ggml_feats_arch64_runtime {
     bool has_sve;
     bool has_sve2;
     bool has_i8mm;
+    bool has_svei8mm;
     bool has_sme;
     bool has_sme2;
     int sve_cnt;
@@ -104,6 +113,7 @@ static inline ggml_feats_arch64_runtime_t ggml_feats_get_arch64_runtime(void) {
     runtime_feat.has_sve     = !!(hwcap & HWCAP_SVE);
     runtime_feat.has_sve2    = !!(hwcap2 & HWCAP2_SVE2);
     runtime_feat.has_i8mm    = !!(hwcap2 & HWCAP2_I8MM);
+    runtime_feat.has_svei8mm = !!(hwcap2 & HWCAP2_SVEI8MM);
     runtime_feat.has_sme     = !!(hwcap2 & HWCAP2_SME);
     runtime_feat.has_sme2    = !!(hwcap2 & HWCAP2_SME2);
 
@@ -153,6 +163,7 @@ static inline ggml_feats_arch64_runtime_t ggml_feats_get_arch64_runtime(void) {
     runtime_feat.has_sve     = IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) != 0;
     runtime_feat.has_sve2    = IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE) != 0;
     runtime_feat.has_i8mm    = IsProcessorFeaturePresent(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE) != 0;
+    runtime_feat.has_svei8mm = IsProcessorFeaturePresent(PF_ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE) != 0;
     runtime_feat.has_sme     = IsProcessorFeaturePresent(PF_ARM_SME_INSTRUCTIONS_AVAILABLE) != 0;
     runtime_feat.has_sme2    = IsProcessorFeaturePresent(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE) != 0;
 

ggml/src/ggml-opencl/CMakeLists.txt

@@ -64,6 +64,7 @@ set(GGML_OPENCL_KERNELS
     add
     add_id
     moe_add_id_glu
+    argmax
     argsort
     tri
     fill

ggml/src/ggml-opencl/ggml-opencl.cpp

@@ -646,8 +646,8 @@ struct ggml_backend_opencl_context {
     ggml_cl_version platform_version;
     ggml_cl_version opencl_c_version;
 
-    // argsort is loaded in supports_op because its availability depends on how
-    // many workgroups are allowed, which requires kernel compilation.
+    // argmax and argsort are loaded in supports_op; argsort availability depends on the compiled kernel's workgroup limit.
+    bool kernels_loaded_argmax = false;
     bool kernels_loaded_argsort = false;
     // rest of the kernels are currently always loaded in alloc_buffer.
     bool kernels_loaded = false;
@@ -972,6 +972,7 @@ struct ggml_backend_opencl_context {
     cl_kernel kernel_solve_tri_f32;
     cl_kernel kernel_im2col_f32, kernel_im2col_f16;
     cl_kernel kernel_im2col_3d_f32, kernel_im2col_3d_f16;
+    cl_kernel kernel_argmax_f32_i32;
     cl_kernel kernel_argsort_f32_i32;
     cl_kernel kernel_sum_rows_f32, kernel_sum_rows_f32_4;
     cl_kernel kernel_cumsum_blk, kernel_cumsum_add;
@@ -1485,6 +1486,27 @@ static cl_program build_program_from_binary(cl_context ctx, cl_device_id dev, co
     return p;
 }
 
+static void load_cl_kernels_argmax(ggml_backend_opencl_context *backend_ctx) {
+    auto opencl_c_std =
+        std::string("CL") + std::to_string(backend_ctx->opencl_c_version.major) + "." + std::to_string(backend_ctx->opencl_c_version.minor);
+    std::string compile_opts = std::string("-cl-std=") + opencl_c_std;
+
+    if (!backend_ctx->kernels_loaded_argmax) {
+        cl_int err;
+#ifdef GGML_OPENCL_EMBED_KERNELS
+        const std::string kernel_src {
+            #include "argmax.cl.h"
+        };
+#else
+        const std::string kernel_src = read_file("argmax.cl");
+#endif
+        cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+        CL_CHECK((backend_ctx->kernel_argmax_f32_i32 = clCreateKernel(prog, "kernel_argmax_f32_i32", &err), err));
+        CL_CHECK(clReleaseProgram(prog));
+        backend_ctx->kernels_loaded_argmax = true;
+    }
+}
+
 static void load_cl_kernels_argsort(ggml_backend_opencl_context *backend_ctx) {
     // compiler options for general kernels
     auto opencl_c_std =
@@ -9693,6 +9715,11 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
             return true;
         case GGML_OP_IM2COL_3D:
             return true;
+        case GGML_OP_ARGMAX:
+            load_cl_kernels_argmax(backend_ctx);
+            return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_I32 &&
+                   ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op) &&
+                   backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_argmax_f32_i32) >= 64;
         case GGML_OP_ARGSORT: {
             load_cl_kernels_argsort(backend_ctx);
 
@@ -30164,6 +30191,41 @@ static void ggml_cl_im2col_3d(ggml_backend_t backend, const ggml_tensor * src0,
     backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
 }
 
+static void ggml_cl_argmax(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+    GGML_ASSERT(src0);
+    GGML_ASSERT(src0->extra);
+    GGML_ASSERT(dst);
+    GGML_ASSERT(dst->extra);
+    GGML_UNUSED(src1);
+
+    GGML_ASSERT(src0->type == GGML_TYPE_F32);
+    GGML_ASSERT(dst->type == GGML_TYPE_I32);
+    GGML_ASSERT(ggml_is_contiguous(src0));
+    GGML_ASSERT(ggml_is_contiguous(dst));
+
+    ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context;
+    ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) src0->extra;
+    ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra;
+
+    const cl_ulong offset0 = extra0->offset + src0->view_offs;
+    const cl_ulong offsetd = extrad->offset + dst->view_offs;
+    const int ne00 = src0->ne[0];
+    const int nrows = ggml_nrows(src0);
+    constexpr int nth = 64;
+
+    cl_kernel kernel = backend_ctx->kernel_argmax_f32_i32;
+    CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem),   &extra0->data_device));
+    CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
+    CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem),   &extrad->data_device));
+    CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));
+    CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int),      &ne00));
+    CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int),      &nrows));
+
+    size_t global_work_size[] = { (size_t) nth, (size_t) CEIL_DIV(nrows, 2), 1 };
+    size_t local_work_size[] = { (size_t) nth, 1, 1 };
+    backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+}
+
 static void ggml_cl_argsort(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
     GGML_ASSERT(src0);
     GGML_ASSERT(src0->extra);
@@ -31071,6 +31133,12 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor
             }
             func = ggml_cl_im2col_3d;
             break;
+        case GGML_OP_ARGMAX:
+            if (!any_on_device) {
+                return false;
+            }
+            func = ggml_cl_argmax;
+            break;
         case GGML_OP_ARGSORT:
             if (!any_on_device) {
                 return false;

ggml/src/ggml-opencl/kernels/argmax.cl

@@ -0,0 +1,55 @@
+#define ARGMAX_LANES_PER_ROW 32
+#define ARGMAX_ROWS_PER_GROUP 2
+#define ARGMAX_BLOCK_SIZE (ARGMAX_LANES_PER_ROW * ARGMAX_ROWS_PER_GROUP)
+
+kernel void kernel_argmax_f32_i32(
+        global const char * src0,
+        ulong               offset0,
+        global char *       dst,
+        ulong               offsetd,
+        int                 ne00,
+        int                 nrows) {
+    const int local_id = get_local_id(0);
+    const int col = local_id % ARGMAX_LANES_PER_ROW;
+    const int row = get_group_id(1) * ARGMAX_ROWS_PER_GROUP + local_id / ARGMAX_LANES_PER_ROW;
+
+    float max_value = -INFINITY;
+    int max_index = 0;
+
+    if (row < nrows) {
+        global const float * src_row = (global const float *) (src0 + offset0) + row * ne00;
+        for (int i = col; i < ne00; i += ARGMAX_LANES_PER_ROW) {
+            const float value = src_row[i];
+            if (value > max_value || (value == max_value && i > max_index)) {
+                max_value = value;
+                max_index = i;
+            }
+        }
+    }
+
+    local float max_values[ARGMAX_BLOCK_SIZE];
+    local int max_indices[ARGMAX_BLOCK_SIZE];
+
+    max_values[local_id] = max_value;
+    max_indices[local_id] = max_index;
+
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    const int row_offset = local_id / ARGMAX_LANES_PER_ROW * ARGMAX_LANES_PER_ROW;
+    for (int stride = ARGMAX_LANES_PER_ROW / 2; stride > 0; stride /= 2) {
+        if (col < stride) {
+            const float value = max_values[row_offset + col + stride];
+            const int index = max_indices[row_offset + col + stride];
+            if (value > max_values[row_offset + col] || (value == max_values[row_offset + col] && index > max_indices[row_offset + col])) {
+                max_values[row_offset + col] = value;
+                max_indices[row_offset + col] = index;
+            }
+        }
+        barrier(CLK_LOCAL_MEM_FENCE);
+    }
+
+    if (col == 0 && row < nrows) {
+        global int * dst_row = (global int *) (dst + offsetd) + row;
+        dst_row[0] = max_indices[row_offset];
+    }
+}

ggml/src/ggml-sycl/common.hpp

@@ -68,6 +68,9 @@ extern int g_ggml_sycl_enable_optimize;
 extern int g_ggml_sycl_enable_fusion;
 extern int g_ggml_sycl_enable_esimd;
 extern int g_ggml_sycl_mmvq_wide;
+extern int g_ggml_sycl_enable_xmx_esimd;
+extern int g_ggml_sycl_xmx_max_cols;
+extern int g_ggml_sycl_xmx_glu_max_cols;
 extern int g_ggml_sycl_prioritize_dmmv;
 
 // Which quantized weight formats may take the XMX dequant-GEMM paths. A bitmask rather than one
@@ -227,6 +230,41 @@ typedef sycl::float2 dfloat2;
 
 #define MMVQ_MAX_BATCH_SIZE  8
 
+#if defined(__INTEL_LLVM_COMPILER) && __has_include(<sycl/ext/intel/esimd/xmx/dpas.hpp>)
+#    define GGML_SYCL_MMVQ_HAS_XMX
+#endif // __INTEL_LLVM_COMPILER
+
+// default of the most columns the XMX mul_mat_vec_q handles (GGML_SYCL_XMX_MAX_COLS), wider batches use other kernels
+#define GGML_SYCL_XMX_MAX_COLS_DEFAULT 80
+
+// most columns the fused gate and up XMX kernel is built for, GGML_SYCL_XMX_GLU_MAX_COLS can only lower this limit
+#define GGML_SYCL_XMX_GLU_MAX_COLS 16
+
+static inline bool ggml_sycl_xmx_supports_type(ggml_type type) {
+    switch (type) {
+        case GGML_TYPE_Q2_K:
+        case GGML_TYPE_Q3_K:
+        case GGML_TYPE_Q4_K:
+        case GGML_TYPE_Q5_K:
+        case GGML_TYPE_Q6_K:
+        case GGML_TYPE_Q8_0:
+            return true;
+        default:
+            return false;
+    }
+}
+
+// the DPAS shapes and 2D block loads used by the XMX kernel are those of Xe2
+static inline bool ggml_sycl_xmx_supports_arch(gpu_arch arch) {
+    return arch == gpu_arch::intel_gpu_bmg_g21 || arch == gpu_arch::intel_gpu_bmg_g31 ||
+           arch == gpu_arch::intel_gpu_lnl_m;
+}
+
+// fewest columns for which the XMX kernel is faster than the other kernels
+static inline int ggml_sycl_xmx_min_cols(ggml_type type) {
+    return type == GGML_TYPE_Q2_K || type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q4_K ? 1 : 2;
+}
+
 static int g_all_sycl_device_count = -1;
 static bool g_ggml_backend_sycl_buffer_type_initialized = false;
 
@@ -310,6 +348,10 @@ struct ggml_sycl_device_info {
 
 const ggml_sycl_device_info & ggml_sycl_info();
 
+static inline bool ggml_sycl_xmx_enabled(int device) {
+    return g_ggml_sycl_enable_xmx_esimd && ggml_sycl_xmx_supports_arch(ggml_sycl_info().devices[device].hw_info.arch);
+}
+
 static constexpr size_t SYCL_BUFFER_ALIGNMENT = 128;
 
 struct ggml_sycl_pool {

ggml/src/ggml-sycl/conv2d.cpp

@@ -53,7 +53,16 @@ static inline int64_t whcn_output_index(int64_t n, int64_t c, int64_t y, int64_t
     return n * (P.OC * P.OW * P.OH) + c * P.OW * P.OH + y * P.OW + x;
 }
 
-template <typename T>
+// cwhn layout helpers: buffers are [N][H][W][C] input and [KH][KW][IC][OC] kernel
+static inline int64_t cwhn_input_index(int64_t n, int64_t c, int64_t y, int64_t x, const conv2d_params & P) {
+    return ((n * P.IH + y) * P.IW + x) * P.IC + c;
+}
+
+static inline int64_t cwhn_kernel_index(int64_t c_out, int64_t c_in, int64_t ky, int64_t kx, const conv2d_params & P) {
+    return ((ky * P.KW + kx) * P.IC + c_in) * P.OC + c_out;
+}
+
+template <typename T, bool CWHN>
 static void conv2d_kernel(const float * input, const T * kernel, float * output,
                           const conv2d_params P, const sycl::nd_item<3> & item_ct1) {
     const int64_t global_idx = item_ct1.get_local_id(2) +
@@ -77,8 +86,8 @@ static void conv2d_kernel(const float * input, const T * kernel, float * output,
             const int64_t in_y = calculate_input_coord(out_y, ky, P.ST_Y, P.DL_Y, P.PD_Y);
             for (int64_t kx = bounds.x_min; kx < bounds.x_max; ++kx) {
                 const int64_t in_x = calculate_input_coord(out_x, kx, P.ST_X, P.DL_X, P.PD_X);
-                const float input_val  = input[whcn_input_index(n, c_in, in_y, in_x, P)];
-                const T     kernel_val = kernel[whcn_kernel_index(c_out, c_in, ky, kx, P)];
+                const float input_val  = input[CWHN ? cwhn_input_index(n, c_in, in_y, in_x, P) : whcn_input_index(n, c_in, in_y, in_x, P)];
+                const T     kernel_val = kernel[CWHN ? cwhn_kernel_index(c_out, c_in, ky, kx, P) : whcn_kernel_index(c_out, c_in, ky, kx, P)];
                 acc += input_val * ggml_sycl_cast<float>(kernel_val);
             }
         }
@@ -87,7 +96,7 @@ static void conv2d_kernel(const float * input, const T * kernel, float * output,
     output[whcn_output_index(n, c_out, out_y, out_x, P)] = acc;
 }
 
-template <typename T>
+template <typename T, bool CWHN>
 static void conv2d_sycl(const float * X_D, const T * K_D, float * Y_D,
                         const conv2d_params P, const queue_ptr & stream) {
     const int num_blocks = (P.TOTAL + SYCL_CONV2D_BLOCK_SIZE - 1) / SYCL_CONV2D_BLOCK_SIZE;
@@ -95,7 +104,7 @@ static void conv2d_sycl(const float * X_D, const T * K_D, float * Y_D,
     const sycl::range<3> block_nums(1, 1, num_blocks);
     stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
         [=](sycl::nd_item<3> item_ct1) {
-            conv2d_kernel<T>(X_D, K_D, Y_D, P, item_ct1);
+            conv2d_kernel<T, CWHN>(X_D, K_D, Y_D, P, item_ct1);
         });
 }
 
@@ -108,7 +117,14 @@ void ggml_sycl_op_conv2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
     const float *       X_D    = (const float *) input->data;
     float *             Y_D    = (float *) dst->data;
 
-    GGML_ASSERT(ggml_is_contiguous(kernel));
+    // WHCN: contiguous [KW,KH,IC,OC] kernel and [W,H,C,N] input
+    // CWHN: channel-fastest buffer [N][H][W][C] input and [KH][KW][IC][OC] kernel (OC fastest)
+    const bool whcn = ggml_is_contiguous(kernel) && ggml_is_contiguous(input);
+    const bool cwhn = input->nb[2]  == ggml_type_size(input->type) &&
+                      kernel->nb[3] == ggml_type_size(kernel->type);
+    GGML_ASSERT(whcn || cwhn);
+    const bool cwhn_layout = !whcn && cwhn; // prefer whcn when both match (degenerate size-1 dims)
+
     GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32);
     GGML_ASSERT(input->type == GGML_TYPE_F32);
     GGML_ASSERT(dst->type == GGML_TYPE_F32);
@@ -126,9 +142,6 @@ void ggml_sycl_op_conv2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
     const int       DL_X = p[4];
     const int       DL_Y = p[5];
 
-    // no cwhn layout support
-    GGML_ASSERT(p[6] == 0);
-
     const int IW = input->ne[0];
     const int IH = input->ne[1];
     const int OW = dst->ne[0];
@@ -142,9 +155,17 @@ void ggml_sycl_op_conv2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
     const int64_t     total  = (int64_t) B * OC * OH * OW;
     const conv2d_params params = { IW, IH, OW, OH, KW, KH, ST_X, ST_Y, PD_X, PD_Y, DL_X, DL_Y, IC, OC, B, total };
 
-    if (kernel->type == GGML_TYPE_F16) {
-        conv2d_sycl<sycl::half>(X_D, (const sycl::half *) K_D, Y_D, params, stream);
+    if (cwhn_layout) {
+        if (kernel->type == GGML_TYPE_F16) {
+            conv2d_sycl<sycl::half, true>(X_D, (const sycl::half *) K_D, Y_D, params, stream);
+        } else {
+            conv2d_sycl<float, true>(X_D, K_D, Y_D, params, stream);
+        }
     } else {
-        conv2d_sycl<float>(X_D, K_D, Y_D, params, stream);
+        if (kernel->type == GGML_TYPE_F16) {
+            conv2d_sycl<sycl::half, false>(X_D, (const sycl::half *) K_D, Y_D, params, stream);
+        } else {
+            conv2d_sycl<float, false>(X_D, K_D, Y_D, params, stream);
+        }
     }
 }

ggml/src/ggml-sycl/convert.cpp

@@ -657,58 +657,68 @@ static void convert_unary_sycl(const void * vx, dst_t * y, const int64_t k, dpct
 }
 
 
-to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) {
+static bool is_reordered_src0(const ggml_tensor * dst) {
+    return dst->src[0]->extra &&
+           ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder;
+}
+
+template <typename dst_t>
+static to_t_sycl_t<dst_t> ggml_get_dequant_sycl(ggml_type type, ggml_tensor * dst) {
     switch (type) {
         case GGML_TYPE_Q1_0:
-            return dequantize_block_sycl<QK1_0, QR1_0, dequantize_q1_0>;
+            return dequantize_block_sycl<QK1_0, QR1_0, dequantize_q1_0, dst_t>;
         case GGML_TYPE_Q2_0:
-            return dequantize_block_sycl<QK2_0, QR2_0, dequantize_q2_0>;
+            return dequantize_block_sycl<QK2_0, QR2_0, dequantize_q2_0, dst_t>;
         case GGML_TYPE_Q4_0:
-            if (dst->src[0]->extra &&
-                ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) {
+            if (is_reordered_src0(dst)) {
                 return dequantize_row_q4_0_sycl_reorder;
+            } else if constexpr (std::is_same_v<dst_t, float>) {
+                return dequantize_row_q4_0_sycl;
             } else {
-                return dequantize_block_sycl<QK4_0, QR4_0, dequantize_q4_0>;
+                return dequantize_block_sycl<QK4_0, QR4_0, dequantize_q4_0, dst_t>;
             }
         case GGML_TYPE_Q4_1:
-            return dequantize_block_sycl<QK4_1, QR4_1, dequantize_q4_1>;
+            if constexpr (std::is_same_v<dst_t, float>) {
+                return dequantize_row_q4_1_sycl;
+            } else {
+                return dequantize_block_sycl<QK4_1, QR4_1, dequantize_q4_1, dst_t>;
+            }
         case GGML_TYPE_Q5_0:
-            return dequantize_block_sycl<QK5_0, QR5_0, dequantize_q5_0>;
+            return dequantize_block_sycl<QK5_0, QR5_0, dequantize_q5_0, dst_t>;
         case GGML_TYPE_Q5_1:
-            return dequantize_block_sycl<QK5_1, QR5_1, dequantize_q5_1>;
+            return dequantize_block_sycl<QK5_1, QR5_1, dequantize_q5_1, dst_t>;
         case GGML_TYPE_Q8_0:
-            if (dst->src[0]->extra &&
-                ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+            if (is_reordered_src0(dst)) {
                 return dequantize_row_q8_0_sycl_reorder;
             } else {
-                return dequantize_block_sycl<QK8_0, QR8_0, dequantize_q8_0>;
+                return dequantize_block_sycl<QK8_0, QR8_0, dequantize_q8_0, dst_t>;
             }
         case GGML_TYPE_Q2_K:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+            if (is_reordered_src0(dst)) {
                 return dequantize_row_q2_K_sycl_reorder;
             } else {
                 return dequantize_row_q2_K_sycl;
             }
         case GGML_TYPE_Q3_K:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+            if (is_reordered_src0(dst)) {
                 return dequantize_row_q3_K_sycl_reorder;
             } else {
                 return dequantize_row_q3_K_sycl;
             }
         case GGML_TYPE_Q4_K:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+            if (is_reordered_src0(dst)) {
                 return dequantize_row_q4_K_sycl_reorder;
             } else {
                 return dequantize_row_q4_K_sycl;
             }
         case GGML_TYPE_Q5_K:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+            if (is_reordered_src0(dst)) {
                 return dequantize_row_q5_K_sycl_reorder;
             } else {
                 return dequantize_row_q5_K_sycl;
             }
         case GGML_TYPE_Q6_K:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+            if (is_reordered_src0(dst)) {
                 return dequantize_row_q6_K_sycl_reorder;
             } else {
                 return dequantize_row_q6_K_sycl;
@@ -724,15 +734,13 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) {
         case GGML_TYPE_IQ2_S:
             return dequantize_row_iq2_s_sycl;
         case GGML_TYPE_IQ3_XXS:
-            if (dst->src[0]->extra &&
-                ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+            if (is_reordered_src0(dst)) {
                 return dequantize_row_iq3_xxs_sycl_reorder;
             } else {
                 return dequantize_row_iq3_xxs_sycl;
             }
         case GGML_TYPE_IQ3_S:
-            if (dst->src[0]->extra &&
-                ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+            if (is_reordered_src0(dst)) {
                 return dequantize_row_iq3_s_sycl_reorder;
             } else {
                 return dequantize_row_iq3_s_sycl;
@@ -742,124 +750,38 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) {
         case GGML_TYPE_IQ4_NL:
             return dequantize_row_iq4_nl_sycl;
         case GGML_TYPE_MXFP4:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+            if (is_reordered_src0(dst)) {
                 return dequantize_row_mxfp4_sycl_reorder;
             }
             return dequantize_row_mxfp4_sycl;
         case GGML_TYPE_NVFP4:
             return dequantize_row_nvfp4_sycl;
         case GGML_TYPE_F32:
-            return convert_unary_sycl<float>;
+            if constexpr (std::is_same_v<dst_t, sycl::half>) {
+                return convert_unary_sycl<float>;
+            }
+            break;
+        case GGML_TYPE_F16:
+            if constexpr (std::is_same_v<dst_t, float>) {
+                return convert_unary_sycl<sycl::half>;
+            }
+            break;
 #ifdef GGML_SYCL_HAS_BF16
         case GGML_TYPE_BF16:
             return convert_unary_sycl<sycl::ext::oneapi::bfloat16>;
 #endif
         default:
-            GGML_ABORT("fatal error: unsupport data type=%s\n", ggml_type_name(type));
+            break;
     }
+    GGML_ABORT("fatal error: unsupport data type=%s\n", ggml_type_name(type));
+}
+
+to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) {
+    return ggml_get_dequant_sycl<sycl::half>(type, dst);
 }
 
 to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) {
-    switch (type) {
-        case GGML_TYPE_Q1_0:
-            return dequantize_block_sycl<QK1_0, QR1_0, dequantize_q1_0>;
-        case GGML_TYPE_Q2_0:
-            return dequantize_block_sycl<QK2_0, QR2_0, dequantize_q2_0>;
-        case GGML_TYPE_Q4_0:
-            if (dst->src[0]->extra &&
-                ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_q4_0_sycl_reorder;
-            } else {
-                return dequantize_row_q4_0_sycl;
-            }
-        case GGML_TYPE_Q4_1:
-            return dequantize_row_q4_1_sycl;
-        case GGML_TYPE_Q5_0:
-            return dequantize_block_sycl<QK5_0, QR5_0, dequantize_q5_0>;
-        case GGML_TYPE_Q5_1:
-            return dequantize_block_sycl<QK5_1, QR5_1, dequantize_q5_1>;
-        case GGML_TYPE_Q8_0:
-            if (dst->src[0]->extra &&
-                ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_q8_0_sycl_reorder;
-            } else {
-                return dequantize_block_sycl<QK8_0, QR8_0, dequantize_q8_0>;
-            }
-        case GGML_TYPE_Q2_K:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_q2_K_sycl_reorder;
-            } else {
-                return dequantize_row_q2_K_sycl;
-            }
-        case GGML_TYPE_Q3_K:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_q3_K_sycl_reorder;
-            } else {
-                return dequantize_row_q3_K_sycl;
-            }
-        case GGML_TYPE_Q4_K:
-            if (dst->src[0]->extra &&
-                ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_q4_K_sycl_reorder;
-            } else {
-                return dequantize_row_q4_K_sycl;
-            }
-        case GGML_TYPE_Q5_K:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_q5_K_sycl_reorder;
-            } else {
-                return dequantize_row_q5_K_sycl;
-            }
-        case GGML_TYPE_Q6_K:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_q6_K_sycl_reorder;
-            } else {
-                return dequantize_row_q6_K_sycl;
-            }
-        case GGML_TYPE_IQ1_S:
-            return dequantize_row_iq1_s_sycl;
-        case GGML_TYPE_IQ1_M:
-            return dequantize_row_iq1_m_sycl;
-        case GGML_TYPE_IQ2_XXS:
-            return dequantize_row_iq2_xxs_sycl;
-        case GGML_TYPE_IQ2_XS:
-            return dequantize_row_iq2_xs_sycl;
-        case GGML_TYPE_IQ2_S:
-            return dequantize_row_iq2_s_sycl;
-        case GGML_TYPE_IQ3_XXS:
-            if (dst->src[0]->extra &&
-                ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_iq3_xxs_sycl_reorder;
-            } else {
-                return dequantize_row_iq3_xxs_sycl;
-            }
-        case GGML_TYPE_IQ3_S:
-            if (dst->src[0]->extra &&
-                ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_iq3_s_sycl_reorder;
-            } else {
-                return dequantize_row_iq3_s_sycl;
-            }
-        case GGML_TYPE_IQ4_XS:
-            return dequantize_row_iq4_xs_sycl;
-        case GGML_TYPE_IQ4_NL:
-            return dequantize_row_iq4_nl_sycl;
-        case GGML_TYPE_MXFP4:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_mxfp4_sycl_reorder;
-            }
-            return dequantize_row_mxfp4_sycl;
-        case GGML_TYPE_NVFP4:
-            return dequantize_row_nvfp4_sycl;
-        case GGML_TYPE_F16:
-            return convert_unary_sycl<sycl::half>;
-#ifdef GGML_SYCL_HAS_BF16
-        case GGML_TYPE_BF16:
-            return convert_unary_sycl<sycl::ext::oneapi::bfloat16>;
-#endif
-        default:
-            GGML_ABORT("fatal error: unsupport data type=%s\n", ggml_type_name(type));
-    }
+    return ggml_get_dequant_sycl<float>(type, dst);
 }
 
 

ggml/src/ggml-sycl/esimd.hpp

@@ -2,6 +2,8 @@
 #define GGML_SYCL_ESIMD_HPP
 
 #include <sycl/ext/intel/esimd.hpp>
+#include <sycl/ext/intel/esimd/xmx/dpas.hpp>
+#include <sycl/ext/intel/experimental/grf_size_properties.hpp>
 
 #include "common.hpp"
 
@@ -584,6 +586,814 @@ template <> struct esimd_reorder_q_traits<GGML_TYPE_Q6_K> {
     }
 };
 
+//
+// Multi-column mul_mat_vec for reordered K-quant weights on the XMX engines (int8 DPAS).
+// A thread owns 16 weight rows (B operand, one dword is 4 weights of a row), the q8_1 activation
+// columns are the A operand. A super-block is 8 groups of 32, run as 2 parts of 4 groups. A quant
+// type supplies its loads and unpack as a traits struct.
+//
+using sycl::ext::intel::esimd::simd;
+constexpr int XMX_ROWS        = 16;  // weight rows per thread (DPAS N)
+constexpr int XMX_CHUNK       = 64;  // widest single launch, more columns run as repeated launches
+constexpr int XMX_FUSED_CHUNK = GGML_SYCL_XMX_GLU_MAX_COLS;  // widest launch of the fused gate and up kernel
+
+// threads that split K for one tile, sized so that the shared memory stays small
+template <int NC> constexpr int xmx_n_split() {
+    return NC <= 8 ? 32 : (NC == 16 ? 16 : 8);
+}
+
+inline bool xmx_supported(const void * vx, const int ncols) {
+    // 2D block loads need a 64 byte aligned base
+    return ncols % QK_K == 0 && (uintptr_t) vx % 64 == 0;
+}
+
+// transposed 2D block load of 16 rows, dword (kd, n) lands at [kd * 16 + n]
+// 2D block loads need rows of at least 64 bytes, shorter rows (SR) are read row by row
+template <int W, bool SR>
+ESIMD_INLINE simd<uint32_t, W * XMX_ROWS> xmx_load_t(const void * base, unsigned pitch, unsigned h, int x, int y) {
+    using namespace sycl::ext::intel::esimd;
+    if constexpr (SR) {
+        simd<uint32_t, W * XMX_ROWS> r;
+        for (int n = 0; n < XMX_ROWS; ++n) {
+            const unsigned row                = y + n < (int) h ? y + n : h;
+            const char *   p                  = (const char *) base + (size_t) row * (pitch + 1);
+            r.template select<W, XMX_ROWS>(n) = block_load<uint32_t, W>((const uint32_t *) p + x);
+        }
+        return r;
+    } else {
+        return load_2d<uint32_t, W, XMX_ROWS, 1, true, false>((const uint32_t *) base, pitch, h, pitch, x, y);
+    }
+}
+
+// per byte (q - zero) as int8 for q in 0..127, no borrow between bytes
+ESIMD_INLINE simd<uint32_t, 128> xmx_bytes_sub(simd<uint32_t, 128> x, uint32_t zero_x4) {
+    return ((x | 0x80808080u) - zero_x4) ^ 0x80808080u;
+}
+
+// How the scales of a quant type enter the DPAS epilogue of a group of 32 (see xmx_group)
+enum class xmx_scale_kind {
+    split,  // two scales per group, one per 16 wide half: the weights are masked to one half for each DPAS
+    min,    // one scale and one min per group: a second DPAS sums the activations for the min term
+    min16,  // a scale and a min per 16 wide half: the scales are folded into the DPAS operands
+    plain,  // one scale per group
+};
+
+// What every quant type states about itself. nc_wide is the tile width from which it takes the large register file.
+template <bool SR, int NC_WIDE, xmx_scale_kind KIND> struct xmx_traits_base {
+    static constexpr bool short_rows = SR;
+    static constexpr int  nc_wide    = NC_WIDE;
+    static constexpr bool has_min    = KIND == xmx_scale_kind::min;
+    static constexpr bool plain      = KIND == xmx_scale_kind::plain;
+    static constexpr bool min16      = KIND == xmx_scale_kind::min16;
+};
+
+// The part of a context every quant type shares: the last row (the clamp of 2D loads), this thread's first row and
+// the number of super-blocks per row. A type adds its array pointers and 2D load pitches.
+struct xmx_ctx_base {
+    unsigned h;
+    int      row0;
+    int      bpr;
+};
+
+ESIMD_INLINE void xmx_ctx_init(xmx_ctx_base & c, int nrows, int bpr, int row0) {
+    c.h    = nrows - 1;
+    c.row0 = row0;
+    c.bpr  = bpr;
+}
+
+// pitch of a 2D load over an array with `bytes` bytes per super-block
+constexpr unsigned xmx_pitch(int bpr, int bytes) {
+    return bpr * bytes - 1;
+}
+
+// per group row scales of the 16 rows, one vector per group
+template <int N> struct xmx_row_scales {
+    simd<float, 16> v[N];
+
+    ESIMD_INLINE simd<float, 16> get(int i) { return v[i]; }
+};
+
+// the 12 packed scale bytes of a super-block (Q3_K, Q4_K, Q5_K) of the 16 rows as three dwords
+ESIMD_INLINE void xmx_load_sc12(const uint8_t *            sc,
+                                const simd<uint32_t, 16> & bi,
+                                simd<uint32_t, 16> &       w0,
+                                simd<uint32_t, 16> &       w1,
+                                simd<uint32_t, 16> &       w2) {
+    using namespace sycl::ext::intel::esimd;
+    w0 = gather<uint32_t, 16>((const uint32_t *) sc, bi * 12);
+    w1 = gather<uint32_t, 16>((const uint32_t *) sc, bi * 12 + 4);
+    w2 = gather<uint32_t, 16>((const uint32_t *) sc, bi * 12 + 8);
+}
+
+// d and dmin (a half2 per super-block, Q2_K, Q4_K, Q5_K) of the 16 rows
+ESIMD_INLINE void xmx_load_dm(const sycl::half *          dm,
+                              const simd<uint32_t, 16> & bi,
+                              simd<float, 16> &          d,
+                              simd<float, 16> &          dmin) {
+    using namespace sycl::ext::intel::esimd;
+    d    = convert<float>(gather<sycl::half, 16>(dm, bi * (2 * (uint32_t) sizeof(sycl::half))));
+    dmin = convert<float>(
+        gather<sycl::half, 16>(dm, bi * (2 * (uint32_t) sizeof(sycl::half)) + (uint32_t) sizeof(sycl::half)));
+}
+
+// Q6_K reorder layout: [ql: nb*128] [qh: nb*64] [scales int8: nb*16] [d: nb*2]
+template <bool SR> struct xmx_traits_q6_k : xmx_traits_base<SR, 7, xmx_scale_kind::split> {
+    struct ctx : xmx_ctx_base {
+        const uint8_t *    ql;
+        const uint8_t *    qh;
+        const int8_t *     scales;
+        const sycl::half * dh;
+        unsigned           pl, ph, ps;
+    };
+
+    static ESIMD_INLINE ctx make(const void * vx, int nrows, int bpr, int row0) {
+        const size_t nb = (size_t) nrows * bpr;
+        ctx          c;
+        xmx_ctx_init(c, nrows, bpr, row0);
+        c.ql     = (const uint8_t *) vx;
+        c.qh     = c.ql + nb * (QK_K / 2);
+        c.scales = (const int8_t *) (c.qh + nb * (QK_K / 4));
+        c.dh     = (const sycl::half *) (c.scales + nb * (QK_K / 16));
+        c.pl     = xmx_pitch(bpr, QK_K / 2);
+        c.ph     = xmx_pitch(bpr, QK_K / 4);
+        c.ps     = xmx_pitch(bpr, QK_K / 16);
+        return c;
+    }
+
+    // row scales of the 16 scale groups of a super-block
+    struct scales_t {
+        simd<int8_t, 256> sc8;
+        simd<float, 16>   dv;
+
+        ESIMD_INLINE simd<float, 16> get(int gr) {
+            simd<int8_t, 16> s = sc8.template select<16, 4>(64 * (gr / 4) + (gr % 4));
+            return sycl::ext::intel::esimd::convert<float>(s) * dv;
+        }
+    };
+
+    // rows: clamped row index of each of the 16 lanes
+    static ESIMD_INLINE scales_t load_scales(const ctx & c, int sb, const simd<uint32_t, 16> & rows) {
+        using namespace sycl::ext::intel::esimd;
+        scales_t s;
+        s.dv = convert<float>(gather<sycl::half, 16>(c.dh, (rows * c.bpr + sb) * (uint32_t) sizeof(sycl::half)));
+        simd<uint32_t, 64> raw = xmx_load_t<4, SR>(c.scales, c.ps, c.h, sb * 4, c.row0);
+        s.sc8                  = raw.template bit_cast_view<int8_t>();
+        return s;
+    }
+
+    struct part_t {
+        simd<uint32_t, 128> a, b, h;
+    };
+
+    static ESIMD_INLINE part_t load_part(const ctx & c, int sb, int part) {
+        part_t p;
+        p.a = xmx_load_t<8, SR>(c.ql, c.pl, c.h, sb * 32 + part * 16, c.row0);
+        p.b = xmx_load_t<8, SR>(c.ql, c.pl, c.h, sb * 32 + part * 16 + 8, c.row0);
+        p.h = xmx_load_t<8, SR>(c.qh, c.ph, c.h, sb * 16 + part * 8, c.row0);
+        return p;
+    }
+
+    static ESIMD_INLINE simd<uint32_t, 128> unpack(const part_t & p, int g, int /*part*/) {
+        simd<uint32_t, 128> qs = (g & 1) ? p.b : p.a;
+        simd<uint32_t, 128> x;
+        if (g == 0) {
+            x = (qs & 0x0F0F0F0Fu) | ((p.h << 4) & 0x30303030u);
+        } else if (g == 1) {
+            x = (qs & 0x0F0F0F0Fu) | ((p.h << 2) & 0x30303030u);
+        } else if (g == 2) {
+            x = ((qs >> 4) & 0x0F0F0F0Fu) | (p.h & 0x30303030u);
+        } else {
+            x = ((qs >> 4) & 0x0F0F0F0Fu) | ((p.h >> 2) & 0x30303030u);
+        }
+        return xmx_bytes_sub(x, 0x20202020u);
+    }
+};
+
+// Q3_K reorder layout: [qs: nb*64] [hmask: nb*32] [scales: nb*12] [d: nb*2]
+// chunk s of 32: 2 bits from qs byte 32*(s/4)+l at shift 2*(s%4), high bit is hmask[l] bit s, value is q - 4
+template <bool SR> struct xmx_traits_q3_k : xmx_traits_base<SR, 16, xmx_scale_kind::split> {
+    struct ctx : xmx_ctx_base {
+        const uint8_t *    qs;
+        const uint8_t *    hm;
+        const uint8_t *    sc;
+        const sycl::half * dh;
+        unsigned           pq, ph;
+    };
+
+    static ESIMD_INLINE ctx make(const void * vx, int nrows, int bpr, int row0) {
+        const size_t nb = (size_t) nrows * bpr;
+        ctx          c;
+        xmx_ctx_init(c, nrows, bpr, row0);
+        c.qs = (const uint8_t *) vx;
+        c.hm = c.qs + nb * (QK_K / 4);
+        c.sc = c.hm + nb * (QK_K / 8);
+        c.dh = (const sycl::half *) (c.sc + nb * 12);
+        c.pq = xmx_pitch(bpr, QK_K / 4);
+        c.ph = xmx_pitch(bpr, QK_K / 8);
+        return c;
+    }
+
+    using scales_t = xmx_row_scales<16>;
+
+    static ESIMD_INLINE scales_t load_scales(const ctx & c, int sb, const simd<uint32_t, 16> & rows) {
+        using namespace sycl::ext::intel::esimd;
+        scales_t           s;
+        simd<uint32_t, 16> bi = rows * c.bpr + sb;
+        simd<uint32_t, 16> w0, w1, w2;
+        xmx_load_sc12(c.sc, bi, w0, w1, w2);
+        simd<float, 16> dv = convert<float>(gather<sycl::half, 16>(c.dh, bi * (uint32_t) sizeof(sycl::half)));
+#pragma unroll
+        for (int i = 0; i < 16; ++i) {
+            const simd<uint32_t, 16> lo   = (i % 8) < 4 ? w0 : w1;
+            const int                bl   = i % 4;
+            simd<uint32_t, 16>       nib  = (lo >> (8 * bl + (i < 8 ? 0 : 4))) & 0xFu;
+            simd<uint32_t, 16>       hi   = ((w2 >> (8 * bl + 2 * (i / 4))) & 3u) << 4;
+            simd<int, 16>            code = nib | hi;
+            s.v[i]                        = convert<float>(code - 32) * dv;
+        }
+        return s;
+    }
+
+    struct part_t {
+        simd<uint32_t, 128> q, h;
+    };
+
+    static ESIMD_INLINE part_t load_part(const ctx & c, int sb, int part) {
+        part_t p;
+        p.q = xmx_load_t<8, SR>(c.qs, c.pq, c.h, sb * 16 + part * 8, c.row0);
+        p.h = xmx_load_t<8, SR>(c.hm, c.ph, c.h, sb * 8, c.row0);
+        return p;
+    }
+
+    static ESIMD_INLINE simd<uint32_t, 128> unpack(const part_t & p, int g, int part) {
+        const int           s = 4 * part + g;
+        simd<uint32_t, 128> x = ((p.q >> (2 * g)) & 0x03030303u) | (((p.h >> s) & 0x01010101u) << 2);
+        return xmx_bytes_sub(x, 0x04040404u);
+    }
+};
+
+// Q4_K and Q5_K (QH) reorder layout: [qs: nb*128] ([qh: nb*32]) [scales: nb*12] [dm: nb*4]
+// group j of 32: nibble of qs byte 32*(j/2)+l (low for even j), Q5_K adds 16 * bit j of qh[l]
+// value is dall * sc_j * q - dmin * m_j, one scale and one min per group
+template <bool QH, bool SR> struct xmx_traits_q45_k : xmx_traits_base<SR, QH ? 16 : 7, xmx_scale_kind::min> {
+    struct ctx : xmx_ctx_base {
+        const uint8_t *    qs;
+        const uint8_t *    qh;
+        const uint8_t *    sc;
+        const sycl::half * dm;
+        unsigned           pq, ph;
+    };
+
+    static ESIMD_INLINE ctx make(const void * vx, int nrows, int bpr, int row0) {
+        const size_t nb = (size_t) nrows * bpr;
+        ctx          c;
+        xmx_ctx_init(c, nrows, bpr, row0);
+        c.qs = (const uint8_t *) vx;
+        c.qh = c.qs + nb * (QK_K / 2);
+        c.sc = QH ? c.qh + nb * (QK_K / 8) : c.qh;
+        c.dm = (const sycl::half *) (c.sc + nb * 12);
+        c.pq = xmx_pitch(bpr, QK_K / 2);
+        c.ph = xmx_pitch(bpr, QK_K / 8);
+        return c;
+    }
+
+    struct scales_t {
+        xmx_row_scales<8> d;  // dall * scale of each group
+        xmx_row_scales<8> m;  // dmin * min of each group
+    };
+
+    static ESIMD_INLINE scales_t load_scales(const ctx & c, int sb, const simd<uint32_t, 16> & rows) {
+        using namespace sycl::ext::intel::esimd;
+        scales_t           s;
+        simd<uint32_t, 16> bi = rows * c.bpr + sb;
+        simd<uint32_t, 16> w0, w1, w2;
+        xmx_load_sc12(c.sc, bi, w0, w1, w2);
+        simd<float, 16> d, dmin;
+        xmx_load_dm(c.dm, bi, d, dmin);
+#pragma unroll
+        for (int j = 0; j < 8; ++j) {
+            const int          jj = j % 4;
+            simd<uint32_t, 16> sc, m;
+            if (j < 4) {
+                sc = (w0 >> (8 * jj)) & 63u;
+                m  = (w1 >> (8 * jj)) & 63u;
+            } else {
+                sc = ((w2 >> (8 * jj)) & 0xFu) | (((w0 >> (8 * jj + 6)) & 3u) << 4);
+                m  = ((w2 >> (8 * jj + 4)) & 0xFu) | (((w1 >> (8 * jj + 6)) & 3u) << 4);
+            }
+            s.d.v[j] = convert<float>(simd<int, 16>(sc)) * d;
+            s.m.v[j] = convert<float>(simd<int, 16>(m)) * dmin;
+        }
+        return s;
+    }
+
+    struct part_t {
+        simd<uint32_t, 128> a, b, h;
+    };
+
+    static ESIMD_INLINE part_t load_part(const ctx & c, int sb, int part) {
+        part_t p;
+        p.a = xmx_load_t<8, SR>(c.qs, c.pq, c.h, sb * 32 + part * 16, c.row0);
+        p.b = xmx_load_t<8, SR>(c.qs, c.pq, c.h, sb * 32 + part * 16 + 8, c.row0);
+        if constexpr (QH) {
+            p.h = xmx_load_t<8, SR>(c.qh, c.ph, c.h, sb * 8, c.row0);
+        }
+        return p;
+    }
+
+    static ESIMD_INLINE simd<uint32_t, 128> unpack(const part_t & p, int g, int part) {
+        simd<uint32_t, 128> qs = (g < 2) ? p.a : p.b;
+        simd<uint32_t, 128> x  = (qs >> (4 * (g & 1))) & 0x0F0F0F0Fu;
+        if constexpr (QH) {
+            x |= ((p.h >> (4 * part + g)) & 0x01010101u) << 4;
+        }
+        return x;
+    }
+};
+
+// Q2_K reorder layout: [qs: nb*64] [scales: nb*16] [dm: nb*4]
+// chunk s of 32: 2 bits from qs byte 32*(s/4)+l at shift 2*(s%4), each 16 wide half has a 4 bit scale and min
+// value is d * (code & 15) * q - dmin * (code >> 4)
+template <bool SR> struct xmx_traits_q2_k : xmx_traits_base<SR, 16, xmx_scale_kind::min16> {
+    struct ctx : xmx_ctx_base {
+        const uint8_t *    qs;
+        const uint8_t *    sc;
+        const sycl::half * dm;
+        unsigned           pq, ps;
+    };
+
+    static ESIMD_INLINE ctx make(const void * vx, int nrows, int bpr, int row0) {
+        const size_t nb = (size_t) nrows * bpr;
+        ctx          c;
+        xmx_ctx_init(c, nrows, bpr, row0);
+        c.qs = (const uint8_t *) vx;
+        c.sc = c.qs + nb * (QK_K / 4);
+        c.dm = (const sycl::half *) (c.sc + nb * (QK_K / 16));
+        c.pq = xmx_pitch(bpr, QK_K / 4);
+        c.ps = xmx_pitch(bpr, QK_K / 16);
+        return c;
+    }
+
+    struct scales_t {
+        simd<uint8_t, 256> code;  // 4 bit scale (low) and min (high) of each 16 wide half
+        simd<float, 16>    d, dmin;
+
+        ESIMD_INLINE simd<uint8_t, 16> byte(int gr) { return code.template select<16, 4>(64 * (gr / 4) + (gr % 4)); }
+    };
+
+    static ESIMD_INLINE scales_t load_scales(const ctx & c, int sb, const simd<uint32_t, 16> & rows) {
+        using namespace sycl::ext::intel::esimd;
+        scales_t           s;
+        simd<uint32_t, 16> bi  = rows * c.bpr + sb;
+        simd<uint32_t, 64> raw = xmx_load_t<4, SR>(c.sc, c.ps, c.h, sb * 4, c.row0);
+        s.code                 = raw.template bit_cast_view<uint8_t>();
+        xmx_load_dm(c.dm, bi, s.d, s.dmin);
+        return s;
+    }
+
+    struct part_t {
+        simd<uint32_t, 128> q;
+    };
+
+    static ESIMD_INLINE part_t load_part(const ctx & c, int sb, int part) {
+        part_t p;
+        p.q = xmx_load_t<8, SR>(c.qs, c.pq, c.h, sb * 16 + part * 8, c.row0);
+        return p;
+    }
+
+    static ESIMD_INLINE simd<uint32_t, 128> unpack(const part_t & p, int g, int /*part*/) {
+        return (p.q >> (2 * g)) & 0x03030303u;
+    }
+};
+
+// Q8_0 reorder layout: [qs: nrows*K bytes] [d: one half per 32 weights]
+// the int8 weights feed DPAS as they are, a group is one block with one scale
+template <bool SR> struct xmx_traits_q8_0 : xmx_traits_base<SR, 16, xmx_scale_kind::plain> {
+    struct ctx : xmx_ctx_base {
+        const uint8_t *    qs;
+        const sycl::half * dh;
+        unsigned           pq;
+    };
+
+    static ESIMD_INLINE ctx make(const void * vx, int nrows, int bpr, int row0) {
+        ctx c;
+        xmx_ctx_init(c, nrows, bpr, row0);
+        c.qs = (const uint8_t *) vx;
+        c.dh = (const sycl::half *) (c.qs + (size_t) nrows * bpr * QK_K);
+        c.pq = xmx_pitch(bpr, QK_K);
+        return c;
+    }
+
+    struct scales_t {
+        xmx_row_scales<8> d;  // scale of each of the 8 blocks of a super-block
+    };
+
+    static ESIMD_INLINE scales_t load_scales(const ctx & c, int sb, const simd<uint32_t, 16> & rows) {
+        using namespace sycl::ext::intel::esimd;
+        scales_t           s;
+        simd<uint32_t, 16> blk = (rows * c.bpr + sb) * (QK_K / QK8_0);
+#pragma unroll
+        for (int j = 0; j < 8; ++j) {
+            s.d.v[j] = convert<float>(gather<sycl::half, 16>(c.dh, (blk + j) * (uint32_t) sizeof(sycl::half)));
+        }
+        return s;
+    }
+
+    struct part_t {
+        simd<uint32_t, 128> q[4];
+    };
+
+    static ESIMD_INLINE part_t load_part(const ctx & c, int sb, int part) {
+        part_t p;
+#pragma unroll
+        for (int g = 0; g < 4; ++g) {
+            p.q[g] = xmx_load_t<8, SR>(c.qs, c.pq, c.h, sb * 64 + 8 * (4 * part + g), c.row0);
+        }
+        return p;
+    }
+
+    static ESIMD_INLINE simd<uint32_t, 128> unpack(const part_t & p, int g, int /*part*/) { return p.q[g]; }
+};
+
+template <bool SR> using xmx_traits_q4_k = xmx_traits_q45_k<false, SR>;
+template <bool SR> using xmx_traits_q5_k = xmx_traits_q45_k<true, SR>;
+
+// registers per thread, the large file halves the resident threads but avoids spills in wide tiles, the fused
+// kernel keeps two sets of weights and scales and always takes it
+template <typename T, int NC, bool FUSED> constexpr int xmx_grf() {
+    return FUSED || NC >= T::nc_wide ? 256 : 128;
+}
+
+// One DPAS of a block of RC columns: the int32 products of the weights B and the activations A, as float and scaled
+// per weight row. Every quant type sums one or two of these per group of 32.
+template <int RC, int NT>
+ESIMD_INLINE simd<float, RC * NT> xmx_term(const simd<int8_t, 512> &    B,
+                                           const simd<int8_t, RC * 32> & A,
+                                           const simd<float, NT> &       s) {
+    using namespace sycl::ext::intel::esimd;
+    namespace xmx = sycl::ext::intel::esimd::xmx;
+
+    simd<int, RC * NT> C = xmx::dpas<8, RC, int>(B, A);
+    return convert<float>(C) * s.template replicate<RC>();
+}
+
+// one group of 32 of one weight set, accumulated into Cf for all the columns of the tile
+template <typename T, int NC, typename A2T, typename D8T>
+ESIMD_INLINE void xmx_group(simd<float, NC * XMX_ROWS> & Cf,
+                            const typename T::part_t &   pd,
+                            typename T::scales_t &       sc,
+                            A2T &                        A2,
+                            D8T &                        d8f,
+                            simd<int8_t, 512> &          Bl,
+                            simd<int8_t, 512> &          Bh,
+                            const simd<int8_t, 512> &    Ol,
+                            const int                    g,
+                            const int                    part) {
+    using namespace sycl::ext::intel::esimd;
+
+    constexpr int  NT    = XMX_ROWS;
+    constexpr int  RC    = NC < 8 ? NC : 8;
+    constexpr int  NB    = NC / RC;
+    constexpr bool split = !T::has_min && !T::plain && !T::min16;  // two scales per group of 32
+    constexpr bool pair  = (RC * 32) % 64 == 0;                    // a block of RC rows fills whole registers
+
+    const int gi = 4 * part + g;
+
+    simd<uint32_t, 128> x = T::unpack(pd, g, part);
+    simd<int8_t, 512>   B = x.template bit_cast_view<int8_t>();
+    if constexpr (split) {
+        Bl.template select<256, 1>(0)   = B.template select<256, 1>(0);
+        Bh.template select<256, 1>(256) = B.template select<256, 1>(256);
+    }
+
+    simd<float, NT> rd;
+    simd<float, NT> rm;
+    simd<float, NT> rs0;
+    simd<float, NT> rs1;
+
+    // Q2_K: scale and min of each 16 wide half are 4 bit codes, the weight is d * scl * q - dmin * mc. scl * q
+    // is at most 45, so a dword multiply keeps the bytes apart and the scaled weights go to DPAS as int8. The
+    // min term sums a * mc over K, the codes mc take the place of the weights in a second DPAS.
+    simd<int8_t, 512> Bs;
+    simd<int8_t, 512> Bm;
+    if constexpr (T::min16) {
+        const simd<uint8_t, NT> c0  = sc.byte(2 * gi);
+        const simd<uint8_t, NT> c1  = sc.byte(2 * gi + 1);
+        simd<uint32_t, NT>      sl0 = convert<uint32_t>(simd<uint8_t, NT>(c0 & (uint8_t) 15));
+        simd<uint32_t, NT>      sl1 = convert<uint32_t>(simd<uint8_t, NT>(c1 & (uint8_t) 15));
+        simd<uint32_t, NT>      mc0 = convert<uint32_t>(simd<uint8_t, NT>(c0 >> (uint8_t) 4)) * 0x01010101u;
+        simd<uint32_t, NT>      mc1 = convert<uint32_t>(simd<uint8_t, NT>(c1 >> (uint8_t) 4)) * 0x01010101u;
+        simd<uint32_t, 128>     ws;
+        simd<uint32_t, 128>     wm;
+        ws.template select<64, 1>(0)  = x.template select<64, 1>(0) * sl0.template replicate<4>();
+        ws.template select<64, 1>(64) = x.template select<64, 1>(64) * sl1.template replicate<4>();
+        wm.template select<64, 1>(0)  = mc0.template replicate<4>();
+        wm.template select<64, 1>(64) = mc1.template replicate<4>();
+        Bs                            = ws.template bit_cast_view<int8_t>();
+        Bm                            = wm.template bit_cast_view<int8_t>();
+    }
+    if constexpr (T::plain) {
+        rd = sc.d.get(gi);
+    } else if constexpr (T::has_min) {
+        rd = sc.d.get(gi);
+        rm = sc.m.get(gi);
+    } else if constexpr (T::min16) {
+        // scales are applied through Bs and Bm
+    } else {
+        rs0 = sc.get(2 * gi);
+        rs1 = sc.get(2 * gi + 1);
+    }
+
+#pragma unroll
+    for (int b = 0; b < NB; ++b) {
+        simd<int8_t, RC * 32> A =
+            A2[b].template select<RC * 8, 1>(pair ? (g % 2) * RC * 8 : 0).template bit_cast_view<int8_t>();
+        simd<float, RC * NT> d8r =
+            d8f[b].template select<RC, RC == 1 ? 1 : 8>(gi).template replicate_vs_w_hs<RC, 1, NT, 0>(0);
+        auto Cb = Cf.template select<RC * NT, 1>(b * RC * NT);
+
+        simd<float, RC * NT> z;
+        if constexpr (T::plain) {
+            z = xmx_term<RC, NT>(B, A, rd);
+        } else if constexpr (T::has_min) {
+            // the second DPAS (all ones against the activations) is the exact integer sum of the activations
+            z = xmx_term<RC, NT>(B, A, rd) - xmx_term<RC, NT>(Ol, A, rm);
+        } else if constexpr (T::min16) {
+            z = xmx_term<RC, NT>(Bs, A, sc.d) - xmx_term<RC, NT>(Bm, A, sc.dmin);
+        } else {
+            z = xmx_term<RC, NT>(Bl, A, rs0) + xmx_term<RC, NT>(Bh, A, rs1);
+        }
+        Cb += z * d8r;
+    }
+}
+
+// activation of the GLU applied to the gate, uniform across the launch
+template <int N> ESIMD_INLINE simd<float, N> xmx_glu_act(const simd<float, N> & gate, const int glu_op) {
+    using namespace sycl::ext::intel::esimd;
+    if (glu_op == GGML_GLU_OP_SWIGLU) {
+        return gate / (1.0f + exp(-gate));
+    }
+    // GEGLU, the tanh is written with exp
+    simd<float, N> a = 0.79788456f * gate * (1.0f + 0.044715f * gate * gate);
+    simd<float, N> t = 1.0f - 2.0f / (exp(2.0f * a) + 1.0f);
+    return 0.5f * gate * (1.0f + t);
+}
+
+// activations come in through 2D block loads, one for the (d, s) pairs of a super-block (s is unused) and one per two
+// groups for the quants, the DPAS A operand needs no repacking. Groups with two scales (16 wide) mask the
+// weights instead, zeros in B drop the other half of K.
+// KS is the most threads that split K for one tile, the launch may use fewer.
+// FUSED computes up * act(gate) from the weights vx (up) and vg (gate), reading the activations once.
+template <typename T, int NC, int KS, bool FUSED>
+ESIMD_INLINE void xmx_mul_mat(const void * vx,
+                              const void * vg,
+                              const char * vy,
+                              float *      dst,
+                              const int    ncols,
+                              const int    nrows,
+                              const int    ncv,
+                              const int    stride_y_bytes,
+                              const int    ldd,
+                              const int    glu_op,
+                              const int    tile,
+                              const int    lid,
+                              const int    ks) {
+    using namespace sycl::ext::intel::esimd;
+
+    constexpr int  NT    = XMX_ROWS;
+    constexpr int  RC    = NC < 8 ? NC : 8;
+    constexpr int  NB    = NC / RC;
+    constexpr int  N_ACC = FUSED ? 2 : 1;
+    constexpr bool pair  = (RC * 32) % 64 == 0;  // a block of RC rows fills whole registers
+
+    if constexpr (KS > 1) {
+        slm_init<N_ACC * KS * NC * NT * sizeof(float)>();
+    }
+
+    const int  bpr  = ncols / QK_K;
+    const int  row0 = tile * NT;
+    const auto c    = T::make(vx, nrows, bpr, row0);
+    const auto cg   = T::make(FUSED ? vg : vx, nrows, bpr, row0);
+
+    simd<uint32_t, NT> rows;
+#pragma unroll
+    for (int n = 0; n < NT; ++n) {
+        rows[n] = row0 + n < nrows ? row0 + n : nrows - 1;
+    }
+
+    const uint32_t * yq  = (const uint32_t *) vy;
+    const uint32_t * yd  = (const uint32_t *) (vy + ncols);
+    const unsigned   yqw = ncols - 1;
+    const unsigned   ydw = ncols / QK8_1 * sizeof(sycl::half2) - 1;
+    const unsigned   yh  = ncv - 1;
+    const unsigned   yp  = stride_y_bytes - 1;
+
+    simd<float, NC * NT> Cf  = 0.0f;
+    simd<float, NC * NT> Cfg = 0.0f;  // gate, only used when fused
+    simd<int8_t, 512>    Bl  = int8_t(0);
+    simd<int8_t, 512>    Bh  = int8_t(0);
+    simd<int8_t, 512>    Ol  = int8_t(0);
+    if constexpr (T::has_min) {
+        Ol = int8_t(1);
+    }
+
+    for (int sb = lid; sb < bpr; sb += ks) {
+        auto                 sc = T::load_scales(c, sb, rows);
+        typename T::scales_t scg;
+        if constexpr (FUSED) {
+            scg = T::load_scales(cg, sb, rows);
+        }
+
+        // d of the 8 groups of the super-block, [column * 8 + group]
+        simd<float, 8 * RC> d8f[NB];
+#pragma unroll
+        for (int b = 0; b < NB; ++b) {
+            simd<uint32_t, 8 * RC>    ds = load_2d<uint32_t, 8, RC, 1, false, false>(yd, ydw, yh, yp, sb * 8, b * RC);
+            simd<sycl::half, 16 * RC> dh = ds.template bit_cast_view<sycl::half>();
+            d8f[b]                       = convert<float>(simd<sycl::half, 8 * RC>(dh.template select<8 * RC, 2>(0)));
+        }
+
+#pragma unroll
+        for (int part = 0; part < 2; ++part) {
+            const auto         pd = T::load_part(c, sb, part);
+            typename T::part_t pdg;
+            if constexpr (FUSED) {
+                pdg = T::load_part(cg, sb, part);
+            }
+
+            simd<uint32_t, 16 * RC> A2[NB];  // quants of two groups
+
+#pragma unroll
+            for (int g = 0; g < 4; ++g) {
+                const int gi = 4 * part + g;
+                if constexpr (pair) {
+                    if (g % 2 == 0) {
+#pragma unroll
+                        for (int b = 0; b < NB; ++b) {
+                            A2[b] =
+                                load_2d<uint32_t, 8, RC, 2, false, false>(yq, yqw, yh, yp, sb * 64 + gi * 8, b * RC);
+                        }
+                    }
+                } else {
+#pragma unroll
+                    for (int b = 0; b < NB; ++b) {
+                        A2[b].template select<RC * 8, 1>(0) =
+                            load_2d<uint32_t, 8, RC, 1, false, false>(yq, yqw, yh, yp, sb * 64 + gi * 8, b * RC);
+                    }
+                }
+
+                xmx_group<T, NC>(Cf, pd, sc, A2, d8f, Bl, Bh, Ol, g, part);
+                if constexpr (FUSED) {
+                    xmx_group<T, NC>(Cfg, pdg, scg, A2, d8f, Bl, Bh, Ol, g, part);
+                }
+            }
+        }
+    }
+
+    if constexpr (KS > 1) {
+        slm_block_store<float, NC * NT>(lid * NC * NT * sizeof(float), Cf);
+        if constexpr (FUSED) {
+            slm_block_store<float, NC * NT>((KS + lid) * NC * NT * sizeof(float), Cfg);
+        }
+        barrier();
+        if (lid != 0) {
+            return;
+        }
+        Cf  = 0.0f;
+        Cfg = 0.0f;
+#pragma unroll 4
+        for (int j = 0; j < ks; ++j) {
+            Cf += slm_block_load<float, NC * NT>(j * NC * NT * sizeof(float));
+            if constexpr (FUSED) {
+                Cfg += slm_block_load<float, NC * NT>((KS + j) * NC * NT * sizeof(float));
+            }
+        }
+    }
+
+    if constexpr (FUSED) {
+        Cf *= xmx_glu_act<NC * NT>(Cfg, glu_op);
+    }
+
+    if (row0 + NT <= nrows && ldd % 4 == 0 && (uintptr_t) dst % 16 == 0) {
+#pragma unroll
+        for (int m = 0; m < NC; ++m) {
+            if (NC <= 8 || m < ncv) {
+                block_store<float, NT>(dst + (size_t) m * ldd + row0, Cf.template select<NT, 1>(m * NT));
+            }
+        }
+    } else {
+        // tail tile or unaligned output
+        simd<uint32_t, NT> lane(0, 1);
+        simd_mask<NT>      in_rows = (lane + row0) < (uint32_t) nrows;
+#pragma unroll
+        for (int m = 0; m < NC; ++m) {
+            if (NC <= 8 || m < ncv) {
+                scatter<float, NT>(dst + (size_t) m * ldd + row0, lane * (uint32_t) sizeof(float),
+                                   Cf.template select<NT, 1>(m * NT), in_rows);
+            }
+        }
+    }
+}
+
+// KS_MAX bounds the threads that split K for one tile, the launch uses as few as keep every thread to the same
+// number of super-blocks as the busiest one
+template <typename T, int NC, int KS_MAX, bool FUSED>
+static void xmx_mul_mat_launch_ks(const void *    vx,
+                                  const void *    vg,
+                                  const char *    vy,
+                                  float *         dst,
+                                  const int       ncols,
+                                  const int       nrows,
+                                  const int       ncv,
+                                  const int       stride_y_bytes,
+                                  const int       ldd,
+                                  const int       glu_op,
+                                  dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const size_t n_tiles = (nrows + XMX_ROWS - 1) / XMX_ROWS;
+    const int    bpr     = ncols / QK_K;
+    const int    rounds  = (bpr + KS_MAX - 1) / KS_MAX;
+    const int    ks      = (bpr + rounds - 1) / rounds;
+    stream->submit([&](sycl::handler & cgh) {
+        cgh.parallel_for(sycl::nd_range<1>(sycl::range<1>(n_tiles * ks), sycl::range<1>(ks)),
+                         sycl::ext::oneapi::experimental::properties{
+                             sycl::ext::intel::experimental::grf_size<xmx_grf<T, NC, FUSED>()> },
+                         [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
+                             xmx_mul_mat<T, NC, KS_MAX, FUSED>(vx, vg, vy, dst, ncols, nrows, ncv, stride_y_bytes, ldd,
+                                                               glu_op, (int) it.get_group(0), (int) it.get_local_id(0),
+                                                               ks);
+                         });
+    });
+}
+
+template <typename T, int NC, bool FUSED>
+static void xmx_mul_mat_launch(const void *    vx,
+                               const void *    vg,
+                               const char *    vy,
+                               float *         dst,
+                               const int       ncols,
+                               const int       nrows,
+                               const int       ncv,
+                               const int       stride_y_bytes,
+                               const int       ldd,
+                               const int       glu_op,
+                               dpct::queue_ptr stream) {
+    // the fused kernel keeps two sets of accumulators in shared memory, half the threads for the same size
+    constexpr int ks_max = xmx_n_split<NC>() / (FUSED ? 2 : 1);
+    xmx_mul_mat_launch_ks<T, NC, ks_max, FUSED>(vx, vg, vy, dst, ncols, nrows, ncv, stride_y_bytes, ldd, glu_op,
+                                                stream);
+}
+
+// FUSED: vx and vg are the up and gate weights, dst gets up * act(gate) with glu_op a ggml_glu_op
+template <typename T, bool FUSED = false>
+static void xmx_mul_mat_ncols(const void *    vx,
+                              const void *    vg,
+                              const char *    vy,
+                              float *         dst,
+                              const int       ncols,
+                              const int       nrows,
+                              const int       ncols_dst,
+                              const int       stride_y_bytes,
+                              const int       ldd,
+                              const int       glu_op,
+                              dpct::queue_ptr stream) {
+    constexpr int chunk_max = FUSED ? XMX_FUSED_CHUNK : XMX_CHUNK;
+    for (int c0 = 0; c0 < ncols_dst; c0 += chunk_max) {
+        const int    n  = std::min(chunk_max, ncols_dst - c0);
+        const char * y  = vy + (size_t) c0 * stride_y_bytes;
+        float *      d  = dst + (size_t) c0 * ldd;
+        const int    nc = n > 32 ? 64 : (n > 16 ? 32 : (n > 8 ? 16 : n));
+        switch (nc) {
+#define XMX_CASE(N)                                                                                          \
+    case N:                                                                                                  \
+        xmx_mul_mat_launch<T, N, FUSED>(vx, vg, y, d, ncols, nrows, n, stride_y_bytes, ldd, glu_op, stream); \
+        break
+            XMX_CASE(1);
+            XMX_CASE(2);
+            XMX_CASE(3);
+            XMX_CASE(4);
+            XMX_CASE(5);
+            XMX_CASE(6);
+            XMX_CASE(7);
+            XMX_CASE(8);
+#undef XMX_CASE
+            // short row variants are for tests, not built for the wide tiles
+#define XMX_CASE_WIDE(N)                                                                                         \
+    case N:                                                                                                      \
+        if constexpr (!T::short_rows && (!FUSED || N <= XMX_FUSED_CHUNK)) {                                      \
+            xmx_mul_mat_launch<T, N, FUSED>(vx, vg, y, d, ncols, nrows, n, stride_y_bytes, ldd, glu_op, stream); \
+        }                                                                                                        \
+        break
+            XMX_CASE_WIDE(16);
+            XMX_CASE_WIDE(32);
+            XMX_CASE_WIDE(64);
+#undef XMX_CASE_WIDE
+            default:
+                GGML_ABORT("unsupported ncols_dst=%d for XMX MMV", n);
+        }
+    }
+}
+
 } // namespace ggml_sycl_esimd
 
 #endif // GGML_SYCL_ESIMD_HPP

ggml/src/ggml-sycl/fusion.cpp

@@ -37,6 +37,13 @@ static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const gg
     const bool reorder_pair = wu->type == GGML_TYPE_Q4_K && wg->type == GGML_TYPE_Q4_K;
     const bool plain_pair   = (wu->type == GGML_TYPE_Q5_K || wu->type == GGML_TYPE_IQ4_XS) &&
                             (wg->type == GGML_TYPE_Q5_K || wg->type == GGML_TYPE_IQ4_XS);
+#ifdef GGML_SYCL_MMVQ_HAS_XMX
+    // same-type pairs of the types of the XMX kernels, which also take a few more columns
+    // (the XMX kernels walk whole QK_K super-blocks, so they need rows of a multiple of QK_K)
+    const bool xmx_pair = wu->type == wg->type && ggml_sycl_xmx_supports_type(wu->type) && wu->ne[0] % QK_K == 0;
+#else
+    const bool xmx_pair = false;
+#endif // GGML_SYCL_MMVQ_HAS_XMX
     // the ESIMD kernel covers same-type q2_K..q6_K and q8_0, single column only (as unfused DMMV)
 #ifdef GGML_SYCL_DMMV_HAS_ESIMD
     const bool esimd_pair   = g_ggml_sycl_enable_esimd && wu->type == wg->type && act->ne[1] == 1 &&
@@ -47,7 +54,7 @@ static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const gg
 #endif
     // q8_0 rows are walked in 32-wide blocks, so they only need to be a multiple of QK8_0
     const int64_t row_align = esimd_pair && wu->type == GGML_TYPE_Q8_0 ? QK8_0 : QK_K;
-    if ((!reorder_pair && !plain_pair && !esimd_pair) || wu->ne[0] % row_align != 0) {
+    if ((!reorder_pair && !plain_pair && !xmx_pair && !esimd_pair) || wu->ne[0] % row_align != 0) {
         return false;
     }
 
@@ -67,7 +74,8 @@ static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const gg
         return false;
     }
     // mat-vec only: one column per decoded token, up to the batch the reorder kernels cover
-    if (act->ne[1] > MMVQ_MAX_BATCH_SIZE) {
+    const int max_cols = xmx_pair ? std::max(MMVQ_MAX_BATCH_SIZE, g_ggml_sycl_xmx_glu_max_cols) : MMVQ_MAX_BATCH_SIZE;
+    if (act->ne[1] > max_cols) {
         return false;
     }
 

ggml/src/ggml-sycl/ggml-sycl.cpp

@@ -103,6 +103,9 @@ int g_ggml_sycl_enable_vmm = 1;
 int g_ggml_sycl_enable_fusion = 1;
 int g_ggml_sycl_enable_esimd = 1;
 int g_ggml_sycl_mmvq_wide = 1;
+int g_ggml_sycl_enable_xmx_esimd = 1;
+int g_ggml_sycl_xmx_max_cols = GGML_SYCL_XMX_MAX_COLS_DEFAULT;
+int g_ggml_sycl_xmx_glu_max_cols = GGML_SYCL_XMX_GLU_MAX_COLS;
 int g_ggml_sycl_prioritize_dmmv = 0;
 int g_ggml_sycl_xmx_gather_types = GGML_SYCL_XMX_GATHER_TYPES_DEFAULT;
 int g_ggml_sycl_xmx_gather_shapes = GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT;
@@ -424,6 +427,10 @@ static void ggml_check_sycl() try {
         g_ggml_sycl_enable_fusion = ggml_sycl_get_env("GGML_SYCL_ENABLE_FUSION", 1);
         g_ggml_sycl_enable_esimd = ggml_sycl_get_env("GGML_SYCL_ENABLE_ESIMD", 1);
         g_ggml_sycl_mmvq_wide = ggml_sycl_get_env("GGML_SYCL_MMVQ_WIDE", 1);
+        g_ggml_sycl_enable_xmx_esimd = ggml_sycl_get_env("GGML_SYCL_ENABLE_XMX_ESIMD", 1);
+        g_ggml_sycl_xmx_max_cols = std::max(0, ggml_sycl_get_env("GGML_SYCL_XMX_MAX_COLS", GGML_SYCL_XMX_MAX_COLS_DEFAULT));
+        g_ggml_sycl_xmx_glu_max_cols =
+            std::clamp(ggml_sycl_get_env("GGML_SYCL_XMX_GLU_MAX_COLS", GGML_SYCL_XMX_GLU_MAX_COLS), 0, GGML_SYCL_XMX_GLU_MAX_COLS);
         g_ggml_sycl_prioritize_dmmv = ggml_sycl_get_env("GGML_SYCL_PRIORITIZE_DMMV", 0);
         g_ggml_sycl_xmx_gather_types = ggml_sycl_get_env("GGML_SYCL_XMX_GATHER_TYPES", GGML_SYCL_XMX_GATHER_TYPES_DEFAULT);
         g_ggml_sycl_xmx_gather_shapes = ggml_sycl_get_env("GGML_SYCL_XMX_GATHER_SHAPES", GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT);
@@ -561,7 +568,13 @@ static void ggml_check_sycl() try {
 #else
         GGML_LOG_INFO("  GGML_SYCL_ENABLE_ESIMD: %d disabled by compile flag\n", g_ggml_sycl_enable_esimd);
 #endif
+
         GGML_LOG_INFO("  GGML_SYCL_MMVQ_WIDE: %d\n", g_ggml_sycl_mmvq_wide);
+
+        GGML_LOG_INFO("  GGML_SYCL_ENABLE_XMX_ESIMD: %d\n", g_ggml_sycl_enable_xmx_esimd);
+        GGML_LOG_INFO("  GGML_SYCL_XMX_MAX_COLS: %d\n", g_ggml_sycl_xmx_max_cols);
+        GGML_LOG_INFO("  GGML_SYCL_XMX_GLU_MAX_COLS: %d\n", g_ggml_sycl_xmx_glu_max_cols);
+
         GGML_LOG_INFO("  GGML_SYCL_PRIORITIZE_DMMV: %d\n", g_ggml_sycl_prioritize_dmmv);
 
         g_ggml_sycl_use_async_mem_op_requested = ggml_sycl_get_env("GGML_SYCL_USE_ASYNC_MEM_OP", 1);
@@ -5165,9 +5178,24 @@ static bool can_use_dequantize_mul_mat_vec(const ggml_tensor * src0, const ggml_
            src0->ne[0] % dmmv_x_required == 0 && src1->ne[1] == 1;
 }
 
-static bool can_use_mul_mat_vec_q(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+// reordered weights of the types in ggml_sycl_xmx_supports_type() take the XMX kernel for wider batches
+static bool can_use_xmx_batch(int device, const ggml_tensor * src0, const ggml_tensor * src1) {
+#ifdef GGML_SYCL_MMVQ_HAS_XMX
+    const auto * extra = static_cast<const ggml_tensor_extra_gpu *>(src0->extra);
+    return ggml_sycl_xmx_enabled(device) && ggml_sycl_xmx_supports_type(src0->type) && extra &&
+           extra->optimized_feature.reorder && src1->ne[1] <= g_ggml_sycl_xmx_max_cols && src1->ne[2] == 1 &&
+           src1->ne[3] == 1;
+#else
+    GGML_UNUSED(device);
+    GGML_UNUSED(src0);
+    GGML_UNUSED(src1);
+    return false;
+#endif // GGML_SYCL_MMVQ_HAS_XMX
+}
+
+static bool can_use_mul_mat_vec_q(int device, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
     return ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 &&
-           src1->ne[1] <= MMVQ_MAX_BATCH_SIZE;
+           (src1->ne[1] <= MMVQ_MAX_BATCH_SIZE || can_use_xmx_batch(device, src0, src1));
 }
 
 static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
@@ -5207,12 +5235,15 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor
 
     // check data types and tensor shapes for custom matrix multiplication kernels:
     bool use_dequantize_mul_mat_vec = can_use_dequantize_mul_mat_vec(src0, src1, dst);
+    if (src1->ne[1] == 1 && ggml_sycl_xmx_min_cols(src0->type) == 1 && can_use_xmx_batch(ctx.device, src0, src1)) {
+        use_dequantize_mul_mat_vec = false;
+    }
 #ifdef GGML_SYCL_F16
     // dmmv may convert src1 to f16 in this build [TAG_GGML_PREC]
     use_dequantize_mul_mat_vec = use_dequantize_mul_mat_vec && ggml_sycl_src1_f16_ok(dst);
 #endif
 
-    bool use_mul_mat_vec_q = can_use_mul_mat_vec_q(src0, src1, dst);
+    bool use_mul_mat_vec_q = can_use_mul_mat_vec_q(ctx.device, src0, src1, dst);
 
     bool use_mul_mat_q =  ggml_sycl_supports_mmq(src0->type)
         && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;
@@ -5321,6 +5352,35 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, gg
     GGML_UNUSED(gate);
 }
 
+// Gate and up weights of one type on the XMX engines: one launch reads the activations once and applies the GLU.
+// Both weights must already be in the reorder layout. Returns false if the kernel declined.
+static bool ggml_sycl_mul_mat_glu_xmx(ggml_backend_sycl_context & ctx,
+                                      ggml_tensor *               glu,
+                                      ggml_tensor *               up,
+                                      const ggml_tensor *         wu,
+                                      const ggml_tensor *         wg,
+                                      const ggml_tensor *         act) {
+    const int64_t ne00 = wu->ne[0];
+    const int64_t ne11 = act->ne[1];
+
+    const queue_ptr stream           = ctx.stream();
+    const int       src1_padded_cols = GGML_PAD((int) ne00, MATRIX_ROW_PADDING);
+    const int       stride_y_bytes   = src1_padded_cols * (int) sizeof(block_q8_1) / QK8_1;
+
+    // log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises
+    scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (XMX)");
+
+    ggml_sycl_pool_alloc<char> src1_q8_alloc(ctx.pool(), (size_t) ne11 * stride_y_bytes);
+    char *                     src1_ddq = src1_q8_alloc.get();
+
+    quantize_row_q8_1_sycl<quantize_and_reorder_q8_1_soa>((const float *) act->data, src1_ddq, (int) ne00, (int) ne11,
+                                                          src1_padded_cols, stream);
+
+    return ggml_sycl_mul_mat_vec_q_glu_xmx(ctx.device, wu->type, ggml_get_glu_op(glu), wu->data, wg->data, src1_ddq,
+                                           (float *) glu->data, (int) ne00, (int) wu->ne[1], (int) ne11, stride_y_bytes,
+                                           /*stride_col_dst=*/(int) glu->ne[0], stream);
+}
+
 #ifdef GGML_SYCL_DMMV_HAS_ESIMD
 // {mul_mat(gate), mul_mat(up), GLU} on the reorder layout via the ESIMD DMMV kernel, for
 // same-type K-quant or Q8_0 pairs and a single activation column. It reads the F32 activation
@@ -5389,11 +5449,33 @@ static bool ggml_sycl_mul_mat_glu_fused(ggml_backend_sycl_context & ctx, ggml_cg
         return false;
     }
 
+    // gate and up of one type the XMX kernel handles, it needs the reorder layout like the unfused mmvq path
+    if (wg->type == wu->type && gate->src[1] == act && act->ne[1] >= ggml_sycl_xmx_min_cols(wu->type) &&
+        act->ne[1] <= g_ggml_sycl_xmx_glu_max_cols) {
+        opt_for_reorder(&ctx, wu, act, up, mul_mat_algo::MMVQ);
+        opt_for_reorder(&ctx, wg, act, gate, mul_mat_algo::MMVQ);
+        if (can_use_xmx_batch(ctx.device, wu, act) && can_use_xmx_batch(ctx.device, wg, act) &&
+            ggml_sycl_mul_mat_glu_xmx(ctx, glu, up, wu, wg, act)) {
+            return true;
+        }
+    }
+
+    // the kernels below take up to MMVQ_MAX_BATCH_SIZE columns, a wider batch got here for the XMX kernel
+    if (act->ne[1] > MMVQ_MAX_BATCH_SIZE) {
+        return false;
+    }
+
     // quant pairs the reorder kernel does not serve (mixed gate/up types, q5_K off BMG) take the
     // standard-layout fused path instead; same-type q4_K / q5_K keep the reorder path below
     const bool reorder_pair = wg->type == wu->type &&
         (wu->type == GGML_TYPE_Q4_K || (wu->type == GGML_TYPE_Q5_K && ggml_sycl_q5_k_mmvq_reuse(ctx.device)));
     if (!reorder_pair) {
+        const auto is_plain_type = [](ggml_type type) {
+            return type == GGML_TYPE_Q5_K || type == GGML_TYPE_IQ4_XS;
+        };
+        if (!is_plain_type(wu->type) || !is_plain_type(wg->type)) {
+            return false;
+        }
         return ggml_sycl_mul_mat_glu_mmvq_plain(ctx, glu, gate, up, wu, wg, act);
     }
 
@@ -6912,6 +6994,11 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
                 return false;
             }
         case GGML_OP_CONV_2D:
+            // WHCN: contiguous [KW,KH,IC,OC] kernel and [W,H,C,N] input
+            // CWHN: channel-fastest input (C fastest) and kernel with OC fastest
+            return (ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) ||
+                   (op->src[1]->nb[2] == ggml_type_size(op->src[1]->type) &&
+                    op->src[0]->nb[3] == ggml_type_size(op->src[0]->type));
         case GGML_OP_CONV_2D_DW:
         case GGML_OP_CONV_TRANSPOSE_2D:
             return true;

ggml/src/ggml-sycl/mmvq.cpp

@@ -6,6 +6,10 @@
 #include "quants.hpp"
 #include "vecdotq.hpp"
 
+#ifdef GGML_SYCL_MMVQ_HAS_XMX
+#    include "esimd.hpp"
+#endif // GGML_SYCL_MMVQ_HAS_XMX
+
 // vec_dot_q_sycl_t adapters for the IQ vec_dots that take their codebook tables as extra
 // arguments: bind the constant tables here (as vec_dot_iq2_s_q8_1 / vec_dot_iq1_m_q8_1 already do
 // internally) so they can be used as template arguments of mul_mat_vec_q_moe.
@@ -2557,6 +2561,158 @@ static void mul_mat_vec_iq4_xs_q8_1_sycl_switch_ncols(
     }
 }
 
+#ifdef GGML_SYCL_MMVQ_HAS_XMX
+// vgate is the gate weight of the fused gate and up kernel, nullptr for a plain mat-vec
+template <template <bool> class Traits>
+static void ggml_sycl_mul_mat_vec_q_xmx_run(bool            short_rows,
+                                            const void *    vx,
+                                            const void *    vgate,
+                                            const char *    vy,
+                                            float *         dst,
+                                            int             ne00,
+                                            int64_t         nrows,
+                                            int64_t         ncols_dst,
+                                            int             stride_y_bytes,
+                                            int             ldd,
+                                            int             glu_op,
+                                            dpct::queue_ptr stream) {
+    namespace esimd = ggml_sycl_esimd;
+    if (vgate) {
+        if (short_rows) {
+            esimd::xmx_mul_mat_ncols<Traits<true>, true>(vx, vgate, vy, dst, ne00, nrows, ncols_dst, stride_y_bytes,
+                                                         ldd, glu_op, stream);
+        } else {
+            esimd::xmx_mul_mat_ncols<Traits<false>, true>(vx, vgate, vy, dst, ne00, nrows, ncols_dst, stride_y_bytes,
+                                                          ldd, glu_op, stream);
+        }
+    } else if (short_rows) {
+        esimd::xmx_mul_mat_ncols<Traits<true>>(vx, nullptr, vy, dst, ne00, nrows, ncols_dst, stride_y_bytes, ldd, 0,
+                                               stream);
+    } else {
+        esimd::xmx_mul_mat_ncols<Traits<false>>(vx, nullptr, vy, dst, ne00, nrows, ncols_dst, stride_y_bytes, ldd, 0,
+                                                stream);
+    }
+}
+
+static bool ggml_sycl_mul_mat_vec_q_xmx_type(ggml_type       type,
+                                             bool            short_rows,
+                                             const void *    vx,
+                                             const void *    vgate,
+                                             const char *    vy,
+                                             float *         dst,
+                                             int             ne00,
+                                             int64_t         nrows,
+                                             int64_t         ncols_dst,
+                                             int             stride_y_bytes,
+                                             int             ldd,
+                                             int             glu_op,
+                                             dpct::queue_ptr stream) {
+#    define XMX_RUN(TRAITS)                                                                                       \
+        ggml_sycl_mul_mat_vec_q_xmx_run<ggml_sycl_esimd::TRAITS>(short_rows, vx, vgate, vy, dst, ne00, nrows,     \
+                                                                 ncols_dst, stride_y_bytes, ldd, glu_op, stream); \
+        return true
+    switch (type) {
+        case GGML_TYPE_Q2_K:
+            XMX_RUN(xmx_traits_q2_k);
+        case GGML_TYPE_Q3_K:
+            XMX_RUN(xmx_traits_q3_k);
+        case GGML_TYPE_Q4_K:
+            XMX_RUN(xmx_traits_q4_k);
+        case GGML_TYPE_Q5_K:
+            XMX_RUN(xmx_traits_q5_k);
+        case GGML_TYPE_Q6_K:
+            XMX_RUN(xmx_traits_q6_k);
+        case GGML_TYPE_Q8_0:
+            XMX_RUN(xmx_traits_q8_0);
+        default:
+            return false;
+    }
+#    undef XMX_RUN
+}
+
+// multi-column XMX path for reordered weights, returns false when it does not apply
+static bool ggml_sycl_mul_mat_vec_q_xmx(int                 device,
+                                        const ggml_tensor * src0,
+                                        const ggml_tensor * dst,
+                                        const char *        src0_dd_i,
+                                        const char *        src1_ddq_i,
+                                        float *             dst_dd_i,
+                                        int64_t             row_diff,
+                                        int64_t             src1_ncols,
+                                        int64_t             src1_padded_col_size,
+                                        dpct::queue_ptr     stream) {
+    const auto * extra = static_cast<const ggml_tensor_extra_gpu *>(src0->extra);
+    if (!ggml_sycl_xmx_enabled(device) || !ggml_sycl_xmx_supports_type(src0->type) || !extra ||
+        !extra->optimized_feature.reorder) {
+        return false;
+    }
+    if (src1_ncols < ggml_sycl_xmx_min_cols(src0->type) || src1_ncols > g_ggml_sycl_xmx_max_cols) {
+        return false;
+    }
+
+    const int ne00 = src0->ne[0];
+    const int ldd  = dst->ne[0];
+    if (!ggml_sycl_esimd::xmx_supported(src0_dd_i, ne00)) {
+        return false;
+    }
+
+    // rows under 64 bytes use the variant without 2D block loads, it is not built for the widest tile
+    const bool short_rows = ne00 < 4 * QK_K;
+    if (short_rows && src1_ncols > 8) {
+        return false;
+    }
+
+    const int stride_y_bytes = src1_padded_col_size * sizeof(block_q8_1) / QK8_1;
+    // the 2D block loads of the activations need a 64 byte aligned base and a pitch of 16 bytes
+    if ((uintptr_t) src1_ddq_i % 64 != 0 || stride_y_bytes % 16 != 0) {
+        return false;
+    }
+    return ggml_sycl_mul_mat_vec_q_xmx_type(src0->type, short_rows, src0_dd_i, nullptr, src1_ddq_i, dst_dd_i, ne00,
+                                            row_diff, src1_ncols, stride_y_bytes, ldd, 0, stream);
+}
+#endif // GGML_SYCL_MMVQ_HAS_XMX
+
+bool ggml_sycl_mul_mat_vec_q_glu_xmx(int              device,
+                                     enum ggml_type   src0_type,
+                                     enum ggml_glu_op glu_op,
+                                     const void *     vx,
+                                     const void *     vgate,
+                                     const void *     vy,
+                                     float *          dst,
+                                     int              ncols,
+                                     int              nrows,
+                                     int              ncols_dst,
+                                     int              stride_col_y_bytes,
+                                     int              stride_col_dst,
+                                     dpct::queue_ptr  stream) {
+#ifdef GGML_SYCL_MMVQ_HAS_XMX
+    if (!ggml_sycl_xmx_enabled(device) || !ggml_sycl_xmx_supports_type(src0_type)) {
+        return false;
+    }
+    if (glu_op != GGML_GLU_OP_SWIGLU && glu_op != GGML_GLU_OP_GEGLU) {
+        return false;
+    }
+    if (ncols_dst < ggml_sycl_xmx_min_cols(src0_type) || ncols_dst > g_ggml_sycl_xmx_glu_max_cols) {
+        return false;
+    }
+    if (!ggml_sycl_esimd::xmx_supported(vx, ncols) || !ggml_sycl_esimd::xmx_supported(vgate, ncols) ||
+        (uintptr_t) vy % 64 != 0 || stride_col_y_bytes % 16 != 0) {
+        return false;
+    }
+    // rows under 64 bytes use the variant without 2D block loads, it is not built for the wide tiles
+    const bool short_rows = ncols < 4 * QK_K;
+    if (short_rows && ncols_dst > 8) {
+        return false;
+    }
+    return ggml_sycl_mul_mat_vec_q_xmx_type(src0_type, short_rows, vx, vgate, (const char *) vy, dst, ncols, nrows,
+                                            ncols_dst, stride_col_y_bytes, stride_col_dst, (int) glu_op, stream);
+#else
+    GGML_UNUSED_VARS(device, src0_type, glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, stride_col_y_bytes,
+                     stride_col_dst, stream);
+    return false;
+#endif // GGML_SYCL_MMVQ_HAS_XMX
+}
+
 void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1,
                                 ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
                                 const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low,
@@ -2568,6 +2724,13 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens
     const int64_t ne00     = src0->ne[0];
     const int64_t row_diff = row_high - row_low;
 
+#ifdef GGML_SYCL_MMVQ_HAS_XMX
+    if (ggml_sycl_mul_mat_vec_q_xmx(ctx.device, src0, dst, src0_dd_i, src1_ddq_i, dst_dd_i, row_diff, src1_ncols,
+                                    src1_padded_col_size, stream)) {
+        return;
+    }
+#endif // GGML_SYCL_MMVQ_HAS_XMX
+
     int id;
     SYCL_CHECK(CHECK_TRY_ERROR(id = get_current_device_id()));
     const size_t q8_1_ts = sizeof(block_q8_1);

ggml/src/ggml-sycl/mmvq.hpp

@@ -80,6 +80,24 @@ bool ggml_sycl_mul_mat_vec_q_glu_reorder(
     dpct::queue_ptr    stream);
 
 
+// Fused dense-FFN GEMV on the XMX engines for reordered weights of one type, up * act(gate) for up to
+// GGML_SYCL_XMX_GLU_MAX_COLS (at most) activation columns. Returns false if it does not apply, the caller falls back.
+// vy: src1 quantized with quantize_and_reorder_q8_1_soa.
+bool ggml_sycl_mul_mat_vec_q_glu_xmx(
+    int                device,
+    enum ggml_type     src0_type,
+    enum ggml_glu_op   glu_op,
+    const void *       vx,                   // up weights
+    const void *       vgate,                // gate weights
+    const void *       vy,
+    float *            dst,
+    int                ncols,                // K, shared by both weights
+    int                nrows,                // output rows, i.e. weight ne[1]
+    int                ncols_dst,            // activation columns
+    int                stride_col_y_bytes,   // bytes between activation columns in vy
+    int                stride_col_dst,       // floats between output columns in dst
+    dpct::queue_ptr    stream);
+
 // Fused dense-FFN GEMV + GLU over the standard (non-reorder) layout; the gate and up
 // weights may carry different block types (q5_K / iq4_xs, mixed included).
 // vy: src1 quantized with plain quantize_q8_1 (padded rows). stride_col_y is in

ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp

@@ -9,7 +9,11 @@ static void * portable_aligned_malloc(size_t align, size_t size) {
 #ifdef _WIN32
     return _aligned_malloc(size, align);
 #else
-    return std::aligned_alloc(align, size);
+    void * ptr = nullptr;
+    if (posix_memalign(&ptr, align, size) != 0) {
+        return nullptr;
+    }
+    return ptr;
 #endif
 }
 

ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp

@@ -1655,6 +1655,7 @@ class ggml_webgpu_shader_lib {
 
                     switch (key.src_type) {
                         case GGML_TYPE_Q1_0:
+                        case GGML_TYPE_Q2_0:
                         case GGML_TYPE_Q4_0:
                         case GGML_TYPE_Q5_0:
                         case GGML_TYPE_Q8_0:
@@ -1699,7 +1700,7 @@ class ggml_webgpu_shader_lib {
                     } else if ((key.src_type >= GGML_TYPE_Q4_0 && key.src_type <= GGML_TYPE_Q8_1) ||
                                key.src_type == GGML_TYPE_IQ4_NL || key.src_type == GGML_TYPE_MXFP4) {
                         defines.push_back("BLOCK_SIZE=32u");
-                    } else if (key.src_type == GGML_TYPE_NVFP4) {
+                    } else if (key.src_type == GGML_TYPE_Q2_0 || key.src_type == GGML_TYPE_NVFP4) {
                         defines.push_back("BLOCK_SIZE=64u");
                     } else if (key.src_type >= GGML_TYPE_Q2_K) {
                         defines.push_back("BLOCK_SIZE=256u");

ggml/src/ggml-webgpu/ggml-webgpu.cpp

@@ -4395,6 +4395,7 @@ static bool ggml_backend_webgpu_device_supports_buft(ggml_backend_dev_t dev, ggm
 static bool ggml_webgpu_supported_qtype(ggml_type type) {
     switch (type) {
         case GGML_TYPE_Q1_0:
+        case GGML_TYPE_Q2_0:
         case GGML_TYPE_Q4_0:
         case GGML_TYPE_Q4_1:
         case GGML_TYPE_Q5_0:
@@ -4500,6 +4501,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
                             case GGML_TYPE_F16:
                             case GGML_TYPE_BF16:
                             case GGML_TYPE_Q1_0:
+                            case GGML_TYPE_Q2_0:
                             case GGML_TYPE_Q4_0:
                             case GGML_TYPE_Q4_1:
                             case GGML_TYPE_Q5_0:

ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl

@@ -51,6 +51,20 @@ fn copy_elements(src_base: u32, dst_base: u32, offset: u32) {
 }
 #endif
 
+#ifdef Q2_0
+fn copy_elements(src_base: u32, dst_base: u32, offset: u32) {
+    let block_byte_base = (src_base + offset) * 18u;
+    let d = load_f16_as_f32_at_src(block_byte_base);
+    for (var j = 0u; j < 4u; j++) {
+        let q_packed = load_u32_at_src(block_byte_base + 2u + j * 4u);
+        for (var k = 0u; k < 16u; k++) {
+            let q = (q_packed >> (2u * k)) & 3u;
+            dst[dst_base + offset * 64u + j * 16u + k] = (f32(q) - 1.0) * d;
+        }
+    }
+}
+#endif
+
 #ifdef Q4_0
 fn copy_elements(src_base: u32, dst_base: u32, offset: u32) {
     let block_byte_base = (src_base + offset) * 18; // Block stride: 18 bytes

ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl

@@ -111,6 +111,29 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
 }
 #endif // INIT_SRC0_SHMEM_Q1_0
 
+#ifdef INIT_SRC0_SHMEM_Q2_0
+const BLOCK_SIZE = 64u;
+const BLOCK_SIZE_BYTES = 18u;
+const NQ = 16u;
+
+fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u32) {
+    for (var i = thread_id * NQ; i < TILE_SRC0_SHMEM; i += TOTAL_WORKGROUP_SIZE * NQ) {
+        let global_m = offset_m + i / TILE_K;
+        let global_k = k_outer + i % TILE_K;
+        if (global_m < params.m && global_k < params.k) {
+            let block_idx = batch_offset + global_m * params.stride_01 + global_k / BLOCK_SIZE;
+            let block_byte_base = block_idx * BLOCK_SIZE_BYTES;
+            let d = load_f16_at_src0(block_byte_base);
+            let q_packed = load_u32_at_src0(block_byte_base + 2u + (global_k % BLOCK_SIZE) / 4u);
+            for (var k = 0u; k < NQ; k++) {
+                let q = (q_packed >> (2u * k)) & 3u;
+                shmem[i + k] = (f16(q) - 1.0) * d;
+            }
+        }
+    }
+}
+#endif // INIT_SRC0_SHMEM_Q2_0
+
 // legacy-quants
 #if defined(INIT_SRC0_SHMEM_Q4_0) || defined(INIT_SRC0_SHMEM_Q4_1) || defined(INIT_SRC0_SHMEM_Q5_0) || defined(INIT_SRC0_SHMEM_Q5_1) || defined(INIT_SRC0_SHMEM_Q8_0) || defined(INIT_SRC0_SHMEM_Q8_1) || defined(INIT_SRC0_SHMEM_MXFP4)
 const BLOCK_SIZE = 32u;

ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl

@@ -119,6 +119,46 @@ fn accumulate_vec_dot(thread_id: u32, row_base: u32, src0_batch_offset: u32, src
 }
 #endif
 
+#ifdef MUL_ACC_Q2_0
+#define BLOCK_SIZE 64
+#define BLOCK_SIZE_BYTES 18
+#define THREADS_PER_BLOCK 4
+#define ELEMS_PER_THREAD (BLOCK_SIZE/THREADS_PER_BLOCK)
+fn accumulate_vec_dot(thread_id: u32, row_base: u32, src0_batch_offset: u32, src1_idx_base: u32) -> array<array<f32, OUTPUTS_PER_WG>, NUM_COLS> {
+    var acc: array<array<f32, OUTPUTS_PER_WG>, NUM_COLS>;
+
+    let num_blocks = params.k / BLOCK_SIZE;
+    let thread_within_block = thread_id % THREADS_PER_BLOCK;
+    for (var block = thread_id / THREADS_PER_BLOCK; block < num_blocks; block += WG_SIZE / THREADS_PER_BLOCK) {
+        let x_base = src1_idx_base + block * BLOCK_SIZE + thread_within_block * ELEMS_PER_THREAD;
+        var x_block: array<array<f32, ELEMS_PER_THREAD>, NUM_COLS>;
+        for (var col = 0u; col < NUM_COLS; col++) {
+            for (var i = 0u; i < ELEMS_PER_THREAD; i++) {
+                x_block[col][i] = f32(SRC1[x_base + col * params.stride_11 + i]);
+            }
+        }
+        for (var row = 0u; row < OUTPUTS_PER_WG; row++) {
+            let output_row = row_base + row;
+            if (output_row < params.m) {
+                let block_byte_base = (src0_batch_offset + output_row * params.stride_01 + block) * BLOCK_SIZE_BYTES;
+                let d = f32(load_f16_at_src0(block_byte_base));
+                let q_packed = load_u32_at_src0(block_byte_base + 2u + 4u * thread_within_block);
+                for (var col = 0u; col < NUM_COLS; col++) {
+                    var row_sum = 0.0;
+                    for (var i = 0u; i < ELEMS_PER_THREAD; i++) {
+                        let q = (q_packed >> (2u * i)) & 3u;
+                        row_sum += (f32(q) - 1.0) * x_block[col][i];
+                    }
+                    acc[col][row] += d * row_sum;
+                }
+            }
+        }
+    }
+
+    return acc;
+}
+#endif
+
 #ifdef MUL_ACC_Q4_0
 #define BLOCK_SIZE 32
 #define BLOCK_SIZE_BYTES 18

gguf-py/gguf/constants.py

@@ -371,6 +371,7 @@ class Keys:
         DNA_AUTO_TAGS        = "tokenizer.ggml.dna_auto_tags"
         ADD_PREFIX           = "tokenizer.ggml.add_space_prefix"
         REMOVE_EXTRA_WS      = "tokenizer.ggml.remove_extra_whitespaces"
+        CLEAN_SPACES         = "tokenizer.ggml.clean_spaces"
         PRECOMPILED_CHARSMAP = "tokenizer.ggml.precompiled_charsmap"
         SUPPRESS_TOKENS      = "tokenizer.ggml.suppress_tokens"
         HF_JSON              = "tokenizer.huggingface.json"
@@ -382,6 +383,8 @@ class Keys:
         NORMALIZER_NFC           = "tokenizer.ggml.normalizer.nfc"
         NORMALIZER_LOWERCASE     = "tokenizer.ggml.normalizer.lowercase"
         NORMALIZER_STRIP_ACCENTS = "tokenizer.ggml.normalizer.strip_accents"
+        # Metaspace constants
+        METASPACE_PREPEND_SCHEME = "tokenizer.ggml.metaspace.prepend_scheme"
         # FIM/Infill special tokens constants
         FIM_PRE_ID           = "tokenizer.ggml.fim_pre_token_id"
         FIM_SUF_ID           = "tokenizer.ggml.fim_suf_token_id"
@@ -577,6 +580,7 @@ class MODEL_ARCH(IntEnum):
     SENSENOVA_U1     = auto()
     QWEN3            = auto()
     QWEN3MOE         = auto()
+    FLINT            = auto()
     QWEN3NEXT        = auto()
     HRM_TEXT         = auto()
     BEAM             = auto()
@@ -1394,6 +1398,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.QWEN2VL:          "qwen2vl",
     MODEL_ARCH.SENSENOVA_U1:     "sensenova_u1",
     MODEL_ARCH.QWEN3:            "qwen3",
+    MODEL_ARCH.FLINT:            "flint",
     MODEL_ARCH.QWEN3MOE:         "qwen3moe",
     MODEL_ARCH.QWEN3NEXT:        "qwen3next",
     MODEL_ARCH.HRM_TEXT:         "hrm_text",
@@ -2974,6 +2979,33 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_DOWN_EXP,
         MODEL_TENSOR.FFN_UP_EXP,
     ],
+
+    MODEL_ARCH.FLINT: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_QKV,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_Q_NORM,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_K_NORM,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_GATE_INP,
+        MODEL_TENSOR.FFN_GATE_EXP,
+        MODEL_TENSOR.FFN_DOWN_EXP,
+        MODEL_TENSOR.FFN_UP_EXP,
+        MODEL_TENSOR.ATTN_SINKS,
+        MODEL_TENSOR.FFN_GATE,
+        MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_GATE_SHEXP,
+        MODEL_TENSOR.FFN_UP_SHEXP,
+        MODEL_TENSOR.FFN_DOWN_SHEXP,
+        MODEL_TENSOR.FFN_EXP_PROBS_B,
+    ],
     MODEL_ARCH.QWEN3NEXT: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT_NORM,
@@ -6420,6 +6452,7 @@ class VisionProjectorType:
     MINIMAXM3      = "minimax_m3"
     MINICPMV4_6    = "minicpmv4_6"
     MINICPMV4_7    = "minicpmv4_7"
+    MINICPM_OCR_1_0 = "minicpm_ocr_1_0"
     GRANITE_SPEECH = "granite_speech"  # audio
     MIMOVL         = "mimovl"
     MIMO_AUDIO     = "mimo_audio"

gguf-py/gguf/gguf_writer.py

@@ -1326,6 +1326,9 @@ class GGUFWriter:
     def add_remove_extra_whitespaces(self, value: bool) -> None:
         self.add_bool(Keys.Tokenizer.REMOVE_EXTRA_WS, value)
 
+    def add_clean_spaces(self, value: bool) -> None:
+        self.add_bool(Keys.Tokenizer.CLEAN_SPACES, value)
+
     def add_precompiled_charsmap(self, charsmap: bytes) -> None:
         self.add_array(Keys.Tokenizer.PRECOMPILED_CHARSMAP, charsmap)
 
@@ -1374,6 +1377,9 @@ class GGUFWriter:
     def add_normalizer_strip_accents(self, value: bool) -> None:
         self.add_bool(Keys.Tokenizer.NORMALIZER_STRIP_ACCENTS, value)
 
+    def add_metaspace_prepend_scheme(self, value: str) -> None:
+        self.add_string(Keys.Tokenizer.METASPACE_PREPEND_SCHEME, value)
+
     def add_eot_token_id(self, id: int) -> None:
         self.add_uint32(Keys.Tokenizer.EOT_ID, id)
 

gguf-py/gguf/tensor_mapping.py

@@ -429,6 +429,7 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.ATTN_SINKS: (
+            "model.layers.{bid}.self_attn.softmax_offset", # flint
             "model.layers.{bid}.self_attn.sinks", # openai-moe
             "model.layers.{bid}.self_attn.attention_sink_bias", # mimov2
             "model.layers.{bid}.self_attn.learnable_sink_param", # hy-v4

gguf-py/gguf/vocab.py

@@ -55,6 +55,8 @@ class SpecialVocab:
     normalizer_nfc: bool | None
     normalizer_lowercase: bool | None
     normalizer_strip_accents: bool | None
+    clean_spaces: bool | None
+    metaspace_prepend_scheme: str | None
 
     def __init__(
         self, path: str | os.PathLike[str], load_merges: bool = False,
@@ -70,6 +72,8 @@ class SpecialVocab:
         self.normalizer_nfc = None
         self.normalizer_lowercase = None
         self.normalizer_strip_accents = None
+        self.clean_spaces = None
+        self.metaspace_prepend_scheme = None
         if special_token_types is not None:
             self.special_token_types = special_token_types
         else:
@@ -120,6 +124,14 @@ class SpecialVocab:
             if not quiet:
                 logger.info(f'Setting normalizer_strip_accents to {self.normalizer_strip_accents}')
             gw.add_normalizer_strip_accents(self.normalizer_strip_accents)
+        if self.clean_spaces is not None:
+            if not quiet:
+                logger.info(f'Setting clean_spaces to {self.clean_spaces}')
+            gw.add_clean_spaces(self.clean_spaces)
+        if self.metaspace_prepend_scheme is not None:
+            if not quiet:
+                logger.info(f'Setting metaspace_prepend_scheme to {self.metaspace_prepend_scheme}')
+            gw.add_metaspace_prepend_scheme(self.metaspace_prepend_scheme)
 
     def _load(self, path: Path) -> None:
         self._try_load_from_tokenizer_json(path)
@@ -188,6 +200,16 @@ class SpecialVocab:
             for norm in normalizer.get('normalizers', []):
                 self._parse_normalizer(norm)
 
+    def _parse_pre_tokenizer(self, pre_tokenizer: dict) -> None:
+        # ref: https://huggingface.co/docs/tokenizers/api/pre-tokenizers
+        pre_tokenizer_type = pre_tokenizer.get('type')
+        if pre_tokenizer_type == 'Metaspace':
+            if 'prepend_scheme' in pre_tokenizer:
+                self.metaspace_prepend_scheme = pre_tokenizer['prepend_scheme']
+        elif pre_tokenizer_type == 'Sequence':
+            for pre in pre_tokenizer.get('pretokenizers', []):
+                self._parse_pre_tokenizer(pre)
+
     def _try_load_from_tokenizer_json(self, path: Path) -> bool:
         tokenizer = None
         tokenizer_file = path / 'tokenizer.json'
@@ -223,6 +245,8 @@ class SpecialVocab:
             # Parse normalizer configuration (e.g. Lowercase) into metadata
             if normalizer := tokenizer.get('normalizer'):
                 self._parse_normalizer(normalizer)
+            if pre_tokenizer := tokenizer.get('pre_tokenizer'):
+                self._parse_pre_tokenizer(pre_tokenizer)
             added_tokens = tokenizer.get('added_tokens', {})
         else:
             added_tokens = {}
@@ -345,6 +369,10 @@ class SpecialVocab:
             self.chat_template = chat_template
         else:
             logger.warning(f'Bad type for chat_template field in {tokenizer_config_file!r} - ignoring')
+        # decode clean-up of spaces before punctuation (transformers clean_up_tokenization_spaces)
+        clean_spaces = tokenizer_config.get('clean_up_tokenization_spaces')
+        if isinstance(clean_spaces, bool):
+            self.clean_spaces = clean_spaces
         for typ in self.special_token_types:
             add_entry = tokenizer_config.get(f'add_{typ}_token')
             if isinstance(add_entry, bool):

src/llama-adapter.cpp

@@ -334,9 +334,10 @@ static void llama_adapter_lora_init_impl(llama_model & model, FILE * file, llama
         if (!model_tensor) {
             throw std::runtime_error("LoRA tensor '" + name + "' does not exist in base model (hint: maybe wrong base model?)");
         }
-        // the LoRA delta of a Hadamard-folded weight reads the transformed input, so the result is wrong
-        if (model.hdmd.weight_blocks.count(name) || model.hdmd.inverse_blocks.count(name)) {
-            throw std::runtime_error("LoRA tensor '" + name + "' targets a prism.hadamard folded weight, which is not supported");
+        // build_lora_mm feeds a folded weight's LoRA delta the primal input, so folded matmul weights are fine;
+        // a latent lookup table (token_embd, also the head when the output is tied) stores rotated rows, refuse it
+        if (model.hdmd.inverse_blocks.count(name)) {
+            throw std::runtime_error("LoRA tensor '" + name + "' targets a prism.hadamard latent lookup table, which is not supported");
         }
 
         auto * buft = ggml_backend_buffer_get_type(model_tensor->buffer);

src/llama-arch.cpp

@@ -35,6 +35,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_QWEN2VL,          "qwen2vl"          },
     { LLM_ARCH_SENSENOVA_U1,     "sensenova_u1"     },
     { LLM_ARCH_QWEN3,            "qwen3"            },
+    { LLM_ARCH_FLINT,            "flint"            },
     { LLM_ARCH_QWEN3MOE,         "qwen3moe"         },
     { LLM_ARCH_QWEN3NEXT,        "qwen3next"        },
     { LLM_ARCH_QWEN3VL,          "qwen3vl"          },
@@ -436,6 +437,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
     { LLM_KV_TOKENIZER_DNA_AUTO_TAGS,            "tokenizer.ggml.dna_auto_tags"            },
     { LLM_KV_TOKENIZER_ADD_PREFIX,               "tokenizer.ggml.add_space_prefix"         },
     { LLM_KV_TOKENIZER_REMOVE_EXTRA_WS,          "tokenizer.ggml.remove_extra_whitespaces" },
+    { LLM_KV_TOKENIZER_CLEAN_SPACES,             "tokenizer.ggml.clean_spaces"             },
     { LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP,     "tokenizer.ggml.precompiled_charsmap"     },
     { LLM_KV_TOKENIZER_HF_JSON,                  "tokenizer.huggingface.json"              },
     { LLM_KV_TOKENIZER_RWKV,                     "tokenizer.rwkv.world"                    },
@@ -443,6 +445,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
     { LLM_KV_TOKENIZER_NORMALIZER_NFC,           "tokenizer.ggml.normalizer.nfc"           },
     { LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE,     "tokenizer.ggml.normalizer.lowercase"     },
     { LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, "tokenizer.ggml.normalizer.strip_accents" },
+    { LLM_KV_TOKENIZER_METASPACE_PREPEND_SCHEME, "tokenizer.ggml.metaspace.prepend_scheme" },
     { LLM_KV_TOKENIZER_FIM_PRE_ID,               "tokenizer.ggml.fim_pre_token_id"         },
     { LLM_KV_TOKENIZER_FIM_SUF_ID,               "tokenizer.ggml.fim_suf_token_id"         },
     { LLM_KV_TOKENIZER_FIM_MID_ID,               "tokenizer.ggml.fim_mid_token_id"         },

src/llama-arch.h

@@ -41,6 +41,7 @@ enum llm_arch {
     LLM_ARCH_SENSENOVA_U1,
     LLM_ARCH_QWEN3,
     LLM_ARCH_QWEN3MOE,
+    LLM_ARCH_FLINT,
     LLM_ARCH_QWEN3NEXT,
     LLM_ARCH_QWEN3VL,
     LLM_ARCH_QWEN3VLMOE,
@@ -399,6 +400,7 @@ enum llm_kv {
     LLM_KV_TOKENIZER_DNA_AUTO_TAGS,
     LLM_KV_TOKENIZER_ADD_PREFIX,
     LLM_KV_TOKENIZER_REMOVE_EXTRA_WS,
+    LLM_KV_TOKENIZER_CLEAN_SPACES,
     LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP,
     LLM_KV_TOKENIZER_HF_JSON,
     LLM_KV_TOKENIZER_RWKV,
@@ -406,6 +408,7 @@ enum llm_kv {
     LLM_KV_TOKENIZER_NORMALIZER_NFC,
     LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE,
     LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS,
+    LLM_KV_TOKENIZER_METASPACE_PREPEND_SCHEME,
     LLM_KV_TOKENIZER_FIM_PRE_ID,
     LLM_KV_TOKENIZER_FIM_SUF_ID,
     LLM_KV_TOKENIZER_FIM_MID_ID,

src/llama-graph.cpp

@@ -1630,8 +1630,9 @@ ggml_tensor * llm_graph_context::build_lora_mm(
           ggml_tensor * w,
           ggml_tensor * cur,
           ggml_tensor * w_s) const {
-    cur = build_hadamard_input(w, cur);
-    ggml_tensor * res = ggml_mul_mat(ctx0, w, cur);
+    // a Hadamard-folded weight reads the transformed input; a LoRA delta lives in the primal basis and reads `cur`
+    ggml_tensor * cur_mm = build_hadamard_input(w, cur);
+    ggml_tensor * res = ggml_mul_mat(ctx0, w, cur_mm);
 
     if (prec_policy) {
         prec_policy->apply(res);
@@ -1684,7 +1685,10 @@ llm_graph_amax_src1 llm_graph_context::build_amax_src1_f16(
     constexpr float k_ln_2   = 0.6931471805599453f;
 
     ggml_tensor * e;
-    e = ggml_scale(ctx0, amax, 1.0f/32768.0f);
+    // note: this can technically be:
+    // e = ggml_scale(ctx0, amax, 1.0f/32768.0f);
+    // but we leave x32 headroom for the F16 sums of 32 values in Q8 activation blocks
+    e = ggml_scale(ctx0, amax, 1.0f/1024.0f);
     e = ggml_log  (ctx0, e);
     e = ggml_scale(ctx0, e, k_log2_e);
     e = ggml_ceil (ctx0, e);
@@ -1706,11 +1710,12 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
           ggml_tensor * ids,
           ggml_tensor * w_s,
           ggml_tensor * slots) const {
-    cur = build_hadamard_input(w, cur);
+    // as in build_lora_mm: the folded weight reads the transformed input, the LoRA delta reads `cur`
+    ggml_tensor * cur_mm = build_hadamard_input(w, cur);
     // the experts in the MoE cache are selected by their slots
     ggml_tensor * res = slots == nullptr ?
-        ggml_mul_mat_id(ctx0, w, cur, ids) :
-        ggml_mul_mat_id(ctx0, moe_cache->get_experts(w), cur, slots);
+        ggml_mul_mat_id(ctx0, w, cur_mm, ids) :
+        ggml_mul_mat_id(ctx0, moe_cache->get_experts(w), cur_mm, slots);
 
     if (prec_policy) {
         prec_policy->apply(res);
@@ -2505,7 +2510,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
         //       ref: https://github.com/ggml-org/llama.cpp/pull/30656#issuecomment-6269597922
         llm_graph_amax_src1 amax_src1 = { nullptr, nullptr };
 
-        if (arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MELLUM) {
+        if (arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MELLUM || arch == LLM_ARCH_FLINT) {
             amax_src1 = build_amax_src1_f16(cur);
             cur = amax_src1.res;
         }

src/llama-graph.h

@@ -1122,7 +1122,7 @@ struct llm_graph_context {
               ggml_tensor * slots = nullptr) const;
 
     // rescale `cur` (the src1 of a matmul) by a power of two, computed from its maximum absolute
-    // when the values already fit in F16, the factors are exactly 1.0f
+    // keep both individual values and Q8 activation block sums within F16 range
     llm_graph_amax_src1 build_amax_src1_f16(
               ggml_tensor * cur) const;
 

src/llama-kv-cache.cpp

@@ -882,7 +882,10 @@ bool llama_kv_cache::update(llama_context * lctx, bool do_shift, const stream_co
         LLAMA_LOG_DEBUG("%s: applying K-shift\n", __func__);
 
         // apply K-shift if needed
-        if (hparams.rope_type != LLAMA_ROPE_TYPE_NONE) {
+        if (hparams.rope_type != LLAMA_ROPE_TYPE_NONE &&
+                std::any_of(layers.begin(), layers.end(), [&](const auto & layer) {
+                    return hparams.has_rope(layer.il);
+                })) {
             ggml_backend_sched_reset(sched);
 
             auto * res = lctx->get_gf_res_reserve();

src/llama-memory-hybrid-idx.cpp

@@ -758,7 +758,7 @@ void llama_memory_hybrid_idx_context::set_input_kpool(ggml_tensor * pool_cells,
     GGML_ASSERT(tail_idxs->ne[0] == (int64_t) kpool - 1 && tail_idxs->ne[1] == (int64_t) n_tokens);
     GGML_ASSERT(pool_idxs->ne[0] == (int64_t) kpool && pool_idxs->ne[1] == (int64_t) n_pool);
     GGML_ASSERT(ggml_backend_buffer_is_host(new_pool_idxs->buffer));
-    GGML_ASSERT(new_pool_idxs->ne[0] == (int64_t) kpool && new_pool_idxs->ne[1] == (int64_t) n_new_g);
+    GGML_ASSERT(ggml_nelements(new_pool_idxs) == (int64_t) kpool*n_new_g);
     // the graph always scatters the fresh pooled keys back into the cache, see build_qsa_sel
     GGML_ASSERT(new_pool_rep != nullptr && ggml_backend_buffer_is_host(new_pool_rep->buffer));
     GGML_ASSERT(new_pool_rep->ne[0] == (int64_t) n_new_g);

src/llama-model-saver.cpp

@@ -424,6 +424,7 @@ void llama_model_saver::add_kv_from_model() {
     add_kv(LLM_KV_TOKENIZER_ADD_SEP,                 vocab.get_add_sep());
     add_kv(LLM_KV_TOKENIZER_ADD_PREFIX,              vocab.get_add_space_prefix());
     add_kv(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS,         vocab.get_remove_extra_whitespaces());
+    add_kv(LLM_KV_TOKENIZER_CLEAN_SPACES,            vocab.get_clean_spaces());
     add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP,    vocab.get_precompiled_charsmap());
     // add_kv(LLM_KV_TOKENIZER_HF_JSON,                 ???);
     // add_kv(LLM_KV_TOKENIZER_RWKV,                    ???);

src/llama-model.cpp

@@ -114,6 +114,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_sensenova_u1(params);
         case LLM_ARCH_QWEN3:
             return new llama_model_qwen3(params);
+        case LLM_ARCH_FLINT:
+            return new llama_model_flint(params);
         case LLM_ARCH_QWEN3MOE:
             return new llama_model_qwen3moe(params);
         case LLM_ARCH_QWEN3VL:
@@ -1966,12 +1968,20 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
             if (buf == nullptr) {
                 throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft)));
             }
-            // a multi_buffer has no base to lock, its parts are usually pinned memory (host buffer types with a max size)
-            if (use_mlock && ggml_backend_buffer_is_host(buf) && ggml_backend_buffer_get_base(buf) != nullptr) {
-                pimpl->mlock_bufs.emplace_back(new llama_mlock);
-                auto & mlock_buf = pimpl->mlock_bufs.back();
-                mlock_buf->init   (ggml_backend_buffer_get_base(buf));
-                mlock_buf->grow_to(ggml_backend_buffer_get_size(buf));
+            // lock the buffers that hold the tensors: a multi_buffer has no base, and its parts are pageable memory when pinning fails
+            if (use_mlock && !ml.no_alloc && ggml_backend_buffer_is_host(buf)) {
+                std::vector<ggml_backend_buffer_t> lock_bufs;
+                for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
+                    if (t->buffer != nullptr && std::find(lock_bufs.begin(), lock_bufs.end(), t->buffer) == lock_bufs.end()) {
+                        lock_bufs.push_back(t->buffer);
+                    }
+                }
+                for (ggml_backend_buffer_t lock_buf : lock_bufs) {
+                    pimpl->mlock_bufs.emplace_back(new llama_mlock);
+                    auto & mlock_buf = pimpl->mlock_bufs.back();
+                    mlock_buf->init   (ggml_backend_buffer_get_base(lock_buf));
+                    mlock_buf->grow_to(ggml_backend_buffer_get_size(lock_buf));
+                }
             }
             bufs.emplace_back(buf);
         }
@@ -3571,6 +3581,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_QWEN2MOE:
         case LLM_ARCH_QWEN3:
         case LLM_ARCH_VLLM_DECISION2:
+        case LLM_ARCH_FLINT:
         case LLM_ARCH_QWEN3MOE:
         case LLM_ARCH_LLADA_MOE:
         case LLM_ARCH_RND1:

src/llama-model.h

@@ -779,8 +779,9 @@ struct llama_model {
     struct ggml_tensor * per_layer_proj_norm  = nullptr;
 
     // eagle3 / dflash feature fusion layer
-    struct ggml_tensor * fc   = nullptr;
-    struct ggml_tensor * fc_s = nullptr;
+    struct ggml_tensor * fc      = nullptr;
+    struct ggml_tensor * fc_s    = nullptr;
+    struct ggml_tensor * fc_in_s = nullptr;
     struct ggml_tensor * d2t = nullptr;  // draft to target vocabulary mapping
 
     // dspark

src/llama-vocab.cpp

@@ -281,6 +281,13 @@ struct llm_tokenizer_bpe : llm_tokenizer {
     llm_tokenizer_bpe(const llama_vocab & vocab) {
         GGML_ASSERT(vocab.get_type() == LLAMA_VOCAB_TYPE_BPE);
         switch (vocab.get_pre_type()) {
+            case LLAMA_VOCAB_PRE_TYPE_FLINT:
+                regex_exprs = {
+                    "\\p{N}{1,3}",
+                    "[一-龥぀-ゟ゠-ヿ]+",
+                    "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?| ?[\\p{P}\\p{S}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+                };
+                break;
             case LLAMA_VOCAB_PRE_TYPE_LLAMA3:
                 regex_exprs = {
                     // original regex from tokenizer.json
@@ -536,6 +543,7 @@ struct llm_tokenizer_bpe : llm_tokenizer {
                 byte_encode = false; // uses raw UTF-8, not GPT-2 byte encoding
                 break;
             case LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE:
+            case LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL:
                 // Sarvam uses SPM-style BPE (same shape as Gemma4): spaces replaced with U+2581
                 // by the normalizer, BPE merges over the whole text on raw UTF-8.
                 regex_exprs = {
@@ -2090,6 +2098,8 @@ struct llama_vocab::impl {
     // Tokenizer normalizer options
     llama_vocab::normalizer_options normalizer_opts;
 
+    llama_metaspace_prepend_scheme metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_NEVER;
+
     std::unordered_map<std::string, llama_token> token_to_id;
     std::vector<token_data>                      id_to_token;
 
@@ -2417,24 +2427,32 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                 LLAMA_LOG_WARN("%s: ************************************        \n", __func__);
                 LLAMA_LOG_WARN("%s:                                             \n", __func__);
                 pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+            } else if (tokenizer_pre == "flint") {
+                pre_type = LLAMA_VOCAB_PRE_TYPE_FLINT;
+                clean_spaces = false;
             } else if (tokenizer_pre == "default") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
             } else if (tokenizer_pre == "minicpm5") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_MINICPM5;
                 ignore_merges = true;
+                clean_spaces = false;
             } else if (
                     tokenizer_pre == "llama3"   ||
                     tokenizer_pre == "llama-v3" ||
                     tokenizer_pre == "llama-bpe"||
                     tokenizer_pre == "falcon3"  ||
                     tokenizer_pre == "falcon-h1" ||
-                    tokenizer_pre == "pixtral"  ||
                     tokenizer_pre == "midm-2.0" ||
-                    tokenizer_pre == "lfm2"     ||
                     tokenizer_pre == "jina-v5-nano") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_LLAMA3;
                 ignore_merges = true;
                 add_bos = true;
+            } else if (
+                    tokenizer_pre == "lfm2") {
+                pre_type = LLAMA_VOCAB_PRE_TYPE_LLAMA3;
+                ignore_merges = true;
+                add_bos = true;
+                clean_spaces = false;
             } else if (
                     tokenizer_pre == "deepseek-llm") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM;
@@ -2485,6 +2503,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                     tokenizer_pre == "granite-embed-multi-311m") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_GEMMA4;
                 escape_whitespaces = true;
+                clean_spaces = false;
             } else if (
                     tokenizer_pre == "mmbert") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_MMBERT;
@@ -2531,6 +2550,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
             } else if (
                 tokenizer_pre == "dbrx") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_DBRX;
+                clean_spaces = false;
             } else if (
                 tokenizer_pre == "smaug-bpe") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_SMAUG;
@@ -2542,12 +2562,14 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                 tokenizer_pre == "chatglm-bpe") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_CHATGLM4;
                 special_bos_id = LLAMA_TOKEN_NULL;
+                clean_spaces = false;
             } else if (
                 tokenizer_pre == "glm4" ||
                 tokenizer_pre == "glm5") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_CHATGLM4;
                 special_bos_id = LLAMA_TOKEN_NULL;
                 ignore_merges = true;
+                clean_spaces = false;
             } else if (
                 tokenizer_pre == "viking") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_VIKING;
@@ -2556,7 +2578,8 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                 tokenizer_pre == "jais") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_JAIS;
             } else if (
-                tokenizer_pre == "tekken") {
+                tokenizer_pre == "tekken" ||
+                tokenizer_pre == "pixtral") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_TEKKEN;
                 clean_spaces = false;
                 ignore_merges = true;
@@ -2580,6 +2603,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
             } else if (
                 tokenizer_pre == "exaone4") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_GPT2;
+                clean_spaces = false;
             } else if (
                 tokenizer_pre == "exaone-moe") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE;
@@ -2591,6 +2615,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
             } else if (
                 tokenizer_pre == "minerva-7b") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_MINERVA;
+            } else if (
+                tokenizer_pre == "domyn-small") {
+                pre_type = LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL;
+                escape_whitespaces = true;
             } else if (
                 tokenizer_pre == "megrez") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_QWEN2;
@@ -2680,6 +2708,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
             } else if (
                 tokenizer_pre == "mellum2") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_MELLUM2;
+                clean_spaces = false;
             } else if (
                 tokenizer_pre == "k2-horizon") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_K2_HORIZON;
@@ -2716,6 +2745,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
 
         ml.get_key(LLM_KV_TOKENIZER_ADD_PREFIX,      add_space_prefix,         false);
         ml.get_key(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, remove_extra_whitespaces, false);
+        ml.get_key(LLM_KV_TOKENIZER_CLEAN_SPACES,    clean_spaces,             false);
     }
 
     const int token_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_LIST).c_str());
@@ -2933,6 +2963,24 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
         normalizer_opts.strip_accents = normalizer_opts.lowercase;
         ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, normalizer_opts.strip_accents, false);
 
+        {
+            std::string scheme;
+            if (ml.get_key(LLM_KV_TOKENIZER_METASPACE_PREPEND_SCHEME, scheme, false)) {
+                if (scheme == "never") {
+                    metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_NEVER;
+                } else if (scheme == "first") {
+                    metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_FIRST;
+                } else if (scheme == "always") {
+                    metaspace_prepend_scheme = LLAMA_METASPACE_PREPEND_SCHEME_ALWAYS;
+                } else {
+                    throw std::runtime_error(format("unknown metaspace prepend_scheme: '%s'", scheme.c_str()));
+                }
+                if (type == LLAMA_VOCAB_TYPE_BPE && metaspace_prepend_scheme == LLAMA_METASPACE_PREPEND_SCHEME_ALWAYS) {
+                    throw std::runtime_error("metaspace prepend_scheme 'always' is not supported for BPE vocab");
+                }
+            }
+        }
+
         // suppress tokens
         {
             const int suppress_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SUPPRESS_TOKENS).c_str());
@@ -3147,7 +3195,6 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
         // this is currently determined based on the token text, which is obviously not ideal
         // ref: https://github.com/ggml-org/llama.cpp/issues/9606
         special_eog_ids.clear();
-
         if (special_fim_pad_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_fim_pad_id) == 0) {
             special_eog_ids.insert(special_fim_pad_id);
         }
@@ -3181,6 +3228,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                     || t.first == "[EOT]" // Kimi-K2
                     || t.first == "[EOS]" // Kimi-K2
                     || t.first == "<|end_of_text|>"
+                    || (pre_type == LLAMA_VOCAB_PRE_TYPE_FLINT && (t.first == "<|user|>" || t.first == "<|observation|>"))
                     || t.first == "<end_of_utterance>" // smoldocling
                     || t.first == "<eos>"            // gemma4
                     || t.first == "<turn|>"          // gemma4
@@ -3720,6 +3768,7 @@ std::vector<llama_token> llama_vocab::impl::tokenize(
                 // tokenizer.encode('', add_special_tokens=True)  returns [1]
                 // tokenizer.encode('', add_special_tokens=False) returns []
 
+                // TODO: honor metaspace_prepend_scheme (add_space_prefix currently behaves like "always")
                 bool is_prev_special = true;  // prefix with space if first token
 
                 if (add_special && add_bos) {
@@ -3790,6 +3839,13 @@ std::vector<llama_token> llama_vocab::impl::tokenize(
                             llama_escape_whitespace(text);
                         }
 
+                        // Metaspace prepend_scheme "first": add U+2581 only at the start of the input
+                        static const std::string escaped_space = "\xE2\x96\x81";
+                        if (metaspace_prepend_scheme == LLAMA_METASPACE_PREPEND_SCHEME_FIRST &&
+                            &fragment == &fragment_buffer.front() && text.rfind(escaped_space, 0) != 0) {
+                            text.insert(0, escaped_space);
+                        }
+
 #ifdef PRETOKENIZERDEBUG
                         LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", text.length(), fragment.offset, fragment.length, text.c_str());
 #endif
@@ -3837,6 +3893,7 @@ std::vector<llama_token> llama_vocab::impl::tokenize(
                     GGML_ASSERT(special_bos_id != LLAMA_TOKEN_NULL);
                     output.push_back(special_bos_id);
                 }
+                // TODO: honor metaspace_prepend_scheme
                 llm_tokenizer_ugm_session session(vocab, *static_cast<const llm_tokenizer_ugm *>(tokenizer.get()));
 
                 for (const auto & fragment : fragment_buffer) {
@@ -4120,10 +4177,17 @@ int32_t llama_vocab::impl::detokenize(
         }
     }
 
+    // Metaspace prepend_scheme "first": remove the leading space of the first rendered piece, so a skipped BOS does not consume it
+    // TODO: honor metaspace_prepend_scheme for the other vocab types
+    bool remove_first_space = type == LLAMA_VOCAB_TYPE_BPE && metaspace_prepend_scheme == LLAMA_METASPACE_PREPEND_SCHEME_FIRST;
+
     for (int32_t i = 0; i < n_tokens; ++i) {
         GGML_ASSERT(avail >= 0);
-        int32_t n_chars = token_to_piece(tokens[i], text, avail, remove_space, unparse_special);
+        int32_t n_chars = token_to_piece(tokens[i], text, avail, remove_space || remove_first_space, unparse_special);
         remove_space = false;
+        if (unparse_special || !(token_get_attr(tokens[i]) & (LLAMA_TOKEN_ATTR_UNKNOWN | LLAMA_TOKEN_ATTR_CONTROL))) {
+            remove_first_space = false;
+        }
         if (n_chars < 0) {
             avail = 0;
             total -= n_chars;

src/llama-vocab.h

@@ -70,6 +70,15 @@ enum llama_vocab_pre_type {
     LLAMA_VOCAB_PRE_TYPE_UFAKZEKA          = 59,
     LLAMA_VOCAB_PRE_TYPE_MMBERT            = 60,
     LLAMA_VOCAB_PRE_TYPE_K2_HORIZON        = 61,
+    LLAMA_VOCAB_PRE_TYPE_FLINT             = 62,
+    LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL       = 63,
+};
+
+// HF Metaspace pre-tokenizer prepend_scheme
+enum llama_metaspace_prepend_scheme {
+    LLAMA_METASPACE_PREPEND_SCHEME_NEVER  = 0,
+    LLAMA_METASPACE_PREPEND_SCHEME_FIRST  = 1,
+    LLAMA_METASPACE_PREPEND_SCHEME_ALWAYS = 2,
 };
 
 struct LLM_KV;

src/models/clef.cpp

@@ -2,6 +2,7 @@
 
 #include "llama-ext.h"
 
+#include <algorithm>
 #include <cmath>
 
 void llama_model_clef::load_arch_hparams(llama_model_loader & ml) {

src/models/dflash.cpp

@@ -238,6 +238,7 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {
 
     fc              = create_tensor(tn(LLM_TENSOR_FC,              "weight"), { n_embd_inp, n_embd }, 0);
     fc_s            = create_tensor(tn(LLM_TENSOR_FC,              "scale"),  { 1 }, TENSOR_NOT_REQUIRED);
+    fc_in_s         = create_tensor(tn(LLM_TENSOR_FC,              "input_scale"), { 1 }, TENSOR_NOT_REQUIRED);
     output_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_OUTPUT_NORM, "weight"), { n_embd }, 0); // encoder hidden_norm (after fc)
     output_norm     = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM,    "weight"), { n_embd }, 0); // decoder final norm
 

src/models/flint.cpp

@@ -0,0 +1,194 @@
+#include "models.h"
+
+void llama_model_flint::load_arch_hparams(llama_model_loader & ml) {
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+    ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,   hparams.n_layer_dense_lead);
+    ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
+    ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,         hparams.n_expert_shared);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,        hparams.expert_weights_scale, false);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,         hparams.expert_weights_norm);
+    ml.get_key(LLM_KV_EXPERT_GATING_FUNC,          hparams.expert_gating_func);
+
+    ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);
+    ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+    ml.get_arr(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern);
+    ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
+    type = LLM_TYPE_UNKNOWN;
+}
+
+void llama_model_flint::load_arch_tensors(llama_model_loader &) {
+    LLAMA_LOAD_LOCALS;
+
+    const int64_t n_ff_exp        = hparams.n_ff_exp();
+
+    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, 0);
+
+    for (int i = 0; i < n_layer; ++i) {
+        auto & layer = layers[i];
+
+        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+
+        create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
+        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
+
+        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
+        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
+
+        layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, 0);
+
+        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+
+        if (i < (int) hparams.n_layer_dense_lead) {
+            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd,   n_ff}, 0);
+            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {  n_ff, n_embd}, 0);
+            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd,   n_ff}, 0);
+        } else {
+            layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
+            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
+
+            if (n_expert == 0) {
+                throw std::runtime_error("n_expert must be > 0");
+            }
+            if (n_expert_used == 0) {
+                throw std::runtime_error("n_expert_used must be > 0");
+            }
+
+            // MoE branch
+            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {  n_embd, n_ff_exp, n_expert}, 0);
+            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp,   n_embd, n_expert}, 0);
+            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {  n_embd, n_ff_exp, n_expert}, 0);
+
+            // Shared expert branch
+            layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, hparams.n_ff_shexp}, 0);
+            layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {        hparams.n_ff_shexp, n_embd}, 0);
+            layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   "weight", i), {n_embd, hparams.n_ff_shexp}, 0);
+        }
+    }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_flint::build_arch_graph(const llm_graph_params & params) const {
+    return std::make_unique<graph>(*this, params);
+}
+
+llama_model_flint::graph::graph(const llama_model & model, const llm_graph_params & params) :
+    llm_graph_context(params) {
+    const int64_t n_embd_head = hparams.n_embd_head_v();
+
+    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+    GGML_ASSERT(n_embd_head == n_rot);
+
+    ggml_tensor * cur;
+    ggml_tensor * inpL;
+
+    inpL = build_inp_embd(model.tok_embd);
+
+    // inp_pos - contains the positions
+    ggml_tensor * inp_pos = build_inp_pos();
+
+    auto * inp_attn = build_attn_inp_kv_iswa();
+
+    ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+    for (int il = 0; il < n_layer; ++il) {
+        res->t_layer_inp[il] = inpL;
+        ggml_tensor * inpSA = inpL;
+
+        // norm
+        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "attn_norm", il);
+
+        // self_attention
+        {
+            // compute Q and K and RoPE them
+            auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+                    n_embd_head, n_head, n_head_kv, il);
+
+            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);
+            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);
+            if (hparams.has_rope(il)) {
+                Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
+                        freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
+                Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
+                        freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
+            }
+            cb(Qcur, "Qcur", il);
+            cb(Kcur, "Kcur", il);
+            cb(Vcur, "Vcur", il);
+
+            cur = build_attn(inp_attn,
+                    model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
+                    Qcur, Kcur, Vcur, nullptr, model.layers[il].attn_sinks, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
+        }
+        if (il == n_layer - 1 && inp_out_ids) {
+            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);
+            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+        }
+        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+        cb(ffn_inp, "ffn_inp", il);
+
+        // MoE branch
+        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "ffn_norm", il);
+
+        if ((uint32_t) il < hparams.n_layer_dense_lead) {
+            cur = build_ffn(cur,
+                    model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s,
+                    model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s,
+                    model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s,
+                    NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);
+            cb(cur, "ffn_out", il);
+        } else {
+            ggml_tensor * moe_out = build_moe_ffn(cur,
+                model.layers[il].ffn_gate_inp,
+                model.layers[il].ffn_up_exps,
+                model.layers[il].ffn_gate_exps,
+                model.layers[il].ffn_down_exps,
+                model.layers[il].ffn_exp_probs_b,
+                n_expert, n_expert_used,
+                LLM_FFN_SILU, hparams.expert_weights_norm,
+                hparams.expert_weights_scale,
+                (llama_expert_gating_func_type) hparams.expert_gating_func,
+                il, nullptr, nullptr,
+                model.layers[il].ffn_up_exps_s,
+                model.layers[il].ffn_gate_exps_s,
+                model.layers[il].ffn_down_exps_s);
+            cb(moe_out, "ffn_moe_out", il);
+
+            {
+                ggml_tensor * ffn_shexp =
+                    build_ffn(cur,
+                        model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s,
+                        model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s,
+                        model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s,
+                        NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);
+                cb(ffn_shexp, "ffn_shexp", il);
+
+                cur = ggml_add(ctx0, moe_out, ffn_shexp);
+                cb(cur, "ffn_out", il);
+            }
+        }
+        cur = ggml_add(ctx0, cur, ffn_inp);
+
+        cur = build_cvec(cur, il);
+        cb(cur, "l_out", il);
+
+        // input for next layer
+        inpL = cur;
+    }
+    cur = inpL;
+
+    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
+
+    cb(cur, "result_norm", -1);
+    res->t_embd = cur;
+
+    // lm_head
+    cur = build_lora_mm(model.output, cur, model.output_s);
+
+    cb(cur, "result_output", -1);
+    res->t_logits = cur;
+}

src/models/glm5-next.cpp

@@ -271,10 +271,8 @@ public:
     ggml_tensor * pool_mask     = nullptr; // F32/F16 [n_pool, n_tokens]
     ggml_tensor * tail_idxs     = nullptr; // I32     [kpool - 1, n_tokens]
     ggml_tensor * sel_mask      = nullptr; // F32     [n_sel, 1, 1, n_tokens] 0 for live selection slots, -inf for dead ones
-    ggml_tensor * sel_mask_2d   = nullptr; // F32     [n_sel, n_tokens] view of sel_mask, shared by the layers
     // n_new is never below 1, see build_inp_kpool
-    ggml_tensor * new_pool_idxs = nullptr; // I32     [kpool, n_new]   members of the pools completed this ubatch
-    ggml_tensor * new_pool_flat = nullptr; // I32     [kpool*n_new]    view of new_pool_idxs, shared by the layers
+    ggml_tensor * new_pool_idxs = nullptr; // I32     [kpool*n_new]    members of the pools completed this ubatch
     ggml_tensor * new_pool_rep  = nullptr; // I64     [n_new]          cell to write each new pooled key into
 
     const llama_memory_hybrid_idx_context * mctx;
@@ -329,16 +327,13 @@ llama_model_glm5_next::llm_graph_input_kpool * llama_model_glm5_next::graph::bui
             ggml_set_input(inp->sel_mask);
             // set_input_kpool always fills the mask, so it stays allocated in every graph
             ggml_build_forward_expand(gf, inp->sel_mask);
-            // one view for all layers: each view of a host input is a separate upload
-            inp->sel_mask_2d = ggml_reshape_2d(ctx0, inp->sel_mask, n_sel, n_tokens);
         }
     }
 
     inp->n_new = n_new;
 
-    inp->new_pool_idxs = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, kpool, n_new);
+    inp->new_pool_idxs = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, kpool*n_new);
     ggml_set_input(inp->new_pool_idxs);
-    inp->new_pool_flat = ggml_reshape_1d(ctx0, inp->new_pool_idxs, kpool*n_new);
     // one scatter row per new pool, each a distinct rep row (see kpool_build_state)
     inp->new_pool_rep = ggml_new_tensor_1d(ctx0, GGML_TYPE_I64, n_new);
     ggml_set_input(inp->new_pool_rep);
@@ -924,7 +919,7 @@ ggml_tensor * llama_model_glm5_next::graph::build_kpool_select(
     const int64_t n_kv = mctx_lid->get_n_kv();
 
     // Pool the entries completed by this ubatch. The last one is a dummy when the ubatch completes none.
-    ggml_tensor * rows = kpool_cache.gather_key_gate(inp_kpool->new_pool_flat);
+    ggml_tensor * rows = kpool_cache.gather_key_gate(inp_kpool->new_pool_idxs);
     rows = ggml_reshape_3d(ctx0, rows, 2*n_embd_indexer, kpool, n_new);
 
     ggml_tensor * pk = ggml_view_3d(ctx0, rows, n_embd_indexer, kpool, n_new, rows->nb[1], rows->nb[2], 0);
@@ -1023,7 +1018,7 @@ ggml_tensor * llama_model_glm5_next::graph::build_kpool_select(
     // Live slots (visible pools, real tail cells) address disjoint cells. Each dead slot writes its own dump row
     // n_kv + slot, so the scatter indices of a token are unique: idx = dump + live*(idx - dump), live = exp(mask).
     GGML_ASSERT(inp_kpool->sel_mask->ne[0] == n_sel && inp_kpool->sel_mask->ne[3] == n_tokens);
-    ggml_tensor * live  = ggml_exp(ctx0, inp_kpool->sel_mask_2d);
+    ggml_tensor * live  = ggml_reshape_2d(ctx0, ggml_exp(ctx0, inp_kpool->sel_mask), n_sel, n_tokens);
     ggml_tensor * dump  = ggml_arange(ctx0, (float) n_kv, (float) (n_kv + n_sel), 1.0f);
     ggml_tensor * idx_f = ggml_cast(ctx0, sel_idx, GGML_TYPE_F32);
     idx_f   = ggml_add(ctx0, ggml_mul(ctx0, ggml_sub(ctx0, idx_f, dump), live), dump);

src/models/granite-hybrid.cpp

@@ -153,6 +153,8 @@ llama_model_granite_hybrid::graph::graph(const llama_model & model, const llm_gr
     }
 
     for (int il = 0; il < n_layer; ++il) {
+        res->t_layer_inp[il] = inpL;
+
         struct ggml_tensor * inpSA = inpL;
 
         // norm

src/models/granite-swa.cpp

@@ -154,6 +154,8 @@ llama_model_granite_swa::graph::graph(
             cb(inpL, "deepstack_in", il);
         }
 
+        res->t_layer_inp[il] = inpL;
+
         ggml_tensor * inpSA = inpL;
 
         // norm

src/models/granite-switch.cpp

@@ -293,6 +293,8 @@ llama_model_granite_switch::graph::graph(
     ggml_tensor * cur;
 
     for (int il = 0; il < n_layer; ++il) {
+        res->t_layer_inp[il] = inpL;
+
         ggml_tensor * inpSA = inpL;
 
         cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);

src/models/granite.cpp

@@ -159,6 +159,8 @@ llama_model_granite::graph::graph(
             cb(inpL, "deepstack_in", il);
         }
 
+        res->t_layer_inp[il] = inpL;
+
         ggml_tensor * inpSA = inpL;
 
         // norm

src/models/models.h

@@ -693,6 +693,19 @@ struct llama_model_qwen3moe : public llama_model_base {
 };
 
 
+struct llama_model_flint : public llama_model_base {
+    llama_model_flint(const struct llama_model_params & params) : llama_model_base(params) {}
+    void load_arch_hparams(llama_model_loader & ml) override;
+    void load_arch_tensors(llama_model_loader & ml) override;
+
+    struct graph : public llm_graph_context {
+        graph(const llama_model & model, const llm_graph_params & params);
+    };
+
+    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
 struct llama_model_qwen3vl : public llama_model_base {
     llama_model_qwen3vl(const struct llama_model_params & params) : llama_model_base(params) {}
     void load_arch_hparams(llama_model_loader & ml) override;

src/models/nemotron.cpp

@@ -25,8 +25,11 @@ void llama_model_nemotron::load_arch_tensors(llama_model_loader &) {
         layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
         layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, 0);
 
-        create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
-        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
+        // n_embd_head_k * n_head equals n_embd for standard Nemotron (32 heads),
+        // but differs for models with non-standard GQA (e.g. 48 Q heads, head_dim=128).
+        const int64_t n_embd_q = n_embd_head_k * n_head;
+        create_tensor_qkv(layer, i, n_embd, n_embd_q, n_embd_gqa, n_embd_gqa, 0);
+        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q, n_embd}, 0);
 
         // optional bias tensors
         layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);

src/models/qwen4exp.cpp

@@ -308,6 +308,20 @@ std::unique_ptr<llm_graph_context> llama_model_qwen4exp::build_arch_graph(const
     return std::make_unique<graph>(*this, params);
 }
 
+// DFlash drafters read the layer inputs as [n_embd, T] rows, so collapse the streams
+static ggml_tensor * hc_stream_mean(ggml_context * ctx, ggml_tensor * x) {
+    const int64_t n_embd = x->ne[0];
+    const int64_t hc     = x->ne[1];
+    const int64_t nt     = x->ne[2];
+
+    ggml_tensor * acc = ggml_view_2d(ctx, x, n_embd, nt, x->nb[2], 0);
+    for (int64_t s = 1; s < hc; ++s) {
+        acc = ggml_add(ctx, acc, ggml_view_2d(ctx, x, n_embd, nt, x->nb[2], s*x->nb[1]));
+    }
+
+    return ggml_scale(ctx, acc, 1.0f / (float) hc);
+}
+
 // Hyper-connections keep hc parallel residual streams [n_embd, hc, T] in place of layer norms.
 // Returns the mixed [n_embd, T] stream; `inject` gets the [hc, T] scatter weights.
 ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix(
@@ -447,7 +461,11 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa
     ggml_build_forward_expand(gf, res_hc);
 
     for (int il = 0; il < n_layer; ++il) {
-        res->t_layer_inp[il] = res_hc;
+        if ((size_t) il < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[il]) {
+            res->t_layer_inp[il] = hc_stream_mean(ctx0, res_hc);
+            cb(res->t_layer_inp[il], "layer_inp", il);
+            ggml_build_forward_expand(gf, res->t_layer_inp[il]);
+        }
 
         if (hparams.is_ple(il)) {
             res_hc = build_ple(inp->get_recr(), ple_emb, res_hc, il);

tests/CMakeLists.txt

@@ -142,7 +142,7 @@ llama_test(test-tokenizer-0 NAME test-tokenizer-0-command-r         ARGS ${PROJE
 llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-coder    ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-coder.gguf)
 llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-llm      ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-llm.gguf)
 llama_test(test-tokenizer-0 NAME test-tokenizer-0-falcon            ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-falcon.gguf)
-llama_test(test-tokenizer-0 NAME test-tokenizer-0-gemma-4           ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gemma-4.gguf)
+llama_test(test-tokenizer-0 NAME test-tokenizer-0-gemma-4           ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gemma-4.gguf --check-decode)
 llama_test(test-tokenizer-0 NAME test-tokenizer-0-gpt-2             ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gpt-2.gguf)
 llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-bpe         ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-bpe.gguf)
 llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-spm         ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-spm.gguf)

tests/fusion/MTL.csv

@@ -71,6 +71,9 @@ falcon              ,0   ,any     ,ADD+ADD                     ,      2
 falcon              ,0   ,any     ,NORM+MUL+ADD                ,      5
 falcon-h1           ,0   ,any     ,ADD+ADD                     ,      2
 falcon-h1           ,0   ,any     ,RMS_NORM+MUL                ,      9
+flint               ,0   ,any     ,ADD+ADD                     ,      1
+flint               ,0   ,any     ,MUL+ADD                     ,      2
+flint               ,0   ,any     ,RMS_NORM+MUL                ,      9
 gemma               ,0   ,any     ,RMS_NORM+MUL                ,      5
 gemma2              ,0   ,any     ,RMS_NORM+MUL                ,      5
 gemma2              ,0   ,any     ,RMS_NORM+MUL+ADD            ,      4

tests/test-backend-ops.cpp

@@ -12246,6 +12246,18 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
             true, 16, 8, b, false, true, false));
     }
 
+    // Gate and up on the SYCL XMX kernels: rows long enough for 2D block loads, a last tile of 8 rows, and
+    // columns on both sides of the 8 and 16 column tiles
+    for (ggml_type type : {GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K,
+                           GGML_TYPE_Q8_0}) {
+        for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) {
+            for (int64_t m_batch : {1, 2, 3, 5, 8, 9, 16, 17}) {
+                test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, m_batch, 1000, 1024,
+                    false, 16, 8, false, false, true, false, { 1, 1 }));
+            }
+        }
+    }
+
     for (int64_t k : {96, 896, 1024, 2880}) {
         for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) {
             test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q8_0, glu_op, 1, 32, k,

tests/test-jinja.cpp

@@ -1299,6 +1299,18 @@ static void test_namespace(testing & t) {
         json::object(),
         "3"
     );
+
+    test_template(t, "namespace attribute named like a dict method",
+        "{% set ns = namespace(items=[]) %}{{ 'yes' if ns.items else 'no' }}{% set ns.items = ns.items + [1] %}{{ ns.items | length }}{{ 'yes' if ns.items else 'no' }}",
+        json::object(),
+        "no1yes"
+    );
+
+    test_template(t, "dict keeps its methods",
+        "{% set d = dict(items=1) %}{% for k, v in d.items() %}{{ k }}={{ v }}{% endfor %}",
+        json::object(),
+        "items=1"
+    );
 }
 
 static void test_tests(testing & t) {

tests/test-llama-archs.cpp

@@ -361,13 +361,16 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5));
     } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 ||
             arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE ||
-            arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE) {
+            arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT) {
         std::vector<uint32_t> pattern;
         pattern.reserve(n_layer);
         for (uint32_t il = 0; il < n_layer; il++) {
             pattern.push_back(il % 2);
         }
         ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);
+        if (arch == LLM_ARCH_FLINT) {
+            ms.add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, pattern);
+        }
     } else {
         ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(2));
     }
@@ -525,6 +528,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         ms.add_kv(LLM_KV_EXPERT_GATING_FUNC,         gating_func); // sigmoid : softmax : sqrtsoftplus
         ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE,         1.0f);
         ms.add_kv(LLM_KV_EXPERTS_PER_GROUP,          uint32_t(1));
+        if (arch == LLM_ARCH_FLINT) {
+            ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
+        }
         if (arch == LLM_ARCH_K2_HORIZON) {
             ms.add_kv(LLM_KV_ATTENTION_VALUE_EXPERT_COUNT,      uint32_t(2));
             ms.add_kv(LLM_KV_ATTENTION_VALUE_EXPERT_USED_COUNT, uint32_t(2));
@@ -754,6 +760,7 @@ static bool check_causal_attn_toggle(
 
 static bool moe_mandatory(const llm_arch arch) {
     switch (arch) {
+        case LLM_ARCH_FLINT:
         case LLM_ARCH_LLAMA4:
         case LLM_ARCH_MAION_CODER:
         case LLM_ARCH_COHERE2MOE:

tests/test-tokenizer-0.cpp

@@ -125,7 +125,7 @@ static llama_tests read_tests(const std::string & fname_inp, const std::string &
 
 int main(int argc, char **argv) {
     if (argc < 2) {
-        fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]);
+        fprintf(stderr, "Usage: %s vocab-file [text-file] [--check-decode]\n", argv[0]);
         return 1;
     }
 
@@ -135,8 +135,13 @@ int main(int argc, char **argv) {
     const std::string fname_out = fname + ".out";
 
     std::string fname_text;
-    if (argc > 2) {
-        fname_text = argv[2];
+    bool check_decode = false; // also check that the expected tokens detokenize to the input text
+    for (int i = 2; i < argc; i++) {
+        if (std::string(argv[i]) == "--check-decode") {
+            check_decode = true;
+        } else {
+            fname_text = argv[i];
+        }
     }
 
     fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());
@@ -244,6 +249,14 @@ int main(int argc, char **argv) {
 
                     success = false;
                 }
+
+                if (check_decode) {
+                    const std::string dec = common_detokenize(ctx, test_kv.second);
+                    if (dec != test_kv.first) {
+                        fprintf(stderr, "%s : failed decode:  '%s' detokenized to '%s'\n", __func__, test_kv.first.c_str(), dec.c_str());
+                        success = false;
+                    }
+                }
             }
         });
     }

tools/mtmd/clip-impl.h

@@ -537,6 +537,7 @@ enum projector_type {
     PROJECTOR_TYPE_EXAONE4_5,
     PROJECTOR_TYPE_MINICPMV4_6,
     PROJECTOR_TYPE_MINICPMV4_7,
+    PROJECTOR_TYPE_MINICPM_OCR_1_0,
     PROJECTOR_TYPE_GRANITE_SPEECH,
     PROJECTOR_TYPE_MIMOVL,
     PROJECTOR_TYPE_MINIMAX_M3,
@@ -611,6 +612,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
     { PROJECTOR_TYPE_HUNYUANVL,         "hunyuanvl"},
     { PROJECTOR_TYPE_MINICPMV4_6,       "minicpmv4_6"},
     { PROJECTOR_TYPE_MINICPMV4_7,       "minicpmv4_7"},
+    { PROJECTOR_TYPE_MINICPM_OCR_1_0,   "minicpm_ocr_1_0"},
     { PROJECTOR_TYPE_GRANITE_SPEECH,    "granite_speech"},
     { PROJECTOR_TYPE_MIMOVL,            "mimovl"},
     { PROJECTOR_TYPE_MINIMAX_M3,        "minimax_m3"},

tools/mtmd/clip.cpp

@@ -1031,6 +1031,7 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
             } break;
         case PROJECTOR_TYPE_MINICPMV4_6:
         case PROJECTOR_TYPE_MINICPMV4_7:
+        case PROJECTOR_TYPE_MINICPM_OCR_1_0:
             {
                 builder = std::make_unique<clip_graph_minicpmv4_6>(ctx, img);
             } break;
@@ -1496,6 +1497,7 @@ struct clip_model_loader {
                     } break;
                 case PROJECTOR_TYPE_MINICPMV4_6:
                 case PROJECTOR_TYPE_MINICPMV4_7:
+                case PROJECTOR_TYPE_MINICPM_OCR_1_0:
                     {
                         // MiniCPM-V 4.6/4.7 unified merger projector
                         // ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension
@@ -2527,6 +2529,7 @@ struct clip_model_loader {
                     || model.proj_type == PROJECTOR_TYPE_MINICPMV
                     || model.proj_type == PROJECTOR_TYPE_MINICPMV4_6
                     || model.proj_type == PROJECTOR_TYPE_MINICPMV4_7
+                    || model.proj_type == PROJECTOR_TYPE_MINICPM_OCR_1_0
                 ) && layer.ff_up_w && layer.ff_down_w && layer.ff_down_w->ne[0] == hparams.n_embd;
             if (is_ffn_swapped) {
                 // swap up and down weights
@@ -2630,6 +2633,7 @@ struct clip_model_loader {
                 } break;
             case PROJECTOR_TYPE_MINICPMV4_6:
             case PROJECTOR_TYPE_MINICPMV4_7:
+            case PROJECTOR_TYPE_MINICPM_OCR_1_0:
                 {
                     const bool merger_required = hparams.n_merge == 4;
                     auto get_merger_tensor = [&](const std::string & name, bool required = true) {
@@ -4348,6 +4352,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
         case PROJECTOR_TYPE_STEP3VL:
         case PROJECTOR_TYPE_MINICPMV4_6:
         case PROJECTOR_TYPE_MINICPMV4_7:
+        case PROJECTOR_TYPE_MINICPM_OCR_1_0:
             return img->nx() / (params.patch_size * params.n_merge);
         case PROJECTOR_TYPE_DEEPSEEKOCR:
         case PROJECTOR_TYPE_DEEPSEEKOCR2:
@@ -4379,6 +4384,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) {
         case PROJECTOR_TYPE_STEP3VL:
         case PROJECTOR_TYPE_MINICPMV4_6:
         case PROJECTOR_TYPE_MINICPMV4_7:
+        case PROJECTOR_TYPE_MINICPM_OCR_1_0:
             return img->ny() / (params.patch_size * params.n_merge);
         default:
             break;
@@ -4445,6 +4451,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
             } break;
         case PROJECTOR_TYPE_MINICPMV4_6:
         case PROJECTOR_TYPE_MINICPMV4_7:
+        case PROJECTOR_TYPE_MINICPM_OCR_1_0:
             {
                 n_patches /= params.n_merge * params.n_merge;
             } break;
@@ -5169,6 +5176,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
             } break;
         case PROJECTOR_TYPE_MINICPMV4_6:
         case PROJECTOR_TYPE_MINICPMV4_7:
+        case PROJECTOR_TYPE_MINICPM_OCR_1_0:
             {
                 const bool is_4x = hparams.n_merge == 2;
 
@@ -6437,6 +6445,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
             return ctx->model.mm_model_proj->ne[0];
         case PROJECTOR_TYPE_MINICPMV4_6:
         case PROJECTOR_TYPE_MINICPMV4_7:
+        case PROJECTOR_TYPE_MINICPM_OCR_1_0:
             return ctx->model.mm_ffn_down_w->ne[1];
         case PROJECTOR_TYPE_GLM_EDGE:
             return ctx->model.mm_model_mlp_3_w->ne[1];

tools/mtmd/mtmd-image.h

@@ -190,6 +190,17 @@ protected:
     }
 };
 
+// MiniCPM-OCR 1.0: find_best_resize aligns to patch_size * n_merge (28 in 4x).
+struct mtmd_image_preprocessor_minicpm_ocr_1_0 : mtmd_image_preprocessor_minicpmv {
+    using mtmd_image_preprocessor_minicpmv::mtmd_image_preprocessor_minicpmv;
+
+protected:
+    int get_slice_align() const override {
+        const int merge = hparams.n_merge > 0 ? hparams.n_merge : 2;
+        return hparams.patch_size * merge;
+    }
+};
+
 // custom llava-uhd slicing logic for LFM2
 // tiling params (min/max tiles, tile size, pixels tolerance) come from hparams
 // ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py

tools/mtmd/mtmd.cpp

@@ -722,6 +722,7 @@ struct mtmd_context {
                 } break;
             case PROJECTOR_TYPE_MINICPMV4_6:
             case PROJECTOR_TYPE_MINICPMV4_7:
+            case PROJECTOR_TYPE_MINICPM_OCR_1_0:
                 {
                     slice_tmpl        = MTMD_SLICE_TMPL_MINICPMV_2_6;
                     tok_ov_img_start  = {lookup_token("<image>")};
@@ -732,7 +733,11 @@ struct mtmd_context {
                     tok_row_end_trail = false; // no trailing end-of-row token
                     ov_img_first      = true;
                     use_image_id      = true;
-                    image_preproc     = std::make_unique<mtmd_image_preprocessor_minicpmv>(ctx_v);
+                    if (clip_get_projector_type(ctx_v) == PROJECTOR_TYPE_MINICPM_OCR_1_0) {
+                        image_preproc = std::make_unique<mtmd_image_preprocessor_minicpm_ocr_1_0>(ctx_v);
+                    } else {
+                        image_preproc = std::make_unique<mtmd_image_preprocessor_minicpmv>(ctx_v);
+                    }
                 } break;
             case PROJECTOR_TYPE_QWEN2VL:
             case PROJECTOR_TYPE_QWEN25VL:

tools/server/README.md

@@ -771,37 +771,81 @@ This endpoint also supports multimodal embeddings. See the documentation for the
 
 ### POST `/reranking`: Rerank documents according to a given query
 
+> [!IMPORTANT]
+>
+> This endpoint is **not** OAI-compatible.
 Similar to https://jina.ai/reranker/ but might change in the future.
-Requires a reranker model (such as [bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3)) and the `--embedding --pooling rank` options.
+Requires a reranker model (such as [bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3)) and the `--embedding --pooling rank` or `--reranking` options.
 
-*Options:*
+`query` and each `documents`/`texts` element accept a plain string, or
+(for multimodal rerankers such as Qwen3-VL-Reranker) an object of the form
+`{"text": string | [strings], "image": url | data-uri | raw-base64 | [urls]}`.
+All keys are optional; a plain string is treated as `{"text": "..."}`.
 
-`query`: The query against which the documents will be ranked.
-
-`documents`: An array strings representing the documents to be ranked.
+Options: `top_n` (default: all), `model` (echoed in the Jina response), `return_text` (TEI only).
 
 *Aliases:*
   - `/rerank`
   - `/v1/rerank`
   - `/v1/reranking`
 
-*Examples:*
+The format is selected by which key carries the documents: `texts` (TEI, same input shape as Jina,
+set `return_text: true` to echo document text) or `documents` (Jina, multimodal supported).
+
+**Text only (TEI format)** - `query` + `texts`; set `return_text: true` to echo document text:
 
 ```shell
 curl http://127.0.0.1:8012/v1/rerank \
     -H "Content-Type: application/json" \
     -d '{
-        "model": "some-model",
-            "query": "What is panda?",
-            "top_n": 3,
-            "documents": [
-                "hi",
-            "it is a bear",
-            "The giant panda (Ailuropoda melanoleuca), sometimes called a panda bear or simply panda, is a bear species endemic to China."
-            ]
+        "query": "What is machine learning?",
+        "texts": [
+            "Machine learning is a subset of artificial intelligence.",
+            "A python is a large snake."
+        ],
+        "return_text": true
     }' | jq
 ```
 
+**Multimodal (Jina Format)** - mix text and images in `query` and/or `documents`:
+
+```shell
+curl http://127.0.0.1:8012/v1/rerank \
+    -H "Content-Type: application/json" \
+    -d '{
+        "model": "Qwen/Qwen3-VL-Reranker-2B",
+        "query": {
+            "text": "A city skyline with tall buildings and a river",
+            "image": "data:image/jpeg;base64,<base64>"
+        },
+        "documents": [
+            "a plain string document is also valid",
+            {
+                "text": "A photograph of the Boston skyline at dusk",
+                "image": "https://example.com/skyline.jpg"
+            },
+            {
+                "text": "caption for the two-figure spread",
+                "image": ["https://example.com/fig1.png", "data:image/jpeg;base64,<base64>"]
+            }
+        ],
+        "top_n": 2
+    }' | jq
+```
+
+Jina format returns a wrapped object with `model`, `object`, `usage` (`prompt_tokens` is the sum
+of `tokens_evaluated` across all scored pairs), and a `results` array of `{index, relevance_score}`.
+TEI format returns a bare array of `{index, score}` (+ `text` when `return_text: true`).
+Both are sorted descending by score and truncated to `top_n`; the score is the match probability in `[0, 1]`.
+
+Images never appear in any response shape.
+
+> **NOTE:** **Input size limit:** A bidirectional-encoder reranker (e.g. `bge-reranker-v2-m3`) must fit the whole prompt within
+`--ubatch-size`; if the input is too large to process, increase `--ubatch-size`.
+Causal decoder rerankers (e.g. `Qwen3-VL-Reranker`) have no such limitation.
+
+
+
 ### POST `/tts`: Generate speech audio from text
 
 Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info, see [tts/README.md](../tts/README.md)

tools/server/server-common.cpp

@@ -1265,7 +1265,6 @@ void handle_media(
         out_files.push_back(decoded_data);
     }
 }
-
 // load media files from an OAI content array, then replace each media part with a media marker text part
 static void oaicompat_content_load_media(json & content, const server_chat_params & opt, std::vector<raw_buffer> & out_files) {
     for (auto & p : content) {
@@ -1336,6 +1335,7 @@ server_tokens tokenize_oai_content_array(const llama_vocab * vocab, mtmd_context
     if (files.empty()) {
         return server_tokens(common_tokenize(vocab, prompt, add_special, parse_special), false);
     }
+
     return process_mtmd_prompt(mctx, prompt, files, init_opt);
 }
 
@@ -1951,18 +1951,49 @@ server_tokens format_prompt_rerank(
         mtmd_context * mctx,
         const std::string & query,
         const std::string & doc,
+        const std::vector<raw_buffer> & query_files,
+        const std::vector<raw_buffer> & doc_files,
         const mtmd_helper_init_opt & init_opt) {
     server_tokens result = {};
 
+    // media must be spliced through the mtmd pipeline
+    if ((!query_files.empty() || !doc_files.empty()) && mctx == nullptr) {
+        throw std::invalid_argument("multimodal rerank input requires --mmproj");
+    }
+
     const char * rerank_prompt = llama_model_chat_template(model, "rerank");
 
     if (rerank_prompt != nullptr) {
+        // multimodal: append one media marker per image (arrays already flattened by caller)
+        std::string q = query;
+        for (size_t i = 0; i < query_files.size(); i++) {
+            q += get_media_marker();
+        }
+        std::string d = doc;
+        for (size_t i = 0; i < doc_files.size(); i++) {
+            d += get_media_marker();
+        }
+
         std::string prompt = rerank_prompt;
-        string_replace_all(prompt, "{query}"   , query);
-        string_replace_all(prompt, "{document}", doc  );
-        server_tokens tokens = tokenize_input_subprompt(vocab, mctx, prompt, false, true, init_opt);
-        result.push_back(tokens);
+        string_replace_all(prompt, "{query}"   , q);
+        string_replace_all(prompt, "{document}", d  );
+
+        if (query_files.empty() && doc_files.empty()) {
+            server_tokens tokens = tokenize_input_subprompt(vocab, mctx, prompt, false, true, init_opt);
+            result.push_back(tokens);
+        } else {
+            // process_mtmd_prompt returns a complete mtmd server_tokens; return it directly
+            // (push_back into a non-mtmd result would hit the NULL placeholder tokens)
+            std::vector<raw_buffer> files;
+            files.insert(files.end(), query_files.begin(), query_files.end());
+            files.insert(files.end(), doc_files.begin(),   doc_files.end());
+            return process_mtmd_prompt(mctx, prompt, files, init_opt);
+        }
     } else {
+        if (!query_files.empty() || !doc_files.empty()) {
+            throw std::invalid_argument("multimodal rerank input requires a model with a rerank chat template");
+        }
+
         // Get EOS token - use SEP token as fallback if EOS is not available
         server_tokens query_tokens = tokenize_input_subprompt(vocab, mctx, query, false, false, init_opt);
         server_tokens doc_tokens   = tokenize_input_subprompt(vocab, mctx, doc,   false, false, init_opt);

tools/server/server-common.h

@@ -303,8 +303,11 @@ server_tokens process_mtmd_prompt(
                                         mtmd_context * mctx,
                                         const std::string & prompt,
                                         const std::vector<raw_buffer> & files,
-                                        const mtmd_helper_init_opt & init_opt,
-                                        bool is_placeholder = false);
+                                         const mtmd_helper_init_opt & init_opt,
+                                          bool is_placeholder = false);
+
+// decode one media url (http/file/data-uri) into out_files; throws on failure
+void handle_media(std::vector<raw_buffer> & out_files, const std::string & url, const std::string & media_path);
 
 /**
  * break the input "prompt" object into multiple prompt if needed, then tokenize them
@@ -590,12 +593,16 @@ llama_tokens format_prompt_infill(
         const llama_tokens & tokens_prompt);
 
 // format rerank task: [BOS]query[EOS][SEP]doc[EOS].
+// query_files/doc_files carry decoded media for multimodal (Qwen) inputs; when non-empty the
+// prompt is built with get_media_marker() sentinels and spliced via process_mtmd_prompt.
 server_tokens format_prompt_rerank(
         const struct llama_model * model,
         const struct llama_vocab * vocab,
         mtmd_context * mctx,
         const std::string & query,
         const std::string & doc,
+        const std::vector<raw_buffer> & query_files,
+        const std::vector<raw_buffer> & doc_files,
         const mtmd_helper_init_opt & init_opt);
 
 // simple implementation of a pipe

tools/server/server-context.cpp

@@ -6284,83 +6284,7 @@ void server_routes::init_routes() {
     };
 
     this->post_rerank = [this](const server_http_req & req) {
-        auto res = create_response();
-        if (!params.embedding || params.pooling_type != LLAMA_POOLING_TYPE_RANK) {
-            res->error(format_error_response("This server does not support reranking. Start it with `--reranking`", ERROR_TYPE_NOT_SUPPORTED));
-            return res;
-        }
-
-        const json body = json::parse(req.body);
-
-        // if true, use TEI API format, otherwise use Jina API format
-        // Jina: https://jina.ai/reranker/
-        // TEI: https://huggingface.github.io/text-embeddings-inference/#/Text%20Embeddings%20Inference/rerank
-        bool is_tei_format = body.contains("texts");
-
-        json query;
-        if (body.count("query") == 1) {
-            query = body.at("query");
-            if (!query.is_string()) {
-                res->error(format_error_response("\"query\" must be a string", ERROR_TYPE_INVALID_REQUEST));
-                return res;
-            }
-        } else {
-            res->error(format_error_response("\"query\" must be provided", ERROR_TYPE_INVALID_REQUEST));
-            return res;
-        }
-
-        std::vector<std::string> documents = json_value(body, "documents",
-                                             json_value(body, "texts", std::vector<std::string>()));
-        if (documents.empty()) {
-            res->error(format_error_response("\"documents\" must be a non-empty string array", ERROR_TYPE_INVALID_REQUEST));
-            return res;
-        }
-
-        int top_n = json_value(body, "top_n", (int)documents.size());
-
-        // create and queue the task
-        json responses = json::array();
-        auto & rd = res->rd;
-        {
-            std::vector<server_task> tasks;
-            tasks.reserve(documents.size());
-            for (size_t i = 0; i < documents.size(); i++) {
-                auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query, documents[i], ctx_server.init_opt);
-                server_task task = server_task(SERVER_TASK_TYPE_RERANK);
-                task.id     = rd.get_new_id();
-                task.tokens = std::move(tmp);
-                tasks.push_back(std::move(task));
-            }
-            rd.post_tasks(std::move(tasks));
-        }
-
-        // wait for the results
-        auto all_results = rd.wait_for_all(req.should_stop);
-
-        // collect results
-        if (all_results.is_terminated) {
-            return res; // connection is closed
-        } else if (all_results.error) {
-            res->error(all_results.error->to_json());
-            return res;
-        } else {
-            for (auto & res : all_results.results) {
-                GGML_ASSERT(dynamic_cast<server_task_result_rerank*>(res.get()) != nullptr);
-                responses.push_back(res->to_json());
-            }
-        }
-
-        // write JSON response
-        json root = format_response_rerank(
-            body,
-            meta->model_name,
-            responses,
-            is_tei_format,
-            documents,
-            top_n);
-
-        res->ok(root);
-        return res;
+        return handle_rerank_impl(req);
     };
 
     this->post_tts = [this](const server_http_req & req) {
@@ -6822,6 +6746,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(cons
         return tokenize_input_subprompt(ctx_server.vocab, ctx_server.mctx, p, true, true, ctx_server.init_opt);
     };
 
+    // dispatch to the appropriate tokenizer based on the shape of the input
     std::vector<server_tokens> tokenized_prompts;
     if (prompt.is_array() && !json_is_array_and_contains_numbers(prompt)) {
         for (const auto & p : prompt) {
@@ -6895,6 +6820,154 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(cons
     return res;
 }
 
+std::unique_ptr<server_res_generator> server_routes::handle_rerank_impl(const server_http_req & req) {
+    auto res = create_response();
+    if (!params.embedding || params.pooling_type != LLAMA_POOLING_TYPE_RANK) {
+        res->error(format_error_response("This server does not support reranking. Start it with `--reranking`", ERROR_TYPE_NOT_SUPPORTED));
+        return res;
+    }
+
+    const json body = json::parse(req.body);
+
+    // if true, use TEI API format, otherwise use Jina API format
+    // Jina: https://jina.ai/reranker/
+    // TEI: https://huggingface.github.io/text-embeddings-inference/#/Text%20Embeddings%20Inference/rerank
+    bool is_tei_format = body.contains("texts");
+
+    // expand one rerank input item: plain string or {"text": str|[str], "image": url|data-uri|raw-base64|[urls]}
+    auto parse_item = [&](const json & item, std::string & out_text, std::vector<raw_buffer> & out_files) {
+        if (item.is_string()) {
+            out_text = item.get<std::string>();
+            return;
+        }
+        if (!item.is_object()) {
+            throw std::invalid_argument("rerank input must be a string or an object of the form {\"text\": ..., \"image\": ...}");
+        }
+        if (item.contains("video") || item.contains("fps") || item.contains("max_frames")) {
+            throw std::invalid_argument("video/fps/max_frames are not supported for rerank input");
+        }
+        json text = item.value("text", json());
+        if (text.is_string()) {
+            out_text += text.get<std::string>();
+        } else if (text.is_array()) {
+            for (const auto & t : text) {
+                if (!t.is_string()) {
+                    throw std::invalid_argument("\"text\" array elements must be strings");
+                }
+                out_text += t.get<std::string>();
+            }
+        } else if (!text.is_null()) {
+            throw std::invalid_argument("\"text\" must be a string or an array of strings");
+        }
+        json image = item.value("image", json());
+        auto add_image = [&](const std::string & url) {
+            // marker is appended by format_prompt_rerank, one per decoded file
+            try {
+                handle_media(out_files, url, params.media_path);
+            } catch (const std::runtime_error & e) {
+                throw std::invalid_argument(e.what());
+            }
+        };
+        if (image.is_string()) {
+            add_image(image.get<std::string>());
+        } else if (image.is_array()) {
+            for (const auto & u : image) {
+                if (!u.is_string()) {
+                    throw std::invalid_argument("\"image\" array elements must be strings");
+                }
+                add_image(u.get<std::string>());
+            }
+        } else if (!image.is_null()) {
+            throw std::invalid_argument("\"image\" must be a string or an array of strings");
+        }
+    };
+
+    // Parse query
+    std::string query_str;
+    std::vector<raw_buffer> query_files;
+    if (body.count("query") == 1) {
+        try {
+            parse_item(body.at("query"), query_str, query_files);
+        } catch (const std::invalid_argument & e) {
+            res->error(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST));
+            return res;
+        }
+    } else {
+        res->error(format_error_response("\"query\" must be provided", ERROR_TYPE_INVALID_REQUEST));
+        return res;
+    }
+
+    // Resolve Jina/TEI format ambiguity
+    json documents_json = json_value(body, "documents",
+        json_value(body, "texts", json::array()));
+    if (!documents_json.is_array() || documents_json.empty()) {
+        res->error(format_error_response("\"documents\" must be a non-empty array of strings or objects", ERROR_TYPE_INVALID_REQUEST));
+        return res;
+    }
+
+    // Parse documents
+    std::vector<std::string>          documents(documents_json.size());
+    std::vector<std::vector<raw_buffer>> document_files(documents_json.size());
+    for (size_t i = 0; i < documents_json.size(); i++) {
+        try {
+            parse_item(documents_json[i], documents[i], document_files[i]);
+        } catch (const std::invalid_argument & e) {
+            res->error(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST));
+            return res;
+        }
+    }
+
+    int top_n = json_value(body, "top_n", (int)documents.size());
+    if (top_n < 0) {
+        res->error(format_error_response("\"top_n\" must be >= 0", ERROR_TYPE_INVALID_REQUEST));
+        return res;
+    }
+
+    // create and queue the task
+    json responses = json::array();
+    auto & rd = res->rd;
+    {
+        std::vector<server_task> tasks;
+        tasks.reserve(documents.size());
+        for (size_t i = 0; i < documents.size(); i++) {
+            auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query_str, documents[i], query_files, document_files[i], ctx_server.init_opt);
+            server_task task = server_task(SERVER_TASK_TYPE_RERANK);
+            task.id     = rd.get_new_id();
+            task.tokens = std::move(tmp);
+            tasks.push_back(std::move(task));
+        }
+        rd.post_tasks(std::move(tasks));
+    }
+
+    // wait for the results
+    auto all_results = rd.wait_for_all(req.should_stop);
+
+    // collect results
+    if (all_results.is_terminated) {
+        return res; // connection is closed
+    } else if (all_results.error) {
+        res->error(all_results.error->to_json());
+        return res;
+    } else {
+        for (auto & res : all_results.results) {
+            GGML_ASSERT(dynamic_cast<server_task_result_rerank*>(res.get()) != nullptr);
+            responses.push_back(res->to_json());
+        }
+    }
+
+    // write JSON response
+    json root = format_response_rerank(
+        body,
+        meta->model_name,
+        responses,
+        is_tei_format,
+        documents,
+        top_n);
+
+    res->ok(root);
+    return res;
+}
+
 std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const server_http_req & req, task_response_type res_type) {
     auto res = create_response();
     std::vector<raw_buffer> files;

tools/server/server-context.h

@@ -177,6 +177,7 @@ private:
     std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
     std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
     std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
+    std::unique_ptr<server_res_generator> handle_rerank_impl(const server_http_req & req);
     std::unique_ptr<server_res_generator> handle_count_tokens(const server_http_req & req, task_response_type res_type);
 
     // using unique_ptr to allow late initialization of const

tools/server/tests/conftest.py

@@ -7,9 +7,9 @@ from utils import *
 @pytest.fixture(scope="session", autouse=True)
 def configure_worker_port(request):
     worker_id = getattr(request.config, "workerinput", {}).get("workerid", "master")
-    if worker_id != "master":
-        worker_num = int(worker_id[2:])
-        os.environ["PORT"] = str(8080 + worker_num * 10)
+    worker_num = 0 if worker_id == "master" else int(worker_id[2:])
+    port_base = int(os.environ.get("LLAMA_SERVER_TEST_PORT_BASE", "8080"))
+    os.environ["PORT"] = str(port_base + worker_num * 10)
 
 
 # ref: https://stackoverflow.com/questions/22627659/run-code-before-and-after-each-test-in-py-test

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte

@@ -23,13 +23,15 @@
 	interface Props {
 		section: AgenticSection;
 		attachments?: DatabaseMessageExtra[];
+		conversationCwd?: string;
 		open: boolean;
 		isStreaming: boolean;
 		isExecuting?: boolean;
 		onToggle?: () => void;
 	}
 
-	let { attachments, isExecuting, isStreaming, onToggle, open, section }: Props = $props();
+	let { attachments, conversationCwd, isExecuting, isStreaming, onToggle, open, section }: Props =
+		$props();
 
 	// Runs for every tool block on mount, before the body renders: the cheap
 	// content prefilter and the tool-name allow-list come first so blobs from
@@ -60,6 +62,7 @@
 {:else if section.toolName === BuiltInTool.SERVER_EXEC_SHELL_COMMAND}
 	<ChatMessageToolCallBlockExecShellCommand
 		{attachments}
+		{conversationCwd}
 		{isExecuting}
 		{isStreaming}
 		{onToggle}

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte

@@ -2,10 +2,12 @@
 	import { parseEditFileMeta, parseEditFileTitleMeta } from './parsers/edit-file';
 	import ToolCallBlock from './ToolCallBlock.svelte';
 	import { XCircle } from '@lucide/svelte';
+	import { SyntaxHighlightedCode } from '$lib/components/app';
 	import { MAX_HEIGHT_CODE_BLOCK, RESULT_STAT_SEPARATOR } from '$lib/constants';
+	import { FileTypeText } from '$lib/enums';
 	import { toolsStore } from '$lib/stores';
 	import type { AgenticSection } from '$lib/types';
-	import { abbreviateHome, computeLineDiff, prefixFor } from '$lib/utils';
+	import { abbreviateHome, computeLineDiff, formatJsonPretty, prefixFor } from '$lib/utils';
 
 	interface Props {
 		section: AgenticSection;
@@ -44,16 +46,16 @@
 		</span>
 	{/snippet}
 
-	{#snippet children(meta, _ctx)}
-		{#if meta?.errorMessage}
+	{#snippet children(_meta, ctx)}
+		{#if editFileBody?.errorMessage}
 			<div
 				class="flex items-start gap-2 rounded bg-red-500/10 p-2 text-xs text-red-600 italic dark:text-red-400"
 			>
 				<XCircle class="mt-0.5 h-3 w-3 shrink-0" />
 
-				<span>{meta.errorMessage}</span>
+				<span>{editFileBody.errorMessage}</span>
 			</div>
-		{:else if meta && editFileBody && editFileBody.edits.length > 0}
+		{:else if editFileBody?.edits.length}
 			{#each editDiffs as diffLines, ei (ei)}
 				<div class={ei === 0 ? '' : 'mt-3'}>
 					<div class="mb-1.5 text-xs text-muted-foreground/70 italic">
@@ -79,14 +81,28 @@
 			{/each}
 
 			<div class="mt-1.5 text-xs text-muted-foreground/70 italic">
-				{#if meta.resultMessage}
-					{meta.resultMessage}{meta.editsApplied != null ? RESULT_STAT_SEPARATOR : ''}{/if}
+				{#if editFileBody.resultMessage}
+					{editFileBody.resultMessage}{editFileBody.editsApplied != null
+						? RESULT_STAT_SEPARATOR
+						: ''}{/if}
 
-				{#if meta.editsApplied != null}
-					<span class="font-mono">{meta.editsApplied}</span>
-					{meta.editsApplied === 1 ? 'edit' : 'edits'}&nbsp;applied
+				{#if editFileBody.editsApplied != null}
+					<span class="font-mono">{editFileBody.editsApplied}</span>
+					{editFileBody.editsApplied === 1 ? 'edit' : 'edits'}&nbsp;applied
 				{/if}
 			</div>
+		{:else if section.toolArgs}
+			<!-- no usable edits: not streamed yet, or a same-named tool with other arg names -->
+			<SyntaxHighlightedCode
+				code={formatJsonPretty(section.toolArgs)}
+				language={FileTypeText.JSON}
+				maxHeight={MAX_HEIGHT_CODE_BLOCK}
+				streaming={ctx.isCodeStreaming}
+			/>
+		{:else if ctx.isStreaming}
+			<div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic">
+				Receiving arguments...
+			</div>
 		{:else}
 			<div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic">No edits</div>
 		{/if}

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte

@@ -11,7 +11,7 @@
 	import { AlertTriangle, Check, Loader2, XCircle } from '@lucide/svelte';
 	import { CollapsibleTerminalBlock } from '$lib/components/app';
 	import { SETTINGS_KEYS, TOOL_RUNTIME_SCROLL_AT_BOTTOM_THRESHOLD_PX } from '$lib/constants';
-	import { AttachmentType } from '$lib/enums';
+	import { AgenticSectionType, AttachmentType } from '$lib/enums';
 	import { settingsStore, toolsStore } from '$lib/stores';
 	import type { AgenticSection, DatabaseMessageExtra, ToolResultLine } from '$lib/types';
 	import {
@@ -32,17 +32,40 @@
 		 *  tool call. Drives max-height + auto-scroll while true; releases
 		 *  them when the loop reports this call as done. */
 		isExecuting?: boolean;
+		/** Working directory of the conversation this call belongs to; stands
+		 *  in for `section.toolCwd` until the result message persists the
+		 *  real one. */
+		conversationCwd?: string;
 		attachments?: DatabaseMessageExtra[];
 		onToggle?: () => void;
 	}
 
-	let { attachments, isExecuting = false, isStreaming, onToggle, open, section }: Props = $props();
+	let {
+		attachments,
+		conversationCwd,
+		isExecuting = false,
+		isStreaming,
+		onToggle,
+		open,
+		section
+	}: Props = $props();
 
 	// `isLive` covers all in-flight phases: pre-chunk spinner and
 	// streaming itself. Frozen output (tool done while agent continues)
 	// is not live.
 	const isLive = $derived(isExecuting);
 
+	const isPreResult = $derived(
+		section.type === AgenticSectionType.TOOL_CALL_PENDING ||
+			section.type === AgenticSectionType.TOOL_CALL_STREAMING
+	);
+
+	// The model is still typing the call while the args JSON doesn't parse;
+	// the title keeps a terminal cursor until it does.
+	const commandStreaming = $derived(
+		isStreaming && isPreResult && !isParsableJson(section.toolArgs)
+	);
+
 	const execShellMeta = $derived(parseExecShellCommandMeta(section));
 	const execShellError = $derived(parseExecShellCommandError(section.toolResult));
 	const execShellExitStatus: ExecShellExitStatus | undefined = $derived(
@@ -76,9 +99,11 @@
 
 	// The working directory the command ran with, persisted per call on the
 	// tool result message (it travels via the x-tool-cwd header, not the tool
-	// args). Reading it from the section keeps it accurate even if the
-	// conversation cwd changes later.
-	const cwd = $derived(section.toolCwd);
+	// args). Until that message exists, the conversation cwd is the best known
+	// value - it is what the agentic loop passes when executing the call.
+	const cwd = $derived(
+		section.toolCwd ?? (isStreaming && isPreResult ? conversationCwd : undefined)
+	);
 	const home = $derived(toolsStore.serverHome);
 	const wdDisplay = $derived(abbreviateHome(cwd ?? '', home));
 
@@ -112,6 +137,18 @@
 		);
 	}
 
+	function isParsableJson(value: string | undefined): boolean {
+		if (!value) return false;
+
+		try {
+			JSON.parse(value);
+
+			return true;
+		} catch {
+			return false;
+		}
+	}
+
 	function scrollToBottomOnFrame() {
 		if (pendingFrame !== null || !scrollEl || userScrolledUp) return;
 
@@ -185,6 +222,10 @@
 	{:else}
 		<span class="font-mono">{execShellMeta?.command}</span>
 	{/if}
+
+	{#if commandStreaming}
+		<span class="exec-cursor"></span>
+	{/if}
 {/snippet}
 
 <ToolCallBlock
@@ -202,10 +243,10 @@
 	{/snippet}
 
 	{#snippet children(_meta, ctx)}
-		{#if ctx.isPending}
+		{#if (ctx.isPending && !commandStreaming) || (isLive && !section.toolResult)}
 			<div class="flex items-start gap-2 text-xs text-muted-foreground/70">
 				<Loader2 class="h-3 w-3 animate-spin" />
-				Running...
+				Executing command...
 			</div>
 		{:else if execShellError}
 			<div class="flex items-start gap-2 text-xs text-red-600 italic dark:text-red-400">
@@ -277,6 +318,22 @@
 		margin-right: var(--exec-wd-margin);
 	}
 
+	.exec-cursor {
+		display: inline-block;
+		width: 0.5em;
+		height: 1.05em;
+		margin-left: 0.15rem;
+		vertical-align: text-bottom;
+		background: var(--foreground);
+		animation: exec-cursor-blink 1.1s steps(1) infinite;
+	}
+
+	@keyframes exec-cursor-blink {
+		50% {
+			opacity: 0;
+		}
+	}
+
 	.terminal-output {
 		overscroll-behavior: contain;
 	}

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockWriteFile.svelte

@@ -4,9 +4,10 @@
 	import { XCircle } from '@lucide/svelte';
 	import { SyntaxHighlightedCode } from '$lib/components/app';
 	import { MAX_HEIGHT_CODE_BLOCK, RESULT_STAT_SEPARATOR } from '$lib/constants';
+	import { FileTypeText } from '$lib/enums';
 	import { toolsStore } from '$lib/stores';
 	import type { AgenticSection } from '$lib/types';
-	import { abbreviateHome } from '$lib/utils';
+	import { abbreviateHome, formatJsonPretty } from '$lib/utils';
 
 	interface Props {
 		section: AgenticSection;
@@ -42,32 +43,48 @@
 		</span>
 	{/snippet}
 
-	{#snippet children(meta, ctx)}
-		{#if meta?.errorMessage}
+	{#snippet children(_meta, ctx)}
+		{#if writeFileBody?.errorMessage}
 			<div
 				class="flex items-start gap-2 rounded bg-red-500/10 p-2 text-xs text-red-600 italic dark:text-red-400"
 			>
 				<XCircle class="mt-0.5 h-3 w-3 shrink-0" />
 
-				<span>{meta.errorMessage}</span>
+				<span>{writeFileBody.errorMessage}</span>
 			</div>
-		{:else if meta}
+		{:else if writeFileBody?.content !== undefined}
 			<SyntaxHighlightedCode
-				code={writeFileBody?.content ?? ''}
-				language={meta.language}
+				code={writeFileBody.content}
+				language={writeFileBody.language}
 				maxHeight={MAX_HEIGHT_CODE_BLOCK}
 				streaming={ctx.isCodeStreaming}
 			/>
 
 			<div class="mt-1.5 text-xs text-muted-foreground/70 italic">
-				{#if meta.resultMessage}
-					{meta.resultMessage}{meta.bytesWritten != null ? RESULT_STAT_SEPARATOR : ''}{/if}
+				{#if writeFileBody.resultMessage}
+					{writeFileBody.resultMessage}{writeFileBody.bytesWritten != null
+						? RESULT_STAT_SEPARATOR
+						: ''}{/if}
 
-				{#if meta.bytesWritten != null}
-					<span class="font-mono">{meta.bytesWritten}</span>
+				{#if writeFileBody.bytesWritten != null}
+					<span class="font-mono">{writeFileBody.bytesWritten}</span>
 					bytes
 				{/if}
 			</div>
+		{:else if section.toolArgs}
+			<!-- no content arg: not streamed yet, or a same-named tool with other arg names -->
+			<SyntaxHighlightedCode
+				code={formatJsonPretty(section.toolArgs)}
+				language={FileTypeText.JSON}
+				maxHeight={MAX_HEIGHT_CODE_BLOCK}
+				streaming={ctx.isCodeStreaming}
+			/>
+		{:else if ctx.isStreaming}
+			<div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic">
+				Receiving arguments...
+			</div>
+		{:else}
+			<div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic">No content</div>
 		{/if}
 	{/snippet}
 </ToolCallBlock>

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/_shared.ts

@@ -1,10 +1,14 @@
 // Helpers shared by the per-tool meta parsers under
 // `src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/`.
 // Each tool needs the same first three steps (tool-name check,
-// args-present check, JSON parse) - keeping them here lets each parser
-// stay focused on its own format quirks.
+// args-present check, JSON parse), plus the same path aliases - keeping
+// them here lets each parser stay focused on its own format quirks.
 
-import { TOOL_ARG_STRING_FIELD_PATTERN_TEMPLATE } from '$lib/constants';
+import {
+	FILE_PATH_SEPARATOR_REGEX,
+	TOOL_ARG_PATH_KEYS,
+	TOOL_ARG_STRING_FIELD_PATTERN_TEMPLATE
+} from '$lib/constants';
 import { BuiltInTool } from '$lib/enums';
 import type { AgenticSection } from '$lib/types/agentic';
 import { parsePartialJsonArgs } from '$lib/utils/parse-partial-json-args';
@@ -68,6 +72,34 @@ export function extractToolArgString(
 	return undefined;
 }
 
+/**
+ * File name shown in a block title, from a tool-call path. Undefined when
+ * the call carries no path; the title then shows the tool label alone.
+ */
+export function fileNameFromPath(filePath: string | undefined): string | undefined {
+	if (!filePath) return undefined;
+
+	return filePath.split(FILE_PATH_SEPARATOR_REGEX).pop() || filePath;
+}
+
+/**
+ * Read the path argument out of an already parsed args object, checking
+ * the aliases in `TOOL_ARG_PATH_KEYS`. Returns undefined when the call
+ * carries no usable path - the path is display-only, so callers must
+ * still render whatever else the call streamed.
+ */
+export function extractToolArgPath(args?: Record<string, unknown> | null): string | undefined {
+	if (!args) return undefined;
+
+	for (const key of TOOL_ARG_PATH_KEYS) {
+		const value = args[key];
+
+		if (typeof value === 'string' && value) return value;
+	}
+
+	return undefined;
+}
+
 /**
  * Parse a section's toolArgs against an expected tool name. Returns
  * `null` when:

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/edit-file.ts

@@ -1,10 +1,16 @@
 // Meta parser for `edit_file` tool calls. Reads the file path and the
 // array of edits from the streamed args (partial JSON for incremental
 // rendering), plus the result blob for `result` / `edits_applied` /
-// `error` fields.
+// `error` fields. The path is display-only (block title), so a call
+// that has not streamed one still shows its diffs.
 
-import { extractToolArgString, parseToolArgs } from './_shared';
-import { FILE_PATH_SEPARATOR_REGEX, TOOL_ARG_PATH_KEYS } from '$lib/constants';
+import {
+	extractToolArgPath,
+	extractToolArgString,
+	fileNameFromPath,
+	parseToolArgs
+} from './_shared';
+import { TOOL_ARG_PATH_KEYS } from '$lib/constants';
 import { BuiltInTool } from '$lib/enums';
 import type { AgenticSection, EditFileEdit, EditFileMeta, EditFileTitleMeta } from '$lib/types';
 import { tryParseToolResultObject } from '$lib/utils';
@@ -14,11 +20,8 @@ export function parseEditFileMeta(section: AgenticSection): EditFileMeta | null
 
 	if (!args) return null;
 
-	const rawPath = args.path ?? args.file_path ?? args.filePath;
-
-	if (typeof rawPath !== 'string' || !rawPath) return null;
-
-	const fileName = rawPath.split(FILE_PATH_SEPARATOR_REGEX).pop() || rawPath;
+	const filePath = extractToolArgPath(args);
+	const fileName = fileNameFromPath(filePath);
 	// Filter the streamed edits array strictly: each entry must be an
 	// object with a non-empty `old_text`. Edits without an old_text
 	// would diff against empty and render as a full re-write.
@@ -61,7 +64,7 @@ export function parseEditFileMeta(section: AgenticSection): EditFileMeta | null
 		editsApplied,
 		errorMessage,
 		fileName,
-		filePath: rawPath,
+		filePath,
 		resultMessage
 	};
 }
@@ -75,18 +78,15 @@ export function parseEditFileMeta(section: AgenticSection): EditFileMeta | null
 export function parseEditFileTitleMeta(section: AgenticSection): EditFileTitleMeta | null {
 	if (section.toolName !== BuiltInTool.SERVER_EDIT_FILE || !section.toolArgs) return null;
 
-	let rawPath: string | undefined = extractToolArgString(section.toolArgs, TOOL_ARG_PATH_KEYS);
+	let filePath = extractToolArgString(section.toolArgs, TOOL_ARG_PATH_KEYS);
 
-	if (!rawPath) {
-		const args = parseToolArgs(BuiltInTool.SERVER_EDIT_FILE, section, { partial: true });
-		const fallbackPath = args?.path ?? args?.file_path ?? args?.filePath;
-
-		if (typeof fallbackPath === 'string' && fallbackPath) rawPath = fallbackPath;
+	if (!filePath) {
+		filePath = extractToolArgPath(
+			parseToolArgs(BuiltInTool.SERVER_EDIT_FILE, section, { partial: true })
+		);
 	}
 
-	if (!rawPath) return null;
-
-	const fileName = rawPath.split(FILE_PATH_SEPARATOR_REGEX).pop() || rawPath;
+	const fileName = fileNameFromPath(filePath);
 	const resultObj = tryParseToolResultObject(section.toolResult);
 
 	let resultMessage: string | undefined;
@@ -105,5 +105,5 @@ export function parseEditFileTitleMeta(section: AgenticSection): EditFileTitleMe
 		}
 	}
 
-	return { editsApplied, errorMessage, fileName, filePath: rawPath, resultMessage };
+	return { editsApplied, errorMessage, fileName, filePath, resultMessage };
 }

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/exec-shell-command.ts

@@ -1,5 +1,7 @@
 // Meta parser for `exec_shell_command` tool calls. Surfaces the
 // command text from args `command` / `cmd` / `shell_command` aliases.
+// Args are parsed partially so the title can render the command while
+// it is still streaming.
 // The exit-status and error parsing live in their own utilities
 // (`parse-exec-shell-status.ts` / `parse-exec-shell-error.ts`) - this
 // file only deals with what's strictly about *calling* the tool, since
@@ -14,7 +16,7 @@ export type ExecShellCommandMeta = {
 };
 
 export function parseExecShellCommandMeta(section: AgenticSection): ExecShellCommandMeta | null {
-	const args = parseToolArgs(BuiltInTool.SERVER_EXEC_SHELL_COMMAND, section);
+	const args = parseToolArgs(BuiltInTool.SERVER_EXEC_SHELL_COMMAND, section, { partial: true });
 
 	if (!args) return null;
 

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/write-file.ts

@@ -1,30 +1,40 @@
 // Meta parser for `write_file` tool calls. Reads the path/content from
 // the streamed args (partial JSON so we can render before the call
 // finishes) and surfaces `bytes`, `result`, and `error` from the
-// result blob.
+// result blob. The path is display-only (title + highlight language),
+// so a call that has not streamed one still shows its content.
 
-import { extractToolArgString, parseToolArgs } from './_shared';
-import { CODE_BLOCK, FILE_PATH_SEPARATOR_REGEX, TOOL_ARG_PATH_KEYS } from '$lib/constants';
+import {
+	extractToolArgPath,
+	extractToolArgString,
+	fileNameFromPath,
+	parseToolArgs
+} from './_shared';
+import { CODE_BLOCK, TOOL_ARG_PATH_KEYS } from '$lib/constants';
 import { BuiltInTool } from '$lib/enums';
 import type { AgenticSection, WriteFileMeta, WriteFileTitleMeta } from '$lib/types';
 import { getFileTypeByExtension, tryParseToolResultObject } from '$lib/utils';
 
+// Plain text when the call carries no path, or one with no known extension.
+function languageForPath(filePath: string | undefined): string {
+	if (!filePath) return CODE_BLOCK.DEFAULT_LANGUAGE;
+
+	return (
+		getFileTypeByExtension(filePath)?.replace(CODE_BLOCK.TEXT_LANGUAGE_PREFIX_REGEX, '') ??
+		CODE_BLOCK.DEFAULT_LANGUAGE
+	);
+}
+
 export function parseWriteFileMeta(section: AgenticSection): WriteFileMeta | null {
 	const args = parseToolArgs(BuiltInTool.SERVER_WRITE_FILE, section, { partial: true });
 
 	if (!args) return null;
 
-	// Tool contracts drifted over time: some models emit `path`,
-	// others `file_path` / `filePath`. Accept all three.
-	const rawPath = args.path ?? args.file_path ?? args.filePath;
-
-	if (typeof rawPath !== 'string' || !rawPath) return null;
-
-	const fileName = rawPath.split(FILE_PATH_SEPARATOR_REGEX).pop() || rawPath;
-	const content = typeof args.content === 'string' ? args.content : '';
-	const language =
-		getFileTypeByExtension(rawPath)?.replace(CODE_BLOCK.TEXT_LANGUAGE_PREFIX_REGEX, '') ??
-		CODE_BLOCK.DEFAULT_LANGUAGE;
+	const filePath = extractToolArgPath(args);
+	const fileName = fileNameFromPath(filePath);
+	// undefined, not '', so an intentionally empty write still renders
+	const content = typeof args.content === 'string' ? args.content : undefined;
+	const language = languageForPath(filePath);
 	const resultObj = tryParseToolResultObject(section.toolResult);
 	const bytesWritten =
 		resultObj && Number.isFinite(Number(resultObj.bytes)) ? Number(resultObj.bytes) : undefined;
@@ -36,7 +46,7 @@ export function parseWriteFileMeta(section: AgenticSection): WriteFileMeta | nul
 		content,
 		errorMessage,
 		fileName,
-		filePath: rawPath,
+		filePath,
 		language,
 		resultMessage
 	};
@@ -51,21 +61,14 @@ export function parseWriteFileMeta(section: AgenticSection): WriteFileMeta | nul
 export function parseWriteFileTitleMeta(section: AgenticSection): WriteFileTitleMeta | null {
 	if (section.toolName !== BuiltInTool.SERVER_WRITE_FILE || !section.toolArgs) return null;
 
-	let rawPath: string | undefined = extractToolArgString(section.toolArgs, TOOL_ARG_PATH_KEYS);
+	let filePath = extractToolArgString(section.toolArgs, TOOL_ARG_PATH_KEYS);
 
-	if (!rawPath) {
-		const args = parseToolArgs(BuiltInTool.SERVER_WRITE_FILE, section, { partial: true });
-		const fallbackPath = args?.path ?? args?.file_path ?? args?.filePath;
-
-		if (typeof fallbackPath === 'string' && fallbackPath) rawPath = fallbackPath;
+	if (!filePath) {
+		filePath = extractToolArgPath(
+			parseToolArgs(BuiltInTool.SERVER_WRITE_FILE, section, { partial: true })
+		);
 	}
 
-	if (!rawPath) return null;
-
-	const fileName = rawPath.split(FILE_PATH_SEPARATOR_REGEX).pop() || rawPath;
-	const language =
-		getFileTypeByExtension(rawPath)?.replace(CODE_BLOCK.TEXT_LANGUAGE_PREFIX_REGEX, '') ??
-		CODE_BLOCK.DEFAULT_LANGUAGE;
 	const resultObj = tryParseToolResultObject(section.toolResult);
 	const bytesWritten =
 		resultObj && Number.isFinite(Number(resultObj.bytes)) ? Number(resultObj.bytes) : undefined;
@@ -75,9 +78,9 @@ export function parseWriteFileTitleMeta(section: AgenticSection): WriteFileTitle
 	return {
 		bytesWritten,
 		errorMessage,
-		fileName,
-		filePath: rawPath,
-		language,
+		fileName: fileNameFromPath(filePath),
+		filePath,
+		language: languageForPath(filePath),
 		resultMessage
 	};
 }

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte

@@ -8,14 +8,14 @@
 		MarkdownContent
 	} from '$lib/components/app';
 	import { AgenticSectionType, ChatMessageStatsView, ToolPermissionDecision } from '$lib/enums';
-	import { agenticStore, settingsStore } from '$lib/stores';
+	import { agenticStore, conversationsStore, settingsStore } from '$lib/stores';
 	import type {
 		AgenticSection,
 		ChatMessageAgenticTimings,
 		ChatMessageAgenticTurnStats,
 		DatabaseMessage
 	} from '$lib/types';
-	import { deriveAgenticSections } from '$lib/utils';
+	import { deriveAgenticSections, isSectionExpandedByDefault } from '$lib/utils';
 
 	interface Props {
 		message: DatabaseMessage;
@@ -34,6 +34,7 @@
 	let expandedStates: Record<number, boolean> = $state({});
 
 	const showThoughtInProgress = $derived(Boolean(settingsStore.config.showThoughtInProgress));
+	const showToolCallInProgress = $derived(Boolean(settingsStore.config.showToolCallInProgress));
 	const alwaysShowToolCallContent = $derived(
 		Boolean(settingsStore.config.alwaysShowToolCallContent)
 	);
@@ -95,6 +96,14 @@
 		isStreaming ? agenticStore.getExecutingToolCallId(message.convId) : null
 	);
 
+	// Cwd the agentic loop passes to tool calls in this conversation; the exec
+	// block shows it until the result message persists the real one.
+	const conversationCwd = $derived(
+		conversationsStore.activeConversation?.id === message.convId
+			? conversationsStore.activeConversation?.cwd
+			: undefined
+	);
+
 	type TurnGroup = {
 		sections: AgenticSection[];
 		flatIndices: number[];
@@ -132,28 +141,17 @@
 		return groups;
 	});
 
-	function getDefaultExpanded(section: AgenticSection): boolean {
-		if (
-			section.type === AgenticSectionType.TOOL_CALL ||
-			section.type === AgenticSectionType.TOOL_CALL_PENDING ||
-			section.type === AgenticSectionType.TOOL_CALL_STREAMING
-		) {
-			return alwaysShowToolCallContent;
-		}
-
-		if (section.type === AgenticSectionType.REASONING_PENDING) {
-			return showThoughtInProgress;
-		}
-
-		return false;
-	}
-
 	function isExpanded(index: number, section: AgenticSection): boolean {
 		if (expandedStates[index] !== undefined) {
 			return expandedStates[index];
 		}
 
-		return getDefaultExpanded(section);
+		return isSectionExpandedByDefault(section, {
+			alwaysShowToolCallContent,
+			executingToolCallId: currentlyExecutingToolCallId,
+			showThoughtInProgress,
+			showToolCallInProgress
+		});
 	}
 
 	function toggleExpanded(index: number, section: AgenticSection) {
@@ -190,6 +188,7 @@
 	{:else if section.type === AgenticSectionType.TOOL_CALL || section.type === AgenticSectionType.TOOL_CALL_PENDING || section.type === AgenticSectionType.TOOL_CALL_STREAMING}
 		<ChatMessageToolCallBlock
 			attachments={section.toolResultExtras}
+			{conversationCwd}
 			isExecuting={section.toolCallId !== undefined &&
 				section.toolCallId === currentlyExecutingToolCallId}
 			{isStreaming}

tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte

@@ -98,9 +98,17 @@
 		<!-- See CollapsibleContentBlock: bits-ui keeps closed content mounted, which
 		     makes a collapsed tool result re-render on every streamed token. -->
 		{#if open}
-			<div class="p-3 pt-1">
+			<div class="terminal-body p-3 pt-1">
 				{@render children()}
 			</div>
 		{/if}
 	</Collapsible.Content>
 </Collapsible.Root>
+
+<style>
+	/* A body that renders no element (exec block while its args stream)
+	   must not leave its padding as a gap under the title. */
+	.terminal-body:not(:has(*)) {
+		padding: 0;
+	}
+</style>

tools/ui/src/lib/constants/settings-keys.constants.ts

@@ -65,6 +65,7 @@ export const SETTINGS_KEYS = {
 	SHOW_RAW_OUTPUT_SWITCH: 'showRawOutputSwitch',
 	SHOW_SYSTEM_MESSAGE: 'showSystemMessage',
 	SHOW_THOUGHT_IN_PROGRESS: 'showThoughtInProgress',
+	SHOW_TOOL_CALL_IN_PROGRESS: 'showToolCallInProgress',
 	SYMBOLIC_MATH_ENABLED: 'symbolicMathEnabled',
 	SYSTEM_MESSAGE: 'systemMessage',
 	// Sampling

tools/ui/src/lib/constants/settings.constants.ts

@@ -227,9 +227,16 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [
 				label: 'Show thought in progress',
 				type: SettingsFieldType.CHECKBOX
 			},
+			{
+				defaultValue: true,
+				help: 'Expand tool call details while the call is being generated and executed, and collapse them again once its result is in. Mirrors "Show thought in progress". Tool calls you expand or collapse by hand keep your choice.',
+				key: SETTINGS_KEYS.SHOW_TOOL_CALL_IN_PROGRESS,
+				label: 'Show tool call in progress',
+				type: SettingsFieldType.CHECKBOX
+			},
 			{
 				defaultValue: false,
-				help: 'Automatically expand tool call details while executing and keep them expanded after completion.',
+				help: 'Keep tool call details expanded after completion. Takes precedence over "Show tool call in progress", which only covers the in-flight phases.',
 				key: SETTINGS_KEYS.ALWAYS_SHOW_TOOL_CALL_CONTENT,
 				label: 'Always show tool call content',
 				type: SettingsFieldType.CHECKBOX

tools/ui/src/lib/stores/models/index.svelte.ts

@@ -578,7 +578,6 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
 					const displayNameSource =
 						details?.name && details.name.trim().length > 0 ? details.name : item.id;
 					const modelId = details?.model || item.id;
-					const repo = repoOf(modelId);
 
 					return {
 						aliases: item.aliases ?? [],
@@ -594,7 +593,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
 								: undefined),
 						description: details?.description,
 						details: details?.details,
-						draftSidecars: this._status.getDraftSidecars(repo),
+						draftSidecars: this._status.getDraftSidecars(modelId),
 						id: item.id,
 						meta: item.meta ?? null,
 						modalities: this.props.buildArchitectureModalities(item.architecture),

tools/ui/src/lib/stores/models/status.svelte.ts

@@ -97,10 +97,11 @@ export class ModelStatusManager {
 	/** Sidecar kinds each in-flight or paused download pulls with its model. */
 	private downloadSidecars = new SvelteMap<string, ModelSidecar[]>();
 	/**
-	 * Draft sidecars the registered models pull with them, keyed by the repo of the model
-	 * that pulls them: the router records them in the model's `--model-draft` args.
+	 * Draft sidecars the registered models pull with them, keyed by the model that pulls
+	 * them: the router records them in the model's `--model-draft` args, and each quant
+	 * of a repo can pull a different draft.
 	 */
-	private draftSidecarsByRepo = $derived.by(() => {
+	private draftSidecarsByModel = $derived.by(() => {
 		const result = new SvelteMap<string, ModelSidecarBadge[]>();
 
 		for (const m of this.host.routerModels) {
@@ -108,8 +109,6 @@ export class ModelStatusManager {
 
 			if (!args) continue;
 
-			const baseRepo = repoOf(m.id);
-
 			for (let i = 0; i < args.length - 1; i++) {
 				if (args[i] !== CLI_FLAGS.MODEL_DRAFT && args[i] !== CLI_FLAGS.MODEL_DRAFT_SHORT) {
 					continue;
@@ -128,7 +127,7 @@ export class ModelStatusManager {
 					quant: meta.quant,
 					repo: parsed.repo
 				};
-				const kinds = result.get(baseRepo);
+				const kinds = result.get(m.id);
 
 				if (kinds) {
 					if (
@@ -139,7 +138,7 @@ export class ModelStatusManager {
 						kinds.push(badge);
 					}
 				} else {
-					result.set(baseRepo, [badge]);
+					result.set(m.id, [badge]);
 				}
 			}
 		}
@@ -404,17 +403,19 @@ export class ModelStatusManager {
 	}
 
 	/**
-	 * Draft sidecars a repo pulls with its model, e.g. `mtp` at `Q8_0`: the
-	 * router records loaded models' sidecars in their `--model-draft` args, and
-	 * lists the sidecars each cached model has on disk.
+	 * Draft sidecars a model pulls with it, e.g. `mtp` at `Q8_0`: the router
+	 * records each model's sidecars in its `--model-draft` args, and lists the
+	 * sidecars each cached repo has on disk.
 	 */
-	getDraftSidecars(repoId: string): ModelSidecarBadge[] {
-		// one badge per kind: the args know the quant, the listing carries kinds only,
-		// and a draft still coming down is pending rather than on disk
-		const merged = [...(this.draftSidecarsByRepo.get(repoId) ?? [])];
+	getDraftSidecars(modelId: string): ModelSidecarBadge[] {
+		const repo = repoOf(modelId);
+		// one badge per kind: the args know the quant of this model's draft, the
+		// listing carries repo-wide kinds only, and a draft still coming down is
+		// pending rather than on disk
+		const merged = [...(this.draftSidecarsByModel.get(modelId) ?? [])];
 
 		for (const source of [this.listedSidecarsByRepo, this.pendingSidecarsByRepo]) {
-			for (const badge of source.get(repoId) ?? []) {
+			for (const badge of source.get(repo) ?? []) {
 				if (!merged.some((b) => b.kind === badge.kind && b.repo === badge.repo)) {
 					merged.push(badge);
 				}

tools/ui/src/lib/types/tools.d.ts

@@ -33,10 +33,12 @@ export interface ToolGroup {
 }
 
 export interface WriteFileMeta {
-	fileName: string;
-	filePath: string;
+	/** Undefined until the call streams a path; content renders without it. */
+	fileName?: string;
+	filePath?: string;
 	language: string;
-	content: string;
+	/** Undefined until the args carry a content string; an empty write is ''. */
+	content?: string;
 	bytesWritten?: number;
 	resultMessage?: string;
 	errorMessage?: string;
@@ -46,8 +48,8 @@ export interface WriteFileMeta {
  *  ( with the embedded file content ) stays body-only so collapsed blocks
  *  never parse the content blob. */
 export interface WriteFileTitleMeta {
-	fileName: string;
-	filePath: string;
+	fileName?: string;
+	filePath?: string;
 	language: string;
 	bytesWritten?: number;
 	resultMessage?: string;
@@ -60,8 +62,9 @@ export interface EditFileEdit {
 }
 
 export interface EditFileMeta {
-	fileName: string;
-	filePath: string;
+	/** Undefined until the call streams a path; the edits render without it. */
+	fileName?: string;
+	filePath?: string;
 	edits: EditFileEdit[];
 	resultMessage?: string;
 	editsApplied?: number;
@@ -72,8 +75,8 @@ export interface EditFileMeta {
  *  ( with the embedded edit strings ) stays body-only so collapsed blocks
  *  never parse the args blob. */
 export interface EditFileTitleMeta {
-	fileName: string;
-	filePath: string;
+	fileName?: string;
+	filePath?: string;
 	resultMessage?: string;
 	editsApplied?: number;
 	errorMessage?: string;

tools/ui/src/lib/utils/agentic.ts

@@ -540,6 +540,49 @@ export function hasAgenticContent(
 	return toolMessages.length > 0;
 }
 
+/**
+ * Default expansion of a section, used until the user toggles it by hand.
+ *
+ * @param section - The section to judge
+ * @param flags - Display settings plus the tool call the agentic loop is running now.
+ *   A running call outranks its section type: `exec_shell_command` persists an empty
+ *   result before reading output, so a live call can already be TOOL_CALL.
+ */
+export function isSectionExpandedByDefault(
+	section: AgenticSection,
+	{
+		alwaysShowToolCallContent,
+		executingToolCallId,
+		showThoughtInProgress,
+		showToolCallInProgress
+	}: {
+		alwaysShowToolCallContent: boolean;
+		executingToolCallId?: string | null;
+		showThoughtInProgress: boolean;
+		showToolCallInProgress: boolean;
+	}
+): boolean {
+	const isRunning = section.toolCallId !== undefined && section.toolCallId === executingToolCallId;
+
+	if (
+		isRunning ||
+		section.type === AgenticSectionType.TOOL_CALL_PENDING ||
+		section.type === AgenticSectionType.TOOL_CALL_STREAMING
+	) {
+		return showToolCallInProgress || alwaysShowToolCallContent;
+	}
+
+	if (section.type === AgenticSectionType.TOOL_CALL) {
+		return alwaysShowToolCallContent;
+	}
+
+	if (section.type === AgenticSectionType.REASONING_PENDING) {
+		return showThoughtInProgress;
+	}
+
+	return false;
+}
+
 /**
  * Decide how a Continue click on messages[idx] should resume generation.
  * Pure function over the persisted history snapshot.

tools/ui/src/lib/utils/index.ts

@@ -282,6 +282,7 @@ export {
 	parseToolResultWithMedia,
 	splitSearchSummaryList,
 	hasAgenticContent,
+	isSectionExpandedByDefault,
 	classifyToolResult,
 	classifyContinueIntent
 } from './agentic';

tools/ui/tests/client/exec-shell-command-streaming.svelte.test.ts

@@ -0,0 +1,216 @@
+// Guards the exec_shell_command title while the call is still being
+// generated: the command must stream into the terminal title from the
+// first character (args arrive as partial JSON), with a blinking cursor
+// while the args are incomplete, and the conversation cwd standing in
+// for the x-tool-cwd value that only arrives with the result message.
+
+import ChatMessageToolCallBlockExecShellCommand from '$lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte';
+import ChatMessageAgenticContent from '$lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte';
+import { SETTINGS_KEYS } from '$lib/constants';
+import { AgenticSectionType, BuiltInTool, MessageRole, MessageType } from '$lib/enums';
+import { conversationsStore, settingsStore } from '$lib/stores';
+import type { AgenticSection, DatabaseMessage } from '$lib/types';
+import { tick } from 'svelte';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { render } from 'vitest-browser-svelte';
+
+const CONV_ID = 'conv-1';
+const TOOL_CALL_ID = 'call-1';
+const TOOL_NAME = BuiltInTool.SERVER_EXEC_SHELL_COMMAND;
+const CWD = '/home/user/project';
+
+function messageWithArgs(args: string): DatabaseMessage {
+	return {
+		children: [],
+		content: '',
+		convId: CONV_ID,
+		id: 'assistant-1',
+		parent: null,
+		role: MessageRole.ASSISTANT,
+		timestamp: 0,
+		toolCalls: JSON.stringify([
+			{ function: { arguments: args, name: TOOL_NAME }, id: TOOL_CALL_ID }
+		]),
+		type: MessageType.TEXT
+	};
+}
+
+function toolResultMessage(content: string, toolCwd?: string): DatabaseMessage {
+	return {
+		children: [],
+		content,
+		convId: CONV_ID,
+		id: 'tool-1',
+		parent: null,
+		role: MessageRole.TOOL,
+		timestamp: 0,
+		toolCallId: TOOL_CALL_ID,
+		toolCalls: '',
+		toolCwd,
+		type: MessageType.TEXT
+	};
+}
+
+function triggerText(container: Element): string {
+	return container.querySelector('[data-slot="collapsible-trigger"]')?.textContent ?? '';
+}
+
+function hasCursor(container: Element): boolean {
+	return container.querySelector('.exec-cursor') !== null;
+}
+
+describe('exec_shell_command streaming title', () => {
+	beforeEach(() => {
+		settingsStore.updateMultipleConfig({
+			[SETTINGS_KEYS.ALWAYS_SHOW_TOOL_CALL_CONTENT]: false,
+			[SETTINGS_KEYS.SHOW_TOOL_CALL_IN_PROGRESS]: true
+		});
+		conversationsStore.activeConversation = {
+			currNode: null,
+			cwd: CWD,
+			id: CONV_ID,
+			lastModified: 0,
+			name: 'test'
+		};
+	});
+
+	afterEach(() => {
+		conversationsStore.activeConversation = null;
+		conversationsStore.activeMessages = [];
+	});
+
+	it('streams the partial command into the title with a cursor', async () => {
+		const { container } = render(ChatMessageAgenticContent, {
+			isStreaming: true,
+			message: messageWithArgs('{"command":"git sta'),
+			toolMessages: []
+		});
+
+		await tick();
+
+		const title = triggerText(container);
+
+		expect(title).toContain('git sta');
+		expect(title).toContain('$');
+		expect(title).toContain(CWD);
+		expect(hasCursor(container)).toBe(true);
+	});
+
+	it('drops the cursor once the args JSON is complete', async () => {
+		const screen = render(ChatMessageAgenticContent, {
+			isStreaming: true,
+			message: messageWithArgs('{"command":"git sta'),
+			toolMessages: []
+		});
+
+		await tick();
+
+		expect(hasCursor(screen.container)).toBe(true);
+
+		screen.rerender({
+			isStreaming: true,
+			message: messageWithArgs('{"command":"git status"}'),
+			toolMessages: []
+		});
+		await tick();
+
+		expect(triggerText(screen.container)).toContain('git status');
+		expect(hasCursor(screen.container)).toBe(false);
+	});
+
+	it('switches the prompt from the conversation cwd to the persisted one', async () => {
+		const screen = render(ChatMessageAgenticContent, {
+			isStreaming: true,
+			message: messageWithArgs('{"command":"git status"}'),
+			toolMessages: []
+		});
+
+		await tick();
+
+		expect(triggerText(screen.container)).toContain(CWD);
+
+		// Result message lands: its x-tool-cwd takes over the prompt.
+		screen.rerender({
+			isStreaming: true,
+			message: messageWithArgs('{"command":"git status"}'),
+			toolMessages: [toolResultMessage('', '/other/dir')]
+		});
+		await tick();
+
+		expect(triggerText(screen.container)).toContain('/other/dir');
+		expect(triggerText(screen.container)).not.toContain(CWD);
+	});
+
+	it('shows the executing label only after the command finishes streaming', async () => {
+		const screen = render(ChatMessageAgenticContent, {
+			isStreaming: true,
+			message: messageWithArgs('{"command":"git sta'),
+			toolMessages: []
+		});
+
+		await tick();
+
+		// While the args stream, the title cursor alone signals activity.
+		expect(screen.container.textContent).not.toContain('Executing command...');
+
+		screen.rerender({
+			isStreaming: true,
+			message: messageWithArgs('{"command":"git status"}'),
+			toolMessages: []
+		});
+		await tick();
+
+		expect(screen.container.textContent).toContain('Executing command...');
+	});
+
+	it('shows Executing command... while executing before the first output chunk', async () => {
+		// The result message is created empty right before execution, so
+		// the block must not go blank while the process produces nothing.
+		const section: AgenticSection = {
+			content: '',
+			toolArgs: '{"command":"sleep 1"}',
+			toolCallId: TOOL_CALL_ID,
+			toolName: TOOL_NAME,
+			toolResult: '',
+			type: AgenticSectionType.TOOL_CALL
+		};
+		const { container } = render(ChatMessageToolCallBlockExecShellCommand, {
+			isExecuting: true,
+			isStreaming: true,
+			open: true,
+			section
+		});
+
+		await tick();
+
+		expect(container.textContent).toContain('Executing command...');
+	});
+
+	it('updates the title when the store mutates toolCalls in place, as the stream does', async () => {
+		// The live flow never swaps the message object: onToolCallsStreaming
+		// writes growing toolCalls strings into the same store row per delta.
+		conversationsStore.activeMessages = [messageWithArgs('')];
+
+		const screen = render(ChatMessageAgenticContent, {
+			isStreaming: true,
+			message: conversationsStore.activeMessages[0]!,
+			toolMessages: []
+		});
+
+		await tick();
+
+		const idx = conversationsStore.findMessageIndex('assistant-1');
+
+		for (const args of ['{"command', '{"command":"git', '{"command":"git sta']) {
+			conversationsStore.updateMessageAtIndex(idx, {
+				toolCalls: JSON.stringify([
+					{ function: { arguments: args, name: TOOL_NAME }, id: TOOL_CALL_ID }
+				])
+			});
+			await tick();
+
+			expect(triggerText(screen.container)).toContain(args.slice('{"command":"'.length));
+			expect(hasCursor(screen.container)).toBe(true);
+		}
+	});
+});

tools/ui/tests/client/models-sidecar-badges.svelte.test.ts

@@ -1,6 +1,7 @@
-// Guards the draft badges a repo row shows: a loaded model records its draft in
-// the `--model-draft` args, which know the file's quant, and the listing reports
-// the kinds of every cached model, which do not. The row shows each kind once.
+// Guards the draft badges a model row shows: a registered model records its draft
+// in its `--model-draft` args, which know the file's quant, and the listing reports
+// the kinds every cached model of the repo has on disk, which do not. The row shows
+// each kind once, and two quants of one repo keep their own drafts apart.
 
 import { ModelAuxSidecar, ModelDraftSidecar, ServerModelStatus } from '$lib/enums';
 import { modelsStore } from '$lib/stores';
@@ -9,6 +10,11 @@ import { beforeEach, describe, expect, it } from 'vitest';
 
 const REPO = 'ggml-org/Qwen3.8-Flash-Next-GGUF';
 
+/** Draft file of a cached model, as a preset records it in `--model-draft`. */
+function draftPath(quant: string): string {
+	return `/cache/models--ggml-org--Qwen3.8-Flash-Next-GGUF/snapshots/${'a'.repeat(40)}/mtp-Qwen3.8-Flash-Next-${quant}.gguf`;
+}
+
 /** A router entry; `args` stand in for the args of a loaded instance. */
 function entry(
 	id: string,
@@ -36,18 +42,28 @@ describe('draft sidecar badges', () => {
 	it('shows one badge for a draft the args and the listing both report', () => {
 		modelsStore.routerModels = [
 			entry(
-				`${REPO}:IQ4_NL`,
-				[],
-				[
-					'--model-draft',
-					`/cache/models--ggml-org--Qwen3.8-Flash-Next-GGUF/snapshots/${'a'.repeat(40)}/mtp-Qwen3.8-Flash-Next-Q8_0.gguf`
-				]
-			),
-			entry(`${REPO}:Q8_0`, [ModelAuxSidecar.MMPROJ, ModelDraftSidecar.MTP])
+				`${REPO}:Q8_0`,
+				[ModelAuxSidecar.MMPROJ, ModelDraftSidecar.MTP],
+				['--model-draft', draftPath('Q8_0')]
+			)
 		];
 
 		// the args badge wins: it knows the quant the listing does not
-		expect(modelsStore.status.getDraftSidecars(REPO)).toEqual([
+		expect(modelsStore.status.getDraftSidecars(`${REPO}:Q8_0`)).toEqual([
+			{ kind: ModelDraftSidecar.MTP, quant: 'Q8_0', repo: REPO }
+		]);
+	});
+
+	it('gives each quant of a repo the draft it pulls itself', () => {
+		modelsStore.routerModels = [
+			entry(`${REPO}:IQ4_NL`, [], ['--model-draft', draftPath('Q4_0')]),
+			entry(`${REPO}:Q8_0`, [], ['--model-draft', draftPath('Q8_0')])
+		];
+
+		expect(modelsStore.status.getDraftSidecars(`${REPO}:IQ4_NL`)).toEqual([
+			{ kind: ModelDraftSidecar.MTP, quant: 'Q4_0', repo: REPO }
+		]);
+		expect(modelsStore.status.getDraftSidecars(`${REPO}:Q8_0`)).toEqual([
 			{ kind: ModelDraftSidecar.MTP, quant: 'Q8_0', repo: REPO }
 		]);
 	});
@@ -58,7 +74,7 @@ describe('draft sidecar badges', () => {
 			entry(`${REPO}:Q8_0`, [ModelDraftSidecar.MTP])
 		];
 
-		expect(modelsStore.status.getDraftSidecars(REPO)).toEqual([
+		expect(modelsStore.status.getDraftSidecars(`${REPO}:IQ4_NL`)).toEqual([
 			{ kind: ModelDraftSidecar.MTP, quant: null, repo: REPO }
 		]);
 	});
@@ -74,7 +90,7 @@ describe('draft sidecar badges', () => {
 		];
 
 		// the kinds are not on disk yet, so the row reads them as in flight
-		expect(modelsStore.status.getDraftSidecars(REPO)).toEqual([
+		expect(modelsStore.status.getDraftSidecars(`${REPO}:Q4_0`)).toEqual([
 			{ kind: ModelDraftSidecar.MTP, pending: true, quant: null, repo: REPO }
 		]);
 	});
@@ -82,17 +98,10 @@ describe('draft sidecar badges', () => {
 	it('keeps the on-disk badge of a kind a download also pulls', () => {
 		modelsStore.routerModels = [
 			entry(`${REPO}:IQ4_NL`, [ModelDraftSidecar.MTP], [], ServerModelStatus.INCOMPLETE),
-			entry(
-				`${REPO}:Q8_0`,
-				[],
-				[
-					'--model-draft',
-					`/cache/models--ggml-org--Qwen3.8-Flash-Next-GGUF/snapshots/${'a'.repeat(40)}/mtp-Qwen3.8-Flash-Next-Q8_0.gguf`
-				]
-			)
+			entry(`${REPO}:Q8_0`, [], ['--model-draft', draftPath('Q8_0')])
 		];
 
-		expect(modelsStore.status.getDraftSidecars(REPO)).toEqual([
+		expect(modelsStore.status.getDraftSidecars(`${REPO}:Q8_0`)).toEqual([
 			{ kind: ModelDraftSidecar.MTP, quant: 'Q8_0', repo: REPO }
 		]);
 	});

tools/ui/tests/client/tool-call-block-streamed-body.svelte.test.ts

@@ -0,0 +1,82 @@
+// A file tool call must show what it streamed even when it carries no
+// path: the path only names the file in the block title, so a call that
+// streams its body first (or a same-named tool with other arg names) used
+// to render an empty block.
+
+import ChatMessageToolCallBlockEditFile from '$lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte';
+import ChatMessageToolCallBlockWriteFile from '$lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockWriteFile.svelte';
+import { AgenticSectionType, BuiltInTool } from '$lib/enums';
+import type { AgenticSection } from '$lib/types';
+import { tick } from 'svelte';
+import { describe, expect, it } from 'vitest';
+import { render } from 'vitest-browser-svelte';
+
+const WRITE_FILE_BODY = 'body streamed without a path';
+const EDIT_FILE_NEW_TEXT = 'replacement text';
+
+function makeSection(toolName: BuiltInTool, toolArgs: string): AgenticSection {
+	return {
+		content: '',
+		toolArgs,
+		toolName,
+		toolResult: undefined,
+		type: AgenticSectionType.TOOL_CALL
+	};
+}
+
+describe('file tool calls render their body without a path', () => {
+	it('write_file shows the content it streamed', async () => {
+		render(ChatMessageToolCallBlockWriteFile, {
+			isStreaming: true,
+			open: true,
+			section: makeSection(BuiltInTool.SERVER_WRITE_FILE, `{"content":"${WRITE_FILE_BODY}"`)
+		});
+
+		await tick();
+
+		expect(document.body.textContent).toContain(WRITE_FILE_BODY);
+	});
+
+	it('edit_file shows the diff it streamed', async () => {
+		render(ChatMessageToolCallBlockEditFile, {
+			isStreaming: true,
+			open: true,
+			section: makeSection(
+				BuiltInTool.SERVER_EDIT_FILE,
+				`{"edits":[{"old_text":"stale text","new_text":"${EDIT_FILE_NEW_TEXT}"}]}`
+			)
+		});
+
+		await tick();
+
+		expect(document.body.textContent).toContain(EDIT_FILE_NEW_TEXT);
+		expect(document.body.textContent).toContain('stale text');
+	});
+
+	it('write_file falls back to the raw args when no known content key is present', async () => {
+		render(ChatMessageToolCallBlockWriteFile, {
+			isStreaming: false,
+			open: true,
+			section: makeSection(BuiltInTool.SERVER_WRITE_FILE, '{"contents":"x"}')
+		});
+
+		await tick();
+
+		expect(document.body.textContent).toContain('contents');
+	});
+
+	it('write_file renders an empty code block for a finished empty write', async () => {
+		// content of '' used to fall through to the raw-args fallback
+		const section = makeSection(BuiltInTool.SERVER_WRITE_FILE, '{"content":""}');
+
+		section.toolResult = '{"result":"File written","bytes":0}';
+
+		render(ChatMessageToolCallBlockWriteFile, { isStreaming: false, open: true, section });
+
+		await tick();
+
+		expect(document.querySelector('pre code.hljs')).toBeTruthy();
+		expect(document.body.textContent).not.toContain('"content"');
+		expect(document.body.textContent).toMatch(/0\s*bytes/);
+	});
+});

tools/ui/tests/client/tool-call-in-progress.svelte.test.ts

@@ -0,0 +1,212 @@
+// Guards the two Settings > Display toggles that drive the default expansion of
+// a tool call block.
+//
+// "Show tool call in progress" mirrors "Show thought in progress": a call is
+// expanded by default only while it is in flight and folds back on its own once
+// the result lands. "Always show tool call content" is the one that keeps
+// finished calls open and wins on its own. A manual toggle always overrides the
+// default, in both directions.
+
+import ChatMessageAgenticContent from '$lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte';
+import { SETTINGS_KEYS } from '$lib/constants';
+import { MessageRole, MessageType } from '$lib/enums';
+import { settingsStore } from '$lib/stores';
+import type { DatabaseMessage } from '$lib/types';
+import { tick } from 'svelte';
+import { beforeEach, describe, expect, it } from 'vitest';
+import { userEvent } from 'vitest/browser';
+import { render } from 'vitest-browser-svelte';
+
+const TOOL_NAME = 'custom_tool';
+const TOOL_CALL_ID = 'call-1';
+// Both markers sit inside the collapsible body, which CollapsibleContentBlock
+// only mounts while open - so their presence is exactly "the block is expanded".
+const PENDING_BODY = 'Waiting for result...';
+const RESULT_BODY = 'marker-tool-output';
+const TRIGGER_SELECTOR = '[data-slot="collapsible-trigger"]';
+
+function assistantMessage(): DatabaseMessage {
+	return {
+		children: [],
+		content: '',
+		convId: 'conv-1',
+		id: 'assistant-1',
+		parent: null,
+		role: MessageRole.ASSISTANT,
+		timestamp: 0,
+		toolCalls: JSON.stringify([
+			{ function: { arguments: '{"path":"a.txt"}', name: TOOL_NAME }, id: TOOL_CALL_ID }
+		]),
+		type: MessageType.TEXT
+	};
+}
+
+function toolResultMessage(): DatabaseMessage {
+	return {
+		...assistantMessage(),
+		content: RESULT_BODY,
+		id: 'tool-1',
+		role: MessageRole.TOOL,
+		toolCallId: TOOL_CALL_ID
+	};
+}
+
+function setSettings({
+	alwaysShow,
+	inProgress
+}: {
+	alwaysShow: boolean;
+	inProgress: boolean;
+}): void {
+	settingsStore.updateMultipleConfig({
+		[SETTINGS_KEYS.ALWAYS_SHOW_TOOL_CALL_CONTENT]: alwaysShow,
+		[SETTINGS_KEYS.SHOW_TOOL_CALL_IN_PROGRESS]: inProgress
+	});
+}
+
+function textOf(container: Element): string {
+	return container.textContent ?? '';
+}
+
+async function collapseBlock(screen: { container: Element }): Promise<void> {
+	const trigger = screen.container.querySelector(TRIGGER_SELECTOR);
+
+	if (!(trigger instanceof HTMLButtonElement)) throw new Error('trigger not found');
+
+	await userEvent.click(trigger);
+	await tick();
+}
+
+describe('Show tool call in progress', () => {
+	beforeEach(() => {
+		setSettings({ alwaysShow: false, inProgress: false });
+	});
+
+	it('expands a call that is still executing when the setting is on', async () => {
+		setSettings({ alwaysShow: false, inProgress: true });
+
+		const { container } = render(ChatMessageAgenticContent, {
+			isStreaming: true,
+			message: assistantMessage(),
+			toolMessages: []
+		});
+
+		await tick();
+
+		expect(textOf(container)).toContain(PENDING_BODY);
+	});
+
+	it('keeps the call collapsed while executing when the setting is off', async () => {
+		const { container } = render(ChatMessageAgenticContent, {
+			isStreaming: true,
+			message: assistantMessage(),
+			toolMessages: []
+		});
+
+		await tick();
+
+		// The header is still there, only the body stays hidden.
+		expect(textOf(container)).toContain(TOOL_NAME);
+		expect(textOf(container)).not.toContain(PENDING_BODY);
+	});
+
+	it('folds the call back when its result arrives', async () => {
+		setSettings({ alwaysShow: false, inProgress: true });
+
+		const screen = render(ChatMessageAgenticContent, {
+			isStreaming: true,
+			message: assistantMessage(),
+			toolMessages: []
+		});
+
+		await tick();
+
+		// Open while it waits for the result.
+		expect(textOf(screen.container)).toContain(PENDING_BODY);
+
+		// Same block, now finished: it must close on its own, no toggle given.
+		screen.rerender({
+			isStreaming: true,
+			message: assistantMessage(),
+			toolMessages: [toolResultMessage()]
+		});
+		await tick();
+
+		expect(textOf(screen.container)).toContain(TOOL_NAME);
+		expect(textOf(screen.container)).not.toContain(PENDING_BODY);
+		expect(textOf(screen.container)).not.toContain(RESULT_BODY);
+	});
+
+	it('keeps the manual choice once the result arrives', async () => {
+		setSettings({ alwaysShow: false, inProgress: true });
+
+		const screen = render(ChatMessageAgenticContent, {
+			isStreaming: true,
+			message: assistantMessage(),
+			toolMessages: []
+		});
+
+		await tick();
+
+		// Collapsed by hand while running, so the result must not reopen it.
+		await collapseBlock(screen);
+
+		screen.rerender({
+			isStreaming: true,
+			message: assistantMessage(),
+			toolMessages: [toolResultMessage()]
+		});
+		await tick();
+
+		expect(textOf(screen.container)).toContain(TOOL_NAME);
+		expect(textOf(screen.container)).not.toContain(RESULT_BODY);
+	});
+
+	it('still expands a finished call when "always show" is on', async () => {
+		setSettings({ alwaysShow: true, inProgress: false });
+
+		const { container } = render(ChatMessageAgenticContent, {
+			isStreaming: true,
+			message: assistantMessage(),
+			toolMessages: [toolResultMessage()]
+		});
+
+		await tick();
+
+		expect(textOf(container)).toContain(RESULT_BODY);
+	});
+
+	it('lets a manual collapse win over the in-progress default', async () => {
+		setSettings({ alwaysShow: false, inProgress: true });
+
+		const screen = render(ChatMessageAgenticContent, {
+			isStreaming: true,
+			message: assistantMessage(),
+			toolMessages: []
+		});
+
+		await tick();
+
+		expect(textOf(screen.container)).toContain(PENDING_BODY);
+
+		await collapseBlock(screen);
+
+		expect(textOf(screen.container)).not.toContain(PENDING_BODY);
+	});
+
+	it('lets a manual expand win over the collapsed default', async () => {
+		const screen = render(ChatMessageAgenticContent, {
+			isStreaming: true,
+			message: assistantMessage(),
+			toolMessages: []
+		});
+
+		await tick();
+
+		expect(textOf(screen.container)).not.toContain(PENDING_BODY);
+
+		await collapseBlock(screen);
+
+		expect(textOf(screen.container)).toContain(PENDING_BODY);
+	});
+});

tools/ui/tests/unit/agentic-section-expansion.test.ts

@@ -0,0 +1,122 @@
+// Truth table for the default expansion of agentic sections - what a tool call
+// block does before the user touches it.
+//
+// The interesting row is a call the agentic loop reports as running while its
+// section is already TOOL_CALL: exec_shell_command persists an empty result
+// before any output is read, so the section type alone cannot tell a live call
+// from a finished one.
+
+import { AgenticSectionType } from '$lib/enums';
+import type { AgenticSection } from '$lib/types';
+import { isSectionExpandedByDefault } from '$lib/utils';
+import { describe, expect, it } from 'vitest';
+
+const CALL_ID = 'call-1';
+
+function section(overrides: Partial<AgenticSection> = {}): AgenticSection {
+	return { content: '', type: AgenticSectionType.TOOL_CALL, ...overrides };
+}
+
+function expanded(section: AgenticSection, flags: Record<string, unknown> = {}): boolean {
+	return isSectionExpandedByDefault(section, {
+		alwaysShowToolCallContent: false,
+		executingToolCallId: null,
+		showThoughtInProgress: false,
+		showToolCallInProgress: false,
+		...flags
+	});
+}
+
+describe('isSectionExpandedByDefault', () => {
+	it('expands a call whose arguments are streaming when the setting is on', () => {
+		const streaming = section({
+			toolCallId: CALL_ID,
+			toolName: 'read_file',
+			type: AgenticSectionType.TOOL_CALL_STREAMING
+		});
+
+		expect(expanded(streaming, { showToolCallInProgress: true })).toBe(true);
+		expect(expanded(streaming)).toBe(false);
+	});
+
+	it('expands a call waiting for its result when the setting is on', () => {
+		const pending = section({ toolCallId: CALL_ID, type: AgenticSectionType.TOOL_CALL_PENDING });
+
+		expect(expanded(pending, { showToolCallInProgress: true })).toBe(true);
+		expect(expanded(pending)).toBe(false);
+	});
+
+	it('collapses a finished call unless "always show" is on', () => {
+		const finished = section({ toolCallId: CALL_ID, toolResult: 'output' });
+
+		expect(expanded(finished, { showToolCallInProgress: true })).toBe(false);
+		expect(expanded(finished, { alwaysShowToolCallContent: true })).toBe(true);
+	});
+
+	it('keeps the running call expanded even when its result is already persisted', () => {
+		// exec_shell_command: empty result created, output still streaming
+		const live = section({ toolCallId: CALL_ID, toolResult: '' });
+
+		expect(expanded(live, { executingToolCallId: CALL_ID, showToolCallInProgress: true })).toBe(
+			true
+		);
+	});
+
+	it('does not treat another call finishing as the running one', () => {
+		const other = section({ toolCallId: 'call-2', toolResult: 'output' });
+
+		expect(expanded(other, { executingToolCallId: CALL_ID, showToolCallInProgress: true })).toBe(
+			false
+		);
+	});
+
+	it('still leaves the running call collapsed when both settings are off', () => {
+		const live = section({ toolCallId: CALL_ID, toolResult: '' });
+
+		expect(expanded(live, { executingToolCallId: CALL_ID })).toBe(false);
+	});
+
+	it('never reads a missing executing id as a running call', () => {
+		expect(expanded(section({ toolCallId: CALL_ID }), { executingToolCallId: null })).toBe(false);
+		expect(expanded(section({ type: AgenticSectionType.TEXT }), { executingToolCallId: '' })).toBe(
+			false
+		);
+	});
+
+	it('expands a running call when "always show" is on and the in-progress one is off', () => {
+		const pending = section({ toolCallId: CALL_ID, type: AgenticSectionType.TOOL_CALL_PENDING });
+
+		expect(expanded(pending, { alwaysShowToolCallContent: true })).toBe(true);
+	});
+
+	it('follows the thought setting for reasoning and never for finished thoughts', () => {
+		expect(
+			expanded(
+				{ content: 'think', type: AgenticSectionType.REASONING_PENDING },
+				{ showThoughtInProgress: true }
+			)
+		).toBe(true);
+		expect(
+			expanded(
+				{ content: 'think', type: AgenticSectionType.REASONING },
+				{
+					showThoughtInProgress: true
+				}
+			)
+		).toBe(false);
+	});
+
+	it('never expands plain text', () => {
+		expect(
+			expanded(
+				{ content: 'hello', type: AgenticSectionType.TEXT },
+				{
+					alwaysShowToolCallContent: true,
+					executingToolCallId: CALL_ID,
+					showThoughtInProgress: true,
+					showToolCallInProgress: true
+				}
+			)
+		).toBe(false);
+	});
+});

tools/ui/tests/unit/settings-tool-call-in-progress.test.ts

@@ -0,0 +1,45 @@
+// Guards the Settings > Display wiring of "Show tool call in progress": the
+// flat config default and its presence as a checkbox in the Display section
+// come from the settings registry, so the key/label/type/section pairing is the
+// contract the dialog renders.
+
+import {
+	SETTING_CONFIG_DEFAULT,
+	SETTINGS_CHAT_SECTIONS,
+	SETTINGS_KEYS,
+	SETTINGS_SECTION_SLUGS
+} from '$lib/constants';
+import { SettingsFieldType } from '$lib/enums';
+import { describe, expect, it } from 'vitest';
+
+function displayField(key: string) {
+	const section = SETTINGS_CHAT_SECTIONS.find((s) => s.slug === SETTINGS_SECTION_SLUGS.DISPLAY);
+
+	return section?.fields?.find((field) => field?.key === key);
+}
+
+describe('showToolCallInProgress setting', () => {
+	it('is a checkbox in the Display section', () => {
+		const field = displayField(SETTINGS_KEYS.SHOW_TOOL_CALL_IN_PROGRESS);
+
+		expect(field).toBeDefined();
+		expect(field?.type).toBe(SettingsFieldType.CHECKBOX);
+		expect(field?.label).toBe('Show tool call in progress');
+	});
+
+	it('ships enabled, like "Show thought in progress"', () => {
+		expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.SHOW_TOOL_CALL_IN_PROGRESS]).toBe(true);
+		expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.SHOW_THOUGHT_IN_PROGRESS]).toBe(true);
+	});
+
+	it('is listed next to its "always show" counterpart', () => {
+		const section = SETTINGS_CHAT_SECTIONS.find(
+			(s) => s.slug === SETTINGS_SECTION_SLUGS.DISPLAY
+		) as (typeof SETTINGS_CHAT_SECTIONS)[number];
+		const keys = section.fields?.map((field) => field?.key) ?? [];
+		const inProgressIndex = keys.indexOf(SETTINGS_KEYS.SHOW_TOOL_CALL_IN_PROGRESS);
+
+		expect(inProgressIndex).toBeGreaterThanOrEqual(0);
+		expect(keys[inProgressIndex + 1]).toBe(SETTINGS_KEYS.ALWAYS_SHOW_TOOL_CALL_CONTENT);
+	});
+});

tools/ui/tests/unit/tool-calls.test.ts

@@ -1,4 +1,7 @@
-import { parseToolArgs } from '$lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/_shared';
+import {
+	extractToolArgPath,
+	parseToolArgs
+} from '$lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/_shared';
 import {
 	parseEditFileMeta,
 	parseEditFileTitleMeta
@@ -12,6 +15,7 @@ import {
 	parseWriteFileMeta,
 	parseWriteFileTitleMeta
 } from '$lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/write-file';
+import { CODE_BLOCK } from '$lib/constants';
 import { AgenticSectionType, BuiltInTool } from '$lib/enums';
 import type { AgenticSection, WriteFileMeta } from '$lib/types';
 import { abbreviateHome, formatCwdMessage, lastPathSegment, parseCwdMessage } from '$lib/utils';
@@ -164,6 +168,22 @@ describe('parseToolArgs (shared)', () => {
 	});
 });
 
+describe('extractToolArgPath', () => {
+	it('reads every path alias', () => {
+		expect(extractToolArgPath({ path: '/a' })).toBe('/a');
+		expect(extractToolArgPath({ file_path: '/b' })).toBe('/b');
+		expect(extractToolArgPath({ filePath: '/c' })).toBe('/c');
+	});
+
+	it('returns undefined when no path alias is a usable string', () => {
+		expect(extractToolArgPath({ content: 'x' })).toBeUndefined();
+		expect(extractToolArgPath({ path: '' })).toBeUndefined();
+		expect(extractToolArgPath({ path: 7 })).toBeUndefined();
+		expect(extractToolArgPath(null)).toBeUndefined();
+		expect(extractToolArgPath(undefined)).toBeUndefined();
+	});
+});
+
 describe('parseWriteFileMeta', () => {
 	it('returns null for sections with a different tool name', () => {
 		expect(
@@ -176,12 +196,23 @@ describe('parseWriteFileMeta', () => {
 		).toBeNull();
 	});
 
-	it('returns null when args have no path-like field', () => {
-		expect(
-			parseWriteFileMeta(
-				makeSection({ toolArgs: '{"content":"x"}', toolName: BuiltInTool.SERVER_WRITE_FILE })
-			)
-		).toBeNull();
+	it('returns content without a path when args have no path-like field', () => {
+		const meta = parseWriteFileMeta(
+			makeSection({ toolArgs: '{"content":"x"}', toolName: BuiltInTool.SERVER_WRITE_FILE })
+		);
+
+		expect(meta?.content).toBe('x');
+		expect(meta?.filePath).toBeUndefined();
+		expect(meta?.fileName).toBeUndefined();
+		expect(meta?.language).toBe(CODE_BLOCK.DEFAULT_LANGUAGE);
+	});
+
+	it('returns content streamed before the path has arrived', () => {
+		const meta = parseWriteFileMeta(
+			makeSection({ toolArgs: '{"content":"hello ', toolName: BuiltInTool.SERVER_WRITE_FILE })
+		);
+
+		expect(meta?.content).toBe('hello ');
 	});
 
 	it('accepts partial args (renders incrementally as content streams in)', () => {
@@ -279,6 +310,16 @@ describe('parseWriteFileTitleMeta', () => {
 		expect(parseWriteFileTitleMeta(section)?.filePath).toBe('/foo.t');
 	});
 
+	it('returns meta without a path when args have no path-like field', () => {
+		const title = parseWriteFileTitleMeta(
+			makeSection({ toolArgs: '{"content":"x"}', toolName: BuiltInTool.SERVER_WRITE_FILE })
+		);
+
+		expect(title?.filePath).toBeUndefined();
+		expect(title?.fileName).toBeUndefined();
+		expect(title?.language).toBe(CODE_BLOCK.DEFAULT_LANGUAGE);
+	});
+
 	it('returns null for sections with a different tool name', () => {
 		expect(
 			parseWriteFileTitleMeta(
@@ -324,12 +365,13 @@ describe('parseEditFileTitleMeta', () => {
 		expect(parseEditFileTitleMeta(section)?.errorMessage).toBe('permission denied');
 	});
 
-	it('returns null when args have no path-like field', () => {
-		expect(
-			parseEditFileTitleMeta(
-				makeSection({ toolArgs: '{"edits":[]}', toolName: BuiltInTool.SERVER_EDIT_FILE })
-			)
-		).toBeNull();
+	it('returns meta without a path when args have no path-like field', () => {
+		const title = parseEditFileTitleMeta(
+			makeSection({ toolArgs: '{"edits":[]}', toolName: BuiltInTool.SERVER_EDIT_FILE })
+		);
+
+		expect(title?.filePath).toBeUndefined();
+		expect(title?.fileName).toBeUndefined();
 	});
 });
 
@@ -383,6 +425,33 @@ describe('parseEditFileMeta', () => {
 		expect(meta?.errorMessage).toBe('bad path');
 		expect(meta?.resultMessage).toBeUndefined();
 	});
+
+	it('still parses edits when args have no path-like field', () => {
+		const meta = parseEditFileMeta(
+			makeSection({
+				toolArgs: '{"edits":[{"old_text":"a","new_text":"b"}]}',
+				toolName: BuiltInTool.SERVER_EDIT_FILE
+			})
+		);
+
+		expect(meta?.edits).toEqual([{ newText: 'b', oldText: 'a' }]);
+		expect(meta?.filePath).toBeUndefined();
+	});
+
+	it('parses edits streamed before the path has arrived', () => {
+		const meta = parseEditFileMeta(
+			makeSection({
+				// second edit is still mid-stream: only its old_text has landed
+				toolArgs: '{"edits":[{"old_text":"a","new_text":"b"},{"old_text":"c',
+				toolName: BuiltInTool.SERVER_EDIT_FILE
+			})
+		);
+
+		expect(meta?.edits).toEqual([
+			{ newText: 'b', oldText: 'a' },
+			{ newText: '', oldText: 'c' }
+		]);
+	});
 });
 
 describe('parseReadFileMeta', () => {
@@ -647,4 +716,37 @@ describe('parseExecShellCommandMeta', () => {
 			)
 		).toBeNull();
 	});
+
+	it('surfaces a partially streamed command before the JSON closes', () => {
+		const meta = parseExecShellCommandMeta(
+			makeSection(
+				{ toolArgs: '{"command":"git sta', toolName: BuiltInTool.SERVER_EXEC_SHELL_COMMAND },
+				BuiltInTool.SERVER_EXEC_SHELL_COMMAND
+			)
+		);
+
+		expect(meta?.command).toBe('git sta');
+	});
+
+	it('keeps a dangling escape from breaking the partial parse', () => {
+		const meta = parseExecShellCommandMeta(
+			makeSection(
+				{ toolArgs: '{"command":"echo \\"he', toolName: BuiltInTool.SERVER_EXEC_SHELL_COMMAND },
+				BuiltInTool.SERVER_EXEC_SHELL_COMMAND
+			)
+		);
+
+		expect(meta?.command).toBe('echo "he');
+	});
+
+	it('returns null while the command value is still empty', () => {
+		expect(
+			parseExecShellCommandMeta(
+				makeSection(
+					{ toolArgs: '{"command":"', toolName: BuiltInTool.SERVER_EXEC_SHELL_COMMAND },
+					BuiltInTool.SERVER_EXEC_SHELL_COMMAND
+				)
+			)
+		).toBeNull();
+	});
 });