tests : catch vanished rows in the fusion baseline (#30618 )

* tests : update the Metal fusion baseline Re-record the MTL fusion baseline with the current build: 276 rows over 131 dummy models. Assisted-by: pi:llama.cpp/ggml-org/Qwen3.8-Flash-Next-GGUF (MacBook, M5 Max, 128GB) * tests : fail test-fusion when a baseline row stops matching The report was built from the counters the run produced, and the baseline was only ever used as a lookup table, so a fusion that stopped matching produced no row and was never compared. qwen4exp and mellum lost their fusions this way while the Fusion / metal job stayed green. Report a baseline row that the run does not reproduce as a regression. Scope the pass to the arches that were actually run so --model keeps working. Assisted-by: pi:llama.cpp/ggml-org/Qwen3.8-Flash-Next-GGUF (MacBook, M5 Max, 128GB) * pi : never write inline comments unless the user asks Assisted-by: pi:llama.cpp/ggml-org/Qwen3.8-Flash-Next-GGUF (MacBook, M5 Max, 128GB) * tests : keep the mellum dummy model softmax-gated mellum reads expert_gating_func from the GGUF since #30578, while the fixture writes sigmoid for every MoE arch, so the Mellum dummy model stopped building the softmax MoE routing fusion. Write softmax for mellum, matching the arch default and the GGUFs that conversion/mellum.py produces. Re-record the MTL baseline: the routing row is back, 277 ok / 0 failed. Assisted-by: pi:llama.cpp/ggml-org/Qwen3.8-Flash-Next-GGUF (MacBook, M5 Max, 128GB) * pi : use emoji in the AI stats disclosure Assisted-by: pi:llama.cpp/ggml-org/Qwen3.8-Flash-Next-GGUF (MacBook, M5 Max, 128GB) * pi : list the disclosure model after the hardware Assisted-by: pi:llama.cpp/ggml-org/Qwen3.8-Flash-Next-GGUF (MacBook, M5 Max, 128GB) * pi : use monochrome symbols in the AI stats disclosure Assisted-by: pi:llama.cpp/ggml-org/Qwen3.8-Flash-Next-GGUF (MacBook, M5 Max, 128GB) * tests : add muse-spark to the Metal fusion baseline Assisted-by: pi:llama.cpp/ggml-org/Qwen3.8-Flash-Next-GGUF (MacBook, M5 Max, 128GB)
vulkan: fix gpu-visible host memory issues (#30468 )
2026-11-10 21:07:41 -06:00 · 2026-10-16 09:35:19 +03:00 · 2026-10-16 08:29:23 +02:00 · 2026-10-16 12:46:55 +08:00 · 2026-10-16 11:32:26 +08:00 · 2026-10-15 20:30:11 -07:00
174 changed files with 9675 additions and 4556 deletions
@@ -145,7 +145,7 @@ ENTRYPOINT ["/app/tools.sh"]
 # ==============================================================================
 FROM base AS light

-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/

 ENTRYPOINT [ "/app/llama-cli" ]

@@ -157,7 +157,7 @@ FROM base AS server
 ENV LLAMA_ARG_HOST=0.0.0.0
 ENV LLAMA_ARG_PORT=8080

-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/

 HEALTHCHECK --interval=5m CMD [ "curl", "-f", "http://localhost:8080/health" ]

@@ -104,7 +104,7 @@ ENTRYPOINT ["/app/tools.sh"]
 ### Light, CLI only
 FROM base AS light

-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/

 WORKDIR /app

@@ -116,7 +116,7 @@ FROM base AS server
 ENV LLAMA_ARG_HOST=0.0.0.0
 ENV LLAMA_ARG_PORT=8080

-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/

 WORKDIR /app

@@ -113,7 +113,7 @@ ENTRYPOINT ["/app/tools.sh"]
 ### Light, CLI only
 FROM base AS light

-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/

 WORKDIR /app

@@ -125,7 +125,7 @@ FROM base AS server
 ENV LLAMA_ARG_HOST=0.0.0.0
 ENV LLAMA_ARG_PORT=8080

-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/

 WORKDIR /app

@@ -19,7 +19,12 @@ RUN npm ci
 COPY tools/ui/ ./
 RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build

-FROM docker.io/intel/oneapi-toolkit:$ONEAPI_VERSION AS build
+FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build
+
+RUN echo "Download oneDNN installation package" && \
+    wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/94c3dbac-0852-45be-a57d-21c204cada3e/intel-onednn-2026.0.2.46_offline.sh -O intel-onednn_offline.sh && \
+    echo "Install oneDNN components" && \
+    bash intel-onednn_offline.sh -s -a --silent --eula accept

 ARG GGML_SYCL_F16=ON
 ARG LEVEL_ZERO_VERSION=1.28.2
@@ -102,7 +107,7 @@ RUN mkdir /tmp/neo/ && cd /tmp/neo/ \
  && dpkg --install *.deb

 RUN apt-get update \
-    && apt-get install -y libgomp1 curl ffmpeg \
+    && apt-get install -y libgomp1 curl ffmpeg openssl libssl-dev \
    && apt autoremove -y \
    && apt clean -y \
    && rm -rf /tmp/* /var/tmp/* \
@@ -141,7 +146,7 @@ ENTRYPOINT ["/app/tools.sh"]
 FROM base AS light

 COPY --from=build /app/lib/ /app
-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/

 WORKDIR /app

@@ -154,7 +159,7 @@ ENV LLAMA_ARG_HOST=0.0.0.0
 ENV LLAMA_ARG_PORT=8080

 COPY --from=build /app/lib/ /app
-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/

 WORKDIR /app

@@ -120,7 +120,7 @@ ENTRYPOINT ["/app/tools.sh"]
 ### Light, CLI only
 FROM base AS light

-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/

 WORKDIR /app

@@ -132,7 +132,7 @@ FROM base AS server
 ENV LLAMA_ARG_HOST=0.0.0.0
 ENV LLAMA_ARG_PORT=8080

-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/

 WORKDIR /app

@@ -126,7 +126,7 @@ ENTRYPOINT ["/app/tools.sh"]
 ### Light, CLI only
 FROM base AS light

-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/

 WORKDIR /app

@@ -138,7 +138,7 @@ FROM base AS server
 ENV LLAMA_ARG_HOST=0.0.0.0
 ENV LLAMA_ARG_PORT=8080

-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/

 WORKDIR /app

@@ -107,7 +107,7 @@ ENTRYPOINT ["/app/tools.sh"]
 ### Light, CLI only
 FROM base AS light

-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/

 WORKDIR /app

@@ -119,7 +119,7 @@ FROM base AS server
 ENV LLAMA_ARG_HOST=0.0.0.0
 ENV LLAMA_ARG_PORT=8080

-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/

 WORKDIR /app

@@ -97,7 +97,7 @@ ENTRYPOINT ["/app/tools.sh"]
 ### Light, CLI only
 FROM base AS light

-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/

 WORKDIR /app

@@ -109,7 +109,7 @@ FROM base AS server
 ENV LLAMA_ARG_HOST=0.0.0.0
 ENV LLAMA_ARG_PORT=8080

-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/

 WORKDIR /app

@@ -67,8 +67,15 @@ jobs:
        shell: bash
        run: |
          cd /tmp
-          wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/5996e26b-f48a-42b1-8db0-b002ad0bd8d7/intel-oneapi-toolkit-2026.1.1.33_offline.sh -O intel-oneapi-toolkit_offline.sh
-          sudo bash intel-oneapi-toolkit_offline.sh -s -a --silent --eula accept
+          echo "Download oneAPI Deep Learning Essentials installation package"
+          wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b5c100a7-8052-4e88-90e2-63d9916a998b/intel-deep-learning-essentials-2026.1.4.17_offline.sh -O intel-deep-learning-essentials_offline.sh
+          echo "Install oneAPI Deep Learning Essentials components"
+          sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept
+
+          echo "Download oneDNN installation package"
+          wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/94c3dbac-0852-45be-a57d-21c204cada3e/intel-onednn-2026.0.2.46_offline.sh -O intel-onednn_offline.sh
+          echo "Install oneDNN components"
+          sudo bash intel-onednn_offline.sh -s -a --silent --eula accept

      - name: Install Level Zero SDK
        shell: bash
@@ -79,6 +86,12 @@ jobs:
          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze-dev_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze-dev.deb
          sudo apt-get install -y ./libze1.deb ./libze-dev.deb

+      - name: Dependencies
+        id: depends
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libssl-dev
+
      - name: ccache
        uses: ggml-org/ccache-action@v1.2.24
        with:
@@ -104,7 +117,6 @@ jobs:
            -DGGML_SYCL=ON \
            -DCMAKE_C_COMPILER=icx \
            -DCMAKE_CXX_COMPILER=icpx \
-            -DLLAMA_OPENSSL=OFF \
            -DGGML_NATIVE=OFF \
            -DGGML_SYCL_F16=${{ matrix.fp16 }} \
            -DGGML_SYCL_SUPPORT_LEVEL_ZERO_API=ON \
@@ -138,8 +150,10 @@ jobs:
        shell: bash

    env:
-      WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0cb67a0d-67f6-410b-868b-f4a0a17ff0cf/intel-oneapi-toolkit-2026.1.1.32_offline.exe
-      WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
+      WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/a51104eb-6f4a-4a0e-9dae-1b81b0c81ae1/intel-deep-learning-essentials-2026.1.4.16_offline.exe
+      WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/34e28083-d04b-46a3-826c-8c1d00bf1e1c/intel-onednn-2026.0.2.47_offline.exe
+      WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.tbb.devel
+      WINDOWS_DNNL: intel.oneapi.win.dnnl
      LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip
      ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
      ONEAPI_INSTALLER_VERSION: "2026.1"
@@ -151,7 +165,8 @@ jobs:
      - name: Download & Install oneAPI
        shell: bash
        run: |
-          scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL
+          scripts/install-oneapi.bat $WINDOWS_DLE_URL $WINDOWS_DPCPP_MKL
+          scripts/install-oneapi.bat $WINDOWS_ONEDNN_URL $WINDOWS_DNNL

      - name: Install Level Zero SDK
        shell: pwsh
@@ -35,7 +35,7 @@ jobs:
        uses: actions/setup-python@v6
        with:
          python-version: "3.11"
-          pip-install: -r requirements/requirements-all.txt ty==0.0.84
+          pip-install: -r requirements/requirements-all.txt ty==0.0.88
      # - name: Type-check with Pyright
      #   uses: jakebailey/pyright-action@v2
      #   with:
@@ -1300,8 +1300,11 @@ jobs:
        shell: bash

    env:
-      WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0cb67a0d-67f6-410b-868b-f4a0a17ff0cf/intel-oneapi-toolkit-2026.1.1.32_offline.exe
-      WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
+      WINDOWS_DLE_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/a51104eb-6f4a-4a0e-9dae-1b81b0c81ae1/intel-deep-learning-essentials-2026.1.4.16_offline.exe
+      WINDOWS_ONEDNN_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/34e28083-d04b-46a3-826c-8c1d00bf1e1c/intel-onednn-2026.0.2.47_offline.exe
+      WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.tbb.devel
+      WINDOWS_DNNL: intel.oneapi.win.dnnl
+
      LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip
      ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
      ONEAPI_INSTALLER_VERSION: "2026.1"
@@ -1314,7 +1317,8 @@ jobs:
      - name: Download & Install oneAPI
        shell: bash
        run: |
-          scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL
+          scripts/install-oneapi.bat $WINDOWS_DLE_URL $WINDOWS_DPCPP_MKL
+          scripts/install-oneapi.bat $WINDOWS_ONEDNN_URL $WINDOWS_DNNL

      - name: Install Level Zero SDK
        shell: pwsh
@@ -1425,8 +1429,15 @@ jobs:
        shell: bash
        run: |
          cd /tmp
-          wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/5996e26b-f48a-42b1-8db0-b002ad0bd8d7/intel-oneapi-toolkit-2026.1.1.33_offline.sh -O intel-oneapi-toolkit_offline.sh
-          sudo bash intel-oneapi-toolkit_offline.sh -s -a --silent --eula accept
+          echo "Download oneAPI Deep Learning Essentials installation package"
+          wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b5c100a7-8052-4e88-90e2-63d9916a998b/intel-deep-learning-essentials-2026.1.4.17_offline.sh -O intel-deep-learning-essentials_offline.sh
+          echo "Install oneAPI Deep Learning Essentials components"
+          sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept
+
+          echo "Download oneDNN installation package"
+          wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/94c3dbac-0852-45be-a57d-21c204cada3e/intel-onednn-2026.0.2.46_offline.sh -O intel-onednn_offline.sh
+          echo "Install oneDNN components"
+          sudo bash intel-onednn_offline.sh -s -a --silent --eula accept

      - name: Install Level Zero SDK
        shell: bash
@@ -1442,6 +1453,12 @@ jobs:
          name: llama-ui.zip
          path: tools/ui/dist

+      - name: Dependencies
+        id: depends
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libssl-dev
+
      - name: ccache
        uses: ggml-org/ccache-action@v1.2.24
        with:
@@ -1460,7 +1477,6 @@ jobs:
            -DCMAKE_CXX_COMPILER=icpx \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
-            -DLLAMA_OPENSSL=OFF \
            -DGGML_NATIVE=OFF \
            -DGGML_SYCL_F16=${{ matrix.fp16 }}
          time cmake --build build --config Release -j $(nproc)
@@ -6,10 +6,12 @@ General:
 - PR and commit titles format: `<module> : <title>`. Lookup recents for examples
 - Don't try to build or run the code unless you are explicitly asked to do so
 - Use the `gh` CLI tool when querying PRs, issues, or other GitHub resources
- When [MODEL] is needed, first try to get it from the `PI_MODEL_NAME` env var before asking the user
+- When a model name is needed, first try to get it from the `PI_MODEL_NAME` env var before asking the user
+- When a hardware description is needed, first try to get it from the `PI_HARDWARE` env var before asking the user
 - Never read the `AGENTS.md` file

 Coding:
+- Never write inline comments in the code unless the user tells you to
 - When in doubt, always refer to the CONTRIBUTING.md file of the project
 - In `test-backend-ops.cpp`, do not mention specific backends (e.g. Metal, CUDA) in comments
 - When referencing issues or PRs in comments, use the format:
@@ -21,13 +23,22 @@ Pull requests (PRs):
 - Before opening a pull request, ask the user to confirm the description
 - Don't explicitly wrap lines in the PR description (each paragraph and bullet is a single line)
 - When creating a pull request, look for the repository's PR template and follow it
- For the AI usage disclosure section, write "YES. pi:llama.cpp/[MODEL]"
- If `PI_MODEL_NAME` env var is not set, ask the user to tell you what model was used and write it in place of [MODEL]
+- For the AI usage disclosure section, write "YES", then these 4 bullets:
+  - Harness: `pi:llama.cpp`
+  - Hardware: `<hardware>`
+  - Model: `<model>`, linked to its Hugging Face page - the link text is the plain model name
+  - Stats: `<stats>`
+- Run `~/.pi/stats.sh "$PI_SESSION_FILE"` to get `<stats>` and write it as
+  `◷ <time> · ↑ <input tokens> · ↓ <output tokens> · ≡ <messages> · ⌨ <user messages>`,
+  e.g. `◷ 12.3m · ↑ 120k · ↓ 45k · ≡ 87 · ⌨ 6`
+- Write these symbols as literal unicode in the PR description, never `:shortcodes:`, and keep
+  commit trailers ASCII
+- If `PI_MODEL_NAME` or `PI_HARDWARE` is not set, ask the user and use their answer for `<model>` or `<hardware>`
 - Always create the pull requests in draft mode
 - Never reply to review comments or post comments on issues/PRs without explicit permission from the user

 Commits:
- On every commit that you make, include a "Assisted-by: pi:llama.cpp/[MODEL]" tag
+- On every commit that you make, include an `Assisted-by: pi:llama.cpp/<model> (<hardware>)` tag
 - Do not explicitly set the git author in commits - rely on the default git config
 - Always use `--no-gpg-sign` when committing
 - Never `git push` without explicit confirmation from the user
@@ -4,7 +4,7 @@ include(CheckIncludeFileCXX)

 ### llama.cpp version
 set(LLAMA_VERSION_MAJOR 0)
-set(LLAMA_VERSION_MINOR 6)
+set(LLAMA_VERSION_MINOR 7)
 set(LLAMA_VERSION_PATCH 0)
 set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")

@@ -1717,10 +1717,11 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
    ).set_env("LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT").set_examples({LLAMA_EXAMPLE_SERVER}));
    add_opt(common_arg(
        {"-cram", "--cache-ram"}, "N",
-        string_format("set the maximum cache size in MiB (default: %d, -1 - no limit, 0 - disable)"
+        string_format("set the maximum cache size in MiB (default: %d, 0 for decision models, -1 - no limit, 0 - disable)"
            "[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)", params.cache_ram_mib),
        [](common_params & params, int value) {
            params.cache_ram_mib = value;
+            params.cache_ram_explicit = true;
        }
    ).set_env("LLAMA_ARG_CACHE_RAM").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
    add_opt(common_arg(
@@ -1110,6 +1110,12 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
        return common_chat_params_init_gpt_oss;
    }

+    // Muse Spark shares the Muse recipient protocol and adds a commentary recipient.
+    if (src.find("to=commentary") != std::string::npos && src.find("<|eom|>") != std::string::npos) {
+        LOG_DBG("Using specialized template: Muse Spark\n");
+        return common_chat_params_init_muse_spark;
+    }
+
    // Muse Glimmer format using " to=<recipient>" recipients and <|eom|>/<|eot|> message terminators.
    if (src.find("<atem:function_calls>") != std::string::npos && src.find("<|eom|>") != std::string::npos) {
        LOG_DBG("Using specialized template: Muse Glimmer\n");
@@ -640,6 +640,7 @@ struct common_params {
    int32_t kv_unified_per_slot = 0;     // max context per parallel slot; 0 = unset
    int32_t checkpoint_min_step = 8192;  // minimum spacing between context checkpoints
    int32_t cache_ram_mib       = 8192;  // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
+    bool    cache_ram_explicit  = false; // set by --cache-ram, decision models disable the cache otherwise

    std::string public_path   = "";                                                                         // NOLINT
    std::string api_prefix    = "";                                                                         // NOLINT
@@ -600,6 +600,7 @@ value for_statement::execute_impl(context & ctx) const {
    bool noIteration = true;
    for (size_t i = 0; i < filtered_items.size(); i++) {
        JJ_DEBUG("For loop iteration %zu/%zu", i + 1, filtered_items.size());
+        context loop_scope(scope);
        value_object loop_obj = mk_val<value_object>();
        loop_obj->has_builtins = false; // loop object has no builtins
        loop_obj->insert("index", mk_val<value_int>(i + 1));
@@ -611,11 +612,11 @@ value for_statement::execute_impl(context & ctx) const {
        loop_obj->insert("length", mk_val<value_int>(filtered_items.size()));
        loop_obj->insert("previtem", i > 0 ? filtered_items[i - 1] : mk_val<value_undefined>("previtem"));
        loop_obj->insert("nextitem", i < filtered_items.size() - 1 ? filtered_items[i + 1] : mk_val<value_undefined>("nextitem"));
-        scope.set_val("loop", loop_obj);
-        scope_update_fns[i](scope);
+        loop_scope.set_val("loop", loop_obj);
+        scope_update_fns[i](loop_scope);
        try {
            for (auto & stmt : body) {
-                value val = stmt->execute(scope);
+                value val = stmt->execute(loop_scope);
                result->push_back(val);
            }
        } catch (const continue_statement::signal &) {
@@ -74,16 +74,12 @@ struct context {
    }
    ~context() = default;

-    context(const context & parent) : context() {
+    context(const context & parent) :
+        src(parent.src),
+        current_time(parent.current_time),
+        is_get_stats(parent.is_get_stats),
        // inherit variables (for example, when entering a new scope)
-        auto & pvar = parent.env->as_ordered_object();
-        for (const auto & pair : pvar) {
-            set_val(pair.first, pair.second);
-        }
-        current_time = parent.current_time;
-        is_get_stats = parent.is_get_stats;
-        src = parent.src;
-    }
+        env(mk_val<value_object>(*parent.env)) {}

    value get_val(const std::string & name) {
        value default_val = mk_val<value_undefined>(name);
@@ -244,6 +244,8 @@ static std::string json_ensure_ascii_preserving_format(const std::string & json_
    return output;
 }

+static std::string float_repr(double val);
+
 static value tojson(const func_args & args) {
    args.ensure_count(1, 5);
    value val_ascii      = args.get_kwarg_or_pos("ensure_ascii", 1);
@@ -634,6 +636,11 @@ const func_builtins & value_int_t::get_builtins() const {


 const func_builtins & value_float_t::get_builtins() const {
+    // Python's str() of a float, nan and inf stay as they are (tojson writes NaN and Infinity)
+    static const func_handler tostring = [](const func_args & args) -> value {
+        args.ensure_vals<value_float>();
+        return mk_val<value_string>(float_repr(args.get_pos(0)->as_float()));
+    };
    static const func_builtins builtins = {
        {"default", default_value},
        {"abs", [](const func_args & args) -> value {
@@ -650,8 +657,8 @@ const func_builtins & value_float_t::get_builtins() const {
            args.ensure_vals<value_float>();
            return mk_val<value_float>(args.get_pos(0)->as_float());
        }},
-        {"safe", tojson},
-        {"string", tojson},
+        {"safe", tostring},
+        {"string", tostring},
        {"tojson", tojson},
    };
    return builtins;
@@ -1522,10 +1529,10 @@ void global_from_json(context & ctx, const common_json & json_obj, bool mark_inp
    }
 }

-// a float as Python writes it (repr, json.dumps): the shortest digits that read back as the same value
-static std::string float_to_json(double val) {
+// a float as Python's repr() and str() write it: the shortest digits that read back as the same value
+static std::string float_repr(double val) {
    if (!std::isfinite(val)) {
-        return std::isnan(val) ? "nan" : val < 0 ? "-inf" : "inf"; // as before (json.dumps writes NaN and Infinity, str() writes nan and inf)
+        return std::isnan(val) ? "nan" : val < 0 ? "-inf" : "inf";
    }
    const double mag = std::fabs(val);

@@ -1551,6 +1558,7 @@ static std::string float_to_json(double val) {
        if (prec == 16 || reads_back(d)) {
            digits = d;
        } else if (is_pow2 && d.back() != '9') {
+            // a last digit 9 is not bumped (it would carry): no power of 2, from 2^-1074 to 2^1023, needs it
            d.back()++;
            if (reads_back(d)) {
                digits = d;
@@ -1581,6 +1589,17 @@ static std::string float_to_json(double val) {
    return out;
 }

+// a float as json.dumps writes it: nan and inf as NaN and Infinity, which are not valid JSON
+static std::string float_json(double val) {
+    if (std::isnan(val)) {
+        return "NaN";
+    }
+    if (std::isinf(val)) {
+        return val < 0 ? "-Infinity" : "Infinity";
+    }
+    return float_repr(val);
+}
+
 // recursively convert value to JSON string
 // TODO: avoid circular references
 static void value_to_json_internal(std::ostringstream & oss, const value & val, int curr_lvl, int indent, const std::string_view item_sep, const std::string_view key_sep) {
@@ -1598,7 +1617,7 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
    } else if (is_val<value_int>(val)) {
        oss << val->as_int();
    } else if (is_val<value_float>(val)) {
-        oss << float_to_json(val->as_float());
+        oss << float_json(val->as_float());
    } else if (is_val<value_string>(val)) {
        oss << "\"";
        for (char c : val->as_string().str()) {
@@ -1645,7 +1664,8 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
            size_t i = 0;
            for (const auto & pair : obj) {
                oss << indent_str() << (indent > 0 ? std::string(indent, ' ') : "");
-                value_to_json_internal(oss, mk_val<value_string>(pair.first->as_string().str()), curr_lvl + 1, indent, item_sep, key_sep);
+                const std::string key = is_val<value_float>(pair.first) ? float_json(pair.first->as_float()) : pair.first->as_string().str();
+                value_to_json_internal(oss, mk_val<value_string>(key), curr_lvl + 1, indent, item_sep, key_sep);
                oss << key_sep;
                value_to_json_internal(oss, pair.second, curr_lvl + 1, indent, item_sep, key_sep);
                if (i < obj.size() - 1) {
@@ -0,0 +1,146 @@
+#include "parsers.h"
+
+// An assistant turn is rendered as one or more messages, each
+// "<|start|>assistant to=<recipient><|message|>{content}{END}" where END is
+// <|eom|> (more messages follow) or <|eot|> (end of turn):
+//   - chain-of-thought:  to=self, terminated by <|eom|>
+//   - user facing aside: to=commentary, terminated by <|eom|>
+//   - final answer:      to=user, terminated by <|eot|>
+// A commentary message carries content the user sees while the assistant keeps
+// working, typically before or between tool calls.
+// The generation prompt is just "<|start|>assistant"; the model emits its own
+// " to=...<|message|>".
+common_chat_params common_chat_params_init_muse_spark(const common_chat_template &          tmpl,
+                                                               const autoparser::generation_params & inputs) {
+    common_chat_params data;
+
+    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);
+    data.generation_prompt = "<|start|>assistant";
+    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
+    data.supports_thinking = true;
+
+    data.preserved_tokens = {
+        "<|start|>", "<|message|>", "<|eom|>", "<|eot|>",
+        // ATEM tool-call markup emitted on " to=<tool>" turns.
+        "<atem:function_calls>", "<atem:invoke", "<atem:parameter", "</atem:parameter>",
+        "</atem:invoke>", "</atem:function_calls>",
+    };
+
+    data.message_delimiters = {
+        { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },
+        { COMMON_CHAT_ROLE_USER,      "<|start|>user"      },
+        { COMMON_CHAT_ROLE_SYSTEM,    "<|start|>system"    },
+        { COMMON_CHAT_ROLE_TOOL,      "<|start|>tool"      },
+    };
+
+    if (inputs.has_continuation()) {
+        const auto & msg = inputs.continue_msg;
+
+        data.generation_prompt = "<|start|>assistant to=self<|message|>" + msg.reasoning_content;
+        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+            data.generation_prompt += "<|eom|><|start|>assistant to=user<|message|>" + msg.render_content();
+        }
+
+        data.prompt.append(data.generation_prompt);
+    }
+
+    auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
+
+    auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
+    // Constrained grammar whenever tools are offered.
+    auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
+
+    data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+        auto start = p.rule("start", p.literal("<|start|>assistant"));
+
+        if (!extract_reasoning && !include_grammar) {
+            return start + p.content(p.rest());
+        }
+
+        if (extract_reasoning) {
+            p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>"));
+        } else {
+            p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>"));
+        }
+        auto analysis = p.ref("analysis");
+
+        // Commentary is user facing content emitted mid turn, so it reads as
+        // content and always ends on <|eom|>.
+        auto preamble = p.rule("preamble", p.literal(" to=commentary<|message|>") +
+                                               p.content(p.until("<|eom|>")) + p.literal("<|eom|>"));
+
+        auto aside = p.rule("aside", analysis | preamble);
+
+        auto recipient  = p.optional(p.literal(" to=user"));
+        auto final_msg  = p.rule("final", recipient + p.literal("<|message|>") +
+                                              p.content(p.until_one_of({ "<|eot|>", "<|eom|>" })));
+
+        if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
+            auto string_value = p.ac(
+                p.tool_arg_string_value(p.until("</atem:parameter>")) + p.tool_arg_close(p.literal("</atem:parameter>")),
+                "</atem:parameter>");
+
+            auto tool_choice = p.choice();
+            foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
+                const auto &      function = tool.at("function");
+                const std::string name     = function.at("name");
+
+                std::vector<common_peg_parser> arg_rules;
+                foreach_parameter(function, [&](size_t param_index, const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) {
+                    auto value_parser = p.eps();
+                    if (prop.schema->may_be_string()) {
+                        value_parser = string_value;
+                    } else {
+                        value_parser = p.tool_arg_json_value(
+                                p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index) + "-schema", doc, *prop.schema))
+                            + p.tool_arg_close(p.literal("</atem:parameter>"));
+                    }
+
+                    arg_rules.push_back(p.tool_arg(
+                        p.tool_arg_open(p.literal("<atem:parameter name=\"") + p.tool_arg_name(p.literal(prop.name)) + p.literal("\">")) +
+                        value_parser));
+                });
+
+                auto args = p.eps();
+                if (!arg_rules.empty()) {
+                    args = p.zero_or_more(p.choice(arg_rules) + p.space());
+                }
+
+                auto tool_parser = p.tool(
+                    p.tool_open(p.literal(" to=") + p.until("<|message|>") +
+                                p.literal("<|message|><atem:function_calls>") + p.space() +
+                                p.literal("<atem:invoke name=\"") + p.tool_name(p.literal(name)) + p.literal("\">") + p.space())
+                    << p.tool_args(args)
+                    << p.tool_close(p.literal("</atem:invoke>") + p.space() + p.literal("</atem:function_calls>")));
+
+                tool_choice |= p.rule("tool-" + std::to_string(tool_index), tool_parser);
+            });
+
+            auto tool_calls = inputs.parallel_tool_calls
+                ? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice))
+                : p.trigger_rule("tool-call", tool_choice);
+
+
+            if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
+                return p.zero_or_more(start + aside) + start + tool_calls;
+            }
+            auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls);
+            return p.zero_or_more(start + aside) + start + (tool_calls | (final_msg + trailing_calls));
+        }
+
+        return p.zero_or_more(start + aside) + start + final_msg;
+    });
+
+    if (include_grammar) {
+        data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
+        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
+            data.parser.build_grammar(builder, data.grammar_lazy);
+        });
+        data.grammar_triggers = {
+            { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN,
+              "<\\|start\\|>assistant( to=(?!self<\\|message\\|>)(?!commentary<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" },
+        };
+    }
+
+    return data;
+}
@@ -79,6 +79,7 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
 common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);

 common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
+common_chat_params common_chat_params_init_muse_spark(const common_chat_template & tmpl, const autoparser::generation_params & inputs);

 common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, const autoparser::generation_params & inputs);

@@ -19,6 +19,7 @@ set(LLAMA_CHAT_PARSERS_SOURCES
    ${CMAKE_CURRENT_LIST_DIR}/minimax-m3.cpp
    ${CMAKE_CURRENT_LIST_DIR}/ministral3.cpp
    ${CMAKE_CURRENT_LIST_DIR}/muse-glimmer.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/muse-spark.cpp
    ${CMAKE_CURRENT_LIST_DIR}/qwen3-coder.cpp
    ${CMAKE_CURRENT_LIST_DIR}/translate-gemma.cpp
 )
@@ -999,6 +999,12 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
    int32_t     block_size    = 0;
    llama_token mask_token_id = 0;

+    bool    is_lilicorr    = false;
+    int32_t lilicorr_top_k = 0;
+    size_t lilicorr_rows = 0;
+    std::vector<float> lilicorr_embd;
+    std::vector<llama_pos> lilicorr_pos;
+
    bool    is_dflash2     = false;
    bool    is_mrope       = false;
    int32_t selector_top_k = 0;
@@ -1053,6 +1059,16 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {

        selector_top_k = llama_model_dflash_selector_top_k(model_dft);
        is_dflash2     = selector_top_k > 0;
+        char lilicorr_meta[32] = {};
+        lilicorr_top_k = llama_model_meta_val_str(model_dft, "lilicorr.candidate_topk", lilicorr_meta, sizeof(lilicorr_meta)) >= 0
+                ? std::atoi(lilicorr_meta) : 0;
+        is_lilicorr = lilicorr_top_k > 0;
+        if (is_lilicorr) {
+            lilicorr_rows = (size_t) block_size + 1;
+            const size_t lilicorr_slots = (size_t) n_seq * lilicorr_rows;
+            lilicorr_embd.resize(lilicorr_slots * (size_t) n_embd_dec);
+            lilicorr_pos.assign(lilicorr_slots, -1);
+        }
        mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft));

        if (is_dspark && this->params.p_min > 0.0f) {
@@ -1098,7 +1114,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {

        // offload draft sampling to the backend
        backend_chains.assign(n_seq, nullptr);
-        if (this->params.backend_sampling && !is_dflash2) {
+        if (this->params.backend_sampling && !is_dflash2 && !is_lilicorr) {
            for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) {
                llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params());
                llama_sampler_chain_add(chain, llama_sampler_init_top_k(10));
@@ -1118,7 +1134,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
        }

        // DFlash2 reads its selector lattice from h_nextn and never consumes raw logits.
-        llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ !is_dflash2);
+        llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ !is_dflash2 && !is_lilicorr);
        llama_set_causal_attn(ctx_dft, causal_attn); // DFlash needs non-causal attention unless the model says otherwise
    }

@@ -1233,6 +1249,24 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
                            __func__, rc, (int) n_chunk, (int) offset);
                    return false;
                }
+                if (is_lilicorr) {
+                    // Ordinary DFlash can write to the KV cache, but LiliCorr needs to store embeddings across sequences.
+                    const float * projected = llama_get_embeddings_nextn(ctx_dft);
+                    if (projected) {
+                        const size_t seq_offset = (size_t) seq_id * lilicorr_rows;
+                        const llama_pos pos_min = std::max(0, batch_inject.tokens.back().pos[0] - block_size);
+                        for (int32_t i = 0; i < n_chunk; ++i) {
+                            const llama_pos pos = batch_inject.tokens[i].pos[0];
+                            if (pos < pos_min) {
+                                continue;
+                            }
+                            const size_t slot = seq_offset + (size_t) pos % lilicorr_rows;
+                            std::memcpy(lilicorr_embd.data() + slot * (size_t) n_embd_dec,
+                                    projected + (size_t) i * n_embd_dec, (size_t) n_embd_dec * sizeof(float));
+                            lilicorr_pos[slot] = pos;
+                        }
+                    }
+                }
            }
        }

@@ -1260,12 +1294,17 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
            const int32_t n = (int32_t) dp.pos0;

            const int32_t n_draft = params.n_max;
-
-            const int32_t n_block_tokens = n_draft + (is_dspark && sample_from_anchor ? 0 : 1);
+            const int32_t n_block_tokens = is_lilicorr ? block_size : n_draft + (is_dspark && sample_from_anchor ? 0 : 1);
            i_block_beg[seq_id] = batch.size();
            n_block    [seq_id] = n_block_tokens;
            for (int32_t i = 0; i < n_block_tokens; ++i) {
-                batch.add(i == 0 ? dp.id_last : mask_token_id, n + i, seq_id, !is_dflash2);
+                const int32_t idx = batch.add(i == 0 ? dp.id_last : mask_token_id, n + i, seq_id, !is_dflash2 && !is_lilicorr);
+                if (is_lilicorr) {
+                    // For LiliCorr drafting batch, set state embeddings for use in the associated transformer head.
+                    const llama_pos pos = dp.pos0 - 1;
+                    const size_t slot = (size_t) seq_id * lilicorr_rows + (size_t) pos % lilicorr_rows;
+                    batch.set_embd_state(idx, { lilicorr_embd.data() + slot * (size_t) n_embd_dec, 1, (size_t) n_embd_dec });
+                }
            }
        }

@@ -1293,21 +1332,23 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {

            auto & result = *dp.result;

-            if (is_dflash2) {
+            if (is_dflash2 || is_lilicorr) {
+                // Both DFlash and LiliCorr greedily walk the candidate lattice.
                const float * lattice = llama_get_embeddings_nextn(ctx_dft);
                GGML_ASSERT(lattice && "DFlash2 selector produced no lattice");

                int32_t predecessor = 0;
-                for (int32_t i = 1; i < n_block_tokens; ++i) {
+                for (int32_t i = 1; i < (is_dflash2 ? n_block_tokens : params.n_max + 1); ++i) {
                    const float * row = lattice + (size_t) (beg + i) * n_embd_dec;
-                    const float * scores = row + selector_top_k + (size_t) predecessor * selector_top_k;
+                    const int32_t top_k = is_dflash2 ? selector_top_k : lilicorr_top_k;
+                    const float * scores = row + top_k + (size_t) predecessor * top_k;

                    predecessor = (int32_t) std::distance(scores,
-                            std::max_element(scores, scores + selector_top_k));
+                            std::max_element(scores, scores + top_k));
                    if (params.p_min > 0.0f) {
                        // softmax(scores) at the argmax, i.e. 1 / sum(exp(s_k - s_max))
                        float sum = 0.0f;
-                        for (int32_t k = 0; k < selector_top_k; ++k) {
+                        for (int32_t k = 0; k < top_k; ++k) {
                            sum += std::exp(scores[k] - scores[predecessor]);
                        }
                        if (1.0f / sum < params.p_min) {
@@ -59,6 +59,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
    "DeepseekV32ForCausalLM": "deepseek",
    "DFlashDraftModel": "qwen",
    "DFlash2DraftModel": "qwen",
+    "LiLiCorrDraftModel": "nemotron",
    "Qwen3DSparkModel": "qwen",
    "DSparkDraftModel": "qwen",
    "DSparkSpeculator": "qwen",
@@ -198,6 +199,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
    "MiniMaxM3SparseForConditionalGeneration": "minimax",
    "Ministral3ForCausalLM": "mistral3",
    "Mistral3ForConditionalGeneration": "mistral3",
+    "VoxtralRealtimeForConditionalGeneration": "mistral3",
    "MistralForCausalLM": "llama",
    "MixtralForCausalLM": "llama",
    "ModernBertDecisionModel": "bert",
@@ -219,6 +221,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
    "OlmoeForCausalLM": "olmo",
    "MuseGlimmerAssistantModel": "muse_glimmer",
    "MuseGlimmerForConditionalGeneration": "muse_glimmer",
+    "MuseSparkForConditionalGeneration": "muse_spark",
    "OpenELMForCausalLM": "openelm",
    "OpenJevModel": "qwen",
    "OrionForCausalLM": "orion",
@@ -352,8 +355,10 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
    "MiniCPMV4_6ForConditionalGeneration": "minicpm",
    "MiniCPMV4_7ForConditionalGeneration": "minicpm",
    "Mistral3ForConditionalGeneration": "llava",
+    "NemotronH_Omni_Reasoning_V3": "nemotron",
    "NemotronH_Nano_VL_V2": "nemotron",
    "MuseGlimmerForConditionalGeneration": "muse_glimmer",
+    "MuseSparkForConditionalGeneration": "muse_spark",
    "PaddleOCRVisionModel": "ernie",
    "Phi4ForCausalLMV": "phi",
    "Qwen2AudioForConditionalGeneration": "ultravox",
@@ -379,6 +384,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
    "UltravoxModel": "ultravox",
    "UnlimitedOCRForCausalLM": "deepseek",
    "VoxtralForConditionalGeneration": "ultravox",
+    "VoxtralRealtimeForConditionalGeneration": "ultravox",
    "YoutuVLForConditionalGeneration": "youtuvl",
 }

@@ -568,7 +568,7 @@ class ModelBase:
                                tensors_to_remove.append(base_name + "_zero_point")
                else:
                    raise NotImplementedError(f"Quant format {quant_format!r} for method {quant_method!r} is not yet supported")
-            elif quant_method == "modelopt":
+            elif quant_method in ("modelopt", "modelopt_mixed"):
                dequant_fp8()
            elif quant_method is not None:
                raise NotImplementedError(f"Quant method is not yet supported: {quant_method!r}")
@@ -1869,6 +1869,9 @@ class TextModel(ModelBase):
        if chkhsh == "66b8d4e19ab16c3bfd89bce5d785fb7e0155e8648708a1f42077cb9fe002c273":
            # ref: https://huggingface.co/alvarobartt/grok-2-tokenizer
            res = "grok-2"
+        if chkhsh == "f5c965e4198da6e7525072d87d1ecf4c51d981fd486c7c4ec728ef9cec7e03fc":
+            # ref: https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B
+            res = "gpt-4o"
        if chkhsh == "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df":
            # ref: https://huggingface.co/aari1995/German_Semantic_V3
            res = "jina-v2-de"
@@ -239,8 +239,21 @@ class GraniteSwitchModel(GraniteMoeModel):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
-        # the weightless switch reserves one cache slot: one fewer block than num_hidden_layers
-        self.block_count = self.block_count - 1
+        # The weightless switch layer(s) are counted in num_hidden_layers but carry no tensors.
+        # The count is not fixed, nor is it present in hparams, so we derive it here based on tensor block names.
+        layer_ids = {
+            int(m.group(1))
+            for name in self.model_tensors
+            if (m := re.match(r"(?:model\.)?layers\.(\d+)\.", name))
+        }
+        if not layer_ids:
+            raise ValueError("graniteswitch: no 'model.layers.N' tensors found in checkpoint")
+        n_weight_layers = max(layer_ids) + 1
+        logger.info(
+            "gguf: (graniteswitch) num_hidden_layers %s counts %d weightless switch layer(s); using %d blocks",
+            self.hparams["num_hidden_layers"], self.block_count - n_weight_layers, n_weight_layers,
+        )
+        self.block_count = n_weight_layers
        self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)

        self._n_adapters = int(self.hparams["num_adapters"])
@@ -29,6 +29,9 @@ class MellumModel(TextModel):
            self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in self.hparams["layer_types"]])
            logger.info(f"gguf: sliding window pattern length = {len(self.hparams['layer_types'])}")

+        if (norm_topk_prob := self.hparams.get("norm_topk_prob")) is not None:
+            self.gguf_writer.add_expert_weights_norm(norm_topk_prob)
+
    _experts: list[dict[str, Tensor]] | None = None

    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
@@ -66,3 +66,13 @@ class Mistral3Model(TextModel):

    def write(self):
        self.impl.write()
+
+
+@ModelBase.register("VoxtralRealtimeForConditionalGeneration")
+@ModelBase.example("mistralai/Voxtral-Mini-4B-Realtime-2602")
+class VoxtralRealtimeModel(LlamaModel):
+    model_arch = gguf.MODEL_ARCH.MISTRAL3
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        self.gguf_writer.add_realtime_delay_tokens(self.hparams["default_num_delay_tokens"])
@@ -94,7 +94,7 @@ class MuseGlimmerVisionModel(MmprojModel):
        assert self.hparams_vision is not None
        c = self.hparams_vision  # enriched vision_config from get_vision_config()

-        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MUSE_GLIMMER)
+        self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.MUSE_GLIMMER)
        self.gguf_writer.add_vision_attention_layernorm_eps(float(c["layer_norm_eps"]))
        self.gguf_writer.add_vision_spatial_merge_size(int(c["merge_size"]))

@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+from typing import Any, Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+    from torch import Tensor
+
+from .base import ModelBase, MmprojModel, TextModel, gguf
+from .muse_glimmer import MuseGlimmerVisionModel
+
+
+# The reference clamps the squared stream coefficient with this floor.
+RESIDUAL_GATE_EPS = 1e-3
+
+# Block norms hold a delta around one; the output norm holds an absolute gain.
+OFFSET_NORM_SUFFIXES = (
+    "input_layernorm.weight",
+    "pre_feedforward_layernorm.weight",
+    "post_feedforward_layernorm.weight",
+    "pre_expert_norm.weight",
+    "post_expert_norm.weight",
+)
+
+
+def _unpermute_for_rope(tensor: "Tensor", n_heads: int) -> "Tensor":
+    """Invert transformers' `_permute_for_rope`: HF stores Q/K in rotate_half layout,
+    llama.cpp consumes the interleaved (NORM) layout. Vision Q/K carry a bias too."""
+    if tensor.ndim == 1:
+        dim1, = tensor.shape
+        return tensor.view(n_heads, 2, dim1 // n_heads // 2).transpose(1, 2).reshape(dim1)
+    dim1, dim2 = tensor.shape
+    return tensor.view(n_heads, 2, dim1 // n_heads // 2, dim2).transpose(1, 2).reshape(dim1, dim2)
+
+
+@ModelBase.register("MuseSparkForConditionalGeneration")
+@ModelBase.example("meta-models/Muse-Spark-Open-816B-A42B")
+class MuseSparkModel(TextModel):
+    model_arch = gguf.MODEL_ARCH.MUSE_SPARK
+
+    def set_vocab(self):
+        self._set_vocab_gpt2()
+
+        from transformers import AutoTokenizer
+        tok = AutoTokenizer.from_pretrained(self.dir_model)
+        eot_id = tok.convert_tokens_to_ids("<|eot|>")  # ty: ignore[unresolved-attribute]
+        if isinstance(eot_id, int) and eot_id >= 0:
+            self.gguf_writer.add_eot_token_id(eot_id)
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        hparams = self.hparams
+
+        # MetaP resolves to two scalars
+        # The output multiplier lands here and the attention scale rides on the synthesized QK-norm weights
+        self.gguf_writer.add_logit_scale(hparams["output_multiplier"])
+        self.gguf_writer.add_final_logit_softcapping(hparams["final_logit_softcapping"])
+
+        self.gguf_writer.add_post_norm_eps(hparams["post_norm_eps"])
+
+        # Only the sliding layers carry rope, so the graph takes it from the pattern
+        sliding = [t == "sliding_attention" for t in hparams["layer_types"]]
+        theta = hparams["rope_parameters"]["rope_theta"]
+        assert sliding == [bool(t) for t in hparams["layer_rope_theta"]]
+        assert all(t in (0, theta) for t in hparams["layer_rope_theta"])
+        assert all(t == "sparse" for t in hparams["mlp_layer_types"])
+
+        self.gguf_writer.add_sliding_window(hparams["sliding_window"])
+        self.gguf_writer.add_rope_freq_base_swa(theta)
+        self.gguf_writer.add_sliding_window_pattern(sliding)
+
+        # Experts live in a latent space narrower than the residual stream.
+        self.gguf_writer.add_moe_latent_size(hparams["moe_hidden_size"])
+        self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
+        self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"])
+        self.gguf_writer.add_expert_weights_norm(True)
+        # the config carries no score function, the reference hardcodes sigmoid
+        self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        if name.endswith(OFFSET_NORM_SUFFIXES):
+            data_torch = data_torch + 1.0
+
+        if name.endswith(".self_attn.q_proj.weight"):
+            assert bid is not None
+            n_head = self.hparams["num_attention_heads"]
+            head_dim = self.hparams["head_dim"]
+
+            yield self.map_tensor_name(name), _unpermute_for_rope(data_torch, n_head)
+
+            # Synthesize QK-norm to carry the MetaP scale so the graph keeps the plain 1 / sqrt(head_dim)
+            ones = torch.ones(head_dim, dtype=torch.float32)
+            q_norm = ones * self.hparams["qk_scale_factor"]
+            yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q_NORM, bid), q_norm
+            yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K_NORM, bid), ones
+            return
+
+        if name.endswith(".self_attn.k_proj.weight"):
+            n_head_kv = self.hparams["num_key_value_heads"]
+            yield self.map_tensor_name(name), _unpermute_for_rope(data_torch, n_head_kv)
+            return
+
+        # gate_up_proj fuses both projections over the output dim, gate first.
+        if name.endswith(".mlp.experts.gate_up_proj"):
+            assert bid is not None
+            n_ff = data_torch.shape[1] // 2
+            yield self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid), data_torch[:, :n_ff, :]
+            yield self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid), data_torch[:, n_ff:, :]
+            return
+
+        if name.endswith(".mlp.experts.down_proj"):
+            assert bid is not None
+            yield self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN_EXP, bid), data_torch
+            return
+
+        # The router scales its logits by the MetaP output multiplier before the sigmoid
+        # Fold it into the weight to keep a plain matmul
+        if name.endswith(".mlp.gate.weight"):
+            data_torch = data_torch.float() * self.hparams["output_multiplier"]
+            yield from super().modify_tensors(data_torch, name, bid)
+            return
+
+        # Norm preserving residual gate: alpha scales the stream, beta the branch, with alpha^2 + beta^2 = 1
+        # Both follow from the weight, so resolve them here
+        if name.endswith("_residual_gate.gate"):
+            assert bid is not None
+            attn = "post_attention" in name
+
+            w = data_torch.float() / self.hparams["residual_gate_temperature"]
+            beta = torch.sigmoid(w)
+            alpha_sq = torch.sigmoid(-w) * (1.0 + beta)
+            alpha = torch.sqrt(torch.clamp(alpha_sq, min=RESIDUAL_GATE_EPS))
+
+            t_alpha = gguf.MODEL_TENSOR.ATTN_RESID_ALPHA if attn else gguf.MODEL_TENSOR.FFN_RESID_ALPHA
+            t_beta = gguf.MODEL_TENSOR.ATTN_RESID_BETA if attn else gguf.MODEL_TENSOR.FFN_RESID_BETA
+            yield self.format_tensor_name(t_alpha, bid), alpha
+            yield self.format_tensor_name(t_beta, bid), beta
+            return
+
+        yield from super().modify_tensors(data_torch, name, bid)
+
+
+@ModelBase.register("MuseSparkForConditionalGeneration")
+@ModelBase.example("meta-models/Muse-Spark-Open-816B-A42B")
+class MuseSparkVisionModel(MuseGlimmerVisionModel):
+    has_vision_encoder = True # same vision encoder as Muse Glimmer
+    has_audio_encoder = True
+
+    def get_audio_config(self) -> dict[str, Any]:
+        c = self.global_config["audio_config"]
+        # the base class wants one width and one layer count: use the last stage width and the total number of conv blocks
+        return {
+            **c,
+            "hidden_size": c["hidden_sizes"][-1],
+            "intermediate_size": int(c["hidden_sizes"][-1] * c["mlp_ratio"]),
+            "num_attention_heads": 1,
+            "num_hidden_layers": sum(c["depths"]),
+        }
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+
+        assert self.hparams_audio
+        a = self.hparams_audio
+
+        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.MUSE_SPARK_AUDIO)
+        # the input is the raw waveform, one channel takes the place of the mel bins
+        self.gguf_writer.add_audio_num_mel_bins(1)
+        self.gguf_writer.add_audio_attention_layernorm_eps(float(a["rms_norm_eps"]))
+        self.gguf_writer.add_audio_conv_kernel_size(int(a["conv_kernel_size"]))
+        self.gguf_writer.add_audio_hidden_sizes([int(x) for x in a["hidden_sizes"]])
+        self.gguf_writer.add_audio_patch_sizes([int(x) for x in a["patch_sizes"]])
+        self.gguf_writer.add_audio_depths([int(x) for x in a["depths"]])
+
+    @classmethod
+    def filter_tensors(cls, item):
+        name, gen = item
+        if name.startswith("model.audio_tower."):
+            return MmprojModel.filter_tensors((name, gen))
+        return super().filter_tensors((name, gen))
+
+    _AUDIO_HEAD = {
+        "model.audio_tower.norm": gguf.MODEL_TENSOR.A_PRE_NORM,
+        "model.audio_tower.proj": gguf.MODEL_TENSOR.A_MMPROJ_FC,
+        "model.audio_tower.post_proj_norm": gguf.MODEL_TENSOR.A_POST_NORM,
+    }
+
+    def _audio_tensor(self, data_torch, name, suffix):
+        assert self.hparams_audio
+
+        stem = name[: -len(suffix) - 1]
+
+        if stem in self._AUDIO_HEAD:
+            return self.format_tensor_name(self._AUDIO_HEAD[stem], suffix="." + suffix), data_torch
+
+        parts = stem.split(".")
+        stage = int(parts[3])
+
+        if parts[4] == "patch_embedder":
+            key = (gguf.MODEL_TENSOR.A_ENC_PATCH_PROJ if parts[5] == "proj"
+                   else gguf.MODEL_TENSOR.A_ENC_PATCH_NORM)
+            return self.format_tensor_name(key, bid=stage, suffix="." + suffix), data_torch
+
+        # conv blocks are numbered per stage upstream and flat here
+        depths = [int(d) for d in self.hparams_audio["depths"]]
+        bid = sum(depths[:stage]) + int(parts[5])
+        tail = ".".join(parts[6:])
+        key = {
+            "conv": gguf.MODEL_TENSOR.A_ENC_CONV1D,
+            "norm": gguf.MODEL_TENSOR.A_ENC_CONV1D_NORM,
+            "mlp.fc1": gguf.MODEL_TENSOR.A_ENC_FFN_UP,
+            "mlp.fc2": gguf.MODEL_TENSOR.A_ENC_FFN_DOWN,
+        }[tail]
+        return self.format_tensor_name(key, bid=bid, suffix="." + suffix), data_torch
+
+    def modify_tensors(self, data_torch, name, bid):
+        if name.startswith("model.audio_tower."):
+            yield self._audio_tensor(data_torch, name, name.rpartition(".")[2])
+            return
+        yield from super().modify_tensors(data_torch, name, bid)
@@ -1,16 +1,118 @@
 from __future__ import annotations

-from typing import Any, Callable, Iterable, TYPE_CHECKING
+from typing import Any, Callable, Iterable, Sequence, TYPE_CHECKING

+import numpy as np
 import torch

 if TYPE_CHECKING:
    from pathlib import Path
    from torch import Tensor

-from .base import MmprojModel, ModelBase, TextModel, gguf, logger
+from .base import MmprojModel, ModelBase, ModelType, TextModel, get_model_architecture, gguf, logger

 from .granite import GraniteHybridModel
+from .qwen import DFlashModel
+
+
+@ModelBase.register("LiLiCorrDraftModel")
+class LiLiCorrDraftModel(DFlashModel):
+    model_arch = gguf.MODEL_ARCH.DFLASH
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        prefix = "model.lilicorr."
+        if not any(name.startswith(prefix) for name in self.model_tensors):
+            raise ValueError("no lilicorr.* tensors found")
+        dflash_config = self.hparams["dflash_config"]
+        self.lilicorr_geometry = {
+            "hidden_size": int(dflash_config["lilicorr_hidden_size"]),
+            "candidate_topk": int(dflash_config["lilicorr_candidate_topk"]),
+            "num_heads": int(dflash_config["lilicorr_num_heads"]),
+            "num_layers": int(dflash_config["lilicorr_num_layers"]),
+            "factor_dim": int(dflash_config["lilicorr_factor_dim"]),
+            "mlp_hidden_size": self.model_tensors[prefix + "layers.0.mlp.up_proj.weight"]().shape[0],
+        }
+        self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count + self.lilicorr_geometry["num_layers"] + 1)
+        block = int(dflash_config.get("block_size", self.hparams.get("block_size", 16)))
+        topk = self.lilicorr_geometry["candidate_topk"]
+        heads = self.lilicorr_geometry["num_heads"]
+        nodes = (block - 1) * topk
+        relative = gguf.LazyBase.to_eager(self.model_tensors.pop(prefix + "relative_slot_bias")()).float().cpu().numpy()
+        same = gguf.LazyBase.to_eager(self.model_tensors.pop(prefix + "same_slot_bias")()).float().cpu().numpy()
+        bias = np.empty((heads, nodes, nodes), dtype=np.float32)
+        for q in range(nodes):
+            q_slot = q // topk
+            for k in range(nodes):
+                k_slot = k // topk
+                bias[:, q, k] = relative[:, q_slot - k_slot + block - 1]
+                if q_slot == k_slot:
+                    bias[:, q, k] += same
+        self.model_tensors[prefix + "attention_bias.bias"] = lambda bias=bias: torch.from_numpy(bias)
+        for name in ("slot_embedding", "rank_embedding"):
+            gen = self.model_tensors.pop(prefix + name)
+            self.model_tensors[prefix + name + ".weight"] = lambda gen=gen: gen().reshape(-1, self.lilicorr_geometry["hidden_size"])
+        for i in range(self.lilicorr_geometry["num_layers"]):
+            layer_prefix = prefix + f"layers.{i}.attn.in_proj"
+            for suffix in ("weight", "bias"):
+                self.model_tensors[layer_prefix + "." + suffix] = self.model_tensors.pop(layer_prefix + "_" + suffix)
+        dflash_config["selector_rank"] = 0
+        dflash_config["selector_top_k"] = 0
+
+    def set_vocab(self):
+        if self.target_model_dir is None:
+            raise ValueError("LiLiCorr draft requires --target-model-dir")
+        from . import get_model_class
+        target_text_config = ModelBase.load_hparams(self.target_model_dir, False)
+        target_text_config = {**target_text_config, **target_text_config.get("text_config", {})}
+        target_arch = get_model_architecture(target_text_config, ModelType.TEXT)
+        target_cls = get_model_class(target_arch)
+        original_dir = self.dir_model
+        self.dir_model = self.target_model_dir
+        try:
+            if target_arch == "NemotronHForCausalLM":
+                self.is_moe = "num_experts_per_tok" in target_text_config
+            target_cls.set_vocab(self)  # ty: ignore[unresolved-attribute]
+        finally:
+            self.dir_model = original_dir
+        self.gguf_writer.add_mask_token_id(self.hparams["dflash_config"]["mask_token_id"])
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        dflash_config = self.hparams["dflash_config"]
+        self.gguf_writer.add_lilicorr_hidden_size(int(self.lilicorr_geometry["hidden_size"]))
+        self.gguf_writer.add_lilicorr_candidate_topk(int(self.lilicorr_geometry["candidate_topk"]))
+        self.gguf_writer.add_lilicorr_num_heads(int(self.lilicorr_geometry["num_heads"]))
+        self.gguf_writer.add_lilicorr_num_layers(int(self.lilicorr_geometry["num_layers"]))
+        self.gguf_writer.add_lilicorr_factor_dim(int(self.lilicorr_geometry["factor_dim"]))
+        self.gguf_writer.add_lilicorr_mlp_hidden_size(int(self.lilicorr_geometry["mlp_hidden_size"]))
+        self.gguf_writer.add_lilicorr_vector_eps(float(dflash_config["lilicorr_vector_eps"]))
+        self.gguf_writer.add_lilicorr_logit_scale(float(dflash_config["lilicorr_logit_scale"]))
+
+    def map_tensor_name(self, name: str, try_suffixes: Sequence[str] = (".weight", ".bias")) -> str:
+        if name.startswith("model.lilicorr.layers."):
+            parts = name.split(".", 4)
+            if not parts[4].startswith("attn.in_proj."):
+                parts[3] = str(self.block_count + int(parts[3]))
+            name = ".".join(parts)
+        elif name.startswith("model.lilicorr.feature_norm."):
+            feature_bid = self.block_count + self.lilicorr_geometry["num_layers"]
+            name = name.replace("model.lilicorr.feature_norm.", f"model.lilicorr.feature_mlp.{feature_bid}.norm.")
+        elif name.startswith("model.lilicorr.feature_mlp."):
+            feature_bid = self.block_count + self.lilicorr_geometry["num_layers"]
+            name = name.replace("model.lilicorr.feature_mlp.", f"model.lilicorr.feature_mlp.{feature_bid}.")
+        return super().map_tensor_name(name, try_suffixes)
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        if name.startswith("model.lilicorr."):
+            yield self.map_tensor_name(name), data_torch
+            return
+        yield from super().modify_tensors(data_torch, name, bid)
+
+    def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
+        if name.startswith("model.lilicorr."):
+            return gguf.GGMLQuantizationType.F32
+        return super().tensor_force_quant(name, new_name, bid, n_dims)


@ModelBase.register(
@@ -152,6 +254,113 @@ class NemotronNanoV2VLModel(MmprojModel):
            yield mapped_name, tensor


+@ModelBase.register("NemotronH_Omni_Reasoning_V3")
+class NemotronOmniVLModel(NemotronNanoV2VLModel):
+    _omni_tensor_prefixes = (
+        "vision_model.embeddings.",
+        "vision_model.encoder.layer.",
+        "vision_projector.mlp1.",
+    )
+
+    def get_vision_config(self) -> dict[str, Any] | None:
+        vision_config = dict(self.global_config["vision_config"])
+        vision_config["intermediate_size"] = int(vision_config["hidden_size"] * vision_config["mlp_ratio"])
+        vision_config["image_size"] = 512  # warmup only; the full position table is resized at runtime
+        return vision_config
+
+    def set_gguf_parameters(self):
+        preprocessor_config = self.preprocessor_config
+
+        preprocessor_config["image_mean"] = preprocessor_config["norm_mean"]
+        preprocessor_config["image_std"] = preprocessor_config["norm_std"]
+
+        MmprojModel.set_gguf_parameters(self)
+        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.NEMOTRON_OMNI)
+        self.gguf_writer.add_vision_attention_layernorm_eps(1e-6)
+        self.gguf_writer.add_vision_use_gelu(True)
+        downsample_ratio = self.global_config.get("downsample_ratio", 0.5)
+        self.gguf_writer.add_vision_projector_scale_factor(int(1.0 / downsample_ratio))
+        patch_area = self.hparams["patch_size"] ** 2
+        self.gguf_writer.add_vision_min_pixels(preprocessor_config["min_num_patches"] * patch_area)
+        self.gguf_writer.add_vision_max_pixels(preprocessor_config["max_num_patches"] * patch_area)
+
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        name, gen = item
+
+        # Transformers can rename RADIO and projector tensors in Omni checkpoints.
+        # Accept both checkpoint layouts.
+        if name == "vision_model.summary_idxs":
+            # This buffer is only used by RadioModel.summary. The multimodal
+            # projector consumes RadioModel.features for both images and video.
+            return None
+        if name.startswith(cls._omni_tensor_prefixes):
+            return name, gen
+
+        if "radio_model.model.patch_generator.video_embedder" in name or name.startswith("vision_projector.vision_final_layernorm."):
+            return MmprojModel.filter_tensors((name, gen))
+
+        return super().filter_tensors(item)
+
+    def tensor_force_quant(self, name, new_name, bid, n_dims):
+        if new_name in ("v.position_embd.weight", "v.class_embd"):
+            return gguf.GGMLQuantizationType.F32
+        return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        if name.startswith(self._omni_tensor_prefixes):
+            if name == "vision_model.embeddings.video_patch_projection.weight":
+                data_torch = data_torch.reshape(
+                    self.hparams["hidden_size"],
+                    6,
+                    self.hparams["patch_size"],
+                    self.hparams["patch_size"],
+                )
+                yield gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1", data_torch
+                return
+
+            if name.startswith("vision_projector.mlp1."):
+                projector_mapping = {
+                    "vision_projector.mlp1.norm.weight": "mlp1.0.weight",
+                    "vision_projector.mlp1.linear1.weight": "mlp1.1.weight",
+                    "vision_projector.mlp1.linear2.weight": "mlp1.3.weight",
+                }
+                name = projector_mapping[name]
+            elif name == "vision_model.embeddings.patch_projection.weight":
+                data_torch = data_torch.reshape(
+                    self.hparams["hidden_size"],
+                    3,
+                    self.hparams["patch_size"],
+                    self.hparams["patch_size"],
+                )
+            elif name == "vision_model.embeddings.position_embedding":
+                data_torch = data_torch.squeeze(0)
+                name += ".weight"
+            elif ".layer_scale" in name and name.endswith(".lambda1"):
+                name = name.removesuffix("lambda1") + "weight"
+
+            yield from MmprojModel.modify_tensors(self, data_torch, name, bid)
+            return
+
+        if "patch_generator.pos_embed" in name:
+            # Keep the full table for runtime interpolation.
+            yield from MmprojModel.modify_tensors(self, data_torch, name, bid)
+            return
+
+        if "patch_generator.video_embedder" in name:
+            patch_size = self.hparams["patch_size"]
+            n_embd = self.hparams["hidden_size"]
+            temporal_patch_size = self.global_config["video_temporal_patch_size"]
+            if temporal_patch_size != 2:
+                raise ValueError("Current implementation only supports a temporal patch size of 2")
+            data_torch = data_torch.reshape(n_embd, temporal_patch_size * 3, patch_size, patch_size)
+            new_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + ".weight.1"
+            yield new_name, data_torch
+            return
+
+        yield from super().modify_tensors(data_torch, name, bid)
+
+
@ModelBase.register("NemotronForCausalLM")
@ModelBase.example("nvidia/Minitron-4B-Base")
 class NemotronModel(TextModel):
@@ -283,6 +492,8 @@ class NemotronHModel(GraniteHybridModel):
    @classmethod
    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
        name, gen = item
+        if name.startswith("language_model."):
+            name = name[len("language_model."):]
        if name.startswith("mtp."):
            # --no-mtp: drop the MTP head entirely
            if cls.no_mtp:
@@ -432,6 +643,14 @@ class NemotronHModel(GraniteHybridModel):
        "mtp.layers.1.final_layernorm.weight": "model.layers.{bid}.shared_head.norm.weight",
    }

+    def _generate_nvfp4_tensors(self):
+        if self._mtp_bid is not None:
+            for name in list(self.model_tensors):
+                if name.startswith("mtp.layers.1.mixer"):
+                    new_name = f"backbone.layers.{self._mtp_bid}.{name.split('.', 3)[3]}"
+                    self.model_tensors[new_name] = self.model_tensors.pop(name)
+        super()._generate_nvfp4_tensors()
+
    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
        #   mtp.layers.0: NextN input fusion + attention
        #   mtp.layers.1: MoE + final head norm
@@ -6,6 +6,7 @@ if TYPE_CHECKING:
    from torch import Tensor

 from .base import MmprojModel, ModelBase, TextModel, gguf
+from .llama import LlamaModel


@ModelBase.register("UltravoxModel")
@@ -208,3 +209,47 @@ class AudioFlamingo3WhisperEncoderModel(WhisperEncoderModel):
            # Was trained in BF16, being safe, avoiding quantizing to FP16
            return gguf.GGMLQuantizationType.F32
        return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+
+@ModelBase.register("VoxtralRealtimeForConditionalGeneration")
+@ModelBase.example("mistralai/Voxtral-Mini-4B-Realtime-2602")
+class VoxtralRealtimeEncoderModel(MmprojModel):
+    has_vision_encoder = False
+    has_audio_encoder = True
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        assert self.hparams_audio is not None
+        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.VOXTRAL_RT)
+        self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["num_mel_bins"])
+        self.gguf_writer.add_audio_head_dim(self.hparams_audio["head_dim"])
+        self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams_audio.get("rms_norm_eps", 1e-5))
+        self.gguf_writer.add_audio_window_size(self.hparams_audio["sliding_window"])
+        self.gguf_writer.add_audio_stack_factor(self.global_config["downsample_factor"])
+
+    def tensor_force_quant(self, name, new_name, bid, n_dims):
+        if ".conv" in name and ".weight" in name:
+            return gguf.GGMLQuantizationType.F16
+        return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        name, gen = item
+
+        # prevent clash naming with vision tensors
+        if name.startswith("multi_modal_projector"):
+            name = "audio." + name
+
+        return super().filter_tensors((name, gen))
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        if "conv1.bias" in name or "conv2.bias" in name:
+            data_torch = data_torch.unsqueeze(-1)
+
+        # HF stores q/k in rotate-half layout, the encoder uses interleaved (normal) RoPE
+        if ".self_attn.q_proj." in name or ".self_attn.k_proj." in name:
+            assert self.hparams_audio is not None
+            n_head = self.hparams_audio["num_attention_heads"]
+            data_torch = LlamaModel.permute(data_torch, n_head, n_head)
+
+        yield from super().modify_tensors(data_torch, name, bid)
@@ -189,6 +189,8 @@ pre_computed_hashes = [
    {"name": "qwen2",     "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B", "chkhsh": "d4540891389ea895b53b399da6ac824becc30f2fba0e9ddbb98f92e55ca0e97c"},
    {"name": "qwen35",    "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openbmb/MiniCPM-V-4_6", "chkhsh": "1444df51289cfa8063b96f0e62b1125440111bc79a52003ea14b6eac7016fd5f"},
    {"name": "grok-2",    "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/alvarobartt/grok-2-tokenizer", "chkhsh": "66b8d4e19ab16c3bfd89bce5d785fb7e0155e8648708a1f42077cb9fe002c273"},
+    # muse-spark uses the l4_202k tokenizer, whose pre-tokenizer regex is gpt-4o's
+    {"name": "gpt-4o",    "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "chkhsh": "f5c965e4198da6e7525072d87d1ecf4c51d981fd486c7c4ec728ef9cec7e03fc"},
    # jina-v2-de variants
    {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"},
    {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"},
@@ -134,6 +134,18 @@ Writing high-performance operators for Hexagon requires following specific guide
 - **Distinction**: `octx->src[N]` pointers *can* be NULL by design and must be checked when optional.
  Examples include attention masks, optional bias or weights in fused kernels, and frequency factors.

+### `supports_op` Independence from Buffer and Repack State
+
+- `supports_op` (and op-specific checkers such as `supported_mul_mat()`, `supported_get_rows()`) runs during GGML graph scheduling **before** buffers are allocated and before tensor data is populated.
+- **Never check buffer usage or repacking state in `supports_op`**:
+  - `t->buffer` may be NULL or uninitialized.
+  - `t->data` may be NULL.
+  - `buffer->usage` is unknown at scheduling time.
+  - `is_tiled` or `extra->flags & GGML_HEXAGON_TENSOR_TILED` is not set yet.
+- Operator support decisions must depend strictly on static tensor properties: operand types (`t->type`), shapes (`t->ne`), strides (`t->nb`), and operation parameters.
+- Repacking (converting weights to tiled layout) is an internal optimization applied during `set_tensor()` or rt-repack. Non-weight tensors or test harness tensors with the same type remain in flat layout.
+- Therefore, any supported type (such as BF16, F16, quantized types) must be supported both when repacked into tiles and when stored flat in memory.
+
 ### Multiline Macro Formatting

 - Keep trailing backslashes in multiline `#define` macros cleanly aligned to a consistent column.
@@ -190,7 +202,7 @@ sessions.
 Common partitioning logic is factored into reusable inline helpers in
 [`htp-tensor.h`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h):

-1. [`htp_tensor_mdev_rows_per_chunk`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L71):
+1. [`htp_tensor_mdev_rows_per_chunk`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L104):
   Determines the minimum number of rows per chunk so that the chunk byte size is a multiple of 128 bytes:

   ```
@@ -201,7 +213,7 @@ Common partitioning logic is factored into reusable inline helpers in
   Returns `false` if the tensor cannot be safely row-partitioned (such as unaligned base pointer, permuted layout,
   or non-128-byte aligned outer strides).

-2. [`htp_tensor_mdev_partition`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L98):
+2. [`htp_tensor_mdev_partition`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L131):
   Calculates the per-device work range `struct htp_tensor_mdev_range { uint32_t start; uint32_t count; }` given
   `total_units`, `units_per_chunk`, `mdev_idx`, `mdev_count`, and the precomputed `mdev_count_div`.
   Handles chunk distribution across devices, assigns remainder units to the last device, and automatically triggers
@@ -240,11 +252,11 @@ if (nrows == 0) {
 For flat element-wise operations that write directly to DDR:
 - Partition total linear elements N = ne0 * ne1 * ne2 * ne3 in 128-byte cache line chunks (`elems_per_line = (elem_size == 4) ? 32 : 64`).
 - Requires strict 1D contiguity:
-  [`htp_tensor_is_contiguous(dst, elem_size)`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L32)
+  [`htp_tensor_is_contiguous(dst, elem_size)`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L65)
  and 128-byte aligned destination pointer
-  [`htp_tensor_mdev_data_aligned(dst)`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L51).
+  [`htp_tensor_mdev_data_aligned(dst)`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L84).
 - If contiguous and aligned, pass `elems_per_line` to
-  [`htp_tensor_mdev_partition`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L98);
+  [`htp_tensor_mdev_partition`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h#L131);
  otherwise pass 0 to trigger Device 0 fallback.

 ### Single-Device Fallback (Device 0)
@@ -312,7 +324,7 @@ Multi-device execution synchronizes worker sessions through atomic fence slots a
  (Output Slice 0)                             (Output Slice 1)
       |                                            |
  Tensor Cache Flush                           Tensor Cache Flush
-  (htp_tensor_flush_all)                       (htp_tensor_flush_all)
+  (htp_dcache_flush_dirty_tensors)             (htp_dcache_flush_dirty_tensors)
       |                                            |
  Post-Op/Batch Barrier ---------------------- Post-Op/Batch Barrier
  (htp_mdev_group_barrier)                     (htp_mdev_group_barrier)
@@ -350,8 +362,8 @@ Multi-device execution synchronizes worker sessions through atomic fence slots a
 ### Tensor Cache Flush and Pipeline Completion

 - In the kernel, ensure all pushed DMA operations have been popped in strict FIFO order to drain the queue.
- Use [`htp_tensor_flush_all()`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h) to flush specific dirty tensors back to DDR:
-  - [`htp_tensor_flush_all()`](../../../ggml/src/ggml-hexagon/htp/htp-tensor.h) flushes modified tensor address ranges, or the
+- Use [`htp_dcache_flush_dirty_tensors()`](../../../ggml/src/ggml-hexagon/htp/htp-dcache.h) to flush specific dirty tensors back to DDR:
+  - [`htp_dcache_flush_dirty_tensors()`](../../../ggml/src/ggml-hexagon/htp/htp-dcache.h) flushes modified tensor address ranges, or the
    full D-cache when their total size exceeds the flush threshold, ensuring peer devices and the host CPU observe consistent
    data in DDR.
 - Never signal completion before all DMA transfers are drained and dirty tensor flushes have completed.
@@ -4,15 +4,18 @@
 #  SPDX-License-Identifier: MIT

 print_usage() {
-    echo "Usage: ./build.sh [fp32|fp16] [--help]"
+    echo "Usage: ./build.sh [fp32|fp16] [ssl|nossl] [--help]"
    echo ""
    echo "Options:"
    echo "  fp32    Build with FP32 precision (default)"
    echo "  fp16    Build with FP16 precision (faster for long-prompt inference)"
+    echo "  ssl     Build with OpenSSL (default)"
+    echo "  nossl   Build without OpenSSL"
    echo "  --help  Print this help message"
 }

 PRECISION=fp32
+SSL=ON

 for arg in "$@"; do
    case "$arg" in
@@ -23,6 +26,12 @@ for arg in "$@"; do
        fp32|fp16)
            PRECISION="$arg"
            ;;
+        ssl)
+            SSL=ON
+            ;;
+        nossl)
+            SSL=OFF
+            ;;
        *)
            echo "Error: unknown option '$arg'"
            print_usage
@@ -37,10 +46,13 @@ source /opt/intel/oneapi/setvars.sh

 if [ "$PRECISION" = "fp16" ]; then
    #for FP16
-    cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON -DLLAMA_OPENSSL=ON # faster for long-prompt inference
+    echo "cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON -DLLAMA_OPENSSL=$SSL # faster for long-prompt inference
+"
+    cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON -DLLAMA_OPENSSL="$SSL" # faster for long-prompt inference
 else
    #for FP32
-    cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_OPENSSL=ON
+    echo "cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_OPENSSL=$SSL"
+    cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_OPENSSL="$SSL"
 fi

 #build example/main
@@ -50,4 +62,5 @@ fi
 #cmake --build . --config Release --target llama-bench

 #build all binary
+echo "cmake --build . --config Release -j$((($(nproc)+1)/2)) -v"
 cmake --build . --config Release -j$((($(nproc)+1)/2)) -v
@@ -3,23 +3,62 @@
 ::  Copyright (C) 2024 Intel Corporation
 ::  SPDX-License-Identifier: MIT

-IF /I "%1"=="--help" (
-    echo Usage: win-build-sycl.bat [fp32^|fp16] [--help]
-    echo.
-    echo Options:
-    echo   fp32    Build with FP32 precision ^(default^)
-    echo   fp16    Build with FP16 precision ^(faster for long-prompt inference^)
-    echo   --help  Print this help message
-    exit /B 0
-)
+SET "PRECISION=fp32"
+SET "SSL=ON"
+SET "OPENSSL_OPTION="

-SET PRECISION=%1
-IF "%PRECISION%"=="" SET PRECISION=fp32
-IF /I NOT "%PRECISION%"=="fp32" IF /I NOT "%PRECISION%"=="fp16" (
-    echo Error: invalid value '%PRECISION%'. Use 'fp32' or 'fp16'.
-    echo Usage: win-build-sycl.bat [fp32^|fp16] [--help]
-    exit /B 1
-)
+:PARSE_ARGS
+IF "%~1"=="" GOTO ARGS_DONE
+IF /I "%~1"=="--help" GOTO HELP
+IF /I "%~1"=="fp32" GOTO ARG_FP32
+IF /I "%~1"=="fp16" GOTO ARG_FP16
+IF /I "%~1"=="ssl" GOTO ARG_SSL
+IF /I "%~1"=="nossl" GOTO ARG_NOSSL
+GOTO INVALID_ARG
+
+:ARG_FP32
+SET "PRECISION=fp32"
+GOTO NEXT_ARG
+
+:ARG_FP16
+SET "PRECISION=fp16"
+GOTO NEXT_ARG
+
+:ARG_SSL
+SET "SSL=ON"
+SET "OPENSSL_OPTION="
+GOTO NEXT_ARG
+
+:ARG_NOSSL
+SET "SSL=OFF"
+SET "OPENSSL_OPTION=-DLLAMA_OPENSSL=OFF"
+GOTO NEXT_ARG
+
+:NEXT_ARG
+SHIFT
+GOTO PARSE_ARGS
+
+:HELP
+SET "ARG_ERROR=0"
+GOTO PRINT_USAGE
+
+:INVALID_ARG
+SET "ARG_ERROR=1"
+echo Error: unknown option '%~1'
+
+:PRINT_USAGE
+echo Usage: win-build-sycl.bat [fp32^|fp16] [ssl^|nossl] [--help]
+echo.
+echo Options:
+echo   fp32      Build with FP32 precision ^(default^)
+echo   fp16      Build with FP16 precision ^(faster for long-prompt inference^)
+echo   ssl       Build with BoringSSL ^(default^)
+echo   nossl     Build without BoringSSL
+echo   --help    Print this help message
+IF "%ARG_ERROR%"=="1" exit /B 1
+exit /B 0
+
+:ARGS_DONE

 IF not exist build (mkdir build)
 cd build
@@ -31,10 +70,10 @@ if %errorlevel% neq 0 goto ERROR
 IF /I "%PRECISION%"=="fp16" (
    ::  for FP16
    ::  faster for long-prompt inference
-    cmake -G "MinGW Makefiles" .. -DLLAMA_OPENSSL=OFF -DGGML_SYCL=ON -DCMAKE_CXX_COMPILER=icx -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DGGML_SYCL_F16=ON
+    cmake -G "MinGW Makefiles" .. -DLLAMA_BUILD_BORINGSSL=%SSL% %OPENSSL_OPTION% -DGGML_SYCL=ON -DCMAKE_CXX_COMPILER=icx -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DGGML_SYCL_F16=ON
 ) ELSE (
    ::  for FP32
-    cmake -G "Ninja" .. -DLLAMA_OPENSSL=OFF -DGGML_SYCL=ON -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=icx -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release
+    cmake -G "Ninja" .. -DLLAMA_BUILD_BORINGSSL=%SSL% %OPENSSL_OPTION% -DGGML_SYCL=ON -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=icx -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release
 )
 if %errorlevel% neq 0 goto ERROR

@@ -4,7 +4,7 @@ project("ggml" C CXX ASM)

 ### GGML Version
 set(GGML_VERSION_MAJOR 0)
-set(GGML_VERSION_MINOR 26)
+set(GGML_VERSION_MINOR 27)
 set(GGML_VERSION_PATCH 0)
 set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")

@@ -3548,6 +3548,73 @@ static void ggml_compute_forward_swiglu_oai_f32(
    }
 }

+static void ggml_compute_forward_swiglu_oai_f16(
+        const ggml_compute_params * params,
+        ggml_tensor * dst) {
+
+    const ggml_tensor * src0 = dst->src[0];
+    const ggml_tensor * src1 = dst->src[1];
+    char * src0_d = (char *) src0->data;
+    char * src1_d = (char *) (src1 ? src1->data : src0->data);
+    const size_t src0_o = src0->nb[1];
+    const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
+
+    GGML_ASSERT(ggml_is_contiguous_1(src0));
+    GGML_ASSERT(ggml_is_contiguous_1(dst));
+
+    if (src1) {
+        GGML_ASSERT(ggml_is_contiguous_1(src1));
+        GGML_ASSERT(src0->type == src1->type);
+    }
+
+    const int ith = params->ith;
+    const int nth = params->nth;
+
+    const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
+    const int nr = ggml_nrows(src0);
+
+    GGML_ASSERT(dst->ne[0] == nc);
+    GGML_ASSERT(ggml_nrows(dst) == nr);
+
+    const int32_t swapped = ggml_get_op_params_i32(dst, 1);
+    const float alpha = ggml_get_op_params_f32(dst, 2);
+    const float limit = ggml_get_op_params_f32(dst, 3);
+
+    // rows per thread
+    const int dr = (nr + nth - 1)/nth;
+
+    // row range for this thread
+    const int ir0 = dr*ith;
+    const int ir1 = MIN(ir0 + dr, nr);
+
+    for (int i1 = ir0; i1 < ir1; i1++) {
+        ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);
+        ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);
+        ggml_fp16_t * dst_p  = (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1]));
+
+        if (!src1) {
+            src0_p += swapped ? nc : 0;
+            src1_p += swapped ? 0 : nc;
+        }
+
+        for (int k = 0; k < nc; k++) {
+            const float x = std::min(GGML_CPU_FP16_TO_FP32(src0_p[k]), limit);
+            const float y = std::clamp(GGML_CPU_FP16_TO_FP32(src1_p[k]), -limit, limit);
+            const float out_glu = x / (1.f + expf(alpha * (-x)));
+            dst_p[k] = GGML_CPU_FP32_TO_FP16(out_glu * (y + 1.f));
+        }
+
+#ifndef NDEBUG
+        for (int k = 0; k < nc; k++) {
+            const float x = GGML_CPU_FP16_TO_FP32(dst_p[k]);
+            GGML_UNUSED(x);
+            assert(!isnan(x));
+            assert(!isinf(x));
+        }
+#endif // NDEBUG
+    }
+}
+
 static void ggml_compute_forward_swiglu_oai(
        const ggml_compute_params * params,
        ggml_tensor * dst) {
@@ -3559,6 +3626,10 @@ static void ggml_compute_forward_swiglu_oai(
            {
                ggml_compute_forward_swiglu_oai_f32(params, dst);
            } break;
+        case GGML_TYPE_F16:
+            {
+                ggml_compute_forward_swiglu_oai_f16(params, dst);
+            } break;
        default:
            {
                GGML_ABORT("fatal error");
@@ -8444,40 +8515,47 @@ static void ggml_compute_forward_pad_f32(
    const int32_t lp3 = ggml_get_op_params_i32(dst, 6);
    const int32_t rp3 = ggml_get_op_params_i32(dst, 7);

-    // TODO: optimize
+    const int64_t nrows = ne1 * ne2 * ne3;
+    const bool src_row_contig = src0->nb[0] == sizeof(float);
+
+    for (int64_t row = ith; row < nrows; row += nth) {
+        const int64_t i1 = row % ne1;
+        const int64_t i2 = (row / ne1) % ne2;
+        const int64_t i3 = row / (ne1 * ne2);
+
+        float * dst_row = dst_ptr + row * ne0;
+
+        if constexpr (circular_t) {
+            // circular padding wraps around on a torus
+            const int64_t src_i1 = ggml_wrap_around(i1 - lp1, ne01);
+            const int64_t src_i2 = ggml_wrap_around(i2 - lp2, ne02);
+            const int64_t src_i3 = ggml_wrap_around(i3 - lp3, ne03);

-    for (int64_t i2 = 0; i2 < ne2; ++i2) {
-        for (int64_t i1 = ith; i1 < ne1; i1 += nth) {
            for (int64_t i0 = 0; i0 < ne0; ++i0) {
-                for (int64_t i3 = 0; i3 < ne3; ++i3) {
-                    // circular means wrap around on a torus, so x and y loop around
-                    if constexpr (circular_t) {
-                        const int64_t dst_idx = i3*(ne0*ne1*ne2) + i2*(ne0*ne1) + i1*ne0 + i0;
-                        const int64_t src_i0 = ggml_wrap_around(i0 - lp0, ne00);
-                        const int64_t src_i1 = ggml_wrap_around(i1 - lp1, ne01);
-                        const int64_t src_i2 = ggml_wrap_around(i2 - lp2, ne02);
-                        const int64_t src_i3 = ggml_wrap_around(i3 - lp3, ne03);
+                const int64_t src_i0 = ggml_wrap_around(i0 - lp0, ne00);
+                dst_row[i0] = *(const float *)((const char *) src0->data + src_i3*nb03 + src_i2*nb02 + src_i1*nb01 + src_i0*nb00);
+            }
+        } else {
+            const bool interior = (i1 >= lp1 && i1 < ne1 - rp1)
+                               && (i2 >= lp2 && i2 < ne2 - rp2)
+                               && (i3 >= lp3 && i3 < ne3 - rp3);
+            if (!interior) {
+                memset(dst_row, 0, ne0 * sizeof(float));
+                continue;
+            }

-                        const int64_t src_idx =
-                            src_i3*nb03 +
-                            src_i2*nb02 +
-                            src_i1*nb01 +
-                            src_i0*nb00;
+            const char * src_row = (const char *) src0->data + (i3 - lp3)*nb03 + (i2 - lp2)*nb02 + (i1 - lp1)*nb01;

-                        const float * src_ptr = (const float *)((char *) src0->data + src_idx);
-                        dst_ptr[dst_idx] = *src_ptr;
+            if (src_row_contig && lp0 >= 0 && rp0 >= 0) {
+                memset(dst_row, 0, lp0 * sizeof(float));
+                memcpy(dst_row + lp0, src_row, ne00 * sizeof(float));
+                memset(dst_row + lp0 + ne00, 0, rp0 * sizeof(float));
+            } else {
+                for (int64_t i0 = 0; i0 < ne0; ++i0) {
+                    if (i0 >= lp0 && i0 < ne0 - rp0) {
+                        dst_row[i0] = *(const float *)(src_row + (i0 - lp0)*nb00);
                    } else {
-                        const int64_t dst_idx = i3*(ne0*ne1*ne2) + i2*(ne0*ne1) + i1*ne0 + i0;
-                        if ((i0 >= lp0 && i0 < ne0 - rp0) \
-                            && (i1 >= lp1 && i1 < ne1 - rp1) \
-                            && (i2 >= lp2 && i2 < ne2 - rp2) \
-                            && (i3 >= lp3 && i3 < ne3 - rp3)) {
-                            const int64_t src_idx = (i3 - lp3)*nb03 + (i2 - lp2)*nb02 + (i1 - lp1)*nb01 + (i0 - lp0)*nb00;
-                            const float * src_ptr = (const float *)((char *) src0->data + src_idx);
-                            dst_ptr[dst_idx] = *src_ptr;
-                        } else {
-                            dst_ptr[dst_idx] = 0;
-                        }
+                        dst_row[i0] = 0.0f;
                    }
                }
            }
@@ -1348,6 +1348,12 @@ static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggm
    return buffer;
 }

+static size_t ggml_backend_cuda_host_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
+    GGML_UNUSED(buft);
+    // very large pinned allocations can fail (e.g. on Windows) and fall back to pageable memory, smaller ones succeed
+    return 4ull*1024*1024*1024;
+}
+
 ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {
    static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_type_host = {
        /* .iface    = */ {
@@ -1355,7 +1361,7 @@ ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {
            /* .alloc_buffer        = */ ggml_backend_cuda_host_buffer_type_alloc_buffer,
            /* .alloc_buffer_n      = */ NULL,
            /* .get_alignment       = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,
-            /* .get_max_size        = */ NULL, // defaults to SIZE_MAX
+            /* .get_max_size        = */ ggml_backend_cuda_host_buffer_type_get_max_size,
            /* .get_alloc_size      = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,
            /* .get_alloc_size_n    = */ NULL,
            /* .is_host             = */ ggml_backend_cpu_buffer_type()->iface.is_host,
@@ -5474,6 +5480,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
                            (ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_OAI || ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_CLAMP)) {
                        return false;
                    }
+                    if (op->src[0]->type == GGML_TYPE_F16 && ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_OAI) {
+                        return false;
+                    }
                    return ggml_is_contiguous_1(op->src[0]);
                default:
                    return false;
@@ -50,6 +50,8 @@ endif()
 set(TARGET_NAME ggml-hexagon)
 ggml_add_backend_library(${TARGET_NAME}
    ggml-hexagon.cpp
+    htp-repack.cpp
+    htp-repack.h
    htp-drv.cpp
    htp-drv.h
    libdl.h
@@ -326,10 +326,12 @@ struct htp_opformat {
            const auto * kparams = (const struct htp_mm_kernel_params *) node.kernel_params;
            const char * path = "unknown";
            int32_t type = kparams->kernel_type;
-            if (type == HTP_MM_KERNEL_HMX_2D || type == HTP_MM_KERNEL_HMX_F16_BATCHED) {
-                path = "hmx-tiled";
+            if (type == HTP_MM_KERNEL_HMX_2D || type == HTP_MM_KERNEL_HMX_BATCHED) {
+                path = (kparams->tile_size > 0) ? "hmx-tiled" : "hmx-flat";
            } else if (type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || type == HTP_MM_KERNEL_HVX_F32_F32_VTCM ||
-                       type == HTP_MM_KERNEL_HVX_QUANT_ROW    || type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {
+                       type == HTP_MM_KERNEL_HVX_BF16_F32_VTCM) {
+                path = "hvx-flat";
+            } else if (type == HTP_MM_KERNEL_HVX_QUANT_ROW || type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {
                path = "hvx-tiled";
            }
            snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);
@@ -0,0 +1,64 @@
+#ifndef HTP_REPACK_H
+#define HTP_REPACK_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "ggml.h"
+#include "htp/htp-tensor.h"
+
+static inline bool htp_is_tiled_weight_type(enum ggml_type type) {
+    return type == GGML_TYPE_Q4_0  || type == GGML_TYPE_Q4_1   ||
+           type == GGML_TYPE_Q8_0  || type == GGML_TYPE_IQ4_NL ||
+           type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q6_K   ||
+           type == GGML_TYPE_Q4_K  || type == GGML_TYPE_Q5_K   ||
+           type == GGML_TYPE_Q3_K  || type == GGML_TYPE_Q2_K   ||
+           type == GGML_TYPE_F16   || type == GGML_TYPE_F32    ||
+           type == GGML_TYPE_BF16;
+}
+
+static inline bool htp_is_tiled_weight_tensor(const struct ggml_tensor * t) {
+    if (!ggml_is_contiguous(t) || ggml_is_view(t)) {
+        return false;
+    }
+    if (!htp_is_tiled_weight_type(t->type)) {
+        return false;
+    }
+    if (t->ne[0] < 32) {
+        return false;
+    }
+    if (!ggml_is_quantized(t->type)) {
+        return (t->ne[0] % 32 == 0) && (t->ne[1] % 32 == 0) && (t->ne[2] == 1) && (t->ne[3] == 1);
+    }
+    return t->ne[3] == 1;
+}
+
+// Size of one tiled row in the DSP tiled layout. The Q3_K..Q6_K tiles store uncompressed scales/mins,
+// so they are larger than the ggml blocks. For the other tiled types (Q2_K included) the tile has the same size.
+static inline size_t htp_tiled_row_size(enum ggml_type type, int64_t ne0) {
+    if (type == GGML_TYPE_F16 || type == GGML_TYPE_F32 || type == GGML_TYPE_BF16) {
+        return (size_t) ne0 * sizeof(ggml_fp16_t);
+    }
+    if (type == GGML_TYPE_Q6_K) {
+        return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q6_K / 32);
+    }
+    if (type == GGML_TYPE_Q4_K) {
+        return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q4_K / 32);
+    }
+    if (type == GGML_TYPE_Q5_K) {
+        return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q5_K / 32);
+    }
+    if (type == GGML_TYPE_Q3_K) {
+        return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q3_K / 32);
+    }
+    return ggml_row_size(type, ne0);
+}
+
+static inline bool htp_is_hmx_weight_type(enum ggml_type type) {
+    return htp_is_tiled_weight_type(type);
+}
+
+void htp_repack_tensor_tiled(struct ggml_tensor * t, const void * data, size_t size);
+void htp_unpack_tensor_tiled(void * data, const struct ggml_tensor * t, size_t offset, size_t size);
+
+#endif // HTP_REPACK_H
@@ -20,7 +20,7 @@ add_library(${HTP_LIB} SHARED
    work-queue.c
    dma-queue.c
    hmx-queue.c
-    htp-tensor.c
+    htp-dcache.c
    matmul-ops.c
    flash-attn-ops.c
    gated-delta-net-ops.c
@@ -361,7 +361,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
                htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir);
                for (uint32_t r = 0; r < current_block_size; ++r) {
                    __fp16 * row_k = (__fp16 *)(k_base + r * factx->size_k_row_padded);
-                    hvx_dequantize_row_q8_0_f16(row_k, row_k, DK);
+                    hvx_dequantize_row_flat_q8_0_f16(row_k, row_k, DK);
                }
                htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir);
            }
@@ -369,7 +369,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
                htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir);
                for (uint32_t r = 0; r < current_block_size; ++r) {
                    __fp16 * row_v = (__fp16 *)(v_base + r * factx->size_v_row_padded);
-                    hvx_dequantize_row_q8_0_f16(row_v, row_v, DV);
+                    hvx_dequantize_row_flat_q8_0_f16(row_v, row_v, DV);
                }
                htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir);
            }
@@ -625,7 +625,7 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data)
    if (factx->k->type == HTP_TYPE_Q8_0) {
        for (uint32_t r = start; r < end; ++r) {
            __fp16 * row_k = (__fp16 *)((char *)args->curr_k + r * args->src_stride * sizeof(__fp16));
-            hvx_dequantize_row_q8_0_f16(row_k, row_k, factx->DK);
+            hvx_dequantize_row_flat_q8_0_f16(row_k, row_k, factx->DK);
        }
    }
    hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles[args->buf_idx], (const __fp16 *) args->curr_k, total_rows, factx->DK_pad,
@@ -679,7 +679,7 @@ static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data)
    if (factx->v->type == HTP_TYPE_Q8_0) {
        for (uint32_t r = start; r < end; ++r) {
            __fp16 * row_v = (__fp16 *)((char *)args->v_src + r * args->src_stride * sizeof(__fp16));
-            hvx_dequantize_row_q8_0_f16(row_v, row_v, factx->DV);
+            hvx_dequantize_row_flat_q8_0_f16(row_v, row_v, factx->DV);
        }
    }
    hmx_interleave_cols_to_tiles(v_tiles_dst, (const __fp16 *) args->v_src, total_rows, factx->DV_pad,
@@ -17,21 +17,32 @@
 #include "htp-tensor.h"
 #include "hvx-utils.h"
 #include "hvx-quant.h"
-#include "matmul-ops.h"
 #include "get-rows-ops.h"
 #include "work-queue.h"

+struct get_rows_context;
+
+typedef uint32_t (*get_rows_read_index_fn_t)(const void * src1, uint32_t idx, int idx_type);
+typedef void (*get_rows_flat_compute_fn_t)(float * dst_spad, const void * src_spad, uint32_t cur_elems);
+typedef void (*get_rows_tiled_compute_fn_t)(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row);
+
 struct get_rows_context {
    struct htp_ops_context * octx;
    const struct htp_get_rows_kernel_params * kparams;
    struct htp_get_rows_vtcm_layout vtcm_layout;
-    uint8_t * vtcm_base;
+    uint8_t    * vtcm_base;
+    const void * src1_data;
+    get_rows_read_index_fn_t    read_index;
+    get_rows_flat_compute_fn_t  compute_flat;
+    get_rows_tiled_compute_fn_t compute_tiled;
    uint32_t task_start;
    uint32_t tasks;
    uint32_t tasks_per_thread;
    uint32_t tile_size;
    uint32_t tile_stride;
-    bool index_i32;
+    uint32_t src0_row_size;
+    uint32_t dst_row_size;
+    int idx_type;
 };

 #define get_rows_preamble                      \
@@ -64,229 +75,169 @@ struct get_rows_context {
                                               \
    const uint32_t nr = ne10 * ne11 * ne12;

-#define GET_ROWS_THREAD_ST_FN(IDX_TYPE)                                                                                 \
-static void get_rows_thread_st_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) {                             \
-    struct get_rows_context * grctx = (struct get_rows_context *)data;                                                  \
-    struct htp_ops_context * octx = grctx->octx;                                                                        \
-    const struct htp_get_rows_kernel_params * kparams = grctx->kparams;                                                 \
-    get_rows_preamble;                                                                                                  \
-    const uint32_t dr  = grctx->tasks_per_thread;                                                                       \
-    const uint32_t ir0 = grctx->task_start + dr * ith;                                                                  \
-    if (ir0 >= grctx->task_start + grctx->tasks) {                                                                      \
-        return;                                                                                                         \
-    }                                                                                                                   \
-    const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);                                               \
-    const uint32_t row_size_bytes = htp_tensor_get_row_size(octx->src[0]->type, ne00);                                  \
-    dma_queue * dma_q = octx->ctx->dma[ith];                                                                            \
-    for (uint32_t i = ir0; i < ir1; ++i) {                                                                              \
-        const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11);                                                       \
-        const uint32_t rem = i - i12 * ne11 * ne10;                                                                     \
-        const uint32_t i11 = fastdiv(rem, &kparams->div_ne10);                                                          \
-        const uint32_t i10 = rem - i11 * ne10;                                                                          \
-        const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(uintptr_t)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12); \
-        const uint32_t i01 = (uint32_t)*src1_ptr;                                                                       \
-        assert(i01 < ne01);                                                                                             \
-        const uint32_t q02 = fastdiv(i11, &kparams->div_ne02);                                                          \
-        const uint32_t i02 = i11 - q02 * ne02;                                                                          \
-        const uint32_t q03 = fastdiv(i12, &kparams->div_ne03);                                                          \
-        const uint32_t i03 = i12 - q03 * ne03;                                                                          \
-        const dma_addr_t src0_data = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03;                               \
-        const dma_addr_t dst_data  = octx->dst->data    + i10*nb1  + i11*nb2  + i12*nb3;                                \
-        while (!dma_queue_push(dma_q, dma_make_data(dst_data, src0_data), nb1, nb01,                                    \
-                               row_size_bytes, 1)) {                                                                    \
-            dma_queue_pop(dma_q);                                                                                       \
-        }                                                                                                               \
-    }                                                                                                                   \
-    dma_queue_flush(dma_q);                                                                                             \
+static uint32_t get_rows_read_index_ddr(
+    const void * src1,
+    uint32_t idx,
+    int idx_type
+) {
+    const uint32_t off = idx * ((idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t));
+    return *(const uint32_t *) ((const uint8_t *) src1 + off);
 }

-GET_ROWS_THREAD_ST_FN(int32_t)
-GET_ROWS_THREAD_ST_FN(int64_t)
-
-#define GET_ROWS_THREAD_DT_FN(TYPE_NAME, SRC0_SIZE_EXPR, IDX_TYPE, COMPUTE_EXPR)                                                \
-static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) {                          \
-    struct get_rows_context * grctx = (struct get_rows_context *)data;                                                          \
-    struct htp_ops_context * octx = grctx->octx;                                                                                \
-    const struct htp_get_rows_kernel_params * kparams = grctx->kparams;                                                         \
-    get_rows_preamble;                                                                                                          \
-    struct htp_thread_trace * tr = &octx->ctx->trace[ith];                                                                      \
-    const uint32_t dr  = grctx->tasks_per_thread;                                                                               \
-    const uint32_t ir0 = grctx->task_start + dr * ith;                                                                          \
-    if (ir0 >= grctx->task_start + grctx->tasks) {                                                                              \
-        return;                                                                                                                 \
-    }                                                                                                                           \
-    const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);                                                       \
-    const uint32_t chunks_per_row = kparams->chunks_per_row;                                                                    \
-    const uint32_t chunk_size     = kparams->chunk_size;                                                                        \
-    dma_queue * dma_q = octx->ctx->dma[ith];                                                                                    \
-    const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout;                                                  \
-    uint8_t * vtcm_src0 = grctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread;                  \
-    uint8_t * vtcm_dst  = grctx->vtcm_base + vtcm_layout->off_dst  + ith * vtcm_layout->dst_bytes_per_thread;                   \
-    for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, spad_idx++) {                               \
-        const uint32_t i = ir0 + step;                                                                                          \
-        const uint32_t row_idx   = fastdiv(i, &kparams->div_chunks_per_row);                                                    \
-        const uint32_t chunk_idx = i - row_idx * chunks_per_row;                                                                \
-        const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11);                                                         \
-        const uint32_t rem = row_idx - i12 * ne11 * ne10;                                                                       \
-        const uint32_t i11 = fastdiv(rem, &kparams->div_ne10);                                                                  \
-        const uint32_t i10 = rem - i11 * ne10;                                                                                  \
-        const IDX_TYPE * src1_ptr = (const IDX_TYPE *)(uintptr_t)(octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12);         \
-        const uint32_t i01 = (uint32_t)*src1_ptr;                                                                               \
-        assert(i01 < ne01);                                                                                                     \
-        const uint32_t q02 = fastdiv(i11, &kparams->div_ne02);                                                                  \
-        const uint32_t i02 = i11 - q02 * ne02;                                                                                  \
-        const uint32_t q03 = fastdiv(i12, &kparams->div_ne03);                                                                  \
-        const uint32_t i03 = i12 - q03 * ne03;                                                                                  \
-        const uint32_t offset = chunk_idx * chunk_size;                                                                         \
-        const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0;                                        \
-        const uint32_t cur_src0_bytes = SRC0_SIZE_EXPR(cur_elems);                                                              \
-        const uint32_t cur_dst_bytes  = cur_elems * sizeof(float);                                                              \
-        const dma_addr_t src0_data = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03 + SRC0_SIZE_EXPR(offset);              \
-        dma_queue_push(dma_q,                                                                                                   \
-                       dma_make_data(octx->dst->data,                                                                           \
-                                     vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size),                                    \
-                       cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 0);                                       \
-        dma_queue_push(dma_q,                                                                                                   \
-                       dma_make_data(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size, src0_data),                       \
-                       vtcm_layout->src0_spad_half_size, cur_src0_bytes, cur_src0_bytes, 1);                                    \
-    }                                                                                                                           \
-    for (uint32_t step = 0; step < ir1 - ir0; ++step) {                                                                         \
-        const uint32_t i = ir0 + step;                                                                                          \
-        void * dst_spad = (void *) dma_queue_pop(dma_q).src;                                                                    \
-        void * src_spad = (void *) dma_queue_pop(dma_q).dst;                                                                    \
-        const uint32_t row_idx   = fastdiv(i, &kparams->div_chunks_per_row);                                                    \
-        const uint32_t chunk_idx = i - row_idx * chunks_per_row;                                                                \
-        const uint32_t i12 = fastdiv(row_idx, &kparams->div_ne10_ne11);                                                         \
-        const uint32_t rem = row_idx - i12 * ne11 * ne10;                                                                       \
-        const uint32_t i11 = fastdiv(rem, &kparams->div_ne10);                                                                  \
-        const uint32_t i10 = rem - i11 * ne10;                                                                                  \
-        const uint32_t offset = chunk_idx * chunk_size;                                                                         \
-        const uint32_t cur_elems = (offset < ne00) ? MIN(chunk_size, ne00 - offset) : 0;                                        \
-        const uint32_t cur_dst_bytes  = cur_elems * sizeof(float);                                                              \
-        htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, i);                                                                   \
-        COMPUTE_EXPR;                                                                                                           \
-        htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, i);                                                                    \
-        const dma_addr_t dst_data  = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3 + offset * sizeof(float);                    \
-        dma_queue_push(dma_q,                                                                                                   \
-                       dma_make_data(dst_data, dst_spad),                                                                       \
-                       cur_dst_bytes, vtcm_layout->dst_spad_half_size, cur_dst_bytes, 1);                                       \
-        const uint32_t next_step = step + 2;                                                                                    \
-        if (next_step < ir1 - ir0) {                                                                                            \
-            const uint32_t pi = ir0 + next_step;                                                                                \
-            const uint32_t prow_idx   = fastdiv(pi, &kparams->div_chunks_per_row);                                              \
-            const uint32_t pchunk_idx = pi - prow_idx * chunks_per_row;                                                         \
-            const uint32_t pi12 = fastdiv(prow_idx, &kparams->div_ne10_ne11);                                                   \
-            const uint32_t prem = prow_idx - pi12 * ne11 * ne10;                                                                \
-            const uint32_t pi11 = fastdiv(prem, &kparams->div_ne10);                                                            \
-            const uint32_t pi10 = prem - pi11 * ne10;                                                                           \
-            const IDX_TYPE * psrc1_ptr = (const IDX_TYPE *)(uintptr_t)(octx->src[1]->data + pi10*nb10 + pi11*nb11 + pi12*nb12); \
-            const uint32_t pi01 = (uint32_t)*psrc1_ptr;                                                                         \
-            assert(pi01 < ne01);                                                                                                \
-            const uint32_t pq02 = fastdiv(pi11, &kparams->div_ne02);                                                            \
-            const uint32_t pi02 = pi11 - pq02 * ne02;                                                                           \
-            const uint32_t pq03 = fastdiv(pi12, &kparams->div_ne03);                                                            \
-            const uint32_t pi03 = pi12 - pq03 * ne03;                                                                           \
-            const uint32_t poffset = pchunk_idx * chunk_size;                                                                   \
-            const uint32_t pcur_elems = (poffset < ne00) ? MIN(chunk_size, ne00 - poffset) : 0;                                 \
-            const uint32_t pcur_src0_bytes = SRC0_SIZE_EXPR(pcur_elems);                                                        \
-            const dma_addr_t psrc0_data =                                                                                       \
-                octx->src[0]->data + pi01*nb01 + pi02*nb02 + pi03*nb03 + SRC0_SIZE_EXPR(poffset);                               \
-            dma_queue_push(dma_q,                                                                                               \
-                           dma_make_data(src_spad, psrc0_data),                                                                 \
-                           vtcm_layout->src0_spad_half_size, pcur_src0_bytes, pcur_src0_bytes, 1);                              \
-        }                                                                                                                       \
-    }                                                                                                                           \
-    dma_queue_flush(dma_q);                                                                                                     \
+static __attribute__((noinline)) uint32_t get_rows_read_index_vtcm(
+    const void * src1,
+    uint32_t idx,
+    int idx_type
+) {
+    const uint32_t off = idx * ((idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t));
+    const HVX_Vector v = hvx_vmem((const uint8_t *) src1 + off); // off & ~127u in HW
+    // vextract is normally slow, but here it avoids an HVX->L2 store and L1D miss
+    return (uint32_t) Q6_R_vextract_VR(v, off);
 }

-#define F16_BYTES(n)  ((n) * sizeof(__fp16))
-#define Q8_0_BYTES(n) (((n) / 32) * sizeof(block_q8_0))
+static void get_rows_thread_sametype(unsigned int nth, unsigned int ith, void * data) {
+    struct get_rows_context * grctx = (struct get_rows_context *) data;
+    struct htp_ops_context * octx = grctx->octx;
+    const struct htp_get_rows_kernel_params * kparams = grctx->kparams;
+    get_rows_preamble;

-static __attribute__((noinline)) void compute_get_rows_f16(float * dst_spad, const void * src_spad, uint32_t cur_elems) {
-    hvx_dequantize_row_f16_f32(dst_spad, src_spad, cur_elems);
-}
+    const uint32_t dr  = grctx->tasks_per_thread;
+    const uint32_t ir0 = grctx->task_start + dr * ith;
+    if (ir0 >= grctx->task_start + grctx->tasks) {
+        return;
+    }
+    const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);
+    const uint32_t row_size_bytes = grctx->src0_row_size;
+    dma_queue * dma_q = octx->ctx->dma[ith];

-static __attribute__((noinline)) void compute_get_rows_q8_0(float * dst_spad, const void * src_spad, uint32_t cur_elems) {
-    hvx_dequantize_row_q8_0_f32(dst_spad, src_spad, cur_elems);
-}
+    for (uint32_t i = ir0; i < ir1; ++i) {
+        const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11);
+        const uint32_t rem = i - i12 * ne11 * ne10;
+        const uint32_t i11 = fastdiv(rem, &kparams->div_ne10);
+        const uint32_t i10 = rem - i11 * ne10;

-GET_ROWS_THREAD_DT_FN(f16,  F16_BYTES,  int32_t, { compute_get_rows_f16((float *)dst_spad, src_spad, cur_elems); })
-GET_ROWS_THREAD_DT_FN(f16,  F16_BYTES,  int64_t, { compute_get_rows_f16((float *)dst_spad, src_spad, cur_elems); })
+        const uint32_t i01 = grctx->read_index(grctx->src1_data, i, grctx->idx_type);
+        assert(i01 < ne01);

-GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { compute_get_rows_q8_0((float *)dst_spad, src_spad, cur_elems); })
-GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { compute_get_rows_q8_0((float *)dst_spad, src_spad, cur_elems); })
+        const uint32_t q02 = fastdiv(i11, &kparams->div_ne02);
+        const uint32_t i02 = i11 - q02 * ne02;
+        const uint32_t q03 = fastdiv(i12, &kparams->div_ne03);
+        const uint32_t i03 = i12 - q03 * ne03;

+        const dma_addr_t src0_data = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03;
+        const dma_addr_t dst_data  = octx->dst->data    + i10*nb1  + i11*nb2  + i12*nb3;

-static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const uint8_t * tile, uint32_t row, bool q4, bool q4_k) {
-    const HVX_VectorPred first2 = Q6_Q_vsetq_R(2);
-    const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
-    HVX_Vector vq = Q6_V_vzero();
-    if (q4) {
-        const HVX_VectorPred first1 = Q6_Q_vsetq_R(1);
-        const HVX_VectorPred first3 = Q6_Q_vsetq_R(3);
-        for (int group = 3; group >= 0; --group) {
-            const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), row);
-            // Four planes contribute bytes at 0, 32, 64 and 96 after rotation.
-            HVX_Vector packed = Q6_V_vmux_QVV(first1, v, Q6_V_vror_VR(v, 31));
-            packed = Q6_V_vmux_QVV(first2, packed, Q6_V_vror_VR(v, 62));
-            packed = Q6_V_vmux_QVV(first3, packed, Q6_V_vror_VR(v, 93));
-            vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4));
-        }
-        const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F));
-        const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4);
-        vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1));
-        if (!q4_k) {
-            vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8));
-        }
-    } else {
-        for (int group = 7; group >= 0; --group) {
-            const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), 2 * row);
-            // Two planes contribute halfwords at 0 and 64 after rotation.
-            const HVX_Vector packed = Q6_V_vmux_QVV(first2, v, Q6_V_vror_VR(v, 62));
-            vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4));
+        while (!dma_queue_push(dma_q, dma_make_data(dst_data, src0_data), nb1, nb01, row_size_bytes, 1)) {
+            dma_queue_pop(dma_q);
        }
    }
-    const HVX_Vector scales = hvx_vmem(tile + (q4 ? 512 : 1024));
-    const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, (q4_k ? 4 : 2) * row));
-    const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
-    const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);
-    const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
-    HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
-    if (q4_k) {
-        const HVX_Vector offset_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row + 2));
-        const HVX_Vector offset = Q6_V_lo_W(hvx_vec_f16_to_f32(offset_hf));
-        values = hvx_vec_add_f32_f32(values, offset);
-    }
-    *(HVX_Vector *) dst = values;
+    dma_queue_flush(dma_q);
 }

-static __attribute__((noinline)) void compute_get_rows_q6_k(float * dst, const uint8_t * tile, uint32_t row) {
-    const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
-    const HVX_VectorPred first16 = Q6_Q_vsetq_R(16 * sizeof(float));
-    const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F);
-    const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
-    HVX_Vector vq = Q6_V_vzero();

-    for (int group = 7; group >= 0; --group) {
-        const HVX_Vector lo_plane = Q6_V_vror_VR(hvx_vmem(tile + (group >> 1) * VLEN), 4 * row);
-        const HVX_Vector hi_plane = Q6_V_vror_VR(hvx_vmem(tile + 512 + (group >> 2) * VLEN), 4 * row);
-        const HVX_Vector lo = (group & 1) ? Q6_Vub_vlsr_VubR(lo_plane, 4) : Q6_V_vand_VV(lo_plane, mask_0f);
-        const HVX_Vector hi = Q6_Vub_vlsr_VubR(hi_plane, 2 * (group & 3));
-        const HVX_Vector packed = Q6_V_vor_VV(lo, Q6_Vw_vasl_VwR(Q6_V_vand_VV(hi, mask_03), 4));
-        vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4));
+
+struct get_rows_flat_task {
+    dma_addr_t src0_data;
+    dma_addr_t dst_data;
+};
+
+static inline struct get_rows_flat_task get_rows_flat_calc_task(
+    const struct htp_ops_context * octx,
+    const struct get_rows_context * grctx,
+    uint32_t i,
+    bool need_src0
+) {
+    const struct htp_get_rows_kernel_params * kparams = grctx->kparams;
+    get_rows_preamble;
+
+    const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11);
+    const uint32_t rem = i - i12 * ne11 * ne10;
+    const uint32_t i11 = fastdiv(rem, &kparams->div_ne10);
+    const uint32_t i10 = rem - i11 * ne10;
+
+    struct get_rows_flat_task task;
+    task.src0_data = 0;
+    task.dst_data  = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3;
+
+    if (need_src0) {
+        const uint32_t i01 = grctx->read_index(grctx->src1_data, i, grctx->idx_type);
+        assert(i01 < ne01);
+
+        const uint32_t q02 = fastdiv(i11, &kparams->div_ne02);
+        const uint32_t i02 = i11 - q02 * ne02;
+        const uint32_t q03 = fastdiv(i12, &kparams->div_ne03);
+        const uint32_t i03 = i12 - q03 * ne03;
+
+        task.src0_data = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03;
    }

-    const HVX_Vector scales = hvx_vmem(tile + 768);
-    const HVX_Vector scale_lo_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
-    const HVX_Vector scale_hi_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 64 + 2 * row));
-    const HVX_Vector scale_lo = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_lo_hf));
-    const HVX_Vector scale_hi = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hi_hf));
-    const HVX_Vector scale = Q6_V_vmux_QVV(first16, scale_lo, scale_hi);
-    const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(32)));
-    const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
-    *(HVX_Vector *) dst = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
+    return task;
 }

+static void get_rows_thread_flat(unsigned int nth, unsigned int ith, void * data) {
+    struct get_rows_context * grctx = (struct get_rows_context *) data;
+    struct htp_ops_context * octx = grctx->octx;
+    get_rows_preamble;
+
+    const uint32_t dr  = grctx->tasks_per_thread;
+    const uint32_t ir0 = grctx->task_start + dr * ith;
+    if (ir0 >= grctx->task_start + grctx->tasks) {
+        return;
+    }
+    const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);
+
+    dma_queue * dma_q = octx->ctx->dma[ith];
+    struct htp_thread_trace * tr = &octx->ctx->trace[ith];
+    const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout;
+    uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread;
+    uint8_t * vtcm_dst  = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst)  + ith * vtcm_layout->dst_bytes_per_thread;
+    const uint32_t src0_bytes = grctx->src0_row_size;
+    const uint32_t dst_bytes  = grctx->dst_row_size;
+    get_rows_flat_compute_fn_t compute = grctx->compute_flat;
+
+    for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, spad_idx++) {
+        const uint32_t i = ir0 + step;
+        struct get_rows_flat_task task = get_rows_flat_calc_task(octx, grctx, i, true);
+
+        dma_queue_push(dma_q,
+                       dma_make_data(octx->dst->data,
+                                     vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size),
+                       dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 0);
+        dma_queue_push(dma_q,
+                       dma_make_data(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size, task.src0_data),
+                       vtcm_layout->src0_spad_half_size, src0_bytes, src0_bytes, 1);
+    }
+
+    for (uint32_t step = 0; step < ir1 - ir0; ++step) {
+        const uint32_t i = ir0 + step;
+        void * dst_spad = (void *) dma_queue_pop(dma_q).src;
+        void * src_spad = (void *) dma_queue_pop(dma_q).dst;
+
+        struct get_rows_flat_task task = get_rows_flat_calc_task(octx, grctx, i, false);
+
+        htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
+        compute((float *) dst_spad, src_spad, ne00);
+        htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
+
+        dma_queue_push(dma_q,
+                       dma_make_data(task.dst_data, dst_spad),
+                       dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 1);
+
+        const uint32_t next_step = step + 2;
+        if (next_step < ir1 - ir0) {
+            const uint32_t pi = ir0 + next_step;
+            struct get_rows_flat_task next_task = get_rows_flat_calc_task(octx, grctx, pi, true);
+            dma_queue_push(dma_q,
+                           dma_make_data(src_spad, next_task.src0_data),
+                           vtcm_layout->src0_spad_half_size, src0_bytes, src0_bytes, 1);
+        }
+    }
+
+    dma_queue_flush(dma_q);
+}
+
+
+
 struct get_rows_tiled_task {
    dma_addr_t tile_src_base;
    dma_addr_t dst_data;
@@ -307,16 +258,15 @@ static inline struct get_rows_tiled_task get_rows_tiled_calc_task(
    const uint32_t rem = i - i12 * ne11 * ne10;
    const uint32_t i11 = fastdiv(rem, &kparams->div_ne10);
    const uint32_t i10 = rem - i11 * ne10;
-    const dma_addr_t src1_data = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12;
-    const uint32_t i01 = grctx->index_i32 ? *(const int32_t *)(uintptr_t) src1_data : (uint32_t) *(const int64_t *)(uintptr_t) src1_data;
+    const uint32_t i01 = grctx->read_index(grctx->src1_data, i, grctx->idx_type);
    assert(i01 < ne01);

    const uint32_t q02 = fastdiv(i11, &kparams->div_ne02);
    const uint32_t i02 = i11 - q02 * ne02;
    const uint32_t q03 = fastdiv(i12, &kparams->div_ne03);
    const uint32_t i03 = i12 - q03 * ne03;
-    const uint32_t column_tile = i01 / HTP_MM_HMX_TILE_N_ROWS;
-    const uint32_t row = i01 % HTP_MM_HMX_TILE_N_ROWS;
+    const uint32_t column_tile = i01 / HTP_TENSOR_TILE_N_ROWS;
+    const uint32_t row = i01 % HTP_TENSOR_TILE_N_ROWS;
    const dma_addr_t matrix = octx->src[0]->data + i02*nb02 + i03*nb03;

    struct get_rows_tiled_task task;
@@ -329,7 +279,6 @@ static inline struct get_rows_tiled_task get_rows_tiled_calc_task(
 static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * data) {
    struct get_rows_context * grctx = (struct get_rows_context *) data;
    struct htp_ops_context * octx = grctx->octx;
-    const struct htp_get_rows_kernel_params * kparams = grctx->kparams;
    get_rows_preamble;

    const uint32_t dr  = grctx->tasks_per_thread;
@@ -339,19 +288,17 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat
    }

    const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);
-    const uint32_t n_k_tiles = ne00 / HTP_MM_HMX_TILE_N_COLS;
+    const uint32_t n_k_tiles = ne00 / HTP_TENSOR_TILE_N_COLS;
    const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout;
-    uint8_t * src_spad_base = grctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread;
-    uint8_t * dst_spad_base = grctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread;
+    uint8_t * src_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread;
+    uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst)  + ith * vtcm_layout->dst_bytes_per_thread;
    dma_queue * dma_q = octx->ctx->dma[ith];
    struct htp_thread_trace * tr = &octx->ctx->trace[ith];

    const uint32_t tile_size   = grctx->tile_size;
    const uint32_t tile_stride = grctx->tile_stride;
-    const uint32_t dst_bytes   = ne00 * sizeof(float);
-    const bool is_q4 = octx->src[0]->type == HTP_TYPE_Q4_0 || octx->src[0]->type == HTP_TYPE_Q4_K;
-    const bool is_q4_k = octx->src[0]->type == HTP_TYPE_Q4_K;
-    const bool is_q6_k = octx->src[0]->type == HTP_TYPE_Q6_K;
+    const uint32_t dst_bytes   = grctx->dst_row_size;
+    get_rows_tiled_compute_fn_t compute_tiled = grctx->compute_tiled;

    for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, ++spad_idx) {
        const uint32_t i = ir0 + step;
@@ -376,15 +323,7 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat
        struct get_rows_tiled_task task = get_rows_tiled_calc_task(octx, grctx, i, n_k_tiles, tile_size);

        htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
-        for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
-            const uint8_t * tile = src_spad + k_tile * tile_stride;
-            float * dst_block = dst_spad + k_tile * HTP_MM_HMX_TILE_N_COLS;
-            if (is_q6_k) {
-                compute_get_rows_q6_k(dst_block, tile, task.row);
-            } else {
-                compute_get_rows_tiled(dst_block, tile, task.row, is_q4, is_q4_k);
-            }
-        }
+        compute_tiled(dst_spad, src_spad, n_k_tiles, tile_stride, task.row);
        htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);

        // Real writeback of dst_spad
@@ -405,16 +344,40 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat
    dma_queue_flush(dma_q);
 }

+static __attribute__((noinline)) void get_rows_dma_src1(
+    struct htp_ops_context * octx,
+    uint8_t * vtcm_src1,
+    uint32_t elem_size
+) {
+    const uint32_t ne10 = octx->src[1]->ne[0];
+    const uint32_t ne11 = octx->src[1]->ne[1];
+    const uint32_t ne12 = octx->src[1]->ne[2];
+    const uint32_t ne13 = octx->src[1]->ne[3];
+    const uint32_t nb11 = octx->src[1]->nb[1];
+    const uint32_t nb12 = octx->src[1]->nb[2];
+    const uint32_t nb13 = octx->src[1]->nb[3];
+
+    dma_queue * dma_q = octx->ctx->dma[0];
+    uint8_t * cur_dst = vtcm_src1;
+    for (uint32_t i13 = 0; i13 < ne13; ++i13) {
+        for (uint32_t i12 = 0; i12 < ne12; ++i12) {
+            const dma_addr_t src_plane = octx->src[1]->data + i12 * nb12 + i13 * nb13;
+            while (!dma_queue_push(dma_q, dma_make_data(cur_dst, src_plane),
+                                   ne10 * elem_size, nb11, ne10 * elem_size, ne11)) {
+                dma_queue_pop(dma_q);
+            }
+            cur_dst += ne10 * ne11 * elem_size;
+        }
+    }
+    dma_queue_flush(dma_q);
+}
+
 int op_get_rows(struct htp_ops_context * octx) {
    const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params *) octx->kernel_params;

-    if (octx->src[0]->type != HTP_TYPE_F32 &&
-        octx->src[0]->type != HTP_TYPE_F16 &&
-        octx->src[0]->type != HTP_TYPE_Q4_0 &&
-        octx->src[0]->type != HTP_TYPE_Q4_K &&
-        octx->src[0]->type != HTP_TYPE_Q6_K &&
-        octx->src[0]->type != HTP_TYPE_Q8_0 &&
-        octx->src[0]->type != HTP_TYPE_I32) {
+    const uint32_t ne00          = octx->src[0]->ne[0];
+    const uint32_t src0_row_size = htp_tensor_get_row_size(octx->src[0]->type, ne00);
+    if (src0_row_size == 0) {
        return HTP_STATUS_NO_SUPPORT;
    }

@@ -432,20 +395,16 @@ int op_get_rows(struct htp_ops_context * octx) {
        return HTP_STATUS_NO_SUPPORT;
    }

-    if (htp_tensor_is_extended(octx->src[1])) {
-        return HTP_STATUS_NO_SUPPORT;
-    }
-
    const struct htp_tensor * dst = octx->dst;
    const uint32_t total_tasks    = kparams->total_tasks;
-    const size_t dst_row_size     = htp_tensor_get_row_size(dst->type, dst->ne[0]);
+    const uint32_t dst_row_size   = (kparams->kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) ? src0_row_size : ne00 * sizeof(float);

    uint32_t task_start = 0;
    uint32_t tasks      = total_tasks;

    if (octx->ctx->mdev.count > 1) {
        uint32_t tasks_per_chunk = 1;
-        htp_tensor_mdev_rows_per_chunk(dst, dst_row_size / dst->ne[0], (uint32_t) dst_row_size, &tasks_per_chunk);
+        htp_tensor_mdev_rows_per_chunk(dst, dst_row_size / dst->ne[0], dst_row_size, &tasks_per_chunk);
        const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, tasks_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
        task_start = range.start;
        tasks      = range.count;
@@ -465,15 +424,18 @@ int op_get_rows(struct htp_ops_context * octx) {
    grctx.octx = octx;
    grctx.kparams = kparams;
    grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base;
+    grctx.compute_flat = NULL;
+    grctx.compute_tiled = NULL;
    grctx.task_start = task_start;
    grctx.tasks = tasks;
    grctx.tasks_per_thread = octx->ctx->mdev.count == 1 ? kparams->tasks_per_thread : fastdiv(tasks + n_threads - 1, &octx->n_threads_div);
-    grctx.tile_size = htp_mm_get_weight_tile_size(octx->src[0]->type);
-    grctx.tile_stride = (grctx.tile_size + 127) & ~127;
-    grctx.index_i32 = octx->src[1]->type == HTP_TYPE_I32;
+    grctx.tile_size   = 0;
+    grctx.tile_stride = 0;
+    grctx.src0_row_size = src0_row_size;
+    grctx.dst_row_size  = dst_row_size;
+    grctx.idx_type      = octx->src[1]->type;

-    const uint32_t ne00 = octx->src[0]->ne[0];
-    htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, n_threads);
+    htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, total_tasks, grctx.idx_type, n_threads);

    if (grctx.vtcm_layout.total_bytes > octx->ctx->vtcm_size) {
        FARF(ERROR, "get-rows: VTCM reservation %zu is too small, needed %zu\n",
@@ -481,27 +443,56 @@ int op_get_rows(struct htp_ops_context * octx) {
        return HTP_STATUS_INVAL_PARAMS;
    }

-    const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
-
    work_queue_func_t q_func = NULL;
    switch (kparams->kernel_type) {
        case HTP_GET_ROWS_KERNEL_SAMETYPE:
-            q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t);
+            q_func = get_rows_thread_sametype;
            break;
        case HTP_GET_ROWS_KERNEL_TILED:
+            switch (octx->src[0]->type) {
+                case HTP_TYPE_F16:  grctx.compute_tiled = hvx_dequantize_row_tiled_f16_f32; break;
+                case HTP_TYPE_Q8_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q8_0_f32; break;
+                case HTP_TYPE_Q4_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_0_f32; break;
+                case HTP_TYPE_Q4_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_k_f32; break;
+                case HTP_TYPE_Q5_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q5_k_f32; break;
+                case HTP_TYPE_Q6_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q6_k_f32; break;
+                default:            return HTP_STATUS_NO_SUPPORT;
+            }
+            grctx.tile_size   = htp_tensor_get_tile_size(octx->src[0]->type);
+            grctx.tile_stride = htp_tensor_get_tile_padded_size(octx->src[0]->type);
            q_func = get_rows_thread_tiled;
            break;
        case HTP_GET_ROWS_KERNEL_FLAT:
            switch (octx->src[0]->type) {
-                case HTP_TYPE_F16:  q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f16_int32_t  : get_rows_thread_f16_int64_t);  break;
-                case HTP_TYPE_Q8_0: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_q8_0_int32_t : get_rows_thread_q8_0_int64_t); break;
+                case HTP_TYPE_F16:  grctx.compute_flat = hvx_dequantize_row_flat_f16_f32;  break;
+                case HTP_TYPE_BF16: grctx.compute_flat = hvx_dequantize_row_flat_bf16_f32; break;
+                case HTP_TYPE_Q8_0: grctx.compute_flat = hvx_dequantize_row_flat_q8_0_f32; break;
+                case HTP_TYPE_Q4_0: grctx.compute_flat = hvx_dequantize_row_flat_q4_0_f32; break;
+                case HTP_TYPE_Q4_K: grctx.compute_flat = hvx_dequantize_row_flat_q4_k_f32; break;
+                case HTP_TYPE_Q5_K: grctx.compute_flat = hvx_dequantize_row_flat_q5_k_f32; break;
+                case HTP_TYPE_Q6_K: grctx.compute_flat = hvx_dequantize_row_flat_q6_k_f32; break;
                default:            return HTP_STATUS_NO_SUPPORT;
            }
+            q_func = get_rows_thread_flat;
            break;
        default:
            return HTP_STATUS_NO_SUPPORT;
    }

+    const uint32_t elem_size = (grctx.idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t);
+    const size_t src1_bytes  = (size_t) total_tasks * elem_size;
+
+    if (!htp_tensor_is_extended(octx->src[1]) && src1_bytes <= 128 &&
+        htp_tensor_is_contiguous(octx->src[1], elem_size)) {
+        grctx.src1_data  = (const void *) (uintptr_t) octx->src[1]->data;
+        grctx.read_index = get_rows_read_index_ddr;
+    } else {
+        uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, grctx.vtcm_base, grctx.vtcm_layout.off_src1);
+        get_rows_dma_src1(octx, vtcm_src1, elem_size);
+        grctx.src1_data  = vtcm_src1;
+        grctx.read_index = get_rows_read_index_vtcm;
+    }
+
    FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu kernel-type %d n-threads %d\n",
         octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3],
         octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3],
@@ -5,7 +5,8 @@
 #include <string.h>

 #include "hex-fastdiv.h"
-#include "matmul-ops.h"
+#include "htp-tensor.h"
+#include "htp-vtcm.h"

 enum htp_get_rows_kernel_type {
    HTP_GET_ROWS_KERNEL_SAMETYPE = 0,
@@ -16,8 +17,6 @@ enum htp_get_rows_kernel_type {
 struct htp_get_rows_kernel_params {
    int32_t  n_threads;
    int32_t  kernel_type;
-    int32_t  chunks_per_row;
-    int32_t  chunk_size;
    int32_t  total_tasks;
    int32_t  tasks_per_thread;
    int32_t  vtcm_size;
@@ -25,13 +24,14 @@ struct htp_get_rows_kernel_params {
    // Fastdiv helpers
    struct fastdiv_values div_ne10;
    struct fastdiv_values div_ne10_ne11;
-    struct fastdiv_values div_chunks_per_row;
    struct fastdiv_values div_ne02;
    struct fastdiv_values div_ne03;
 };

 struct htp_get_rows_vtcm_layout {
    size_t total_bytes;
+    size_t off_src1;
+    size_t src1_bytes;
    size_t off_src0;
    size_t off_dst;

@@ -43,57 +43,44 @@ struct htp_get_rows_vtcm_layout {
 };

 static inline void htp_get_rows_vtcm_layout_build(
-    struct htp_get_rows_vtcm_layout * vtcm_layout,
+    struct htp_get_rows_vtcm_layout * L,
    int kernel_type,
    int type,
    uint32_t ne00,
+    uint32_t total_tasks,
+    int idx_type,
    uint32_t n_threads) {

-    if (kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) {
-        memset(vtcm_layout, 0, sizeof(*vtcm_layout));
-        return;
+    memset(L, 0, sizeof(*L));
+
+    const size_t elem_size = (idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t);
+    const size_t src1_raw_bytes = total_tasks * elem_size;
+    const size_t src1_bytes = ((src1_raw_bytes + 127) & ~127) + 128;
+
+    L->src1_bytes = src1_bytes;
+    const size_t src1_alloc_bytes = (src1_bytes + 255) & ~255;
+
+    if (kernel_type != HTP_GET_ROWS_KERNEL_SAMETYPE) {
+        size_t src0_row_size = 0;
+        if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) {
+            const size_t tile_stride = htp_tensor_get_tile_padded_size(type);
+            const uint32_t n_k_tiles = ne00 / HTP_TENSOR_TILE_N_COLS;
+            src0_row_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride;
+        } else {
+            src0_row_size = htp_tensor_get_row_size(type, ne00);
+        }
+
+        L->src0_spad_half_size   = (src0_row_size + 255) & ~255;
+        L->dst_spad_half_size    = (ne00 * sizeof(float) + 255) & ~255;
+        L->src0_bytes_per_thread = 2 * L->src0_spad_half_size;
+        L->dst_bytes_per_thread  = 2 * L->dst_spad_half_size;
    }

-    if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) {
-        const size_t tile_size   = htp_mm_get_weight_tile_size(type);
-        const size_t tile_stride = (tile_size + 127) & ~127;
-        const uint32_t n_k_tiles = ne00 / HTP_MM_HMX_TILE_N_COLS;
-        const size_t row_tiles_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride;
-        vtcm_layout->src0_spad_half_size = (row_tiles_size + 255) & ~255;
-        vtcm_layout->dst_spad_half_size  = (ne00 * sizeof(float) + 255) & ~255;
-        vtcm_layout->src0_bytes_per_thread = 2 * vtcm_layout->src0_spad_half_size;
-        vtcm_layout->dst_bytes_per_thread  = 2 * vtcm_layout->dst_spad_half_size;
-        vtcm_layout->off_src0 = 0;
-        vtcm_layout->off_dst  = vtcm_layout->src0_bytes_per_thread * n_threads;
-        vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads;
-        return;
-    }
-
-    uint32_t src0_row_size = 0;
-    switch (type) {
-        case HTP_TYPE_F16:
-            src0_row_size = ne00 * 2;
-            break;
-        case HTP_TYPE_Q8_0:
-            src0_row_size = (ne00 / 32) * 34;
-            break;
-        default:
-            src0_row_size = 0;
-            break;
-    }
-
-    size_t src0_row_size_aligned = (src0_row_size + 255) & ~255;
-    size_t dst_row_size_aligned  = (ne00 * sizeof(float) + 255) & ~255;
-
-    vtcm_layout->src0_spad_half_size = src0_row_size_aligned;
-    vtcm_layout->dst_spad_half_size  = dst_row_size_aligned;
-
-    vtcm_layout->src0_bytes_per_thread = src0_row_size_aligned * 2;
-    vtcm_layout->dst_bytes_per_thread  = dst_row_size_aligned * 2;
-
-    vtcm_layout->off_src0 = 0;
-    vtcm_layout->off_dst  = vtcm_layout->off_src0 + vtcm_layout->src0_bytes_per_thread * n_threads;
-    vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads;
+    size_t off = 0;
+    VTCM_LAYOUT_ALLOC(off, off_src1, src1_alloc_bytes);
+    VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
+    VTCM_LAYOUT_ALLOC(off, off_dst,  L->dst_bytes_per_thread  * n_threads);
+    L->total_bytes = off;
 }

 #if defined(__cplusplus)
@@ -14,11 +14,15 @@ static const __fp16 iq4_nl_to_fp16_lut[64] __attribute__((aligned(VLEN))) = {

 // --- tiled format dequantizers ---

-typedef struct {
+typedef struct tiled_dequantize_state tiled_dequantize_state_t;
+typedef void (*tiled_dequant_task_fn_t)(const tiled_dequantize_state_t * state, uint32_t start_tile, uint32_t end_tile);
+
+struct tiled_dequantize_state {
    struct htp_context      * ctx;
    struct htp_thread_trace * traces;
    __fp16                  * dst;
    const uint8_t           * src;
+    tiled_dequant_task_fn_t   task_fn;

    struct fastdiv_values     n_k_tiles_div;
    uint32_t                  n_k_tiles;
@@ -31,7 +35,7 @@ typedef struct {
    uint32_t                  k_block;
    size_t                    row_stride;
    uint32_t                  weight_type;
-} tiled_dequantize_state_t;
+};

 // Dequantize a single tile from tiled weight data (already in VTCM) to tile-major FP16.
 static void dequantize_tiled_weight_to_fp16_task_q4_0(
@@ -611,7 +615,7 @@ static void dequantize_tiled_weight_to_fp16_task_q5_k(
    }
 }

-// Q6_K stores 6-bit weights and one fp16 scale per 16 k, see HTP_MM_WEIGHT_TILE_SIZE_Q6_K.
+// Q6_K stores 6-bit weights and one fp16 scale per 16 k, see HTP_TENSOR_TILE_SIZE_Q6_K.
 // A k-group holds 4 k per row, the HMX tile holds 2, so each group is dealt into two tiles.
 static void dequantize_tiled_weight_to_fp16_task_q6_k(
        const tiled_dequantize_state_t *state,
@@ -669,7 +673,7 @@ static void dequantize_tiled_weight_to_fp16_task_q6_k(
    }
 }

-// Q3_K stores 3-bit weights and one fp16 scale per 16 k, see HTP_MM_WEIGHT_TILE_SIZE_Q3_K.
+// Q3_K stores 3-bit weights and one fp16 scale per 16 k, see HTP_TENSOR_TILE_SIZE_Q3_K.
 static void dequantize_tiled_weight_to_fp16_task_q3_k(
        const tiled_dequantize_state_t *state,
        uint32_t start_tile, uint32_t end_tile) {
@@ -701,7 +705,7 @@ static void dequantize_tiled_weight_to_fp16_task_q3_k(
    }
 }

-// Q2_K stores 2-bit weights, the ggml scale bytes and d / -dmin per super-block, see HTP_MM_WEIGHT_TILE_SIZE_Q2_K.
+// Q2_K stores 2-bit weights, the ggml scale bytes and d / -dmin per super-block, see HTP_TENSOR_TILE_SIZE_Q2_K.
 static void dequantize_tiled_weight_to_fp16_task_q2_k(
        const tiled_dequantize_state_t *state,
        uint32_t start_tile, uint32_t end_tile) {
@@ -837,6 +841,56 @@ void quantize_f32_weight_to_fp16_tiles_task(
    }
 }

+static __attribute__((noinline))
+void convert_bf16_weight_to_fp16_tiles_task(
+        const tiled_dequantize_state_t *state,
+        uint32_t start_tile, uint32_t end_tile) {
+
+    const uint32_t n_k_tiles = state->n_k_tiles;
+    const struct fastdiv_values n_k_tiles_div = state->n_k_tiles_div;
+
+    const HVX_Vector v_scat_base  = hvx_vmem(hmx_transpose_scatter_offsets);
+    const HVX_Vector v_scat_step  = Q6_V_vsplat_R(4);
+    const HVX_VectorPred q_mask64 = Q6_Q_vsetq_R(64);
+
+    unsigned ct = fastdiv((unsigned)start_tile, &n_k_tiles_div);
+    unsigned kt = fastmodulo((unsigned)start_tile, n_k_tiles, &n_k_tiles_div);
+
+    for (unsigned t = start_tile; t < (unsigned)end_tile; ) {
+        if (kt >= (unsigned)n_k_tiles) { kt = 0; ct++; }
+
+        __fp16 *tile_base = state->dst + t * HTP_MM_HMX_TILE_N_ELMS;
+        {
+            uint32_t byte_off = kt * 32 * sizeof(uint16_t);
+
+            HVX_Vector v_off = v_scat_base;
+            for (uint32_t r = 0; r < HTP_MM_HMX_TILE_N_ROWS; r += 2) {
+                uint32_t row0 = ct * HTP_MM_HMX_TILE_N_COLS + r;
+                uint32_t row1 = row0 + 1;
+
+                const uint8_t *r0 = state->src + row0 * state->row_stride;
+                const uint8_t *r1 = state->src + row1 * state->row_stride;
+
+                HVX_Vector v0_raw = (row0 < state->n_cols) ? hvx_vmemu((const void *)(r0 + byte_off)) : Q6_V_vzero();
+                HVX_Vector v1_raw = (row1 < state->n_cols) ? hvx_vmemu((const void *)(r1 + byte_off)) : Q6_V_vzero();
+
+                HVX_Vector v0_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(Q6_Ww_vunpack_Vh(v0_raw)), 16);
+                HVX_Vector v1_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(Q6_Ww_vunpack_Vh(v1_raw)), 16);
+
+                HVX_Vector v_out = hvx_vec_f32_to_f16(v0_f32, v1_f32);
+
+                Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v_out);
+                v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step);
+
+                HVX_Vector v_out_hi = Q6_V_vror_VR(v_out, 64);
+                Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v_out_hi);
+                v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step);
+            }
+        }
+        ++t; ++kt;
+    }
+}
+
 // --- End tiled dequantizers ---

 // dot-chunk functions require external HMX lock
@@ -3,6 +3,7 @@

 #include "dma-queue.h"
 #include "hmx-queue.h"
+#include "htp-dcache.h"
 #include "htp-ops.h"
 #include "hex-profile.h"
 #include "work-queue.h"
@@ -18,8 +19,7 @@
 #define HTP_MAX_NTHREADS 10
 #endif

-#define HTP_MAX_MMAPS        64
-#define HTP_MAX_DIRTY_RANGES 64
+#define HTP_MAX_MMAPS 64

 // Memory mapping
 struct htp_mmap {
@@ -29,11 +29,6 @@ struct htp_mmap {
    uint32_t flags;
 };

-struct htp_dirty_range {
-    uint32_t start;
-    uint32_t end;
-};
-
 // Scratchpad state
 struct htp_spad {
    const struct htp_tensor * src;             // original src of the data (for reuse)
@@ -1,4 +1,4 @@
-#include "htp-tensor.h"
+#include "htp-dcache.h"

 #include <qurt.h>
 #include <qurt_memory.h>
@@ -94,7 +94,7 @@ static void merge_dirty_ranges(struct htp_context * ctx) {
    }
 }

-void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) {
+void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) {
    const struct htp_tensor * pending[HTP_OP_MAX_OUTPUTS];
    uint32_t n_pending = 0;

@@ -301,7 +301,7 @@ static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_
    }
 }

-void htp_flush_dirty_ranges(struct htp_context * ctx) {
+void htp_dcache_flush_dirty_ranges(struct htp_context * ctx) {
    struct htp_dirty_range ranges[HTP_MAX_DIRTY_RANGES];
    uint32_t n_ranges = 0;
    uint64_t total_dirty = 0;
@@ -328,7 +328,7 @@ void htp_flush_dirty_ranges(struct htp_context * ctx) {
    memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges));
 }

-void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) {
+void htp_dcache_flush_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) {
    const struct htp_tensor * dirty_tensors[HTP_OP_MAX_INPUTS];
    struct htp_dirty_range ranges[HTP_OP_MAX_INPUTS];
    uint32_t n_dirty = 0;
@@ -0,0 +1,20 @@
+#ifndef HTP_DCACHE_H
+#define HTP_DCACHE_H
+
+#include <stdint.h>
+#include "htp-ops.h"
+
+#define HTP_MAX_DIRTY_RANGES 64
+
+struct htp_dirty_range {
+    uint32_t start;
+    uint32_t end;
+};
+
+struct htp_context;
+
+void htp_dcache_flush_dirty_ranges(struct htp_context * ctx);
+void htp_dcache_flush_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n);
+void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n);
+
+#endif // HTP_DCACHE_H
@@ -30,21 +30,14 @@ enum htp_data_type {
    HTP_TYPE_IQ4_NL = 20,
    HTP_TYPE_I32    = 26,
    HTP_TYPE_I64    = 27,
+    HTP_TYPE_BF16   = 30,
    HTP_TYPE_MXFP4  = 39,

-    // types used internally for repack, dyn.quant, etc
-    HTP_TYPE_Q4_0_TILED = 200,
-    HTP_TYPE_Q4_1_TILED,
-    HTP_TYPE_Q8_0_TILED,
-    HTP_TYPE_MXFP4_TILED,
-
    HTP_TYPE_INVALID
 };

-// Constats for internal types
-#define QK_Q4_0_TILED  256  // 32x32 Q4_0 tiled layout
+// Constants for internal types
 #define QK_Q8_0_TILED  128  // 32x32 Q8_0 tiled layout
-#define QK_MXFP4_TILED 256  // 32x32 MXFP4 tiled layout



@@ -141,7 +134,7 @@ enum htp_op_code {

 enum htp_tensor_flags {
    HTP_TENSOR_WEIGHT  = (1U << 0), // Tensor buffer model weight data (not compute)
-    HTP_TENSOR_REPACK  = (1U << 1), // Tensor is in repacked tiled format
+    HTP_TENSOR_TILED   = (1U << 1), // Tensor is in tiled format
    HTP_TENSOR_FENCE   = (1U << 2)  // Tensor is synchronization fence (explicitly managed)
 };

@@ -8,27 +8,59 @@
 #include "hex-common.h"
 #include "hex-fastdiv.h"

-enum {
-    HTP_TENSOR_MDEV_LINE_SIZE = 128,
-};
+#define HTP_TENSOR_MDEV_LINE_SIZE   128
+
+// Currently HMX requires 32x32 tile formats (do not change)
+#define HTP_TENSOR_TILE_N_COLS      32
+#define HTP_TENSOR_TILE_N_ROWS      32
+
+#define HTP_TENSOR_TILE_SIZE_F16    2048
+#define HTP_TENSOR_TILE_SIZE_Q4_0   576
+#define HTP_TENSOR_TILE_SIZE_Q4_1   640
+#define HTP_TENSOR_TILE_SIZE_Q8_0   1088
+#define HTP_TENSOR_TILE_SIZE_IQ4_NL 576
+#define HTP_TENSOR_TILE_SIZE_MXFP4  544
+#define HTP_TENSOR_TILE_SIZE_Q4_K   640
+#define HTP_TENSOR_TILE_SIZE_Q5_K   768
+#define HTP_TENSOR_TILE_SIZE_Q6_K   896
+#define HTP_TENSOR_TILE_SIZE_Q3_K   512
+#define HTP_TENSOR_TILE_SIZE_Q2_K   336
+
+// Q2_K, Q3_K, Q6_K use vrmpy byte order within each 128B HVX vector:
+//   k = 0..31 is split into 8 groups g = k/4 of 4 k each; byte 4*r + (k%4) holds row r
+//
+// Q2_K (336B, lossless tiling with same size as block_q2_K):
+//   bytes   0..255 (vec 0..1): unsigned 2-bit quants, vec m holds groups 4m..4m+3 at bit offsets 0,2,4,6
+//   bytes 256..319:            raw scales[] bytes (min << 4 | sc): [0..31] for k 0..15, [32..63] for k 16..31
+//   bytes 320..335:            8 fp16 in tile j (0..7) of super-block: d (j < 4) or -dmin (j >= 4) for rows 8*(j%4)..8*(j%4)+7
+//
+// Q3_K (512B):
+//   bytes   0..255 (vec 0..1): low 2 bits, vec m holds groups 4m..4m+3 at bit offsets 0,2,4,6
+//   bytes 256..383 (vec 2):    sign mask, bit g set when hmask is 0 (quant = low2 - 4)
+//   bytes 384..511 (vec 3):    fp16 scales d * (scales[] - 32): fp16[0..31] for k 0..15, fp16[32..63] for k 16..31
+//
+// Q4_K (640B, same layout as Q4_1 tile; each 256-k super-block splits into 8 tiles of 32 k):
+//   bytes   0..511 (vec 0..3): 4-bit quants, byte 32*cp + r holds k = 2*cp (low nibble) and 2*cp + 1 (high nibble)
+//   bytes 512..639 (vec 4):    32x2 fp16 per row: (d * sc, -dmin * m)
+//
+// Q5_K (768B):
+//   bytes   0..639 (vec 0..4): Q4_K tile (low 4 bits + fp16 scale/min pairs)
+//   bytes 640..767 (vec 5):    5th-bit plane; byte l holds the 8 high bits for lane l across vec i = 0..3 (bit 2i = lo, 2i+1 = hi)
+//
+// Q6_K (896B):
+//   bytes   0..511 (vec 0..3): low 4 bits, vec i holds group 2i (low nibble) and 2i+1 (high nibble)
+//   bytes 512..767 (vec 4..5): high 2 bits, vec 4+m holds groups 4m..4m+3 at bit offsets 0,2,4,6
+//   bytes 768..895 (vec 6):    fp16 scales d * scales[]: fp16[0..31] for k 0..15, fp16[32..63] for k 16..31

 struct htp_tensor_mdev_range {
    uint32_t start;
    uint32_t count;
 };

-static inline void * htp_tensor_data(const struct htp_tensor * t) {
-    return (void *) (uintptr_t) t->data;
-}
-
 static inline bool htp_tensor_is_extended(const struct htp_tensor * t) {
    return t && (t->data >> 32) != 0;
 }

-static inline uint32_t * htp_tensor_flags(const struct htp_tensor * t) {
-    return (uint32_t *) &t->flags;
-}
-
 static inline bool htp_tensor_is_contiguous(const struct htp_tensor * t, uint32_t type_size) {
    uint32_t next_nb = type_size;
    if (t->ne[0] != 1 && t->nb[0] != next_nb) {
@@ -125,19 +157,50 @@ static inline struct htp_tensor_mdev_range htp_tensor_mdev_partition(uint32_t to
    return range;
 }

-static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) {
+static inline uint32_t htp_tensor_get_tile_size(int type) {
    switch (type) {
-        case HTP_TYPE_F32:  return ne00 * 4;
-        case HTP_TYPE_F16:  return ne00 * 2;
-        case HTP_TYPE_Q8_0: return (ne00 / 32) * 34;
-        case HTP_TYPE_I32:  return ne00 * 4;
-        default:            return 0;
+        case HTP_TYPE_Q4_0:   return HTP_TENSOR_TILE_SIZE_Q4_0;
+        case HTP_TYPE_IQ4_NL: return HTP_TENSOR_TILE_SIZE_IQ4_NL;
+        case HTP_TYPE_Q4_1:   return HTP_TENSOR_TILE_SIZE_Q4_1;
+        case HTP_TYPE_Q4_K:   return HTP_TENSOR_TILE_SIZE_Q4_K;
+        case HTP_TYPE_Q8_0:   return HTP_TENSOR_TILE_SIZE_Q8_0;
+        case HTP_TYPE_Q5_K:   return HTP_TENSOR_TILE_SIZE_Q5_K;
+        case HTP_TYPE_Q6_K:   return HTP_TENSOR_TILE_SIZE_Q6_K;
+        case HTP_TYPE_Q3_K:   return HTP_TENSOR_TILE_SIZE_Q3_K;
+        case HTP_TYPE_Q2_K:   return HTP_TENSOR_TILE_SIZE_Q2_K;
+        case HTP_TYPE_MXFP4:  return HTP_TENSOR_TILE_SIZE_MXFP4;
+        case HTP_TYPE_F16:    return HTP_TENSOR_TILE_SIZE_F16;
+        default:              return 0;
    }
 }

-struct htp_context;
-void htp_flush_dirty_ranges(struct htp_context * ctx);
-void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n);
-void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n);
+static inline uint32_t htp_tensor_get_tile_padded_size(int type) {
+    return (htp_tensor_get_tile_size(type) + 127u) & ~127u;
+}
+
+static inline bool htp_tensor_is_tiled(const struct htp_tensor * t) {
+    return t && (t->flags & HTP_TENSOR_TILED) != 0;
+}
+
+static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) {
+    switch (type) {
+        case HTP_TYPE_F32:    return ne00 * 4;
+        case HTP_TYPE_F16:
+        case HTP_TYPE_BF16:   return ne00 * 2;
+        case HTP_TYPE_Q8_0:   return (ne00 / 32) * 34;
+        case HTP_TYPE_Q4_0:
+        case HTP_TYPE_IQ4_NL: return (ne00 / 32) * 18;
+        case HTP_TYPE_Q4_1:   return (ne00 / 32) * 20;
+        case HTP_TYPE_MXFP4:  return (ne00 / 32) * 17;
+        case HTP_TYPE_Q2_K:   return (ne00 / 256) * 84;
+        case HTP_TYPE_Q3_K:   return (ne00 / 256) * 110;
+        case HTP_TYPE_Q4_K:   return (ne00 / 256) * 144;
+        case HTP_TYPE_Q5_K:   return (ne00 / 256) * 176;
+        case HTP_TYPE_Q6_K:   return (ne00 / 256) * 210;
+        case HTP_TYPE_I32:    return ne00 * 4;
+        case HTP_TYPE_I64:    return ne00 * 8;
+        default:              return 0;
+    }
+}

 #endif // HTP_TENSOR_H
@@ -76,7 +76,7 @@ static inline void quantize_f16_f16_kernel(
 #define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b)
 #endif

-static inline void vec_dot_f32_f32_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) {
+static inline void flat_vec_dot_f32_f32_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) {
    const HVX_Vector * restrict x = (const HVX_Vector *) vx;
    const HVX_Vector * restrict y = (const HVX_Vector *) vy;

@@ -104,7 +104,7 @@ static inline void vec_dot_f32_f32_aa_1x1(const uint32_t n, float * restrict s,
    *s = hvx_vec_get_f32(hvx_vec_reduce_sum_f32(rsum));
 }

-static inline void vec_dot_f32_f32_aa_2x1(const uint32_t n, float * restrict s0,
+static inline void flat_vec_dot_f32_f32_aa_2x1(const uint32_t n, float * restrict s0,
                                const void * restrict vx0, const void * restrict vx1,
                                const void * restrict vy0) {
    const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0;
@@ -143,7 +143,7 @@ static inline void vec_dot_f32_f32_aa_2x1(const uint32_t n, float * restrict s0,
    hvx_vec_store_u(s0, 8, rsum);
 }

-static inline void vec_dot_f32_f32_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1,
+static inline void flat_vec_dot_f32_f32_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1,
                                const void * restrict vx0, const void * restrict vx1,
                                const void * restrict vy0, const void * restrict vy1) {
    const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0;
@@ -196,10 +196,203 @@ static inline void vec_dot_f32_f32_aa_2x2(const uint32_t n, float * restrict s0,
    hvx_vec_store_u(s1, 8, r0_r1_c1_sum);
 }

+static inline void flat_vec_dot_bf16_f32_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) {
+    const HVX_Vector * restrict x = (const HVX_Vector *) vx;
+    const HVX_Vector * restrict y = (const HVX_Vector *) vy;
+
+    uint32_t nvec = n / 64;
+    uint32_t rem  = n % 64;
+
+    HVX_Vector rsum0 = Q6_V_vzero();
+    HVX_Vector rsum1 = Q6_V_vzero();
+
+    uint32_t i = 0;
+    #pragma unroll(2)
+    for (i = 0; i < nvec; i++) {
+        HVX_Vector v_bf16 = x[i];
+        HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16);
+        HVX_Vector w0_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16);
+        HVX_Vector w1_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p), 16);
+
+        rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w0_f32, y[2 * i + 0]));
+        rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w1_f32, y[2 * i + 1]));
+    }
+
+    if (rem) {
+        HVX_VectorPred bmask = Q6_Q_vsetq2_R(rem * 2);
+        HVX_Vector v_bf16 = Q6_V_vand_QV(bmask, x[i]);
+        HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16);
+        HVX_Vector w0_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16);
+        HVX_Vector w1_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p), 16);
+
+        if (rem <= 32) {
+            HVX_VectorPred a_mask = Q6_Q_vsetq2_R(rem * 4);
+            HVX_Vector y0 = Q6_V_vand_QV(a_mask, y[2 * i + 0]);
+            rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w0_f32, y0));
+        } else {
+            HVX_Vector y0 = y[2 * i + 0];
+            rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w0_f32, y0));
+
+            HVX_VectorPred a_mask = Q6_Q_vsetq2_R((rem - 32) * 4);
+            HVX_Vector y1 = Q6_V_vand_QV(a_mask, y[2 * i + 1]);
+            rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w1_f32, y1));
+        }
+    }
+
+    HVX_Vector rsum = HVX_OP_ADD_F32(rsum0, rsum1);
+    *s = hvx_vec_get_f32(hvx_vec_reduce_sum_f32(rsum));
+}
+
+static inline void flat_vec_dot_bf16_f32_aa_2x1(const uint32_t n, float * restrict s0,
+                                const void * restrict vx0, const void * restrict vx1,
+                                const void * restrict vy0) {
+    const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0;
+    const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1;
+    const HVX_Vector * restrict y  = (const HVX_Vector *) vy0;
+
+    uint32_t nvec = n / 64;
+    uint32_t rem  = n % 64;
+
+    HVX_Vector rsum0 = Q6_V_vzero();
+    HVX_Vector rsum1 = Q6_V_vzero();
+
+    uint32_t i = 0;
+    #pragma unroll(2)
+    for (i = 0; i < nvec; i++) {
+        HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(x0[i]);
+        HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(x1[i]);
+        HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16);
+        HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16);
+        HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16);
+        HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16);
+
+        HVX_Vector y0 = y[2 * i + 0];
+        HVX_Vector y1 = y[2 * i + 1];
+
+        rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w00_f32, y0));
+        rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w10_f32, y0));
+        rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w01_f32, y1));
+        rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w11_f32, y1));
+    }
+
+    if (rem) {
+        HVX_VectorPred bmask = Q6_Q_vsetq2_R(rem * 2);
+        HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x0[i]));
+        HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x1[i]));
+        HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16);
+        HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16);
+        HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16);
+        HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16);
+
+        if (rem <= 32) {
+            HVX_VectorPred a_mask = Q6_Q_vsetq2_R(rem * 4);
+            HVX_Vector y0 = Q6_V_vand_QV(a_mask, y[2 * i + 0]);
+            rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w00_f32, y0));
+            rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w10_f32, y0));
+        } else {
+            HVX_Vector y0 = y[2 * i + 0];
+            rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w00_f32, y0));
+            rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w10_f32, y0));
+
+            HVX_VectorPred a_mask = Q6_Q_vsetq2_R((rem - 32) * 4);
+            HVX_Vector y1 = Q6_V_vand_QV(a_mask, y[2 * i + 1]);
+            rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w01_f32, y1));
+            rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w11_f32, y1));
+        }
+    }
+
+    HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1);
+    hvx_vec_store_u(s0, 8, rsum);
+}
+
+static inline void flat_vec_dot_bf16_f32_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1,
+                                const void * restrict vx0, const void * restrict vx1,
+                                const void * restrict vy0, const void * restrict vy1) {
+    const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0;
+    const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1;
+    const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0;
+    const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1;
+
+    uint32_t nvec = n / 64;
+    uint32_t rem  = n % 64;
+
+    HVX_Vector r0_c0_sum = Q6_V_vzero();
+    HVX_Vector r0_c1_sum = Q6_V_vzero();
+    HVX_Vector r1_c0_sum = Q6_V_vzero();
+    HVX_Vector r1_c1_sum = Q6_V_vzero();
+
+    uint32_t i = 0;
+    #pragma unroll(2)
+    for (i = 0; i < nvec; i++) {
+        HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(x0[i]);
+        HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(x1[i]);
+        HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16);
+        HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16);
+        HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16);
+        HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16);
+
+        HVX_Vector c0_0 = y0[2 * i + 0];
+        HVX_Vector c0_1 = y0[2 * i + 1];
+        HVX_Vector c1_0 = y1[2 * i + 0];
+        HVX_Vector c1_1 = y1[2 * i + 1];
+
+        r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w00_f32, c0_0));
+        r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w00_f32, c1_0));
+        r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w10_f32, c0_0));
+        r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w10_f32, c1_0));
+
+        r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w01_f32, c0_1));
+        r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w01_f32, c1_1));
+        r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w11_f32, c0_1));
+        r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w11_f32, c1_1));
+    }
+
+    if (rem) {
+        HVX_VectorPred bmask = Q6_Q_vsetq2_R(rem * 2);
+        HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x0[i]));
+        HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x1[i]));
+        HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16);
+        HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16);
+        HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16);
+        HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16);
+
+        if (rem <= 32) {
+            HVX_VectorPred a_mask = Q6_Q_vsetq2_R(rem * 4);
+            HVX_Vector c0_0 = Q6_V_vand_QV(a_mask, y0[2 * i + 0]);
+            HVX_Vector c1_0 = Q6_V_vand_QV(a_mask, y1[2 * i + 0]);
+            r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w00_f32, c0_0));
+            r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w00_f32, c1_0));
+            r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w10_f32, c0_0));
+            r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w10_f32, c1_0));
+        } else {
+            HVX_Vector c0_0 = y0[2 * i + 0];
+            HVX_Vector c1_0 = y1[2 * i + 0];
+            r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w00_f32, c0_0));
+            r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w00_f32, c1_0));
+            r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w10_f32, c0_0));
+            r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w10_f32, c1_0));
+
+            HVX_VectorPred a_mask = Q6_Q_vsetq2_R((rem - 32) * 4);
+            HVX_Vector c0_1 = Q6_V_vand_QV(a_mask, y0[2 * i + 1]);
+            HVX_Vector c1_1 = Q6_V_vand_QV(a_mask, y1[2 * i + 1]);
+            r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w01_f32, c0_1));
+            r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w01_f32, c1_1));
+            r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w11_f32, c0_1));
+            r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w11_f32, c1_1));
+        }
+    }
+
+    HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum);
+    HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum);
+
+    hvx_vec_store_u(s0, 8, r0_r1_c0_sum);
+    hvx_vec_store_u(s1, 8, r0_r1_c1_sum);
+}
+
 #undef HVX_OP_ADD_F32
 #undef HVX_OP_MUL_F32

-static inline void vec_dot_f16_f16_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) {
+static inline void flat_vec_dot_f16_f16_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) {
    const HVX_Vector * restrict x = (const HVX_Vector *) vx;
    const HVX_Vector * restrict y = (const HVX_Vector *) vy;

@@ -226,7 +419,7 @@ static inline void vec_dot_f16_f16_aa_1x1(const uint32_t n, float * restrict s,
    hvx_vec_store_u(s, 4, hvx_vec_reduce_sum_f32(rsum));
 }

-static inline void vec_dot_f16_f16_aa_2x1(const uint32_t n, float * restrict s0,
+static inline void flat_vec_dot_f16_f16_aa_2x1(const uint32_t n, float * restrict s0,
                                const void * restrict vx0, const void * restrict vx1,
                                const void * restrict vy0) {
    const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0;
@@ -263,7 +456,7 @@ static inline void vec_dot_f16_f16_aa_2x1(const uint32_t n, float * restrict s0,
    hvx_vec_store_u(s0, 8, rsum);
 }

-static inline void vec_dot_f16_f16_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1,
+static inline void flat_vec_dot_f16_f16_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1,
                                const void * restrict vx0, const void * restrict vx1,
                                const void * restrict vy0, const void * restrict vy1) {
    const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0;
@@ -274,6 +274,72 @@ static void quantize_row_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restri
    }
 }

+static inline void quantize_half_f16_tiled_splat(HVX_Vector v_src, HVX_Vector * restrict dst) {
+    static const uint8_t __attribute__((aligned(128))) repl[128] = {
+        0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+        0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+        0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+        0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+        0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+        0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+        0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+        0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+    };
+    HVX_Vector v_repl_ctrl = *(const HVX_Vector *) repl;
+
+    #pragma unroll(4)
+    for (int cp = 0; cp < 32; cp++) {
+        dst[cp] = Q6_V_vdelta_VV(cp == 0 ? v_src : Q6_V_vror_VR(v_src, cp * 4), v_repl_ctrl);
+    }
+}
+
+static inline void quantize_block_f32_f16_tiled(const float * restrict x, uint8_t * restrict y_block) {
+    assert((unsigned long) x % 128 == 0);
+    assert((unsigned long) y_block % 128 == 0);
+
+    const HVX_Vector * vx = (const HVX_Vector *) x;
+    HVX_Vector * restrict dst = (HVX_Vector *) y_block;
+
+    HVX_Vector vx01_hf = hvx_vec_f32_to_f16(vx[0], vx[1]);
+    quantize_half_f16_tiled_splat(vx01_hf, dst);
+
+    HVX_Vector vx23_hf = hvx_vec_f32_to_f16(vx[2], vx[3]);
+    quantize_half_f16_tiled_splat(vx23_hf, dst + 32);
+}
+
+static inline void quantize_block_f16_f16_tiled(const __fp16 * restrict x, uint8_t * restrict y_block) {
+    assert((unsigned long) x % 128 == 0);
+    assert((unsigned long) y_block % 128 == 0);
+
+    const HVX_Vector * vx = (const HVX_Vector *) x;
+    HVX_Vector * restrict dst = (HVX_Vector *) y_block;
+
+    quantize_half_f16_tiled_splat(vx[0], dst);
+    quantize_half_f16_tiled_splat(vx[1], dst + 32);
+}
+
+static void quantize_row_f32_f16_tiled(const float * restrict x, uint8_t * restrict y, uint32_t k) {
+    assert(k % 32 == 0);
+    const uint32_t qk = QK_Q8_0_TILED;
+    const uint32_t nb = (k + qk - 1) / qk;
+
+    for (uint32_t i = 0; i < nb; i++) {
+        uint8_t * restrict y_block = y + i * 4 * 2048;
+        quantize_block_f32_f16_tiled(x + i * qk, y_block);
+    }
+}
+
+static void quantize_row_f16_f16_tiled(const __fp16 * restrict x, uint8_t * restrict y, uint32_t k) {
+    assert(k % 32 == 0);
+    const uint32_t qk = QK_Q8_0_TILED;
+    const uint32_t nb = (k + qk - 1) / qk;
+
+    for (uint32_t i = 0; i < nb; i++) {
+        uint8_t * restrict y_block = y + i * 4 * 2048;
+        quantize_block_f16_f16_tiled(x + i * qk, y_block);
+    }
+}
+
 // Dot kernels & helpers that consume tiled activations

 static inline HVX_Vector hvx_vec_mul_f16_f16_to_f32_lower32(HVX_Vector v1, HVX_Vector v2) {
@@ -454,7 +520,7 @@ static inline HVX_VectorPair accum_q8_0_32x2(
    return Q6_W_vcombine_VV(v_sum1, v_sum0);
 }

-// Q5_K: OR 0x10 into every lane of v whose flag j is set in the plane (see HTP_MM_WEIGHT_TILE_SIZE_Q5_K)
+// Q5_K: OR 0x10 into every lane of v whose flag j is set in the plane (see HTP_TENSOR_TILE_SIZE_Q5_K)
 static inline HVX_Vector hvx_q5k_or_hibit(HVX_Vector v, HVX_Vector v_plane, int j) {
    HVX_VectorPred q = Q6_Q_vand_VR(v_plane, 0x01010101u << j);
    return Q6_V_vandor_VQR(v, q, 0x10101010);
@@ -526,7 +592,7 @@ static inline HVX_VectorPair accum_5bit_32x2(
    return Q6_W_vcombine_VV(v_sum1, v_sum0);
 }

-// Q6_K weights are stored unsigned (0..63), see HTP_MM_WEIGHT_TILE_SIZE_Q6_K. Unpack k-group g of a tile to signed bytes (q - 32)
+// Q6_K weights are stored unsigned (0..63), see HTP_TENSOR_TILE_SIZE_Q6_K. Unpack k-group g of a tile to signed bytes (q - 32)
 static inline HVX_Vector unpack_q6_k_group(const HVX_Vector * restrict vptr, int g, HVX_Vector mask_0f, HVX_Vector mask_03, HVX_Vector i32) {
    HVX_Vector v_lo = (g & 1) ? Q6_Vub_vlsr_VubR(vptr[g >> 1], 4) : Q6_V_vand_VV(vptr[g >> 1], mask_0f);
    HVX_Vector v_hi = (g & 3) ? Q6_Vub_vlsr_VubR(vptr[4 + (g >> 2)], 2 * (g & 3)) : vptr[4 + (g >> 2)];
@@ -593,7 +659,7 @@ static inline HVX_Vector scale_q6_k_32x1(HVX_VectorPair v_sums, HVX_Vector v_sca
    return hvx_vec_add_f32_f32(v_lo, v_hi);
 }

-// Q3_K / Q2_K: low 2 bits of k-group g, see HTP_MM_WEIGHT_TILE_SIZE_Q3_K
+// Q3_K / Q2_K: low 2 bits of k-group g, see HTP_TENSOR_TILE_SIZE_Q3_K
 static inline HVX_Vector unpack_q3_k_low2(const HVX_Vector * restrict vptr, int g, HVX_Vector mask_03) {
    HVX_Vector v = vptr[g >> 2];
    if ((g & 3) == 3) {
@@ -1482,6 +1548,59 @@ static void tiled_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, floa
    }
 }

+static void tiled_vec_dot_f16_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
+    const HVX_Vector * restrict vptr  = (const HVX_Vector *) vx;
+    const HVX_Vector * restrict v_act = (const HVX_Vector *) vy;
+    const uint32_t n_vecs = ((n + 31) / 32) * 16;
+
+    HVX_VectorPair acc = Q6_W_vzero();
+
+    #pragma unroll(4)
+    for (uint32_t i = 0; i < n_vecs; i++) {
+        acc = hvx_vec_mpyacc_f32_f16(acc, vptr[i], v_act[i]);
+    }
+
+    HVX_Vector v_sum = hvx_vec_add_f32_f32(Q6_V_lo_W(acc), Q6_V_hi_W(acc));
+
+    if (sz) {
+        hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum, hvx_vmemu(sz)));
+    } else {
+        hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum);
+    }
+}
+
+static void tiled_vec_dot_f16_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) {
+    const HVX_Vector * restrict vptr   = (const HVX_Vector *) vx;
+    const HVX_Vector * restrict v_act0 = (const HVX_Vector *) vy0;
+    const HVX_Vector * restrict v_act1 = (const HVX_Vector *) vy1;
+    const uint32_t n_vecs = ((n + 31) / 32) * 16;
+
+    HVX_VectorPair acc0 = Q6_W_vzero();
+    HVX_VectorPair acc1 = Q6_W_vzero();
+
+    #pragma unroll(4)
+    for (uint32_t i = 0; i < n_vecs; i++) {
+        HVX_Vector v_w = vptr[i];
+        acc0 = hvx_vec_mpyacc_f32_f16(acc0, v_w, v_act0[i]);
+        acc1 = hvx_vec_mpyacc_f32_f16(acc1, v_w, v_act1[i]);
+    }
+
+    HVX_Vector v_sum0 = hvx_vec_add_f32_f32(Q6_V_lo_W(acc0), Q6_V_hi_W(acc0));
+    HVX_Vector v_sum1 = hvx_vec_add_f32_f32(Q6_V_lo_W(acc1), Q6_V_hi_W(acc1));
+
+    if (sz0) {
+        hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum0, hvx_vmemu(sz0)));
+    } else {
+        hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum0);
+    }
+    if (sz1) {
+        hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum1, hvx_vmemu(sz1)));
+    } else {
+        hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum1);
+    }
+}
+
+
 static inline void quantize_f32_q8_0_tiled_kernel(
    const uint8_t * restrict src_data,
    uint8_t * restrict dst_data,
@@ -1533,6 +1652,40 @@ static inline void quantize_f32_q8_1_s16_tiled_kernel(
    }
 }

+static inline void quantize_f32_f16_tiled_kernel(
+    const uint8_t * restrict src_data,
+    uint8_t * restrict dst_data,
+    uint8_t * restrict tmp_data,
+    uint32_t ne0,
+    uint32_t nrows,
+    size_t src_row_size,
+    size_t dst_row_size
+) {
+    (void) tmp_data;
+    for (uint32_t i = 0; i < nrows; ++i) {
+        quantize_row_f32_f16_tiled((const float *) src_data, dst_data, ne0);
+        dst_data += dst_row_size;
+        src_data += src_row_size;
+    }
+}
+
+static inline void quantize_f16_f16_tiled_kernel(
+    const uint8_t * restrict src_data,
+    uint8_t * restrict dst_data,
+    uint8_t * restrict tmp_data,
+    uint32_t ne0,
+    uint32_t nrows,
+    size_t src_row_size,
+    size_t dst_row_size
+) {
+    (void) tmp_data;
+    for (uint32_t i = 0; i < nrows; ++i) {
+        quantize_row_f16_f16_tiled((const __fp16 *) src_data, dst_data, ne0);
+        dst_data += dst_row_size;
+        src_data += src_row_size;
+    }
+}
+
 static inline void quantize_f32_q8_0_tiled_block_kernel(
    const float * restrict src,
    uint8_t * restrict dst,
@@ -1622,3 +1775,63 @@ static inline void quantize_f32_q8_1_s16_tiled_block_kernel(
        }
    }
 }
+
+static inline void quantize_f32_f16_tiled_block_kernel(
+    const float * restrict src,
+    uint8_t * restrict dst,
+    uint8_t * restrict tmp_data,
+    uint32_t ne0,
+    uint32_t ib_first,
+    uint32_t ib_last,
+    size_t src_row_size,
+    size_t dst_row_size,
+    uint32_t r,
+    uint32_t c
+) {
+    (void) tmp_data;
+    const uint32_t qk = QK_Q8_0_TILED;
+    const uint32_t nb = (ne0 + qk - 1) / qk;
+
+    for (uint32_t ib = ib_first; ib < ib_last; ++ib) {
+        const float * restrict src_ptr = (const float *) ((const uint8_t *) src + r * src_row_size + c * qk * sizeof(float));
+        uint8_t * restrict dst_ptr = dst + r * dst_row_size + c * 4 * 2048;
+
+        quantize_block_f32_f16_tiled(src_ptr, dst_ptr);
+
+        c++;
+        if (c == nb) {
+            c = 0;
+            r++;
+        }
+    }
+}
+
+static inline void quantize_f16_f16_tiled_block_kernel(
+    const __fp16 * restrict src,
+    uint8_t * restrict dst,
+    uint8_t * restrict tmp_data,
+    uint32_t ne0,
+    uint32_t ib_first,
+    uint32_t ib_last,
+    size_t src_row_size,
+    size_t dst_row_size,
+    uint32_t r,
+    uint32_t c
+) {
+    (void) tmp_data;
+    const uint32_t qk = QK_Q8_0_TILED;
+    const uint32_t nb = (ne0 + qk - 1) / qk;
+
+    for (uint32_t ib = ib_first; ib < ib_last; ++ib) {
+        const __fp16 * restrict src_ptr = (const __fp16 *) ((const uint8_t *) src + r * src_row_size + c * qk * sizeof(__fp16));
+        uint8_t * restrict dst_ptr = dst + r * dst_row_size + c * 4 * 2048;
+
+        quantize_block_f16_f16_tiled(src_ptr, dst_ptr);
+
+        c++;
+        if (c == nb) {
+            c = 0;
+            r++;
+        }
+    }
+}
@@ -5,6 +5,7 @@
 #include <stdint.h>
 #include <string.h>

+#include "htp-tensor.h"
 #include "hvx-arith.h"
 #include "hvx-base.h"
 #include "hvx-reduce.h"
@@ -96,7 +97,7 @@ static inline void hvx_quantize_row_q8_0_f32(void * restrict dst_ptr, const floa
    }
 }

-static inline void hvx_dequantize_row_q8_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, int n) {
+static inline void hvx_dequantize_row_flat_q8_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
    const int nb = n / QK8_0;
    const block_q8_0 * src = (const block_q8_0 *) src_ptr;

@@ -120,7 +121,7 @@ static inline void hvx_dequantize_row_q8_0_f32(float * restrict dst_ptr, const v
    }
 }

-static inline void hvx_dequantize_row_q8_0_f16(__fp16 * restrict dst_ptr, const void * restrict src_ptr, int n) {
+static inline void hvx_dequantize_row_flat_q8_0_f16(__fp16 * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
    const int nb = n / QK8_0;
    const block_q8_0 * src = (const block_q8_0 *) src_ptr;

@@ -146,20 +147,460 @@ static inline void hvx_dequantize_row_q8_0_f16(__fp16 * restrict dst_ptr, const
    }
 }

-static inline void hvx_dequantize_row_f16_f32(float * restrict dst_ptr, const void * restrict src_ptr, int n) {
-    const int nb = n / 32;
+static inline void hvx_dequantize_row_flat_f16_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
+    const uint32_t nvec = n / VLEN_FP32;
+    const uint32_t nloe = n % VLEN_FP32;
    const _Float16 * src = (const _Float16 *) src_ptr;

-    for (int i = 0; i < nb; i++) {
-        HVX_Vector v_f16 = *(const HVX_UVector *) (src + i * 32);
+    for (uint32_t i = 0; i < nvec; i++) {
+        HVX_Vector v_f16 = *(const HVX_UVector *) (src + i * VLEN_FP32);
        HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(v_f16);
        HVX_Vector res = Q6_V_lo_W(vp_f32);

-        float * block_dst = dst_ptr + i * 32;
-        hvx_vmem(block_dst) = res;
+        hvx_vmem(dst_ptr + i * VLEN_FP32) = res;
+    }
+
+    if (nloe) {
+        HVX_Vector v_f16 = *(const HVX_UVector *) (src + nvec * VLEN_FP32);
+        HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(v_f16);
+        HVX_Vector res = Q6_V_lo_W(vp_f32);
+
+        hvx_vec_store_a(dst_ptr + nvec * VLEN_FP32, nloe * sizeof(float), res);
    }
 }

+static inline void hvx_dequantize_row_flat_bf16_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
+    const uint32_t nvec = n / VLEN_FP32;
+    const uint32_t nloe = n % VLEN_FP32;
+    const uint16_t * src = (const uint16_t *) src_ptr;

+    for (uint32_t i = 0; i < nvec; i++) {
+        HVX_Vector v_bf16 = *(const HVX_UVector *) (src + i * VLEN_FP32);
+        HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16);
+        HVX_Vector res = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16);
+
+        hvx_vmem(dst_ptr + i * VLEN_FP32) = res;
+    }
+
+    if (nloe) {
+        HVX_Vector v_bf16 = *(const HVX_UVector *) (src + nvec * VLEN_FP32);
+        HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16);
+        HVX_Vector res = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16);
+
+        hvx_vec_store_a(dst_ptr + nvec * VLEN_FP32, nloe * sizeof(float), res);
+    }
+}
+
+static inline void hvx_dequantize_row_flat_q4_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
+    const int nb = n / QK4_0;
+    const block_q4_0 * src = (const block_q4_0 *) src_ptr;
+
+    const HVX_VectorPred q16 = Q6_Q_vsetq_R(16);
+    const HVX_Vector v_eight = Q6_Vb_vsplat_R(8);
+    const HVX_Vector v_zero  = Q6_V_vzero();
+
+    for (int i = 0; i < nb; i++) {
+        HVX_Vector vd_f16     = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
+        HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16);
+        HVX_Vector vd         = Q6_V_lo_W(vp_f32);
+
+        HVX_Vector vq = *(const HVX_UVector *) src[i].qs;
+        HVX_Vector v_lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F));
+        HVX_Vector v_hi = Q6_Vub_vlsr_VubR(vq, 4);
+
+        HVX_Vector v_lo_s8 = Q6_Vb_vsub_VbVb(v_lo, v_eight);
+        HVX_Vector v_hi_s8 = Q6_Vb_vsub_VbVb(v_hi, v_eight);
+
+        HVX_Vector v32 = Q6_V_vmux_QVV(q16, v_lo_s8, Q6_V_vlalign_VVR(v_hi_s8, v_zero, 16));
+
+        HVX_VectorPair p16   = Q6_Wh_vunpack_Vb(v32);
+        HVX_Vector     v_i16 = Q6_V_lo_W(p16);
+        HVX_VectorPair p32   = Q6_Ww_vunpack_Vh(v_i16);
+        HVX_Vector     v_i32 = Q6_V_lo_W(p32);
+
+        HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32);
+        HVX_Vector res   = hvx_vec_mul_f32_f32(v_f32, vd);
+
+        hvx_vmem(dst_ptr + i * QK4_0) = res;
+    }
+}
+
+static inline void hvx_dequantize_row_flat_q4_k_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
+    const int nb = n / QK_K;
+    const block_q4_K * src = (const block_q4_K *) src_ptr;
+
+    for (int i = 0; i < nb; i++) {
+        const float d   = GGML_FP16_TO_FP32(src[i].d);
+        const float min = GGML_FP16_TO_FP32(src[i].dmin);
+        const uint8_t * sc = src[i].scales;
+
+        float * y = dst_ptr + i * QK_K;
+
+        for (int step = 0; step < 4; ++step) {
+            uint8_t sc1, m1_u, sc2, m2_u;
+            if (step < 2) {
+                sc1  = sc[2 * step + 0] & 63;
+                m1_u = sc[2 * step + 4] & 63;
+                sc2  = sc[2 * step + 1] & 63;
+                m2_u = sc[2 * step + 5] & 63;
+            } else {
+                const int s = step - 2;
+                sc1  = (sc[8 + 2 * s + 0] & 0xF) | ((sc[2 * s + 0] >> 6) << 4);
+                m1_u = (sc[8 + 2 * s + 0] >>  4) | ((sc[4 + 2 * s + 0] >> 6) << 4);
+                sc2  = (sc[8 + 2 * s + 1] & 0xF) | ((sc[2 * s + 1] >> 6) << 4);
+                m2_u = (sc[8 + 2 * s + 1] >>  4) | ((sc[4 + 2 * s + 1] >> 6) << 4);
+            }
+
+            HVX_Vector vq = *(const HVX_UVector *) (src[i].qs + step * 32);
+            HVX_Vector v_lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F));
+            HVX_Vector v_hi = Q6_Vub_vlsr_VubR(vq, 4);
+
+            HVX_VectorPair p16_lo = Q6_Wuh_vunpack_Vub(v_lo);
+            HVX_VectorPair p32_lo = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_lo));
+            HVX_Vector     v_i32_lo = Q6_V_lo_W(p32_lo);
+            HVX_Vector     v_f32_lo = Q6_Vsf_equals_Vw(v_i32_lo);
+
+            HVX_Vector vd1 = hvx_vec_splat_f32(d * sc1);
+            HVX_Vector vm1 = hvx_vec_splat_f32(min * m1_u);
+            HVX_Vector v_out1 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_lo, vd1), vm1);
+            hvx_vmem(y + 0) = v_out1;
+
+            HVX_VectorPair p16_hi = Q6_Wuh_vunpack_Vub(v_hi);
+            HVX_VectorPair p32_hi = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_hi));
+            HVX_Vector     v_i32_hi = Q6_V_lo_W(p32_hi);
+            HVX_Vector     v_f32_hi = Q6_Vsf_equals_Vw(v_i32_hi);
+
+            HVX_Vector vd2 = hvx_vec_splat_f32(d * sc2);
+            HVX_Vector vm2 = hvx_vec_splat_f32(min * m2_u);
+            HVX_Vector v_out2 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_hi, vd2), vm2);
+            hvx_vmem(y + 32) = v_out2;
+
+            y += 64;
+        }
+    }
+}
+
+static inline void hvx_dequantize_row_flat_q5_k_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
+    const int nb = n / QK_K;
+    const block_q5_K * src = (const block_q5_K *) src_ptr;
+
+    for (int i = 0; i < nb; i++) {
+        const float d   = GGML_FP16_TO_FP32(src[i].d);
+        const float min = GGML_FP16_TO_FP32(src[i].dmin);
+        const uint8_t * sc = src[i].scales;
+
+        HVX_Vector v_qh = *(const HVX_UVector *) src[i].qh;
+        float * y = dst_ptr + i * QK_K;
+
+        for (int step = 0; step < 4; ++step) {
+            uint8_t sc1, m1_u, sc2, m2_u;
+            if (step < 2) {
+                sc1  = sc[2 * step + 0] & 63;
+                m1_u = sc[2 * step + 4] & 63;
+                sc2  = sc[2 * step + 1] & 63;
+                m2_u = sc[2 * step + 5] & 63;
+            } else {
+                const int s = step - 2;
+                sc1  = (sc[8 + 2 * s + 0] & 0xF) | ((sc[2 * s + 0] >> 6) << 4);
+                m1_u = (sc[8 + 2 * s + 0] >>  4) | ((sc[4 + 2 * s + 0] >> 6) << 4);
+                sc2  = (sc[8 + 2 * s + 1] & 0xF) | ((sc[2 * s + 1] >> 6) << 4);
+                m2_u = (sc[8 + 2 * s + 1] >>  4) | ((sc[4 + 2 * s + 1] >> 6) << 4);
+            }
+
+            HVX_Vector v_ql = *(const HVX_UVector *) (src[i].qs + step * 32);
+            HVX_Vector v_lo = Q6_V_vand_VV(v_ql, Q6_Vb_vsplat_R(0x0F));
+            HVX_Vector v_hi = Q6_Vub_vlsr_VubR(v_ql, 4);
+
+            HVX_Vector v_h1 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 2 * step), Q6_Vb_vsplat_R(1)), 4);
+            HVX_Vector v_val1 = Q6_V_vor_VV(v_lo, v_h1);
+
+            HVX_VectorPair p16_1 = Q6_Wuh_vunpack_Vub(v_val1);
+            HVX_VectorPair p32_1 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_1));
+            HVX_Vector     v_i32_1 = Q6_V_lo_W(p32_1);
+            HVX_Vector     v_f32_1 = Q6_Vsf_equals_Vw(v_i32_1);
+
+            HVX_Vector vd1 = hvx_vec_splat_f32(d * sc1);
+            HVX_Vector vm1 = hvx_vec_splat_f32(min * m1_u);
+            HVX_Vector v_out1 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_1, vd1), vm1);
+            hvx_vmem(y + 0) = v_out1;
+
+            HVX_Vector v_h2 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 2 * step + 1), Q6_Vb_vsplat_R(1)), 4);
+            HVX_Vector v_val2 = Q6_V_vor_VV(v_hi, v_h2);
+
+            HVX_VectorPair p16_2 = Q6_Wuh_vunpack_Vub(v_val2);
+            HVX_VectorPair p32_2 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_2));
+            HVX_Vector     v_i32_2 = Q6_V_lo_W(p32_2);
+            HVX_Vector     v_f32_2 = Q6_Vsf_equals_Vw(v_i32_2);
+
+            HVX_Vector vd2 = hvx_vec_splat_f32(d * sc2);
+            HVX_Vector vm2 = hvx_vec_splat_f32(min * m2_u);
+            HVX_Vector v_out2 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_2, vd2), vm2);
+            hvx_vmem(y + 32) = v_out2;
+
+            y += 64;
+        }
+    }
+}
+
+static inline void hvx_dequantize_row_flat_q6_k_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
+    const int nb = n / QK_K;
+    const block_q6_K * src = (const block_q6_K *) src_ptr;
+
+    const HVX_VectorPred q64 = Q6_Q_vsetq_R(64);
+    const HVX_Vector v32_b   = Q6_Vb_vsplat_R(32);
+    const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F);
+    const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
+
+    for (int i = 0; i < nb; i++) {
+        const float d = GGML_FP16_TO_FP32(src[i].d);
+
+        const uint8_t * restrict ql = src[i].ql;
+        const uint8_t * restrict qh = src[i].qh;
+        const int8_t  * restrict sc = src[i].scales;
+
+        float * y = dst_ptr + i * QK_K;
+
+        for (int half = 0; half < 2; ++half) {
+            float s[8];
+            for (int j = 0; j < 8; ++j) {
+                s[j] = d * sc[j];
+            }
+
+            HVX_Vector vscale1 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[0]), hvx_vec_splat_f32(s[1]));
+            HVX_Vector vscale2 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[2]), hvx_vec_splat_f32(s[3]));
+            HVX_Vector vscale3 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[4]), hvx_vec_splat_f32(s[5]));
+            HVX_Vector vscale4 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[6]), hvx_vec_splat_f32(s[7]));
+
+            HVX_Vector v_ql0 = *(const HVX_UVector *) (ql);
+            HVX_Vector v_ql1 = *(const HVX_UVector *) (ql + 32);
+            HVX_Vector v_qh  = *(const HVX_UVector *) (qh);
+
+            HVX_Vector ql0_lo = Q6_V_vand_VV(v_ql0, mask_0f);
+            HVX_Vector qh_0   = Q6_Vw_vasl_VwR(Q6_V_vand_VV(v_qh, mask_03), 4);
+            HVX_Vector v_q1   = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql0_lo, qh_0), v32_b);
+
+            HVX_Vector ql1_lo = Q6_V_vand_VV(v_ql1, mask_0f);
+            HVX_Vector qh_1   = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 2), mask_03), 4);
+            HVX_Vector v_q2   = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql1_lo, qh_1), v32_b);
+
+            HVX_Vector ql0_hi = Q6_Vub_vlsr_VubR(v_ql0, 4);
+            HVX_Vector qh_2   = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 4), mask_03), 4);
+            HVX_Vector v_q3   = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql0_hi, qh_2), v32_b);
+
+            HVX_Vector ql1_hi = Q6_Vub_vlsr_VubR(v_ql1, 4);
+            HVX_Vector qh_3   = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 6), mask_03), 4);
+            HVX_Vector v_q4   = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql1_hi, qh_3), v32_b);
+
+            HVX_VectorPair p16_1 = Q6_Wh_vunpack_Vb(v_q1);
+            HVX_VectorPair p32_1 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_1));
+            HVX_Vector     v_f32_1 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_1));
+            hvx_vmem(y + 0) = hvx_vec_mul_f32_f32(v_f32_1, vscale1);
+
+            HVX_VectorPair p16_2 = Q6_Wh_vunpack_Vb(v_q2);
+            HVX_VectorPair p32_2 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_2));
+            HVX_Vector     v_f32_2 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_2));
+            hvx_vmem(y + 32) = hvx_vec_mul_f32_f32(v_f32_2, vscale2);
+
+            HVX_VectorPair p16_3 = Q6_Wh_vunpack_Vb(v_q3);
+            HVX_VectorPair p32_3 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_3));
+            HVX_Vector     v_f32_3 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_3));
+            hvx_vmem(y + 64) = hvx_vec_mul_f32_f32(v_f32_3, vscale3);
+
+            HVX_VectorPair p16_4 = Q6_Wh_vunpack_Vb(v_q4);
+            HVX_VectorPair p32_4 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_4));
+            HVX_Vector     v_f32_4 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_4));
+            hvx_vmem(y + 96) = hvx_vec_mul_f32_f32(v_f32_4, vscale4);
+
+            y  += 128;
+            ql += 64;
+            qh += 32;
+            sc += 8;
+        }
+    }
+}
+
+// bytes row, row + 32, row + 64 and row + 96 of v, to bytes 0..3
+static inline HVX_Vector hvx_dequantize_gather4(HVX_Vector v, uint32_t row) {
+    v = Q6_V_vror_VR(v, row);
+    HVX_Vector packed = Q6_V_vmux_QVV(Q6_Q_vsetq_R(1), v, Q6_V_vror_VR(v, 31));
+    packed = Q6_V_vmux_QVV(Q6_Q_vsetq_R(2), packed, Q6_V_vror_VR(v, 62));
+    return Q6_V_vmux_QVV(Q6_Q_vsetq_R(3), packed, Q6_V_vror_VR(v, 93));
+}
+
+static inline void hvx_dequantize_row_tiled_q8_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
+    const HVX_VectorPred first2 = Q6_Q_vsetq_R(2);
+    const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
+
+    for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
+        const uint8_t * tile = src_spad + k_tile * tile_stride;
+        float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS;
+
+        HVX_Vector vq = Q6_V_vzero();
+        for (int group = 7; group >= 0; --group) {
+            const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), 2 * row);
+            const HVX_Vector packed = Q6_V_vmux_QVV(first2, v, Q6_V_vror_VR(v, 62));
+            vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4));
+        }
+
+        const HVX_Vector scales = hvx_vmem(tile + 1024);
+        const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
+        const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
+        const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);
+        const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
+        HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
+        *(HVX_Vector *) dst = values;
+    }
+}
+
+static inline void hvx_dequantize_row_tiled_q4_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
+    const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
+
+    for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
+        const uint8_t * tile = src_spad + k_tile * tile_stride;
+        float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS;
+
+        HVX_Vector vq = Q6_V_vzero();
+        for (int group = 3; group >= 0; --group) {
+            vq = Q6_V_vmux_QVV(first4, hvx_dequantize_gather4(hvx_vmem(tile + group * VLEN), row), Q6_V_vror_VR(vq, VLEN - 4));
+        }
+
+        const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F));
+        const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4);
+        vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1));
+        vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8));
+
+        const HVX_Vector scales = hvx_vmem(tile + 512);
+        const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
+        const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
+        const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);
+        const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
+        HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
+        *(HVX_Vector *) dst = values;
+    }
+}
+
+static inline void hvx_dequantize_row_tiled_q4_k_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
+    const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
+
+    for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
+        const uint8_t * tile = src_spad + k_tile * tile_stride;
+        float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS;
+
+        HVX_Vector vq = Q6_V_vzero();
+        for (int group = 3; group >= 0; --group) {
+            vq = Q6_V_vmux_QVV(first4, hvx_dequantize_gather4(hvx_vmem(tile + group * VLEN), row), Q6_V_vror_VR(vq, VLEN - 4));
+        }
+
+        const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F));
+        const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4);
+        vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1));
+
+        const HVX_Vector scales = hvx_vmem(tile + 512);
+        const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row));
+        const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
+        const HVX_Vector offset_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row + 2));
+        const HVX_Vector offset = Q6_V_lo_W(hvx_vec_f16_to_f32(offset_hf));
+
+        const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);
+        const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
+        HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
+        *(HVX_Vector *) dst = hvx_vec_add_f32_f32(values, offset);
+    }
+}
+
+static inline void hvx_dequantize_row_tiled_q5_k_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
+    const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
+    const HVX_Vector one = Q6_Vb_vsplat_R(1);
+
+    for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
+        const uint8_t * tile = src_spad + k_tile * tile_stride;
+        float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS;
+
+        const HVX_Vector p5 = hvx_dequantize_gather4(hvx_vmem(tile + 640), row);
+        HVX_Vector vq = Q6_V_vzero();
+        HVX_Vector vh = Q6_V_vzero();
+        for (int group = 3; group >= 0; --group) {
+            vq = Q6_V_vmux_QVV(first4, hvx_dequantize_gather4(hvx_vmem(tile + group * VLEN), row), Q6_V_vror_VR(vq, VLEN - 4));
+            vh = Q6_V_vmux_QVV(first4, Q6_Vub_vlsr_VubR(p5, 2 * group), Q6_V_vror_VR(vh, VLEN - 4));
+        }
+
+        const HVX_Vector lo = Q6_V_vor_VV(Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F)), Q6_Vw_vasl_VwR(Q6_V_vand_VV(vh, one), 4));
+        const HVX_Vector hi = Q6_V_vor_VV(Q6_Vub_vlsr_VubR(vq, 4), Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vh, 1), one), 4));
+        vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1));
+
+        const HVX_Vector scales = hvx_vmem(tile + 512);
+        const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row));
+        const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
+        const HVX_Vector offset_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row + 2));
+        const HVX_Vector offset = Q6_V_lo_W(hvx_vec_f16_to_f32(offset_hf));
+
+        const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);
+        const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
+        HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
+        *(HVX_Vector *) dst = hvx_vec_add_f32_f32(values, offset);
+    }
+}
+
+static inline void hvx_dequantize_row_tiled_q6_k_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
+    const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
+    const HVX_VectorPred first16 = Q6_Q_vsetq_R(16 * sizeof(float));
+    const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F);
+    const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
+
+    for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
+        const uint8_t * tile = src_spad + k_tile * tile_stride;
+        float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS;
+
+        HVX_Vector vq = Q6_V_vzero();
+        for (int group = 7; group >= 0; --group) {
+            const HVX_Vector lo_plane = Q6_V_vror_VR(hvx_vmem(tile + (group >> 1) * VLEN), 4 * row);
+            const HVX_Vector hi_plane = Q6_V_vror_VR(hvx_vmem(tile + 512 + (group >> 2) * VLEN), 4 * row);
+            const HVX_Vector lo = (group & 1) ? Q6_Vub_vlsr_VubR(lo_plane, 4) : Q6_V_vand_VV(lo_plane, mask_0f);
+            const HVX_Vector hi = Q6_Vub_vlsr_VubR(hi_plane, 2 * (group & 3));
+            const HVX_Vector packed = Q6_V_vor_VV(lo, Q6_Vw_vasl_VwR(Q6_V_vand_VV(hi, mask_03), 4));
+            vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4));
+        }
+
+        const HVX_Vector scales = hvx_vmem(tile + 768);
+        const HVX_Vector scale_lo_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
+        const HVX_Vector scale_hi_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 64 + 2 * row));
+        const HVX_Vector scale_lo = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_lo_hf));
+        const HVX_Vector scale_hi = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hi_hf));
+        const HVX_Vector scale = Q6_V_vmux_QVV(first16, scale_lo, scale_hi);
+        const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(32)));
+        const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
+        *(HVX_Vector *) dst = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
+    }
+}
+
+static inline HVX_Vector hvx_dequantize_tiled_f16_gather4(const uint8_t * p, uint32_t r_byte) {
+    HVX_Vector w0  = Q6_V_vror_VR(hvx_vmem(p + 0 * VLEN), r_byte);
+    HVX_Vector w1  = Q6_V_vror_VR(hvx_vmem(p + 1 * VLEN), r_byte);
+    HVX_Vector w01 = Q6_V_lo_W(Q6_W_vshuff_VVR(w1, w0, 4));
+    HVX_Vector w2  = Q6_V_vror_VR(hvx_vmem(p + 2 * VLEN), r_byte);
+    HVX_Vector w3  = Q6_V_vror_VR(hvx_vmem(p + 3 * VLEN), r_byte);
+    HVX_Vector w23 = Q6_V_lo_W(Q6_W_vshuff_VVR(w3, w2, 4));
+    return Q6_V_lo_W(Q6_W_vshuff_VVR(w23, w01, 8));
+}
+
+static inline void hvx_dequantize_row_tiled_f16_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
+    const uint32_t r_byte = 4 * row;
+
+    for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
+        const uint8_t * tile = src_spad + k_tile * tile_stride;
+        float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS;
+
+        HVX_Vector g0 = hvx_dequantize_tiled_f16_gather4(tile + 0 * VLEN, r_byte);
+        HVX_Vector g1 = hvx_dequantize_tiled_f16_gather4(tile + 4 * VLEN, r_byte);
+        HVX_Vector g01 = Q6_V_lo_W(Q6_W_vshuff_VVR(g1, g0, 16));
+
+        HVX_Vector g2 = hvx_dequantize_tiled_f16_gather4(tile + 8 * VLEN, r_byte);
+        HVX_Vector g3 = hvx_dequantize_tiled_f16_gather4(tile + 12 * VLEN, r_byte);
+        HVX_Vector g23 = Q6_V_lo_W(Q6_W_vshuff_VVR(g3, g2, 16));
+
+        HVX_Vector g = Q6_V_lo_W(Q6_W_vshuff_VVR(g23, g01, 32));
+        *(HVX_Vector *) dst = Q6_V_lo_W(hvx_vec_f16_to_f32(g));
+    }
+}

 #endif // HVX_QUANT_H
@@ -28,8 +28,8 @@
 #include "ggml-common.h"
 #include "hex-bitmap.h"
 #include "htp-ctx.h"
+#include "htp-dcache.h"
 #include "htp-ops.h"
-#include "htp-tensor.h"
 #include "htp_iface.h"
 #include "work-queue.h"
 #include "hex-profile.h"
@@ -835,7 +835,7 @@ static int op_fence(struct htp_ops_context * octx) {
    atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data;

    if (mode == 1) {
-        htp_flush_dirty_ranges(ctx);
+        htp_dcache_flush_dirty_ranges(ctx);

        htp_mdev_group_barrier(octx);

@@ -1211,7 +1211,7 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs
            src->ne[0], src->ne[1], src->ne[2], src->ne[3]);
    }

-    htp_tensor_flush_all(octx->ctx, octx->src, HTP_OP_MAX_INPUTS);
+    htp_dcache_flush_dirty_tensors(octx->ctx, octx->src, HTP_OP_MAX_INPUTS);

    // Prep output tensors
    for (uint32_t i = 0; i < HTP_OP_MAX_OUTPUTS; i++) {
@@ -1227,7 +1227,7 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs
            dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);
    }

-    htp_tensor_dirty_all(octx->ctx, octx->dsts, HTP_OP_MAX_OUTPUTS);
+    htp_dcache_mark_dirty_tensors(octx->ctx, octx->dsts, HTP_OP_MAX_OUTPUTS);

    htp_mdev_group_barrier(octx);

@@ -4,6 +4,7 @@
 #include <stdint.h>
 #include <stddef.h>
 #include "htp-ops.h"
+#include "htp-tensor.h"
 #include "hex-fastdiv.h"
 #include "hex-common.h"
 #include "htp-vtcm.h"
@@ -19,45 +20,10 @@ extern "C" {
 #define HTP_MM_HMX_TILE_N_ELMS 1024
 #define HTP_MM_HMX_MIN_NROWS   4

-// --- Weight Repacked Tile Sizes ---
-#define HTP_MM_WEIGHT_TILE_SIZE_Q4_0   576
-#define HTP_MM_WEIGHT_TILE_SIZE_Q4_1   640
-#define HTP_MM_WEIGHT_TILE_SIZE_Q8_0   1088
-#define HTP_MM_WEIGHT_TILE_SIZE_IQ4_NL 576
-#define HTP_MM_WEIGHT_TILE_SIZE_MXFP4  544
-// Q5_K: the Q4_1 tile (640) followed by a 128-byte plane with the 5th bit of every quant, transposed so that
-//   plane byte l holds the eight flags of lane l: bit 2i = low nibble of nibble vector i, bit 2i+1 = high nibble
-#define HTP_MM_WEIGHT_TILE_SIZE_Q5_K   768
-// Q6_K native 6-bit tile (32 rows x 32 k), vrmpy-ready: byte 4*row+b of a vector holds k = 4*group+b
-//   vectors 0..3: low nibbles, vector i holds group 2i (low nibble) and group 2i+1 (high nibble)
-//   vectors 4..5: high 2 bits, vector m holds groups 4m..4m+3 at bit offsets 0,2,4,6
-//   vector 6: fp16 scales per row, d * scales[]: k 0..15 in lanes 0..31, k 16..31 in lanes 32..63
-#define HTP_MM_WEIGHT_TILE_SIZE_Q6_K   896
-// Q3_K native 3-bit tile, vrmpy-ready like Q6_K
-//   vectors 0..1: low 2 bits, vector m holds groups 4m..4m+3 at bit offsets 0,2,4,6
-//   vector 2: bit g set where the hmask bit of group g is clear (quant = low 2 bits - 4)
-//   vector 3: fp16 scales per row, d * (scales[] - 32): k 0..15 in lanes 0..31, k 16..31 in lanes 32..63
-#define HTP_MM_WEIGHT_TILE_SIZE_Q3_K   512
-// Q2_K native 2-bit tile with the ggml scale bytes, vrmpy-ready like Q6_K (same bits per weight as block_q2_K)
-//   bytes   0..255: unsigned 2-bit quants, vector m holds groups 4m..4m+3 at bit offsets 0,2,4,6
-//   bytes 256..319: scales[] bytes (min << 4 | scale), sub-block s of row r at 32 * s + r
-//   bytes 320..335: 8 fp16, tile j of a super-block: d (j < 4) or -dmin (j >= 4) of rows 8 * (j % 4) .. 8 * (j % 4) + 7
-#define HTP_MM_WEIGHT_TILE_SIZE_Q2_K   336
-
-// --- Weight Repacked Aligned Tile Sizes ---
-#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_0   640
-#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_1   640
-#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q8_0   1152
-#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_IQ4_NL 640
-#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_MXFP4  640
-#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q5_K   768
-#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q6_K   896
-#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q3_K   512
-#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_K   384
-
 // --- Activation Tiled Block Sizes (including padding) ---
 #define HTP_MM_ACT_TILE_SIZE_Q8_0      1152
 #define HTP_MM_ACT_TILE_SIZE_Q8_1      1280
+#define HTP_MM_ACT_TILE_SIZE_F16       2048

 #define HTP_MM_MAX_PREFETCH 16

@@ -74,11 +40,12 @@ enum htp_mm_kernel_type {

    // HMX paths
    HTP_MM_KERNEL_HMX_2D,
-    HTP_MM_KERNEL_HMX_F16_BATCHED,
+    HTP_MM_KERNEL_HMX_BATCHED,

    // HVX floating-point paths
    HTP_MM_KERNEL_HVX_F16_F16_VTCM,
    HTP_MM_KERNEL_HVX_F32_F32_VTCM,
+    HTP_MM_KERNEL_HVX_BF16_F32_VTCM,

    // HVX quantized paths
    HTP_MM_KERNEL_HVX_QUANT_ROW,      // standard row-wise parallel quantization
@@ -212,57 +179,6 @@ next_nc:
    return 0;
 }

-// --- Tile Size Helpers ---
-static inline uint32_t htp_mm_get_weight_tile_size(int weight_type) {
-    switch (weight_type) {
-        case HTP_TYPE_Q4_0:
-        case HTP_TYPE_IQ4_NL:
-            return HTP_MM_WEIGHT_TILE_SIZE_Q4_0;
-        case HTP_TYPE_Q4_1:
-        case HTP_TYPE_Q4_K:
-            return HTP_MM_WEIGHT_TILE_SIZE_Q4_1;
-        case HTP_TYPE_Q8_0:
-            return HTP_MM_WEIGHT_TILE_SIZE_Q8_0;
-        case HTP_TYPE_Q5_K:
-            return HTP_MM_WEIGHT_TILE_SIZE_Q5_K;
-        case HTP_TYPE_Q6_K:
-            return HTP_MM_WEIGHT_TILE_SIZE_Q6_K;
-        case HTP_TYPE_Q3_K:
-            return HTP_MM_WEIGHT_TILE_SIZE_Q3_K;
-        case HTP_TYPE_Q2_K:
-            return HTP_MM_WEIGHT_TILE_SIZE_Q2_K;
-        case HTP_TYPE_MXFP4:
-            return HTP_MM_WEIGHT_TILE_SIZE_MXFP4;
-        default:
-            return 0;
-    }
-}
-
-static inline uint32_t htp_mm_get_weight_aligned_tile_size(int weight_type) {
-    switch (weight_type) {
-        case HTP_TYPE_Q4_0:
-        case HTP_TYPE_IQ4_NL:
-            return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_0;
-        case HTP_TYPE_Q4_1:
-        case HTP_TYPE_Q4_K:
-            return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_1;
-        case HTP_TYPE_Q8_0:
-            return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q8_0;
-        case HTP_TYPE_Q5_K:
-            return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q5_K;
-        case HTP_TYPE_Q6_K:
-            return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q6_K;
-        case HTP_TYPE_Q3_K:
-            return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q3_K;
-        case HTP_TYPE_Q2_K:
-            return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_K;
-        case HTP_TYPE_MXFP4:
-            return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_MXFP4;
-        default:
-            return 0;
-    }
-}
-
 // weight types whose tiles carry a per-block offset (x = d * q + m): the activations need block sums (q8_1)
 // (Q2_K: per-16 k sums, q8_1_s16)
 static inline bool htp_mm_weight_has_offset(int weight_type) {
@@ -283,27 +199,17 @@ static inline size_t htp_mm_q8_1_tiled_row_size(uint32_t ne) {
    return nb_32 * HTP_MM_ACT_TILE_SIZE_Q8_1;
 }

-static inline size_t htp_mm_get_tiled_row_stride(int weight_type, uint32_t k) {
-    uint32_t nb = (k + QK_Q4_0_TILED - 1) / QK_Q4_0_TILED;
-    switch (weight_type) {
-        case HTP_TYPE_Q4_0:
-        case HTP_TYPE_IQ4_NL:
-        case HTP_TYPE_Q4_1:
-        case HTP_TYPE_Q4_K:
-        case HTP_TYPE_Q8_0:
-        case HTP_TYPE_Q5_K:
-        case HTP_TYPE_Q6_K:
-        case HTP_TYPE_Q3_K:
-        case HTP_TYPE_Q2_K:
-        case HTP_TYPE_MXFP4:
-            return (size_t) nb * htp_mm_get_weight_tile_size(weight_type);
-        case HTP_TYPE_F16:
-            return (size_t) k * sizeof(__fp16);
-        case HTP_TYPE_F32:
-            return (size_t) k * sizeof(float);
-        default:
-            return 0;
+static inline size_t htp_mm_f16_tiled_row_size(uint32_t ne) {
+    const uint32_t ne_padded = ((ne + 127) / 128) * 128;
+    const uint32_t nb_32 = ne_padded / 32;
+    return nb_32 * HTP_MM_ACT_TILE_SIZE_F16;
+}
+
+static inline size_t htp_mm_tiled_act_row_size(int weight_type, uint32_t ne) {
+    if (weight_type == HTP_TYPE_F16) {
+        return htp_mm_f16_tiled_row_size(ne);
    }
+    return htp_mm_weight_has_offset(weight_type) ? htp_mm_q8_1_tiled_row_size(ne) : htp_mm_q8_0_tiled_row_size(ne);
 }

 static inline size_t htp_mm_round_up(size_t n, size_t m) {
@@ -318,8 +224,8 @@ static inline void htp_mm_hmx_get_2d_chunk_costs(
    int wtype, uint32_t k, bool pipeline, uint32_t aligned_tile_size,
    size_t * size_per_n_out, size_t * size_per_m_out, size_t * size_per_mn_out
 ) {
-    const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32);
-    const size_t row_stride = htp_mm_get_tiled_row_stride(wtype, k);
+    const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32 && wtype != HTP_TYPE_BF16);
+    const size_t row_stride = htp_tensor_get_row_size(wtype, k);
    const size_t vec_dot_size = k * sizeof(uint16_t);
    const uint32_t n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS;
    const size_t qweight_row_stride = is_quant ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0;
@@ -404,7 +310,7 @@ static inline void htp_mm_hmx_vtcm_layout_build(
 ) {
    size_t off = 0;

-    if (kernel_type == HTP_MM_KERNEL_HMX_F16_BATCHED) {
+    if (kernel_type == HTP_MM_KERNEL_HMX_BATCHED) {
        const size_t vec_dot_size     = k * sizeof(uint16_t);
        const size_t act_head_stride   = mc * k;
        const size_t weight_area_size  = hex_align_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE);
@@ -450,8 +356,8 @@ static inline void htp_mm_hmx_vtcm_layout_build(
        off = off_group_a + hex_smax(group_b_size, group_c_size);
    } else {
        // HTP_MM_KERNEL_HMX_2D
-        const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32);
-        const size_t row_stride = htp_mm_get_tiled_row_stride(wtype, k);
+        const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32 && wtype != HTP_TYPE_BF16);
+        const size_t row_stride = htp_tensor_get_row_size(wtype, k);
        const size_t vec_dot_size = k * sizeof(uint16_t);
        const uint32_t n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS;

@@ -527,28 +433,13 @@ static inline void htp_mm_hvx_vtcm_layout_build(
    size_t dst_sz     = 0;
    size_t act_raw_sz = 0;

-    const bool is_repack = (wtype == HTP_TYPE_Q4_0 || wtype == HTP_TYPE_Q4_1 ||
-                            wtype == HTP_TYPE_Q8_0 || wtype == HTP_TYPE_IQ4_NL ||
-                            wtype == HTP_TYPE_MXFP4 || wtype == HTP_TYPE_Q6_K ||
-                            wtype == HTP_TYPE_Q4_K || wtype == HTP_TYPE_Q5_K ||
-                            wtype == HTP_TYPE_Q3_K || wtype == HTP_TYPE_Q2_K);
-
    if (is_fused_nx) {
-        const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128);
+        uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
+        uint32_t n_k_tiles = hex_round_up(ne10, 32) / 32;
+        uint32_t tile_row_size = n_k_tiles * aligned_tile_size;
+        size_t weight_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128);

-        size_t weight_sz_per_thread = 0;
-
-        if (is_repack) {
-            uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype);
-            uint32_t n_k_tiles = hex_round_up(ne10, 32) / 32;
-            uint32_t tile_row_size = n_k_tiles * aligned_tile_size;
-
-            weight_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128);
-        } else {
-            weight_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128);
-        }
-
-        size_t tiled_act_row_size = htp_mm_weight_has_offset(wtype) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
+        size_t tiled_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
        size_t q_act_sz = hex_round_up(tiled_act_row_size * act_nrows, 128);
        size_t raw_row_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float));

@@ -558,20 +449,13 @@ static inline void htp_mm_hvx_vtcm_layout_build(
        dst_sz     = 0;
        act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128);
    } else if (is_matmul_id) {
-        const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128);
-        const size_t act_row_size_tiled = htp_mm_weight_has_offset(wtype) ? htp_mm_q8_1_tiled_row_size(ne10)
-                                                                          : htp_mm_q8_0_tiled_row_size(ne10);
+        const size_t act_row_size_tiled = htp_mm_tiled_act_row_size(wtype, ne10);
+        const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
+        const uint32_t n_k_tiles         = ne10 / 32;
+        const uint32_t tile_row_size     = n_k_tiles * aligned_tile_size;
+        size_t src0_sz_per_thread        = htp_mm_round_up(n_prefetch * tile_row_size, 256);

-        size_t src0_sz_per_thread = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256);
-        act_sz                    = htp_mm_round_up(act_row_size_tiled * act_nrows, 256);
-
-        if (is_repack) {
-            const uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype);
-            const uint32_t n_k_tiles         = ne10 / 32;
-            const uint32_t tile_row_size     = n_k_tiles * aligned_tile_size;
-            size_t repacked_vtcm_size        = htp_mm_round_up(n_prefetch * tile_row_size, 256);
-            src0_sz_per_thread               = repacked_vtcm_size;
-        }
+        act_sz = htp_mm_round_up(act_row_size_tiled * act_nrows, 256);

        size_t raw_row_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float));

@@ -592,7 +476,8 @@ static inline void htp_mm_hvx_vtcm_layout_build(
                act_raw_sz = hex_round_up(hex_round_up(ne10 * sizeof(float), 128) * act_nrows, 128);
                break;
            }
-            case HTP_MM_KERNEL_HVX_F32_F32_VTCM: {
+            case HTP_MM_KERNEL_HVX_F32_F32_VTCM:
+            case HTP_MM_KERNEL_HVX_BF16_F32_VTCM: {
                size_t f32_act_row_size = htp_mm_round_up(ne10 * 4, 128);
                act_sz     = htp_mm_round_up(f32_act_row_size * act_nrows, 256);
                src0_sz    = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads;
@@ -602,21 +487,15 @@ static inline void htp_mm_hvx_vtcm_layout_build(
            }
            case HTP_MM_KERNEL_HVX_QUANT_BLOCK:
            case HTP_MM_KERNEL_HVX_QUANT_ROW: {
-                size_t q_act_row_size = htp_mm_weight_has_offset(wtype) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
+                size_t q_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
+                uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
+                uint32_t n_k_tiles = (ne10 + 31) / 32;
+                uint32_t tile_row_size = n_k_tiles * aligned_tile_size;
+                size_t tiled_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256);

-                src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256);
+                src0_sz = tiled_vtcm_size * n_threads;
                act_sz  = htp_mm_round_up(q_act_row_size * act_nrows, 256);

-                src0_sz = src0_sz * n_threads;
-
-                if (is_repack) {
-                    uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype);
-                    uint32_t n_k_tiles = ne10 / 32;
-                    uint32_t tile_row_size = n_k_tiles * aligned_tile_size;
-                    size_t repacked_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256);
-                    src0_sz = repacked_vtcm_size * n_threads;
-                }
-
                size_t dst_slice_per_thread = (dst_nrows > 0 && act_nrows == 1) ? htp_mm_round_up((dst_row_size + n_threads - 1) / n_threads, 128) : 0;
                dst_sz = dst_slice_per_thread * n_threads;
                size_t raw_row_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float));
@@ -687,9 +566,7 @@ static inline bool htp_mm_hvx_solve_vtcm_params(
    const size_t avail_act = vtcm_budget - fixed_bytes;
    size_t row_size = 0;
    if (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {
-        row_size = htp_mm_weight_has_offset(wtype)
-                 ? htp_mm_q8_1_tiled_row_size(ne10)
-                 : htp_mm_q8_0_tiled_row_size(ne10);
+        row_size = htp_mm_tiled_act_row_size(wtype, ne10);
    } else if (kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM) {
        row_size = hex_round_up(ne10 * 2, 128);
    } else {
@@ -749,7 +626,7 @@ static inline size_t htp_mm_hmx_get_batched_vtcm_size(
    int wtype, uint32_t k, size_t mc, size_t nc, uint32_t group_size, bool pipeline, uint32_t act_threads, size_t bias_size) {
    (void)pipeline;
    struct htp_mm_hmx_vtcm_layout L;
-    htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_F16_BATCHED, wtype, k, mc, nc, group_size, false, act_threads, 0, bias_size);
+    htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, wtype, k, mc, nc, group_size, false, act_threads, 0, bias_size);
    return L.total_bytes;
 }

@@ -45,6 +45,7 @@ set(METALLIB_KERNEL_SOURCES
    kernels/fa_q5_0.metal
    kernels/fa_q5_1.metal
    kernels/fa_q8_0.metal
+    kernels/fa_tensor.metal
    kernels/fa_vec_f16.metal
    kernels/fa_vec_f32.metal
    kernels/fa_vec_q4_0.metal
@@ -233,12 +234,12 @@ else()
            VERBATIM
        )

-        set(AIR_FA_TENSOR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/fa_f16_tensor.air")
+        set(AIR_FA_TENSOR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/fa_tensor_tensor.air")
        add_custom_command(
            OUTPUT ${AIR_FA_TENSOR}
-            COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS_TENSOR} -DGGML_METAL_HAS_TENSOR -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/fa_f16.metal -o ${AIR_FA_TENSOR}
-            DEPENDS kernels/fa_f16.metal ${METALLIB_KERNELS_FA_SHARED} kernels/common.h kernels/dequantize.h ${METALLIB_COMMON} ggml-metal-impl.h
-            COMMENT "Compiling kernels/fa_f16.metal (tensor API)"
+            COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS_TENSOR} -DGGML_METAL_HAS_TENSOR -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/fa_tensor.metal -o ${AIR_FA_TENSOR}
+            DEPENDS kernels/fa_tensor.metal kernels/common.h ${METALLIB_COMMON} ggml-metal-impl.h
+            COMMENT "Compiling kernels/fa_tensor.metal (tensor API)"
            VERBATIM
        )

@@ -1866,7 +1866,9 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_t
    const int32_t dk = (int32_t) op->src[1]->ne[0];
    const int32_t dv = (int32_t) op->src[2]->ne[0];

-    snprintf(base, 256, "kernel_flash_attn_ext_tensor_f16_dk%d_dv%d", dk, dv);
+    const char * type = op->src[1]->type == GGML_TYPE_BF16 ? "bf16" : "f16";
+
+    snprintf(base, 256, "kernel_flash_attn_ext_tensor_%s_dk%d_dv%d", type, dk, dv);
    snprintf(name, 256, "%s_mask=%d_sinks=%d_bias=%d_scap=%d", base, has_mask, has_sinks, has_bias, has_scap);

    ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
@@ -118,6 +118,7 @@ int ggml_metal_pipeline_max_theads_per_threadgroup(struct ggml_metal_pipeline_wi
    X(FA_Q5_0,         fa_q5_0)        \
    X(FA_Q5_1,         fa_q5_1)        \
    X(FA_Q8_0,         fa_q8_0)        \
+    X(FA_TENSOR,       fa_tensor)      \
    X(FA_VEC_F16,      fa_vec_f16)     \
    X(FA_VEC_F32,      fa_vec_f32)     \
    X(FA_VEC_Q4_0,     fa_vec_q4_0)    \
@@ -3161,7 +3161,7 @@ static bool ggml_metal_op_flash_attn_ext_use_tensor(const ggml_tensor * op, bool
        return false;
    }

-    if (op->src[1]->type != GGML_TYPE_F16 && !ggml_metal_op_flash_attn_ext_use_kv_f16(op)) {
+    if (op->src[1]->type != GGML_TYPE_F16 && op->src[1]->type != GGML_TYPE_BF16 && !ggml_metal_op_flash_attn_ext_use_kv_f16(op)) {
        return false;
    }

@@ -3678,7 +3678,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
            /*.logit_softcap =*/ logit_softcap,
        };

-        // shared memory layout: queries (half), scores (float), probabilities (half), row scale (float), rescale flag (int)
+        // shared memory layout: queries (16-bit), scores (float), probabilities (16-bit), row scale (float), rescale flag (int)
        const size_t smem = GGML_PAD(nqptg*ne00*sizeof(ggml_fp16_t) + nqptg*ncpsg*(sizeof(float) + sizeof(ggml_fp16_t)) + nqptg*sizeof(float) + sizeof(int32_t), 16);

        auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_tensor(lib, op, has_mask, has_sinks, has_bias, has_scap);
@@ -75,291 +75,3 @@ template [[host_name("kernel_flash_attn_ext_bf16_dk576_dv512")]] kernel flash_at
 #undef FA_TYPES
 #undef FA_TYPES_BF
 #undef FA_TYPES_F32
-
-#ifdef GGML_METAL_HAS_TENSOR
-
-constant bool FC_flash_attn_ext_tensor_has_mask  [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 0)]];
-constant bool FC_flash_attn_ext_tensor_has_sinks [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 1)]];
-constant bool FC_flash_attn_ext_tensor_has_bias  [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 2)]];
-constant bool FC_flash_attn_ext_tensor_has_scap  [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 3)]];
-
-// ref: https://arxiv.org/pdf/2307.08691.pdf
-template<
-    short DK,                                   // K head size
-    short DV,                                   // V head size
-    short Q   = OP_FLASH_ATTN_EXT_TENSOR_NQPSG, // queries per threadgroup
-    short C   = OP_FLASH_ATTN_EXT_TENSOR_NCPSG, // cache items per threadgroup
-    short NSG = OP_FLASH_ATTN_EXT_TENSOR_NSG>   // number of simd groups
-kernel void kernel_flash_attn_ext_tensor(
-        constant ggml_metal_kargs_flash_attn_ext & args,
-        device const char * q,
-        device const char * k,
-        device const char * v,
-        device const char * mask,
-        device const char * sinks,
-        device const char * blk,
-        device       char * dst,
-        threadgroup  char * shmem [[threadgroup(0)]],
-        uint3  tgpig [[threadgroup_position_in_grid]],
-        ushort tiisg [[thread_index_in_simdgroup]],
-        ushort sgitg [[simdgroup_index_in_threadgroup]]) {
-    constexpr short NW = N_SIMDWIDTH;
-    constexpr short NT = NW*NSG;
-    constexpr short NQ = Q/NSG;
-    constexpr short NC = C/NW; // columns per thread
-
-    static_assert(DK % 4 == 0,  "DK must be divisible by 4");
-    static_assert(Q % NSG == 0, "Q must be divisible by NSG");
-    static_assert(C % NW == 0,  "C must be divisible by NW");
-
-    const int iq3 = tgpig[2];
-    const int iq2 = tgpig[1];
-    const int iq1 = tgpig[0]*Q;
-
-    const short tiitg = sgitg*NW + tiisg;
-
-    threadgroup half  * sq = (threadgroup half  *) shmem;       // [Q, DK] queries
-    threadgroup float * ss = (threadgroup float *) (sq + Q*DK); // [Q, C]  scores
-    threadgroup half  * sp = (threadgroup half  *) (ss + Q*C);  // [Q, C]  probabilities
-    threadgroup float * sr = (threadgroup float *) (sp + Q*C);  // [Q]     per-row scale of O
-    threadgroup int   * sf = (threadgroup int   *) (sr + Q);    // [1]     last iteration (ic0 + 1) that rescaled O
-
-    q += iq1*args.nb01 + iq2*args.nb02 + iq3*args.nb03;
-
-    {
-        const int ikv2 = iq2/(args.ne02/args.ne_12_2);
-        const int ikv3 = iq3/(args.ne03/args.ne_12_3);
-
-        k += ikv2*args.nb12 + ikv3*args.nb13;
-        v += ikv2*args.nb22 + ikv3*args.nb23;
-    }
-
-    // with softcap the scale is small (scale/softcap), so it is applied to the scores to keep the precision of Q
-    const float qscale = FC_flash_attn_ext_tensor_has_scap ? 1.0f : args.scale;
-
-    // load the queries, with the scale folded in
-    for (int i = tiitg; i < Q*DK/4; i += NT) {
-        const int j = i/(DK/4);
-
-        float4 q4 = 0.0f;
-        if (iq1 + j < args.ne01) {
-            q4 = ((device const float4 *) (q + j*args.nb01))[i%(DK/4)];
-        }
-
-        ((threadgroup half4 *) sq)[i] = (half4) (q4*qscale);
-    }
-
-    device const half * pm[NQ];
-
-    FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
-        const short j = jj*NSG + sgitg;
-
-        pm[jj] = (device const half *) (mask + (iq1 + j)*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33);
-    }
-
-    {
-        const int nblk1 = (args.ne01 + Q - 1)/Q;
-        const int nblk0 = (args.ne11 + C - 1)/C;
-
-        blk += (((iq3%args.ne33)*args.ne32 + (iq2%args.ne32))*nblk1 + iq1/Q)*nblk0;
-    }
-
-    float M[NQ];
-    float S[NQ];
-
-    FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
-        M[jj] = -FLT_MAX/2;
-        S[jj] = 0.0f;
-    }
-
-    float slope = 1.0f;
-
-    // ALiBi
-    if (FC_flash_attn_ext_tensor_has_bias) {
-        const short h = iq2;
-
-        const float base = h < args.n_head_log2 ? args.m0 : args.m1;
-        const short exph = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1;
-
-        slope = pow(base, exph);
-    }
-
-    const int sk = args.ns10;
-    const int sv = args.ns20;
-
-    auto tq = tensor(sq, dextents<int32_t, 2>(DK, Q));
-    auto ts = tensor(ss, dextents<int32_t, 2>(C,  Q));
-    auto tp = tensor(sp, dextents<int32_t, 2>(C,  Q));
-
-    mpp::tensor_ops::matmul2d<
-        mpp::tensor_ops::matmul2d_descriptor(Q, C, DK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply),
-        execution_simdgroups<NSG>> mm_qk;
-
-    mpp::tensor_ops::matmul2d<
-        mpp::tensor_ops::matmul2d_descriptor(Q, DV, C, false, false, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
-        execution_simdgroups<NSG>> mm_pv;
-
-    auto tv0 = tensor((device half *) v, dextents<int32_t, 2>(DV, C), array<int, 2>({1, sv}));
-
-    // the O matrix from the paper
-    auto co = mm_pv.template get_destination_cooperative_tensor<decltype(tp), decltype(tv0), float>();
-
-    FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) {
-        if (co.is_valid_element(i)) {
-            co[i] = 0.0f;
-        }
-    }
-
-    if (tiitg == 0) {
-        sf[0] = 0;
-    }
-
-    threadgroup_barrier(mem_flags::mem_threadgroup);
-
-    // the host guarantees ne11 % C == 0
-    for (int ic0 = 0, ic = 0; ic < args.ne11; ++ic0, ic += C) {
-        char blk_cur = 1;
-
-        if (FC_flash_attn_ext_tensor_has_mask) {
-            blk_cur = blk[ic0];
-
-            if (blk_cur == 0) {
-                continue;
-            }
-        }
-
-        // Q*K^T
-        {
-            auto tk = tensor((device half *) (k + (uint64_t) ic*args.nb11), dextents<int32_t, 2>(DK, C), array<int, 2>({1, sk}));
-
-            mm_qk.run(tq, tk, ts);
-        }
-
-        threadgroup_barrier(mem_flags::mem_threadgroup);
-
-        // online softmax
-        FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
-            const short j = jj*NSG + sgitg;
-
-            float s[NC];
-
-            FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
-                s[ii] = ss[j*C + ii*NW + tiisg];
-            }
-
-            if (FC_flash_attn_ext_tensor_has_scap) {
-                FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
-                    s[ii] = args.logit_softcap*precise::tanh(s[ii]*args.scale);
-                }
-            }
-
-            if (FC_flash_attn_ext_tensor_has_mask && blk_cur != 2 && iq1 + j < args.ne31) {
-                FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
-                    s[ii] += slope*(float) pm[jj][ic + ii*NW + tiisg];
-                }
-            }
-
-            float m = M[jj];
-
-            FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
-                m = max(m, s[ii]);
-            }
-
-            m = simd_max(m);
-
-            // lazy rescaling: move the running max only when it grows by more than 8 (e^8 fits in half)
-            float ms = 1.0f;
-
-            if (m > M[jj] + 8.0f) {
-                ms    = exp(M[jj] - m);
-                M[jj] = m;
-
-                if (tiisg == 0) {
-                    sf[0] = ic0 + 1;
-                }
-            }
-
-            float sum = 0.0f;
-
-            FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
-                // the sum uses the same rounded values as P*V
-                const half p = (half) exp(s[ii] - M[jj]);
-
-                sp[j*C + ii*NW + tiisg] = p;
-
-                sum += (float) p;
-            }
-
-            S[jj] = S[jj]*ms + simd_sum(sum);
-
-            if (tiisg == 0) {
-                sr[j] = ms;
-            }
-        }
-
-        threadgroup_barrier(mem_flags::mem_threadgroup);
-
-        // O = diag(ms)*O + P*V
-        if (sf[0] == ic0 + 1) {
-            FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) {
-                if (co.is_valid_element(i)) {
-                    co[i] *= sr[co.get_multidimensional_index(i)[1]];
-                }
-            }
-        }
-
-        {
-            auto tv = tensor((device half *) (v + (uint64_t) ic*args.nb21), dextents<int32_t, 2>(DV, C), array<int, 2>({1, sv}));
-
-            mm_pv.run(tp, tv, co);
-        }
-
-        threadgroup_barrier(mem_flags::mem_threadgroup);
-    }
-
-    FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
-        const short j = jj*NSG + sgitg;
-
-        // the sink only adds to the denominator - its rescale of O is folded into the final scale
-        float ms = 1.0f;
-
-        if (FC_flash_attn_ext_tensor_has_sinks) {
-            const float s = ((device const float *) sinks)[iq2];
-            const float m = max(M[jj], s);
-
-            ms = exp(M[jj] - m);
-
-            S[jj] = S[jj]*ms + exp(s - m);
-        }
-
-        if (tiisg == 0) {
-            sr[j] = S[jj] == 0.0f ? 0.0f : ms/S[jj];
-        }
-    }
-
-    threadgroup_barrier(mem_flags::mem_threadgroup);
-
-    FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) {
-        if (co.is_valid_element(i)) {
-            co[i] *= sr[co.get_multidimensional_index(i)[1]];
-        }
-    }
-
-    // store to global memory - rows past ne01 are clipped by the tensor extents
-    device float * pdst = (device float *) dst + ((uint64_t) iq3*args.ne2*args.ne1 + iq2 + (uint64_t) iq1*args.ne1)*DV;
-
-    auto td = tensor(pdst, dextents<int32_t, 2>(DV, args.ne01 - iq1), array<int, 2>({1, args.ne1*DV}));
-
-    co.store(td);
-}
-
-typedef decltype(kernel_flash_attn_ext_tensor<64, 64>) flash_attn_ext_tensor_t;
-
-template [[host_name("kernel_flash_attn_ext_tensor_f16_dk64_dv64"  )]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<64,  64>;
-template [[host_name("kernel_flash_attn_ext_tensor_f16_dk128_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<128, 128>;
-template [[host_name("kernel_flash_attn_ext_tensor_f16_dk192_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<192, 128>;
-template [[host_name("kernel_flash_attn_ext_tensor_f16_dk256_dv256")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<256, 256>;
-template [[host_name("kernel_flash_attn_ext_tensor_f16_dk512_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<512, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>;
-template [[host_name("kernel_flash_attn_ext_tensor_f16_dk576_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<576, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>;
-
-#endif // GGML_METAL_HAS_TENSOR
@@ -0,0 +1,298 @@
+#include "common.h"
+
+#ifdef GGML_METAL_HAS_TENSOR
+
+constant bool FC_flash_attn_ext_tensor_has_mask  [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 0)]];
+constant bool FC_flash_attn_ext_tensor_has_sinks [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 1)]];
+constant bool FC_flash_attn_ext_tensor_has_bias  [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 2)]];
+constant bool FC_flash_attn_ext_tensor_has_scap  [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 3)]];
+
+// ref: https://arxiv.org/pdf/2307.08691.pdf
+template<
+    typename T,                                 // Q, K, V and P types
+    typename T4,
+    short DK,                                   // K head size
+    short DV,                                   // V head size
+    short Q   = OP_FLASH_ATTN_EXT_TENSOR_NQPSG, // queries per threadgroup
+    short C   = OP_FLASH_ATTN_EXT_TENSOR_NCPSG, // cache items per threadgroup
+    short NSG = OP_FLASH_ATTN_EXT_TENSOR_NSG>   // number of simd groups
+kernel void kernel_flash_attn_ext_tensor(
+        constant ggml_metal_kargs_flash_attn_ext & args,
+        device const char * q,
+        device const char * k,
+        device const char * v,
+        device const char * mask,
+        device const char * sinks,
+        device const char * blk,
+        device       char * dst,
+        threadgroup  char * shmem [[threadgroup(0)]],
+        uint3  tgpig [[threadgroup_position_in_grid]],
+        ushort tiisg [[thread_index_in_simdgroup]],
+        ushort sgitg [[simdgroup_index_in_threadgroup]]) {
+    constexpr short NW = N_SIMDWIDTH;
+    constexpr short NT = NW*NSG;
+    constexpr short NQ = Q/NSG;
+    constexpr short NC = C/NW; // columns per thread
+
+    static_assert(DK % 4 == 0,  "DK must be divisible by 4");
+    static_assert(Q % NSG == 0, "Q must be divisible by NSG");
+    static_assert(C % NW == 0,  "C must be divisible by NW");
+
+    const int iq3 = tgpig[2];
+    const int iq2 = tgpig[1];
+    const int iq1 = tgpig[0]*Q;
+
+    const short tiitg = sgitg*NW + tiisg;
+
+    threadgroup T     * sq = (threadgroup T     *) shmem;       // [Q, DK] queries
+    threadgroup float * ss = (threadgroup float *) (sq + Q*DK); // [Q, C]  scores
+    threadgroup T     * sp = (threadgroup T     *) (ss + Q*C);  // [Q, C]  probabilities
+    threadgroup float * sr = (threadgroup float *) (sp + Q*C);  // [Q]     per-row scale of O
+    threadgroup int   * sf = (threadgroup int   *) (sr + Q);    // [1]     last iteration (ic0 + 1) that rescaled O
+
+    q += iq1*args.nb01 + iq2*args.nb02 + iq3*args.nb03;
+
+    {
+        const int ikv2 = iq2/(args.ne02/args.ne_12_2);
+        const int ikv3 = iq3/(args.ne03/args.ne_12_3);
+
+        k += ikv2*args.nb12 + ikv3*args.nb13;
+        v += ikv2*args.nb22 + ikv3*args.nb23;
+    }
+
+    // with softcap the scale is small (scale/softcap), so it is applied to the scores to keep the precision of Q
+    const float qscale = FC_flash_attn_ext_tensor_has_scap ? 1.0f : args.scale;
+
+    // load the queries, with the scale folded in
+    for (int i = tiitg; i < Q*DK/4; i += NT) {
+        const int j = i/(DK/4);
+
+        float4 q4 = 0.0f;
+        if (iq1 + j < args.ne01) {
+            q4 = ((device const float4 *) (q + j*args.nb01))[i%(DK/4)];
+        }
+
+        ((threadgroup T4 *) sq)[i] = (T4) (q4*qscale);
+    }
+
+    device const half * pm[NQ];
+
+    FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
+        const short j = jj*NSG + sgitg;
+
+        pm[jj] = (device const half *) (mask + (iq1 + j)*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33);
+    }
+
+    {
+        const int nblk1 = (args.ne01 + Q - 1)/Q;
+        const int nblk0 = (args.ne11 + C - 1)/C;
+
+        blk += (((iq3%args.ne33)*args.ne32 + (iq2%args.ne32))*nblk1 + iq1/Q)*nblk0;
+    }
+
+    float M[NQ];
+    float S[NQ];
+
+    FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
+        M[jj] = -FLT_MAX/2;
+        S[jj] = 0.0f;
+    }
+
+    float slope = 1.0f;
+
+    // ALiBi
+    if (FC_flash_attn_ext_tensor_has_bias) {
+        const short h = iq2;
+
+        const float base = h < args.n_head_log2 ? args.m0 : args.m1;
+        const short exph = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1;
+
+        slope = pow(base, exph);
+    }
+
+    const int sk = args.ns10;
+    const int sv = args.ns20;
+
+    auto tq = tensor(sq, dextents<int32_t, 2>(DK, Q));
+    auto ts = tensor(ss, dextents<int32_t, 2>(C,  Q));
+    auto tp = tensor(sp, dextents<int32_t, 2>(C,  Q));
+
+    mpp::tensor_ops::matmul2d<
+        mpp::tensor_ops::matmul2d_descriptor(Q, C, DK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply),
+        execution_simdgroups<NSG>> mm_qk;
+
+    mpp::tensor_ops::matmul2d<
+        mpp::tensor_ops::matmul2d_descriptor(Q, DV, C, false, false, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
+        execution_simdgroups<NSG>> mm_pv;
+
+    auto tv0 = tensor((device T *) v, dextents<int32_t, 2>(DV, C), array<int, 2>({1, sv}));
+
+    // the O matrix from the paper
+    auto co = mm_pv.template get_destination_cooperative_tensor<decltype(tp), decltype(tv0), float>();
+
+    FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) {
+        if (co.is_valid_element(i)) {
+            co[i] = 0.0f;
+        }
+    }
+
+    if (tiitg == 0) {
+        sf[0] = 0;
+    }
+
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+
+    // the host guarantees ne11 % C == 0
+    for (int ic0 = 0, ic = 0; ic < args.ne11; ++ic0, ic += C) {
+        char blk_cur = 1;
+
+        if (FC_flash_attn_ext_tensor_has_mask) {
+            blk_cur = blk[ic0];
+
+            if (blk_cur == 0) {
+                continue;
+            }
+        }
+
+        // Q*K^T
+        {
+            auto tk = tensor((device T *) (k + (uint64_t) ic*args.nb11), dextents<int32_t, 2>(DK, C), array<int, 2>({1, sk}));
+
+            mm_qk.run(tq, tk, ts);
+        }
+
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+
+        // online softmax
+        FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
+            const short j = jj*NSG + sgitg;
+
+            float s[NC];
+
+            FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
+                s[ii] = ss[j*C + ii*NW + tiisg];
+            }
+
+            if (FC_flash_attn_ext_tensor_has_scap) {
+                FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
+                    s[ii] = args.logit_softcap*precise::tanh(s[ii]*args.scale);
+                }
+            }
+
+            if (FC_flash_attn_ext_tensor_has_mask && blk_cur != 2 && iq1 + j < args.ne31) {
+                FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
+                    s[ii] += slope*(float) pm[jj][ic + ii*NW + tiisg];
+                }
+            }
+
+            float m = M[jj];
+
+            FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
+                m = max(m, s[ii]);
+            }
+
+            m = simd_max(m);
+
+            // lazy rescaling: move the running max only when it grows by more than 8 (e^8 fits in half)
+            float ms = 1.0f;
+
+            if (m > M[jj] + 8.0f) {
+                ms    = exp(M[jj] - m);
+                M[jj] = m;
+
+                if (tiisg == 0) {
+                    sf[0] = ic0 + 1;
+                }
+            }
+
+            float sum = 0.0f;
+
+            FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
+                // the sum uses the same rounded values as P*V
+                const T p = (T) exp(s[ii] - M[jj]);
+
+                sp[j*C + ii*NW + tiisg] = p;
+
+                sum += (float) p;
+            }
+
+            S[jj] = S[jj]*ms + simd_sum(sum);
+
+            if (tiisg == 0) {
+                sr[j] = ms;
+            }
+        }
+
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+
+        // O = diag(ms)*O + P*V
+        if (sf[0] == ic0 + 1) {
+            FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) {
+                if (co.is_valid_element(i)) {
+                    co[i] *= sr[co.get_multidimensional_index(i)[1]];
+                }
+            }
+        }
+
+        {
+            auto tv = tensor((device T *) (v + (uint64_t) ic*args.nb21), dextents<int32_t, 2>(DV, C), array<int, 2>({1, sv}));
+
+            mm_pv.run(tp, tv, co);
+        }
+
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+    }
+
+    FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
+        const short j = jj*NSG + sgitg;
+
+        // the sink only adds to the denominator - its rescale of O is folded into the final scale
+        float ms = 1.0f;
+
+        if (FC_flash_attn_ext_tensor_has_sinks) {
+            const float s = ((device const float *) sinks)[iq2];
+            const float m = max(M[jj], s);
+
+            ms = exp(M[jj] - m);
+
+            S[jj] = S[jj]*ms + exp(s - m);
+        }
+
+        if (tiisg == 0) {
+            sr[j] = S[jj] == 0.0f ? 0.0f : ms/S[jj];
+        }
+    }
+
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+
+    FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) {
+        if (co.is_valid_element(i)) {
+            co[i] *= sr[co.get_multidimensional_index(i)[1]];
+        }
+    }
+
+    // store to global memory - rows past ne01 are clipped by the tensor extents
+    device float * pdst = (device float *) dst + ((uint64_t) iq3*args.ne2*args.ne1 + iq2 + (uint64_t) iq1*args.ne1)*DV;
+
+    auto td = tensor(pdst, dextents<int32_t, 2>(DV, args.ne01 - iq1), array<int, 2>({1, args.ne1*DV}));
+
+    co.store(td);
+}
+
+typedef decltype(kernel_flash_attn_ext_tensor<half, half4, 64, 64>) flash_attn_ext_tensor_t;
+
+template [[host_name("kernel_flash_attn_ext_tensor_f16_dk64_dv64"  )]]  kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 64,  64>;
+template [[host_name("kernel_flash_attn_ext_tensor_f16_dk128_dv128")]]  kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 128, 128>;
+template [[host_name("kernel_flash_attn_ext_tensor_f16_dk192_dv128")]]  kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 192, 128>;
+template [[host_name("kernel_flash_attn_ext_tensor_f16_dk256_dv256")]]  kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 256, 256>;
+template [[host_name("kernel_flash_attn_ext_tensor_f16_dk512_dv512")]]  kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 512, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>;
+template [[host_name("kernel_flash_attn_ext_tensor_f16_dk576_dv512")]]  kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 576, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>;
+
+template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk64_dv64"  )]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 64,  64>;
+template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk128_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 128, 128>;
+template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk192_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 192, 128>;
+template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk256_dv256")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 256, 256>;
+template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk512_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 512, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>;
+template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk576_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 576, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>;
+
+#endif // GGML_METAL_HAS_TENSOR
@@ -847,7 +847,7 @@ struct ggml_backend_opencl_context {
    cl_kernel kernel_sigmoid_f32, kernel_sigmoid_f16;
    cl_kernel kernel_tri;
    cl_kernel kernel_fill;
-    cl_kernel kernel_clamp;
+    cl_kernel kernel_clamp, kernel_clamp_nc;
    cl_kernel kernel_geglu, kernel_reglu, kernel_swiglu, kernel_swiglu_oai, kernel_swiglu_clamp, kernel_geglu_erf,
              kernel_geglu_quick, kernel_geglu_f16, kernel_reglu_f16, kernel_swiglu_f16, kernel_swiglu_clamp_f16,
              kernel_geglu_erf_f16, kernel_geglu_quick_f16;
@@ -1613,6 +1613,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
            build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);

        CL_CHECK((backend_ctx->kernel_clamp = clCreateKernel(backend_ctx->program_clamp, "kernel_clamp", &err), err));
+        CL_CHECK((backend_ctx->kernel_clamp_nc = clCreateKernel(backend_ctx->program_clamp, "kernel_clamp_nc", &err), err));
        GGML_LOG_CONT(".");
    }

@@ -15137,6 +15138,46 @@ static void ggml_cl_clamp(ggml_backend_t backend, const ggml_tensor * src0, cons
    memcpy(&min, ((int32_t *) dst->op_params) + 0, sizeof(float));
    memcpy(&max, ((int32_t *) dst->op_params) + 1, sizeof(float));

+    if (!ggml_is_contiguous(src0)) {
+        const int      ne00 = src0->ne[0];
+        const int      ne01 = src0->ne[1];
+        const int      ne02 = src0->ne[2];
+        const int      ne03 = src0->ne[3];
+        const cl_ulong nb00 = src0->nb[0];
+        const cl_ulong nb01 = src0->nb[1];
+        const cl_ulong nb02 = src0->nb[2];
+        const cl_ulong nb03 = src0->nb[3];
+        const cl_ulong nb0  = dst->nb[0];
+        const cl_ulong nb1  = dst->nb[1];
+        const cl_ulong nb2  = dst->nb[2];
+        const cl_ulong nb3  = dst->nb[3];
+
+        cl_kernel kernel = backend_ctx->kernel_clamp_nc;
+
+        CL_CHECK(clSetKernelArg(kernel,  0, sizeof(cl_mem),   &extra0->data_device));
+        CL_CHECK(clSetKernelArg(kernel,  1, sizeof(cl_ulong), &offset0));
+        CL_CHECK(clSetKernelArg(kernel,  2, sizeof(cl_mem),   &extrad->data_device));
+        CL_CHECK(clSetKernelArg(kernel,  3, sizeof(cl_ulong), &offsetd));
+        CL_CHECK(clSetKernelArg(kernel,  4, sizeof(float),    &min));
+        CL_CHECK(clSetKernelArg(kernel,  5, sizeof(float),    &max));
+        CL_CHECK(clSetKernelArg(kernel,  6, sizeof(int),      &ne00));
+        CL_CHECK(clSetKernelArg(kernel,  7, sizeof(cl_ulong), &nb00));
+        CL_CHECK(clSetKernelArg(kernel,  8, sizeof(cl_ulong), &nb01));
+        CL_CHECK(clSetKernelArg(kernel,  9, sizeof(cl_ulong), &nb02));
+        CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));
+        CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb0));
+        CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb1));
+        CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb2));
+        CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb3));
+
+        const int nth = 64;
+        size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};
+        size_t local_work_size[]  = {(size_t)nth, 1, 1};
+
+        backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+        return;
+    }
+
    cl_kernel kernel = backend_ctx->kernel_clamp;

    CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem),   &extra0->data_device));
@@ -18,3 +18,33 @@ kernel void kernel_clamp(
        min :
        (src0[get_global_id(0)] > max ? max : src0[get_global_id(0)]);
 }
+
+kernel void kernel_clamp_nc(
+        global const char * src0,
+        ulong               offset0,
+        global       char * dst,
+        ulong               offsetd,
+        float min,
+        float max,
+        int   ne00,
+        ulong nb00,
+        ulong nb01,
+        ulong nb02,
+        ulong nb03,
+        ulong nb0,
+        ulong nb1,
+        ulong nb2,
+        ulong nb3
+) {
+    src0 = src0 + offset0;
+    dst  = dst + offsetd;
+
+    const int i3 = get_group_id(2);
+    const int i2 = get_group_id(1);
+    const int i1 = get_group_id(0);
+
+    for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {
+        const float x = *(global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
+        *(global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = x < min ? min : (x > max ? max : x);
+    }
+}
@@ -175,6 +175,12 @@ __kernel void flash_attn_f16(
            ACC_TYPE s1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale;
            ACC_TYPE s2 = (dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3) * scale;
            ACC_TYPE s3 = (dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3) * scale;
+            if (logit_softcap > 0.0f) {
+                s0 = logit_softcap * tanh(s0 / logit_softcap);
+                s1 = logit_softcap * tanh(s1 / logit_softcap);
+                s2 = logit_softcap * tanh(s2 / logit_softcap);
+                s3 = logit_softcap * tanh(s3 / logit_softcap);
+            }

            if (is_causal) {
                const int causal_limit = n_kv - n_q + my_query_row;
@@ -196,13 +202,6 @@ __kernel void flash_attn_f16(
                if (k_row3 < n_kv) s3 += slope * (ACC_TYPE)mask_ptr[k_row3];
            }

-            if (logit_softcap > 0.0f) {
-                s0 = logit_softcap * tanh(s0 / logit_softcap);
-                s1 = logit_softcap * tanh(s1 / logit_softcap);
-                s2 = logit_softcap * tanh(s2 / logit_softcap);
-                s3 = logit_softcap * tanh(s3 / logit_softcap);
-            }
-
            const ACC_TYPE m_new      = max(m_i, max(max(s0, s1), max(s2, s3)));
            const ACC_TYPE scale_prev = native_exp(m_i - m_new);
            const ACC_TYPE p0         = native_exp(s0 - m_new);
@@ -331,13 +330,13 @@ __kernel void flash_attn_f16_q1(
            dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc);
        }
        ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;
+        if (logit_softcap > 0.0f) {
+            score = logit_softcap * tanh(score / logit_softcap);
+        }
        if (mask_base != NULL) {
            const global DATA_TYPE* mask_ptr = (const global DATA_TYPE*)(mask_base);
            score += slope * (ACC_TYPE)mask_ptr[k_idx];
        }
-        if (logit_softcap > 0.0f) {
-            score = logit_softcap * tanh(score / logit_softcap);
-        }
        m_i = max(m_i, score);
    }

@@ -367,13 +366,13 @@ __kernel void flash_attn_f16_q1(
            dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc);
        }
        ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;
+        if (logit_softcap > 0.0f) {
+            score = logit_softcap * tanh(score / logit_softcap);
+        }
        if (mask_base != NULL) {
            const global DATA_TYPE* mask_ptr = (const global DATA_TYPE*)(mask_base);
            score += slope * (ACC_TYPE)mask_ptr[k_idx];
        }
-        if (logit_softcap > 0.0f) {
-            score = logit_softcap * tanh(score / logit_softcap);
-        }
        const ACC_TYPE p = exp(score - m_final);
        l_i += p;
        FA_UNROLL
@@ -176,6 +176,12 @@ __kernel void flash_attn_f32(
            ACC_TYPE s1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale;
            ACC_TYPE s2 = (dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3) * scale;
            ACC_TYPE s3 = (dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3) * scale;
+            if (logit_softcap > 0.0f) {
+                s0 = logit_softcap * tanh(s0 / logit_softcap);
+                s1 = logit_softcap * tanh(s1 / logit_softcap);
+                s2 = logit_softcap * tanh(s2 / logit_softcap);
+                s3 = logit_softcap * tanh(s3 / logit_softcap);
+            }

            if (is_causal) {
                const int causal_limit = n_kv - n_q + my_query_row;
@@ -197,13 +203,6 @@ __kernel void flash_attn_f32(
                if (k_row3 < n_kv) s3 += slope * (ACC_TYPE)mask_ptr[k_row3];
            }

-            if (logit_softcap > 0.0f) {
-                s0 = logit_softcap * tanh(s0 / logit_softcap);
-                s1 = logit_softcap * tanh(s1 / logit_softcap);
-                s2 = logit_softcap * tanh(s2 / logit_softcap);
-                s3 = logit_softcap * tanh(s3 / logit_softcap);
-            }
-
            const ACC_TYPE m_new      = max(m_i, max(max(s0, s1), max(s2, s3)));
            const ACC_TYPE scale_prev = native_exp(m_i - m_new);
            const ACC_TYPE p0         = native_exp(s0 - m_new);
@@ -332,13 +331,13 @@ __kernel void flash_attn_f32_q1(
            dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc);
        }
        ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;
+        if (logit_softcap > 0.0f) {
+            score = logit_softcap * tanh(score / logit_softcap);
+        }
        if (mask_base != NULL) {
            const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base);
            score += slope * (ACC_TYPE)mask_ptr[k_idx];
        }
-        if (logit_softcap > 0.0f) {
-            score = logit_softcap * tanh(score / logit_softcap);
-        }
        m_i = max(m_i, score);
    }

@@ -368,13 +367,13 @@ __kernel void flash_attn_f32_q1(
            dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc);
        }
        ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;
+        if (logit_softcap > 0.0f) {
+            score = logit_softcap * tanh(score / logit_softcap);
+        }
        if (mask_base != NULL) {
            const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base);
            score += slope * (ACC_TYPE)mask_ptr[k_idx];
        }
-        if (logit_softcap > 0.0f) {
-            score = logit_softcap * tanh(score / logit_softcap);
-        }
        const ACC_TYPE p = exp(score - m_final);
        l_i += p;
        FA_UNROLL
@@ -336,6 +336,10 @@ __kernel void FA_TILE_NAME(

                ACC_TYPE score0 = partial0 * scale;
                ACC_TYPE score1 = partial1 * scale;
+                if (logit_softcap > 0.0f) {
+                    score0 = logit_softcap * tanh(score0 / logit_softcap);
+                    score1 = logit_softcap * tanh(score1 / logit_softcap);
+                }

                if (!query_valid) { score0 = FA_M_INIT; score1 = FA_M_INIT; }
                if (is_causal) {
@@ -359,11 +363,6 @@ __kernel void FA_TILE_NAME(
                    }
                }

-                if (logit_softcap > 0.0f) {
-                    score0 = logit_softcap * tanh(score0 / logit_softcap);
-                    score1 = logit_softcap * tanh(score1 / logit_softcap);
-                }
-
                const ACC_TYPE m_new = max(m_i, max(score0, score1));
                // Whole tile masked (m_new == FA_M_INIT): force the exp() args
                // far negative so the tile contributes 0, not exp(0)=1.
@@ -408,6 +407,9 @@ __kernel void FA_TILE_NAME(
                        score += local_partial[j][q_lane * N_SPLIT + s];
                    }
                    score *= scale;
+                    if (logit_softcap > 0.0f) {
+                        score = logit_softcap * tanh(score / logit_softcap);
+                    }

                    if (is_causal && k_row > (n_kv - n_q + my_query_row)) score = FA_M_INIT;
                    if (k_row >= n_kv) score = FA_M_INIT;
@@ -424,10 +426,6 @@ __kernel void FA_TILE_NAME(
                        }
                    }

-                    if (logit_softcap > 0.0f) {
-                        score = logit_softcap * tanh(score / logit_softcap);
-                    }
-
                    m_new = max(m_new, score);
                    local_p[q_lane][j] = score;
                }
@@ -502,6 +500,12 @@ __kernel void FA_TILE_NAME(
                ACC_TYPE s1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale;
                ACC_TYPE s2 = (dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3) * scale;
                ACC_TYPE s3 = (dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3) * scale;
+                if (logit_softcap > 0.0f) {
+                    s0 = logit_softcap * tanh(s0 / logit_softcap);
+                    s1 = logit_softcap * tanh(s1 / logit_softcap);
+                    s2 = logit_softcap * tanh(s2 / logit_softcap);
+                    s3 = logit_softcap * tanh(s3 / logit_softcap);
+                }

                if (is_causal) {
                    const int causal_limit = n_kv - n_q + my_query_row;
@@ -531,13 +535,6 @@ __kernel void FA_TILE_NAME(
                    }
                }

-                if (logit_softcap > 0.0f) {
-                    s0 = logit_softcap * tanh(s0 / logit_softcap);
-                    s1 = logit_softcap * tanh(s1 / logit_softcap);
-                    s2 = logit_softcap * tanh(s2 / logit_softcap);
-                    s3 = logit_softcap * tanh(s3 / logit_softcap);
-                }
-
                const ACC_TYPE m_new      = max(m_i, max(max(s0, s1), max(s2, s3)));
                // Whole tile masked (m_new == FA_M_INIT): force the exp() args
                // far negative so the tile contributes 0, not exp(0)=1.
@@ -748,13 +745,13 @@ __kernel void flash_attn_f32_f16_q1(
            dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc);
        }
        ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;
+        if (logit_softcap > 0.0f) {
+            score = logit_softcap * tanh(score / logit_softcap);
+        }
        if (mask_base != NULL) {
            const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base);
            score += slope * (ACC_TYPE)mask_ptr[k_idx];
        }
-        if (logit_softcap > 0.0f) {
-            score = logit_softcap * tanh(score / logit_softcap);
-        }
        m_i = max(m_i, score);
    }

@@ -776,13 +773,13 @@ __kernel void flash_attn_f32_f16_q1(
            dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc);
        }
        ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;
+        if (logit_softcap > 0.0f) {
+            score = logit_softcap * tanh(score / logit_softcap);
+        }
        if (mask_base != NULL) {
            const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base);
            score += slope * (ACC_TYPE)mask_ptr[k_idx];
        }
-        if (logit_softcap > 0.0f) {
-            score = logit_softcap * tanh(score / logit_softcap);
-        }
        const ACC_TYPE p = exp(score - m_final);
        l_i += p;
        FA_UNROLL
@@ -1122,13 +1119,13 @@ __kernel void flash_attn_f32_f16_q1_local_tile(

            if (tid == 0) {
                float s = red[0] * scale;
+                if (logit_softcap > 0.0f) {
+                    s = logit_softcap * tanh(s / logit_softcap);
+                }
                if (mask_base != NULL) {
                    const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base;
                    s += slope * (float) mask_ptr[kv_idx];
                }
-                if (logit_softcap > 0.0f) {
-                    s = logit_softcap * tanh(s / logit_softcap);
-                }
                score_shared = s;
            }
            barrier(CLK_LOCAL_MEM_FENCE);
@@ -1323,13 +1320,13 @@ __kernel void flash_attn_f32_f16_q1_local_mq_split(
                    contrib += q_shared[h * DK + d] * (float) k_tile[j * DK + d];
                }
                float s = sub_group_reduce_add(contrib) * scale;
+                if (logit_softcap > 0.0f) {
+                    s = logit_softcap * tanh(s / logit_softcap);
+                }
                if (mask_base[h] != NULL) {
                    const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
                    s += slope[h] * (float) mask_ptr[kv_idx];
                }
-                if (logit_softcap > 0.0f) {
-                    s = logit_softcap * tanh(s / logit_softcap);
-                }
                score[h] = s;
            }

@@ -1510,13 +1507,13 @@ __kernel void flash_attn_f32_f16_q1_vec_mq(
        for (int h = 0; h < MQ_GQA; ++h) {
            const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3;
            ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale;
+            if (logit_softcap > 0.0f) {
+                s = logit_softcap * tanh(s / logit_softcap);
+            }
            if (mask_base[h] != NULL) {
                const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
                s += slope[h] * (ACC_TYPE) mask_ptr[k_idx];
            }
-            if (logit_softcap > 0.0f) {
-                s = logit_softcap * tanh(s / logit_softcap);
-            }
            score[h] = s;
        }

@@ -2448,13 +2445,13 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_k_img(
        for (int h = 0; h < MQ_GQA; ++h) {
            const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3;
            ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale;
+            if (logit_softcap > 0.0f) {
+                s = logit_softcap * tanh(s / logit_softcap);
+            }
            if (mask_base[h] != NULL) {
                const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
                s += slope[h] * (ACC_TYPE) mask_ptr[k_idx];
            }
-            if (logit_softcap > 0.0f) {
-                s = logit_softcap * tanh(s / logit_softcap);
-            }
            score[h] = s;
        }

@@ -2633,13 +2630,13 @@ __kernel void flash_attn_f32_f16_q1_split(
            dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc);
        }
        ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;
+        if (logit_softcap > 0.0f) {
+            score = logit_softcap * tanh(score / logit_softcap);
+        }
        if (mask_base != NULL) {
            const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base);
            score += slope * (ACC_TYPE) mask_ptr[k_idx];
        }
-        if (logit_softcap > 0.0f) {
-            score = logit_softcap * tanh(score / logit_softcap);
-        }
        m_i = max(m_i, score);
    }

@@ -2662,13 +2659,13 @@ __kernel void flash_attn_f32_f16_q1_split(
            dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc);
        }
        ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;
+        if (logit_softcap > 0.0f) {
+            score = logit_softcap * tanh(score / logit_softcap);
+        }
        if (mask_base != NULL) {
            const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base);
            score += slope * (ACC_TYPE) mask_ptr[k_idx];
        }
-        if (logit_softcap > 0.0f) {
-            score = logit_softcap * tanh(score / logit_softcap);
-        }
        const ACC_TYPE p = exp(score - m_c);
        l_i += p;
        #pragma unroll
@@ -287,14 +287,14 @@ __kernel void flash_attn_f32_q4_0_q1(
 #endif
        }
        score *= scale;
+        if (logit_softcap > 0.0f) {
+            score = logit_softcap * tanh(score / logit_softcap);
+        }

        if (mask_base != NULL) {
            const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base);
            score += slope * (ACC_TYPE)mask_ptr[k_idx];
        }
-        if (logit_softcap > 0.0f) {
-            score = logit_softcap * tanh(score / logit_softcap);
-        }

        // Online softmax step.
        const ACC_TYPE m_new = max(m_i, score);
@@ -554,14 +554,14 @@ __kernel void flash_attn_f32_q4_0_q1_vec(
        ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3;
        ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale;
 #endif
+        if (logit_softcap > 0.0f) {
+            score = logit_softcap * tanh(score / logit_softcap);
+        }

        if (mask_base != NULL) {
            const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base;
            score += slope * (ACC_TYPE) mask_ptr[k_idx];
        }
-        if (logit_softcap > 0.0f) {
-            score = logit_softcap * tanh(score / logit_softcap);
-        }

        const ACC_TYPE m_new      = max(m_i, score);
        const ACC_TYPE scale_prev = native_exp(m_i - m_new);
@@ -745,13 +745,13 @@ __kernel void flash_attn_f32_q4_0_q1_split(
 #endif
        }
        score *= scale;
+        if (logit_softcap > 0.0f) {
+            score = logit_softcap * tanh(score / logit_softcap);
+        }
        if (mask_base != NULL) {
            const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base);
            score += slope * (ACC_TYPE) mask_ptr[k_idx];
        }
-        if (logit_softcap > 0.0f) {
-            score = logit_softcap * tanh(score / logit_softcap);
-        }

        // Online softmax step.
        const ACC_TYPE m_new = max(m_i, score);
@@ -1037,13 +1037,13 @@ __kernel void flash_attn_f32_q4_0_q1_vec_mq_split(
        #pragma unroll
        for (int h = 0; h < MQ_GQA; ++h) {
            ACC_TYPE s = sub_group_reduce_add(lane_contrib[h]) * scale;
+            if (logit_softcap > 0.0f) {
+                s = logit_softcap * tanh(s / logit_softcap);
+            }
            if (mask_base[h] != NULL) {
                const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
                s += slope[h] * (ACC_TYPE) mask_ptr[k_idx];
            }
-            if (logit_softcap > 0.0f) {
-                s = logit_softcap * tanh(s / logit_softcap);
-            }
            score[h] = s;
        }
 #else
@@ -1067,13 +1067,13 @@ __kernel void flash_attn_f32_q4_0_q1_vec_mq_split(
        for (int h = 0; h < MQ_GQA; ++h) {
            const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3;
            ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale;
+            if (logit_softcap > 0.0f) {
+                s = logit_softcap * tanh(s / logit_softcap);
+            }
            if (mask_base[h] != NULL) {
                const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
                s += slope[h] * (ACC_TYPE) mask_ptr[k_idx];
            }
-            if (logit_softcap > 0.0f) {
-                s = logit_softcap * tanh(s / logit_softcap);
-            }
            score[h] = s;
        }
 #endif
@@ -1385,13 +1385,13 @@ __kernel void flash_attn_f32_q4_0_q1_vec_mq_split_c8(
                s += sub_group_shuffle_xor(s, step);
            }
            s *= scale;
+            if (logit_softcap > 0.0f) {
+                s = logit_softcap * tanh(s / logit_softcap);
+            }
            if (mask_base[h] != NULL) {
                const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
                s += slope[h] * (ACC_TYPE) mask_ptr[k_safe];
            }
-            if (logit_softcap > 0.0f) {
-                s = logit_softcap * tanh(s / logit_softcap);
-            }
            score[h] = valid ? s : FA_M_INIT;
        }

@@ -1852,9 +1852,15 @@ __kernel void flash_attn_f32_q4_0(
                    s2 = sub_group_shuffle(s2, tri_base + 0) + sub_group_shuffle(s2, tri_base + 1) + sub_group_shuffle(s2, tri_base + 2);
                    s3 = sub_group_shuffle(s3, tri_base + 0) + sub_group_shuffle(s3, tri_base + 1) + sub_group_shuffle(s3, tri_base + 2);
                #endif
-                if (!query_valid) { s0 = FA_M_INIT; s1 = FA_M_INIT; s2 = FA_M_INIT; s3 = FA_M_INIT; }
 #endif
                s0 *= scale; s1 *= scale; s2 *= scale; s3 *= scale;
+                if (logit_softcap > 0.0f) {
+                    s0 = logit_softcap * tanh(s0 / logit_softcap);
+                    s1 = logit_softcap * tanh(s1 / logit_softcap);
+                    s2 = logit_softcap * tanh(s2 / logit_softcap);
+                    s3 = logit_softcap * tanh(s3 / logit_softcap);
+                }
+                if (!query_valid) { s0 = FA_M_INIT; s1 = FA_M_INIT; s2 = FA_M_INIT; s3 = FA_M_INIT; }

                if (is_causal) {
                    const int causal_limit = n_kv - n_q + my_query_row;
@@ -1876,12 +1882,6 @@ __kernel void flash_attn_f32_q4_0(
                    if (k_row2 < n_kv) s2 += slope * (ACC_TYPE) mask_ptr[k_row2];
                    if (k_row3 < n_kv) s3 += slope * (ACC_TYPE) mask_ptr[k_row3];
                }
-                if (logit_softcap > 0.0f) {
-                    s0 = logit_softcap * tanh(s0 / logit_softcap);
-                    s1 = logit_softcap * tanh(s1 / logit_softcap);
-                    s2 = logit_softcap * tanh(s2 / logit_softcap);
-                    s3 = logit_softcap * tanh(s3 / logit_softcap);
-                }

                const ACC_TYPE m_new      = max(m_i, max(max(s0, s1), max(s2, s3)));
                // Whole tile masked (m_new == FA_M_INIT): force the exp() args
@@ -232,14 +232,14 @@ __kernel void flash_attn_f32_q8_0_q1(
 #endif
        }
        score *= scale;
+        if (logit_softcap > 0.0f) {
+            score = logit_softcap * tanh(score / logit_softcap);
+        }

        if (mask_base != NULL) {
            const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base);
            score += slope * (ACC_TYPE)mask_ptr[k_idx];
        }
-        if (logit_softcap > 0.0f) {
-            score = logit_softcap * tanh(score / logit_softcap);
-        }

        // Online softmax step.
        const ACC_TYPE m_new = max(m_i, score);
@@ -437,14 +437,14 @@ __kernel void flash_attn_f32_q8_0_q1_vec(
        }
        ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3;
        ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale;
+        if (logit_softcap > 0.0f) {
+            score = logit_softcap * tanh(score / logit_softcap);
+        }

        if (mask_base != NULL) {
            const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base;
            score += slope * (ACC_TYPE) mask_ptr[k_idx];
        }
-        if (logit_softcap > 0.0f) {
-            score = logit_softcap * tanh(score / logit_softcap);
-        }

        const ACC_TYPE m_new      = max(m_i, score);
        const ACC_TYPE scale_prev = native_exp(m_i - m_new);
@@ -627,13 +627,13 @@ __kernel void flash_attn_f32_q8_0_q1_split(
 #endif
        }
        score *= scale;
+        if (logit_softcap > 0.0f) {
+            score = logit_softcap * tanh(score / logit_softcap);
+        }
        if (mask_base != NULL) {
            const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base);
            score += slope * (ACC_TYPE) mask_ptr[k_idx];
        }
-        if (logit_softcap > 0.0f) {
-            score = logit_softcap * tanh(score / logit_softcap);
-        }

        // Online softmax step.
        const ACC_TYPE m_new = max(m_i, score);
@@ -882,13 +882,13 @@ __kernel void flash_attn_f32_q8_0_q1_vec_mq_split(
        for (int h = 0; h < MQ_GQA; ++h) {
            const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3;
            ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale;
+            if (logit_softcap > 0.0f) {
+                s = logit_softcap * tanh(s / logit_softcap);
+            }
            if (mask_base[h] != NULL) {
                const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
                s += slope[h] * (ACC_TYPE) mask_ptr[k_idx];
            }
-            if (logit_softcap > 0.0f) {
-                s = logit_softcap * tanh(s / logit_softcap);
-            }
            score[h] = s;
        }

@@ -1151,13 +1151,13 @@ __kernel void flash_attn_f32_q8_0_q1_vec_mq_split_c8(
                s += sub_group_shuffle_xor(s, step);
            }
            s *= scale;
+            if (logit_softcap > 0.0f) {
+                s = logit_softcap * tanh(s / logit_softcap);
+            }
            if (mask_base[h] != NULL) {
                const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
                s += slope[h] * (ACC_TYPE) mask_ptr[k_safe];
            }
-            if (logit_softcap > 0.0f) {
-                s = logit_softcap * tanh(s / logit_softcap);
-            }
            score[h] = valid ? s : FA_M_INIT;
        }

@@ -1652,9 +1652,15 @@ __kernel void flash_attn_f32_q8_0(
                    s2 = sub_group_shuffle(s2, tri_base + 0) + sub_group_shuffle(s2, tri_base + 1) + sub_group_shuffle(s2, tri_base + 2);
                    s3 = sub_group_shuffle(s3, tri_base + 0) + sub_group_shuffle(s3, tri_base + 1) + sub_group_shuffle(s3, tri_base + 2);
                #endif
-                if (!query_valid) { s0 = FA_M_INIT; s1 = FA_M_INIT; s2 = FA_M_INIT; s3 = FA_M_INIT; }
 #endif
                s0 *= scale; s1 *= scale; s2 *= scale; s3 *= scale;
+                if (logit_softcap > 0.0f) {
+                    s0 = logit_softcap * tanh(s0 / logit_softcap);
+                    s1 = logit_softcap * tanh(s1 / logit_softcap);
+                    s2 = logit_softcap * tanh(s2 / logit_softcap);
+                    s3 = logit_softcap * tanh(s3 / logit_softcap);
+                }
+                if (!query_valid) { s0 = FA_M_INIT; s1 = FA_M_INIT; s2 = FA_M_INIT; s3 = FA_M_INIT; }

                if (is_causal) {
                    const int causal_limit = n_kv - n_q + my_query_row;
@@ -1676,12 +1682,6 @@ __kernel void flash_attn_f32_q8_0(
                    if (k_row2 < n_kv) s2 += slope * (ACC_TYPE) mask_ptr[k_row2];
                    if (k_row3 < n_kv) s3 += slope * (ACC_TYPE) mask_ptr[k_row3];
                }
-                if (logit_softcap > 0.0f) {
-                    s0 = logit_softcap * tanh(s0 / logit_softcap);
-                    s1 = logit_softcap * tanh(s1 / logit_softcap);
-                    s2 = logit_softcap * tanh(s2 / logit_softcap);
-                    s3 = logit_softcap * tanh(s3 / logit_softcap);
-                }

                const ACC_TYPE m_new      = max(m_i, max(max(s0, s1), max(s2, s3)));
                // Whole tile masked (m_new == FA_M_INIT): force the exp() args
@@ -230,6 +230,8 @@ kernel void kernel_gemv_noshuffle_iq4_nl_f32(
    uint M = ne01;

    uint LINE_STRIDE_A = M / 2;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, LINE_STRIDE_A - 1);
    uint BLOCK_STRIDE_A = NSUBGROUPS * M;

    private uint4     regA;
@@ -240,7 +242,7 @@ kernel void kernel_gemv_noshuffle_iq4_nl_f32(

    // loop along K in block granularity, skip 4 blocks every iter
    for (uint k = groupId; k < (K / QK4_NL); k += NSUBGROUPS) {
-        regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of two rows
+        regS = src0_d[gid_s + k * LINE_STRIDE_A]; // each fiber loads scale of two rows
        // first 4 fibers in each wave load 8 B values to its private scope
        if (slid < 4) {
            regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));
@@ -248,20 +250,20 @@ kernel void kernel_gemv_noshuffle_iq4_nl_f32(
        }

        // load half weights for two blocks in consecutive rows
-        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
-        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
-        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
-        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
 #ifdef VECTOR_SUB_GROUP_BROADCAST
        dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regB);
 #else
        dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regB);
 #endif // VECTOR_SUB_GROUP_BROADCAST

-        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
-        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
-        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
-        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+        regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
 #ifdef VECTOR_SUB_GROUP_BROADCAST
        dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regB);
 #else
@@ -73,6 +73,8 @@ __kernel void kernel_gemv_noshuffle_q1_0_f32(
    uint M = ne01;

    uint LINE_STRIDE_A  = M;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, LINE_STRIDE_A - 1);
    uint BLOCK_STRIDE_A = 4 * M;

    uint4  regA;
@@ -83,7 +85,7 @@ __kernel void kernel_gemv_noshuffle_q1_0_f32(

    #pragma unroll 1
    for (uint kb = groupId; kb < (K / QK1_0); kb += N_SIMDGROUP) {
-        regS = src0_d[gid + kb * LINE_STRIDE_A]; // each fiber loads its row's scale
+        regS = src0_d[gid_s + kb * LINE_STRIDE_A]; // each fiber loads its row's scale

        // first 16 fibers load 8 B values each -> 128 activations for this block
        if (slid < 16) {
@@ -92,10 +94,10 @@ __kernel void kernel_gemv_noshuffle_q1_0_f32(
        }

        // load this row's 4 uint32 (128 sign bits)
-        regA.s0 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
-        regA.s1 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
-        regA.s2 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
-        regA.s3 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA.s0 = read_imageui(src0_q, (gid_s + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA.s1 = read_imageui(src0_q, (gid_s + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA.s2 = read_imageui(src0_q, (gid_s + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA.s3 = read_imageui(src0_q, (gid_s + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;

        float scale = (float)regS;
        dequantizeBlockAccum_q1(totalSum, regA.s0, scale, regB, 0);
@@ -224,6 +224,8 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32(
    uint M = ne01;

    uint LINE_STRIDE_A = M / 2;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, LINE_STRIDE_A - 1);
    uint BLOCK_STRIDE_A = N_SIMDGROUP * M;

    __private uint4     regA;
@@ -234,7 +236,7 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32(

    // loop along K in block granularity, skip 4 blocks every iter
    for (uint k = groupId; k < (K / QK4_0); k += N_SIMDGROUP) {
-        regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of two rows
+        regS = src0_d[gid_s + k * LINE_STRIDE_A]; // each fiber loads scale of two rows
        // first 4 fibers in each wave load 8 B values to its private scope
        if (slid < 4) {
            regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));
@@ -242,20 +244,20 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32(
        }

        // load half weights for two blocks in consecutive rows
-        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
-        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
-        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
-        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
 #ifdef VECTOR_SUB_GROUP_BROADCAST
        dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regB);
 #else
        dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regB);
 #endif // VECTOR_SUB_GROUP_BROADCAST

-        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
-        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
-        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
-        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+        regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
 #ifdef VECTOR_SUB_GROUP_BROADCAST
        dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regB);
 #else
@@ -329,6 +331,8 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32_mc3(
    uint M = ne01;

    uint LINE_STRIDE_A  = M / 2;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, LINE_STRIDE_A - 1);
    // BLOCK_STRIDE_A is the LAYOUT stride between consecutive K-blocks = 4 uints
    // per q4_0 block * M (set by the trans4_ns convert). The "4" is uints/block, NOT
    // the subgroup count — keep it fixed so the K-split count (nsg) can vary.
@@ -347,17 +351,17 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32_mc3(
    __private float2 ts3 = (float2)(0.0f);

    for (uint k = groupId; k < (K / QK4_0); k += nsg) {
-        regS = src0_d[gid + k * LINE_STRIDE_A];
+        regS = src0_d[gid_s + k * LINE_STRIDE_A];

        // weights loaded ONCE, reused across the columns
-        regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
-        regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
-        regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
-        regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
-        regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
-        regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
-        regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
-        regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+        regA_hi.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA_hi.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA_hi.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA_hi.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA_lo.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA_lo.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA_lo.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA_lo.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;

        MC_COL_Q40(ts0, 0);
        MC_COL_Q40(ts1, 1);
@@ -381,10 +385,12 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32_mc3(
        }
        dst = (global float*)((global char*)dst + offsetd);
        // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row
-        vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
-        vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
-        if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
-        if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2]));
+        if (gid < LINE_STRIDE_A) {
+            vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
+            vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
+            if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
+            if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2]));
+        }
    }
 }
 #undef MC_COL_Q40
@@ -210,6 +210,8 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32(
 {
    uint groupId = get_local_id(1);
    uint gid     = get_global_id(0);
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, (uint)LINE_STRIDE_A - 1u);
    ushort slid    = get_sub_group_local_id();

    __private uint4     regA;
@@ -220,7 +222,7 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32(

    // loop along K in block granularity, skip 4 blocks every iter
    for (uint k = groupId; k < (K / QK4_0); k += N_SIMDGROUP) {
-        regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of two rows
+        regS = src0_d[gid_s + k * LINE_STRIDE_A]; // each fiber loads scale of two rows
        // first 4 fibers in each wave load 8 B values to its private scope
        if (slid < 4) {
            regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));
@@ -228,20 +230,20 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32(
        }

        // load half weights for two blocks in consecutive rows
-        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
-        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
-        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
-        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
 #ifdef VECTOR_SUB_GROUP_BROADCAST
        dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regB);
 #else
        dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regB);
 #endif // VECTOR_SUB_GROUP_BROADCAST

-        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
-        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
-        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
-        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+        regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
 #ifdef VECTOR_SUB_GROUP_BROADCAST
        dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regB);
 #else
@@ -209,6 +209,8 @@ kernel void kernel_gemv_noshuffle_q4_1_f32(
    uint M = ne01;

    uint LINE_STRIDE_A = M / 2;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, LINE_STRIDE_A - 1);
    uint BLOCK_STRIDE_A = NSUBGROUPS * M;

    private uint4     regA;
@@ -220,8 +222,8 @@ kernel void kernel_gemv_noshuffle_q4_1_f32(

    // loop along K in block granularity, skip 4 blocks every iter
    for (uint k = groupId; k < (K / QK4_0); k += NSUBGROUPS) {
-        regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of two rows
-        regM = src0_m[gid + k * LINE_STRIDE_A]; // each fiber loads min of two rows
+        regS = src0_d[gid_s + k * LINE_STRIDE_A]; // each fiber loads scale of two rows
+        regM = src0_m[gid_s + k * LINE_STRIDE_A]; // each fiber loads min of two rows
        // first 4 fibers in each wave load 8 B values to its private scope
        if (slid < 4) {
            regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));
@@ -229,20 +231,20 @@ kernel void kernel_gemv_noshuffle_q4_1_f32(
        }

        // load half weights for two blocks in consecutive rows
-        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
-        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
-        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
-        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
 #ifdef VECTOR_SUB_GROUP_BROADCAT
        dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regM, regB);
 #else
        dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regM, regB);
 #endif // VECTOR_SUB_GROUP_BROADCAT

-        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
-        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
-        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
-        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+        regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
 #ifdef VECTOR_SUB_GROUP_BROADCAT
        dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regM, regB);
 #else
@@ -326,6 +328,8 @@ kernel void kernel_gemv_noshuffle_q4_1_f32_mc3(
    uint M = ne01;

    uint LINE_STRIDE_A  = M / 2;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, LINE_STRIDE_A - 1);
    uint BLOCK_STRIDE_A = NSUBGROUPS * M;
    uint COL_STRIDE     = K / 4;   // float4 pixels per activation column

@@ -339,18 +343,18 @@ kernel void kernel_gemv_noshuffle_q4_1_f32_mc3(
    private float2 ts3 = (float2)(0.0f);

    for (uint k = groupId; k < (K / QK4_0); k += NSUBGROUPS) {
-        regS = src0_d[gid + k * LINE_STRIDE_A];
-        regM = src0_m[gid + k * LINE_STRIDE_A];
+        regS = src0_d[gid_s + k * LINE_STRIDE_A];
+        regM = src0_m[gid_s + k * LINE_STRIDE_A];

        // weights loaded ONCE, reused across the columns
-        regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
-        regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
-        regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
-        regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
-        regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
-        regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
-        regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
-        regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+        regA_hi.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA_hi.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA_hi.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA_hi.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA_lo.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA_lo.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA_lo.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA_lo.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;

        MC_COL_Q41(ts0, 0);
        MC_COL_Q41(ts1, 1);
@@ -373,10 +377,12 @@ kernel void kernel_gemv_noshuffle_q4_1_f32_mc3(
        acc += reduceLM[SUBGROUP_SIZE * 2 + slid];
        dst = (global float*)((global char*)dst + offsetd);
        // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row
-        vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
-        vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
-        if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
-        if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2]));
+        if (gid < LINE_STRIDE_A) {
+            vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
+            vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
+            if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
+            if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2]));
+        }
    }
 }
 #undef MC_COL_Q41
@@ -415,6 +415,8 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_glu(
    uint M = ne01;

    uint LINE_STRIDE_A  = M / 2;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, LINE_STRIDE_A - 1);
    uint BLOCK_STRIDE_A = 4 * M;

    private uint4  regA;
@@ -434,9 +436,9 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_glu(
    for (uint k = groupId; k < (K / 32); k += nsg) {                                           \
        uint sb = k / 8;                                                                       \
        uint j  = k % 8;                                                                       \
-        half2 d   = DD[gid + sb * LINE_STRIDE_A];                                              \
-        half2 dm  = MM[gid + sb * LINE_STRIDE_A];                                              \
-        global const uchar * sc0 = SS + sb * 12 * M + 2 * gid;                                 \
+        half2 d   = DD[gid_s + sb * LINE_STRIDE_A];                                            \
+        half2 dm  = MM[gid_s + sb * LINE_STRIDE_A];                                            \
+        global const uchar * sc0 = SS + sb * 12 * M + 2 * gid_s;                               \
        global const uchar * sc1 = sc0 + 1;                                                    \
        uchar sv0, mn0, sv1, mn1;                                                              \
        get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);                      \
@@ -447,15 +449,15 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_glu(
            regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));                                \
            regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));                            \
        }                                                                                      \
-        regA.s0 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;           \
-        regA.s1 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;           \
-        regA.s2 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;           \
-        regA.s3 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;           \
+        regA.s0 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;           \
+        regA.s1 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;           \
+        regA.s2 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;           \
+        regA.s3 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;           \
        DEQ_HI(SUM, as_ushort8(regA), regS, regM, regB);                                       \
-        regA.s0 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;           \
-        regA.s1 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;           \
-        regA.s2 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;           \
-        regA.s3 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;           \
+        regA.s0 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;           \
+        regA.s1 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;           \
+        regA.s2 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;           \
+        regA.s3 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;           \
        DEQ_LO(SUM, as_ushort8(regA), regS, regM, regB);                                       \
    }

@@ -489,8 +491,10 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_glu(
            upSum   += p.zw;
        }
        dst = (global float*)((global char*)dst + offsetd);
-        dst[gid * 2 + 0] = glu_apply(glu_op, gateSum.s0, upSum.s0);
-        dst[gid * 2 + 1] = glu_apply(glu_op, gateSum.s1, upSum.s1);
+        if (gid < LINE_STRIDE_A) {
+            dst[gid * 2 + 0] = glu_apply(glu_op, gateSum.s0, upSum.s0);
+            dst[gid * 2 + 1] = glu_apply(glu_op, gateSum.s1, upSum.s1);
+        }
    }
 }

@@ -533,6 +537,9 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_splitk(
    uint LINE_STRIDE_A  = M / 2;
    uint BLOCK_STRIDE_A = 4 * M;      // physical, independent of the K-split

+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, LINE_STRIDE_A - 1);
+
    private uint4  regA;
    private half2  regS, regM;
    private float8 regB;
@@ -542,9 +549,9 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_splitk(
    for (uint k = kslice * nsg + groupId; k < (K / 32); k += ksplit * nsg) {
        uint sb = k / 8;
        uint j  = k % 8;
-        half2 d   = src0_d[gid + sb * LINE_STRIDE_A];
-        half2 dm  = src0_m[gid + sb * LINE_STRIDE_A];
-        global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid;
+        half2 d   = src0_d[gid_s + sb * LINE_STRIDE_A];
+        half2 dm  = src0_m[gid_s + sb * LINE_STRIDE_A];
+        global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid_s;
        global const uchar * sc1 = sc0 + 1;
        uchar sv0, mn0, sv1, mn1;
        get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
@@ -555,19 +562,19 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_splitk(
            regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));
            regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));
        }
-        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
-        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
-        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
-        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
 #ifdef VECTOR_SUB_GROUP_BROADCAST
        dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regM, regB);
 #else
        dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regM, regB);
 #endif
-        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
-        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
-        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
-        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+        regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
 #ifdef VECTOR_SUB_GROUP_BROADCAST
        dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regM, regB);
 #else
@@ -584,7 +591,9 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_splitk(
        for (uint i = 0; i < nsg - 1; ++i) {
            totalSum += reduceLM[SUBGROUP_SIZE * i + slid];
        }
-        vstore2(totalSum, 0, &(partial[kslice * M + gid * 2]));
+        // Keep tail stores inside this K-slice.
+        if (gid * 2 + 0 < M) partial[kslice * M + gid * 2 + 0] = totalSum.s0;
+        if (gid * 2 + 1 < M) partial[kslice * M + gid * 2 + 1] = totalSum.s1;
    }
 }

@@ -689,6 +698,8 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_mc3(
    uint M = ne01;

    uint LINE_STRIDE_A  = M / 2;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, LINE_STRIDE_A - 1);
    uint BLOCK_STRIDE_A = NSUBGROUPS * M;
    uint COL_STRIDE     = K / 4;   // float4 pixels per activation column

@@ -713,10 +724,10 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_mc3(
        uint sb = k / 8;
        uint j  = k % 8;

-        half2 d   = src0_d[gid + sb * LINE_STRIDE_A];
-        half2 dm  = src0_m[gid + sb * LINE_STRIDE_A];
+        half2 d   = src0_d[gid_s + sb * LINE_STRIDE_A];
+        half2 dm  = src0_m[gid_s + sb * LINE_STRIDE_A];

-        global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid;
+        global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid_s;
        global const uchar * sc1 = sc0 + 1;

        uchar sv0, mn0, sv1, mn1;
@@ -727,14 +738,14 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_mc3(
        regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1)));

        // weights loaded ONCE, reused across the 3 columns
-        regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
-        regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
-        regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
-        regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
-        regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
-        regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
-        regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
-        regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+        regA_hi.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA_hi.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA_hi.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA_hi.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA_lo.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA_lo.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA_lo.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA_lo.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;

 #ifdef Q4K_MC3_DEQUANT_ONCE
        // Dequant the 32 weights/row (16 hi + 16 lo) ONCE into half2[] (byte-
@@ -827,8 +838,10 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_mc3(
        acc += reduceLM[SUBGROUP_SIZE * 2 + slid];
        dst = (global float*)((global char*)dst + offsetd);
        // dst is column-major [M rows x 3 cols]: (row, col) at col*M + row
-        vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
-        vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
-        vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
+        if (gid < LINE_STRIDE_A) {
+            vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
+            vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
+            vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
+        }
    }
 }
@@ -231,8 +231,9 @@ kernel void kernel_gemv_noshuffle_q4_k_f32_o4(

    // Two consecutive pair-indices (each the same access pattern the 2-output
    // kernel uses); together they cover 4 consecutive output rows.
-    uint gid_a = gid * 2;
-    uint gid_b = gid * 2 + 1;
+    // Keep tail lanes active for subgroup ops, but read valid weight pairs.
+    uint gid_a = min(gid * 2,     (uint)(ne01 / 2) - 1u);
+    uint gid_b = min(gid * 2 + 1, (uint)(ne01 / 2) - 1u);

    uint K = ne00;
    uint M = ne01;
@@ -209,6 +209,8 @@ __kernel void kernel_gemv_noshuffle_q5_0_f32(
    uint M = ne01;

    uint LINE_STRIDE_A    = M / 2;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, LINE_STRIDE_A - 1);
    uint BLOCK_STRIDE_A  = NSUBGROUPS * M;

    private uint4     regA;
@@ -218,13 +220,13 @@ __kernel void kernel_gemv_noshuffle_q5_0_f32(
    private float2 totalSum = (float2)(0.0f);

    for (uint k = groupId; k < (K / QK5_0); k += NSUBGROUPS) {
-        regS = src0_d[gid + k * LINE_STRIDE_A];
+        regS = src0_d[gid_s + k * LINE_STRIDE_A];

        ushort4 qh_raw;
-        qh_raw.s0 = src0_qh[gid + (4*k + 0) * LINE_STRIDE_A];
-        qh_raw.s1 = src0_qh[gid + (4*k + 1) * LINE_STRIDE_A];
-        qh_raw.s2 = src0_qh[gid + (4*k + 2) * LINE_STRIDE_A];
-        qh_raw.s3 = src0_qh[gid + (4*k + 3) * LINE_STRIDE_A];
+        qh_raw.s0 = src0_qh[gid_s + (4*k + 0) * LINE_STRIDE_A];
+        qh_raw.s1 = src0_qh[gid_s + (4*k + 1) * LINE_STRIDE_A];
+        qh_raw.s2 = src0_qh[gid_s + (4*k + 2) * LINE_STRIDE_A];
+        qh_raw.s3 = src0_qh[gid_s + (4*k + 3) * LINE_STRIDE_A];

        uchar8 raw = as_uchar8(qh_raw);
        uchar8 qh_bytes = (uchar8)(raw.s0, raw.s2, raw.s4, raw.s6,
@@ -236,10 +238,10 @@ __kernel void kernel_gemv_noshuffle_q5_0_f32(
            regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));
        }

-        regA.s0 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
-        regA.s1 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
-        regA.s2 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
-        regA.s3 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA.s0 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA.s1 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA.s2 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA.s3 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;

 #ifdef VECTOR_SUB_GROUP_BROADCAST
        dequantizeBlockAccum_ns_q5_0_sgbroadcast_8_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regB);
@@ -247,10 +249,10 @@ __kernel void kernel_gemv_noshuffle_q5_0_f32(
        dequantizeBlockAccum_ns_q5_0_sgbroadcast_1_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regB);
 #endif // VECTOR_SUB_GROUP_BROADCAST

-        regA.s0 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
-        regA.s1 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
-        regA.s2 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
-        regA.s3 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+        regA.s0 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA.s1 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA.s2 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA.s3 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
 #ifdef VECTOR_SUB_GROUP_BROADCAST
        dequantizeBlockAccum_ns_q5_0_sgbroadcast_8_lo(totalSum, as_ushort8(regA), qh_bytes, regS, regB);
 #else
@@ -210,6 +210,8 @@ __kernel void kernel_gemv_noshuffle_q5_1_f32(
    uint M = ne01;

    uint LINE_STRIDE_A    = M / 2;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, LINE_STRIDE_A - 1);
   uint BLOCK_STRIDE_A  = NSUBGROUPS * M;

    __private uint4     regA;
@@ -220,14 +222,14 @@ __kernel void kernel_gemv_noshuffle_q5_1_f32(
    __private float2 totalSum = (float2)(0.0f);

    for (uint k = groupId; k < (K / QK5_1); k += NSUBGROUPS) {
-        regS = src0_d[gid + k * LINE_STRIDE_A];
-        regM = src0_m[gid + k * LINE_STRIDE_A];
+        regS = src0_d[gid_s + k * LINE_STRIDE_A];
+        regM = src0_m[gid_s + k * LINE_STRIDE_A];

        ushort4 qh_raw;
-        qh_raw.s0 = src0_qh[gid + (4*k + 0) * LINE_STRIDE_A];
-        qh_raw.s1 = src0_qh[gid + (4*k + 1) * LINE_STRIDE_A];
-        qh_raw.s2 = src0_qh[gid + (4*k + 2) * LINE_STRIDE_A];
-        qh_raw.s3 = src0_qh[gid + (4*k + 3) * LINE_STRIDE_A];
+        qh_raw.s0 = src0_qh[gid_s + (4*k + 0) * LINE_STRIDE_A];
+        qh_raw.s1 = src0_qh[gid_s + (4*k + 1) * LINE_STRIDE_A];
+        qh_raw.s2 = src0_qh[gid_s + (4*k + 2) * LINE_STRIDE_A];
+        qh_raw.s3 = src0_qh[gid_s + (4*k + 3) * LINE_STRIDE_A];

        uchar8 raw = as_uchar8(qh_raw);
        uchar8 qh_bytes = (uchar8)(raw.s0, raw.s2, raw.s4, raw.s6,
@@ -239,10 +241,10 @@ __kernel void kernel_gemv_noshuffle_q5_1_f32(
            regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));
        }

-        regA.s0 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
-        regA.s1 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
-        regA.s2 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
-        regA.s3 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA.s0 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA.s1 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA.s2 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA.s3 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;

 #ifdef VECTOR_SUB_GROUP_BROADCAST
        dequantizeBlockAccum_ns_q5_1_sgbroadcast_8_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regM, regB);
@@ -250,10 +252,10 @@ __kernel void kernel_gemv_noshuffle_q5_1_f32(
        dequantizeBlockAccum_ns_q5_1_sgbroadcast_1_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regM, regB);
 #endif // VECTOR_SUB_GROUP_BROADCAST

-        regA.s0 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
-        regA.s1 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
-        regA.s2 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
-        regA.s3 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+        regA.s0 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA.s1 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA.s2 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA.s3 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
 #ifdef VECTOR_SUB_GROUP_BROADCAST
        dequantizeBlockAccum_ns_q5_1_sgbroadcast_8_lo(totalSum, as_ushort8(regA), qh_bytes, regS, regM, regB);
 #else
@@ -232,6 +232,8 @@ kernel void kernel_gemv_noshuffle_q5_k_f32(
    uint M = ne01;

    uint LINE_STRIDE_A     = M / 2;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, LINE_STRIDE_A - 1);
    uint BLOCK_STRIDE_A    = NSUBGROUPS * M;

    uint LINE_STRIDE_A_QH  = M / 2;
@@ -250,11 +252,11 @@ kernel void kernel_gemv_noshuffle_q5_k_f32(
        uint sb = k / 8;
        uint j  = k % 8;

-        half2 d   = src0_d[gid + sb * LINE_STRIDE_A];
-        half2 dm  = src0_m[gid + sb * LINE_STRIDE_A];
+        half2 d   = src0_d[gid_s + sb * LINE_STRIDE_A];
+        half2 dm  = src0_m[gid_s + sb * LINE_STRIDE_A];

-        global const uchar * sc0 = src0_s + 2 * gid * scales_per_row + sb * 12;
-        global const uchar * sc1 = src0_s + (2 * gid + 1) * scales_per_row + sb * 12;
+        global const uchar * sc0 = src0_s + 2 * gid_s * scales_per_row + sb * 12;
+        global const uchar * sc1 = src0_s + (2 * gid_s + 1) * scales_per_row + sb * 12;

        uchar sv0, mn0, sv1, mn1;
        get_scale_min_k4(j, sc0, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
@@ -268,25 +270,25 @@ kernel void kernel_gemv_noshuffle_q5_k_f32(
            regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));
        }

-        regH.s0 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 0)).x);
-        regH.s1 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 1)).x);
-        regH.s2 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 2)).x);
-        regH.s3 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 3)).x);
+        regH.s0 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 0)).x);
+        regH.s1 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 1)).x);
+        regH.s2 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 2)).x);
+        regH.s3 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 3)).x);

-        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
-        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
-        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
-        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
 #ifdef VECTOR_SUB_GROUP_BROADCAST
        dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), as_uchar8(regH), regS, regM, regB);
 #else
        dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), as_uchar8(regH), regS, regM, regB);
 #endif // VECTOR_SUB_GROUP_BROADCAST

-        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
-        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
-        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
-        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+        regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
 #ifdef VECTOR_SUB_GROUP_BROADCAST
        dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), as_uchar8(regH), regS, regM, regB);
 #else
@@ -373,6 +375,8 @@ kernel void kernel_gemv_noshuffle_q5_k_f32_mc3(
    uint M = ne01;

    uint LINE_STRIDE_A     = M / 2;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, LINE_STRIDE_A - 1);
    uint BLOCK_STRIDE_A    = NSUBGROUPS * M;
    uint LINE_STRIDE_A_QH  = M / 2;
    uint BLOCK_STRIDE_A_QH = NSUBGROUPS * M / 2;
@@ -393,11 +397,11 @@ kernel void kernel_gemv_noshuffle_q5_k_f32_mc3(
        uint sb = k / 8;
        uint j  = k % 8;

-        half2 d   = src0_d[gid + sb * LINE_STRIDE_A];
-        half2 dm  = src0_m[gid + sb * LINE_STRIDE_A];
+        half2 d   = src0_d[gid_s + sb * LINE_STRIDE_A];
+        half2 dm  = src0_m[gid_s + sb * LINE_STRIDE_A];

-        global const uchar * sc0 = src0_s + 2 * gid * scales_per_row + sb * 12;
-        global const uchar * sc1 = src0_s + (2 * gid + 1) * scales_per_row + sb * 12;
+        global const uchar * sc0 = src0_s + 2 * gid_s * scales_per_row + sb * 12;
+        global const uchar * sc1 = src0_s + (2 * gid_s + 1) * scales_per_row + sb * 12;

        uchar sv0, mn0, sv1, mn1;
        get_scale_min_k4(j, sc0, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
@@ -407,19 +411,19 @@ kernel void kernel_gemv_noshuffle_q5_k_f32_mc3(
        regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1)));

        // high-bit plane + weights loaded ONCE, reused across the columns
-        regH.s0 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 0)).x);
-        regH.s1 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 1)).x);
-        regH.s2 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 2)).x);
-        regH.s3 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 3)).x);
+        regH.s0 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 0)).x);
+        regH.s1 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 1)).x);
+        regH.s2 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 2)).x);
+        regH.s3 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 3)).x);

-        regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
-        regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
-        regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
-        regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
-        regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
-        regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
-        regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
-        regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+        regA_hi.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA_hi.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA_hi.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA_hi.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA_lo.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA_lo.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA_lo.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA_lo.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;

        MC_COL_Q5K(ts0, 0);
        MC_COL_Q5K(ts1, 1);
@@ -442,10 +446,12 @@ kernel void kernel_gemv_noshuffle_q5_k_f32_mc3(
        acc += reduceLM[SUBGROUP_SIZE * 2 + slid];
        dst = (global float*)((global char*)dst + offsetd);
        // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row
-        vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
-        vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
-        if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
-        if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2]));
+        if (gid < LINE_STRIDE_A) {
+            vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
+            vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
+            if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
+            if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2]));
+        }
    }
 }
 #undef MC_COL_Q5K
@@ -219,26 +219,28 @@ kernel void kernel_gemv_noshuffle_q6_K_f32(
    float2  total_sum = 0.0f;

    int line_stride_a = ne01 / 2;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    int gid_s = min(gid, line_stride_a - 1);
    int block_stride_a = NSUBGROUPS * ne01;

    for (int k = grp; k < nb; k += NSUBGROUPS) {
-        reg_d = src0_d[gid + k/8 * line_stride_a];
-        reg_s = as_char4(src0_s[gid + k * line_stride_a]);
+        reg_d = src0_d[gid_s + k/8 * line_stride_a];
+        reg_s = as_char4(src0_s[gid_s + k * line_stride_a]);

        if (slid < 4) {
            reg_b.s0123 = read_imagef(src1, 0 + slid*2 + k*8);
            reg_b.s4567 = read_imagef(src1, 1 + slid*2 + k*8);
        }

-        reg_a_l.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*0).x;
-        reg_a_l.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*1).x;
-        reg_a_l.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*2).x;
-        reg_a_l.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*3).x;
+        reg_a_l.s0 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*0).x;
+        reg_a_l.s1 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*1).x;
+        reg_a_l.s2 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*2).x;
+        reg_a_l.s3 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*3).x;

-        reg_a_h.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*0).x);
-        reg_a_h.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*1).x);
-        reg_a_h.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*2).x);
-        reg_a_h.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*3).x);
+        reg_a_h.s0 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*0).x);
+        reg_a_h.s1 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*1).x);
+        reg_a_h.s2 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*2).x);
+        reg_a_h.s3 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*3).x);

 #ifdef VECTOR_SUB_GROUP_BROADCAT
        dequantize_block_acc_bcast_8_hi(total_sum, as_ushort8(reg_a_l), as_uchar8(reg_a_h), reg_d, reg_s, reg_b);
@@ -246,15 +248,15 @@ kernel void kernel_gemv_noshuffle_q6_K_f32(
        dequantize_block_acc_bcast_1_hi(total_sum, as_ushort8(reg_a_l), as_uchar8(reg_a_h), reg_d, reg_s, reg_b);
 #endif // VECTOR_SUB_GROUP_BROADCAT

-        reg_a_l.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*4).x;
-        reg_a_l.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*5).x;
-        reg_a_l.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*6).x;
-        reg_a_l.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*7).x;
+        reg_a_l.s0 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*4).x;
+        reg_a_l.s1 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*5).x;
+        reg_a_l.s2 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*6).x;
+        reg_a_l.s3 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*7).x;

-        reg_a_h.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*4).x);
-        reg_a_h.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*5).x);
-        reg_a_h.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*6).x);
-        reg_a_h.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*7).x);
+        reg_a_h.s0 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*4).x);
+        reg_a_h.s1 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*5).x);
+        reg_a_h.s2 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*6).x);
+        reg_a_h.s3 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*7).x);

 #ifdef VECTOR_SUB_GROUP_BROADCAT
        dequantize_block_acc_bcast_8_lo(total_sum, as_ushort8(reg_a_l), as_uchar8(reg_a_h), reg_d, reg_s, reg_b);
@@ -323,6 +325,8 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_mc3(

    int nb = ne00 / 32;
    int line_stride_a  = ne01 / 2;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    int gid_s = min(gid, line_stride_a - 1);
    int block_stride_a = NSUBGROUPS * ne01;
    int COL_STRIDE     = ne00 / 4;   // float4 pixels per activation column

@@ -335,27 +339,27 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_mc3(
    float2  ts0 = 0.0f, ts1 = 0.0f, ts2 = 0.0f;

    for (int k = grp; k < nb; k += NSUBGROUPS) {
-        reg_d = src0_d[gid + k/8 * line_stride_a];
-        reg_s = as_char4(src0_s[gid + k * line_stride_a]);
+        reg_d = src0_d[gid_s + k/8 * line_stride_a];
+        reg_s = as_char4(src0_s[gid_s + k * line_stride_a]);

        // weights loaded ONCE (hi: blocks 0-3, lo: blocks 4-7), reused x3 cols
-        ql_hi.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*0).x;
-        ql_hi.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*1).x;
-        ql_hi.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*2).x;
-        ql_hi.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*3).x;
-        qh_hi.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*0).x);
-        qh_hi.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*1).x);
-        qh_hi.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*2).x);
-        qh_hi.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*3).x);
+        ql_hi.s0 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*0).x;
+        ql_hi.s1 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*1).x;
+        ql_hi.s2 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*2).x;
+        ql_hi.s3 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*3).x;
+        qh_hi.s0 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*0).x);
+        qh_hi.s1 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*1).x);
+        qh_hi.s2 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*2).x);
+        qh_hi.s3 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*3).x);

-        ql_lo.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*4).x;
-        ql_lo.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*5).x;
-        ql_lo.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*6).x;
-        ql_lo.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*7).x;
-        qh_lo.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*4).x);
-        qh_lo.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*5).x);
-        qh_lo.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*6).x);
-        qh_lo.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*7).x);
+        ql_lo.s0 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*4).x;
+        ql_lo.s1 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*5).x;
+        ql_lo.s2 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*6).x;
+        ql_lo.s3 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*7).x;
+        qh_lo.s0 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*4).x);
+        qh_lo.s1 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*5).x);
+        qh_lo.s2 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*6).x);
+        qh_lo.s3 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*7).x);

        // Per-column: load only this column's activation (single reg_b live) ->
        // 1/3 the activation register pressure, cutting the private-mem spill.
@@ -402,8 +406,10 @@ kernel void kernel_gemv_noshuffle_q6_K_f32_mc3(
        acc += reduce_lm[SUBGROUP_SIZE*2 + slid];
        dst = (global float*)((global char*)dst + offsetd);
        // dst column-major [ne01 rows x 3 cols]: (row, col) at col*ne01 + row
-        vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0*ne01 + gid*2]));
-        vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1*ne01 + gid*2]));
-        vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2*ne01 + gid*2]));
+        if (gid < line_stride_a) {
+            vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0*ne01 + gid*2]));
+            vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1*ne01 + gid*2]));
+            vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2*ne01 + gid*2]));
+        }
    }
 }
@@ -249,8 +249,9 @@ kernel void Q6K_O4_NAME(
    // ne01/4 apart) is slower because two distant cache-line streams have worse
    // locality than the adjacent pair whose reads interleave into the same lines
    // each iteration.
-    int gid_a = gid * 2;
-    int gid_b = gid * 2 + 1;
+    // Keep tail lanes active for subgroup ops, but read valid weight pairs.
+    int gid_a = min(gid * 2,     ne01 / 2 - 1);
+    int gid_b = min(gid * 2 + 1, ne01 / 2 - 1);

    int nb = ne00 / 32;

@@ -155,6 +155,8 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32_splitk(
    uint M = ne01;

    uint LINE_STRIDE_A  = M;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, LINE_STRIDE_A - 1);
    uint BLOCK_STRIDE_A = 8 * M;   // physical, independent of the K-split

    __private uint8  regA;
@@ -164,19 +166,19 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32_splitk(

    #pragma unroll 1
    for (uint k = kslice * nsg + groupId; k < (K / QK8_0); k += ksplit * nsg) {
-        regS = src0_d[gid + k * LINE_STRIDE_A];
+        regS = src0_d[gid_s + k * LINE_STRIDE_A];
        if (slid < 4) {
            regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));
            regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));
        }
-        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
-        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
-        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
-        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
-        regA.s4 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
-        regA.s5 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
-        regA.s6 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
-        regA.s7 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+        regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA.s4 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA.s5 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA.s6 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA.s7 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;

        dequantizeBlockAccum_ns_sgbroadcast_1(totalSum, regA, convert_float(regS), regB);
    }
@@ -227,6 +229,8 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32(
    uint M = ne01;

    uint LINE_STRIDE_A = M;
+    // Keep tail lanes active for subgroup ops, but read a valid weight row.
+    uint gid_s = min(gid, LINE_STRIDE_A - 1);
    uint BLOCK_STRIDE_A = 8 * M;   // 32 / 4 = 8

    __private uint8     regA;
@@ -238,7 +242,7 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32(
    // loop along K in block granularity, skip 4 blocks every iter
    #pragma unroll 1 /* tell compiler not to unroll */
    for (uint k = groupId; k < (K / QK8_0); k += N_SIMDGROUP) {
-        regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of one rows
+        regS = src0_d[gid_s + k * LINE_STRIDE_A]; // each fiber loads the scale of one row
        // first 4 fibers in each wave load 8 B values to its private scope
        if (slid < 4) {
            regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));
@@ -246,14 +250,14 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32(
        }

        // load weights for one block in consecutive rows
-        regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
-        regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
-        regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
-        regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
-        regA.s4 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
-        regA.s5 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
-        regA.s6 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
-        regA.s7 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+        regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+        regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+        regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+        regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+        regA.s4 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+        regA.s5 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+        regA.s6 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+        regA.s7 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;

        dequantizeBlockAccum_ns_sgbroadcast_1(totalSum, regA, convert_float(regS), regB);
    }
@@ -75,6 +75,15 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) {
    const float alpha = reinterpret_cast<const float *>(params)[2];
    const float limit = reinterpret_cast<const float *>(params)[3];

+    // Compute in f32: alpha_const/one are f32, and Swish/Add need matching input types.
+    auto output_type = context.get_output_type();
+    if (src0.get_element_type() != ov::element::f32) {
+        src0 = std::make_shared<ov::op::v0::Convert>(src0, ov::element::f32);
+    }
+    if (src1.get_element_type() != ov::element::f32) {
+        src1 = std::make_shared<ov::op::v0::Convert>(src1, ov::element::f32);
+    }
+
    auto gate = std::make_shared<ov::op::v0::Clamp>(src0, -std::numeric_limits<float>::infinity(), limit);
    auto alpha_const = ov::op::v0::Constant::create(ov::element::f32, {}, {alpha});
    auto out_glu = std::make_shared<ov::op::v4::Swish>(gate, alpha_const);
@@ -82,7 +91,10 @@ OutputVector translate_glu_swiglu_oai(const NodeContext & context) {
    auto up = std::make_shared<ov::op::v0::Clamp>(src1, -limit, limit);
    auto one = ov::op::v0::Constant::create(ov::element::f32, {}, {1.0f});
    auto up_plus_one = std::make_shared<ov::op::v1::Add>(up, one);
-    auto res = std::make_shared<ov::op::v1::Multiply>(out_glu, up_plus_one);
+    ov::Output<ov::Node> res = std::make_shared<ov::op::v1::Multiply>(out_glu, up_plus_one);
+    if (res.get_element_type() != output_type) {
+        res = std::make_shared<ov::op::v0::Convert>(res, output_type);
+    }

    return rename_outputs_with_suffix({res}, context.get_name());
 }
@@ -131,33 +131,65 @@ endif()
 # Link against oneDNN
 set(GGML_SYCL_DNNL 0)
 if(GGML_SYCL_DNN)
+    # oneAPI installs dnnl-config.cmake under a versioned path CMake does not
+    # search by default. When the user did not point CMake at it explicitly
+    # (DNNL_DIR), auto-detect the oneAPI-bundled GPU build via the DNNLROOT
+    # environment variable or the standard oneAPI install prefixes. A shell
+    # with setvars.sh sourced already exports CMAKE_PREFIX_PATH and finds it
+    # without any of this.
+    if(NOT DEFINED DNNL_DIR)
+        if(DEFINED ENV{DNNLROOT} AND EXISTS "$ENV{DNNLROOT}/lib/cmake/dnnl/dnnl-config.cmake")
+            set(DNNL_DIR "$ENV{DNNLROOT}/lib/cmake/dnnl")
+        elseif(EXISTS "/opt/intel/oneapi/dnnl/latest/lib/cmake/dnnl/dnnl-config.cmake")
+            set(DNNL_DIR "/opt/intel/oneapi/dnnl/latest/lib/cmake/dnnl")
+        elseif(EXISTS "/usr/intel/oneapi/dnnl/latest/lib/cmake/dnnl/dnnl-config.cmake")
+            set(DNNL_DIR "/usr/intel/oneapi/dnnl/latest/lib/cmake/dnnl")
+        endif()
+        if(DEFINED DNNL_DIR)
+            message(STATUS "oneDNN: auto-detected config at ${DNNL_DIR}")
+        endif()
+    endif()
    find_package(DNNL)
-    if(DNNL_FOUND)
-        if (NOT DEFINED DNNL_GPU_VENDOR)
-            # default to intel target
-            set(DNNL_GPU_VENDOR "INTEL")
-            if(NOT "${GGML_SYCL_TARGET}" STREQUAL "INTEL")
-                message(WARNING "oneDNN builds bundled with oneapi release only support INTEL target")
-            endif()
-        endif()
+    if(NOT DNNL_FOUND)
+        # GGML_SYCL_DNN defaults to ON, so a plain build used to continue
+        # silently without oneDNN: every flash-attention then runs the much
+        # slower MKL-FA path (20-40% prefill loss on long prompts). Fail
+        # loudly instead of producing a degraded binary.
+        message(FATAL_ERROR
+            "oneDNN (DNNL) was requested (GGML_SYCL_DNN=ON) but not found. "
+            "The SYCL backend needs the oneAPI-bundled GPU build: install "
+            "intel-oneapi-dnnl and intel-oneapi-dnnl-devel from the oneAPI "
+            "repo, source setvars.sh, or pass "
+            "-DDNNL_DIR=<dnnl-prefix>/lib/cmake/dnnl (or export DNNLROOT). "
+            "To build without oneDNN on purpose, pass -DGGML_SYCL_DNN=OFF.")
+    endif()
+    if(NOT DEFINED DNNL_GPU_VENDOR)
+        # A DNNL package that does not declare DNNL_GPU_VENDOR has no GPU/SYCL
+        # support compiled in (e.g. Debian/Ubuntu's CPU-only libdnnl-dev).
+        # The old code defaulted the vendor to INTEL here, which fake-passed
+        # the target check below and silently linked a CPU-only library into
+        # the SYCL backend.
+        message(FATAL_ERROR
+            "Found oneDNN at ${DNNL_DIR} but it does not declare "
+            "DNNL_GPU_VENDOR - it is a CPU-only build (Debian/Ubuntu "
+            "libdnnl-dev?). Install the oneAPI-bundled GPU build "
+            "(intel-oneapi-dnnl-devel) instead.")
+    endif()

-        # Verify oneDNN was compiled for the same target as llama
-        if("${GGML_SYCL_TARGET}" STREQUAL "${DNNL_GPU_VENDOR}")
-            target_link_libraries(ggml-sycl PRIVATE DNNL::dnnl)
-            set(GGML_SYCL_DNNL 1)
-            get_target_property(CONFIGS DNNL::dnnl IMPORTED_CONFIGURATIONS)
-            foreach(CONFIG ${CONFIGS})
-                get_target_property(DNNL_LIB DNNL::dnnl IMPORTED_LOCATION_${CONFIG})
-                message(STATUS "Found oneDNN: ${DNNL_LIB}")
-            endforeach()
-        else()
-            message(WARNING
-                "oneDNN must be compiled for the same target as llama.cpp.
-                 llama.cpp: ${GGML_SYCL_TARGET}, oneDNN: ${DNNL_GPU_VENDOR}.
-                 Disabling oneDNN support.")
-        endif()
+    # Verify oneDNN was compiled for the same target as llama
+    if("${GGML_SYCL_TARGET}" STREQUAL "${DNNL_GPU_VENDOR}")
+        target_link_libraries(ggml-sycl PRIVATE DNNL::dnnl)
+        set(GGML_SYCL_DNNL 1)
+        get_target_property(CONFIGS DNNL::dnnl IMPORTED_CONFIGURATIONS)
+        foreach(CONFIG ${CONFIGS})
+            get_target_property(DNNL_LIB DNNL::dnnl IMPORTED_LOCATION_${CONFIG})
+            message(STATUS "Found oneDNN: ${DNNL_LIB}")
+        endforeach()
    else()
-        message(STATUS "oneDNN not found, disabling oneDNN support")
+        message(FATAL_ERROR
+            "oneDNN must be compiled for the same target as llama.cpp: "
+            "llama.cpp: ${GGML_SYCL_TARGET}, oneDNN: ${DNNL_GPU_VENDOR}. "
+            "Pass -DGGML_SYCL_DNN=OFF to build without oneDNN.")
    endif()
 else()
    message(STATUS "oneDNN support disabled by the user")
@@ -463,6 +463,19 @@ static void dequantize_row_iq3_xxs_sycl(const void *vx, dst_t *y, const int64_t
    }
 }

+template <typename dst_t>
+static void dequantize_row_iq3_xxs_sycl_reorder(const void * vx, dst_t * y, const int64_t k,
+                                                dpct::queue_ptr stream) {
+    const int64_t nb = k / QK_K;
+    dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 });
+    stream->parallel_for(
+        sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 32), sycl::range<3>(1, 1, 32)),
+        [=](sycl::nd_item<3> item_ct1) {
+            dequantize_block_iq3_xxs_reorder(
+                vx, y, item_ct1, nb, iq3xxs_grid, ksigns_iq2xs, kmask_iq2xs);
+        });
+}
+
 template <typename dst_t>
 static void dequantize_row_iq3_s_sycl(const void *vx, dst_t *y, const int64_t k,
                                        dpct::queue_ptr stream) {
@@ -483,6 +496,18 @@ static void dequantize_row_iq3_s_sycl(const void *vx, dst_t *y, const int64_t k,
    }
 }

+template <typename dst_t>
+static void dequantize_row_iq3_s_sycl_reorder(const void * vx, dst_t * y, const int64_t k,
+                                              dpct::queue_ptr stream) {
+    const int64_t nb = k / QK_K;
+    dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 });
+    stream->parallel_for(
+        sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 32), sycl::range<3>(1, 1, 32)),
+        [=](sycl::nd_item<3> item_ct1) {
+            dequantize_block_iq3_s_reorder(vx, y, item_ct1, nb, kmask_iq2xs, iq3s_grid);
+        });
+}
+
 template <typename dst_t>
 static void dequantize_row_iq4_xs_sycl(const void *vx, dst_t *y, const int64_t k,
                                       dpct::queue_ptr stream) {
@@ -732,9 +757,19 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) {
        case GGML_TYPE_IQ2_S:
            return dequantize_row_iq2_s_sycl;
        case GGML_TYPE_IQ3_XXS:
-            return dequantize_row_iq3_xxs_sycl;
+            if (dst->src[0]->extra &&
+                ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+                return dequantize_row_iq3_xxs_sycl_reorder;
+            } else {
+                return dequantize_row_iq3_xxs_sycl;
+            }
        case GGML_TYPE_IQ3_S:
-            return dequantize_row_iq3_s_sycl;
+            if (dst->src[0]->extra &&
+                ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+                return dequantize_row_iq3_s_sycl_reorder;
+            } else {
+                return dequantize_row_iq3_s_sycl;
+            }
        case GGML_TYPE_IQ4_XS:
            return dequantize_row_iq4_xs_sycl;
        case GGML_TYPE_IQ4_NL:
@@ -826,9 +861,19 @@ to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) {
        case GGML_TYPE_IQ2_S:
            return dequantize_row_iq2_s_sycl;
        case GGML_TYPE_IQ3_XXS:
-            return dequantize_row_iq3_xxs_sycl;
+            if (dst->src[0]->extra &&
+                ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+                return dequantize_row_iq3_xxs_sycl_reorder;
+            } else {
+                return dequantize_row_iq3_xxs_sycl;
+            }
        case GGML_TYPE_IQ3_S:
-            return dequantize_row_iq3_s_sycl;
+            if (dst->src[0]->extra &&
+                ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+                return dequantize_row_iq3_s_sycl_reorder;
+            } else {
+                return dequantize_row_iq3_s_sycl;
+            }
        case GGML_TYPE_IQ4_XS:
            return dequantize_row_iq4_xs_sycl;
        case GGML_TYPE_IQ4_NL:
@@ -1490,6 +1490,45 @@ static void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __res

 }

+template<typename dst_t>
+static void dequantize_block_iq3_xxs_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy,
+                                             const sycl::nd_item<3> & item_ct1, const int64_t nb,
+                                             const uint32_t * iq3xxs_grid,
+                                             const uint8_t * ksigns_iq2xs,
+                                             const uint8_t * kmask_iq2xs) {
+    const int64_t i   = item_ct1.get_group(2);
+    const int64_t tid = item_ct1.get_local_id(2);
+#if QK_K == 256
+    const int64_t il = tid / 8;
+    const int64_t ib = tid % 8;
+    const uint8_t * base = static_cast<const uint8_t *>(vx);
+    const uint8_t * qs = base + i * (3 * QK_K / 8);
+    const ggml_half * d = reinterpret_cast<const ggml_half *>(base + nb * (3 * QK_K / 8));
+    const uint8_t * q3 = qs + 8 * ib;
+    const uint16_t * gas = reinterpret_cast<const uint16_t *>(qs + QK_K / 4) + 2 * ib;
+    const uint8_t * grid1 = reinterpret_cast<const uint8_t *>(iq3xxs_grid + q3[2*il+0]);
+    const uint8_t * grid2 = reinterpret_cast<const uint8_t *>(iq3xxs_grid + q3[2*il+1]);
+    const uint32_t aux32 = gas[0] | (gas[1] << 16);
+    const float dl = (float) d[i] * (0.5f + (aux32 >> 28)) * 0.5f;
+    const uint8_t signs = ksigns_iq2xs[(aux32 >> (7 * il)) & 127];
+    dst_t * y = yy + i * QK_K + 32 * ib + 8 * il;
+#pragma unroll
+    for (int j = 0; j < 4; ++j) {
+        y[j+0] = dl * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);
+        y[j+4] = dl * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);
+    }
+#else
+    GGML_UNUSED(vx);
+    GGML_UNUSED(yy);
+    GGML_UNUSED(item_ct1);
+    GGML_UNUSED(nb);
+    GGML_UNUSED(iq3xxs_grid);
+    GGML_UNUSED(ksigns_iq2xs);
+    GGML_UNUSED(kmask_iq2xs);
+    GGML_ABORT("IQ3_XXS reorder dequantization requires QK_K == 256");
+#endif
+}
+
 template <typename dst_t>
 __dpct_inline__ static void
 dequantize_block_iq3_s(const void *__restrict__ vx, dst_t *__restrict__ yy,
@@ -1520,6 +1559,48 @@ dequantize_block_iq3_s(const void *__restrict__ vx, dst_t *__restrict__ yy,

 }

+template <typename dst_t>
+__dpct_inline__ static void
+dequantize_block_iq3_s_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy,
+                               const sycl::nd_item<3> & item_ct1, const int64_t nb,
+                               const uint8_t * kmask_iq2xs, const uint32_t * iq3s_grid) {
+    const int64_t i   = item_ct1.get_group(2);
+    const int64_t tid = item_ct1.get_local_id(2);
+#if QK_K == 256
+    const int64_t il = tid / 8;
+    const int64_t ib = tid % 8;
+    const uint8_t * base = static_cast<const uint8_t *>(vx);
+    const uint8_t * qs = base + i * (QK_K / 4);
+    const uint8_t * qh = base + nb * (QK_K / 4) + i * (QK_K / 32);
+    const uint8_t * signs_base = base + nb * (QK_K / 4 + QK_K / 32) + i * (QK_K / 8);
+    const uint8_t * metadata = base + nb * (QK_K / 4 + QK_K / 32 + QK_K / 8) +
+                               i * (sizeof(ggml_half) + IQ3S_N_SCALE);
+    const ggml_half d = *reinterpret_cast<const ggml_half *>(metadata);
+    const uint8_t * scales = metadata + sizeof(ggml_half);
+    const uint8_t * qs_ib = qs + 8 * ib;
+    const uint8_t * grid1 = reinterpret_cast<const uint8_t *>(
+        iq3s_grid + (qs_ib[2*il+0] | ((qh[ib] << (8 - 2*il)) & 256)));
+    const uint8_t * grid2 = reinterpret_cast<const uint8_t *>(
+        iq3s_grid + (qs_ib[2*il+1] | ((qh[ib] << (7 - 2*il)) & 256)));
+    const float dl = (float) d * (1 + 2 * ((scales[ib/2] >> (4 * (ib%2))) & 0xf));
+    const uint8_t signs = signs_base[4 * ib + il];
+    dst_t * y = yy + i * QK_K + 32 * ib + 8 * il;
+#pragma unroll
+    for (int j = 0; j < 4; ++j) {
+        y[j+0] = dl * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);
+        y[j+4] = dl * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);
+    }
+#else
+    GGML_UNUSED(vx);
+    GGML_UNUSED(yy);
+    GGML_UNUSED(item_ct1);
+    GGML_UNUSED(nb);
+    GGML_UNUSED(kmask_iq2xs);
+    GGML_UNUSED(iq3s_grid);
+    GGML_ABORT("IQ3_S reorder dequantization requires QK_K == 256");
+#endif
+}
+
 template <typename dst_t>
 __dpct_inline__ static void
 dequantize_block_iq1_s(const void *__restrict__ vx, dst_t *__restrict__ yy,
@@ -774,6 +774,8 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer,
            case GGML_TYPE_Q4_K:
            case GGML_TYPE_Q5_K:
            case GGML_TYPE_Q6_K:
+            case GGML_TYPE_IQ3_XXS:
+            case GGML_TYPE_IQ3_S:
            case GGML_TYPE_MXFP4:{
                ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};
                tensor->extra                 = extra;
@@ -4242,6 +4244,8 @@ inline bool ggml_sycl_supports_reorder_mmvq(enum ggml_type type) {
        case GGML_TYPE_Q4_K:
        case GGML_TYPE_Q5_K:
        case GGML_TYPE_Q6_K:
+        case GGML_TYPE_IQ3_XXS:
+        case GGML_TYPE_IQ3_S:
            return true;
        default:
            return false;
@@ -4438,6 +4442,47 @@ static bool reorder_qw_q8_0(uint8_t * data_device, const int ncols, const int nr
    return true;
 }

+// Reorder each Q8_0 expert slice into a self-contained [qs][d] SoA layout.
+static bool reorder_qw_q8_0_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) {
+    GGML_ASSERT(expert_bytes % sizeof(block_q8_0) == 0);
+    const int    blocks_per_expert = (int) (expert_bytes / sizeof(block_q8_0));
+    const size_t total_bytes       = expert_bytes * (size_t) n_expert;
+
+    sycl_reorder_temp_buffer tmp(stream, total_bytes);
+    if (!tmp) {
+        GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, total_bytes);
+        return false;
+    }
+    uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr);
+
+    sycl::event copy_event;
+    SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, total_bytes)));
+    if (!g_ggml_sycl_use_async_mem_op) {
+        copy_event.wait();
+    }
+
+    const int total_blocks = blocks_per_expert * (int) n_expert;
+    auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) {
+        const int          gb   = gb_;
+        const int          e    = gb / blocks_per_expert;
+        const int          ib   = gb % blocks_per_expert;
+        const block_q8_0 * x    = (const block_q8_0 *) (tmp_buf + (size_t) e * expert_bytes);
+        uint8_t *          base = data_device + (size_t) e * expert_bytes;
+
+        auto * qs_ptr = base;
+        auto * d_ptr  = (sycl::half *) (qs_ptr + (size_t) blocks_per_expert * QK8_0);
+
+        for (int j = 0; j < QK8_0; ++j) {
+            *((int8_t *) qs_ptr + ib * QK8_0 + j) = x[ib].qs[j];
+        }
+        d_ptr[ib] = x[ib].d;
+    });
+    if (!g_ggml_sycl_use_async_mem_op) {
+        reorder_event.wait_and_throw();
+    }
+    return true;
+}
+
 static bool reorder_qw_q4_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) {
    GGML_ASSERT(size % sizeof(block_q4_K) == 0);
    GGML_ASSERT(offset % sizeof(block_q4_K) == 0);
@@ -4868,6 +4913,89 @@ static bool reorder_qw_q6_k(uint8_t * data_device, size_t size, size_t offset, d
    return true;
 }

+static bool reorder_qw_iq3_s(uint8_t * data_device, size_t size, dpct::queue_ptr stream) {
+    GGML_ASSERT(size % sizeof(block_iq3_s) == 0);
+
+    const int nblocks = size / sizeof(block_iq3_s);
+    sycl_reorder_temp_buffer tmp(stream, size);
+    if (!tmp) {
+        GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size);
+        return false;
+    }
+    uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr);
+
+    sycl::event copy_event;
+    SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size)));
+    if (!g_ggml_sycl_use_async_mem_op) {
+        copy_event.wait();
+    }
+
+    auto * qs_ptr       = data_device;
+    auto * qh_ptr       = qs_ptr + (QK_K / 4) * nblocks;
+    auto * signs_ptr    = qh_ptr + (QK_K / 32) * nblocks;
+    auto * metadata_ptr = signs_ptr + (QK_K / 8) * nblocks;
+
+    auto reorder_event = stream->parallel_for(nblocks, [=](auto i) {
+        const block_iq3_s * x = (const block_iq3_s *) tmp_buf;
+        const int ib = i;
+
+        for (int j = 0; j < QK_K / 4; ++j) {
+            qs_ptr[ib * (QK_K / 4) + j] = x[ib].qs[j];
+        }
+        for (int j = 0; j < QK_K / 32; ++j) {
+            qh_ptr[ib * (QK_K / 32) + j] = x[ib].qh[j];
+        }
+        for (int j = 0; j < QK_K / 8; ++j) {
+            signs_ptr[ib * (QK_K / 8) + j] = x[ib].signs[j];
+        }
+
+        uint8_t * metadata = metadata_ptr + ib * (sizeof(ggml_half) + IQ3S_N_SCALE);
+        *reinterpret_cast<ggml_half *>(metadata) = x[ib].d;
+        for (int j = 0; j < IQ3S_N_SCALE; ++j) {
+            metadata[sizeof(ggml_half) + j] = x[ib].scales[j];
+        }
+    });
+    if (!g_ggml_sycl_use_async_mem_op) {
+        reorder_event.wait_and_throw();
+    }
+    return true;
+}
+
+static bool reorder_qw_iq3_xxs(uint8_t * data_device, size_t size, dpct::queue_ptr stream) {
+    GGML_ASSERT(size % sizeof(block_iq3_xxs) == 0);
+
+    const int nblocks = size / sizeof(block_iq3_xxs);
+    sycl_reorder_temp_buffer tmp(stream, size);
+    if (!tmp) {
+        GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size);
+        return false;
+    }
+    uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr);
+
+    sycl::event copy_event;
+    SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size)));
+    if (!g_ggml_sycl_use_async_mem_op) {
+        copy_event.wait();
+    }
+
+    auto * qs_ptr = data_device;
+    auto * d_ptr  = reinterpret_cast<ggml_half *>(qs_ptr + (3 * QK_K / 8) * nblocks);
+
+    auto reorder_event = stream->parallel_for(nblocks, [=](auto i) {
+        const block_iq3_xxs * x = (const block_iq3_xxs *) tmp_buf;
+        const int ib = i;
+
+        for (int j = 0; j < 3 * QK_K / 8; ++j) {
+            qs_ptr[ib * (3 * QK_K / 8) + j] = x[ib].qs[j];
+        }
+        d_ptr[ib] = x[ib].d;
+    });
+    if (!g_ggml_sycl_use_async_mem_op) {
+        reorder_event.wait_and_throw();
+    }
+    return true;
+}
+
 static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) {
    uint8_t * data_device = (uint8_t *) src0->data;
    size_t ncols = src0->ne[0];
@@ -4879,6 +5007,8 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) {
    if (src0->ne[2] > 1) {
        GGML_ASSERT((size_t) size == (size_t) src0->ne[2] * src0->nb[2]);
        switch (src0->type) {
+            case GGML_TYPE_Q8_0:
+                return reorder_qw_q8_0_moe(data_device, src0->nb[2], src0->ne[2], stream);
            case GGML_TYPE_Q4_K:
                return reorder_qw_q4_k_moe(data_device, src0->nb[2], src0->ne[2], stream);
            case GGML_TYPE_Q5_K:
@@ -4907,6 +5037,10 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) {
            return reorder_qw_q5_k(data_device, size, 0, stream);
        case GGML_TYPE_Q6_K:
            return reorder_qw_q6_k(data_device, size, 0, stream);
+        case GGML_TYPE_IQ3_XXS:
+            return reorder_qw_iq3_xxs(data_device, size, stream);
+        case GGML_TYPE_IQ3_S:
+            return reorder_qw_iq3_s(data_device, size, stream);
        default:
            return false;
    }
@@ -4960,8 +5094,8 @@ static void opt_for_reorder_id(ggml_backend_sycl_context * ctx, const ggml_tenso
    if (!g_ggml_sycl_enable_optimize || !ctx->opt_feature.reorder) {
        return;
    }
-    if (src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K &&
-        src0->type != GGML_TYPE_MXFP4) {
+    if (src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_Q4_K &&
+        src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_MXFP4) {
        return;
    }
    // The MXFP4 reorder kernels use 8-byte vector loads, so every expert slice must stay aligned.
@@ -5347,7 +5481,7 @@ static bool ggml_sycl_mul_mat_id_mmvq_fused(
    const int       n_experts_used   = (int) n_ids_per_group;
    const int       nrows            = (int) src0->ne[1];

-    // Lazily reorder the (Q4_K) expert weights into a per-expert SoA layout, then run the reorder
+    // Lazily reorder supported expert weights into a per-expert SoA layout, then run the reorder
    // GEMV. Placed after the bail checks so a non-dispatchable op does not pay the reorder cost.
    opt_for_reorder_id(&ctx, src0);
    const ggml_tensor_extra_gpu * src0_extra =
@@ -18,13 +18,13 @@ static __dpct_inline__ float vec_dot_iq2_xxs_q8_1_moe(const void * __restrict__
 static __dpct_inline__ float vec_dot_iq2_xs_q8_1_moe(const void * __restrict__ vbq,
                                                     const block_q8_1 * __restrict__ bq8_1,
                                                     const int & iqs) {
-    return vec_dot_iq2_xs_q8_1(vbq, bq8_1, iqs, iq2xs_grid, ksigns64);
+    return vec_dot_iq2_xs_q8_1(vbq, bq8_1, iqs, iq2xs_grid);
 }

 static __dpct_inline__ float vec_dot_iq3_xxs_q8_1_moe(const void * __restrict__ vbq,
                                                      const block_q8_1 * __restrict__ bq8_1,
                                                      const int & iqs) {
-    return vec_dot_iq3_xxs_q8_1(vbq, bq8_1, iqs, iq3xxs_grid, ksigns64);
+    return vec_dot_iq3_xxs_q8_1(vbq, bq8_1, iqs, iq3xxs_grid);
 }

 static __dpct_inline__ float vec_dot_iq3_s_q8_1_moe(const void * __restrict__ vbq,
@@ -447,7 +447,7 @@ static void mul_mat_vec_q_iq2_xs_q8_1(const void *__restrict__ vx,
            (item_ct1.get_local_id(2) %
             (qi / vdr)); // x block quant index when casting the quants to int

-        tmp += vec_dot_iq2_xs_q8_1(&x[ibx], &y[iby], iqs, iq2xs_grid, ksigns64);
+        tmp += vec_dot_iq2_xs_q8_1(&x[ibx], &y[iby], iqs, iq2xs_grid);
    }

    // sum up partial sums and write back result
@@ -543,7 +543,7 @@ static void mul_mat_vec_q_iq3_xxs_q8_1(const void *__restrict__ vx,
            (item_ct1.get_local_id(2) %
             (qi / vdr)); // x block quant index when casting the quants to int

-        tmp += vec_dot_iq3_xxs_q8_1(&x[ibx], &y[iby], iqs, iq3xxs_grid, ksigns64);
+        tmp += vec_dot_iq3_xxs_q8_1(&x[ibx], &y[iby], iqs, iq3xxs_grid);
    }

    // sum up partial sums and write back result
@@ -558,6 +558,11 @@ static void mul_mat_vec_q_iq3_xxs_q8_1(const void *__restrict__ vx,
    }
 }

+static __dpct_inline__ float vec_dot_iq3_s_q8_1_adapter(
+        const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
+    return vec_dot_iq3_s_q8_1(vbq, bq8_1, iqs, iq3s_grid);
+}
+
 template <int qk, int qi, typename block_q_t, int vdr>
 static void mul_mat_vec_q_iq3_s_q8_1(const void *__restrict__ vx,
                                     const void *__restrict__ vy,
@@ -2276,6 +2281,67 @@ static void mul_mat_vec_iq3_xxs_q8_1_sycl(const void *vx, const void *vy,
    }
 }

+template <ggml_type type>
+static void reorder_mul_mat_vec_iq3_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
+                                              const int nrows, dpct::queue_ptr stream) {
+    static_assert(type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S);
+    GGML_ASSERT(ncols % QK_K == 0);
+    constexpr size_t num_subgroups = WARP_SIZE;
+    const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
+
+    stream->submit([&](sycl::handler & cgh) {
+        cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
+                         [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                             mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<type>>(
+                                 vx, vy, dst, ncols, nrows, nd_item);
+                         });
+    });
+}
+
+template <ggml_type type, int ncols_dst>
+static void reorder_mul_mat_vec_iq3_q8_1_sycl_ncols(
+        const void * vx, const void * vy, float * dst,
+        const int ncols, const int nrows,
+        const int stride_col_y_bytes, const int stride_col_dst,
+        dpct::queue_ptr stream) {
+    static_assert(type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S);
+    GGML_ASSERT(ncols % QK_K == 0);
+    constexpr size_t num_subgroups = WARP_SIZE;
+    const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
+
+    stream->submit([&](sycl::handler & cgh) {
+        cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
+                         [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                             mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<type>, ncols_dst>(
+                                 vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
+                                 /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
+                         });
+    });
+}
+
+template <ggml_type type>
+static void reorder_mul_mat_vec_iq3_q8_1_sycl_switch_ncols(
+        const void * vx, const void * vy, float * dst,
+        const int ncols, const int nrows, const int ncols_dst,
+        const int stride_col_y_bytes, const int stride_col_dst,
+        dpct::queue_ptr stream) {
+    switch (ncols_dst) {
+        case 1: reorder_mul_mat_vec_iq3_q8_1_sycl<type>(vx, vy, dst, ncols, nrows, stream); break;
+        case 2: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+        case 3: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+        case 4: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+        case 5: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+        case 6: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+        case 7: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+        case 8: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols<type, 8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
+        default: GGML_ABORT("unsupported ncols_dst=%d for reordered IQ3 multi-col MMVQ", ncols_dst);
+    }
+}
+
 static void mul_mat_vec_iq3_s_q8_1_sycl(const void *vx, const void *vy,
                                          float *dst, const int ncols,
                                          const int nrows,
@@ -2793,15 +2859,49 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens
                mul_mat_vec_iq2_s_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
                break;
            case GGML_TYPE_IQ3_XXS:
-                mul_mat_vec_iq3_xxs_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
+                if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
+                    ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+                    if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
+                        const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
+                        const int stride_col_dst     = dst->ne[0];
+                        GGML_SYCL_DEBUG("Calling reordered IQ3_XXS multi-col MMVQ ncols=%d\n", (int) src1_ncols);
+                        reorder_mul_mat_vec_iq3_q8_1_sycl_switch_ncols<GGML_TYPE_IQ3_XXS>(
+                            src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
+                            src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
+                        return;
+                    } else {
+                        GGML_SYCL_DEBUG("Calling reordered IQ3_XXS MMVQ\n");
+                        reorder_mul_mat_vec_iq3_q8_1_sycl<GGML_TYPE_IQ3_XXS>(
+                            src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
+                    }
+                } else {
+                    mul_mat_vec_iq3_xxs_q8_1_sycl(
+                        src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
+                }
                break;
            case GGML_TYPE_IQ3_S:
-                if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
+                if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
+                    ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
+                    if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
+                        const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
+                        const int stride_col_dst     = dst->ne[0];
+                        GGML_SYCL_DEBUG("Calling reordered IQ3_S multi-col MMVQ ncols=%d\n", (int) src1_ncols);
+                        reorder_mul_mat_vec_iq3_q8_1_sycl_switch_ncols<GGML_TYPE_IQ3_S>(
+                            src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
+                            src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
+                        return;
+                    } else {
+                        GGML_SYCL_DEBUG("Calling reordered IQ3_S MMVQ\n");
+                        reorder_mul_mat_vec_iq3_q8_1_sycl<GGML_TYPE_IQ3_S>(
+                            src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
+                    }
+                } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
                    const int stride_col_y   = src1_padded_col_size / QK8_1;
                    const int stride_col_dst = dst->ne[0];
                    GGML_SYCL_DEBUG("Calling mul_mat_vec_iq3_s_q8_1_sycl_switch_ncols ncols=%d\n", (int) src1_ncols);
-                    mul_mat_vec_iq3_s_q8_1_sycl_switch_ncols(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
-                                                             src1_ncols, stride_col_y, stride_col_dst, stream);
+                    mul_mat_vec_iq3_s_q8_1_sycl_switch_ncols(
+                        src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
+                        src1_ncols, stride_col_y, stride_col_dst, stream);
                    return;
                } else if (i == 0 || src1_ncols == 1) {
                    mul_mat_vec_iq3_s_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
@@ -3045,7 +3145,7 @@ bool ggml_sycl_mul_mat_vec_q_id(
                expert_weight_stride, dst_row_stride, src1_row_stride, stream);
            return true;
        case GGML_TYPE_IQ3_S:
-            launch_mul_mat_vec_q_moe<QK_K, QI3_S/2, block_iq3_s, VDR_IQ3_S_Q8_1_MMVQ, vec_dot_iq3_s_q8_1_moe>(
+            launch_mul_mat_vec_q_moe<QK_K, QI3_S/2, block_iq3_s, VDR_IQ3_S_Q8_1_MMVQ, vec_dot_iq3_s_q8_1_adapter>(
                vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
                expert_weight_stride, dst_row_stride, src1_row_stride, stream);
            return true;
@@ -3102,9 +3202,11 @@ static void mul_mat_vec_q_moe_reorder(

    const auto sg = item_ct1.get_sub_group();

+    constexpr int vdr_mmvq = reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0 ?
+                            VDR_Q8_0_Q8_1_MMVQ : block_traits::vdr_mmvq;
    const int     blocks_per_row              = ncols / block_traits::qk;
-    constexpr int blocks_per_subgroup         = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi);
-    constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq;
+    constexpr int blocks_per_subgroup         = ceil_div(vdr_mmvq * WARP_SIZE, block_traits::qi);
+    constexpr int block_elements_per_subgroup = block_traits::qi / vdr_mmvq;
    const int     nblocks                     = nrows * (ncols / block_traits::qk);

    static_assert(blocks_per_subgroup > 0);
@@ -3123,12 +3225,24 @@ static void mul_mat_vec_q_moe_reorder(

 #pragma unroll
        for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) {
-            const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);
-            partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
+            const int iqs = elem + vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);
+            if constexpr (reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0) {
+                partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs, vdr_mmvq);
+            } else {
+                partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
+            }
        }
    }

-    auto sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>());
+    float sum = partial_sum;
+    if constexpr (reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0) {
+        // Keep the accumulation order of the non-reordered MoE kernel.
+        for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
+            sum += dpct::permute_sub_group_by_xor(sg, sum, mask);
+        }
+    } else {
+        sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>());
+    }
    if (sg.leader()) {
        dst[row] = sum;
    }
@@ -3169,6 +3283,11 @@ bool ggml_sycl_mul_mat_vec_q_id_reorder(
    size_t             src1_row_stride,
    dpct::queue_ptr    stream) {
    switch (src0_type) {
+        case GGML_TYPE_Q8_0:
+            launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0>>(
+                vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
+                expert_weight_stride, dst_row_stride, src1_row_stride, stream);
+            return true;
        case GGML_TYPE_Q4_K:
            launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>>(
                vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
@@ -178,6 +178,53 @@ template <> struct block_q_t<GGML_TYPE_Q6_K> {
    static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
 };

+template <> struct block_q_t<GGML_TYPE_IQ3_XXS> {
+    struct traits {
+        static constexpr uint32_t qk       = QK_K;
+        static constexpr uint32_t qi       = QI3_XXS / 2;
+        static constexpr uint32_t qr       = QR3_XXS;
+        static constexpr uint32_t vdr_mmvq = 1;
+    };
+
+    // Reordered layout: [96-byte qs records for all blocks][2-byte d records for all blocks].
+    static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) {
+        return { block_index * (3 * QK_K / 8), 0 };
+    }
+
+    static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
+        const int nblocks = nrows * (ncols / QK_K);
+        return { nblocks * (3 * QK_K / 8) + block_index * sizeof(ggml_half), 0 };
+    }
+
+    static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
+};
+
+template <> struct block_q_t<GGML_TYPE_IQ3_S> {
+    struct traits {
+        static constexpr uint32_t qk       = QK_K;
+        static constexpr uint32_t qi       = QI3_S / 2;
+        static constexpr uint32_t qr       = QR3_S;
+        static constexpr uint32_t vdr_mmvq = 1;
+    };
+
+    // Reordered layout: [qs][qh][signs][{d, scales} per block]. Keeping the small
+    // metadata together leaves the generic reordered dot interface at two offset pairs.
+    static constexpr std::pair<int, int> get_block_offset(const int block_index, const int nblocks) {
+        return { block_index * (QK_K / 4),
+                 nblocks * (QK_K / 4) + block_index * (QK_K / 32) };
+    }
+
+    static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
+        const int nblocks = nrows * (ncols / QK_K);
+        const int signs_base = nblocks * (QK_K / 4 + QK_K / 32);
+        const int metadata_base = signs_base + nblocks * (QK_K / 8);
+        return { signs_base + block_index * (QK_K / 8),
+                 metadata_base + block_index * (sizeof(ggml_half) + IQ3S_N_SCALE) };
+    }
+
+    static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
+};
+
 template <> struct block_q_t<GGML_TYPE_Q8_0> {
    struct traits {
        static constexpr uint32_t qk       = QK8_0;      // 32
@@ -483,7 +483,8 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0> {

    __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
                                     const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
-                                     const sycl::half2 * q8_1_ds, const int & iqs) {
+                                     const sycl::half2 * q8_1_ds, const int & iqs,
+                                     const int vdr = q8_0_traits::vdr_mmvq) {
        const uint8_t * base = static_cast<const uint8_t *>(vbq);
        const int8_t *  qs   = reinterpret_cast<const int8_t *>(base + ibx_offset.first);
        const ggml_half  d   = *reinterpret_cast<const ggml_half *>(base + d_offset.first);
@@ -492,14 +493,14 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0> {
        int u[q8_0_traits::vdr_mmvq];

 #pragma unroll
-        for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) {
+        for (int i = 0; i < vdr; ++i) {
            v[i] = get_int_from_int8(qs, iqs + i);
            u[i] = get_int_from_int8_aligned(q8_1_quant_ptr, iqs + i);
        }

        int sumi = 0;
 #pragma unroll
-        for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) {
+        for (int i = 0; i < vdr; ++i) {
            sumi = dpct::dp4a(v[i], u[i], sumi);
        }

@@ -1514,6 +1515,20 @@ vec_dot_q6_K_q8_1(const void *__restrict__ vbq,
 // interchangeable and must not be copied across backends.
 #define VDR_IQ2_XXS_Q8_1_MMVQ 1

+static __dpct_inline__ uint32_t iq_sign_mask(const uint32_t signs) {
+    const uint32_t bits =
+        (signs & 0x01) | ((signs & 0x02) << 7) |
+        ((signs & 0x04) << 14) | ((signs & 0x08) << 21);
+
+    return bits * 0xff;
+}
+
+static __dpct_inline__ uint32_t iq_sign_bits(const uint32_t encoded) {
+    const uint32_t signs = encoded & 127;
+
+    return signs | ((sycl::popcount(signs) & 1) << 7);
+}
+
 static __dpct_inline__ float
 vec_dot_iq2_xxs_q8_1(const void *__restrict__ vbq,
                     const block_q8_1 *__restrict__ bq8_1, const int &iqs,
@@ -1550,7 +1565,7 @@ vec_dot_iq2_xxs_q8_1(const void *__restrict__ vbq,
 static __dpct_inline__ float
 vec_dot_iq2_xs_q8_1(const void *__restrict__ vbq,
                    const block_q8_1 *__restrict__ bq8_1, const int &iqs,
-                    const uint64_t *iq2xs_grid, const uint64_t *ksigns64) {
+                    const uint64_t *iq2xs_grid) {
 #if DPCT_COMPATIBILITY_TEMP >=                                                 \
    MIN_CC_DP4A // lowest compute capability for integer intrinsics
 #if QK_K == 256
@@ -1562,25 +1577,27 @@ vec_dot_iq2_xs_q8_1(const void *__restrict__ vbq,
    const uint8_t ls1 = bq2->scales[ib32] & 0xf;
    const uint8_t ls2 = bq2->scales[ib32] >>  4;
    int sumi1 = 0;
+#pragma unroll
    for (int l = 0; l < 2; ++l) {
-        const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511));
-        const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9));
-        const int grid_l = dpct::vectorized_binary<sycl::uchar4>(
-            grid[0] ^ signs[0], signs[0], std::minus<>());
-        const int grid_h = dpct::vectorized_binary<sycl::uchar4>(
-            grid[1] ^ signs[1], signs[1], std::minus<>());
+        const uint64_t grid = iq2xs_grid[q2[l] & 511];
+        const uint32_t sign_bits = iq_sign_bits(q2[l] >> 9);
+        const uint32_t signs0 = iq_sign_mask(sign_bits);
+        const uint32_t signs1 = iq_sign_mask(sign_bits >> 4);
+        const int grid_l = ((uint32_t)grid ^ signs0) + (signs0 & 0x01010101);
+        const int grid_h = ((uint32_t)(grid >> 32) ^ signs1) + (signs1 & 0x01010101);
        sumi1 = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi1);
        sumi1 = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi1);
        q8 += 8;
    }
    int sumi2 = 0;
+#pragma unroll
    for (int l = 2; l < 4; ++l) {
-        const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511));
-        const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9));
-        const int grid_l = dpct::vectorized_binary<sycl::uchar4>(
-            grid[0] ^ signs[0], signs[0], std::minus<>());
-        const int grid_h = dpct::vectorized_binary<sycl::uchar4>(
-            grid[1] ^ signs[1], signs[1], std::minus<>());
+        const uint64_t grid = iq2xs_grid[q2[l] & 511];
+        const uint32_t sign_bits = iq_sign_bits(q2[l] >> 9);
+        const uint32_t signs0 = iq_sign_mask(sign_bits);
+        const uint32_t signs1 = iq_sign_mask(sign_bits >> 4);
+        const int grid_l = ((uint32_t)grid ^ signs0) + (signs0 & 0x01010101);
+        const int grid_h = ((uint32_t)(grid >> 32) ^ signs1) + (signs1 & 0x01010101);
        sumi2 = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi2);
        sumi2 = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi2);
        q8 += 8;
@@ -1656,7 +1673,7 @@ vec_dot_iq2_s_q8_1(const void *__restrict__ vbq,
 static __dpct_inline__ float
 vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq,
                     const block_q8_1 *__restrict__ bq8_1, const int &iqs,
-                     const uint32_t *iq3xxs_grid, const uint64_t *ksigns64) {
+                     const uint32_t *iq3xxs_grid) {
 #if DPCT_COMPATIBILITY_TEMP >=                                                 \
    MIN_CC_DP4A // lowest compute capability for integer intrinsics
 #if QK_K == 256
@@ -1668,14 +1685,15 @@ vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq,
    const int8_t   * q8 = bq8_1[ib32].qs;
    uint32_t aux32 = gas[0] | (gas[1] << 16);
    int sumi = 0;
+#pragma unroll
    for (int l = 0; l < 4; ++l) {
-        const uint32_t * grid1 = iq3xxs_grid + q3[2*l+0];
-        const uint32_t * grid2 = iq3xxs_grid + q3[2*l+1];
-        const uint32_t * signs = (const uint32_t *)(ksigns64 + (aux32 & 127));
-        const int grid_l = dpct::vectorized_binary<sycl::uchar4>(
-            grid1[0] ^ signs[0], signs[0], std::minus<>());
-        const int grid_h = dpct::vectorized_binary<sycl::uchar4>(
-            grid2[0] ^ signs[1], signs[1], std::minus<>());
+        const uint32_t grid1 = iq3xxs_grid[q3[2*l+0]];
+        const uint32_t grid2 = iq3xxs_grid[q3[2*l+1]];
+        const uint32_t sign_bits = iq_sign_bits(aux32);
+        const uint32_t signs0 = iq_sign_mask(sign_bits);
+        const uint32_t signs1 = iq_sign_mask(sign_bits >> 4);
+        const int grid_l = (grid1 ^ signs0) + (signs0 & 0x01010101);
+        const int grid_h = (grid2 ^ signs1) + (signs1 & 0x01010101);
        sumi = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi);
        sumi = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi);
        q8 += 8;
@@ -1693,6 +1711,50 @@ vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq,
 #endif
 }

+template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_IQ3_XXS> {
+    static constexpr ggml_type gtype = GGML_TYPE_IQ3_XXS;
+
+    __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
+                                     const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
+                                     const sycl::half2 * q8_1_ds, const int & iqs) {
+#if DPCT_COMPATIBILITY_TEMP >= MIN_CC_DP4A && QK_K == 256
+        const uint8_t * base = static_cast<const uint8_t *>(vbq);
+        const uint8_t * qs   = base + ibx_offset.first;
+        const uint8_t * q3   = qs + 8 * iqs;
+        const uint16_t * gas = reinterpret_cast<const uint16_t *>(qs + QK_K / 4) + 2 * iqs;
+        const int8_t * q8    = q8_1_quant_ptr + iqs * QK8_1;
+        uint32_t aux32       = gas[0] | (gas[1] << 16);
+
+        int sumi = 0;
+#pragma unroll
+        for (int l = 0; l < 4; ++l) {
+            const uint32_t grid1 = iq3xxs_grid[q3[2*l+0]];
+            const uint32_t grid2 = iq3xxs_grid[q3[2*l+1]];
+            const uint32_t sign_bits = iq_sign_bits(aux32);
+            const uint32_t signs0 = iq_sign_mask(sign_bits);
+            const uint32_t signs1 = iq_sign_mask(sign_bits >> 4);
+            const int grid_l = (grid1 ^ signs0) + (signs0 & 0x01010101);
+            const int grid_h = (grid2 ^ signs1) + (signs1 & 0x01010101);
+            sumi = dpct::dp4a(grid_l, *((const int *) q8 + 0), sumi);
+            sumi = dpct::dp4a(grid_h, *((const int *) q8 + 1), sumi);
+            q8 += 8;
+            aux32 >>= 7;
+        }
+
+        const ggml_half d = *reinterpret_cast<const ggml_half *>(base + d_offset.first);
+        return (float) d * (0.5f + aux32) * q8_1_ds[iqs][0] * 0.5f * sumi;
+#else
+        GGML_UNUSED(vbq);
+        GGML_UNUSED(ibx_offset);
+        GGML_UNUSED(d_offset);
+        GGML_UNUSED(q8_1_quant_ptr);
+        GGML_UNUSED(q8_1_ds);
+        GGML_UNUSED(iqs);
+        GGML_ABORT("IQ3_XXS reorder vec dot requires dp4a and QK_K == 256");
+#endif
+    }
+};
+
 #define VDR_IQ3_S_Q8_1_MMVQ 1

 static __dpct_inline__ float
@@ -1707,18 +1769,13 @@ vec_dot_iq3_s_q8_1(const void *__restrict__ vbq,
    const int8_t   * q8 = bq8_1[ib32].qs;
    int sumi = 0;
    for (int l = 0; l < 4; ++l) {
-        const uint32_t * grid1 = iq3s_grid + (qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256));
-        const uint32_t * grid2 = iq3s_grid + (qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256));
-        uint32_t signs0 = dpct::vectorized_binary<sycl::uchar4>(
-            ((bq2->signs[4 * ib32 + l] & 0xf) * 0x01010101) & 0x08040201,
-            0x08040201, std::equal_to<>());
-        uint32_t signs1 = dpct::vectorized_binary<sycl::uchar4>(
-            ((bq2->signs[4 * ib32 + l] >> 4) * 0x01010101) & 0x08040201,
-            0x08040201, std::equal_to<>());
-        const int grid_l = dpct::vectorized_binary<sycl::uchar4>(
-            grid1[0] ^ signs0, signs0, std::minus<>());
-        const int grid_h = dpct::vectorized_binary<sycl::uchar4>(
-            grid2[0] ^ signs1, signs1, std::minus<>());
+        const uint32_t grid1 = iq3s_grid[qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256)];
+        const uint32_t grid2 = iq3s_grid[qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256)];
+        const uint32_t signs = bq2->signs[4 * ib32 + l];
+        const uint32_t signs0 = iq_sign_mask(signs);
+        const uint32_t signs1 = iq_sign_mask(signs >> 4);
+        const int grid_l = (grid1 ^ signs0) + (signs0 & 0x01010101);
+        const int grid_h = (grid2 ^ signs1) + (signs1 & 0x01010101);
        sumi = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi);
        sumi = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi);
        q8 += 8;
@@ -1733,6 +1790,50 @@ vec_dot_iq3_s_q8_1(const void *__restrict__ vbq,
 #endif
 }

+template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_IQ3_S> {
+    static constexpr ggml_type gtype = GGML_TYPE_IQ3_S;
+
+    __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
+                                     const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
+                                     const sycl::half2 * q8_1_ds, const int & iqs) {
+#if QK_K == 256
+        const uint8_t * base     = static_cast<const uint8_t *>(vbq);
+        const uint8_t * qs       = base + ibx_offset.first + 8 * iqs;
+        const uint8_t * qh       = base + ibx_offset.second;
+        const uint8_t * signs    = base + d_offset.first + 4 * iqs;
+        const uint8_t * metadata = base + d_offset.second;
+        const ggml_half d        = *reinterpret_cast<const ggml_half *>(metadata);
+        const uint8_t * scales   = metadata + sizeof(ggml_half);
+        const int8_t * q8        = q8_1_quant_ptr + iqs * QK8_1;
+
+        int sumi = 0;
+#pragma unroll
+        for (int l = 0; l < 4; ++l) {
+            const uint32_t grid1 = iq3s_grid[qs[2*l+0] | ((qh[iqs] << (8 - 2*l)) & 256)];
+            const uint32_t grid2 = iq3s_grid[qs[2*l+1] | ((qh[iqs] << (7 - 2*l)) & 256)];
+            const uint32_t signs0 = iq_sign_mask(signs[l]);
+            const uint32_t signs1 = iq_sign_mask(signs[l] >> 4);
+            const int grid_l = (grid1 ^ signs0) + (signs0 & 0x01010101);
+            const int grid_h = (grid2 ^ signs1) + (signs1 & 0x01010101);
+            sumi = dpct::dp4a(grid_l, *((const int *) q8 + 0), sumi);
+            sumi = dpct::dp4a(grid_h, *((const int *) q8 + 1), sumi);
+            q8 += 8;
+        }
+
+        const float scale = 1 + 2 * ((scales[iqs / 2] >> (4 * (iqs % 2))) & 0xf);
+        return (float) d * scale * q8_1_ds[iqs][0] * sumi;
+#else
+        GGML_UNUSED(vbq);
+        GGML_UNUSED(ibx_offset);
+        GGML_UNUSED(d_offset);
+        GGML_UNUSED(q8_1_quant_ptr);
+        GGML_UNUSED(q8_1_ds);
+        GGML_UNUSED(iqs);
+        GGML_ABORT("IQ3_S reorder vec dot not supported for QK_K != 256");
+#endif
+    }
+};
+
 #define VDR_IQ1_S_Q8_1_MMVQ 1

 static __dpct_inline__ float
@@ -1,6 +1,5 @@
 #include "ggml-vulkan-common.h"

-#include <atomic>
 #include <cstdint>
 #include <cstdlib>
 #include <mutex>
@@ -363,21 +362,16 @@ void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf,
    vk_host_allocation & entry = vk_instance.host_allocations[index];

    if (entry.views[device->idx] == nullptr) {
-        if (!vk_instance.multi_device) {
-            return;
-        }
-        if (!device->external_memory_host) {
-            static std::atomic<uint32_t> warned_mask{0};
-            const uint32_t bit = 1u << device->idx;
-            if (!(warned_mask.fetch_or(bit) & bit)) {
-                GGML_LOG_WARN("ggml_vulkan: %s does not support VK_EXT_external_memory_host; host memory "
-                    "will be staged, not pinned, on this device\n", device->name.c_str());
-            }
+        const uint32_t bit = 1u << device->idx;
+        if (!vk_instance.multi_device || !vk_instance.device_host_buffer_import[device->idx] || (vk_instance.host_import_failed & bit)) {
            return;
        }
+        GGML_ASSERT(device->external_memory_host);
        // const only propagates from the const context; the pointee is mutable
        entry.views[device->idx] = ggml_vk_buffer_from_host_ptr(const_cast<vk_device&>(device), entry.base, entry.size);
        if (entry.views[device->idx] == nullptr) {
+            vk_instance.host_import_failed |= bit;
+            GGML_LOG_WARN("ggml_vulkan: %s: host memory import failed, using staging\n", device->name.c_str());
            return;
        }
    }
@@ -835,8 +835,8 @@ struct vk_device_struct {
    vk_pipeline pipeline_roll_f32;
    vk_pipeline pipeline_repeat_i32, pipeline_repeat_back_f32;
    vk_pipeline pipeline_repeat_i16;
-    vk_pipeline pipeline_cpy_f32_f32, pipeline_cpy_f32_f16, pipeline_cpy_f16_f16, pipeline_cpy_f16_f32, pipeline_cpy_f32_bf16, pipeline_cpy_bf16_f32, pipeline_cpy_bf16_f16, pipeline_cpy_f32_i32, pipeline_cpy_i32_f32;
-    vk_pipeline pipeline_contig_cpy_f32_f32, pipeline_contig_cpy_f32_f16, pipeline_contig_cpy_f16_f16, pipeline_contig_cpy_f16_f32, pipeline_contig_cpy_f32_bf16, pipeline_contig_cpy_bf16_f32, pipeline_contig_cpy_bf16_f16, pipeline_contig_cpy_f32_i32, pipeline_contig_cpy_i32_f32;
+    vk_pipeline pipeline_cpy_32, pipeline_cpy_f32_f16, pipeline_cpy_16, pipeline_cpy_f16_f32, pipeline_cpy_f32_bf16, pipeline_cpy_bf16_f32, pipeline_cpy_bf16_f16, pipeline_cpy_f32_i32, pipeline_cpy_i32_f32;
+    vk_pipeline pipeline_contig_cpy_32, pipeline_contig_cpy_f32_f16, pipeline_contig_cpy_16, pipeline_contig_cpy_f16_f32, pipeline_contig_cpy_f32_bf16, pipeline_contig_cpy_bf16_f32, pipeline_contig_cpy_bf16_f16, pipeline_contig_cpy_f32_i32, pipeline_contig_cpy_i32_f32;
    vk_pipeline pipeline_cpy_f32_quant[GGML_TYPE_COUNT];
    vk_pipeline pipeline_cpy_quant_f32[GGML_TYPE_COUNT];
    vk_pipeline pipeline_cpy_transpose_16, pipeline_cpy_transpose_32;
@@ -1329,12 +1329,14 @@ struct vk_instance_t {

    std::vector<size_t> device_indices;
    std::vector<bool>   device_supports_membudget;
+    std::vector<bool>   device_host_buffer_import;
    vk_device devices[GGML_VK_MAX_DEVICES];

    std::vector<vk_host_allocation> host_allocations;
    std::shared_mutex               host_allocations_mutex;

    size_t host_import_alignment = 0;
+    uint32_t host_import_failed = 0;  // bitmask by device->idx, guarded by host_allocations_mutex

    bool multi_device = false;
 };
@@ -3273,9 +3273,9 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
    ggml_vk_create_pipeline(device, device->pipeline_rms_norm_back_f32, "rms_norm_back_f32", rms_norm_back_f32_len, rms_norm_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1);
    ggml_vk_create_pipeline(device, device->pipeline_l2_norm_f32, "l2_norm_f32", l2_norm_f32_len, l2_norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1);

-    ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f32, "cpy_f32_f32", cpy_f32_f32_len, cpy_f32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
+    ggml_vk_create_pipeline(device, device->pipeline_cpy_32, "cpy_32", cpy_32_len, cpy_32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
    ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_f16, "cpy_f32_f16", cpy_f32_f16_len, cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
-    ggml_vk_create_pipeline(device, device->pipeline_cpy_f16_f16, "cpy_f16_f16", cpy_f16_f16_len, cpy_f16_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
+    ggml_vk_create_pipeline(device, device->pipeline_cpy_16, "cpy_16", cpy_16_len, cpy_16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
    ggml_vk_create_pipeline(device, device->pipeline_cpy_f16_f32, "cpy_f16_f32", cpy_f16_f32_len, cpy_f16_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
    ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_bf16,"cpy_f32_bf16",cpy_f32_bf16_len,cpy_f32_bf16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
    ggml_vk_create_pipeline(device, device->pipeline_cpy_bf16_f32,"cpy_bf16_f32",cpy_bf16_f32_len,cpy_bf16_f32_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
@@ -3283,9 +3283,9 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
    ggml_vk_create_pipeline(device, device->pipeline_cpy_i32_f32, "cpy_i32_f32", cpy_i32_f32_len, cpy_i32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
    ggml_vk_create_pipeline(device, device->pipeline_cpy_f32_i32, "cpy_f32_i32", cpy_f32_i32_len, cpy_f32_i32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);

-    ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_f32, "contig_cpy_f32_f32", contig_cpy_f32_f32_len, contig_cpy_f32_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
+    ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_32, "contig_cpy_32", contig_cpy_32_len, contig_cpy_32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
    ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_f16, "contig_cpy_f32_f16", contig_cpy_f32_f16_len, contig_cpy_f32_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
-    ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f16_f16, "contig_cpy_f16_f16", contig_cpy_f16_f16_len, contig_cpy_f16_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
+    ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_16, "contig_cpy_16", contig_cpy_16_len, contig_cpy_16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
    ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f16_f32, "contig_cpy_f16_f32", contig_cpy_f16_f32_len, contig_cpy_f16_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
    ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_f32_bf16,"contig_cpy_f32_bf16",contig_cpy_f32_bf16_len,contig_cpy_f32_bf16_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
    ggml_vk_create_pipeline(device, device->pipeline_contig_cpy_bf16_f32,"contig_cpy_bf16_f32",contig_cpy_bf16_f32_len,contig_cpy_bf16_f32_data,"main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
@@ -3987,13 +3987,26 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
    }
 }

+static bool ggml_vk_external_memory_host_usable(vk::DriverId driver_id) {
+    // Disable external_memory_host until https://github.com/KhronosGroup/MoltenVK/pull/2622
+    // is available in the Vulkan SDK.
+    return driver_id != vk::DriverId::eMoltenvk;
+}
+
+static bool ggml_vk_host_buffer_import_usable(uint32_t vendor_id) {
+    // amdgpu re-faults all pages of every imported host allocation on each submission
+    return vendor_id != VK_VENDOR_ID_AMD;
+}
+
 vk_device ggml_vk_get_device(size_t idx) {
    VK_LOG_DEBUG("ggml_vk_get_device(" << idx << ")");

+    static std::mutex mutex;
+    std::lock_guard<std::mutex> lock(mutex);
+
    if (vk_instance.devices[idx] == nullptr) {
        VK_LOG_DEBUG("Initializing new vk_device");
        vk_device device = std::make_shared<vk_device_struct>();
-        vk_instance.devices[idx] = device;

        device->memory_logger = std::unique_ptr<vk_memory_logger>(new vk_memory_logger());

@@ -4172,11 +4185,7 @@ vk_device ggml_vk_get_device(size_t idx) {
        device->vendor_id = device->properties.vendorID;
        device->driver_id = driver_props.driverID;

-        if (device->driver_id == vk::DriverId::eMoltenvk) {
-            // Disable external_memory_host until https://github.com/KhronosGroup/MoltenVK/pull/2622
-            // is available in the Vulkan SDK.
-            device->external_memory_host = false;
-        }
+        device->external_memory_host = device->external_memory_host && ggml_vk_external_memory_host_usable(device->driver_id);

        // Implementing the async backend interfaces seems broken on older Intel HW,
        // see https://github.com/ggml-org/llama.cpp/issues/17302.
@@ -4891,14 +4900,14 @@ vk_device ggml_vk_get_device(size_t idx) {
            device->async_use_transfer_queue = false;
        }

+        device->fence = device->device.createFence({});
+
        device->buffer_type = {
            /* .iface    = */ ggml_backend_vk_buffer_type_interface,
            /* .device   = */ ggml_backend_reg_dev_get(ggml_backend_vk_reg(), idx),
            /* .context  = */ new ggml_backend_vk_buffer_type_context{ device->name, device },
        };

-        device->fence = device->device.createFence({});
-
        device->idx = idx;

        device->serialize_submissions = getenv("GGML_VK_SERIALIZE_SUBMISSIONS") != nullptr;
@@ -4920,6 +4929,9 @@ vk_device ggml_vk_get_device(size_t idx) {
            device->mmvq_mode = 1;
        }

+        // publish the device only when it is fully built
+        vk_instance.devices[idx] = device;
+
        return device;
    }

@@ -5416,7 +5428,7 @@ void ggml_vk_instance_init() {
    }
    GGML_LOG_DEBUG("ggml_vulkan: Found %zu Vulkan devices:\n", vk_instance.device_indices.size());

-    bool all_external_memory_host = vk_instance.device_indices.size() > 1;
+    size_t host_buffer_import_count = 0;
    for (size_t i = 0; i < vk_instance.device_indices.size(); i++) {
        vk::PhysicalDevice vkdev = devices[vk_instance.device_indices[i]];
        std::vector<vk::ExtensionProperties> extensionprops = vkdev.enumerateDeviceExtensionProperties();
@@ -5433,20 +5445,27 @@ void ggml_vk_instance_init() {

        vk_instance.device_supports_membudget.push_back(membudget_supported);

+        bool host_buffer_import = false;
        if (external_memory_host_supported) {
            vk::PhysicalDeviceProperties2 props2;
+            vk::PhysicalDeviceDriverProperties driver_props;
            vk::PhysicalDeviceExternalMemoryHostPropertiesEXT ext_props;
-            props2.pNext = &ext_props;
+            props2.pNext = &driver_props;
+            driver_props.pNext = &ext_props;
            vkdev.getProperties2(&props2);
-            vk_instance.host_import_alignment = std::max<size_t>(vk_instance.host_import_alignment, ext_props.minImportedHostPointerAlignment);
-        } else {
-            all_external_memory_host = false;
+            host_buffer_import = ggml_vk_external_memory_host_usable(driver_props.driverID) &&
+                                 ggml_vk_host_buffer_import_usable(props2.properties.vendorID);
+            if (host_buffer_import) {
+                vk_instance.host_import_alignment = std::max<size_t>(vk_instance.host_import_alignment, ext_props.minImportedHostPointerAlignment);
+                host_buffer_import_count++;
+            }
        }
+        vk_instance.device_host_buffer_import.push_back(host_buffer_import);

        ggml_vk_print_gpu_info(i);
    }

-    vk_instance.multi_device = all_external_memory_host;
+    vk_instance.multi_device = host_buffer_import_count > 1 && vk_instance.device_host_buffer_import[0];
 }

 void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) {
@@ -6100,13 +6119,6 @@ vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_t
        }
    }

-    if (src->type == GGML_TYPE_F32 && to == GGML_TYPE_F32) {
-        if (contig) {
-            return ctx->device->pipeline_contig_cpy_f32_f32;
-        } else {
-            return ctx->device->pipeline_cpy_f32_f32;
-        }
-    }
    if (src->type == GGML_TYPE_F32 && to == GGML_TYPE_F16) {
        if (contig) {
            return ctx->device->pipeline_contig_cpy_f32_f16;
@@ -6114,13 +6126,6 @@ vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_t
            return ctx->device->pipeline_cpy_f32_f16;
        }
    }
-    if (src->type == GGML_TYPE_F16 && to == GGML_TYPE_F16) {
-        if (contig) {
-            return ctx->device->pipeline_contig_cpy_f16_f16;
-        } else {
-            return ctx->device->pipeline_cpy_f16_f16;
-        }
-    }
    if (src->type == GGML_TYPE_F16 && to == GGML_TYPE_F32) {
        if (contig) {
            return ctx->device->pipeline_contig_cpy_f16_f32;
@@ -6203,15 +6208,15 @@ vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_t
        GGML_ASSERT(ggml_is_quantized(to) || ggml_type_size(src->type) == 2 || ggml_type_size(src->type) == 4);
        if ((ggml_type_size(src->type) % 4) == 0) {
            if (contig) {
-                return ctx->device->pipeline_contig_cpy_f32_f32;
+                return ctx->device->pipeline_contig_cpy_32;
            } else {
-                return ctx->device->pipeline_cpy_f32_f32;
+                return ctx->device->pipeline_cpy_32;
            }
        } else {
            if (contig) {
-                return ctx->device->pipeline_contig_cpy_f16_f16;
+                return ctx->device->pipeline_contig_cpy_16;
            } else {
-                return ctx->device->pipeline_cpy_f16_f16;
+                return ctx->device->pipeline_cpy_16;
            }
        }
    }
@@ -8134,19 +8139,30 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
    };
    const bool k_quant = k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_BF16 && k->type != GGML_TYPE_F32;
    const bool v_quant = v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_BF16 && v->type != GGML_TYPE_F32;
-    const bool use_dequant_kv = k_quant && v_quant && neq1 >= 64 &&
+    const bool kv_raw = (k->type == GGML_TYPE_F16 || k->type == GGML_TYPE_BF16) && k->type == v->type;
+    // gated to AMD, except RDNA4, which regressed in benchmarks
+    const bool kv_raw_strided = ctx->device->vendor_id == VK_VENDOR_ID_AMD &&
+                                ctx->device->architecture != vk_device_architecture::AMD_RDNA4 &&
+                                neq1 >= 256 &&
+                                kv_raw &&
+                                (k->nb[1] != (uint64_t)HSK * ggml_type_size(k->type) ||
+                                 v->nb[1] != (uint64_t)HSV * ggml_type_size(v->type)) &&
+                                (HSK % 8) == 0 && (HSV % 8) == 0;
+    const size_t kv_scratch_ts = kv_raw_strided ? ggml_type_size(k->type) : sizeof(ggml_fp16_t);
+    const bool use_dequant_kv = ((k_quant && v_quant) || kv_raw_strided) && neq1 >= 64 &&
                                is_dense_kv_cache(k) && is_dense_kv_cache(v) &&
-                                (uint64_t)ggml_nelements(k) * sizeof(ggml_fp16_t) <= ctx->device->properties.limits.maxStorageBufferRange &&
-                                (uint64_t)ggml_nelements(v) * sizeof(ggml_fp16_t) <= ctx->device->properties.limits.maxStorageBufferRange &&
-                                ctx->device->pipeline_dequant_transpose[k->type] != nullptr &&
-                                ctx->device->pipeline_dequant_transpose[v->type] != nullptr &&
+                                (uint64_t)ggml_nelements(k) * kv_scratch_ts <= ctx->device->properties.limits.maxStorageBufferRange &&
+                                (uint64_t)ggml_nelements(v) * kv_scratch_ts <= ctx->device->properties.limits.maxStorageBufferRange &&
+                                (kv_raw_strided ||
+                                 (ctx->device->pipeline_dequant_transpose[k->type] != nullptr &&
+                                  ctx->device->pipeline_dequant_transpose[v->type] != nullptr)) &&
                                // coopmat2 path does not benefit from the f16 scratch
                                !ctx->device->coopmat2 &&
                                // Intel Xe1 regresses, see PR 25494
                                (ctx->device->vendor_id != VK_VENDOR_ID_INTEL ||
                                 (ctx->device->coopmat_support && ctx->device->architecture != vk_device_architecture::INTEL_XE1));
-    const ggml_type k_type_eff = use_dequant_kv ? GGML_TYPE_F16 : k->type;
-    const ggml_type v_type_eff = use_dequant_kv ? GGML_TYPE_F16 : v->type;
+    const ggml_type k_type_eff = (use_dequant_kv && !kv_raw_strided) ? GGML_TYPE_F16 : k->type;
+    const ggml_type v_type_eff = (use_dequant_kv && !kv_raw_strided) ? GGML_TYPE_F16 : v->type;

    // For scalar/coopmat1 FA, we can use the "large" size to accommodate qga.
    // For coopmat2 FA, we always use the small size (which is still pretty large for gqa).
@@ -8422,28 +8438,56 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const
    }

    if (use_dequant_kv) {
-        const uint64_t fp = sizeof(ggml_fp16_t);
-        const uint64_t k_f16_sz = (uint64_t)ggml_nelements(k) * fp;
-        const uint64_t v_f16_sz = (uint64_t)ggml_nelements(v) * fp;
-        if (ctx->prealloc_size_x < k_f16_sz + v_f16_sz) {
-            ctx->prealloc_size_x = k_f16_sz + v_f16_sz;
+        const uint64_t k_sz = (uint64_t)ggml_nelements(k) * kv_scratch_ts;
+        const uint64_t v_sz = (uint64_t)ggml_nelements(v) * kv_scratch_ts;
+        if (ctx->prealloc_size_x < k_sz + v_sz) {
+            ctx->prealloc_size_x = k_sz + v_sz;
            ggml_vk_preallocate_buffers(ctx, subctx);
        }
-        vk_pipeline tr_k = ctx->device->pipeline_dequant_transpose[k->type];
-        vk_pipeline tr_v = ctx->device->pipeline_dequant_transpose[v->type];
-        ggml_pipeline_request_descriptor_sets(ctx, tr_k, 1);
-        ggml_pipeline_request_descriptor_sets(ctx, tr_v, 1);
-        if (ctx->prealloc_x_need_sync) {
-            ggml_vk_sync_buffers(ctx, subctx);
-        }
-        vk_subbuffer k_dst = vk_subbuffer{ ctx->prealloc_x, 0,        k_f16_sz };
-        vk_subbuffer v_dst = vk_subbuffer{ ctx->prealloc_x, k_f16_sz, v_f16_sz };
+        vk_subbuffer k_dst = vk_subbuffer{ ctx->prealloc_x, 0,    k_sz };
+        vk_subbuffer v_dst = vk_subbuffer{ ctx->prealloc_x, k_sz, v_sz };
        const uint32_t k_nel = (uint32_t)ggml_nelements(k);
        const uint32_t v_nel = (uint32_t)ggml_nelements(v);
-        { const std::vector<uint32_t> pc = { (uint32_t)HSK, (uint32_t)nek2, (uint32_t)KV, 0, k_nel };
-          ggml_vk_dispatch_pipeline(ctx, subctx, tr_k, { k_buf, k_dst }, pc, { k_nel, 1, 1 }); }
-        { const std::vector<uint32_t> pc = { (uint32_t)HSV, (uint32_t)nev2, (uint32_t)KV, 0, v_nel };
-          ggml_vk_dispatch_pipeline(ctx, subctx, tr_v, { v_buf, v_dst }, pc, { v_nel, 1, 1 }); }
+        if (kv_raw_strided) {
+            vk_pipeline cp_k = ggml_vk_get_cpy_pipeline(ctx, k, nullptr, k->type);
+            vk_pipeline cp_v = ggml_vk_get_cpy_pipeline(ctx, v, nullptr, v->type);
+            ggml_pipeline_request_descriptor_sets(ctx, cp_k, 1);
+            ggml_pipeline_request_descriptor_sets(ctx, cp_v, 1);
+            if (ctx->prealloc_x_need_sync) {
+                ggml_vk_sync_buffers(ctx, subctx);
+            }
+            auto make_pc = [](uint32_t hs, uint32_t nh, uint32_t kv, uint32_t nel) {
+                vk_op_unary_push_constants pc{};
+                pc.ne = nel;
+                pc.ne00 = hs; pc.ne01 = nh;      pc.ne02 = kv; pc.ne03 = nel / (hs * nh * kv);
+                pc.nb00 = 1;  pc.nb01 = hs;      pc.nb02 = hs * nh; pc.nb03 = hs * nh * kv;
+                pc.ne10 = hs; pc.ne11 = nh;      pc.ne12 = kv; pc.ne13 = pc.ne03;
+                pc.nb10 = 1;  pc.nb11 = hs * kv; pc.nb12 = hs; pc.nb13 = hs * kv * nh;
+                init_pushconst_fastdiv(pc);
+                return pc;
+            };
+            auto cpy_elems = [](uint32_t ne) -> std::array<uint32_t, 3> {
+                if (ne > 262144) { return { 512, 512, CEIL_DIV(ne, 262144) }; }
+                if (ne > 512)    { return { 512, CEIL_DIV(ne, 512), 1 }; }
+                return { ne, 1, 1 };
+            };
+            ggml_vk_dispatch_pipeline(ctx, subctx, cp_k, { k_buf, k_dst },
+                make_pc((uint32_t)HSK, (uint32_t)nek2, (uint32_t)KV, k_nel), cpy_elems(k_nel));
+            ggml_vk_dispatch_pipeline(ctx, subctx, cp_v, { v_buf, v_dst },
+                make_pc((uint32_t)HSV, (uint32_t)nev2, (uint32_t)KV, v_nel), cpy_elems(v_nel));
+        } else {
+            vk_pipeline tr_k = ctx->device->pipeline_dequant_transpose[k->type];
+            vk_pipeline tr_v = ctx->device->pipeline_dequant_transpose[v->type];
+            ggml_pipeline_request_descriptor_sets(ctx, tr_k, 1);
+            ggml_pipeline_request_descriptor_sets(ctx, tr_v, 1);
+            if (ctx->prealloc_x_need_sync) {
+                ggml_vk_sync_buffers(ctx, subctx);
+            }
+            { const std::vector<uint32_t> pc = { (uint32_t)HSK, (uint32_t)nek2, (uint32_t)KV, 0, k_nel };
+              ggml_vk_dispatch_pipeline(ctx, subctx, tr_k, { k_buf, k_dst }, pc, { k_nel, 1, 1 }); }
+            { const std::vector<uint32_t> pc = { (uint32_t)HSV, (uint32_t)nev2, (uint32_t)KV, 0, v_nel };
+              ggml_vk_dispatch_pipeline(ctx, subctx, tr_v, { v_buf, v_dst }, pc, { v_nel, 1, 1 }); }
+        }
        ggml_vk_sync_buffers(ctx, subctx);
        k_buf = k_dst;
        v_buf = v_dst;
@@ -16438,6 +16482,11 @@ void vk_queue_handle_unsynchronized::submit(vk::ArrayProxy<const vk::SubmitInfo>
 }

 vk_device_struct::~vk_device_struct() {
+    // the build failed before the device was created, nothing to destroy
+    if (device == VK_NULL_HANDLE) {
+        return;
+    }
+
    VK_LOG_DEBUG("destroy device " << name);

    device.destroyFence(fence);
@@ -45,6 +45,7 @@ const uint BNover2 = BN / 2;
 const uint BNover4 = enable_smaller_matrices ? (BN / 4) : (BN / 2);
 layout (constant_id = 5) const uint ALIGNED = 0;
 layout (constant_id = 6) const uint subgroup_size = 32;
+#define GROUP_A_BUDGET (8u * 1024u * 1024u)

 #ifdef MULMAT_QUANT

@@ -258,8 +259,6 @@ void main() {
    const uint row_begin = tile_idx * BN;
    _ne1 = data_row_map[2 + 2 * tile_idx];
    const uint ic = 0;
-#else
-    const uint ic = gl_WorkGroupID.y;
 #endif

 #ifdef MUL_MAT_ID
@@ -289,9 +288,33 @@ void main() {
 #endif

    const uint blocks_m = (p.M + BM - 1) / BM;
-    const uint ir = gl_WorkGroupID.x % blocks_m;
    const uint ik = gl_WorkGroupID.x / blocks_m;

+#ifdef MUL_MAT_ID
+    const uint ir = gl_WorkGroupID.x % blocks_m;
+#else
+    // L2-friendly workgroup scheduling
+    const uint blocks_n = (p.N + BN - 1) / BN;
+#if defined(MULMAT_QUANT)
+    const uint a_panel_bytes = (BM * p.K / mm_quant_k()) * MmABlockBytes;
+#elif defined(DATA_A_F32)
+    const uint a_panel_bytes = BM * p.K * 4;
+#elif QUANT_K == 1
+    const uint a_panel_bytes = BM * p.K * 2;
+#else
+    const uint a_panel_bytes = (BM * p.K) / 2 + (BM * p.K) / 16;
+#endif
+    const uint group_m = clamp(GROUP_A_BUDGET / max(a_panel_bytes, 1u), 1u, min(blocks_m, 32u));
+    const uint tiles_per_group = group_m * blocks_n;
+    const uint lin = gl_WorkGroupID.y * blocks_m + (gl_WorkGroupID.x % blocks_m);
+    const uint group_id = lin / tiles_per_group;
+    const uint first_m = group_id * group_m;
+    const uint gsize = min(blocks_m - first_m, group_m);
+    const uint in_group = lin - group_id * tiles_per_group;
+    const uint ir = first_m + in_group % gsize;
+    const uint ic = in_group / gsize;
+#endif
+
 #ifdef MUL_MAT_ID
    load_row_ids(row_begin);
 #endif
@@ -864,18 +864,18 @@ void process_shaders() {
    string_to_spv("rms_norm_back_f32", "rms_norm_back.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}));
    string_to_spv("l2_norm_f32", "l2_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));

-    string_to_spv("cpy_f32_f32", "copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
+    string_to_spv("cpy_32", "copy.comp", {{"A_TYPE", "uint"}, {"D_TYPE", "uint"}});
    string_to_spv("cpy_f32_f16", "copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float16_t"}});
-    string_to_spv("cpy_f16_f16", "copy.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OPTIMIZATION_ERROR_WORKAROUND", "1"}});
+    string_to_spv("cpy_16", "copy.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}});
    string_to_spv("cpy_f16_f32", "copy.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float"}, {"OPTIMIZATION_ERROR_WORKAROUND", "1"}});
    string_to_spv("cpy_f32_bf16","copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "uint16_t"}, {"DATA_D_BF16", "1"}});
    string_to_spv("cpy_bf16_f32","copy.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "float"}, {"DATA_A_BF16", "1"}});
    string_to_spv("cpy_bf16_f16","copy.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "float16_t"}, {"DATA_A_BF16", "1"}});
-    string_to_spv("contig_cpy_f32_f32", "contig_copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
+    string_to_spv("contig_cpy_32", "contig_copy.comp", {{"A_TYPE", "uint"}, {"D_TYPE", "uint"}});
    string_to_spv("contig_cpy_f32_i32", "contig_copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "int"}});
    string_to_spv("contig_cpy_i32_f32", "contig_copy.comp", {{"A_TYPE", "int"}, {"D_TYPE", "float"}});
    string_to_spv("contig_cpy_f32_f16", "contig_copy.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float16_t"}});
-    string_to_spv("contig_cpy_f16_f16", "contig_copy.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OPTIMIZATION_ERROR_WORKAROUND", "1"}});
+    string_to_spv("contig_cpy_16", "contig_copy.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}});
    string_to_spv("contig_cpy_f16_f32", "contig_copy.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float"}, {"OPTIMIZATION_ERROR_WORKAROUND", "1"}});
    string_to_spv("contig_cpy_f32_bf16","contig_copy.comp",{{"A_TYPE", "float"}, {"D_TYPE", "uint16_t"}, {"DATA_D_BF16", "1"}});
    string_to_spv("contig_cpy_bf16_f32","contig_copy.comp",{{"A_TYPE", "uint16_t"}, {"D_TYPE", "float"}, {"DATA_A_BF16", "1"}});
@@ -132,6 +132,7 @@ class Keys:
        EXPERTS_PER_GROUP                 = "{arch}.experts_per_group"
        MOE_EVERY_N_LAYERS                = "{arch}.moe_every_n_layers"
        MOE_LATENT_SIZE                   = "{arch}.moe_latent_size"
+        POST_NORM_EPS                     = "{arch}.post_norm_epsilon"
        NEXTN_PREDICT_LAYERS              = "{arch}.nextn_predict_layers"
        NUM_DEEPSTACK_LAYERS              = "{arch}.n_deepstack_layers"
        DEEPSTACK_MAPPING                 = "{arch}.deepstack_mapping"
@@ -174,6 +175,7 @@ class Keys:
        HAS_CONFIDENCE_HEAD               = "{arch}.has_confidence_head"
        NORM_BEFORE_RESIDUAL              = "{arch}.norm_before_residual"
        NORM_BEFORE_FC                    = "{arch}.norm_before_fc"
+        REALTIME_DELAY_TOKENS             = "{arch}.realtime_delay_tokens"

    class Adapters:
        COUNT                = "{arch}.adapters.count"
@@ -309,6 +311,16 @@ class Keys:
    class WKV:
        HEAD_SIZE = "{arch}.wkv.head_size"

+    class LiliCorr:
+        HIDDEN_SIZE     = "lilicorr.hidden_size"
+        CANDIDATE_TOPK  = "lilicorr.candidate_topk"
+        NUM_HEADS       = "lilicorr.num_heads"
+        NUM_LAYERS      = "lilicorr.num_layers"
+        FACTOR_DIM      = "lilicorr.factor_dim"
+        MLP_HIDDEN_SIZE = "lilicorr.mlp_hidden_size"
+        VECTOR_EPS      = "lilicorr.vector_eps"
+        LOGIT_SCALE     = "lilicorr.logit_scale"
+
    class PosNet:
        EMBEDDING_LENGTH = "{arch}.posnet.embedding_length"
        BLOCK_COUNT      = "{arch}.posnet.block_count"
@@ -456,6 +468,9 @@ class Keys:
        SUBSAMPLING_FACTOR  = "clip.audio.subsampling_factor"
        CHUNK_SIZE          = "clip.audio.chunk_size"
        CONV_KERNEL_SIZE    = "clip.audio.conv_kernel_size"
+        HIDDEN_SIZES        = "clip.audio.hidden_sizes"   # muse-spark, per stage
+        PATCH_SIZES         = "clip.audio.patch_sizes"    # muse-spark, per stage
+        DEPTHS              = "clip.audio.depths"         # muse-spark, per stage
        MAX_POS_EMB         = "clip.audio.max_pos_emb"
        FEATURE_LAYERS      = "clip.audio.feature_layer" # Granite Speech Plus
        RVQ_NUM_QUANTIZERS  = "clip.audio.rvq.num_quantizers"
@@ -467,6 +482,7 @@ class Keys:

        class Attention:
            HEAD_COUNT      = "clip.audio.attention.head_count"
+            HEAD_DIM        = "clip.audio.attention.head_dim" # set when qkv width != n_embd
            LAYERNORM_EPS   = "clip.audio.attention.layer_norm_epsilon"

        class Projector:
@@ -601,6 +617,7 @@ class MODEL_ARCH(IntEnum):
    OLMO2            = auto()
    OLMOE            = auto()
    MUSE_GLIMMER     = auto()
+    MUSE_SPARK       = auto()
    OPENELM          = auto()
    ARCTIC           = auto()
    DEEPSEEK         = auto()
@@ -674,6 +691,7 @@ class MODEL_ARCH(IntEnum):
    SPARK2_5           = auto()
    LLAMA_EMBED      = auto()
    MAINCODER        = auto()
+    MAION_CODER      = auto()
    KIMI_LINEAR      = auto()
    KIMI_K3          = auto()
    TALKIE           = auto()
@@ -732,6 +750,8 @@ class MODEL_TENSOR(IntEnum):
    ATTN_ROT_EMBD        = auto()
    ATTN_SINKS           = auto()
    ATTN_GATE            = auto()
+    ATTN_RESID_ALPHA     = auto()
+    ATTN_RESID_BETA      = auto()
    FFN_GATE_INP         = auto()
    FFN_GATE_INP_SHEXP   = auto()
    FFN_NORM             = auto()
@@ -740,6 +760,8 @@ class MODEL_TENSOR(IntEnum):
    FFN_POST_NORM        = auto()
    FFN_POST_NORM_1      = auto() # gemma4
    FFN_POST_NORM_2      = auto() # gemma4
+    FFN_NORM_ADA_DOWN    = auto() # voxtral realtime
+    FFN_NORM_ADA_UP      = auto() # voxtral realtime
    FFN_GATE             = auto()
    FFN_DOWN             = auto()
    FFN_UP               = auto()
@@ -760,6 +782,10 @@ class MODEL_TENSOR(IntEnum):
    FFN_GATE_TID2EID     = auto()
    MOE_LATENT_DOWN      = auto() # nemotron 3 super
    MOE_LATENT_UP        = auto() # nemotron 3 super
+    FFN_RESID_ALPHA      = auto()
+    FFN_RESID_BETA       = auto()
+    FFN_PRE_EXP_NORM     = auto()
+    FFN_POST_EXP_NORM    = auto()
    ATTN_Q_NORM          = auto()
    ATTN_K_NORM          = auto()
    LAYER_OUT_NORM       = auto()
@@ -1110,6 +1136,8 @@ class MODEL_TENSOR(IntEnum):
    A_ENC_EMBD_TO_LOGITS  = auto() # lfm2
    A_ENC_INP_PROJ        = auto() # gemma4
    A_ENC_CONV1D          = auto()
+    A_ENC_PATCH_PROJ      = auto() # muse-spark
+    A_ENC_PATCH_NORM      = auto() # muse-spark
    A_ENC_CONV1D_NORM     = auto() # gemma3n
    A_ENC_CONV2D          = auto()
    A_ENC_CONV_OUT        = auto()
@@ -1279,6 +1307,19 @@ class MODEL_TENSOR(IntEnum):
    DFLASH_SELECTOR_PREV   = auto()
    DFLASH_SELECTOR_NEXT   = auto()
    DFLASH_SELECTOR_HIDDEN = auto()
+    LILICORR_TOKEN_PROJ           = auto()
+    LILICORR_PASS_HIDDEN_PROJ     = auto()
+    LILICORR_SLOT_EMBD            = auto()
+    LILICORR_RANK_EMBD            = auto()
+    LILICORR_ATTN_BIAS            = auto()
+    LILICORR_CONTEXT_PROJ         = auto()
+    LILICORR_ATTN_IN_PROJ         = auto()
+    LILICORR_OUTPUT_NORM          = auto()
+    LILICORR_ANCHOR_NORM          = auto()
+    LILICORR_FACTOR_INPUT_PROJ    = auto()
+    LILICORR_OUT_HEAD             = auto()
+    LILICORR_IN_HEAD              = auto()
+    LILICORR_ANCHOR_OUT_HEAD      = auto()
    # lfm2 audio
    A_ENC_NORM_CONV        = auto()
    A_ENC_LINEAR_POS       = auto()
@@ -1391,6 +1432,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
    MODEL_ARCH.OLMO2:            "olmo2",
    MODEL_ARCH.OLMOE:            "olmoe",
    MODEL_ARCH.MUSE_GLIMMER:     "muse-glimmer",
+    MODEL_ARCH.MUSE_SPARK:       "muse-spark",
    MODEL_ARCH.OPENELM:          "openelm",
    MODEL_ARCH.ARCTIC:           "arctic",
    MODEL_ARCH.DEEPSEEK:         "deepseek",
@@ -1465,6 +1507,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
    MODEL_ARCH.SPARK2_5:         "spark2_5",
    MODEL_ARCH.LLAMA_EMBED:      "llama-embed",
    MODEL_ARCH.MAINCODER:        "maincoder",
+    MODEL_ARCH.MAION_CODER:      "maion-coder",
    MODEL_ARCH.KIMI_LINEAR:      "kimi-linear",
    MODEL_ARCH.KIMI_K3:          "kimi-k3",
    MODEL_ARCH.TALKIE:           "talkie",
@@ -1519,6 +1562,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
    MODEL_TENSOR.ATTN_ROT_EMBD:             "blk.{bid}.attn_rot_embd",
    MODEL_TENSOR.ATTN_SINKS:                "blk.{bid}.attn_sinks",
    MODEL_TENSOR.ATTN_GATE:                 "blk.{bid}.attn_gate",
+    MODEL_TENSOR.ATTN_RESID_ALPHA:          "blk.{bid}.attn_resid_alpha",
+    MODEL_TENSOR.ATTN_RESID_BETA:           "blk.{bid}.attn_resid_beta",
    MODEL_TENSOR.ATTN_Q_NORM:               "blk.{bid}.attn_q_norm",
    MODEL_TENSOR.ATTN_K_NORM:               "blk.{bid}.attn_k_norm",
    MODEL_TENSOR.ATTN_OUT_NORM:             "blk.{bid}.attn_output_norm",
@@ -1531,6 +1576,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
    MODEL_TENSOR.FFN_PRE_NORM_2:            "blk.{bid}.pre_ffw_norm_2",  # gemma4
    MODEL_TENSOR.FFN_POST_NORM_1:           "blk.{bid}.post_ffw_norm_1", # gemma4
    MODEL_TENSOR.FFN_POST_NORM_2:           "blk.{bid}.post_ffw_norm_2", # gemma4
+    MODEL_TENSOR.FFN_NORM_ADA_DOWN:         "blk.{bid}.ffn_norm_ada_down",
+    MODEL_TENSOR.FFN_NORM_ADA_UP:           "blk.{bid}.ffn_norm_ada_up",
    MODEL_TENSOR.FFN_GATE:                  "blk.{bid}.ffn_gate",
    MODEL_TENSOR.FFN_DOWN:                  "blk.{bid}.ffn_down",
    MODEL_TENSOR.FFN_UP:                    "blk.{bid}.ffn_up",
@@ -1551,6 +1598,10 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
    MODEL_TENSOR.FFN_GATE_TID2EID:          "blk.{bid}.ffn_gate_tid2eid",
    MODEL_TENSOR.MOE_LATENT_DOWN:           "blk.{bid}.ffn_latent_down",      # nemotron 3 super
    MODEL_TENSOR.MOE_LATENT_UP:             "blk.{bid}.ffn_latent_up",        # nemotron 3 super
+    MODEL_TENSOR.FFN_RESID_ALPHA:           "blk.{bid}.ffn_resid_alpha",
+    MODEL_TENSOR.FFN_RESID_BETA:            "blk.{bid}.ffn_resid_beta",
+    MODEL_TENSOR.FFN_PRE_EXP_NORM:          "blk.{bid}.ffn_pre_exp_norm",
+    MODEL_TENSOR.FFN_POST_EXP_NORM:         "blk.{bid}.ffn_post_exp_norm",
    MODEL_TENSOR.LAYER_OUT_NORM:            "blk.{bid}.layer_output_norm",
    MODEL_TENSOR.LAYER_OUT_SCALE:           "blk.{bid}.layer_output_scale",
    MODEL_TENSOR.PER_LAYER_TOKEN_EMBD:      "per_layer_token_embd",           # gemma3n
@@ -1896,6 +1947,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
    MODEL_TENSOR.A_ENC_EMBD_TO_LOGITS:      "a.embd_to_logits",
    MODEL_TENSOR.A_ENC_INP_PROJ:            "a.input_projection",
    MODEL_TENSOR.A_ENC_CONV1D:              "a.conv1d.{bid}",
+    MODEL_TENSOR.A_ENC_PATCH_PROJ:          "a.patch.{bid}",
+    MODEL_TENSOR.A_ENC_PATCH_NORM:          "a.patch.{bid}.norm",
    MODEL_TENSOR.A_ENC_CONV2D:              "a.conv2d.{bid}",
    MODEL_TENSOR.A_ENC_CONV_OUT:            "a.conv_out",
    MODEL_TENSOR.A_ENC_CONV1D_NORM:         "a.conv1d.{bid}.norm",
@@ -2093,6 +2146,19 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
    MODEL_TENSOR.DFLASH_SELECTOR_PREV:      "selector_predecessor",
    MODEL_TENSOR.DFLASH_SELECTOR_NEXT:      "selector_successor",
    MODEL_TENSOR.DFLASH_SELECTOR_HIDDEN:    "selector_hidden",
+    MODEL_TENSOR.LILICORR_TOKEN_PROJ:                   "lilicorr.token_proj",
+    MODEL_TENSOR.LILICORR_PASS_HIDDEN_PROJ:             "lilicorr.pass_hidden_proj",
+    MODEL_TENSOR.LILICORR_SLOT_EMBD:                    "lilicorr.slot_embedding",
+    MODEL_TENSOR.LILICORR_RANK_EMBD:                    "lilicorr.rank_embedding",
+    MODEL_TENSOR.LILICORR_ATTN_BIAS:                    "lilicorr.attention_bias",
+    MODEL_TENSOR.LILICORR_CONTEXT_PROJ:                 "lilicorr.context_proj",
+    MODEL_TENSOR.LILICORR_ATTN_IN_PROJ:                 "lilicorr.layers.{bid}.attn.in_proj",
+    MODEL_TENSOR.LILICORR_OUTPUT_NORM:                  "lilicorr.output_norm",
+    MODEL_TENSOR.LILICORR_ANCHOR_NORM:                  "lilicorr.anchor_norm",
+    MODEL_TENSOR.LILICORR_FACTOR_INPUT_PROJ:            "lilicorr.factor_input_proj",
+    MODEL_TENSOR.LILICORR_OUT_HEAD:                     "lilicorr.out_head",
+    MODEL_TENSOR.LILICORR_IN_HEAD:                      "lilicorr.in_head",
+    MODEL_TENSOR.LILICORR_ANCHOR_OUT_HEAD:              "lilicorr.anchor_out_head",
    MODEL_TENSOR.D2T:                       "d2t",
    MODEL_TENSOR.ATTN_V_GATE:               "blk.{bid}.attn_v_gate",
    MODEL_TENSOR.ATTN_V_EXP:                "blk.{bid}.attn_v_exps",
@@ -2235,6 +2301,8 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
        MODEL_TENSOR.A_ENC_EMBD_TO_LOGITS,
        MODEL_TENSOR.A_ENC_INP_PROJ,
        MODEL_TENSOR.A_ENC_CONV1D,
+        MODEL_TENSOR.A_ENC_PATCH_PROJ,
+        MODEL_TENSOR.A_ENC_PATCH_NORM,
        MODEL_TENSOR.A_ENC_CONV2D,
        MODEL_TENSOR.A_ENC_CONV_OUT,
        MODEL_TENSOR.A_ENC_CONV1D_NORM,
@@ -3941,6 +4009,34 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
        MODEL_TENSOR.FFN_PRE_NORM,
        MODEL_TENSOR.FFN_POST_NORM,
    ],
+    MODEL_ARCH.MUSE_SPARK: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_Q_NORM,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_K_NORM,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.ATTN_GATE,
+        MODEL_TENSOR.ATTN_RESID_ALPHA,
+        MODEL_TENSOR.ATTN_RESID_BETA,
+        MODEL_TENSOR.FFN_PRE_NORM,
+        MODEL_TENSOR.FFN_POST_NORM,
+        MODEL_TENSOR.FFN_RESID_ALPHA,
+        MODEL_TENSOR.FFN_RESID_BETA,
+        MODEL_TENSOR.FFN_GATE_INP,
+        MODEL_TENSOR.FFN_EXP_PROBS_B,
+        MODEL_TENSOR.MOE_LATENT_DOWN,
+        MODEL_TENSOR.FFN_PRE_EXP_NORM,
+        MODEL_TENSOR.FFN_GATE_EXP,
+        MODEL_TENSOR.FFN_UP_EXP,
+        MODEL_TENSOR.FFN_DOWN_EXP,
+        MODEL_TENSOR.FFN_POST_EXP_NORM,
+        MODEL_TENSOR.MOE_LATENT_UP,
+    ],
    MODEL_ARCH.OPENELM: [
        MODEL_TENSOR.TOKEN_EMBD,
        MODEL_TENSOR.OUTPUT_NORM,
@@ -5464,6 +5560,8 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
        MODEL_TENSOR.ATTN_ROT_EMBD,
        MODEL_TENSOR.FFN_GATE_INP,
        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_NORM_ADA_DOWN,
+        MODEL_TENSOR.FFN_NORM_ADA_UP,
        MODEL_TENSOR.FFN_GATE,
        MODEL_TENSOR.FFN_DOWN,
        MODEL_TENSOR.FFN_UP,
@@ -5548,6 +5646,19 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
        MODEL_TENSOR.DFLASH_SELECTOR_PREV,
        MODEL_TENSOR.DFLASH_SELECTOR_NEXT,
        MODEL_TENSOR.DFLASH_SELECTOR_HIDDEN,
+        MODEL_TENSOR.LILICORR_TOKEN_PROJ,
+        MODEL_TENSOR.LILICORR_PASS_HIDDEN_PROJ,
+        MODEL_TENSOR.LILICORR_SLOT_EMBD,
+        MODEL_TENSOR.LILICORR_RANK_EMBD,
+        MODEL_TENSOR.LILICORR_ATTN_BIAS,
+        MODEL_TENSOR.LILICORR_CONTEXT_PROJ,
+        MODEL_TENSOR.LILICORR_ATTN_IN_PROJ,
+        MODEL_TENSOR.LILICORR_OUTPUT_NORM,
+        MODEL_TENSOR.LILICORR_ANCHOR_NORM,
+        MODEL_TENSOR.LILICORR_FACTOR_INPUT_PROJ,
+        MODEL_TENSOR.LILICORR_OUT_HEAD,
+        MODEL_TENSOR.LILICORR_IN_HEAD,
+        MODEL_TENSOR.LILICORR_ANCHOR_OUT_HEAD,
    ],
    MODEL_ARCH.MISTRAL4: [
        MODEL_TENSOR.TOKEN_EMBD,
@@ -5693,6 +5804,31 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
        MODEL_TENSOR.FFN_DOWN,
        MODEL_TENSOR.FFN_UP,
    ],
+    MODEL_ARCH.MAION_CODER: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_QKV,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_Q_NORM,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_K_NORM,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.ATTN_POST_NORM,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_POST_NORM,
+        MODEL_TENSOR.FFN_GATE,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.FFN_GATE_INP,
+        MODEL_TENSOR.FFN_GATE_EXP,
+        MODEL_TENSOR.FFN_DOWN_EXP,
+        MODEL_TENSOR.FFN_UP_EXP,
+        MODEL_TENSOR.MOE_LATENT_DOWN,
+        MODEL_TENSOR.MOE_LATENT_UP,
+    ],
    MODEL_ARCH.KIMI_LINEAR: [
        MODEL_TENSOR.TOKEN_EMBD,
        MODEL_TENSOR.OUTPUT_NORM,
@@ -6169,6 +6305,7 @@ class VisionProjectorType:
    GLMA = "glma" # audio
    QWEN25O = "qwen2.5o" # omni
    VOXTRAL = "voxtral"
+    VOXTRAL_RT = "voxtral_rt"
    MERALION = "meralion"  # audio: Whisper + gated MLP adaptor
    LFM2 = "lfm2"
    KIMIVL = "kimivl"
@@ -6189,6 +6326,7 @@ class VisionProjectorType:
    GLM5V = "glm5v"
    YOUTUVL = "youtuvl"
    NEMOTRON_V2_VL = "nemotron_v2_vl"
+    NEMOTRON_OMNI = "nemotron_omni"
    QWEN3TTS_SPKENC = "qwen3tts_spkenc" # audio: ECAPA-TDNN speaker encoder
    QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor
    POCKETTTS_SPKENC = "pockettts_spkenc" # audio: mimi encoder as voice-prompt encoder
@@ -6206,6 +6344,7 @@ class VisionProjectorType:
    COHERE2V       = "cohere2v"
    D1OMNI_V       = "d1omni_v"  # lfm2 vision, without separator tokens
    D1OMNI_A       = "d1omni_a"  # lfm2a audio, with a residual block after the projector
+    MUSE_SPARK_AUDIO = "muse-spark-audio"


 # Items here are (block size, type size)
--- a/Show More
+++ b/Show More