improve

At a given code point a candidate only differs from its input by the
position of its code point pointer, so the levels now carry indices
into the input candidates instead of copies. The first level needs no
special case and the rejects map back to the input directly.

Grammar sampling gets faster than master on GLM-4.7, Laguna, Qwen3.6
and gemma-4 with the json, list and spaces grammars, same masks.

.github/workflows/behavior-check.yml

@@ -107,52 +107,3 @@ jobs:
           source .venv/bin/activate
           cd build
           ctest -L behavior -LE 'model|backend' --verbose --timeout 900
-
-  windows:
-    runs-on: windows-2025
-
-    cache-mode: write
-    permissions:
-      actions: write
-      contents: read
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          key: behavior-check-windows-2025
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-
-      - name: Python Dependencies
-        id: python_depends
-        shell: bash
-        run: |
-          python3 -m venv .venv
-          source .venv/Scripts/activate
-          pip install jinja2==3.1.6
-
-      - name: Build
-        id: cmake_build
-        shell: cmd
-        run: |
-          call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
-          cmake -B build -G "Ninja Multi-Config" ^
-            -DCMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake ^
-            -DCMAKE_BUILD_TYPE=Release ^
-            -DGGML_NATIVE=OFF ^
-            -DGGML_BACKEND_DL=OFF
-          set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
-          cmake --build build --config Release -j %NINJA_JOBS%
-
-      - name: Test
-        id: cmake_test
-        shell: bash
-        run: |
-          source .venv/Scripts/activate
-          cd build
-          ctest -C Release -L behavior -LE 'model|backend' --verbose --timeout 900

.github/workflows/build-apple.yml

@@ -107,6 +107,62 @@ jobs:
           # ref: https://github.com/ggml-org/llama.cpp/pull/19802#issuecomment-4013704023
           ctest -L main -E "test-llama-archs|test-llama-context|test-speculative-rejection-sampling" --verbose --timeout 900
 
+  macos-latest-x64:
+    runs-on: macos-15-intel
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: ccache
+        uses: ggml-org/ccache-action@v1.2.24
+        with:
+          restore: false
+          save: false
+
+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+        with:
+          key: apple-x64
+          folder: llama.cpp
+          hf_bucket: ggml-org/cache
+
+      - name: Build
+        id: cmake_build
+        run: |
+          sysctl -a
+          # Metal is disabled due to intermittent failures with Github runners not having a GPU:
+          # https://github.com/ggml-org/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
+          cmake -B build \
+            -DCMAKE_BUILD_RPATH="@loader_path" \
+            -DLLAMA_FATAL_WARNINGS=ON \
+            -DLLAMA_BUILD_BORINGSSL=ON \
+            -DGGML_METAL=OFF \
+            -DGGML_RPC=ON \
+            -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
+          time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
+
+      - name: ccache-buckets-save
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+        with:
+          key: apple-x64
+          folder: llama.cpp
+          evict-old-files: 1d
+          hf_bucket: ggml-org/cache
+          save: true
+
+      - name: Test
+        id: cmake_test
+        run: |
+          cd build
+          ctest -L main --verbose --timeout 900
+
   macos-latest-ios-xcode:
     runs-on: macos-latest
 

.github/workflows/ci-self-hosted-webgpu.yml

@@ -58,12 +58,7 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt update
-          sudo apt install -y software-properties-common
-          sudo add-apt-repository ppa:deadsnakes/ppa
-          sudo apt update
-          sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3.12 python3.12-venv
-          sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1
-          sudo update-alternatives --config python3
+          sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
 
       - name: ccache
         uses: ggml-org/ccache-action@v1.2.24

.github/workflows/release.yml

@@ -968,11 +968,6 @@ jobs:
           evict-old-files: 1d
           max-size: "1G"
 
-      - name: Tune ccache for ROCm
-        run: |
-          # since ROCm 10.1, clang -E output has no GPU code, so the preprocessor fallback can reuse objects with old GPU code
-          ccache --set-config=depend_mode=true
-
       # - name: Cache ROCm Installation
       #   id: cache-rocm
       #   uses: actions/cache@v5
@@ -1550,8 +1545,6 @@ jobs:
           # builds hit.
           ccache --set-config=compiler_check=content
           ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime
-          # since ROCm 10.1, clang -E output has no GPU code, so the preprocessor fallback can reuse objects with old GPU code
-          ccache --set-config=depend_mode=true
 
       - name: Dependencies
         id: depends

CMakeLists.txt

@@ -103,7 +103,7 @@ endif()
 
 # subprocess spawning isn't a supported/sandbox-friendly operation on mobile OSes or in WASM
 if (CMAKE_SYSTEM_NAME STREQUAL "iOS" OR CMAKE_SYSTEM_NAME STREQUAL "Android" OR ANDROID
-        OR CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR EMSCRIPTEN OR CMAKE_SYSTEM_NAME STREQUAL "WASI")
+        OR CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR EMSCRIPTEN)
     set(LLAMA_SUBPROCESS_DEFAULT OFF)
 else()
     set(LLAMA_SUBPROCESS_DEFAULT ON)

ci/run.sh

@@ -460,10 +460,10 @@ function gg_run_qwen3_0_6b {
     #       https://github.com/ggml-org/llama.cpp/actions/runs/39639985732
     if [ -z ${GG_BUILD_WEBGPU} ]; then
         (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+        (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa on  --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
         (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa off                ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+        (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa on                 ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
     fi
-    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa on  --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
-    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa on                 ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
 
     check_ppl "f16"  "$(cat $OUT/${ci}-tg-f16.log  | grep "^\[1\]")"
     if [ -z ${GG_BUILD_NO_BF16} ]; then

common/arg.cpp

@@ -924,7 +924,7 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
         // TODO @ngxson : maybe show a list of available models in CLI in this case
         bool can_skip_model = params.usage || params.completion || !params.server_base.empty();
         if (!can_skip_model && params.model.path.empty()) {
-            throw std::invalid_argument("error: --model or -hf is required\n");
+            throw std::invalid_argument("error: --model is required\n");
         }
     }
 
@@ -3708,13 +3708,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.models_max = value;
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MODELS_MAX"));
-    add_opt(common_arg(
-        {"--models-memory-margin"}, "N",
-        string_format("for router server, MiB of device memory to keep free per device, unload LRU models until a new one fits (default: %d, 0 = disabled)", params.models_memory_margin),
-        [](common_params & params, int value) {
-            params.models_memory_margin = value;
-        }
-    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MODELS_MEMORY_MARGIN"));
     add_opt(common_arg(
         {"--models-autoload"},
         {"--no-models-autoload"},
@@ -4512,14 +4505,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.tts_lang = value;
         }
     ).set_examples({LLAMA_EXAMPLE_TTS}));
-    add_opt(common_arg(
-        {"--tts-voice"}, "VOICE",
-        "voice for audio generation (model-dependent)\n"
-        "see tts/README.md for per-model usage notes",
-        [](common_params & params, const std::string & value) {
-            params.tts_voice = value;
-        }
-    ).set_examples({LLAMA_EXAMPLE_TTS}));
     add_opt(common_arg(
         {"--tts-speaker-file"}, "FNAME",
         "speaker file path for audio generation",

common/chat.cpp

@@ -190,17 +190,10 @@ json common_chat_msg::to_json_oaicompat(bool concat_typed_text) const {
         } else {
             auto & parts = jmsg["content"] = json::array();
             for (const auto & part : content_parts) {
-                if (part.type == "tool_reference") {
-                    parts.push_back({
-                        {"type", part.type},
-                        {"name", part.name},
-                    });
-                } else {
-                    parts.push_back({
-                        {"type", part.type},
-                        {"text", part.text},
-                    });
-                }
+                parts.push_back({
+                    {"type", part.type},
+                    {"text", part.text},
+                });
             }
         }
     } else {
@@ -316,7 +309,6 @@ struct common_chat_templates {
     bool has_explicit_template;  // Model had builtin template or template overridden was specified.
     std::unique_ptr<common_chat_template> template_default;  // always set (defaults to chatml)
     std::unique_ptr<common_chat_template> template_tool_use;
-    std::unique_ptr<common_chat_template> template_tts;  // the prompt of a text to speak
 };
 
 common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {
@@ -378,16 +370,12 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
                             throw std::invalid_argument("Missing content part type: " + part.dump());
                         }
                         const auto & type = part.at("type");
-                        common_chat_msg_content_part msg_part;
-                        msg_part.type = type;
-                        if (type == "text" || type == "media_marker") {
-                            msg_part.text = part.at("text");
-                        } else if (type == "tool_reference") {
-                            // names a request tool, the template renders its definition
-                            msg_part.name = part.at("name");
-                        } else {
+                        if (type != "text" && type != "media_marker") {
                             throw std::invalid_argument("Unsupported content part type: " + type.dump());
                         }
+                        common_chat_msg_content_part msg_part;
+                        msg_part.type = type;
+                        msg_part.text = part.at("text");
                         msg.content_parts.push_back(msg_part);
                     }
                 } else if (!content.is_null()) {
@@ -622,17 +610,13 @@ json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & t
 
     auto result = json::array();
     for (const auto & tool : tools) {
-        json function {
-            { "name",        tool.name },
-            { "description", tool.description },
-            { "parameters",  json::parse(tool.parameters) },
-        };
-        if (tool.defer_loading) {
-            function["defer_loading"] = true;
-        }
         result.push_back({
             { "type",     "function" },
-            { "function", function },
+            { "function", {
+                { "name", tool.name },
+                { "description", tool.description },
+                { "parameters", json::parse(tool.parameters) },
+            }},
         });
     }
     return result;
@@ -669,19 +653,10 @@ std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & too
                 }
 
                 const auto & function = tool.at("function");
-
-                // "defer_loading" sits on the function, or on the tool next to "type"; the function wins
-                const json & defer = function.contains("defer_loading") ? function.at("defer_loading")
-                                                                        : tool.value("defer_loading", json(false));
-                if (!defer.is_boolean()) {
-                    throw std::invalid_argument("Expected 'defer_loading' to be a boolean, got " + defer.dump());
-                }
-
                 result.push_back({
                     /* .name = */ function.at("name"),
                     /* .description = */ function.value("description", ""),
                     /* .parameters = */ function.value("parameters", json::object()).dump(),
-                    /* .defer_loading = */ defer.get<bool>(),
                 });
             }
         }
@@ -804,70 +779,19 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
     return tmpls->has_explicit_template;
 }
 
-// renders a task template (e.g. "tts") with the given variables
-static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) {
-    jinja::context ctx(tmpl.source());
-    jinja::global_from_json(ctx, vars, false);
-    ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
-    ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
-    jinja::runtime runtime(ctx);
-    return jinja::runtime::gather_string_parts(runtime.execute(tmpl.prog))->as_string().str();
-}
-
-// LFM3 format detection: template renders deferred tools inside <additional_tools>...</additional_tools>
-static bool is_lfm3_template(const std::string & src) {
-    return src.find("<additional_tools>") != std::string::npos;
-}
-
-// LFM3-Audio: a known language code maps to its name, no language lets the model detect it
-static std::string lfm3_asr_system_prompt(const std::string & language) {
-    static const std::map<std::string, std::string> names = {
-        { "ar", "Arabic" }, { "da", "Danish" }, { "de", "German" }, { "el", "Greek" }, { "en", "English" },
-        { "es", "Spanish" }, { "fi", "Finnish" }, { "fr", "French" }, { "he", "Hebrew" }, { "it", "Italian" },
-        { "ja", "Japanese" }, { "ko", "Korean" }, { "nl", "Dutch" }, { "pl", "Polish" }, { "pt", "Portuguese" },
-        { "sv", "Swedish" }, { "tr", "Turkish" }, { "zh", "Chinese" },
-    };
-    if (language.empty()) {
-        return "Perform ASR.";
-    }
-    std::string code = language;
-    std::transform(code.begin(), code.end(), code.begin(), [](unsigned char c) { return (char) std::tolower(c); });
-    const auto it = names.find(code);
-    return "Perform ASR in " + (it != names.end() ? it->second : language) + ".";
-}
-
-common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) {
+common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates) {
     common_chat_prompt_preset asr_preset;
     asr_preset.system = "";
     asr_preset.user   = "Transcribe audio to text";
 
-    if (chat_templates && chat_templates->template_default) {
-        const std::string src = chat_templates->template_default->source();
-        if (is_lfm2_template(src)) {
-            asr_preset.system = "Perform ASR.";
-            asr_preset.user   = "";
-        } else if (is_lfm3_template(src)) {
-            asr_preset.system = lfm3_asr_system_prompt(language);
-            asr_preset.user   = "";
-            asr_preset.has_language = true;
-        }
+    if (chat_templates && chat_templates->template_default && is_lfm2_template(chat_templates->template_default->source())) {
+        asr_preset.system = "Perform ASR.";
+        asr_preset.user   = "";
     }
 
     return asr_preset;
 }
 
-std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
-                                       const std::string & language, const std::string & voice) {
-    if (!chat_templates || !chat_templates->template_tts) {
-        return "";
-    }
-    return common_chat_template_render(*chat_templates->template_tts, {
-        {"messages", json::array({{{"role", "user"}, {"content", text}}})},
-        {"language", language},
-        {"voice",    voice},
-    });
-}
-
 std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant) {
     if (!variant.empty()) {
         if (variant == "tool_use") {
@@ -978,16 +902,6 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
             LOG_ERR("%s: failed to parse tool use chat template (ignoring it): %s\n", __func__, e.what());
         }
     }
-    // task templates of the model, kept with a chat template override
-    if (model) {
-        if (const char * src = llama_model_chat_template(model, "tts")) {
-            try {
-                tmpls->template_tts = std::make_unique<common_chat_template>(src, token_bos, token_eos);
-            } catch (const std::exception & e) {
-                LOG_ERR("%s: failed to parse the tts chat template (ignoring it): %s\n", __func__, e.what());
-            }
-        }
-    }
     return tmpls;
 }
 
@@ -1940,13 +1854,12 @@ const common_chat_msg & common_chat_session::finish(const common_chat_input & ch
     return result;
 }
 
-common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
+std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
     GGML_ASSERT(chat_templates != nullptr);
     GGML_ASSERT(chat_templates->template_default != nullptr);
-    // take the more expressive template when available
-    const auto & caps = chat_templates->template_tool_use != nullptr ? chat_templates->template_tool_use->caps
-                                                                      : chat_templates->template_default->caps;
-    common_json result = caps.to_map();
-    result["reasoning_effort_levels"] = caps.reasoning_effort_levels;
-    return result;
+    if (chat_templates->template_tool_use != nullptr) {
+        // take the more expressive template when available
+        return chat_templates->template_tool_use->caps.to_map();
+    }
+    return chat_templates->template_default->caps.to_map();
 }

common/chat.h

@@ -45,7 +45,6 @@ struct common_chat_tool_call {
 struct common_chat_msg_content_part {
     std::string type;
     std::string text;
-    std::string name; // tool_reference: name of a tool from the request tools
 
     // TODO @ngxson : no known chat templates support reasoning_content in content parts yet
     //                this can be useful for models with interleaved thinking (like Kimi-K2)
@@ -53,7 +52,7 @@ struct common_chat_msg_content_part {
     // std::string reasoning_content;
 
     bool operator==(const common_chat_msg_content_part & other) const {
-        return type == other.type && text == other.text && name == other.name;
+        return type == other.type && text == other.text;
     }
 };
 
@@ -227,9 +226,6 @@ struct common_chat_tool {
     std::string name;
     std::string description;
     std::string parameters;
-
-    // withheld from the system prompt, a tool_reference reveals it later in the conversation
-    bool defer_loading = false;
 };
 
 enum common_chat_tool_choice {
@@ -472,7 +468,7 @@ common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_to
 common_json common_chat_tool_parameters(const common_json & function);
 
 // get template caps, useful for reporting to server /props endpoint
-common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates);
+std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);
 
 common_chat_prompt common_chat_template_direct_apply(
     const common_chat_template & tmpl,
@@ -491,12 +487,6 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
 struct common_chat_prompt_preset {
     std::string system;
     std::string user;
-    bool has_language = false; // the preset already names the requested language
 };
 
-common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "");
-
-// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
-// language and voice are optional, the template defines their defaults
-std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
-                                       const std::string & language = "", const std::string & voice = "");
+common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);

common/common.cpp

@@ -1187,9 +1187,6 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
     { COMMON_DECISION_TYPE_LFM2_D1_OMNI,   "lfm2-d1-omni"  },
     { COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"},
     { COMMON_DECISION_TYPE_H2O_LIGHTNING,  "h2o-lightning" },
-    { COMMON_DECISION_TYPE_MINICPM_OCR_JEV, "openbmb-minicpm-ocr-jev" },
-    { COMMON_DECISION_TYPE_LFM3_D1,        "lfm3-d1"       },
-    { COMMON_DECISION_TYPE_LFM3_D1_OMNI,   "lfm3-d1-omni"  },
     // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
 };
 
@@ -1255,50 +1252,44 @@ common_gguf_info common_get_gguf_info(const std::string & fname) {
     return info;
 }
 
-void common_fit_apply(common_params & params, llama_model_params & mparams, llama_context_params & cparams) {
-    if (!params.fit_params) {
-        return;
-    }
-
-    COM_TRC("%s", "fitting params to device memory ...\n");
-    COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n");
-
-    // the draft context is created from the same base params and follows the main context, fit both together
-    const bool has_draft = params.speculative.has_dft();
-    const bool spec_mtp  = std::find(params.speculative.types.begin(), params.speculative.types.end(),
-        COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
-
-    common_params params_dft = common_base_params_to_speculative(params);
-
-    auto mparams_dft = common_model_params_to_llama(params_dft);
-    auto cparams_dft = common_context_params_to_llama(params_dft);
-    if (spec_mtp) {
-        cparams_dft.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
-    }
-    cparams_dft.n_rs_seq = 0;
-
-    const common_fit_extra_model extra = {
-        /*.path_model   =*/ params_dft.model.path.c_str(),
-        /*.mparams      =*/ &mparams_dft,
-        /*.cparams      =*/ &cparams_dft,
-        /*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model
-    };
-
-    common_fit_params(params.model.path.c_str(), &mparams, &cparams,
-        params.tensor_split,
-        params.tensor_buft_overrides.data(),
-        params.fit_params_target.data(),
-        params.fit_params_min_ctx,
-        has_draft || spec_mtp ? &extra : nullptr,
-        params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
-}
-
 common_init_result::common_init_result(common_params & params, bool model_only) :
     pimpl(new impl{}) {
     auto mparams = common_model_params_to_llama(params);
     auto cparams = common_context_params_to_llama(params);
 
-    common_fit_apply(params, mparams, cparams);
+    if (params.fit_params) {
+        COM_TRC("%s", "fitting params to device memory ...\n");
+        COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n");
+
+        // the draft context is created from the same base params and follows the main context, fit both together
+        const bool has_draft = params.speculative.has_dft();
+        const bool spec_mtp  = std::find(params.speculative.types.begin(), params.speculative.types.end(),
+            COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
+
+        common_params params_dft = common_base_params_to_speculative(params);
+
+        auto mparams_dft = common_model_params_to_llama(params_dft);
+        auto cparams_dft = common_context_params_to_llama(params_dft);
+        if (spec_mtp) {
+            cparams_dft.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
+        }
+        cparams_dft.n_rs_seq = 0;
+
+        const common_fit_extra_model extra = {
+            /*.path_model   =*/ params_dft.model.path.c_str(),
+            /*.mparams      =*/ &mparams_dft,
+            /*.cparams      =*/ &cparams_dft,
+            /*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model
+        };
+
+        common_fit_params(params.model.path.c_str(), &mparams, &cparams,
+            params.tensor_split,
+            params.tensor_buft_overrides.data(),
+            params.fit_params_target.data(),
+            params.fit_params_min_ctx,
+            has_draft || spec_mtp ? &extra : nullptr,
+            params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
+    }
 
     llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
     if (model == NULL) {
@@ -1316,17 +1307,13 @@ common_init_result::common_init_result(common_params & params, bool model_only)
     // these decision models return a score for each token via the embeddings output
     // TODO: maybe improve this in the future
     const auto decision_type = common_get_decision_type(model);
-    // minicpm-ocr-jev reads its class scores from the pooled output, the pooling type is the one of the GGUF
     if (decision_type == COMMON_DECISION_TYPE_LAYA || decision_type == COMMON_DECISION_TYPE_KEV || decision_type == COMMON_DECISION_TYPE_CLEF ||
-        decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || decision_type == COMMON_DECISION_TYPE_VLLM_DECISION2 ||
-        decision_type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV) {
-        const auto pooling_type = decision_type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV ? LLAMA_POOLING_TYPE_UNSPECIFIED : LLAMA_POOLING_TYPE_NONE;
-
+        decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || decision_type == COMMON_DECISION_TYPE_VLLM_DECISION2) {
         params.embedding    = true;
-        params.pooling_type = pooling_type;
+        params.pooling_type = LLAMA_POOLING_TYPE_NONE;
 
         cparams.embeddings            = true;
-        cparams.pooling_type          = pooling_type;
+        cparams.pooling_type          = LLAMA_POOLING_TYPE_NONE;
         cparams.n_outputs_max         = cparams.n_batch;
         cparams.n_outputs_max_per_seq = 1;
 

common/common.h

@@ -701,7 +701,6 @@ struct common_params {
     std::string models_dir    = "";     // directory containing models for the router server
     std::string models_preset = "";     // directory containing model presets for the router server
     int models_max = 4;                 // maximum number of models to load simultaneously
-    int models_memory_margin = 0;       // MiB of device memory to keep free when loading models (0 = no memory accounting)
     bool models_autoload = true;        // automatically load models when requested via the router server
     std::string models_preset_hf = "";  // show a warning about remote presets on router loaded (if not empty)
 
@@ -772,7 +771,6 @@ struct common_params {
 
     // TTS params
     std::string tts_lang = "";
-    std::string tts_voice = "";
     std::string tts_speaker_file = "";
 
     bool is_gen_docs = false; // whether we are running inside llama-gen-docs
@@ -980,9 +978,6 @@ enum common_decision_type {
     COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout
     COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch
     COMMON_DECISION_TYPE_H2O_LIGHTNING, // same as openjev, the labels are read after an answer prefill
-    COMMON_DECISION_TYPE_MINICPM_OCR_JEV, // one score per class, the pooled embeddings output of the last token
-    COMMON_DECISION_TYPE_LFM3_D1,       // same as lfm2-d1, non-causal, label codes A..Z then AA, AB..
-    COMMON_DECISION_TYPE_LFM3_D1_OMNI,  // same as lfm3-d1
     COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
 };
 
@@ -1022,9 +1017,6 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
 struct llama_model_params   common_model_params_to_llama  (      common_params & params);
 struct llama_context_params common_context_params_to_llama(const common_params & params);
 
-// apply --fit to mparams and cparams, no-op if params.fit_params is false
-void common_fit_apply(common_params & params, struct llama_model_params & mparams, struct llama_context_params & cparams);
-
 // clear LoRA adapters from context, then apply new list of adapters
 void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora);
 

common/fit.cpp

@@ -1,8 +1,6 @@
 #include "fit.h"
 
-#include "common.h"
 #include "json.h"
-#include "llama-cpp.h"
 #include "log.h"
 
 #include "../src/llama-ext.h"
@@ -29,32 +27,6 @@ class common_params_fit_exception : public std::runtime_error {
     using std::runtime_error::runtime_error;
 };
 
-// RAII: send llama logs below min_level to the debug log
-struct common_fit_log_scope {
-    common_fit_log_scope(ggml_log_level min_level) : min_level(min_level) {
-        llama_log_get(&original_logger.callback, &original_logger.user_data);
-        llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
-            const common_fit_log_scope * self = (const common_fit_log_scope *) user_data;
-            const ggml_log_level level_eff = level >= self->min_level ? level : GGML_LOG_LEVEL_DEBUG;
-            self->original_logger.callback(level_eff, text, self->original_logger.user_data);
-        }, this);
-    }
-
-    ~common_fit_log_scope() {
-        llama_log_set(original_logger.callback, original_logger.user_data);
-    }
-
-    common_fit_log_scope(const common_fit_log_scope &) = delete;
-    common_fit_log_scope & operator=(const common_fit_log_scope &) = delete;
-
-private:
-    struct {
-        ggml_log_callback callback;
-        void * user_data;
-    } original_logger;
-    ggml_log_level min_level;
-};
-
 static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
         const char * path_model,
         const llama_model_params * mparams,
@@ -64,7 +36,22 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
         uint32_t & hp_n_ctx_train,
         uint32_t & hp_n_expert,
         ggml_log_level log_level) {
-    common_fit_log_scope log_scope(log_level);
+    struct user_data_t {
+        struct {
+            ggml_log_callback callback;
+            void * user_data;
+        } original_logger;
+        ggml_log_level min_level; // prints below this log level go to debug log
+    };
+    user_data_t ud;
+    llama_log_get(&ud.original_logger.callback, &ud.original_logger.user_data);
+    ud.min_level = log_level;
+
+    llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
+        const user_data_t * ud = (const user_data_t *) user_data;
+        const ggml_log_level level_eff = level >= ud->min_level ? level : GGML_LOG_LEVEL_DEBUG;
+        ud->original_logger.callback(level_eff, text, ud->original_logger.user_data);
+    }, &ud);
 
     llama_model_params mparams_copy = *mparams;
     mparams_copy.no_alloc  = true;
@@ -72,12 +59,14 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
 
     llama_model * model = llama_model_load_from_file(path_model, mparams_copy);
     if (model == nullptr) {
+        llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
         throw std::runtime_error("failed to load model");
     }
 
     llama_context * ctx = llama_init_from_model(model, *cparams);
     if (ctx == nullptr) {
         llama_model_free(model);
+        llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
         throw std::runtime_error("failed to create llama_context from model");
     }
 
@@ -159,81 +148,11 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
 
     llama_free(ctx);
     llama_model_free(model);
+    llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
 
     return ret;
 }
 
-std::vector<common_memory_breakdown_row> common_fit_measure(
-        const char * path_model,
-        const llama_model_params * mparams,
-        const llama_context_params * cparams,
-        uint32_t * n_ctx_train,
-        uint32_t * n_ctx,
-        ggml_log_level log_level) {
-    common_fit_log_scope log_scope(log_level);
-
-    llama_model_params mparams_copy = *mparams;
-    mparams_copy.no_alloc  = true;
-    mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;
-
-    llama_model_ptr model(llama_model_load_from_file(path_model, mparams_copy));
-    if (!model) {
-        throw std::runtime_error("failed to load model");
-    }
-
-    llama_context_ptr ctx(llama_init_from_model(model.get(), *cparams));
-    if (!ctx) {
-        throw std::runtime_error("failed to create llama_context from model");
-    }
-
-    if (n_ctx_train) {
-        *n_ctx_train = llama_model_n_ctx_train(model.get());
-    }
-    if (n_ctx) {
-        *n_ctx = llama_n_ctx(ctx.get());
-    }
-
-    return common_memory_breakdown_rows(ctx.get());
-}
-
-common_memory_reports common_measure_memory(common_params & params) {
-    const ggml_log_level log_level = params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR;
-    const char * path_model = params.model.path.c_str();
-
-    common_memory_reports res;
-
-    auto mparams = common_model_params_to_llama(params);
-    auto cparams = common_context_params_to_llama(params);
-
-    // max: what --fit picks now, no-op with -fit off
-    {
-        auto mparams_fit = mparams;
-        auto cparams_fit = cparams;
-        common_fit_apply(params, mparams_fit, cparams_fit);
-        res.info_max = common_fit_measure(path_model, &mparams_fit, &cparams_fit, &res.n_ctx_train, &res.n_ctx_max, log_level);
-    }
-
-    // min: the smallest context --fit may pick (--fit only changes n_ctx when it is 0)
-    uint32_t n_ctx_min = cparams.n_ctx;
-    if (n_ctx_min == 0 && params.fit_params) {
-        const uint32_t n_streams = cparams.kv_unified ? 1 : std::max<uint32_t>(1, cparams.n_seq_max);
-        // -c 0 sets fit_params_min_ctx to UINT32_MAX, cap at n_ctx_train like --fit
-        n_ctx_min = (uint32_t) std::min<uint64_t>(
-            uint64_t(params.fit_params_min_ctx) * n_streams,
-            uint64_t(res.n_ctx_train) * n_streams);
-    }
-    if (n_ctx_min == 0 || n_ctx_min >= res.n_ctx_max) {
-        // nothing to reduce, or -fit off with the trained context
-        res.n_ctx_min = res.n_ctx_max;
-        res.info_min  = res.info_max;
-    } else {
-        cparams.n_ctx = n_ctx_min;
-        res.info_min  = common_fit_measure(path_model, &mparams, &cparams, nullptr, &res.n_ctx_min, log_level);
-    }
-
-    return res;
-}
-
 common_device_memory_data_vec common_get_device_memory_data(
         const char * path_model,
         const llama_model_params * mparams,
@@ -984,7 +903,8 @@ enum common_params_fit_status common_fit_params(
     return status;
 }
 
-std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const struct llama_context * ctx) {
+void common_memory_breakdown_print(const struct llama_context * ctx) {
+    //const auto & devices = ctx->get_model().devices;
     const auto * model = llama_get_model(ctx);
 
     std::vector<ggml_backend_dev_t> devices;
@@ -994,6 +914,18 @@ std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const stru
 
     llama_memory_breakdown memory_breakdown = llama_get_memory_breakdown(ctx);
 
+    std::vector<std::array<std::string, 9>> table_data;
+    table_data.reserve(devices.size());
+
+    // same data as the table below, for --log-jsonl consumers
+    common_json rows = common_json::array();
+    const std::string template_header = "%s: | %s | %s   %s    %s   %s   %s   %s    %s |\n";
+    const std::string template_gpu    = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";
+    const std::string template_other  = "%s: | %s | %s   %s    %s = %s + %s + %s    %s |\n";
+
+    table_data.push_back({template_header, "memory breakdown [MiB]", "total", "free", "self", "model", "context", "compute", "unaccounted"});
+
+    constexpr size_t MiB = 1024 * 1024;
     const std::vector<std::string> desc_prefixes_strip = {"NVIDIA ", "GeForce ", "Tesla ", "AMD ", "Radeon ", "Instinct "};
 
     // track seen buffer types to avoid double counting:
@@ -1032,130 +964,104 @@ std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const stru
         }
     }
 
-    std::vector<common_memory_breakdown_row> rows;
-    rows.reserve(devices.size() + 1 + memory_breakdown.size());
-
+    // print memory breakdown for each device:
     for (size_t i = 0; i < devices.size(); i++) {
         ggml_backend_dev_t dev = devices[i];
+        llama_memory_breakdown_data mb = mb_dev[i];
 
-        common_memory_breakdown_row row;
-        row.kind        = common_memory_breakdown_row::DEVICE;
-        row.name        = ggml_backend_dev_name(dev);
-        row.description = ggml_backend_dev_description(dev);
+        const std::string name = ggml_backend_dev_name(dev);
+        std::string desc = ggml_backend_dev_description(dev);
         for (const std::string & prefix : desc_prefixes_strip) {
-            if (row.description.length() >= prefix.length() && row.description.substr(0, prefix.length()) == prefix) {
-                row.description = row.description.substr(prefix.length());
+            if (desc.length() >= prefix.length() && desc.substr(0, prefix.length()) == prefix) {
+                desc = desc.substr(prefix.length());
             }
         }
 
         size_t free, total;
         ggml_backend_dev_memory(dev, &free, &total);
-        row.total   = total;
-        row.free    = free;
-        row.model   = mb_dev[i].model;
-        row.context = mb_dev[i].context;
-        row.compute = mb_dev[i].compute;
-        rows.push_back(std::move(row));
+
+        const size_t self = mb.model + mb.context + mb.compute;
+        const int64_t unaccounted = static_cast<int64_t>(total) - static_cast<int64_t>(free) - static_cast<int64_t>(self);
+
+        table_data.push_back({
+            template_gpu,
+            "  - " + name + " (" + desc + ")",
+            std::to_string(total / MiB),
+            std::to_string(free / MiB),
+            std::to_string(self / MiB),
+            std::to_string(mb.model / MiB),
+            std::to_string(mb.context / MiB),
+            std::to_string(mb.compute / MiB),
+            std::to_string(unaccounted / static_cast<int64_t>(MiB))});
+
+        rows.push_back({
+            {"kind",        "device"},
+            {"name",        name},
+            {"description", desc},
+            {"total",       total / MiB},
+            {"free",        free / MiB},
+            {"self",        self / MiB},
+            {"model",       mb.model / MiB},
+            {"context",     mb.context / MiB},
+            {"compute",     mb.compute / MiB},
+            {"unaccounted", unaccounted / static_cast<int64_t>(MiB)},
+        });
     }
 
+    // print memory breakdown for host:
     {
-        common_memory_breakdown_row row;
-        row.kind    = common_memory_breakdown_row::HOST;
-        row.name    = "Host";
-        row.model   = mb_host.model;
-        row.context = mb_host.context;
-        row.compute = mb_host.compute;
-        rows.push_back(std::move(row));
+        const size_t self = mb_host.model + mb_host.context + mb_host.compute;
+        table_data.push_back({
+            template_other,
+            "  - Host",
+            "", // total
+            "", // free
+            std::to_string(self / MiB),
+            std::to_string(mb_host.model / MiB),
+            std::to_string(mb_host.context / MiB),
+            std::to_string(mb_host.compute / MiB),
+            ""}); // unaccounted
+
+        rows.push_back({
+            {"kind",    "host"},
+            {"name",    "Host"},
+            {"self",    self / MiB},
+            {"model",   mb_host.model / MiB},
+            {"context", mb_host.context / MiB},
+            {"compute", mb_host.compute / MiB},
+        });
     }
 
-    // all remaining buffer types:
+    // print memory breakdown for all remaining buffer types:
     for (const auto & buft_mb : memory_breakdown) {
         ggml_backend_buffer_type_t          buft = buft_mb.first;
         const llama_memory_breakdown_data & mb   = buft_mb.second;
         if (seen_buffer_types.count(buft) == 1) {
             continue;
         }
-        common_memory_breakdown_row row;
-        row.kind    = common_memory_breakdown_row::BUFFER_TYPE;
-        row.name    = ggml_backend_buft_name(buft);
-        row.model   = mb.model;
-        row.context = mb.context;
-        row.compute = mb.compute;
-        rows.push_back(std::move(row));
-
-        seen_buffer_types.insert(buft);
-    }
-
-    return rows;
-}
-
-void common_memory_breakdown_print(const struct llama_context * ctx) {
-    const std::vector<common_memory_breakdown_row> rows = common_memory_breakdown_rows(ctx);
-
-    std::vector<std::array<std::string, 9>> table_data;
-    table_data.reserve(rows.size() + 1);
-
-    // same data as the table below, for --log-jsonl consumers
-    common_json json_rows = common_json::array();
-    const std::string template_header = "%s: | %s | %s   %s    %s   %s   %s   %s    %s |\n";
-    const std::string template_gpu    = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";
-    const std::string template_other  = "%s: | %s | %s   %s    %s = %s + %s + %s    %s |\n";
-
-    table_data.push_back({template_header, "memory breakdown [MiB]", "total", "free", "self", "model", "context", "compute", "unaccounted"});
-
-    constexpr size_t MiB = 1024 * 1024;
-
-    for (const auto & row : rows) {
-        const size_t self = row.model + row.context + row.compute;
-
-        if (row.kind == common_memory_breakdown_row::DEVICE) {
-            const int64_t unaccounted = row.total - row.free - static_cast<int64_t>(self);
-
-            table_data.push_back({
-                template_gpu,
-                "  - " + row.name + " (" + row.description + ")",
-                std::to_string(row.total / MiB),
-                std::to_string(row.free / MiB),
-                std::to_string(self / MiB),
-                std::to_string(row.model / MiB),
-                std::to_string(row.context / MiB),
-                std::to_string(row.compute / MiB),
-                std::to_string(unaccounted / static_cast<int64_t>(MiB))});
-
-            json_rows.push_back({
-                {"kind",        "device"},
-                {"name",        row.name},
-                {"description", row.description},
-                {"total",       row.total / MiB},
-                {"free",        row.free / MiB},
-                {"self",        self / MiB},
-                {"model",       row.model / MiB},
-                {"context",     row.context / MiB},
-                {"compute",     row.compute / MiB},
-                {"unaccounted", unaccounted / static_cast<int64_t>(MiB)},
-            });
-            continue;
-        }
-
+        const std::string name = ggml_backend_buft_name(buft);
+        const size_t self = mb.model + mb.context + mb.compute;
         table_data.push_back({
             template_other,
-            "  - " + row.name,
+            "  - " + name,
             "", // total
             "", // free
             std::to_string(self / MiB),
-            std::to_string(row.model / MiB),
-            std::to_string(row.context / MiB),
-            std::to_string(row.compute / MiB),
+            std::to_string(mb.model / MiB),
+            std::to_string(mb.context / MiB),
+            std::to_string(mb.compute / MiB),
             ""}); // unaccounted
 
-        json_rows.push_back({
-            {"kind",    row.kind == common_memory_breakdown_row::HOST ? "host" : "buffer_type"},
-            {"name",    row.name},
+        rows.push_back({
+            {"kind",    "buffer_type"},
+            {"name",    name},
             {"self",    self / MiB},
-            {"model",   row.model / MiB},
-            {"context", row.context / MiB},
-            {"compute", row.compute / MiB},
+            {"model",   mb.model / MiB},
+            {"context", mb.context / MiB},
+            {"compute", mb.compute / MiB},
         });
+
+        seen_buffer_types.insert(buft);
     }
 
     for (size_t j = 1; j < table_data[0].size(); j++) {
@@ -1175,7 +1081,7 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
 
     LOG_JSON("fit_memory_breakdown", common_json({
         {"unit", "MiB"},
-        {"rows", json_rows},
+        {"rows", rows},
     }));
 }
 

common/fit.h

@@ -3,7 +3,6 @@
 #include "ggml.h"
 #include "llama.h"
 
-#include <string>
 #include <vector>
 
 enum common_params_fit_status {
@@ -46,47 +45,6 @@ void common_fit_print(
 
 void common_memory_breakdown_print(const llama_context * ctx);
 
-// one row of the memory breakdown: a device, the host or another buffer type
-struct common_memory_breakdown_row {
-    enum kind_t { DEVICE, HOST, BUFFER_TYPE };
-
-    kind_t      kind = DEVICE;
-    std::string name;        // device or buffer type name, "Host" for the host row
-    std::string description; // device description, empty otherwise
-    int64_t     total = 0;   // device memory in bytes, 0 unless kind == DEVICE
-    int64_t     free  = 0;   // free device memory in bytes, 0 unless kind == DEVICE
-    size_t      model   = 0; // bytes held by this context, by kind of buffer
-    size_t      context = 0;
-    size_t      compute = 0;
-};
-
-// memory a model needs at the min and max context --fit may pick
-struct common_memory_reports {
-    uint32_t n_ctx_train = 0;
-    uint32_t n_ctx_max   = 0; // the largest context that fits the free device memory right now
-    uint32_t n_ctx_min   = 0; // --fit-ctx (the smallest context --fit may pick), with the placement set in params
-    std::vector<common_memory_breakdown_row> info_max; // breakdown at n_ctx_max
-    std::vector<common_memory_breakdown_row> info_min; // breakdown at n_ctx_min
-};
-
-// rows of common_memory_breakdown_print: model devices first, then host
-std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const llama_context * ctx);
-
-// load model + context with no_alloc and return the memory breakdown, params are used as-is
-// n_ctx_train and n_ctx are optional outputs; throws on failure
-std::vector<common_memory_breakdown_row> common_fit_measure(
-                         const char * path_model,
-           const llama_model_params * mparams,
-         const llama_context_params * cparams,
-                           uint32_t * n_ctx_train,
-                           uint32_t * n_ctx,
-                     ggml_log_level   log_level);
-
-struct common_params;
-
-// measure min and max without allocating, params may be modified as with --fit; throws on failure
-common_memory_reports common_measure_memory(common_params & params);
-
 struct common_device_memory_data {
     int64_t total;
     int64_t free;

common/jinja/caps.cpp

@@ -6,9 +6,7 @@
 // we can remove it in the future when we figure out a better way to define inputs using jinja::value
 #include "json.h"
 
-#include <algorithm>
 #include <functional>
-#include <optional>
 #include <sstream>
 
 #define FILENAME "jinja-caps"
@@ -186,10 +184,6 @@ caps caps_get(jinja::program & prog) {
                     {
                         {"role", "user"},
                         {"content", json::array({
-                            {
-                                {"type", "text"},
-                                {"text", content_marker}
-                            }
                         })}
                     }
                 });
@@ -590,58 +584,6 @@ caps caps_get(jinja::program & prog) {
         }
     );
 
-    if (result.supports_reasoning_effort) {
-        JJ_DEBUG("%s\n", ">>> Running capability check: reasoning effort levels");
-
-        static const std::vector<std::string> candidates = { "minimal", "low", "medium", "high", "xhigh", "max" };
-
-        // renders a single user turn with the given effort, nothing when the template raises
-        const auto render = [&](const std::string & effort) {
-            std::optional<std::string> rendered;
-            caps_try_execute(
-                prog,
-                [&]() {
-                    // messages
-                    return json::array({
-                        {
-                            {"role", "user"},
-                            {"content", "User message"}
-                        },
-                    });
-                },
-                [&](context & ctx) {
-                    ctx.set_val("enable_thinking", mk_val<value_bool>(true));
-                    caps_apply_reasoning_effort(ctx, effort);
-                },
-                nullptr, // tools_fn
-                [&](context &, bool success, value &, value &, const std::string & output) {
-                    if (success) {
-                        rendered = output;
-                    }
-                }
-            );
-            return rendered;
-        };
-
-        // an unknown level shows the render of the template fallback, a level that renders the same is not honored
-        // levels sharing a render are aliases and the highest one is kept
-        const auto unknown = render("REASONING_EFFORT_UNKNOWN");
-        std::vector<std::string> renders;
-        for (const auto & level : candidates) {
-            const auto rendered = render(level);
-            if (!rendered || rendered == unknown) {
-                continue;
-            }
-            const auto it = std::find(renders.begin(), renders.end(), *rendered);
-            if (it != renders.end()) {
-                result.reasoning_effort_levels[it - renders.begin()] = level;
-            } else {
-                renders.push_back(*rendered);
-                result.reasoning_effort_levels.push_back(level);
-            }
-        }
-    }
-
     JJ_DEBUG("%s\n", result.to_string().c_str());
 
     return result;

common/jinja/caps.h

@@ -4,7 +4,6 @@
 
 #include <string>
 #include <map>
-#include <vector>
 
 namespace jinja {
 
@@ -20,9 +19,6 @@ struct caps {
     // supports reasoning effort levels
     bool supports_reasoning_effort = false;
 
-    // reasoning effort levels the template renders distinctly, from lowest to highest
-    std::vector<std::string> reasoning_effort_levels;
-
     // one of the 2 content capabilities must be true
     bool supports_string_content = true;
     bool supports_typed_content = false;

conversion/__init__.py

@@ -172,10 +172,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "D1OmniModel": "lfm2",
     "Lfm2Model": "lfm2",
     "Lfm2MoeForCausalLM": "lfm2",
-    "Lfm3D1Model": "lfm3",
-    "Lfm3ForCausalLM": "lfm3",
-    "Lfm3OmniForConditionalGeneration": "lfm3",
-    "Lfm3VlForConditionalGeneration": "lfm3",
     "Llama4ForCausalLM": "llama",
     "Llama4ForConditionalGeneration": "llama",
     "LlamaBidirectionalModel": "llama",
@@ -201,7 +197,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "MiniCPMForCausalLM": "minicpm",
     "MiniCPMV4_6ForConditionalGeneration": "minicpm",
     "MiniCPMV4_7ForConditionalGeneration": "minicpm",
-    "MiniCPMOCRForSequenceClassification": "qwen",
     "MiniCPMOcrForConditionalGeneration": "minicpm",
     "MiniMaxText01ForCausalLM": "minimax",
     "MiniMaxM1ForCausalLM": "minimax",
@@ -362,9 +357,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "Lfm2AudioForConditionalGeneration": "lfm2",
     "D1OmniModel": "lfm2",
     "Lfm2VlForConditionalGeneration": "lfm2",
-    "Lfm3AudioForConditionalGeneration": "lfm3",
-    "Lfm3OmniForConditionalGeneration": "lfm3",
-    "Lfm3VlForConditionalGeneration": "lfm3",
     "LightOnOCRForConditionalGeneration": "lighton_ocr",
     "Llama4ForConditionalGeneration": "llama4",
     "LlavaForConditionalGeneration": "llava",
@@ -373,7 +365,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "MiniMaxM3SparseForConditionalGeneration": "minimax",
     "MiniCPMV4_6ForConditionalGeneration": "minicpm",
     "MiniCPMV4_7ForConditionalGeneration": "minicpm",
-    "MiniCPMOCRForSequenceClassification": "qwen3vl",
     "MiniCPMOcrForConditionalGeneration": "minicpm",
     "Mistral3ForConditionalGeneration": "llava",
     "Mistral3Model": "llava",

conversion/deepseek.py

@@ -803,6 +803,9 @@ class DeepseekV4Model(TextModel):
         self._dsv4_mxfp4_generated = True
         return ()
 
+    def _format_dsv4_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None, suffix: str = ".weight") -> str:
+        return self.format_tensor_name(key, bid, suffix)
+
     def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]:
         root_map: dict[str, tuple[gguf.MODEL_TENSOR, str]] = {
             "embed.weight": (gguf.MODEL_TENSOR.TOKEN_EMBD, ".weight"),
@@ -876,17 +879,17 @@ class DeepseekV4Model(TextModel):
 
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         if re.match(r"layers\.\d+\.ffn\.experts\.\d+\.w[123]\.(weight|scale)$", name):
-            return
+            return []
 
         # hash layers route text tokens via tid2eid and image tokens via bias_vl; gate.bias is unused
         if name.endswith(".ffn.gate.bias") and bid is not None and bid < self.hparams["num_hash_layers"]:
-            return
+            return []
 
         tensor_key, suffix = self._map_dsv4_tensor_name(name, bid)
         if tensor_key == gguf.MODEL_TENSOR.FFN_GATE_TID2EID:
-            return
+            return []
 
-        yield self.format_tensor_name(tensor_key, bid, suffix), data_torch
+        return [(self._format_dsv4_tensor_name(tensor_key, bid, suffix), data_torch)]
 
     def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
         del bid  # unused

conversion/deepseek41.py

@@ -145,6 +145,12 @@ class DeepseekV41Model(DeepseekV4Model):
         # num_hidden_layers lives in text_config, which is merged only after indexing; MTP stages are skipped anyway
         return TextModel.index_tensors(self, remote_hf_model_id=remote_hf_model_id)
 
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        if item[0].startswith("vision."):
+            return None
+        return super().filter_tensors(item)
+
     def set_gguf_parameters(self):
         super().set_gguf_parameters()
         hp = self.hparams
@@ -276,13 +282,12 @@ class DeepseekV41Model(DeepseekV4Model):
 
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         if name.endswith(".engram.k_weight"):
-            return  # folded into engram_gate
+            return []  # folded into engram_gate
         if name.endswith(".engram.q_weight"):
             # Engram gate weight = q_weight * k_weight (bf16 * bf16 is exact in f32)
             k = self.model_tensors[name.removesuffix("q_weight") + "k_weight"]
-            yield self.format_tensor_name(gguf.MODEL_TENSOR.ENGRAM_GATE, bid), data_torch.float() * k().float()
-            return
-        yield from super().modify_tensors(data_torch, name, bid)
+            return [(self.format_tensor_name(gguf.MODEL_TENSOR.ENGRAM_GATE, bid), data_torch.float() * k().float())]
+        return super().modify_tensors(data_torch, name, bid)
 
     def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
         if new_name.endswith((".engram_gate.weight", ".ffn_gate_inp.weight")) or name in self._dsv4_f32_tensors:
@@ -406,15 +411,13 @@ class DeepseekV41DSparkModel(DeepseekV41Model):
 
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         if name.endswith(".ffn.gate.bias_vl"):
-            return
+            return []
         if name in self._DSPARK_ROOT:
-            yield self.format_tensor_name(self._DSPARK_ROOT[name], None), data_torch
-            return
+            return [(self.format_tensor_name(self._DSPARK_ROOT[name], None), data_torch)]
         m = re.match(r"layers\.\d+\.(main_proj\.weight|main_norm\.weight|norm\.weight)$", name)
         if m is not None:
-            yield self.format_tensor_name(self._DSPARK_ROOT[m.group(1)], None), data_torch
-            return
-        yield from super().modify_tensors(data_torch, name, bid)
+            return [(self.format_tensor_name(self._DSPARK_ROOT[m.group(1)], None), data_torch)]
+        return super().modify_tensors(data_torch, name, bid)
 
     def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
         if new_name == "conf_proj.weight":

conversion/lfm2.py

@@ -15,30 +15,25 @@ from .base import MmprojModel, ModelBase, TextModel, gguf, jinja_str_or_json, lo
 from .gemma import ConformerAudioModel
 
 
-def lfm2_feed_forward_length(hparams: dict[str, Any]) -> int:
-    ff_dim = hparams.get("block_ff_dim", hparams.get("intermediate_size"))
-    assert ff_dim is not None
-    auto_adjust_ff_dim = hparams["block_auto_adjust_ff_dim"]
-    ffn_dim_multiplier = hparams["block_ffn_dim_multiplier"]
-    multiple_of = hparams["block_multiple_of"]
-
-    if auto_adjust_ff_dim:
-        ff_dim = int(2 * ff_dim / 3)
-        # custom dim factor multiplier
-        if ffn_dim_multiplier is not None:
-            ff_dim = int(ffn_dim_multiplier * ff_dim)
-        ff_dim = multiple_of * ((ff_dim + multiple_of - 1) // multiple_of)
-
-    return ff_dim
-
-
 @ModelBase.register("Lfm2ForCausalLM", "LFM2ForCausalLM")
 @ModelBase.example("LiquidAI/LFM2-1.2B", "LiquidAI/LFM2.5-350M")
 class LFM2Model(TextModel):
     model_arch = gguf.MODEL_ARCH.LFM2
 
     def _add_feed_forward_length(self):
-        self.gguf_writer.add_feed_forward_length(lfm2_feed_forward_length(self.hparams))
+        ff_dim = self.find_hparam(["block_ff_dim", "intermediate_size"])
+        auto_adjust_ff_dim = self.hparams["block_auto_adjust_ff_dim"]
+        ffn_dim_multiplier = self.hparams["block_ffn_dim_multiplier"]
+        multiple_of = self.hparams["block_multiple_of"]
+
+        if auto_adjust_ff_dim:
+            ff_dim = int(2 * ff_dim / 3)
+            # custom dim factor multiplier
+            if ffn_dim_multiplier is not None:
+                ff_dim = int(ffn_dim_multiplier * ff_dim)
+            ff_dim = multiple_of * ((ff_dim + multiple_of - 1) // multiple_of)
+
+        self.gguf_writer.add_feed_forward_length(ff_dim)
 
     def set_gguf_parameters(self):
         # set num_key_value_heads only for attention layers
@@ -77,9 +72,7 @@ def _is_d1_checkpoint(dir_model: Path) -> bool:
     if not (dir_model / "config.json").is_file():
         return False
     with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    # d1 on LFM3 is converted by conversion/lfm3.py
-    return config.get("auto_map", {}).get("AutoModel", "").endswith(".D1Model") and not config.get("model_type", "").startswith("lfm3")
+        return json.load(f).get("auto_map", {}).get("AutoModel", "").endswith(".D1Model")
 
 
 @ModelBase.register_hparams_loader(_is_d1_checkpoint)

conversion/lfm3.py

@@ -1,613 +0,0 @@
-from __future__ import annotations
-
-import json
-import math
-
-from pathlib import Path
-from typing import Any, Callable, Iterable, TYPE_CHECKING
-
-import torch
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import MmprojModel, ModelBase, TextModel, gguf, logger
-from .lfm2 import D1Model, lfm2_feed_forward_length
-
-
-@ModelBase.register("Lfm3ForCausalLM", "Lfm3VlForConditionalGeneration", "Lfm3OmniForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-1.7B", "LiquidAI/LFM3-1.7B-Base")
-class LFM3Model(TextModel):
-    model_arch = gguf.MODEL_ARCH.LFM3
-
-    # LFM3-Audio/Omni: encoder, projector and audio-output heads live next to the text model
-    _media_prefixes = ("encoder.", "mlp.", "codebook_embedding.", "depth_embeddings.", "depth_linear.", "depthformer.",
-                       "mask_embedding", "model.codebook_embedding.", "model.depth_decoder.")
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-
-        if name.startswith(cls._media_prefixes):
-            return None
-
-        name = name.removeprefix("lfm.")
-
-        return super().filter_tensors((name, gen))
-
-    # LFM3-Audio live ASR prompt of the model card, the audio follows in the assistant turn
-    _asr_template = (
-        "{{- bos_token -}}"
-        "{{- '<|im_start|>system\\nPerform live ASR.<|im_end|>\\n' -}}"
-        "{%- if add_generation_prompt -%}{{- '<|im_start|>assistant\\n' -}}{%- endif -%}"
-    )
-
-    # TTS prompt of the model card: a language code is looked up, anything else is taken as the language name
-    _tts_langs = {
-        "ar": "Arabic", "da": "Danish", "de": "German", "el": "Greek", "en": "American English", "en-us": "American English",
-        "en-gb": "British English", "fi": "Finnish", "fr": "French", "he": "Hebrew", "it": "Italian", "ja": "Japanese",
-        "ko": "Korean", "nl": "Dutch", "pl": "Polish", "pt": "Portuguese", "es": "Spanish", "sv": "Swedish", "tr": "Turkish",
-        "zh": "Chinese",
-    }
-
-    # "language" and "voice" are optional, the text to speak is the first message
-    def _tts_template(self, think: bool) -> str:
-        return (
-            "{%- set names = " + json.dumps(self._tts_langs) + " -%}"
-            "{%- set code = (language | lower) if language is defined and language else 'en' -%}"
-            "{%- set lang = names[code] if code in names else language -%}"
-            "{%- set v = (voice | lower) if voice is defined and voice and (voice | lower) in ['female', 'male'] else 'female' -%}"
-            "{{- bos_token -}}"
-            "{{- '<|im_start|>system\\nPerform TTS in ' + lang + ' using a ' + v + ' voice.<|im_end|>\\n' -}}"
-            "{{- '<|im_start|>user\\n' + messages[0]['content'] + '<|im_end|>\\n' -}}"
-            "{{- '<|im_start|>assistant\\n" + ("<think></think>" if think else "") + "<|audio_start|>' -}}"
-        )
-
-    def set_vocab(self):
-        super().set_vocab()
-        # hparams hold the merged text config, the checkpoint's own model_type is in config.json
-        with open(self.dir_model / "config.json", encoding="utf-8") as f:
-            model_type = json.load(f).get("model_type")
-        is_audio = model_type == "lfm3_audio"
-        is_omni  = model_type == "lfm3_omni"
-        templates = []
-        if is_audio:
-            templates.append({"name": "asr", "template": self._asr_template})
-        if is_audio or is_omni:
-            # LFM3-Omni opens every assistant turn with an empty think block
-            templates.append({"name": "tts", "template": self._tts_template(think=is_omni)})
-        if templates:
-            self.gguf_writer.add_chat_template(templates)
-
-    def set_gguf_parameters(self):
-        layer_types = self.hparams["layer_types"]
-        n_kv_heads = self.hparams["num_key_value_heads"]
-
-        # set num_key_value_heads only for attention layers
-        self.hparams["num_key_value_heads"] = [
-            n_kv_heads if lt != "conv" else 0
-            for lt in layer_types
-        ]
-
-        super().set_gguf_parameters()
-
-        self.gguf_writer.add_rope_dimension_count(self.hparams["head_dim"])
-
-        self.gguf_writer.add_sliding_window_pattern([lt == "sliding_attention" for lt in layer_types])
-        self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
-
-        self.gguf_writer.add_shortconv_l_cache(self.hparams["conv_L_cache"])
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        # Partial RoPE is represented by masking trailing frequency pairs.
-        # rope_parameters is keyed by layer type; only full_attention layers use partial RoPE.
-        rope_parameters = self.hparams["rope_parameters"]["full_attention"]
-        partial = rope_parameters.get("partial_rotary_factor", 1.0)
-        head_dim = self.hparams["head_dim"]
-        rope_angles = int(partial * head_dim) // 2
-        n_pairs = head_dim // 2
-        assert 0 < rope_angles <= n_pairs
-        values = [1.0] * rope_angles + [1e30] * (n_pairs - rope_angles)
-        rope_freqs = torch.tensor(values, dtype=torch.float32)
-        yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), rope_freqs)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # conv op requires 2d tensor
-        if "conv.conv" in name:
-            data_torch = data_torch.squeeze(1)
-
-        yield from super().modify_tensors(data_torch, name, bid)
-
-
-# d1 on LFM3-VL / LFM3-Omni: the stock weights, the d1 prompt read at its last token
-_D1_TYPES = {"lfm3_vl": gguf.DecisionType.LFM3_D1, "lfm3_omni": gguf.DecisionType.LFM3_D1_OMNI}
-
-
-def _is_lfm3_d1_checkpoint(dir_model: Path) -> bool:
-    if not (dir_model / "config.json").is_file():
-        return False
-    with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    return config.get("architectures") == ["D1Model"] and config.get("model_type") in _D1_TYPES
-
-
-@ModelBase.register_hparams_loader(_is_lfm3_d1_checkpoint)
-def _load_lfm3_d1_hparams(dir_model: Path) -> dict[str, Any]:
-    logger.info("gguf: detected LFM3 d1 checkpoint")
-    hparams = ModelBase.load_hparams(dir_model, False, guess=False)
-    hparams["d1_family"] = hparams["model_type"]
-    # the mmproj is the base model's, the text model writes the decision
-    hparams["architectures"] = ["Lfm3VlForConditionalGeneration" if hparams["model_type"] == "lfm3_vl" else "Lfm3OmniForConditionalGeneration"]
-    hparams["text_config"]["architectures"] = ["Lfm3D1Model"]
-    return hparams
-
-
-@ModelBase.register("Lfm3D1Model")
-@ModelBase.example("LiquidAI/d1-2B", "LiquidAI/d1-omni-2B")
-class LFM3D1Model(LFM3Model):
-    model_arch = gguf.MODEL_ARCH.LFM3
-
-    # how the checkpoint serves, hard-coded in the server: assert the checkpoint records the same
-    _served = {"d1_codes": 2, "d1_attention": "set"}
-
-    def set_vocab(self):
-        # the vocab without the speech task templates of LFM3-Audio/Omni, d1 only decides
-        TextModel.set_vocab(self)
-        # the d1 prompt of LFM2, the LFM3 template writes an empty thought before the answer
-        self.gguf_writer.add_chat_template([{"name": "systemone", "template": D1Model._systemone_template() + "<think></think>"}])
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-
-        for key, value in self._served.items():
-            if self.hparams.get(key) != value:
-                raise ValueError(f"unsupported d1 checkpoint: {key} = {self.hparams.get(key)!r}, expected {value!r}")
-
-        # a lone question reads the whole prompt both ways on the full attention layers
-        self.gguf_writer.add_causal_attention(False)
-        self.gguf_writer.add_decision_type(_D1_TYPES[self.hparams["d1_family"]])
-
-        # the option answers share one temperature
-        with open(self.dir_model / "calibration.json", encoding="utf-8") as f:
-            calibration = json.load(f)
-        if calibration.get("temperatures"):
-            raise ValueError("unsupported d1 calibration: per-bucket temperatures")
-        for name in ("noul", "choice", "score"):
-            self.gguf_writer.add_decision_temperature(name, calibration["default"])
-
-
-class LFM3VLVisionMixin(MmprojModel):
-    """LFM3-VL vision tower and projector, shared with LFM3-Omni."""
-
-    def _set_lfm3vl_gguf_parameters(self):
-        assert self.hparams_vision is not None
-        # clip.cpp hardcodes the GELU projector and its input norm eps
-        if (self.global_config["projector_hidden_act"] != "gelu" or not self.global_config["projector_bias"]
-                or self.global_config["projector_layer_norm_eps"] != 1e-6):
-            raise ValueError("unsupported projector_hidden_act / projector_bias / projector_layer_norm_eps")
-        # clip.cpp hardcodes the vision rope base, the RGB patchifier and the 3x3 spatial conv
-        if (self.hparams_vision["rope_parameters"]["rope_theta"] != 1e6 or self.hparams_vision["in_channels"] != 3
-                or self.hparams_vision["conv_kernel_size"] != 3):
-            raise ValueError("unsupported vision rope_theta / in_channels / conv_kernel_size")
-        self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.LFM3VL)
-        self.gguf_writer.add_vision_head_count_kv(self.hparams_vision["num_key_value_heads"])
-        self.gguf_writer.add_vision_spatial_merge_size(self.global_config["downsample_factor"])
-        self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision["norm_eps"])
-        self.gguf_writer.add_vision_use_silu(True)
-        size = self.preprocessor_config["size"]
-        self.gguf_writer.add_vision_min_pixels(size["shortest_edge"])
-        self.gguf_writer.add_vision_max_pixels(size["longest_edge"])
-        # PIL resample enum, the torchvision backend has no lanczos
-        resample = self.preprocessor_config["resample"]
-        resize_algo = {2: "bilinear", 3: "bicubic"}.get(resample)
-        if resize_algo is None:
-            raise ValueError(f"unsupported resample: {resample}")
-        self.gguf_writer.add_vision_image_resize_algo(resize_algo)
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        if "patch_embd" in new_name:
-            return gguf.GGMLQuantizationType.F16
-        if "conv2d_spatial" in new_name:
-            # CUDA CONV_2D_DW requires an f32 kernel; the tensor is tiny
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    def _lfm3vl_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name == "model.vision_tower.patchifier.weight":
-            # qwen2vl-style layout [channel, temporal, ph, pw], one conv kernel per temporal slice
-            assert self.hparams_vision is not None
-            t = self.hparams_vision["temporal_patch_size"]
-            p = self.hparams_vision["patch_size"]
-            n_embd = data_torch.shape[0]
-            w = data_torch.view(n_embd, 3, t, p, p)
-            yield ("v.patch_embd.weight", w[:, :, 0].contiguous())
-            if t == 2:
-                yield ("v.patch_embd.weight.1", w[:, :, 1].contiguous())
-            else:
-                assert t == 1, f"unsupported vision_temporal_patch_size {t}"
-            return
-        if name == "model.vision_tower.embedding_norm.weight":
-            yield ("v.embedding_norm.weight", data_torch)
-            return
-        if "conv.in_proj.weight" in name:
-            yield (f"v.blk.{bid}.conv2d_in_proj.weight", data_torch)
-            return
-        if "conv.spatial_conv.weight" in name:
-            yield (f"v.blk.{bid}.conv2d_spatial.weight", data_torch)
-            return
-        if "conv.out_proj.weight" in name:
-            yield (f"v.blk.{bid}.conv2d_out_proj.weight", data_torch)
-            return
-
-        # projector: input LayerNorm -> linear_1 -> GELU -> linear_2
-        if name.startswith("model.multi_modal_projector."):
-            layer, suffix = name.removeprefix("model.multi_modal_projector.").split(".")
-            yield ({"layer_norm": "mm.input_norm", "linear_1": "mm.0", "linear_2": "mm.2"}[layer] + "." + suffix, data_torch)
-            return
-
-        yield (self.map_tensor_name(name), data_torch)
-
-
-@ModelBase.register("Lfm3VlForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-VL-2B")
-class LFM3VLMmprojModel(LFM3VLVisionMixin):
-    has_vision_encoder = True
-    has_audio_encoder = False
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        assert self.hparams_vision is not None
-        # dynamic resolution, image_size is only set for MmprojModel compatibility
-        self.hparams_vision["image_size"] = 512
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        self._set_lfm3vl_gguf_parameters()
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.startswith(("model.vision_tower.", "model.multi_modal_projector.")):
-            yield from self._lfm3vl_tensors(data_torch, name, bid)
-
-
-def _is_lfm3_audio_checkpoint(dir_model: Path) -> bool:
-    if not (dir_model / "config.json").is_file():
-        return False
-    with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    return config.get("model_type") == "lfm3_audio" and "lfm" in config
-
-
-# read the release config.json as is: a transformers that knows lfm3_audio rewrites it into another layout
-@ModelBase.register_hparams_loader(_is_lfm3_audio_checkpoint)
-def _load_lfm3_audio_hparams(dir_model: Path) -> dict[str, Any]:
-    with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    config["text_config"] = config["lfm"]
-    return config
-
-
-@ModelBase.register("Lfm3AudioForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-Audio-2B")
-class LFM3AudioMmprojModel(MmprojModel):
-    has_vision_encoder = False
-    has_audio_encoder = True
-
-    # the downsampler has the LFM2-Audio pre-encoder layout: conv, dw conv, pw conv, dw conv, pw conv
-    _downsample_idx = {"conv_in": 0, "conv_mid": 2, "lin_mid": 3, "conv_end": 5, "lin_end": 6}
-
-    # audio output: MaskGIT depthformer (a.gen.code.*) and the codec decoder (a.gen.wav.*)
-    _depth_layer_map = {
-        "operator_norm":      gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM,
-        "self_attn.q_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q,
-        "self_attn.q_norm":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM,
-        "self_attn.k_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K,
-        "self_attn.k_norm":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM,
-        "self_attn.v_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_V,
-        "self_attn.o_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT,
-        "self_attn.gate_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_GATE,
-        "ffn_norm":           gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM,
-        "feed_forward.w1":    gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE,
-        "feed_forward.w3":    gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP,
-        "feed_forward.w2":    gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
-    }
-    _codec_layer_map = {
-        "operator_norm":         gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM,
-        "post_operator_norm":    gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_POST_NORM,
-        "self_attn.q_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q,
-        "self_attn.q_norm":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q_NORM,
-        "self_attn.k_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K,
-        "self_attn.k_norm":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K_NORM,
-        "self_attn.v_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V,
-        "self_attn.o_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT,
-        "self_attn.gate_proj":   gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_GATE,
-        "conv.in_proj":          gguf.MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW1,
-        "conv.conv":             gguf.MODEL_TENSOR.A_GEN_WAV_TFM_CONV_DW,
-        "conv.out_proj":         gguf.MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW2,
-        "ffn_norm":              gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM,
-        "post_ffn_norm":         gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_POST_NORM,
-        "feed_forward.w1":       gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_GATE,
-        "feed_forward.w3":       gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP,
-        "feed_forward.w2":       gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN,
-    }
-    # the codec graph in clip.cpp hardcodes the liquid-codec FSQ front end and decoder shape, conversion fails on a mismatch
-    _codec = {
-        "model_type": "liquid_codec", "fsq_levels": [8, 7, 6, 6], "n_q": 12, "quantizer_dim": 256,
-        "encoder_dim": 512, "stft_n_fft": 1280, "stft_hop_length": 320, "stft_win_length": 1280, "stft_frame_stack": 6,
-        "decoder_window": 30, "sampling_rate": 24000,
-    }
-    # the lfm3a pipeline in mtmd-helper-gen.cpp hardcodes these
-    _generation = {"codebooks": 12, "maskgit_schedule": [4, 4, 4], "interleaved_n_text": 6, "interleaved_n_audio": 12}
-
-    # speech-code sampling of the model card, generation_config.json overrides it
-    _audio_sampling = {
-        "audio_temperature": 1.0, "audio_top_k": 128, "audio_repetition_penalty": 1.1, "audio_repetition_window": 4096,
-    }
-
-    # mtmd_audio_preprocessor_lfm3a hardcodes this front end, conversion fails on a mismatch
-    _feature_extractor = {
-        "sampling_rate": 16000, "feature_size": 128, "n_fft": 400, "win_length": 400, "hop_length": 160,
-        "log_floor": -12.0, "normalization_offset": 4.0, "normalization_divisor": 4.0, "encoder_stride": 8,
-    }
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        # audio output is exported when the checkpoint ships the codec
-        self._audio_output = (self.dir_model / "audio_tokenizer").is_dir()
-        gen = self.get_generation_config()
-        self._n_codebooks: int = gen["codebooks"]
-        self._audio_vocab: int = gen["audio_vocab_size"]
-        self._depth_embd: dict[int, Tensor] = {}
-        self._depth_norm: dict[int, Tensor] = {}
-
-    def get_audio_config(self) -> dict[str, Any] | None:
-        enc = self.global_config["encoder"]
-        cfg = dict(enc["lfm"])
-        cfg["num_mel_bins"] = enc["dim_in"]
-        cfg["sliding_window"] = enc["sliding_window"]
-        return cfg
-
-    def get_generation_config(self) -> dict[str, Any]:
-        c = self.global_config
-        steps = c["maskgit_steps"]
-        return {
-            "codebooks": c["codebooks"], "audio_vocab_size": c["audio_vocab_size"],
-            "maskgit_schedule": [c["codebooks"] // steps] * steps,
-            "interleaved_n_text": c["interleaved_n_text"], "interleaved_n_audio": c["interleaved_n_audio"],
-        }
-
-    def get_depth_config(self) -> dict[str, Any]:
-        return self.global_config["depthformer"]
-
-    def set_gguf_parameters(self):
-        assert self.hparams_audio is not None
-        fe = self.preprocessor_config["feature_extractor"]
-        if fe["feature_extractor_type"] not in ("Lfm3AudioFeatureExtractor", "Lfm3OmniFeatureExtractor"):
-            raise ValueError(f"unsupported feature_extractor_type {fe['feature_extractor_type']!r}")
-        for k, v in self._feature_extractor.items():
-            if fe[k] != v:
-                raise ValueError(f"unsupported feature_extractor.{k}={fe[k]!r}, expected {v!r}")
-
-        # clip.cpp hardcodes the encoder rope base
-        if self.hparams_audio["rope_parameters"]["rope_theta"] != 1e6:
-            raise ValueError("unsupported audio encoder rope_theta")
-
-        self.hparams_audio["intermediate_size"] = lfm2_feed_forward_length(self.hparams_audio)
-        super().set_gguf_parameters()
-        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.LFM3A)
-        self.gguf_writer.add_audio_head_count_kv(self.hparams_audio["num_key_value_heads"])
-        self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams_audio["norm_eps"])
-        self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["num_mel_bins"])
-        self.gguf_writer.add_audio_window_size(self.hparams_audio["sliding_window"])
-
-        if not self._audio_output:
-            return
-
-        # audio generation: depthformer hparams, the codec ones are constant and live in clip.cpp
-        gen = self.get_generation_config()
-        for k, v in self._generation.items():
-            if gen[k] != v:
-                raise ValueError(f"unsupported {k}={gen[k]!r}, expected {v!r}")
-        depth = self.get_depth_config()
-        if depth.get("is_causal", False) or depth["use_sandwich_norm"] or set(depth["layer_types"]) != {"full_attention"}:
-            raise ValueError("unsupported depthformer, expected a bidirectional all-attention stack")
-        if depth["rope_parameters"]["full_attention"]["rope_theta"] != 1e6:
-            raise ValueError("unsupported depthformer rope_theta")
-        self.gguf_writer.add_clip_has_gen_audio_encoder(True)
-        self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.LFM3A_GEN)
-        self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
-        self.gguf_writer.add_gen_audio_embedding_length(depth["hidden_size"])
-        self.gguf_writer.add_gen_audio_feed_forward_length(depth["intermediate_size"])
-        self.gguf_writer.add_gen_audio_block_count(depth["num_hidden_layers"])
-        self.gguf_writer.add_gen_audio_head_count(depth["num_attention_heads"])
-        self.gguf_writer.add_gen_audio_head_count_kv(depth["num_key_value_heads"])
-        self.gguf_writer.add_gen_audio_attention_layernorm_eps(depth["norm_eps"])
-
-        # sampling defaults of the speech codes, next to the text ones that become general.sampling.*
-        gen = gguf.Metadata.load_generation_config(self.dir_model)
-        for key, add in (("audio_temperature",        self.gguf_writer.add_gen_audio_sampling_temp),
-                         ("audio_top_k",              self.gguf_writer.add_gen_audio_sampling_top_k),
-                         ("audio_repetition_penalty", self.gguf_writer.add_gen_audio_sampling_penalty_repeat),
-                         ("audio_repetition_window",  self.gguf_writer.add_gen_audio_sampling_penalty_last_n)):
-            value = gen.get(key, self._audio_sampling.get(key))
-            if value is not None:
-                add(value)
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        if new_name.startswith("a.conv1d.") or "conv_dw" in new_name or new_name.startswith("a.gen.wav.quant."):
-            return gguf.GGMLQuantizationType.F32
-        # the codec decoder is small and its log-magnitude/phase output does not survive quantization
-        if new_name.startswith("a.gen.wav."):
-            return gguf.GGMLQuantizationType.F16
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        if self._audio_output:
-            yield from self._codec_tensors()
-
-    def _codec_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        import json
-        from safetensors.torch import load_file
-
-        T = gguf.MODEL_TENSOR
-        codec_dir = self.dir_model / "audio_tokenizer"
-        with open(codec_dir / "config.json", encoding="utf-8") as f:
-            cfg = json.load(f)
-        for k, v in self._codec.items():
-            if cfg.get(k) != v:
-                raise ValueError(f"unsupported audio_tokenizer {k}={cfg.get(k)!r}, expected {v!r}")
-        if math.prod(cfg["fsq_levels"]) != self._audio_vocab - 1:
-            raise ValueError(f"audio_vocab_size {self._audio_vocab} does not match the codec, expected codebook size + 1")
-
-        state_dict = load_file(codec_dir / "model.safetensors")
-
-        # FSQ code table: index -> the len(levels) quantized values, see FiniteScalarQuantization._indices_to_codes
-        levels = torch.tensor(cfg["fsq_levels"], dtype=torch.int64)
-        basis = torch.cumprod(torch.cat([torch.ones(1, dtype=torch.int64), levels[:-1]]), dim=0)
-        half = levels // 2
-        idx = torch.arange(int(levels.prod()), dtype=torch.int64)[:, None]
-        codes = ((idx // basis) % levels - half) / half
-        yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), codes.to(torch.float32))
-
-        # one projection over the concatenated code vectors of all codebooks, under quantizer.fsq. in newer exports
-        quantizer = "quantizer.fsq." if "quantizer.fsq.project_out.weight" in state_dict else "quantizer."
-        for suffix in ("weight", "bias"):
-            yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_OUT, suffix=f".{suffix}"), state_dict[f"{quantizer}project_out.{suffix}"])
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_IN_PROJ, suffix=f".{suffix}"), state_dict[f"from_quantizer.{suffix}"])
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUT_PROJ, suffix=f".{suffix}"), state_dict[f"decoder.lin.{suffix}"])
-        yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUTPUT_NORM), state_dict["decoder.lfm.embedding_norm.weight"])
-
-        for name, data_torch in state_dict.items():
-            if not name.startswith("decoder.lfm.layers."):
-                continue
-            bid_str, key = name.removeprefix("decoder.lfm.layers.").rsplit(".weight", 1)[0].split(".", 1)
-            if key == "conv.conv":
-                data_torch = data_torch.squeeze(1)
-            yield (self.format_tensor_name(self._codec_layer_map[key], int(bid_str)), data_torch)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.startswith("a.gen.wav."):
-            yield (name, data_torch)
-            return
-
-        if name.startswith(("depthformer.", "depth_", "codebook_embedding.", "mask_embedding")):
-            if self._audio_output:
-                yield from self._depthformer_tensor(data_torch, name, bid)
-            return
-
-        if not name.startswith(("encoder.", "mlp.")):
-            return
-
-        if name.startswith("encoder.downsample."):
-            layer, suffix = name.removeprefix("encoder.downsample.").split(".")
-            if layer == "lin_out":
-                if suffix == "weight":
-                    data_torch = data_torch.squeeze(-1)
-                yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_ENC_OUT, suffix=f".{suffix}"), data_torch)
-                return
-            if suffix == "bias":
-                data_torch = data_torch[:, None, None]
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_ENC_CONV1D, self._downsample_idx[layer], f".{suffix}"), data_torch)
-            return
-
-        # conv op requires 2d tensor
-        if name.endswith("conv.conv.weight"):
-            data_torch = data_torch.squeeze(1)
-
-        yield (self.map_tensor_name(name), data_torch)
-
-    def _depthformer_tensor(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        T = gguf.MODEL_TENSOR
-
-        if name.startswith("depthformer.model.layers."):
-            key = name.removeprefix("depthformer.model.layers.").split(".", 1)[1].rsplit(".weight", 1)[0]
-            yield (self.format_tensor_name(self._depth_layer_map[key], bid), data_torch)
-            return
-        if name == "depthformer.model.embedding_norm.weight":
-            yield (self.format_tensor_name(T.A_GEN_CODE_OUTPUT_NORM), data_torch)
-            return
-        if name.startswith("depth_linear."):
-            yield (self.format_tensor_name(T.A_GEN_CODE_PROJ_IN, suffix="." + name.rsplit(".", 1)[1]), data_torch)
-            return
-        if name == "mask_embedding":
-            yield (self.format_tensor_name(T.A_GEN_CODE_MASK_EMBD, suffix=""), data_torch)
-            return
-        if name == "codebook_embedding.weight":
-            assert data_torch.shape[0] == self._n_codebooks * self._audio_vocab
-            yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch)
-            return
-
-        # per-codebook heads: the embedding table is tied to to_logits, merge the 12 tables into one tensor
-        assert name.startswith("depth_embeddings.")
-        idx = int(name.split(".")[1])
-        key = name.split(".", 2)[2]
-        if key == "embedding.weight":
-            self._depth_embd[idx] = data_torch
-            if len(self._depth_embd) == self._n_codebooks:
-                yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), torch.stack([self._depth_embd[i] for i in range(self._n_codebooks)], dim=0))
-        elif key == "embedding_norm.weight":
-            self._depth_norm[idx] = data_torch
-            if len(self._depth_norm) == self._n_codebooks:
-                yield (self.format_tensor_name(T.A_GEN_CODE_HEAD_NORM), torch.stack([self._depth_norm[i] for i in range(self._n_codebooks)], dim=0))
-        elif key == "to_logits.weight":
-            if not torch.equal(data_torch, self._depth_embd[idx]):
-                raise ValueError(f"depth_embeddings.{idx} is not tied")
-        else:
-            raise ValueError(f"unexpected tensor {name}")
-
-
-@ModelBase.register("Lfm3OmniForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-Omni-2B")
-class LFM3OmniMmprojModel(LFM3VLVisionMixin, LFM3AudioMmprojModel):
-    has_vision_encoder = True
-    has_audio_encoder = True
-
-    _audio_sampling = {"audio_temperature": 0.4, "audio_repetition_penalty": 1.2, "audio_repetition_window": 64}
-
-    # the audio side is the LFM3-Audio model under the HF-native names
-    _audio_renames = (
-        ("model.audio_encoder.lfm.", "encoder.lfm.lfm."),
-        ("model.audio_encoder.", "encoder."),
-        ("model.audio_projector.layer_norm.", "mlp.model.0."),
-        ("model.audio_projector.linear_1.", "mlp.model.1."),
-        ("model.audio_projector.linear_2.", "mlp.model.3."),
-        ("model.depth_decoder.model.", "depthformer.model."),
-        ("model.depth_decoder.", ""),
-        ("model.codebook_embedding.", "codebook_embedding."),
-    )
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        assert self.hparams_vision is not None
-        # dynamic resolution, image_size is only set for MmprojModel compatibility
-        self.hparams_vision["image_size"] = 512
-
-    def get_audio_config(self) -> dict[str, Any] | None:
-        enc = self.global_config["audio_encoder_config"]
-        cfg = dict(enc["text_config"])
-        cfg["num_mel_bins"] = enc["num_mel_bins"]
-        cfg["sliding_window"] = enc["sliding_window"]
-        return cfg
-
-    def get_generation_config(self) -> dict[str, Any]:
-        c = self.global_config
-        return {
-            "codebooks": c["num_codebooks"], "audio_vocab_size": c["codebook_size"], "maskgit_schedule": c["maskgit_schedule"],
-            "interleaved_n_text": c["interleaved_n_text"], "interleaved_n_audio": c["interleaved_n_audio"],
-        }
-
-    def get_depth_config(self) -> dict[str, Any]:
-        return self.global_config["depth_config"]
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        self._set_lfm3vl_gguf_parameters()
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.startswith(("model.vision_tower.", "model.multi_modal_projector.")):
-            yield from self._lfm3vl_tensors(data_torch, name, bid)
-            return
-        for old, new in self._audio_renames:
-            if name.startswith(old):
-                yield from super().modify_tensors(data_torch, new + name.removeprefix(old), bid)
-                return
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/qwen.py

@@ -690,52 +690,6 @@ class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
         self._try_set_pooling_type()
 
 
-@ModelBase.register("MiniCPMOCRForSequenceClassification")
-class MiniCPMOCRSeqClsModel(Qwen3_5TextModel):
-    # OCR consistency judge: Qwen3.5 VL trunk with a value head read at the last token
-    # the class scores are [inconsistent, consistent], the softmax temperature is 1
-    model_arch = gguf.MODEL_ARCH.QWEN35
-    no_mtp = True  # the checkpoint has no MTP head
-
-    CLASS_LABELS = ["inconsistent", "consistent"]
-
-    def set_vocab(self):
-        super().set_vocab()
-        self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])
-
-    def _systemone_template(self) -> str:
-        # the prompt of the training data, the two images are the original page and the one
-        # rendered from the OCR text; the task is fixed, there is nothing to fill in
-        # the trailing newlines are in an expression, so the template does not drop them
-        return (
-            "<|im_start|>user\n{{ images[0] }}\nOCR文本：\n{{ state }}\n{{ images[1] }}\n"
-            "第一张图是原图，第二张图是上述 OCR 文本渲染得到的图片。\n"
-            "请判断该 OCR 文本及其对应渲染图是否与原图内容一致。\n"
-            "这是一个二分类任务：一致标为 1，不一致标为 0。"
-            "{{ '<|im_end|>\\n<|im_start|>assistant\\n<think>\\n\\n</think>\\n\\n' }}"
-        )
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-
-        # the config.json of the checkpoint keeps the one of the base model, it has no num_labels / id2label
-        if "score.weight" not in self.model_tensors:
-            raise ValueError("the checkpoint has no score.weight classification head")
-
-        self.gguf_writer.add_pooling_type(gguf.PoolingType.LAST)
-        self.gguf_writer.add_embedding_length_out(len(self.CLASS_LABELS))
-        self.gguf_writer.add_classifier_output_labels(self.CLASS_LABELS)
-        self.gguf_writer.add_decision_type(gguf.DecisionType.MINICPM_OCR_JEV)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name == "score.weight":
-            if data_torch.shape[0] != len(self.CLASS_LABELS):
-                raise ValueError(f"score.weight must have {len(self.CLASS_LABELS)} rows, got {data_torch.shape[0]}")
-            yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.CLS_OUT] + ".weight", data_torch)
-            return
-        yield from super().modify_tensors(data_torch, name, bid)
-
-
 def _is_openjev_checkpoint(dir_model: Path) -> bool:
     return (dir_model / "helper" / "shim.py").is_file() and (dir_model / "config.json").is_file()
 

conversion/qwen3vl.py

@@ -144,16 +144,6 @@ class Qwen3VLVisionModel(MmprojModel):
         yield from MmprojModel.modify_tensors(self, data_torch, name, bid)
 
 
-@ModelBase.register("MiniCPMOCRForSequenceClassification")
-class MiniCPMOCRSeqClsVisionModel(Qwen3VLVisionModel):
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        # trained with the image budget of its processor, wider than the mtmd default of Qwen-VL
-        size = self.preprocessor_config["size"]
-        self.gguf_writer.add_vision_min_pixels(int(size["shortest_edge"]))
-        self.gguf_writer.add_vision_max_pixels(int(size["longest_edge"]))
-
-
 @ModelBase.register("Qwen3OmniMoeForConditionalGeneration")
 @ModelBase.example("Qwen/Qwen3-Omni-30B-A3B-Instruct")
 class Qwen3OmniMmprojModel(Qwen3VLVisionModel, Qwen25AudioModel):

conversion/qwen4exp.py

@@ -41,8 +41,6 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
         self._ple_shards: dict[int, str] = {}
         self._ple_row_dim: int | None = None
         self._mtp_fc: dict[str, Tensor] = {}
-        # halves of the merged weights, keyed by (bid, merged tensor), see _merge_shared_input
-        self._merge_parts: dict[tuple[int, gguf.MODEL_TENSOR], dict[int, Tensor]] = {}
 
     @classmethod
     def filter_tensors(cls, item):
@@ -133,79 +131,44 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
         # int64 hash constants must stay exact; 1-D tensors force F32, so use KV
         if name.endswith("ple_embedding.layer_multipliers"):
             self._ple_multipliers = [int(x) for x in data_torch.tolist()]
-            return
+            return []
         if name.endswith("ple_embedding.ngram_heads_offsets"):
             self._ple_head_offsets = [int(x) for x in data_torch.tolist()]
-            return
+            return []
         if name.endswith("ple_embedding.ngram_heads_vocab_sizes"):
             self._ple_head_vocab_sizes = [int(x) for x in data_torch.tolist()]
-            return
+            return []
 
         if ".ngram_embedding.shard_" in name:
-            yield from self._place_ple_shard(data_torch, name)
-            return
+            return self._place_ple_shard(data_torch, name)
 
         # eh_proj([e ; h_s]) = fc_embedding(e) + fc_hidden(h_s) for every hc stream s
         if name.endswith((".nextn_fc_embedding.weight", ".nextn_fc_hidden.weight")):
             self._mtp_fc[name.rsplit(".", 2)[1]] = data_torch
             if len(self._mtp_fc) < 2:
-                return
+                return []
             eh = torch.cat([self._mtp_fc.pop("nextn_fc_embedding"), self._mtp_fc.pop("nextn_fc_hidden")], dim=1)
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.NEXTN_EH_PROJ, bid, ".weight"), eh)
-            return
+            return [(self.format_tensor_name(gguf.MODEL_TENSOR.NEXTN_EH_PROJ, bid, ".weight"), eh)]
 
         # one projection feeds indexer q and k; split it, as minimax-m3 does
         if ".indexer.index_qk_proj.weight" in name:
             n_q = self.hparams["indexer_n_heads"] * self.hparams["indexer_head_dim"]
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_Q_PROJ, bid, ".weight"), data_torch[:n_q])
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_K_PROJ, bid, ".weight"), data_torch[n_q:])
-            return
+            q = data_torch[:n_q]
+            k = data_torch[n_q:]
+            return [
+                (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_Q_PROJ, bid, ".weight"), q),
+                (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_K_PROJ, bid, ".weight"), k),
+            ]
 
         # Gemma zero-centred gammas the inherited norm.weight rule misses
         if name.endswith((".ple.norm_key.weight", ".ple.norm_query.weight", ".ple.norm_conv.weight",
                           ".indexer.q_layernorm.weight", ".indexer.k_layernorm.weight")):
-            yield (self.map_tensor_name(name), data_torch + 1)
-            return
+            return [(self.map_tensor_name(name), data_torch + 1)]
 
         if name.endswith(".ple.conv1d.weight"):
-            yield (self.map_tensor_name(name), data_torch.squeeze())
-            return
+            return [(self.map_tensor_name(name), data_torch.squeeze())]
 
-        for new_name, data in super().modify_tensors(data_torch, name, bid):
-            yield from self._merge_shared_input(new_name, data, bid)
-
-    # matmuls that read the same input, written as one weight [a | b] so the runtime does one matmul:
-    # (merged tensor, (a, b), linear-attention layers only)
-    _SHARED_INPUT_MERGES = (
-        (gguf.MODEL_TENSOR.HC_ATTN_DOWN_INJECT, (gguf.MODEL_TENSOR.HC_ATTN_DOWN, gguf.MODEL_TENSOR.HC_ATTN_INJECT), False),
-        (gguf.MODEL_TENSOR.HC_FFN_DOWN_INJECT,  (gguf.MODEL_TENSOR.HC_FFN_DOWN,  gguf.MODEL_TENSOR.HC_FFN_INJECT),  False),
-        (gguf.MODEL_TENSOR.ATTN_QKV_GATE,       (gguf.MODEL_TENSOR.ATTN_QKV,     gguf.MODEL_TENSOR.ATTN_GATE),      True),
-        (gguf.MODEL_TENSOR.SSM_BETA_ALPHA_CAT,  (gguf.MODEL_TENSOR.SSM_BETA,     gguf.MODEL_TENSOR.SSM_ALPHA),      True),
-    )
-
-    # caches one half of a merged weight until the other half arrives, then yields [a | b]
-    def _merge_shared_input(self, new_name: str, data: Tensor, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # a pre-quantized checkpoint carries per-tensor scales, which one merged weight cannot hold
-        if bid is None or self.hparams.get("quantization_config"):
-            yield (new_name, data)
-            return
-
-        layer_types = self.hparams.get("layer_types") or []
-        is_linear = bid < len(layer_types) and layer_types[bid] == "linear_attention"
-
-        match = next(((merged, parts.index(part)) for merged, parts, linear_only in self._SHARED_INPUT_MERGES
-                      for part in parts
-                      if (is_linear or not linear_only) and self.match_model_tensor_name(new_name, part, bid)), None)
-        if match is None:
-            yield (new_name, data)
-            return
-
-        merged, idx = match
-        halves = self._merge_parts.setdefault((bid, merged), {})
-        halves[idx] = data
-        if len(halves) == 2:
-            del self._merge_parts[(bid, merged)]
-            yield (self.format_tensor_name(merged, bid), torch.cat([halves[0], halves[1]], dim=0))
+        return super().modify_tensors(data_torch, name, bid)
 
     # the shards concatenate into a tensor of well over 100 GB
     # use LazyChunkedTensor here, a single shard resident at a time
@@ -251,8 +214,6 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
         super().prepare_tensors()
         if self._mtp_fc:
             raise ValueError(f"MTP projection missing its other half: {sorted(self._mtp_fc)}")
-        if self._merge_parts:
-            raise ValueError(f"merged weights missing a half: {sorted((bid, t.name) for bid, t in self._merge_parts)}")
         n_parts = self.hparams.get("split_ngram_parts", 0)
         if self._ple_shards and len(self._ple_shards) != n_parts:
             raise ValueError(

docs/backend/OPENCL.md

@@ -72,8 +72,6 @@ However, A6x GPUs in phones are likely not supported due to the outdated driver
 | MXFP4                  | Support                    |
 | IQ4_NL                 | Support                    |
 
-`MUL_MAT_ID` accepts an expert id of `-1` and writes zeros to that slot. `ADD_ID` adds no bias for `-1`, including in fused MoE operations.
-
 ## Model Preparation
 
 Since common quantizations are supported now, it is recommanded to download GGUF models directly from Huggingface.

docs/backend/snapdragon/README.md

@@ -324,9 +324,6 @@ on 4 physical NPUs, or `--devices 'HTP0[0-1:0],HTP1[0-1:1]'` on 2 physical NPUs
 - `GGML_HEXAGON_NHVX=0`
   Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version).
 
-- `GGML_HEXAGON_VTCM_SIZE=0`
-  Overrides the VTCM size used by each session. Accepts `K`, `M` and `G` suffixes; a plain number is in MB. The default (0) uses the size the hardware reports. Values above the hardware size will fail to start the session.
-
 - `GGML_HEXAGON_HOSTBUF=1` (default: 0, disabled)
   Enables allocating host buffers for debugging. By default, host buffers are disabled.
 

examples/realtime_client.py

@@ -1,172 +0,0 @@
-#!/usr/bin/env python3
-"""Example client for the llama-server /v1/realtime websocket, with the openai SDK.
-
-Start a server with an audio model, e.g. LFM3-Audio:
-
-    llama-server -hf LiquidAI/LFM3-Audio-2B-GGUF
-
-Spoken conversation, the audio of each wav file is one user turn. It is sent in chunks as it would come from a
-microphone, the server prefills it as it arrives so the response starts right after the last chunk:
-
-    python examples/realtime_client.py --instructions "Respond with interleaved text and audio." question.wav
-
-Output modalities: left out, the model picks; "text", "audio" or "text,audio" (interleaved) force the response.
-Without --instructions, a forced mode comes with the system prompt of the LFM3-Audio or LFM3-Omni model card:
-
-    python examples/realtime_client.py --output-modalities text,audio --out answer.wav question.wav
-
-Text turns go in with --text, transcription sessions transcribe each file:
-
-    python examples/realtime_client.py --text "What is the capital of France?"
-    python examples/realtime_client.py --type transcription --language en speech.wav
-
-Live transcription models (Voxtral Realtime, LFM3-Audio live ASR) transcribe while the audio is sent, a transcription
-session opens with ?intent=transcription:
-
-    python examples/realtime_client.py --type transcription speech.wav
-
-Requirements: pip install "openai[realtime]" numpy
-"""
-
-import argparse
-import asyncio
-import base64
-import time
-import wave
-
-import numpy as np
-from openai import AsyncOpenAI
-
-
-# system prompts of the LiquidAI/LFM3-Audio-2B and LiquidAI/LFM3-Omni-2B model cards for a forced output mode
-MODE_INSTRUCTIONS = {
-    "audio": {"text,audio": "Respond with interleaved text and audio."},
-    "omni":  {"text": "Respond in text.", "text,audio": "Respond in interleaved text and audio."},
-}
-
-
-def read_wav(path, rate):
-    # PCM16 wav -> mono int16 at the given rate
-    with wave.open(path, "rb") as f:
-        assert f.getsampwidth() == 2, "the example reads PCM16 wav files"
-        x = np.frombuffer(f.readframes(f.getnframes()), dtype=np.int16).astype(np.float32)
-        x = x.reshape(-1, f.getnchannels()).mean(axis=1)
-        sr = f.getframerate()
-    if sr != rate:
-        n = int(round(len(x) * rate / sr))
-        x = np.interp(np.arange(n) * sr / rate, np.arange(len(x)), x)
-    return np.clip(x, -32768, 32767).astype(np.int16)
-
-
-def write_wav(path, pcm, rate):
-    with wave.open(path, "wb") as f:
-        f.setnchannels(1)
-        f.setsampwidth(2)
-        f.setframerate(rate)
-        f.writeframes(pcm.tobytes())
-
-
-async def main(args):
-    # the SDK derives a wss:// URL from base_url, a plain http server needs ws://
-    client = AsyncOpenAI(base_url=args.url, api_key=args.api_key, websocket_base_url=args.url.replace("http", "ws", 1))
-    query = {"intent": "transcription"} if args.type == "transcription" else {}
-    async with client.realtime.connect(model=args.model, extra_query=query) as conn:
-        ev = await conn.recv()
-        if ev.type == "error":
-            print("error:", ev.error.message)
-            return
-        # the audio goes in at the rate of the model, PCM16 mono
-        session_info = ev.model_dump(warnings=False)["session"]
-        audio_fmt = session_info["audio"]
-        card = "omni" if "omni" in str(session_info.get("model", "")).lower() else "audio"
-        in_rate = audio_fmt["input"]["format"]["rate"]
-        out_rate = (audio_fmt.get("output") or {}).get("format", {}).get("rate", 24000)
-
-        session = {"type": args.type}
-        if args.type == "realtime":
-            instructions = args.instructions
-            if instructions is None:
-                instructions = MODE_INSTRUCTIONS[card].get(args.output_modalities)
-            if instructions is not None:
-                session["instructions"] = instructions
-            if args.output_modalities:
-                session["output_modalities"] = args.output_modalities.split(",")
-        elif args.language:
-            session["audio"] = {"input": {"transcription": {"language": args.language}}}
-        await conn.session.update(session=session)  # ty: ignore[invalid-argument-type]
-        ev = await conn.recv()
-        if ev.type == "error":
-            print("error:", ev.error.message)
-            return
-
-        turns = [("audio", path) for path in args.wav] + [("text", text) for text in args.text]
-        for i, (kind, src) in enumerate(turns):
-            t_last = 0.0
-
-            # the input is sent while the events are read: live transcription arrives while the audio is sent
-            async def send_turn():
-                nonlocal t_last
-                if kind == "audio":
-                    pcm = read_wav(src, in_rate)
-                    step = in_rate * args.chunk_ms // 1000
-                    for j in range(0, len(pcm), step):
-                        chunk = pcm[j:j + step]
-                        await conn.input_audio_buffer.append(audio=base64.b64encode(chunk.tobytes()).decode())
-                        if args.realtime:
-                            await asyncio.sleep(len(chunk) / in_rate)
-                    if args.type == "transcription":
-                        # "final" ends the input, models that transcribe with a delay then complete the last item
-                        await conn.send({"type": "input_audio_buffer.commit", "final": i == len(args.wav) - 1})  # ty: ignore[invalid-argument-type]
-                else:
-                    await conn.conversation.item.create(item={
-                        "type": "message", "role": "user", "content": [{"type": "input_text", "text": src}]})
-                if args.type == "realtime":
-                    # the audio not committed yet ends the user turn
-                    await conn.response.create()
-                t_last = time.perf_counter()
-
-            print(f"--- turn {i} ({kind}: {src})")
-            sender = asyncio.create_task(send_turn())
-            t_first = None
-            audio = []
-            async for ev in conn:
-                if ev.type == "response.output_text.delta" or ev.type == "conversation.item.input_audio_transcription.delta":
-                    t_first = t_first or time.perf_counter()
-                    print(ev.delta, end="", flush=True)
-                elif ev.type == "response.output_audio.delta":
-                    t_first = t_first or time.perf_counter()
-                    audio.append(np.frombuffer(base64.b64decode(ev.delta), dtype=np.int16))
-                elif ev.type in ("response.done", "conversation.item.input_audio_transcription.completed"):
-                    break
-                elif ev.type == "error":
-                    print("\nerror:", ev.error.message)
-                    return
-            await sender
-            print()
-            if t_first is not None and t_first > t_last:
-                print(f"first output {1000 * (t_first - t_last):.0f} ms after the end of the input")
-            elif t_first is not None:
-                print(f"output started {1000 * (t_last - t_first):.0f} ms before the end of the input, "
-                      f"done {1000 * (time.perf_counter() - t_last):.0f} ms after it")
-            if audio:
-                pcm_out = np.concatenate(audio)
-                path = f"{args.out.removesuffix('.wav')}_{i}.wav" if len(turns) > 1 else args.out
-                write_wav(path, pcm_out, out_rate)
-                print(f"audio: {len(pcm_out) / out_rate:.2f} s -> {path}")
-
-
-if __name__ == "__main__":
-    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    p.add_argument("wav", nargs="*", help="PCM16 wav files, one user turn each")
-    p.add_argument("--text", action="append", default=[], help="a user turn as text, after the audio turns, repeatable")
-    p.add_argument("--type", choices=["realtime", "transcription"], default="realtime", help="session type")
-    p.add_argument("--instructions", help="system prompt of the conversation")
-    p.add_argument("--output-modalities", choices=["text", "audio", "text,audio"], help="default: the model picks")
-    p.add_argument("--language", help="transcription language, e.g. en")
-    p.add_argument("--chunk-ms", type=int, default=100, help="audio chunk size")
-    p.add_argument("--realtime", action=argparse.BooleanOptionalAction, default=True, help="send the audio in real time")
-    p.add_argument("--out", default="response.wav", help="where the audio of the responses goes")
-    p.add_argument("--url", default="http://127.0.0.1:9931/v1")
-    p.add_argument("--api-key", default="none")
-    p.add_argument("--model", default="default", help="model name, used by the router")
-    asyncio.run(main(p.parse_args()))

ggml/src/ggml-cpu/arch/arm/quants.c

@@ -287,9 +287,6 @@ void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
 
     sumf = vaddvq_f32(sumv);
 #else
-    UNUSED(nb);
-    UNUSED(x);
-    UNUSED(y);
     ggml_vec_dot_q2_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
     return;
 #endif

ggml/src/ggml-cpu/ops.cpp

@@ -1914,87 +1914,213 @@ void ggml_compute_forward_repeat_back(
 }
 
 // ggml_compute_forward_concat
-
-// rows of dst copied together, so that a strided source is read along neighboring rows
-static constexpr int64_t GGML_CONCAT_TILE = 32;
-
-// copy n_rows rows of n elements of es bytes from a source with element stride nb0, element by element
-template <size_t es>
-static void ggml_concat_copy_strided(char * y, size_t nby, const char * x, size_t nbx, size_t nb0, int64_t n, int64_t n_rows) {
-    for (int64_t i0 = 0; i0 < n; i0++) {
-        for (int64_t r = 0; r < n_rows; r++) {
-            memcpy(y + r*nby + i0*es, x + r*nbx + i0*nb0, es);
-        }
-    }
-}
-
-// copy n_rows rows of n elements of src into dst, rows are nbx bytes apart in src and nby bytes apart in dst
-static void ggml_concat_copy_rows(char * y, size_t nby, const ggml_tensor * src, const char * x, size_t nbx, int64_t n, int64_t n_rows) {
-    const size_t es = ggml_type_size(src->type);
-
-    if (src->nb[0] == es) {
-        const size_t row_size = ggml_row_size(src->type, n);
-        for (int64_t r = 0; r < n_rows; r++) {
-            memcpy(y + r*nby, x + r*nbx, row_size);
-        }
-        return;
-    }
-
-    switch (es) {
-        case 1: ggml_concat_copy_strided<1>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
-        case 2: ggml_concat_copy_strided<2>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
-        case 4: ggml_concat_copy_strided<4>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
-        case 8: ggml_concat_copy_strided<8>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
-        default: GGML_ABORT("unsupported element size");
-    }
-}
-
-// each dst row comes from src0 or src1, or for dim 0 from the row of src0 followed by the row of src1
-void ggml_compute_forward_concat(
+static void ggml_compute_forward_concat_any(
     const ggml_compute_params * params,
     ggml_tensor * dst) {
 
     const ggml_tensor * src0 = dst->src[0];
     const ggml_tensor * src1 = dst->src[1];
 
+    const int ith = params->ith;
+    const int nth = params->nth;
+
     GGML_TENSOR_BINARY_OP_LOCALS
 
     const int32_t dim = ggml_get_op_params_i32(dst, 0);
 
     GGML_ASSERT(dim >= 0 && dim < 4);
-    GGML_ASSERT(nb0 == ggml_type_size(dst->type));
-    GGML_ASSERT(!ggml_is_quantized(dst->type) || (nb00 == nb0 && nb10 == nb0));
-    GGML_ASSERT(dim != 0 || (ne00 % ggml_blck_size(dst->type) == 0 && ne10 % ggml_blck_size(dst->type) == 0));
+    GGML_ASSERT(ggml_is_contiguous_rows(src0));
+    GGML_ASSERT(ggml_is_contiguous_rows(src1));
 
-    const size_t y1 = dim == 0 ? ggml_row_size(dst->type, ne00) : 0;
+    int64_t o[4] = {0, 0, 0, 0};
 
-    const int64_t nr = ne1*ne2*ne3;
-    const int64_t dr = (nr + params->nth - 1)/params->nth;
+    if (dim == 0) {
+        GGML_ASSERT(src0->ne[0] % ggml_blck_size(src0->type) == 0);
+        GGML_ASSERT(src1->ne[0] % ggml_blck_size(src1->type) == 0);
 
-    const int64_t ir0 = dr*params->ith;
-    const int64_t ir1 = MIN(ir0 + dr, nr);
+        o[dim] = src0->ne[dim]/ggml_blck_size(src0->type);
+    } else {
+        o[dim] = src0->ne[dim];
+    }
 
-    for (int64_t ir = ir0; ir < ir1; ) {
-        int64_t i[4] = { 0, ir % ne1, (ir/ne1) % ne2, ir/(ne1*ne2) };
-
-        // a tile of consecutive rows sharing i2, i3 and their source
-        int64_t nt = MIN(MIN(ir1 - ir, ne1 - i[1]), GGML_CONCAT_TILE);
-        if (dim == 1 && i[1] < ne01) {
-            nt = MIN(nt, ne01 - i[1]);
+    // Region 1: copy rows from src0
+    for (int i3 = 0; i3 < ne03; i3++) {
+        for (int i2 = ith; i2 < ne02; i2 += nth) {
+            for (int i1 = 0; i1 < ne01; i1++) {
+                const char * x = (const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03;
+                      char * y = (      char *) dst->data  + i1*nb1  + i2*nb2  + i3*nb3;
+                memcpy(y, x, ggml_row_size(src0->type, ne00));
+            }
         }
+    }
 
-        char * y = (char *) dst->data + i[1]*nb1 + i[2]*nb2 + i[3]*nb3;
-
-        if (dim == 0 || i[dim] < src0->ne[dim]) {
-            ggml_concat_copy_rows(y, nb1, src0, (const char *) src0->data + i[1]*nb01 + i[2]*nb02 + i[3]*nb03, nb01, ne00, nt);
+    // Region 2: copy rows from src1, offset into dst by o[]
+    for (int i3 = 0; i3 < ne13; i3++) {
+        for (int i2 = ith; i2 < ne12; i2 += nth) {
+            for (int i1 = 0; i1 < ne11; i1++) {
+                const char * x = (const char *) src1->data + i1*nb11         + i2*nb12         + i3*nb13;
+                      char * y = (      char *)  dst->data + (i1 + o[1])*nb1 + (i2 + o[2])*nb2 + (i3 + o[3])*nb3 + o[0]*nb0;
+                memcpy(y, x, ggml_row_size(src1->type, ne10));
+            }
         }
+    }
+}
 
-        if (dim == 0 || i[dim] >= src0->ne[dim]) {
-            i[dim] -= src0->ne[dim]; // i[0] does not address rows
-            ggml_concat_copy_rows(y + y1, nb1, src1, (const char *) src1->data + i[1]*nb11 + i[2]*nb12 + i[3]*nb13, nb11, ne10, nt);
+static void ggml_compute_forward_concat_i8(
+    const ggml_compute_params * params,
+    ggml_tensor * dst) {
+
+    const ggml_tensor * src0 = dst->src[0];
+    const ggml_tensor * src1 = dst->src[1];
+
+    GGML_ASSERT(ggml_type_size(src0->type) == sizeof(int8_t));
+
+    const int ith = params->ith;
+    const int nth = params->nth;
+
+    GGML_TENSOR_BINARY_OP_LOCALS
+
+    const int32_t dim = ggml_get_op_params_i32(dst, 0);
+
+    GGML_ASSERT(dim >= 0 && dim < 4);
+
+    int64_t o[4] = {0, 0, 0, 0};
+    o[dim] = src0->ne[dim];
+
+    const int8_t * x;
+
+    // TODO: smarter multi-theading
+    for (int i3 = 0; i3 < ne3; i3++) {
+        for (int i2 = ith; i2 < ne2; i2 += nth) {
+            for (int i1 = 0; i1 < ne1; i1++) {
+                for (int i0 = 0; i0 < ne0; i0++) {
+                    if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
+                        x = (const int8_t *) ((const char *)src0->data + (i0       )*nb00 + (i1       )*nb01 + (i2       )*nb02 + (i3       )*nb03);
+                    } else {
+                        x = (const int8_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
+                    }
+
+                    int8_t * y = (int8_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
+
+                    *y = *x;
+                }
+            }
         }
+    }
+}
 
-        ir += nt;
+static void ggml_compute_forward_concat_f16(
+    const ggml_compute_params * params,
+    ggml_tensor * dst) {
+
+    const ggml_tensor * src0 = dst->src[0];
+    const ggml_tensor * src1 = dst->src[1];
+
+    GGML_ASSERT(ggml_type_size(src0->type) == sizeof(ggml_fp16_t));
+
+    const int ith = params->ith;
+    const int nth = params->nth;
+
+    GGML_TENSOR_BINARY_OP_LOCALS
+
+    const int32_t dim = ggml_get_op_params_i32(dst, 0);
+
+    GGML_ASSERT(dim >= 0 && dim < 4);
+
+    int64_t o[4] = {0, 0, 0, 0};
+    o[dim] = src0->ne[dim];
+
+    const ggml_fp16_t * x;
+
+    // TODO: smarter multi-theading
+    for (int i3 = 0; i3 < ne3; i3++) {
+        for (int i2 = ith; i2 < ne2; i2 += nth) {
+            for (int i1 = 0; i1 < ne1; i1++) {
+                for (int i0 = 0; i0 < ne0; i0++) {
+                    if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
+                        x = (const ggml_fp16_t *) ((const char *)src0->data + (i0       )*nb00 + (i1       )*nb01 + (i2       )*nb02 + (i3       )*nb03);
+                    } else {
+                        x = (const ggml_fp16_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
+                    }
+
+                    ggml_fp16_t * y = (ggml_fp16_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
+
+                    *y = *x;
+                }
+            }
+        }
+    }
+}
+
+static void ggml_compute_forward_concat_f32(
+    const ggml_compute_params * params,
+    ggml_tensor * dst) {
+
+    const ggml_tensor * src0 = dst->src[0];
+    const ggml_tensor * src1 = dst->src[1];
+
+    GGML_ASSERT(ggml_type_size(src0->type) == sizeof(float));
+
+    const int ith = params->ith;
+    const int nth = params->nth;
+
+    GGML_TENSOR_BINARY_OP_LOCALS
+
+    const int32_t dim = ggml_get_op_params_i32(dst, 0);
+
+    GGML_ASSERT(dim >= 0 && dim < 4);
+
+    int64_t o[4] = {0, 0, 0, 0};
+    o[dim] = src0->ne[dim];
+
+    const float * x;
+
+    // TODO: smarter multi-theading
+    for (int i3 = 0; i3 < ne3; i3++) {
+        for (int i2 = ith; i2 < ne2; i2 += nth) {
+            for (int i1 = 0; i1 < ne1; i1++) {
+                for (int i0 = 0; i0 < ne0; i0++) {
+                    if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
+                        x = (const float *) ((const char *)src0->data + (i0       )*nb00 + (i1       )*nb01 + (i2       )*nb02 + (i3       )*nb03);
+                    } else {
+                        x = (const float *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
+                    }
+
+                    float * y = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
+
+                    *y = *x;
+                }
+            }
+        }
+    }
+}
+
+void ggml_compute_forward_concat(
+    const ggml_compute_params * params,
+    ggml_tensor * dst) {
+
+    const ggml_tensor * src0 = dst->src[0];
+
+    switch (src0->type) {
+        case GGML_TYPE_F16:
+        case GGML_TYPE_BF16:
+        case GGML_TYPE_I16:
+            {
+                ggml_compute_forward_concat_f16(params, dst);
+            } break;
+        case GGML_TYPE_I8:
+            {
+                ggml_compute_forward_concat_i8(params, dst);
+            } break;
+        case GGML_TYPE_F32:
+        case GGML_TYPE_I32:
+            {
+                ggml_compute_forward_concat_f32(params, dst);
+            } break;
+        default:
+            {
+                ggml_compute_forward_concat_any(params, dst);
+            }
     }
 }
 
@@ -4846,8 +4972,8 @@ static void ggml_compute_forward_scale_f32(
 
     const ggml_tensor * src0 = dst->src[0];
 
-    GGML_ASSERT(ggml_is_contiguous_rows(src0));
-    GGML_ASSERT(ggml_is_contiguous_rows(dst));
+    GGML_ASSERT(ggml_is_contiguous(src0));
+    GGML_ASSERT(ggml_is_contiguous(dst));
     GGML_ASSERT(ggml_are_same_shape(src0, dst));
 
     float s; // scale factor
@@ -4862,8 +4988,6 @@ static void ggml_compute_forward_scale_f32(
     const int nc = src0->ne[0];
     const int nr = ggml_nrows(src0);
 
-    GGML_TENSOR_UNARY_OP_LOCALS
-
     // rows per thread
     const int dr = (nr + nth - 1)/nth;
 
@@ -4871,23 +4995,25 @@ static void ggml_compute_forward_scale_f32(
     const int ir0 = dr*ith;
     const int ir1 = MIN(ir0 + dr, nr);
 
-    for (int i = ir0; i < ir1; i++) {
-        const int64_t i1 = i % ne01;
-        const int64_t i2 = (i / ne01) % ne02;
-        const int64_t i3 = i / (ne01*ne02);
+    const size_t nb01 = src0->nb[1];
 
-        float *       y = (float *)       ((char *) dst->data  + i1*nb1  + i2*nb2  + i3*nb3);
-        const float * x = (const float *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
+    const size_t nb1 = dst->nb[1];
 
-        if (b == 0.0f) {
+    if (b == 0.0f) {
+        for (int i1 = ir0; i1 < ir1; i1++) {
             if (dst->data != src0->data) {
                 // src0 is same shape as dst => same indices
                 // TODO: add x parameter to ggml_vec_scale_f32 and remove this memcpy
-                memcpy(y, x, nc * sizeof(float));
+                memcpy((char *)dst->data + i1*nb1, (char *)src0->data + i1*nb01, nc * sizeof(float));
             }
-            ggml_vec_scale_f32(nc, y, s);
-        } else {
-            ggml_vec_mad1_f32(nc, y, x, s, b);
+            ggml_vec_scale_f32(nc, (float *) ((char *) dst->data + i1*nb1), s);
+        }
+    } else {
+        for (int i1 = ir0; i1 < ir1; i1++) {
+            ggml_vec_mad1_f32(nc,
+                (float *) ((char *) dst->data  + i1*nb1),
+                (float *) ((char *) src0->data + i1*nb1),
+                s, b);
         }
     }
 }
@@ -4898,8 +5024,8 @@ static void ggml_compute_forward_scale_bf16(
 
     const ggml_tensor * src0 = dst->src[0];
 
-    GGML_ASSERT(ggml_is_contiguous_rows(src0));
-    GGML_ASSERT(ggml_is_contiguous_rows(dst));
+    GGML_ASSERT(ggml_is_contiguous(src0));
+    GGML_ASSERT(ggml_is_contiguous(dst));
     GGML_ASSERT(ggml_are_same_shape(src0, dst));
 
     float s; // scale factor
@@ -4914,8 +5040,6 @@ static void ggml_compute_forward_scale_bf16(
     const int nc = src0->ne[0];
     const int nr = ggml_nrows(src0);
 
-    GGML_TENSOR_UNARY_OP_LOCALS
-
     // rows per thread
     const int dr = (nr + nth - 1)/nth;
 
@@ -4923,13 +5047,9 @@ static void ggml_compute_forward_scale_bf16(
     const int ir0 = dr*ith;
     const int ir1 = MIN(ir0 + dr, nr);
 
-    for (int i = ir0; i < ir1; i++) {
-        const int64_t i1 = i % ne01;
-        const int64_t i2 = (i / ne01) % ne02;
-        const int64_t i3 = i / (ne01*ne02);
-
-        const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
-        ggml_bf16_t       * y = (ggml_bf16_t *)       ((char *)       dst->data  + i1*nb1  + i2*nb2  + i3*nb3);
+    for (int i1 = ir0; i1 < ir1; i1++) {
+        const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*src0->nb[1]);
+        ggml_bf16_t       * y = (ggml_bf16_t *)       ((char *)       dst->data  + i1*dst->nb[1]);
         for (int k = 0; k < nc; k++) {
             y[k] = GGML_FP32_TO_BF16(s*GGML_BF16_TO_FP32(x[k]) + b);
         }

ggml/src/ggml-cuda/conv2d.cu

@@ -180,6 +180,7 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
     const int                  end            = int(int64_t(tiles) * (split + 1) / split_k) * BK;
     for (int k0 = begin; k0 < end; k0 += BK) {
         if (k_total % 8 == 0 && uintptr_t(weight) % 16 == 0) {
+#pragma unroll
             for (int i = tid; i < BM * BK / 8; i += nthreads) {
                 const int  row = i / (BK / 8), col = 8 * (i % (BK / 8));
                 const int4 v                 = m0 + row < oc && k0 + col < k_total ?
@@ -188,6 +189,7 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
                 *(int4 *) &a_s[row][col / 2] = v;
             }
         } else {
+#pragma unroll
             for (int i = tid; i < BM * BK / 2; i += nthreads) {
                 const int row = i / (BK / 2), col = 2 * (i % (BK / 2));
                 half      lo = __float2half(0.0f), hi = lo;
@@ -200,6 +202,7 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
                 a_s[row][col / 2] = __halves2half2(lo, hi);
             }
         }
+#pragma unroll
         for (int k = load_row; k < BK; k += nthreads / (BN / 2)) {
             const int ki = k0 + k;
             const int ci = ki / (kw * kh), ky = ki / kw % kh, kx = ki % kw;

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -214,36 +214,52 @@ static cudaError_t ggml_cuda_device_malloc(void ** ptr, size_t size, int device)
 }
 
 #if defined(GGML_USE_HIP)
-static int ggml_cuda_parse_id(const char * devName) {
+static int ggml_cuda_parse_id(char devName[]) {
     // A list of possible Target IDs can be found under the rocclr/clr repo in device.cpp
     // these values are not stable so this is susceptible to breakage
     // https://github.com/ROCm/clr/blob/amd-staging/rocclr/device/device.cpp
-    // Extract the base architecture model ("gfx908:sramecc+:xnack-" -> "908")
-    std::string archName = devName;
-    archName.erase(0, 3);
-    archName = archName.substr(0, archName.find(':'));
+    int archMajor = 0x0;
+    int archMinor = 0x0;
+    int archNum = GGML_CUDA_CC_OFFSET_AMD;
+    int archLen = strlen(devName);
+    char archName[archLen + 1];
 
-    const auto hex = [](const std::string & s) { return (int) strtoul(s.c_str(), nullptr, 16); };
-
-    int cc = 0;
-
-    // tease out the version information
-    if (archName.size() > 8) {
-        // versions labeled generic use '-' as delimiter
-        if (archName.compare(archName.size() - 8, 8, "-generic") == 0) {
-            archName.resize(archName.size() - 8);
-            const size_t sep = archName.find('-');
-            cc = 0x100 * hex(archName.substr(0, sep));
-            if (sep != std::string::npos) {
-                cc += 0x10 * hex(archName.substr(sep + 1));
-            }
-        }
-    } else if (archName.size() >= 3) {
-        // ordinary IDs are the version in hex
-        cc = hex(archName);
+    // strip leading 'gfx' while copying into our buffer
+    if (archLen > 3) {
+        strcpy(archName, &devName[3]);
+        archLen -= 3;
     }
 
-    return GGML_CUDA_CC_OFFSET_AMD + cc;
+    // trim trailing :xnack- or :sramecc- statuses
+    archLen = strcspn(archName, ":");
+    archName[archLen] = '\0';
+
+    // tease out the version information
+    if (archLen > 8) {
+        // versions labeled generic use '-' as delimiter
+        // strip the trailing "-generic" then iterate through what remains
+        if ((strstr(archName, "-generic"))) {
+            archName[archLen - 8] = '\0';
+            char * pch;
+            if ((pch = strtok(archName, "-"))) {
+                archMajor = (int)strtoul(pch, 0, 16);
+                if ((pch = strtok(NULL, "-"))) {
+                    archMinor = 0x10 * (int)strtoul(pch, 0, 16);
+                }
+            }
+        }
+    } else if (archLen >= 3) {
+        // last two digits should be the minor * 0x10 + stepping
+        archMinor = (int)strtoul(&archName[archLen - 2], 0, 16);
+        archName[archLen - 2] = '\0';
+
+        // only the major version remains
+        archMajor = (int)strtoul(archName, 0, 16);
+    }
+    archNum += archMajor * 0x100;
+    archNum += archMinor;
+
+    return archNum;
 }
 #endif // defined(GGML_USE_HIP)
 
@@ -3822,11 +3838,6 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph *                cgraph,
             return false;
         }
 
-        // TODO: make the fused softcap kernel stride aware, it reads src and writes dst as dense
-        if (!ggml_is_contiguous(scale->src[0]) || !ggml_is_contiguous(scale2)) {
-            return false;
-        }
-
         if (ggml_get_unary_op(tanh) != GGML_UNARY_OP_TANH) {
             return false;
         }
@@ -5977,8 +5988,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
         case GGML_OP_LOG:
             return true;
         case GGML_OP_SCALE:
-            return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type &&
-                   ggml_is_contiguous_rows(op->src[0]);
+            return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type;
         case GGML_OP_CLAMP:
             return ggml_is_contiguous_rows(op->src[0]);
         case GGML_OP_ADD:

ggml/src/ggml-cuda/scale.cu

@@ -3,45 +3,23 @@
 
 #define MAX_GRIDDIM_X 0x7FFFFFFF
 
-// src and dst may be views: rows are contiguous, dims 1..3 follow the strides (in elements).
 template <typename T>
-static __global__ void scale_f32(const T * x, T * dst, const float scale, const float bias, const uint32_t k,
-                                 const uint3 ne0, const uint3 ne1, const uint3 ne2,
-                                 const uint32_t s01, const uint32_t s02, const uint32_t s03,
-                                 const uint32_t s1,  const uint32_t s2,  const uint32_t s3) {
+static __global__ void scale_f32(const T * x, T * dst, const float scale, const float bias, const int64_t nelements) {
     ggml_cuda_pdl_lc();
-    const uint32_t i = blockIdx.x * blockDim.x + threadIdx.x;
+    int64_t tid = (int64_t)blockIdx.x * (int64_t)blockDim.x + (int64_t)threadIdx.x;
+    int64_t stride = (int64_t)blockDim.x * (int64_t)gridDim.x;
 
     ggml_cuda_pdl_sync();
-    if (i >= k) {
-        return;
+    for (int64_t i = tid; i < nelements; i += stride) {
+        dst[i] = ggml_cuda_cast<T>(scale * ggml_cuda_cast<float>(x[i]) + bias);
     }
-
-    const uint2 d0 = fast_div_modulo(i,    ne0); // <i / ne0, i0>
-    const uint2 d1 = fast_div_modulo(d0.x, ne1); // <i / (ne0*ne1), i1>
-    const uint2 d2 = fast_div_modulo(d1.x, ne2); // <i3, i2>
-
-    const size_t i_src = d0.y + size_t(d1.y)*s01 + size_t(d2.y)*s02 + size_t(d2.x)*s03;
-    const size_t i_dst = d0.y + size_t(d1.y)*s1  + size_t(d2.y)*s2  + size_t(d2.x)*s3;
-
-    dst[i_dst] = ggml_cuda_cast<T>(scale * ggml_cuda_cast<float>(x[i_src]) + bias);
 }
 
 template <typename T>
-static void scale_f32_cuda(const T * x, T * dst, const float scale, const float bias, const ggml_tensor * src0, const ggml_tensor * t, cudaStream_t stream) {
-    const int64_t k  = ggml_nelements(src0);
-    const size_t  ts = sizeof(T);
-    GGML_ASSERT(k <= std::numeric_limits<uint32_t>::max());
-
-    const uint3 ne0 = init_fastdiv_values(src0->ne[0]);
-    const uint3 ne1 = init_fastdiv_values(src0->ne[1]);
-    const uint3 ne2 = init_fastdiv_values(src0->ne[2]);
-
-    const int64_t num_blocks = (k + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE;
+static void scale_f32_cuda(const T * x, T * dst, const float scale, const float bias, const int64_t nelements, cudaStream_t stream) {
+    const int64_t num_blocks = (nelements + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE;
     const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(MIN(MAX_GRIDDIM_X, num_blocks), CUDA_SCALE_BLOCK_SIZE, 0, stream);
-    ggml_cuda_kernel_launch(scale_f32<T>, launch_params, x, dst, scale, bias, (uint32_t) k, ne0, ne1, ne2,
-        src0->nb[1]/ts, src0->nb[2]/ts, src0->nb[3]/ts,
-        t->nb[1]/ts,    t->nb[2]/ts,    t->nb[3]/ts);
+    ggml_cuda_kernel_launch(scale_f32<T>, launch_params, x, dst, scale, bias, nelements);
 }
 
 void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
@@ -50,7 +28,6 @@ void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
 
     GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16);
     GGML_ASSERT(dst->type == src0->type);
-    GGML_ASSERT(ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(dst));
 
     float scale;
     float bias;
@@ -58,8 +35,8 @@ void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
     memcpy(&bias,  (float *) dst->op_params + 1, sizeof(float));
 
     if (src0->type == GGML_TYPE_BF16) {
-        scale_f32_cuda((const nv_bfloat16 *) src0->data, (nv_bfloat16 *) dst->data, scale, bias, src0, dst, stream);
+        scale_f32_cuda((const nv_bfloat16 *) src0->data, (nv_bfloat16 *) dst->data, scale, bias, ggml_nelements(src0), stream);
     } else {
-        scale_f32_cuda((const float *) src0->data, (float *) dst->data, scale, bias, src0, dst, stream);
+        scale_f32_cuda((const float *) src0->data, (float *) dst->data, scale, bias, ggml_nelements(src0), stream);
     }
 }

ggml/src/ggml-hexagon/ggml-hexagon.cpp

@@ -85,7 +85,6 @@ int    opt_arch       = 0; // autodetect
 size_t opt_ndev       = 1;
 size_t opt_nhvx       = 0; // use all
 int    opt_nhmx       = 1; // when set, enable HMX; when 0, use HVX only
-size_t opt_vtcm_size  = 0; // VTCM size override in bytes; 0 = auto-detect
 size_t opt_vmem       = HTP_OP_MAX_VMEM_DEFAULT;  // max available va space for buffer mappings
 int    opt_etm        = 0;
 int    opt_verbose    = 0;
@@ -2153,12 +2152,6 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
             this->n_hvx     = this->n_threads;
             this->n_hmx     = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0;
             this->vtcm_size = (uint64_t)hw_vtcm_size;
-            if (opt_vtcm_size > 0) {
-                if (opt_vtcm_size > hw_vtcm_size) {
-                    GGML_LOG_WARN("ggml-hex: %s VTCM override %zu exceeds HW size %llu\n", this->c_name(), opt_vtcm_size, hw_vtcm_size);
-                }
-                this->vtcm_size = opt_vtcm_size;
-            }
             GGML_LOG_INFO("ggml-hex: %s hwinfo: threads %u, hvx %u, hmx %u, vtcm %llu MB\n",
                           this->c_name(), this->n_threads, this->n_hvx, this->n_hmx,
                           (unsigned long long)(this->vtcm_size / (1024 * 1024)));
@@ -2168,7 +2161,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
             this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads;
             this->n_hvx     = this->n_threads;
             this->n_hmx     = (opt_nhmx != 0) ? 1 : 0;
-            this->vtcm_size = opt_vtcm_size > 0 ? opt_vtcm_size : 8 * 1024 * 1024;
+            this->vtcm_size = 8 * 1024 * 1024;
         }
     }
 
@@ -2237,8 +2230,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
     this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem);
 
     // Start dspqueue/opbatch processing
-    err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, opt_vtcm_size, this->max_vmem,
-                          (uint32_t) opt_power_mode);
+    err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, this->max_vmem, (uint32_t) opt_power_mode);
     if (err != 0) {
         GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err);
         throw std::runtime_error("ggml-hex: iface start failed (see log for details)");
@@ -3399,7 +3391,6 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
     const char * str_etm      = getenv("GGML_HEXAGON_ETM");
     const char * str_nhvx     = getenv("GGML_HEXAGON_NHVX");
     const char * str_nhmx     = getenv("GGML_HEXAGON_NHMX");
-    const char * str_vtcm_size = getenv("GGML_HEXAGON_VTCM_SIZE");
     const char * str_mm_select = getenv("GGML_HEXAGON_MM_SELECT");
     const char * str_fa_select = getenv("GGML_HEXAGON_FA_SELECT");
     const char * str_fa_split  = getenv("GGML_HEXAGON_FA_SPLIT");
@@ -3465,7 +3456,6 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
     }
     opt_nhvx      = str_nhvx     ? strtoul(str_nhvx, NULL, 0)             : opt_nhvx;
     opt_nhmx      = str_nhmx     ? atoi(str_nhmx)                         : opt_nhmx;
-    opt_vtcm_size = str_vtcm_size ? parse_size(str_vtcm_size)                 : opt_vtcm_size;
     opt_mm_select = str_mm_select ? atoi(str_mm_select)                   : opt_mm_select;
     opt_fa_select = str_fa_select ? atoi(str_fa_select)                   : opt_fa_select;
     opt_fa_split  = str_fa_split  ? atoi(str_fa_split)                   : opt_fa_split;

ggml/src/ggml-hexagon/htp/htp_iface.idl

@@ -11,7 +11,7 @@ struct htp_iface_pmu_conf {
 };
 
 interface htp_iface : remote_handle64 {
-    AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 vtcm_size, in uint64 max_vmem, in uint32 power_mode);
+    AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 max_vmem, in uint32 power_mode);
     AEEResult stop();
     AEEResult mmap(in uint32 fd, in uint64 size);
     AEEResult munmap(in uint32 fd);

ggml/src/ggml-hexagon/htp/main.c

@@ -325,11 +325,8 @@ static unsigned int vtcm_query_size(void) {
     return vtcm_size;
 }
 
-static int vtcm_alloc(struct htp_context * ctx, uint64_t size_override) {
+static int vtcm_alloc(struct htp_context * ctx) {
     unsigned int vtcm_size = vtcm_query_size();
-    if (size_override) {
-        vtcm_size = (unsigned int) size_override;
-    }
 
     compute_res_attr_t attr;
     HAP_compute_res_attr_init(&attr);
@@ -441,7 +438,7 @@ static const struct htp_power_profile htp_power_profiles[HTP_POWER_MODE_COUNT] =
 
 #undef HTP_POWER_PROFILE_ENTRY
 
-AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t vtcm_size, uint64_t max_vmem, uint32_t power_mode) {
+AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t max_vmem, uint32_t power_mode) {
     struct htp_handle * h = (struct htp_handle *) handle;
     if (!h) {
         return AEE_EBADPARM;
@@ -560,7 +557,7 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp
     ctx->max_vmem    = max_vmem;
     ctx->dsp_queue   = dsp_queue;
 
-    err = vtcm_alloc(ctx, vtcm_size);
+    err = vtcm_alloc(ctx);
     if (err != AEE_SUCCESS) {
         FARF(ERROR, "Unable to allocate VTCM");
         htp_iface_stop(handle);

ggml/src/ggml-hexagon/htp/matmul-ops.c

@@ -16,7 +16,6 @@
 #include "hvx-dump.h"
 #include "hvx-arith.h"
 #include "hvx-reduce.h"
-#include "hvx-copy.h"
 
 #define GGML_COMMON_DECL_C
 #include "ggml-common.h"
@@ -1414,7 +1413,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) {
 
     for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) {  // for each expert
         const int32_t eid = *(const int32_t *) ((const uint8_t *) src2->data + ie1 * src2->nb[0]);
-        if (eid < 0) {
+        if (eid == -1) {
             continue;
         }
         assert(eid >= 0 && eid < (int32_t) n_ids);
@@ -4112,7 +4111,7 @@ static int hvx_mm_matmul_id_nx(
     return HTP_STATUS_OK;
 }
 
-static __attribute__((noinline)) uint32_t scan_expert_ids_n(
+static inline void scan_expert_ids_n(
     const struct htp_tensor * ids,
     const uint32_t n_ids,
     uint32_t n_as,
@@ -4122,17 +4121,15 @@ static __attribute__((noinline)) uint32_t scan_expert_ids_n(
 ) {
     const size_t ids_nb1 = ids->nb[1];
     const uint8_t * ids_data = (const uint8_t *) ids->data;
-    uint32_t n_skipped = 0;
 
     for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
         const int32_t * row_ptr = (const int32_t *) (ids_data + iid1 * ids_nb1);
         for (uint32_t id = 0; id < n_ids; ++id) {
             const int32_t i02 = row_ptr[id];
-            if (i02 < 0) {
-                n_skipped++;
+            if (i02 == -1) {
                 continue;
             }
-            assert(i02 >= 0 && i02 < (int32_t) n_as);
+            assert(i02 >= 0 && i02 < n_as);
 
             if (matrix_rows) {
                 matrix_rows[i02 * mapping_stride + counts[i02]] = (struct mmid_row_mapping) { id, iid1 };
@@ -4140,10 +4137,31 @@ static __attribute__((noinline)) uint32_t scan_expert_ids_n(
             counts[i02] += 1;
         }
     }
-    return n_skipped;
 }
 
-static __attribute__((noinline)) uint32_t scan_expert_ids(
+// the matmuls only write the rows that an expert owns, so the rows of the skipped slots are zeroed here
+static inline void zero_skipped_dst_rows(
+    const struct htp_tensor * ids,
+    uint32_t n_ids,
+    const struct htp_tensor * dst,
+    uint32_t ne0,
+    size_t nb1,
+    size_t nb2
+) {
+    const uint8_t * ids_data = (const uint8_t *) ids->data;
+
+    for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
+        for (uint32_t id = 0; id < n_ids; ++id) {
+            const int32_t i02 = *(const int32_t *) (ids_data + iid1 * ids->nb[1] + id * ids->nb[0]);
+            if (i02 != -1) {
+                continue;
+            }
+            memset((uint8_t *) (dst->data + id * nb1 + iid1 * nb2), 0, ne0 * sizeof(float));
+        }
+    }
+}
+
+static inline void scan_expert_ids(
     const struct htp_tensor * ids,
     uint32_t n_ids,
     uint32_t n_as,
@@ -4155,25 +4173,23 @@ static __attribute__((noinline)) uint32_t scan_expert_ids(
 
     if (ids_nb0 == 4) {
         switch (n_ids) {
-            case 8:  return scan_expert_ids_n(ids, 8,     n_as, counts, matrix_rows, mapping_stride);
-            case 4:  return scan_expert_ids_n(ids, 4,     n_as, counts, matrix_rows, mapping_stride);
-            case 2:  return scan_expert_ids_n(ids, 2,     n_as, counts, matrix_rows, mapping_stride);
-            default: return scan_expert_ids_n(ids, n_ids, n_as, counts, matrix_rows, mapping_stride);
+            case 8:  scan_expert_ids_n(ids, 8,     n_as, counts, matrix_rows, mapping_stride); break;
+            case 4:  scan_expert_ids_n(ids, 4,     n_as, counts, matrix_rows, mapping_stride); break;
+            case 2:  scan_expert_ids_n(ids, 2,     n_as, counts, matrix_rows, mapping_stride); break;
+            default: scan_expert_ids_n(ids, n_ids, n_as, counts, matrix_rows, mapping_stride); break;
         }
     } else {
         // Strided fallback
         const size_t ids_nb1 = ids->nb[1];
         const uint8_t * ids_data = (const uint8_t *) ids->data;
-        uint32_t n_skipped = 0;
         for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
             const int32_t * row_ptr = (const int32_t *) (ids_data + iid1 * ids_nb1);
             for (uint32_t id = 0; id < n_ids; ++id) {
                 const int32_t i02 = *(const int32_t *) ((const uint8_t *) row_ptr + id * ids_nb0);
-                if (i02 < 0) {
-                    n_skipped++;
+                if (i02 == -1) {
                     continue;
                 }
-                assert(i02 >= 0 && i02 < (int32_t) n_as);
+                assert(i02 >= 0 && i02 < n_as);
 
                 if (matrix_rows) {
                     matrix_rows[i02 * mapping_stride + counts[i02]] = (struct mmid_row_mapping) { id, iid1 };
@@ -4181,124 +4197,9 @@ static __attribute__((noinline)) uint32_t scan_expert_ids(
                 counts[i02] += 1;
             }
         }
-        return n_skipped;
     }
 }
 
-static __attribute__((noinline)) uint32_t count_skipped_ids_single(const struct htp_tensor * ids, uint32_t n_ids) {
-    const uint8_t * ids_data = (const uint8_t *) ids->data;
-    const size_t ids_nb0 = ids->nb[0];
-    uint32_t n_skipped = 0;
-    for (uint32_t id = 0; id < n_ids; ++id) {
-        const int32_t i02 = *(const int32_t *) (ids_data + id * ids_nb0);
-        if (i02 < 0) {
-            n_skipped++;
-        }
-    }
-    return n_skipped;
-}
-
-// Zero rows for skipped slots (ids < 0). Matmuls only write rows that an expert owns.
-static void zero_skipped_dst_rows(
-    struct htp_ops_context * octx,
-    const struct htp_tensor * ids,
-    uint32_t n_ids,
-    const struct htp_tensor * dst
-) {
-    const uint32_t elem_size = (dst->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float);
-    uint32_t col_start = 0;
-    uint32_t col_count = dst->ne[0];
-
-    if (octx->ctx->mdev.count > 1) {
-        const bool can_split = htp_tensor_can_row_partition(dst, elem_size);
-        const uint32_t chunk_elems = (dst->type == HTP_TYPE_F16) ? 64 : 32;
-        const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
-            dst->ne[0], can_split ? chunk_elems : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
-        col_start = range.start;
-        col_count = range.count;
-    }
-
-    if (col_count == 0) {
-        return;
-    }
-
-    const uint8_t * ids_data = (const uint8_t *) ids->data;
-    const size_t ids_nb0 = ids->nb[0];
-    const size_t ids_nb1 = ids->nb[1];
-    const size_t dst_nb1 = dst->nb[1];
-    const size_t dst_nb2 = dst->nb[2];
-    const uint32_t col_offset_bytes = col_start * elem_size;
-
-    for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
-        const uint8_t * ids_row = ids_data + iid1 * ids_nb1;
-        uint8_t * dst_row_group = (uint8_t *) dst->data + iid1 * dst_nb2 + col_offset_bytes;
-        for (uint32_t id = 0; id < n_ids; ++id) {
-            const int32_t i02 = *(const int32_t *) (ids_row + id * ids_nb0);
-            if (i02 < 0) {
-                hvx_splat_u(dst_row_group + id * dst_nb1, Q6_V_vzero(), col_count, elem_size);
-            }
-        }
-    }
-}
-
-static int htp_mmid_prepare_mapping(
-    struct htp_ops_context * octx,
-    const struct htp_tensor * ids,
-    uint32_t n_ids,
-    uint32_t n_as,
-    uint32_t act_nrows,
-    uint8_t ** out_mapping_buf,
-    uint32_t ** out_matrix_row_counts,
-    struct mmid_row_mapping ** out_matrix_rows,
-    uint32_t * out_mapping_stride,
-    uint32_t * out_n_skipped
-) {
-    uint8_t  * mapping_buf       = octx->ctx->ddr_spad_base;
-    uint32_t   mapping_stride    = 1;
-    uint32_t * matrix_row_counts = (uint32_t *) mapping_buf;
-    struct mmid_row_mapping * matrix_rows = NULL;
-    uint32_t n_skipped = 0;
-
-    if (act_nrows > 1) {
-        const size_t matrix_row_counts_size = n_as * sizeof(uint32_t);
-        assert(octx->ctx->ddr_spad_size >= matrix_row_counts_size);
-
-        hex_l2fetch_block((const void *) ids->data, ids->ne[1] * ids->nb[1]);
-
-        hvx_splat_u(matrix_row_counts, Q6_V_vzero(), n_as, sizeof(uint32_t));
-        n_skipped = scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0);
-
-        uint32_t max_count = hvx_reduce_max_i32((const uint8_t *) matrix_row_counts, n_as);
-        mapping_stride = max_count > 0 ? max_count : 1;
-
-        size_t matrix_row_map_size  = n_as * mapping_stride * sizeof(struct mmid_row_mapping);
-        const size_t total_map_size = matrix_row_counts_size + matrix_row_map_size;
-
-        if (total_map_size > octx->ctx->ddr_spad_size) {
-            mapping_buf = memalign(128, total_map_size);
-            if (!mapping_buf) {
-                return HTP_STATUS_INTERNAL_ERR;
-            }
-        }
-
-        matrix_row_counts = (uint32_t *) mapping_buf;
-        matrix_rows       = (struct mmid_row_mapping *) (mapping_buf + matrix_row_counts_size);
-
-        hvx_splat_u(matrix_row_counts, Q6_V_vzero(), n_as, sizeof(uint32_t));
-        scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride);
-    } else {
-        n_skipped = count_skipped_ids_single(ids, n_ids);
-    }
-
-    *out_mapping_buf       = mapping_buf;
-    *out_matrix_row_counts = matrix_row_counts;
-    *out_matrix_rows       = matrix_rows;
-    *out_mapping_stride    = mapping_stride;
-    *out_n_skipped         = n_skipped;
-
-    return HTP_STATUS_OK;
-}
-
 int op_matmul_id(struct htp_ops_context * octx) {
     htp_matmul_tensors_preamble;
 
@@ -4334,22 +4235,41 @@ int op_matmul_id(struct htp_ops_context * octx) {
     const int n_ids = ids->ne[0];  // n_expert_used
     const int n_as  = ne02;        // n_expert
 
-    uint8_t  * mapping_buf       = NULL;
-    uint32_t * matrix_row_counts = NULL;
-    struct mmid_row_mapping * matrix_rows = NULL;
+    uint8_t  * mapping_buf       = octx->ctx->ddr_spad_base;
     uint32_t   mapping_stride    = 1;
-    uint32_t   n_skipped         = 0;
+    uint32_t * matrix_row_counts = (uint32_t *) mapping_buf;
+    struct mmid_row_mapping * matrix_rows = NULL;
 
-    int map_status = htp_mmid_prepare_mapping(octx, ids, n_ids, n_as, act_nrows,
-                                              &mapping_buf, &matrix_row_counts, &matrix_rows,
-                                              &mapping_stride, &n_skipped);
-    if (map_status != HTP_STATUS_OK) {
-        return map_status;
+    if (act_nrows > 1) {
+        const size_t matrix_row_counts_size = n_as * sizeof(uint32_t);
+        assert(octx->ctx->ddr_spad_size >= matrix_row_counts_size);
+
+        hex_l2fetch_block((const void *) ids->data, ids->ne[1] * ids->nb[1]);
+
+        memset(matrix_row_counts, 0, matrix_row_counts_size);
+        scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0);
+
+        uint32_t max_count = hvx_reduce_max_i32((const uint8_t *) matrix_row_counts, n_as);
+        mapping_stride = max_count > 0 ? max_count : 1;
+
+        size_t matrix_row_map_size  = n_as * mapping_stride * sizeof(struct mmid_row_mapping);
+        const size_t total_map_size = matrix_row_counts_size + matrix_row_map_size;
+
+        if (total_map_size > octx->ctx->ddr_spad_size) {
+            mapping_buf = memalign(128, total_map_size);
+            if (!mapping_buf) {
+                return HTP_STATUS_INTERNAL_ERR;
+            }
+        }
+
+        matrix_row_counts = (uint32_t *) mapping_buf;
+        matrix_rows       = (struct mmid_row_mapping *) (mapping_buf + matrix_row_counts_size);
+
+        memset(matrix_row_counts, 0, n_as * sizeof(uint32_t));
+        scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride);
     }
 
-    if (n_skipped > 0) {
-        zero_skipped_dst_rows(octx, ids, n_ids, dst);
-    }
+    zero_skipped_dst_rows(ids, n_ids, dst, ne0, nb1, nb2);
 
     mmctx->matrix_row_counts    = matrix_row_counts;
     mmctx->matrix_rows          = matrix_rows;
@@ -4440,26 +4360,38 @@ int op_matmul_id_nx(struct htp_ops_context * octx) {
     const int n_ids = ids->ne[0];
     const int n_as  = src0->ne[2];
 
-    uint8_t  * mapping_buf       = NULL;
-    uint32_t * matrix_row_counts = NULL;
-    struct mmid_row_mapping * matrix_rows = NULL;
+    uint8_t  * mapping_buf       = octx->ctx->ddr_spad_base;
     uint32_t   mapping_stride    = 1;
-    uint32_t   n_skipped         = 0;
+    uint32_t * matrix_row_counts = (uint32_t *) mapping_buf;
+    struct mmid_row_mapping * matrix_rows = NULL;
 
-    int map_status = htp_mmid_prepare_mapping(octx, ids, n_ids, n_as, act_nrows,
-                                              &mapping_buf, &matrix_row_counts, &matrix_rows,
-                                              &mapping_stride, &n_skipped);
-    if (map_status != HTP_STATUS_OK) {
-        return map_status;
-    }
+    if (act_nrows > 1) {
+        const size_t matrix_row_counts_size = n_as * sizeof(uint32_t);
+        assert(octx->ctx->ddr_spad_size >= matrix_row_counts_size);
 
-    if (n_skipped > 0) {
-        for (uint32_t p = 0; p < n_weights; ++p) {
-            const struct htp_tensor * restrict d = octx->dsts[p];
-            if (d) {
-                zero_skipped_dst_rows(octx, ids, n_ids, d);
+        hex_l2fetch_block((const void *) ids->data, ids->ne[1] * ids->nb[1]);
+
+        memset(matrix_row_counts, 0, matrix_row_counts_size);
+        scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0);
+
+        uint32_t max_count = hvx_reduce_max_i32((const uint8_t *) matrix_row_counts, n_as);
+        mapping_stride = max_count > 0 ? max_count : 1;
+
+        size_t matrix_row_map_size  = n_as * mapping_stride * sizeof(struct mmid_row_mapping);
+        const size_t total_map_size = matrix_row_counts_size + matrix_row_map_size;
+
+        if (total_map_size > octx->ctx->ddr_spad_size) {
+            mapping_buf = memalign(128, total_map_size);
+            if (!mapping_buf) {
+                return HTP_STATUS_INTERNAL_ERR;
             }
         }
+
+        matrix_row_counts = (uint32_t *) mapping_buf;
+        matrix_rows       = (struct mmid_row_mapping *) (mapping_buf + matrix_row_counts_size);
+
+        memset(matrix_row_counts, 0, n_as * sizeof(uint32_t));
+        scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride);
     }
 
     mmctx->matrix_row_counts    = matrix_row_counts;

ggml/src/ggml-metal/ggml-metal-device.cpp

@@ -507,47 +507,6 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexe
     return res;
 }
 
-ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_mask(
-        ggml_metal_library_t lib,
-        const ggml_tensor * op,
-        bool    fill,
-        bool    has_score,
-        bool    has_blocks,
-        bool    has_extra,
-        bool    vec) {
-    GGML_ASSERT(op->op == GGML_OP_TOP_K_MASK);
-
-    char base[256];
-    char name[256];
-
-    snprintf(base, 256, "kernel_top_k_mask%s_%s", fill ? "_fill" : "", ggml_type_name(op->type));
-
-    ggml_metal_pipeline_with_params res;
-
-    ggml_metal_cv_t cv = ggml_metal_cv_init();
-
-    if (fill) {
-        snprintf(name, 256, "%s_vec=%d", base, vec ? 1 : 0);
-
-        ggml_metal_cv_set_bool(cv, vec, FC_TOP_K_MASK + 3);
-    } else {
-        snprintf(name, 256, "%s_sc=%d_bl=%d_ex=%d", base, has_score ? 1 : 0, has_blocks ? 1 : 0, has_extra ? 1 : 0);
-
-        ggml_metal_cv_set_bool(cv, has_score,  FC_TOP_K_MASK + 0);
-        ggml_metal_cv_set_bool(cv, has_blocks, FC_TOP_K_MASK + 1);
-        ggml_metal_cv_set_bool(cv, has_extra,  FC_TOP_K_MASK + 2);
-    }
-
-    res = ggml_metal_library_get_pipeline(lib, name);
-    if (!res.pipeline) {
-        res = ggml_metal_library_compile_pipeline(lib, base, name, cv);
-    }
-
-    ggml_metal_cv_free(cv);
-
-    return res;
-}
-
 ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_metal_library_t lib, const ggml_tensor * op) {
     char name[256];
     const char * base = nullptr;

ggml/src/ggml-metal/ggml-metal-device.h

@@ -126,7 +126,6 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_ad
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_tri               (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max          (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexer (ggml_metal_library_t lib, const struct ggml_tensor * op);
-struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_mask        (ggml_metal_library_t lib, const struct ggml_tensor * op, bool fill, bool has_score, bool has_blocks, bool has_extra, bool vec);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc           (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv          (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t nc, bool use_silu);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched  (ggml_metal_library_t lib, const struct ggml_tensor * op, int ssm_conv_bs, int32_t nc, bool use_silu);

ggml/src/ggml-metal/ggml-metal-device.m

@@ -1555,7 +1555,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
 
     switch (op->op) {
         case GGML_OP_SCALE:
-            return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16);
         case GGML_OP_FILL:
         case GGML_OP_CLAMP:
         case GGML_OP_SQR:
@@ -1835,15 +1834,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
                 ggml_is_contiguous_rows(op->src[1]) &&
                 ggml_is_contiguous_rows(op->src[2]) &&
                 (op->src[3] == NULL || ggml_is_contiguous_rows(op->src[3]));
-        case GGML_OP_TOP_K_MASK:
-            return (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32) &&
-                op->src[0]->type == op->type &&
-                op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) &&
-                op->nb[0] == ggml_type_size(op->type) &&
-                op->src[1]->type == GGML_TYPE_I32 &&
-                (op->src[2] == NULL || op->src[2]->type == GGML_TYPE_F32) &&
-                (op->src[3] == NULL || op->src[3]->type == GGML_TYPE_I32) &&
-                (op->src[4] == NULL || op->src[4]->type == GGML_TYPE_I32);
         case GGML_OP_SSM_SCAN:
             return has_simdgroup_reduction;
         case GGML_OP_SSM_CONV:

ggml/src/ggml-metal/ggml-metal-impl.h

@@ -125,7 +125,6 @@
 #define FC_FLASH_ATTN_EXT_TENSOR       2200
 #define FC_LIGHTNING_INDEXER           2200
 #define FC_MUL_MV_MMA                  2300
-#define FC_TOP_K_MASK                  2400
 
 // op-specific constants
 #define OP_FLASH_ATTN_EXT_NQPSG 8
@@ -1296,38 +1295,6 @@ typedef struct {
     uint64_t nbm3;
 } ggml_metal_kargs_lightning_indexer;
 
-typedef struct {
-    int32_t  n_kv;
-    int32_t  m1;
-    int32_t  m2;
-    int32_t  m3;
-    int32_t  n_top;
-    int32_t  n_pick;
-    int32_t  n_blk;
-    int32_t  n_extra;
-    int64_t  n_elems;
-    uint64_t nb1;
-    uint64_t nb2;
-    uint64_t nb3;
-    uint64_t nbm1;
-    uint64_t nbm2;
-    uint64_t nbm3;
-    uint64_t nbk0;
-    uint64_t nbk1;
-    uint64_t nbk2;
-    uint64_t nbk3;
-    uint64_t nbs0;
-    uint64_t nbs1;
-    uint64_t nbs2;
-    uint64_t nbs3;
-    uint64_t nbb0;
-    uint64_t nbb1;
-    uint64_t nbe0;
-    uint64_t nbe1;
-    uint64_t nbe2;
-    uint64_t nbe3;
-} ggml_metal_kargs_top_k_mask;
-
 typedef struct {
     int32_t  n_tokens;
     int32_t  n_iter;

ggml/src/ggml-metal/ggml-metal-ops.cpp

@@ -345,10 +345,6 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {
             {
                 n_fuse = ggml_metal_op_lightning_indexer(ctx, idx);
             } break;
-        case GGML_OP_TOP_K_MASK:
-            {
-                n_fuse = ggml_metal_op_top_k_mask(ctx, idx);
-            } break;
         case GGML_OP_DSV4_HC_COMB:
         case GGML_OP_DSV4_HC_PRE:
         case GGML_OP_DSV4_HC_POST:
@@ -1428,124 +1424,6 @@ int ggml_metal_op_lightning_indexer(ggml_metal_op_t ctx, int idx) {
     return 1;
 }
 
-int ggml_metal_op_top_k_mask(ggml_metal_op_t ctx, int idx) {
-    ggml_tensor * op = ctx->node(idx);
-
-    ggml_metal_library_t lib = ctx->lib;
-    ggml_metal_encoder_t enc = ctx->enc;
-
-    const ggml_tensor * mask   = op->src[0];
-    const ggml_tensor * top_k  = op->src[1];
-    const ggml_tensor * score  = op->src[2];
-    const ggml_tensor * blocks = op->src[3];
-    const ggml_tensor * extra  = op->src[4];
-
-    GGML_ASSERT(op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32);
-    GGML_ASSERT(mask->type == op->type);
-    GGML_ASSERT(top_k->type == GGML_TYPE_I32);
-    GGML_ASSERT(!score  || score->type  == GGML_TYPE_F32);
-    GGML_ASSERT(!blocks || blocks->type == GGML_TYPE_I32);
-    GGML_ASSERT(!extra  || extra->type  == GGML_TYPE_I32);
-
-    const int32_t n_kv    = mask->ne[0];
-    const int32_t n_top   = (int32_t) top_k->ne[0];
-    const int32_t n_pick  = blocks ? blocks->ne[1] : score ? score->ne[0] : n_kv;
-    const int32_t n_blk   = blocks ? blocks->ne[0] : 0;
-    const int32_t n_extra = extra  ? extra->ne[0]  : 0;
-
-    const ggml_metal_buffer_id bid = ggml_metal_get_buffer_id(op);
-
-    // the vector fill stores 4 elements at a time
-    const int32_t ts      = ggml_type_size(op->type);
-    const int64_t n_elems = ggml_nelements(op);
-
-    const bool has_score  = score  != NULL;
-    const bool has_blocks = blocks != NULL;
-    const bool has_extra  = extra  != NULL;
-
-    const bool vec = n_elems % 4 == 0 && bid.offs % (4*ts) == 0;
-
-    ggml_metal_kargs_top_k_mask args = {
-        /*.n_kv    =*/ n_kv,
-        /*.m1      =*/ (int32_t) mask->ne[1],
-        /*.m2      =*/ (int32_t) mask->ne[2],
-        /*.m3      =*/ (int32_t) mask->ne[3],
-        /*.n_top   =*/ n_top,
-        /*.n_pick  =*/ n_pick,
-        /*.n_blk   =*/ n_blk,
-        /*.n_extra =*/ n_extra,
-        /*.n_elems =*/ n_elems,
-        /*.nb1     =*/ op->nb[1],
-        /*.nb2     =*/ op->nb[2],
-        /*.nb3     =*/ op->nb[3],
-        /*.nbm1    =*/ mask->nb[1],
-        /*.nbm2    =*/ mask->nb[2],
-        /*.nbm3    =*/ mask->nb[3],
-        /*.nbk0    =*/ top_k->nb[0],
-        /*.nbk1    =*/ top_k->nb[1],
-        /*.nbk2    =*/ top_k->nb[2],
-        /*.nbk3    =*/ top_k->nb[3],
-        /*.nbs0    =*/ score  ? score->nb[0]  : 0,
-        /*.nbs1    =*/ score  ? score->nb[1]  : 0,
-        /*.nbs2    =*/ score  ? score->nb[2]  : 0,
-        /*.nbs3    =*/ score  ? score->nb[3]  : 0,
-        /*.nbb0    =*/ blocks ? blocks->nb[0] : 0,
-        /*.nbb1    =*/ blocks ? blocks->nb[1] : 0,
-        /*.nbe0    =*/ extra  ? extra->nb[0]  : 0,
-        /*.nbe1    =*/ extra  ? extra->nb[1]  : 0,
-        /*.nbe2    =*/ extra  ? extra->nb[2]  : 0,
-        /*.nbe3    =*/ extra  ? extra->nb[3]  : 0,
-    };
-
-    // the fill covers the whole dst, the scatter overwrites the kept cells with the mask values
-    {
-        auto pipeline_fill = ggml_metal_library_get_pipeline_top_k_mask(lib, op, true, false, false, false, vec);
-
-        const int64_t n_fill  = n_elems/(vec ? 4 : 1);
-        const int32_t nth_max = ggml_metal_pipeline_max_theads_per_threadgroup(pipeline_fill);
-
-        int nth = 32; // SIMD width
-
-        while (nth < nth_max && nth < n_fill) {
-            nth *= 2;
-        }
-
-        ggml_metal_encoder_set_pipeline(enc, pipeline_fill);
-        ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0);
-        ggml_metal_encoder_set_buffer(enc, bid, 1);
-
-        ggml_metal_encoder_dispatch_threadgroups(enc, (n_fill + nth - 1)/nth, 1, 1, nth, 1, 1);
-
-        ggml_metal_op_concurrency_reset(ctx);
-    }
-
-    {
-        auto pipeline = ggml_metal_library_get_pipeline_top_k_mask(lib, op, false, has_score, has_blocks, has_extra, false);
-
-        const int32_t n_max   = n_top + n_extra;
-        const int32_t nth_max = ggml_metal_pipeline_max_theads_per_threadgroup(pipeline);
-
-        int nth = 32; // SIMD width
-
-        while (nth < nth_max && nth < n_max) {
-            nth *= 2;
-        }
-
-        ggml_metal_encoder_set_pipeline(enc, pipeline);
-        ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0);
-        ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(mask),  1);
-        ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(top_k), 2);
-        ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(score  ? score  : mask), 3);
-        ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(blocks ? blocks : mask), 4);
-        ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(extra  ? extra  : mask), 5);
-        ggml_metal_encoder_set_buffer(enc, bid, 6);
-
-        ggml_metal_encoder_dispatch_threadgroups(enc, op->ne[1], op->ne[2], op->ne[3], nth, 1, 1);
-    }
-
-    return 1;
-}
-
 int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) {
     ggml_tensor * op = ctx->node(idx);
 

ggml/src/ggml-metal/ggml-metal-ops.h

@@ -60,7 +60,6 @@ int ggml_metal_op_get_rows          (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_set_rows          (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_diag              (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_lightning_indexer (ggml_metal_op_t ctx, int idx);
-int ggml_metal_op_top_k_mask        (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_dsv4_hc           (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_soft_max          (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_ssm_conv          (ggml_metal_op_t ctx, int idx);

ggml/src/ggml-metal/kernels/misc.metal

@@ -906,99 +906,3 @@ kernel void kernel_dsv4_hc_post_f32(
         *(device float *) (dst + i0*args.nb_d0 + idst*args.nb_d1 + it*args.nb_d2) = result[idst];
     }
 }
-
-constant bool FC_top_k_mask_has_score  [[function_constant(FC_TOP_K_MASK + 0)]];
-constant bool FC_top_k_mask_has_blocks [[function_constant(FC_TOP_K_MASK + 1)]];
-constant bool FC_top_k_mask_has_extra  [[function_constant(FC_TOP_K_MASK + 2)]];
-constant bool FC_top_k_mask_vec        [[function_constant(FC_TOP_K_MASK + 3)]];
-
-// the fill runs before the scatter, both are fully parallel over the whole dst and over the picks of one row
-template <typename T, typename T4>
-kernel void kernel_top_k_mask_fill(
-        constant ggml_metal_kargs_top_k_mask & args,
-        device char * dst,
-        uint  tid[[thread_position_in_grid]],
-        uint ntot[[threads_per_grid]]) {
-    const T ninf = T(-INFINITY);
-
-    if (FC_top_k_mask_vec) {
-        device T4 * dst_4 = (device T4 *) dst;
-        for (int64_t i = tid; i < args.n_elems/4; i += ntot) {
-            dst_4[i] = T4(ninf);
-        }
-    } else {
-        device T * dst_t = (device T *) dst;
-        for (int64_t i = tid; i < args.n_elems; i += ntot) {
-            dst_t[i] = ninf;
-        }
-    }
-}
-
-typedef decltype(kernel_top_k_mask_fill<half,  half4>)  kernel_top_k_mask_fill_h_t;
-typedef decltype(kernel_top_k_mask_fill<float, float4>) kernel_top_k_mask_fill_f_t;
-
-template [[host_name("kernel_top_k_mask_fill_f16")]] kernel kernel_top_k_mask_fill_h_t kernel_top_k_mask_fill<half,  half4>;
-template [[host_name("kernel_top_k_mask_fill_f32")]] kernel kernel_top_k_mask_fill_f_t kernel_top_k_mask_fill<float, float4>;
-
-// one threadgroup per row: copy the mask cells of the live picks and of the extra cells over the fill
-template <typename T, typename T4>
-kernel void kernel_top_k_mask(
-        constant ggml_metal_kargs_top_k_mask & args,
-        device const  char * mask,
-        device const  char * top_k,
-        device const  char * score,
-        device const  char * blocks,
-        device const  char * extra,
-        device        char * dst,
-        uint3   tgpig[[threadgroup_position_in_grid]],
-        uint3   tpitg[[thread_position_in_threadgroup]],
-        uint3 ntg_max[[threads_per_threadgroup]]) {
-    const int i1 = tgpig.x;
-    const int i2 = tgpig.y;
-    const int i3 = tgpig.z;
-
-    const int   tiitg = tpitg.x;
-    const int     ntg = ntg_max.x;
-
-    device const char * mask_row  = mask  + (i1 % args.m1)*args.nbm1 + (i2 % args.m2)*args.nbm2 + (i3 % args.m3)*args.nbm3;
-    device       char * dst_row   = dst   + i1*args.nb1 + i2*args.nb2 + i3*args.nb3;
-    device const char * top_k_row = top_k + i1*args.nbk1 + i2*args.nbk2 + i3*args.nbk3;
-
-    device const T * mask_row_t = (device const T *) mask_row;
-    device       T * dst_row_t  = (device       T *) dst_row;
-
-    for (int i = tiitg; i < args.n_top; i += ntg) {
-        const int32_t p = *(device const int32_t *) (top_k_row + i*args.nbk0);
-        if (p < 0 || p >= args.n_pick) {
-            continue;
-        }
-        if (FC_top_k_mask_has_score && !(*(device const float *) (score + p*args.nbs0 + i1*args.nbs1 + i2*args.nbs2 + i3*args.nbs3) > -INFINITY)) {
-            continue;
-        }
-        if (FC_top_k_mask_has_blocks) {
-            FOR_UNROLL (int k = 0; k < args.n_blk; ++k) {
-                const int32_t c = *(device const int32_t *) (blocks + k*args.nbb0 + p*args.nbb1);
-                if (c >= 0 && c < args.n_kv) {
-                    dst_row_t[c] = mask_row_t[c];
-                }
-            }
-        } else if (p < args.n_kv) {
-            dst_row_t[p] = mask_row_t[p];
-        }
-    }
-
-    if (FC_top_k_mask_has_extra) {
-        for (int i = tiitg; i < args.n_extra; i += ntg) {
-            const int32_t c = *(device const int32_t *) (extra + i*args.nbe0 + i1*args.nbe1 + i2*args.nbe2 + i3*args.nbe3);
-            if (c >= 0 && c < args.n_kv) {
-                dst_row_t[c] = mask_row_t[c];
-            }
-        }
-    }
-}
-
-typedef decltype(kernel_top_k_mask<half,  half4>)  kernel_top_k_mask_h_t;
-typedef decltype(kernel_top_k_mask<float, float4>) kernel_top_k_mask_f_t;
-
-template [[host_name("kernel_top_k_mask_f16")]] kernel kernel_top_k_mask_h_t kernel_top_k_mask<half,  half4>;
-template [[host_name("kernel_top_k_mask_f32")]] kernel kernel_top_k_mask_f_t kernel_top_k_mask<float, float4>;

ggml/src/ggml-metal/kernels/mul_mm.metal

@@ -323,10 +323,10 @@ kernel void kernel_mul_mm(
         // if no bounds checks on the output are needed, we can directly write to device memory
         device float * C = (device float *) dst +
             (r0 + 32*(sgitg &  1)) + \
-            (uint64_t) (r1 + 16*(sgitg >> 1))*args.ne0 + (uint64_t) im*args.ne1*args.ne0;
+            (r1 + 16*(sgitg >> 1)) * args.ne0 + im*args.ne1*args.ne0;
 
         for (short i = 0; i < 8; i++) {
-            simdgroup_store(mc[i], C + 8*(i%4) + (uint64_t) 8*args.ne0*(i/4), args.ne0, 0, false);
+            simdgroup_store(mc[i], C + 8*(i%4) + 8*args.ne0*(i/4), args.ne0, 0, false);
         }
     } else {
         // block is smaller than 64x32, we should avoid writing data outside of the matrix
@@ -342,7 +342,7 @@ kernel void kernel_mul_mm(
 
         if (sgitg == 0) {
             for (int j = tiitg; j < nr1; j += NR1) {
-                device float  * D  = (device float  *) dst + r0 + (uint64_t) (r1 + j)*args.ne0 + (uint64_t) im*args.ne1*args.ne0;
+                device float  * D  = (device float  *) dst + r0 + (r1 + j)*args.ne0 + im*args.ne1*args.ne0;
                 device float4 * D4 = (device float4 *) D;
 
                 threadgroup float  * C  = temp_str + (j*NR0);
@@ -633,7 +633,7 @@ kernel void kernel_mul_mm_id(
     const int id = ids_i32[idb + lr1];
 
     const short i11 = (id % args.ne20) % args.ne11;
-    const int   i12 = (id / args.ne20);
+    const short i12 = (id / args.ne20);
 
     device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1;
 
@@ -866,9 +866,9 @@ kernel void kernel_mul_mm_id(
         const int id = ids_i32[idb + j];
 
         const short ide = id % args.ne20;
-        const int   idt = id / args.ne20;
+        const short idt = id / args.ne20;
 
-        device float  * D  = (device float  *) dst + r0 + (uint64_t) ide*args.ne0 + (uint64_t) idt*args.ne1*args.ne0;
+        device float  * D  = (device float  *) dst + r0 + ide*args.ne0 + idt*args.ne1*args.ne0;
         device float4 * D4 = (device float4 *) D;
 
         threadgroup float  * C  = (threadgroup float  *) shmem + j*NR0;

ggml/src/ggml-metal/kernels/unary.metal

@@ -3,24 +3,6 @@
 constant short FC_unary_op [[function_constant(FC_UNARY + 0)]];
 constant bool  FC_unary_cnt[[function_constant(FC_UNARY + 1)]];
 
-template <typename T, typename U>
-inline constexpr T ggml_metal_unary_cast(U x) {
-    return (T) x;
-}
-
-#if defined(GGML_METAL_HAS_BF16)
-// MSL has no C-style cast from float vectors to bfloat vectors
-template <>
-inline constexpr bfloat4 ggml_metal_unary_cast<bfloat4, float4>(float4 x) {
-    return bfloat4(bfloat(x[0]), bfloat(x[1]), bfloat(x[2]), bfloat(x[3]));
-}
-
-template <>
-inline constexpr bfloat4 ggml_metal_unary_cast<bfloat4, float>(float x) {
-    return bfloat4(bfloat(x), bfloat(x), bfloat(x), bfloat(x));
-}
-#endif
-
 template <typename T0, typename T, typename TC>
 kernel void kernel_unary_impl(
         constant ggml_metal_kargs_unary & args,
@@ -66,79 +48,79 @@ kernel void kernel_unary_impl(
         const TC x = (TC) src0_ptr[i0];
 
         if (FC_OP == OP_UNARY_NUM_SCALE) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(args.scale * x + args.bias);
+            dst_ptr[i0] = (T) (args.scale * x + args.bias);
         }
 
         if (FC_OP == OP_UNARY_NUM_FILL) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(args.val);
+            dst_ptr[i0] = (T) args.val;
         }
 
         if (FC_OP == OP_UNARY_NUM_CLAMP) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(clamp(x, args.min, args.max));
+            dst_ptr[i0] = (T) clamp(x, args.min, args.max);
         }
 
         if (FC_OP == OP_UNARY_NUM_SQR) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(x * x);
+            dst_ptr[i0] = (T) (x * x);
         }
 
         if (FC_OP == OP_UNARY_NUM_SQRT) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(sqrt(x));
+            dst_ptr[i0] = (T) sqrt(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_SIN) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(sin(x));
+            dst_ptr[i0] = (T) sin(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_COS) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(cos(x));
+            dst_ptr[i0] = (T) cos(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_LOG) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(log(x));
+            dst_ptr[i0] = (T) log(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_LEAKY_RELU) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(TC(x > 0)*x + TC(x <= 0)*(x * args.slope));
+            dst_ptr[i0] = (T) (TC(x > 0)*x + TC(x <= 0)*(x * args.slope));
         }
 
         if (FC_OP == OP_UNARY_NUM_TANH) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(precise::tanh(x));
+            dst_ptr[i0] = (T) precise::tanh(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_RELU) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(fmax(0, x));
+            dst_ptr[i0] = (T) fmax(0, x);
         }
 
         if (FC_OP == OP_UNARY_NUM_SIGMOID) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(1 / (1 + exp(-x)));
+            dst_ptr[i0] = (T) (1 / (1 + exp(-x)));
         }
 
         if (FC_OP == OP_UNARY_NUM_GELU) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(0.5*x*(1 + precise::tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x))));
+            dst_ptr[i0] = (T) (0.5*x*(1 + precise::tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x))));
         }
 
         if (FC_OP == OP_UNARY_NUM_GELU_ERF) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(0.5*x*(1 + erf_approx(SQRT_2_INV*x)));
+            dst_ptr[i0] = (T) (0.5*x*(1 + erf_approx(SQRT_2_INV*x)));
         }
 
         if (FC_OP == OP_UNARY_NUM_GELU_QUICK) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(x * (1/(1 + exp(GELU_QUICK_COEF*x))));
+            dst_ptr[i0] = (T) (x * (1/(1 + exp(GELU_QUICK_COEF*x))));
         }
 
         if (FC_OP == OP_UNARY_NUM_SILU) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(x / (1 + exp(-x)));
+            dst_ptr[i0] = (T) (x / (1 + exp(-x)));
         }
 
         if (FC_OP == OP_UNARY_NUM_ELU) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(elu_approx(x));
+            dst_ptr[i0] = (T) elu_approx(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_NEG) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(-x);
+            dst_ptr[i0] = (T) -x;
         }
 
         if (FC_OP == OP_UNARY_NUM_ABS) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(fabs(x));
+            dst_ptr[i0] = (T) fabs(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_SGN) {
@@ -150,40 +132,40 @@ kernel void kernel_unary_impl(
         }
 
         if (FC_OP == OP_UNARY_NUM_HARDSWISH) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(x * fmax(0, fmin(1, x/6 + 0.5)));
+            dst_ptr[i0] = (T) (x * fmax(0, fmin(1, x/6 + 0.5)));
         }
 
         if (FC_OP == OP_UNARY_NUM_HARDSIGMOID) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(fmax(0, fmin(1, x/6 + 0.5)));
+            dst_ptr[i0] = (T) fmax(0, fmin(1, x/6 + 0.5));
         }
 
         if (FC_OP == OP_UNARY_NUM_EXP) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(exp(x));
+            dst_ptr[i0] = (T) exp(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_SOFTPLUS) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(select(log(1 + exp(x)), x, x > 20));
+            dst_ptr[i0] = (T) select(log(1 + exp(x)), x, x > 20);
         }
 
         if (FC_OP == OP_UNARY_NUM_EXPM1) {
             // TODO: precise implementation
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(exp(x) - 1);
+            dst_ptr[i0] = (T) (exp(x) - 1);
         }
 
         if (FC_OP == OP_UNARY_NUM_FLOOR) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(floor(x));
+            dst_ptr[i0] = (T) floor(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_CEIL) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(ceil(x));
+            dst_ptr[i0] = (T) ceil(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_ROUND) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(round(x));
+            dst_ptr[i0] = (T) round(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_TRUNC) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(trunc(x));
+            dst_ptr[i0] = (T) trunc(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_XIELU) {
@@ -192,7 +174,7 @@ kernel void kernel_unary_impl(
             const TC clamped = fmin(xi, TC(args.val));
             const TC y_pos   = TC(args.scale) * xi * xi + TC(args.bias) * xi;
             const TC y_neg   = (exp(clamped) - TC(1.0f) - xi) * TC(args.slope) + TC(args.bias) * xi;
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(gate * y_pos + (TC(1.0f) - gate) * y_neg);
+            dst_ptr[i0] = (T) (gate * y_pos + (TC(1.0f) - gate) * y_neg);
         }
     }
 
@@ -206,10 +188,6 @@ template [[host_name("kernel_unary_f32_f32")]]   kernel kernel_unary_t kernel_un
 template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_unary_impl<float4, float4, float4>;
 template [[host_name("kernel_unary_f16_f16")]]   kernel kernel_unary_t kernel_unary_impl<half,   half,   float>;
 template [[host_name("kernel_unary_f16_f16_4")]] kernel kernel_unary_t kernel_unary_impl<half4,  half4,  float4>;
-#if defined(GGML_METAL_HAS_BF16)
-template [[host_name("kernel_unary_bf16_bf16")]]   kernel kernel_unary_t kernel_unary_impl<bfloat,  bfloat,  float>;
-template [[host_name("kernel_unary_bf16_bf16_4")]] kernel kernel_unary_t kernel_unary_impl<bfloat4, bfloat4, float4>;
-#endif
 
 kernel void kernel_silu_back_f32(
         constant ggml_metal_kargs_silu_back & args,

ggml/src/ggml-opencl/CMakeLists.txt

@@ -104,13 +104,7 @@ set(GGML_OPENCL_KERNELS
     gemm_noshuffle_q2_k_q8_1_dp4a
     gemv_noshuffle_q2_k_f32
     gemm_noshuffle_q2_k_f32
-    gemm_noshuffle_q3_k_f32
-    gemm_noshuffle_q3_k_q8_1_dp4a
-    gemv_noshuffle_q3_k_f32
     mul_mv_q2_k_f32
-    mul_mv_q3_k_f32
-    mul_mv_q3_k_f32_flat
-    mul_mm_q3_k_f32_l4_lm
     mul_mv_q2_k_f32_flat
     mul_mm_q2_k_f32_l4_lm
     mul_mv_q4_k_f32
@@ -134,7 +128,6 @@ set(GGML_OPENCL_KERNELS
     mul_mv_id_q8_0_f32_flat
     mul_mv_id_mxfp4_f32
     mul_mv_id_mxfp4_f32_flat
-    mul_mat_id_utils
     gemm_moe_q4_0_f32_ns
     gemm_moe_q4_0_q8_1_dp4a
     gemv_moe_q4_0_f32_ns

ggml/src/ggml-opencl/ggml-opencl.cpp

@@ -975,17 +975,6 @@ struct ggml_backend_opencl_context {
     int q2_k_mv_rows = 4;
     int q2_k_mv_nsg = 4;
     cl_kernel kernel_mul_mv_q2_K_f32;
-    cl_kernel kernel_convert_block_q3_K, kernel_restore_block_q3_K;
-    cl_kernel kernel_mul_mv_q3_K_f32_flat;
-    cl_kernel kernel_mul_mm_q3_k_f32_l4_lm;
-    cl_kernel kernel_convert_block_q3_k_noshuffle, kernel_restore_block_q3_k_noshuffle;
-    cl_kernel kernel_gemv_noshuffle_q3_k_f32 = nullptr;
-    cl_kernel kernel_gemm_noshuffle_q3_k_f32 = nullptr;
-    cl_kernel kernel_gemm_noshuffle_q3_k_q8_1_dp4a = nullptr;
-    cl_kernel kernel_gemm_noshuffle_q3_k_q8_1_dp4a_narrow = nullptr;
-    int q3_k_mv_rows = 2;
-    int q3_k_mv_nsg = 8;
-    cl_kernel kernel_mul_mv_q3_K_f32;
     cl_kernel kernel_mul_mv_q4_K_f32;
     cl_kernel kernel_mul_mv_q4_K_f32_flat;
     cl_kernel kernel_mul_mv_q5_K_f32;
@@ -1553,23 +1542,6 @@ static std::string ggml_cl_q2_k_mm_options(const ggml_backend_opencl_context * b
     return " -DBK=" + std::to_string(bk) + " -DLM_HALF=" + std::to_string(half ? 1 : 0);
 }
 
-static std::string ggml_cl_q3_k_mm_options(const ggml_backend_opencl_context * backend_ctx) {
-    const bool x2 = backend_ctx->gpu_family == ADRENO &&
-                   (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
-    int bk = x2 ? 16 : 32;
-    bool half = x2;
-    if (const char * env = getenv("GGML_OPENCL_LM_BK")) {
-        const int value = atoi(env);
-        if (value == 16 || value == 32) {
-            bk = value;
-        }
-    }
-    if (const char * env = getenv("GGML_OPENCL_LM_HALF")) {
-        half = atoi(env) != 0;
-    }
-    return " -DBK=" + std::to_string(bk) + " -DLM_HALF=" + std::to_string(half ? 1 : 0);
-}
-
 static void load_cl_kernels_argsort(ggml_backend_opencl_context *backend_ctx) {
     // compiler options for general kernels
     auto opencl_c_std =
@@ -1843,21 +1815,6 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
         GGML_LOG_CONT(".");
     }
 
-    // mul_mat_id_utils
-    {
-#ifdef GGML_OPENCL_EMBED_KERNELS
-        const std::string kernel_src {
-            #include "mul_mat_id_utils.cl.h"
-        };
-#else
-        const std::string kernel_src = read_file("mul_mat_id_utils.cl");
-#endif
-        cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
-        CL_CHECK((backend_ctx->kernel_moe_zero_dst = clCreateKernel(prog, "kernel_moe_zero_dst", &err), err));
-        CL_CHECK(clReleaseProgram(prog));
-        GGML_LOG_CONT(".");
-    }
-
     // moe_add_id_glu
     {
 #ifdef GGML_OPENCL_EMBED_KERNELS
@@ -2035,10 +1992,6 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
         CL_CHECK((backend_ctx->kernel_restore_block_q5_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_K_noshuffle", &err), err));
         CL_CHECK((backend_ctx->kernel_convert_block_q2_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_k_noshuffle", &err), err));
         CL_CHECK((backend_ctx->kernel_restore_block_q2_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_k_noshuffle", &err), err));
-        CL_CHECK((backend_ctx->kernel_convert_block_q3_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_k_noshuffle", &err), err));
-        CL_CHECK((backend_ctx->kernel_restore_block_q3_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_k_noshuffle", &err), err));
-        CL_CHECK((backend_ctx->kernel_convert_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_K", &err), err));
-        CL_CHECK((backend_ctx->kernel_restore_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_K", &err), err));
         CL_CHECK((backend_ctx->kernel_convert_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_K", &err), err));
         CL_CHECK((backend_ctx->kernel_restore_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_K", &err), err));
         CL_CHECK((backend_ctx->kernel_convert_block_q6_K  = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_K", &err), err));
@@ -2445,53 +2398,6 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
         GGML_LOG_CONT(".");
     }
 
-    // mul_mv_q3_k_f32_flat
-    {
-#ifdef GGML_OPENCL_EMBED_KERNELS
-        const std::string kernel_src {
-            #include "mul_mv_q3_k_f32_flat.cl.h"
-        };
-#else
-        const std::string kernel_src = read_file("mul_mv_q3_k_f32_flat.cl");
-#endif
-        cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
-        CL_CHECK((backend_ctx->kernel_mul_mv_q3_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q3_K_f32_flat", &err), err));
-        CL_CHECK(clReleaseProgram(prog));
-        GGML_LOG_CONT(".");
-    }
-
-    // mul_mm_q3_k_f32_l4_lm
-    {
-#ifdef GGML_OPENCL_EMBED_KERNELS
-        const std::string kernel_src {
-            #include "mul_mm_q3_k_f32_l4_lm.cl.h"
-        };
-#else
-        const std::string kernel_src = read_file("mul_mm_q3_k_f32_l4_lm.cl");
-#endif
-        cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + ggml_cl_q3_k_mm_options(backend_ctx));
-        CL_CHECK((backend_ctx->kernel_mul_mm_q3_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q3_k_f32_l4_lm", &err), err));
-        CL_CHECK(clReleaseProgram(prog));
-        GGML_LOG_CONT(".");
-    }
-
-    // mul_mv_q3_k_f32
-    {
-#ifdef GGML_OPENCL_EMBED_KERNELS
-        const std::string kernel_src {
-            #include "mul_mv_q3_k_f32.cl.h"
-        };
-#else
-        const std::string kernel_src = read_file("mul_mv_q3_k_f32.cl");
-#endif
-        cl_program prog =
-            build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
-
-        CL_CHECK((backend_ctx->kernel_mul_mv_q3_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q3_K_f32", &err), err));
-        CL_CHECK(clReleaseProgram(prog));
-        GGML_LOG_CONT(".");
-    }
-
     // mul_mv_q4_k_f32
     {
 #ifdef GGML_OPENCL_EMBED_KERNELS
@@ -4300,80 +4206,6 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
         GGML_LOG_CONT(".");
     }
 
-    // gemm_noshuffle_q3_k_q8_1_dp4a
-    {
-#ifdef GGML_OPENCL_EMBED_KERNELS
-        const std::string kernel_src {
-            #include "gemm_noshuffle_q3_k_q8_1_dp4a.cl.h"
-        };
-#else
-        const std::string kernel_src = read_file("gemm_noshuffle_q3_k_q8_1_dp4a.cl");
-#endif
-        cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
-        CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_q8_1_dp4a", &err), err));
-        CL_CHECK(clReleaseProgram(prog));
-        prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + " -DTILESIZE_N=8");
-        CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_q8_1_dp4a", &err), err));
-        CL_CHECK(clReleaseProgram(prog));
-        GGML_LOG_CONT(".");
-    }
-
-    // gemm_noshuffle_q3_k_f32
-    {
-#ifdef GGML_OPENCL_EMBED_KERNELS
-        const std::string kernel_src {
-            #include "gemm_noshuffle_q3_k_f32.cl.h"
-        };
-#else
-        const std::string kernel_src = read_file("gemm_noshuffle_q3_k_f32.cl");
-#endif
-        cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
-        CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_f32", &err), err));
-        CL_CHECK(clReleaseProgram(prog));
-        GGML_LOG_CONT(".");
-    }
-
-    // gemv_noshuffle_q3_k_f32
-    {
-#ifdef GGML_OPENCL_EMBED_KERNELS
-        const std::string kernel_src {
-            #include "gemv_noshuffle_q3_k_f32.cl.h"
-        };
-#else
-        const std::string kernel_src = read_file("gemv_noshuffle_q3_k_f32.cl");
-#endif
-        if (const char * env = getenv("GGML_OPENCL_Q3_K_MV_R")) {
-            const int rows = atoi(env);
-            if (rows == 1 || rows == 2 || rows == 4) {
-                backend_ctx->q3_k_mv_rows = rows;
-            }
-        }
-        if (const char * env = getenv("GGML_OPENCL_Q3_K_MV_NSG")) {
-            const int nsg = atoi(env);
-            if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) {
-                backend_ctx->q3_k_mv_nsg = nsg;
-            }
-        }
-        for (;;) {
-            const std::string opts = compile_opts + " -DQ3_K_MV_R=" + std::to_string(backend_ctx->q3_k_mv_rows) +
-                                     " -DQ3_K_MV_NSG=" + std::to_string(backend_ctx->q3_k_mv_nsg);
-            cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts);
-            cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q3_k_f32", &err);
-            CL_CHECK(err);
-            CL_CHECK(clReleaseProgram(prog));
-            size_t max_workgroup;
-            CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr));
-            if (64*(size_t)backend_ctx->q3_k_mv_nsg <= max_workgroup) {
-                backend_ctx->kernel_gemv_noshuffle_q3_k_f32 = kernel;
-                break;
-            }
-            CL_CHECK(clReleaseKernel(kernel));
-            GGML_ASSERT(backend_ctx->q3_k_mv_nsg > 1);
-            backend_ctx->q3_k_mv_nsg /= 2;
-        }
-        GGML_LOG_CONT(".");
-    }
-
     // gemm_noshuffle_q2_k_f32
     {
 #ifdef GGML_OPENCL_EMBED_KERNELS
@@ -5905,6 +5737,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
             build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts);
 
         CL_CHECK((backend_ctx->kernel_moe_histogram = clCreateKernel(prog, "kernel_moe_histogram", &err), err));
+        CL_CHECK((backend_ctx->kernel_moe_zero_dst = clCreateKernel(prog, "kernel_moe_zero_dst", &err), err));
         CL_CHECK((backend_ctx->kernel_moe_scan = clCreateKernel(prog, "kernel_moe_scan", &err), err));
         CL_CHECK((backend_ctx->kernel_moe_fill = clCreateKernel(prog, "kernel_moe_fill", &err), err));
         CL_CHECK((backend_ctx->kernel_moe_scatter = clCreateKernel(prog, "kernel_moe_scatter", &err), err));
@@ -8412,27 +8245,6 @@ struct ggml_tensor_extra_cl_q2_K {
     }
 };
 
-struct ggml_tensor_extra_cl_q3_K {
-    bool noshuffle = false;
-    cl_mem qs = nullptr;   // quantized values
-    cl_mem hm = nullptr;   // high bits
-    cl_mem sc = nullptr;   // packed 6-bit scales
-    cl_mem d = nullptr;    // super-block scales
-
-    size_t size_qs = 0, size_hm = 0, size_sc = 0, size_d = 0;
-
-    ~ggml_tensor_extra_cl_q3_K() { reset(); }
-
-    void reset() {
-        noshuffle = false;
-        if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; }
-        if (hm != nullptr) { CL_CHECK(clReleaseMemObject(hm)); hm = nullptr; }
-        if (sc != nullptr) { CL_CHECK(clReleaseMemObject(sc)); sc = nullptr; }
-        if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; }
-        size_qs = size_hm = size_sc = size_d = 0;
-    }
-};
-
 struct ggml_tensor_extra_cl_q6_K {
     // Lower 4 bits of quantized weights.
     cl_mem ql = nullptr;
@@ -9533,24 +9345,6 @@ static bool ggml_cl_q2_k_use_noshuffle(const ggml_backend_opencl_context * backe
 #endif
 }
 
-static bool ggml_cl_q3_k_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) {
-#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS)
-    if (const char * env = getenv("GGML_OPENCL_Q3_K_NOSHUFFLE")) {
-        if (env[0] == '0') {
-            return false;
-        }
-    }
-    return tensor->type == GGML_TYPE_Q3_K &&
-           backend_ctx->gpu_family == ADRENO &&
-           ggml_is_contiguous(tensor) && !tensor->view_src &&
-           use_adreno_kernels(backend_ctx, tensor);
-#else
-    GGML_UNUSED(backend_ctx);
-    GGML_UNUSED(tensor);
-    return false;
-#endif
-}
-
 inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
     // The moe weight repack kernels *_trans4_ns alias a private ushort8 through a uchar*.
     // Certain compilers (found with some A7x and A6x) miscompiles this, corrupting the weights.
@@ -10074,12 +9868,6 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
             } else if (op->src[0]->type == GGML_TYPE_Q4_0) {
                 // Non-contig src0 routes through on-device dequant-to-f16.
                 return op->src[1]->type == GGML_TYPE_F32;
-            } else if (op->src[0]->type == GGML_TYPE_Q3_K) {
-                const ggml_tensor * weight = op->src[0]->view_src;
-                if (weight && ggml_cl_q3_k_use_noshuffle(backend_ctx, weight)) {
-                    return false;
-                }
-                return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]);
             } else if (op->src[0]->type == GGML_TYPE_Q2_K) {
                 const ggml_tensor * weight = op->src[0]->view_src;
                 if (weight && ggml_cl_q2_k_use_noshuffle(backend_ctx, weight)) {
@@ -10466,18 +10254,12 @@ struct ggml_backend_opencl_buffer_context {
         for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K) {
             delete e;
         }
-        for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K) {
-            delete e;
-        }
         for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) {
             delete e;
         }
         for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K_in_use) {
             delete e;
         }
-        for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K_in_use) {
-            delete e;
-        }
         for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K) {
             delete e;
         }
@@ -10666,21 +10448,6 @@ struct ggml_backend_opencl_buffer_context {
         return extra;
     }
 
-    ggml_tensor_extra_cl_q3_K * ggml_opencl_alloc_temp_tensor_extra_q3_K() {
-        ggml_tensor_extra_cl_q3_K * extra;
-        if (temp_tensor_extras_q3_K.empty()) {
-            extra = new ggml_tensor_extra_cl_q3_K();
-        } else {
-            extra = temp_tensor_extras_q3_K.back();
-            temp_tensor_extras_q3_K.pop_back();
-        }
-
-        temp_tensor_extras_q3_K_in_use.push_back(extra);
-
-        extra->reset();
-        return extra;
-    }
-
     ggml_tensor_extra_cl_q6_K * ggml_opencl_alloc_temp_tensor_extra_q6_K() {
         ggml_tensor_extra_cl_q6_K * extra;
         if (temp_tensor_extras_q6_K.empty()) {
@@ -10762,11 +10529,6 @@ struct ggml_backend_opencl_buffer_context {
         }
         temp_tensor_extras_q2_K_in_use.clear();
 
-        for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K_in_use) {
-            temp_tensor_extras_q3_K.push_back(e);
-        }
-        temp_tensor_extras_q3_K_in_use.clear();
-
         q8_0_soa_tensors.clear();
         q4_0_soa_tensors.clear();
     }
@@ -10799,9 +10561,7 @@ struct ggml_backend_opencl_buffer_context {
     std::vector<ggml_tensor_extra_cl_q5_K *> temp_tensor_extras_q5_K;
     std::vector<ggml_tensor_extra_cl_q5_K *> temp_tensor_extras_q5_K_in_use;
     std::vector<ggml_tensor_extra_cl_q2_K *> temp_tensor_extras_q2_K;
-    std::vector<ggml_tensor_extra_cl_q3_K *> temp_tensor_extras_q3_K;
     std::vector<ggml_tensor_extra_cl_q2_K *> temp_tensor_extras_q2_K_in_use;
-    std::vector<ggml_tensor_extra_cl_q3_K *> temp_tensor_extras_q3_K_in_use;
     std::vector<ggml_tensor_extra_cl_q6_K *> temp_tensor_extras_q6_K;
     std::vector<ggml_tensor_extra_cl_q6_K *> temp_tensor_extras_q6_K_in_use;
 
@@ -11058,91 +10818,6 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
         return;
     }
 
-    if (tensor->type == GGML_TYPE_Q3_K) {
-        GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor));
-        ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;
-        GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized");
-
-        ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;
-        ggml_tensor_extra_cl_q3_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q3_K();
-        extra->noshuffle = ggml_cl_q3_k_use_noshuffle(backend_ctx, tensor);
-
-        const size_t blck  = (size_t)ggml_blck_size(tensor->type);
-        const size_t n_blk = ggml_nelements(tensor) / blck;
-
-        const size_t size_qs = n_blk * (blck/4);
-        const size_t size_hm = n_blk * (blck/8);
-        const size_t size_sc = n_blk * 12;
-        const size_t size_d = n_blk * sizeof(ggml_fp16_t);
-        GGML_ASSERT(size_qs + size_hm + size_sc + size_d == ggml_nbytes(tensor) && "Incorrect tensor size");
-
-        cl_int err;
-        cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,
-            ggml_nbytes(tensor), NULL, &err);
-        CL_CHECK(err);
-        CL_CHECK(clEnqueueWriteBuffer(
-            queue, data_device, CL_TRUE, 0,
-            ggml_nbytes(tensor), data, 0, NULL, NULL));
-
-        cl_buffer_region region;
-        region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment);
-        region.size   = size_qs;
-        CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
-        auto previous_origin = region.origin;
-
-        region.origin = align_to(previous_origin + size_qs, backend_ctx->alignment);
-        region.size   = size_hm;
-        CL_CHECK((extra->hm = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
-        previous_origin = region.origin;
-
-        region.origin = align_to(previous_origin + size_hm, backend_ctx->alignment);
-        region.size   = size_sc;
-        CL_CHECK((extra->sc = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
-        previous_origin = region.origin;
-
-        region.origin = align_to(previous_origin + size_sc, backend_ctx->alignment);
-        region.size   = size_d;
-        CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
-
-        cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_convert_block_q3_k_noshuffle
-                                           : backend_ctx->kernel_convert_block_q3_K;
-        cl_ulong nb_arg = (cl_ulong)n_blk;
-        CL_CHECK(clSetKernelArg(kernel,  0, sizeof(cl_mem),    &data_device));
-        CL_CHECK(clSetKernelArg(kernel,  1, sizeof(cl_mem),    &extra->qs));
-        CL_CHECK(clSetKernelArg(kernel,  2, sizeof(cl_mem),    &extra->hm));
-        CL_CHECK(clSetKernelArg(kernel,  3, sizeof(cl_mem),    &extra->sc));
-        CL_CHECK(clSetKernelArg(kernel,  4, sizeof(cl_mem),    &extra->d));
-        CL_CHECK(clSetKernelArg(kernel,  5, sizeof(cl_ulong),  &nb_arg));
-
-        size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 };
-        size_t local_work_size[]  = { 64, 1, 1 };
-
-        cl_event evt;
-        CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));
-        CL_CHECK(clWaitForEvents(1, &evt));
-        CL_CHECK(clReleaseEvent(evt));
-        CL_CHECK(clReleaseMemObject(data_device));
-
-#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
-        if (extra->noshuffle) {
-            const int m = tensor->ne[1];
-            const int k = tensor->ne[0];
-            transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true);
-            transpose_2d_as_8b(backend_ctx, extra->hm, extra->hm, size_hm, k/8, m, true, true);
-            transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->sc, extra->sc, size_sc, 3*k/(int)blck, m, true, true);
-            transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, extra->d, size_d, k/(int)blck, m, true, true);
-        }
-#endif
-
-        extra->size_qs = size_qs;
-        extra->size_hm = size_hm;
-        extra->size_sc = size_sc;
-        extra->size_d = size_d;
-
-        tensor->extra = extra;
-        return;
-    }
-
     if (tensor->type == GGML_TYPE_Q1_0) {
         ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;
         GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized");
@@ -11226,16 +10901,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
     // conversion kernel.
     if (tensor->type == GGML_TYPE_Q4_0) {
         // Views can't SoA-ify here — parent owns the layout (see q8_0 guard).
-        // A view of an AoS tensor (the KV cache, written by a state restore)
-        // is written in place.
         if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) {
-            if (!ggml_cl_is_q4_0_soa(tensor)) {
-                ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra;
-                CL_CHECK(clEnqueueWriteBuffer(
-                    queue, extra_aos->data_device, CL_TRUE,
-                    extra_aos->offset + tensor->view_offs + offset,
-                    size, data, 0, NULL, NULL));
-            }
             return;
         }
         // Tensors should have been preallocated, therefore they should
@@ -11939,16 +11605,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
     }
     if (tensor->type == GGML_TYPE_Q8_0) {
         // Views share the parent's buffer; parent owns SoA conversion.
-        // A view of an AoS tensor (the KV cache, written by a state restore)
-        // is written in place.
         if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) {
-            if (!ggml_cl_is_q8_0_soa(tensor)) {
-                ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra;
-                CL_CHECK(clEnqueueWriteBuffer(
-                    queue, extra_aos->data_device, CL_TRUE,
-                    extra_aos->offset + tensor->view_offs + offset,
-                    size, data, 0, NULL, NULL));
-            }
             return;
         }
 
@@ -12783,7 +12440,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
         GGML_ASSERT(extra);
 
         cl_ulong n_elements = size / sizeof(ggml_fp16_t);
-        cl_ulong off_dst = (extra->offset + tensor->view_offs + offset) / sizeof(ggml_fp16_t);
+        cl_ulong off_dst = (extra->offset + offset) / sizeof(ggml_fp16_t);
 
         cl_int err;
         cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR,
@@ -12811,9 +12468,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
     ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra;
     GGML_ASSERT(extra);
 
-    // a view reuses its parent's extra, whose offset does not include view_offs
     CL_CHECK(clEnqueueWriteBuffer(
-        queue, extra->data_device, CL_TRUE, extra->offset + tensor->view_offs + offset,
+        queue, extra->data_device, CL_TRUE, extra->offset + offset,
         size, data, 0, NULL, NULL));
 
     GGML_UNUSED(buffer);
@@ -12878,56 +12534,6 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,
         return;
     }
 
-    if (tensor->type == GGML_TYPE_Q3_K) {
-        ggml_tensor_extra_cl_q3_K * extra = (ggml_tensor_extra_cl_q3_K *)tensor->extra;
-
-        const size_t blck = (size_t)ggml_blck_size(tensor->type);
-
-        ggml_cl_buffer buf_unpacked;
-        buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor));
-
-        const size_t n_blk = (size_t)ggml_nelements(tensor) / blck;
-        cl_ulong nb_arg = (cl_ulong)n_blk;
-
-        cl_mem qs = extra->qs, hm = extra->hm, sc = extra->sc, d = extra->d;
-#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
-        ggml_cl_buffer buf_qs, buf_hm, buf_sc, buf_d;
-        if (extra->noshuffle) {
-            buf_qs.allocate(context, extra->size_qs);
-            buf_hm.allocate(context, extra->size_hm);
-            buf_sc.allocate(context, extra->size_sc);
-            buf_d.allocate(context, extra->size_d);
-            qs = buf_qs.buffer;
-            hm = buf_hm.buffer;
-            sc = buf_sc.buffer;
-            d = buf_d.buffer;
-            const int m = tensor->ne[1];
-            const int k = tensor->ne[0];
-            transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true);
-            transpose_2d_as_8b(backend_ctx, extra->hm, hm, extra->size_hm, m, k/8, true, true);
-            transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->sc, sc, extra->size_sc, m, 3*k/(int)blck, true, true);
-            transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, d, extra->size_d, m, k/(int)blck, true, true);
-        }
-#endif
-        cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_restore_block_q3_k_noshuffle
-                                           : backend_ctx->kernel_restore_block_q3_K;
-        CL_CHECK(clSetKernelArg(kernel,  0, sizeof(cl_mem),    &qs));
-        CL_CHECK(clSetKernelArg(kernel,  1, sizeof(cl_mem),    &hm));
-        CL_CHECK(clSetKernelArg(kernel,  2, sizeof(cl_mem),    &sc));
-        CL_CHECK(clSetKernelArg(kernel,  3, sizeof(cl_mem),    &d));
-        CL_CHECK(clSetKernelArg(kernel,  4, sizeof(cl_mem),    &buf_unpacked.buffer));
-        CL_CHECK(clSetKernelArg(kernel,  5, sizeof(cl_ulong),  &nb_arg));
-
-        size_t gws[] = { (n_blk + 63) / 64 * 64 };
-        size_t lws[] = { 64 };
-        cl_event evt;
-        CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt));
-        CL_CHECK(clWaitForEvents(1, &evt));
-        CL_CHECK(clReleaseEvent(evt));
-        CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL));
-        return;
-    }
-
     if (tensor->type == GGML_TYPE_Q1_0) {
         ggml_tensor_extra_cl_q1_0 * extra = (ggml_tensor_extra_cl_q1_0 *)tensor->extra;
 
@@ -25299,65 +24905,6 @@ static bool ggml_cl_mul_mat_q2_K_dp4a(ggml_backend_t backend, const ggml_tensor
     return true;
 }
 
-static bool ggml_cl_mul_mat_q3_K_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
-    ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
-    ggml_tensor_extra_cl_q3_K * extra0 = (ggml_tensor_extra_cl_q3_K *)src0->extra;
-    ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
-    ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
-
-    cl_ulong offset1 = extra1->offset + src1->view_offs;
-    cl_ulong offsetd = extrad->offset + dst->view_offs;
-    const int m = src0->ne[1];
-    const int n = src1->ne[1];
-    const int k = src0->ne[0];
-
-    const int tile_n = n <= 16 ? 8 : 32;
-    cl_kernel gemm = n <= 16 ? backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a_narrow
-                             : backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a;
-    const char * env = getenv("GGML_OPENCL_Q3_K_DENSE_DP4A");
-    if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') ||
-        n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
-        return false;
-    }
-
-    cl_int err;
-    cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) };
-    cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
-    CL_CHECK(err);
-    const size_t n_blocks = (size_t)n*(k/32);
-    backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k);
-    backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half));
-    backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half));
-
-    cl_int nb = n_blocks;
-    cl_kernel quant = backend_ctx->kernel_quant_a_q8_1;
-    CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation));
-    CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
-    CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
-    CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
-    CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb));
-    size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 };
-    size_t q_local[] = { 64 };
-    backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst);
-
-    CL_CHECK(clSetKernelArg(gemm,  0, sizeof(cl_mem),    &extra0->qs));
-    CL_CHECK(clSetKernelArg(gemm,  1, sizeof(cl_mem),    &extra0->hm));
-    CL_CHECK(clSetKernelArg(gemm,  2, sizeof(cl_mem),    &extra0->sc));
-    CL_CHECK(clSetKernelArg(gemm,  3, sizeof(cl_mem),    &extra0->d));
-    CL_CHECK(clSetKernelArg(gemm,  4, sizeof(cl_mem),    &backend_ctx->prealloc_moe_qa.buffer));
-    CL_CHECK(clSetKernelArg(gemm,  5, sizeof(cl_mem),    &backend_ctx->prealloc_moe_da.buffer));
-    CL_CHECK(clSetKernelArg(gemm,  6, sizeof(cl_mem),    &extrad->data_device));
-    CL_CHECK(clSetKernelArg(gemm,  7, sizeof(cl_ulong),  &offsetd));
-    CL_CHECK(clSetKernelArg(gemm,  8, sizeof(int),       &m));
-    CL_CHECK(clSetKernelArg(gemm,  9, sizeof(int),       &n));
-    CL_CHECK(clSetKernelArg(gemm, 10, sizeof(int),       &k));
-    size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) };
-    size_t local[] = { 64, 1, 1 };
-    backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst);
-    CL_CHECK(clReleaseMemObject(activation));
-    return true;
-}
-
 static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
     ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
     ggml_tensor_extra_cl_q2_K * extra0 = (ggml_tensor_extra_cl_q2_K *)src0->extra;
@@ -25457,108 +25004,6 @@ static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_t
     CL_CHECK(clReleaseMemObject(b_img));
     CL_CHECK(clReleaseMemObject(b_sub_buf));
 }
-
-static void ggml_cl_mul_mat_q3_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
-    ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
-    ggml_tensor_extra_cl_q3_K * extra0 = (ggml_tensor_extra_cl_q3_K *)src0->extra;
-    ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
-    ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
-
-    cl_ulong offset1 = extra1->offset + src1->view_offs;
-    cl_ulong offsetd = extrad->offset + dst->view_offs;
-    const int m = src0->ne[1];
-    const int n = src1->ne[1];
-    const int k = src0->ne[0];
-
-    if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
-        cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_q3_k_f32;
-        const int stride_d = dst->ne[0];
-        const int stride_b = src1->nb[1]/sizeof(float);
-        CL_CHECK(clSetKernelArg(gemv,  0, sizeof(cl_mem),    &extra0->qs));
-        CL_CHECK(clSetKernelArg(gemv,  1, sizeof(cl_mem),    &extra0->hm));
-        CL_CHECK(clSetKernelArg(gemv,  2, sizeof(cl_mem),    &extra0->sc));
-        CL_CHECK(clSetKernelArg(gemv,  3, sizeof(cl_mem),    &extra0->d));
-        CL_CHECK(clSetKernelArg(gemv,  4, sizeof(cl_mem),    &extra1->data_device));
-        CL_CHECK(clSetKernelArg(gemv,  5, sizeof(cl_ulong),  &offset1));
-        CL_CHECK(clSetKernelArg(gemv,  6, sizeof(cl_mem),    &extrad->data_device));
-        CL_CHECK(clSetKernelArg(gemv,  7, sizeof(cl_ulong),  &offsetd));
-        CL_CHECK(clSetKernelArg(gemv,  8, sizeof(int),       &k));
-        CL_CHECK(clSetKernelArg(gemv,  9, sizeof(int),       &m));
-        CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int),       &stride_b));
-        CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int),       &stride_d));
-        const size_t rows = 64*(size_t)backend_ctx->q3_k_mv_rows;
-        const size_t nsg = backend_ctx->q3_k_mv_nsg;
-        size_t global[] = { CEIL_DIV((size_t)m, rows)*64, (size_t)n*nsg, 1 };
-        size_t local[] = { 64, nsg, 1 };
-        backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst);
-        return;
-    }
-
-    if (ggml_cl_mul_mat_q3_K_dp4a(backend, src0, src1, dst)) {
-        return;
-    }
-
-    cl_context context = backend_ctx->context;
-    cl_int err;
-    cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) };
-    cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
-    CL_CHECK(err);
-
-    cl_image_format img_fmt = { CL_RGBA, CL_FLOAT };
-    cl_image_desc img_desc = {};
-    img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
-    img_desc.image_width = (size_t)k*n/4;
-    img_desc.buffer = b_sub_buf;
-    cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err);
-    CL_CHECK(err);
-
-    const int padded_n = CEIL_DIV(n, 8)*8;
-    region.origin = 0;
-    region.size = (size_t)k*padded_n*sizeof(cl_half);
-    backend_ctx->prealloc_act_trans.allocate(context, region.size);
-    cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
-    CL_CHECK(err);
-
-    img_fmt = { CL_RGBA, CL_HALF_FLOAT };
-    img_desc.image_width = (size_t)k*padded_n/4;
-    img_desc.buffer = b_sub_buf_trans;
-    cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err);
-    CL_CHECK(err);
-
-    const int height_b = std::max(n/4, 1);
-    const int width_b = k/4;
-    const int padded_height_b = padded_n/4;
-    cl_kernel kernel = backend_ctx->kernel_transpose_32_16;
-    CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img));
-    CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans));
-    CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b));
-    CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b));
-    CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b));
-    size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b };
-    size_t t_local[] = { 1, 16 };
-    backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst);
-
-    kernel = backend_ctx->kernel_gemm_noshuffle_q3_k_f32;
-    CL_CHECK(clSetKernelArg(kernel,  0, sizeof(cl_mem),    &extra0->qs));
-    CL_CHECK(clSetKernelArg(kernel,  1, sizeof(cl_mem),    &extra0->hm));
-    CL_CHECK(clSetKernelArg(kernel,  2, sizeof(cl_mem),    &extra0->sc));
-    CL_CHECK(clSetKernelArg(kernel,  3, sizeof(cl_mem),    &extra0->d));
-    CL_CHECK(clSetKernelArg(kernel,  4, sizeof(cl_mem),    &b_img_trans));
-    CL_CHECK(clSetKernelArg(kernel,  5, sizeof(cl_mem),    &extrad->data_device));
-    CL_CHECK(clSetKernelArg(kernel,  6, sizeof(cl_ulong),  &offsetd));
-    CL_CHECK(clSetKernelArg(kernel,  7, sizeof(int),       &m));
-    CL_CHECK(clSetKernelArg(kernel,  8, sizeof(int),       &padded_n));
-    CL_CHECK(clSetKernelArg(kernel,  9, sizeof(int),       &k));
-    CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int),       &n));
-    size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 };
-    size_t local[] = { 1, 128, 1 };
-    backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst);
-
-    CL_CHECK(clReleaseMemObject(b_img_trans));
-    CL_CHECK(clReleaseMemObject(b_sub_buf_trans));
-    CL_CHECK(clReleaseMemObject(b_img));
-    CL_CHECK(clReleaseMemObject(b_sub_buf));
-}
 #endif
 
 static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
@@ -25630,7 +25075,6 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
     ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra;
     ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)soa0_src->extra;
     ggml_tensor_extra_cl_q2_K * extra0_q2_K = (ggml_tensor_extra_cl_q2_K *)soa0_src->extra;
-    ggml_tensor_extra_cl_q3_K * extra0_q3_K = (ggml_tensor_extra_cl_q3_K *)soa0_src->extra;
     ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)soa0_src->extra;
     ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)soa0_src->extra;
     ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)soa0_src->extra;
@@ -26032,12 +25476,6 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
             return;
         }
 
-        // q3_k x fp32
-        if (src0t == GGML_TYPE_Q3_K && src1t == GGML_TYPE_F32 && extra0_q3_K->noshuffle) {
-            ggml_cl_mul_mat_q3_k_f32_adreno(backend, src0, src1, dst);
-            return;
-        }
-
         // q4_k x fp32
         if (src0t == GGML_TYPE_Q4_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q4_K(backend_ctx, src0)) {
             ggml_cl_mul_mat_q4_k_f32_adreno(backend, src0, src1, dst);
@@ -26588,51 +26026,6 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
                 backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
                 return;
             }
-            case GGML_TYPE_Q3_K: {
-                if (ne11 < 32) {
-                    break;
-                }
-                if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {
-                    break;
-                }
-
-                kernel = backend_ctx->kernel_mul_mm_q3_k_f32_l4_lm;
-                nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128;
-                offset0 = src0->view_offs;
-
-                int batch_stride_a = ne00*ne01;
-                int batch_stride_b = ne10*ne11;
-                int batch_stride_d = ne0*ne1;
-
-                CL_CHECK(clSetKernelArg(kernel,  0, sizeof(cl_mem),    &extra0_q3_K->qs));
-                CL_CHECK(clSetKernelArg(kernel,  1, sizeof(cl_mem),    &extra0_q3_K->hm));
-                CL_CHECK(clSetKernelArg(kernel,  2, sizeof(cl_mem),    &extra0_q3_K->sc));
-                CL_CHECK(clSetKernelArg(kernel,  3, sizeof(cl_mem),    &extra0_q3_K->d));
-                CL_CHECK(clSetKernelArg(kernel,  4, sizeof(cl_ulong),  &offset0));
-                CL_CHECK(clSetKernelArg(kernel,  5, sizeof(cl_mem),    &extra1->data_device));
-                CL_CHECK(clSetKernelArg(kernel,  6, sizeof(cl_ulong),  &offset1));
-                CL_CHECK(clSetKernelArg(kernel,  7, sizeof(cl_mem),    &extrad->data_device));
-                CL_CHECK(clSetKernelArg(kernel,  8, sizeof(cl_ulong),  &offsetd));
-                CL_CHECK(clSetKernelArg(kernel,  9, sizeof(int),       &ne00));
-                CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int),       &ne01));
-                CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int),       &ne02));
-                CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int),       &ne11));
-                CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int),       &ne12));
-                CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int),       &ne00));
-                CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int),       &ne10));
-                CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int),       &ne0));
-                CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int),       &batch_stride_a));
-                CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int),       &batch_stride_b));
-                CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int),       &batch_stride_d));
-                CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int),       &r2));
-                CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int),       &r3));
-
-                size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13};
-                size_t local_work_size[] = {(size_t)nth0, 1, 1};
-
-                backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
-                return;
-            }
             case GGML_TYPE_Q4_K: {
                 if (ne11 < 32) {
                     break;
@@ -27612,70 +27005,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
 #endif // GGML_OPENCL_SOA_Q
             break;
         }
-        case GGML_TYPE_Q3_K: {
-            if (backend_ctx->gpu_family == INTEL) {
-                nth0 = 16;
-                nth1 = 1;
-                ndst = 2;
-            } else if (backend_ctx->gpu_family == ADRENO) {
-                nth0 = 64;
-                nth1 = 1;
-                ndst = 2;
-            } else {
-                GGML_ASSERT(false && "TODO: Unknown GPU");
-            }
-
-#ifdef GGML_OPENCL_SOA_Q
-            kernel = backend_ctx->kernel_mul_mv_q3_K_f32_flat;
-            offset0 = src0->view_offs;
-
-            CL_CHECK(clSetKernelArg(kernel,  0, sizeof(cl_mem),    &extra0_q3_K->qs));
-            CL_CHECK(clSetKernelArg(kernel,  1, sizeof(cl_mem),    &extra0_q3_K->hm));
-            CL_CHECK(clSetKernelArg(kernel,  2, sizeof(cl_mem),    &extra0_q3_K->sc));
-            CL_CHECK(clSetKernelArg(kernel,  3, sizeof(cl_mem),    &extra0_q3_K->d));
-            CL_CHECK(clSetKernelArg(kernel,  4, sizeof(cl_ulong),  &offset0));
-            CL_CHECK(clSetKernelArg(kernel,  5, sizeof(cl_mem),    &extra1->data_device));
-            CL_CHECK(clSetKernelArg(kernel,  6, sizeof(cl_ulong),  &offset1));
-            CL_CHECK(clSetKernelArg(kernel,  7, sizeof(cl_mem),    &extrad->data_device));
-            CL_CHECK(clSetKernelArg(kernel,  8, sizeof(cl_ulong),  &offsetd));
-            CL_CHECK(clSetKernelArg(kernel,  9, sizeof(int),       &ne00));
-            CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int),       &ne01));
-            CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong),  &nb01));
-            CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong),  &nb02));
-            CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong),  &nb03));
-            CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int),       &ne12));
-            CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong),  &nb11));
-            CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong),  &nb12));
-            CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong),  &nb13));
-            CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int),       &ne0));
-            CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int),       &ne1));
-            CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int),       &r2));
-            CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int),       &r3));
-#else
-            kernel = backend_ctx->kernel_mul_mv_q3_K_f32;
-
-            CL_CHECK(clSetKernelArg(kernel,  0, sizeof(cl_mem),   &extra0->data_device));
-            CL_CHECK(clSetKernelArg(kernel,  1, sizeof(cl_ulong), &offset0));
-            CL_CHECK(clSetKernelArg(kernel,  2, sizeof(cl_mem),   &extra1->data_device));
-            CL_CHECK(clSetKernelArg(kernel,  3, sizeof(cl_ulong), &offset1));
-            CL_CHECK(clSetKernelArg(kernel,  4, sizeof(cl_mem),   &extrad->data_device));
-            CL_CHECK(clSetKernelArg(kernel,  5, sizeof(cl_ulong), &offsetd));
-            CL_CHECK(clSetKernelArg(kernel,  6, sizeof(int),      &ne00));
-            CL_CHECK(clSetKernelArg(kernel,  7, sizeof(int),      &ne01));
-            CL_CHECK(clSetKernelArg(kernel,  8, sizeof(cl_ulong), &nb01));
-            CL_CHECK(clSetKernelArg(kernel,  9, sizeof(cl_ulong), &nb02));
-            CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));
-            CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int),      &ne12));
-            CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11));
-            CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12));
-            CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13));
-            CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int),      &ne0));
-            CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int),      &ne1));
-            CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int),      &r2));
-            CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int),      &r3));
-#endif // GGML_OPENCL_SOA_Q
-            break;
-        }
+        case GGML_TYPE_Q3_K:
         case GGML_TYPE_Q4_K: {
 #ifdef GGML_OPENCL_SOA_Q
             kernel = backend_ctx->kernel_mul_mv_q4_K_f32_flat;
@@ -27983,7 +27313,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
         src0t == GGML_TYPE_Q8_0 ||
         src0t == GGML_TYPE_Q1_0 ||
         src0t == GGML_TYPE_IQ4_NL ||
-        src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_Q3_K) {
+        src0t == GGML_TYPE_Q2_K) {
         // Each SIMD group produces N_DST values in the result. Assuming each
         // workgroup has N_SIMDGROUP SIMD groups, then each workgroup will
         // produce N_DST*N_SIMDGROUP values in the result. Hence, the grid size
@@ -27999,6 +27329,8 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
         size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1};
 
         backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+    } else if (src0t == GGML_TYPE_Q3_K) {
+        GGML_ASSERT(false && "not implemented");
     } else if (src0t == GGML_TYPE_Q5_K) {
         size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13};
         size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1};
@@ -28298,21 +27630,35 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
 
     // the matmul kernels only write the rows an expert owns, so zero the skipped rows here
     {
-        const cl_ulong nb1 = dst->nb[1];
-        const cl_ulong nb2 = dst->nb[2];
+        cl_int err;
+
+        cl_buffer_region region;
+        region.origin = offset2;
+        region.size   = nb21 * ne21;
+        cl_mem buf_ids = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
+        CL_CHECK(err);
+
+        region.origin = offsetd;
+        region.size   = (size_t) ne0 * ne1 * ne2 * sizeof(float);
+        cl_mem buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
+        CL_CHECK(err);
+
+        const cl_uint ids_stride = nb21 / sizeof(int);
+
         cl_kernel kernel_zero = backend_ctx->kernel_moe_zero_dst;
-        CL_CHECK(clSetKernelArg(kernel_zero, 0, sizeof(cl_mem),   &extra2->data_device));
-        CL_CHECK(clSetKernelArg(kernel_zero, 1, sizeof(cl_ulong), &offset2));
-        CL_CHECK(clSetKernelArg(kernel_zero, 2, sizeof(cl_ulong), &nb21));
-        CL_CHECK(clSetKernelArg(kernel_zero, 3, sizeof(cl_mem),   &extrad->data_device));
-        CL_CHECK(clSetKernelArg(kernel_zero, 4, sizeof(cl_ulong), &offsetd));
-        CL_CHECK(clSetKernelArg(kernel_zero, 5, sizeof(cl_ulong), &nb1));
-        CL_CHECK(clSetKernelArg(kernel_zero, 6, sizeof(cl_ulong), &nb2));
-        CL_CHECK(clSetKernelArg(kernel_zero, 7, sizeof(int),      &ne0));
+        CL_CHECK(clSetKernelArg(kernel_zero, 0, sizeof(cl_mem),  &buf_ids));
+        CL_CHECK(clSetKernelArg(kernel_zero, 1, sizeof(cl_mem),  &buf_dst));
+        CL_CHECK(clSetKernelArg(kernel_zero, 2, sizeof(cl_uint), &ne21));
+        CL_CHECK(clSetKernelArg(kernel_zero, 3, sizeof(cl_uint), &ne20));
+        CL_CHECK(clSetKernelArg(kernel_zero, 4, sizeof(cl_uint), &ids_stride));
+        CL_CHECK(clSetKernelArg(kernel_zero, 5, sizeof(cl_uint), &ne0));
 
         size_t zero_local[3]  = {1, 1, 64};
-        size_t zero_global[3] = {(size_t) ne20, (size_t) ne21, 64};
+        size_t zero_global[3] = {(size_t) ne21, (size_t) ne20, 64};
         backend_ctx->enqueue_ndrange_kernel(kernel_zero, 3, zero_global, zero_local, dst);
+
+        CL_CHECK(clReleaseMemObject(buf_ids));
+        CL_CHECK(clReleaseMemObject(buf_dst));
     }
 
     int sgs   = 32; // subgroup size

ggml/src/ggml-opencl/kernels/cvt.cl

@@ -2773,156 +2773,3 @@ kernel void kernel_restore_block_q2_k_noshuffle(
         }
     }
 }
-
-struct block_q3_K {
-    uchar hmask[QK_K/8];
-    uchar qs[QK_K/4];
-    uchar scales[12];
-    half d;
-};
-
-kernel void kernel_convert_block_q3_K(
-    global struct block_q3_K * src0,
-    global uchar * dst_qs,
-    global uchar * dst_hm,
-    global uchar * dst_sc,
-    global half * dst_d,
-    ulong n_blk
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blk) {
-        return;
-    }
-    global struct block_q3_K * b = src0 + i;
-    for (int j = 0; j < 64; ++j) {
-        dst_qs[64*i + j] = b->qs[j];
-    }
-    for (int j = 0; j < 32; ++j) {
-        dst_hm[32*i + j] = b->hmask[j];
-    }
-    for (int j = 0; j < 12; ++j) {
-        dst_sc[12*i + j] = b->scales[j];
-    }
-    dst_d[i] = b->d;
-}
-
-kernel void kernel_restore_block_q3_K(
-    global uchar * src_qs,
-    global uchar * src_hm,
-    global uchar * src_sc,
-    global half * src_d,
-    global struct block_q3_K * dst,
-    ulong n_blk
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blk) {
-        return;
-    }
-    global struct block_q3_K * b = dst + i;
-    for (int j = 0; j < 64; ++j) {
-        b->qs[j] = src_qs[64*i + j];
-    }
-    for (int j = 0; j < 32; ++j) {
-        b->hmask[j] = src_hm[32*i + j];
-    }
-    for (int j = 0; j < 12; ++j) {
-        b->scales[j] = src_sc[12*i + j];
-    }
-    b->d = src_d[i];
-}
-
-kernel void kernel_convert_block_q3_k_noshuffle(
-    global struct block_q3_K * src0,
-    global uchar * dst_qs,      // QK_K/4 per block
-    global uchar * dst_hm,      // QK_K/8 per block
-    global uint  * dst_sc,      // 3 per block
-    global half  * dst_d,       // 1
-    ulong          n_blk
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blk) {
-        return;
-    }
-    global struct block_q3_K * b = (global struct block_q3_K *) src0 + i;
-
-    dst_d[i] = b->d;
-
-    for (int w = 0; w < 3; ++w) {
-        global const uchar * p = b->scales + 4*w;
-        dst_sc[3*i + w] = (uint)p[0] | ((uint)p[1] << 8) | ((uint)p[2] << 16) | ((uint)p[3] << 24);
-    }
-
-    for (int g = 0; g < QK_K/4; ++g) {
-        uint pk = 0;
-        for (int t = 0; t < 4; ++t) {
-            const int e  = 4*g + t;
-            const int sb = e >> 5;
-            const int ee = e & 31;
-            pk |= (uint)(((b->qs[32*(sb >> 2) + ee] >> (2*(sb & 3))) & 3) << (2*t));
-        }
-        dst_qs[(QK_K/4) * i + g] = (uchar)pk;
-    }
-
-    // two groups per hmask byte, so this loop owns the byte and cannot race
-    for (int t = 0; t < QK_K/8; ++t) {
-        uint v = 0;
-        for (int s = 0; s < 2; ++s) {
-            const int g = 2*t + s;
-            for (int u = 0; u < 4; ++u) {
-                const int e  = 4*g + u;
-                const int sb = e >> 5;
-                const int ee = e & 31;
-                v |= (uint)(((b->hmask[ee] >> sb) & 1) << (4*s + u));
-            }
-        }
-        dst_hm[(QK_K/8) * i + t] = (uchar)v;
-    }
-}
-
-kernel void kernel_restore_block_q3_k_noshuffle(
-    global uchar * src_qs,
-    global uchar * src_hm,
-    global uint  * src_sc,
-    global half  * src_d,
-    global struct block_q3_K * dst,
-    ulong          n_blk
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blk) {
-        return;
-    }
-    global struct block_q3_K * b = (global struct block_q3_K *) dst + i;
-
-    b->d = src_d[i];
-
-    for (int w = 0; w < 3; ++w) {
-        const uint x = src_sc[3*i + w];
-        global uchar * p = b->scales + 4*w;
-        p[0] = (uchar)( x        & 0xFF);
-        p[1] = (uchar)((x >>  8) & 0xFF);
-        p[2] = (uchar)((x >> 16) & 0xFF);
-        p[3] = (uchar)((x >> 24) & 0xFF);
-    }
-
-    for (int n2 = 0; n2 < 2; ++n2) {
-        for (int ee = 0; ee < 32; ++ee) {
-            uint v = 0;
-            for (int j = 0; j < 4; ++j) {
-                const int sb = 4*n2 + j;
-                const int e  = 32*sb + ee;
-                v |= ((uint)(src_qs[(QK_K/4) * i + (e >> 2)] >> (2*(e & 3))) & 3u) << (2*j);
-            }
-            b->qs[32*n2 + ee] = (uchar)v;
-        }
-    }
-
-    for (int ee = 0; ee < 32; ++ee) {
-        uint v = 0;
-        for (int sb = 0; sb < 8; ++sb) {
-            const int e = 32*sb + ee;
-            const int g = e >> 2;
-            v |= ((uint)(src_hm[(QK_K/8) * i + (g >> 1)] >> (4*(g & 1) + (e & 3))) & 1u) << sb;
-        }
-        b->hmask[ee] = (uchar)v;
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_f32.cl

@@ -1,92 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_qcom_reqd_sub_group_size
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-__attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-kernel void kernel_gemm_noshuffle_q3_k_f32(
-    global const uchar * src0_qs,
-    global const uchar * src0_hm,
-    global const uint  * src0_sc,
-    global const half  * src0_d,
-    read_only image1d_buffer_t src1,
-    global float * dst,
-    ulong offsetd,
-    int m,
-    int n,
-    int k,
-    int n_no_padding
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-    const int gy = get_global_id(0);
-    const int row = get_global_id(1) * 4;
-    const int n_4 = n / 4;
-    if (row >= m) {
-        return;
-    }
-
-    half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
-    for (int i = 0; i < k; i += 16) {
-        const int is = (i / 16) & 15;
-        const int word = is >> 2;
-        const int shift = 8 * (is & 3);
-        const int sc_base = row + 3 * (i / 256) * m;
-        const uint4 low = vload4(0, src0_sc + sc_base + (word & 1) * m) >> (uint4)(4 * (word >> 1));
-        const uint4 high = vload4(0, src0_sc + sc_base + 2 * m) >> (uint4)(2 * word);
-        const int4 sc = convert_int4(((low >> (uint4)(shift)) & (uint4)(15)) | (((high >> (uint4)(shift)) & (uint4)(3)) << (uint4)(4))) - (int4)(32);
-        const half4 d = vload4(0, src0_d + row + (i / 256) * m);
-        const half4 scale = convert_half4(convert_float4(d) * convert_float4(sc));
-
-        for (int l = 0; l < 16; l += 4) {
-            const int ki = i + l;
-            const uchar4 bits = vload4(0, src0_qs + row + (ki / 4) * m);
-            const uchar4 hbits = vload4(0, src0_hm + row + (ki / 8) * m);
-            #pragma unroll
-            for (int j = 0; j < 4; ++j) {
-                half8 b;
-                b.s0123 = read_imageh(src1, gy * 2     + (ki + j) * n_4);
-                b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
-                const int4 low = convert_int4((bits >> (uchar4)(2 * j)) & (uchar4)(3));
-                const int4 high = convert_int4((hbits >> (uchar4)((ki & 7) + j)) & (uchar4)(1));
-                const half4 w = convert_half4(low - (int4)(4) + (int4)(4) * high) * scale;
-                c0 += b * w.s0;
-                c1 += b * w.s1;
-                c2 += b * w.s2;
-                c3 += b * w.s3;
-            }
-        }
-    }
-
-    int idx = gy * 8 * m + row;
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_q8_1_dp4a.cl

@@ -1,160 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#pragma OPENCL EXTENSION cl_khr_subgroups : enable
-#ifdef cl_khr_integer_dot_product
-#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
-#endif
-
-#define QK_K 256
-
-#ifndef TILESIZE_N
-#define TILESIZE_N 32
-#endif
-
-inline int q3_k_scale(uint sc0, uint sc1, uint sc2, uint is) {
-    const uint w  = is >> 2;
-    const uint sh = 8u * (is & 3u);
-    const uint tb = (sc2 >> sh) & 0xFFu;
-    uint v;
-    if      (w == 0u) { v = ( (sc0 >> sh)       & 0xFu) | (((tb >> 0) & 3u) << 4); }
-    else if (w == 1u) { v = ( (sc1 >> sh)       & 0xFu) | (((tb >> 2) & 3u) << 4); }
-    else if (w == 2u) { v = (((sc0 >> sh) >> 4) & 0xFu) | (((tb >> 4) & 3u) << 4); }
-    else              { v = (((sc1 >> sh) >> 4) & 0xFu) | (((tb >> 6) & 3u) << 4); }
-    return (int)v - 32;
-}
-
-inline uint q3_k_pack(uint pk, uint hb) {
-    int v0 = (int)((pk      ) & 3u) - 4 + 4*(int)((hb     ) & 1u);
-    int v1 = (int)((pk >> 2) & 3u) - 4 + 4*(int)((hb >> 1) & 1u);
-    int v2 = (int)((pk >> 4) & 3u) - 4 + 4*(int)((hb >> 2) & 1u);
-    int v3 = (int)((pk >> 6) & 3u) - 4 + 4*(int)((hb >> 3) & 1u);
-    return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) <<  8)
-         | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24);
-}
-
-// Pass staged operands by value; local dp4a operands miscompile on X2.
-#define KQ_STAGE(p) vload4(0, (p))
-
-inline int dot4_q8a_v(uint4 qw, uint4 a) {
-    int r = 0;
-    r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r);
-    return r;
-}
-
-__attribute__((qcom_wave_pair_mode(1)))
-kernel void kernel_gemm_noshuffle_q3_k_q8_1_dp4a(
-        __global const uchar  * src0_qs,
-        __global const uchar  * src0_hm,
-        __global const uint   * src0_sc,
-        __global const half   * src0_d,
-        __global const uint   * src1_qa,   // q8_1 activations int8 (as uint, 4/elem) [N, K]
-        __global const half   * src1_da,   // q8_1 per-block scale [N, K/32]
-        __global       float  * dst,
-        ulong  offsetd,
-        int    m,                          // output features (rows)
-        int    n_no_padding,               // tokens (cols)
-        int    k                           // K (== ne00)
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-
-    const uint lid = get_local_id(0);
-    const uint block_id_m = get_global_id(1);
-    const uint block_id_n = get_global_id(2);
-
-    const uint row      = block_id_m * 64 + lid;
-    const uint col_base = block_id_n * TILESIZE_N;
-    const bool row_valid = row < (uint)m;
-    const uint rrow     = row_valid ? row : 0;
-
-    const uint k_u = (uint)k >> 2;
-    const uint k_b = (uint)k >> 5;
-
-    __local uint4 sh_qa4[TILESIZE_N][2];
-    __local half sh_d[TILESIZE_N];
-
-#define NGROUPS (TILESIZE_N / 4)
-    float4 acc[NGROUPS];
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
-
-    for (uint step = 0; step < (uint)k; step += 32) {
-        const uint sub = step >> 5;
-        const uint ib  = sub >> 3;                // super-block along K
-        const uint sb  = sub & 7u;
-
-        const uint scb = rrow + (3u * ib) * (uint)m;
-        const uint sc0 = src0_sc[scb + 0u * (uint)m];
-        const uint sc1 = src0_sc[scb + 1u * (uint)m];
-        const uint sc2 = src0_sc[scb + 2u * (uint)m];
-
-        const float d_w = (float)src0_d[rrow + ib * (uint)m];
-        const float dl0 = d_w * (float)q3_k_scale(sc0, sc1, sc2, 2u*sb + 0u);
-        const float dl1 = d_w * (float)q3_k_scale(sc0, sc1, sc2, 2u*sb + 1u);
-
-        const uint qsb = rrow + (step >> 2) * (uint)m;
-        const uint hmb = rrow + (step >> 3) * (uint)m;
-
-        uint4 qlo, qhi;
-        {
-            const uint h0 = (uint)src0_hm[hmb + 0u * (uint)m];
-            const uint h1 = (uint)src0_hm[hmb + 1u * (uint)m];
-            const uint h2 = (uint)src0_hm[hmb + 2u * (uint)m];
-            const uint h3 = (uint)src0_hm[hmb + 3u * (uint)m];
-            qlo.s0 = q3_k_pack((uint)src0_qs[qsb + 0u * (uint)m],  h0       & 0xFu);
-            qlo.s1 = q3_k_pack((uint)src0_qs[qsb + 1u * (uint)m], (h0 >> 4) & 0xFu);
-            qlo.s2 = q3_k_pack((uint)src0_qs[qsb + 2u * (uint)m],  h1       & 0xFu);
-            qlo.s3 = q3_k_pack((uint)src0_qs[qsb + 3u * (uint)m], (h1 >> 4) & 0xFu);
-            qhi.s0 = q3_k_pack((uint)src0_qs[qsb + 4u * (uint)m],  h2       & 0xFu);
-            qhi.s1 = q3_k_pack((uint)src0_qs[qsb + 5u * (uint)m], (h2 >> 4) & 0xFu);
-            qhi.s2 = q3_k_pack((uint)src0_qs[qsb + 6u * (uint)m],  h3       & 0xFu);
-            qhi.s3 = q3_k_pack((uint)src0_qs[qsb + 7u * (uint)m], (h3 >> 4) & 0xFu);
-        }
-
-        for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
-            const uint t = idx >> 1;
-            const uint v = idx & 1;
-            const uint c = col_base + t;
-            sh_qa4[t][v] = (c < (uint)n_no_padding)
-                         ? KQ_STAGE(src1_qa + c * k_u + (step >> 2) + (v << 2))
-                         : (uint4)(0u);
-        }
-        if (lid < TILESIZE_N) {
-            const uint c = col_base + lid;
-            sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
-        #pragma unroll
-        for (int g = 0; g < NGROUPS; ++g) {
-            const int b = g * 4;
-            float4 rf;
-#define Q3_K_COL(T) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[T][0]))  \
-                  + dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[T][1])))
-            rf.s0 = Q3_K_COL(b+0);  rf.s1 = Q3_K_COL(b+1);
-            rf.s2 = Q3_K_COL(b+2);  rf.s3 = Q3_K_COL(b+3);
-#undef Q3_K_COL
-            acc[g] += LD4(sh_d, b) * rf;
-        }
-#undef LD4
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    if (!row_valid) {
-        return;
-    }
-
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) {
-        const uint b = (uint)(g * 4);
-        const float4 a = acc[g];
-        const uint c0 = col_base + b;
-        if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
-        if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
-        if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
-        if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
-    }
-#undef NGROUPS
-}

ggml/src/ggml-opencl/kernels/gemv_noshuffle_q3_k_f32.cl

@@ -1,249 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#define QK_K 256
-
-#ifndef Q3_K_MV_NSG
-#define Q3_K_MV_NSG 8
-#endif
-
-#ifndef Q3_K_MV_R
-#define Q3_K_MV_R 2
-#endif
-
-inline int q3_k_scale(uint sc0, uint sc1, uint sc2, uint is) {
-    const uint w  = is >> 2;
-    const uint sh = 8u * (is & 3u);
-    const uint tb = (sc2 >> sh) & 0xFFu;
-    uint v;
-    if      (w == 0u) { v = ( (sc0 >> sh)       & 0xFu) | (((tb >> 0) & 3u) << 4); }
-    else if (w == 1u) { v = ( (sc1 >> sh)       & 0xFu) | (((tb >> 2) & 3u) << 4); }
-    else if (w == 2u) { v = (((sc0 >> sh) >> 4) & 0xFu) | (((tb >> 4) & 3u) << 4); }
-    else              { v = (((sc1 >> sh) >> 4) & 0xFu) | (((tb >> 6) & 3u) << 4); }
-    return (int)v - 32;
-}
-
-inline float4 q3_k_vals(uint pk, uint hb) {
-    float4 v;
-    v.s0 = (float)((int)((pk      ) & 3u) - 4 + 4*(int)((hb     ) & 1u));
-    v.s1 = (float)((int)((pk >> 2) & 3u) - 4 + 4*(int)((hb >> 1) & 1u));
-    v.s2 = (float)((int)((pk >> 4) & 3u) - 4 + 4*(int)((hb >> 2) & 1u));
-    v.s3 = (float)((int)((pk >> 6) & 3u) - 4 + 4*(int)((hb >> 3) & 1u));
-    return v;
-}
-
-kernel void kernel_gemv_noshuffle_q3_k_f32(
-        global const uchar * src0_qs,
-        global const uchar * src0_hm,
-        global const uint  * src0_sc,
-        global const half  * src0_d,
-        global const float * src1,
-        ulong offset1,
-        global float * dst,
-        ulong offsetd,
-        int ne00,      // K
-        int ne01,      // M
-        int ne10,      // activation row stride
-        int ne0        // dst row stride
-) {
-    src1 = (global const float *)((global const char *)src1 + offset1);
-    dst  = (global float       *)((global char       *)dst  + offsetd);
-
-    const uint m   = (uint)ne01;
-    const uint K   = (uint)ne00;
-    const uint nsb = K / QK_K;
-
-    const uint lid = get_local_id(0);
-    const uint sgi = get_local_id(1);
-    const uint col = get_group_id(1);
-
-    global const float * y = src1 + (ulong)col * (uint)ne10;
-
-#if Q3_K_MV_R == 4
-    const uint mq  = m >> 2;
-    const uint j   = get_group_id(0) * 64u + lid;
-    const uint row = j << 2;
-
-    float sumf = 0.f, sumf1 = 0.f, sumf2 = 0.f, sumf3 = 0.f;
-
-    if (j < mq) {
-        global const uint * qsw = (global const uint *)src0_qs;
-        global const uint * hmw = (global const uint *)src0_hm;
-
-        for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) {
-            const half4 dh = vload4(j + ib * mq, src0_d);
-            const uint4 s0 = vload4(j + (3u * ib + 0u) * mq, src0_sc);
-            const uint4 s1 = vload4(j + (3u * ib + 1u) * mq, src0_sc);
-            const uint4 s2 = vload4(j + (3u * ib + 2u) * mq, src0_sc);
-
-            float acc0 = 0.f, acc1 = 0.f, acc2 = 0.f, acc3 = 0.f;
-            for (uint sb = 0; sb < 8u; ++sb) {
-                const uint grp = ib * 64u + sb * 8u;
-                const uint qsb = j + grp * mq;
-                const uint hmb = j + (grp >> 1) * mq;
-
-                for (uint h = 0; h < 2u; ++h) {         // two 16-weight halves
-                    const int l0 = q3_k_scale(s0.s0, s1.s0, s2.s0, 2u*sb + h);
-                    const int l1 = q3_k_scale(s0.s1, s1.s1, s2.s1, 2u*sb + h);
-                    const int l2 = q3_k_scale(s0.s2, s1.s2, s2.s2, 2u*sb + h);
-                    const int l3 = q3_k_scale(s0.s3, s1.s3, s2.s3, 2u*sb + h);
-
-                    float a0 = 0.f, a1 = 0.f, a2 = 0.f, a3 = 0.f;
-                    for (uint u = 0; u < 4u; ++u) {
-                        const uint gg  = 4u*h + u;
-                        const uint qsv = qsw[qsb + gg * mq];        // four rows, one load
-                        const uint hmv = hmw[hmb + (gg >> 1) * mq];
-                        const uint hsh = 4u * (gg & 1u);
-                        const float4 yv = vload4(grp + gg, y);      // shared by all four
-                        a0 += dot(yv, q3_k_vals((qsv      ) & 0xFFu, ((hmv      ) >> hsh) & 0xFu));
-                        a1 += dot(yv, q3_k_vals((qsv >>  8) & 0xFFu, ((hmv >>  8) >> hsh) & 0xFu));
-                        a2 += dot(yv, q3_k_vals((qsv >> 16) & 0xFFu, ((hmv >> 16) >> hsh) & 0xFu));
-                        a3 += dot(yv, q3_k_vals((qsv >> 24) & 0xFFu, ((hmv >> 24) >> hsh) & 0xFu));
-                    }
-                    acc0 += (float)l0 * a0;
-                    acc1 += (float)l1 * a1;
-                    acc2 += (float)l2 * a2;
-                    acc3 += (float)l3 * a3;
-                }
-            }
-            sumf  += (float)dh.s0 * acc0;
-            sumf1 += (float)dh.s1 * acc1;
-            sumf2 += (float)dh.s2 * acc2;
-            sumf3 += (float)dh.s3 * acc3;
-        }
-    }
-#elif Q3_K_MV_R == 2
-    const uint mh  = m >> 1;
-    const uint j   = get_group_id(0) * 64u + lid;
-    const uint row = j << 1;
-
-    float sumf  = 0.f;
-    float sumf1 = 0.f;
-
-    if (j < mh) {
-        global const ushort * qsu = (global const ushort *)src0_qs;
-        global const ushort * hmu = (global const ushort *)src0_hm;
-
-        for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) {
-            const half2 dh = vload2(j + ib * mh, src0_d);
-            const uint2 s0 = vload2(j + (3u * ib + 0u) * mh, src0_sc);
-            const uint2 s1 = vload2(j + (3u * ib + 1u) * mh, src0_sc);
-            const uint2 s2 = vload2(j + (3u * ib + 2u) * mh, src0_sc);
-
-            float acc0 = 0.f, acc1 = 0.f;
-            for (uint sb = 0; sb < 8u; ++sb) {
-                const uint grp = ib * 64u + sb * 8u;
-                const uint qsb = j + grp * mh;
-                const uint hmb = j + (grp >> 1) * mh;
-
-                for (uint h = 0; h < 2u; ++h) {         // two 16-weight halves
-                    const int l0 = q3_k_scale(s0.s0, s1.s0, s2.s0, 2u*sb + h);
-                    const int l1 = q3_k_scale(s0.s1, s1.s1, s2.s1, 2u*sb + h);
-
-                    float a0 = 0.f, a1 = 0.f;
-                    for (uint u = 0; u < 4u; ++u) {
-                        const uint gg  = 4u*h + u;
-                        const uint qsv = (uint)qsu[qsb + gg * mh];
-                        const uint hmv = (uint)hmu[hmb + (gg >> 1) * mh];
-                        const uint hsh = 4u * (gg & 1u);
-                        const float4 yv = vload4(grp + gg, y);
-                        a0 += dot(yv, q3_k_vals( qsv       & 0xFFu, ( hmv        >> hsh) & 0xFu));
-                        a1 += dot(yv, q3_k_vals((qsv >> 8) & 0xFFu, ((hmv >> 8)  >> hsh) & 0xFu));
-                    }
-                    acc0 += (float)l0 * a0;
-                    acc1 += (float)l1 * a1;
-                }
-            }
-            sumf  += (float)dh.s0 * acc0;
-            sumf1 += (float)dh.s1 * acc1;
-        }
-    }
-#else
-    const uint row = get_group_id(0) * 64u + lid;
-
-    float sumf = 0.f;
-
-    if (row < m) {
-        for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) {
-            const float d  = (float)src0_d[row + ib * m];
-            const uint  s0 = src0_sc[row + (3u * ib + 0u) * m];
-            const uint  s1 = src0_sc[row + (3u * ib + 1u) * m];
-            const uint  s2 = src0_sc[row + (3u * ib + 2u) * m];
-
-            float acc = 0.f;
-            for (uint sb = 0; sb < 8u; ++sb) {
-                const uint grp = ib * 64u + sb * 8u;
-                const uint qsb = row + grp * m;
-                const uint hmb = row + (grp >> 1) * m;
-
-                for (uint h = 0; h < 2u; ++h) {
-                    const int l = q3_k_scale(s0, s1, s2, 2u*sb + h);
-                    float a = 0.f;
-                    for (uint u = 0; u < 4u; ++u) {
-                        const uint gg  = 4u*h + u;
-                        const uint qsv = (uint)src0_qs[qsb + gg * m];
-                        const uint hmv = (uint)src0_hm[hmb + (gg >> 1) * m];
-                        const float4 yv = vload4(grp + gg, y);
-                        a += dot(yv, q3_k_vals(qsv, (hmv >> (4u * (gg & 1u))) & 0xFu));
-                    }
-                    acc += (float)l * a;
-                }
-            }
-            sumf += d * acc;
-        }
-    }
-#endif
-
-#if Q3_K_MV_NSG > 1
-#if Q3_K_MV_R == 4
-    __local float4 part[Q3_K_MV_NSG][64];
-    part[sgi][lid] = (float4)(sumf, sumf1, sumf2, sumf3);
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < Q3_K_MV_NSG; ++s) {
-        const float4 p = part[s][lid];
-        sumf  += p.s0;
-        sumf1 += p.s1;
-        sumf2 += p.s2;
-        sumf3 += p.s3;
-    }
-#elif Q3_K_MV_R == 2
-    __local float2 part[Q3_K_MV_NSG][64];
-    part[sgi][lid] = (float2)(sumf, sumf1);
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < Q3_K_MV_NSG; ++s) {
-        const float2 p = part[s][lid];
-        sumf  += p.s0;
-        sumf1 += p.s1;
-    }
-#else
-    __local float part[Q3_K_MV_NSG][64];
-    part[sgi][lid] = sumf;
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < Q3_K_MV_NSG; ++s) {
-        sumf += part[s][lid];
-    }
-#endif
-#endif
-
-#if Q3_K_MV_R == 4
-    if (j < mq) {
-        vstore4((float4)(sumf, sumf1, sumf2, sumf3), 0, dst + (ulong)col * (uint)ne0 + row);
-    }
-#elif Q3_K_MV_R == 2
-    if (j < mh) {
-        vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
-    }
-#else
-    if (row < m) {
-        dst[(ulong)col * (uint)ne0 + row] = sumf;
-    }
-#endif
-}

ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl

@@ -57,17 +57,13 @@ kernel void kernel_add_id_add_id_swiglu_oai(
 
     global const float * g_row  = (global const float *)(src_g  + i1*nb01_g + i2*nb02_g);
     global const float * u_row  = (global const float *)(src_u  + i1*nb01_u + i2*nb02_u);
+    global const float * gb_row = (global const float *)(src_gb + i11*nb11_g);
+    global const float * ub_row = (global const float *)(src_ub + i11*nb11_u);
     global       float * d_row  = (global       float *)(dst    + i1*nbd1   + i2*nbd2);
 
     for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {
-        float x0 = g_row[i0];
-        float x1 = u_row[i0];
-        if (i11 != -1) {
-            global const float * gb_row = (global const float *)(src_gb + i11*nb11_g);
-            global const float * ub_row = (global const float *)(src_ub + i11*nb11_u);
-            x0 += gb_row[i0];
-            x1 += ub_row[i0];
-        }
+        float x0 = g_row[i0] + gb_row[i0];
+        float x1 = u_row[i0] + ub_row[i0];
 
         x0 = min(x0, limit);
         x1 = max(min(x1, limit), -limit);

ggml/src/ggml-opencl/kernels/moe_combine.cl

@@ -42,11 +42,8 @@ __kernel void kernel_moe_combine_bias_f32(
     float4 acc = (float4)(0.0f);
     for (int e = 0; e < k; ++e) {
         const int i11 = *((__global const int *)(I + (ulong)e*sizeof(int)));
-        float4 v = vload4(0, E + (uint)e*e1);
-        if (i11 != -1) {
-            __global const float * Brow = (__global const float *)(B + (ulong)i11*nb_b1) + r4*4u;
-            v += vload4(0, Brow);
-        }
+        __global const float * Brow = (__global const float *)(B + (ulong)i11*nb_b1) + r4*4u;
+        const float4 v = vload4(0, E + (uint)e*e1) + vload4(0, Brow);
         acc = mad(v, (float4)(W[(uint)e*w1]), acc);
     }
 

ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl

@@ -80,6 +80,32 @@ __kernel void kernel_moe_scatter(
     emap[tile_idx] = val;
 }
 
+// the gemm only writes the rows an expert owns, so the skipped rows (negative ids) are zeroed here
+__kernel void kernel_moe_zero_dst(
+    __global const int * input,
+    __global float * dst,
+    uint N,
+    uint topK,
+    uint n_experts,
+    uint ne0
+) {
+    uint n = get_global_id(0);
+    uint k = get_global_id(1);
+
+    if (n >= N || k >= topK) {
+        return;
+    }
+
+    if (input[n * n_experts + k] != -1) {
+        return;
+    }
+
+    __global float * dst_row = dst + ((ulong)n * topK + k) * ne0;
+    for (uint i = get_local_id(2); i < ne0; i += get_local_size(2)) {
+        dst_row[i] = 0.0f;
+    }
+}
+
 // Deterministic replacement for kernel_moe_scatter.
 //
 // kernel_moe_scatter takes each token's slot from atomic_inc(slot_counter[expert]),

ggml/src/ggml-opencl/kernels/mul_mat_id_utils.cl

@@ -1,22 +0,0 @@
-kernel void kernel_moe_zero_dst(
-    global const char * ids,
-    ulong offset_ids,
-    ulong nb_ids1,
-    global char * dst,
-    ulong offset_dst,
-    ulong nb1,
-    ulong nb2,
-    int ne0
-) {
-    const size_t i1 = get_group_id(0);
-    const size_t i2 = get_group_id(1);
-    const int expert = *((global const int *)(ids + offset_ids + i1*sizeof(int) + i2*nb_ids1));
-    if (expert != -1) {
-        return;
-    }
-
-    global float * row = (global float *)(dst + offset_dst + i1*nb1 + i2*nb2);
-    for (size_t i0 = get_local_id(2); i0 < (size_t)ne0; i0 += get_local_size(2)) {
-        row[i0] = 0.0f;
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mm_q3_k_f32_l4_lm.cl

@@ -1,219 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#define INTEL_GPU 1
-#endif
-
-#define QK_K 256
-
-struct block_q3_K {
-    uchar hmask[QK_K/8];
-    uchar qs[QK_K/4];
-    uchar scales[12];
-    half d;
-};
-
-#define LOAD_VEC_A 4
-#define LOAD_VEC_B 4
-
-#define BM 64
-#define BN 64
-#ifndef BK
-#define BK 32
-#endif
-#ifdef INTEL_GPU
-#define TM 8
-#else
-#define TM 4
-#endif
-#define TN 8
-
-#ifndef LM_HALF
-#define LM_HALF 0
-#endif
-#if LM_HALF
-typedef half lm_st;
-#define LM_LD4(p, i) convert_float4(vload4((i), (p)))
-#else
-typedef float lm_st;
-#define LM_LD4(p, i) vload4((i), (p))
-#endif
-
-kernel void kernel_mul_mm_q3_k_f32_l4_lm(
-    global uchar * src0_qs,
-    global uchar * src0_hm,
-    global uchar * src0_sc,
-    global half * src0_d,
-    ulong offset0,
-    global float4 * src1,
-    ulong offset1,
-    global float  * dst,
-    ulong offsetd,
-
-    int ne00,
-    int ne01,
-    int ne02,
-    int ne11,
-    int ne12,
-
-    int stride_a,
-    int stride_b,
-    int stride_d,
-
-    int batch_stride_a,
-    int batch_stride_b,
-    int batch_stride_d,
-
-    int r2,
-    int r3
-) {
-    src1 = (global float4*)((global char*)src1 + offset1);
-    dst  = (global float *)((global char*)dst  + offsetd);
-
-    local lm_st buf_a[BM * BK];
-    local lm_st buf_b[BN * BK];
-
-    const int batch_idx = get_global_id(2);
-
-    const int i13 = batch_idx / ne12;
-    const int i12 = batch_idx % ne12;
-
-    const int i03 = i13 / r3;
-    const int i02 = i12 / r2;
-
-    const int batch_idx_a = i03 * ne02 + i02;
-
-    const int ir = get_group_id(0);
-    const int ic = get_group_id(1);
-
-    const int tid = get_local_id(0);
-    const int th_r  = tid % (BM / TM);
-    const int th_c  = tid / (BM / TM);
-
-    const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
-    const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
-    const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
-    const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
-
-    const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
-    const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
-
-    // pos_a counts elements, not blocks - one q3_K super block holds QK_K of them
-    int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
-    int pos_b = (batch_idx   * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
-
-    float4 sums4[(TM/4) * TN];
-    float4 cache_a4[TM/4];
-
-    for (int i = 0; i < (TM/4) * TN; i++) {
-        sums4[i] = (float4)(0.0f);
-    }
-
-    for (int block = 0; block < ne00; block += BK) {
-        for (int l = 0; l < BM; l += loadstride_a) {
-            if (ir*BM + loadc_a + l < ne01) {
-                int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
-                int ib  = idx / QK_K;
-                ib += offset0/sizeof(struct block_q3_K);
-                int e   = idx % QK_K;
-
-                // Only qs advances in the second half of a super-block.
-                int n     = e >> 7;
-                int rem   = e & 127;
-                int j     = rem >> 5;
-                int s     = (rem >> 4) & 1;
-                int l16   = rem & 15;
-                int is    = 8*n + 2*j + s;
-                int shift = 2*j;
-                uchar m   = (uchar)(1u << (4*n + j));
-
-                global ushort * a = (global ushort *)(src0_sc + 12*ib);
-                uint a0  = (uint)a[0] | ((uint)a[1] << 16);
-                uint a1  = (uint)a[2] | ((uint)a[3] << 16);
-                uint tmp = (uint)a[4] | ((uint)a[5] << 16);
-
-                uint g   = is >> 2;
-                uint w   = (g & 1u) ? a1 : a0;
-                w = ((w >> ((g >> 1)*4)) & 0x0f0f0f0fu) | (((tmp >> (2*g)) & 0x03030303u) << 4);
-
-                uint  sc6 = (w >> (8*(is & 3))) & 0xFFu;
-                float dl  = (float)src0_d[ib] * ((float)sc6 - 32.0f);
-
-                uchar4 q  = vload4(0, src0_qs + 64*ib + 32*n + 16*s + l16);
-                uchar4 hb = vload4(0, src0_hm + 32*ib + 16*s + l16);
-
-                float4 qv = convert_float4((uchar4)((q.s0 >> shift) & 3,
-                                                    (q.s1 >> shift) & 3,
-                                                    (q.s2 >> shift) & 3,
-                                                    (q.s3 >> shift) & 3));
-                float4 hv = (float4)((hb.s0 & m) ? 0.f : 4.f,
-                                     (hb.s1 & m) ? 0.f : 4.f,
-                                     (hb.s2 & m) ? 0.f : 4.f,
-                                     (hb.s3 & m) ? 0.f : 4.f);
-                float4 v1 = (qv - hv) * dl;
-
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
-            } else {
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
-            }
-        }
-
-        for (int l = 0; l < BN; l += loadstride_b) {
-            if (ic*BN + loadc_b + l < ne11) {
-                int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
-            } else {
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
-            }
-        }
-
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-        pos_a += BK;
-        pos_b += BK / LOAD_VEC_B;
-
-        for (int i = 0; i < BK; i++) {
-            for (int a = 0; a < TM/4; a++) {
-                cache_a4[a] = LM_LD4(buf_a + (i) * BM + th_r * TM, a);
-            }
-
-            for (int cc = 0; cc < TN; cc++) {
-                const float cache_b = buf_b[(i) * BN + th_c * TN + cc];
-                for (int a = 0; a < TM/4; a++) {
-                    const int sums_idx = cc*(TM/4) + a;
-                    sums4[sums_idx] = mad(cache_a4[a], (float4)cache_b, sums4[sums_idx]);
-                }
-            }
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    const int dr = ir * BM + th_r * TM;
-    const int dc = ic * BN + th_c * TN;
-
-    const ulong offsets = batch_idx * batch_stride_d;
-
-    for (int cc = 0; cc < TN; cc++) {
-        for (int a = 0; a < TM/4; a++) {
-            const float4 v = sums4[cc * (TM/4) + a];
-            const float  vs[4] = { v.s0, v.s1, v.s2, v.s3 };
-            for (int k = 0; k < 4; k++) {
-                if (dr + 4*a + k < ne01 && dc + cc < ne11) {
-                    dst[offsets + (dc + cc) * stride_d + dr + 4*a + k] = vs[k];
-                }
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32.cl

@@ -1,197 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-//------------------------------------------------------------------------------
-// block_q3_K
-//------------------------------------------------------------------------------
-#define QK_K 256
-
-// 16 blocks of 16 elements each
-// weight is represented as x = a * q
-typedef struct {
-    uchar hmask[QK_K/8]; // quants - high bit
-    uchar qs[QK_K/4];    // quants - low 2 bits
-    uchar scales[12];    // scales, quantized with 6 bits
-    half  d;             // super-block scale
-} block_q3_K;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 2 // number of rows each SIMD group works on
-#define N_SIMDGROUP 1 // number of SIMD groups in a thread group
-#define N_SIMDWIDTH 16 // SIMD group size
-#elif defined (ADRENO_GPU)
-#define N_DST 2
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_q3_K_f32(
-        global char * src0,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src0 = src0 + offset0;
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global block_q3_K * x  = (global block_q3_K *) (src0 + offset_src0);
-    global float      * yy = (global float      *) (src1 + offset_src1);
-
-    float yl[32];
-
-    // Eight lanes cover one super-block.
-    int ix  = get_sub_group_local_id() % BLOCK_STRIDE;
-    int tid = get_sub_group_local_id() / BLOCK_STRIDE;
-
-    int ip = tid/4;         // 0 or 1
-    int il = 2*((tid%4)/2); // 0 or 2
-    int ir = tid%2;
-    int l0 = 8*ir;
-
-    // Shift the masks instead of indexing private arrays to avoid spills.
-    ushort4 hm = (ushort4)(0x0001, 0x0100, 0x0002, 0x0200) << (ushort4)(2*(2*ip + il/2));
-    int4    qm = (int4)   (0x0003, 0x0300, 0x000c, 0x0c00) << (int4)   (4*(il/2));
-
-    int shift = 2*il;
-
-    float v1 = il == 0 ? 4.f : 64.f;
-    float v2 = 4.f * v1;
-
-    int s_shift1 = 4*ip;
-    int s_shift2 = s_shift1 + il;
-
-    int q_offset = 32*ip + l0;
-    int y_offset = 128*ip + 32*il + l0;
-
-    global float * y1 = yy + ix*QK_K + y_offset;
-
-    float sumf1[N_DST] = {0.f};
-    float sumf2[N_DST] = {0.f};
-    float all_sum;
-
-    for (int i = ix; i < nb; i += BLOCK_STRIDE) {
-        for (int l = 0; l < 8; ++l) {
-            yl[l+ 0] = y1[l+ 0];
-            yl[l+ 8] = y1[l+16];
-            yl[l+16] = y1[l+32];
-            yl[l+24] = y1[l+48];
-        }
-
-        global ushort * q = (global ushort *)(x[i].qs + q_offset);
-        global ushort * h = (global ushort *)(x[i].hmask + l0);
-        global ushort * a = (global ushort *)(x[i].scales);
-        global half   * dh = &x[i].d;
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; ++row) {
-            float d_all = dh[0];
-
-            uint s32   = (uint)a[4] | ((uint)a[5] << 16);
-            uint aux32 = ((s32 >> s_shift2) << 4) & 0x30303030u;
-            s32 = (uint)a[il+0] | ((uint)a[il+1] << 16);
-            s32 = ((s32 >> s_shift1) & 0x0f0f0f0fu) | aux32;
-            char4 sc = as_char4(s32);
-
-            float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0;
-            for (int l = 0; l < 8; l += 2) {
-                int qs = q[l/2];
-                s1 += yl[l+0] * (qs & qm.s0);
-                s2 += yl[l+1] * (qs & qm.s1);
-                s3 += ((h[l/2] & hm.s0) ? 0.f : yl[l+0]) + ((h[l/2] & hm.s1) ? 0.f : yl[l+1]);
-                s4 += yl[l+16] * (qs & qm.s2);
-                s5 += yl[l+17] * (qs & qm.s3);
-                s6 += ((h[l/2] & hm.s2) ? 0.f : yl[l+16]) + ((h[l/2] & hm.s3) ? 0.f : yl[l+17]);
-            }
-            float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
-            float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
-            sumf1[row] += d1 * (sc.s0 - 32);
-            sumf2[row] += d2 * (sc.s2 - 32);
-
-            s1 = s2 = s3 = s4 = s5 = s6 = 0;
-            for (int l = 0; l < 8; l += 2) {
-                int qs = q[l/2+8];
-                s1 += yl[l+8] * (qs & qm.s0);
-                s2 += yl[l+9] * (qs & qm.s1);
-                s3 += ((h[l/2+8] & hm.s0) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm.s1) ? 0.f : yl[l+9]);
-                s4 += yl[l+24] * (qs & qm.s2);
-                s5 += yl[l+25] * (qs & qm.s3);
-                s6 += ((h[l/2+8] & hm.s2) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm.s3) ? 0.f : yl[l+25]);
-            }
-            d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
-            d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
-            sumf1[row] += d1 * (sc.s1 - 32);
-            sumf2[row] += d2 * (sc.s3 - 32);
-
-            q  += nb01/2;
-            h  += nb01/2;
-            a  += nb01/2;
-            dh += nb01/2;
-        }
-
-        y1 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add((sumf1[row] + 0.25f * sumf2[row]) / (1 << shift));
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32_flat.cl

@@ -1,199 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-//------------------------------------------------------------------------------
-// block_q3_K
-//------------------------------------------------------------------------------
-#define QK_K 256
-
-// 16 blocks of 16 elements each
-// weight is represented as x = a * q
-typedef struct {
-    uchar hmask[QK_K/8]; // quants - high bit
-    uchar qs[QK_K/4];    // quants - low 2 bits
-    uchar scales[12];    // scales, quantized with 6 bits
-    half  d;             // super-block scale
-} block_q3_K;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 2 // number of rows each SIMD group works on
-#define N_SIMDGROUP 1 // number of SIMD groups in a thread group
-#define N_SIMDWIDTH 16 // SIMD group size
-#elif defined (ADRENO_GPU)
-#define N_DST 2
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_q3_K_f32_flat(
-        global uchar * src0_qs,
-        global uchar * src0_hm,
-        global uchar * src0_sc,
-        global half * src0_d,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_q3_K);
-    ulong row_stride = nb01/sizeof(block_q3_K);
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global float      * yy = (global float      *) (src1 + offset_src1);
-
-    float yl[32];
-
-    // Eight lanes cover one super-block.
-    int ix  = get_sub_group_local_id() % BLOCK_STRIDE;
-    int tid = get_sub_group_local_id() / BLOCK_STRIDE;
-
-    int ip = tid/4;         // 0 or 1
-    int il = 2*((tid%4)/2); // 0 or 2
-    int ir = tid%2;
-    int l0 = 8*ir;
-
-    // Shift the masks instead of indexing private arrays to avoid spills.
-    ushort4 hm = (ushort4)(0x0001, 0x0100, 0x0002, 0x0200) << (ushort4)(2*(2*ip + il/2));
-    int4    qm = (int4)   (0x0003, 0x0300, 0x000c, 0x0c00) << (int4)   (4*(il/2));
-
-    int shift = 2*il;
-
-    float v1 = il == 0 ? 4.f : 64.f;
-    float v2 = 4.f * v1;
-
-    int s_shift1 = 4*ip;
-    int s_shift2 = s_shift1 + il;
-
-    int q_offset = 32*ip + l0;
-    int y_offset = 128*ip + 32*il + l0;
-
-    global float * y1 = yy + ix*QK_K + y_offset;
-
-    float sumf1[N_DST] = {0.f};
-    float sumf2[N_DST] = {0.f};
-    float all_sum;
-
-    for (int i = ix; i < nb; i += BLOCK_STRIDE) {
-        for (int l = 0; l < 8; ++l) {
-            yl[l+ 0] = y1[l+ 0];
-            yl[l+ 8] = y1[l+16];
-            yl[l+16] = y1[l+32];
-            yl[l+24] = y1[l+48];
-        }
-
-        global ushort * q = (global ushort *)(src0_qs + 64*(offset_src0 + i) + q_offset);
-        global ushort * h = (global ushort *)(src0_hm + 32*(offset_src0 + i) + l0);
-        global ushort * a = (global ushort *)(src0_sc + 12*(offset_src0 + i));
-        global half   * dh = src0_d + offset_src0 + i;
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; ++row) {
-            float d_all = dh[0];
-
-            uint s32   = (uint)a[4] | ((uint)a[5] << 16);
-            uint aux32 = ((s32 >> s_shift2) << 4) & 0x30303030u;
-            s32 = (uint)a[il+0] | ((uint)a[il+1] << 16);
-            s32 = ((s32 >> s_shift1) & 0x0f0f0f0fu) | aux32;
-            char4 sc = as_char4(s32);
-
-            float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0;
-            for (int l = 0; l < 8; l += 2) {
-                int qs = q[l/2];
-                s1 += yl[l+0] * (qs & qm.s0);
-                s2 += yl[l+1] * (qs & qm.s1);
-                s3 += ((h[l/2] & hm.s0) ? 0.f : yl[l+0]) + ((h[l/2] & hm.s1) ? 0.f : yl[l+1]);
-                s4 += yl[l+16] * (qs & qm.s2);
-                s5 += yl[l+17] * (qs & qm.s3);
-                s6 += ((h[l/2] & hm.s2) ? 0.f : yl[l+16]) + ((h[l/2] & hm.s3) ? 0.f : yl[l+17]);
-            }
-            float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
-            float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
-            sumf1[row] += d1 * (sc.s0 - 32);
-            sumf2[row] += d2 * (sc.s2 - 32);
-
-            s1 = s2 = s3 = s4 = s5 = s6 = 0;
-            for (int l = 0; l < 8; l += 2) {
-                int qs = q[l/2+8];
-                s1 += yl[l+8] * (qs & qm.s0);
-                s2 += yl[l+9] * (qs & qm.s1);
-                s3 += ((h[l/2+8] & hm.s0) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm.s1) ? 0.f : yl[l+9]);
-                s4 += yl[l+24] * (qs & qm.s2);
-                s5 += yl[l+25] * (qs & qm.s3);
-                s6 += ((h[l/2+8] & hm.s2) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm.s3) ? 0.f : yl[l+25]);
-            }
-            d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
-            d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
-            sumf1[row] += d1 * (sc.s1 - 32);
-            sumf2[row] += d2 * (sc.s3 - 32);
-
-            q += row_stride*32;
-            h += row_stride*16;
-            a += row_stride*6;
-            dh += row_stride*1;
-        }
-
-        y1 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add((sumf1[row] + 0.25f * sumf2[row]) / (1 << shift));
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-rpc/ggml-rpc.cpp

@@ -125,16 +125,11 @@ struct rpc_msg_hello_req {
     uint8_t conn_caps[RPC_CONN_CAPS_SIZE];
 };
 
-// server feature flags in rpc_msg_hello_rsp::flags; old servers send 0
-enum rpc_server_flags : uint8_t {
-    RPC_SERVER_FLAG_CACHE = 1 << 0, // server has a local tensor cache (rpc-server -c)
-};
-
 struct rpc_msg_hello_rsp {
     uint8_t major;
     uint8_t minor;
     uint8_t patch;
-    uint8_t flags;
+    uint8_t padding;
     uint8_t conn_caps[RPC_CONN_CAPS_SIZE];
 };
 
@@ -353,10 +348,7 @@ static bool recv_msg(socket_ptr sock, std::vector<uint8_t> & input) {
     }
     try {
         input.resize(size);
-    } catch (const std::length_error &) {
-        LOG_ERROR("Failed to allocate input buffer of size %" PRIu64 "\n", size);
-        return false;
-    } catch (const std::bad_alloc &) {
+    } catch (const std::bad_alloc & e) {
         LOG_ERROR("Failed to allocate input buffer of size %" PRIu64 "\n", size);
         return false;
     }
@@ -378,30 +370,21 @@ static bool parse_endpoint(const std::string & endpoint, std::string & host, int
 }
 
 // RPC request : | rpc_cmd (1 byte) | request_size (8 bytes) | request_data (request_size bytes) |
-// request_data is input followed by an optional payload, so large data does not need to be copied into input
 // No response
-static bool send_rpc_cmd_payload(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size, const void * payload, size_t payload_size) {
+static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size) {
     uint8_t cmd_byte = cmd;
     if (!sock->send_data(&cmd_byte, sizeof(cmd_byte))) {
         return false;
     }
-    const size_t request_size = input_size + payload_size;
-    if (!sock->send_data(&request_size, sizeof(request_size))) {
+    if (!sock->send_data(&input_size, sizeof(input_size))) {
         return false;
     }
     if (!sock->send_data(input, input_size)) {
         return false;
     }
-    if (payload_size > 0 && !sock->send_data(payload, payload_size)) {
-        return false;
-    }
     return sock->flush();
 }
 
-static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size) {
-    return send_rpc_cmd_payload(sock, cmd, input, input_size, nullptr, 0);
-}
-
 // RPC request : | rpc_cmd (1 byte) | request_size (8 bytes) | request_data (request_size bytes) |
 // RPC response: | response_size (8 bytes) | response_data (response_size bytes) |
 static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size, void * output, size_t output_size) {
@@ -437,7 +420,7 @@ static inline void rpc_cpu_relax() {
 // Performs HELLO handshake with transport auto-negotiation.
 // Advertises local capabilities via conn_caps; if the server responds with
 // matching capabilities, the socket is upgraded transparently.
-static bool negotiate_hello(const std::shared_ptr<socket_t> & sock, uint8_t & server_flags) {
+static bool negotiate_hello(const std::shared_ptr<socket_t> & sock) {
     rpc_msg_hello_req request = {};
     rpc_msg_hello_rsp response = {};
 
@@ -453,7 +436,6 @@ static bool negotiate_hello(const std::shared_ptr<socket_t> & sock, uint8_t & se
     }
 
     sock->update_caps(response.conn_caps);
-    server_flags = response.flags;
     LOG_DBG("[%s] handshake ok: server version %d.%d.%d, transport: %s\n",
             __func__, response.major, response.minor, response.patch, sock->transport_name());
     return true;
@@ -518,8 +500,6 @@ public:
     void send(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, void * output, size_t output_size);
     void send_async(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size);
     void send_async(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, void * output, size_t output_size);
-    // payload is sent after input without a copy, the call returns once it is sent
-    void send_payload(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, const void * payload, size_t payload_size);
 
     ggml_backend_event_t event_new(ggml_backend_dev_t dev);
     void event_free(ggml_backend_event_t event);
@@ -533,8 +513,6 @@ public:
     void start(const std::string & endpoint);
     void work();
 
-    bool server_has_cache() const { return server_flags & RPC_SERVER_FLAG_CACHE; }
-
     ~rpc_dispatcher();
 
 private:
@@ -544,8 +522,6 @@ private:
         size_t                        input_size;
         void                        * output;
         size_t                        output_size;
-        const void                  * payload      = nullptr;
-        size_t                        payload_size = 0;
         std::promise<void>            completion;
     };
     using rpc_msg_ptr   = std::shared_ptr<rpc_msg>;
@@ -558,7 +534,6 @@ private:
     std::unordered_map<uint32_t, std::unordered_set<uint64_t>> graph_uids;
     rpc_msg_queue    queue;
     socket_ptr       sock;
-    uint8_t          server_flags = 0;
     std::atomic_uint busy_spin_users = 0;
     std::atomic_bool running;
     std::thread      thread;
@@ -591,20 +566,6 @@ void rpc_dispatcher::send_async(enum rpc_cmd cmd, std::shared_ptr<const void> in
     GGML_ASSERT(queue.push(msg));
 }
 
-void rpc_dispatcher::send_payload(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, const void * payload, size_t payload_size) {
-    auto msg = std::make_shared<rpc_msg>();
-    msg->cmd = cmd;
-    msg->input = input;
-    msg->input_size = input_size;
-    msg->output = nullptr;
-    msg->output_size = 0;
-    msg->payload = payload;
-    msg->payload_size = payload_size;
-    GGML_ASSERT(queue.push(msg));
-    auto future = msg->completion.get_future();
-    future.wait();
-}
-
 void rpc_dispatcher::send(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, void * output, size_t output_size) {
     auto msg = std::make_shared<rpc_msg>();
     msg->cmd = cmd;
@@ -688,7 +649,7 @@ void rpc_dispatcher::start(const std::string & endpoint) {
     if (sock == nullptr) {
         GGML_ABORT("Failed to connect to %s\n", endpoint.c_str());
     }
-    if (!negotiate_hello(sock, server_flags)) {
+    if (!negotiate_hello(sock)) {
         GGML_ABORT("RPC handshake failed for %s\n", endpoint.c_str());
     }
     LOG_DBG("[%s] connected to %s\n", __func__, endpoint.c_str());
@@ -720,7 +681,7 @@ void rpc_dispatcher::work() {
                 bool status = send_rpc_cmd(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size, msg_ptr->output, msg_ptr->output_size);
                 RPC_STATUS_ASSERT(status);
             } else {
-                bool status = send_rpc_cmd_payload(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size, msg_ptr->payload, msg_ptr->payload_size);
+                bool status = send_rpc_cmd(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size);
                 RPC_STATUS_ASSERT(status);
             }
             if (trace) {
@@ -865,17 +826,14 @@ static void ggml_backend_rpc_buffer_memset_tensor(
 }
 
 // input serialization format: | rpc_tensor | cache_flag (1 byte) | offset (8 bytes) | data (size bytes)
-// with size == 0 only the header is serialized and data is sent separately as payload
 static std::shared_ptr<uint8_t> serialize_set_tensor(const rpc_tensor & rpc_tensor, uint8_t cache_flag, uint64_t offset, const void * data, size_t size, size_t & input_size) {
     input_size = sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset) + size;
-    uint8_t * input = new uint8_t[input_size];
+    uint8_t * input = new uint8_t[input_size]();
     uint8_t * p = input;
     memcpy(p, &rpc_tensor, sizeof(rpc_tensor)); p += sizeof(rpc_tensor);
     memcpy(p, &cache_flag, sizeof(cache_flag)); p += sizeof(cache_flag);
     memcpy(p, &offset,     sizeof(offset));     p += sizeof(offset);
-    if (size > 0) {
-        memcpy(p, data, size);
-    }
+    memcpy(p, data, size);
     return std::shared_ptr<uint8_t>(input, std::default_delete<uint8_t[]>());
 }
 
@@ -883,9 +841,8 @@ static std::shared_ptr<uint8_t> serialize_set_tensor(const rpc_tensor & rpc_tens
 // compute-buffer inputs (the activations ggml_backend_sched copies between backends) must not
 // take this path, otherwise with `rpc-server -c` every ubatch above the threshold is written
 // to the cache directory and later served from there.
-// hashing is slow (~1 GB/s), so skip it when the server has no cache to look up.
-static bool rpc_use_hash_cache(const rpc_dispatcher & dispatcher, const ggml_tensor * tensor, size_t size) {
-    return dispatcher.server_has_cache() && size > HASH_THRESHOLD && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS;
+static bool rpc_use_hash_cache(const ggml_tensor * tensor, size_t size) {
+    return size > HASH_THRESHOLD && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS;
 }
 
 static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
@@ -894,7 +851,7 @@ static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggm
     uint8_t cache_flag = 0;
     LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset,
              rpc_log_format_size(size).c_str());
-    if (rpc_use_hash_cache(*ctx->dispatcher, tensor, size)) {
+    if (rpc_use_hash_cache(tensor, size)) {
         auto request = std::make_shared<rpc_msg_set_tensor_hash_req>();
         request->tensor = rpc_tensor;
         request->offset = offset;
@@ -910,9 +867,9 @@ static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggm
         // the server has no cache entry for this tensor - ask it to save one
         cache_flag = 1;
     }
-    size_t header_size;
-    auto header = serialize_set_tensor(rpc_tensor, cache_flag, offset, nullptr, 0, header_size);
-    ctx->dispatcher->send_payload(RPC_CMD_SET_TENSOR, header, header_size, data, size);
+    size_t input_size;
+    auto input = serialize_set_tensor(rpc_tensor, cache_flag, offset, data, size, input_size);
+    ctx->dispatcher->send(RPC_CMD_SET_TENSOR, input, input_size);
 }
 
 static void ggml_backend_rpc_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data,
@@ -1194,7 +1151,7 @@ static void ggml_backend_rpc_set_tensor_async(ggml_backend_t backend, ggml_tenso
     uint8_t cache_flag = 0;
     LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset,
              rpc_log_format_size(size).c_str());
-    if (rpc_use_hash_cache(*ctx->dispatcher, tensor, size)) {
+    if (rpc_use_hash_cache(tensor, size)) {
         auto request = std::make_shared<rpc_msg_set_tensor_hash_req>();
         request->tensor = rpc_tensor;
         request->offset = offset;
@@ -1445,7 +1402,7 @@ public:
     bool free_buffer(const rpc_msg_free_buffer_req & request);
     bool buffer_clear(const rpc_msg_buffer_clear_req & request);
     bool memset_tensor(const rpc_msg_memset_tensor_req & request);
-    bool set_tensor(socket_ptr sock, uint64_t input_size);
+    bool set_tensor(const std::vector<uint8_t> & input);
     bool set_tensor_2d(const std::vector<uint8_t> & input);
     bool set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rpc_msg_set_tensor_hash_rsp & response);
     bool get_tensor(const rpc_msg_get_tensor_req & request, std::vector<uint8_t> & response);
@@ -1492,15 +1449,12 @@ private:
     // computed graphs cached per backend, keyed by uid
     std::vector<std::unordered_map<uint64_t, stored_graph>> stored_graphs;
     std::vector<comm_state> comm_states;
-    // SET_TENSOR data is received in chunks through this buffer
-    std::vector<uint8_t> set_tensor_buf;
 };
 
 void rpc_server::hello(rpc_msg_hello_rsp & response) {
     response.major = RPC_PROTO_MAJOR_VERSION;
     response.minor = RPC_PROTO_MINOR_VERSION;
     response.patch = RPC_PROTO_PATCH_VERSION;
-    response.flags = cache_dir ? RPC_SERVER_FLAG_CACHE : 0;
     LOG_DBG("[%s] version: %d.%d.%d\n", __func__, response.major, response.minor, response.patch);
 }
 
@@ -1741,21 +1695,19 @@ ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rp
 }
 
 
-bool rpc_server::set_tensor(socket_ptr sock, uint64_t input_size) {
+bool rpc_server::set_tensor(const std::vector<uint8_t> & input) {
     sync_all_backends();
     // serialization format: | rpc_tensor | cache_flag (1 byte) | offset (8 bytes) | data (size bytes) |
-    // only the header is read here, the data is streamed from the socket below
     uint8_t  cache_flag;
     uint64_t offset;
-    uint8_t  header[sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset)];
-    const size_t header_size = sizeof(header);
-    if (input_size < header_size || !sock->recv_data(header, header_size)) {
+    const size_t header_size = sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset);
+    if (input.size() < header_size) {
         return false;
     }
-    const rpc_tensor * in_tensor = (const rpc_tensor *)header;
-    memcpy(&cache_flag, header + sizeof(rpc_tensor), sizeof(cache_flag));
-    memcpy(&offset,     header + sizeof(rpc_tensor) + sizeof(cache_flag), sizeof(offset));
-    const size_t size = input_size - header_size;
+    const rpc_tensor * in_tensor = (const rpc_tensor *)input.data();
+    memcpy(&cache_flag, input.data() + sizeof(rpc_tensor), sizeof(cache_flag));
+    memcpy(&offset,     input.data() + sizeof(rpc_tensor) + sizeof(cache_flag), sizeof(offset));
+    const size_t size = input.size() - header_size;
 
     struct ggml_init_params params {
         /*.mem_size   =*/ ggml_tensor_overhead(),
@@ -1786,37 +1738,18 @@ bool rpc_server::set_tensor(socket_ptr sock, uint64_t input_size) {
         }
     }
 
+    const void * data = input.data() + header_size;
     if (cache_dir && cache_flag) {
-        // the whole data is needed to compute the hash
-        std::vector<uint8_t> data(size);
-        if (!sock->recv_data(data.data(), size)) {
-            return false;
-        }
-        uint64_t hash = fnv_hash(data.data(), size);
+        uint64_t hash = fnv_hash((const uint8_t*)data, size);
         char hash_str[17];
         snprintf(hash_str, sizeof(hash_str), "%016" PRIx64, hash);
         // save to cache_dir/hash_str
         fs::path cache_file = fs::path(cache_dir) / hash_str;
         std::ofstream ofs(cache_file, std::ios::binary);
-        ofs.write((const char *)data.data(), size);
+        ofs.write((const char *)data, size);
         LOG_INFO("[%s] saved to '%s'\n", __func__, cache_file.string().c_str());
-        ggml_backend_tensor_set(tensor, data.data(), offset, size);
-        return true;
-    }
-
-    // a buffer of the tensor size would be allocated and page faulted for every tensor, which is slow
-    const size_t chunk_size = 16*1024*1024;
-    if (set_tensor_buf.size() < std::min(size, chunk_size)) {
-        set_tensor_buf.resize(std::min(size, chunk_size));
-    }
-    for (size_t done = 0; done < size; ) {
-        const size_t n = std::min(size - done, chunk_size);
-        if (!sock->recv_data(set_tensor_buf.data(), n)) {
-            return false;
-        }
-        ggml_backend_tensor_set(tensor, set_tensor_buf.data(), offset + done, n);
-        done += n;
     }
+    ggml_backend_tensor_set(tensor, data, offset, size);
     return true;
 }
 
@@ -2665,11 +2598,11 @@ static void rpc_serve_client(const std::vector<ggml_backend_t> & backends, const
                 break;
             }
             case RPC_CMD_SET_TENSOR: {
-                uint64_t input_size;
-                if (!sock->recv_data(&input_size, sizeof(input_size))) {
+                std::vector<uint8_t> input;
+                if (!recv_msg(sock, input)) {
                     return;
                 }
-                if (!server.set_tensor(sock, input_size)) {
+                if (!server.set_tensor(input)) {
                     return;
                 }
                 break;

ggml/src/ggml-rpc/transport.cpp

@@ -71,11 +71,6 @@ struct rdma_conn {
     struct ibv_mr * rx_mr  = nullptr;
     int             rx_head = 0;
 
-    // received slot that is only partly consumed, so recv sizes need not match send sizes
-    int             rx_cur  = -1;
-    size_t          rx_off  = 0;
-    size_t          rx_len  = 0;
-
     uint32_t        max_inline = 0;
 
     uint8_t * rx_slot(int i) const {
@@ -517,24 +512,19 @@ bool socket_t::impl::rdma_recv(void * data, size_t size) {
     uint8_t * dst = (uint8_t *)data;
     size_t rem = size;
     while (rem > 0) {
-        if (c->rx_cur < 0) {
-            struct ibv_wc wc;
-            if (!rdma_poll(c->rcq, &wc)) return false;
+        struct ibv_wc wc;
+        if (!rdma_poll(c->rcq, &wc)) return false;
 
-            c->rx_cur = (int)wc.wr_id;
-            c->rx_off = 0;
-            c->rx_len = wc.byte_len;
+        int slot = (int)wc.wr_id;
+        size_t got = wc.byte_len;
+        if (got > rem) {
+            LOG_ERROR("RDMA recv overrun: got=%zu rem=%zu\n", got, rem);
+            c->post_rx(slot);
+            return false;
         }
+        memcpy(dst, c->rx_slot(slot), got);
 
-        size_t got = std::min(rem, c->rx_len - c->rx_off);
-        memcpy(dst, c->rx_slot(c->rx_cur) + c->rx_off, got);
-        c->rx_off += got;
-
-        // repost the slot only once it is fully consumed
-        if (c->rx_off == c->rx_len) {
-            if (!c->post_rx(c->rx_cur)) return false;
-            c->rx_cur = -1;
-        }
+        if (!c->post_rx(slot)) return false;
 
         dst += got;
         rem -= got;

ggml/src/ggml-sycl/concat.cpp

@@ -196,8 +196,7 @@ void concat_impl_sycl(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
     }
 }
 
-template <typename block_t, int qk, ggml_type qtype>
-static void concat_impl_q_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
+static void concat_impl_q4_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
     scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
     const ggml_tensor * src0   = dst->src[0];
     const ggml_tensor * src1   = dst->src[1];
@@ -205,25 +204,25 @@ static void concat_impl_q_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * ds
 
     const int32_t dim = ((int32_t *) dst->op_params)[0];
 
-    GGML_ASSERT(src0->type == qtype);
-    GGML_ASSERT(src1->type == qtype);
-    GGML_ASSERT(dst->type == qtype);
-    GGML_ASSERT(src0->ne[0] % qk == 0);
-    GGML_ASSERT(src1->ne[0] % qk == 0);
-    GGML_ASSERT(dst->ne[0] % qk == 0);
+    GGML_ASSERT(src0->type == GGML_TYPE_Q4_0);
+    GGML_ASSERT(src1->type == GGML_TYPE_Q4_0);
+    GGML_ASSERT(dst->type == GGML_TYPE_Q4_0);
+    GGML_ASSERT(src0->ne[0] % QK4_0 == 0);
+    GGML_ASSERT(src1->ne[0] % QK4_0 == 0);
+    GGML_ASSERT(dst->ne[0] % QK4_0 == 0);
 
-    const int ne00_blk = src0->ne[0] / qk;
-    const int ne0_blk  = dst->ne[0] / qk;
+    const int ne00_blk = src0->ne[0] / QK4_0;
+    const int ne0_blk  = dst->ne[0] / QK4_0;
 
     if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
-        const block_t * src0_d = (const block_t *) src0->data;
-        const block_t * src1_d = (const block_t *) src1->data;
-        block_t * dst_d        = (block_t *) dst->data;
-        const size_t type_size = sizeof(block_t);
+        const block_q4_0 * src0_d = (const block_q4_0 *) src0->data;
+        const block_q4_0 * src1_d = (const block_q4_0 *) src1->data;
+        block_q4_0 * dst_d        = (block_q4_0 *) dst->data;
+        const size_t type_size    = sizeof(block_q4_0);
 
         if (dim != 3) {
             for (int i3 = 0; i3 < dst->ne[3]; i3++) {
-                concat_T_sycl<block_t>(
+                concat_T_sycl<block_q4_0>(
                     src0_d + i3 * (src0->nb[3] / type_size),
                     src1_d + i3 * (src1->nb[3] / type_size),
                     dst_d + i3 * (dst->nb[3] / type_size),
@@ -238,12 +237,223 @@ static void concat_impl_q_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * ds
             SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
         }
     } else {
-        concat_T_sycl_non_cont<block_t>(
+        concat_T_sycl_non_cont<block_q4_0>(
             stream, (const char *) src0->data, (const char *) src1->data,
             (char *) dst->data,
             ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
             src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
-            src1->ne[0] / qk, src1->ne[1], src1->ne[2], src1->ne[3],
+            src1->ne[0] / QK4_0, src1->ne[1], src1->ne[2], src1->ne[3],
+            src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
+            ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
+            dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
+    }
+}
+
+static void concat_impl_q4_1_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
+    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
+    const ggml_tensor * src0   = dst->src[0];
+    const ggml_tensor * src1   = dst->src[1];
+    queue_ptr           stream = ctx.stream();
+
+    const int32_t dim = ((int32_t *) dst->op_params)[0];
+
+    GGML_ASSERT(src0->type == GGML_TYPE_Q4_1);
+    GGML_ASSERT(src1->type == GGML_TYPE_Q4_1);
+    GGML_ASSERT(dst->type == GGML_TYPE_Q4_1);
+    GGML_ASSERT(src0->ne[0] % QK4_1 == 0);
+    GGML_ASSERT(src1->ne[0] % QK4_1 == 0);
+    GGML_ASSERT(dst->ne[0] % QK4_1 == 0);
+
+    const int ne00_blk = src0->ne[0] / QK4_1;
+    const int ne0_blk  = dst->ne[0] / QK4_1;
+
+    if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
+        const block_q4_1 * src0_d = (const block_q4_1 *) src0->data;
+        const block_q4_1 * src1_d = (const block_q4_1 *) src1->data;
+        block_q4_1 * dst_d        = (block_q4_1 *) dst->data;
+        const size_t type_size    = sizeof(block_q4_1);
+
+        if (dim != 3) {
+            for (int i3 = 0; i3 < dst->ne[3]; i3++) {
+                concat_T_sycl<block_q4_1>(
+                    src0_d + i3 * (src0->nb[3] / type_size),
+                    src1_d + i3 * (src1->nb[3] / type_size),
+                    dst_d + i3 * (dst->nb[3] / type_size),
+                    ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
+                    dst->ne[1], dst->ne[2], dim, stream);
+            }
+        } else {
+            const size_t size0 = ggml_nbytes(src0);
+            const size_t size1 = ggml_nbytes(src1);
+
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
+        }
+    } else {
+        concat_T_sycl_non_cont<block_q4_1>(
+            stream, (const char *) src0->data, (const char *) src1->data,
+            (char *) dst->data,
+            ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
+            src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
+            src1->ne[0] / QK4_1, src1->ne[1], src1->ne[2], src1->ne[3],
+            src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
+            ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
+            dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
+    }
+}
+
+static void concat_impl_q5_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
+    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
+    const ggml_tensor * src0   = dst->src[0];
+    const ggml_tensor * src1   = dst->src[1];
+    queue_ptr           stream = ctx.stream();
+
+    const int32_t dim = ((int32_t *) dst->op_params)[0];
+
+    GGML_ASSERT(src0->type == GGML_TYPE_Q5_0);
+    GGML_ASSERT(src1->type == GGML_TYPE_Q5_0);
+    GGML_ASSERT(dst->type == GGML_TYPE_Q5_0);
+    GGML_ASSERT(src0->ne[0] % QK5_0 == 0);
+    GGML_ASSERT(src1->ne[0] % QK5_0 == 0);
+    GGML_ASSERT(dst->ne[0] % QK5_0 == 0);
+
+    const int ne00_blk = src0->ne[0] / QK5_0;
+    const int ne0_blk  = dst->ne[0] / QK5_0;
+
+    if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
+        const block_q5_0 * src0_d = (const block_q5_0 *) src0->data;
+        const block_q5_0 * src1_d = (const block_q5_0 *) src1->data;
+        block_q5_0 * dst_d        = (block_q5_0 *) dst->data;
+        const size_t type_size    = sizeof(block_q5_0);
+
+        if (dim != 3) {
+            for (int i3 = 0; i3 < dst->ne[3]; i3++) {
+                concat_T_sycl<block_q5_0>(
+                    src0_d + i3 * (src0->nb[3] / type_size),
+                    src1_d + i3 * (src1->nb[3] / type_size),
+                    dst_d + i3 * (dst->nb[3] / type_size),
+                    ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
+                    dst->ne[1], dst->ne[2], dim, stream);
+            }
+        } else {
+            const size_t size0 = ggml_nbytes(src0);
+            const size_t size1 = ggml_nbytes(src1);
+
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
+        }
+    } else {
+        concat_T_sycl_non_cont<block_q5_0>(
+            stream, (const char *) src0->data, (const char *) src1->data,
+            (char *) dst->data,
+            ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
+            src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
+            src1->ne[0] / QK5_0, src1->ne[1], src1->ne[2], src1->ne[3],
+            src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
+            ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
+            dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
+    }
+}
+
+static void concat_impl_q5_1_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
+    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
+    const ggml_tensor * src0   = dst->src[0];
+    const ggml_tensor * src1   = dst->src[1];
+    queue_ptr           stream = ctx.stream();
+
+    const int32_t dim = ((int32_t *) dst->op_params)[0];
+
+    GGML_ASSERT(src0->type == GGML_TYPE_Q5_1);
+    GGML_ASSERT(src1->type == GGML_TYPE_Q5_1);
+    GGML_ASSERT(dst->type == GGML_TYPE_Q5_1);
+    GGML_ASSERT(src0->ne[0] % QK5_1 == 0);
+    GGML_ASSERT(src1->ne[0] % QK5_1 == 0);
+    GGML_ASSERT(dst->ne[0] % QK5_1 == 0);
+
+    const int ne00_blk = src0->ne[0] / QK5_1;
+    const int ne0_blk  = dst->ne[0] / QK5_1;
+
+    if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
+        const block_q5_1 * src0_d = (const block_q5_1 *) src0->data;
+        const block_q5_1 * src1_d = (const block_q5_1 *) src1->data;
+        block_q5_1 * dst_d        = (block_q5_1 *) dst->data;
+        const size_t type_size    = sizeof(block_q5_1);
+
+        if (dim != 3) {
+            for (int i3 = 0; i3 < dst->ne[3]; i3++) {
+                concat_T_sycl<block_q5_1>(
+                    src0_d + i3 * (src0->nb[3] / type_size),
+                    src1_d + i3 * (src1->nb[3] / type_size),
+                    dst_d + i3 * (dst->nb[3] / type_size),
+                    ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
+                    dst->ne[1], dst->ne[2], dim, stream);
+            }
+        } else {
+            const size_t size0 = ggml_nbytes(src0);
+            const size_t size1 = ggml_nbytes(src1);
+
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
+        }
+    } else {
+        concat_T_sycl_non_cont<block_q5_1>(
+            stream, (const char *) src0->data, (const char *) src1->data,
+            (char *) dst->data,
+            ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
+            src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
+            src1->ne[0] / QK5_1, src1->ne[1], src1->ne[2], src1->ne[3],
+            src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
+            ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
+            dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
+    }
+}
+
+static void concat_impl_q8_0_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
+    scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
+    const ggml_tensor * src0   = dst->src[0];
+    const ggml_tensor * src1   = dst->src[1];
+    queue_ptr           stream = ctx.stream();
+
+    const int32_t dim = ((int32_t *) dst->op_params)[0];
+
+    GGML_ASSERT(src0->type == GGML_TYPE_Q8_0);
+    GGML_ASSERT(src1->type == GGML_TYPE_Q8_0);
+    GGML_ASSERT(dst->type == GGML_TYPE_Q8_0);
+    GGML_ASSERT(src0->ne[0] % QK8_0 == 0);
+    GGML_ASSERT(src1->ne[0] % QK8_0 == 0);
+    GGML_ASSERT(dst->ne[0] % QK8_0 == 0);
+
+    const int ne00_blk = src0->ne[0] / QK8_0;
+    const int ne0_blk  = dst->ne[0] / QK8_0;
+
+    if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
+        const block_q8_0 * src0_d = (const block_q8_0 *) src0->data;
+        const block_q8_0 * src1_d = (const block_q8_0 *) src1->data;
+        block_q8_0 * dst_d        = (block_q8_0 *) dst->data;
+        const size_t type_size    = sizeof(block_q8_0);
+
+        if (dim != 3) {
+            for (int i3 = 0; i3 < dst->ne[3]; i3++) {
+                concat_T_sycl<block_q8_0>(
+                    src0_d + i3 * (src0->nb[3] / type_size),
+                    src1_d + i3 * (src1->nb[3] / type_size),
+                    dst_d + i3 * (dst->nb[3] / type_size),
+                    ne00_blk, src0->ne[1], src0->ne[2], ne0_blk,
+                    dst->ne[1], dst->ne[2], dim, stream);
+            }
+        } else {
+            const size_t size0 = ggml_nbytes(src0);
+            const size_t size1 = ggml_nbytes(src1);
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0)));
+            SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1)));
+        }
+    } else {
+        concat_T_sycl_non_cont<block_q8_0>(
+            stream, (const char *) src0->data, (const char *) src1->data,
+            (char *) dst->data,
+            ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3],
+            src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],
+            src1->ne[0] / QK8_0, src1->ne[1], src1->ne[2], src1->ne[3],
             src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],
             ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3],
             dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim);
@@ -277,19 +487,19 @@ void ggml_sycl_op_concat(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
         concat_impl_sycl<int8_t>(ctx, dst);
         break;
     case GGML_TYPE_Q4_0:
-        concat_impl_q_sycl<block_q4_0, QK4_0, GGML_TYPE_Q4_0>(ctx, dst);
+        concat_impl_q4_0_sycl(ctx, dst);
         break;
     case GGML_TYPE_Q4_1:
-        concat_impl_q_sycl<block_q4_1, QK4_1, GGML_TYPE_Q4_1>(ctx, dst);
+        concat_impl_q4_1_sycl(ctx, dst);
         break;
     case GGML_TYPE_Q5_0:
-        concat_impl_q_sycl<block_q5_0, QK5_0, GGML_TYPE_Q5_0>(ctx, dst);
+        concat_impl_q5_0_sycl(ctx, dst);
         break;
     case GGML_TYPE_Q5_1:
-        concat_impl_q_sycl<block_q5_1, QK5_1, GGML_TYPE_Q5_1>(ctx, dst);
+        concat_impl_q5_1_sycl(ctx, dst);
         break;
     case GGML_TYPE_Q8_0:
-        concat_impl_q_sycl<block_q8_0, QK8_0, GGML_TYPE_Q8_0>(ctx, dst);
+        concat_impl_q8_0_sycl(ctx, dst);
         break;
     default:
         fprintf(stderr, "%s: unsupported types: dst: %s\n", __func__, ggml_type_name(dst->type));

ggml/src/ggml-vulkan/ggml-vulkan-types.h

@@ -848,18 +848,18 @@ struct vk_device_struct {
     vk_pipeline pipeline_set_rows_i64[2][GGML_TYPE_COUNT];
     vk_pipeline pipeline_norm_f32;
     vk_pipeline pipeline_group_norm_f32;
-    std::array<vk_pipeline, 3> pipeline_rms_norm_f32;
-    std::array<vk_pipeline, 3> pipeline_rms_norm_mul_f32;
-    std::array<vk_pipeline, 3> pipeline_rms_norm_mul_add_f32;
-    std::array<vk_pipeline, 3> pipeline_rms_norm_mul_add_mul_f32;
+    vk_pipeline pipeline_rms_norm_f32;
+    vk_pipeline pipeline_rms_norm_mul_f32;
+    vk_pipeline pipeline_rms_norm_mul_add_f32;
+    vk_pipeline pipeline_rms_norm_mul_add_mul_f32;
     vk_pipeline pipeline_rms_norm_mul_add_partials_f32;
     vk_pipeline pipeline_rms_norm_mul_add_mul_partials_f32;
     vk_pipeline pipeline_rms_norm_set_rows_f32_f32;
     vk_pipeline pipeline_rms_norm_set_rows_f32_f16;
     vk_pipeline pipeline_rms_norm_partials_f32;
     vk_pipeline pipeline_rms_norm_mul_partials_f32;
-    std::array<vk_pipeline, 3> pipeline_rms_norm_mul_rope_f32_f32;
-    std::array<vk_pipeline, 3> pipeline_rms_norm_mul_rope_f32_f16;
+    vk_pipeline pipeline_rms_norm_mul_rope_f32_f32;
+    vk_pipeline pipeline_rms_norm_mul_rope_f32_f16;
     vk_pipeline pipeline_rms_norm_back_f32;
     vk_pipeline pipeline_l2_norm_f32;
 

ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -3256,34 +3256,21 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
     ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1);
     ggml_vk_create_pipeline(device, device->pipeline_group_norm_f32, "group_norm_f32", group_norm_f32_len, group_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1);
 
-    const bool rms_subgroups = (device->vendor_id == VK_VENDOR_ID_INTEL || device->vendor_id == VK_VENDOR_ID_NVIDIA || device->vendor_id == VK_VENDOR_ID_AMD) && device->subgroup_arithmetic && device->subgroup_require_full_support;
-    const uint32_t rms_set_rows_block_size = 128;
-#define RMS_SHADER(name) (rms_subgroups ? name##_subgroup_len : name##_len), (rms_subgroups ? name##_subgroup_data : name##_data)
-    // 128/256/512 block sizes, picked by row length
-    for (uint32_t i = 0; i < 3; ++i) {
-        const uint32_t block_size = 128u << i;
-        const std::string suffix = "_" + std::to_string(block_size);
-        ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_f32[i], "rms_norm_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, block_size}, 1, true, rms_subgroups);
-        ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_f32[i], "rms_norm_mul_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups);
-        ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_f32[i], "rms_norm_mul_add_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups);
-        ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_mul_f32[i], "rms_norm_mul_add_mul_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1, block_size}, 1, true, rms_subgroups);
-    }
+    ggml_vk_create_pipeline(device, device->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true);
+    ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_f32, "rms_norm_mul_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1}, 1, true);
+    ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_f32, "rms_norm_mul_add_f32", rms_norm_mul_add_f32_len, rms_norm_mul_add_f32_data, "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0}, 1, true);
+    ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_mul_f32, "rms_norm_mul_add_mul_f32", rms_norm_mul_add_f32_len, rms_norm_mul_add_f32_data, "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1}, 1, true);
     ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_partials_f32, "rms_norm_mul_add_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0}, 1, true);
     ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_mul_partials_f32, "rms_norm_mul_add_mul_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1}, 1, true);
-    ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f32, "rms_norm_set_rows_f32_f32", RMS_SHADER(rms_norm_set_rows_f32_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, rms_set_rows_block_size}, 1, true, rms_subgroups);
-    ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f16, "rms_norm_set_rows_f32_f16", RMS_SHADER(rms_norm_set_rows_f32_f16), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, rms_set_rows_block_size}, 1, true, rms_subgroups);
+    ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f32, "rms_norm_set_rows_f32_f32", rms_norm_set_rows_f32_f32_len, rms_norm_set_rows_f32_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true);
+    ggml_vk_create_pipeline(device, device->pipeline_rms_norm_set_rows_f32_f16, "rms_norm_set_rows_f32_f16", rms_norm_set_rows_f32_f16_len, rms_norm_set_rows_f32_f16_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true);
     ggml_vk_create_pipeline(device, device->pipeline_rms_norm_partials_f32, "rms_norm_partials_f32", rms_norm_partials_f32_len, rms_norm_partials_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true);
     ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_partials_f32, "rms_norm_mul_partials_f32", rms_norm_partials_f32_len, rms_norm_partials_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1}, 1, true);
 
     if (sizeof(vk_op_rms_norm_mul_rope_push_constants) <= device->properties.limits.maxPushConstantsSize) {
-        for (uint32_t i = 0; i < 3; ++i) {
-            const uint32_t block_size = 128u << i;
-            const std::string suffix = "_" + std::to_string(block_size);
-            ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_rope_f32_f32[i], "rms_norm_mul_rope_f32_f32" + suffix, RMS_SHADER(rms_norm_mul_rope_f32_f32), "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups);
-            ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_rope_f32_f16[i], "rms_norm_mul_rope_f32_f16" + suffix, RMS_SHADER(rms_norm_mul_rope_f32_f16), "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1, 0, block_size}, 1, true, rms_subgroups);
-        }
+        ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_rope_f32_f32, "rms_norm_mul_rope_f32_f32", rms_norm_mul_rope_f32_f32_len, rms_norm_mul_rope_f32_f32_data, "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1}, 1, true);
+        ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_rope_f32_f16, "rms_norm_mul_rope_f32_f16", rms_norm_mul_rope_f32_f16_len, rms_norm_mul_rope_f32_f16_data, "main", 7, sizeof(vk_op_rms_norm_mul_rope_push_constants), {1, 1, 1}, {0, 1}, 1, true);
     }
-#undef RMS_SHADER
 
     ggml_vk_create_pipeline(device, device->pipeline_rms_norm_back_f32, "rms_norm_back_f32", rms_norm_back_f32_len, rms_norm_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1);
     ggml_vk_create_pipeline(device, device->pipeline_l2_norm_f32, "l2_norm_f32", l2_norm_f32_len, l2_norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1);
@@ -7110,7 +7097,7 @@ static void ggml_vk_mul_mat_vec_nc_f16_f32(ggml_backend_vk_context * ctx, vk_con
     const uint32_t nb23 = (uint32_t)(dst->nb[3] / sizeof(float));
 
     GGML_ASSERT(ne11 == 1);
-    GGML_ASSERT(src0->ne[3] == src1->ne[3]); // checked in ggml_vk_mul_mat
+    GGML_ASSERT(src0->ne[3] == src1->ne[3]); // checked in supports_op
 
     const uint32_t row_stride_x = nb01 / sizeof(ggml_fp16_t);
     const uint32_t channel_stride_x = nb02 / sizeof(ggml_fp16_t);
@@ -7382,7 +7369,7 @@ void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context& subctx, const st
         src1->ne[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]) {
         ggml_vk_mul_mat_vec_p021_f16_f32(ctx, subctx, cgraph, node_idx);
     } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && dst->ne[1] == 1 &&
-               !ggml_is_permuted(src0) && !ggml_is_permuted(src1) && src0->ne[3] == src1->ne[3] &&
+               !ggml_is_permuted(src0) && !ggml_is_permuted(src1) &&
                src0->ne[3] <= ctx->device->properties.limits.maxComputeWorkGroupCount[0] &&
                src0->ne[1] <= ctx->device->properties.limits.maxComputeWorkGroupCount[1] &&
                src1->ne[2] <= ctx->device->properties.limits.maxComputeWorkGroupCount[2]) {
@@ -8753,16 +8740,6 @@ static vk_conv_shapes ggml_vk_conv3d_select_shape(ggml_backend_vk_context * ctx,
     return ggml_vk_conv_select_shape(ctx, K, NPQ);
 }
 
-static vk_pipeline ggml_vk_rms_norm_pipeline(const std::array<vk_pipeline, 3> & pipelines, int64_t ncols) {
-    if (pipelines[0] && ncols <= 128) {
-        return pipelines[0];
-    }
-    if (pipelines[1] && ncols <= 256) {
-        return pipelines[1];
-    }
-    return pipelines[2];
-}
-
 static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * dst, ggml_op op) {
     switch (op) {
     case GGML_OP_GET_ROWS:
@@ -9003,7 +8980,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
             if (ctx->do_add_rms_partials) {
                 return ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_partials_f32 : ctx->device->pipeline_rms_norm_partials_f32;
             }
-            return ggml_vk_rms_norm_pipeline(ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_f32 : ctx->device->pipeline_rms_norm_f32, src0->ne[0]);
+            return ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_f32 : ctx->device->pipeline_rms_norm_f32;
         }
         return nullptr;
     case GGML_OP_RMS_NORM_BACK:
@@ -10965,8 +10942,8 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s
             pipeline = do_post_multiply ?
                 ctx->device->pipeline_rms_norm_mul_add_mul_partials_f32 : ctx->device->pipeline_rms_norm_mul_add_partials_f32;
         } else {
-            pipeline = ggml_vk_rms_norm_pipeline(do_post_multiply ?
-                ctx->device->pipeline_rms_norm_mul_add_mul_f32 : ctx->device->pipeline_rms_norm_mul_add_f32, src0->ne[0]);
+            pipeline = do_post_multiply ?
+                ctx->device->pipeline_rms_norm_mul_add_mul_f32 : ctx->device->pipeline_rms_norm_mul_add_f32;
         }
         ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
         if (ctx->do_add_rms_partials) {
@@ -11039,8 +11016,7 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s
         pc.bin = bin;
         pc.rope = ggml_vk_make_rope_constants(rope, rope->src[0], tensors[4] != nullptr, false, set_rows_stride);
 
-        const auto & pipelines = tensors[5]->type == GGML_TYPE_F16 ? ctx->device->pipeline_rms_norm_mul_rope_f32_f16 : ctx->device->pipeline_rms_norm_mul_rope_f32_f32;
-        vk_pipeline pipeline = ggml_vk_rms_norm_pipeline(pipelines, src0->ne[0]);
+        vk_pipeline pipeline = tensors[5]->type == GGML_TYPE_F16 ? ctx->device->pipeline_rms_norm_mul_rope_f32_f16 : ctx->device->pipeline_rms_norm_mul_rope_f32_f32;
 
         ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
 
@@ -15632,8 +15608,16 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
                     default:
                         return false;
                 }
-                // MUL_MAT broadcasts src0 over dim 3 like dim 2, MUL_MAT_ID needs the ids to match src1
-                if (op->op == GGML_OP_MUL_MAT_ID && op->src[2]->ne[3] != op->src[1]->ne[3]) {
+                struct ggml_tensor * a;
+                struct ggml_tensor * b;
+                if (op->op == GGML_OP_MUL_MAT) {
+                    a = op->src[0];
+                    b = op->src[1];
+                } else {
+                    a = op->src[2];
+                    b = op->src[1];
+                }
+                if (a->ne[3] != b->ne[3]) {
                     return false;
                 }
                 if (!(ggml_vk_dim01_contiguous(op->src[0]) || op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) ||

ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp

@@ -3,15 +3,6 @@
 #include "generic_binary_head.glsl"
 #include "types.glsl"
 
-#extension GL_EXT_control_flow_attributes : enable
-#if USE_SUBGROUP_ADD
-#extension GL_KHR_shader_subgroup_basic : require
-#extension GL_KHR_shader_subgroup_arithmetic : require
-#endif
-
-layout (constant_id = 3) const int BLOCK_SIZE = 512;
-layout(local_size_x_id = 3, local_size_y = 1, local_size_z = 1) in;
-
 #if RMS_NORM_ROPE_FUSION
 
 layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
@@ -20,8 +11,7 @@ layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
 // data is passed from rms_norm -> rope through shared memory.
 // rms_norm calls this data_d, rope calls this rope_data_a.
 // Binding 2 is not used
-// host picks a block size such that ncols <= 2*BLOCK_SIZE
-shared FLOAT_TYPE rope_data_a[2*BLOCK_SIZE];
+shared FLOAT_TYPE rope_data_a[1024];
 #define data_d rope_data_a
 
 layout (binding = 3) readonly buffer R_Y {int rope_data_pos[];};
@@ -48,11 +38,16 @@ layout (binding = 3) readonly buffer I {uvec2 data_i[];};
 
 #endif
 
+#extension GL_EXT_control_flow_attributes : enable
+#define BLOCK_SIZE 512
+
 layout (constant_id = 1) const bool do_multiply = false;
 #if RMS_NORM_ADD_FUSION
 layout (constant_id = 2) const bool do_post_multiply = false;
 #endif
 
+layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
+
 shared FLOAT_TYPE sumsh[BLOCK_SIZE];
 
 void rms_norm(uint num_iters) {
@@ -93,38 +88,17 @@ void rms_norm(uint num_iters) {
                 sum += xi * xi;
             }
 
-            #if USE_SUBGROUP_ADD
-                sum = subgroupAdd(sum);
-                if (subgroupElect()) {
-                    sumsh[gl_SubgroupID] = sum;
+            sumsh[tid] = sum;
+            // sum up partial sums and write back result
+            barrier();
+            [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
+                if (tid < s) {
+                    sum += sumsh[tid + s];
+                    sumsh[tid] = sum;
                 }
                 barrier();
-                sum = FLOAT_TYPE(0.0f);
-                if(gl_NumSubgroups <= 4) { // fast path to skip the 2nd subgroupAdd
-                    [[unroll]] for(uint i=0; i < gl_NumSubgroups; ++i) {
-                        sum += sumsh[i];
-                    }
-                }
-                else {
-                    // Each subgroup reads the partials, so one workgroup barrier is enough.
-                    for (uint i = gl_SubgroupInvocationID; i < gl_NumSubgroups; i += gl_SubgroupSize) {
-                        sum += sumsh[i];
-                    }
-                    sum = subgroupAdd(sum);
-                }
-            #else
-                sumsh[tid] = sum;
-                // sum up partial sums and write back result
-                barrier();
-                [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
-                    if (tid < s) {
-                        sum += sumsh[tid + s];
-                        sumsh[tid] = sum;
-                    }
-                    barrier();
-                }
-                sum = sumsh[0];
-            #endif
+            }
+            sum = sumsh[0];
 
             const FLOAT_TYPE mean = sum / FLOAT_TYPE(ncols);
             const FLOAT_TYPE scale = inversesqrt(mean + FLOAT_TYPE(p.param1));

ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp

@@ -853,18 +853,14 @@ void process_shaders() {
     // Norms
     string_to_spv("norm_f32", "norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
     string_to_spv("group_norm_f32", "group_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
-    for (bool use_subgroups : {false, true}) {
-        const std::string suffix = use_subgroups ? "_subgroup" : "";
-        const auto rms_dict = merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", use_subgroups ? "1" : "0"}});
-        string_to_spv("rms_norm_f32" + suffix, "rms_norm.comp", rms_dict);
-        string_to_spv("rms_norm_mul_add_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_ADD_FUSION", "1"}}));
-        string_to_spv("rms_norm_set_rows_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_SET_ROWS_FUSION", "1"}}));
-        string_to_spv("rms_norm_set_rows_f32_f16" + suffix, "rms_norm.comp", merge_maps({{"D_TYPE", "float16_t"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}}, rms_dict)); // keep rms_dict last so 'merge_maps' overwrites the value for key 'D_TYPE'
-        string_to_spv("rms_norm_mul_rope_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"ROPE_D_TYPE", "float"}, {"RMS_NORM_ROPE_FUSION", "1"}}));
-        string_to_spv("rms_norm_mul_rope_f32_f16" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"ROPE_D_TYPE", "float16_t"}, {"RMS_NORM_ROPE_FUSION", "1"}}));
-    }
+    string_to_spv("rms_norm_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}));
+    string_to_spv("rms_norm_mul_add_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"RMS_NORM_ADD_FUSION", "1"}}));
     string_to_spv("rms_norm_mul_add_partials_f32", "rms_norm_partials.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"RMS_NORM_ADD_FUSION", "1"}}));
+    string_to_spv("rms_norm_set_rows_f32_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}}));
+    string_to_spv("rms_norm_set_rows_f32_f16", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float16_t"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}}));
     string_to_spv("rms_norm_partials_f32", "rms_norm_partials.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}));
+    string_to_spv("rms_norm_mul_rope_f32_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"ROPE_D_TYPE", "float"}, {"RMS_NORM_ROPE_FUSION", "1"}}));
+    string_to_spv("rms_norm_mul_rope_f32_f16", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"ROPE_D_TYPE", "float16_t"}, {"RMS_NORM_ROPE_FUSION", "1"}}));
     string_to_spv("rms_norm_back_f32", "rms_norm_back.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}));
     string_to_spv("l2_norm_f32", "l2_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
 

ggml/src/ggml-webgpu/ggml-webgpu.cpp

@@ -670,8 +670,6 @@ static void ggml_backend_webgpu_buffer_memset(webgpu_global_context & ctx,
                                               uint32_t                value,
                                               size_t                  offset,
                                               size_t                  size) {
-    std::lock_guard<std::recursive_mutex> lock(ctx->mutex);
-
     std::vector<uint32_t>             params  = { (uint32_t) offset, (uint32_t) size, value };
     std::vector<wgpu::BindGroupEntry> entries = { ggml_webgpu_make_bind_group_entry(0, buf, 0, buf.GetSize()) };
     size_t                            bytes_per_wg =

ggml/src/ggml.c

@@ -3503,7 +3503,7 @@ static struct ggml_tensor * ggml_scale_impl(
         float                 s,
         float                 b,
         bool                  inplace) {
-    GGML_ASSERT(ggml_is_contiguous_rows(a));
+    GGML_ASSERT(ggml_is_padded_1d(a));
 
     struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
 

gguf-py/gguf/constants.py

@@ -501,7 +501,6 @@ class Keys:
 
         class Attention:
             HEAD_COUNT      = "clip.audio.attention.head_count"
-            HEAD_COUNT_KV   = "clip.audio.attention.head_count_kv"
             HEAD_DIM        = "clip.audio.attention.head_dim" # set when qkv width != n_embd
             LAYERNORM_EPS   = "clip.audio.attention.layer_norm_epsilon"
 
@@ -519,11 +518,6 @@ class Keys:
         FEED_FORWARD_LENGTH = "clip.gen.audio.feed_forward_length"
         BLOCK_COUNT         = "clip.gen.audio.block_count"
         PROJECTION_DIM      = "clip.gen.audio.projection_dim"
-        # sampling defaults of the generated audio, like general.sampling.* for text
-        SAMPLING_TEMP           = "clip.gen.audio.sampling.temp"
-        SAMPLING_TOP_K          = "clip.gen.audio.sampling.top_k"
-        SAMPLING_PENALTY_REPEAT = "clip.gen.audio.sampling.penalty_repeat"
-        SAMPLING_PENALTY_LAST_N = "clip.gen.audio.sampling.penalty_last_n"
 
         class Attention:
             HEAD_COUNT      = "clip.gen.audio.attention.head_count"
@@ -697,7 +691,6 @@ class MODEL_ARCH(IntEnum):
     GPT_OSS          = auto()
     LFM2             = auto()
     LFM2MOE          = auto()
-    LFM3             = auto()
     DREAM            = auto()
     SMALLTHINKER     = auto()
     LLADA            = auto()
@@ -927,10 +920,6 @@ class MODEL_TENSOR(IntEnum):
     HC_FFN_DOWN          = auto() # qwen4exp
     HC_FFN_UP            = auto() # qwen4exp
     HC_FFN_INJECT        = auto() # qwen4exp
-    HC_ATTN_DOWN_INJECT  = auto() # qwen4exp, [hc_attn_down | hc_attn_inject]
-    HC_FFN_DOWN_INJECT   = auto() # qwen4exp, [hc_ffn_down | hc_ffn_inject]
-    ATTN_QKV_GATE        = auto() # qwen4exp, [attn_qkv | attn_gate]
-    SSM_BETA_ALPHA_CAT   = auto() # qwen4exp, [ssm_beta | ssm_alpha], unlike the grouped qwen3next ssm_ba
     PLE_KEY              = auto() # qwen4exp
     PLE_VALUE            = auto() # qwen4exp
     PLE_NORM_KEY         = auto() # qwen4exp
@@ -1182,8 +1171,6 @@ class MODEL_TENSOR(IntEnum):
     A_ENC_ATTN_Q          = auto()
     A_ENC_ATTN_K          = auto()
     A_ENC_ATTN_V          = auto()
-    A_ENC_ATTN_Q_NORM     = auto() # lfm3a
-    A_ENC_ATTN_K_NORM     = auto() # lfm3a
     A_ENC_ATTN_POST_NORM  = auto()
     A_ENC_ATTN_PRE_NORM   = auto()
     A_ENC_ATTN_K_REL      = auto() # gemma4
@@ -1228,9 +1215,6 @@ class MODEL_TENSOR(IntEnum):
     A_GEN_CODE_FFN_UP      = auto()
     A_GEN_CODE_FFN_DOWN    = auto()
     A_GEN_CODE_OUTPUT_NORM = auto()
-    A_GEN_CODE_ATTN_GATE   = auto() # lfm3a: per-head sigmoid gate on the attention output
-    A_GEN_CODE_HEAD_NORM   = auto() # lfm3a: per-codebook norm before the output head, merged 2D [n_codebooks, dim]
-    A_GEN_CODE_MASK_EMBD   = auto() # lfm3a: MaskGIT mask embedding
     # qwen3tts code2wav: RVQ codes -> raw PCM
     A_GEN_WAV_QUANT_FIRST_IN       = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D)
     A_GEN_WAV_QUANT_FIRST_OUT      = auto() # semantic RVQ, out_proj
@@ -1253,14 +1237,6 @@ class MODEL_TENSOR(IntEnum):
     A_GEN_WAV_TFM_FFN_UP           = auto()
     A_GEN_WAV_TFM_FFN_DOWN         = auto()
     A_GEN_WAV_TFM_FFN_SCALE        = auto() # layer scale (gamma) on the FFN output
-    A_GEN_WAV_TFM_ATTN_Q_NORM      = auto() # lfm3a
-    A_GEN_WAV_TFM_ATTN_K_NORM      = auto() # lfm3a
-    A_GEN_WAV_TFM_CONV_PW1         = auto() # lfm3a: short conv in_proj
-    A_GEN_WAV_TFM_CONV_DW          = auto() # lfm3a: short conv depthwise kernel
-    A_GEN_WAV_TFM_CONV_PW2         = auto() # lfm3a: short conv out_proj
-    A_GEN_WAV_TFM_ATTN_GATE        = auto() # lfm3a: per-head sigmoid gate on the attention output
-    A_GEN_WAV_TFM_ATTN_POST_NORM   = auto() # lfm3a: sandwich norm after the operator
-    A_GEN_WAV_TFM_FFN_POST_NORM    = auto() # lfm3a: sandwich norm after the FFN
     A_GEN_WAV_UP_CONV              = auto() # causal ConvTranspose1d, 2x upsample
     A_GEN_WAV_UP_DWCONV            = auto() # ConvNeXt depthwise conv
     A_GEN_WAV_UP_NORM              = auto() # ConvNeXt LayerNorm
@@ -1543,7 +1519,6 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.GPT_OSS:          "gpt-oss",
     MODEL_ARCH.LFM2:             "lfm2",
     MODEL_ARCH.LFM2MOE:          "lfm2moe",
-    MODEL_ARCH.LFM3:             "lfm3",
     MODEL_ARCH.DREAM:            "dream",
     MODEL_ARCH.SMALLTHINKER:     "smallthinker",
     MODEL_ARCH.LLADA:            "llada",
@@ -1771,10 +1746,6 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.HC_FFN_DOWN:               "blk.{bid}.hc_ffn_down",          # qwen4exp
     MODEL_TENSOR.HC_FFN_UP:                 "blk.{bid}.hc_ffn_up",            # qwen4exp
     MODEL_TENSOR.HC_FFN_INJECT:             "blk.{bid}.hc_ffn_inject",        # qwen4exp
-    MODEL_TENSOR.HC_ATTN_DOWN_INJECT:       "blk.{bid}.hc_attn_down_inject",  # qwen4exp
-    MODEL_TENSOR.HC_FFN_DOWN_INJECT:        "blk.{bid}.hc_ffn_down_inject",   # qwen4exp
-    MODEL_TENSOR.ATTN_QKV_GATE:             "blk.{bid}.attn_qkv_gate",        # qwen4exp
-    MODEL_TENSOR.SSM_BETA_ALPHA_CAT:        "blk.{bid}.ssm_beta_alpha",       # qwen4exp
     MODEL_TENSOR.PLE_KEY:                   "blk.{bid}.ple_key",              # qwen4exp
     MODEL_TENSOR.PLE_VALUE:                 "blk.{bid}.ple_value",            # qwen4exp
     MODEL_TENSOR.PLE_NORM_KEY:              "blk.{bid}.ple_norm_key",         # qwen4exp
@@ -2023,8 +1994,6 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.A_ENC_ATTN_Q:              "a.blk.{bid}.attn_q",
     MODEL_TENSOR.A_ENC_ATTN_K:              "a.blk.{bid}.attn_k",
     MODEL_TENSOR.A_ENC_ATTN_V:              "a.blk.{bid}.attn_v",
-    MODEL_TENSOR.A_ENC_ATTN_Q_NORM:         "a.blk.{bid}.attn_q_norm",
-    MODEL_TENSOR.A_ENC_ATTN_K_NORM:         "a.blk.{bid}.attn_k_norm",
     MODEL_TENSOR.A_ENC_ATTN_POST_NORM:      "a.blk.{bid}.attn_post_norm",
     MODEL_TENSOR.A_ENC_ATTN_PRE_NORM:       "a.blk.{bid}.attn_pre_norm",
     MODEL_TENSOR.A_ENC_ATTN_K_REL:          "a.blk.{bid}.attn_k_rel",
@@ -2068,9 +2037,6 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.A_GEN_CODE_FFN_UP:         "a.gen.code.blk.{bid}.ffn_up",
     MODEL_TENSOR.A_GEN_CODE_FFN_DOWN:       "a.gen.code.blk.{bid}.ffn_down",
     MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM:    "a.gen.code.output_norm",
-    MODEL_TENSOR.A_GEN_CODE_ATTN_GATE:      "a.gen.code.blk.{bid}.attn_gate",
-    MODEL_TENSOR.A_GEN_CODE_HEAD_NORM:      "a.gen.code.head_norm",
-    MODEL_TENSOR.A_GEN_CODE_MASK_EMBD:      "a.gen.code.mask_embd",
     MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN:  "a.gen.wav.quant.first.in_proj",
     MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj",
     MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB:  "a.gen.wav.quant.first.codebook",
@@ -2092,14 +2058,6 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP:      "a.gen.wav.tfm.blk.{bid}.ffn_up",
     MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN:    "a.gen.wav.tfm.blk.{bid}.ffn_down",
     MODEL_TENSOR.A_GEN_WAV_TFM_FFN_SCALE:   "a.gen.wav.tfm.blk.{bid}.ls2",
-    MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q_NORM: "a.gen.wav.tfm.blk.{bid}.attn_q_norm",
-    MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K_NORM: "a.gen.wav.tfm.blk.{bid}.attn_k_norm",
-    MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW1:    "a.gen.wav.tfm.blk.{bid}.conv_pw1",
-    MODEL_TENSOR.A_GEN_WAV_TFM_CONV_DW:     "a.gen.wav.tfm.blk.{bid}.conv_dw",
-    MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW2:    "a.gen.wav.tfm.blk.{bid}.conv_pw2",
-    MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_GATE:   "a.gen.wav.tfm.blk.{bid}.attn_gate",
-    MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_POST_NORM: "a.gen.wav.tfm.blk.{bid}.attn_post_norm",
-    MODEL_TENSOR.A_GEN_WAV_TFM_FFN_POST_NORM:  "a.gen.wav.tfm.blk.{bid}.ffn_post_norm",
     MODEL_TENSOR.A_GEN_WAV_UP_CONV:         "a.gen.wav.up.blk.{bid}.conv",
     MODEL_TENSOR.A_GEN_WAV_UP_DWCONV:       "a.gen.wav.up.blk.{bid}.dwconv",
     MODEL_TENSOR.A_GEN_WAV_UP_NORM:         "a.gen.wav.up.blk.{bid}.norm",
@@ -2396,8 +2354,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.A_ENC_ATTN_Q,
         MODEL_TENSOR.A_ENC_ATTN_K,
         MODEL_TENSOR.A_ENC_ATTN_V,
-        MODEL_TENSOR.A_ENC_ATTN_Q_NORM,
-        MODEL_TENSOR.A_ENC_ATTN_K_NORM,
         MODEL_TENSOR.A_ENC_ATTN_POST_NORM,
         MODEL_TENSOR.A_ENC_ATTN_PRE_NORM,
         MODEL_TENSOR.A_ENC_ATTN_K_REL,
@@ -2465,9 +2421,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.A_GEN_CODE_FFN_UP,
         MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
         MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM,
-        MODEL_TENSOR.A_GEN_CODE_ATTN_GATE,
-        MODEL_TENSOR.A_GEN_CODE_HEAD_NORM,
-        MODEL_TENSOR.A_GEN_CODE_MASK_EMBD,
         MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN,
         MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT,
         MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB,
@@ -2489,14 +2442,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP,
         MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN,
         MODEL_TENSOR.A_GEN_WAV_TFM_FFN_SCALE,
-        MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q_NORM,
-        MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K_NORM,
-        MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW1,
-        MODEL_TENSOR.A_GEN_WAV_TFM_CONV_DW,
-        MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW2,
-        MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_GATE,
-        MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_POST_NORM,
-        MODEL_TENSOR.A_GEN_WAV_TFM_FFN_POST_NORM,
         MODEL_TENSOR.A_GEN_WAV_UP_CONV,
         MODEL_TENSOR.A_GEN_WAV_UP_DWCONV,
         MODEL_TENSOR.A_GEN_WAV_UP_NORM,
@@ -3364,10 +3309,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.HC_FFN_DOWN,
         MODEL_TENSOR.HC_FFN_UP,
         MODEL_TENSOR.HC_FFN_INJECT,
-        MODEL_TENSOR.HC_ATTN_DOWN_INJECT,
-        MODEL_TENSOR.HC_FFN_DOWN_INJECT,
-        MODEL_TENSOR.ATTN_QKV_GATE,
-        MODEL_TENSOR.SSM_BETA_ALPHA_CAT,
         # full attention layers: ATTN_Q holds [q|gate] interleaved per head
         MODEL_TENSOR.ATTN_Q,
         MODEL_TENSOR.ATTN_Q_NORM,
@@ -5569,29 +5510,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_UP_EXP,
         MODEL_TENSOR.FFN_EXP_PROBS_B,
     ],
-    MODEL_ARCH.LFM3: [
-        MODEL_TENSOR.TOKEN_EMBD,
-        MODEL_TENSOR.OUTPUT_NORM,
-        MODEL_TENSOR.OUTPUT,
-        MODEL_TENSOR.ROPE_FREQS,
-        MODEL_TENSOR.SHORTCONV_CONV,
-        MODEL_TENSOR.SHORTCONV_INPROJ,
-        MODEL_TENSOR.SHORTCONV_OUTPROJ,
-        MODEL_TENSOR.FFN_GATE,
-        MODEL_TENSOR.FFN_DOWN,
-        MODEL_TENSOR.FFN_UP,
-        MODEL_TENSOR.FFN_NORM,
-        MODEL_TENSOR.FFN_POST_NORM,
-        MODEL_TENSOR.ATTN_NORM, # operator_norm
-        MODEL_TENSOR.ATTN_POST_NORM,
-        MODEL_TENSOR.ATTN_Q_NORM,
-        MODEL_TENSOR.ATTN_K_NORM,
-        MODEL_TENSOR.ATTN_Q,
-        MODEL_TENSOR.ATTN_K,
-        MODEL_TENSOR.ATTN_V,
-        MODEL_TENSOR.ATTN_OUT,
-        MODEL_TENSOR.ATTN_GATE,
-    ],
     MODEL_ARCH.SMALLTHINKER: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT_NORM,
@@ -6544,11 +6462,8 @@ class DecisionType:
     VLLM_D3 = "vllm-d3"  # same as pplx-decider, non-causal attention
     LFM2_D1 = "lfm2-d1"  # same as openjev, the labels depend on the question type
     LFM2_D1_OMNI = "lfm2-d1-omni"  # same head as laya on a bidirectional LFM2 trunk, other prompt layout
-    LFM3_D1 = "lfm3-d1"  # same as lfm2-d1 on LFM3-VL, full attention layers read the prompt both ways
-    LFM3_D1_OMNI = "lfm3-d1-omni"  # same as lfm3-d1 on LFM3-Omni
     GRANITE_SWITCH = "granite-switch"  # same as openjev, run by the decision aLoRA of granite switch
     H2O_LIGHTNING = "h2o-lightning"  # same as openjev, labels read after an answer prefill
-    MINICPM_OCR_JEV = "openbmb-minicpm-ocr-jev"  # class scores of a value head read at the last token, one fixed task
 
 
 class VisionProjectorType:
@@ -6579,7 +6494,6 @@ class VisionProjectorType:
     VOXTRAL_RT = "voxtral_rt"
     MERALION = "meralion"  # audio: Whisper + gated MLP adaptor
     LFM2 = "lfm2"
-    LFM3VL = "lfm3vl"
     KIMIVL = "kimivl"
     PADDLEOCR = "paddleocr"
     KIMIK25 = "kimik25"
@@ -6594,8 +6508,6 @@ class VisionProjectorType:
     DEEPSEEK4V = "deepseek4v"
     DEEPSEEK41V = "deepseek41v"
     LFM2A = "lfm2a" # audio
-    LFM3A = "lfm3a" # audio
-    LFM3A_GEN = "lfm3a_gen" # audio generation: MaskGIT depthformer + liquid-codec decoder
     MUSIC_FLAMINGO = "musicflamingo" # audio
     GLM4V = "glm4v"
     GLM5V = "glm5v"

gguf-py/gguf/gguf_writer.py

@@ -1628,9 +1628,6 @@ class GGUFWriter:
     def add_audio_head_count(self, value: int) -> None:
         self.add_uint32(Keys.ClipAudio.Attention.HEAD_COUNT, value)
 
-    def add_audio_head_count_kv(self, value: int) -> None:
-        self.add_uint32(Keys.ClipAudio.Attention.HEAD_COUNT_KV, value)
-
     def add_audio_head_dim(self, value: int) -> None:
         self.add_uint32(Keys.ClipAudio.Attention.HEAD_DIM, value)
 
@@ -1711,18 +1708,6 @@ class GGUFWriter:
     def add_gen_audio_block_count(self, value: int) -> None:
         self.add_uint32(Keys.ClipGenAudio.BLOCK_COUNT, value)
 
-    def add_gen_audio_sampling_temp(self, value: float) -> None:
-        self.add_float32(Keys.ClipGenAudio.SAMPLING_TEMP, value)
-
-    def add_gen_audio_sampling_top_k(self, value: int) -> None:
-        self.add_int32(Keys.ClipGenAudio.SAMPLING_TOP_K, value)
-
-    def add_gen_audio_sampling_penalty_repeat(self, value: float) -> None:
-        self.add_float32(Keys.ClipGenAudio.SAMPLING_PENALTY_REPEAT, value)
-
-    def add_gen_audio_sampling_penalty_last_n(self, value: int) -> None:
-        self.add_int32(Keys.ClipGenAudio.SAMPLING_PENALTY_LAST_N, value)
-
     def add_gen_audio_head_count(self, value: int) -> None:
         self.add_uint32(Keys.ClipGenAudio.Attention.HEAD_COUNT, value)
 

gguf-py/gguf/tensor_mapping.py

@@ -119,7 +119,6 @@ class TensorNameMap:
             "model.transformer.ln_f",                  # llada
             "final_norm",                              # modern-bert
             "model.norm",                              # cogvlm
-            "model.embedding_norm",                    # lfm3
         ),
 
         # Rope frequencies
@@ -419,7 +418,6 @@ class TensorNameMap:
             "model.layers.{bid}.post_self_attn_layernorm",       # glm-4-0414
             "model.layers.{bid}.post_attn_norm",                 # kolibri1
             "model.layers.layers.{bid}.post_mixer_norm.weight",  # plamo2
-            "model.layers.{bid}.post_operator_norm",             # lfm3
         ),
 
         # Rotary embeddings
@@ -506,7 +504,6 @@ class TensorNameMap:
             "model.layers.layers.{bid}.post_mlp_norm.weight", # plamo2
             "model.layers.{bid}.feed_forward.up_proj",
             "model.layers.{bid}.post_moe_norm",               # grok-2
-            "model.layers.{bid}.post_ffn_norm",               # lfm3
         ),
 
         MODEL_TENSOR.FFN_POST_NORM_1: (
@@ -1858,7 +1855,6 @@ class TensorNameMap:
             "vision_model.model.layers.{bid}.self_attn.q_proj", # llama4
             "vision_tower.transformer.layers.{bid}.attention.q_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.attention.wq", # pixtral
-            "model.vision_tower.layers.{bid}.self_attn.q_proj", # lfm3vl
             "visual.blocks.{bid}.attn.q", # qwen2vl, generated
             "vision_tower.encoder.blocks.{bid}.wq", # kimi-vl, generated
             "siglip2.vision_model.encoder.layers.{bid}.self_attn.q_proj", # youtuvl
@@ -1874,7 +1870,6 @@ class TensorNameMap:
             "model.vision_tower.encoder.layer.{bid}.attention.q_norm", # Intern-S1
             "visual.blocks.{bid}.attn.q_norm", # GLM-OCR
             "vision_model.model.layers.{bid}.self_attn.q_norm", # gemma4
-            "model.vision_tower.layers.{bid}.self_attn.q_layernorm", # lfm3vl
         ),
 
         MODEL_TENSOR.V_ENC_ATTN_K: (
@@ -1889,7 +1884,6 @@ class TensorNameMap:
             "vision_model.model.layers.{bid}.self_attn.k_proj", # llama4
             "vision_tower.transformer.layers.{bid}.attention.k_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.attention.wk", # pixtral
-            "model.vision_tower.layers.{bid}.self_attn.k_proj", # lfm3vl
             "visual.blocks.{bid}.attn.k", # qwen2vl, generated
             "vision_tower.encoder.blocks.{bid}.wk", # kimi-vl, generated
             "model.vision_model.transformer.layers.{bid}.self_attn.k_proj", # Deepseek-OCR CLIP, generated
@@ -1905,7 +1899,6 @@ class TensorNameMap:
             "model.vision_tower.encoder.layer.{bid}.attention.k_norm", # Intern-S1
             "visual.blocks.{bid}.attn.k_norm", # GLM-OCR
             "vision_model.model.layers.{bid}.self_attn.k_norm", # gemma4
-            "model.vision_tower.layers.{bid}.self_attn.k_layernorm", # lfm3vl
         ),
 
         MODEL_TENSOR.V_ENC_ATTN_V: (
@@ -1920,7 +1913,6 @@ class TensorNameMap:
             "vision_model.model.layers.{bid}.self_attn.v_proj", # llama4
             "vision_tower.transformer.layers.{bid}.attention.v_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.attention.wv", # pixtral
-            "model.vision_tower.layers.{bid}.self_attn.v_proj", # lfm3vl
             "visual.blocks.{bid}.attn.v", # qwen2vl, generated
             "vision_tower.encoder.blocks.{bid}.wv", # kimi-vl, generated
             "siglip2.vision_model.encoder.layers.{bid}.self_attn.v_proj",
@@ -1943,7 +1935,6 @@ class TensorNameMap:
             "vit.layers.{bid}.input_layernorm", # HunyuanVL
             "vision_tower.transformer.layers.{bid}.attention_norm", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.attention_norm", # pixtral
-            "model.vision_tower.layers.{bid}.operator_norm", # lfm3vl
             "vision_model.model.layers.{bid}.input_layernorm", # llama4, gemma4
             "visual.blocks.{bid}.norm1", # qwen2vl
             "vision_tower.encoder.blocks.{bid}.norm0", # kimi-vl (norm0/norm1)
@@ -1973,7 +1964,6 @@ class TensorNameMap:
             "vision_model.model.layers.{bid}.self_attn.o_proj", # llama4
             "vision_tower.transformer.layers.{bid}.attention.o_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.attention.wo", # pixtral
-            "model.vision_tower.layers.{bid}.self_attn.out_proj", # lfm3vl
             "visual.blocks.{bid}.attn.proj", # qwen2vl
             "vision_tower.encoder.blocks.{bid}.wo", # kimi-vl
             "model.vision.transformer.layers.{bid}.attention.dense", # cogvlm
@@ -2006,7 +1996,6 @@ class TensorNameMap:
             "vision_model.model.layers.{bid}.post_attention_layernorm", # llama4
             "vision_tower.transformer.layers.{bid}.ffn_norm", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.ffn_norm", # pixtral
-            "model.vision_tower.layers.{bid}.ffn_norm", # lfm3vl
             "visual.blocks.{bid}.norm2", # qwen2vl
             "vision_tower.encoder.blocks.{bid}.norm1", # kimi-vl (norm0/norm1)
             "model.vision.transformer.layers.{bid}.post_attention_layernorm", # cogvlm
@@ -2034,7 +2023,6 @@ class TensorNameMap:
             "vit.layers.{bid}.mlp.dense_h_to_4h", # HunyuanVL
             "vision_tower.transformer.layers.{bid}.feed_forward.up_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.feed_forward.w3", # pixtral
-            "model.vision_tower.layers.{bid}.feed_forward.w3", # lfm3vl
             "vision_model.model.layers.{bid}.mlp.fc1", # llama4
             "visual.blocks.{bid}.mlp.fc1", # qwen2vl
             "visual.blocks.{bid}.mlp.up_proj", # qwen2.5vl
@@ -2055,7 +2043,6 @@ class TensorNameMap:
             "vision_encoder.blocks.{bid}.mlp.fc1", # dots3note
             "vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral
-            "model.vision_tower.layers.{bid}.feed_forward.w1", # lfm3vl
             "visual.blocks.{bid}.mlp.gate_proj", # qwen2.5vl
             "vision_model.model.layers.{bid}.mlp.gate_proj", # gemma4
             "model.qwen2_model.model.model.layers.{bid}.mlp.gate_proj", # Deepseek-OCR-2 qwen2
@@ -2074,7 +2061,6 @@ class TensorNameMap:
             "vit.layers.{bid}.mlp.dense_4h_to_h", # HunyuanVL
             "vision_tower.transformer.layers.{bid}.feed_forward.down_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.feed_forward.w2", # pixtral
-            "model.vision_tower.layers.{bid}.feed_forward.w2", # lfm3vl
             "vision_model.model.layers.{bid}.mlp.fc2", # llama4
             "visual.blocks.{bid}.mlp.fc2", # qwen2vl
             "visual.blocks.{bid}.mlp.down_proj", # qwen2.5vl
@@ -2554,14 +2540,12 @@ class TensorNameMap:
             "audio_encoder.dots_encoder.speech_encoder.layer_norm", # dots3note
             "audio_tower.ln_post", # qwen2omni
             "encoder.layer_norm", # mimo-audio-tokenizer
-            "encoder.lfm.lfm.embedding_norm", # lfm3a
         ),
 
         MODEL_TENSOR.A_ENC_ATTN_Q: (
             "audio_tower.layers.{bid}.self_attn.q_proj", # ultravox
             "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.q_proj", # dots3note
             "conformer.layers.{bid}.self_attn.linear_q", # lfm2
-            "encoder.lfm.lfm.layers.{bid}.self_attn.q_proj", # lfm3a
             "conformer.layers.{bid}.attention.attn.q_proj", # gemma3n
             "conformer.layers.{bid}.self_attn.q_proj", # gemma4
             "sound_encoder.encoder.layers.{bid}.self_attn.q_proj", # parakeet
@@ -2573,7 +2557,6 @@ class TensorNameMap:
             "audio_tower.layers.{bid}.self_attn.k_proj", # ultravox
             "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.k_proj", # dots3note
             "conformer.layers.{bid}.self_attn.linear_k", # lfm2
-            "encoder.lfm.lfm.layers.{bid}.self_attn.k_proj", # lfm3a
             "conformer.layers.{bid}.attention.attn.k_proj", # gemma3n
             "conformer.layers.{bid}.self_attn.k_proj", # gemma4
             "sound_encoder.encoder.layers.{bid}.self_attn.k_proj", # parakeet
@@ -2585,7 +2568,6 @@ class TensorNameMap:
             "audio_tower.layers.{bid}.self_attn.v_proj", # ultravox
             "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.v_proj", # dots3note
             "conformer.layers.{bid}.self_attn.linear_v", # lfm2
-            "encoder.lfm.lfm.layers.{bid}.self_attn.v_proj", # lfm3a
             "conformer.layers.{bid}.attention.attn.v_proj", # gemma3n
             "conformer.layers.{bid}.self_attn.v_proj", # gemma4
             "sound_encoder.encoder.layers.{bid}.self_attn.v_proj", # parakeet
@@ -2593,14 +2575,6 @@ class TensorNameMap:
             "encoder.layers.{bid}.self_attn.v_proj", # mimo-audio-tokenizer
         ),
 
-        MODEL_TENSOR.A_ENC_ATTN_Q_NORM: (
-            "encoder.lfm.lfm.layers.{bid}.self_attn.q_layernorm", # lfm3a
-        ),
-
-        MODEL_TENSOR.A_ENC_ATTN_K_NORM: (
-            "encoder.lfm.lfm.layers.{bid}.self_attn.k_layernorm", # lfm3a
-        ),
-
         MODEL_TENSOR.A_ENC_ATTN_K_REL: (
             "conformer.layers.{bid}.self_attn.relative_k_proj", # gemma4
         ),
@@ -2626,7 +2600,6 @@ class TensorNameMap:
             "audio_tower.layers.{bid}.self_attn_layer_norm", # ultravox
             "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn_layer_norm", # dots3note
             "conformer.layers.{bid}.norm_self_att", # lfm2
-            "encoder.lfm.lfm.layers.{bid}.operator_norm", # lfm3a
             "conformer.layers.{bid}.attention.pre_attn_norm", # gemma3n
             "sound_encoder.encoder.layers.{bid}.norm_self_att", # parakeet
             "encoder.layers.{bid}.attn.pre_norm", # granite_speech
@@ -2638,7 +2611,6 @@ class TensorNameMap:
             "audio_tower.layers.{bid}.self_attn.o_proj", # voxtral realtime
             "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.out_proj", # dots3note
             "conformer.layers.{bid}.self_attn.linear_out", # lfm2
-            "encoder.lfm.lfm.layers.{bid}.self_attn.out_proj", # lfm3a
             "conformer.layers.{bid}.attention.post", # gemma3n
             "conformer.layers.{bid}.self_attn.post", # gemma4
             "sound_encoder.encoder.layers.{bid}.self_attn.o_proj", # parakeet
@@ -2650,7 +2622,6 @@ class TensorNameMap:
             "audio_tower.layers.{bid}.final_layer_norm", # ultravox
             "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.final_layer_norm", # dots3note
             "conformer.layers.{bid}.norm_out", # lfm2
-            "encoder.lfm.lfm.layers.{bid}.ffn_norm", # lfm3a
             "conformer.layers.{bid}.attention.post_norm", # gemma3n
             "sound_encoder.encoder.layers.{bid}.norm_out", # parakeet
             "encoder.layers.{bid}.post_norm", # granite_speech
@@ -2679,7 +2650,6 @@ class TensorNameMap:
             "audio_tower.layers.{bid}.fc1", # ultravox
             "audio_tower.layers.{bid}.mlp.up_proj", # voxtral realtime
             "conformer.layers.{bid}.feed_forward1.linear1", # lfm2
-            "encoder.lfm.lfm.layers.{bid}.feed_forward.w3", # lfm3a
             "conformer.layers.{bid}.ffw_layer_start.ffw_layer_1", # gemma3n
             "conformer.layers.{bid}.feed_forward1.ffw_layer_1", # gemma4
             "sound_encoder.encoder.layers.{bid}.feed_forward1.linear1", # parakeet
@@ -2689,7 +2659,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_ENC_FFN_GATE: (
             "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.fc1_gate", # dots3note (split from fc1 in conversion code)
-            "encoder.lfm.lfm.layers.{bid}.feed_forward.w1", # lfm3a
             "audio_tower.layers.{bid}.mlp.gate_proj", # voxtral realtime
         ),
 
@@ -2698,7 +2667,6 @@ class TensorNameMap:
             "audio_tower.layers.{bid}.fc2", # ultravox
             "audio_tower.layers.{bid}.mlp.down_proj", # voxtral realtime
             "conformer.layers.{bid}.feed_forward1.linear2", # lfm2
-            "encoder.lfm.lfm.layers.{bid}.feed_forward.w2", # lfm3a
             "conformer.layers.{bid}.ffw_layer_start.ffw_layer_2", # gemma3n
             "conformer.layers.{bid}.feed_forward1.ffw_layer_2", # gemma4
             "sound_encoder.encoder.layers.{bid}.feed_forward1.linear2", # parakeet
@@ -2782,7 +2750,6 @@ class TensorNameMap:
             "audio.multi_modal_projector.linear_{bid}", # ultravox, meralion
             "audio_encoder.audio_adapter.proj.{bid}", # dots3note (proj.1, proj.3)
             "audio_adapter.model.{bid}", # lfm2
-            "mlp.model.{bid}", # lfm3a
             "audio_tower.proj{bid}", # qwen3omni
             "sound_projection.linear{bid}", # parakeet (linear1, linear2)
         ),
@@ -2842,7 +2809,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_ENC_CONV_DW: (
             "conformer.layers.{bid}.conv.depthwise_conv", # lfm2
-            "encoder.lfm.lfm.layers.{bid}.conv.conv", # lfm3a
             "conformer.layers.{bid}.lconv1d.depthwise_conv1d", # gemma3n
             "sound_encoder.encoder.layers.{bid}.conv.depthwise_conv", # parakeet
             "encoder.layers.{bid}.conv.depth_conv.conv", # granite_speech
@@ -2865,7 +2831,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_ENC_CONV_PW1: (
             "conformer.layers.{bid}.conv.pointwise_conv1", # lfm2
-            "encoder.lfm.lfm.layers.{bid}.conv.in_proj", # lfm3a
             "conformer.layers.{bid}.lconv1d.linear_start", # gemma3n
             "sound_encoder.encoder.layers.{bid}.conv.pointwise_conv1", # parakeet
             "encoder.layers.{bid}.conv.up_conv", # granite_speech
@@ -2874,7 +2839,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_ENC_CONV_PW2: (
             "conformer.layers.{bid}.conv.pointwise_conv2", # lfm2
-            "encoder.lfm.lfm.layers.{bid}.conv.out_proj", # lfm3a
             "conformer.layers.{bid}.lconv1d.linear_end", # gemma3n
             "sound_encoder.encoder.layers.{bid}.conv.pointwise_conv2", # parakeet
             "encoder.layers.{bid}.conv.down_conv", # granite_speech

models/templates/LFM3.jinja

@@ -1,280 +0,0 @@
-{{- bos_token -}}
-{%- set preserve_thinking = preserve_thinking | default(false) -%}
-{%- set enable_thinking = enable_thinking | default(true) -%}
-
-{%- macro render_value(v) -%}
-    {%- if v is string -%}
-        {{- v -}}
-    {%- elif v is boolean -%}
-        {{- "true" if v else "false" -}}
-    {%- elif v is none -%}
-        {{- "null" -}}
-    {%- elif v is mapping or (v is sequence and v is not string) -%}
-        {{- v | tojson -}}
-    {%- else -%}
-        {{- v -}}
-    {%- endif -%}
-{%- endmacro -%}
-
-{%- macro parse_content(content) -%}
-    {%- if content is string -%}
-        {{- content -}}
-    {%- elif content is mapping and content.get("type") == "image" -%}
-        {{- "<|vision_start|><|image|><|vision_end|>" -}}
-    {%- elif content is mapping and content.get("type") == "video" -%}
-        {{- "<|vision_start|><|video|><|vision_end|>" -}}
-    {%- elif content is mapping and content.get("type") == "text" -%}
-        {{- (content.get("text") or "") | string -}}
-    {%- elif content is mapping -%}
-        {{- content | tojson -}}
-    {%- elif content is iterable -%}
-        {%- set _ns = namespace(result="") -%}
-        {%- for item in content -%}
-            {%- if item is string -%}
-                {%- set _ns.result = _ns.result + item -%}
-            {%- elif item is mapping and item.get("type") == "image" -%}
-                {%- set _ns.result = _ns.result + "<|vision_start|><|image|><|vision_end|>" -%}
-            {%- elif item is mapping and item.get("type") == "video" -%}
-                {%- set _ns.result = _ns.result + "<|vision_start|><|video|><|vision_end|>" -%}
-            {%- elif item is mapping and item.get("type") == "text" -%}
-                {%- set _ns.result = _ns.result + ((item.get("text") or "") | string) -%}
-            {%- else -%}
-                {%- set _ns.result = _ns.result + (item | tojson) -%}
-            {%- endif -%}
-        {%- endfor -%}
-        {{- _ns.result -}}
-    {%- elif content is not none -%}
-        {{- content | tojson -}}
-    {%- endif -%}
-{%- endmacro -%}
-
-{%- macro render_tool_calls(tool_calls) -%}
-    {%- for tool_call in tool_calls -%}
-        {%- if not loop.first -%}{{- "\n\n" -}}{%- endif -%}
-        {%- set func = tool_call["function"] if "function" in tool_call else tool_call -%}
-        {%- set func_args = func.get("arguments") -%}
-        {%- if func_args is string and (func_args | trim) not in ["", "{}", "null"] -%}
-            {{- raise_exception("Tool call arguments must be a mapping, got a JSON-encoded string: parse arguments with json.loads() before applying the chat template") -}}
-        {%- endif -%}
-        {{- "<tool_call>\n<function=" + func["name"] + ">\n" -}}
-        {%- if func_args is mapping -%}
-            {%- for arg_name, arg_value in func_args.items() -%}
-                {{- "<parameter=" + arg_name + ">\n" + render_value(arg_value) + "\n</parameter>\n" -}}
-            {%- endfor -%}
-        {%- endif -%}
-        {{- "</function>\n</tool_call>" -}}
-    {%- endfor -%}
-{%- endmacro -%}
-
-{%- macro tool_to_json(tool) -%}
-    {%- set fn = tool.function if tool.function is defined else tool -%}
-    {%- set ns_t = namespace(first=true) -%}
-    {{- "{" -}}
-    {%- for k, v in fn.items() -%}
-        {%- if k != "defer_loading" -%}
-            {%- if not ns_t.first -%}{{- ", " -}}{%- endif -%}
-            {%- set ns_t.first = false -%}
-            {{- k | tojson -}}{{- ": " -}}{{- v | tojson -}}
-        {%- endif -%}
-    {%- endfor -%}
-    {{- "}" -}}
-{%- endmacro -%}
-
-{%- macro is_tool_reference(item) -%}
-    {{- "1" if item is mapping and item.get("type") == "tool_reference" else "" -}}
-{%- endmacro -%}
-
-{%- macro referenced_tool_to_json(name) -%}
-    {%- set _ns = namespace(found=none) -%}
-    {%- for tool in tools or [] -%}
-        {%- set fn = tool.function if tool.function is defined else tool -%}
-        {%- if fn.name == name -%}
-            {%- set _ns.found = fn -%}
-        {%- endif -%}
-    {%- endfor -%}
-    {%- if _ns.found is none -%}
-        {{- raise_exception("Tool reference '" + name | string + "' not found in available tools. Every tool referenced by a tool_reference must be in `tools` on every request.") -}}
-    {%- endif -%}
-    {{- tool_to_json(_ns.found) -}}
-{%- endmacro -%}
-
-{%- macro render_tool_response(content) -%}
-    {%- if content is string and "<tool_response>" in content -%}
-        {{- content -}}
-    {%- else -%}
-    {{- "<tool_response>\n" -}}
-    {%- if content is string -%}
-        {{- content -}}
-    {%- elif content is mapping -%}
-        {{- parse_content(content) -}}
-    {%- elif content is iterable -%}
-        {%- for item in content -%}
-            {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
-            {%- if is_tool_reference(item) -%}
-                {%- if loop.first or not is_tool_reference(loop.previtem) -%}
-                    {{- "<additional_tools>\n" -}}
-                {%- endif -%}
-                {{- referenced_tool_to_json(item.name) -}}
-                {%- if loop.last or not is_tool_reference(loop.nextitem) -%}
-                    {{- "\n</additional_tools>" -}}
-                {%- endif -%}
-            {%- elif item is mapping and item.result is defined and item.result is mapping and item.result.tools is defined -%}
-                {{- "<additional_tools>\n" -}}
-                {%- set _tl = item.result.tools -%}
-                {%- if _tl is string -%}
-                    {{- _tl -}}
-                {%- elif _tl is mapping -%}
-                    {{- tool_to_json(_tl) -}}
-                {%- else -%}
-                    {%- for t in _tl -%}
-                        {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
-                        {{- tool_to_json(t) -}}
-                    {%- endfor -%}
-                {%- endif -%}
-                {{- "\n</additional_tools>" -}}
-            {%- else -%}
-                {{- parse_content(item) -}}
-            {%- endif -%}
-        {%- endfor -%}
-    {%- elif content is not none -%}
-        {{- content | tojson -}}
-    {%- endif -%}
-    {{- "\n</tool_response>" -}}
-    {%- endif -%}
-{%- endmacro -%}
-
-{%- set ns = namespace(system_prompt="", last_user_index=-1) -%}
-{%- if messages and messages[0]["role"] == "system" -%}
-    {%- if messages[0].get("content") -%}
-        {%- set ns.system_prompt = parse_content(messages[0]["content"]) -%}
-    {%- endif -%}
-    {%- set messages = messages[1:] -%}
-{%- endif -%}
-{%- if tools -%}
-    {%- set tools_ns = namespace(lines=[]) -%}
-    {%- for tool in tools -%}
-        {%- set fn = tool.function if tool.function is defined else tool -%}
-        {%- set _dyn = (fn.defer_loading is defined and fn.defer_loading) or (tool.defer_loading is defined and tool.defer_loading) -%}
-        {%- if not _dyn -%}
-            {%- set tools_ns.lines = tools_ns.lines + [tool_to_json(fn)] -%}
-        {%- endif -%}
-    {%- endfor -%}
-    {%- if tools_ns.lines -%}
-        {%- set ns.system_prompt = ns.system_prompt + ("\n\n" if ns.system_prompt else "") + "<tools>\n" + (tools_ns.lines | join("\n")) + "\n</tools>" -%}
-    {%- endif -%}
-{%- endif -%}
-{%- if ns.system_prompt -%}
-    {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
-{%- endif -%}
-{%- for message in messages -%}
-    {%- if message["role"] == "user" -%}
-        {%- set ns.last_user_index = loop.index0 -%}
-    {%- endif -%}
-{%- endfor -%}
-{%- set carry = namespace(pending=[]) -%}
-{%- for message in messages -%}
-    {%- set _next = messages[loop.index0 + 1] if not loop.last else none -%}
-    {%- set _merge = message.role == "tool" and not message.get("additional_tools")
-        and _next is not none and _next.role == "tool" and not _next.get("additional_tools")
-        and message.get("tool_call_id") and _next.get("tool_call_id") == message.get("tool_call_id") -%}
-    {%- set _items = [] -%}
-    {%- if message.role == "tool" and not message.get("additional_tools") -%}
-        {%- set _c = message.get("content") -%}
-        {%- if _c is string -%}
-            {%- set _items = [{"type": "text", "text": _c}] if _c else [] -%}
-        {%- elif _c is mapping -%}
-            {%- set _items = [_c] -%}
-        {%- elif _c is iterable -%}
-            {%- set _items = _c | list -%}
-        {%- elif _c is not none -%}
-            {%- set _items = [_c] -%}
-        {%- endif -%}
-    {%- endif -%}
-    {%- if _merge -%}
-        {%- set carry.pending = carry.pending + _items -%}
-    {%- else -%}
-    {{- "<|im_start|>" + message.role + "\n" -}}
-    {%- if message.role == "assistant" -%}
-        {%- generation -%}
-        {%- set thinking = message.thinking if message.thinking is defined else message.reasoning if message.reasoning is defined else message.reasoning_content -%}
-        {%- set keep_thinking = preserve_thinking or loop.index0 > ns.last_user_index -%}
-        {%- if thinking is defined and keep_thinking -%}
-            {{- "<think>" + (thinking or "") + "</think>" -}}
-        {%- elif keep_thinking and "</think>" not in (message.get("content") if message.get("content") is string else "") -%}
-            {{- "<think></think>" -}}
-        {%- endif -%}
-        {%- set _cfm_tag = "CONTINUE_FINAL_MESSAGE_TAG " -%}
-        {%- set _has_cfm = false -%}
-        {%- set content = "" -%}
-        {%- if message.get("content") -%}
-            {%- set content = parse_content(message.content) -%}
-        {%- endif -%}
-        {%- if not keep_thinking -%}
-            {%- if "</think>" in content -%}
-                {%- set content = content.split("</think>")[-1] | trim -%}
-            {%- endif -%}
-            {%- set content = "<think></think>" + content -%}
-        {%- endif -%}
-        {%- if content.endswith(_cfm_tag) -%}
-            {%- set _has_cfm = true -%}
-            {%- set _trunc_len = (content | length) - (_cfm_tag | length) -%}
-            {%- set content = content[:_trunc_len] -%}
-        {%- endif -%}
-        {{- content -}}
-        {%- if message.tool_calls -%}
-            {{- render_tool_calls(message.tool_calls) -}}
-        {%- endif -%}
-        {%- if _has_cfm -%}
-            {{- _cfm_tag -}}
-        {%- endif -%}
-        {{- "<|im_end|>\n" -}}
-        {%- endgeneration -%}
-    {%- elif message.role == "tool" -%}
-        {%- if message.get("additional_tools") -%}
-            {%- set _at = message["additional_tools"] -%}
-            {%- if _at is mapping or (_at is sequence and _at is not string) -%}
-                {%- set _src = _at -%}
-            {%- else -%}
-                {%- set _src = message.get("content") -%}
-            {%- endif -%}
-            {%- if _src is string and "<tool_response>" in _src -%}
-                {{- _src -}}
-            {%- elif _src is string and "<additional_tools>" in _src -%}
-                {{- "<tool_response>\n" + _src + "\n</tool_response>" -}}
-            {%- elif _src -%}
-                {{- "<tool_response>\n<additional_tools>\n" -}}
-                {%- if _src is string -%}
-                    {{- _src -}}
-                {%- elif _src is mapping -%}
-                    {{- tool_to_json(_src) -}}
-                {%- else -%}
-                    {%- for t in _src -%}
-                        {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
-                        {{- tool_to_json(t) -}}
-                    {%- endfor -%}
-                {%- endif -%}
-                {{- "\n</additional_tools>\n</tool_response>" -}}
-            {%- endif -%}
-        {%- elif carry.pending -%}
-            {{- render_tool_response(carry.pending + _items) -}}
-            {%- set carry.pending = [] -%}
-        {%- else -%}
-            {{- render_tool_response("" if message.get("content") is none else message.get("content")) -}}
-        {%- endif -%}
-        {{- "<|im_end|>\n" -}}
-    {%- else %}
-        {%- if message.get("content") -%}
-            {{- parse_content(message["content"]) -}}
-        {%- endif -%}
-        {{- "<|im_end|>\n" -}}
-    {%- endif %}
-    {%- endif -%}
-{%- endfor -%}
-{%- if add_generation_prompt -%}
-    {{- "<|im_start|>assistant\n" -}}
-    {%- if enable_thinking -%}
-        {{- "<think>" -}}
-    {%- else -%}
-        {{- "<think></think>" -}}
-    {%- endif -%}
-{%- endif -%}

scripts/snapdragon/qdc/run_qdc_jobs.py

@@ -36,7 +36,6 @@ import sys
 import tempfile
 import time
 import xml.etree.ElementTree as ET
-import zipfile
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Callable
@@ -407,19 +406,11 @@ def fetch_logs_and_parse_tests(
 
     with tempfile.TemporaryDirectory() as tmpdir:
         for lf in log_files:
-            dl_path = os.path.join(tmpdir, "log.bin")
+            zip_path = os.path.join(tmpdir, "log.zip")
             log.info("Downloading log file: %s", lf.filename)
+            qdc_api.download_job_log_files(client, lf.filename, zip_path)
             try:
-                qdc_api.download_job_log_files_v2(client, lf.filename, dl_path)
-            except Exception as e:
-                log.warning("Could not download %s: %s", lf.filename, e)
-                continue
-            # v2 returns some files raw instead of zipped
-            if not zipfile.is_zipfile(dl_path):
-                os.replace(dl_path, os.path.join(tmpdir, os.path.basename(lf.filename)))
-                continue
-            try:
-                shutil.unpack_archive(dl_path, tmpdir, "zip")
+                shutil.unpack_archive(zip_path, tmpdir, "zip")
             except Exception as e:
                 log.warning("Could not unpack %s as zip: %s", lf.filename, e)
 
@@ -626,7 +617,7 @@ def main() -> int:
             args.pkg_dir, Path(tmpdir), args.test, args.model_url
         )
         log.info("Uploading artifact (%d MB) ...", zip_path.stat().st_size // 1_000_000)
-        artifact_id = qdc_api.upload_file_v2(client, str(zip_path), ArtifactType.TESTSCRIPT)
+        artifact_id = qdc_api.upload_file(client, str(zip_path), ArtifactType.TESTSCRIPT)
 
     if artifact_id is None:
         log.error("Artifact upload failed")

src/llama-arch.cpp

@@ -1,6 +1,5 @@
 #include "llama-arch.h"
 
-#include "llama-ext.h"
 #include "llama-impl.h"
 
 #include <map>
@@ -135,7 +134,6 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_OPENAI_MOE,       "gpt-oss"          },
     { LLM_ARCH_LFM2,             "lfm2"             },
     { LLM_ARCH_LFM2MOE,          "lfm2moe"          },
-    { LLM_ARCH_LFM3,             "lfm3"             },
     { LLM_ARCH_DREAM,            "dream"            },
     { LLM_ARCH_SMALLTHINKER,     "smallthinker"     },
     { LLM_ARCH_LLADA,            "llada"            },
@@ -637,10 +635,6 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
     { LLM_TENSOR_HC_FFN_DOWN,                            "blk.%d.hc_ffn_down" },
     { LLM_TENSOR_HC_FFN_UP,                              "blk.%d.hc_ffn_up" },
     { LLM_TENSOR_HC_FFN_INJECT,                          "blk.%d.hc_ffn_inject" },
-    { LLM_TENSOR_HC_ATTN_DOWN_INJECT,                    "blk.%d.hc_attn_down_inject" },
-    { LLM_TENSOR_HC_FFN_DOWN_INJECT,                     "blk.%d.hc_ffn_down_inject" },
-    { LLM_TENSOR_ATTN_QKV_GATE,                          "blk.%d.attn_qkv_gate" },
-    { LLM_TENSOR_SSM_BETA_ALPHA_CAT,                     "blk.%d.ssm_beta_alpha" },
     { LLM_TENSOR_PLE_KEY,                                "blk.%d.ple_key" },
     { LLM_TENSOR_PLE_VALUE,                              "blk.%d.ple_value" },
     { LLM_TENSOR_PLE_NORM_KEY,                           "blk.%d.ple_norm_key" },
@@ -913,10 +907,6 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
     {LLM_TENSOR_HC_FFN_DOWN,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
     {LLM_TENSOR_HC_FFN_UP,                  {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
     {LLM_TENSOR_HC_FFN_INJECT,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
-    {LLM_TENSOR_HC_ATTN_DOWN_INJECT,        {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
-    {LLM_TENSOR_HC_FFN_DOWN_INJECT,         {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
-    {LLM_TENSOR_ATTN_QKV_GATE,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
-    {LLM_TENSOR_SSM_BETA_ALPHA_CAT,         {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
     {LLM_TENSOR_PLE_KEY,                    {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
     {LLM_TENSOR_PLE_VALUE,                  {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
     {LLM_TENSOR_PLE_NORM_KEY,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
@@ -1252,7 +1242,6 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
         case LLM_ARCH_GRANITE_HYBRID:
         case LLM_ARCH_LFM2:
         case LLM_ARCH_LFM2MOE:
-        case LLM_ARCH_LFM3:
         case LLM_ARCH_NEMOTRON_H:
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_QWEN3NEXT:
@@ -1297,7 +1286,6 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) {
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_LFM2:
         case LLM_ARCH_LFM2MOE:
-        case LLM_ARCH_LFM3:
         case LLM_ARCH_BAILINGMOE3:
             return true;
         default:
@@ -1344,14 +1332,12 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
         case LLM_ARCH_NEMOTRON_H:
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_GRANITE_HYBRID:
-        case LLM_ARCH_LFM3:
         case LLM_ARCH_MINIMAX_01:
         case LLM_ARCH_MINIMAX_M2:
         case LLM_ARCH_MINIMAX_M3:
         case LLM_ARCH_MISTRAL4:
         case LLM_ARCH_MELLUM: // [TAG_MOE_AMAX_SCALE]
         case LLM_ARCH_FLINT:  // [TAG_MOE_AMAX_SCALE]
-        case LLM_ARCH_LAGUNA: // [TAG_MOE_AMAX_SCALE]
         case LLM_ARCH_KIMI_LINEAR:
         case LLM_ARCH_BAILINGMOE3:
         case LLM_ARCH_KIMI_K3:
@@ -1362,43 +1348,3 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
             return true;
     }
 }
-
-bool llama_arch_supports_save(llm_arch arch) {
-    switch (arch) {
-        case LLM_ARCH_SENSENOVA_U1:
-        case LLM_ARCH_GEMMA3N:
-        case LLM_ARCH_BITNET:
-        case LLM_ARCH_T5:
-        case LLM_ARCH_APERTUS:
-        case LLM_ARCH_STEP35:
-        case LLM_ARCH_CLEF: // the head tensors are not saved
-        case LLM_ARCH_VLLM_DECISION2:
-            return false;
-        default:
-            return true;
-    }
-}
-
-std::vector<llm_arch> llama_arch_all() {
-    return llm_arch_all();
-}
-
-const char * llama_arch_name(llm_arch arch) {
-    return llm_arch_name(arch);
-}
-
-llm_arch llama_arch_from_string(const std::string & name) {
-    return llm_arch_from_string(name);
-}
-
-std::string llama_arch_kv_name(llm_arch arch, llm_kv key) {
-    return LLM_KV(arch)(key);
-}
-
-bool llama_arch_supports_sm_tensor(llm_arch arch) {
-    return llm_arch_supports_sm_tensor(arch);
-}
-
-bool llama_arch_supports_mixed_batch(llm_arch arch) {
-    return llm_arch_supports_mixed_batch(arch);
-}

src/llama-arch.h

@@ -139,7 +139,6 @@ enum llm_arch {
     LLM_ARCH_OPENAI_MOE,
     LLM_ARCH_LFM2,
     LLM_ARCH_LFM2MOE,
-    LLM_ARCH_LFM3,
     LLM_ARCH_DREAM,
     LLM_ARCH_SMALLTHINKER,
     LLM_ARCH_LLADA,
@@ -680,10 +679,6 @@ enum llm_tensor {
     LLM_TENSOR_HC_FFN_DOWN,    // qwen4exp
     LLM_TENSOR_HC_FFN_UP,      // qwen4exp
     LLM_TENSOR_HC_FFN_INJECT,  // qwen4exp
-    LLM_TENSOR_HC_ATTN_DOWN_INJECT, // qwen4exp
-    LLM_TENSOR_HC_FFN_DOWN_INJECT,  // qwen4exp
-    LLM_TENSOR_ATTN_QKV_GATE,       // qwen4exp
-    LLM_TENSOR_SSM_BETA_ALPHA_CAT,  // qwen4exp
     LLM_TENSOR_PLE_KEY,        // qwen4exp
     LLM_TENSOR_PLE_VALUE,      // qwen4exp
     LLM_TENSOR_PLE_NORM_KEY,   // qwen4exp

src/llama-context.cpp

@@ -489,7 +489,6 @@ llama_context::llama_context(
             /*.swa_full  =*/ params.swa_full,
             /*.ctx_type  =*/ cparams.ctx_type,
             /*.mem_other =*/ llama_get_memory(cparams.ctx_other),
-            /*.no_alloc  =*/ model.no_alloc(),
         };
 
         memory.reset(model.create_memory(params_mem, cparams));
@@ -560,7 +559,7 @@ llama_context::llama_context(
         }
 
         if (cparams.moe_cache_size > 0) {
-            moe_cache = std::make_unique<llama_moe_cache>(model, model.no_alloc(), backend_ptrs, backend_buft, cparams.moe_cache_size);
+            moe_cache = std::make_unique<llama_moe_cache>(model, backend_ptrs, backend_buft, cparams.moe_cache_size);
         }
 
         sched_reserve();
@@ -588,7 +587,7 @@ llama_context::~llama_context() {
     synchronize();
 
     // when training, ggml_opt allocates extra buffers through the scheduler, so the sizes no longer match the expectation
-    if (!model.no_alloc() && !opt_ctx) {
+    if (!model.hparams.no_alloc && !opt_ctx) {
         for (size_t i = 0; i < backend_ptrs.size(); ++i) {
             ggml_backend_t             backend = backend_ptrs[i];
             ggml_backend_buffer_type_t buft    = backend_buft[i];
@@ -789,7 +788,7 @@ void llama_context::sched_reserve() {
     // reserve pp (prompt processing) graph first so that buffers are only allocated once
     {
         auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(),
-                model.no_alloc(), model.no_alloc() ? backend_buf_exp_size.data() : nullptr);
+                model.hparams.no_alloc, model.hparams.no_alloc ? backend_buf_exp_size.data() : nullptr);
         if (!gf) {
             if (cparams.pipeline_parallel) {
                 LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__);
@@ -816,7 +815,7 @@ void llama_context::sched_reserve() {
         n_splits_tg = n_splits_pp;
         n_nodes_tg  = n_nodes_pp;
     } else {
-        auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.no_alloc());
+        auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.hparams.no_alloc);
         if (!gf) {
             throw std::runtime_error("failed to allocate compute tg buffers");
         }
@@ -838,10 +837,10 @@ void llama_context::sched_reserve() {
                 // [TAG_RESERVE_DIAG_DECAY]
                 // the `inp_diag_decay` tensor size scales with `n_seq_tokens^2` which
                 // makes `n_seqs == 1` use more memory for the compute graph compared to `n_seqs > 1`
-                gf = graph_reserve(n_tokens, 1,      n_outputs_pp, mctx.get(), model.no_alloc());
+                gf = graph_reserve(n_tokens, 1,      n_outputs_pp, mctx.get(), model.hparams.no_alloc);
                 break;
             default:
-                gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.no_alloc());
+                gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc);
         };
 
         if (!gf) {
@@ -852,7 +851,7 @@ void llama_context::sched_reserve() {
     for (size_t i = 0; i < backend_ptrs.size(); ++i) {
         ggml_backend_t             backend = backend_ptrs[i];
         ggml_backend_buffer_type_t buft    = backend_buft[i];
-        if (!model.no_alloc()) {
+        if (!model.hparams.no_alloc) {
             backend_buf_exp_size[i] = ggml_backend_sched_get_buffer_size(sched.get(), backend);
         }
         if (backend_buf_exp_size[i] > 1) {
@@ -1614,9 +1613,8 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
     // eagle3/DFlash: features as encoder input, and non-draft paths fall back to model's input dim
     const int64_t n_vocab = model.vocab.n_tokens();
 
-    // note: during encode, we skip position continuity checks and output all tokens
-    //       a decoder without memory (see decode()) outputs the marked tokens, unless it extracts embeddings
-    if (!balloc->init(batch_inp, model.vocab, memory || cparams.embeddings)) {
+    // note: during encode, we always output all tokens and skip position continuity checks (output_all=true)
+    if (!balloc->init(batch_inp, model.vocab, true)) {
         LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__);
         return -1;
     }
@@ -1645,20 +1643,17 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
 
     n_queued_tokens += n_tokens;
 
-    const uint32_t n_outputs_all = balloc->get_n_outputs();
-
     // reserve output buffer
-    if (output_reserve(n_outputs_all) < n_outputs_all) {
-        LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_outputs_all);
+    if (output_reserve(n_tokens) < n_tokens) {
+        LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_tokens);
         return -2;
     };
 
-    const auto & out_ids = balloc->get_out_ids();
-    for (uint32_t i = 0; i < n_outputs_all; ++i) {
-        output_ids[out_ids[i]] = i;
+    for (uint32_t i = 0; i < n_tokens; ++i) {
+        output_ids[i] = i;
     }
 
-    n_outputs = n_outputs_all;
+    n_outputs = n_tokens;
 
     const auto causal_attn_org = cparams.causal_attn;
 
@@ -1691,7 +1686,7 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
         GGML_ASSERT(backend_res != nullptr);
         GGML_ASSERT(logits.data != nullptr);
 
-        ggml_backend_tensor_get_async(backend_res, t_logits, logits.data, 0, n_outputs*n_vocab*sizeof(float));
+        ggml_backend_tensor_get_async(backend_res, t_logits, logits.data, 0, n_tokens*n_vocab*sizeof(float));
     }
 
     // extract embeddings
@@ -1706,8 +1701,8 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
                     GGML_ASSERT(embd.data != nullptr);
                     const uint32_t n_embd_out = hparams.n_embd_out();
 
-                    GGML_ASSERT(n_outputs*n_embd_out <= (int64_t) embd.size);
-                    ggml_backend_tensor_get_async(backend_embd, t_embd, embd.data, 0, n_outputs*n_embd_out*sizeof(float));
+                    GGML_ASSERT(n_tokens*n_embd_out <= (int64_t) embd.size);
+                    ggml_backend_tensor_get_async(backend_embd, t_embd, embd.data, 0, n_tokens*n_embd_out*sizeof(float));
                 } break;
             case LLAMA_POOLING_TYPE_MEAN:
             case LLAMA_POOLING_TYPE_CLS:
@@ -1755,8 +1750,8 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
         GGML_ASSERT(backend_h != nullptr);
 
         const uint32_t n_embd = hparams.n_embd_out();
-        GGML_ASSERT(n_outputs*n_embd <= (int64_t) embd_nextn.size);
-        ggml_backend_tensor_get_async(backend_h, t_h_nextn, embd_nextn.data, 0, n_outputs*n_embd*sizeof(float));
+        GGML_ASSERT(n_tokens*n_embd <= (int64_t) embd_nextn.size);
+        ggml_backend_tensor_get_async(backend_h, t_h_nextn, embd_nextn.data, 0, n_tokens*n_embd*sizeof(float));
     }
 
     // TODO: hacky solution
@@ -3735,7 +3730,7 @@ llama_memory_breakdown llama_context::memory_breakdown() const {
             ret[buft].context += size;
         }
     }
-    if (model.no_alloc()) {
+    if (model.hparams.no_alloc) {
         for (size_t i = 0; i < backends.size(); ++i) {
             ggml_backend_t             backend = backends[i].get();
             ggml_backend_buffer_type_t buft    = ggml_backend_sched_get_buffer_type(sched.get(), backend);

src/llama-ext.h

@@ -5,36 +5,11 @@
 // try as much as possible to not include this header in the rest of the codebase
 
 #include "llama.h"
-#include "llama-arch.h"
 
 #include <cstdint>
 #include <map>
-#include <string>
 #include <vector>
 
-//
-// llama architecture queries
-//
-
-// all the LLM architectures known to the library
-LLAMA_API std::vector<llm_arch> llama_arch_all();
-
-// the canonical name of an architecture (e.g. "llama"), or "unknown"
-LLAMA_API const char * llama_arch_name(llm_arch arch);
-
-// the architecture with the given canonical name, or LLM_ARCH_UNKNOWN
-LLAMA_API llm_arch llama_arch_from_string(const std::string & name);
-
-// the arch-resolved GGUF key of a KV id, e.g. LLM_KV_CONTEXT_LENGTH -> "llama.context_length"
-LLAMA_API std::string llama_arch_kv_name(llm_arch arch, llm_kv key);
-
-LLAMA_API bool llama_arch_supports_sm_tensor(llm_arch arch);
-
-LLAMA_API bool llama_arch_supports_mixed_batch(llm_arch arch);
-
-// FIXME temporary function for better error messages
-LLAMA_API bool llama_arch_supports_save(llm_arch arch);
-
 // Reserve a new compute graph. It is valid until the next call to llama_graph_reserve.
 LLAMA_API struct ggml_cgraph * llama_graph_reserve(
         struct llama_context * ctx,

src/llama-graph.cpp

@@ -479,7 +479,7 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
     const int64_t n_kv     = self_kq_mask->ne[0];
     const int64_t n_tokens = ubatch->n_tokens;
 
-    const auto fill_mask = [&](auto * data, int64_t ne, int n_swa, llama_swa_type swa_type, bool causal) {
+    const auto fill_mask = [&](auto * data, int64_t ne, int n_swa, llama_swa_type swa_type) {
         using T = std::remove_reference_t<decltype(*data)>;
         std::fill(data, data + ne, llama_cast<T>(-INFINITY));
 
@@ -499,7 +499,7 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
                 }
 
                 // mask future tokens
-                if (causal && p0 > p1) {
+                if (cparams.causal_attn && p0 > p1) {
                     continue;
                 }
 
@@ -517,25 +517,21 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
         }
     };
 
-    // see llama_non_causal_type
-    const bool causal     = cparams.causal_attn;
-    const bool causal_swa = cparams.causal_attn || hparams.non_causal_type == LLAMA_NON_CAUSAL_TYPE_DENSE_ONLY;
-
     GGML_ASSERT(self_kq_mask);
     GGML_ASSERT(ggml_backend_buffer_is_host(self_kq_mask->buffer));
     if (self_kq_mask->type == GGML_TYPE_F16) {
-        fill_mask((ggml_fp16_t *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE, causal);
+        fill_mask((ggml_fp16_t *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE);
     } else {
-        fill_mask((float       *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE, causal);
+        fill_mask((float       *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE);
     }
 
     if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
         GGML_ASSERT(self_kq_mask_swa);
         GGML_ASSERT(ggml_backend_buffer_is_host(self_kq_mask_swa->buffer));
         if (self_kq_mask_swa->type == GGML_TYPE_F16) {
-            fill_mask((ggml_fp16_t *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type, causal_swa);
+            fill_mask((ggml_fp16_t *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type);
         } else {
-            fill_mask((float       *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type, causal_swa);
+            fill_mask((float       *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type);
         }
     }
 }
@@ -638,10 +634,7 @@ bool llm_graph_input_attn_kv_msa::can_reuse(const llm_graph_params & params) {
 void llm_graph_input_attn_k_dsa::set_input(const llama_ubatch * ubatch) {
     mctx->get_mla()->set_input_k_idxs(self_k_idxs_mla, ubatch);
 
-    // leave unallocated when the top-k layers use the indexer mask
-    if (self_kq_mask_mla->buffer) {
-        mctx->get_mla()->set_input_kq_mask(self_kq_mask_mla, ubatch, cparams.causal_attn);
-    }
+    mctx->get_mla()->set_input_kq_mask(self_kq_mask_mla, ubatch, cparams.causal_attn);
 
     mctx->get_lid()->set_input_k_idxs(self_k_idxs_lid, ubatch);
 
@@ -1225,10 +1218,7 @@ bool llm_graph_input_mem_hybrid::can_reuse(const llm_graph_params & params) {
 void llm_graph_input_mem_hybrid_k::set_input(const llama_ubatch * ubatch) {
     mctx->get_attn()->set_input_k_idxs(inp_attn->self_k_idxs, ubatch);
 
-    // leave the mask unallocated if all attention layers have their own (e.g. glm5-next DSA)
-    if (inp_attn->self_kq_mask->buffer) {
-        mctx->get_attn()->set_input_kq_mask(inp_attn->self_kq_mask, ubatch, cparams.causal_attn);
-    }
+    mctx->get_attn()->set_input_kq_mask(inp_attn->self_kq_mask, ubatch, cparams.causal_attn);
 
     const int64_t n_rs = mctx->get_recr()->get_n_rs();
 
@@ -2529,7 +2519,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
         //       ref: https://github.com/ggml-org/llama.cpp/pull/30656#issuecomment-6269597922
         llm_graph_amax_src1 amax_src1 = { nullptr, nullptr };
 
-        if (arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MELLUM || arch == LLM_ARCH_FLINT || arch == LLM_ARCH_LAGUNA) {
+        if (arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MELLUM || arch == LLM_ARCH_FLINT) {
             amax_src1 = build_amax_src1_f16(cur);
             cur = amax_src1.res;
         }
@@ -3402,10 +3392,20 @@ ggml_tensor * llm_graph_context::build_attn(
 ggml_tensor * llm_graph_context::build_fused_top_k_mask(
         ggml_tensor * kq_mask,
         ggml_tensor * top_k,
+        ggml_tensor * score,
+        ggml_tensor * blocks,
+        ggml_tensor * extra,
+        ggml_tensor * mask,
                 int   il) const {
-    top_k = ggml_reshape_4d(ctx0, top_k, top_k->ne[0], kq_mask->ne[1], kq_mask->ne[2], kq_mask->ne[3]);
+    const int64_t m1 = kq_mask->ne[1];
+    const int64_t m2 = kq_mask->ne[2];
+    const int64_t m3 = kq_mask->ne[3];
 
-    ggml_tensor * cur = ggml_top_k_mask(ctx0, kq_mask, top_k, nullptr, nullptr, nullptr);
+    auto rows = [&](ggml_tensor * t) {
+        return t ? ggml_reshape_4d(ctx0, t, t->ne[0], m1, m2, m3) : nullptr;
+    };
+
+    ggml_tensor * cur = ggml_top_k_mask(ctx0, mask ? mask : kq_mask, rows(top_k), rows(score), blocks, rows(extra));
     res->add_fused_node({LLM_FUSED_OP_TOP_K_MASK, cur, il});
 
     return cur;
@@ -3441,16 +3441,11 @@ ggml_tensor * llm_graph_context::build_attn(
         ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il));
     }
 
-    // top_k indexes the cells, and the indexer mask of the same type is built the same way
-    // let's use it, so when no other layer reads the MLA mask is not allocated
-    ggml_tensor * kq_mask = inp->get_kq_mask_mla();
-    if (inp->get_kq_mask_lid()->type == kq_mask->type) {
-        kq_mask = inp->get_kq_mask_lid();
-    }
+    const auto & kq_mask = inp->get_kq_mask_mla();
 
     ggml_tensor * kq_mask_top_k = nullptr;
     if (cparams.fused_top_k_mask) {
-        kq_mask_top_k = build_fused_top_k_mask(kq_mask, top_k, il);
+        kq_mask_top_k = build_fused_top_k_mask(kq_mask, top_k, nullptr, nullptr, nullptr, nullptr, il);
     } else {
         // prepare new kq mask - starts filled with -INFINITY
         ggml_tensor * kq_mask_all = ggml_fill(ctx0, kq_mask, -INFINITY);

src/llama-graph.h

@@ -1336,10 +1336,15 @@ struct llm_graph_context {
                   float   kq_scale,
                     int   il) const;
 
-    // ggml_top_k_mask with top_k viewed in the row layout of kq_mask, see ggml.h
+    // ggml_top_k_mask with top_k, score and extra viewed in the row layout of kq_mask, see ggml.h
+    // mask replaces kq_mask as the source of the kept values, it broadcasts to the rows
     ggml_tensor * build_fused_top_k_mask(
             ggml_tensor * kq_mask, // [n_kv, n_tokens/n_stream, 1, n_stream]
             ggml_tensor * top_k,   // [n_top, n_tokens]
+            ggml_tensor * score,   // [n_pick, n_tokens], can be null
+            ggml_tensor * blocks,  // [n_blk, n_pick], can be null
+            ggml_tensor * extra,   // [n_extra, n_tokens], can be null
+            ggml_tensor * mask,    // [n_kv], kq_mask if null
                     int   il) const;
 
     llm_graph_input_attn_kv_iswa * build_attn_inp_kv_iswa() const;

src/llama-hparams.h

@@ -34,7 +34,6 @@ enum llama_non_causal_type {
     LLAMA_NON_CAUSAL_TYPE_ALL      = 0, // all layers non-causal, SWA still applied (gemma 3, qwen-vl, ...)
     LLAMA_NON_CAUSAL_TYPE_SWA_ONLY = 1, // SWA layers non-causal, dense layers stay causal (gemma 4)
     LLAMA_NON_CAUSAL_TYPE_SWA_FULL = 2, // all layers non-causal, SWA not applied between tokens of the current ubatch (deepseek 4)
-    LLAMA_NON_CAUSAL_TYPE_DENSE_ONLY = 3, // dense layers non-causal, SWA layers stay causal (lfm3 d1)
 };
 
 // M-RoPE: which input position slot feeds each RoPE section
@@ -62,6 +61,7 @@ struct llama_hparams {
     //       for example: n_embd_out() vs n_embd_out_impl
 
     bool vocab_only;
+    bool no_alloc; // TODO: no_alloc is not a model hyperparameter; move it out of llama_hparams
     bool rope_finetuned;
     bool use_par_res;
     bool swin_norm;
@@ -358,8 +358,6 @@ struct llama_hparams {
     llama_token ple_embd_token   = LLAMA_TOKEN_NULL;
     // hash constant sets, stored back to back: 1 shared by all PLE layers (qwen4exp), or 1 per PLE layer in layer order (deepseek41)
     uint32_t ple_n_modules       = 1;
-    // deepseek41 Engram: the token that pads the n-gram history, an index into llama_model::engram_token_map
-    uint32_t engram_pad_id       = 0;
     // the file lists PLE layer indices, so this is never a per-layer gguf array and can hold one bit per layer
     std::bitset<LLAMA_MAX_LAYERS> is_ple_impl;
     // the hash multipliers reach ~2e13 and have to stay 64-bit

src/llama-kv-cache-dsa-iswa.cpp

@@ -13,7 +13,6 @@
 
 llama_kv_cache_dsa_iswa::llama_kv_cache_dsa_iswa(
         const llama_model & model,
-                     bool   no_alloc,
                 ggml_type   type_k,
                 ggml_type   type_v,
                      bool   v_trans,
@@ -23,6 +22,7 @@ llama_kv_cache_dsa_iswa::llama_kv_cache_dsa_iswa(
                  uint32_t   kv_size,
                  uint32_t   n_seq_max,
                  uint32_t   n_ubatch,
+                 uint32_t   n_pad,
     const layer_filter_cb & filter_mla,
     const layer_filter_cb & filter_lid,
     const  layer_reuse_cb & reuse) : unified(unified) {
@@ -63,15 +63,15 @@ llama_kv_cache_dsa_iswa::llama_kv_cache_dsa_iswa(
     LLAMA_LOG_INFO("%s: creating DSA KV cache, size = %u cells\n", __func__, size_dsa);
 
     kv_dsa = std::make_unique<llama_kv_cache_dsa>(
-            model, no_alloc, type_k, type_v,
-            v_trans, offload, unified, size_dsa, n_seq_max,
+            model, type_k, type_v,
+            v_trans, offload, unified, size_dsa, n_seq_max, n_pad,
             0, LLAMA_SWA_TYPE_NONE, filter_dsa, filter_lid, reuse);
 
     LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa);
 
     kv_swa = std::make_unique<llama_kv_cache>(
-            model, no_alloc, hparams, type_k, type_v,
-            v_trans, offload, unified, size_swa, n_seq_max,
+            model, hparams, type_k, type_v,
+            v_trans, offload, unified, size_swa, n_seq_max, n_pad,
             hparams.n_swa, hparams.swa_type, nullptr, filter_swa, reuse, nullptr);
 }
 

src/llama-kv-cache-dsa-iswa.h

@@ -14,7 +14,6 @@ class llama_kv_cache_dsa_iswa : public llama_memory_i {
 public:
     llama_kv_cache_dsa_iswa(
             const llama_model & model,
-                         bool   no_alloc,
                     ggml_type   type_k,
                     ggml_type   type_v,
                          bool   v_trans,
@@ -24,6 +23,7 @@ public:
                      uint32_t   kv_size,
                      uint32_t   n_seq_max,
                      uint32_t   n_ubatch,
+                     uint32_t   n_pad,
         const layer_filter_cb & filter_mla,
         const layer_filter_cb & filter_lid,
         const  layer_reuse_cb & reuse);

src/llama-kv-cache-dsa.cpp

@@ -13,7 +13,6 @@
 
 llama_kv_cache_dsa::llama_kv_cache_dsa(
         const llama_model & model,
-                     bool   no_alloc,
                 ggml_type   type_k,
                 ggml_type   type_v,
                      bool   v_trans,
@@ -21,6 +20,7 @@ llama_kv_cache_dsa::llama_kv_cache_dsa(
                      bool   unified,
                  uint32_t   kv_size,
                  uint32_t   n_seq_max,
+                 uint32_t   n_pad,
                  uint32_t   n_swa,
            llama_swa_type   swa_type,
     const layer_filter_cb & filter_mla,
@@ -31,8 +31,8 @@ llama_kv_cache_dsa::llama_kv_cache_dsa(
     LLAMA_LOG_INFO("%s: creating main KV cache, size = %u cells\n", __func__, kv_size);
 
     kv_mla = std::make_unique<llama_kv_cache>(
-            model, no_alloc, model.hparams, type_k, type_v,
-            v_trans, offload, unified, kv_size, n_seq_max,
+            model, model.hparams, type_k, type_v,
+            v_trans, offload, unified, kv_size, n_seq_max, n_pad,
             n_swa, swa_type, nullptr, filter_mla, reuse, nullptr);
 
     // we use llama_kv_cache for caching indexer keys
@@ -48,8 +48,8 @@ llama_kv_cache_dsa::llama_kv_cache_dsa(
     LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size);
 
     kv_lid = std::make_unique<llama_kv_cache>(
-            model, no_alloc, hparams_lid, type_k, type_v,
-            v_trans, offload, unified, kv_size, n_seq_max,
+            model, hparams_lid, type_k, type_v,
+            v_trans, offload, unified, kv_size, n_seq_max, n_pad,
             n_swa, swa_type, nullptr, filter_lid, reuse, nullptr);
 }
 

src/llama-kv-cache-dsa.h

@@ -16,7 +16,6 @@ class llama_kv_cache_dsa : public llama_memory_i {
 public:
     llama_kv_cache_dsa(
             const llama_model & model,
-                         bool   no_alloc,
                     ggml_type   type_k,
                     ggml_type   type_v,
                          bool   v_trans,
@@ -24,6 +23,7 @@ public:
                          bool   unified,
                      uint32_t   kv_size,
                      uint32_t   n_seq_max,
+                     uint32_t   n_pad,
                      uint32_t   n_swa,
                llama_swa_type   swa_type,
         const layer_filter_cb & filter_mla,

src/llama-kv-cache-dsv4.cpp

@@ -428,11 +428,8 @@ static void dsv4_state_read_k_cache(
         dsv4_state_read_tensor_streams(io, kv->get_k_storage(il), kv_size, n_rows_ref, s0, ns);
     }
 
-    if (n_rows_ref == 0) {
-        return;
-    }
-
-    // refill the mirrors from their sources; a mirror gets the same rows as its source, so a whole copy is exact
+    // refill the mirrors from their source rows just read
+    std::vector<uint8_t> buf;
     for (uint32_t il : kv->get_layer_ids()) {
         const int32_t src = kv_src[il];
         if (src == (int32_t) il) {
@@ -440,11 +437,20 @@ static void dsv4_state_read_k_cache(
         }
         ggml_tensor * t_src = kv->get_k_storage(src);
         ggml_tensor * t_dst = kv->get_k_storage(il);
-        if (t_src->type != t_dst->type || !ggml_are_same_shape(t_src, t_dst)) {
-            throw std::runtime_error("DSV4 K-cache mirror layout mismatch");
-        }
+        GGML_ASSERT(t_src->type == t_dst->type && t_src->ne[0] == t_dst->ne[0]);
 
-        ggml_backend_tensor_copy(t_src, t_dst);
+        const size_t row_size      = ggml_row_size(t_src->type, t_src->ne[0]);
+        const size_t stream_stride = (size_t) kv_size*row_size;
+        const size_t size          = (size_t) n_rows_ref*row_size;
+        if (size == 0) {
+            continue;
+        }
+        buf.resize(size);
+        for (uint32_t s = 0; s < ns; ++s) {
+            const size_t offset = (size_t) (s0 + s)*stream_stride;
+            ggml_backend_tensor_get(t_src, buf.data(), offset, size);
+            ggml_backend_tensor_set(t_dst, buf.data(), offset, size);
+        }
     }
 }
 
@@ -931,24 +937,21 @@ static void dsv4_make_k_only(llama_hparams & hparams) {
 
 llama_dsv4_comp_state::llama_dsv4_comp_state(
         const llama_model & model,
-                     bool   no_alloc,
-                     bool   offload,
-                     bool   unified,
-                 uint32_t   n_seq_max,
-                 uint32_t   ratio,
-                 uint32_t   state_size,
-                 uint32_t   n_embd_state,
-                 uint32_t   n_rs_seq,
-               const char * name,
+                bool        offload,
+                bool        unified,
+            uint32_t        n_seq_max,
+            uint32_t        ratio,
+            uint32_t        state_size,
+            uint32_t        n_embd_state,
+            uint32_t        n_rs_seq,
+        const char    * name,
         const llama_memory_i::layer_filter_cb & filter) :
+    hparams(model.hparams),
     ratio(ratio),
     state_size(state_size),
     n_embd_state(n_embd_state),
     n_stream(unified ? 1 : n_seq_max),
-    n_rs_seq(n_rs_seq),
-    ctxs_bufs(no_alloc) {
-    const llama_hparams & hparams = model.hparams;
-
+    n_rs_seq(n_rs_seq) {
     struct ggml_backend_buft_comparator {
         bool operator()(const ggml_backend_buffer_type_t & lhs, const ggml_backend_buffer_type_t & rhs) const {
             return strcmp(ggml_backend_buft_name(lhs), ggml_backend_buft_name(rhs)) < 0;
@@ -1023,10 +1026,25 @@ llama_dsv4_comp_state::llama_dsv4_comp_state(
     }
 
     for (auto & [buft, ctx] : ctx_map) {
-        ggml_backend_buffer_t buf = ctxs_bufs.emplace(std::move(ctx), buft, "DSV4 compressor state");
+        ggml_backend_buffer_t buf;
+        if (hparams.no_alloc) {
+            buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer
+            for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) {
+                t->buffer = buf; // set dummy buffer for DSV4 state so that the backend scheduler won't try to allocate it
+            }
+        } else {
+            buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer
+        }
+        if (!buf) {
+            throw std::runtime_error("failed to allocate buffer for DSV4 compressor state");
+        }
+
+        ggml_backend_buffer_clear(buf, 0);
 
         LLAMA_LOG_INFO("%s: %10s DSV4 %s state buffer size = %8.2f MiB\n",
                 __func__, ggml_backend_buffer_name(buf), name, ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
+
+        ctxs_bufs.emplace_back(std::move(ctx), buf);
     }
 
     LLAMA_LOG_INFO("%s: %s ratio = %u, state = %u x %u, streams = %u, rs_seq = %u, layers = %zu, size = %7.2f MiB\n",
@@ -1107,7 +1125,18 @@ uint32_t llama_dsv4_comp_state::get_n_rows() const {
 }
 
 std::map<ggml_backend_buffer_type_t, size_t> llama_dsv4_comp_state::memory_breakdown() const {
-    return ctxs_bufs.memory_breakdown();
+    std::map<ggml_backend_buffer_type_t, size_t> ret;
+    for (const auto & [ctx, buf] : ctxs_bufs) {
+        ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
+
+        if (hparams.no_alloc) {
+            GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr);
+            ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
+        } else {
+            ret[buft] += ggml_backend_buffer_get_size(buf.get());
+        }
+    }
+    return ret;
 }
 
 void llama_dsv4_comp_state::state_write(
@@ -1242,7 +1271,6 @@ size_t llama_dsv4_comp_state::total_size() const {
 
 llama_kv_cache_dsv4::llama_kv_cache_dsv4(
         const llama_model & model,
-                     bool   no_alloc,
                 ggml_type   type_k,
                 ggml_type   type_v,
                      bool   v_trans,
@@ -1252,6 +1280,7 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
                  uint32_t   kv_size,
                  uint32_t   n_seq_max,
                  uint32_t   n_ubatch,
+                 uint32_t   n_pad,
                  uint32_t   n_rs_seq,
     const layer_filter_cb & filter,
     const  layer_reuse_cb & reuse) :
@@ -1286,8 +1315,8 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
     dsv4_make_k_only(hparams_raw);
 
     kv_raw = std::make_unique<llama_kv_cache_iswa>(
-            model, no_alloc, hparams_raw, type_k, type_v,
-            v_trans, offload, swa_full, unified_raw, kv_size, n_seq_max, n_ubatch,
+            model, hparams_raw, type_k, type_v,
+            v_trans, offload, swa_full, unified_raw, kv_size, n_seq_max, n_ubatch, n_pad,
             nullptr, filter_raw, reuse, nullptr);
 
     // V4.1: CSA slot = ratio 2, HCA slot = ratio 1, no overlap and no indexer compressor; the index K is stored
@@ -1323,10 +1352,10 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
     // crosses devices. no mirrors without offload
     const uint32_t n_layer = model.hparams.n_layer();
     comp_read.assign(n_layer, -1);
-    comp_il_src.resize(n_layer);
+    comp_kv_src.resize(n_layer);
     for (uint32_t il = 0; il < n_layer; ++il) {
         const int32_t src = is_v41 ? model.hparams.dsv41_kv_src[il] : (int32_t) il;
-        comp_il_src[il] = src >= 0 ? src : (int32_t) il;
+        comp_kv_src[il] = src >= 0 ? src : (int32_t) il;
         if (model.hparams.dsv4_compress_ratios[il] == 0) {
             continue;
         }
@@ -1337,7 +1366,7 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
         // the first layer of this source on this device holds the mirror
         comp_read[il] = il;
         for (int32_t j = src + 1; j < (int32_t) il; ++j) {
-            if (comp_read[j] == j && comp_il_src[j] == src && model.dev_layer(j) == model.dev_layer(il)) {
+            if (comp_read[j] == j && comp_kv_src[j] == src && model.dev_layer(j) == model.dev_layer(il)) {
                 comp_read[il] = j;
                 break;
             }
@@ -1390,24 +1419,24 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
             __func__, dsv4_comp_size(kv_size, ratio_csa));
 
     kv_csa = std::make_unique<llama_kv_cache>(
-            model, no_alloc, hparams_csa, type_k, type_v,
-            v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max,
+            model, hparams_csa, type_k, type_v,
+            v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max, n_pad,
             0, LLAMA_SWA_TYPE_NONE, nullptr, filter_csa, nullptr, nullptr);
 
     LLAMA_LOG_INFO("%s: creating DSV4 HCA compressed KV cache, size = %u cells\n",
             __func__, dsv4_comp_size(kv_size, ratio_hca));
 
     kv_hca = std::make_unique<llama_kv_cache>(
-            model, no_alloc, hparams_hca, type_k, type_v,
-            v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_hca), 256u), n_seq_max,
+            model, hparams_hca, type_k, type_v,
+            v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_hca), 256u), n_seq_max, n_pad,
             0, LLAMA_SWA_TYPE_NONE, nullptr, filter_hca, nullptr, nullptr);
 
     LLAMA_LOG_INFO("%s: creating DSV4 lightning-indexer KV cache, size = %u cells\n",
             __func__, dsv4_comp_size(kv_size, ratio_csa));
 
     kv_lid = std::make_unique<llama_kv_cache>(
-            model, no_alloc, hparams_lid, type_k, type_v,
-            v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max,
+            model, hparams_lid, type_k, type_v,
+            v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max, n_pad,
             0, LLAMA_SWA_TYPE_NONE, nullptr, filter_lid, nullptr, nullptr);
 
     LLAMA_LOG_INFO("%s: creating DSV4 CSA compressor state\n", __func__);
@@ -1416,19 +1445,19 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
     const uint32_t csa_state_embd = is_v41 ? model.hparams.n_embd_head_k() : 2*model.hparams.n_embd_head_k();
 
     csa_state = std::make_unique<llama_dsv4_comp_state>(
-            model, no_alloc, offload, unified_compressed, n_seq_max, ratio_csa, csa_state_size,
+            model, offload, unified_compressed, n_seq_max, ratio_csa, csa_state_size,
             csa_state_embd, n_rs_seq, "csa", filter_csa_state);
 
     LLAMA_LOG_INFO("%s: creating DSV4 HCA compressor state\n", __func__);
 
     hca_state = std::make_unique<llama_dsv4_comp_state>(
-            model, no_alloc, offload, unified_compressed, n_seq_max, ratio_hca, ratio_hca,
+            model, offload, unified_compressed, n_seq_max, ratio_hca, ratio_hca,
             model.hparams.n_embd_head_k(), n_rs_seq, "hca", filter_hca_state);
 
     LLAMA_LOG_INFO("%s: creating DSV4 lightning-indexer compressor state\n", __func__);
 
     lid_state = std::make_unique<llama_dsv4_comp_state>(
-            model, no_alloc, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO,
+            model, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO,
             2*model.hparams.indexer_head_size, n_rs_seq, "lid", filter_lid);
 
     // DSV4 attention reads compressed-K / compressor-state rows that the current
@@ -1782,9 +1811,9 @@ void llama_kv_cache_dsv4::state_write(llama_io_write_i & io, llama_seq_id seq_id
         const uint32_t n_rows_lid = seq_id >= 0 ?
             dsv4_state_n_used_k_rows(pos_max, lid_state->get_ratio(), kv_lid->get_size()) : kv_lid->get_size();
 
-        dsv4_state_write_k_cache(io, kv_csa.get(), seq_id, flags, n_rows_csa, comp_il_src);
-        dsv4_state_write_k_cache(io, kv_hca.get(), seq_id, flags, n_rows_hca, comp_il_src);
-        dsv4_state_write_k_cache(io, kv_lid.get(), seq_id, flags, n_rows_lid, comp_il_src);
+        dsv4_state_write_k_cache(io, kv_csa.get(), seq_id, flags, n_rows_csa, comp_kv_src);
+        dsv4_state_write_k_cache(io, kv_hca.get(), seq_id, flags, n_rows_hca, comp_kv_src);
+        dsv4_state_write_k_cache(io, kv_lid.get(), seq_id, flags, n_rows_lid, comp_kv_src);
     }
 
     csa_state->state_write(io, seq_id, flags, rs_idx);
@@ -1822,9 +1851,9 @@ void llama_kv_cache_dsv4::state_read(llama_io_read_i & io, llama_seq_id seq_id,
     if (!partial_only) {
         clear_compressed(seq_id, true);
 
-        dsv4_state_read_k_cache(io, kv_csa.get(), seq_id, flags, comp_il_src);
-        dsv4_state_read_k_cache(io, kv_hca.get(), seq_id, flags, comp_il_src);
-        dsv4_state_read_k_cache(io, kv_lid.get(), seq_id, flags, comp_il_src);
+        dsv4_state_read_k_cache(io, kv_csa.get(), seq_id, flags, comp_kv_src);
+        dsv4_state_read_k_cache(io, kv_hca.get(), seq_id, flags, comp_kv_src);
+        dsv4_state_read_k_cache(io, kv_lid.get(), seq_id, flags, comp_kv_src);
     }
 
     csa_state->state_read(io, seq_id, flags);

src/llama-kv-cache-dsv4.h

@@ -16,15 +16,14 @@ public:
 
     llama_dsv4_comp_state(
             const llama_model & model,
-                         bool   no_alloc,
-                         bool   offload,
-                         bool   unified,
-                     uint32_t   n_seq_max,
-                     uint32_t   ratio,
-                     uint32_t   state_size,
-                     uint32_t   n_embd_state,
-                     uint32_t   n_rs_seq,
-                   const char * name,
+            bool            offload,
+            bool            unified,
+            uint32_t        n_seq_max,
+            uint32_t        ratio,
+            uint32_t        state_size,
+            uint32_t        n_embd_state,
+            uint32_t        n_rs_seq,
+            const char    * name,
         const llama_memory_i::layer_filter_cb & filter);
 
     void clear(llama_seq_id seq_id, bool data);
@@ -62,13 +61,15 @@ private:
         std::vector<ggml_tensor *> score_stream;
     };
 
+    const llama_hparams & hparams;
+
     const uint32_t ratio;
     const uint32_t state_size;
     const uint32_t n_embd_state;
     const uint32_t n_stream;
     const uint32_t n_rs_seq;
 
-    llama_memory_bufs ctxs_bufs;
+    std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;
 
     std::vector<layer> layers;
 
@@ -91,7 +92,6 @@ class llama_kv_cache_dsv4 : public llama_memory_i {
 public:
     llama_kv_cache_dsv4(
             const llama_model & model,
-                         bool   no_alloc,
                     ggml_type   type_k,
                     ggml_type   type_v,
                          bool   v_trans,
@@ -101,6 +101,7 @@ public:
                      uint32_t   kv_size,
                      uint32_t   n_seq_max,
                      uint32_t   n_ubatch,
+                     uint32_t   n_pad,
                      uint32_t   n_rs_seq,
         const layer_filter_cb & filter,
         const  layer_reuse_cb & reuse);
@@ -169,7 +170,7 @@ private:
     std::vector<uint32_t> rs_idx;
 
     // [n_layer] storage layer -> layer whose rows it holds (itself, or the source of a mirror)
-    std::vector<int32_t> comp_il_src;
+    std::vector<int32_t> comp_kv_src;
 
     // [n_layer] layer -> layer whose compressed-KV storage it reads (itself, its V4.1 kv source or a mirror), -1 = none
     std::vector<int32_t> comp_read;

src/llama-kv-cache-iswa.cpp

@@ -13,7 +13,6 @@
 
 llama_kv_cache_iswa::llama_kv_cache_iswa(
         const llama_model & model,
-                     bool   no_alloc,
                 ggml_type   type_k,
                 ggml_type   type_v,
                      bool   v_trans,
@@ -23,18 +22,18 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
                  uint32_t   kv_size,
                  uint32_t   n_seq_max,
                  uint32_t   n_ubatch,
+                 uint32_t   n_pad,
            llama_memory_t   mem_other,
     const layer_filter_cb & filter,
     const  layer_reuse_cb & reuse,
     const  layer_share_cb & share) :
-    llama_kv_cache_iswa(model, no_alloc, model.hparams, type_k, type_v, v_trans, offload, swa_full, unified,
-            kv_size, n_seq_max, n_ubatch, mem_other, filter, reuse, share) {
+    llama_kv_cache_iswa(model, model.hparams, type_k, type_v, v_trans, offload, swa_full, unified,
+            kv_size, n_seq_max, n_ubatch, n_pad, mem_other, filter, reuse, share) {
 }
 
 llama_kv_cache_iswa::llama_kv_cache_iswa(
         const llama_model & model,
-                     bool   no_alloc,
-      const llama_hparams & hparams,
+        const llama_hparams & hparams,
                 ggml_type   type_k,
                 ggml_type   type_v,
                      bool   v_trans,
@@ -44,6 +43,7 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
                  uint32_t   kv_size,
                  uint32_t   n_seq_max,
                  uint32_t   n_ubatch,
+                 uint32_t   n_pad,
            llama_memory_t   mem_other,
     const layer_filter_cb & filter,
     const  layer_reuse_cb & reuse,
@@ -93,15 +93,15 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
     }
 
     kv_base = std::make_unique<llama_kv_cache>(
-            model, no_alloc, hparams, type_k, type_v,
-            v_trans, offload, unified, size_base, n_seq_max,
+            model, hparams, type_k, type_v,
+            v_trans, offload, unified, size_base, n_seq_max, n_pad,
             0, LLAMA_SWA_TYPE_NONE, mem_other_base, filter_base, reuse, share);
 
     LLAMA_LOG_INFO("%s: creating     SWA KV cache, size = %u cells\n", __func__, size_swa);
 
     kv_swa = std::make_unique<llama_kv_cache>(
-            model, no_alloc, hparams, type_k, type_v,
-            v_trans, offload, unified, size_swa, n_seq_max,
+            model, hparams, type_k, type_v,
+            v_trans, offload, unified, size_swa, n_seq_max, n_pad,
             hparams.n_swa, hparams.swa_type, mem_other_swa, filter_swa, reuse, share);
 }
 

src/llama-kv-cache-iswa.h

@@ -15,7 +15,6 @@ class llama_kv_cache_iswa : public llama_memory_i {
 public:
     llama_kv_cache_iswa(
             const llama_model & model,
-                         bool   no_alloc,
                     ggml_type   type_k,
                     ggml_type   type_v,
                          bool   v_trans,
@@ -25,6 +24,7 @@ public:
                      uint32_t   kv_size,
                      uint32_t   n_seq_max,
                      uint32_t   n_ubatch,
+                     uint32_t   n_pad,
                llama_memory_t   mem_other,
         const layer_filter_cb & filter,
         const  layer_reuse_cb & reuse,
@@ -32,8 +32,7 @@ public:
 
     llama_kv_cache_iswa(
             const llama_model & model,
-                         bool   no_alloc,
-          const llama_hparams & hparams,
+            const llama_hparams & hparams,
                     ggml_type   type_k,
                     ggml_type   type_v,
                          bool   v_trans,
@@ -43,6 +42,7 @@ public:
                      uint32_t   kv_size,
                      uint32_t   n_seq_max,
                      uint32_t   n_ubatch,
+                     uint32_t   n_pad,
                llama_memory_t   mem_other,
         const layer_filter_cb & filter,
         const  layer_reuse_cb & reuse,

src/llama-kv-cache-msa.cpp

@@ -12,7 +12,6 @@
 
 llama_kv_cache_msa::llama_kv_cache_msa(
         const llama_model & model,
-                     bool   no_alloc,
                 ggml_type   type_k,
                 ggml_type   type_v,
                      bool   v_trans,
@@ -20,20 +19,21 @@ llama_kv_cache_msa::llama_kv_cache_msa(
                      bool   unified,
                  uint32_t   kv_size,
                  uint32_t   n_seq_max,
+                 uint32_t   n_pad,
                  uint32_t   n_swa,
            llama_swa_type   swa_type,
     const layer_filter_cb & filter,
     const layer_filter_cb & filter_idx,
     const  layer_reuse_cb & reuse) :
     hparams_idx(model.hparams),
-    n_stream(unified ? 1 : n_seq_max), n_seq_max(n_seq_max),
+    n_stream(unified ? 1 : n_seq_max), n_seq_max(n_seq_max), n_pad(n_pad),
     n_swa(n_swa), swa_type(swa_type) {
 
     LLAMA_LOG_INFO("%s: creating main KV cache, size = %u cells\n", __func__, kv_size);
 
     kv_base = std::make_unique<llama_kv_cache>(
-            model, no_alloc, model.hparams, type_k, type_v,
-            v_trans, offload, unified, kv_size, n_seq_max,
+            model, model.hparams, type_k, type_v,
+            v_trans, offload, unified, kv_size, n_seq_max, n_pad,
             n_swa, swa_type, nullptr, filter, reuse, nullptr);
 
     // the MSA indexer uses a single key head per layer
@@ -44,8 +44,8 @@ llama_kv_cache_msa::llama_kv_cache_msa(
     LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size);
 
     kv_idx = std::make_unique<llama_kv_cache>(
-            model, no_alloc, hparams_idx, type_k, type_v,
-            v_trans, offload, unified, kv_size, n_seq_max,
+            model, hparams_idx, type_k, type_v,
+            v_trans, offload, unified, kv_size, n_seq_max, n_pad,
             n_swa, swa_type, nullptr, filter_idx, reuse, nullptr);
 }
 
@@ -261,7 +261,7 @@ const llama_kv_cache_context * llama_kv_cache_msa_context::get_idx() const {
 
 uint32_t llama_kv_cache_msa_context::get_n_pos() const {
     // pad the value so that the graph remains constant across batches and can be reused
-    const uint32_t n_pad = 256;
+    const uint32_t n_pad_cur = std::max(kv->get_n_pad(), 256u);
 
     llama_pos pos_max = -1;
 
@@ -269,7 +269,7 @@ uint32_t llama_kv_cache_msa_context::get_n_pos() const {
         pos_max = std::max(pos_max, kv->seq_pos_max(seq_id));
     }
 
-    return std::max(n_pad, GGML_PAD((uint32_t) (pos_max + 1), n_pad));
+    return std::max(n_pad_cur, GGML_PAD((uint32_t) (pos_max + 1), n_pad_cur));
 }
 
 void llama_kv_cache_msa_context::set_input_cell_pos(ggml_tensor * dst, const llama_ubatch * ubatch, int32_t div) const {

src/llama-kv-cache-msa.h

@@ -15,7 +15,6 @@ class llama_kv_cache_msa : public llama_memory_i {
 public:
     llama_kv_cache_msa(
             const llama_model & model,
-                         bool   no_alloc,
                     ggml_type   type_k,
                     ggml_type   type_v,
                          bool   v_trans,
@@ -23,6 +22,7 @@ public:
                          bool   unified,
                      uint32_t   kv_size,
                      uint32_t   n_seq_max,
+                     uint32_t   n_pad,
                      uint32_t   n_swa,
                llama_swa_type   swa_type,
         const layer_filter_cb & filter,
@@ -67,6 +67,7 @@ public:
     llama_kv_cache * get_base() const;
     llama_kv_cache * get_idx () const;
 
+    uint32_t       get_n_pad()    const { return n_pad; }
     uint32_t       get_n_seq_max() const { return n_seq_max; }
     uint32_t       get_n_swa()    const { return n_swa; }
     llama_swa_type get_swa_type() const { return swa_type; }
@@ -77,6 +78,7 @@ private:
 
     const uint32_t n_stream  = 1;
     const uint32_t n_seq_max = 1;
+    const uint32_t n_pad     = 1;
 
     const uint32_t       n_swa    = 0;
     const llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE;

src/llama-kv-cache.cpp

@@ -64,8 +64,7 @@ static void ggml_gen_hadamard(ggml_tensor * tensor) {
 
 llama_kv_cache::llama_kv_cache(
         const llama_model & model,
-                     bool   no_alloc,
-      const llama_hparams & hparams,
+        const llama_hparams & hparams,
                 ggml_type   type_k,
                 ggml_type   type_v,
                      bool   v_trans,
@@ -73,16 +72,16 @@ llama_kv_cache::llama_kv_cache(
                      bool   unified,
                  uint32_t   kv_size,
                  uint32_t   n_seq_max,
+                 uint32_t   n_pad,
                  uint32_t   n_swa,
            llama_swa_type   swa_type,
            llama_memory_t   mem_other,
     const layer_filter_cb & filter,
     const  layer_reuse_cb & reuse,
     const  layer_share_cb & share,
-               const char * name_tag) :
+             const char *   name_tag) :
     model(model), hparams(hparams), v_trans(v_trans),
-    n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_swa(n_swa), swa_type(swa_type),
-    ctxs_bufs(no_alloc),
+    n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_pad(n_pad), n_swa(n_swa), swa_type(swa_type),
     other(static_cast<llama_kv_cache *>(mem_other)),
     v_cells_impl(other ? other->v_cells_impl : std::make_shared<llama_kv_cells_vec>()),
     v_cells(*v_cells_impl) {
@@ -98,6 +97,8 @@ llama_kv_cache::llama_kv_cache(
         }
     }
 
+    GGML_ASSERT(kv_size % n_pad == 0);
+
     const uint32_t n_layer = hparams.n_layer_all;
 
     // define a comparator for the buft -> ctx map to ensure that the order is well-defined:
@@ -272,11 +273,25 @@ llama_kv_cache::llama_kv_cache(
         }
     }
 
-    // allocate the tensors and initialize the buffers to avoid NaNs in the padding
+    // allocate tensors and initialize the buffers to avoid NaNs in the padding
     for (auto & [buft, ctx] : ctx_map) {
-        ggml_backend_buffer_t buf = ctxs_bufs.emplace(std::move(ctx), buft, "kv cache");
+        ggml_backend_buffer_t buf;
+        if (hparams.no_alloc) {
+            buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer
+            for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) {
+                t->buffer = buf; // set dummy buffer for KV cache so that the backend scheduler won't try to allocate it
+            }
+        } else {
+            buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer
+        }
+        if (!buf) {
+            throw std::runtime_error("failed to allocate buffer for kv cache");
+        }
 
         LLAMA_LOG_INFO("%s: %10s KV buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
+
+        ggml_backend_buffer_clear(buf, 0);
+        ctxs_bufs.emplace_back(std::move(ctx), buf);
     }
 
     {
@@ -690,7 +705,20 @@ llama_pos llama_kv_cache::seq_pos_max(llama_seq_id seq_id) const {
 }
 
 std::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache::memory_breakdown() const {
-    return ctxs_bufs.memory_breakdown();
+    std::map<ggml_backend_buffer_type_t, size_t> ret;
+    for (const auto & [ctx, buf] : ctxs_bufs) {
+        ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
+
+        if (hparams.no_alloc) {
+            GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr);
+            ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
+        } else {
+            // GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) != nullptr); // multi_buffer does not have a defined base
+            ret[buft] += ggml_backend_buffer_get_size(buf.get());
+        }
+    }
+
+    return ret;
 }
 
 llama_memory_context_ptr llama_kv_cache::init_batch(
@@ -1257,12 +1285,12 @@ uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const {
 
     // pad the n_kv value so that the graph remains constant across batches and can be reused
     // note: this also helps some backends with performance (f.ex https://github.com/ggml-org/llama.cpp/pull/16812#issuecomment-3455112220)
-    const uint32_t n_pad = 256;
+    const uint32_t n_pad_cur = std::max(n_pad, 256u);
 
     for (uint32_t s = 0; s < sinfo.n_stream(); ++s) {
         const auto & cells = v_cells[sinfo.strm[s]];
 
-        result = std::max(std::min(cells.size(), std::max(n_pad, GGML_PAD(cells.used_max_p1(), n_pad))), result);
+        result = std::max(std::min(cells.size(), std::max(n_pad_cur, GGML_PAD(cells.used_max_p1(), n_pad_cur))), result);
     }
 
     return result;
@@ -1749,9 +1777,6 @@ void llama_kv_cache::set_input_kq_mask(ggml_tensor * dst, const llama_ubatch * u
     if (!causal_attn && hparams.non_causal_type == LLAMA_NON_CAUSAL_TYPE_SWA_ONLY) {
         causal_attn = swa_type == LLAMA_SWA_TYPE_NONE;
     }
-    if (!causal_attn && hparams.non_causal_type == LLAMA_NON_CAUSAL_TYPE_DENSE_ONLY) {
-        causal_attn = swa_type != LLAMA_SWA_TYPE_NONE;
-    }
 
     //const int64_t t_start = ggml_time_us();
 

src/llama-kv-cache.h

@@ -98,7 +98,6 @@ public:
     //       likely through `struct llama_memory_params`
     llama_kv_cache(
             const llama_model & model,
-                         bool   no_alloc,
           const llama_hparams & hparams,
                     ggml_type   type_k,
                     ggml_type   type_v,
@@ -107,6 +106,7 @@ public:
                          bool   unified,
                      uint32_t   kv_size,
                      uint32_t   n_seq_max,
+                     uint32_t   n_pad,
                      uint32_t   n_swa,
                llama_swa_type   swa_type,
                llama_memory_t   mem_other,
@@ -271,6 +271,9 @@ private:
     const uint32_t n_seq_max = 1;
     const uint32_t n_stream  = 1;
 
+    // required padding
+    const uint32_t n_pad = 1;
+
     // SWA
     const uint32_t n_swa = 0;
 
@@ -293,7 +296,7 @@ private:
     const llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE;
 
     // ggml contexts for the KV cache along with the allocated backend buffers:
-    llama_memory_bufs ctxs_bufs;
+    std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;
 
     // the current index from where we start searching for a free slot in the ring buffer of KV cells (see find_slot())
     // note: this is not part of the KV state and it's only used to speed-up the find_slot() method

src/llama-memory-hybrid-idx.cpp

@@ -1,16 +1,20 @@
 #include "llama-memory-hybrid-idx.h"
 
+#include <algorithm>
+#include <cmath>
+#include <type_traits>
+
 #include "llama-impl.h"
 #include "llama-batch.h"
 #include "llama-io.h"
 #include "llama-model.h"
 
+
 #include <algorithm>
 #include <cassert>
 #include <cmath>
 #include <iterator>
 #include <stdexcept>
-#include <type_traits>
 
 //
 // llama_memory_hybrid_idx
@@ -18,12 +22,12 @@
 
 llama_memory_hybrid_idx::llama_memory_hybrid_idx(
         const llama_model & model,
-                     bool   no_alloc,
                             /* attn */
                 ggml_type   type_k,
                 ggml_type   type_v,
                      bool   v_trans,
                  uint32_t   kv_size,
+                 uint32_t   n_pad,
                  uint32_t   n_swa,
            llama_swa_type   swa_type,
                             /* recurrent */
@@ -41,8 +45,7 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx(
     const layer_filter_cb & filter_idx) :
     llama_memory_hybrid(
         model,
-        no_alloc,
-        type_k, type_v, v_trans, kv_size, n_swa, swa_type,
+        type_k, type_v, v_trans, kv_size, n_pad, n_swa, swa_type,
         type_r, type_s, rs_size,
         n_seq_max, n_rs_seq, offload, unified,
         filter_attn, filter_recr),
@@ -65,8 +68,8 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx(
         LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size);
 
         return new llama_kv_cache(
-            model, no_alloc, hparams_idx, type_k, type_v, v_trans, offload, unified,
-            kv_size, n_seq_max, n_swa, swa_type,
+            model, hparams_idx, type_k, type_v, v_trans, offload, unified,
+            kv_size, n_seq_max, n_pad, n_swa, swa_type,
             nullptr, filter_idx, nullptr, nullptr, "idx_");
     }()) {}
 

src/llama-memory-hybrid-idx.h

@@ -18,12 +18,12 @@ class llama_memory_hybrid_idx : public llama_memory_hybrid {
 public:
     llama_memory_hybrid_idx(
         const llama_model & model,
-                     bool   no_alloc,
                             /* attn */
                 ggml_type   type_k,
                 ggml_type   type_v,
                      bool   v_trans,
                  uint32_t   kv_size,
+                 uint32_t   n_pad,
                  uint32_t   n_swa,
            llama_swa_type   swa_type,
                             /* recurrent */

src/llama-memory-hybrid-iswa.cpp

@@ -10,7 +10,6 @@
 
 llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
         const llama_model & model,
-                      bool   no_alloc,
                             /* attn */
                 ggml_type   type_k,
                 ggml_type   type_v,
@@ -18,6 +17,7 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
                      bool   swa_full,
                  uint32_t   kv_size,
                  uint32_t   n_ubatch,
+                 uint32_t   n_pad,
                             /* recurrent */
                 ggml_type   type_r,
                 ggml_type   type_s,
@@ -33,7 +33,6 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
     hparams(model.hparams),
     mem_attn(new llama_kv_cache_iswa(
         model,
-        no_alloc,
         type_k,
         type_v,
         v_trans,
@@ -43,6 +42,7 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
         kv_size,
         n_seq_max,
         n_ubatch,
+        n_pad,
         nullptr,
         filter_attn == nullptr ?
             [&](int32_t il) { return !hparams.is_recr(il); }
@@ -52,7 +52,6 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
     )),
     mem_recr(new llama_memory_recurrent(
         model,
-        no_alloc,
         type_r,
         type_s,
         offload,

src/llama-memory-hybrid-iswa.h

@@ -20,7 +20,6 @@ class llama_memory_hybrid_iswa : public llama_memory_i {
 public:
     llama_memory_hybrid_iswa(
         const llama_model & model,
-                      bool   no_alloc,
                             /* attn */
                 ggml_type   type_k,
                 ggml_type   type_v,
@@ -28,6 +27,7 @@ public:
                      bool   swa_full,
                  uint32_t   kv_size,
                  uint32_t   n_ubatch,
+                 uint32_t   n_pad,
                             /* recurrent */
                 ggml_type   type_r,
                 ggml_type   type_s,

src/llama-memory-hybrid.cpp

@@ -10,12 +10,12 @@
 
 llama_memory_hybrid::llama_memory_hybrid(
         const llama_model & model,
-                     bool   no_alloc,
                             /* attn */
                 ggml_type   type_k,
                 ggml_type   type_v,
                      bool   v_trans,
                  uint32_t   kv_size,
+                 uint32_t   n_pad,
                  uint32_t   n_swa,
            llama_swa_type   swa_type,
                             /* recurrent */
@@ -33,7 +33,6 @@ llama_memory_hybrid::llama_memory_hybrid(
     hparams(model.hparams),
     mem_attn(new llama_kv_cache(
         model,
-        no_alloc,
         model.hparams,
         type_k,
         type_v,
@@ -42,6 +41,7 @@ llama_memory_hybrid::llama_memory_hybrid(
         unified,
         kv_size,
         n_seq_max,
+        n_pad,
         n_swa,
         swa_type,
         nullptr,
@@ -53,7 +53,6 @@ llama_memory_hybrid::llama_memory_hybrid(
     )),
     mem_recr(new llama_memory_recurrent(
         model,
-        no_alloc,
         type_r,
         type_s,
         offload,

src/llama-memory-hybrid.h

@@ -20,12 +20,12 @@ class llama_memory_hybrid : public llama_memory_i {
 public:
     llama_memory_hybrid(
         const llama_model & model,
-                     bool   no_alloc,
                             /* attn */
                 ggml_type   type_k,
                 ggml_type   type_v,
                      bool   v_trans,
                  uint32_t   kv_size,
+                 uint32_t   n_pad,
                  uint32_t   n_swa,
            llama_swa_type   swa_type,
                             /* recurrent */

src/llama-memory-recurrent.cpp

@@ -19,14 +19,13 @@
 
 llama_memory_recurrent::llama_memory_recurrent(
         const llama_model & model,
-                 bool   no_alloc,
                 ggml_type   type_r,
                 ggml_type   type_s,
                      bool   offload,
                  uint32_t   mem_size,
                  uint32_t   n_seq_max,
                  uint32_t   n_rs_seq,
-    const layer_filter_cb & filter) : hparams(model.hparams), n_seq_max(n_seq_max), ctxs_bufs(no_alloc) {
+    const layer_filter_cb & filter) : hparams(model.hparams), n_seq_max(n_seq_max) {
     const int32_t n_layer = hparams.n_layer();
 
     head = 0;
@@ -115,11 +114,23 @@ llama_memory_recurrent::llama_memory_recurrent(
         }
     }
 
-    // allocate the tensors and initialize the buffers to avoid NaNs in the padding
+    // allocate tensors and initialize the buffers to avoid NaNs in the padding
     for (auto & [buft, ctx] : ctx_map) {
-        ggml_backend_buffer_t buf = ctxs_bufs.emplace(std::move(ctx), buft, "rs cache");
-
+        ggml_backend_buffer_t buf;
+        if (hparams.no_alloc) {
+            buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer
+            for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) {
+                t->buffer = buf; // set dummy buffer for RS cache so that the backend scheduler won't try to allocate it
+            }
+        } else {
+            buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer
+        }
+        if (!buf) {
+            throw std::runtime_error("failed to allocate buffer for rs cache");
+        }
+        ggml_backend_buffer_clear(buf, 0);
         LLAMA_LOG_INFO("%s: %10s RS buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
+        ctxs_bufs.emplace_back(std::move(ctx), buf);
     }
 
     if (is_empty()) {
@@ -431,7 +442,18 @@ void llama_memory_recurrent::set_rs_idx(llama_seq_id seq_id, uint32_t idx) {
 }
 
 std::map<ggml_backend_buffer_type_t, size_t> llama_memory_recurrent::memory_breakdown() const {
-    return ctxs_bufs.memory_breakdown();
+    std::map<ggml_backend_buffer_type_t, size_t> ret;
+    for (const auto & [ctx, buf] : ctxs_bufs) {
+        ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
+
+        if (hparams.no_alloc) {
+            GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr);
+            ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
+        } else {
+            ret[buft] += ggml_backend_buffer_get_size(buf.get());
+        }
+    }
+    return ret;
 }
 
 llama_memory_context_ptr llama_memory_recurrent::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) {

src/llama-memory-recurrent.h

@@ -18,7 +18,6 @@ class llama_memory_recurrent : public llama_memory_i {
 public:
     llama_memory_recurrent(
             const llama_model & model,
-                         bool   no_alloc,
                     ggml_type   type_r,
                     ggml_type   type_s,
                          bool   offload,
@@ -121,8 +120,8 @@ private:
 
     const uint32_t n_seq_max = 1;
 
-    // ggml contexts for the RS cache along with the allocated backend buffers:
-    llama_memory_bufs ctxs_bufs;
+    // ggml contexts for the KV cache along with the allocated backend buffers:
+    std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;
 
     // true if no layers - can happen if the layer filter removes all layers
     bool is_empty() const;