cont

.github/workflows/behavior-check.yml

@@ -107,52 +107,3 @@ jobs:
           source .venv/bin/activate
           cd build
           ctest -L behavior -LE 'model|backend' --verbose --timeout 900
-
-  windows:
-    runs-on: windows-2025
-
-    cache-mode: write
-    permissions:
-      actions: write
-      contents: read
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          key: behavior-check-windows-2025
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-
-      - name: Python Dependencies
-        id: python_depends
-        shell: bash
-        run: |
-          python3 -m venv .venv
-          source .venv/Scripts/activate
-          pip install jinja2==3.1.6
-
-      - name: Build
-        id: cmake_build
-        shell: cmd
-        run: |
-          call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
-          cmake -B build -G "Ninja Multi-Config" ^
-            -DCMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake ^
-            -DCMAKE_BUILD_TYPE=Release ^
-            -DGGML_NATIVE=OFF ^
-            -DGGML_BACKEND_DL=OFF
-          set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
-          cmake --build build --config Release -j %NINJA_JOBS%
-
-      - name: Test
-        id: cmake_test
-        shell: bash
-        run: |
-          source .venv/Scripts/activate
-          cd build
-          ctest -C Release -L behavior -LE 'model|backend' --verbose --timeout 900

.github/workflows/build-apple.yml

@@ -107,6 +107,62 @@ jobs:
           # ref: https://github.com/ggml-org/llama.cpp/pull/19802#issuecomment-4013704023
           ctest -L main -E "test-llama-archs|test-llama-context|test-speculative-rejection-sampling" --verbose --timeout 900
 
+  macos-latest-x64:
+    runs-on: macos-15-intel
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: ccache
+        uses: ggml-org/ccache-action@v1.2.24
+        with:
+          restore: false
+          save: false
+
+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+        with:
+          key: apple-x64
+          folder: llama.cpp
+          hf_bucket: ggml-org/cache
+
+      - name: Build
+        id: cmake_build
+        run: |
+          sysctl -a
+          # Metal is disabled due to intermittent failures with Github runners not having a GPU:
+          # https://github.com/ggml-org/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
+          cmake -B build \
+            -DCMAKE_BUILD_RPATH="@loader_path" \
+            -DLLAMA_FATAL_WARNINGS=ON \
+            -DLLAMA_BUILD_BORINGSSL=ON \
+            -DGGML_METAL=OFF \
+            -DGGML_RPC=ON \
+            -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
+          time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
+
+      - name: ccache-buckets-save
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+        with:
+          key: apple-x64
+          folder: llama.cpp
+          evict-old-files: 1d
+          hf_bucket: ggml-org/cache
+          save: true
+
+      - name: Test
+        id: cmake_test
+        run: |
+          cd build
+          ctest -L main --verbose --timeout 900
+
   macos-latest-ios-xcode:
     runs-on: macos-latest
 

.github/workflows/build-cmake-pkg.yml

@@ -9,7 +9,7 @@ permissions:
 
 jobs:
   linux:
-    runs-on: [self-hosted, Linux, fast]
+    runs-on: [self-hosted, Linux, CPU]
     steps:
       - uses: actions/checkout@v6
         with:

.github/workflows/build-sanitize.yml

@@ -49,7 +49,7 @@ jobs:
         - sanitizer: THREAD
           machine: ubuntu-24.04
         - sanitizer: UNDEFINED
-          machine: [self-hosted, X64, Linux, CPU]
+          machine: [self-hosted, X64, Linux]
 
     runs-on: ${{ matrix.machine }}
 

.github/workflows/ci-self-hosted-cpu.yml

@@ -46,7 +46,7 @@ env:
 
 jobs:
   cpu-x64-high-perf:
-    runs-on: [self-hosted, Linux, X64, CPU]
+    runs-on: [self-hosted, Linux, X64]
 
     steps:
       - name: Clone

.github/workflows/ci-self-hosted-webgpu.yml

@@ -58,12 +58,7 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt update
-          sudo apt install -y software-properties-common
-          sudo add-apt-repository ppa:deadsnakes/ppa
-          sudo apt update
-          sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3.12 python3.12-venv
-          sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1
-          sudo update-alternatives --config python3
+          sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
 
       - name: ccache
         uses: ggml-org/ccache-action@v1.2.24

.github/workflows/release.yml

@@ -968,11 +968,6 @@ jobs:
           evict-old-files: 1d
           max-size: "1G"
 
-      - name: Tune ccache for ROCm
-        run: |
-          # since ROCm 10.1, clang -E output has no GPU code, so the preprocessor fallback can reuse objects with old GPU code
-          ccache --set-config=depend_mode=true
-
       # - name: Cache ROCm Installation
       #   id: cache-rocm
       #   uses: actions/cache@v5
@@ -1550,8 +1545,6 @@ jobs:
           # builds hit.
           ccache --set-config=compiler_check=content
           ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime
-          # since ROCm 10.1, clang -E output has no GPU code, so the preprocessor fallback can reuse objects with old GPU code
-          ccache --set-config=depend_mode=true
 
       - name: Dependencies
         id: depends

CMakeLists.txt

@@ -103,7 +103,7 @@ endif()
 
 # subprocess spawning isn't a supported/sandbox-friendly operation on mobile OSes or in WASM
 if (CMAKE_SYSTEM_NAME STREQUAL "iOS" OR CMAKE_SYSTEM_NAME STREQUAL "Android" OR ANDROID
-        OR CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR EMSCRIPTEN OR CMAKE_SYSTEM_NAME STREQUAL "WASI")
+        OR CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR EMSCRIPTEN)
     set(LLAMA_SUBPROCESS_DEFAULT OFF)
 else()
     set(LLAMA_SUBPROCESS_DEFAULT ON)

ci/run.sh

@@ -460,10 +460,10 @@ function gg_run_qwen3_0_6b {
     #       https://github.com/ggml-org/llama.cpp/actions/runs/39639985732
     if [ -z ${GG_BUILD_WEBGPU} ]; then
         (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+        (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa on  --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
         (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa off                ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+        (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa on                 ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
     fi
-    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa on  --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
-    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa on                 ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
 
     check_ppl "f16"  "$(cat $OUT/${ci}-tg-f16.log  | grep "^\[1\]")"
     if [ -z ${GG_BUILD_NO_BF16} ]; then

common/arg.cpp

@@ -924,7 +924,7 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
         // TODO @ngxson : maybe show a list of available models in CLI in this case
         bool can_skip_model = params.usage || params.completion || !params.server_base.empty();
         if (!can_skip_model && params.model.path.empty()) {
-            throw std::invalid_argument("error: --model or -hf is required\n");
+            throw std::invalid_argument("error: --model is required\n");
         }
     }
 
@@ -3708,13 +3708,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.models_max = value;
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MODELS_MAX"));
-    add_opt(common_arg(
-        {"--models-memory-margin"}, "N",
-        string_format("for router server, MiB of device memory to keep free per device, unload LRU models until a new one fits (default: %d, 0 = disabled)", params.models_memory_margin),
-        [](common_params & params, int value) {
-            params.models_memory_margin = value;
-        }
-    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MODELS_MEMORY_MARGIN"));
     add_opt(common_arg(
         {"--models-autoload"},
         {"--no-models-autoload"},
@@ -4512,14 +4505,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.tts_lang = value;
         }
     ).set_examples({LLAMA_EXAMPLE_TTS}));
-    add_opt(common_arg(
-        {"--tts-voice"}, "VOICE",
-        "voice for audio generation (model-dependent)\n"
-        "see tts/README.md for per-model usage notes",
-        [](common_params & params, const std::string & value) {
-            params.tts_voice = value;
-        }
-    ).set_examples({LLAMA_EXAMPLE_TTS}));
     add_opt(common_arg(
         {"--tts-speaker-file"}, "FNAME",
         "speaker file path for audio generation",

common/chat.cpp

@@ -190,17 +190,10 @@ json common_chat_msg::to_json_oaicompat(bool concat_typed_text) const {
         } else {
             auto & parts = jmsg["content"] = json::array();
             for (const auto & part : content_parts) {
-                if (part.type == "tool_reference") {
-                    parts.push_back({
-                        {"type", part.type},
-                        {"name", part.name},
-                    });
-                } else {
-                    parts.push_back({
-                        {"type", part.type},
-                        {"text", part.text},
-                    });
-                }
+                parts.push_back({
+                    {"type", part.type},
+                    {"text", part.text},
+                });
             }
         }
     } else {
@@ -316,7 +309,6 @@ struct common_chat_templates {
     bool has_explicit_template;  // Model had builtin template or template overridden was specified.
     std::unique_ptr<common_chat_template> template_default;  // always set (defaults to chatml)
     std::unique_ptr<common_chat_template> template_tool_use;
-    std::unique_ptr<common_chat_template> template_tts;  // the prompt of a text to speak
 };
 
 common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {
@@ -378,16 +370,12 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
                             throw std::invalid_argument("Missing content part type: " + part.dump());
                         }
                         const auto & type = part.at("type");
-                        common_chat_msg_content_part msg_part;
-                        msg_part.type = type;
-                        if (type == "text" || type == "media_marker") {
-                            msg_part.text = part.at("text");
-                        } else if (type == "tool_reference") {
-                            // names a request tool, the template renders its definition
-                            msg_part.name = part.at("name");
-                        } else {
+                        if (type != "text" && type != "media_marker") {
                             throw std::invalid_argument("Unsupported content part type: " + type.dump());
                         }
+                        common_chat_msg_content_part msg_part;
+                        msg_part.type = type;
+                        msg_part.text = part.at("text");
                         msg.content_parts.push_back(msg_part);
                     }
                 } else if (!content.is_null()) {
@@ -440,41 +428,6 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
             if (message.contains("tool_call_id")) {
                 msg.tool_call_id = message.at("tool_call_id");
             }
-            {
-                json extra = json::object();
-                for (const auto & [k, v] : message.items()) {
-                    if (k != "role" && k != "content" && k != "tool_calls" && k != "reasoning_content" &&
-                        k != "name" && k != "tool_call_id") {
-                        extra[k] = v;
-                    }
-                }
-                if (has_tool_calls) {
-                    // per tool call: keys besides type/function/id, and function keys besides name/arguments
-                    json tc_extra = json::array();
-                    bool any      = false;
-                    for (const auto & tool_call : message.at("tool_calls")) {
-                        json e = json::object();
-                        for (const auto & [k, v] : tool_call.items()) {
-                            if (k != "type" && k != "function" && k != "id") {
-                                e[k] = v;
-                            }
-                        }
-                        for (const auto & [k, v] : tool_call.at("function").items()) {
-                            if (k != "name" && k != "arguments") {
-                                e["function"][k] = v;
-                            }
-                        }
-                        any = any || !e.empty();
-                        tc_extra.push_back(e);
-                    }
-                    if (any) {
-                        extra["__tool_call_extras"] = tc_extra;
-                    }
-                }
-                if (!extra.empty()) {
-                    msg.extra_json = extra.dump();
-                }
-            }
 
             msgs.push_back(msg);
         }
@@ -563,50 +516,6 @@ static json render_message_to_json(const std::vector<common_chat_msg> & msgs, co
     return messages_inp_normalizer(c).normalize(messages);
 }
 
-// DeepSeek V4.1 reads non-OpenAI message keys and joins content blocks with "\n\n" around images,
-// so keep media markers as separate text parts and restore the extra keys
-static json render_message_to_json_deepseek_v4_1(const std::vector<common_chat_msg> & msgs, const jinja::caps & c) {
-    json messages = json::array();
-    for (const auto & msg : msgs) {
-        json jmsg;
-        if (msg.contains_media() && msg.content.empty()) {
-            common_chat_msg typed = msg;
-            for (auto & part : typed.content_parts) {
-                part.type = "text";
-            }
-            jmsg = typed.to_json_oaicompat(/* concat_typed_text= */ false);
-        } else {
-            jmsg = msg.to_json_oaicompat(/* concat_typed_text= */ false);
-        }
-        if (!msg.extra_json.empty()) {
-            const json extra = json::parse(msg.extra_json);
-            for (const auto & [k, v] : extra.items()) {
-                if (k == "__tool_call_extras") {
-                    if (!jmsg.contains("tool_calls")) {
-                        continue;
-                    }
-                    auto & tcs = jmsg.at("tool_calls");
-                    for (size_t i = 0; i < tcs.size() && i < v.size(); i++) {
-                        for (const auto & [tk, tv] : v[i].items()) {
-                            if (tk == "function") {
-                                for (const auto & [fk, fv] : tv.items()) {
-                                    tcs[i]["function"][fk] = fv;
-                                }
-                            } else {
-                                tcs[i][tk] = tv;
-                            }
-                        }
-                    }
-                } else if (!jmsg.contains(k)) {
-                    jmsg[k] = v;
-                }
-            }
-        }
-        messages.push_back(std::move(jmsg));
-    }
-    return messages_inp_normalizer(c).normalize(messages);
-}
-
 // DEPRECATED: only used in tests
 json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text) {
     jinja::caps c;
@@ -622,17 +531,13 @@ json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & t
 
     auto result = json::array();
     for (const auto & tool : tools) {
-        json function {
-            { "name",        tool.name },
-            { "description", tool.description },
-            { "parameters",  json::parse(tool.parameters) },
-        };
-        if (tool.defer_loading) {
-            function["defer_loading"] = true;
-        }
         result.push_back({
             { "type",     "function" },
-            { "function", function },
+            { "function", {
+                { "name", tool.name },
+                { "description", tool.description },
+                { "parameters", json::parse(tool.parameters) },
+            }},
         });
     }
     return result;
@@ -669,19 +574,10 @@ std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & too
                 }
 
                 const auto & function = tool.at("function");
-
-                // "defer_loading" sits on the function, or on the tool next to "type"; the function wins
-                const json & defer = function.contains("defer_loading") ? function.at("defer_loading")
-                                                                        : tool.value("defer_loading", json(false));
-                if (!defer.is_boolean()) {
-                    throw std::invalid_argument("Expected 'defer_loading' to be a boolean, got " + defer.dump());
-                }
-
                 result.push_back({
                     /* .name = */ function.at("name"),
                     /* .description = */ function.value("description", ""),
                     /* .parameters = */ function.value("parameters", json::object()).dump(),
-                    /* .defer_loading = */ defer.get<bool>(),
                 });
             }
         }
@@ -804,70 +700,19 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
     return tmpls->has_explicit_template;
 }
 
-// renders a task template (e.g. "tts") with the given variables
-static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) {
-    jinja::context ctx(tmpl.source());
-    jinja::global_from_json(ctx, vars, false);
-    ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
-    ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
-    jinja::runtime runtime(ctx);
-    return jinja::runtime::gather_string_parts(runtime.execute(tmpl.prog))->as_string().str();
-}
-
-// LFM3 format detection: template renders deferred tools inside <additional_tools>...</additional_tools>
-static bool is_lfm3_template(const std::string & src) {
-    return src.find("<additional_tools>") != std::string::npos;
-}
-
-// LFM3-Audio: a known language code maps to its name, no language lets the model detect it
-static std::string lfm3_asr_system_prompt(const std::string & language) {
-    static const std::map<std::string, std::string> names = {
-        { "ar", "Arabic" }, { "da", "Danish" }, { "de", "German" }, { "el", "Greek" }, { "en", "English" },
-        { "es", "Spanish" }, { "fi", "Finnish" }, { "fr", "French" }, { "he", "Hebrew" }, { "it", "Italian" },
-        { "ja", "Japanese" }, { "ko", "Korean" }, { "nl", "Dutch" }, { "pl", "Polish" }, { "pt", "Portuguese" },
-        { "sv", "Swedish" }, { "tr", "Turkish" }, { "zh", "Chinese" },
-    };
-    if (language.empty()) {
-        return "Perform ASR.";
-    }
-    std::string code = language;
-    std::transform(code.begin(), code.end(), code.begin(), [](unsigned char c) { return (char) std::tolower(c); });
-    const auto it = names.find(code);
-    return "Perform ASR in " + (it != names.end() ? it->second : language) + ".";
-}
-
-common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) {
+common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates) {
     common_chat_prompt_preset asr_preset;
     asr_preset.system = "";
     asr_preset.user   = "Transcribe audio to text";
 
-    if (chat_templates && chat_templates->template_default) {
-        const std::string src = chat_templates->template_default->source();
-        if (is_lfm2_template(src)) {
-            asr_preset.system = "Perform ASR.";
-            asr_preset.user   = "";
-        } else if (is_lfm3_template(src)) {
-            asr_preset.system = lfm3_asr_system_prompt(language);
-            asr_preset.user   = "";
-            asr_preset.has_language = true;
-        }
+    if (chat_templates && chat_templates->template_default && is_lfm2_template(chat_templates->template_default->source())) {
+        asr_preset.system = "Perform ASR.";
+        asr_preset.user   = "";
     }
 
     return asr_preset;
 }
 
-std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
-                                       const std::string & language, const std::string & voice) {
-    if (!chat_templates || !chat_templates->template_tts) {
-        return "";
-    }
-    return common_chat_template_render(*chat_templates->template_tts, {
-        {"messages", json::array({{{"role", "user"}, {"content", text}}})},
-        {"language", language},
-        {"voice",    voice},
-    });
-}
-
 std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant) {
     if (!variant.empty()) {
         if (variant == "tool_use") {
@@ -978,16 +823,6 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
             LOG_ERR("%s: failed to parse tool use chat template (ignoring it): %s\n", __func__, e.what());
         }
     }
-    // task templates of the model, kept with a chat template override
-    if (model) {
-        if (const char * src = llama_model_chat_template(model, "tts")) {
-            try {
-                tmpls->template_tts = std::make_unique<common_chat_template>(src, token_bos, token_eos);
-            } catch (const std::exception & e) {
-                LOG_ERR("%s: failed to parse the tts chat template (ignoring it): %s\n", __func__, e.what());
-            }
-        }
-    }
     return tmpls;
 }
 
@@ -1373,13 +1208,11 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
     // DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls.
     // The template source contains the token as a variable assignment, not as a literal in markup.
     // V3.2 names the tool call block "function_calls", V4 names it "tool_calls".
-    // V4.1 tag names have a leading space and its template does not name the dsml_token variable.
-    if ((src.find("dsml_token") != std::string::npos &&
-         src.find("DSML") != std::string::npos &&
-         (src.find("function_calls") != std::string::npos ||
-          src.find("tool_calls") != std::string::npos)) ||
-        src.find("｜DSML｜ calls>") != std::string::npos) {
-        LOG_DBG("Using specialized template: DeepSeek V3.2/V4/V4.1\n");
+    if (src.find("dsml_token") != std::string::npos &&
+        src.find("DSML") != std::string::npos &&
+        (src.find("function_calls") != std::string::npos ||
+         src.find("tool_calls") != std::string::npos)) {
+        LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n");
         return common_chat_params_init_deepseek_v3_2;
     }
 
@@ -1480,9 +1313,7 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
         workaround::trim_all_content(trimmed_messages);
         messages_to_render = &trimmed_messages;
     }
-    params.messages              = src.find("｜DSML｜ calls>") != std::string::npos ?
-                                       render_message_to_json_deepseek_v4_1(*messages_to_render, tmpl.original_caps()) :
-                                       render_message_to_json(*messages_to_render, tmpl.original_caps());
+    params.messages              = render_message_to_json(*messages_to_render, tmpl.original_caps());
     params.tool_choice           = inputs.tool_choice;
     params.reasoning_format      = inputs.reasoning_format;
     params.enable_thinking       = inputs.enable_thinking;
@@ -1940,13 +1771,12 @@ const common_chat_msg & common_chat_session::finish(const common_chat_input & ch
     return result;
 }
 
-common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
+std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
     GGML_ASSERT(chat_templates != nullptr);
     GGML_ASSERT(chat_templates->template_default != nullptr);
-    // take the more expressive template when available
-    const auto & caps = chat_templates->template_tool_use != nullptr ? chat_templates->template_tool_use->caps
-                                                                      : chat_templates->template_default->caps;
-    common_json result = caps.to_map();
-    result["reasoning_effort_levels"] = caps.reasoning_effort_levels;
-    return result;
+    if (chat_templates->template_tool_use != nullptr) {
+        // take the more expressive template when available
+        return chat_templates->template_tool_use->caps.to_map();
+    }
+    return chat_templates->template_default->caps.to_map();
 }

common/chat.h

@@ -45,7 +45,6 @@ struct common_chat_tool_call {
 struct common_chat_msg_content_part {
     std::string type;
     std::string text;
-    std::string name; // tool_reference: name of a tool from the request tools
 
     // TODO @ngxson : no known chat templates support reasoning_content in content parts yet
     //                this can be useful for models with interleaved thinking (like Kimi-K2)
@@ -53,7 +52,7 @@ struct common_chat_msg_content_part {
     // std::string reasoning_content;
 
     bool operator==(const common_chat_msg_content_part & other) const {
-        return type == other.type && text == other.text && name == other.name;
+        return type == other.type && text == other.text;
     }
 };
 
@@ -95,8 +94,6 @@ struct common_chat_msg {
     std::string                               reasoning_content;
     std::string                               tool_name;
     std::string                               tool_call_id;
-    // non-OpenAI message keys as a JSON object string, only for templates that read them (DeepSeek V4.1)
-    std::string                               extra_json;
 
     common_json to_json_oaicompat(bool concat_typed_text = false) const;
 
@@ -227,9 +224,6 @@ struct common_chat_tool {
     std::string name;
     std::string description;
     std::string parameters;
-
-    // withheld from the system prompt, a tool_reference reveals it later in the conversation
-    bool defer_loading = false;
 };
 
 enum common_chat_tool_choice {
@@ -472,7 +466,7 @@ common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_to
 common_json common_chat_tool_parameters(const common_json & function);
 
 // get template caps, useful for reporting to server /props endpoint
-common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates);
+std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);
 
 common_chat_prompt common_chat_template_direct_apply(
     const common_chat_template & tmpl,
@@ -491,12 +485,6 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
 struct common_chat_prompt_preset {
     std::string system;
     std::string user;
-    bool has_language = false; // the preset already names the requested language
 };
 
-common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "");
-
-// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
-// language and voice are optional, the template defines their defaults
-std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
-                                       const std::string & language = "", const std::string & voice = "");
+common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);

common/common.cpp

@@ -742,56 +742,50 @@ static inline bool glob_class_match(const char c, const char * pattern, const ch
 }
 
 // simple glob: * matches non-/ chars, ** matches anything including /, [] matches character class
-// only wildcards recurse, too many wildcards: no match
-static bool glob_match(const char * pattern, const char * str, int depth = 0) {
-    if (depth >= COMMON_RECURSIVE_LIMIT) {
+static inline bool glob_match(const char * pattern, const char * str) {
+    if (*pattern == '\0') {
+        return *str == '\0';
+    }
+    if (pattern[0] == '*' && pattern[1] == '*') {
+        const char * p = pattern + 2;
+        if (glob_match(p, str)) return true;
+        if (*str != '\0') return glob_match(pattern, str + 1);
         return false;
     }
-    while (*pattern != '\0') {
-        if (pattern[0] == '*' && pattern[1] == '*') {
-            const char * p = pattern + 2;
-            for (;; str++) {
-                if (glob_match(p, str, depth + 1)) return true;
-                if (*str == '\0') return false;
-            }
+    if (*pattern == '*') {
+        const char * p = pattern + 1;
+        for (; *str != '\0' && *str != '/'; str++) {
+            if (glob_match(p, str)) return true;
         }
-        if (*pattern == '*') {
-            const char * p = pattern + 1;
-            for (; *str != '\0' && *str != '/'; str++) {
-                if (glob_match(p, str, depth + 1)) return true;
-            }
-            return glob_match(p, str, depth + 1);
-        }
-        if (*pattern == '?' && *str != '\0' && *str != '/') {
-            pattern++;
-            str++;
-            continue;
-        }
-        if (*pattern == '[') {
-            const char * class_end = pattern + 1;
-            // If first character after '[' is ']' or '-', treat it as literal
-            if (*class_end == ']' || *class_end == '-') {
-                class_end++;
-            }
-            while (*class_end != '\0' && *class_end != ']') {
-                class_end++;
-            }
-            if (*class_end == ']') {
-                if (*str == '\0' || !glob_class_match(*str, pattern + 1, class_end)) return false;
-                pattern = class_end + 1;
-                str++;
-                continue;
-            }
-            if (*str != '[') return false;
-            pattern++;
-            str++;
-            continue;
-        }
-        if (*pattern != *str) return false;
-        pattern++;
-        str++;
+        return glob_match(p, str);
     }
-    return *str == '\0';
+    if (*pattern == '?' && *str != '\0' && *str != '/') {
+        return glob_match(pattern + 1, str + 1);
+    }
+    if (*pattern == '[') {
+        const char * class_end = pattern + 1;
+        // If first character after '[' is ']' or '-', treat it as literal
+        if (*class_end == ']' || *class_end == '-') {
+            class_end++;
+        }
+        while (*class_end != '\0' && *class_end != ']') {
+            class_end++;
+        }
+        if (*class_end == ']') {
+            if (*str == '\0') return false;
+            bool matched = glob_class_match(*str, pattern + 1, class_end);
+            return matched && glob_match(class_end + 1, str + 1);
+        } else {
+            if (*str == '[') {
+                return glob_match(pattern + 1, str + 1);
+            }
+            return false;
+        }
+    }
+    if (*pattern == *str) {
+        return glob_match(pattern + 1, str + 1);
+    }
+    return false;
 }
 
 bool glob_match(const std::string & pattern, const std::string & str) {
@@ -1187,9 +1181,6 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
     { COMMON_DECISION_TYPE_LFM2_D1_OMNI,   "lfm2-d1-omni"  },
     { COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"},
     { COMMON_DECISION_TYPE_H2O_LIGHTNING,  "h2o-lightning" },
-    { COMMON_DECISION_TYPE_MINICPM_OCR_JEV, "openbmb-minicpm-ocr-jev" },
-    { COMMON_DECISION_TYPE_LFM3_D1,        "lfm3-d1"       },
-    { COMMON_DECISION_TYPE_LFM3_D1_OMNI,   "lfm3-d1-omni"  },
     // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
 };
 
@@ -1255,50 +1246,44 @@ common_gguf_info common_get_gguf_info(const std::string & fname) {
     return info;
 }
 
-void common_fit_apply(common_params & params, llama_model_params & mparams, llama_context_params & cparams) {
-    if (!params.fit_params) {
-        return;
-    }
-
-    COM_TRC("%s", "fitting params to device memory ...\n");
-    COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n");
-
-    // the draft context is created from the same base params and follows the main context, fit both together
-    const bool has_draft = params.speculative.has_dft();
-    const bool spec_mtp  = std::find(params.speculative.types.begin(), params.speculative.types.end(),
-        COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
-
-    common_params params_dft = common_base_params_to_speculative(params);
-
-    auto mparams_dft = common_model_params_to_llama(params_dft);
-    auto cparams_dft = common_context_params_to_llama(params_dft);
-    if (spec_mtp) {
-        cparams_dft.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
-    }
-    cparams_dft.n_rs_seq = 0;
-
-    const common_fit_extra_model extra = {
-        /*.path_model   =*/ params_dft.model.path.c_str(),
-        /*.mparams      =*/ &mparams_dft,
-        /*.cparams      =*/ &cparams_dft,
-        /*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model
-    };
-
-    common_fit_params(params.model.path.c_str(), &mparams, &cparams,
-        params.tensor_split,
-        params.tensor_buft_overrides.data(),
-        params.fit_params_target.data(),
-        params.fit_params_min_ctx,
-        has_draft || spec_mtp ? &extra : nullptr,
-        params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
-}
-
 common_init_result::common_init_result(common_params & params, bool model_only) :
     pimpl(new impl{}) {
     auto mparams = common_model_params_to_llama(params);
     auto cparams = common_context_params_to_llama(params);
 
-    common_fit_apply(params, mparams, cparams);
+    if (params.fit_params) {
+        COM_TRC("%s", "fitting params to device memory ...\n");
+        COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n");
+
+        // the draft context is created from the same base params and follows the main context, fit both together
+        const bool has_draft = params.speculative.has_dft();
+        const bool spec_mtp  = std::find(params.speculative.types.begin(), params.speculative.types.end(),
+            COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
+
+        common_params params_dft = common_base_params_to_speculative(params);
+
+        auto mparams_dft = common_model_params_to_llama(params_dft);
+        auto cparams_dft = common_context_params_to_llama(params_dft);
+        if (spec_mtp) {
+            cparams_dft.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
+        }
+        cparams_dft.n_rs_seq = 0;
+
+        const common_fit_extra_model extra = {
+            /*.path_model   =*/ params_dft.model.path.c_str(),
+            /*.mparams      =*/ &mparams_dft,
+            /*.cparams      =*/ &cparams_dft,
+            /*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model
+        };
+
+        common_fit_params(params.model.path.c_str(), &mparams, &cparams,
+            params.tensor_split,
+            params.tensor_buft_overrides.data(),
+            params.fit_params_target.data(),
+            params.fit_params_min_ctx,
+            has_draft || spec_mtp ? &extra : nullptr,
+            params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
+    }
 
     llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
     if (model == NULL) {
@@ -1316,17 +1301,13 @@ common_init_result::common_init_result(common_params & params, bool model_only)
     // these decision models return a score for each token via the embeddings output
     // TODO: maybe improve this in the future
     const auto decision_type = common_get_decision_type(model);
-    // minicpm-ocr-jev reads its class scores from the pooled output, the pooling type is the one of the GGUF
     if (decision_type == COMMON_DECISION_TYPE_LAYA || decision_type == COMMON_DECISION_TYPE_KEV || decision_type == COMMON_DECISION_TYPE_CLEF ||
-        decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || decision_type == COMMON_DECISION_TYPE_VLLM_DECISION2 ||
-        decision_type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV) {
-        const auto pooling_type = decision_type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV ? LLAMA_POOLING_TYPE_UNSPECIFIED : LLAMA_POOLING_TYPE_NONE;
-
+        decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || decision_type == COMMON_DECISION_TYPE_VLLM_DECISION2) {
         params.embedding    = true;
-        params.pooling_type = pooling_type;
+        params.pooling_type = LLAMA_POOLING_TYPE_NONE;
 
         cparams.embeddings            = true;
-        cparams.pooling_type          = pooling_type;
+        cparams.pooling_type          = LLAMA_POOLING_TYPE_NONE;
         cparams.n_outputs_max         = cparams.n_batch;
         cparams.n_outputs_max_per_seq = 1;
 

common/common.h

@@ -31,8 +31,6 @@
 #define DIRECTORY_SEPARATOR '/'
 #endif // _WIN32
 
-#define COMMON_RECURSIVE_LIMIT 128
-
 #define COM_DBG(fmt, ...) LOG_DBG("cmn  %12.*s: " fmt, 12, __func__, __VA_ARGS__)
 #define COM_TRC(fmt, ...) LOG_TRC("cmn  %12.*s: " fmt, 12, __func__, __VA_ARGS__)
 #define COM_INF(fmt, ...) LOG_INF("cmn  %12.*s: " fmt, 12, __func__, __VA_ARGS__)
@@ -701,7 +699,6 @@ struct common_params {
     std::string models_dir    = "";     // directory containing models for the router server
     std::string models_preset = "";     // directory containing model presets for the router server
     int models_max = 4;                 // maximum number of models to load simultaneously
-    int models_memory_margin = 0;       // MiB of device memory to keep free when loading models (0 = no memory accounting)
     bool models_autoload = true;        // automatically load models when requested via the router server
     std::string models_preset_hf = "";  // show a warning about remote presets on router loaded (if not empty)
 
@@ -772,7 +769,6 @@ struct common_params {
 
     // TTS params
     std::string tts_lang = "";
-    std::string tts_voice = "";
     std::string tts_speaker_file = "";
 
     bool is_gen_docs = false; // whether we are running inside llama-gen-docs
@@ -980,9 +976,6 @@ enum common_decision_type {
     COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout
     COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch
     COMMON_DECISION_TYPE_H2O_LIGHTNING, // same as openjev, the labels are read after an answer prefill
-    COMMON_DECISION_TYPE_MINICPM_OCR_JEV, // one score per class, the pooled embeddings output of the last token
-    COMMON_DECISION_TYPE_LFM3_D1,       // same as lfm2-d1, non-causal, label codes A..Z then AA, AB..
-    COMMON_DECISION_TYPE_LFM3_D1_OMNI,  // same as lfm3-d1
     COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
 };
 
@@ -1022,9 +1015,6 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
 struct llama_model_params   common_model_params_to_llama  (      common_params & params);
 struct llama_context_params common_context_params_to_llama(const common_params & params);
 
-// apply --fit to mparams and cparams, no-op if params.fit_params is false
-void common_fit_apply(common_params & params, struct llama_model_params & mparams, struct llama_context_params & cparams);
-
 // clear LoRA adapters from context, then apply new list of adapters
 void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora);
 

common/fit.cpp

@@ -1,8 +1,6 @@
 #include "fit.h"
 
-#include "common.h"
 #include "json.h"
-#include "llama-cpp.h"
 #include "log.h"
 
 #include "../src/llama-ext.h"
@@ -29,32 +27,6 @@ class common_params_fit_exception : public std::runtime_error {
     using std::runtime_error::runtime_error;
 };
 
-// RAII: send llama logs below min_level to the debug log
-struct common_fit_log_scope {
-    common_fit_log_scope(ggml_log_level min_level) : min_level(min_level) {
-        llama_log_get(&original_logger.callback, &original_logger.user_data);
-        llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
-            const common_fit_log_scope * self = (const common_fit_log_scope *) user_data;
-            const ggml_log_level level_eff = level >= self->min_level ? level : GGML_LOG_LEVEL_DEBUG;
-            self->original_logger.callback(level_eff, text, self->original_logger.user_data);
-        }, this);
-    }
-
-    ~common_fit_log_scope() {
-        llama_log_set(original_logger.callback, original_logger.user_data);
-    }
-
-    common_fit_log_scope(const common_fit_log_scope &) = delete;
-    common_fit_log_scope & operator=(const common_fit_log_scope &) = delete;
-
-private:
-    struct {
-        ggml_log_callback callback;
-        void * user_data;
-    } original_logger;
-    ggml_log_level min_level;
-};
-
 static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
         const char * path_model,
         const llama_model_params * mparams,
@@ -64,7 +36,22 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
         uint32_t & hp_n_ctx_train,
         uint32_t & hp_n_expert,
         ggml_log_level log_level) {
-    common_fit_log_scope log_scope(log_level);
+    struct user_data_t {
+        struct {
+            ggml_log_callback callback;
+            void * user_data;
+        } original_logger;
+        ggml_log_level min_level; // prints below this log level go to debug log
+    };
+    user_data_t ud;
+    llama_log_get(&ud.original_logger.callback, &ud.original_logger.user_data);
+    ud.min_level = log_level;
+
+    llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
+        const user_data_t * ud = (const user_data_t *) user_data;
+        const ggml_log_level level_eff = level >= ud->min_level ? level : GGML_LOG_LEVEL_DEBUG;
+        ud->original_logger.callback(level_eff, text, ud->original_logger.user_data);
+    }, &ud);
 
     llama_model_params mparams_copy = *mparams;
     mparams_copy.no_alloc  = true;
@@ -72,12 +59,14 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
 
     llama_model * model = llama_model_load_from_file(path_model, mparams_copy);
     if (model == nullptr) {
+        llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
         throw std::runtime_error("failed to load model");
     }
 
     llama_context * ctx = llama_init_from_model(model, *cparams);
     if (ctx == nullptr) {
         llama_model_free(model);
+        llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
         throw std::runtime_error("failed to create llama_context from model");
     }
 
@@ -159,81 +148,11 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
 
     llama_free(ctx);
     llama_model_free(model);
+    llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
 
     return ret;
 }
 
-std::vector<common_memory_breakdown_row> common_fit_measure(
-        const char * path_model,
-        const llama_model_params * mparams,
-        const llama_context_params * cparams,
-        uint32_t * n_ctx_train,
-        uint32_t * n_ctx,
-        ggml_log_level log_level) {
-    common_fit_log_scope log_scope(log_level);
-
-    llama_model_params mparams_copy = *mparams;
-    mparams_copy.no_alloc  = true;
-    mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;
-
-    llama_model_ptr model(llama_model_load_from_file(path_model, mparams_copy));
-    if (!model) {
-        throw std::runtime_error("failed to load model");
-    }
-
-    llama_context_ptr ctx(llama_init_from_model(model.get(), *cparams));
-    if (!ctx) {
-        throw std::runtime_error("failed to create llama_context from model");
-    }
-
-    if (n_ctx_train) {
-        *n_ctx_train = llama_model_n_ctx_train(model.get());
-    }
-    if (n_ctx) {
-        *n_ctx = llama_n_ctx(ctx.get());
-    }
-
-    return common_memory_breakdown_rows(ctx.get());
-}
-
-common_memory_reports common_measure_memory(common_params & params) {
-    const ggml_log_level log_level = params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR;
-    const char * path_model = params.model.path.c_str();
-
-    common_memory_reports res;
-
-    auto mparams = common_model_params_to_llama(params);
-    auto cparams = common_context_params_to_llama(params);
-
-    // max: what --fit picks now, no-op with -fit off
-    {
-        auto mparams_fit = mparams;
-        auto cparams_fit = cparams;
-        common_fit_apply(params, mparams_fit, cparams_fit);
-        res.info_max = common_fit_measure(path_model, &mparams_fit, &cparams_fit, &res.n_ctx_train, &res.n_ctx_max, log_level);
-    }
-
-    // min: the smallest context --fit may pick (--fit only changes n_ctx when it is 0)
-    uint32_t n_ctx_min = cparams.n_ctx;
-    if (n_ctx_min == 0 && params.fit_params) {
-        const uint32_t n_streams = cparams.kv_unified ? 1 : std::max<uint32_t>(1, cparams.n_seq_max);
-        // -c 0 sets fit_params_min_ctx to UINT32_MAX, cap at n_ctx_train like --fit
-        n_ctx_min = (uint32_t) std::min<uint64_t>(
-            uint64_t(params.fit_params_min_ctx) * n_streams,
-            uint64_t(res.n_ctx_train) * n_streams);
-    }
-    if (n_ctx_min == 0 || n_ctx_min >= res.n_ctx_max) {
-        // nothing to reduce, or -fit off with the trained context
-        res.n_ctx_min = res.n_ctx_max;
-        res.info_min  = res.info_max;
-    } else {
-        cparams.n_ctx = n_ctx_min;
-        res.info_min  = common_fit_measure(path_model, &mparams, &cparams, nullptr, &res.n_ctx_min, log_level);
-    }
-
-    return res;
-}
-
 common_device_memory_data_vec common_get_device_memory_data(
         const char * path_model,
         const llama_model_params * mparams,
@@ -984,7 +903,8 @@ enum common_params_fit_status common_fit_params(
     return status;
 }
 
-std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const struct llama_context * ctx) {
+void common_memory_breakdown_print(const struct llama_context * ctx) {
+    //const auto & devices = ctx->get_model().devices;
     const auto * model = llama_get_model(ctx);
 
     std::vector<ggml_backend_dev_t> devices;
@@ -994,6 +914,18 @@ std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const stru
 
     llama_memory_breakdown memory_breakdown = llama_get_memory_breakdown(ctx);
 
+    std::vector<std::array<std::string, 9>> table_data;
+    table_data.reserve(devices.size());
+
+    // same data as the table below, for --log-jsonl consumers
+    common_json rows = common_json::array();
+    const std::string template_header = "%s: | %s | %s   %s    %s   %s   %s   %s    %s |\n";
+    const std::string template_gpu    = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";
+    const std::string template_other  = "%s: | %s | %s   %s    %s = %s + %s + %s    %s |\n";
+
+    table_data.push_back({template_header, "memory breakdown [MiB]", "total", "free", "self", "model", "context", "compute", "unaccounted"});
+
+    constexpr size_t MiB = 1024 * 1024;
     const std::vector<std::string> desc_prefixes_strip = {"NVIDIA ", "GeForce ", "Tesla ", "AMD ", "Radeon ", "Instinct "};
 
     // track seen buffer types to avoid double counting:
@@ -1032,130 +964,104 @@ std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const stru
         }
     }
 
-    std::vector<common_memory_breakdown_row> rows;
-    rows.reserve(devices.size() + 1 + memory_breakdown.size());
-
+    // print memory breakdown for each device:
     for (size_t i = 0; i < devices.size(); i++) {
         ggml_backend_dev_t dev = devices[i];
+        llama_memory_breakdown_data mb = mb_dev[i];
 
-        common_memory_breakdown_row row;
-        row.kind        = common_memory_breakdown_row::DEVICE;
-        row.name        = ggml_backend_dev_name(dev);
-        row.description = ggml_backend_dev_description(dev);
+        const std::string name = ggml_backend_dev_name(dev);
+        std::string desc = ggml_backend_dev_description(dev);
         for (const std::string & prefix : desc_prefixes_strip) {
-            if (row.description.length() >= prefix.length() && row.description.substr(0, prefix.length()) == prefix) {
-                row.description = row.description.substr(prefix.length());
+            if (desc.length() >= prefix.length() && desc.substr(0, prefix.length()) == prefix) {
+                desc = desc.substr(prefix.length());
             }
         }
 
         size_t free, total;
         ggml_backend_dev_memory(dev, &free, &total);
-        row.total   = total;
-        row.free    = free;
-        row.model   = mb_dev[i].model;
-        row.context = mb_dev[i].context;
-        row.compute = mb_dev[i].compute;
-        rows.push_back(std::move(row));
+
+        const size_t self = mb.model + mb.context + mb.compute;
+        const int64_t unaccounted = static_cast<int64_t>(total) - static_cast<int64_t>(free) - static_cast<int64_t>(self);
+
+        table_data.push_back({
+            template_gpu,
+            "  - " + name + " (" + desc + ")",
+            std::to_string(total / MiB),
+            std::to_string(free / MiB),
+            std::to_string(self / MiB),
+            std::to_string(mb.model / MiB),
+            std::to_string(mb.context / MiB),
+            std::to_string(mb.compute / MiB),
+            std::to_string(unaccounted / static_cast<int64_t>(MiB))});
+
+        rows.push_back({
+            {"kind",        "device"},
+            {"name",        name},
+            {"description", desc},
+            {"total",       total / MiB},
+            {"free",        free / MiB},
+            {"self",        self / MiB},
+            {"model",       mb.model / MiB},
+            {"context",     mb.context / MiB},
+            {"compute",     mb.compute / MiB},
+            {"unaccounted", unaccounted / static_cast<int64_t>(MiB)},
+        });
     }
 
+    // print memory breakdown for host:
     {
-        common_memory_breakdown_row row;
-        row.kind    = common_memory_breakdown_row::HOST;
-        row.name    = "Host";
-        row.model   = mb_host.model;
-        row.context = mb_host.context;
-        row.compute = mb_host.compute;
-        rows.push_back(std::move(row));
+        const size_t self = mb_host.model + mb_host.context + mb_host.compute;
+        table_data.push_back({
+            template_other,
+            "  - Host",
+            "", // total
+            "", // free
+            std::to_string(self / MiB),
+            std::to_string(mb_host.model / MiB),
+            std::to_string(mb_host.context / MiB),
+            std::to_string(mb_host.compute / MiB),
+            ""}); // unaccounted
+
+        rows.push_back({
+            {"kind",    "host"},
+            {"name",    "Host"},
+            {"self",    self / MiB},
+            {"model",   mb_host.model / MiB},
+            {"context", mb_host.context / MiB},
+            {"compute", mb_host.compute / MiB},
+        });
     }
 
-    // all remaining buffer types:
+    // print memory breakdown for all remaining buffer types:
     for (const auto & buft_mb : memory_breakdown) {
         ggml_backend_buffer_type_t          buft = buft_mb.first;
         const llama_memory_breakdown_data & mb   = buft_mb.second;
         if (seen_buffer_types.count(buft) == 1) {
             continue;
         }
-        common_memory_breakdown_row row;
-        row.kind    = common_memory_breakdown_row::BUFFER_TYPE;
-        row.name    = ggml_backend_buft_name(buft);
-        row.model   = mb.model;
-        row.context = mb.context;
-        row.compute = mb.compute;
-        rows.push_back(std::move(row));
-
-        seen_buffer_types.insert(buft);
-    }
-
-    return rows;
-}
-
-void common_memory_breakdown_print(const struct llama_context * ctx) {
-    const std::vector<common_memory_breakdown_row> rows = common_memory_breakdown_rows(ctx);
-
-    std::vector<std::array<std::string, 9>> table_data;
-    table_data.reserve(rows.size() + 1);
-
-    // same data as the table below, for --log-jsonl consumers
-    common_json json_rows = common_json::array();
-    const std::string template_header = "%s: | %s | %s   %s    %s   %s   %s   %s    %s |\n";
-    const std::string template_gpu    = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";
-    const std::string template_other  = "%s: | %s | %s   %s    %s = %s + %s + %s    %s |\n";
-
-    table_data.push_back({template_header, "memory breakdown [MiB]", "total", "free", "self", "model", "context", "compute", "unaccounted"});
-
-    constexpr size_t MiB = 1024 * 1024;
-
-    for (const auto & row : rows) {
-        const size_t self = row.model + row.context + row.compute;
-
-        if (row.kind == common_memory_breakdown_row::DEVICE) {
-            const int64_t unaccounted = row.total - row.free - static_cast<int64_t>(self);
-
-            table_data.push_back({
-                template_gpu,
-                "  - " + row.name + " (" + row.description + ")",
-                std::to_string(row.total / MiB),
-                std::to_string(row.free / MiB),
-                std::to_string(self / MiB),
-                std::to_string(row.model / MiB),
-                std::to_string(row.context / MiB),
-                std::to_string(row.compute / MiB),
-                std::to_string(unaccounted / static_cast<int64_t>(MiB))});
-
-            json_rows.push_back({
-                {"kind",        "device"},
-                {"name",        row.name},
-                {"description", row.description},
-                {"total",       row.total / MiB},
-                {"free",        row.free / MiB},
-                {"self",        self / MiB},
-                {"model",       row.model / MiB},
-                {"context",     row.context / MiB},
-                {"compute",     row.compute / MiB},
-                {"unaccounted", unaccounted / static_cast<int64_t>(MiB)},
-            });
-            continue;
-        }
-
+        const std::string name = ggml_backend_buft_name(buft);
+        const size_t self = mb.model + mb.context + mb.compute;
         table_data.push_back({
             template_other,
-            "  - " + row.name,
+            "  - " + name,
             "", // total
             "", // free
             std::to_string(self / MiB),
-            std::to_string(row.model / MiB),
-            std::to_string(row.context / MiB),
-            std::to_string(row.compute / MiB),
+            std::to_string(mb.model / MiB),
+            std::to_string(mb.context / MiB),
+            std::to_string(mb.compute / MiB),
             ""}); // unaccounted
 
-        json_rows.push_back({
-            {"kind",    row.kind == common_memory_breakdown_row::HOST ? "host" : "buffer_type"},
-            {"name",    row.name},
+        rows.push_back({
+            {"kind",    "buffer_type"},
+            {"name",    name},
             {"self",    self / MiB},
-            {"model",   row.model / MiB},
-            {"context", row.context / MiB},
-            {"compute", row.compute / MiB},
+            {"model",   mb.model / MiB},
+            {"context", mb.context / MiB},
+            {"compute", mb.compute / MiB},
         });
+
+        seen_buffer_types.insert(buft);
     }
 
     for (size_t j = 1; j < table_data[0].size(); j++) {
@@ -1175,7 +1081,7 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
 
     LOG_JSON("fit_memory_breakdown", common_json({
         {"unit", "MiB"},
-        {"rows", json_rows},
+        {"rows", rows},
     }));
 }
 

common/fit.h

@@ -3,7 +3,6 @@
 #include "ggml.h"
 #include "llama.h"
 
-#include <string>
 #include <vector>
 
 enum common_params_fit_status {
@@ -46,47 +45,6 @@ void common_fit_print(
 
 void common_memory_breakdown_print(const llama_context * ctx);
 
-// one row of the memory breakdown: a device, the host or another buffer type
-struct common_memory_breakdown_row {
-    enum kind_t { DEVICE, HOST, BUFFER_TYPE };
-
-    kind_t      kind = DEVICE;
-    std::string name;        // device or buffer type name, "Host" for the host row
-    std::string description; // device description, empty otherwise
-    int64_t     total = 0;   // device memory in bytes, 0 unless kind == DEVICE
-    int64_t     free  = 0;   // free device memory in bytes, 0 unless kind == DEVICE
-    size_t      model   = 0; // bytes held by this context, by kind of buffer
-    size_t      context = 0;
-    size_t      compute = 0;
-};
-
-// memory a model needs at the min and max context --fit may pick
-struct common_memory_reports {
-    uint32_t n_ctx_train = 0;
-    uint32_t n_ctx_max   = 0; // the largest context that fits the free device memory right now
-    uint32_t n_ctx_min   = 0; // --fit-ctx (the smallest context --fit may pick), with the placement set in params
-    std::vector<common_memory_breakdown_row> info_max; // breakdown at n_ctx_max
-    std::vector<common_memory_breakdown_row> info_min; // breakdown at n_ctx_min
-};
-
-// rows of common_memory_breakdown_print: model devices first, then host
-std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const llama_context * ctx);
-
-// load model + context with no_alloc and return the memory breakdown, params are used as-is
-// n_ctx_train and n_ctx are optional outputs; throws on failure
-std::vector<common_memory_breakdown_row> common_fit_measure(
-                         const char * path_model,
-           const llama_model_params * mparams,
-         const llama_context_params * cparams,
-                           uint32_t * n_ctx_train,
-                           uint32_t * n_ctx,
-                     ggml_log_level   log_level);
-
-struct common_params;
-
-// measure min and max without allocating, params may be modified as with --fit; throws on failure
-common_memory_reports common_measure_memory(common_params & params);
-
 struct common_device_memory_data {
     int64_t total;
     int64_t free;

common/jinja/caps.cpp

@@ -6,9 +6,7 @@
 // we can remove it in the future when we figure out a better way to define inputs using jinja::value
 #include "json.h"
 
-#include <algorithm>
 #include <functional>
-#include <optional>
 #include <sstream>
 
 #define FILENAME "jinja-caps"
@@ -186,10 +184,6 @@ caps caps_get(jinja::program & prog) {
                     {
                         {"role", "user"},
                         {"content", json::array({
-                            {
-                                {"type", "text"},
-                                {"text", content_marker}
-                            }
                         })}
                     }
                 });
@@ -590,58 +584,6 @@ caps caps_get(jinja::program & prog) {
         }
     );
 
-    if (result.supports_reasoning_effort) {
-        JJ_DEBUG("%s\n", ">>> Running capability check: reasoning effort levels");
-
-        static const std::vector<std::string> candidates = { "minimal", "low", "medium", "high", "xhigh", "max" };
-
-        // renders a single user turn with the given effort, nothing when the template raises
-        const auto render = [&](const std::string & effort) {
-            std::optional<std::string> rendered;
-            caps_try_execute(
-                prog,
-                [&]() {
-                    // messages
-                    return json::array({
-                        {
-                            {"role", "user"},
-                            {"content", "User message"}
-                        },
-                    });
-                },
-                [&](context & ctx) {
-                    ctx.set_val("enable_thinking", mk_val<value_bool>(true));
-                    caps_apply_reasoning_effort(ctx, effort);
-                },
-                nullptr, // tools_fn
-                [&](context &, bool success, value &, value &, const std::string & output) {
-                    if (success) {
-                        rendered = output;
-                    }
-                }
-            );
-            return rendered;
-        };
-
-        // an unknown level shows the render of the template fallback, a level that renders the same is not honored
-        // levels sharing a render are aliases and the highest one is kept
-        const auto unknown = render("REASONING_EFFORT_UNKNOWN");
-        std::vector<std::string> renders;
-        for (const auto & level : candidates) {
-            const auto rendered = render(level);
-            if (!rendered || rendered == unknown) {
-                continue;
-            }
-            const auto it = std::find(renders.begin(), renders.end(), *rendered);
-            if (it != renders.end()) {
-                result.reasoning_effort_levels[it - renders.begin()] = level;
-            } else {
-                renders.push_back(*rendered);
-                result.reasoning_effort_levels.push_back(level);
-            }
-        }
-    }
-
     JJ_DEBUG("%s\n", result.to_string().c_str());
 
     return result;

common/jinja/caps.h

@@ -4,7 +4,6 @@
 
 #include <string>
 #include <map>
-#include <vector>
 
 namespace jinja {
 
@@ -20,9 +19,6 @@ struct caps {
     // supports reasoning effort levels
     bool supports_reasoning_effort = false;
 
-    // reasoning effort levels the template renders distinctly, from lowest to highest
-    std::vector<std::string> reasoning_effort_levels;
-
     // one of the 2 content capabilities must be true
     bool supports_string_content = true;
     bool supports_typed_content = false;

common/jinja/parser.cpp

@@ -21,17 +21,6 @@ static bool is_type(const statement_ptr & ptr) {
 class parser {
     const std::vector<token> & tokens;
     size_t current = 0;
-    int depth = 0;
-
-    struct depth_guard {
-        parser & p;
-        depth_guard(parser & p) : p(p) {
-            if (++p.depth > JINJA_RECURSIVE_LIMIT) {
-                throw parser_exception("Parser Error: template is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep", p.source, p.peek().pos);
-            }
-        }
-        ~depth_guard() { p.depth--; }
-    };
 
     std::string source; // for error reporting
 
@@ -105,7 +94,6 @@ private:
     }
 
     statement_ptr parse_any() {
-        depth_guard guard(*this);
         size_t start_pos = current;
         switch (peek().t) {
             case token::comment:
@@ -266,7 +254,6 @@ private:
             size_t pos0 = current;
             ++current; // consume {%
             ++current; // consume 'elif'
-            depth_guard guard(*this);
             alternate.push_back(parse_if_statement(pos0)); // nested If
         } else if (is_statement({"else"})) {
             ++current; // consume {%
@@ -343,7 +330,6 @@ private:
     }
 
     statement_ptr parse_if_expression() {
-        depth_guard guard(*this);
         auto a = parse_logical_or_expression();
         if (is_identifier("if")) {
             // Ternary expression
@@ -387,7 +373,6 @@ private:
     statement_ptr parse_logical_negation_expression() {
         // Try parse unary operators
         if (is_identifier("not")) {
-            depth_guard guard(*this);
             size_t start_pos = current;
             auto op = next();
             return mk_stmt<unary_expression>(start_pos, op, parse_logical_negation_expression());
@@ -473,7 +458,6 @@ private:
 
     statement_ptr parse_unary_expression() {
         if (is(token::unary_operator)) {
-            depth_guard guard(*this);
             size_t start_pos = current;
             auto op = next();
             return mk_stmt<unary_expression>(start_pos, op, parse_unary_expression());
@@ -493,11 +477,9 @@ private:
         size_t start_pos = current;
         auto expr = mk_stmt<call_expression>(start_pos, std::move(callee), parse_args());
         auto member = parse_member_expression(std::move(expr)); // foo.x().y
-        if (!is(token::open_paren)) {
-            return member;
-        }
-        depth_guard guard(*this);
-        return parse_call_expression(std::move(member)); // foo.x()()
+        return is(token::open_paren)
+            ? parse_call_expression(std::move(member)) // foo.x()()
+            : std::move(member);
     }
 
     statements parse_args() {

common/jinja/runtime.cpp

@@ -3,7 +3,6 @@
 #include "value.h"
 #include "utils.h"
 
-#include <exception>
 #include <string>
 #include <vector>
 #include <memory>
@@ -53,39 +52,32 @@ static void ensure_key_type_allowed(const value & val) {
 
 // execute with error handling
 value statement::execute(context & ctx) const {
-    std::exception_ptr pending;
-    std::string message;
     try {
         return execute_impl(ctx);
     } catch (const continue_statement::signal & /* ex */) {
-        pending = std::current_exception();
+        throw;
     } catch (const break_statement::signal & /* ex */) {
-        pending = std::current_exception();
+        throw;
     } catch (const rethrown_exception & /* ex */) {
-        pending = std::current_exception();
+        throw;
     } catch (const not_implemented_exception & /* ex */) {
-        pending = std::current_exception();
+        throw;
     } catch (const std::exception & e) {
         const std::string & source = *ctx.src;
         if (source.empty()) {
             std::ostringstream oss;
             oss << "\nError executing " << type() << " at position " << pos << ": " << e.what();
-            message = oss.str();
+            throw rethrown_exception(oss.str());
         } else {
             std::ostringstream oss;
             oss << "\n------------\n";
             oss << "While executing " << type() << " at " << get_line_col(source, pos) << " in source:\n";
             oss << peak_source(source, pos) << "\n";
             oss << "Error: " << e.what();
-            message = oss.str();
+            // throw as another exception to avoid repeated formatting
+            throw rethrown_exception(oss.str());
         }
     }
-    // throw outside the catch: MSVC keeps the unwound frames on the stack while a catch block runs
-    if (pending) {
-        std::rethrow_exception(pending);
-    }
-    // throw as another exception to avoid repeated formatting
-    throw rethrown_exception(message);
 }
 
 value identifier::execute_impl(context & ctx) const {
@@ -755,11 +747,7 @@ value macro_statement::execute_impl(context & ctx) const {
     std::string name = cast_stmt<identifier>(this->name)->val;
 
     const func_handler func = [this, name](const func_args & args) -> value {
-        if (args.ctx.macro_depth >= JINJA_RECURSIVE_LIMIT) {
-            throw std::runtime_error("macro calls are nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
-        }
         context macro_ctx(args.ctx); // new scope for macro execution
-        macro_ctx.macro_depth = args.ctx.macro_depth + 1;
 
         bind_parameters(name, this->args, args, macro_ctx);
 
@@ -790,11 +778,7 @@ value call_statement::execute_impl(context & ctx) const {
     context caller_ctx(ctx); // new scope for caller execution
 
     const func_handler func = [this, caller_ctx = std::move(caller_ctx)](const func_args & args) -> value {
-        if (args.ctx.macro_depth >= JINJA_RECURSIVE_LIMIT) {
-            throw std::runtime_error("macro calls are nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
-        }
         context block_ctx(caller_ctx); // new scope for block execution
-        block_ctx.macro_depth = args.ctx.macro_depth + 1;
 
         bind_parameters("caller", this->caller_args, args, block_ctx);
 

common/jinja/runtime.h

@@ -58,8 +58,6 @@ struct context {
 
     bool is_get_stats = false; // whether to collect stats
 
-    int macro_depth = 0;
-
     visitor_fn visitor;
 
     // src is optional, used for error reporting
@@ -80,7 +78,6 @@ struct context {
         src(parent.src),
         current_time(parent.current_time),
         is_get_stats(parent.is_get_stats),
-        macro_depth(parent.macro_depth),
         // inherit variables (for example, when entering a new scope)
         env(mk_val<value_object>(*parent.env)) {}
 
@@ -713,7 +710,7 @@ struct rethrown_exception : public std::exception {
 
 //////////////////////
 
-static void gather_string_parts_recursive(const value & val, value_string & parts, int depth = 0) {
+static void gather_string_parts_recursive(const value & val, value_string & parts) {
     // TODO: probably allow print value_none as "None" string? currently this breaks some templates
     if (is_val<value_string>(val)) {
         const auto & str_val = cast_val<value_string>(val)->val_str;
@@ -721,12 +718,10 @@ static void gather_string_parts_recursive(const value & val, value_string & part
     } else if (is_val<value_int>(val) || is_val<value_float>(val) || is_val<value_bool>(val)) {
         std::string str_val = val->as_string().str();
         parts->val_str.append(str_val);
-    } else if (is_val<value_array>(val) && depth >= JINJA_RECURSIVE_LIMIT) {
-        throw std::runtime_error("value is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
     } else if (is_val<value_array>(val)) {
         auto items = cast_val<value_array>(val)->as_array();
         for (const auto & item : items) {
-            gather_string_parts_recursive(item, parts, depth + 1);
+            gather_string_parts_recursive(item, parts);
         }
     }
 }

common/jinja/utils.h

@@ -6,8 +6,6 @@
 #include <cstdint>
 #include <cstring>
 
-#define JINJA_RECURSIVE_LIMIT 128
-
 namespace jinja {
 
 static void string_replace_all(std::string & s, const std::string & search, const std::string & replace) {

common/jinja/value.cpp

@@ -6,7 +6,6 @@
 #include "json.h"
 
 #include <sstream>
-#include <stdexcept>
 #include <string>
 #include <cctype>
 #include <cfloat>
@@ -300,53 +299,74 @@ static value & get_attribute(const value & val, const value & attr, value & defa
     return default_val;
 }
 
-template<bool is_reject, bool lookup_attr>
-static value select_or_reject(const func_args & args) {
-    // example: array | select("odd") -> test_is_odd(item)
-    // example: array | selectattr("status", "equalto", "active") -> test_is_equalto(item.status, "active")
-    constexpr size_t test_pos = lookup_attr ? 2 : 1;
-    args.ensure_count(test_pos);
-    args.ensure_val<value_array>(args.get_pos(0));
-    value attr;
-    if constexpr (lookup_attr) {
-        attr = args.get_pos(1);
-        args.ensure_val<value_string>(attr);
-    }
-
-    auto & builtins = global_builtins();
-    auto test_it = builtins.end();
-    if (args.count() > test_pos) {
-        args.ensure_val<value_string>(args.get_pos(test_pos));
-        std::string test_name = args.get_pos(test_pos)->as_string().str();
-        test_it = builtins.find("test_is_" + test_name);
-        if (test_it == builtins.end()) {
-            throw raised_exception(args.func_name + ": unknown test '" + test_name + "'");
-        }
-    }
+template<bool is_reject>
+static value selectattr(const func_args & args) {
+    args.ensure_count(2, 4);
+    args.ensure_vals<value_array, value_string, value_string, value_string>(true, true, false, false);
 
     auto arr = args.get_pos(0)->as_array();
+    auto attribute = args.get_pos(1);
     auto out = mk_val<value_array>();
     value val_default = mk_val<value_undefined>();
 
-    for (const auto & item : arr) {
-        value val = item;
-        if constexpr (lookup_attr) {
-            val = get_attribute(item, attr, val_default);
+    if (args.count() == 2) {
+        // example: array | selectattr("active")
+        for (const auto & item : arr) {
+            value attr_val = get_attribute(item, attribute, val_default);
+            bool is_selected = attr_val->as_bool();
+            if constexpr (is_reject) is_selected = !is_selected;
+            if (is_selected) out->push_back(item);
         }
-        bool is_selected;
-        if (test_it == builtins.end()) {
-            is_selected = val->as_bool();
-        } else {
+        return out;
+
+    } else if (args.count() == 3) {
+        // example: array | selectattr("equalto", "text")
+        // translated to: test_is_equalto(item, "text")
+        std::string test_name = args.get_pos(1)->as_string().str();
+        value test_val = args.get_pos(2);
+        auto & builtins = global_builtins();
+        auto it = builtins.find("test_is_" + test_name);
+        if (it == builtins.end()) {
+            throw raised_exception("selectattr: unknown test '" + test_name + "'");
+        }
+        auto test_fn = it->second;
+        for (const auto & item : arr) {
             func_args test_args(args.ctx);
-            test_args.push_back(val);
-            for (size_t i = test_pos + 1; i < args.count(); i++) {
-                test_args.push_back(args.get_pos(i));
-            }
-            is_selected = test_it->second(test_args)->as_bool();
+            test_args.push_back(item); // current object
+            test_args.push_back(test_val); // extra argument
+            value test_result = test_fn(test_args);
+            bool is_selected = test_result->as_bool();
+            if constexpr (is_reject) is_selected = !is_selected;
+            if (is_selected) out->push_back(item);
         }
-        if constexpr (is_reject) is_selected = !is_selected;
-        if (is_selected) out->push_back(item);
+        return out;
+
+    } else if (args.count() == 4) {
+        // example: array | selectattr("status", "equalto", "active")
+        // translated to: test_is_equalto(item.status, "active")
+        std::string test_name = args.get_pos(2)->as_string().str();
+        auto extra_arg = args.get_pos(3);
+        auto & builtins = global_builtins();
+        auto it = builtins.find("test_is_" + test_name);
+        if (it == builtins.end()) {
+            throw raised_exception("selectattr: unknown test '" + test_name + "'");
+        }
+        auto test_fn = it->second;
+        for (const auto & item : arr) {
+            value attr_val = get_attribute(item, attribute, val_default);
+            func_args test_args(args.ctx);
+            test_args.push_back(attr_val); // attribute value
+            test_args.push_back(extra_arg); // extra argument
+            value test_result = test_fn(test_args);
+            bool is_selected = test_result->as_bool();
+            if constexpr (is_reject) is_selected = !is_selected;
+            if (is_selected) out->push_back(item);
+        }
+        return out;
+    } else {
+        throw raised_exception("selectattr: invalid number of arguments");
     }
+
     return out;
 }
 
@@ -1075,10 +1095,10 @@ const func_builtins & value_array_t::get_builtins() const {
             auto arr = slice(val->as_array(), start, stop, step);
             return is_val<value_tuple>(val) ? mk_val<value_tuple>(std::move(arr)) : mk_val<value_array>(std::move(arr));
         }},
-        {"selectattr", select_or_reject<false, true>},
-        {"select", select_or_reject<false, false>},
-        {"rejectattr", select_or_reject<true, true>},
-        {"reject", select_or_reject<true, false>},
+        {"selectattr", selectattr<false>},
+        {"select", selectattr<false>},
+        {"rejectattr", selectattr<true>},
+        {"reject", selectattr<true>},
         {"join", [](const func_args & args) -> value {
             args.ensure_count(1, 3);
             if (!is_val<value_array>(args.get_pos(0))) {
@@ -1440,8 +1460,7 @@ const func_builtins & value_undefined_t::get_builtins() const {
 //////////////////////////////////
 
 
-// the runtime walks values with recursion, a deeper input overflows the stack
-static value from_json(const common_json & j, bool mark_input, int depth = 0) {
+static value from_json(const common_json & j, bool mark_input) {
     if (j.is_null()) {
         return mk_val<value_none>();
     } else if (j.is_boolean()) {
@@ -1456,18 +1475,16 @@ static value from_json(const common_json & j, bool mark_input, int depth = 0) {
             str->mark_input();
         }
         return str;
-    } else if ((j.is_array() || j.is_object()) && depth >= JINJA_RECURSIVE_LIMIT) {
-        throw std::invalid_argument("JSON is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
     } else if (j.is_array()) {
         auto arr = mk_val<value_array>();
         for (const auto & item : j) {
-            arr->push_back(from_json(item, mark_input, depth + 1));
+            arr->push_back(from_json(item, mark_input));
         }
         return arr;
     } else if (j.is_object()) {
         auto obj = mk_val<value_object>();
         for (auto it = j.begin(); it != j.end(); ++it) {
-            obj->insert(it.key(), from_json(it.value(), mark_input, depth + 1));
+            obj->insert(it.key(), from_json(it.value(), mark_input));
         }
         return obj;
     } else {
@@ -1662,8 +1679,6 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
             }
         }
         oss << "\"";
-    } else if ((is_val<value_array>(val) || is_val<value_object>(val)) && curr_lvl >= JINJA_RECURSIVE_LIMIT) {
-        throw std::runtime_error("value is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
     } else if (is_val<value_array>(val)) {
         const auto & arr = val->as_array();
         oss << "[";
@@ -1729,20 +1744,17 @@ std::string value_to_string_repr(const value & val) {
 }
 
 // stats utility
-void value_t::stats_t::mark_used(value & val, bool deep, int depth) {
+void value_t::stats_t::mark_used(value & val, bool deep) {
     val->stats.used = true;
-    if (deep && (is_val<value_array>(val) || is_val<value_object>(val)) && depth >= JINJA_RECURSIVE_LIMIT) {
-        throw std::runtime_error("value is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
-    }
     if (deep) {
         if (is_val<value_array>(val)) {
             for (auto & item : val->val_arr) {
-                mark_used(item, deep, depth + 1);
+                mark_used(item, deep);
             }
         } else if (is_val<value_object>(val)) {
             for (auto & pair : val->val_obj) {
-                mark_used(pair.first, deep, depth + 1);
-                mark_used(pair.second, deep, depth + 1);
+                mark_used(pair.first, deep);
+                mark_used(pair.second, deep);
             }
         }
     }

common/jinja/value.h

@@ -119,7 +119,7 @@ struct value_t {
         // ops can be builtin calls or operators: "array_access", "object_access"
         std::set<std::string> ops;
         // utility to recursively mark value and its children as used
-        static void mark_used(value & val, bool deep = false, int depth = 0);
+        static void mark_used(value & val, bool deep = false);
     } stats;
 
     value_t() = default;

common/json-schema-to-grammar.cpp

@@ -272,12 +272,15 @@ static bool is_reserved_name(const std::string & name) {
     return RESERVED_NAMES.find(name) != RESERVED_NAMES.end();
 }
 
+static std::regex INVALID_RULE_CHARS_RE("[^a-zA-Z0-9-]+");
 static std::regex GRAMMAR_LITERAL_ESCAPE_RE("[\r\n\"\\\\]");
 static std::regex GRAMMAR_RANGE_LITERAL_ESCAPE_RE("[\r\n\"\\]\\-\\\\]");
 static std::unordered_map<char, std::string> GRAMMAR_LITERAL_ESCAPES = {
     {'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"}, {'\\', "\\\\"}
 };
 
+static const int MAX_PATTERN_DEPTH = 100;
+
 static std::unordered_set<char> NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?', '^', '$'};
 static std::unordered_set<char> ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = {'^', '$', '.', '[', ']', '(', ')', '|', '{', '}', '*', '+', '?'};
 
@@ -309,22 +312,6 @@ static std::string format_literal(const std::string & literal) {
 
 std::string gbnf_format_literal(const std::string & literal) { return format_literal(literal); }
 
-// same as std::regex_replace with [^a-zA-Z0-9-]+ -> "-", no std::regex: libstdc++ recurses per char
-std::string gbnf_format_rule_name(const std::string & name) {
-    std::string out;
-    bool in_invalid = false;
-    for (char c : name) {
-        if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-') {
-            out += c;
-            in_invalid = false;
-        } else if (!in_invalid) {
-            out += '-';
-            in_invalid = true;
-        }
-    }
-    return out;
-}
-
 static size_t gbnf_escape_length(const std::string & pattern, size_t pos) {
     if (pos + 1 >= pattern.length() || pattern[pos] != '\\') {
         return 0;
@@ -360,17 +347,6 @@ private:
     std::unordered_set<std::string> _refs_being_resolved;
     std::vector<std::string> _errors;
     std::vector<std::string> _warnings;
-    int _depth = 0;
-
-    struct depth_guard {
-        int & depth;
-        depth_guard(int & depth) : depth(depth) {
-            if (++depth > COMMON_RECURSIVE_LIMIT) {
-                throw std::invalid_argument("JSON schema is nested more than " + std::to_string(COMMON_RECURSIVE_LIMIT) + " levels deep");
-            }
-        }
-        ~depth_guard() { depth--; }
-    };
 
     template <typename T>
     static const T & as(const common_chat_schema & node) {
@@ -378,7 +354,7 @@ private:
     }
 
     std::string _add_rule(const std::string & name, const std::string & rule) {
-        std::string esc_name = gbnf_format_rule_name(name);
+        std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-");
         if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) {
             _rules[esc_name] = rule;
             return esc_name;
@@ -506,7 +482,7 @@ private:
                         }
                     }
                     paren_depth++;
-                    if (paren_depth > COMMON_RECURSIVE_LIMIT) {
+                    if (paren_depth > MAX_PATTERN_DEPTH) {
                         throw unsupported_pattern("pattern nesting too deep");
                     }
                     seq.emplace_back("(" + to_rule(transform()) + ")", false);
@@ -668,11 +644,7 @@ private:
         std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
         std::ostringstream out;
         out << "[\"] ( ";
-        // one level per char of the longest name
-        std::function<void(size_t, int)> visit = [&](size_t idx, int depth) {
-            if (depth >= COMMON_RECURSIVE_LIMIT) {
-                throw std::invalid_argument("JSON schema property name is longer than " + std::to_string(COMMON_RECURSIVE_LIMIT) + " chars");
-            }
+        std::function<void(size_t)> visit = [&](size_t idx) {
             const auto & node = trie.nodes[idx];
             std::string rejects;
             auto first = true;
@@ -687,7 +659,7 @@ private:
                 out << "[" << c << "]";
                 if (!trie.nodes[child].children.empty()) {
                     out << " (";
-                    visit(child, depth + 1);
+                    visit(child);
                     out << ")";
                 } else {
                     out << " " << char_rule << "+";
@@ -697,7 +669,7 @@ private:
                 out << " | [^\"" << rejects << "] " << char_rule << "*";
             }
         };
-        visit(0, 0);
+        visit(0);
 
         out << " )";
         if (trie.nodes[0].pattern < 0) {
@@ -710,7 +682,8 @@ private:
     std::string _resolve_ref(const common_chat_schema_ref & schema) {
         auto it = schema.ref.find('#');
         std::string ref_fragment = it != std::string::npos ? schema.ref.substr(it + 1) : schema.ref;
-        std::string ref_name = "ref" + gbnf_format_rule_name(ref_fragment);
+        static const std::regex nonalphanumeric_regex(R"([^a-zA-Z0-9-]+)");
+        std::string ref_name = "ref" + std::regex_replace(ref_fragment, nonalphanumeric_regex, "-");
         if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(schema.ref) == _refs_being_resolved.end()) {
             if (!schema.target) {
                 _errors.push_back("Unresolved $ref " + schema.ref);
@@ -860,13 +833,10 @@ public:
         std::unordered_set<std::string> required;
         std::vector<std::pair<std::string, const common_chat_schema *>> properties;
         std::map<std::string, size_t> enum_values;
-        std::function<void(const common_chat_schema &, bool, int)> add_component = [&](const common_chat_schema & comp, bool is_required, int depth) {
-            if (depth >= COMMON_RECURSIVE_LIMIT) {
-                throw std::invalid_argument("JSON schema is nested more than " + std::to_string(COMMON_RECURSIVE_LIMIT) + " levels deep");
-            }
+        std::function<void(const common_chat_schema &, bool)> add_component = [&](const common_chat_schema & comp, bool is_required) {
             if (comp.kind() == common_chat_schema::KIND_REF) {
                 if (const auto * target = as<common_chat_schema_ref>(comp).target) {
-                    add_component(*target, is_required, depth + 1);
+                    add_component(*target, is_required);
                 }
             } else if (comp.kind() == common_chat_schema::KIND_OBJECT) {
                 for (const auto & prop : as<common_chat_schema_object>(comp).properties) {
@@ -884,10 +854,10 @@ public:
         for (const auto & child : schema.children) {
             if (child->kind() == common_chat_schema::KIND_ANY_OF) {
                 for (const auto & alt : as<common_chat_schema_any_of>(*child).children) {
-                    add_component(*alt, false, 0);
+                    add_component(*alt, false);
                 }
             } else {
-                add_component(*child, true, 0);
+                add_component(*child, true);
             }
         }
         if (!enum_values.empty()) {
@@ -905,8 +875,6 @@ public:
     }
 
     std::string visit(const common_chat_schema & schema, const std::string & name) {
-        // the $ref targets are visited too, a $ref chain is not bounded by the builder
-        depth_guard guard(_depth);
         std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name;
         std::string sub_name  = name + (name.empty() ? "" : "-");
 

common/json-schema-to-grammar.h

@@ -19,6 +19,5 @@ struct common_grammar_options {
 };
 
 std::string gbnf_format_literal(const std::string & literal);
-std::string gbnf_format_rule_name(const std::string & name);
 
 std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options = {});

common/json-schema.cpp

@@ -9,18 +9,6 @@
 #include <utility>
 #include <vector>
 
-namespace {
-struct depth_guard {
-    int & depth;
-    depth_guard(int & depth) : depth(depth) {
-        if (++depth > COMMON_RECURSIVE_LIMIT) {
-            throw std::invalid_argument("JSON schema is nested more than " + std::to_string(COMMON_RECURSIVE_LIMIT) + " levels deep");
-        }
-    }
-    ~depth_guard() { depth--; }
-};
-} // namespace
-
 class common_chat_schema_builder {
     const common_json &      root_;
     common_chat_schema_document & doc_;
@@ -236,14 +224,7 @@ class common_chat_schema_builder {
         return node;
     }
 
-    int depth_ = 0;
-
     common_chat_schema_ptr build_node(const common_json & schema, const std::string & path) {
-        depth_guard guard(depth_);
-        return build_node_impl(schema, path);
-    }
-
-    common_chat_schema_ptr build_node_impl(const common_json & schema, const std::string & path) {
         if (!schema.is_object()) {
             fail(path, "schema must be an object");
         }
@@ -384,9 +365,7 @@ static common_chat_schema::value_type json_type(const common_json & value) {
     return common_chat_schema::TYPE_OBJECT;
 }
 
-// depth counts the $ref, anyOf and allOf hops, a $ref chain is not bounded by the builder
-static common_chat_schema::type_set value_types_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited, int & depth) {
-    depth_guard guard(depth);
+static common_chat_schema::type_set value_types_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited) {
     switch (s.kind()) {
         case common_chat_schema::KIND_ANY:
             return common_chat_schema::type_set::all();
@@ -420,21 +399,21 @@ static common_chat_schema::type_set value_types_impl(const common_chat_schema &
                 // a cycle contributes no type, to be safe
                 return {};
             }
-            auto types = value_types_impl(*target, visited, depth);
+            auto types = value_types_impl(*target, visited);
             visited.erase(target);
             return types;
         }
         case common_chat_schema::KIND_ANY_OF: {
             common_chat_schema::type_set types;
             for (const auto & child : static_cast<const common_chat_schema_any_of &>(s).children) {
-                types |= value_types_impl(*child, visited, depth);
+                types |= value_types_impl(*child, visited);
             }
             return types;
         }
         case common_chat_schema::KIND_ALL_OF: {
             auto types = common_chat_schema::type_set::all();
             for (const auto & child : static_cast<const common_chat_schema_all_of &>(s).children) {
-                types &= value_types_impl(*child, visited, depth);
+                types &= value_types_impl(*child, visited);
             }
             return types;
         }
@@ -444,12 +423,10 @@ static common_chat_schema::type_set value_types_impl(const common_chat_schema &
 
 common_chat_schema::type_set common_chat_schema::value_types() const {
     std::unordered_set<const common_chat_schema *> visited;
-    int depth = 0;
-    return value_types_impl(*this, visited, depth);
+    return value_types_impl(*this, visited);
 }
 
-static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited, int & depth) {
-    depth_guard guard(depth);
+static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited) {
     switch (s.kind()) {
         case common_chat_schema::KIND_STRING:
             return true;
@@ -468,13 +445,13 @@ static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set<
             if (!target || !visited.insert(target).second) {
                 return false;
             }
-            bool result = may_be_string_impl(*target, visited, depth);
+            bool result = may_be_string_impl(*target, visited);
             visited.erase(target);
             return result;
         }
         case common_chat_schema::KIND_ANY_OF:
             for (const auto & child : static_cast<const common_chat_schema_any_of &>(s).children) {
-                if (may_be_string_impl(*child, visited, depth)) {
+                if (may_be_string_impl(*child, visited)) {
                     return true;
                 }
             }
@@ -486,7 +463,7 @@ static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set<
                 if (child->kind() == common_chat_schema::KIND_ANY) {
                     continue;
                 }
-                if (!may_be_string_impl(*child, visited, depth)) {
+                if (!may_be_string_impl(*child, visited)) {
                     return false;
                 }
                 any_string = true;
@@ -500,8 +477,7 @@ static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set<
 
 bool common_chat_schema::may_be_string() const {
     std::unordered_set<const common_chat_schema *> visited;
-    int depth = 0;
-    return may_be_string_impl(*this, visited, depth);
+    return may_be_string_impl(*this, visited);
 }
 
 const char * common_chat_schema::kind_name(node_kind kind) {

common/parsers/deepseek.cpp

@@ -1,5 +1,4 @@
 #include "parsers.h"
-#include "log.h"
 
 // The DeepSeek V4 reference implementation renders consecutive tool results into a single
 // user block, ordered by the tool call order of the preceding assistant message (matched
@@ -76,30 +75,13 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
     // non-thinking generation prompt ends with a bare </think> instead of an empty
     // <think></think> pair.
     const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos;
-    // V4.1: DSML tag names carry a leading space, e.g. <｜DSML｜ calls>
-    const bool is_v41 = is_v4 && tmpl.source().find("｜DSML｜ calls>") != std::string::npos;
 
     std::optional<json> adjusted_messages;
     if (is_v4) {
         adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages);
     }
 
-    // V4.1 can declare tools on a system message instead of top-level "tools": parse calls to those too.
-    json tools = inputs.tools;
-    if (is_v41 && !(tools.is_array() && !tools.empty()) && inputs.messages.is_array()) {
-        for (const auto & msg : inputs.messages) {
-            if (msg.value("role", "") == "system" && msg.contains("tools") && msg.at("tools").is_array()) {
-                if (!tools.is_array()) {
-                    tools = json::array();
-                }
-                for (const auto & tool : msg.at("tools")) {
-                    tools.push_back(tool);
-                }
-            }
-        }
-    }
-
-    auto has_tools           = tools.is_array() && !tools.empty();
+    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
     auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
     auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
     auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
@@ -108,39 +90,17 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
     if (is_v4 && has_response_format) {
         additional_context = json{ { "response_format", inputs.json_schema } };
     }
-    // V4.1 raises on anything but low/high/max or 1-100: map minimal -> low, xhigh -> max, medium -> default
-    if (is_v41 && inputs.extra_context.is_object() && inputs.extra_context.contains("reasoning_effort") &&
-        inputs.extra_context.at("reasoning_effort").is_string()) {
-        const std::string effort = inputs.extra_context.at("reasoning_effort").get<std::string>();
-        json mapped;
-        if (effort == "minimal") {
-            mapped = "low";
-        } else if (effort == "xhigh") {
-            mapped = "max";
-        } else if (effort == "medium") {
-            mapped = nullptr;
-        }
-        if (effort == "minimal" || effort == "xhigh" || effort == "medium") {
-            LOG_DBG("%s: DeepSeek V4.1 reasoning_effort '%s' -> %s\n", __func__, effort.c_str(),
-                    mapped.is_null() ? "unset (template default)" : mapped.get<std::string>().c_str());
-            if (!additional_context.has_value()) {
-                additional_context = json::object();
-            }
-            (*additional_context)["reasoning_effort"] = mapped;
-        }
-    }
 
     const std::string DSML         = "｜DSML｜";
     const std::string THINK_START  = "<think>";
     const std::string THINK_END    = "</think>";
-    const std::string TAG_SP       = is_v41 ? " " : "";
-    const std::string TC_BLOCK     = is_v41 ? "calls" : is_v4 ? "tool_calls" : "function_calls";
-    const std::string FC_START     = "<" + DSML + TAG_SP + TC_BLOCK + ">";
-    const std::string FC_END       = "</" + DSML + TAG_SP + TC_BLOCK + ">";
-    const std::string INVOKE_START = "<" + DSML + TAG_SP + "invoke";
-    const std::string INVOKE_END   = "</" + DSML + TAG_SP + "invoke>";
-    const std::string PARAM_START  = "<" + DSML + TAG_SP + "parameter";
-    const std::string PARAM_END    = "</" + DSML + TAG_SP + "parameter>";
+    const std::string TC_BLOCK     = is_v4 ? "tool_calls" : "function_calls";
+    const std::string FC_START     = "<" + DSML + TC_BLOCK + ">";
+    const std::string FC_END       = "</" + DSML + TC_BLOCK + ">";
+    const std::string INVOKE_START = "<" + DSML + "invoke";
+    const std::string INVOKE_END   = "</" + DSML + "invoke>";
+    const std::string PARAM_START  = "<" + DSML + "parameter";
+    const std::string PARAM_END    = "</" + DSML + "parameter>";
     const std::string GEN_PROMPT   = "<｜Assistant｜>";
     const std::string TC_SEPARATOR = "\n\n";
 
@@ -192,7 +152,7 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
         // build tool call section first since we might need it in reasoning
         auto tool_choice = p.choice();
         if (has_tool_calls) {
-            foreach_function(tools, [&](size_t tool_index, const json & tool) {
+            foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
                 const auto & function = tool.at("function");
                 std::string  name     = function.at("name");
 
@@ -263,10 +223,7 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
         }
 
         if (extract_reasoning && inputs.enable_thinking) {
-            // V4.1 renders a continuation without reasoning as "<｜Assistant｜></think>" even in thinking mode,
-            // so consume the bare THINK_END too or it leaks into the content
-            reasoning = is_v41 ? p.optional((THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END) | p.literal(THINK_END))
-                               : p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
+            reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
             reasoning_with_tc = THINK_START +
                 p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) +
                 p.space() + obligatory_tool_calls;

common/peg-parser.cpp

@@ -10,12 +10,10 @@
 #include <initializer_list>
 #include <map>
 #include <memory>
+#include <regex>
 #include <set>
 #include <stdexcept>
 
-// The parser can run with a lower limit than COMMON_RECURSIVE_LIMIT
-static constexpr int RULE_RECURSIVE_LIMIT = 64;
-
 // Trick to catch missing branches
 template <typename T>
 inline constexpr bool is_always_false_v = false;
@@ -188,17 +186,13 @@ std::string common_peg_ast_node::sanitized_text() const {
 }
 
 void common_peg_ast_arena::visit(common_peg_ast_id id, const common_peg_ast_visitor & visitor) const {
-    // no recursion: the AST is several nodes deep per level of the input
-    std::vector<common_peg_ast_id> stack = { id };
-    while (!stack.empty()) {
-        const auto cur = stack.back();
-        stack.pop_back();
-        if (cur == COMMON_PEG_INVALID_AST_ID) {
-            continue;
-        }
-        const auto & node = get(cur);
-        visitor(node);
-        stack.insert(stack.end(), node.children.rbegin(), node.children.rend());
+    if (id == COMMON_PEG_INVALID_AST_ID) {
+        return;
+    }
+    const auto & node = get(id);
+    visitor(node);
+    for (const auto & child : node.children) {
+        visit(child, visitor);
     }
 }
 
@@ -742,15 +736,8 @@ struct parser_executor {
     }
 
     common_peg_parse_result operator()(const common_peg_rule_parser & p) {
-        // only rules recurse, a recursive rule is entered once per input level
-        if (ctx.rule_depth[p.child] >= RULE_RECURSIVE_LIMIT) {
-            throw common_peg_recursion_error("input is nested more than " + std::to_string(RULE_RECURSIVE_LIMIT) + " levels deep");
-        }
-
         // Parse the child
-        ctx.rule_depth[p.child]++;
         auto result = arena.parse(p.child, ctx, start_pos);
-        ctx.rule_depth[p.child]--;
 
         if (!result.fail()) {
             std::string_view text;
@@ -832,10 +819,6 @@ common_peg_parse_result common_peg_arena::parse(common_peg_parse_context & ctx,
     if (root_ == COMMON_PEG_INVALID_PARSER_ID) {
         throw std::runtime_error("No root parser set");
     }
-
-    // Initialize rule depth for all parsers
-    ctx.rule_depth.assign(parsers_.size(), 0);
-
     return parse(root_, ctx, start);
 }
 
@@ -1086,6 +1069,11 @@ common_peg_parser operator|(const std::string & str, const common_peg_parser & p
     return operator|(str.c_str(), p);
 }
 
+static std::string rule_name(const std::string & name) {
+    static const std::regex invalid_rule_chars_re("[^a-zA-Z0-9-]+");
+    return std::regex_replace(name, invalid_rule_chars_re, "-");
+}
+
 common_peg_parser_builder::common_peg_parser_builder() {}
 
 common_peg_parser common_peg_parser_builder::sequence(const std::vector<common_peg_parser_id> & parsers) {
@@ -1167,14 +1155,14 @@ common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p,
 }
 
 common_peg_parser common_peg_parser_builder::rule(const std::string & name, const common_peg_parser & p, bool trigger) {
-    auto clean_name = gbnf_format_rule_name(name);
+    auto clean_name = rule_name(name);
     auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, p.id(), trigger});
     arena_.add_rule(clean_name, rule_id);
     return ref(clean_name);
 }
 
 common_peg_parser common_peg_parser_builder::rule(const std::string & name, const std::function<common_peg_parser()> & builder_fn, bool trigger) {
-    auto clean_name = gbnf_format_rule_name(name);
+    auto clean_name = rule_name(name);
     if (arena_.has_rule(clean_name)) {
         return ref(clean_name);
     }

common/peg-parser.h

@@ -6,7 +6,6 @@
 
 #include <memory>
 #include <set>
-#include <stdexcept>
 #include <unordered_map>
 #include <string>
 #include <string_view>
@@ -183,10 +182,6 @@ inline common_peg_parse_flags operator~(common_peg_parse_flags a) {
     return static_cast<common_peg_parse_flags>(~int(a));
 }
 
-struct common_peg_recursion_error : std::invalid_argument {
-    using std::invalid_argument::invalid_argument;
-};
-
 struct common_peg_parse_context {
     std::string input;               // [h,   e,  l,  l,  o,  _,  w,  o,  r,  l,  d]
     std::vector<llama_token> tokens; // [id, -1, -1, -1, -1, id, -1, -1, -1, -1, -1]
@@ -194,7 +189,6 @@ struct common_peg_parse_context {
     common_peg_ast_arena ast;
 
     int parse_depth;
-    std::vector<int> rule_depth; // index: rule child id
 
     common_peg_parse_context(common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE)
         : flags(flags), parse_depth(0) {}

conversion/__init__.py

@@ -70,7 +70,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "LingDSparkModel": "qwen",
     "DeepseekV4ForCausalLM": "deepseek",
     "DeepseekV4DSparkModel": "deepseek",
-    "DeepseekV41ForCausalLM": "deepseek41",
     "DistilBertForMaskedLM": "bert",
     "DistilBertForSequenceClassification": "bert",
     "DistilBertModel": "bert",
@@ -172,10 +171,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "D1OmniModel": "lfm2",
     "Lfm2Model": "lfm2",
     "Lfm2MoeForCausalLM": "lfm2",
-    "Lfm3D1Model": "lfm3",
-    "Lfm3ForCausalLM": "lfm3",
-    "Lfm3OmniForConditionalGeneration": "lfm3",
-    "Lfm3VlForConditionalGeneration": "lfm3",
     "Llama4ForCausalLM": "llama",
     "Llama4ForConditionalGeneration": "llama",
     "LlamaBidirectionalModel": "llama",
@@ -201,7 +196,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "MiniCPMForCausalLM": "minicpm",
     "MiniCPMV4_6ForConditionalGeneration": "minicpm",
     "MiniCPMV4_7ForConditionalGeneration": "minicpm",
-    "MiniCPMOCRForSequenceClassification": "qwen",
     "MiniCPMOcrForConditionalGeneration": "minicpm",
     "MiniMaxText01ForCausalLM": "minimax",
     "MiniMaxM1ForCausalLM": "minimax",
@@ -332,7 +326,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "VllmD3Model": "vllm_d3",
     "DeepseekOCR2ForCausalLM": "deepseek",
     "DeepseekOCRForCausalLM": "deepseek",
-    "DeepseekV41ForCausalLM": "deepseek41",
     "DeepseekV4ForCausalLM": "deepseek",
     "Dots3NoteForCausalLM": "dots3",
     "Dots3NoteForConditionalGeneration": "dots3",
@@ -362,9 +355,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "Lfm2AudioForConditionalGeneration": "lfm2",
     "D1OmniModel": "lfm2",
     "Lfm2VlForConditionalGeneration": "lfm2",
-    "Lfm3AudioForConditionalGeneration": "lfm3",
-    "Lfm3OmniForConditionalGeneration": "lfm3",
-    "Lfm3VlForConditionalGeneration": "lfm3",
     "LightOnOCRForConditionalGeneration": "lighton_ocr",
     "Llama4ForConditionalGeneration": "llama4",
     "LlavaForConditionalGeneration": "llava",
@@ -373,7 +363,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "MiniMaxM3SparseForConditionalGeneration": "minimax",
     "MiniCPMV4_6ForConditionalGeneration": "minicpm",
     "MiniCPMV4_7ForConditionalGeneration": "minicpm",
-    "MiniCPMOCRForSequenceClassification": "qwen3vl",
     "MiniCPMOcrForConditionalGeneration": "minicpm",
     "Mistral3ForConditionalGeneration": "llava",
     "Mistral3Model": "llava",

conversion/deepseek.py

@@ -803,6 +803,9 @@ class DeepseekV4Model(TextModel):
         self._dsv4_mxfp4_generated = True
         return ()
 
+    def _format_dsv4_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None, suffix: str = ".weight") -> str:
+        return self.format_tensor_name(key, bid, suffix)
+
     def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]:
         root_map: dict[str, tuple[gguf.MODEL_TENSOR, str]] = {
             "embed.weight": (gguf.MODEL_TENSOR.TOKEN_EMBD, ".weight"),
@@ -876,17 +879,17 @@ class DeepseekV4Model(TextModel):
 
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         if re.match(r"layers\.\d+\.ffn\.experts\.\d+\.w[123]\.(weight|scale)$", name):
-            return
+            return []
 
         # hash layers route text tokens via tid2eid and image tokens via bias_vl; gate.bias is unused
         if name.endswith(".ffn.gate.bias") and bid is not None and bid < self.hparams["num_hash_layers"]:
-            return
+            return []
 
         tensor_key, suffix = self._map_dsv4_tensor_name(name, bid)
         if tensor_key == gguf.MODEL_TENSOR.FFN_GATE_TID2EID:
-            return
+            return []
 
-        yield self.format_tensor_name(tensor_key, bid, suffix), data_torch
+        return [(self._format_dsv4_tensor_name(tensor_key, bid, suffix), data_torch)]
 
     def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
         del bid  # unused

conversion/deepseek41.py

@@ -1,464 +0,0 @@
-from __future__ import annotations
-
-import json
-import math
-import re
-from pathlib import Path
-from typing import Any, Callable, Iterable, TYPE_CHECKING, cast
-
-import numpy as np
-import torch
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import LazyTorchTensor, ModelBase, TextModel, gguf
-from .deepseek import DeepseekV4FlashVisionModel, DeepseekV4Model
-
-
-# Engram hash constants from inference/engram.py, stored in the GGUF: the token map needs the HF tokenizers
-# normalizers and the multipliers numpy's PCG64 stream, neither of which the runtime has.
-
-def _is_prime(n: int) -> bool:
-    if n < 2 or n % 2 == 0:
-        return n == 2
-    return all(n % d for d in range(3, math.isqrt(n) + 1, 2))
-
-
-def engram_primes(n_layers: int, max_ngram_size: int, n_heads: int, vocab_size: int) -> list[list[list[int]]]:
-    """EngramLayout.from_args: [layer][ngram order 2..max][head] bucket moduli, never reused across layers."""
-    seen: set[int] = set()
-    out: list[list[list[int]]] = []
-    for _ in range(n_layers):
-        per_ngram = []
-        for _ in range(max_ngram_size - 1):
-            sizes, cur = [], vocab_size - 1
-            for _ in range(n_heads):
-                cand = cur + 1
-                while not _is_prime(cand) or cand in seen:
-                    cand += 1
-                cur = cand
-                seen.add(cur)
-                sizes.append(cur)
-            per_ngram.append(sizes)
-        out.append(per_ngram)
-    return out
-
-
-def engram_multipliers(layer_ids: list[int], max_ngram_size: int, compressed_vocab_size: int) -> list[list[int]]:
-    """compute_hash_multipliers: odd, bounded so that token_id * multiplier cannot overflow int64."""
-    bound = max(1, (np.iinfo(np.int64).max // compressed_vocab_size) // 2)
-    rows = []
-    for layer_id in layer_ids:
-        rng = np.random.default_rng(10007 * layer_id)
-        values = rng.integers(low=0, high=bound, size=(max_ngram_size,), dtype=np.int64)
-        rows.append([int(v) * 2 + 1 for v in values])
-    return rows
-
-
-def engram_compressed_token_map(tokenizer_dir: Path) -> tuple[list[int], int]:
-    """build_compressed_token_map: token id -> compressed id (tokens that normalize alike collapse)."""
-    from tokenizers import Regex, normalizers
-    from transformers import AutoTokenizer
-
-    tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir)
-    sentinel = "\ue000"
-    normalizer = normalizers.Sequence([
-        normalizers.NFKC(),
-        normalizers.NFD(),
-        normalizers.StripAccents(),
-        normalizers.Lowercase(),
-        normalizers.Replace(Regex(r"[ \t\r\n]+"), " "),
-        normalizers.Replace(Regex(r"^ $"), sentinel),
-        normalizers.Strip(),
-        normalizers.Replace(sentinel, " "),
-    ])
-    backend = tokenizer.backend_tokenizer
-    key_to_new: dict[str, int] = {}
-    lookup = [0] * len(tokenizer)
-    for token_id in range(len(tokenizer)):
-        text = backend.decode([token_id], skip_special_tokens=False)
-        if "\ufffd" in text:
-            key = backend.id_to_token(token_id)
-        else:
-            normalized = normalizer.normalize_str(text)
-            key = normalized if normalized else text
-        lookup[token_id] = key_to_new.setdefault(key, len(key_to_new))
-    return lookup, len(key_to_new)
-
-
-class _StreamedBytes:
-    """Raw tensor bytes produced chunk by chunk at write time (an Engram table is ~200 GB in BF16)."""
-    def __init__(self, shape: tuple[int, ...], chunks: Callable[[], Iterable[np.ndarray]]):
-        self.shape = shape
-        self.dtype = np.dtype(np.uint8)
-        self.nbytes = int(np.prod(shape))
-        self._chunks = chunks
-
-    def tofile(self, fout) -> None:
-        written = 0
-        for chunk in self._chunks():
-            chunk = np.ascontiguousarray(chunk, dtype=np.uint8)
-            chunk.tofile(fout)
-            written += chunk.nbytes
-        if written != self.nbytes:
-            raise ValueError(f"streamed {written} bytes, expected {self.nbytes}")
-
-    def byteswap(self, inplace: bool = False):
-        raise NotImplementedError("streamed tensors are written little-endian only")
-
-
-def _e8m0_check(scale_u8: np.ndarray, name: str) -> None:
-    # e4m3 (|x| in [2^-9, 448]) * 2^(e - 127) is exact in BF16 for e in [3, 246]
-    if scale_u8.size and (int(scale_u8.min()) < 3 or int(scale_u8.max()) > 246):
-        raise ValueError(f"{name}: E8M0 scale exponents outside [3, 246], the dequantized values would not be exact")
-
-
-@ModelBase.register("DeepseekV41ForCausalLM")
-class DeepseekV41Model(DeepseekV4Model):
-    """DeepSeek-V4.1 text model, lossless: FP8 (e4m3, 32x32 ue8m0 blocks) linears dequantized to BF16 (F32 with
-    --outtype f32), routed experts copied bit-exact as MXFP4, Engram tables (e4m3 rows, one e8m0 scale per 32) as the
-    exact BF16 lookup (F32 with --outtype f32), router and Engram gate F32. DSpark (mtp.*) is not converted here
-    (--dspark, DeepseekV41DSparkModel); the vision tower goes to an mmproj (--mmproj, DeepseekV41VisionModel).
-    The Engram tables of all layers are written as one per_layer_token_embd (rows concatenated in layer order) with
-    the PLE keys of qwen4exp: absolute head offsets and bucket counts, so the runtime can read it lazily."""
-    model_arch = gguf.MODEL_ARCH.DEEPSEEK41
-    supports_mtp_export = False
-
-    _ENGRAM_EMBED_RE = re.compile(r"layers\.(\d+)\.engram\.embed\.(weight|scale)$")
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.hparams.setdefault("num_hash_layers", 0)  # no hash-routed layers in V4.1
-        self._f32 = self.ftype == gguf.LlamaFileType.ALL_F32
-        qc = self.hparams.get("quantization_config") or {}
-        self._fp8_block = tuple(qc.get("weight_block_size") or (32, 32))
-        if qc.get("scale_fmt", "ue8m0") != "ue8m0":
-            raise ValueError(f"unsupported FP8 scale format {qc.get('scale_fmt')!r}")
-        if self.hparams.get("gate_temp", 1.0) != 1.0:
-            raise ValueError("gate_temp != 1 is not supported (the router logits would need dividing by it)")
-
-        # V4.1 does not use the DeepSeek-V4 template; a template shipped with the model is still added by set_vocab
-        self.gguf_writer.remove_key(gguf.Keys.Tokenizer.CHAT_TEMPLATE)
-
-    def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
-        # num_hidden_layers lives in text_config, which is merged only after indexing; MTP stages are skipped anyway
-        return TextModel.index_tensors(self, remote_hf_model_id=remote_hf_model_id)
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        hp = self.hparams
-        w = self.gguf_writer
-        w.add_attention_kv_source_layers(hp["kv_source_layer_ids"])
-        w.add_indexer_source_layers(hp["index_source_layer_ids"])
-        if hp.get("candidate_source_layer_id", -1) >= 0:
-            w.add_indexer_candidate_source_layer(hp["candidate_source_layer_id"])
-            w.add_indexer_candidate_top_k_blocks(hp["candidate_topk_blocks"])
-            w.add_indexer_candidate_block_size(hp["candidate_block_size"])
-
-        # GGUF cannot hold empty arrays: the Engram keys are omitted when the model has no Engram layers
-        layer_ids = list(hp.get("engram_layer_ids") or [])
-        if layer_ids:
-            max_ngram, n_heads = hp["engram_max_ngram_size"], hp["engram_n_heads"]
-            primes = engram_primes(len(layer_ids), max_ngram, n_heads, hp["engram_vocab_size"])
-            for li, layer_primes in enumerate(primes):
-                if sum(p for per in layer_primes for p in per) != hp["engram_num_embeddings"][li]:
-                    raise ValueError(f"engram layer {layer_ids[li]}: prime sum != num_embeddings")
-            token_map, cvocab = engram_compressed_token_map(self.dir_model)
-            if cvocab != hp["engram_compressed_vocab_size"] or len(token_map) != hp["vocab_size"]:
-                raise ValueError(f"compressed token map ({len(token_map)} -> {cvocab}) does not match the config")
-            # absolute row offsets into the concatenated table
-            sizes = [p for layer in primes for per in layer for p in per]
-            offsets = [sum(sizes[:i]) for i in range(len(sizes))]
-            w.add_key_value(gguf.Keys.PerLayerEmbedding.LAYERS.format(arch=w.arch), layer_ids,
-                            gguf.GGUFValueType.ARRAY, gguf.GGUFValueType.UINT32)
-            w.add_ple_ngram_size(max_ngram)
-            w.add_ple_heads_per_ngram(n_heads)
-            w.add_embedding_length_per_layer_input(hp["engram_head_dim"])
-            w.add_ple_layer_multipliers([m for row in engram_multipliers(layer_ids, max_ngram, cvocab) for m in row])
-            w.add_ple_head_offsets(offsets)
-            w.add_ple_head_vocab_sizes(sizes)
-            w.add_engram_pad_token_id(hp["engram_pad_token_id"])
-            w.add_engram_token_map(token_map)
-
-    def dequant_model(self):
-        bo, bi = self._fp8_block
-
-        def dequant(weight: Tensor, scale: Tensor) -> Tensor:
-            out_f, in_f = weight.shape
-            s = self._e8m0_to_float(scale).repeat_interleave(bo, 0)[:out_f].repeat_interleave(bi, 1)[:, :in_f]
-            return weight.float() * s
-
-        for name in [n for n in self.model_tensors if n.endswith(".scale") and not self._ENGRAM_EMBED_RE.match(n)]:
-            weight_name = name.removesuffix(".scale") + ".weight"
-            if weight_name not in self.model_tensors or self.model_tensors[weight_name]().dtype not in self._float8_dtypes():
-                continue
-            w, s = self.model_tensors[weight_name], self.model_tensors.pop(name)
-            out_f, in_f = w().shape
-            if tuple(s().shape) != (-(-out_f // bo), -(-in_f // bi)):
-                raise ValueError(f"{weight_name}: scale shape {tuple(s().shape)} does not match {(bo, bi)} blocks")
-            _e8m0_check(LazyTorchTensor.to_eager(s()).view(torch.uint8).numpy(), name)
-            self.model_tensors[weight_name] = lambda w=w, s=s: dequant(w(), s())
-            self._dsv4_fp8_dequantized.add(weight_name)
-
-    def _tensor_part(self, name: str) -> Path:
-        index = self.dir_model / "model.safetensors.index.json"
-        if not index.is_file():
-            return self.dir_model / "model.safetensors"
-        with open(index, "r", encoding="utf-8") as f:
-            return self.dir_model / json.load(f)["weight_map"][name]
-
-    def _write_mxfp4_experts(self, bid: int, proj: str, key: gguf.MODEL_TENSOR, n_expert: int | None = None) -> None:
-        n_expert = self.hparams["n_routed_experts"] if n_expert is None else n_expert
-        names = [f"layers.{bid}.ffn.experts.{e}.{proj}" for e in range(n_expert)]
-        pairs = [(self.model_tensors.pop(n + ".weight"), self.model_tensors.pop(n + ".scale")) for n in names]
-        rows, packed_cols = pairs[0][0]().shape
-
-        def chunks() -> Iterable[np.ndarray]:
-            for w, s in pairs:
-                yield self.repack_mxfp4_blocks(LazyTorchTensor.to_eager(w()), LazyTorchTensor.to_eager(s()))
-        data = cast(np.ndarray, _StreamedBytes((len(pairs), rows, packed_cols * 2 // 32 * 17), chunks))
-        self.gguf_writer.add_tensor(self.format_tensor_name(key, bid), data, raw_dtype=gguf.GGMLQuantizationType.MXFP4)
-
-    def _write_engram_tables(self, layer_ids: list[int]) -> None:
-        # official lookup e4m3 * 2^(e8m0 - 127), exact in BF16
-        parts, dim = [], None
-        for bid in layer_ids:
-            wn, sn = f"layers.{bid}.engram.embed.weight", f"layers.{bid}.engram.embed.scale"
-            rows, d = self.model_tensors.pop(wn)().shape
-            if d % 32 != 0 or tuple(self.model_tensors.pop(sn)().shape) != (rows, d // 32) or d != (dim or d):
-                raise ValueError(f"{wn}: expected one e8m0 scale per 32 elements and the same width in every layer")
-            if rows != self.hparams["engram_num_embeddings"][len(parts)]:
-                raise ValueError(f"{wn}: {rows} rows, config says {self.hparams['engram_num_embeddings'][len(parts)]}")
-            dim = d
-            parts.append((wn, sn, rows, self._tensor_part(wn), self._tensor_part(sn)))
-        assert dim is not None
-        name = self.format_tensor_name(gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD)
-        n_rows = sum(p[2] for p in parts)
-
-        def chunks() -> Iterable[np.ndarray]:
-            from safetensors import safe_open
-            for wn, sn, rows, wpart, spart in parts:
-                with safe_open(wpart, framework="pt") as fw, safe_open(spart, framework="pt") as fs:
-                    ws, ss = fw.get_slice(wn), fs.get_slice(sn)
-                    for r0 in range(0, rows, 1 << 18):
-                        q = ws[r0:r0 + (1 << 18)].view(torch.uint8).reshape(-1, 32)
-                        e = ss[r0:r0 + (1 << 18)].view(torch.uint8).reshape(-1, 1)
-                        _e8m0_check(e.numpy(), sn)
-                        v = (q.view(torch.float8_e4m3fn).float() * torch.exp2(e.float() - 127)).reshape(-1, dim)
-                        yield (v if self._f32 else v.bfloat16()).view(torch.uint8).numpy()
-        qt = gguf.GGMLQuantizationType.F32 if self._f32 else gguf.GGMLQuantizationType.BF16
-        data = cast(np.ndarray, _StreamedBytes((n_rows, dim * (4 if self._f32 else 2)), chunks))
-        self.gguf_writer.add_tensor(name, data, raw_dtype=qt)
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        if not self._dsv4_mxfp4_generated:
-            for bid in range(self.block_count):
-                self._write_mxfp4_experts(bid, "w1", gguf.MODEL_TENSOR.FFN_GATE_EXP)
-                self._write_mxfp4_experts(bid, "w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP)
-                self._write_mxfp4_experts(bid, "w3", gguf.MODEL_TENSOR.FFN_UP_EXP)
-            if self.hparams.get("engram_layer_ids"):
-                self._write_engram_tables(list(self.hparams["engram_layer_ids"]))
-            self._dsv4_mxfp4_generated = True
-        return ()
-
-    _LAYER_MAP: dict[str, tuple[gguf.MODEL_TENSOR, str]] = {
-        "attn.indexer.wk.weight": (gguf.MODEL_TENSOR.INDEXER_ATTN_K, ".weight"),
-        "attn.indexer.k_norm.weight": (gguf.MODEL_TENSOR.INDEXER_K_NORM, ".weight"),
-        "engram.wkv.weight": (gguf.MODEL_TENSOR.ENGRAM_KV, ".weight"),
-    }
-
-    def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]:
-        m = re.match(r"layers\.\d+\.(.+)$", name)
-        if m is not None and m.group(1) in self._LAYER_MAP:
-            return self._LAYER_MAP[m.group(1)]
-        return super()._map_dsv4_tensor_name(name, bid)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.endswith(".engram.k_weight"):
-            return  # folded into engram_gate
-        if name.endswith(".engram.q_weight"):
-            # Engram gate weight = q_weight * k_weight (bf16 * bf16 is exact in f32)
-            k = self.model_tensors[name.removesuffix("q_weight") + "k_weight"]
-            yield self.format_tensor_name(gguf.MODEL_TENSOR.ENGRAM_GATE, bid), data_torch.float() * k().float()
-            return
-        yield from super().modify_tensors(data_torch, name, bid)
-
-    def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
-        if new_name.endswith((".engram_gate.weight", ".ffn_gate_inp.weight")) or name in self._dsv4_f32_tensors:
-            return gguf.GGMLQuantizationType.F32
-        if n_dims >= 2 and (name in self._dsv4_fp8_dequantized or name in self._dsv4_bf16_tensors):
-            return gguf.GGMLQuantizationType.F32 if self._f32 else gguf.GGMLQuantizationType.BF16
-        return False
-
-
-class DeepseekV41DSparkModel(DeepseekV41Model):
-    """DeepSeek-V4.1 DSpark speculative draft (mtp.0..2 of the checkpoint) as a DFLASH GGUF (`--dspark`).
-
-    Layout = the V4 DSpark draft (conversion/deepseek.py DeepseekV4DSparkModel) with V4.1's differences flagged by
-    `dflash.hyper_connection.single_pass`: stages blk.0..2 (MXFP4 experts bit-exact, dense BF16), mtp.0.main_proj /
-    main_norm -> fc / enc.output_norm, mtp.2.norm -> output_norm, markov_head.embed / head -> markov_w1 / markov_w2,
-    confidence_head.proj -> conf_proj (F32), no hc_head. Token embeddings and the lm head are the target's (shared at
-    runtime through ctx_other). target_layers are the V4.1 ids as-is: the draft reads the attention input of those
-    layers, which is what llama_set_embeddings_layer_inp extracts.
-    """
-    model_arch = gguf.MODEL_ARCH.DFLASH
-    mtp_only = True
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        hp = self.hparams
-        self._n_main = hp["num_hidden_layers"]
-        self._n_stage = hp["num_nextn_predict_layers"]
-        if self._n_stage <= 0 or "dspark_block_size" not in hp:
-            raise ValueError("DeepSeek-V4.1 DSpark export needs num_nextn_predict_layers > 0 and the dspark_* config")
-        # stages keep main-model block ids (mtp.M -> layers.n_main+M); format_tensor_name() maps them back
-        self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self._n_stage)
-
-    def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
-        hp = {**self.hparams, **self.hparams.get("text_config", {})}  # text_config is merged only after indexing
-        type(self)._dsv4_main_layers = hp["num_hidden_layers"]
-        type(self)._dsv4_nextn_layers = hp.get("num_nextn_predict_layers", 0)
-        return TextModel.index_tensors(self, remote_hf_model_id=remote_hf_model_id)
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-        m = re.match(r"mtp\.(\d+)\.(.+)$", name)
-        if m is None:
-            return None  # shared with the target at runtime
-        stage, rest = int(m.group(1)), m.group(2)
-        assert cls._dsv4_main_layers is not None
-        if stage >= cls._dsv4_nextn_layers:
-            raise ValueError(f"Unexpected DSpark stage {stage}")
-        if rest.startswith(("markov_head.", "confidence_head.")):
-            return f"dspark.{rest}", gen
-        return f"layers.{cls._dsv4_main_layers + stage}.{rest}", gen
-
-    def format_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None = None, suffix: str = ".weight") -> str:
-        if bid is not None and bid >= self._n_main:
-            bid -= self._n_main
-        return super().format_tensor_name(key, bid, suffix)
-
-    def set_vocab(self):
-        super().set_vocab()
-        self.gguf_writer.add_mask_token_id(self.hparams["dspark_noise_token_id"])
-
-    def set_gguf_parameters(self):
-        hp = self.hparams
-        n = self._n_stage
-        w = self.gguf_writer
-        w.add_block_count(n)
-        w.add_context_length(hp["max_position_embeddings"])
-        w.add_embedding_length(hp["hidden_size"])
-        w.add_feed_forward_length(hp["moe_intermediate_size"])
-        w.add_head_count(hp["num_attention_heads"])
-        w.add_head_count_kv(1)
-        w.add_key_length(hp["head_dim"])
-        w.add_value_length(hp["head_dim"])
-        w.add_rope_dimension_count(hp["qk_rope_head_dim"])
-        # compress-ratio-0 layers: plain rope at rope_theta, no YaRN
-        w.add_rope_freq_base(float(hp["rope_theta"]))
-        w.add_layer_norm_rms_eps(hp["rms_norm_eps"])
-        w.add_q_lora_rank(hp["q_lora_rank"])
-        w.add_sliding_window(hp["sliding_window"])
-        w.add_expert_count(hp["dspark_n_routed_experts"])
-        w.add_expert_used_count(hp["dspark_num_experts_per_tok"])
-        w.add_expert_feed_forward_length(hp["moe_intermediate_size"])
-        w.add_expert_shared_count(hp["n_shared_experts"])
-        w.add_expert_weights_scale(hp["routed_scaling_factor"])
-        w.add_expert_weights_norm(hp["norm_topk_prob"])
-        w.add_expert_gating_func(gguf.ExpertGatingFuncType.SQRTSOFTPLUS)
-        w.add_swiglu_clamp_exp([hp["swiglu_limit"]] * n)
-        w.add_swiglu_clamp_shexp([hp["swiglu_limit"]] * n)
-        w.add_attention_output_group_count(hp["o_groups"])
-        w.add_attention_output_lora_rank(hp["o_lora_rank"])
-        w.add_attention_compress_ratios([0] * n)
-        w.add_hyper_connection_count(hp["hc_mult"])
-        w.add_hyper_connection_sinkhorn_iterations(hp["hc_sinkhorn_iters"])
-        w.add_hyper_connection_epsilon(hp["hc_eps"])
-        w.add_hyper_connection_single_pass(True)
-        w.add_block_size(hp["dspark_block_size"])
-        w.add_target_layers(hp["dspark_target_layer_ids"])
-        w.add_sample_from_anchor(True)
-        w.add_has_confidence_head(True)
-        w.add_file_type(self.ftype)
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        # no Engram tables in the draft
-        if not self._dsv4_mxfp4_generated:
-            n_expert = self.hparams["dspark_n_routed_experts"]
-            for bid in range(self._n_main, self._n_main + self._n_stage):
-                self._write_mxfp4_experts(bid, "w1", gguf.MODEL_TENSOR.FFN_GATE_EXP, n_expert)
-                self._write_mxfp4_experts(bid, "w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP, n_expert)
-                self._write_mxfp4_experts(bid, "w3", gguf.MODEL_TENSOR.FFN_UP_EXP, n_expert)
-            self._dsv4_mxfp4_generated = True
-        return ()
-
-    _DSPARK_ROOT: dict[str, gguf.MODEL_TENSOR] = {
-        "main_proj.weight": gguf.MODEL_TENSOR.FC,
-        "main_norm.weight": gguf.MODEL_TENSOR.ENC_OUTPUT_NORM,
-        "norm.weight": gguf.MODEL_TENSOR.OUTPUT_NORM,
-        "dspark.markov_head.embed.weight": gguf.MODEL_TENSOR.DSPARK_MARKOV_W1,
-        "dspark.markov_head.head.weight": gguf.MODEL_TENSOR.DSPARK_MARKOV_W2,
-        "dspark.confidence_head.proj.weight": gguf.MODEL_TENSOR.DSPARK_CONF_PROJ,
-    }
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.endswith(".ffn.gate.bias_vl"):
-            return
-        if name in self._DSPARK_ROOT:
-            yield self.format_tensor_name(self._DSPARK_ROOT[name], None), data_torch
-            return
-        m = re.match(r"layers\.\d+\.(main_proj\.weight|main_norm\.weight|norm\.weight)$", name)
-        if m is not None:
-            yield self.format_tensor_name(self._DSPARK_ROOT[m.group(1)], None), data_torch
-            return
-        yield from super().modify_tensors(data_torch, name, bid)
-
-    def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
-        if new_name == "conf_proj.weight":
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-
-@ModelBase.register("DeepseekV41ForCausalLM")
-@ModelBase.example("deepseek-ai/DeepSeek-V4.1-Flash-1017")
-class DeepseekV41VisionModel(DeepseekV4FlashVisionModel):
-    """DeepSeek-V4.1 vision tower (projector deepseek41v): the V4-Flash-Vision ViT + 3x3 aligner, with the V4.1 token
-    block ([START] ([IMAGE]*w [NEWLINE])*h [END], no pads) and resize plan. ref: inference/vision.py,
-    inference/image_processor.py of deepseek-ai/DeepSeek-V4.1-Flash-1017. The checkpoint config is HF-style
-    (vision_config); image_start / image_end / image_newline are written like V4's sentinels, there is no image_pad."""
-
-    def get_vision_config(self) -> dict[str, Any] | None:
-        vc = self.global_config.get("vision_config")
-        if not vc or vc.get("num_hidden_layers", 0) == 0:
-            raise ValueError("DeepseekV41VisionModel requires vision_config.num_hidden_layers > 0")
-        return {
-            "num_hidden_layers":   vc["num_hidden_layers"],
-            "hidden_size":         vc["hidden_size"],
-            "num_attention_heads": vc["num_attention_heads"],
-            "intermediate_size":   vc["intermediate_size"],
-            "patch_size":          vc["patch_size"],
-            # dynamic resolution; only used for compat / warmup
-            "image_size":          vc["patch_size"] * vc["downsample_ratio"] * 16,
-            "rope_theta":          vc.get("rope_theta", 10000.0),
-            "downsample_ratio":    vc["downsample_ratio"],
-            "min_pixels":          vc["min_pixels"],
-            "max_image_tokens":    vc["max_image_tokens"],
-            "max_wh_ratio":        vc.get("max_wh_ratio"),
-        }
-
-    def set_gguf_parameters(self):
-        assert self.hparams_vision is not None
-        super(DeepseekV4FlashVisionModel, self).set_gguf_parameters()
-        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.DEEPSEEK41V)
-        # pytorch default eps (vision.py), not the LLM rms_norm_eps (1e-20)
-        self.gguf_writer.add_vision_attention_layernorm_eps(1e-6)
-        self.gguf_writer.add_vision_use_silu(True)
-        self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"])
-        self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"])
-        # hardcoded in clip.cpp (PROJECTOR_TYPE_DEEPSEEK41V); add GGUF keys if a later model differs
-        assert self.hparams_vision["max_image_tokens"] == 1024, self.hparams_vision["max_image_tokens"]
-        assert self.hparams_vision["max_wh_ratio"] is None, self.hparams_vision["max_wh_ratio"]
-        assert self.hparams_vision["rope_theta"] == 10000, self.hparams_vision["rope_theta"]

conversion/lfm2.py

@@ -15,30 +15,25 @@ from .base import MmprojModel, ModelBase, TextModel, gguf, jinja_str_or_json, lo
 from .gemma import ConformerAudioModel
 
 
-def lfm2_feed_forward_length(hparams: dict[str, Any]) -> int:
-    ff_dim = hparams.get("block_ff_dim", hparams.get("intermediate_size"))
-    assert ff_dim is not None
-    auto_adjust_ff_dim = hparams["block_auto_adjust_ff_dim"]
-    ffn_dim_multiplier = hparams["block_ffn_dim_multiplier"]
-    multiple_of = hparams["block_multiple_of"]
-
-    if auto_adjust_ff_dim:
-        ff_dim = int(2 * ff_dim / 3)
-        # custom dim factor multiplier
-        if ffn_dim_multiplier is not None:
-            ff_dim = int(ffn_dim_multiplier * ff_dim)
-        ff_dim = multiple_of * ((ff_dim + multiple_of - 1) // multiple_of)
-
-    return ff_dim
-
-
 @ModelBase.register("Lfm2ForCausalLM", "LFM2ForCausalLM")
 @ModelBase.example("LiquidAI/LFM2-1.2B", "LiquidAI/LFM2.5-350M")
 class LFM2Model(TextModel):
     model_arch = gguf.MODEL_ARCH.LFM2
 
     def _add_feed_forward_length(self):
-        self.gguf_writer.add_feed_forward_length(lfm2_feed_forward_length(self.hparams))
+        ff_dim = self.find_hparam(["block_ff_dim", "intermediate_size"])
+        auto_adjust_ff_dim = self.hparams["block_auto_adjust_ff_dim"]
+        ffn_dim_multiplier = self.hparams["block_ffn_dim_multiplier"]
+        multiple_of = self.hparams["block_multiple_of"]
+
+        if auto_adjust_ff_dim:
+            ff_dim = int(2 * ff_dim / 3)
+            # custom dim factor multiplier
+            if ffn_dim_multiplier is not None:
+                ff_dim = int(ffn_dim_multiplier * ff_dim)
+            ff_dim = multiple_of * ((ff_dim + multiple_of - 1) // multiple_of)
+
+        self.gguf_writer.add_feed_forward_length(ff_dim)
 
     def set_gguf_parameters(self):
         # set num_key_value_heads only for attention layers
@@ -77,9 +72,7 @@ def _is_d1_checkpoint(dir_model: Path) -> bool:
     if not (dir_model / "config.json").is_file():
         return False
     with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    # d1 on LFM3 is converted by conversion/lfm3.py
-    return config.get("auto_map", {}).get("AutoModel", "").endswith(".D1Model") and not config.get("model_type", "").startswith("lfm3")
+        return json.load(f).get("auto_map", {}).get("AutoModel", "").endswith(".D1Model")
 
 
 @ModelBase.register_hparams_loader(_is_d1_checkpoint)

conversion/lfm3.py

@@ -1,613 +0,0 @@
-from __future__ import annotations
-
-import json
-import math
-
-from pathlib import Path
-from typing import Any, Callable, Iterable, TYPE_CHECKING
-
-import torch
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import MmprojModel, ModelBase, TextModel, gguf, logger
-from .lfm2 import D1Model, lfm2_feed_forward_length
-
-
-@ModelBase.register("Lfm3ForCausalLM", "Lfm3VlForConditionalGeneration", "Lfm3OmniForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-1.7B", "LiquidAI/LFM3-1.7B-Base")
-class LFM3Model(TextModel):
-    model_arch = gguf.MODEL_ARCH.LFM3
-
-    # LFM3-Audio/Omni: encoder, projector and audio-output heads live next to the text model
-    _media_prefixes = ("encoder.", "mlp.", "codebook_embedding.", "depth_embeddings.", "depth_linear.", "depthformer.",
-                       "mask_embedding", "model.codebook_embedding.", "model.depth_decoder.")
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-
-        if name.startswith(cls._media_prefixes):
-            return None
-
-        name = name.removeprefix("lfm.")
-
-        return super().filter_tensors((name, gen))
-
-    # LFM3-Audio live ASR prompt of the model card, the audio follows in the assistant turn
-    _asr_template = (
-        "{{- bos_token -}}"
-        "{{- '<|im_start|>system\\nPerform live ASR.<|im_end|>\\n' -}}"
-        "{%- if add_generation_prompt -%}{{- '<|im_start|>assistant\\n' -}}{%- endif -%}"
-    )
-
-    # TTS prompt of the model card: a language code is looked up, anything else is taken as the language name
-    _tts_langs = {
-        "ar": "Arabic", "da": "Danish", "de": "German", "el": "Greek", "en": "American English", "en-us": "American English",
-        "en-gb": "British English", "fi": "Finnish", "fr": "French", "he": "Hebrew", "it": "Italian", "ja": "Japanese",
-        "ko": "Korean", "nl": "Dutch", "pl": "Polish", "pt": "Portuguese", "es": "Spanish", "sv": "Swedish", "tr": "Turkish",
-        "zh": "Chinese",
-    }
-
-    # "language" and "voice" are optional, the text to speak is the first message
-    def _tts_template(self, think: bool) -> str:
-        return (
-            "{%- set names = " + json.dumps(self._tts_langs) + " -%}"
-            "{%- set code = (language | lower) if language is defined and language else 'en' -%}"
-            "{%- set lang = names[code] if code in names else language -%}"
-            "{%- set v = (voice | lower) if voice is defined and voice and (voice | lower) in ['female', 'male'] else 'female' -%}"
-            "{{- bos_token -}}"
-            "{{- '<|im_start|>system\\nPerform TTS in ' + lang + ' using a ' + v + ' voice.<|im_end|>\\n' -}}"
-            "{{- '<|im_start|>user\\n' + messages[0]['content'] + '<|im_end|>\\n' -}}"
-            "{{- '<|im_start|>assistant\\n" + ("<think></think>" if think else "") + "<|audio_start|>' -}}"
-        )
-
-    def set_vocab(self):
-        super().set_vocab()
-        # hparams hold the merged text config, the checkpoint's own model_type is in config.json
-        with open(self.dir_model / "config.json", encoding="utf-8") as f:
-            model_type = json.load(f).get("model_type")
-        is_audio = model_type == "lfm3_audio"
-        is_omni  = model_type == "lfm3_omni"
-        templates = []
-        if is_audio:
-            templates.append({"name": "asr", "template": self._asr_template})
-        if is_audio or is_omni:
-            # LFM3-Omni opens every assistant turn with an empty think block
-            templates.append({"name": "tts", "template": self._tts_template(think=is_omni)})
-        if templates:
-            self.gguf_writer.add_chat_template(templates)
-
-    def set_gguf_parameters(self):
-        layer_types = self.hparams["layer_types"]
-        n_kv_heads = self.hparams["num_key_value_heads"]
-
-        # set num_key_value_heads only for attention layers
-        self.hparams["num_key_value_heads"] = [
-            n_kv_heads if lt != "conv" else 0
-            for lt in layer_types
-        ]
-
-        super().set_gguf_parameters()
-
-        self.gguf_writer.add_rope_dimension_count(self.hparams["head_dim"])
-
-        self.gguf_writer.add_sliding_window_pattern([lt == "sliding_attention" for lt in layer_types])
-        self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
-
-        self.gguf_writer.add_shortconv_l_cache(self.hparams["conv_L_cache"])
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        # Partial RoPE is represented by masking trailing frequency pairs.
-        # rope_parameters is keyed by layer type; only full_attention layers use partial RoPE.
-        rope_parameters = self.hparams["rope_parameters"]["full_attention"]
-        partial = rope_parameters.get("partial_rotary_factor", 1.0)
-        head_dim = self.hparams["head_dim"]
-        rope_angles = int(partial * head_dim) // 2
-        n_pairs = head_dim // 2
-        assert 0 < rope_angles <= n_pairs
-        values = [1.0] * rope_angles + [1e30] * (n_pairs - rope_angles)
-        rope_freqs = torch.tensor(values, dtype=torch.float32)
-        yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), rope_freqs)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # conv op requires 2d tensor
-        if "conv.conv" in name:
-            data_torch = data_torch.squeeze(1)
-
-        yield from super().modify_tensors(data_torch, name, bid)
-
-
-# d1 on LFM3-VL / LFM3-Omni: the stock weights, the d1 prompt read at its last token
-_D1_TYPES = {"lfm3_vl": gguf.DecisionType.LFM3_D1, "lfm3_omni": gguf.DecisionType.LFM3_D1_OMNI}
-
-
-def _is_lfm3_d1_checkpoint(dir_model: Path) -> bool:
-    if not (dir_model / "config.json").is_file():
-        return False
-    with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    return config.get("architectures") == ["D1Model"] and config.get("model_type") in _D1_TYPES
-
-
-@ModelBase.register_hparams_loader(_is_lfm3_d1_checkpoint)
-def _load_lfm3_d1_hparams(dir_model: Path) -> dict[str, Any]:
-    logger.info("gguf: detected LFM3 d1 checkpoint")
-    hparams = ModelBase.load_hparams(dir_model, False, guess=False)
-    hparams["d1_family"] = hparams["model_type"]
-    # the mmproj is the base model's, the text model writes the decision
-    hparams["architectures"] = ["Lfm3VlForConditionalGeneration" if hparams["model_type"] == "lfm3_vl" else "Lfm3OmniForConditionalGeneration"]
-    hparams["text_config"]["architectures"] = ["Lfm3D1Model"]
-    return hparams
-
-
-@ModelBase.register("Lfm3D1Model")
-@ModelBase.example("LiquidAI/d1-2B", "LiquidAI/d1-omni-2B")
-class LFM3D1Model(LFM3Model):
-    model_arch = gguf.MODEL_ARCH.LFM3
-
-    # how the checkpoint serves, hard-coded in the server: assert the checkpoint records the same
-    _served = {"d1_codes": 2, "d1_attention": "set"}
-
-    def set_vocab(self):
-        # the vocab without the speech task templates of LFM3-Audio/Omni, d1 only decides
-        TextModel.set_vocab(self)
-        # the d1 prompt of LFM2, the LFM3 template writes an empty thought before the answer
-        self.gguf_writer.add_chat_template([{"name": "systemone", "template": D1Model._systemone_template() + "<think></think>"}])
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-
-        for key, value in self._served.items():
-            if self.hparams.get(key) != value:
-                raise ValueError(f"unsupported d1 checkpoint: {key} = {self.hparams.get(key)!r}, expected {value!r}")
-
-        # a lone question reads the whole prompt both ways on the full attention layers
-        self.gguf_writer.add_causal_attention(False)
-        self.gguf_writer.add_decision_type(_D1_TYPES[self.hparams["d1_family"]])
-
-        # the option answers share one temperature
-        with open(self.dir_model / "calibration.json", encoding="utf-8") as f:
-            calibration = json.load(f)
-        if calibration.get("temperatures"):
-            raise ValueError("unsupported d1 calibration: per-bucket temperatures")
-        for name in ("noul", "choice", "score"):
-            self.gguf_writer.add_decision_temperature(name, calibration["default"])
-
-
-class LFM3VLVisionMixin(MmprojModel):
-    """LFM3-VL vision tower and projector, shared with LFM3-Omni."""
-
-    def _set_lfm3vl_gguf_parameters(self):
-        assert self.hparams_vision is not None
-        # clip.cpp hardcodes the GELU projector and its input norm eps
-        if (self.global_config["projector_hidden_act"] != "gelu" or not self.global_config["projector_bias"]
-                or self.global_config["projector_layer_norm_eps"] != 1e-6):
-            raise ValueError("unsupported projector_hidden_act / projector_bias / projector_layer_norm_eps")
-        # clip.cpp hardcodes the vision rope base, the RGB patchifier and the 3x3 spatial conv
-        if (self.hparams_vision["rope_parameters"]["rope_theta"] != 1e6 or self.hparams_vision["in_channels"] != 3
-                or self.hparams_vision["conv_kernel_size"] != 3):
-            raise ValueError("unsupported vision rope_theta / in_channels / conv_kernel_size")
-        self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.LFM3VL)
-        self.gguf_writer.add_vision_head_count_kv(self.hparams_vision["num_key_value_heads"])
-        self.gguf_writer.add_vision_spatial_merge_size(self.global_config["downsample_factor"])
-        self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision["norm_eps"])
-        self.gguf_writer.add_vision_use_silu(True)
-        size = self.preprocessor_config["size"]
-        self.gguf_writer.add_vision_min_pixels(size["shortest_edge"])
-        self.gguf_writer.add_vision_max_pixels(size["longest_edge"])
-        # PIL resample enum, the torchvision backend has no lanczos
-        resample = self.preprocessor_config["resample"]
-        resize_algo = {2: "bilinear", 3: "bicubic"}.get(resample)
-        if resize_algo is None:
-            raise ValueError(f"unsupported resample: {resample}")
-        self.gguf_writer.add_vision_image_resize_algo(resize_algo)
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        if "patch_embd" in new_name:
-            return gguf.GGMLQuantizationType.F16
-        if "conv2d_spatial" in new_name:
-            # CUDA CONV_2D_DW requires an f32 kernel; the tensor is tiny
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    def _lfm3vl_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name == "model.vision_tower.patchifier.weight":
-            # qwen2vl-style layout [channel, temporal, ph, pw], one conv kernel per temporal slice
-            assert self.hparams_vision is not None
-            t = self.hparams_vision["temporal_patch_size"]
-            p = self.hparams_vision["patch_size"]
-            n_embd = data_torch.shape[0]
-            w = data_torch.view(n_embd, 3, t, p, p)
-            yield ("v.patch_embd.weight", w[:, :, 0].contiguous())
-            if t == 2:
-                yield ("v.patch_embd.weight.1", w[:, :, 1].contiguous())
-            else:
-                assert t == 1, f"unsupported vision_temporal_patch_size {t}"
-            return
-        if name == "model.vision_tower.embedding_norm.weight":
-            yield ("v.embedding_norm.weight", data_torch)
-            return
-        if "conv.in_proj.weight" in name:
-            yield (f"v.blk.{bid}.conv2d_in_proj.weight", data_torch)
-            return
-        if "conv.spatial_conv.weight" in name:
-            yield (f"v.blk.{bid}.conv2d_spatial.weight", data_torch)
-            return
-        if "conv.out_proj.weight" in name:
-            yield (f"v.blk.{bid}.conv2d_out_proj.weight", data_torch)
-            return
-
-        # projector: input LayerNorm -> linear_1 -> GELU -> linear_2
-        if name.startswith("model.multi_modal_projector."):
-            layer, suffix = name.removeprefix("model.multi_modal_projector.").split(".")
-            yield ({"layer_norm": "mm.input_norm", "linear_1": "mm.0", "linear_2": "mm.2"}[layer] + "." + suffix, data_torch)
-            return
-
-        yield (self.map_tensor_name(name), data_torch)
-
-
-@ModelBase.register("Lfm3VlForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-VL-2B")
-class LFM3VLMmprojModel(LFM3VLVisionMixin):
-    has_vision_encoder = True
-    has_audio_encoder = False
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        assert self.hparams_vision is not None
-        # dynamic resolution, image_size is only set for MmprojModel compatibility
-        self.hparams_vision["image_size"] = 512
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        self._set_lfm3vl_gguf_parameters()
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.startswith(("model.vision_tower.", "model.multi_modal_projector.")):
-            yield from self._lfm3vl_tensors(data_torch, name, bid)
-
-
-def _is_lfm3_audio_checkpoint(dir_model: Path) -> bool:
-    if not (dir_model / "config.json").is_file():
-        return False
-    with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    return config.get("model_type") == "lfm3_audio" and "lfm" in config
-
-
-# read the release config.json as is: a transformers that knows lfm3_audio rewrites it into another layout
-@ModelBase.register_hparams_loader(_is_lfm3_audio_checkpoint)
-def _load_lfm3_audio_hparams(dir_model: Path) -> dict[str, Any]:
-    with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    config["text_config"] = config["lfm"]
-    return config
-
-
-@ModelBase.register("Lfm3AudioForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-Audio-2B")
-class LFM3AudioMmprojModel(MmprojModel):
-    has_vision_encoder = False
-    has_audio_encoder = True
-
-    # the downsampler has the LFM2-Audio pre-encoder layout: conv, dw conv, pw conv, dw conv, pw conv
-    _downsample_idx = {"conv_in": 0, "conv_mid": 2, "lin_mid": 3, "conv_end": 5, "lin_end": 6}
-
-    # audio output: MaskGIT depthformer (a.gen.code.*) and the codec decoder (a.gen.wav.*)
-    _depth_layer_map = {
-        "operator_norm":      gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM,
-        "self_attn.q_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q,
-        "self_attn.q_norm":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM,
-        "self_attn.k_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K,
-        "self_attn.k_norm":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM,
-        "self_attn.v_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_V,
-        "self_attn.o_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT,
-        "self_attn.gate_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_GATE,
-        "ffn_norm":           gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM,
-        "feed_forward.w1":    gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE,
-        "feed_forward.w3":    gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP,
-        "feed_forward.w2":    gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
-    }
-    _codec_layer_map = {
-        "operator_norm":         gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM,
-        "post_operator_norm":    gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_POST_NORM,
-        "self_attn.q_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q,
-        "self_attn.q_norm":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q_NORM,
-        "self_attn.k_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K,
-        "self_attn.k_norm":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K_NORM,
-        "self_attn.v_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V,
-        "self_attn.o_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT,
-        "self_attn.gate_proj":   gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_GATE,
-        "conv.in_proj":          gguf.MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW1,
-        "conv.conv":             gguf.MODEL_TENSOR.A_GEN_WAV_TFM_CONV_DW,
-        "conv.out_proj":         gguf.MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW2,
-        "ffn_norm":              gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM,
-        "post_ffn_norm":         gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_POST_NORM,
-        "feed_forward.w1":       gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_GATE,
-        "feed_forward.w3":       gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP,
-        "feed_forward.w2":       gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN,
-    }
-    # the codec graph in clip.cpp hardcodes the liquid-codec FSQ front end and decoder shape, conversion fails on a mismatch
-    _codec = {
-        "model_type": "liquid_codec", "fsq_levels": [8, 7, 6, 6], "n_q": 12, "quantizer_dim": 256,
-        "encoder_dim": 512, "stft_n_fft": 1280, "stft_hop_length": 320, "stft_win_length": 1280, "stft_frame_stack": 6,
-        "decoder_window": 30, "sampling_rate": 24000,
-    }
-    # the lfm3a pipeline in mtmd-helper-gen.cpp hardcodes these
-    _generation = {"codebooks": 12, "maskgit_schedule": [4, 4, 4], "interleaved_n_text": 6, "interleaved_n_audio": 12}
-
-    # speech-code sampling of the model card, generation_config.json overrides it
-    _audio_sampling = {
-        "audio_temperature": 1.0, "audio_top_k": 128, "audio_repetition_penalty": 1.1, "audio_repetition_window": 4096,
-    }
-
-    # mtmd_audio_preprocessor_lfm3a hardcodes this front end, conversion fails on a mismatch
-    _feature_extractor = {
-        "sampling_rate": 16000, "feature_size": 128, "n_fft": 400, "win_length": 400, "hop_length": 160,
-        "log_floor": -12.0, "normalization_offset": 4.0, "normalization_divisor": 4.0, "encoder_stride": 8,
-    }
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        # audio output is exported when the checkpoint ships the codec
-        self._audio_output = (self.dir_model / "audio_tokenizer").is_dir()
-        gen = self.get_generation_config()
-        self._n_codebooks: int = gen["codebooks"]
-        self._audio_vocab: int = gen["audio_vocab_size"]
-        self._depth_embd: dict[int, Tensor] = {}
-        self._depth_norm: dict[int, Tensor] = {}
-
-    def get_audio_config(self) -> dict[str, Any] | None:
-        enc = self.global_config["encoder"]
-        cfg = dict(enc["lfm"])
-        cfg["num_mel_bins"] = enc["dim_in"]
-        cfg["sliding_window"] = enc["sliding_window"]
-        return cfg
-
-    def get_generation_config(self) -> dict[str, Any]:
-        c = self.global_config
-        steps = c["maskgit_steps"]
-        return {
-            "codebooks": c["codebooks"], "audio_vocab_size": c["audio_vocab_size"],
-            "maskgit_schedule": [c["codebooks"] // steps] * steps,
-            "interleaved_n_text": c["interleaved_n_text"], "interleaved_n_audio": c["interleaved_n_audio"],
-        }
-
-    def get_depth_config(self) -> dict[str, Any]:
-        return self.global_config["depthformer"]
-
-    def set_gguf_parameters(self):
-        assert self.hparams_audio is not None
-        fe = self.preprocessor_config["feature_extractor"]
-        if fe["feature_extractor_type"] not in ("Lfm3AudioFeatureExtractor", "Lfm3OmniFeatureExtractor"):
-            raise ValueError(f"unsupported feature_extractor_type {fe['feature_extractor_type']!r}")
-        for k, v in self._feature_extractor.items():
-            if fe[k] != v:
-                raise ValueError(f"unsupported feature_extractor.{k}={fe[k]!r}, expected {v!r}")
-
-        # clip.cpp hardcodes the encoder rope base
-        if self.hparams_audio["rope_parameters"]["rope_theta"] != 1e6:
-            raise ValueError("unsupported audio encoder rope_theta")
-
-        self.hparams_audio["intermediate_size"] = lfm2_feed_forward_length(self.hparams_audio)
-        super().set_gguf_parameters()
-        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.LFM3A)
-        self.gguf_writer.add_audio_head_count_kv(self.hparams_audio["num_key_value_heads"])
-        self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams_audio["norm_eps"])
-        self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["num_mel_bins"])
-        self.gguf_writer.add_audio_window_size(self.hparams_audio["sliding_window"])
-
-        if not self._audio_output:
-            return
-
-        # audio generation: depthformer hparams, the codec ones are constant and live in clip.cpp
-        gen = self.get_generation_config()
-        for k, v in self._generation.items():
-            if gen[k] != v:
-                raise ValueError(f"unsupported {k}={gen[k]!r}, expected {v!r}")
-        depth = self.get_depth_config()
-        if depth.get("is_causal", False) or depth["use_sandwich_norm"] or set(depth["layer_types"]) != {"full_attention"}:
-            raise ValueError("unsupported depthformer, expected a bidirectional all-attention stack")
-        if depth["rope_parameters"]["full_attention"]["rope_theta"] != 1e6:
-            raise ValueError("unsupported depthformer rope_theta")
-        self.gguf_writer.add_clip_has_gen_audio_encoder(True)
-        self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.LFM3A_GEN)
-        self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
-        self.gguf_writer.add_gen_audio_embedding_length(depth["hidden_size"])
-        self.gguf_writer.add_gen_audio_feed_forward_length(depth["intermediate_size"])
-        self.gguf_writer.add_gen_audio_block_count(depth["num_hidden_layers"])
-        self.gguf_writer.add_gen_audio_head_count(depth["num_attention_heads"])
-        self.gguf_writer.add_gen_audio_head_count_kv(depth["num_key_value_heads"])
-        self.gguf_writer.add_gen_audio_attention_layernorm_eps(depth["norm_eps"])
-
-        # sampling defaults of the speech codes, next to the text ones that become general.sampling.*
-        gen = gguf.Metadata.load_generation_config(self.dir_model)
-        for key, add in (("audio_temperature",        self.gguf_writer.add_gen_audio_sampling_temp),
-                         ("audio_top_k",              self.gguf_writer.add_gen_audio_sampling_top_k),
-                         ("audio_repetition_penalty", self.gguf_writer.add_gen_audio_sampling_penalty_repeat),
-                         ("audio_repetition_window",  self.gguf_writer.add_gen_audio_sampling_penalty_last_n)):
-            value = gen.get(key, self._audio_sampling.get(key))
-            if value is not None:
-                add(value)
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        if new_name.startswith("a.conv1d.") or "conv_dw" in new_name or new_name.startswith("a.gen.wav.quant."):
-            return gguf.GGMLQuantizationType.F32
-        # the codec decoder is small and its log-magnitude/phase output does not survive quantization
-        if new_name.startswith("a.gen.wav."):
-            return gguf.GGMLQuantizationType.F16
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        if self._audio_output:
-            yield from self._codec_tensors()
-
-    def _codec_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        import json
-        from safetensors.torch import load_file
-
-        T = gguf.MODEL_TENSOR
-        codec_dir = self.dir_model / "audio_tokenizer"
-        with open(codec_dir / "config.json", encoding="utf-8") as f:
-            cfg = json.load(f)
-        for k, v in self._codec.items():
-            if cfg.get(k) != v:
-                raise ValueError(f"unsupported audio_tokenizer {k}={cfg.get(k)!r}, expected {v!r}")
-        if math.prod(cfg["fsq_levels"]) != self._audio_vocab - 1:
-            raise ValueError(f"audio_vocab_size {self._audio_vocab} does not match the codec, expected codebook size + 1")
-
-        state_dict = load_file(codec_dir / "model.safetensors")
-
-        # FSQ code table: index -> the len(levels) quantized values, see FiniteScalarQuantization._indices_to_codes
-        levels = torch.tensor(cfg["fsq_levels"], dtype=torch.int64)
-        basis = torch.cumprod(torch.cat([torch.ones(1, dtype=torch.int64), levels[:-1]]), dim=0)
-        half = levels // 2
-        idx = torch.arange(int(levels.prod()), dtype=torch.int64)[:, None]
-        codes = ((idx // basis) % levels - half) / half
-        yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), codes.to(torch.float32))
-
-        # one projection over the concatenated code vectors of all codebooks, under quantizer.fsq. in newer exports
-        quantizer = "quantizer.fsq." if "quantizer.fsq.project_out.weight" in state_dict else "quantizer."
-        for suffix in ("weight", "bias"):
-            yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_OUT, suffix=f".{suffix}"), state_dict[f"{quantizer}project_out.{suffix}"])
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_IN_PROJ, suffix=f".{suffix}"), state_dict[f"from_quantizer.{suffix}"])
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUT_PROJ, suffix=f".{suffix}"), state_dict[f"decoder.lin.{suffix}"])
-        yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUTPUT_NORM), state_dict["decoder.lfm.embedding_norm.weight"])
-
-        for name, data_torch in state_dict.items():
-            if not name.startswith("decoder.lfm.layers."):
-                continue
-            bid_str, key = name.removeprefix("decoder.lfm.layers.").rsplit(".weight", 1)[0].split(".", 1)
-            if key == "conv.conv":
-                data_torch = data_torch.squeeze(1)
-            yield (self.format_tensor_name(self._codec_layer_map[key], int(bid_str)), data_torch)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.startswith("a.gen.wav."):
-            yield (name, data_torch)
-            return
-
-        if name.startswith(("depthformer.", "depth_", "codebook_embedding.", "mask_embedding")):
-            if self._audio_output:
-                yield from self._depthformer_tensor(data_torch, name, bid)
-            return
-
-        if not name.startswith(("encoder.", "mlp.")):
-            return
-
-        if name.startswith("encoder.downsample."):
-            layer, suffix = name.removeprefix("encoder.downsample.").split(".")
-            if layer == "lin_out":
-                if suffix == "weight":
-                    data_torch = data_torch.squeeze(-1)
-                yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_ENC_OUT, suffix=f".{suffix}"), data_torch)
-                return
-            if suffix == "bias":
-                data_torch = data_torch[:, None, None]
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_ENC_CONV1D, self._downsample_idx[layer], f".{suffix}"), data_torch)
-            return
-
-        # conv op requires 2d tensor
-        if name.endswith("conv.conv.weight"):
-            data_torch = data_torch.squeeze(1)
-
-        yield (self.map_tensor_name(name), data_torch)
-
-    def _depthformer_tensor(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        T = gguf.MODEL_TENSOR
-
-        if name.startswith("depthformer.model.layers."):
-            key = name.removeprefix("depthformer.model.layers.").split(".", 1)[1].rsplit(".weight", 1)[0]
-            yield (self.format_tensor_name(self._depth_layer_map[key], bid), data_torch)
-            return
-        if name == "depthformer.model.embedding_norm.weight":
-            yield (self.format_tensor_name(T.A_GEN_CODE_OUTPUT_NORM), data_torch)
-            return
-        if name.startswith("depth_linear."):
-            yield (self.format_tensor_name(T.A_GEN_CODE_PROJ_IN, suffix="." + name.rsplit(".", 1)[1]), data_torch)
-            return
-        if name == "mask_embedding":
-            yield (self.format_tensor_name(T.A_GEN_CODE_MASK_EMBD, suffix=""), data_torch)
-            return
-        if name == "codebook_embedding.weight":
-            assert data_torch.shape[0] == self._n_codebooks * self._audio_vocab
-            yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch)
-            return
-
-        # per-codebook heads: the embedding table is tied to to_logits, merge the 12 tables into one tensor
-        assert name.startswith("depth_embeddings.")
-        idx = int(name.split(".")[1])
-        key = name.split(".", 2)[2]
-        if key == "embedding.weight":
-            self._depth_embd[idx] = data_torch
-            if len(self._depth_embd) == self._n_codebooks:
-                yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), torch.stack([self._depth_embd[i] for i in range(self._n_codebooks)], dim=0))
-        elif key == "embedding_norm.weight":
-            self._depth_norm[idx] = data_torch
-            if len(self._depth_norm) == self._n_codebooks:
-                yield (self.format_tensor_name(T.A_GEN_CODE_HEAD_NORM), torch.stack([self._depth_norm[i] for i in range(self._n_codebooks)], dim=0))
-        elif key == "to_logits.weight":
-            if not torch.equal(data_torch, self._depth_embd[idx]):
-                raise ValueError(f"depth_embeddings.{idx} is not tied")
-        else:
-            raise ValueError(f"unexpected tensor {name}")
-
-
-@ModelBase.register("Lfm3OmniForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-Omni-2B")
-class LFM3OmniMmprojModel(LFM3VLVisionMixin, LFM3AudioMmprojModel):
-    has_vision_encoder = True
-    has_audio_encoder = True
-
-    _audio_sampling = {"audio_temperature": 0.4, "audio_repetition_penalty": 1.2, "audio_repetition_window": 64}
-
-    # the audio side is the LFM3-Audio model under the HF-native names
-    _audio_renames = (
-        ("model.audio_encoder.lfm.", "encoder.lfm.lfm."),
-        ("model.audio_encoder.", "encoder."),
-        ("model.audio_projector.layer_norm.", "mlp.model.0."),
-        ("model.audio_projector.linear_1.", "mlp.model.1."),
-        ("model.audio_projector.linear_2.", "mlp.model.3."),
-        ("model.depth_decoder.model.", "depthformer.model."),
-        ("model.depth_decoder.", ""),
-        ("model.codebook_embedding.", "codebook_embedding."),
-    )
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        assert self.hparams_vision is not None
-        # dynamic resolution, image_size is only set for MmprojModel compatibility
-        self.hparams_vision["image_size"] = 512
-
-    def get_audio_config(self) -> dict[str, Any] | None:
-        enc = self.global_config["audio_encoder_config"]
-        cfg = dict(enc["text_config"])
-        cfg["num_mel_bins"] = enc["num_mel_bins"]
-        cfg["sliding_window"] = enc["sliding_window"]
-        return cfg
-
-    def get_generation_config(self) -> dict[str, Any]:
-        c = self.global_config
-        return {
-            "codebooks": c["num_codebooks"], "audio_vocab_size": c["codebook_size"], "maskgit_schedule": c["maskgit_schedule"],
-            "interleaved_n_text": c["interleaved_n_text"], "interleaved_n_audio": c["interleaved_n_audio"],
-        }
-
-    def get_depth_config(self) -> dict[str, Any]:
-        return self.global_config["depth_config"]
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        self._set_lfm3vl_gguf_parameters()
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.startswith(("model.vision_tower.", "model.multi_modal_projector.")):
-            yield from self._lfm3vl_tensors(data_torch, name, bid)
-            return
-        for old, new in self._audio_renames:
-            if name.startswith(old):
-                yield from super().modify_tensors(data_torch, new + name.removeprefix(old), bid)
-                return
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/qwen.py

@@ -690,52 +690,6 @@ class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
         self._try_set_pooling_type()
 
 
-@ModelBase.register("MiniCPMOCRForSequenceClassification")
-class MiniCPMOCRSeqClsModel(Qwen3_5TextModel):
-    # OCR consistency judge: Qwen3.5 VL trunk with a value head read at the last token
-    # the class scores are [inconsistent, consistent], the softmax temperature is 1
-    model_arch = gguf.MODEL_ARCH.QWEN35
-    no_mtp = True  # the checkpoint has no MTP head
-
-    CLASS_LABELS = ["inconsistent", "consistent"]
-
-    def set_vocab(self):
-        super().set_vocab()
-        self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])
-
-    def _systemone_template(self) -> str:
-        # the prompt of the training data, the two images are the original page and the one
-        # rendered from the OCR text; the task is fixed, there is nothing to fill in
-        # the trailing newlines are in an expression, so the template does not drop them
-        return (
-            "<|im_start|>user\n{{ images[0] }}\nOCR文本：\n{{ state }}\n{{ images[1] }}\n"
-            "第一张图是原图，第二张图是上述 OCR 文本渲染得到的图片。\n"
-            "请判断该 OCR 文本及其对应渲染图是否与原图内容一致。\n"
-            "这是一个二分类任务：一致标为 1，不一致标为 0。"
-            "{{ '<|im_end|>\\n<|im_start|>assistant\\n<think>\\n\\n</think>\\n\\n' }}"
-        )
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-
-        # the config.json of the checkpoint keeps the one of the base model, it has no num_labels / id2label
-        if "score.weight" not in self.model_tensors:
-            raise ValueError("the checkpoint has no score.weight classification head")
-
-        self.gguf_writer.add_pooling_type(gguf.PoolingType.LAST)
-        self.gguf_writer.add_embedding_length_out(len(self.CLASS_LABELS))
-        self.gguf_writer.add_classifier_output_labels(self.CLASS_LABELS)
-        self.gguf_writer.add_decision_type(gguf.DecisionType.MINICPM_OCR_JEV)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name == "score.weight":
-            if data_torch.shape[0] != len(self.CLASS_LABELS):
-                raise ValueError(f"score.weight must have {len(self.CLASS_LABELS)} rows, got {data_torch.shape[0]}")
-            yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.CLS_OUT] + ".weight", data_torch)
-            return
-        yield from super().modify_tensors(data_torch, name, bid)
-
-
 def _is_openjev_checkpoint(dir_model: Path) -> bool:
     return (dir_model / "helper" / "shim.py").is_file() and (dir_model / "config.json").is_file()
 

conversion/qwen3vl.py

@@ -144,16 +144,6 @@ class Qwen3VLVisionModel(MmprojModel):
         yield from MmprojModel.modify_tensors(self, data_torch, name, bid)
 
 
-@ModelBase.register("MiniCPMOCRForSequenceClassification")
-class MiniCPMOCRSeqClsVisionModel(Qwen3VLVisionModel):
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        # trained with the image budget of its processor, wider than the mtmd default of Qwen-VL
-        size = self.preprocessor_config["size"]
-        self.gguf_writer.add_vision_min_pixels(int(size["shortest_edge"]))
-        self.gguf_writer.add_vision_max_pixels(int(size["longest_edge"]))
-
-
 @ModelBase.register("Qwen3OmniMoeForConditionalGeneration")
 @ModelBase.example("Qwen/Qwen3-Omni-30B-A3B-Instruct")
 class Qwen3OmniMmprojModel(Qwen3VLVisionModel, Qwen25AudioModel):

conversion/qwen4exp.py

@@ -41,8 +41,6 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
         self._ple_shards: dict[int, str] = {}
         self._ple_row_dim: int | None = None
         self._mtp_fc: dict[str, Tensor] = {}
-        # halves of the merged weights, keyed by (bid, merged tensor), see _merge_shared_input
-        self._merge_parts: dict[tuple[int, gguf.MODEL_TENSOR], dict[int, Tensor]] = {}
 
     @classmethod
     def filter_tensors(cls, item):
@@ -133,79 +131,44 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
         # int64 hash constants must stay exact; 1-D tensors force F32, so use KV
         if name.endswith("ple_embedding.layer_multipliers"):
             self._ple_multipliers = [int(x) for x in data_torch.tolist()]
-            return
+            return []
         if name.endswith("ple_embedding.ngram_heads_offsets"):
             self._ple_head_offsets = [int(x) for x in data_torch.tolist()]
-            return
+            return []
         if name.endswith("ple_embedding.ngram_heads_vocab_sizes"):
             self._ple_head_vocab_sizes = [int(x) for x in data_torch.tolist()]
-            return
+            return []
 
         if ".ngram_embedding.shard_" in name:
-            yield from self._place_ple_shard(data_torch, name)
-            return
+            return self._place_ple_shard(data_torch, name)
 
         # eh_proj([e ; h_s]) = fc_embedding(e) + fc_hidden(h_s) for every hc stream s
         if name.endswith((".nextn_fc_embedding.weight", ".nextn_fc_hidden.weight")):
             self._mtp_fc[name.rsplit(".", 2)[1]] = data_torch
             if len(self._mtp_fc) < 2:
-                return
+                return []
             eh = torch.cat([self._mtp_fc.pop("nextn_fc_embedding"), self._mtp_fc.pop("nextn_fc_hidden")], dim=1)
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.NEXTN_EH_PROJ, bid, ".weight"), eh)
-            return
+            return [(self.format_tensor_name(gguf.MODEL_TENSOR.NEXTN_EH_PROJ, bid, ".weight"), eh)]
 
         # one projection feeds indexer q and k; split it, as minimax-m3 does
         if ".indexer.index_qk_proj.weight" in name:
             n_q = self.hparams["indexer_n_heads"] * self.hparams["indexer_head_dim"]
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_Q_PROJ, bid, ".weight"), data_torch[:n_q])
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_K_PROJ, bid, ".weight"), data_torch[n_q:])
-            return
+            q = data_torch[:n_q]
+            k = data_torch[n_q:]
+            return [
+                (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_Q_PROJ, bid, ".weight"), q),
+                (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_K_PROJ, bid, ".weight"), k),
+            ]
 
         # Gemma zero-centred gammas the inherited norm.weight rule misses
         if name.endswith((".ple.norm_key.weight", ".ple.norm_query.weight", ".ple.norm_conv.weight",
                           ".indexer.q_layernorm.weight", ".indexer.k_layernorm.weight")):
-            yield (self.map_tensor_name(name), data_torch + 1)
-            return
+            return [(self.map_tensor_name(name), data_torch + 1)]
 
         if name.endswith(".ple.conv1d.weight"):
-            yield (self.map_tensor_name(name), data_torch.squeeze())
-            return
+            return [(self.map_tensor_name(name), data_torch.squeeze())]
 
-        for new_name, data in super().modify_tensors(data_torch, name, bid):
-            yield from self._merge_shared_input(new_name, data, bid)
-
-    # matmuls that read the same input, written as one weight [a | b] so the runtime does one matmul:
-    # (merged tensor, (a, b), linear-attention layers only)
-    _SHARED_INPUT_MERGES = (
-        (gguf.MODEL_TENSOR.HC_ATTN_DOWN_INJECT, (gguf.MODEL_TENSOR.HC_ATTN_DOWN, gguf.MODEL_TENSOR.HC_ATTN_INJECT), False),
-        (gguf.MODEL_TENSOR.HC_FFN_DOWN_INJECT,  (gguf.MODEL_TENSOR.HC_FFN_DOWN,  gguf.MODEL_TENSOR.HC_FFN_INJECT),  False),
-        (gguf.MODEL_TENSOR.ATTN_QKV_GATE,       (gguf.MODEL_TENSOR.ATTN_QKV,     gguf.MODEL_TENSOR.ATTN_GATE),      True),
-        (gguf.MODEL_TENSOR.SSM_BETA_ALPHA_CAT,  (gguf.MODEL_TENSOR.SSM_BETA,     gguf.MODEL_TENSOR.SSM_ALPHA),      True),
-    )
-
-    # caches one half of a merged weight until the other half arrives, then yields [a | b]
-    def _merge_shared_input(self, new_name: str, data: Tensor, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # a pre-quantized checkpoint carries per-tensor scales, which one merged weight cannot hold
-        if bid is None or self.hparams.get("quantization_config"):
-            yield (new_name, data)
-            return
-
-        layer_types = self.hparams.get("layer_types") or []
-        is_linear = bid < len(layer_types) and layer_types[bid] == "linear_attention"
-
-        match = next(((merged, parts.index(part)) for merged, parts, linear_only in self._SHARED_INPUT_MERGES
-                      for part in parts
-                      if (is_linear or not linear_only) and self.match_model_tensor_name(new_name, part, bid)), None)
-        if match is None:
-            yield (new_name, data)
-            return
-
-        merged, idx = match
-        halves = self._merge_parts.setdefault((bid, merged), {})
-        halves[idx] = data
-        if len(halves) == 2:
-            del self._merge_parts[(bid, merged)]
-            yield (self.format_tensor_name(merged, bid), torch.cat([halves[0], halves[1]], dim=0))
+        return super().modify_tensors(data_torch, name, bid)
 
     # the shards concatenate into a tensor of well over 100 GB
     # use LazyChunkedTensor here, a single shard resident at a time
@@ -251,8 +214,6 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
         super().prepare_tensors()
         if self._mtp_fc:
             raise ValueError(f"MTP projection missing its other half: {sorted(self._mtp_fc)}")
-        if self._merge_parts:
-            raise ValueError(f"merged weights missing a half: {sorted((bid, t.name) for bid, t in self._merge_parts)}")
         n_parts = self.hparams.get("split_ngram_parts", 0)
         if self._ple_shards and len(self._ple_shards) != n_parts:
             raise ValueError(

convert_hf_to_gguf.py

@@ -267,15 +267,11 @@ def main() -> None:
             sys.exit(1)
 
         if args.dspark:
-            if is_mistral_format or model_architecture not in ("DeepseekV4ForCausalLM", "DeepseekV41ForCausalLM"):
-                logger.error("--dspark is only supported for DeepseekV4ForCausalLM and DeepseekV41ForCausalLM")
+            if is_mistral_format or model_architecture != "DeepseekV4ForCausalLM":
+                logger.error("--dspark is only supported for DeepseekV4ForCausalLM")
                 sys.exit(1)
-            if model_architecture == "DeepseekV41ForCausalLM":
-                from conversion.deepseek41 import DeepseekV41DSparkModel
-                model_class = DeepseekV41DSparkModel
-            else:
-                from conversion.deepseek import DeepseekV4DSparkModel
-                model_class = DeepseekV4DSparkModel
+            from conversion.deepseek import DeepseekV4DSparkModel
+            model_class = DeepseekV4DSparkModel
 
         if args.mtp or args.no_mtp:
             if not model_class.supports_mtp_export:

docs/backend/OPENCL.md

@@ -72,8 +72,6 @@ However, A6x GPUs in phones are likely not supported due to the outdated driver
 | MXFP4                  | Support                    |
 | IQ4_NL                 | Support                    |
 
-`MUL_MAT_ID` accepts an expert id of `-1` and writes zeros to that slot. `ADD_ID` adds no bias for `-1`, including in fused MoE operations.
-
 ## Model Preparation
 
 Since common quantizations are supported now, it is recommanded to download GGUF models directly from Huggingface.

docs/backend/snapdragon/README.md

@@ -324,9 +324,6 @@ on 4 physical NPUs, or `--devices 'HTP0[0-1:0],HTP1[0-1:1]'` on 2 physical NPUs
 - `GGML_HEXAGON_NHVX=0`
   Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version).
 
-- `GGML_HEXAGON_VTCM_SIZE=0`
-  Overrides the VTCM size used by each session. Accepts `K`, `M` and `G` suffixes; a plain number is in MB. The default (0) uses the size the hardware reports. Values above the hardware size will fail to start the session.
-
 - `GGML_HEXAGON_HOSTBUF=1` (default: 0, disabled)
   Enables allocating host buffers for debugging. By default, host buffers are disabled.
 

examples/realtime_client.py

@@ -1,172 +0,0 @@
-#!/usr/bin/env python3
-"""Example client for the llama-server /v1/realtime websocket, with the openai SDK.
-
-Start a server with an audio model, e.g. LFM3-Audio:
-
-    llama-server -hf LiquidAI/LFM3-Audio-2B-GGUF
-
-Spoken conversation, the audio of each wav file is one user turn. It is sent in chunks as it would come from a
-microphone, the server prefills it as it arrives so the response starts right after the last chunk:
-
-    python examples/realtime_client.py --instructions "Respond with interleaved text and audio." question.wav
-
-Output modalities: left out, the model picks; "text", "audio" or "text,audio" (interleaved) force the response.
-Without --instructions, a forced mode comes with the system prompt of the LFM3-Audio or LFM3-Omni model card:
-
-    python examples/realtime_client.py --output-modalities text,audio --out answer.wav question.wav
-
-Text turns go in with --text, transcription sessions transcribe each file:
-
-    python examples/realtime_client.py --text "What is the capital of France?"
-    python examples/realtime_client.py --type transcription --language en speech.wav
-
-Live transcription models (Voxtral Realtime, LFM3-Audio live ASR) transcribe while the audio is sent, a transcription
-session opens with ?intent=transcription:
-
-    python examples/realtime_client.py --type transcription speech.wav
-
-Requirements: pip install "openai[realtime]" numpy
-"""
-
-import argparse
-import asyncio
-import base64
-import time
-import wave
-
-import numpy as np
-from openai import AsyncOpenAI
-
-
-# system prompts of the LiquidAI/LFM3-Audio-2B and LiquidAI/LFM3-Omni-2B model cards for a forced output mode
-MODE_INSTRUCTIONS = {
-    "audio": {"text,audio": "Respond with interleaved text and audio."},
-    "omni":  {"text": "Respond in text.", "text,audio": "Respond in interleaved text and audio."},
-}
-
-
-def read_wav(path, rate):
-    # PCM16 wav -> mono int16 at the given rate
-    with wave.open(path, "rb") as f:
-        assert f.getsampwidth() == 2, "the example reads PCM16 wav files"
-        x = np.frombuffer(f.readframes(f.getnframes()), dtype=np.int16).astype(np.float32)
-        x = x.reshape(-1, f.getnchannels()).mean(axis=1)
-        sr = f.getframerate()
-    if sr != rate:
-        n = int(round(len(x) * rate / sr))
-        x = np.interp(np.arange(n) * sr / rate, np.arange(len(x)), x)
-    return np.clip(x, -32768, 32767).astype(np.int16)
-
-
-def write_wav(path, pcm, rate):
-    with wave.open(path, "wb") as f:
-        f.setnchannels(1)
-        f.setsampwidth(2)
-        f.setframerate(rate)
-        f.writeframes(pcm.tobytes())
-
-
-async def main(args):
-    # the SDK derives a wss:// URL from base_url, a plain http server needs ws://
-    client = AsyncOpenAI(base_url=args.url, api_key=args.api_key, websocket_base_url=args.url.replace("http", "ws", 1))
-    query = {"intent": "transcription"} if args.type == "transcription" else {}
-    async with client.realtime.connect(model=args.model, extra_query=query) as conn:
-        ev = await conn.recv()
-        if ev.type == "error":
-            print("error:", ev.error.message)
-            return
-        # the audio goes in at the rate of the model, PCM16 mono
-        session_info = ev.model_dump(warnings=False)["session"]
-        audio_fmt = session_info["audio"]
-        card = "omni" if "omni" in str(session_info.get("model", "")).lower() else "audio"
-        in_rate = audio_fmt["input"]["format"]["rate"]
-        out_rate = (audio_fmt.get("output") or {}).get("format", {}).get("rate", 24000)
-
-        session = {"type": args.type}
-        if args.type == "realtime":
-            instructions = args.instructions
-            if instructions is None:
-                instructions = MODE_INSTRUCTIONS[card].get(args.output_modalities)
-            if instructions is not None:
-                session["instructions"] = instructions
-            if args.output_modalities:
-                session["output_modalities"] = args.output_modalities.split(",")
-        elif args.language:
-            session["audio"] = {"input": {"transcription": {"language": args.language}}}
-        await conn.session.update(session=session)  # ty: ignore[invalid-argument-type]
-        ev = await conn.recv()
-        if ev.type == "error":
-            print("error:", ev.error.message)
-            return
-
-        turns = [("audio", path) for path in args.wav] + [("text", text) for text in args.text]
-        for i, (kind, src) in enumerate(turns):
-            t_last = 0.0
-
-            # the input is sent while the events are read: live transcription arrives while the audio is sent
-            async def send_turn():
-                nonlocal t_last
-                if kind == "audio":
-                    pcm = read_wav(src, in_rate)
-                    step = in_rate * args.chunk_ms // 1000
-                    for j in range(0, len(pcm), step):
-                        chunk = pcm[j:j + step]
-                        await conn.input_audio_buffer.append(audio=base64.b64encode(chunk.tobytes()).decode())
-                        if args.realtime:
-                            await asyncio.sleep(len(chunk) / in_rate)
-                    if args.type == "transcription":
-                        # "final" ends the input, models that transcribe with a delay then complete the last item
-                        await conn.send({"type": "input_audio_buffer.commit", "final": i == len(args.wav) - 1})  # ty: ignore[invalid-argument-type]
-                else:
-                    await conn.conversation.item.create(item={
-                        "type": "message", "role": "user", "content": [{"type": "input_text", "text": src}]})
-                if args.type == "realtime":
-                    # the audio not committed yet ends the user turn
-                    await conn.response.create()
-                t_last = time.perf_counter()
-
-            print(f"--- turn {i} ({kind}: {src})")
-            sender = asyncio.create_task(send_turn())
-            t_first = None
-            audio = []
-            async for ev in conn:
-                if ev.type == "response.output_text.delta" or ev.type == "conversation.item.input_audio_transcription.delta":
-                    t_first = t_first or time.perf_counter()
-                    print(ev.delta, end="", flush=True)
-                elif ev.type == "response.output_audio.delta":
-                    t_first = t_first or time.perf_counter()
-                    audio.append(np.frombuffer(base64.b64decode(ev.delta), dtype=np.int16))
-                elif ev.type in ("response.done", "conversation.item.input_audio_transcription.completed"):
-                    break
-                elif ev.type == "error":
-                    print("\nerror:", ev.error.message)
-                    return
-            await sender
-            print()
-            if t_first is not None and t_first > t_last:
-                print(f"first output {1000 * (t_first - t_last):.0f} ms after the end of the input")
-            elif t_first is not None:
-                print(f"output started {1000 * (t_last - t_first):.0f} ms before the end of the input, "
-                      f"done {1000 * (time.perf_counter() - t_last):.0f} ms after it")
-            if audio:
-                pcm_out = np.concatenate(audio)
-                path = f"{args.out.removesuffix('.wav')}_{i}.wav" if len(turns) > 1 else args.out
-                write_wav(path, pcm_out, out_rate)
-                print(f"audio: {len(pcm_out) / out_rate:.2f} s -> {path}")
-
-
-if __name__ == "__main__":
-    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    p.add_argument("wav", nargs="*", help="PCM16 wav files, one user turn each")
-    p.add_argument("--text", action="append", default=[], help="a user turn as text, after the audio turns, repeatable")
-    p.add_argument("--type", choices=["realtime", "transcription"], default="realtime", help="session type")
-    p.add_argument("--instructions", help="system prompt of the conversation")
-    p.add_argument("--output-modalities", choices=["text", "audio", "text,audio"], help="default: the model picks")
-    p.add_argument("--language", help="transcription language, e.g. en")
-    p.add_argument("--chunk-ms", type=int, default=100, help="audio chunk size")
-    p.add_argument("--realtime", action=argparse.BooleanOptionalAction, default=True, help="send the audio in real time")
-    p.add_argument("--out", default="response.wav", help="where the audio of the responses goes")
-    p.add_argument("--url", default="http://127.0.0.1:9931/v1")
-    p.add_argument("--api-key", default="none")
-    p.add_argument("--model", default="default", help="model name, used by the router")
-    asyncio.run(main(p.parse_args()))

ggml/include/ggml-rpc.h

@@ -6,9 +6,9 @@
 extern "C" {
 #endif
 
-#define RPC_PROTO_MAJOR_VERSION    9
+#define RPC_PROTO_MAJOR_VERSION    8
 #define RPC_PROTO_MINOR_VERSION    0
-#define RPC_PROTO_PATCH_VERSION    0
+#define RPC_PROTO_PATCH_VERSION    1
 
 #ifdef  __cplusplus
 static_assert(GGML_OP_COUNT == 102, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");

ggml/src/ggml-cpu/arch/arm/quants.c

@@ -287,9 +287,6 @@ void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
 
     sumf = vaddvq_f32(sumv);
 #else
-    UNUSED(nb);
-    UNUSED(x);
-    UNUSED(y);
     ggml_vec_dot_q2_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
     return;
 #endif

ggml/src/ggml-cpu/ops.cpp

@@ -1914,87 +1914,213 @@ void ggml_compute_forward_repeat_back(
 }
 
 // ggml_compute_forward_concat
-
-// rows of dst copied together, so that a strided source is read along neighboring rows
-static constexpr int64_t GGML_CONCAT_TILE = 32;
-
-// copy n_rows rows of n elements of es bytes from a source with element stride nb0, element by element
-template <size_t es>
-static void ggml_concat_copy_strided(char * y, size_t nby, const char * x, size_t nbx, size_t nb0, int64_t n, int64_t n_rows) {
-    for (int64_t i0 = 0; i0 < n; i0++) {
-        for (int64_t r = 0; r < n_rows; r++) {
-            memcpy(y + r*nby + i0*es, x + r*nbx + i0*nb0, es);
-        }
-    }
-}
-
-// copy n_rows rows of n elements of src into dst, rows are nbx bytes apart in src and nby bytes apart in dst
-static void ggml_concat_copy_rows(char * y, size_t nby, const ggml_tensor * src, const char * x, size_t nbx, int64_t n, int64_t n_rows) {
-    const size_t es = ggml_type_size(src->type);
-
-    if (src->nb[0] == es) {
-        const size_t row_size = ggml_row_size(src->type, n);
-        for (int64_t r = 0; r < n_rows; r++) {
-            memcpy(y + r*nby, x + r*nbx, row_size);
-        }
-        return;
-    }
-
-    switch (es) {
-        case 1: ggml_concat_copy_strided<1>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
-        case 2: ggml_concat_copy_strided<2>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
-        case 4: ggml_concat_copy_strided<4>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
-        case 8: ggml_concat_copy_strided<8>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
-        default: GGML_ABORT("unsupported element size");
-    }
-}
-
-// each dst row comes from src0 or src1, or for dim 0 from the row of src0 followed by the row of src1
-void ggml_compute_forward_concat(
+static void ggml_compute_forward_concat_any(
     const ggml_compute_params * params,
     ggml_tensor * dst) {
 
     const ggml_tensor * src0 = dst->src[0];
     const ggml_tensor * src1 = dst->src[1];
 
+    const int ith = params->ith;
+    const int nth = params->nth;
+
     GGML_TENSOR_BINARY_OP_LOCALS
 
     const int32_t dim = ggml_get_op_params_i32(dst, 0);
 
     GGML_ASSERT(dim >= 0 && dim < 4);
-    GGML_ASSERT(nb0 == ggml_type_size(dst->type));
-    GGML_ASSERT(!ggml_is_quantized(dst->type) || (nb00 == nb0 && nb10 == nb0));
-    GGML_ASSERT(dim != 0 || (ne00 % ggml_blck_size(dst->type) == 0 && ne10 % ggml_blck_size(dst->type) == 0));
+    GGML_ASSERT(ggml_is_contiguous_rows(src0));
+    GGML_ASSERT(ggml_is_contiguous_rows(src1));
 
-    const size_t y1 = dim == 0 ? ggml_row_size(dst->type, ne00) : 0;
+    int64_t o[4] = {0, 0, 0, 0};
 
-    const int64_t nr = ne1*ne2*ne3;
-    const int64_t dr = (nr + params->nth - 1)/params->nth;
+    if (dim == 0) {
+        GGML_ASSERT(src0->ne[0] % ggml_blck_size(src0->type) == 0);
+        GGML_ASSERT(src1->ne[0] % ggml_blck_size(src1->type) == 0);
 
-    const int64_t ir0 = dr*params->ith;
-    const int64_t ir1 = MIN(ir0 + dr, nr);
+        o[dim] = src0->ne[dim]/ggml_blck_size(src0->type);
+    } else {
+        o[dim] = src0->ne[dim];
+    }
 
-    for (int64_t ir = ir0; ir < ir1; ) {
-        int64_t i[4] = { 0, ir % ne1, (ir/ne1) % ne2, ir/(ne1*ne2) };
-
-        // a tile of consecutive rows sharing i2, i3 and their source
-        int64_t nt = MIN(MIN(ir1 - ir, ne1 - i[1]), GGML_CONCAT_TILE);
-        if (dim == 1 && i[1] < ne01) {
-            nt = MIN(nt, ne01 - i[1]);
+    // Region 1: copy rows from src0
+    for (int i3 = 0; i3 < ne03; i3++) {
+        for (int i2 = ith; i2 < ne02; i2 += nth) {
+            for (int i1 = 0; i1 < ne01; i1++) {
+                const char * x = (const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03;
+                      char * y = (      char *) dst->data  + i1*nb1  + i2*nb2  + i3*nb3;
+                memcpy(y, x, ggml_row_size(src0->type, ne00));
+            }
         }
+    }
 
-        char * y = (char *) dst->data + i[1]*nb1 + i[2]*nb2 + i[3]*nb3;
-
-        if (dim == 0 || i[dim] < src0->ne[dim]) {
-            ggml_concat_copy_rows(y, nb1, src0, (const char *) src0->data + i[1]*nb01 + i[2]*nb02 + i[3]*nb03, nb01, ne00, nt);
+    // Region 2: copy rows from src1, offset into dst by o[]
+    for (int i3 = 0; i3 < ne13; i3++) {
+        for (int i2 = ith; i2 < ne12; i2 += nth) {
+            for (int i1 = 0; i1 < ne11; i1++) {
+                const char * x = (const char *) src1->data + i1*nb11         + i2*nb12         + i3*nb13;
+                      char * y = (      char *)  dst->data + (i1 + o[1])*nb1 + (i2 + o[2])*nb2 + (i3 + o[3])*nb3 + o[0]*nb0;
+                memcpy(y, x, ggml_row_size(src1->type, ne10));
+            }
         }
+    }
+}
 
-        if (dim == 0 || i[dim] >= src0->ne[dim]) {
-            i[dim] -= src0->ne[dim]; // i[0] does not address rows
-            ggml_concat_copy_rows(y + y1, nb1, src1, (const char *) src1->data + i[1]*nb11 + i[2]*nb12 + i[3]*nb13, nb11, ne10, nt);
+static void ggml_compute_forward_concat_i8(
+    const ggml_compute_params * params,
+    ggml_tensor * dst) {
+
+    const ggml_tensor * src0 = dst->src[0];
+    const ggml_tensor * src1 = dst->src[1];
+
+    GGML_ASSERT(ggml_type_size(src0->type) == sizeof(int8_t));
+
+    const int ith = params->ith;
+    const int nth = params->nth;
+
+    GGML_TENSOR_BINARY_OP_LOCALS
+
+    const int32_t dim = ggml_get_op_params_i32(dst, 0);
+
+    GGML_ASSERT(dim >= 0 && dim < 4);
+
+    int64_t o[4] = {0, 0, 0, 0};
+    o[dim] = src0->ne[dim];
+
+    const int8_t * x;
+
+    // TODO: smarter multi-theading
+    for (int i3 = 0; i3 < ne3; i3++) {
+        for (int i2 = ith; i2 < ne2; i2 += nth) {
+            for (int i1 = 0; i1 < ne1; i1++) {
+                for (int i0 = 0; i0 < ne0; i0++) {
+                    if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
+                        x = (const int8_t *) ((const char *)src0->data + (i0       )*nb00 + (i1       )*nb01 + (i2       )*nb02 + (i3       )*nb03);
+                    } else {
+                        x = (const int8_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
+                    }
+
+                    int8_t * y = (int8_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
+
+                    *y = *x;
+                }
+            }
         }
+    }
+}
 
-        ir += nt;
+static void ggml_compute_forward_concat_f16(
+    const ggml_compute_params * params,
+    ggml_tensor * dst) {
+
+    const ggml_tensor * src0 = dst->src[0];
+    const ggml_tensor * src1 = dst->src[1];
+
+    GGML_ASSERT(ggml_type_size(src0->type) == sizeof(ggml_fp16_t));
+
+    const int ith = params->ith;
+    const int nth = params->nth;
+
+    GGML_TENSOR_BINARY_OP_LOCALS
+
+    const int32_t dim = ggml_get_op_params_i32(dst, 0);
+
+    GGML_ASSERT(dim >= 0 && dim < 4);
+
+    int64_t o[4] = {0, 0, 0, 0};
+    o[dim] = src0->ne[dim];
+
+    const ggml_fp16_t * x;
+
+    // TODO: smarter multi-theading
+    for (int i3 = 0; i3 < ne3; i3++) {
+        for (int i2 = ith; i2 < ne2; i2 += nth) {
+            for (int i1 = 0; i1 < ne1; i1++) {
+                for (int i0 = 0; i0 < ne0; i0++) {
+                    if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
+                        x = (const ggml_fp16_t *) ((const char *)src0->data + (i0       )*nb00 + (i1       )*nb01 + (i2       )*nb02 + (i3       )*nb03);
+                    } else {
+                        x = (const ggml_fp16_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
+                    }
+
+                    ggml_fp16_t * y = (ggml_fp16_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
+
+                    *y = *x;
+                }
+            }
+        }
+    }
+}
+
+static void ggml_compute_forward_concat_f32(
+    const ggml_compute_params * params,
+    ggml_tensor * dst) {
+
+    const ggml_tensor * src0 = dst->src[0];
+    const ggml_tensor * src1 = dst->src[1];
+
+    GGML_ASSERT(ggml_type_size(src0->type) == sizeof(float));
+
+    const int ith = params->ith;
+    const int nth = params->nth;
+
+    GGML_TENSOR_BINARY_OP_LOCALS
+
+    const int32_t dim = ggml_get_op_params_i32(dst, 0);
+
+    GGML_ASSERT(dim >= 0 && dim < 4);
+
+    int64_t o[4] = {0, 0, 0, 0};
+    o[dim] = src0->ne[dim];
+
+    const float * x;
+
+    // TODO: smarter multi-theading
+    for (int i3 = 0; i3 < ne3; i3++) {
+        for (int i2 = ith; i2 < ne2; i2 += nth) {
+            for (int i1 = 0; i1 < ne1; i1++) {
+                for (int i0 = 0; i0 < ne0; i0++) {
+                    if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
+                        x = (const float *) ((const char *)src0->data + (i0       )*nb00 + (i1       )*nb01 + (i2       )*nb02 + (i3       )*nb03);
+                    } else {
+                        x = (const float *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
+                    }
+
+                    float * y = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
+
+                    *y = *x;
+                }
+            }
+        }
+    }
+}
+
+void ggml_compute_forward_concat(
+    const ggml_compute_params * params,
+    ggml_tensor * dst) {
+
+    const ggml_tensor * src0 = dst->src[0];
+
+    switch (src0->type) {
+        case GGML_TYPE_F16:
+        case GGML_TYPE_BF16:
+        case GGML_TYPE_I16:
+            {
+                ggml_compute_forward_concat_f16(params, dst);
+            } break;
+        case GGML_TYPE_I8:
+            {
+                ggml_compute_forward_concat_i8(params, dst);
+            } break;
+        case GGML_TYPE_F32:
+        case GGML_TYPE_I32:
+            {
+                ggml_compute_forward_concat_f32(params, dst);
+            } break;
+        default:
+            {
+                ggml_compute_forward_concat_any(params, dst);
+            }
     }
 }
 
@@ -4846,8 +4972,8 @@ static void ggml_compute_forward_scale_f32(
 
     const ggml_tensor * src0 = dst->src[0];
 
-    GGML_ASSERT(ggml_is_contiguous_rows(src0));
-    GGML_ASSERT(ggml_is_contiguous_rows(dst));
+    GGML_ASSERT(ggml_is_contiguous(src0));
+    GGML_ASSERT(ggml_is_contiguous(dst));
     GGML_ASSERT(ggml_are_same_shape(src0, dst));
 
     float s; // scale factor
@@ -4862,8 +4988,6 @@ static void ggml_compute_forward_scale_f32(
     const int nc = src0->ne[0];
     const int nr = ggml_nrows(src0);
 
-    GGML_TENSOR_UNARY_OP_LOCALS
-
     // rows per thread
     const int dr = (nr + nth - 1)/nth;
 
@@ -4871,23 +4995,25 @@ static void ggml_compute_forward_scale_f32(
     const int ir0 = dr*ith;
     const int ir1 = MIN(ir0 + dr, nr);
 
-    for (int i = ir0; i < ir1; i++) {
-        const int64_t i1 = i % ne01;
-        const int64_t i2 = (i / ne01) % ne02;
-        const int64_t i3 = i / (ne01*ne02);
+    const size_t nb01 = src0->nb[1];
 
-        float *       y = (float *)       ((char *) dst->data  + i1*nb1  + i2*nb2  + i3*nb3);
-        const float * x = (const float *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
+    const size_t nb1 = dst->nb[1];
 
-        if (b == 0.0f) {
+    if (b == 0.0f) {
+        for (int i1 = ir0; i1 < ir1; i1++) {
             if (dst->data != src0->data) {
                 // src0 is same shape as dst => same indices
                 // TODO: add x parameter to ggml_vec_scale_f32 and remove this memcpy
-                memcpy(y, x, nc * sizeof(float));
+                memcpy((char *)dst->data + i1*nb1, (char *)src0->data + i1*nb01, nc * sizeof(float));
             }
-            ggml_vec_scale_f32(nc, y, s);
-        } else {
-            ggml_vec_mad1_f32(nc, y, x, s, b);
+            ggml_vec_scale_f32(nc, (float *) ((char *) dst->data + i1*nb1), s);
+        }
+    } else {
+        for (int i1 = ir0; i1 < ir1; i1++) {
+            ggml_vec_mad1_f32(nc,
+                (float *) ((char *) dst->data  + i1*nb1),
+                (float *) ((char *) src0->data + i1*nb1),
+                s, b);
         }
     }
 }
@@ -4898,8 +5024,8 @@ static void ggml_compute_forward_scale_bf16(
 
     const ggml_tensor * src0 = dst->src[0];
 
-    GGML_ASSERT(ggml_is_contiguous_rows(src0));
-    GGML_ASSERT(ggml_is_contiguous_rows(dst));
+    GGML_ASSERT(ggml_is_contiguous(src0));
+    GGML_ASSERT(ggml_is_contiguous(dst));
     GGML_ASSERT(ggml_are_same_shape(src0, dst));
 
     float s; // scale factor
@@ -4914,8 +5040,6 @@ static void ggml_compute_forward_scale_bf16(
     const int nc = src0->ne[0];
     const int nr = ggml_nrows(src0);
 
-    GGML_TENSOR_UNARY_OP_LOCALS
-
     // rows per thread
     const int dr = (nr + nth - 1)/nth;
 
@@ -4923,13 +5047,9 @@ static void ggml_compute_forward_scale_bf16(
     const int ir0 = dr*ith;
     const int ir1 = MIN(ir0 + dr, nr);
 
-    for (int i = ir0; i < ir1; i++) {
-        const int64_t i1 = i % ne01;
-        const int64_t i2 = (i / ne01) % ne02;
-        const int64_t i3 = i / (ne01*ne02);
-
-        const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
-        ggml_bf16_t       * y = (ggml_bf16_t *)       ((char *)       dst->data  + i1*nb1  + i2*nb2  + i3*nb3);
+    for (int i1 = ir0; i1 < ir1; i1++) {
+        const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*src0->nb[1]);
+        ggml_bf16_t       * y = (ggml_bf16_t *)       ((char *)       dst->data  + i1*dst->nb[1]);
         for (int k = 0; k < nc; k++) {
             y[k] = GGML_FP32_TO_BF16(s*GGML_BF16_TO_FP32(x[k]) + b);
         }

ggml/src/ggml-cpu/unary-ops.cpp

@@ -78,7 +78,7 @@ static inline float op_expm1(float x) {
 }
 
 static inline float op_softplus(float x) {
-    return (x > 20.0f) ? x : log1pf(expf(x));
+    return (x > 20.0f) ? x : logf(1.0f + expf(x));
 }
 
 static inline float op_floor(float x) {

ggml/src/ggml-cuda/conv2d.cu

@@ -180,6 +180,7 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
     const int                  end            = int(int64_t(tiles) * (split + 1) / split_k) * BK;
     for (int k0 = begin; k0 < end; k0 += BK) {
         if (k_total % 8 == 0 && uintptr_t(weight) % 16 == 0) {
+#pragma unroll
             for (int i = tid; i < BM * BK / 8; i += nthreads) {
                 const int  row = i / (BK / 8), col = 8 * (i % (BK / 8));
                 const int4 v                 = m0 + row < oc && k0 + col < k_total ?
@@ -188,6 +189,7 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
                 *(int4 *) &a_s[row][col / 2] = v;
             }
         } else {
+#pragma unroll
             for (int i = tid; i < BM * BK / 2; i += nthreads) {
                 const int row = i / (BK / 2), col = 2 * (i % (BK / 2));
                 half      lo = __float2half(0.0f), hi = lo;
@@ -200,6 +202,7 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
                 a_s[row][col / 2] = __halves2half2(lo, hi);
             }
         }
+#pragma unroll
         for (int k = load_row; k < BK; k += nthreads / (BN / 2)) {
             const int ki = k0 + k;
             const int ci = ki / (kw * kh), ky = ki / kw % kh, kx = ki % kw;

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -214,36 +214,52 @@ static cudaError_t ggml_cuda_device_malloc(void ** ptr, size_t size, int device)
 }
 
 #if defined(GGML_USE_HIP)
-static int ggml_cuda_parse_id(const char * devName) {
+static int ggml_cuda_parse_id(char devName[]) {
     // A list of possible Target IDs can be found under the rocclr/clr repo in device.cpp
     // these values are not stable so this is susceptible to breakage
     // https://github.com/ROCm/clr/blob/amd-staging/rocclr/device/device.cpp
-    // Extract the base architecture model ("gfx908:sramecc+:xnack-" -> "908")
-    std::string archName = devName;
-    archName.erase(0, 3);
-    archName = archName.substr(0, archName.find(':'));
+    int archMajor = 0x0;
+    int archMinor = 0x0;
+    int archNum = GGML_CUDA_CC_OFFSET_AMD;
+    int archLen = strlen(devName);
+    char archName[archLen + 1];
 
-    const auto hex = [](const std::string & s) { return (int) strtoul(s.c_str(), nullptr, 16); };
-
-    int cc = 0;
-
-    // tease out the version information
-    if (archName.size() > 8) {
-        // versions labeled generic use '-' as delimiter
-        if (archName.compare(archName.size() - 8, 8, "-generic") == 0) {
-            archName.resize(archName.size() - 8);
-            const size_t sep = archName.find('-');
-            cc = 0x100 * hex(archName.substr(0, sep));
-            if (sep != std::string::npos) {
-                cc += 0x10 * hex(archName.substr(sep + 1));
-            }
-        }
-    } else if (archName.size() >= 3) {
-        // ordinary IDs are the version in hex
-        cc = hex(archName);
+    // strip leading 'gfx' while copying into our buffer
+    if (archLen > 3) {
+        strcpy(archName, &devName[3]);
+        archLen -= 3;
     }
 
-    return GGML_CUDA_CC_OFFSET_AMD + cc;
+    // trim trailing :xnack- or :sramecc- statuses
+    archLen = strcspn(archName, ":");
+    archName[archLen] = '\0';
+
+    // tease out the version information
+    if (archLen > 8) {
+        // versions labeled generic use '-' as delimiter
+        // strip the trailing "-generic" then iterate through what remains
+        if ((strstr(archName, "-generic"))) {
+            archName[archLen - 8] = '\0';
+            char * pch;
+            if ((pch = strtok(archName, "-"))) {
+                archMajor = (int)strtoul(pch, 0, 16);
+                if ((pch = strtok(NULL, "-"))) {
+                    archMinor = 0x10 * (int)strtoul(pch, 0, 16);
+                }
+            }
+        }
+    } else if (archLen >= 3) {
+        // last two digits should be the minor * 0x10 + stepping
+        archMinor = (int)strtoul(&archName[archLen - 2], 0, 16);
+        archName[archLen - 2] = '\0';
+
+        // only the major version remains
+        archMajor = (int)strtoul(archName, 0, 16);
+    }
+    archNum += archMajor * 0x100;
+    archNum += archMinor;
+
+    return archNum;
 }
 #endif // defined(GGML_USE_HIP)
 
@@ -1957,61 +1973,6 @@ static bool ggml_cuda_match_shared_expert(const ggml_cgraph * graph, int routed_
     return true;
 }
 
-// Buffer- and batch-size independent, so graph_optimize produces the same graph topology for every ubatch size (see #29986).
-static bool ggml_cuda_mul_mat_q_fusion_matches(const ggml_tensor * up, const ggml_tensor * gate, const ggml_tensor * glu, const int device) {
-    const ggml_tensor * x_up = up->src[0];
-    const ggml_tensor * y    = up->src[1];
-
-    if (up->op != GGML_OP_MUL_MAT || !ggml_cuda_mmq_fusion_supported(x_up->type) || y->type != GGML_TYPE_F32) {
-        return false;
-    }
-
-    // The fused kernel is only compiled for weights whose rows fill whole tiles (no fallback).
-    if (ggml_cuda_mmq_needs_fallback(x_up->ne[1])) {
-        return false;
-    }
-
-    if (!ggml_cuda_should_fuse_mul_mat(up, gate, glu)) {
-        return false;
-    }
-
-    // The fused write-back applies SWIGLU_OAI with the default alpha and limit of ggml_cuda_op_swiglu_oai_single.
-    if (ggml_get_glu_op(glu) == GGML_GLU_OP_SWIGLU_OAI && (ggml_get_op_params_f32(glu, 2) != 1.702f || ggml_get_op_params_f32(glu, 3) != 7.0f)) {
-        return false;
-    }
-
-    // NVIDIA only: the fused write-back relies on the MMA accumulator layout.
-    return turing_mma_available(ggml_cuda_info().devices[device].cc);
-}
-
-static bool ggml_cuda_should_fuse_mul_mat_q(const ggml_tensor * up, const ggml_tensor * gate, const ggml_tensor * glu) {
-    // A hinted matmul can be fused, but its unfused path may be faster (e.g. FWHT for GGML_HINT_SRC0_IS_HADAMARD).
-    if (ggml_get_op_params_i32(up, 1) != GGML_HINT_NONE || ggml_get_op_params_i32(gate, 1) != GGML_HINT_NONE) {
-        return false;
-    }
-
-    // The padding of a compute-buffer weight that is a view cannot be cleared, see ggml_cuda_mul_mat.
-    const auto bad_padding_clear = [](const ggml_tensor * x) {
-        return ggml_backend_buffer_get_usage(x->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE &&
-            ggml_nbytes(x) != ggml_backend_buffer_get_alloc_size(x->buffer, x) && x->view_src;
-    };
-    const int device = ggml_cuda_get_device();
-    if (!ggml_cuda_mul_mat_q_fusion_matches(up, gate, glu, device) || bad_padding_clear(up->src[0]) || bad_padding_clear(gate->src[0])) {
-        return false;
-    }
-
-    // The batch size checks are done here, not in ggml_cuda_mul_mat_q_fusion_matches.
-    const ggml_tensor * x_up = up->src[0];
-    const ggml_tensor * y    = up->src[1];
-    const int           cc   = ggml_cuda_info().devices[device].cc;
-    if (ggml_cuda_should_use_mmvq(x_up->type, cc, y->ne[1]) || !ggml_cuda_should_use_mmq(x_up->type, cc, y->ne[1], 0)) {
-        return false;
-    }
-
-    // The fused kernel needs one block per tile (no stream-k split), so only fuse if the tiling is efficient.
-    return ggml_cuda_mmq_fusion_is_efficient(x_up->type, x_up->ne[1], y->ne[1], y->ne[2]*y->ne[3], device);
-}
-
 static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
     GGML_TENSOR_BINARY_OP_LOCALS
 
@@ -2920,10 +2881,9 @@ static bool ggml_cuda_should_fuse_rope_set_rows(const ggml_tensor * rope,
         return false;
     }
 
-    // Only norm/neox/multi shaders have the fusion code
+    // Only norm/neox shaders have the fusion code
     const int mode = ((const int32_t *) rope->op_params)[2];
-    if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX &&
-        mode != GGML_ROPE_TYPE_MROPE && mode != GGML_ROPE_TYPE_IMROPE) {
+    if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX) {
         return false;
     }
 
@@ -3822,11 +3782,6 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph *                cgraph,
             return false;
         }
 
-        // TODO: make the fused softcap kernel stride aware, it reads src and writes dst as dense
-        if (!ggml_is_contiguous(scale->src[0]) || !ggml_is_contiguous(scale2)) {
-            return false;
-        }
-
         if (ggml_get_unary_op(tanh) != GGML_UNARY_OP_TANH) {
             return false;
         }
@@ -4422,18 +4377,6 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
                 fused_node_count  = 3;
                 break;
             }
-
-            if (ggml_cuda_should_fuse_mul_mat_q(up, gate, glu)) {
-                ggml_cuda_mm_fusion_args_host fusion_data{};
-                fusion_data.gate      = gate->src[0];
-                fusion_data.glu_op    = ggml_get_glu_op(glu);
-                fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3);
-
-                ggml_cuda_mul_mat_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data);
-                fused_mul_mat_vec = true;
-                fused_node_count  = 3;
-                break;
-            }
         }
     }
 
@@ -5030,19 +4973,6 @@ static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph
             }
         }
         for (int i = 0; i < cgraph->n_nodes; ++i) {
-            // Keep the inputs of the fused gate/up/GLU MMQ kernel alive until the GLU so that they cannot alias its output (under op offload the weights are buffer copies).
-            if (i + 2 < cgraph->n_nodes && cgraph->nodes[i + 2]->op == GGML_OP_GLU) {
-                ggml_tensor * glu  = cgraph->nodes[i + 2];
-                ggml_tensor * gate = glu->src[0];
-                ggml_tensor * up   = glu->src[1];
-                const bool ok = (gate == cgraph->nodes[i] && up == cgraph->nodes[i + 1]) || (gate == cgraph->nodes[i + 1] && up == cgraph->nodes[i]);
-                if (ok && ggml_cuda_mul_mat_q_fusion_matches(up, gate, glu, cuda_ctx->device)) {
-                    add_alloc_deps(i, i + 2);
-                    i += 2;
-                    continue;
-                }
-            }
-
             ggml_cuda_moe_weighted_reduction_match match;
             if (ggml_cuda_match_moe_weighted_reduction(cgraph, i, match)) {
                 params->add_alloc_dep(params->user_data, const_cast<ggml_tensor *>(match.experts), match.dst);
@@ -5977,8 +5907,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
         case GGML_OP_LOG:
             return true;
         case GGML_OP_SCALE:
-            return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type &&
-                   ggml_is_contiguous_rows(op->src[0]);
+            return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type;
         case GGML_OP_CLAMP:
             return ggml_is_contiguous_rows(op->src[0]);
         case GGML_OP_ADD:

ggml/src/ggml-cuda/mmq-load-tiles.cuh

@@ -5,9 +5,7 @@
 #include "mmq.cuh"
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q1_0(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -118,9 +116,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
 }
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_0(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -209,9 +205,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
 }
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_0(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -274,9 +268,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
 }
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_1(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -339,9 +331,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
 }
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_0(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -421,9 +411,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
 }
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_1(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -500,8 +488,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
     }
 }
 
-template <ggml_type type, int J, bool fallback, bool has_fusion> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q8_0(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q8_0(
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -531,7 +519,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
             i = min(i, i_max);
         }
 
-        const block_q8_0 * bxi = ggml_cuda_mmq_x_row<block_q8_0, has_fusion>(x, x_gate, kbx0, i, stride) + kbx;
+        const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbx;
 
 #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
         x_qs[i*sram_stride + 0             + txi] = get_int_b2(bxi[0].qs,                   kqsx);
@@ -554,7 +542,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
             i = min(i, i_max);
         }
 
-        const block_q8_0 * bxi = ggml_cuda_mmq_x_row<block_q8_0, has_fusion>(x, x_gate, kbx0, i, stride) + kbxd;
+        const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbxd;
 
 #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
         x_df[i*sram_stride                           + kbxd] = bxi->d;
@@ -567,9 +555,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
 // ---------------------------------------------------------------------------------------------
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_K(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -630,9 +616,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
 }
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q3_K(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -744,8 +728,8 @@ static __device__ __forceinline__ int unpack_scales_q45_K(const int * scales, co
            ((scales[ksc/2]              >> (2 * (ksc % 2)))       & 0x30303030);  // upper 2 bits
 }
 
-template <ggml_type type, int J, bool fallback, bool has_fusion> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_K(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_K(
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -773,7 +757,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
             i = min(i, i_max);
         }
 
-        const block_q4_K * bxi = ggml_cuda_mmq_x_row<block_q4_K, has_fusion>(x, x_gate, kbx0, i, stride);
+        const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;
         const int qs0 = get_int_b4(bxi->qs, txi);
 
 #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
@@ -802,7 +786,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
                 i = min(i, i_max);
             }
 
-            const block_q4_K * bxi = ggml_cuda_mmq_x_row<block_q4_K, has_fusion>(x, x_gate, kbx0, i, stride);
+            const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;
 
             const int * scales = (const int *) bxi->scales;
             const int ksc = threadIdx.x % 2;
@@ -830,7 +814,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
             i = min(i, i_max);
         }
 
-        const block_q4_K * bxi = ggml_cuda_mmq_x_row<block_q4_K, has_fusion>(x, x_gate, kbx0, i, stride);
+        const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;
 
         x_dm[i] = bxi->dm;
     }
@@ -843,7 +827,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
             i = min(i, i_max);
         }
 
-        const block_q4_K * bxi = ggml_cuda_mmq_x_row<block_q4_K, has_fusion>(x, x_gate, kbx0, i, stride) + (threadIdx.x % (MMQ_TILE_NE_K/8)) / (QI4_K/8);
+        const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / (QI4_K/8);
 
         const int * scales = (const int *) bxi->scales;
 
@@ -855,8 +839,8 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
 #endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
 }
 
-template <ggml_type type, int J, bool fallback, bool has_fusion> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_K(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_K(
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -884,7 +868,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
             i = min(i, i_max);
         }
 
-        const block_q5_K * bxi = ggml_cuda_mmq_x_row<block_q5_K, has_fusion>(x, x_gate, kbx0, i, stride);
+        const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;
         const int ky = QR5_K*txi;
 
         const int ql = get_int_b4(bxi->qs, txi);
@@ -925,7 +909,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
                 i = min(i, i_max);
             }
 
-            const block_q5_K * bxi = ggml_cuda_mmq_x_row<block_q5_K, has_fusion>(x, x_gate, kbx0, i, stride);
+            const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;
 
             const int * scales = (const int *) bxi->scales;
             const int ksc = threadIdx.x % 2;
@@ -953,7 +937,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
             i = min(i, i_max);
         }
 
-        const block_q5_K * bxi = ggml_cuda_mmq_x_row<block_q5_K, has_fusion>(x, x_gate, kbx0, i, stride);
+        const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;
 
         x_dm[i] = bxi->dm;
     }
@@ -967,7 +951,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
             i = min(i, i_max);
         }
 
-        const block_q5_K * bxi = ggml_cuda_mmq_x_row<block_q5_K, has_fusion>(x, x_gate, kbx0, i, stride);
+        const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;
 
         const int * scales = (const int *) bxi->scales;
 
@@ -979,8 +963,8 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
 #endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
 }
 
-template <ggml_type type, int J, bool fallback, bool has_fusion> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q6_K(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q6_K(
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -1009,7 +993,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
             i = min(i, i_max);
         }
 
-        const block_q6_K * bxi = ggml_cuda_mmq_x_row<block_q6_K, has_fusion>(x, x_gate, kbx0, i, stride);
+        const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride;
 
         const int ql = get_int_b2(bxi->ql, txi);
         const int ql0 = (ql >> 0) & 0x0F0F0F0F;
@@ -1039,7 +1023,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
             i = min(i, i_max);
         }
 
-        const block_q6_K * bxi = ggml_cuda_mmq_x_row<block_q6_K, has_fusion>(x, x_gate, kbx0, i, stride);
+        const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride;
 
 #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
         x_df[i*sram_stride]                     = bxi->d;
@@ -1057,7 +1041,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
             i = min(i, i_max);
         }
 
-        const block_q6_K * bxi = ggml_cuda_mmq_x_row<block_q6_K, has_fusion>(x, x_gate, kbx0, i, stride) + (threadIdx.x % (MMQ_TILE_NE_K/8)) / 4;
+        const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / 4;
 
 #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
         x_sc[i*sram_stride + threadIdx.x%4] = get_int_b2(bxi->scales, threadIdx.x % (MMQ_TILE_NE_K/8));
@@ -1070,9 +1054,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
 // ---------------------------------------------------------------------------------------------
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_s(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -1134,9 +1116,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
 }
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xxs(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -1200,9 +1180,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
 }
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xs(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -1267,9 +1245,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
 }
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_s(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -1337,9 +1313,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
 }
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_xxs(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -1403,9 +1377,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
 }
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_s(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -1473,8 +1445,8 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
     }
 }
 
-template <ggml_type type, int J, bool fallback, bool has_fusion> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_xs(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_xs(
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -1501,7 +1473,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
             i = min(i, i_max);
         }
 
-        const block_iq4_xs * bxi = ggml_cuda_mmq_x_row<block_iq4_xs, has_fusion>(x, x_gate, kbx0, i, stride);
+        const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride;
 
         const int aux_q4 = get_int_b4(bxi->qs, kqsx);
         const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);
@@ -1525,7 +1497,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
             i = min(i, i_max);
         }
 
-        const block_iq4_xs * bxi = ggml_cuda_mmq_x_row<block_iq4_xs, has_fusion>(x, x_gate, kbx0, i, stride);
+        const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride;
 
         const float d = __half2float(bxi->d);
 
@@ -1541,9 +1513,7 @@ template <ggml_type type, int J, bool fallback, bool has_fusion> static __device
 }
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_nl(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -1612,9 +1582,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
 // ---------------------------------------------------------------------------------------------
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_mxfp4(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);
@@ -1681,9 +1649,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
 }
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_mxfp4_fp4(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q4) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q4);
@@ -1725,9 +1691,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
 }
 
 template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kb0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kb0, const int i_max, const int stride) {
     constexpr int warp_size   = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps      = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;
     constexpr int I           = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);
@@ -1783,9 +1747,7 @@ template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __de
 }
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4_nvfp4(
-        const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
-    GGML_UNUSED(x_gate);
-
+        const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {
     constexpr int warp_size       = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps          = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q4) / warp_size;
     constexpr int I               = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q4);

ggml/src/ggml-cuda/mmq.cu

@@ -131,51 +131,12 @@ static ggml_prec ggml_cuda_mmq_get_prec_src1(const ggml_tensor * src0, const ggm
     return GGML_PREC_Q4;
 }
 
-// Tile width in ne11 (dense) / ne12 (MoE) direction with the fewest tiles that fits into shared memory, 0 if there is none.
-static int ggml_cuda_mmq_get_J_best(
-        const ggml_type type, const int64_t ncols, const bool fallback, const int cc, const size_t smpbo, const ggml_prec prec_src1,
-        int * nthreads_best = nullptr) {
-    int J_best        = 0;
-    int ntiles_J_best = INT_MAX;
-
-    for (int J = 8; J <= 128 && ntiles_J_best > 1; J += 8) {
-        const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1);
-        if (config.type == GGML_TYPE_COUNT) {
-            continue;
-        }
-
-        if (mmq_get_nbytes_shared(config, cc) > smpbo) {
-            continue;
-        }
-
-        const int ntiles_x = (ncols + config.J - 1) / config.J;
-
-        if (ntiles_x < ntiles_J_best) {
-            J_best = J;
-            if (nthreads_best) {
-                *nthreads_best = config.nthreads;
-            }
-            ntiles_J_best = ntiles_x;
-        }
-    }
-    return J_best;
-}
-
 void ggml_cuda_mul_mat_q(
-        ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,
-        const ggml_cuda_mm_fusion_args_host * fusion) {
+        ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {
     GGML_ASSERT(        src1->type == GGML_TYPE_F32);
     GGML_ASSERT(        dst->type  == GGML_TYPE_F32);
     GGML_ASSERT(!ids || ids->type  == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.
 
-    // Fused gate/up/GLU: src0 is the up weight, dst the GLU output.
-    const ggml_tensor * gate = fusion ? fusion->gate : nullptr;
-    if (fusion) {
-        // TODO MoE support
-        GGML_ASSERT(gate && !ids && !fusion->x_bias && !fusion->gate_bias && !fusion->x_scale && !fusion->gate_scale);
-        GGML_ASSERT(gate->type == src0->type && ggml_are_same_shape(gate, src0) && ggml_are_same_stride(gate, src0));
-    }
-
     GGML_TENSOR_BINARY_OP_LOCALS;
 
     cudaStream_t stream = ctx.stream();
@@ -197,16 +158,14 @@ void ggml_cuda_mul_mat_q(
     const float * src1_d = (const float *) src1->data;
     float       *  dst_d = (float       *)  dst->data;
 
-    // If src0 (or the fused gate weight) is a temporary compute buffer, clear any potential padding.
-    for (const ggml_tensor * t : { src0, gate }) {
-        if (t && ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {
-            const size_t size_data  = ggml_nbytes(t);
-            const size_t size_alloc = ggml_backend_buffer_get_alloc_size(t->buffer, t);
-            if (size_alloc > size_data) {
-                GGML_ASSERT(ggml_is_contiguously_allocated(t));
-                GGML_ASSERT(!t->view_src);
-                CUDA_CHECK(cudaMemsetAsync((char *) t->data + size_data, 0, size_alloc - size_data, stream));
-            }
+    // If src0 is a temporary compute buffer, clear any potential padding.
+    if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {
+        const size_t size_data  = ggml_nbytes(src0);
+        const size_t size_alloc = ggml_backend_buffer_get_alloc_size(src0->buffer, src0);
+        if (size_alloc > size_data) {
+            GGML_ASSERT(ggml_is_contiguously_allocated(src0));
+            GGML_ASSERT(!src0->view_src);
+            CUDA_CHECK(cudaMemsetAsync((char *) src0->data + size_data, 0, size_alloc - size_data, stream));
         }
     }
 
@@ -221,9 +180,7 @@ void ggml_cuda_mul_mat_q(
 
     const bool fallback = ggml_cuda_mmq_needs_fallback(ne01);
 
-    // With fusion dst is the GLU output, not a MUL_MAT; the fused types always quantize src1 to Q8_1.
-    // TODO support for Q4 precision if both tensors allow it
-    const ggml_prec prec_src1 = gate ? GGML_PREC_Q8 : ggml_cuda_mmq_get_prec_src1(src0, dst, cc);
+    const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0, dst, cc);
 
     const bool use_native_fp4 = prec_src1 == GGML_PREC_Q4;
     const size_t y_block_size       = use_native_fp4 ? sizeof(block_fp4_mmq) : sizeof(block_q8_1_mmq);
@@ -244,7 +201,26 @@ void ggml_cuda_mul_mat_q(
             }
         }
 
-        J_best = ggml_cuda_mmq_get_J_best(src0->type, ncols_opt, fallback, cc, smpbo, prec_src1, &nthreads_best);
+        int ntiles_J_best = INT_MAX;
+
+        for (int J = 8; J <= 128 && ntiles_J_best > 1; J += 8) {
+            const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(src0->type, J, fallback, cc, prec_src1);
+            if (config.type == GGML_TYPE_COUNT) {
+                continue;
+            }
+
+            if (mmq_get_nbytes_shared(config, cc) > smpbo) {
+                continue;
+            }
+
+            const int ntiles_x = (ncols_opt + config.J - 1) / config.J;
+
+            if (ntiles_x < ntiles_J_best) {
+                J_best = J;
+                nthreads_best = config.nthreads;
+                ntiles_J_best = ntiles_x;
+            }
+        }
     }
     GGML_ASSERT(J_best > 0);
 
@@ -286,20 +262,13 @@ void ggml_cuda_mul_mat_q(
                                 ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int));
         const int64_t s13 = ne12*s12;
 
-        ggml_cuda_mm_fusion_args_device fusion_device = {};
-        if (gate) {
-            fusion_device.gate      = gate->data;
-            fusion_device.glu_op    = fusion->glu_op;
-            fusion_device.glu_limit = fusion->glu_limit;
-        }
-
         const mmq_args args = {
             src0_d, src0->type, (const int *) src1_q8_1.ptr, nullptr, nullptr, dst_d,
             src0->type == GGML_TYPE_NVFP4 && use_native_fp4 ? src1_scale.ptr : nullptr,
             ne00, ne01, ne1, s01, ne11, s1,
             ne02, ne12, s02, s12, s2,
             ne03, ne13, s03, s13, s3,
-            ne1, J_best, fusion_device};
+            ne1, J_best};
         ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1);
         return;
     }
@@ -516,22 +485,3 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
 
     return (!GGML_CUDA_CC_IS_CDNA(cc)) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;
 }
-
-bool ggml_cuda_mmq_fusion_is_efficient(const ggml_type type, const int64_t nrows_x, const int64_t ncols, const int64_t nchannels, const int device) {
-    const int    cc    = ggml_cuda_info().devices[device].cc;
-    const int    nsm   = ggml_cuda_info().devices[device].nsm;
-    const size_t smpbo = ggml_cuda_info().devices[device].smpbo;
-
-    // Same tile width as ggml_cuda_mul_mat_q; the fused kernel has no fallback, quantizes src1 to Q8_1 and writes I/2 output rows per tile.
-    const bool fallback = false;
-    const int  J_best   = ggml_cuda_mmq_get_J_best(type, ncols, fallback, cc, smpbo, GGML_PREC_Q8);
-    if (J_best == 0) {
-        return false;
-    }
-    const int64_t nrows_tile = ggml_cuda_mmq_get_config(type, J_best, fallback, cc, GGML_PREC_Q8).I / 2;
-    const int64_t ntiles     = (ncols + J_best - 1)/J_best * ((nrows_x + nrows_tile - 1)/nrows_tile) * nchannels;
-    // Same condition as launch_mul_mat_q uses to run the stream-k kernel with one block per tile.
-    const int64_t tiles_nwaves             = (ntiles + nsm - 1) / nsm;
-    const int64_t tiles_efficiency_percent = 100 * ntiles / (nsm*tiles_nwaves);
-    return tiles_efficiency_percent >= 90;
-}

ggml/src/ggml-cuda/mmq.cuh

@@ -1,7 +1,6 @@
 #pragma once
 
 #include "common.cuh"
-#include "unary.cuh"
 
 #include <climits>
 #include <cstdint>
@@ -11,12 +10,10 @@
 #define MMQ_ITER_K_FP4         512
 #define MMQ_NWARPS               8
 
-typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, const char * __restrict__ x_gate, int * x_tile,
-    const int kbx0, const int i_max, const int stride);
+typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride);
 typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00);
 typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted,
-    float * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max,
-    const ggml_glu_op glu_op, const float glu_limit);
+    float * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max);
 
 enum mmq_q8_1_ds_layout {
     MMQ_Q8_1_DS_LAYOUT_D4,
@@ -429,37 +426,12 @@ static __host__ int ggml_cuda_mmq_get_nbytes_shared_x(const ggml_cuda_mmq_config
 
 // ------------------------------------------------------------
 
-// Weight types with a fused gate/up/GLU path (see ggml_cuda_mmq_get_util_funcs).
-static constexpr __host__ __device__ bool ggml_cuda_mmq_fusion_supported(const ggml_type type) {
-    return type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || type == GGML_TYPE_Q6_K || type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_XS;
-}
-
-// Fused gate/up/GLU: the weight tile alternates chunks of this many up and gate rows, so rows r and r + 8 of an MMA tile (same thread) pair up.
-static constexpr int GGML_CUDA_MMQ_FUSION_ROWS = 8;
-
-// Source of weight tile row i; with has_fusion the up or gate row that the interleaved tile holds there.
-template <typename block_t, bool has_fusion>
-static __device__ __forceinline__ const block_t * ggml_cuda_mmq_x_row(
-        const char * __restrict__ x, const char * __restrict__ x_gate, const int kbx0, const int i, const int stride) {
-    if constexpr (has_fusion) {
-        constexpr int R = GGML_CUDA_MMQ_FUSION_ROWS;
-        const char * x_src = (i / R) % 2 == 0 ? x : x_gate;
-        return (const block_t *) x_src + kbx0 + ((i / (2*R))*R + i % R)*stride;
-    } else {
-        GGML_UNUSED(x_gate);
-        return (const block_t *) x + kbx0 + i*stride;
-    }
-}
-
 #include "mmq-load-tiles.cuh"
 #include "mmq-vec-dot.cuh"
 
 template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a(
         const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst,
-        const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max,
-        const ggml_glu_op glu_op, const float glu_limit) {
-    GGML_UNUSED_VARS(glu_op, glu_limit);
-
+        const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {
     constexpr int warp_size = ggml_cuda_get_physical_warp_size();
     constexpr int nwarps    = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;
     constexpr int I         = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);
@@ -497,19 +469,16 @@ template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __de
     }
 }
 
-// has_fusion: accumulators l and l + ne/2 hold the up and gate values of one output element, the GLU of the two is written.
-template<ggml_type type, int J, bool fallback, ggml_prec prec_src1, bool has_fusion>
+template<ggml_type type, int J, bool fallback, ggml_prec prec_src1>
 static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma(
             const float * __restrict__ sum, const int * __restrict__ ids_dst, float * __restrict__ dst,
-            const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max,
-            const ggml_glu_op glu_op, const float glu_limit) {
+            const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {
 
 #if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
     typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;
 #else
     typedef tile<16,  8, int> tile_C;
 #endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
-    static_assert(!has_fusion || tile_C::I == 2*GGML_CUDA_MMQ_FUSION_ROWS, "each C tile must hold one chunk of up rows and one of gate rows");
 
     constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1);
     constexpr int ntx           = rows_per_warp/tile_C::I; // Number of x minitiles per warp.
@@ -523,23 +492,20 @@ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma(
 #pragma unroll
         for (int n = 0; n < ntx; ++n) {
 #pragma unroll
-            for (int l = 0; l < (has_fusion ? tile_C::ne/2 : tile_C::ne); ++l) {
+            for (int l = 0; l < tile_C::ne; ++l) {
                 const int j = j0 + (threadIdx.y % ntx) * tile_C::J + tile_C::get_j(l);
 
                 if (j > j_max) {
                     continue;
                 }
 
-                const int i = (i0 + n*tile_C::I)/(has_fusion ? 2 : 1) + tile_C::get_i(l);
+                const int i = i0 + n*tile_C::I + tile_C::get_i(l);
 
                 if (fallback && i > i_max) {
                     continue;
                 }
 
-                if constexpr (has_fusion) {
-                    const int sum_index = (j0/tile_C::J + n)*tile_C::ne + l;
-                    dst[ids_dst[j]*stride + i] = ggml_cuda_op_glu_single(glu_op, sum[sum_index + tile_C::ne/2], sum[sum_index], glu_limit);
-                } else if constexpr (type == GGML_TYPE_NVFP4) {
+                if constexpr (type == GGML_TYPE_NVFP4) {
                     if (y_scale_used) {
                         dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l];
                     } else {
@@ -552,8 +518,6 @@ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma(
             }
         }
     }
-
-    GGML_UNUSED_VARS(y_scale_used, glu_op, glu_limit);
 }
 
 // -------------------------------------------------------------------------------------------------------------------------------------
@@ -570,9 +534,8 @@ struct ggml_cuda_mmq_util_funcs {
         vdr(vdr), load_tiles(load_tiles), vec_dot(vec_dot), write_back(write_back) {}
 };
 
-template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, bool has_fusion>
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
 static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_funcs() {
-    static_assert(!has_fusion || ggml_cuda_mmq_fusion_supported(type), "no fused loader for this type");
     if (!ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).use_mma_data_layout()) {
         switch (type) {
             case GGML_TYPE_Q1_0:
@@ -614,7 +577,7 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func
             case GGML_TYPE_Q8_0:
                 return ggml_cuda_mmq_util_funcs(
                     VDR_Q8_0_Q8_1_MMQ,
-                    ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback, false>,
+                    ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback>,
                     ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
                     ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
 // ---------------------------------------------------------------------------------------------
@@ -633,19 +596,19 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func
             case GGML_TYPE_Q4_K:
                 return ggml_cuda_mmq_util_funcs(
                     VDR_Q4_K_Q8_1_MMQ,
-                    ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback, false>,
+                    ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback>,
                     ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a<type, J, fallback>,
                     ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
             case GGML_TYPE_Q5_K:
                 return ggml_cuda_mmq_util_funcs(
                     VDR_Q5_K_Q8_1_MMQ,
-                    ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback, false>,
+                    ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback>,
                     ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a<type, J, fallback>,
                     ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
             case GGML_TYPE_Q6_K:
                 return ggml_cuda_mmq_util_funcs(
                     VDR_Q6_K_Q8_1_MMQ,
-                    ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback, false>,
+                    ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback>,
                     ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a<type, J, fallback>,
                     ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
 // ---------------------------------------------------------------------------------------------
@@ -688,7 +651,7 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func
             case GGML_TYPE_IQ4_XS:
                 return ggml_cuda_mmq_util_funcs(
                     VDR_IQ4_XS_Q8_1_MMQ,
-                    ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback, false>,
+                    ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback>,
                     ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
                     ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);
             case GGML_TYPE_IQ4_NL:
@@ -725,7 +688,7 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func
                     -1,
                     ggml_cuda_mmq_load_tiles_mxfp4_fp4<type, J, fallback>,
                     ggml_cuda_mmq_vec_dot_fp4_fp4_mma<type, J, fallback>,
-                    ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                    ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
             }
             break;
         case GGML_TYPE_NVFP4:
@@ -734,7 +697,7 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func
                     -1,
                     ggml_cuda_mmq_load_tiles_nvfp4_nvfp4<type, J, fallback>,
                     ggml_cuda_mmq_vec_dot_fp4_fp4_mma<type, J, fallback>,
-                    ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                    ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
             }
             break;
         default:
@@ -750,136 +713,136 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func
                 -1,
                 ggml_cuda_mmq_load_tiles_q1_0<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_Q2_0:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_q2_0<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_Q4_0:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_q4_0<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_DS4>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_Q4_1:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_q4_1<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_Q5_0:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_q5_0<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_Q5_1:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_q5_1<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_Q8_0:
             return ggml_cuda_mmq_util_funcs(
                 -1,
-                ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback, has_fusion>,
+                ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, has_fusion>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
 // ---------------------------------------------------------------------------------------------
         case GGML_TYPE_Q2_K:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_q2_K<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q2_K_q8_1_mma<type, J, fallback>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_Q3_K:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_q3_K<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_Q4_K:
             return ggml_cuda_mmq_util_funcs(
                 -1,
-                ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback, has_fusion>,
+                ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, has_fusion>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_Q5_K:
             return ggml_cuda_mmq_util_funcs(
                 -1,
-                ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback, has_fusion>,
+                ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, has_fusion>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_Q6_K:
             return ggml_cuda_mmq_util_funcs(
                 -1,
-                ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback, has_fusion>,
+                ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q6_K_q8_1_mma<type, J, fallback>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, has_fusion>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
 // ---------------------------------------------------------------------------------------------
         case GGML_TYPE_IQ1_S:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_IQ2_XXS:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_iq2_xxs<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_IQ2_XS:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_iq2_xs<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_IQ2_S:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_iq2_s<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_IQ3_XXS:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_iq3_xxs<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_IQ3_S:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_iq3_s<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_IQ4_XS:
             return ggml_cuda_mmq_util_funcs(
                 -1,
-                ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback, has_fusion>,
+                ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, has_fusion>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_IQ4_NL:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_iq4_nl<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
 // ---------------------------------------------------------------------------------------------
         case GGML_TYPE_MXFP4:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_mxfp4<type, J, fallback>,
                 ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         case GGML_TYPE_NVFP4:
             return ggml_cuda_mmq_util_funcs(
                 -1,
                 ggml_cuda_mmq_load_tiles_nvfp4<type, J, fallback, prec_src1>,
                 ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,
-                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, false>);
+                ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);
         default:
             return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr);
     }
@@ -887,44 +850,41 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func
 
 template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
 static constexpr __device__ int ggml_cuda_mmq_get_vdr() {
-    return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1, false>().vdr;
+    return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().vdr;
 }
 
-template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, bool has_fusion>
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
 static constexpr __device__ ggml_cuda_mmq_load_tiles_t ggml_cuda_mmq_get_load_tiles() {
-    return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1, has_fusion>().load_tiles;
+    return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().load_tiles;
 }
 
 template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
 static constexpr __device__ ggml_cuda_mmq_vec_dot_t ggml_cuda_mmq_get_vec_dot() {
-    return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1, false>().vec_dot;
+    return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().vec_dot;
 }
 
-template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, bool has_fusion>
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
 static constexpr __device__ ggml_cuda_mmq_write_back_t ggml_cuda_mmq_get_write_back() {
-    return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1, has_fusion>().write_back;
+    return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().write_back;
 }
 
 // ---------------------------------------------------------------------------------------------
 
-// has_fusion: x and x_gate are the up and gate weights.
-template <ggml_type type, int J, bool fallback, bool fixup, ggml_prec prec_src1, bool has_fusion>
+template <ggml_type type, int J, bool fallback, bool fixup, ggml_prec prec_src1>
 static __device__ __forceinline__ void mul_mat_q_process_tile(
-        const char * __restrict__ x, const char * __restrict__ x_gate, const int offset_x, const int * __restrict__ y,
+        const char * __restrict__ x, const int offset_x, const int * __restrict__ y,
         const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup,
-        const float * __restrict__ y_scale, const ggml_glu_op glu_op, const float glu_limit,
+        const float * __restrict__ y_scale,
         const int stride_row_x, const int ncols_y, const int stride_col_dst,
         const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) {
-    static_assert(!(fixup && has_fusion), "the fused gate/up path does not split K");
-    static_assert(!(fallback && has_fusion), "the fused gate/up path needs nrows_x % 128 == 0");
 
     constexpr int              warp_size  = ggml_cuda_get_physical_warp_size();
     constexpr int              nwarps     = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;
     constexpr int              qk         = ggml_cuda_type_traits<type>::qk;
     constexpr int              I          = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);
-    constexpr ggml_cuda_mmq_load_tiles_t load_tiles = ggml_cuda_mmq_get_load_tiles<type, J, fallback, prec_src1, has_fusion>();
+    constexpr ggml_cuda_mmq_load_tiles_t load_tiles = ggml_cuda_mmq_get_load_tiles<type, J, fallback, prec_src1>();
     constexpr ggml_cuda_mmq_vec_dot_t    vec_dot    = ggml_cuda_mmq_get_vec_dot<type, J, fallback, prec_src1>();
-    constexpr ggml_cuda_mmq_write_back_t write_back = ggml_cuda_mmq_get_write_back<type, J, fallback, prec_src1, has_fusion>();
+    constexpr ggml_cuda_mmq_write_back_t write_back = ggml_cuda_mmq_get_write_back<type, J, fallback, prec_src1>();
 
     extern __shared__ int data_mul_mat_q[];
     int * tile_y = data_mul_mat_q + J;
@@ -947,7 +907,7 @@ static __device__ __forceinline__ void mul_mat_q_process_tile(
     constexpr int sz = sizeof(block_q8_1_mmq) / sizeof(int);
 
     for (int kb0 = kb0_start; kb0 < kb0_stop; kb0 += blocks_per_iter) {
-        load_tiles(x, x_gate, tile_x, offset_x + kb0, tile_x_max_i, stride_row_x);
+        load_tiles(x, tile_x, offset_x + kb0, tile_x_max_i, stride_row_x);
         {
             const int * by0 = y + ncols_y * (kb0 * qk / ne_block) * sz;
 #pragma unroll
@@ -982,34 +942,28 @@ static __device__ __forceinline__ void mul_mat_q_process_tile(
     }
 
     if (fixup) {
-        write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J, glu_op, glu_limit);
+        write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J);
     } else {
-        write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j, glu_op, glu_limit);
+        write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);
     }
 }
 
 
 // The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598
 
-template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, bool has_fusion>
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
 __launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1), ggml_cuda_mmq_get_occupancy(type, J, fallback, prec_src1))
 static __global__ void mul_mat_q(
-        const char * __restrict__ x, const char * __restrict__ x_gate, const int * __restrict__ y, const int32_t * __restrict__ ids_dst,
+        const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst,
         const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, float * __restrict__ tmp_fixup,
-        const float * __restrict__ y_scale, const ggml_glu_op glu_op, const float glu_limit,
+        const float * __restrict__ y_scale,
         const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst,
         const uint3 channel_ratio, const uint3 nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
         const uint3 sample_ratio, const uint3 nsamples_y, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,
         const uint3 ntx) {
 
-#ifdef TURING_MMA_AVAILABLE
-    constexpr bool fusion_available = true;
-#else
-    constexpr bool fusion_available = false;
-#endif // TURING_MMA_AVAILABLE
-
     // Skip unused template specializations for faster compilation:
-    if (ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).type == GGML_TYPE_COUNT || (has_fusion && !fusion_available)) {
+    if (ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).type == GGML_TYPE_COUNT) {
         NO_DEVICE_CODE;
         return;
     }
@@ -1037,16 +991,12 @@ static __global__ void mul_mat_q(
     }
     __syncthreads();
 
-    // The fused kernel needs the whole K range of a tile in one block, so it always uses tiling.
-    if constexpr (has_fusion || !ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) {
-        // The fused kernel writes the GLU of I/2 up rows and I/2 gate rows per tile.
-        constexpr int I_dst = has_fusion ? I/2 : I;
-
+    if constexpr (!ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) {
         const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);
         const int wt = tmp2.x;
         const int zt = tmp2.y;
-        const int jt = blockIdx.x;
-        const int it = blockIdx.y;
+        const int jt = blockIdx.y;
+        const int it = blockIdx.x;
 
         // Defaults for regular matrix multiplication:
         int col_low    = 0;
@@ -1091,22 +1041,22 @@ static __global__ void mul_mat_q(
         }
 
         offset_y   += (col_low + jt*J)*(sizeof(block_q8_1_mmq)/sizeof(int));
-        offset_dst += it*I_dst;
+        offset_dst += it*I;
         const float * y_scale_tile = nullptr;
         if constexpr (type == GGML_TYPE_NVFP4) {
             offset_y_scale += col_low + jt*J;
             y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr;
         }
 
-        const int tile_x_max_i = nrows_x  - it*I_dst - 1;
+        const int tile_x_max_i = nrows_x  - it*I - 1;
         const int tile_y_max_j = col_diff - jt*J - 1;
 
-        const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I_dst*stride_row_x;
+        const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;
 
         constexpr bool fixup = false;
-        mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1, has_fusion>
-            (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
-             glu_op, glu_limit, stride_row_x, ncols_y, stride_col_dst,
+        mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1>
+            (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
+             stride_row_x, ncols_y, stride_col_dst,
              tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z);
         return;
     }
@@ -1198,9 +1148,9 @@ static __global__ void mul_mat_q(
         const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;
 
         constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.
-        mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1, false>
-            (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
-             glu_op, glu_limit, stride_row_x, ncols_y, stride_col_dst,
+        mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1>
+            (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
+             stride_row_x, ncols_y, stride_col_dst,
              tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop);
 
         kbc += blocks_per_ne00.z;
@@ -1282,9 +1232,9 @@ static __global__ void mul_mat_q(
     const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;
 
     constexpr bool fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.
-    mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1, false>
-        (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
-         glu_op, glu_limit, stride_row_x, ncols_y, stride_col_dst,
+    mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1>
+        (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
+         stride_row_x, ncols_y, stride_col_dst,
          tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop);
 }
 
@@ -1434,7 +1384,6 @@ struct mmq_args {
     int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst;
     int64_t ncols_max;
     int J_best; // Tile width in ne11(dense)/ne12(MoE) direction to use for optimal performance.
-    ggml_cuda_mm_fusion_args_device fusion = {}; // gate != nullptr selects the fused gate/up/GLU path
 };
 
 static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const int cc) {
@@ -1444,7 +1393,7 @@ static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const i
     return nbs_ids + nbs_x + GGML_PAD(nbs_y, config.nthreads*sizeof(int));
 }
 
-template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, bool has_fusion>
+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
 static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
     const int id = ggml_cuda_get_device();
     const int cc = ggml_cuda_info().devices[id].cc;
@@ -1458,17 +1407,13 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
 
     const dim3 block_dims(warp_size, nwarps, 1);
 
-    CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, /*fallback =*/ false, prec_src1, has_fusion>), nbytes_shared);
-    if constexpr (!has_fusion) {
-        CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, /*fallback =*/ true, prec_src1, false>), nbytes_shared);
-    }
+    CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, false, prec_src1>), nbytes_shared);
+    CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J,  true, prec_src1>), nbytes_shared);
 
-    const int I_dst = has_fusion ? config.I/2 : config.I; // Rows of dst per tile, see mul_mat_q.
-    const int nty  = (args.nrows_x   + I_dst - 1) / I_dst;
+    const int nty  = (args.nrows_x   + config.I - 1) / config.I;
     const int ntx  = (args.ncols_max + config.J - 1) / config.J;
     const int ntzw = args.nchannels_y * args.nsamples_y;
-    // Column tiles innermost so that the concurrently running blocks share the same tile of x and read it from L2.
-    const dim3 block_nums_xy_tiling(ntx, nty, ntzw);
+    const dim3 block_nums_xy_tiling(nty, ntx, ntzw);
 
     GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0);
     GGML_ASSERT(args.nsamples_y  % args.nsamples_x  == 0);
@@ -1482,10 +1427,9 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
     const uint3 channel_ratio_fd   = init_fastdiv_values(channel_ratio);
     const uint3 sample_ratio_fd    = init_fastdiv_values(sample_ratio);
 
-    if (has_fusion || !config.stream_k) {
-        GGML_ASSERT(nty <= 65535); // gridDim.y limit
-        mul_mat_q<type, J, fallback, prec_src1, has_fusion><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>
-            (args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, args.y_scale, args.fusion.glu_op, args.fusion.glu_limit,
+    if (!config.stream_k) {
+        mul_mat_q<type, J, fallback, prec_src1><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>
+            (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, args.y_scale,
              blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,
              channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,
              sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,
@@ -1515,8 +1459,8 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
     const dim3 block_nums_fixup(block_nums_stream_k.x, config.I/warp_size, 1);
     const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z);
 
-    mul_mat_q<type, J, fallback, prec_src1, has_fusion><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>>
-        (args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, args.y_scale, args.fusion.glu_op, args.fusion.glu_limit,
+    mul_mat_q<type, J, fallback, prec_src1><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>>
+        (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, args.y_scale,
          blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,
          channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,
          sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,
@@ -1533,56 +1477,56 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
          ntx_fd);
 }
 
-template <ggml_type type, bool fallback, ggml_prec prec_src1, bool has_fusion>
+template <ggml_type type, bool fallback, ggml_prec prec_src1>
 void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
     switch (args.J_best) {
         case   8:
-            launch_mul_mat_q<type,   8, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type,   8, fallback, prec_src1>(ctx, args, stream);
             break;
         case  16:
-            launch_mul_mat_q<type,  16, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type,  16, fallback, prec_src1>(ctx, args, stream);
             break;
         case  24:
-            launch_mul_mat_q<type,  24, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type,  24, fallback, prec_src1>(ctx, args, stream);
             break;
         case  32:
-            launch_mul_mat_q<type,  32, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type,  32, fallback, prec_src1>(ctx, args, stream);
             break;
         case  40:
-            launch_mul_mat_q<type,  40, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type,  40, fallback, prec_src1>(ctx, args, stream);
             break;
         case  48:
-            launch_mul_mat_q<type,  48, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type,  48, fallback, prec_src1>(ctx, args, stream);
             break;
         case  56:
-            launch_mul_mat_q<type,  56, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type,  56, fallback, prec_src1>(ctx, args, stream);
             break;
         case  64:
-            launch_mul_mat_q<type,  64, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type,  64, fallback, prec_src1>(ctx, args, stream);
             break;
         case  72:
-            launch_mul_mat_q<type,  72, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type,  72, fallback, prec_src1>(ctx, args, stream);
             break;
         case  80:
-            launch_mul_mat_q<type,  80, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type,  80, fallback, prec_src1>(ctx, args, stream);
             break;
         case  88:
-            launch_mul_mat_q<type,  88, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type,  88, fallback, prec_src1>(ctx, args, stream);
             break;
         case  96:
-            launch_mul_mat_q<type,  96, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type,  96, fallback, prec_src1>(ctx, args, stream);
             break;
         case 104:
-            launch_mul_mat_q<type, 104, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type, 104, fallback, prec_src1>(ctx, args, stream);
             break;
         case 112:
-            launch_mul_mat_q<type, 112, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type, 112, fallback, prec_src1>(ctx, args, stream);
             break;
         case 120:
-            launch_mul_mat_q<type, 120, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type, 120, fallback, prec_src1>(ctx, args, stream);
             break;
         case 128:
-            launch_mul_mat_q<type, 128, fallback, prec_src1, has_fusion>(ctx, args, stream);
+            launch_mul_mat_q<type, 128, fallback, prec_src1>(ctx, args, stream);
             break;
         default:
             fprintf(stderr, "J_best=%d\n", args.J_best);
@@ -1590,26 +1534,14 @@ void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args,
     }
 }
 
-template <ggml_type type, bool fallback, ggml_prec prec_src1>
-static void mul_mat_q_switch_fusion(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
-    if constexpr (!fallback && ggml_cuda_mmq_fusion_supported(type)) {
-        if (args.fusion.gate) {
-            mul_mat_q_switch_J<type, fallback, prec_src1, true>(ctx, args, stream);
-            return;
-        }
-    }
-    GGML_ASSERT(!args.fusion.gate);
-    mul_mat_q_switch_J<type, fallback, prec_src1, false>(ctx, args, stream);
-}
-
 template <ggml_type type, ggml_prec prec_src1>
 void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
     if (ggml_cuda_mmq_needs_fallback(args.nrows_x)) {
         constexpr bool fallback = true;
-        mul_mat_q_switch_fusion<type, fallback, prec_src1>(ctx, args, stream);
+        mul_mat_q_switch_J<type, fallback, prec_src1>(ctx, args, stream);
     } else {
         constexpr bool fallback = false;
-        mul_mat_q_switch_fusion<type, fallback, prec_src1>(ctx, args, stream);
+        mul_mat_q_switch_J<type, fallback, prec_src1>(ctx, args, stream);
     }
 }
 
@@ -1651,9 +1583,6 @@ extern DECL_MMQ_CASE_W4A4(GGML_TYPE_NVFP4);
 // -------------------------------------------------------------------------------------------------------------------------
 
 void ggml_cuda_mul_mat_q(
-        ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,
-        const ggml_cuda_mm_fusion_args_host * fusion = nullptr);
+        ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst);
 
 bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts);
-
-bool ggml_cuda_mmq_fusion_is_efficient(ggml_type type, int64_t nrows_x, int64_t ncols, int64_t nchannels, int device);

ggml/src/ggml-cuda/rope.cu

@@ -196,9 +196,9 @@ static __global__ void rope_neox(const T *            x,
     dst[idst + n_offs/2 + n_dims / 2] = ggml_cuda_cast<D>(x0 * sin_theta + x1 * cos_theta);
 }
 
-template <bool forward, bool has_ff, typename T, typename D>
+template <bool forward, bool has_ff, typename T>
 static __global__ void rope_multi(const T *            x,
-                                  D *                  dst,
+                                  T *                  dst,
                                   const int            ne00,
                                   const int            ne01,
                                   const int            ne02,
@@ -219,8 +219,6 @@ static __global__ void rope_multi(const T *            x,
                                   const float *        freq_factors,
                                   const mrope_sections sections,
                                   const bool           is_imrope,
-                                  const int64_t *      row_indices,
-                                  const int            set_rows_stride,
                                   const bool           inplace) {
     const int i0 = 2 * (blockDim.y * blockIdx.y + threadIdx.y);
 
@@ -238,20 +236,12 @@ static __global__ void rope_multi(const T *            x,
     const int ix   = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03;
 
     ggml_cuda_pdl_sync();
-
-    // Fusion optimization: ROPE + VIEW + SET_ROWS.
-    // The rope output is viewed as a 1D tensor and offset based on a row index in row_indices.
-    if (set_rows_stride != 0) {
-        idst = i1 * s1 + i0 / 2;
-        idst += row_indices[i2] * set_rows_stride;
-    }
-
     if (i0 < n_offs || i0 >= n_offs + n_dims) {
         if (inplace) {
             return;
         }
-        dst[idst + i0/2 + 0] = ggml_cuda_cast<D>(x[ix + i0/2 + 0]);
-        dst[idst + i0/2 + 1] = ggml_cuda_cast<D>(x[ix + i0/2 + 1]);
+        dst[idst + i0/2 + 0] = x[ix + i0/2 + 0];
+        dst[idst + i0/2 + 1] = x[ix + i0/2 + 1];
 
         return;
     }
@@ -296,8 +286,8 @@ static __global__ void rope_multi(const T *            x,
     const float x0 = x[ix + n_offs/2 + 0];
     const float x1 = x[ix + n_offs/2 + n_dims/2];
 
-    dst[idst + n_offs/2 + 0]        = ggml_cuda_cast<D>(x0*cos_theta - x1*sin_theta);
-    dst[idst + n_offs/2 + n_dims/2] = ggml_cuda_cast<D>(x0*sin_theta + x1*cos_theta);
+    dst[idst + n_offs/2 + 0]        = x0*cos_theta - x1*sin_theta;
+    dst[idst + n_offs/2 + n_dims/2] = x0*sin_theta + x1*cos_theta;
 }
 
 template <bool forward, bool has_ff, typename T>
@@ -453,9 +443,9 @@ static void rope_neox_cuda(const T *            x,
     }
 }
 
-template <bool forward, typename T, typename D>
+template <bool forward, typename T>
 static void rope_multi_cuda(const T *            x,
-                            D *                  dst,
+                            T *                  dst,
                             const int            ne00,
                             const int            ne01,
                             const int            ne02,
@@ -477,8 +467,6 @@ static void rope_multi_cuda(const T *            x,
                             const float *        freq_factors,
                             const mrope_sections sections,
                             const bool           is_imrope,
-                            const int64_t *      row_indices,
-                            const int            set_rows_stride,
                             const bool           inplace,
                             cudaStream_t         stream) {
     GGML_ASSERT(ne00 % 2 == 0);
@@ -490,14 +478,14 @@ static void rope_multi_cuda(const T *            x,
 
     if (freq_factors == nullptr) {
         const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream);
-        ggml_cuda_kernel_launch(rope_multi<forward, false, T, D>, launch_params,
+        ggml_cuda_kernel_launch(rope_multi<forward, false, T>, launch_params,
             x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, pos, freq_scale, ext_factor,
-            attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope, row_indices, set_rows_stride, inplace);
+            attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope, inplace);
     } else {
         const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream);
-        ggml_cuda_kernel_launch(rope_multi<forward, true, T, D>, launch_params,
+        ggml_cuda_kernel_launch(rope_multi<forward, true, T>, launch_params,
             x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, pos, freq_scale, ext_factor,
-            attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope, row_indices, set_rows_stride, inplace);
+            attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope, inplace);
     }
 }
 
@@ -660,21 +648,14 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
             GGML_ABORT("fatal error");
         }
     } else if (is_mrope && !is_vision) {
-        if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) {
-            rope_multi_cuda<forward, float, float>((const float *) src0_d, (float *) dst_d, ne00, ne01, ne02, s01, s02,
-                                                   s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
-                                                   ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope,
-                                                   row_indices, set_rows_stride, inplace, stream);
-        } else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) {
-            rope_multi_cuda<forward, float, half>((const float *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,
-                                                  s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
-                                                  ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope,
-                                                  row_indices, set_rows_stride, inplace, stream);
-        } else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) {
-            rope_multi_cuda<forward, half, half>((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,
-                                                 s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
-                                                 ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope,
-                                                 row_indices, set_rows_stride, inplace, stream);
+        if (src0->type == GGML_TYPE_F32) {
+            rope_multi_cuda<forward>((const float *) src0_d, (float *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1,
+                                     s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor,
+                                     corr_dims, freq_factors, sections, is_imrope, inplace, stream);
+        } else if (src0->type == GGML_TYPE_F16) {
+            rope_multi_cuda<forward>((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1,
+                                     s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor,
+                                     corr_dims, freq_factors, sections, is_imrope, inplace, stream);
         } else {
             GGML_ABORT("fatal error");
         }

ggml/src/ggml-cuda/scale.cu

@@ -3,45 +3,23 @@
 
 #define MAX_GRIDDIM_X 0x7FFFFFFF
 
-// src and dst may be views: rows are contiguous, dims 1..3 follow the strides (in elements).
 template <typename T>
-static __global__ void scale_f32(const T * x, T * dst, const float scale, const float bias, const uint32_t k,
-                                 const uint3 ne0, const uint3 ne1, const uint3 ne2,
-                                 const uint32_t s01, const uint32_t s02, const uint32_t s03,
-                                 const uint32_t s1,  const uint32_t s2,  const uint32_t s3) {
+static __global__ void scale_f32(const T * x, T * dst, const float scale, const float bias, const int64_t nelements) {
     ggml_cuda_pdl_lc();
-    const uint32_t i = blockIdx.x * blockDim.x + threadIdx.x;
+    int64_t tid = (int64_t)blockIdx.x * (int64_t)blockDim.x + (int64_t)threadIdx.x;
+    int64_t stride = (int64_t)blockDim.x * (int64_t)gridDim.x;
 
     ggml_cuda_pdl_sync();
-    if (i >= k) {
-        return;
+    for (int64_t i = tid; i < nelements; i += stride) {
+        dst[i] = ggml_cuda_cast<T>(scale * ggml_cuda_cast<float>(x[i]) + bias);
     }
-
-    const uint2 d0 = fast_div_modulo(i,    ne0); // <i / ne0, i0>
-    const uint2 d1 = fast_div_modulo(d0.x, ne1); // <i / (ne0*ne1), i1>
-    const uint2 d2 = fast_div_modulo(d1.x, ne2); // <i3, i2>
-
-    const size_t i_src = d0.y + size_t(d1.y)*s01 + size_t(d2.y)*s02 + size_t(d2.x)*s03;
-    const size_t i_dst = d0.y + size_t(d1.y)*s1  + size_t(d2.y)*s2  + size_t(d2.x)*s3;
-
-    dst[i_dst] = ggml_cuda_cast<T>(scale * ggml_cuda_cast<float>(x[i_src]) + bias);
 }
 
 template <typename T>
-static void scale_f32_cuda(const T * x, T * dst, const float scale, const float bias, const ggml_tensor * src0, const ggml_tensor * t, cudaStream_t stream) {
-    const int64_t k  = ggml_nelements(src0);
-    const size_t  ts = sizeof(T);
-    GGML_ASSERT(k <= std::numeric_limits<uint32_t>::max());
-
-    const uint3 ne0 = init_fastdiv_values(src0->ne[0]);
-    const uint3 ne1 = init_fastdiv_values(src0->ne[1]);
-    const uint3 ne2 = init_fastdiv_values(src0->ne[2]);
-
-    const int64_t num_blocks = (k + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE;
+static void scale_f32_cuda(const T * x, T * dst, const float scale, const float bias, const int64_t nelements, cudaStream_t stream) {
+    const int64_t num_blocks = (nelements + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE;
     const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(MIN(MAX_GRIDDIM_X, num_blocks), CUDA_SCALE_BLOCK_SIZE, 0, stream);
-    ggml_cuda_kernel_launch(scale_f32<T>, launch_params, x, dst, scale, bias, (uint32_t) k, ne0, ne1, ne2,
-        src0->nb[1]/ts, src0->nb[2]/ts, src0->nb[3]/ts,
-        t->nb[1]/ts,    t->nb[2]/ts,    t->nb[3]/ts);
+    ggml_cuda_kernel_launch(scale_f32<T>, launch_params, x, dst, scale, bias, nelements);
 }
 
 void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
@@ -50,7 +28,6 @@ void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
 
     GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16);
     GGML_ASSERT(dst->type == src0->type);
-    GGML_ASSERT(ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(dst));
 
     float scale;
     float bias;
@@ -58,8 +35,8 @@ void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
     memcpy(&bias,  (float *) dst->op_params + 1, sizeof(float));
 
     if (src0->type == GGML_TYPE_BF16) {
-        scale_f32_cuda((const nv_bfloat16 *) src0->data, (nv_bfloat16 *) dst->data, scale, bias, src0, dst, stream);
+        scale_f32_cuda((const nv_bfloat16 *) src0->data, (nv_bfloat16 *) dst->data, scale, bias, ggml_nelements(src0), stream);
     } else {
-        scale_f32_cuda((const float *) src0->data, (float *) dst->data, scale, bias, src0, dst, stream);
+        scale_f32_cuda((const float *) src0->data, (float *) dst->data, scale, bias, ggml_nelements(src0), stream);
     }
 }

ggml/src/ggml-cuda/topk-moe.cu

@@ -76,7 +76,7 @@ __device__ void sqrt_softplus_warp_inplace(float (&vals)[experts_per_thread], co
     for (int i = 0; i < experts_per_thread; i++) {
         const int  idx    = lane + i * WARP_SIZE;
         const bool active = !use_limit || (idx < limit);
-        vals[i]           = active ? sqrtf(vals[i] > 20.0f ? vals[i] : log1pf(expf(vals[i]))) : -INFINITY;
+        vals[i]           = active ? sqrtf(vals[i] > 20.0f ? vals[i] : logf(1.0f + expf(vals[i]))) : -INFINITY;
     }
 }
 

ggml/src/ggml-cuda/unary.cu

@@ -91,7 +91,7 @@ static __device__ __forceinline__ float op_expm1(float x) {
 }
 
 static __device__ __forceinline__ float op_softplus(float x) {
-    return (x > 20.0f) ? x : log1pf(expf(x));
+    return (x > 20.0f) ? x : logf(1.0f + expf(x));
 }
 
 static __device__ __forceinline__ float op_elu(float x) {

ggml/src/ggml-cuda/unary.cuh

@@ -121,18 +121,3 @@ __device__ __forceinline__ float ggml_cuda_op_swiglu_clamp_single(float gate, fl
 
     return ggml_cuda_op_silu_single(gate) * up;
 }
-
-__device__ __forceinline__ float ggml_cuda_op_glu_single(ggml_glu_op op, float gate, float up, float limit) {
-    switch (op) {
-        case GGML_GLU_OP_SWIGLU:
-            return up * ggml_cuda_op_silu_single(gate);
-        case GGML_GLU_OP_GEGLU:
-            return up * ggml_cuda_op_gelu_single(gate);
-        case GGML_GLU_OP_SWIGLU_OAI:
-            return ggml_cuda_op_swiglu_oai_single(gate, up);
-        case GGML_GLU_OP_SWIGLU_CLAMP:
-            return ggml_cuda_op_swiglu_clamp_single(gate, up, limit);
-        default:
-            return up * gate;
-    }
-}

ggml/src/ggml-hexagon/ggml-hexagon.cpp

@@ -85,7 +85,6 @@ int    opt_arch       = 0; // autodetect
 size_t opt_ndev       = 1;
 size_t opt_nhvx       = 0; // use all
 int    opt_nhmx       = 1; // when set, enable HMX; when 0, use HVX only
-size_t opt_vtcm_size  = 0; // VTCM size override in bytes; 0 = auto-detect
 size_t opt_vmem       = HTP_OP_MAX_VMEM_DEFAULT;  // max available va space for buffer mappings
 int    opt_etm        = 0;
 int    opt_verbose    = 0;
@@ -134,7 +133,7 @@ size_t opt_mbuf_total  = 0;                        // total buffer space limit (
 
 int    opt_mm_select     = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)
 int    opt_fa_select     = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported)
-int    opt_fa_split      = 1; // 1 = head-split + team-split when n_kv_heads < N_cores (default), 0 = off
+int    opt_fa_head_split = 1; // 1 = partition flash_attn by KV heads in multicore (default on), 0 = token-based (original)
 int    opt_gdn_select    = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported)
 int    opt_ar_select     = 2; // 2 = fused ALLREDUCE+ADD (default), 1 = unfused ALLREDUCE, 0 = fallback to CPY+FENCE
 int    opt_ar_scatter    = 1; // 1 = reduce-scatter the fused ALLREDUCE+ADD (default), 0 = full reduction
@@ -2153,12 +2152,6 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
             this->n_hvx     = this->n_threads;
             this->n_hmx     = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0;
             this->vtcm_size = (uint64_t)hw_vtcm_size;
-            if (opt_vtcm_size > 0) {
-                if (opt_vtcm_size > hw_vtcm_size) {
-                    GGML_LOG_WARN("ggml-hex: %s VTCM override %zu exceeds HW size %llu\n", this->c_name(), opt_vtcm_size, hw_vtcm_size);
-                }
-                this->vtcm_size = opt_vtcm_size;
-            }
             GGML_LOG_INFO("ggml-hex: %s hwinfo: threads %u, hvx %u, hmx %u, vtcm %llu MB\n",
                           this->c_name(), this->n_threads, this->n_hvx, this->n_hmx,
                           (unsigned long long)(this->vtcm_size / (1024 * 1024)));
@@ -2168,7 +2161,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
             this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads;
             this->n_hvx     = this->n_threads;
             this->n_hmx     = (opt_nhmx != 0) ? 1 : 0;
-            this->vtcm_size = opt_vtcm_size > 0 ? opt_vtcm_size : 8 * 1024 * 1024;
+            this->vtcm_size = 8 * 1024 * 1024;
         }
     }
 
@@ -2237,8 +2230,7 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
     this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem);
 
     // Start dspqueue/opbatch processing
-    err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, opt_vtcm_size, this->max_vmem,
-                          (uint32_t) opt_power_mode);
+    err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, this->max_vmem, (uint32_t) opt_power_mode);
     if (err != 0) {
         GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err);
         throw std::runtime_error("ggml-hex: iface start failed (see log for details)");
@@ -2405,7 +2397,7 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
                 }
             }
 
-            GGML_ASSERT(htp_kparams_solve(sess, node));
+            GGML_ASSERT(htp_kparams_solve(sess, node.node, node.kernel_params));
             if (node.opcode == HTP_OP_CPY && node.kparams<htp_copy_kernel_params>()->total_elems == 0) {
                 continue;
             }
@@ -3399,10 +3391,9 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
     const char * str_etm      = getenv("GGML_HEXAGON_ETM");
     const char * str_nhvx     = getenv("GGML_HEXAGON_NHVX");
     const char * str_nhmx     = getenv("GGML_HEXAGON_NHMX");
-    const char * str_vtcm_size = getenv("GGML_HEXAGON_VTCM_SIZE");
     const char * str_mm_select = getenv("GGML_HEXAGON_MM_SELECT");
     const char * str_fa_select = getenv("GGML_HEXAGON_FA_SELECT");
-    const char * str_fa_split  = getenv("GGML_HEXAGON_FA_SPLIT");
+    const char * str_fa_head_split = getenv("GGML_HEXAGON_FA_HEAD_SPLIT");
     const char * str_gdn_select = getenv("GGML_HEXAGON_GDN_SELECT");
     const char * str_ar_select = getenv("GGML_HEXAGON_AR_SELECT");
     const char * str_ar_scatter = getenv("GGML_HEXAGON_AR_SCATTER");
@@ -3465,10 +3456,9 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
     }
     opt_nhvx      = str_nhvx     ? strtoul(str_nhvx, NULL, 0)             : opt_nhvx;
     opt_nhmx      = str_nhmx     ? atoi(str_nhmx)                         : opt_nhmx;
-    opt_vtcm_size = str_vtcm_size ? parse_size(str_vtcm_size)                 : opt_vtcm_size;
     opt_mm_select = str_mm_select ? atoi(str_mm_select)                   : opt_mm_select;
     opt_fa_select = str_fa_select ? atoi(str_fa_select)                   : opt_fa_select;
-    opt_fa_split  = str_fa_split  ? atoi(str_fa_split)                   : opt_fa_split;
+    opt_fa_head_split = str_fa_head_split ? atoi(str_fa_head_split)       : opt_fa_head_split;
     opt_gdn_select    = str_gdn_select    ? atoi(str_gdn_select)          : opt_gdn_select;
     opt_ar_select     = str_ar_select     ? atoi(str_ar_select)           : opt_ar_select;
     opt_ar_scatter    = str_ar_scatter    ? atoi(str_ar_scatter)          : opt_ar_scatter;

ggml/src/ggml-hexagon/htp-common.h

@@ -28,7 +28,7 @@ extern size_t opt_mbuf_total;
 
 extern int    opt_mm_select;
 extern int    opt_fa_select;
-extern int    opt_fa_split;
+extern int    opt_fa_head_split;
 extern int    opt_gdn_select;
 extern int    opt_ar_select;
 extern int    opt_ar_scatter;

ggml/src/ggml-hexagon/htp-fusion.cpp

@@ -192,20 +192,15 @@ static bool fuse_mul_mat_add(const ggml_hexagon_session * sess, htp_opnode & pre
     if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &src2)) return false;
     if (src2->type != GGML_TYPE_F32) return false;
 
-    const ggml_tensor * src0 = prev.node->src[0];
-    const ggml_tensor * src1 = prev.node->src[1];
-    const ggml_tensor * dst  = cur.node;
-    if (!ggml_are_same_shape(prev.node, dst)) return false;
-
-    const bool prev_collapsed = htp_matmul_can_collapse(src0, src1, prev.node);
-    const bool cur_collapsed  = htp_matmul_add_can_collapse(src0, src1, src2, dst);
-    if (prev_collapsed != cur_collapsed) {
-        return false;
-    }
+    const ggml_tensor * src0 = prev.src0();
+    const ggml_tensor * src1 = prev.src1();
+    const ggml_tensor * dst  = cur.dst();
+    if (!ggml_are_same_shape(prev.dst(), dst)) return false;
 
     const auto * orig_kparams = prev.kparams<htp_mm_kernel_params>();
     struct htp_mm_kernel_params kparams;
-    if (!htp_kparams_solve_matmul_add(sess, src0, src1, src2, dst, &kparams)) {
+    if (!htp_kparams_solve_matmul_add(sess, src0, src1, src2, dst, &kparams) ||
+        orig_kparams->collapse != kparams.collapse) {
         return false;
     }
 
@@ -227,7 +222,6 @@ static bool fuse_mul_mat_add(const ggml_hexagon_session * sess, htp_opnode & pre
     prev.opcode  = HTP_OP_MUL_MAT_ADD;
     prev.inputs  = { src0, src1, src2 };
     prev.outputs = { dst };
-    htp_opnode_collapse(sess, prev);
     prev.fused.push_back(cur.node);
     prev.set_kparams(kparams);
     return true;
@@ -239,31 +233,36 @@ static bool fuse_mul_mat_nx(const ggml_hexagon_session * sess, htp_opnode & prev
     const uint32_t curr_n = (uint32_t) prev.outputs.size();
     if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 1 >= HTP_OP_MAX_INPUTS) return false;
 
-    const ggml_tensor * w0     = prev.inputs[0];
-    const ggml_tensor * x_orig = prev.node->src[1];
-    const ggml_tensor * w_in   = cur.node->src[0];
-    const ggml_tensor * x_in   = cur.node->src[1];
-    const ggml_tensor * d_in   = cur.node;
+    const ggml_tensor * w0   = prev.inputs[0];
+    const ggml_tensor * x    = prev.inputs[curr_n];
+    const ggml_tensor * w_in = cur.src0();
+    const ggml_tensor * x_in = cur.src1();
+    const ggml_tensor * d_in = cur.dst();
 
     auto * orig_kparams = prev.kparams<htp_mm_kernel_params>();
     const auto * cur_kparams = cur.kparams<htp_mm_kernel_params>();
 
-    if (x_in != x_orig || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] ||
+    if (x_in != x || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] ||
         w0->ne[2] != 1 || w0->ne[3] != 1 || w_in->ne[2] != 1 || w_in->ne[3] != 1 ||
         htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) ||
         orig_kparams->n_hmx != cur_kparams->n_hmx) {
         return false;
     }
 
-    const bool collapse = htp_matmul_can_collapse(w0, x_orig, prev.node) && htp_matmul_can_collapse(w_in, x_in, d_in);
-    if ((x_orig->ne[2] > 1 || x_orig->ne[3] > 1) && !collapse) return false;
+    const bool collapse = orig_kparams->collapse && htp_matmul_can_collapse(w_in, x, d_in);
+    if ((x->ne[2] > 1 || x->ne[3] > 1) && !collapse) return false;
 
     if (prev.opcode == HTP_OP_MUL_MAT_NX) {
         orig_kparams->n_weights = curr_n + 1;
     } else {
         struct htp_mm_kernel_params kparams;
-        const ggml_tensor x_collapsed = collapse ? htp_tensor_collapse_rows(x_orig) : *x_orig;
-        if (!htp_kparams_solve_mmnx(sess, w0, &x_collapsed, curr_n + 1, &kparams)) return false;
+        if (collapse) {
+            const ggml_tensor x_collapsed = htp_tensor_collapse_rows(x);
+            if (!htp_kparams_solve_mmnx(sess, w0, &x_collapsed, curr_n + 1, &kparams)) return false;
+            kparams.collapse = 1;
+        } else if (!htp_kparams_solve_mmnx(sess, w0, x, curr_n + 1, &kparams)) {
+            return false;
+        }
         if (!is_supported_mul_mat_nx_kernel(w0, &kparams)) return false;
 
         prev.set_kparams(kparams);
@@ -271,7 +270,7 @@ static bool fuse_mul_mat_nx(const ggml_hexagon_session * sess, htp_opnode & prev
     }
 
     prev.inputs.insert(prev.inputs.begin() + curr_n, w_in);
-    prev.outputs.push_back(collapse ? prev.add_dummy(htp_tensor_collapse_rows(d_in)) : d_in);
+    prev.outputs.push_back(d_in);
     prev.fused.push_back(cur.node);
     return true;
 }
@@ -416,13 +415,15 @@ static bool fuse_unary_mul_glu(const ggml_hexagon_session * sess, htp_opnode & p
     const ggml_tensor * dst  = cur.dst();
     if (htp_inputs_overlap_unsafe(prev, dst, up)) return false;
 
+    struct htp_glu_kernel_params kparams;
+    if (!htp_kparams_solve_unary_mul_glu(sess, glu_op, gate, up, dst, &kparams)) return false;
+
     prev.opcode  = glu_op;
     prev.inputs  = { gate, up };
     prev.outputs = { dst };
-    if (!htp_kparams_solve(sess, prev)) return false;
-
     prev.fused.push_back(cur.node);
     memset(prev.params, 0, sizeof(prev.params));
+    prev.set_kparams(kparams);
     return true;
 }
 

ggml/src/ggml-hexagon/htp-kparams.cpp

@@ -1,7 +1,6 @@
 #define _USE_MATH_DEFINES
 
 #include "htp-kparams.h"
-#include "htp-opnode.h"
 #include "ggml-impl.h"
 
 #include <algorithm>
@@ -398,20 +397,9 @@ static bool htp_kparams_solve_flash_attn(
     kparams->scale         = scale;
     kparams->max_bias      = max_bias;
     kparams->logit_softcap = logit_softcap;
-    kparams->head_split    = (opt_fa_split != 0) ? 1 : 0;
-    kparams->flags         = 0;
+    kparams->head_split    = (opt_fa_head_split != 0) ? 1 : 0;
     kparams->G             = G;
 
-    // team-split only when a team has >1 member (n_kv_heads < N_cores)
-    uint32_t fa_teams = 1, fa_team_size = 1;
-    const uint32_t mdev_count = htp_session_mdev_count(sess);
-    if (opt_fa_split && mdev_count > 1) {
-        htp_fa_mdev_teams(n_kv_heads, mdev_count, kparams->head_split != 0, &fa_teams, &fa_team_size);
-        if (fa_team_size > 1) {
-            kparams->flags |= HTP_FA_FLAG_TEAM_SPLIT;
-        }
-    }
-
     const uint32_t n_head = q->ne[2];
     kparams->n_head_log2 = 1u << (uint32_t) std::floor(std::log2(n_head));
     kparams->m0 = std::pow(2.0f, -(max_bias) / kparams->n_head_log2);
@@ -421,7 +409,7 @@ static bool htp_kparams_solve_flash_attn(
         const uint32_t DK_pad = hex_round_up(DK, 64);
         const uint32_t DV_pad = hex_round_up(DV, 64);
         size_t Br = 0, Bc = 0;
-        int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, vtcm_size, n_threads, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head, fa_team_size);
+        int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, vtcm_size, n_threads, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head);
         if (ret == 0) {
             kparams->kernel_type = HTP_FA_KERNEL_HMX;
             if (logit_softcap == 0.0f) {
@@ -1024,12 +1012,14 @@ static bool htp_kparams_solve_matmul(
     if (htp_matmul_can_collapse(src0, src1, dst)) {
         const ggml_tensor src1_collapsed = htp_tensor_collapse_rows(src1);
         const ggml_tensor dst_collapsed  = htp_tensor_collapse_rows(dst);
-        return htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, 0, 0, kparams);
+        const bool ok = htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, 0, 0, kparams);
+        kparams->collapse = 1;
+        return ok;
     }
     return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, 0, 0, kparams);
 }
 
-bool htp_matmul_add_can_collapse(
+static bool htp_matmul_add_can_collapse(
     const struct ggml_tensor * src0,
     const struct ggml_tensor * src1,
     const struct ggml_tensor * src2,
@@ -1067,7 +1057,9 @@ bool htp_kparams_solve_matmul_add(
                                            : (src2 ? *src2 : ggml_tensor{});
         const struct ggml_tensor * p_src2 = src2 ? &src2_collapsed : nullptr;
         const size_t src2_size = p_src2 ? hex_round_up(ggml_nbytes(p_src2), 128) : 0;
-        return htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, p_src2 ? p_src2->nb[1] : 0, src2_size, kparams);
+        const bool ok = htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, p_src2 ? p_src2->nb[1] : 0, src2_size, kparams);
+        kparams->collapse = 1;
+        return ok;
     }
     const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0;
     return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, kparams);
@@ -1402,103 +1394,47 @@ static bool htp_kparams_solve_binary(
 }
 
 // collapse contiguous dims of elementwise ops so that rows fill at least one HVX vector
-static void htp_collapse_dims(struct ggml_tensor ** tensors, size_t n_tensors, size_t elem_size, uint32_t n_threads) {
-    if (n_tensors == 0 || !tensors[0]) {
+static void htp_unary_collapse_dims(struct ggml_tensor * src, struct ggml_tensor * dst, size_t elem_size) {
+    if (src->nb[0] != elem_size || dst->nb[0] != elem_size) {
         return;
     }
-    for (size_t t = 0; t < n_tensors; t++) {
-        if (!tensors[t] || tensors[t]->nb[0] != elem_size) {
-            return;
-        }
-        if (t > 0 && !ggml_are_same_shape(tensors[0], tensors[t])) {
-            return;
-        }
-    }
 
-    struct ggml_tensor * t0 = tensors[0];
-
-    while (t0->ne[1] > 1 || t0->ne[2] > 1 || t0->ne[3] > 1) {
-        const size_t row_size = (size_t) t0->ne[0] * elem_size;
-        bool rows_contig = true;
-        for (size_t t = 0; t < n_tensors; t++) {
-            if (tensors[t]->nb[1] != (size_t) tensors[t]->ne[0] * elem_size) {
-                rows_contig = false;
-                break;
-            }
-        }
-        bool all_ne1_one = true;
-        for (size_t t = 0; t < n_tensors; t++) {
-            if (tensors[t]->ne[1] != 1) {
-                all_ne1_one = false;
-                break;
-            }
-        }
-        const bool can_merge = all_ne1_one ||
-                               (rows_contig && (t0->ne[0] == 1 || (row_size < 128 && row_size * t0->ne[1] <= 2048)));
+    while (src->ne[1] > 1 || src->ne[2] > 1 || src->ne[3] > 1) {
+        const size_t row_size    = (size_t) src->ne[0] * elem_size;
+        const bool   rows_contig = src->nb[1] == row_size && dst->nb[1] == (size_t) dst->ne[0] * elem_size;
+        const bool   can_merge   = (src->ne[1] == 1 && dst->ne[1] == 1) ||
+                                   (rows_contig && (src->ne[0] == 1 || (row_size < 128 && row_size * src->ne[1] <= 2048)));
         if (!can_merge) {
             break;
         }
-        for (size_t t = 0; t < n_tensors; t++) {
-            tensors[t]->ne[0] *= tensors[t]->ne[1];
-            tensors[t]->ne[1]  = tensors[t]->ne[2];
-            tensors[t]->ne[2]  = tensors[t]->ne[3];
-            tensors[t]->ne[3]  = 1;
-            tensors[t]->nb[1]  = tensors[t]->nb[2];
-            tensors[t]->nb[2]  = tensors[t]->nb[3];
-            tensors[t]->nb[3]  = tensors[t]->nb[2] * tensors[t]->ne[2];
-        }
+        src->ne[0] *= src->ne[1];
+        src->ne[1]  = src->ne[2];
+        src->ne[2]  = src->ne[3];
+        src->ne[3]  = 1;
+        dst->ne[0] *= dst->ne[1];
+        dst->ne[1]  = dst->ne[2];
+        dst->ne[2]  = dst->ne[3];
+        dst->ne[3]  = 1;
+        src->nb[1]  = src->nb[2];
+        src->nb[2]  = src->nb[3];
+        dst->nb[1]  = dst->nb[2];
+        dst->nb[2]  = dst->nb[3];
     }
 
     // ne1 too big to merge in full: merge it in chunks of k rows (k divides ne1)
-    const size_t row_size = (size_t) t0->ne[0] * elem_size;
-    bool rows_contig = true;
-    for (size_t t = 0; t < n_tensors; t++) {
-        if (tensors[t]->nb[1] != row_size) {
-            rows_contig = false;
-            break;
-        }
-    }
-    if (t0->ne[1] > 1 && row_size < 128 && rows_contig) {
+    const size_t row_size = (size_t) src->ne[0] * elem_size;
+    if (src->ne[1] > 1 && row_size < 128 && src->nb[1] == row_size && dst->nb[1] == row_size) {
         int64_t k = 2048 / row_size;
-        while (k > 1 && t0->ne[1] % k != 0) {
+        while (k > 1 && src->ne[1] % k != 0) {
             k--;
         }
         if (k > 1) {
-            for (size_t t = 0; t < n_tensors; t++) {
-                tensors[t]->ne[0] *= k;
-                tensors[t]->ne[1] /= k;
-                tensors[t]->nb[1] *= k;
-                tensors[t]->nb[2]  = tensors[t]->nb[1] * tensors[t]->ne[1];
-                tensors[t]->nb[3]  = tensors[t]->nb[2] * tensors[t]->ne[2];
-            }
-        }
-    }
-
-    // nrows too small to saturate threads: split ne0 into k rows (k divides ne0)
-    const size_t   row_size_curr = (size_t) t0->ne[0] * elem_size;
-    const uint32_t nrows         = (uint32_t) (t0->ne[1] * t0->ne[2] * t0->ne[3]);
-    rows_contig = true;
-    for (size_t t = 0; t < n_tensors; t++) {
-        if (tensors[t]->ne[1] != 1 && tensors[t]->nb[1] != row_size_curr) {
-            rows_contig = false;
-            break;
-        }
-    }
-    if (nrows < n_threads && rows_contig && row_size_curr >= 4096) {
-        const size_t vec_elems = 128 / elem_size;
-        int64_t k = std::max<int64_t>(row_size_curr / 16384, (n_threads * 4 + nrows - 1) / nrows);
-        k = std::min<int64_t>(k, row_size_curr / 2048);
-        while (k > 1 && t0->ne[0] % (k * vec_elems) != 0) {
-            k--;
-        }
-        if (k > 1) {
-            for (size_t t = 0; t < n_tensors; t++) {
-                tensors[t]->ne[0] /= k;
-                tensors[t]->ne[1] *= k;
-                tensors[t]->nb[1]  = tensors[t]->ne[0] * elem_size;
-                tensors[t]->nb[2]  = tensors[t]->nb[1] * tensors[t]->ne[1];
-                tensors[t]->nb[3]  = tensors[t]->nb[2] * tensors[t]->ne[2];
-            }
+            src->ne[0] *= k;
+            src->ne[1] /= k;
+            src->nb[1] *= k;
+            dst->ne[0] *= k;
+            dst->ne[1] /= k;
+            dst->nb[1] *= k;
         }
     }
 }
@@ -1560,6 +1496,19 @@ static bool htp_kparams_solve_unary_impl(
 
     const size_t elem_size = ggml_type_size(src0->type);
 
+    struct ggml_tensor src0_c = *src0;
+    struct ggml_tensor dst_c  = *dst;
+    if (htp_unary_is_elementwise(op)) {
+        htp_unary_collapse_dims(&src0_c, &dst_c, elem_size);
+        src0 = &src0_c;
+        dst  = &dst_c;
+        for (int i = 0; i < HTP_OP_MAX_DIMS; i++) {
+            kparams->ne[i]      = (uint32_t) src0->ne[i];
+            kparams->src0_nb[i] = (uint32_t) src0->nb[i];
+            kparams->dst_nb[i]  = (uint32_t) dst->nb[i];
+        }
+    }
+
     const size_t dst_data_row_size  = dst->ne[0]  * ggml_type_size(dst->type);
 
     const uint32_t src0_nrows          = src0->ne[1] * src0->ne[2] * src0->ne[3];
@@ -1573,14 +1522,9 @@ static bool htp_kparams_solve_unary_impl(
     const bool broadcast_weight = (op == HTP_OP_RMS_NORM_MUL) && (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1);
     kparams->broadcast_weight   = broadcast_weight;
 
-    const bool     src0_contig           = ggml_is_contiguous_1(src0);
-    const bool     src1_contig           = (op == HTP_OP_RMS_NORM_MUL && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true;
-    const uint32_t src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;
-    const uint32_t min_block_rows        = (uint32_t) (std::max<size_t>(1, 2048 / dst_data_row_size));
-    uint32_t       max_block             = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1];
-    if (src0_nrows_per_thread >= 2 * min_block_rows) {
-        max_block = (std::min)(max_block, (std::max)(min_block_rows, src0_nrows_per_thread / 4));
-    }
+    const bool     src0_contig = ggml_is_contiguous_1(src0);
+    const bool     src1_contig = (op == HTP_OP_RMS_NORM_MUL && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true;
+    const uint32_t max_block   = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1];
 
     kparams->src0_contig = src0_contig ? 1 : 0;
     kparams->src1_contig = src1_contig ? 1 : 0;
@@ -2045,17 +1989,11 @@ bool htp_kparams_solve_unary_mul_glu(
     const size_t   vtcm_size           = htp_session_vtcm_size(sess);
 
     const size_t   row_size_aligned    = hex_round_up(dst_row_size, 128);
-    uint32_t       vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * 3 * row_size_aligned));
+    const uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * 3 * row_size_aligned));
     if (vtcm_row_per_thread < 2) {
         return false;
     }
 
-    const uint32_t src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;
-    if (src0_nrows_per_thread >= 2 && vtcm_row_per_thread > src0_nrows_per_thread) {
-        uint32_t target_rpt = (src0_nrows_per_thread >= 4) ? (src0_nrows_per_thread / 2) & ~1u : 2;
-        vtcm_row_per_thread = (std::max)(2u, (std::min)(vtcm_row_per_thread, target_rpt));
-    }
-
     struct htp_glu_vtcm_layout L;
     htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, vtcm_row_per_thread);
 
@@ -3297,131 +3235,12 @@ static inline bool htp_kparams_invoke(
     return SolveFn(sess, op, kparams ? (KParams *) kparams : &tmp);
 }
 
-void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode & node) {
-    const uint32_t n_threads = htp_session_n_threads(sess);
-
-    if (htp_op_is_unary(node.opcode) && htp_unary_is_elementwise(node.opcode)) {
-        if (node.inputs.empty() || !node.inputs[0] || node.outputs.empty() || !node.outputs[0]) {
-            return;
-        }
-        if (!ggml_are_same_shape(node.inputs[0], node.outputs[0])) {
-            return;
-        }
-        const size_t elem_size = ggml_type_size(node.inputs[0]->type);
-        struct ggml_tensor src0_c = *node.inputs[0];
-        struct ggml_tensor dst_c  = *node.outputs[0];
-        struct ggml_tensor * tensors[2] = { &src0_c, &dst_c };
-        htp_collapse_dims(tensors, 2, elem_size, n_threads);
-        if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) {
-            node.inputs[0]  = node.add_dummy(src0_c);
-            node.outputs[0] = node.add_dummy(dst_c);
-        }
-        return;
-    }
-
-    if (htp_op_is_glu(node.opcode)) {
-        if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) {
-            return;
-        }
-        if (!ggml_are_same_shape(node.inputs[0], node.outputs[0]) ||
-            !ggml_are_same_shape(node.inputs[1], node.outputs[0])) {
-            return;
-        }
-        const size_t elem_size = ggml_type_size(node.inputs[0]->type);
-        struct ggml_tensor src0_c = *node.inputs[0];
-        struct ggml_tensor src1_c = *node.inputs[1];
-        struct ggml_tensor dst_c  = *node.outputs[0];
-        struct ggml_tensor * tensors[3] = { &src0_c, &src1_c, &dst_c };
-        htp_collapse_dims(tensors, 3, elem_size, n_threads);
-        if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) {
-            node.inputs[0]  = node.add_dummy(src0_c);
-            node.inputs[1]  = node.add_dummy(src1_c);
-            node.outputs[0] = node.add_dummy(dst_c);
-        }
-        return;
-    }
-
-    if (node.opcode == HTP_OP_MUL_MAT) {
-        if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) {
-            return;
-        }
-        const struct ggml_tensor * src0 = node.inputs[0];
-        const struct ggml_tensor * src1 = node.inputs[1];
-        const struct ggml_tensor * dst  = node.outputs[0];
-        if (htp_matmul_can_collapse(src0, src1, dst)) {
-            node.inputs[1]  = node.add_dummy(htp_tensor_collapse_rows(src1));
-            node.outputs[0] = node.add_dummy(htp_tensor_collapse_rows(dst));
-        }
-        return;
-    }
-
-    if (node.opcode == HTP_OP_MUL_MAT_ADD) {
-        if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) {
-            return;
-        }
-        const struct ggml_tensor * src0 = node.inputs[0];
-        const struct ggml_tensor * src1 = node.inputs[1];
-        const struct ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr;
-        const struct ggml_tensor * dst  = node.outputs[0];
-        if (htp_matmul_add_can_collapse(src0, src1, src2, dst)) {
-            node.inputs[1]  = node.add_dummy(htp_tensor_collapse_rows(src1));
-            node.outputs[0] = node.add_dummy(htp_tensor_collapse_rows(dst));
-            if (src2 && (src2->ne[1] * src2->ne[2] * src2->ne[3] > 1)) {
-                node.inputs[2] = node.add_dummy(htp_tensor_collapse_rows(src2));
-            }
-        }
-        return;
-    }
-}
-
-bool htp_kparams_solve(
-    const struct ggml_hexagon_session * sess,
-    htp_opnode & node
-) {
-    htp_opnode_collapse(sess, node);
-
-    const ggml_tensor * src0 = node.inputs.empty() ? nullptr : node.inputs[0];
-    const ggml_tensor * src1 = node.inputs.size() > 1 ? node.inputs[1] : nullptr;
-    const ggml_tensor * dst  = node.dst();
-
-    if (htp_op_is_unary(node.opcode)) {
-        return htp_kparams_solve_unary_impl(sess, node.opcode, src0, src1, dst, node.kparams<htp_unary_kernel_params>());
-    }
-    if (htp_op_is_glu(node.opcode)) {
-        return htp_kparams_solve_unary_mul_glu(sess, node.opcode, src0, src1, dst, node.kparams<htp_glu_kernel_params>());
-    }
-    if (node.opcode == HTP_OP_MUL_MAT) {
-        if (!htp_matmul_check_types(src0, src1, dst)) {
-            return false;
-        }
-        return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, 0, 0, node.kparams<htp_mm_kernel_params>());
-    }
-    if (node.opcode == HTP_OP_MUL_MAT_ADD) {
-        const ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr;
-        const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0;
-        return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, node.kparams<htp_mm_kernel_params>());
-    }
-
-    return htp_kparams_solve(sess, node.node, node.kernel_params);
-}
-
 bool htp_kparams_solve(
     const struct ggml_hexagon_session * sess,
     const struct ggml_tensor * op,
     void * kparams
 ) {
     const htp_op_code opcode = htp_op_remap(op);
-    if (htp_op_is_unary(opcode) || htp_op_is_glu(opcode)) {
-        htp_opnode tmp(opcode, op);
-        if (!htp_kparams_solve(sess, tmp)) {
-            return false;
-        }
-        if (kparams) {
-            memcpy(kparams, tmp.kernel_params, sizeof(tmp.kernel_params));
-        }
-        return true;
-    }
-
     switch (opcode) {
         case HTP_OP_MUL_MAT:
         case HTP_OP_MUL_MAT_ID:

ggml/src/ggml-hexagon/htp-kparams.h

@@ -34,7 +34,6 @@
 
 bool        htp_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride);
 bool        htp_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst);
-bool        htp_matmul_add_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * src2, const struct ggml_tensor * dst);
 ggml_tensor htp_tensor_collapse_rows(const struct ggml_tensor * t);
 
 template <typename... Args>
@@ -46,18 +45,6 @@ static inline bool htp_kparams_check(Args &&... args) {
     return SolveFn(std::forward<Args>(args)..., &kparams);
 }
 
-struct htp_opnode;
-
-bool htp_kparams_solve(
-    const struct ggml_hexagon_session * sess,
-    struct htp_opnode & node
-);
-
-void htp_opnode_collapse(
-    const struct ggml_hexagon_session * sess,
-    struct htp_opnode & node
-);
-
 bool htp_kparams_solve(
     const struct ggml_hexagon_session * sess,
     const struct ggml_tensor * op,

ggml/src/ggml-hexagon/htp/flash-attn-ops.c

@@ -2094,29 +2094,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
         const uint32_t dst_e_size = (dst->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(__fp16);
         const bool can_split      = htp_tensor_can_row_partition(dst, dst_e_size);
 
-        if ((kparams->flags & HTP_FA_FLAG_TEAM_SPLIT) && can_split) {
-            // KV heads split across teams, Q-blocks split within a team
-            uint32_t n_teams, team_size;
-            htp_fa_mdev_teams(n_kv_heads, mdev_count, kparams->head_split != 0, &n_teams, &team_size);
-
-            const uint32_t team        = mdev_idx / team_size;
-            const uint32_t member      = mdev_idx - team * team_size;
-            const uint32_t kv_per_team = n_kv_heads / n_teams;
-            kv_head_min = team * kv_per_team;
-            kv_head_max = kv_head_min + kv_per_team;
-
-            const uint32_t base  = n_q_blocks / team_size;
-            const uint32_t rem   = n_q_blocks - base * team_size;
-            const uint32_t b0    = member * base + MIN(member, rem);
-            const uint32_t nblk  = base + (member < rem ? 1 : 0);
-
-            if (nblk == 0) {
-                return HTP_STATUS_OK;
-            }
-
-            q_start_min = b0 * Br;
-            q_start_max = MIN((b0 + nblk) * Br, neq1);
-        } else if (kparams->head_split && can_split && n_kv_heads >= mdev_count && n_kv_heads % mdev_count == 0) {
+        if (kparams->head_split && can_split && n_kv_heads >= mdev_count && n_kv_heads % mdev_count == 0) {
             const uint32_t kv_per_core = n_kv_heads / mdev_count;
             kv_head_min = mdev_idx * kv_per_core;
             kv_head_max = kv_head_min + kv_per_core;

ggml/src/ggml-hexagon/htp/flash-attn-ops.h

@@ -32,20 +32,9 @@ enum htp_fa_kernel_type {
     HTP_FA_KERNEL_HMX
 };
 
-// Multicore: gcd(n_kv_heads, N) teams own the KV heads, team members share the Q-blocks
-#define HTP_FA_FLAG_TEAM_SPLIT 0x01
-
-static inline void htp_fa_mdev_teams(uint32_t n_kv_heads, uint32_t mdev_count, bool head_split,
-                                     uint32_t * n_teams, uint32_t * team_size) {
-    const uint32_t t = (head_split && n_kv_heads > 0 && mdev_count > 1) ? hex_gcd_u32(n_kv_heads, mdev_count) : 1;
-    *n_teams   = t;
-    *team_size = mdev_count > 1 ? mdev_count / t : 1;
-}
-
 struct htp_fa_kernel_params {
     uint8_t  kernel_type;        // enum htp_fa_kernel_type
     uint8_t  head_split;         // 1 = partition by KV heads in multicore, 0 = token partition
-    uint8_t  flags;              // HTP_FA_FLAG_*
     uint8_t  n_threads;          // Number of threads to run
 
     // Common parameters
@@ -332,8 +321,7 @@ static inline int hmx_fa_find_chunk_size(size_t * Br_out,
                                          size_t   n_threads,
                                          bool     is_q_fp32,
                                          bool     has_sinks,
-                                         size_t   n_heads,
-                                         size_t   q_split) {
+                                         size_t   n_heads) {
     const size_t T       = HMX_FP16_TILE_N_ROWS;  // 32
     const size_t br_unit = hmx_ceil_div(T, gqa_factor);
     const size_t bc_unit = HMX_FP16_TILE_N_COLS * 2;  // 64
@@ -373,8 +361,7 @@ static inline int hmx_fa_find_chunk_size(size_t * Br_out,
                 const size_t vecs_per_t     = n_use > 0 ? (n_row_vec_cnt + n_use - 1) / n_use : 1;
 
                 const size_t c_iter_actual  = c_iter_base + c_softmax * vecs_per_t;
-                const size_t q_blocks_nsp   = (q_blocks + q_split - 1) / (q_split ? q_split : 1);
-                const size_t cost           = q_blocks_nsp * (c_q_fixed + kv_blocks * c_iter_actual);
+                const size_t cost           = q_blocks * (c_q_fixed + kv_blocks * c_iter_actual);
                 const size_t mn             = Br * Bc;
 
                 if (cost < best_cost || (cost == best_cost && mn > best_mn)) {

ggml/src/ggml-hexagon/htp/htp_iface.idl

@@ -11,7 +11,7 @@ struct htp_iface_pmu_conf {
 };
 
 interface htp_iface : remote_handle64 {
-    AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 vtcm_size, in uint64 max_vmem, in uint32 power_mode);
+    AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 max_vmem, in uint32 power_mode);
     AEEResult stop();
     AEEResult mmap(in uint32 fd, in uint64 size);
     AEEResult munmap(in uint32 fd);

ggml/src/ggml-hexagon/htp/main.c

@@ -325,11 +325,8 @@ static unsigned int vtcm_query_size(void) {
     return vtcm_size;
 }
 
-static int vtcm_alloc(struct htp_context * ctx, uint64_t size_override) {
+static int vtcm_alloc(struct htp_context * ctx) {
     unsigned int vtcm_size = vtcm_query_size();
-    if (size_override) {
-        vtcm_size = (unsigned int) size_override;
-    }
 
     compute_res_attr_t attr;
     HAP_compute_res_attr_init(&attr);
@@ -441,7 +438,7 @@ static const struct htp_power_profile htp_power_profiles[HTP_POWER_MODE_COUNT] =
 
 #undef HTP_POWER_PROFILE_ENTRY
 
-AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t vtcm_size, uint64_t max_vmem, uint32_t power_mode) {
+AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t max_vmem, uint32_t power_mode) {
     struct htp_handle * h = (struct htp_handle *) handle;
     if (!h) {
         return AEE_EBADPARM;
@@ -560,7 +557,7 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp
     ctx->max_vmem    = max_vmem;
     ctx->dsp_queue   = dsp_queue;
 
-    err = vtcm_alloc(ctx, vtcm_size);
+    err = vtcm_alloc(ctx);
     if (err != AEE_SUCCESS) {
         FARF(ERROR, "Unable to allocate VTCM");
         htp_iface_stop(handle);

ggml/src/ggml-hexagon/htp/matmul-ops.c

@@ -16,7 +16,6 @@
 #include "hvx-dump.h"
 #include "hvx-arith.h"
 #include "hvx-reduce.h"
-#include "hvx-copy.h"
 
 #define GGML_COMMON_DECL_C
 #include "ggml-common.h"
@@ -1414,7 +1413,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) {
 
     for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) {  // for each expert
         const int32_t eid = *(const int32_t *) ((const uint8_t *) src2->data + ie1 * src2->nb[0]);
-        if (eid < 0) {
+        if (eid == -1) {
             continue;
         }
         assert(eid >= 0 && eid < (int32_t) n_ids);
@@ -3776,6 +3775,16 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k
     return HTP_STATUS_OK;
 }
 
+static inline void htp_mm_tensor_collapse_rows(struct htp_tensor * c, const struct htp_tensor * t, uint32_t stride) {
+    *c = *t;
+    c->ne[1] = t->ne[1] * t->ne[2] * t->ne[3];
+    c->ne[2] = 1;
+    c->ne[3] = 1;
+    c->nb[1] = stride;
+    c->nb[2] = c->nb[1] * c->ne[1];
+    c->nb[3] = c->nb[2];
+}
+
 static int op_matmul_impl(struct htp_ops_context * octx) {
     const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
 
@@ -3792,6 +3801,36 @@ static int op_matmul_impl(struct htp_ops_context * octx) {
 }
 
 int op_matmul(struct htp_ops_context * octx) {
+    const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
+
+    if (kparams->collapse) {
+        const struct htp_tensor * act  = octx->src[1];
+        const struct htp_tensor * dst  = octx->dst;
+        const uint32_t s_act = (act->ne[1] > 1) ? act->nb[1] : ((act->ne[2] > 1) ? act->nb[2] : act->nb[3]);
+        const uint32_t sd    = (dst->ne[1] > 1) ? dst->nb[1] : ((dst->ne[2] > 1) ? dst->nb[2] : dst->nb[3]);
+        struct htp_tensor act_collapsed, dst_collapsed;
+        htp_mm_tensor_collapse_rows(&act_collapsed, act, s_act);
+        htp_mm_tensor_collapse_rows(&dst_collapsed, dst, sd);
+        octx->src[1] = &act_collapsed;
+        octx->dst    = &dst_collapsed;
+
+        struct htp_tensor src2_collapsed;
+        const struct htp_tensor * src2 = octx->src[2];
+        if (src2 && (src2->ne[1] * src2->ne[2] * src2->ne[3] > 1)) {
+            const uint32_t s2 = (src2->ne[1] > 1) ? src2->nb[1] : ((src2->ne[2] > 1) ? src2->nb[2] : src2->nb[3]);
+            htp_mm_tensor_collapse_rows(&src2_collapsed, src2, s2);
+            octx->src[2] = &src2_collapsed;
+        }
+
+        const int status = op_matmul_impl(octx);
+        octx->src[1] = act;
+        octx->dst    = dst;
+        if (src2) {
+            octx->src[2] = src2;
+        }
+        return status;
+    }
+
     return op_matmul_impl(octx);
 }
 
@@ -4112,7 +4151,7 @@ static int hvx_mm_matmul_id_nx(
     return HTP_STATUS_OK;
 }
 
-static __attribute__((noinline)) uint32_t scan_expert_ids_n(
+static inline void scan_expert_ids_n(
     const struct htp_tensor * ids,
     const uint32_t n_ids,
     uint32_t n_as,
@@ -4122,17 +4161,15 @@ static __attribute__((noinline)) uint32_t scan_expert_ids_n(
 ) {
     const size_t ids_nb1 = ids->nb[1];
     const uint8_t * ids_data = (const uint8_t *) ids->data;
-    uint32_t n_skipped = 0;
 
     for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
         const int32_t * row_ptr = (const int32_t *) (ids_data + iid1 * ids_nb1);
         for (uint32_t id = 0; id < n_ids; ++id) {
             const int32_t i02 = row_ptr[id];
-            if (i02 < 0) {
-                n_skipped++;
+            if (i02 == -1) {
                 continue;
             }
-            assert(i02 >= 0 && i02 < (int32_t) n_as);
+            assert(i02 >= 0 && i02 < n_as);
 
             if (matrix_rows) {
                 matrix_rows[i02 * mapping_stride + counts[i02]] = (struct mmid_row_mapping) { id, iid1 };
@@ -4140,10 +4177,31 @@ static __attribute__((noinline)) uint32_t scan_expert_ids_n(
             counts[i02] += 1;
         }
     }
-    return n_skipped;
 }
 
-static __attribute__((noinline)) uint32_t scan_expert_ids(
+// the matmuls only write the rows that an expert owns, so the rows of the skipped slots are zeroed here
+static inline void zero_skipped_dst_rows(
+    const struct htp_tensor * ids,
+    uint32_t n_ids,
+    const struct htp_tensor * dst,
+    uint32_t ne0,
+    size_t nb1,
+    size_t nb2
+) {
+    const uint8_t * ids_data = (const uint8_t *) ids->data;
+
+    for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
+        for (uint32_t id = 0; id < n_ids; ++id) {
+            const int32_t i02 = *(const int32_t *) (ids_data + iid1 * ids->nb[1] + id * ids->nb[0]);
+            if (i02 != -1) {
+                continue;
+            }
+            memset((uint8_t *) (dst->data + id * nb1 + iid1 * nb2), 0, ne0 * sizeof(float));
+        }
+    }
+}
+
+static inline void scan_expert_ids(
     const struct htp_tensor * ids,
     uint32_t n_ids,
     uint32_t n_as,
@@ -4155,25 +4213,23 @@ static __attribute__((noinline)) uint32_t scan_expert_ids(
 
     if (ids_nb0 == 4) {
         switch (n_ids) {
-            case 8:  return scan_expert_ids_n(ids, 8,     n_as, counts, matrix_rows, mapping_stride);
-            case 4:  return scan_expert_ids_n(ids, 4,     n_as, counts, matrix_rows, mapping_stride);
-            case 2:  return scan_expert_ids_n(ids, 2,     n_as, counts, matrix_rows, mapping_stride);
-            default: return scan_expert_ids_n(ids, n_ids, n_as, counts, matrix_rows, mapping_stride);
+            case 8:  scan_expert_ids_n(ids, 8,     n_as, counts, matrix_rows, mapping_stride); break;
+            case 4:  scan_expert_ids_n(ids, 4,     n_as, counts, matrix_rows, mapping_stride); break;
+            case 2:  scan_expert_ids_n(ids, 2,     n_as, counts, matrix_rows, mapping_stride); break;
+            default: scan_expert_ids_n(ids, n_ids, n_as, counts, matrix_rows, mapping_stride); break;
         }
     } else {
         // Strided fallback
         const size_t ids_nb1 = ids->nb[1];
         const uint8_t * ids_data = (const uint8_t *) ids->data;
-        uint32_t n_skipped = 0;
         for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
             const int32_t * row_ptr = (const int32_t *) (ids_data + iid1 * ids_nb1);
             for (uint32_t id = 0; id < n_ids; ++id) {
                 const int32_t i02 = *(const int32_t *) ((const uint8_t *) row_ptr + id * ids_nb0);
-                if (i02 < 0) {
-                    n_skipped++;
+                if (i02 == -1) {
                     continue;
                 }
-                assert(i02 >= 0 && i02 < (int32_t) n_as);
+                assert(i02 >= 0 && i02 < n_as);
 
                 if (matrix_rows) {
                     matrix_rows[i02 * mapping_stride + counts[i02]] = (struct mmid_row_mapping) { id, iid1 };
@@ -4181,124 +4237,9 @@ static __attribute__((noinline)) uint32_t scan_expert_ids(
                 counts[i02] += 1;
             }
         }
-        return n_skipped;
     }
 }
 
-static __attribute__((noinline)) uint32_t count_skipped_ids_single(const struct htp_tensor * ids, uint32_t n_ids) {
-    const uint8_t * ids_data = (const uint8_t *) ids->data;
-    const size_t ids_nb0 = ids->nb[0];
-    uint32_t n_skipped = 0;
-    for (uint32_t id = 0; id < n_ids; ++id) {
-        const int32_t i02 = *(const int32_t *) (ids_data + id * ids_nb0);
-        if (i02 < 0) {
-            n_skipped++;
-        }
-    }
-    return n_skipped;
-}
-
-// Zero rows for skipped slots (ids < 0). Matmuls only write rows that an expert owns.
-static void zero_skipped_dst_rows(
-    struct htp_ops_context * octx,
-    const struct htp_tensor * ids,
-    uint32_t n_ids,
-    const struct htp_tensor * dst
-) {
-    const uint32_t elem_size = (dst->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float);
-    uint32_t col_start = 0;
-    uint32_t col_count = dst->ne[0];
-
-    if (octx->ctx->mdev.count > 1) {
-        const bool can_split = htp_tensor_can_row_partition(dst, elem_size);
-        const uint32_t chunk_elems = (dst->type == HTP_TYPE_F16) ? 64 : 32;
-        const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
-            dst->ne[0], can_split ? chunk_elems : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
-        col_start = range.start;
-        col_count = range.count;
-    }
-
-    if (col_count == 0) {
-        return;
-    }
-
-    const uint8_t * ids_data = (const uint8_t *) ids->data;
-    const size_t ids_nb0 = ids->nb[0];
-    const size_t ids_nb1 = ids->nb[1];
-    const size_t dst_nb1 = dst->nb[1];
-    const size_t dst_nb2 = dst->nb[2];
-    const uint32_t col_offset_bytes = col_start * elem_size;
-
-    for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
-        const uint8_t * ids_row = ids_data + iid1 * ids_nb1;
-        uint8_t * dst_row_group = (uint8_t *) dst->data + iid1 * dst_nb2 + col_offset_bytes;
-        for (uint32_t id = 0; id < n_ids; ++id) {
-            const int32_t i02 = *(const int32_t *) (ids_row + id * ids_nb0);
-            if (i02 < 0) {
-                hvx_splat_u(dst_row_group + id * dst_nb1, Q6_V_vzero(), col_count, elem_size);
-            }
-        }
-    }
-}
-
-static int htp_mmid_prepare_mapping(
-    struct htp_ops_context * octx,
-    const struct htp_tensor * ids,
-    uint32_t n_ids,
-    uint32_t n_as,
-    uint32_t act_nrows,
-    uint8_t ** out_mapping_buf,
-    uint32_t ** out_matrix_row_counts,
-    struct mmid_row_mapping ** out_matrix_rows,
-    uint32_t * out_mapping_stride,
-    uint32_t * out_n_skipped
-) {
-    uint8_t  * mapping_buf       = octx->ctx->ddr_spad_base;
-    uint32_t   mapping_stride    = 1;
-    uint32_t * matrix_row_counts = (uint32_t *) mapping_buf;
-    struct mmid_row_mapping * matrix_rows = NULL;
-    uint32_t n_skipped = 0;
-
-    if (act_nrows > 1) {
-        const size_t matrix_row_counts_size = n_as * sizeof(uint32_t);
-        assert(octx->ctx->ddr_spad_size >= matrix_row_counts_size);
-
-        hex_l2fetch_block((const void *) ids->data, ids->ne[1] * ids->nb[1]);
-
-        hvx_splat_u(matrix_row_counts, Q6_V_vzero(), n_as, sizeof(uint32_t));
-        n_skipped = scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0);
-
-        uint32_t max_count = hvx_reduce_max_i32((const uint8_t *) matrix_row_counts, n_as);
-        mapping_stride = max_count > 0 ? max_count : 1;
-
-        size_t matrix_row_map_size  = n_as * mapping_stride * sizeof(struct mmid_row_mapping);
-        const size_t total_map_size = matrix_row_counts_size + matrix_row_map_size;
-
-        if (total_map_size > octx->ctx->ddr_spad_size) {
-            mapping_buf = memalign(128, total_map_size);
-            if (!mapping_buf) {
-                return HTP_STATUS_INTERNAL_ERR;
-            }
-        }
-
-        matrix_row_counts = (uint32_t *) mapping_buf;
-        matrix_rows       = (struct mmid_row_mapping *) (mapping_buf + matrix_row_counts_size);
-
-        hvx_splat_u(matrix_row_counts, Q6_V_vzero(), n_as, sizeof(uint32_t));
-        scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride);
-    } else {
-        n_skipped = count_skipped_ids_single(ids, n_ids);
-    }
-
-    *out_mapping_buf       = mapping_buf;
-    *out_matrix_row_counts = matrix_row_counts;
-    *out_matrix_rows       = matrix_rows;
-    *out_mapping_stride    = mapping_stride;
-    *out_n_skipped         = n_skipped;
-
-    return HTP_STATUS_OK;
-}
-
 int op_matmul_id(struct htp_ops_context * octx) {
     htp_matmul_tensors_preamble;
 
@@ -4334,22 +4275,41 @@ int op_matmul_id(struct htp_ops_context * octx) {
     const int n_ids = ids->ne[0];  // n_expert_used
     const int n_as  = ne02;        // n_expert
 
-    uint8_t  * mapping_buf       = NULL;
-    uint32_t * matrix_row_counts = NULL;
-    struct mmid_row_mapping * matrix_rows = NULL;
+    uint8_t  * mapping_buf       = octx->ctx->ddr_spad_base;
     uint32_t   mapping_stride    = 1;
-    uint32_t   n_skipped         = 0;
+    uint32_t * matrix_row_counts = (uint32_t *) mapping_buf;
+    struct mmid_row_mapping * matrix_rows = NULL;
 
-    int map_status = htp_mmid_prepare_mapping(octx, ids, n_ids, n_as, act_nrows,
-                                              &mapping_buf, &matrix_row_counts, &matrix_rows,
-                                              &mapping_stride, &n_skipped);
-    if (map_status != HTP_STATUS_OK) {
-        return map_status;
+    if (act_nrows > 1) {
+        const size_t matrix_row_counts_size = n_as * sizeof(uint32_t);
+        assert(octx->ctx->ddr_spad_size >= matrix_row_counts_size);
+
+        hex_l2fetch_block((const void *) ids->data, ids->ne[1] * ids->nb[1]);
+
+        memset(matrix_row_counts, 0, matrix_row_counts_size);
+        scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0);
+
+        uint32_t max_count = hvx_reduce_max_i32((const uint8_t *) matrix_row_counts, n_as);
+        mapping_stride = max_count > 0 ? max_count : 1;
+
+        size_t matrix_row_map_size  = n_as * mapping_stride * sizeof(struct mmid_row_mapping);
+        const size_t total_map_size = matrix_row_counts_size + matrix_row_map_size;
+
+        if (total_map_size > octx->ctx->ddr_spad_size) {
+            mapping_buf = memalign(128, total_map_size);
+            if (!mapping_buf) {
+                return HTP_STATUS_INTERNAL_ERR;
+            }
+        }
+
+        matrix_row_counts = (uint32_t *) mapping_buf;
+        matrix_rows       = (struct mmid_row_mapping *) (mapping_buf + matrix_row_counts_size);
+
+        memset(matrix_row_counts, 0, n_as * sizeof(uint32_t));
+        scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride);
     }
 
-    if (n_skipped > 0) {
-        zero_skipped_dst_rows(octx, ids, n_ids, dst);
-    }
+    zero_skipped_dst_rows(ids, n_ids, dst, ne0, nb1, nb2);
 
     mmctx->matrix_row_counts    = matrix_row_counts;
     mmctx->matrix_rows          = matrix_rows;
@@ -4440,26 +4400,38 @@ int op_matmul_id_nx(struct htp_ops_context * octx) {
     const int n_ids = ids->ne[0];
     const int n_as  = src0->ne[2];
 
-    uint8_t  * mapping_buf       = NULL;
-    uint32_t * matrix_row_counts = NULL;
-    struct mmid_row_mapping * matrix_rows = NULL;
+    uint8_t  * mapping_buf       = octx->ctx->ddr_spad_base;
     uint32_t   mapping_stride    = 1;
-    uint32_t   n_skipped         = 0;
+    uint32_t * matrix_row_counts = (uint32_t *) mapping_buf;
+    struct mmid_row_mapping * matrix_rows = NULL;
 
-    int map_status = htp_mmid_prepare_mapping(octx, ids, n_ids, n_as, act_nrows,
-                                              &mapping_buf, &matrix_row_counts, &matrix_rows,
-                                              &mapping_stride, &n_skipped);
-    if (map_status != HTP_STATUS_OK) {
-        return map_status;
-    }
+    if (act_nrows > 1) {
+        const size_t matrix_row_counts_size = n_as * sizeof(uint32_t);
+        assert(octx->ctx->ddr_spad_size >= matrix_row_counts_size);
 
-    if (n_skipped > 0) {
-        for (uint32_t p = 0; p < n_weights; ++p) {
-            const struct htp_tensor * restrict d = octx->dsts[p];
-            if (d) {
-                zero_skipped_dst_rows(octx, ids, n_ids, d);
+        hex_l2fetch_block((const void *) ids->data, ids->ne[1] * ids->nb[1]);
+
+        memset(matrix_row_counts, 0, matrix_row_counts_size);
+        scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0);
+
+        uint32_t max_count = hvx_reduce_max_i32((const uint8_t *) matrix_row_counts, n_as);
+        mapping_stride = max_count > 0 ? max_count : 1;
+
+        size_t matrix_row_map_size  = n_as * mapping_stride * sizeof(struct mmid_row_mapping);
+        const size_t total_map_size = matrix_row_counts_size + matrix_row_map_size;
+
+        if (total_map_size > octx->ctx->ddr_spad_size) {
+            mapping_buf = memalign(128, total_map_size);
+            if (!mapping_buf) {
+                return HTP_STATUS_INTERNAL_ERR;
             }
         }
+
+        matrix_row_counts = (uint32_t *) mapping_buf;
+        matrix_rows       = (struct mmid_row_mapping *) (mapping_buf + matrix_row_counts_size);
+
+        memset(matrix_row_counts, 0, n_as * sizeof(uint32_t));
+        scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride);
     }
 
     mmctx->matrix_row_counts    = matrix_row_counts;
@@ -4590,5 +4562,35 @@ static int op_matmul_nx_impl(struct htp_ops_context * octx) {
 }
 
 int op_matmul_nx(struct htp_ops_context * octx) {
+    const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
+
+    if (kparams->collapse) {
+        const uint32_t n_weights = kparams->n_weights;
+        const struct htp_tensor * act = octx->src[n_weights];
+        const uint32_t s1 = (act->ne[1] > 1) ? act->nb[1] : ((act->ne[2] > 1) ? act->nb[2] : act->nb[3]);
+        struct htp_tensor act_collapsed;
+        struct htp_tensor dsts_collapsed[HTP_OP_MAX_OUTPUTS];
+        const struct htp_tensor * orig_dsts[HTP_OP_MAX_OUTPUTS];
+
+        htp_mm_tensor_collapse_rows(&act_collapsed, act, s1);
+        octx->src[n_weights] = &act_collapsed;
+
+        for (uint32_t p = 0; p < n_weights; p++) {
+            orig_dsts[p] = octx->dsts[p];
+            const struct htp_tensor * d = octx->dsts[p];
+            const uint32_t sd = (d->ne[1] > 1) ? d->nb[1] : ((d->ne[2] > 1) ? d->nb[2] : d->nb[3]);
+            htp_mm_tensor_collapse_rows(&dsts_collapsed[p], d, sd);
+            octx->dsts[p] = &dsts_collapsed[p];
+        }
+
+        const int status = op_matmul_nx_impl(octx);
+
+        octx->src[n_weights] = act;
+        for (uint32_t p = 0; p < n_weights; p++) {
+            octx->dsts[p] = orig_dsts[p];
+        }
+        return status;
+    }
+
     return op_matmul_nx_impl(octx);
 }

ggml/src/ggml-hexagon/htp/matmul-ops.h

@@ -61,7 +61,7 @@ enum htp_mm_kernel_type {
 struct htp_mm_kernel_params {
     uint8_t  kernel_type;        // enum htp_mm_kernel_type
     uint8_t  pipeline;           // 1 = pipelined execution, 0 = standard
-    uint8_t  unused;             // reserved
+    uint8_t  collapse;           // 1 = collapse outer dims into 2D, 0 = standard
     uint8_t  n_hmx;              // 1 = use HMX, 0 = use HVX
 
     uint8_t  n_threads;          // Number of threads to spawn

ggml/src/ggml-hexagon/htp/unary-ops.c

@@ -1984,8 +1984,29 @@ static int execute_op_unary(struct htp_ops_context * octx) {
 int op_unary(struct htp_ops_context * octx) {
     switch (octx->src[0]->type) {
         case HTP_TYPE_F32:
-        case HTP_TYPE_F16:
-            return execute_op_unary(octx);
+        case HTP_TYPE_F16: {
+            if (!htp_unary_is_elementwise(octx->op)) {
+                return execute_op_unary(octx);
+            }
+
+            const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params;
+
+            struct htp_tensor src0_c = *octx->src[0];
+            struct htp_tensor dst_c  = *octx->dst;
+            memcpy(src0_c.ne, kparams->ne,      sizeof(src0_c.ne));
+            memcpy(dst_c.ne,  kparams->ne,      sizeof(dst_c.ne));
+            memcpy(src0_c.nb, kparams->src0_nb, sizeof(src0_c.nb));
+            memcpy(dst_c.nb,  kparams->dst_nb,  sizeof(dst_c.nb));
+
+            const struct htp_tensor * orig_src0 = octx->src[0];
+            const struct htp_tensor * orig_dst  = octx->dst;
+            octx->src[0] = &src0_c;
+            octx->dst    = &dst_c;
+            const int status = execute_op_unary(octx);
+            octx->src[0] = orig_src0;
+            octx->dst    = orig_dst;
+            return status;
+        }
 
         default:
             return HTP_STATUS_NO_SUPPORT;

ggml/src/ggml-hexagon/htp/unary-ops.h

@@ -24,6 +24,11 @@ struct htp_unary_kernel_params {
     struct fastdiv_values div_ne02;
     struct fastdiv_values div_ne012;
     struct fastdiv_values div_tpr;
+
+    // collapsed shape of elementwise ops
+    uint32_t  ne[HTP_OP_MAX_DIMS];
+    uint32_t  src0_nb[HTP_OP_MAX_DIMS];
+    uint32_t  dst_nb[HTP_OP_MAX_DIMS];
 };
 
 #if defined(__cplusplus)
@@ -67,9 +72,6 @@ static inline bool htp_op_is_unary(uint32_t opcode) {
 }
 
 static inline bool htp_unary_is_elementwise(uint32_t opcode) {
-    if (!htp_op_is_unary(opcode)) {
-        return false;
-    }
     switch (opcode) {
         case HTP_OP_NORM:
         case HTP_OP_RMS_NORM:

ggml/src/ggml-metal/ggml-metal-device.cpp

@@ -507,47 +507,6 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexe
     return res;
 }
 
-ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_mask(
-        ggml_metal_library_t lib,
-        const ggml_tensor * op,
-        bool    fill,
-        bool    has_score,
-        bool    has_blocks,
-        bool    has_extra,
-        bool    vec) {
-    GGML_ASSERT(op->op == GGML_OP_TOP_K_MASK);
-
-    char base[256];
-    char name[256];
-
-    snprintf(base, 256, "kernel_top_k_mask%s_%s", fill ? "_fill" : "", ggml_type_name(op->type));
-
-    ggml_metal_pipeline_with_params res;
-
-    ggml_metal_cv_t cv = ggml_metal_cv_init();
-
-    if (fill) {
-        snprintf(name, 256, "%s_vec=%d", base, vec ? 1 : 0);
-
-        ggml_metal_cv_set_bool(cv, vec, FC_TOP_K_MASK + 3);
-    } else {
-        snprintf(name, 256, "%s_sc=%d_bl=%d_ex=%d", base, has_score ? 1 : 0, has_blocks ? 1 : 0, has_extra ? 1 : 0);
-
-        ggml_metal_cv_set_bool(cv, has_score,  FC_TOP_K_MASK + 0);
-        ggml_metal_cv_set_bool(cv, has_blocks, FC_TOP_K_MASK + 1);
-        ggml_metal_cv_set_bool(cv, has_extra,  FC_TOP_K_MASK + 2);
-    }
-
-    res = ggml_metal_library_get_pipeline(lib, name);
-    if (!res.pipeline) {
-        res = ggml_metal_library_compile_pipeline(lib, base, name, cv);
-    }
-
-    ggml_metal_cv_free(cv);
-
-    return res;
-}
-
 ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_metal_library_t lib, const ggml_tensor * op) {
     char name[256];
     const char * base = nullptr;
@@ -1266,6 +1225,40 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta
     return res;
 }
 
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax_part(ggml_metal_library_t lib) {
+    char base[256];
+    char name[256];
+
+    snprintf(base, 256, "kernel_mul_mm_id_amax_part_f32");
+    snprintf(name, 256, "%s", base);
+
+    ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+    if (!res.pipeline) {
+        res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
+    }
+
+    res.smem = 32*sizeof(float);
+
+    return res;
+}
+
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax(ggml_metal_library_t lib) {
+    char base[256];
+    char name[256];
+
+    snprintf(base, 256, "kernel_mul_mm_id_amax_f32");
+    snprintf(name, 256, "%s", base);
+
+    ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+    if (!res.pipeline) {
+        res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
+    }
+
+    res.smem = 32*sizeof(float);
+
+    return res;
+}
+
 ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0(ggml_metal_library_t lib, int ne02, int ne20) {
     char base[256];
     char name[256];
@@ -1308,14 +1301,18 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id(ggml_m
 
     const bool bc_inp = op->src[0]->ne[0] % 32 != 0;
 
+    // src1 prec [TAG_GGML_PREC]
+    const bool amax = ggml_get_op_params_i32(op, 3) == GGML_PREC_F32;
+
     snprintf(base, 256, "kernel_mul_mm_id_%s_%s", ggml_type_name(tsrc0), ggml_type_name(tsrc1));
-    snprintf(name, 256, "%s_bci=%d", base, bc_inp);
+    snprintf(name, 256, "%s_bci=%d_amax=%d", base, bc_inp, amax);
 
     ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
     if (!res.pipeline) {
         ggml_metal_cv_t cv = ggml_metal_cv_init();
 
         ggml_metal_cv_set_bool(cv, bc_inp, FC_MUL_MM + 0);
+        ggml_metal_cv_set_bool(cv, amax,   FC_MUL_MM + 6);
 
         res = ggml_metal_library_compile_pipeline(lib, base, name, cv);
 

ggml/src/ggml-metal/ggml-metal-device.h

@@ -126,7 +126,6 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_ad
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_tri               (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max          (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexer (ggml_metal_library_t lib, const struct ggml_tensor * op);
-struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_mask        (ggml_metal_library_t lib, const struct ggml_tensor * op, bool fill, bool has_score, bool has_blocks, bool has_extra, bool vec);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc           (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv          (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t nc, bool use_silu);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched  (ggml_metal_library_t lib, const struct ggml_tensor * op, int ssm_conv_bs, int32_t nc, bool use_silu);
@@ -142,6 +141,8 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv            (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0    (ggml_metal_library_t lib, int ne02, int ne20);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_zero    (ggml_metal_library_t lib);
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax(ggml_metal_library_t lib);
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax_part(ggml_metal_library_t lib);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id         (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id         (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax            (ggml_metal_library_t lib, const struct ggml_tensor * op);

ggml/src/ggml-metal/ggml-metal-device.m

@@ -1555,7 +1555,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
 
     switch (op->op) {
         case GGML_OP_SCALE:
-            return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16);
         case GGML_OP_FILL:
         case GGML_OP_CLAMP:
         case GGML_OP_SQR:
@@ -1835,15 +1834,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
                 ggml_is_contiguous_rows(op->src[1]) &&
                 ggml_is_contiguous_rows(op->src[2]) &&
                 (op->src[3] == NULL || ggml_is_contiguous_rows(op->src[3]));
-        case GGML_OP_TOP_K_MASK:
-            return (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32) &&
-                op->src[0]->type == op->type &&
-                op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) &&
-                op->nb[0] == ggml_type_size(op->type) &&
-                op->src[1]->type == GGML_TYPE_I32 &&
-                (op->src[2] == NULL || op->src[2]->type == GGML_TYPE_F32) &&
-                (op->src[3] == NULL || op->src[3]->type == GGML_TYPE_I32) &&
-                (op->src[4] == NULL || op->src[4]->type == GGML_TYPE_I32);
         case GGML_OP_SSM_SCAN:
             return has_simdgroup_reduction;
         case GGML_OP_SSM_CONV:

ggml/src/ggml-metal/ggml-metal-impl.h

@@ -14,6 +14,8 @@
 #define N_MM_SIMD_GROUP_X 2
 #define N_MM_SIMD_GROUP_Y 2
 
+#define N_MM_NPART_AMAX 256
+
 #define N_ARGMAX_NPART_MAX 256
 #define N_ARGMAX_CHUNK 4096
 
@@ -125,7 +127,6 @@
 #define FC_FLASH_ATTN_EXT_TENSOR       2200
 #define FC_LIGHTNING_INDEXER           2200
 #define FC_MUL_MV_MMA                  2300
-#define FC_TOP_K_MASK                  2400
 
 // op-specific constants
 #define OP_FLASH_ATTN_EXT_NQPSG 8
@@ -578,6 +579,14 @@ typedef struct {
     uint64_t nb21;
 } ggml_metal_kargs_mul_mm_id_zero;
 
+typedef struct {
+    int32_t  ne00;
+    int32_t  ne01;
+    int32_t  ne02;
+    uint64_t nb01;
+    uint64_t nb02;
+} ggml_metal_kargs_mul_mm_id_amax;
+
 typedef struct {
     int32_t  ne00;
     int32_t  ne02;
@@ -1296,38 +1305,6 @@ typedef struct {
     uint64_t nbm3;
 } ggml_metal_kargs_lightning_indexer;
 
-typedef struct {
-    int32_t  n_kv;
-    int32_t  m1;
-    int32_t  m2;
-    int32_t  m3;
-    int32_t  n_top;
-    int32_t  n_pick;
-    int32_t  n_blk;
-    int32_t  n_extra;
-    int64_t  n_elems;
-    uint64_t nb1;
-    uint64_t nb2;
-    uint64_t nb3;
-    uint64_t nbm1;
-    uint64_t nbm2;
-    uint64_t nbm3;
-    uint64_t nbk0;
-    uint64_t nbk1;
-    uint64_t nbk2;
-    uint64_t nbk3;
-    uint64_t nbs0;
-    uint64_t nbs1;
-    uint64_t nbs2;
-    uint64_t nbs3;
-    uint64_t nbb0;
-    uint64_t nbb1;
-    uint64_t nbe0;
-    uint64_t nbe1;
-    uint64_t nbe2;
-    uint64_t nbe3;
-} ggml_metal_kargs_top_k_mask;
-
 typedef struct {
     int32_t  n_tokens;
     int32_t  n_iter;

ggml/src/ggml-metal/ggml-metal-ops.cpp

@@ -345,10 +345,6 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {
             {
                 n_fuse = ggml_metal_op_lightning_indexer(ctx, idx);
             } break;
-        case GGML_OP_TOP_K_MASK:
-            {
-                n_fuse = ggml_metal_op_top_k_mask(ctx, idx);
-            } break;
         case GGML_OP_DSV4_HC_COMB:
         case GGML_OP_DSV4_HC_PRE:
         case GGML_OP_DSV4_HC_POST:
@@ -1428,124 +1424,6 @@ int ggml_metal_op_lightning_indexer(ggml_metal_op_t ctx, int idx) {
     return 1;
 }
 
-int ggml_metal_op_top_k_mask(ggml_metal_op_t ctx, int idx) {
-    ggml_tensor * op = ctx->node(idx);
-
-    ggml_metal_library_t lib = ctx->lib;
-    ggml_metal_encoder_t enc = ctx->enc;
-
-    const ggml_tensor * mask   = op->src[0];
-    const ggml_tensor * top_k  = op->src[1];
-    const ggml_tensor * score  = op->src[2];
-    const ggml_tensor * blocks = op->src[3];
-    const ggml_tensor * extra  = op->src[4];
-
-    GGML_ASSERT(op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32);
-    GGML_ASSERT(mask->type == op->type);
-    GGML_ASSERT(top_k->type == GGML_TYPE_I32);
-    GGML_ASSERT(!score  || score->type  == GGML_TYPE_F32);
-    GGML_ASSERT(!blocks || blocks->type == GGML_TYPE_I32);
-    GGML_ASSERT(!extra  || extra->type  == GGML_TYPE_I32);
-
-    const int32_t n_kv    = mask->ne[0];
-    const int32_t n_top   = (int32_t) top_k->ne[0];
-    const int32_t n_pick  = blocks ? blocks->ne[1] : score ? score->ne[0] : n_kv;
-    const int32_t n_blk   = blocks ? blocks->ne[0] : 0;
-    const int32_t n_extra = extra  ? extra->ne[0]  : 0;
-
-    const ggml_metal_buffer_id bid = ggml_metal_get_buffer_id(op);
-
-    // the vector fill stores 4 elements at a time
-    const int32_t ts      = ggml_type_size(op->type);
-    const int64_t n_elems = ggml_nelements(op);
-
-    const bool has_score  = score  != NULL;
-    const bool has_blocks = blocks != NULL;
-    const bool has_extra  = extra  != NULL;
-
-    const bool vec = n_elems % 4 == 0 && bid.offs % (4*ts) == 0;
-
-    ggml_metal_kargs_top_k_mask args = {
-        /*.n_kv    =*/ n_kv,
-        /*.m1      =*/ (int32_t) mask->ne[1],
-        /*.m2      =*/ (int32_t) mask->ne[2],
-        /*.m3      =*/ (int32_t) mask->ne[3],
-        /*.n_top   =*/ n_top,
-        /*.n_pick  =*/ n_pick,
-        /*.n_blk   =*/ n_blk,
-        /*.n_extra =*/ n_extra,
-        /*.n_elems =*/ n_elems,
-        /*.nb1     =*/ op->nb[1],
-        /*.nb2     =*/ op->nb[2],
-        /*.nb3     =*/ op->nb[3],
-        /*.nbm1    =*/ mask->nb[1],
-        /*.nbm2    =*/ mask->nb[2],
-        /*.nbm3    =*/ mask->nb[3],
-        /*.nbk0    =*/ top_k->nb[0],
-        /*.nbk1    =*/ top_k->nb[1],
-        /*.nbk2    =*/ top_k->nb[2],
-        /*.nbk3    =*/ top_k->nb[3],
-        /*.nbs0    =*/ score  ? score->nb[0]  : 0,
-        /*.nbs1    =*/ score  ? score->nb[1]  : 0,
-        /*.nbs2    =*/ score  ? score->nb[2]  : 0,
-        /*.nbs3    =*/ score  ? score->nb[3]  : 0,
-        /*.nbb0    =*/ blocks ? blocks->nb[0] : 0,
-        /*.nbb1    =*/ blocks ? blocks->nb[1] : 0,
-        /*.nbe0    =*/ extra  ? extra->nb[0]  : 0,
-        /*.nbe1    =*/ extra  ? extra->nb[1]  : 0,
-        /*.nbe2    =*/ extra  ? extra->nb[2]  : 0,
-        /*.nbe3    =*/ extra  ? extra->nb[3]  : 0,
-    };
-
-    // the fill covers the whole dst, the scatter overwrites the kept cells with the mask values
-    {
-        auto pipeline_fill = ggml_metal_library_get_pipeline_top_k_mask(lib, op, true, false, false, false, vec);
-
-        const int64_t n_fill  = n_elems/(vec ? 4 : 1);
-        const int32_t nth_max = ggml_metal_pipeline_max_theads_per_threadgroup(pipeline_fill);
-
-        int nth = 32; // SIMD width
-
-        while (nth < nth_max && nth < n_fill) {
-            nth *= 2;
-        }
-
-        ggml_metal_encoder_set_pipeline(enc, pipeline_fill);
-        ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0);
-        ggml_metal_encoder_set_buffer(enc, bid, 1);
-
-        ggml_metal_encoder_dispatch_threadgroups(enc, (n_fill + nth - 1)/nth, 1, 1, nth, 1, 1);
-
-        ggml_metal_op_concurrency_reset(ctx);
-    }
-
-    {
-        auto pipeline = ggml_metal_library_get_pipeline_top_k_mask(lib, op, false, has_score, has_blocks, has_extra, false);
-
-        const int32_t n_max   = n_top + n_extra;
-        const int32_t nth_max = ggml_metal_pipeline_max_theads_per_threadgroup(pipeline);
-
-        int nth = 32; // SIMD width
-
-        while (nth < nth_max && nth < n_max) {
-            nth *= 2;
-        }
-
-        ggml_metal_encoder_set_pipeline(enc, pipeline);
-        ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0);
-        ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(mask),  1);
-        ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(top_k), 2);
-        ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(score  ? score  : mask), 3);
-        ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(blocks ? blocks : mask), 4);
-        ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(extra  ? extra  : mask), 5);
-        ggml_metal_encoder_set_buffer(enc, bid, 6);
-
-        ggml_metal_encoder_dispatch_threadgroups(enc, op->ne[1], op->ne[2], op->ne[3], nth, 1, 1);
-    }
-
-    return 1;
-}
-
 int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) {
     ggml_tensor * op = ctx->node(idx);
 
@@ -2934,6 +2812,15 @@ size_t ggml_metal_op_mul_mat_id_extra_ids(const ggml_tensor * op) {
     return ggml_type_size(GGML_TYPE_I32)*(ne20*ne21 + ne02);
 }
 
+size_t ggml_metal_op_mul_mat_id_extra_amax(const ggml_tensor * op) {
+    assert(op->op == GGML_OP_MUL_MAT_ID);
+
+    GGML_UNUSED(op);
+
+    // 2 scaling factors (8 bytes) + N_MM_NPART_AMAX per-threadgroup scales for stage-1
+    return 8 + N_MM_NPART_AMAX*sizeof(float);
+}
+
 int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
     ggml_tensor * op = ctx->node(idx);
 
@@ -2985,6 +2872,39 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
         ggml_metal_buffer_id bid_ids = bid_tpe;
         bid_ids.offs += ggml_metal_op_mul_mat_id_extra_tpe(op);
 
+        ggml_metal_buffer_id bid_amax = bid_ids;
+        bid_amax.offs += ggml_metal_op_mul_mat_id_extra_ids(op);
+
+        // src1 prec [TAG_GGML_PREC]
+        const bool use_amax = ggml_get_op_params_i32(op, 3) == GGML_PREC_F32;
+
+        // src1 rescale factors, computed before the matmul
+        // ref: https://github.com/ggml-org/llama.cpp/pull/26223
+        if (use_amax) {
+            ggml_metal_kargs_mul_mm_id_amax args = {
+                /*.ne00 =*/ ne10,
+                /*.ne01 =*/ ne11,
+                /*.ne02 =*/ ne12,
+                /*.nb01 =*/ nb11,
+                /*.nb02 =*/ nb12,
+            };
+
+            auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id_amax_part(lib);
+
+            const size_t smem = pipeline.smem;
+
+            GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size);
+
+            ggml_metal_encoder_set_pipeline(enc, pipeline);
+            ggml_metal_encoder_set_bytes   (enc, &args, sizeof(args), 0);
+            ggml_metal_encoder_set_buffer  (enc, bid_src1, 1);
+            ggml_metal_encoder_set_buffer  (enc, bid_amax, 2);
+
+            ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);
+
+            ggml_metal_encoder_dispatch_threadgroups(enc, N_MM_NPART_AMAX, 1, 1, 256, 1, 1);
+        }
+
         {
             ggml_metal_kargs_mul_mm_id_map0 args = {
                 ne10,
@@ -3034,9 +2954,20 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
             ggml_metal_encoder_dispatch_threadgroups(enc, ne20, ne21, 1, 32, 1, 1);
         }
 
-        // this barrier is always needed because the next kernel has to wait for the id maps to be computed
         ggml_metal_op_concurrency_reset(ctx);
 
+        if (use_amax) {
+            auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id_amax(lib);
+
+            ggml_metal_encoder_set_pipeline(enc, pipeline);
+            ggml_metal_encoder_set_buffer  (enc, bid_amax, 0);
+
+            ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, 32, 1, 1);
+
+            // the next kernel has to wait for the amax data
+            ggml_metal_op_concurrency_reset(ctx);
+        }
+
         {
             auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id(lib, op);
 
@@ -3065,6 +2996,7 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
             ggml_metal_encoder_set_buffer  (enc, bid_tpe,  3);
             ggml_metal_encoder_set_buffer  (enc, bid_ids,  4);
             ggml_metal_encoder_set_buffer  (enc, bid_dst,  5);
+            ggml_metal_encoder_set_buffer  (enc, bid_amax, 6);
 
             const size_t smem = pipeline.smem;
 

ggml/src/ggml-metal/ggml-metal-ops.h

@@ -36,6 +36,7 @@ size_t ggml_metal_op_mul_mat_id_extra_tpe(const struct ggml_tensor * op);
 
 // id map [n_tokens, n_expert]
 size_t ggml_metal_op_mul_mat_id_extra_ids(const struct ggml_tensor * op);
+size_t ggml_metal_op_mul_mat_id_extra_amax(const struct ggml_tensor * op);
 
 // return true if we should use the FA vector kernel for this op
 bool ggml_metal_op_flash_attn_ext_use_vec(const struct ggml_tensor * op);
@@ -60,7 +61,6 @@ int ggml_metal_op_get_rows          (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_set_rows          (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_diag              (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_lightning_indexer (ggml_metal_op_t ctx, int idx);
-int ggml_metal_op_top_k_mask        (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_dsv4_hc           (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_soft_max          (ggml_metal_op_t ctx, int idx);
 int ggml_metal_op_ssm_conv          (ggml_metal_op_t ctx, int idx);

ggml/src/ggml-metal/ggml-metal.cpp

@@ -226,6 +226,7 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
             {
                 res += ggml_metal_op_mul_mat_id_extra_tpe(tensor);
                 res += ggml_metal_op_mul_mat_id_extra_ids(tensor);
+                res += ggml_metal_op_mul_mat_id_extra_amax(tensor);
             } break;
         case GGML_OP_FLASH_ATTN_EXT:
             {

ggml/src/ggml-metal/kernels/misc.metal

@@ -906,99 +906,3 @@ kernel void kernel_dsv4_hc_post_f32(
         *(device float *) (dst + i0*args.nb_d0 + idst*args.nb_d1 + it*args.nb_d2) = result[idst];
     }
 }
-
-constant bool FC_top_k_mask_has_score  [[function_constant(FC_TOP_K_MASK + 0)]];
-constant bool FC_top_k_mask_has_blocks [[function_constant(FC_TOP_K_MASK + 1)]];
-constant bool FC_top_k_mask_has_extra  [[function_constant(FC_TOP_K_MASK + 2)]];
-constant bool FC_top_k_mask_vec        [[function_constant(FC_TOP_K_MASK + 3)]];
-
-// the fill runs before the scatter, both are fully parallel over the whole dst and over the picks of one row
-template <typename T, typename T4>
-kernel void kernel_top_k_mask_fill(
-        constant ggml_metal_kargs_top_k_mask & args,
-        device char * dst,
-        uint  tid[[thread_position_in_grid]],
-        uint ntot[[threads_per_grid]]) {
-    const T ninf = T(-INFINITY);
-
-    if (FC_top_k_mask_vec) {
-        device T4 * dst_4 = (device T4 *) dst;
-        for (int64_t i = tid; i < args.n_elems/4; i += ntot) {
-            dst_4[i] = T4(ninf);
-        }
-    } else {
-        device T * dst_t = (device T *) dst;
-        for (int64_t i = tid; i < args.n_elems; i += ntot) {
-            dst_t[i] = ninf;
-        }
-    }
-}
-
-typedef decltype(kernel_top_k_mask_fill<half,  half4>)  kernel_top_k_mask_fill_h_t;
-typedef decltype(kernel_top_k_mask_fill<float, float4>) kernel_top_k_mask_fill_f_t;
-
-template [[host_name("kernel_top_k_mask_fill_f16")]] kernel kernel_top_k_mask_fill_h_t kernel_top_k_mask_fill<half,  half4>;
-template [[host_name("kernel_top_k_mask_fill_f32")]] kernel kernel_top_k_mask_fill_f_t kernel_top_k_mask_fill<float, float4>;
-
-// one threadgroup per row: copy the mask cells of the live picks and of the extra cells over the fill
-template <typename T, typename T4>
-kernel void kernel_top_k_mask(
-        constant ggml_metal_kargs_top_k_mask & args,
-        device const  char * mask,
-        device const  char * top_k,
-        device const  char * score,
-        device const  char * blocks,
-        device const  char * extra,
-        device        char * dst,
-        uint3   tgpig[[threadgroup_position_in_grid]],
-        uint3   tpitg[[thread_position_in_threadgroup]],
-        uint3 ntg_max[[threads_per_threadgroup]]) {
-    const int i1 = tgpig.x;
-    const int i2 = tgpig.y;
-    const int i3 = tgpig.z;
-
-    const int   tiitg = tpitg.x;
-    const int     ntg = ntg_max.x;
-
-    device const char * mask_row  = mask  + (i1 % args.m1)*args.nbm1 + (i2 % args.m2)*args.nbm2 + (i3 % args.m3)*args.nbm3;
-    device       char * dst_row   = dst   + i1*args.nb1 + i2*args.nb2 + i3*args.nb3;
-    device const char * top_k_row = top_k + i1*args.nbk1 + i2*args.nbk2 + i3*args.nbk3;
-
-    device const T * mask_row_t = (device const T *) mask_row;
-    device       T * dst_row_t  = (device       T *) dst_row;
-
-    for (int i = tiitg; i < args.n_top; i += ntg) {
-        const int32_t p = *(device const int32_t *) (top_k_row + i*args.nbk0);
-        if (p < 0 || p >= args.n_pick) {
-            continue;
-        }
-        if (FC_top_k_mask_has_score && !(*(device const float *) (score + p*args.nbs0 + i1*args.nbs1 + i2*args.nbs2 + i3*args.nbs3) > -INFINITY)) {
-            continue;
-        }
-        if (FC_top_k_mask_has_blocks) {
-            FOR_UNROLL (int k = 0; k < args.n_blk; ++k) {
-                const int32_t c = *(device const int32_t *) (blocks + k*args.nbb0 + p*args.nbb1);
-                if (c >= 0 && c < args.n_kv) {
-                    dst_row_t[c] = mask_row_t[c];
-                }
-            }
-        } else if (p < args.n_kv) {
-            dst_row_t[p] = mask_row_t[p];
-        }
-    }
-
-    if (FC_top_k_mask_has_extra) {
-        for (int i = tiitg; i < args.n_extra; i += ntg) {
-            const int32_t c = *(device const int32_t *) (extra + i*args.nbe0 + i1*args.nbe1 + i2*args.nbe2 + i3*args.nbe3);
-            if (c >= 0 && c < args.n_kv) {
-                dst_row_t[c] = mask_row_t[c];
-            }
-        }
-    }
-}
-
-typedef decltype(kernel_top_k_mask<half,  half4>)  kernel_top_k_mask_h_t;
-typedef decltype(kernel_top_k_mask<float, float4>) kernel_top_k_mask_f_t;
-
-template [[host_name("kernel_top_k_mask_f16")]] kernel kernel_top_k_mask_h_t kernel_top_k_mask<half,  half4>;
-template [[host_name("kernel_top_k_mask_f32")]] kernel kernel_top_k_mask_f_t kernel_top_k_mask<float, float4>;

ggml/src/ggml-metal/kernels/mul_mm.metal

@@ -7,6 +7,7 @@ constant short FC_mul_mm_ne12  [[function_constant(FC_MUL_MM + 2)]];
 constant short FC_mul_mm_ne13  [[function_constant(FC_MUL_MM + 3)]];
 constant short FC_mul_mm_r2    [[function_constant(FC_MUL_MM + 4)]];
 constant short FC_mul_mm_r3    [[function_constant(FC_MUL_MM + 5)]];
+constant bool FC_mul_mm_id_amax [[function_constant(FC_MUL_MM + 6)]];
 constant int32_t FC_mul_mm_id_map0_ne02 [[function_constant(FC_MUL_MM + 7)]];
 
 // each block_q contains 16*nl weights
@@ -323,10 +324,10 @@ kernel void kernel_mul_mm(
         // if no bounds checks on the output are needed, we can directly write to device memory
         device float * C = (device float *) dst +
             (r0 + 32*(sgitg &  1)) + \
-            (uint64_t) (r1 + 16*(sgitg >> 1))*args.ne0 + (uint64_t) im*args.ne1*args.ne0;
+            (r1 + 16*(sgitg >> 1)) * args.ne0 + im*args.ne1*args.ne0;
 
         for (short i = 0; i < 8; i++) {
-            simdgroup_store(mc[i], C + 8*(i%4) + (uint64_t) 8*args.ne0*(i/4), args.ne0, 0, false);
+            simdgroup_store(mc[i], C + 8*(i%4) + 8*args.ne0*(i/4), args.ne0, 0, false);
         }
     } else {
         // block is smaller than 64x32, we should avoid writing data outside of the matrix
@@ -342,7 +343,7 @@ kernel void kernel_mul_mm(
 
         if (sgitg == 0) {
             for (int j = tiitg; j < nr1; j += NR1) {
-                device float  * D  = (device float  *) dst + r0 + (uint64_t) (r1 + j)*args.ne0 + (uint64_t) im*args.ne1*args.ne0;
+                device float  * D  = (device float  *) dst + r0 + (r1 + j)*args.ne0 + im*args.ne1*args.ne0;
                 device float4 * D4 = (device float4 *) D;
 
                 threadgroup float  * C  = temp_str + (j*NR0);
@@ -531,6 +532,85 @@ kernel void kernel_mul_mm_id_zero(
     }
 }
 
+kernel void kernel_mul_mm_id_amax_part_f32(
+        constant ggml_metal_kargs_mul_mm_id_amax & args,
+        device   const char * src1,
+        device         char * dst,
+        threadgroup    char * shmem [[threadgroup(0)]],
+        uint  tgpig[[threadgroup_position_in_grid]],
+        ushort tiitg[[thread_index_in_threadgroup]],
+        ushort tiisg[[thread_index_in_simdgroup]],
+        ushort sgitg[[simdgroup_index_in_threadgroup]],
+        ushort   ntg[[threads_per_threadgroup]]) {
+    const int nrow = args.ne01*args.ne02;
+
+    float lmax = 0.0f;
+
+    for (int ir = tgpig; ir < nrow; ir += N_MM_NPART_AMAX) {
+        const int i01 = ir % args.ne01;
+        const int i02 = ir / args.ne01;
+
+        device const float * row = (device const float *) (src1 + i02*args.nb02 + i01*args.nb01);
+
+        for (int i00 = tiitg; i00 < args.ne00; i00 += ntg) {
+            lmax = max(lmax, fabs(row[i00]));
+        }
+    }
+
+    float amax = simd_max(lmax);
+
+    threadgroup float * shared_amax = (threadgroup float *) shmem;
+
+    if (ntg > N_SIMDWIDTH) {
+        if (sgitg == 0) {
+            shared_amax[tiisg] = 0.0f;
+        }
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+
+        if (tiisg == 0) {
+            shared_amax[sgitg] = amax;
+        }
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+
+        amax = shared_amax[tiisg];
+        amax = simd_max(amax);
+    }
+
+    if (tiitg == 0) {
+        ((device float *) (dst + 8))[tgpig] = amax;
+    }
+}
+
+kernel void kernel_mul_mm_id_amax_f32(
+        device char * dst,
+        ushort tiitg[[thread_index_in_threadgroup]]) {
+    device const float * part = (device const float *) (dst + 8);
+
+    float amax = 0.0f;
+
+    for (int i = tiitg; i < N_MM_NPART_AMAX; i += N_SIMDWIDTH) {
+        amax = max(amax, part[i]);
+    }
+
+    amax = simd_max(amax);
+
+    if (tiitg == 0) {
+        // leave a comfortable margin below the f16 max of 65504
+        float scale = 1.0f;
+
+        // isfinite: src1 already inf/nan is not ours to fix - keep the
+        // scale at 1.0 instead of turning it into a different failure
+        if (isfinite(amax) && amax > 32768.0f) {
+            scale = exp2(ceil(log2(amax)) - 15.0f);
+        }
+
+        device float * d = (device float *) dst;
+
+        d[0] = 1.0f/scale; // exact: scale is a power of two
+        d[1] = scale;
+    }
+}
+
 typedef decltype(kernel_mul_mm_id_map0<1>) kernel_mul_mm_id_map0_t;
 
 template [[host_name("kernel_mul_mm_id_map0_ne20_1" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<1>;
@@ -553,6 +633,7 @@ kernel void kernel_mul_mm_id(
         device const char * htpe,
         device const char * hids,
         device       char * dst,
+        device const char * amax,
         threadgroup  char * shmem [[threadgroup(0)]],
         uint3  tgpig[[threadgroup_position_in_grid]],
         ushort tiitg[[thread_index_in_threadgroup]],
@@ -608,6 +689,10 @@ kernel void kernel_mul_mm_id(
     // skip the upper half of the token tile when the expert did not fill it
     constexpr short NR1H = NR1/2;
 
+    // power-of-two rescaling
+    const float s1_inv   = FC_mul_mm_id_amax ? ((device const float *) amax)[0] : 1.0f;
+    const float s1_scale = FC_mul_mm_id_amax ? ((device const float *) amax)[1] : 1.0f;
+
 #ifndef GGML_METAL_HAS_TENSOR
     S0_8x8 ma[4];
     S1_8x8 mb[2];
@@ -633,7 +718,7 @@ kernel void kernel_mul_mm_id(
     const int id = ids_i32[idb + lr1];
 
     const short i11 = (id % args.ne20) % args.ne11;
-    const int   i12 = (id / args.ne20);
+    const short i12 = (id / args.ne20);
 
     device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1;
 
@@ -713,7 +798,7 @@ kernel void kernel_mul_mm_id(
 
                 const short ib = 4*sx + sy;
 
-                *(sb + 64*ib + 8*ly + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0;
+                *(sb + 64*ib + 8*ly + lx) = loop_k + iy + i < args.ne00 ? (S1) (*((device T1 *) y + i) * (T1) s1_inv) : 0;
             }
         } else {
             const short sx = (tiitg%NL1);
@@ -726,7 +811,7 @@ kernel void kernel_mul_mm_id(
 
             const short ib = 4*sx + sy;
 
-            *(threadgroup S1_2x4 *)(sb + 64*ib + 8*ly) = (S1_2x4)(*((device T1_2x4 *) y));
+            *(threadgroup S1_2x4 *)(sb + 64*ib + 8*ly) = (S1_2x4)((*((device T1_2x4 *) y)) * (T1) s1_inv);
         }
 #else
         // load data and store to threadgroup memory
@@ -774,7 +859,7 @@ kernel void kernel_mul_mm_id(
                 //const short lx = (tiitg/NL1)%8;
                 //const short ly = i;
 
-                *(sb + NK*(8*sy + ly) + 8*sx + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0;
+                *(sb + NK*(8*sy + ly) + 8*sx + lx) = loop_k + iy + i < args.ne00 ? (S1) (*((device T1 *) y + i) * (T1) s1_inv) : 0;
             }
         } else {
             const short sx = (tiitg%NL1);
@@ -785,7 +870,7 @@ kernel void kernel_mul_mm_id(
             //const short lx = (tiitg/NL1)%8;
             //const short ly = i;
 
-            *(threadgroup S1_2x4 *)(sb + NK*(8*sy + ly) + 8*sx) = (S1_2x4)(*((device T1_2x4 *) y));
+            *(threadgroup S1_2x4 *)(sb + NK*(8*sy + ly) + 8*sx) = (S1_2x4)((*((device T1_2x4 *) y)) * (T1) s1_inv);
         }
 #endif
 
@@ -866,9 +951,9 @@ kernel void kernel_mul_mm_id(
         const int id = ids_i32[idb + j];
 
         const short ide = id % args.ne20;
-        const int   idt = id / args.ne20;
+        const short idt = id / args.ne20;
 
-        device float  * D  = (device float  *) dst + r0 + (uint64_t) ide*args.ne0 + (uint64_t) idt*args.ne1*args.ne0;
+        device float  * D  = (device float  *) dst + r0 + ide*args.ne0 + idt*args.ne1*args.ne0;
         device float4 * D4 = (device float4 *) D;
 
         threadgroup float  * C  = (threadgroup float  *) shmem + j*NR0;
@@ -876,12 +961,12 @@ kernel void kernel_mul_mm_id(
 
         int i = tiisg;
         for (; i < nr0/4; i += 32) {
-            *(D4 + i) = *(C4 + i);
+            *(D4 + i) = *(C4 + i) * s1_scale;
         }
 
         i = (4*(nr0/4)) + tiisg;
         for (; i < nr0; i += 32) {
-            *(D + i) = *(C + i);
+            *(D + i) = *(C + i) * s1_scale;
         }
     }
 }

ggml/src/ggml-metal/kernels/unary.metal

@@ -3,24 +3,6 @@
 constant short FC_unary_op [[function_constant(FC_UNARY + 0)]];
 constant bool  FC_unary_cnt[[function_constant(FC_UNARY + 1)]];
 
-template <typename T, typename U>
-inline constexpr T ggml_metal_unary_cast(U x) {
-    return (T) x;
-}
-
-#if defined(GGML_METAL_HAS_BF16)
-// MSL has no C-style cast from float vectors to bfloat vectors
-template <>
-inline constexpr bfloat4 ggml_metal_unary_cast<bfloat4, float4>(float4 x) {
-    return bfloat4(bfloat(x[0]), bfloat(x[1]), bfloat(x[2]), bfloat(x[3]));
-}
-
-template <>
-inline constexpr bfloat4 ggml_metal_unary_cast<bfloat4, float>(float x) {
-    return bfloat4(bfloat(x), bfloat(x), bfloat(x), bfloat(x));
-}
-#endif
-
 template <typename T0, typename T, typename TC>
 kernel void kernel_unary_impl(
         constant ggml_metal_kargs_unary & args,
@@ -66,79 +48,79 @@ kernel void kernel_unary_impl(
         const TC x = (TC) src0_ptr[i0];
 
         if (FC_OP == OP_UNARY_NUM_SCALE) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(args.scale * x + args.bias);
+            dst_ptr[i0] = (T) (args.scale * x + args.bias);
         }
 
         if (FC_OP == OP_UNARY_NUM_FILL) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(args.val);
+            dst_ptr[i0] = (T) args.val;
         }
 
         if (FC_OP == OP_UNARY_NUM_CLAMP) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(clamp(x, args.min, args.max));
+            dst_ptr[i0] = (T) clamp(x, args.min, args.max);
         }
 
         if (FC_OP == OP_UNARY_NUM_SQR) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(x * x);
+            dst_ptr[i0] = (T) (x * x);
         }
 
         if (FC_OP == OP_UNARY_NUM_SQRT) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(sqrt(x));
+            dst_ptr[i0] = (T) sqrt(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_SIN) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(sin(x));
+            dst_ptr[i0] = (T) sin(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_COS) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(cos(x));
+            dst_ptr[i0] = (T) cos(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_LOG) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(log(x));
+            dst_ptr[i0] = (T) log(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_LEAKY_RELU) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(TC(x > 0)*x + TC(x <= 0)*(x * args.slope));
+            dst_ptr[i0] = (T) (TC(x > 0)*x + TC(x <= 0)*(x * args.slope));
         }
 
         if (FC_OP == OP_UNARY_NUM_TANH) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(precise::tanh(x));
+            dst_ptr[i0] = (T) precise::tanh(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_RELU) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(fmax(0, x));
+            dst_ptr[i0] = (T) fmax(0, x);
         }
 
         if (FC_OP == OP_UNARY_NUM_SIGMOID) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(1 / (1 + exp(-x)));
+            dst_ptr[i0] = (T) (1 / (1 + exp(-x)));
         }
 
         if (FC_OP == OP_UNARY_NUM_GELU) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(0.5*x*(1 + precise::tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x))));
+            dst_ptr[i0] = (T) (0.5*x*(1 + precise::tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x))));
         }
 
         if (FC_OP == OP_UNARY_NUM_GELU_ERF) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(0.5*x*(1 + erf_approx(SQRT_2_INV*x)));
+            dst_ptr[i0] = (T) (0.5*x*(1 + erf_approx(SQRT_2_INV*x)));
         }
 
         if (FC_OP == OP_UNARY_NUM_GELU_QUICK) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(x * (1/(1 + exp(GELU_QUICK_COEF*x))));
+            dst_ptr[i0] = (T) (x * (1/(1 + exp(GELU_QUICK_COEF*x))));
         }
 
         if (FC_OP == OP_UNARY_NUM_SILU) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(x / (1 + exp(-x)));
+            dst_ptr[i0] = (T) (x / (1 + exp(-x)));
         }
 
         if (FC_OP == OP_UNARY_NUM_ELU) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(elu_approx(x));
+            dst_ptr[i0] = (T) elu_approx(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_NEG) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(-x);
+            dst_ptr[i0] = (T) -x;
         }
 
         if (FC_OP == OP_UNARY_NUM_ABS) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(fabs(x));
+            dst_ptr[i0] = (T) fabs(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_SGN) {
@@ -150,40 +132,40 @@ kernel void kernel_unary_impl(
         }
 
         if (FC_OP == OP_UNARY_NUM_HARDSWISH) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(x * fmax(0, fmin(1, x/6 + 0.5)));
+            dst_ptr[i0] = (T) (x * fmax(0, fmin(1, x/6 + 0.5)));
         }
 
         if (FC_OP == OP_UNARY_NUM_HARDSIGMOID) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(fmax(0, fmin(1, x/6 + 0.5)));
+            dst_ptr[i0] = (T) fmax(0, fmin(1, x/6 + 0.5));
         }
 
         if (FC_OP == OP_UNARY_NUM_EXP) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(exp(x));
+            dst_ptr[i0] = (T) exp(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_SOFTPLUS) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(select(log(1 + exp(x)), x, x > 20));
+            dst_ptr[i0] = (T) select(log(1 + exp(x)), x, x > 20);
         }
 
         if (FC_OP == OP_UNARY_NUM_EXPM1) {
             // TODO: precise implementation
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(exp(x) - 1);
+            dst_ptr[i0] = (T) (exp(x) - 1);
         }
 
         if (FC_OP == OP_UNARY_NUM_FLOOR) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(floor(x));
+            dst_ptr[i0] = (T) floor(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_CEIL) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(ceil(x));
+            dst_ptr[i0] = (T) ceil(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_ROUND) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(round(x));
+            dst_ptr[i0] = (T) round(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_TRUNC) {
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(trunc(x));
+            dst_ptr[i0] = (T) trunc(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_XIELU) {
@@ -192,7 +174,7 @@ kernel void kernel_unary_impl(
             const TC clamped = fmin(xi, TC(args.val));
             const TC y_pos   = TC(args.scale) * xi * xi + TC(args.bias) * xi;
             const TC y_neg   = (exp(clamped) - TC(1.0f) - xi) * TC(args.slope) + TC(args.bias) * xi;
-            dst_ptr[i0] = ggml_metal_unary_cast<T>(gate * y_pos + (TC(1.0f) - gate) * y_neg);
+            dst_ptr[i0] = (T) (gate * y_pos + (TC(1.0f) - gate) * y_neg);
         }
     }
 
@@ -206,10 +188,6 @@ template [[host_name("kernel_unary_f32_f32")]]   kernel kernel_unary_t kernel_un
 template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_unary_impl<float4, float4, float4>;
 template [[host_name("kernel_unary_f16_f16")]]   kernel kernel_unary_t kernel_unary_impl<half,   half,   float>;
 template [[host_name("kernel_unary_f16_f16_4")]] kernel kernel_unary_t kernel_unary_impl<half4,  half4,  float4>;
-#if defined(GGML_METAL_HAS_BF16)
-template [[host_name("kernel_unary_bf16_bf16")]]   kernel kernel_unary_t kernel_unary_impl<bfloat,  bfloat,  float>;
-template [[host_name("kernel_unary_bf16_bf16_4")]] kernel kernel_unary_t kernel_unary_impl<bfloat4, bfloat4, float4>;
-#endif
 
 kernel void kernel_silu_back_f32(
         constant ggml_metal_kargs_silu_back & args,

ggml/src/ggml-opencl/CMakeLists.txt

@@ -66,8 +66,6 @@ set(GGML_OPENCL_KERNELS
     moe_add_id_glu
     argmax
     argsort
-    top_k
-    lightning_indexer
     tri
     fill
     clamp
@@ -101,18 +99,6 @@ set(GGML_OPENCL_KERNELS
     mul_mv_q4_0_f32_1d_16x_flat
     mul_mv_q4_1_f32
     mul_mv_q4_1_f32_flat
-    gemm_noshuffle_q2_k_q8_1_dp4a
-    gemv_noshuffle_q2_k_f32
-    gemm_noshuffle_q2_k_f32
-    gemm_noshuffle_q3_k_f32
-    gemm_noshuffle_q3_k_q8_1_dp4a
-    gemv_noshuffle_q3_k_f32
-    mul_mv_q2_k_f32
-    mul_mv_q3_k_f32
-    mul_mv_q3_k_f32_flat
-    mul_mm_q3_k_f32_l4_lm
-    mul_mv_q2_k_f32_flat
-    mul_mm_q2_k_f32_l4_lm
     mul_mv_q4_k_f32
     mul_mv_q4_k_f32_flat
     mul_mv_q5_0_f32
@@ -134,7 +120,6 @@ set(GGML_OPENCL_KERNELS
     mul_mv_id_q8_0_f32_flat
     mul_mv_id_mxfp4_f32
     mul_mv_id_mxfp4_f32_flat
-    mul_mat_id_utils
     gemm_moe_q4_0_f32_ns
     gemm_moe_q4_0_q8_1_dp4a
     gemv_moe_q4_0_f32_ns

ggml/src/ggml-opencl/kernels/cvt.cl

@@ -2661,268 +2661,3 @@ kernel void kernel_moe_expand_scale_q5_K(
         dst_min[((long)e*ne01 + row)*nblk32 + sub] = (half)(dm * (float)mn);
     }
 }
-
-struct block_q2_K {
-    uint8_t scales[QK_K/16]; // low nibble scale, high nibble min, one per 16 weights
-    uint8_t qs[QK_K/4];      // quants, 2 bits each
-    half d;                  // super-block scale for the quantized scales
-    half dmin;               // super-block scale for the quantized mins
-};
-
-kernel void kernel_convert_block_q2_K(
-    global struct block_q2_K * src0,
-    global uchar * dst_qs,
-    global uchar * dst_sc,
-    global half * dst_dm,
-    ulong n_blk
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blk) {
-        return;
-    }
-    global struct block_q2_K * b = src0 + i;
-    for (int j = 0; j < 64; ++j) {
-        dst_qs[64*i + j] = b->qs[j];
-    }
-    for (int j = 0; j < 16; ++j) {
-        dst_sc[16*i + j] = b->scales[j];
-    }
-    dst_dm[2*i + 0] = b->d;
-    dst_dm[2*i + 1] = b->dmin;
-}
-
-kernel void kernel_restore_block_q2_K(
-    global uchar * src_qs,
-    global uchar * src_sc,
-    global half * src_dm,
-    global struct block_q2_K * dst,
-    ulong n_blk
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blk) {
-        return;
-    }
-    global struct block_q2_K * b = dst + i;
-    for (int j = 0; j < 64; ++j) {
-        b->qs[j] = src_qs[64*i + j];
-    }
-    for (int j = 0; j < 16; ++j) {
-        b->scales[j] = src_sc[16*i + j];
-    }
-    b->d = src_dm[2*i + 0];
-    b->dmin = src_dm[2*i + 1];
-}
-
-kernel void kernel_convert_block_q2_k_noshuffle(
-    global struct block_q2_K * src0,
-    global uchar * dst_qs,      // QK_K/4 per block
-    global uchar * dst_sc,      // QK_K/16 per block
-    global half  * dst_dm,      // 2 per block (d, dmin)
-    ulong          n_blk
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blk) {
-        return;
-    }
-    global struct block_q2_K * b = (global struct block_q2_K *) src0 + i;
-
-    dst_dm[2*i + 0] = b->d;
-    dst_dm[2*i + 1] = b->dmin;
-
-    for (int j = 0; j < QK_K/16; ++j) { dst_sc[(QK_K/16) * i + j] = b->scales[j]; }
-
-    for (int g = 0; g < QK_K/4; ++g) {
-        uint pk = 0;
-        for (int t = 0; t < 4; ++t) {
-            const int e  = 4*g + t;
-            const int sb = e >> 5;
-            const int ee = e & 31;
-            pk |= (uint)(((b->qs[32*(sb >> 2) + ee] >> (2*(sb & 3))) & 3) << (2*t));
-        }
-        dst_qs[(QK_K/4) * i + g] = (uchar)pk;
-    }
-}
-
-kernel void kernel_restore_block_q2_k_noshuffle(
-    global uchar * src_qs,
-    global uchar * src_sc,
-    global half  * src_dm,
-    global struct block_q2_K * dst,
-    ulong          n_blk
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blk) {
-        return;
-    }
-    global struct block_q2_K * b = (global struct block_q2_K *) dst + i;
-
-    b->d    = src_dm[2*i + 0];
-    b->dmin = src_dm[2*i + 1];
-
-    for (int j = 0; j < QK_K/16; ++j) { b->scales[j] = src_sc[(QK_K/16) * i + j]; }
-
-    for (int n2 = 0; n2 < 2; ++n2) {
-        for (int ee = 0; ee < 32; ++ee) {
-            uint v = 0;
-            for (int j = 0; j < 4; ++j) {
-                const int sb = 4*n2 + j;
-                const int e  = 32*sb + ee;
-                v |= ((uint)(src_qs[(QK_K/4) * i + (e >> 2)] >> (2*(e & 3))) & 3u) << (2*j);
-            }
-            b->qs[32*n2 + ee] = (uchar)v;
-        }
-    }
-}
-
-struct block_q3_K {
-    uchar hmask[QK_K/8];
-    uchar qs[QK_K/4];
-    uchar scales[12];
-    half d;
-};
-
-kernel void kernel_convert_block_q3_K(
-    global struct block_q3_K * src0,
-    global uchar * dst_qs,
-    global uchar * dst_hm,
-    global uchar * dst_sc,
-    global half * dst_d,
-    ulong n_blk
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blk) {
-        return;
-    }
-    global struct block_q3_K * b = src0 + i;
-    for (int j = 0; j < 64; ++j) {
-        dst_qs[64*i + j] = b->qs[j];
-    }
-    for (int j = 0; j < 32; ++j) {
-        dst_hm[32*i + j] = b->hmask[j];
-    }
-    for (int j = 0; j < 12; ++j) {
-        dst_sc[12*i + j] = b->scales[j];
-    }
-    dst_d[i] = b->d;
-}
-
-kernel void kernel_restore_block_q3_K(
-    global uchar * src_qs,
-    global uchar * src_hm,
-    global uchar * src_sc,
-    global half * src_d,
-    global struct block_q3_K * dst,
-    ulong n_blk
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blk) {
-        return;
-    }
-    global struct block_q3_K * b = dst + i;
-    for (int j = 0; j < 64; ++j) {
-        b->qs[j] = src_qs[64*i + j];
-    }
-    for (int j = 0; j < 32; ++j) {
-        b->hmask[j] = src_hm[32*i + j];
-    }
-    for (int j = 0; j < 12; ++j) {
-        b->scales[j] = src_sc[12*i + j];
-    }
-    b->d = src_d[i];
-}
-
-kernel void kernel_convert_block_q3_k_noshuffle(
-    global struct block_q3_K * src0,
-    global uchar * dst_qs,      // QK_K/4 per block
-    global uchar * dst_hm,      // QK_K/8 per block
-    global uint  * dst_sc,      // 3 per block
-    global half  * dst_d,       // 1
-    ulong          n_blk
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blk) {
-        return;
-    }
-    global struct block_q3_K * b = (global struct block_q3_K *) src0 + i;
-
-    dst_d[i] = b->d;
-
-    for (int w = 0; w < 3; ++w) {
-        global const uchar * p = b->scales + 4*w;
-        dst_sc[3*i + w] = (uint)p[0] | ((uint)p[1] << 8) | ((uint)p[2] << 16) | ((uint)p[3] << 24);
-    }
-
-    for (int g = 0; g < QK_K/4; ++g) {
-        uint pk = 0;
-        for (int t = 0; t < 4; ++t) {
-            const int e  = 4*g + t;
-            const int sb = e >> 5;
-            const int ee = e & 31;
-            pk |= (uint)(((b->qs[32*(sb >> 2) + ee] >> (2*(sb & 3))) & 3) << (2*t));
-        }
-        dst_qs[(QK_K/4) * i + g] = (uchar)pk;
-    }
-
-    // two groups per hmask byte, so this loop owns the byte and cannot race
-    for (int t = 0; t < QK_K/8; ++t) {
-        uint v = 0;
-        for (int s = 0; s < 2; ++s) {
-            const int g = 2*t + s;
-            for (int u = 0; u < 4; ++u) {
-                const int e  = 4*g + u;
-                const int sb = e >> 5;
-                const int ee = e & 31;
-                v |= (uint)(((b->hmask[ee] >> sb) & 1) << (4*s + u));
-            }
-        }
-        dst_hm[(QK_K/8) * i + t] = (uchar)v;
-    }
-}
-
-kernel void kernel_restore_block_q3_k_noshuffle(
-    global uchar * src_qs,
-    global uchar * src_hm,
-    global uint  * src_sc,
-    global half  * src_d,
-    global struct block_q3_K * dst,
-    ulong          n_blk
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blk) {
-        return;
-    }
-    global struct block_q3_K * b = (global struct block_q3_K *) dst + i;
-
-    b->d = src_d[i];
-
-    for (int w = 0; w < 3; ++w) {
-        const uint x = src_sc[3*i + w];
-        global uchar * p = b->scales + 4*w;
-        p[0] = (uchar)( x        & 0xFF);
-        p[1] = (uchar)((x >>  8) & 0xFF);
-        p[2] = (uchar)((x >> 16) & 0xFF);
-        p[3] = (uchar)((x >> 24) & 0xFF);
-    }
-
-    for (int n2 = 0; n2 < 2; ++n2) {
-        for (int ee = 0; ee < 32; ++ee) {
-            uint v = 0;
-            for (int j = 0; j < 4; ++j) {
-                const int sb = 4*n2 + j;
-                const int e  = 32*sb + ee;
-                v |= ((uint)(src_qs[(QK_K/4) * i + (e >> 2)] >> (2*(e & 3))) & 3u) << (2*j);
-            }
-            b->qs[32*n2 + ee] = (uchar)v;
-        }
-    }
-
-    for (int ee = 0; ee < 32; ++ee) {
-        uint v = 0;
-        for (int sb = 0; sb < 8; ++sb) {
-            const int e = 32*sb + ee;
-            const int g = e >> 2;
-            v |= ((uint)(src_hm[(QK_K/8) * i + (g >> 1)] >> (4*(g & 1) + (e & 3))) & 1u) << sb;
-        }
-        b->hmask[ee] = (uchar)v;
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_f32.cl

@@ -1,83 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_qcom_reqd_sub_group_size
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-__attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-kernel void kernel_gemm_noshuffle_q2_k_f32(
-    global const uchar * src0_qs,
-    global const uchar * src0_sc,
-    global const half  * src0_dm,
-    read_only image1d_buffer_t src1,
-    global float * dst,
-    ulong offsetd,
-    int m,
-    int n,
-    int k,
-    int n_no_padding
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-    const int gy = get_global_id(0);
-    const int row = get_global_id(1) * 4;
-    const int n_4 = n / 4;
-    if (row >= m) {
-        return;
-    }
-
-    half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
-    for (int i = 0; i < k; i += 16) {
-        const half8 dm = vload8(0, src0_dm + 2 * (row + (i / 256) * m));
-        const uchar4 sc = vload4(0, src0_sc + row + (i / 16) * m);
-        const half4 scale = convert_half4(convert_float4(dm.s0246) * convert_float4(sc & (uchar4)(15)));
-        const half4 mval  = convert_half4(convert_float4(dm.s1357) * convert_float4(sc >> (uchar4)(4)));
-
-        for (int l = 0; l < 16; l += 4) {
-            const int ki = i + l;
-            const uchar4 bits = vload4(0, src0_qs + row + (ki / 4) * m);
-            #pragma unroll
-            for (int j = 0; j < 4; ++j) {
-                half8 b;
-                b.s0123 = read_imageh(src1, gy * 2     + (ki + j) * n_4);
-                b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
-                const half4 w = convert_half4((bits >> (uchar4)(2 * j)) & (uchar4)(3)) * scale - mval;
-                c0 += b * w.s0;
-                c1 += b * w.s1;
-                c2 += b * w.s2;
-                c3 += b * w.s3;
-            }
-        }
-    }
-
-    int idx = gy * 8 * m + row;
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_q8_1_dp4a.cl

@@ -1,146 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#pragma OPENCL EXTENSION cl_khr_subgroups : enable
-#ifdef cl_khr_integer_dot_product
-#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
-#endif
-
-#define QK_K 256
-
-#ifndef TILESIZE_N
-#define TILESIZE_N 32
-#endif
-
-inline uint q2_k_pack(uint pk) {
-    return ( (pk      ) & 3u)
-         | (((pk >> 2) & 3u) <<  8)
-         | (((pk >> 4) & 3u) << 16)
-         | (((pk >> 6) & 3u) << 24);
-}
-
-// Pass staged operands by value; local dp4a operands miscompile on X2.
-#define KQ_STAGE(p) vload4(0, (p))
-
-inline int dot4_q8a_v(uint4 qw, uint4 a) {
-    int r = 0;
-    r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r);
-    return r;
-}
-
-__attribute__((qcom_wave_pair_mode(1)))
-kernel void kernel_gemm_noshuffle_q2_k_q8_1_dp4a(
-        __global const uchar  * src0_qs,
-        __global const uchar  * src0_sc,
-        __global const half   * src0_dm,   // half pairs: d, dmin
-        __global const uint   * src1_qa,   // q8_1 activations int8 (as uint, 4/elem) [N, K]
-        __global const half   * src1_da,   // q8_1 per-block scale [N, K/32]
-        __global       float  * dst,
-        ulong  offsetd,
-        int    m,
-        int    n_no_padding,
-        int    k
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-
-    const uint lid = get_local_id(0);
-    const uint block_id_m = get_global_id(1);
-    const uint block_id_n = get_global_id(2);
-
-    const uint row      = block_id_m * 64 + lid;
-    const uint col_base = block_id_n * TILESIZE_N;
-    const bool row_valid = row < (uint)m;
-    const uint rrow     = row_valid ? row : 0;
-
-    const uint k_u = (uint)k >> 2;
-    const uint k_b = (uint)k >> 5;
-
-    __local uint4 sh_qa4[TILESIZE_N][2];
-    __local float sh_s [TILESIZE_N][2];   // per-16 activation sums, in qa units
-    __local half  sh_d [TILESIZE_N];
-
-#define NGROUPS (TILESIZE_N / 4)
-    float4 acc[NGROUPS];
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
-
-    for (uint step = 0; step < (uint)k; step += 32) {
-        const uint sub = step >> 5;
-        const uint ib  = sub >> 3;
-
-        const half2 dm  = vload2(rrow + ib * (uint)m, src0_dm);
-        const float dv  = (float)dm.s0;
-        const float dmv = (float)dm.s1;
-
-        const uint  scb = rrow + (sub << 1) * (uint)m;
-        const uint  sc0 = (uint)src0_sc[scb + 0u * (uint)m];
-        const uint  sc1 = (uint)src0_sc[scb + 1u * (uint)m];
-        const float dl0 = dv  * (float)( sc0       & 0xFu);
-        const float ml0 = dmv * (float)( sc0 >> 4);
-        const float dl1 = dv  * (float)( sc1       & 0xFu);
-        const float ml1 = dmv * (float)( sc1 >> 4);
-
-        const uint qsb = rrow + (step >> 2) * (uint)m;
-        uint4 qlo, qhi;
-        qlo.s0 = q2_k_pack((uint)src0_qs[qsb + 0u * (uint)m]);
-        qlo.s1 = q2_k_pack((uint)src0_qs[qsb + 1u * (uint)m]);
-        qlo.s2 = q2_k_pack((uint)src0_qs[qsb + 2u * (uint)m]);
-        qlo.s3 = q2_k_pack((uint)src0_qs[qsb + 3u * (uint)m]);
-        qhi.s0 = q2_k_pack((uint)src0_qs[qsb + 4u * (uint)m]);
-        qhi.s1 = q2_k_pack((uint)src0_qs[qsb + 5u * (uint)m]);
-        qhi.s2 = q2_k_pack((uint)src0_qs[qsb + 6u * (uint)m]);
-        qhi.s3 = q2_k_pack((uint)src0_qs[qsb + 7u * (uint)m]);
-
-        for (uint idx = lid; idx < TILESIZE_N * 2u; idx += 64u) {
-            const uint t  = idx >> 1;
-            const uint h  = idx & 1u;
-            const uint c  = col_base + t;
-            const bool ok = c < (uint)n_no_padding;
-            const uint4 w = ok ? KQ_STAGE(src1_qa + c * k_u + (step >> 2) + (h << 2))
-                               : (uint4)(0u);
-            sh_qa4[t][h] = w;
-            int s = 0;
-            s = dot_acc_sat_4x8packed_ss_int(w.x, 0x01010101u, s);
-            s = dot_acc_sat_4x8packed_ss_int(w.y, 0x01010101u, s);
-            s = dot_acc_sat_4x8packed_ss_int(w.z, 0x01010101u, s);
-            s = dot_acc_sat_4x8packed_ss_int(w.w, 0x01010101u, s);
-            sh_s[t][h] = (float)s;
-            if (h == 0u) {
-                sh_d[t] = ok ? src1_da[c * k_b + sub] : (half)0;
-            }
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
-#define Q2_K_COL(b) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) - ml0 * sh_s[b][0] + \
-                    dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1])) - ml1 * sh_s[b][1])
-        #pragma unroll
-        for (int g = 0; g < NGROUPS; ++g) {
-            const int b = g * 4;
-            float4 rf;
-            rf.s0 = Q2_K_COL(b+0);  rf.s1 = Q2_K_COL(b+1);
-            rf.s2 = Q2_K_COL(b+2);  rf.s3 = Q2_K_COL(b+3);
-            acc[g] += LD4(sh_d, b) * rf;
-        }
-#undef Q2_K_COL
-#undef LD4
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    if (!row_valid) {
-        return;
-    }
-
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) {
-        const uint b = (uint)(g * 4);
-        const float4 a = acc[g];
-        const uint c0 = col_base + b;
-        if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
-        if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
-        if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
-        if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
-    }
-#undef NGROUPS
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_f32.cl

@@ -1,92 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_qcom_reqd_sub_group_size
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-__attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-kernel void kernel_gemm_noshuffle_q3_k_f32(
-    global const uchar * src0_qs,
-    global const uchar * src0_hm,
-    global const uint  * src0_sc,
-    global const half  * src0_d,
-    read_only image1d_buffer_t src1,
-    global float * dst,
-    ulong offsetd,
-    int m,
-    int n,
-    int k,
-    int n_no_padding
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-    const int gy = get_global_id(0);
-    const int row = get_global_id(1) * 4;
-    const int n_4 = n / 4;
-    if (row >= m) {
-        return;
-    }
-
-    half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
-    for (int i = 0; i < k; i += 16) {
-        const int is = (i / 16) & 15;
-        const int word = is >> 2;
-        const int shift = 8 * (is & 3);
-        const int sc_base = row + 3 * (i / 256) * m;
-        const uint4 low = vload4(0, src0_sc + sc_base + (word & 1) * m) >> (uint4)(4 * (word >> 1));
-        const uint4 high = vload4(0, src0_sc + sc_base + 2 * m) >> (uint4)(2 * word);
-        const int4 sc = convert_int4(((low >> (uint4)(shift)) & (uint4)(15)) | (((high >> (uint4)(shift)) & (uint4)(3)) << (uint4)(4))) - (int4)(32);
-        const half4 d = vload4(0, src0_d + row + (i / 256) * m);
-        const half4 scale = convert_half4(convert_float4(d) * convert_float4(sc));
-
-        for (int l = 0; l < 16; l += 4) {
-            const int ki = i + l;
-            const uchar4 bits = vload4(0, src0_qs + row + (ki / 4) * m);
-            const uchar4 hbits = vload4(0, src0_hm + row + (ki / 8) * m);
-            #pragma unroll
-            for (int j = 0; j < 4; ++j) {
-                half8 b;
-                b.s0123 = read_imageh(src1, gy * 2     + (ki + j) * n_4);
-                b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
-                const int4 low = convert_int4((bits >> (uchar4)(2 * j)) & (uchar4)(3));
-                const int4 high = convert_int4((hbits >> (uchar4)((ki & 7) + j)) & (uchar4)(1));
-                const half4 w = convert_half4(low - (int4)(4) + (int4)(4) * high) * scale;
-                c0 += b * w.s0;
-                c1 += b * w.s1;
-                c2 += b * w.s2;
-                c3 += b * w.s3;
-            }
-        }
-    }
-
-    int idx = gy * 8 * m + row;
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_q8_1_dp4a.cl

@@ -1,160 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#pragma OPENCL EXTENSION cl_khr_subgroups : enable
-#ifdef cl_khr_integer_dot_product
-#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
-#endif
-
-#define QK_K 256
-
-#ifndef TILESIZE_N
-#define TILESIZE_N 32
-#endif
-
-inline int q3_k_scale(uint sc0, uint sc1, uint sc2, uint is) {
-    const uint w  = is >> 2;
-    const uint sh = 8u * (is & 3u);
-    const uint tb = (sc2 >> sh) & 0xFFu;
-    uint v;
-    if      (w == 0u) { v = ( (sc0 >> sh)       & 0xFu) | (((tb >> 0) & 3u) << 4); }
-    else if (w == 1u) { v = ( (sc1 >> sh)       & 0xFu) | (((tb >> 2) & 3u) << 4); }
-    else if (w == 2u) { v = (((sc0 >> sh) >> 4) & 0xFu) | (((tb >> 4) & 3u) << 4); }
-    else              { v = (((sc1 >> sh) >> 4) & 0xFu) | (((tb >> 6) & 3u) << 4); }
-    return (int)v - 32;
-}
-
-inline uint q3_k_pack(uint pk, uint hb) {
-    int v0 = (int)((pk      ) & 3u) - 4 + 4*(int)((hb     ) & 1u);
-    int v1 = (int)((pk >> 2) & 3u) - 4 + 4*(int)((hb >> 1) & 1u);
-    int v2 = (int)((pk >> 4) & 3u) - 4 + 4*(int)((hb >> 2) & 1u);
-    int v3 = (int)((pk >> 6) & 3u) - 4 + 4*(int)((hb >> 3) & 1u);
-    return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) <<  8)
-         | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24);
-}
-
-// Pass staged operands by value; local dp4a operands miscompile on X2.
-#define KQ_STAGE(p) vload4(0, (p))
-
-inline int dot4_q8a_v(uint4 qw, uint4 a) {
-    int r = 0;
-    r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r);
-    return r;
-}
-
-__attribute__((qcom_wave_pair_mode(1)))
-kernel void kernel_gemm_noshuffle_q3_k_q8_1_dp4a(
-        __global const uchar  * src0_qs,
-        __global const uchar  * src0_hm,
-        __global const uint   * src0_sc,
-        __global const half   * src0_d,
-        __global const uint   * src1_qa,   // q8_1 activations int8 (as uint, 4/elem) [N, K]
-        __global const half   * src1_da,   // q8_1 per-block scale [N, K/32]
-        __global       float  * dst,
-        ulong  offsetd,
-        int    m,                          // output features (rows)
-        int    n_no_padding,               // tokens (cols)
-        int    k                           // K (== ne00)
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-
-    const uint lid = get_local_id(0);
-    const uint block_id_m = get_global_id(1);
-    const uint block_id_n = get_global_id(2);
-
-    const uint row      = block_id_m * 64 + lid;
-    const uint col_base = block_id_n * TILESIZE_N;
-    const bool row_valid = row < (uint)m;
-    const uint rrow     = row_valid ? row : 0;
-
-    const uint k_u = (uint)k >> 2;
-    const uint k_b = (uint)k >> 5;
-
-    __local uint4 sh_qa4[TILESIZE_N][2];
-    __local half sh_d[TILESIZE_N];
-
-#define NGROUPS (TILESIZE_N / 4)
-    float4 acc[NGROUPS];
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
-
-    for (uint step = 0; step < (uint)k; step += 32) {
-        const uint sub = step >> 5;
-        const uint ib  = sub >> 3;                // super-block along K
-        const uint sb  = sub & 7u;
-
-        const uint scb = rrow + (3u * ib) * (uint)m;
-        const uint sc0 = src0_sc[scb + 0u * (uint)m];
-        const uint sc1 = src0_sc[scb + 1u * (uint)m];
-        const uint sc2 = src0_sc[scb + 2u * (uint)m];
-
-        const float d_w = (float)src0_d[rrow + ib * (uint)m];
-        const float dl0 = d_w * (float)q3_k_scale(sc0, sc1, sc2, 2u*sb + 0u);
-        const float dl1 = d_w * (float)q3_k_scale(sc0, sc1, sc2, 2u*sb + 1u);
-
-        const uint qsb = rrow + (step >> 2) * (uint)m;
-        const uint hmb = rrow + (step >> 3) * (uint)m;
-
-        uint4 qlo, qhi;
-        {
-            const uint h0 = (uint)src0_hm[hmb + 0u * (uint)m];
-            const uint h1 = (uint)src0_hm[hmb + 1u * (uint)m];
-            const uint h2 = (uint)src0_hm[hmb + 2u * (uint)m];
-            const uint h3 = (uint)src0_hm[hmb + 3u * (uint)m];
-            qlo.s0 = q3_k_pack((uint)src0_qs[qsb + 0u * (uint)m],  h0       & 0xFu);
-            qlo.s1 = q3_k_pack((uint)src0_qs[qsb + 1u * (uint)m], (h0 >> 4) & 0xFu);
-            qlo.s2 = q3_k_pack((uint)src0_qs[qsb + 2u * (uint)m],  h1       & 0xFu);
-            qlo.s3 = q3_k_pack((uint)src0_qs[qsb + 3u * (uint)m], (h1 >> 4) & 0xFu);
-            qhi.s0 = q3_k_pack((uint)src0_qs[qsb + 4u * (uint)m],  h2       & 0xFu);
-            qhi.s1 = q3_k_pack((uint)src0_qs[qsb + 5u * (uint)m], (h2 >> 4) & 0xFu);
-            qhi.s2 = q3_k_pack((uint)src0_qs[qsb + 6u * (uint)m],  h3       & 0xFu);
-            qhi.s3 = q3_k_pack((uint)src0_qs[qsb + 7u * (uint)m], (h3 >> 4) & 0xFu);
-        }
-
-        for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
-            const uint t = idx >> 1;
-            const uint v = idx & 1;
-            const uint c = col_base + t;
-            sh_qa4[t][v] = (c < (uint)n_no_padding)
-                         ? KQ_STAGE(src1_qa + c * k_u + (step >> 2) + (v << 2))
-                         : (uint4)(0u);
-        }
-        if (lid < TILESIZE_N) {
-            const uint c = col_base + lid;
-            sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
-        #pragma unroll
-        for (int g = 0; g < NGROUPS; ++g) {
-            const int b = g * 4;
-            float4 rf;
-#define Q3_K_COL(T) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[T][0]))  \
-                  + dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[T][1])))
-            rf.s0 = Q3_K_COL(b+0);  rf.s1 = Q3_K_COL(b+1);
-            rf.s2 = Q3_K_COL(b+2);  rf.s3 = Q3_K_COL(b+3);
-#undef Q3_K_COL
-            acc[g] += LD4(sh_d, b) * rf;
-        }
-#undef LD4
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    if (!row_valid) {
-        return;
-    }
-
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) {
-        const uint b = (uint)(g * 4);
-        const float4 a = acc[g];
-        const uint c0 = col_base + b;
-        if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
-        if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
-        if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
-        if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
-    }
-#undef NGROUPS
-}

ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_k_f32.cl

@@ -1,231 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#define QK_K 256
-
-#ifndef Q2_K_MV_NSG
-#define Q2_K_MV_NSG 8
-#endif
-
-#ifndef Q2_K_MV_R
-#define Q2_K_MV_R 4
-#endif
-
-inline float4 q2_k_vals(uint pk) {
-    return (float4)((float)( pk       & 3u), (float)((pk >> 2) & 3u),
-                    (float)((pk >> 4) & 3u), (float)((pk >> 6) & 3u));
-}
-
-kernel void kernel_gemv_noshuffle_q2_k_f32(
-        global const uchar * src0_qs,
-        global const uchar * src0_sc,
-        global const half  * src0_dm,
-        global const float * src1,
-        ulong offset1,
-        global float * dst,
-        ulong offsetd,
-        int ne00,      // K
-        int ne01,      // M
-        int ne10,      // activation row stride, == K
-        int ne0        // dst row stride
-) {
-    src1 = (global const float *)((global const char *)src1 + offset1);
-    dst  = (global float       *)((global char       *)dst  + offsetd);
-
-    const uint m   = (uint)ne01;
-    const uint K   = (uint)ne00;
-    const uint nsb = K / QK_K;
-
-    const uint lid = get_local_id(0);
-    const uint sgi = get_local_id(1);
-    const uint col = get_group_id(1);
-
-    global const float * y = src1 + (ulong)col * (uint)ne10;
-
-    const uint mr  = m / Q2_K_MV_R;                  // row groups
-    const uint j   = get_group_id(0) * 64u + lid;   // row group this lane owns
-    const uint row = j * Q2_K_MV_R;
-
-#if Q2_K_MV_R == 4
-    float sumf0 = 0.f;
-    float sumf1 = 0.f;
-    float sumf2 = 0.f;
-    float sumf3 = 0.f;
-
-    if (j < mr) {
-        global const uint * qsu = (global const uint *)src0_qs;
-        global const uint * scu = (global const uint *)src0_sc;
-
-        for (uint ib = sgi; ib < nsb; ib += Q2_K_MV_NSG) {
-            const half4 dma = vload4(2u*(j + ib*mr) + 0u, src0_dm);  // d0, dmin0, d1, dmin1
-            const half4 dmb = vload4(2u*(j + ib*mr) + 1u, src0_dm);  // d2, dmin2, d3, dmin3
-
-            float ad0 = 0.f, am0 = 0.f, ad1 = 0.f, am1 = 0.f;
-            float ad2 = 0.f, am2 = 0.f, ad3 = 0.f, am3 = 0.f;
-
-            for (uint sb = 0; sb < 8u; ++sb) {
-                const uint grp = ib * 64u + sb * 8u;
-                const uint qsb = j + grp * mr;
-
-                for (uint h = 0; h < 2u; ++h) {
-                    const uint scv = scu[j + (2u * (ib * 8u + sb) + h) * mr];
-
-                    float a0 = 0.f, a1 = 0.f, a2 = 0.f, a3 = 0.f;
-                    float4 as = (float4)(0.f);
-                    for (uint u = 0; u < 4u; ++u) {
-                        const uint gg  = 4u*h + u;
-                        const uint qsv = qsu[qsb + gg * mr];   // four rows, one load
-                        const float4 yv = vload4(grp + gg, y);
-                        as += yv;
-                        a0 += dot(yv, q2_k_vals( qsv        & 0xFFu));
-                        a1 += dot(yv, q2_k_vals((qsv >>  8) & 0xFFu));
-                        a2 += dot(yv, q2_k_vals((qsv >> 16) & 0xFFu));
-                        a3 += dot(yv, q2_k_vals((qsv >> 24) & 0xFFu));
-                    }
-                    const float asum = as.s0 + as.s1 + as.s2 + as.s3;
-
-                    const uint s0 =  scv        & 0xFFu;
-                    const uint s1 = (scv >>  8) & 0xFFu;
-                    const uint s2 = (scv >> 16) & 0xFFu;
-                    const uint s3 = (scv >> 24) & 0xFFu;
-                    ad0 += (float)(s0 & 0xFu) * a0;   am0 += (float)(s0 >> 4) * asum;
-                    ad1 += (float)(s1 & 0xFu) * a1;   am1 += (float)(s1 >> 4) * asum;
-                    ad2 += (float)(s2 & 0xFu) * a2;   am2 += (float)(s2 >> 4) * asum;
-                    ad3 += (float)(s3 & 0xFu) * a3;   am3 += (float)(s3 >> 4) * asum;
-                }
-            }
-            sumf0 += (float)dma.s0 * ad0 - (float)dma.s1 * am0;
-            sumf1 += (float)dma.s2 * ad1 - (float)dma.s3 * am1;
-            sumf2 += (float)dmb.s0 * ad2 - (float)dmb.s1 * am2;
-            sumf3 += (float)dmb.s2 * ad3 - (float)dmb.s3 * am3;
-        }
-    }
-
-#if Q2_K_MV_NSG > 1
-    __local float4 part[Q2_K_MV_NSG][64];
-    part[sgi][lid] = (float4)(sumf0, sumf1, sumf2, sumf3);
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < Q2_K_MV_NSG; ++s) {
-        const float4 p = part[s][lid];
-        sumf0 += p.s0;
-        sumf1 += p.s1;
-        sumf2 += p.s2;
-        sumf3 += p.s3;
-    }
-#endif
-
-    if (j < mr) {
-        global float * o = dst + (ulong)col * (uint)ne0 + row;
-        o[0] = sumf0; o[1] = sumf1; o[2] = sumf2; o[3] = sumf3;
-    }
-
-#elif Q2_K_MV_R == 2
-    float sumf0 = 0.f;
-    float sumf1 = 0.f;
-
-    if (j < mr) {
-        global const ushort * qsu = (global const ushort *)src0_qs;
-        global const ushort * scu = (global const ushort *)src0_sc;
-
-        for (uint ib = sgi; ib < nsb; ib += Q2_K_MV_NSG) {
-            const half4 dm = vload4(j + ib * mr, src0_dm);  // d0, dmin0, d1, dmin1
-
-            float ad0 = 0.f, am0 = 0.f, ad1 = 0.f, am1 = 0.f;
-            for (uint sb = 0; sb < 8u; ++sb) {
-                const uint grp = ib * 64u + sb * 8u;
-                const uint qsb = j + grp * mr;
-
-                for (uint h = 0; h < 2u; ++h) {
-                    const uint scv = (uint)scu[j + (2u * (ib * 8u + sb) + h) * mr];
-                    const uint s0  = scv & 0xFFu;
-                    const uint s1  = scv >> 8;
-
-                    float a0 = 0.f, a1 = 0.f;
-                    float4 as = (float4)(0.f);
-                    for (uint u = 0; u < 4u; ++u) {
-                        const uint gg  = 4u*h + u;
-                        const uint qsv = (uint)qsu[qsb + gg * mr];
-                        const float4 yv = vload4(grp + gg, y);
-                        as += yv;
-                        a0 += dot(yv, q2_k_vals( qsv       & 0xFFu));
-                        a1 += dot(yv, q2_k_vals((qsv >> 8) & 0xFFu));
-                    }
-                    const float asum = as.s0 + as.s1 + as.s2 + as.s3;
-                    ad0 += (float)(s0 & 0xFu) * a0;   am0 += (float)(s0 >> 4) * asum;
-                    ad1 += (float)(s1 & 0xFu) * a1;   am1 += (float)(s1 >> 4) * asum;
-                }
-            }
-            sumf0 += (float)dm.s0 * ad0 - (float)dm.s1 * am0;
-            sumf1 += (float)dm.s2 * ad1 - (float)dm.s3 * am1;
-        }
-    }
-
-#if Q2_K_MV_NSG > 1
-    __local float2 part[Q2_K_MV_NSG][64];
-    part[sgi][lid] = (float2)(sumf0, sumf1);
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < Q2_K_MV_NSG; ++s) {
-        const float2 p = part[s][lid];
-        sumf0 += p.s0;
-        sumf1 += p.s1;
-    }
-#endif
-
-    if (j < mr) {
-        vstore2((float2)(sumf0, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
-    }
-
-#else
-    float sumf0 = 0.f;
-
-    if (j < mr) {
-        for (uint ib = sgi; ib < nsb; ib += Q2_K_MV_NSG) {
-            const half2 dm = vload2(row + ib * m, src0_dm);
-
-            float ad = 0.f, am = 0.f;
-            for (uint sb = 0; sb < 8u; ++sb) {
-                const uint grp = ib * 64u + sb * 8u;
-                const uint qsb = row + grp * m;
-
-                for (uint h = 0; h < 2u; ++h) {
-                    const uint sc = (uint)src0_sc[row + (2u * (ib * 8u + sb) + h) * m];
-
-                    float a = 0.f;
-                    float4 as = (float4)(0.f);
-                    for (uint u = 0; u < 4u; ++u) {
-                        const uint gg = 4u*h + u;
-                        const float4 yv = vload4(grp + gg, y);
-                        as += yv;
-                        const uint pkv = (uint)src0_qs[qsb + gg * m];
-                        a += dot(yv, q2_k_vals(pkv));
-                    }
-                    const float asum = as.s0 + as.s1 + as.s2 + as.s3;
-                    ad += (float)(sc & 0xFu) * a;   am += (float)(sc >> 4) * asum;
-                }
-            }
-            sumf0 += (float)dm.s0 * ad - (float)dm.s1 * am;
-        }
-    }
-
-#if Q2_K_MV_NSG > 1
-    __local float part[Q2_K_MV_NSG][64];
-    part[sgi][lid] = sumf0;
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < Q2_K_MV_NSG; ++s) {
-        sumf0 += part[s][lid];
-    }
-#endif
-
-    if (j < mr) {
-        dst[(ulong)col * (uint)ne0 + row] = sumf0;
-    }
-#endif
-}

ggml/src/ggml-opencl/kernels/gemv_noshuffle_q3_k_f32.cl

@@ -1,249 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#define QK_K 256
-
-#ifndef Q3_K_MV_NSG
-#define Q3_K_MV_NSG 8
-#endif
-
-#ifndef Q3_K_MV_R
-#define Q3_K_MV_R 2
-#endif
-
-inline int q3_k_scale(uint sc0, uint sc1, uint sc2, uint is) {
-    const uint w  = is >> 2;
-    const uint sh = 8u * (is & 3u);
-    const uint tb = (sc2 >> sh) & 0xFFu;
-    uint v;
-    if      (w == 0u) { v = ( (sc0 >> sh)       & 0xFu) | (((tb >> 0) & 3u) << 4); }
-    else if (w == 1u) { v = ( (sc1 >> sh)       & 0xFu) | (((tb >> 2) & 3u) << 4); }
-    else if (w == 2u) { v = (((sc0 >> sh) >> 4) & 0xFu) | (((tb >> 4) & 3u) << 4); }
-    else              { v = (((sc1 >> sh) >> 4) & 0xFu) | (((tb >> 6) & 3u) << 4); }
-    return (int)v - 32;
-}
-
-inline float4 q3_k_vals(uint pk, uint hb) {
-    float4 v;
-    v.s0 = (float)((int)((pk      ) & 3u) - 4 + 4*(int)((hb     ) & 1u));
-    v.s1 = (float)((int)((pk >> 2) & 3u) - 4 + 4*(int)((hb >> 1) & 1u));
-    v.s2 = (float)((int)((pk >> 4) & 3u) - 4 + 4*(int)((hb >> 2) & 1u));
-    v.s3 = (float)((int)((pk >> 6) & 3u) - 4 + 4*(int)((hb >> 3) & 1u));
-    return v;
-}
-
-kernel void kernel_gemv_noshuffle_q3_k_f32(
-        global const uchar * src0_qs,
-        global const uchar * src0_hm,
-        global const uint  * src0_sc,
-        global const half  * src0_d,
-        global const float * src1,
-        ulong offset1,
-        global float * dst,
-        ulong offsetd,
-        int ne00,      // K
-        int ne01,      // M
-        int ne10,      // activation row stride
-        int ne0        // dst row stride
-) {
-    src1 = (global const float *)((global const char *)src1 + offset1);
-    dst  = (global float       *)((global char       *)dst  + offsetd);
-
-    const uint m   = (uint)ne01;
-    const uint K   = (uint)ne00;
-    const uint nsb = K / QK_K;
-
-    const uint lid = get_local_id(0);
-    const uint sgi = get_local_id(1);
-    const uint col = get_group_id(1);
-
-    global const float * y = src1 + (ulong)col * (uint)ne10;
-
-#if Q3_K_MV_R == 4
-    const uint mq  = m >> 2;
-    const uint j   = get_group_id(0) * 64u + lid;
-    const uint row = j << 2;
-
-    float sumf = 0.f, sumf1 = 0.f, sumf2 = 0.f, sumf3 = 0.f;
-
-    if (j < mq) {
-        global const uint * qsw = (global const uint *)src0_qs;
-        global const uint * hmw = (global const uint *)src0_hm;
-
-        for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) {
-            const half4 dh = vload4(j + ib * mq, src0_d);
-            const uint4 s0 = vload4(j + (3u * ib + 0u) * mq, src0_sc);
-            const uint4 s1 = vload4(j + (3u * ib + 1u) * mq, src0_sc);
-            const uint4 s2 = vload4(j + (3u * ib + 2u) * mq, src0_sc);
-
-            float acc0 = 0.f, acc1 = 0.f, acc2 = 0.f, acc3 = 0.f;
-            for (uint sb = 0; sb < 8u; ++sb) {
-                const uint grp = ib * 64u + sb * 8u;
-                const uint qsb = j + grp * mq;
-                const uint hmb = j + (grp >> 1) * mq;
-
-                for (uint h = 0; h < 2u; ++h) {         // two 16-weight halves
-                    const int l0 = q3_k_scale(s0.s0, s1.s0, s2.s0, 2u*sb + h);
-                    const int l1 = q3_k_scale(s0.s1, s1.s1, s2.s1, 2u*sb + h);
-                    const int l2 = q3_k_scale(s0.s2, s1.s2, s2.s2, 2u*sb + h);
-                    const int l3 = q3_k_scale(s0.s3, s1.s3, s2.s3, 2u*sb + h);
-
-                    float a0 = 0.f, a1 = 0.f, a2 = 0.f, a3 = 0.f;
-                    for (uint u = 0; u < 4u; ++u) {
-                        const uint gg  = 4u*h + u;
-                        const uint qsv = qsw[qsb + gg * mq];        // four rows, one load
-                        const uint hmv = hmw[hmb + (gg >> 1) * mq];
-                        const uint hsh = 4u * (gg & 1u);
-                        const float4 yv = vload4(grp + gg, y);      // shared by all four
-                        a0 += dot(yv, q3_k_vals((qsv      ) & 0xFFu, ((hmv      ) >> hsh) & 0xFu));
-                        a1 += dot(yv, q3_k_vals((qsv >>  8) & 0xFFu, ((hmv >>  8) >> hsh) & 0xFu));
-                        a2 += dot(yv, q3_k_vals((qsv >> 16) & 0xFFu, ((hmv >> 16) >> hsh) & 0xFu));
-                        a3 += dot(yv, q3_k_vals((qsv >> 24) & 0xFFu, ((hmv >> 24) >> hsh) & 0xFu));
-                    }
-                    acc0 += (float)l0 * a0;
-                    acc1 += (float)l1 * a1;
-                    acc2 += (float)l2 * a2;
-                    acc3 += (float)l3 * a3;
-                }
-            }
-            sumf  += (float)dh.s0 * acc0;
-            sumf1 += (float)dh.s1 * acc1;
-            sumf2 += (float)dh.s2 * acc2;
-            sumf3 += (float)dh.s3 * acc3;
-        }
-    }
-#elif Q3_K_MV_R == 2
-    const uint mh  = m >> 1;
-    const uint j   = get_group_id(0) * 64u + lid;
-    const uint row = j << 1;
-
-    float sumf  = 0.f;
-    float sumf1 = 0.f;
-
-    if (j < mh) {
-        global const ushort * qsu = (global const ushort *)src0_qs;
-        global const ushort * hmu = (global const ushort *)src0_hm;
-
-        for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) {
-            const half2 dh = vload2(j + ib * mh, src0_d);
-            const uint2 s0 = vload2(j + (3u * ib + 0u) * mh, src0_sc);
-            const uint2 s1 = vload2(j + (3u * ib + 1u) * mh, src0_sc);
-            const uint2 s2 = vload2(j + (3u * ib + 2u) * mh, src0_sc);
-
-            float acc0 = 0.f, acc1 = 0.f;
-            for (uint sb = 0; sb < 8u; ++sb) {
-                const uint grp = ib * 64u + sb * 8u;
-                const uint qsb = j + grp * mh;
-                const uint hmb = j + (grp >> 1) * mh;
-
-                for (uint h = 0; h < 2u; ++h) {         // two 16-weight halves
-                    const int l0 = q3_k_scale(s0.s0, s1.s0, s2.s0, 2u*sb + h);
-                    const int l1 = q3_k_scale(s0.s1, s1.s1, s2.s1, 2u*sb + h);
-
-                    float a0 = 0.f, a1 = 0.f;
-                    for (uint u = 0; u < 4u; ++u) {
-                        const uint gg  = 4u*h + u;
-                        const uint qsv = (uint)qsu[qsb + gg * mh];
-                        const uint hmv = (uint)hmu[hmb + (gg >> 1) * mh];
-                        const uint hsh = 4u * (gg & 1u);
-                        const float4 yv = vload4(grp + gg, y);
-                        a0 += dot(yv, q3_k_vals( qsv       & 0xFFu, ( hmv        >> hsh) & 0xFu));
-                        a1 += dot(yv, q3_k_vals((qsv >> 8) & 0xFFu, ((hmv >> 8)  >> hsh) & 0xFu));
-                    }
-                    acc0 += (float)l0 * a0;
-                    acc1 += (float)l1 * a1;
-                }
-            }
-            sumf  += (float)dh.s0 * acc0;
-            sumf1 += (float)dh.s1 * acc1;
-        }
-    }
-#else
-    const uint row = get_group_id(0) * 64u + lid;
-
-    float sumf = 0.f;
-
-    if (row < m) {
-        for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) {
-            const float d  = (float)src0_d[row + ib * m];
-            const uint  s0 = src0_sc[row + (3u * ib + 0u) * m];
-            const uint  s1 = src0_sc[row + (3u * ib + 1u) * m];
-            const uint  s2 = src0_sc[row + (3u * ib + 2u) * m];
-
-            float acc = 0.f;
-            for (uint sb = 0; sb < 8u; ++sb) {
-                const uint grp = ib * 64u + sb * 8u;
-                const uint qsb = row + grp * m;
-                const uint hmb = row + (grp >> 1) * m;
-
-                for (uint h = 0; h < 2u; ++h) {
-                    const int l = q3_k_scale(s0, s1, s2, 2u*sb + h);
-                    float a = 0.f;
-                    for (uint u = 0; u < 4u; ++u) {
-                        const uint gg  = 4u*h + u;
-                        const uint qsv = (uint)src0_qs[qsb + gg * m];
-                        const uint hmv = (uint)src0_hm[hmb + (gg >> 1) * m];
-                        const float4 yv = vload4(grp + gg, y);
-                        a += dot(yv, q3_k_vals(qsv, (hmv >> (4u * (gg & 1u))) & 0xFu));
-                    }
-                    acc += (float)l * a;
-                }
-            }
-            sumf += d * acc;
-        }
-    }
-#endif
-
-#if Q3_K_MV_NSG > 1
-#if Q3_K_MV_R == 4
-    __local float4 part[Q3_K_MV_NSG][64];
-    part[sgi][lid] = (float4)(sumf, sumf1, sumf2, sumf3);
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < Q3_K_MV_NSG; ++s) {
-        const float4 p = part[s][lid];
-        sumf  += p.s0;
-        sumf1 += p.s1;
-        sumf2 += p.s2;
-        sumf3 += p.s3;
-    }
-#elif Q3_K_MV_R == 2
-    __local float2 part[Q3_K_MV_NSG][64];
-    part[sgi][lid] = (float2)(sumf, sumf1);
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < Q3_K_MV_NSG; ++s) {
-        const float2 p = part[s][lid];
-        sumf  += p.s0;
-        sumf1 += p.s1;
-    }
-#else
-    __local float part[Q3_K_MV_NSG][64];
-    part[sgi][lid] = sumf;
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < Q3_K_MV_NSG; ++s) {
-        sumf += part[s][lid];
-    }
-#endif
-#endif
-
-#if Q3_K_MV_R == 4
-    if (j < mq) {
-        vstore4((float4)(sumf, sumf1, sumf2, sumf3), 0, dst + (ulong)col * (uint)ne0 + row);
-    }
-#elif Q3_K_MV_R == 2
-    if (j < mh) {
-        vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
-    }
-#else
-    if (row < m) {
-        dst[(ulong)col * (uint)ne0 + row] = sumf;
-    }
-#endif
-}

ggml/src/ggml-opencl/kernels/lightning_indexer.cl

@@ -1,111 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#ifdef cl_khr_subgroup_shuffle
-#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable
-#endif
-
-kernel void kernel_lightning_indexer(
-        global const char * q,
-        ulong offset_q,
-        global const char * k,
-        ulong offset_k,
-        global const char * w,
-        ulong offset_w,
-        global const char * mask,
-        ulong offset_m,
-        global float * dst,
-        ulong offset_d,
-        int head_size,
-        int n_heads,
-        int n_tokens,
-        int n_keys,
-        int n_masks,
-        int type_k,
-        ulong nbq1,
-        ulong nbq2,
-        ulong nbq3,
-        ulong nbk2,
-        ulong nbk3,
-        ulong nbw1,
-        ulong nbw3,
-        ulong nbm1,
-        ulong nbm3,
-        int use_shuffle
-) {
-    const int tid = get_local_id(0);
-    const int lane = tid & (use_shuffle ? 3 : 15);
-    const int key = use_shuffle ? get_group_id(0)*16 + tid/4 : get_group_id(0)*4 + tid/16;
-    const int token = get_group_id(1);
-    const int stream = get_group_id(2);
-
-    q += offset_q + token*nbq2 + stream*nbq3;
-    k += offset_k + (ulong) key*nbk2 + stream*nbk3;
-    w += offset_w + token*nbw1 + stream*nbw3;
-    mask += offset_m + token*nbm1 + (stream % n_masks)*nbm3;
-    dst = (global float *) ((global char *) dst + offset_d);
-
-#ifdef cl_khr_subgroup_shuffle
-    if (use_shuffle) {
-        // Each group of four lanes keeps one key for all query heads.
-        float4 cached[8];
-        #pragma unroll
-        for (int j = 0; j < 8; ++j) {
-            const int i = 4*(lane + j*4);
-            cached[j] = key < n_keys ? (type_k == 0 ? vload4(0, (global const float *) k + i) : convert_float4(vload4(0, (global const half *) k + i))) : (float4) (0.0f);
-        }
-
-        float score = 0.0f;
-        for (int h = 0; h < n_heads; ++h) {
-            global const float * q_head = (global const float *) (q + h*nbq1);
-            float4 sum = (float4) (0.0f);
-            #pragma unroll
-            for (int j = 0; j < 8; ++j) {
-                sum = mad(vload4(0, q_head + 4*(lane + j*4)), cached[j], sum);
-            }
-            float value = sum.s0 + sum.s1 + sum.s2 + sum.s3;
-            value += sub_group_shuffle_xor(value, 2);
-            value += sub_group_shuffle_xor(value, 1);
-            if (lane == 0) {
-                score = mad(fmax(value, 0.0f), ((global const float *) w)[h], score);
-            }
-        }
-        if (lane == 0 && key < n_keys) {
-            dst[((ulong) stream*n_tokens + token)*n_keys + key] = score + (float) ((global const half *) mask)[key];
-        }
-        return;
-    }
-#endif
-
-    local float sums[64];
-    float score = 0.0f;
-    for (int h = 0; h < n_heads; ++h) {
-        global const float * q_head = (global const float *) (q + h*nbq1);
-        float sum = 0.0f;
-        if (key < n_keys) {
-            for (int i = lane; i < head_size; i += 16) {
-                float kv;
-                if (type_k == 0) {
-                    kv = ((global const float *) k)[i];
-                } else {
-                    kv = (float) ((global const half *) k)[i];
-                }
-                sum = mad(q_head[i], kv, sum);
-            }
-        }
-        sums[tid] = sum;
-        barrier(CLK_LOCAL_MEM_FENCE);
-        for (int stride = 8; stride > 0; stride /= 2) {
-            if (lane < stride) {
-                sums[tid] += sums[tid + stride];
-            }
-            barrier(CLK_LOCAL_MEM_FENCE);
-        }
-        if (lane == 0) {
-            score = mad(fmax(sums[tid], 0.0f), ((global const float *) w)[h], score);
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    if (lane == 0 && key < n_keys) {
-        dst[((ulong) stream*n_tokens + token)*n_keys + key] = score + (float) ((global const half *) mask)[key];
-    }
-}

ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl

@@ -57,17 +57,13 @@ kernel void kernel_add_id_add_id_swiglu_oai(
 
     global const float * g_row  = (global const float *)(src_g  + i1*nb01_g + i2*nb02_g);
     global const float * u_row  = (global const float *)(src_u  + i1*nb01_u + i2*nb02_u);
+    global const float * gb_row = (global const float *)(src_gb + i11*nb11_g);
+    global const float * ub_row = (global const float *)(src_ub + i11*nb11_u);
     global       float * d_row  = (global       float *)(dst    + i1*nbd1   + i2*nbd2);
 
     for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {
-        float x0 = g_row[i0];
-        float x1 = u_row[i0];
-        if (i11 != -1) {
-            global const float * gb_row = (global const float *)(src_gb + i11*nb11_g);
-            global const float * ub_row = (global const float *)(src_ub + i11*nb11_u);
-            x0 += gb_row[i0];
-            x1 += ub_row[i0];
-        }
+        float x0 = g_row[i0] + gb_row[i0];
+        float x1 = u_row[i0] + ub_row[i0];
 
         x0 = min(x0, limit);
         x1 = max(min(x1, limit), -limit);

ggml/src/ggml-opencl/kernels/moe_combine.cl

@@ -42,11 +42,8 @@ __kernel void kernel_moe_combine_bias_f32(
     float4 acc = (float4)(0.0f);
     for (int e = 0; e < k; ++e) {
         const int i11 = *((__global const int *)(I + (ulong)e*sizeof(int)));
-        float4 v = vload4(0, E + (uint)e*e1);
-        if (i11 != -1) {
-            __global const float * Brow = (__global const float *)(B + (ulong)i11*nb_b1) + r4*4u;
-            v += vload4(0, Brow);
-        }
+        __global const float * Brow = (__global const float *)(B + (ulong)i11*nb_b1) + r4*4u;
+        const float4 v = vload4(0, E + (uint)e*e1) + vload4(0, Brow);
         acc = mad(v, (float4)(W[(uint)e*w1]), acc);
     }
 

ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl

@@ -80,6 +80,32 @@ __kernel void kernel_moe_scatter(
     emap[tile_idx] = val;
 }
 
+// the gemm only writes the rows an expert owns, so the skipped rows (negative ids) are zeroed here
+__kernel void kernel_moe_zero_dst(
+    __global const int * input,
+    __global float * dst,
+    uint N,
+    uint topK,
+    uint n_experts,
+    uint ne0
+) {
+    uint n = get_global_id(0);
+    uint k = get_global_id(1);
+
+    if (n >= N || k >= topK) {
+        return;
+    }
+
+    if (input[n * n_experts + k] != -1) {
+        return;
+    }
+
+    __global float * dst_row = dst + ((ulong)n * topK + k) * ne0;
+    for (uint i = get_local_id(2); i < ne0; i += get_local_size(2)) {
+        dst_row[i] = 0.0f;
+    }
+}
+
 // Deterministic replacement for kernel_moe_scatter.
 //
 // kernel_moe_scatter takes each token's slot from atomic_inc(slot_counter[expert]),

ggml/src/ggml-opencl/kernels/mul_mat_id_utils.cl

@@ -1,22 +0,0 @@
-kernel void kernel_moe_zero_dst(
-    global const char * ids,
-    ulong offset_ids,
-    ulong nb_ids1,
-    global char * dst,
-    ulong offset_dst,
-    ulong nb1,
-    ulong nb2,
-    int ne0
-) {
-    const size_t i1 = get_group_id(0);
-    const size_t i2 = get_group_id(1);
-    const int expert = *((global const int *)(ids + offset_ids + i1*sizeof(int) + i2*nb_ids1));
-    if (expert != -1) {
-        return;
-    }
-
-    global float * row = (global float *)(dst + offset_dst + i1*nb1 + i2*nb2);
-    for (size_t i0 = get_local_id(2); i0 < (size_t)ne0; i0 += get_local_size(2)) {
-        row[i0] = 0.0f;
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mm_q2_k_f32_l4_lm.cl

@@ -1,203 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#define INTEL_GPU 1
-#endif
-
-#define QK_K 256
-
-struct block_q2_K {
-    uchar scales[QK_K/16];
-    uchar qs[QK_K/4];
-    half  d;
-    half  dmin;
-};
-
-#define LOAD_VEC_A 4
-#define LOAD_VEC_B 4
-
-#define BM 64
-#define BN 64
-#ifndef BK
-#define BK 32
-#endif
-#ifdef INTEL_GPU
-#define TM 8
-#else
-#define TM 4
-#endif
-#define TN 8
-
-#ifndef LM_HALF
-#define LM_HALF 0
-#endif
-#if LM_HALF
-typedef half lm_st;
-#define LM_LD4(p, i) convert_float4(vload4((i), (p)))
-#else
-typedef float lm_st;
-#define LM_LD4(p, i) vload4((i), (p))
-#endif
-
-kernel void kernel_mul_mm_q2_k_f32_l4_lm(
-    global uchar * src0_qs,
-    global uchar * src0_sc,
-    global half * src0_dm,
-    ulong offset0,
-    global float4 * src1,
-    ulong offset1,
-    global float  * dst,
-    ulong offsetd,
-
-    int ne00,
-    int ne01,
-    int ne02,
-    int ne11,
-    int ne12,
-
-    int stride_a,
-    int stride_b,
-    int stride_d,
-
-    int batch_stride_a,
-    int batch_stride_b,
-    int batch_stride_d,
-
-    int r2,
-    int r3
-) {
-    src1 = (global float4*)((global char*)src1 + offset1);
-    dst  = (global float *)((global char*)dst  + offsetd);
-
-    local lm_st buf_a[BM * BK];
-    local lm_st buf_b[BN * BK];
-
-    const int batch_idx = get_global_id(2);
-
-    const int i13 = batch_idx / ne12;
-    const int i12 = batch_idx % ne12;
-
-    const int i03 = i13 / r3;
-    const int i02 = i12 / r2;
-
-    const int batch_idx_a = i03 * ne02 + i02;
-
-    const int ir = get_group_id(0);
-    const int ic = get_group_id(1);
-
-    const int tid = get_local_id(0);
-    const int th_r  = tid % (BM / TM);
-    const int th_c  = tid / (BM / TM);
-
-    const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
-    const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
-    const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
-    const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
-
-    const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
-    const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
-
-    // pos_a counts elements, not blocks - one q2_K super block holds QK_K of them
-    int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
-    int pos_b = (batch_idx   * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
-
-    float4 sums4[(TM/4) * TN];
-    float4 cache_a4[TM/4];
-
-    for (int i = 0; i < (TM/4) * TN; i++) {
-        sums4[i] = (float4)(0.0f);
-    }
-
-    for (int block = 0; block < ne00; block += BK) {
-        for (int l = 0; l < BM; l += loadstride_a) {
-            if (ir*BM + loadc_a + l < ne01) {
-                int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
-                int ib  = idx / QK_K;
-                ib += offset0/sizeof(struct block_q2_K);
-                int e   = idx % QK_K;
-
-                // dequantize_row_q2_K walks 2 halves of 128, each as 4 shifts of
-                // 2 groups of 16. Recover that position from the element index.
-                int n     = e >> 7;
-                int rem   = e & 127;
-                int j     = rem >> 5;
-                int s     = (rem >> 4) & 1;
-                int l16   = rem & 15;
-                int isc   = 8*n + 2*j + s;
-                int shift = 2*j;
-
-                uchar sc = src0_sc[16*ib + isc];
-                float d  =  (float)src0_dm[2*ib]    * (float)(sc & 0xF);
-                float m  = -(float)src0_dm[2*ib + 1] * (float)(sc >> 4);
-
-                uchar4 q = vload4(0, src0_qs + 64*ib + 32*n + 16*s + l16);
-                float4 v1 = convert_float4((uchar4)((q.s0 >> shift) & 3,
-                                                    (q.s1 >> shift) & 3,
-                                                    (q.s2 >> shift) & 3,
-                                                    (q.s3 >> shift) & 3)) * d + m;
-
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
-            } else {
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
-            }
-        }
-
-        for (int l = 0; l < BN; l += loadstride_b) {
-            if (ic*BN + loadc_b + l < ne11) {
-                int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
-            } else {
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
-            }
-        }
-
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-        pos_a += BK;
-        pos_b += BK / LOAD_VEC_B;
-
-        for (int i = 0; i < BK; i++) {
-            for (int a = 0; a < TM/4; a++) {
-                cache_a4[a] = LM_LD4(buf_a + (i) * BM + th_r * TM, a);
-            }
-
-            for (int cc = 0; cc < TN; cc++) {
-                const float cache_b = buf_b[(i) * BN + th_c * TN + cc];
-                for (int a = 0; a < TM/4; a++) {
-                    const int sums_idx = cc*(TM/4) + a;
-                    sums4[sums_idx] = mad(cache_a4[a], (float4)cache_b, sums4[sums_idx]);
-                }
-            }
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    const int dr = ir * BM + th_r * TM;
-    const int dc = ic * BN + th_c * TN;
-
-    const ulong offsets = batch_idx * batch_stride_d;
-
-    for (int cc = 0; cc < TN; cc++) {
-        for (int a = 0; a < TM/4; a++) {
-            const float4 v = sums4[cc * (TM/4) + a];
-            const float  vs[4] = { v.s0, v.s1, v.s2, v.s3 };
-            for (int k = 0; k < 4; k++) {
-                if (dr + 4*a + k < ne01 && dc + cc < ne11) {
-                    dst[offsets + (dc + cc) * stride_d + dr + 4*a + k] = vs[k];
-                }
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mm_q3_k_f32_l4_lm.cl

@@ -1,219 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#define INTEL_GPU 1
-#endif
-
-#define QK_K 256
-
-struct block_q3_K {
-    uchar hmask[QK_K/8];
-    uchar qs[QK_K/4];
-    uchar scales[12];
-    half d;
-};
-
-#define LOAD_VEC_A 4
-#define LOAD_VEC_B 4
-
-#define BM 64
-#define BN 64
-#ifndef BK
-#define BK 32
-#endif
-#ifdef INTEL_GPU
-#define TM 8
-#else
-#define TM 4
-#endif
-#define TN 8
-
-#ifndef LM_HALF
-#define LM_HALF 0
-#endif
-#if LM_HALF
-typedef half lm_st;
-#define LM_LD4(p, i) convert_float4(vload4((i), (p)))
-#else
-typedef float lm_st;
-#define LM_LD4(p, i) vload4((i), (p))
-#endif
-
-kernel void kernel_mul_mm_q3_k_f32_l4_lm(
-    global uchar * src0_qs,
-    global uchar * src0_hm,
-    global uchar * src0_sc,
-    global half * src0_d,
-    ulong offset0,
-    global float4 * src1,
-    ulong offset1,
-    global float  * dst,
-    ulong offsetd,
-
-    int ne00,
-    int ne01,
-    int ne02,
-    int ne11,
-    int ne12,
-
-    int stride_a,
-    int stride_b,
-    int stride_d,
-
-    int batch_stride_a,
-    int batch_stride_b,
-    int batch_stride_d,
-
-    int r2,
-    int r3
-) {
-    src1 = (global float4*)((global char*)src1 + offset1);
-    dst  = (global float *)((global char*)dst  + offsetd);
-
-    local lm_st buf_a[BM * BK];
-    local lm_st buf_b[BN * BK];
-
-    const int batch_idx = get_global_id(2);
-
-    const int i13 = batch_idx / ne12;
-    const int i12 = batch_idx % ne12;
-
-    const int i03 = i13 / r3;
-    const int i02 = i12 / r2;
-
-    const int batch_idx_a = i03 * ne02 + i02;
-
-    const int ir = get_group_id(0);
-    const int ic = get_group_id(1);
-
-    const int tid = get_local_id(0);
-    const int th_r  = tid % (BM / TM);
-    const int th_c  = tid / (BM / TM);
-
-    const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
-    const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
-    const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
-    const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
-
-    const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
-    const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
-
-    // pos_a counts elements, not blocks - one q3_K super block holds QK_K of them
-    int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
-    int pos_b = (batch_idx   * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
-
-    float4 sums4[(TM/4) * TN];
-    float4 cache_a4[TM/4];
-
-    for (int i = 0; i < (TM/4) * TN; i++) {
-        sums4[i] = (float4)(0.0f);
-    }
-
-    for (int block = 0; block < ne00; block += BK) {
-        for (int l = 0; l < BM; l += loadstride_a) {
-            if (ir*BM + loadc_a + l < ne01) {
-                int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
-                int ib  = idx / QK_K;
-                ib += offset0/sizeof(struct block_q3_K);
-                int e   = idx % QK_K;
-
-                // Only qs advances in the second half of a super-block.
-                int n     = e >> 7;
-                int rem   = e & 127;
-                int j     = rem >> 5;
-                int s     = (rem >> 4) & 1;
-                int l16   = rem & 15;
-                int is    = 8*n + 2*j + s;
-                int shift = 2*j;
-                uchar m   = (uchar)(1u << (4*n + j));
-
-                global ushort * a = (global ushort *)(src0_sc + 12*ib);
-                uint a0  = (uint)a[0] | ((uint)a[1] << 16);
-                uint a1  = (uint)a[2] | ((uint)a[3] << 16);
-                uint tmp = (uint)a[4] | ((uint)a[5] << 16);
-
-                uint g   = is >> 2;
-                uint w   = (g & 1u) ? a1 : a0;
-                w = ((w >> ((g >> 1)*4)) & 0x0f0f0f0fu) | (((tmp >> (2*g)) & 0x03030303u) << 4);
-
-                uint  sc6 = (w >> (8*(is & 3))) & 0xFFu;
-                float dl  = (float)src0_d[ib] * ((float)sc6 - 32.0f);
-
-                uchar4 q  = vload4(0, src0_qs + 64*ib + 32*n + 16*s + l16);
-                uchar4 hb = vload4(0, src0_hm + 32*ib + 16*s + l16);
-
-                float4 qv = convert_float4((uchar4)((q.s0 >> shift) & 3,
-                                                    (q.s1 >> shift) & 3,
-                                                    (q.s2 >> shift) & 3,
-                                                    (q.s3 >> shift) & 3));
-                float4 hv = (float4)((hb.s0 & m) ? 0.f : 4.f,
-                                     (hb.s1 & m) ? 0.f : 4.f,
-                                     (hb.s2 & m) ? 0.f : 4.f,
-                                     (hb.s3 & m) ? 0.f : 4.f);
-                float4 v1 = (qv - hv) * dl;
-
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
-            } else {
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
-            }
-        }
-
-        for (int l = 0; l < BN; l += loadstride_b) {
-            if (ic*BN + loadc_b + l < ne11) {
-                int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
-            } else {
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
-            }
-        }
-
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-        pos_a += BK;
-        pos_b += BK / LOAD_VEC_B;
-
-        for (int i = 0; i < BK; i++) {
-            for (int a = 0; a < TM/4; a++) {
-                cache_a4[a] = LM_LD4(buf_a + (i) * BM + th_r * TM, a);
-            }
-
-            for (int cc = 0; cc < TN; cc++) {
-                const float cache_b = buf_b[(i) * BN + th_c * TN + cc];
-                for (int a = 0; a < TM/4; a++) {
-                    const int sums_idx = cc*(TM/4) + a;
-                    sums4[sums_idx] = mad(cache_a4[a], (float4)cache_b, sums4[sums_idx]);
-                }
-            }
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    const int dr = ir * BM + th_r * TM;
-    const int dc = ic * BN + th_c * TN;
-
-    const ulong offsets = batch_idx * batch_stride_d;
-
-    for (int cc = 0; cc < TN; cc++) {
-        for (int a = 0; a < TM/4; a++) {
-            const float4 v = sums4[cc * (TM/4) + a];
-            const float  vs[4] = { v.s0, v.s1, v.s2, v.s3 };
-            for (int k = 0; k < 4; k++) {
-                if (dr + 4*a + k < ne01 && dc + cc < ne11) {
-                    dst[offsets + (dc + cc) * stride_d + dr + 4*a + k] = vs[k];
-                }
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32.cl

@@ -1,161 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-//------------------------------------------------------------------------------
-// block_q2_K
-//------------------------------------------------------------------------------
-#define QK_K 256
-
-// 16 blocks of 16 elements each
-// weight is represented as x = a * q + b
-typedef struct {
-    uchar scales[QK_K/16]; // scales and mins, quantized with 4 bits
-    uchar qs[QK_K/4];      // 2-bit quants
-    half  d;              // super-block scale for quantized scales
-    half  dmin;            // super-block scale for quantized mins
-} block_q2_K;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4 // number of rows each SIMD group works on
-#define N_SIMDGROUP 1 // number of SIMD groups in a thread group
-#define N_SIMDWIDTH 16 // SIMD group size
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// number of (super) blocks each subgroup processes
-// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_q2_K_f32(
-        global char * src0,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src0 = src0 + offset0;
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // block index (inside super block)
-    int iq = it/4;      // 0 or 1 - first or second half of the super block
-    int ir = it%4;      // 0...3 - block index in the half super block
-    int is = (8*ir)/16; // 0 or 1 - scale pair
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global block_q2_K * x = (global block_q2_K *) (src0 + offset_src0);
-    global float      * y = (global float      *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 128 * iq + 8 * ir;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        float4 sumy = {0.f, 0.f, 0.f, 0.f};
-        for (int i = 0; i < 8; ++i) {
-            yl[i+ 0] = y4[i+ 0]; sumy.s0 += yl[i+ 0];
-            yl[i+ 8] = y4[i+32]; sumy.s1 += yl[i+ 8];
-            yl[i+16] = y4[i+64]; sumy.s2 += yl[i+16];
-            yl[i+24] = y4[i+96]; sumy.s3 += yl[i+24];
-        }
-
-        global uchar  * sc = (global uchar  *)x[ib].scales + 8*iq + is;
-        global ushort * qs = (global ushort *)x[ib].qs + 16 * iq + 4 * ir;
-        global half   * dh = &x[ib].d;
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            float4 acc1 = {0.f, 0.f, 0.f, 0.f};
-            float4 acc2 = {0.f, 0.f, 0.f, 0.f};
-            for (int i = 0; i < 8; i += 2) {
-                acc1.s0 += yl[i+ 0] * (qs[i/2] & 0x0003);
-                acc2.s0 += yl[i+ 1] * (qs[i/2] & 0x0300);
-                acc1.s1 += yl[i+ 8] * (qs[i/2] & 0x000c);
-                acc2.s1 += yl[i+ 9] * (qs[i/2] & 0x0c00);
-                acc1.s2 += yl[i+16] * (qs[i/2] & 0x0030);
-                acc2.s2 += yl[i+17] * (qs[i/2] & 0x3000);
-                acc1.s3 += yl[i+24] * (qs[i/2] & 0x00c0);
-                acc2.s3 += yl[i+25] * (qs[i/2] & 0xc000);
-            }
-
-            float dall = dh[0];
-            float dmin = dh[1] * 1.f/16.f;
-            sumf[row] += dall * ((acc1.s0 + 1.f/256.f * acc2.s0) * (sc[0] & 0xF) * 1.f/ 1.f +
-                                 (acc1.s1 + 1.f/256.f * acc2.s1) * (sc[2] & 0xF) * 1.f/ 4.f +
-                                 (acc1.s2 + 1.f/256.f * acc2.s2) * (sc[4] & 0xF) * 1.f/16.f +
-                                 (acc1.s3 + 1.f/256.f * acc2.s3) * (sc[6] & 0xF) * 1.f/64.f) -
-                        dmin * (sumy.s0 * (sc[0] & 0xF0) + sumy.s1 * (sc[2] & 0xF0) +
-                                sumy.s2 * (sc[4] & 0xF0) + sumy.s3 * (sc[6] & 0xF0));
-
-            qs += nb01/2;
-            sc += nb01;
-            dh += nb01/2;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32_flat.cl

@@ -1,162 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-//------------------------------------------------------------------------------
-// struct block_q2_K
-//------------------------------------------------------------------------------
-#define QK_K 256
-
-// 16 blocks of 16 elements each
-// weight is represented as x = a * q + b
-struct block_q2_K {
-    uchar scales[QK_K/16]; // scales and mins, quantized with 4 bits
-    uchar qs[QK_K/4];      // 2-bit quants
-    half  d;              // super-block scale for quantized scales
-    half  dmin;            // super-block scale for quantized mins
-};
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4 // number of rows each SIMD group works on
-#define N_SIMDGROUP 1 // number of SIMD groups in a thread group
-#define N_SIMDWIDTH 16 // SIMD group size
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// number of (super) blocks each subgroup processes
-// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_q2_K_f32_flat(
-        global uchar * src0_qs,
-        global uchar * src0_sc,
-        global half * src0_dm,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // block index (inside super block)
-    int iq = it/4;      // 0 or 1 - first or second half of the super block
-    int ir = it%4;      // 0...3 - block index in the half super block
-    int is = (8*ir)/16; // 0 or 1 - scale pair
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(struct block_q2_K);
-    ulong row_stride = nb01/sizeof(struct block_q2_K);
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global float      * y = (global float      *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 128 * iq + 8 * ir;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        float4 sumy = {0.f, 0.f, 0.f, 0.f};
-        for (int i = 0; i < 8; ++i) {
-            yl[i+ 0] = y4[i+ 0]; sumy.s0 += yl[i+ 0];
-            yl[i+ 8] = y4[i+32]; sumy.s1 += yl[i+ 8];
-            yl[i+16] = y4[i+64]; sumy.s2 += yl[i+16];
-            yl[i+24] = y4[i+96]; sumy.s3 += yl[i+24];
-        }
-
-        global uchar  * sc = (src0_sc + 16*(offset_src0 + ib)) + 8*iq + is;
-        global ushort * qs = (global ushort *)(src0_qs + 64*(offset_src0 + ib)) + 16 * iq + 4 * ir;
-        global half   * dh = src0_dm + 2*(offset_src0 + ib);
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            float4 acc1 = {0.f, 0.f, 0.f, 0.f};
-            float4 acc2 = {0.f, 0.f, 0.f, 0.f};
-            for (int i = 0; i < 8; i += 2) {
-                acc1.s0 += yl[i+ 0] * (qs[i/2] & 0x0003);
-                acc2.s0 += yl[i+ 1] * (qs[i/2] & 0x0300);
-                acc1.s1 += yl[i+ 8] * (qs[i/2] & 0x000c);
-                acc2.s1 += yl[i+ 9] * (qs[i/2] & 0x0c00);
-                acc1.s2 += yl[i+16] * (qs[i/2] & 0x0030);
-                acc2.s2 += yl[i+17] * (qs[i/2] & 0x3000);
-                acc1.s3 += yl[i+24] * (qs[i/2] & 0x00c0);
-                acc2.s3 += yl[i+25] * (qs[i/2] & 0xc000);
-            }
-
-            float dall = dh[0];
-            float dmin = dh[1] * 1.f/16.f;
-            sumf[row] += dall * ((acc1.s0 + 1.f/256.f * acc2.s0) * (sc[0] & 0xF) * 1.f/ 1.f +
-                                 (acc1.s1 + 1.f/256.f * acc2.s1) * (sc[2] & 0xF) * 1.f/ 4.f +
-                                 (acc1.s2 + 1.f/256.f * acc2.s2) * (sc[4] & 0xF) * 1.f/16.f +
-                                 (acc1.s3 + 1.f/256.f * acc2.s3) * (sc[6] & 0xF) * 1.f/64.f) -
-                        dmin * (sumy.s0 * (sc[0] & 0xF0) + sumy.s1 * (sc[2] & 0xF0) +
-                                sumy.s2 * (sc[4] & 0xF0) + sumy.s3 * (sc[6] & 0xF0));
-
-            qs += row_stride*32;
-            sc += row_stride*16;
-            dh += row_stride*2;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32.cl

@@ -1,197 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-//------------------------------------------------------------------------------
-// block_q3_K
-//------------------------------------------------------------------------------
-#define QK_K 256
-
-// 16 blocks of 16 elements each
-// weight is represented as x = a * q
-typedef struct {
-    uchar hmask[QK_K/8]; // quants - high bit
-    uchar qs[QK_K/4];    // quants - low 2 bits
-    uchar scales[12];    // scales, quantized with 6 bits
-    half  d;             // super-block scale
-} block_q3_K;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 2 // number of rows each SIMD group works on
-#define N_SIMDGROUP 1 // number of SIMD groups in a thread group
-#define N_SIMDWIDTH 16 // SIMD group size
-#elif defined (ADRENO_GPU)
-#define N_DST 2
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_q3_K_f32(
-        global char * src0,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src0 = src0 + offset0;
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global block_q3_K * x  = (global block_q3_K *) (src0 + offset_src0);
-    global float      * yy = (global float      *) (src1 + offset_src1);
-
-    float yl[32];
-
-    // Eight lanes cover one super-block.
-    int ix  = get_sub_group_local_id() % BLOCK_STRIDE;
-    int tid = get_sub_group_local_id() / BLOCK_STRIDE;
-
-    int ip = tid/4;         // 0 or 1
-    int il = 2*((tid%4)/2); // 0 or 2
-    int ir = tid%2;
-    int l0 = 8*ir;
-
-    // Shift the masks instead of indexing private arrays to avoid spills.
-    ushort4 hm = (ushort4)(0x0001, 0x0100, 0x0002, 0x0200) << (ushort4)(2*(2*ip + il/2));
-    int4    qm = (int4)   (0x0003, 0x0300, 0x000c, 0x0c00) << (int4)   (4*(il/2));
-
-    int shift = 2*il;
-
-    float v1 = il == 0 ? 4.f : 64.f;
-    float v2 = 4.f * v1;
-
-    int s_shift1 = 4*ip;
-    int s_shift2 = s_shift1 + il;
-
-    int q_offset = 32*ip + l0;
-    int y_offset = 128*ip + 32*il + l0;
-
-    global float * y1 = yy + ix*QK_K + y_offset;
-
-    float sumf1[N_DST] = {0.f};
-    float sumf2[N_DST] = {0.f};
-    float all_sum;
-
-    for (int i = ix; i < nb; i += BLOCK_STRIDE) {
-        for (int l = 0; l < 8; ++l) {
-            yl[l+ 0] = y1[l+ 0];
-            yl[l+ 8] = y1[l+16];
-            yl[l+16] = y1[l+32];
-            yl[l+24] = y1[l+48];
-        }
-
-        global ushort * q = (global ushort *)(x[i].qs + q_offset);
-        global ushort * h = (global ushort *)(x[i].hmask + l0);
-        global ushort * a = (global ushort *)(x[i].scales);
-        global half   * dh = &x[i].d;
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; ++row) {
-            float d_all = dh[0];
-
-            uint s32   = (uint)a[4] | ((uint)a[5] << 16);
-            uint aux32 = ((s32 >> s_shift2) << 4) & 0x30303030u;
-            s32 = (uint)a[il+0] | ((uint)a[il+1] << 16);
-            s32 = ((s32 >> s_shift1) & 0x0f0f0f0fu) | aux32;
-            char4 sc = as_char4(s32);
-
-            float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0;
-            for (int l = 0; l < 8; l += 2) {
-                int qs = q[l/2];
-                s1 += yl[l+0] * (qs & qm.s0);
-                s2 += yl[l+1] * (qs & qm.s1);
-                s3 += ((h[l/2] & hm.s0) ? 0.f : yl[l+0]) + ((h[l/2] & hm.s1) ? 0.f : yl[l+1]);
-                s4 += yl[l+16] * (qs & qm.s2);
-                s5 += yl[l+17] * (qs & qm.s3);
-                s6 += ((h[l/2] & hm.s2) ? 0.f : yl[l+16]) + ((h[l/2] & hm.s3) ? 0.f : yl[l+17]);
-            }
-            float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
-            float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
-            sumf1[row] += d1 * (sc.s0 - 32);
-            sumf2[row] += d2 * (sc.s2 - 32);
-
-            s1 = s2 = s3 = s4 = s5 = s6 = 0;
-            for (int l = 0; l < 8; l += 2) {
-                int qs = q[l/2+8];
-                s1 += yl[l+8] * (qs & qm.s0);
-                s2 += yl[l+9] * (qs & qm.s1);
-                s3 += ((h[l/2+8] & hm.s0) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm.s1) ? 0.f : yl[l+9]);
-                s4 += yl[l+24] * (qs & qm.s2);
-                s5 += yl[l+25] * (qs & qm.s3);
-                s6 += ((h[l/2+8] & hm.s2) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm.s3) ? 0.f : yl[l+25]);
-            }
-            d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
-            d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
-            sumf1[row] += d1 * (sc.s1 - 32);
-            sumf2[row] += d2 * (sc.s3 - 32);
-
-            q  += nb01/2;
-            h  += nb01/2;
-            a  += nb01/2;
-            dh += nb01/2;
-        }
-
-        y1 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add((sumf1[row] + 0.25f * sumf2[row]) / (1 << shift));
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32_flat.cl

@@ -1,199 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-//------------------------------------------------------------------------------
-// block_q3_K
-//------------------------------------------------------------------------------
-#define QK_K 256
-
-// 16 blocks of 16 elements each
-// weight is represented as x = a * q
-typedef struct {
-    uchar hmask[QK_K/8]; // quants - high bit
-    uchar qs[QK_K/4];    // quants - low 2 bits
-    uchar scales[12];    // scales, quantized with 6 bits
-    half  d;             // super-block scale
-} block_q3_K;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 2 // number of rows each SIMD group works on
-#define N_SIMDGROUP 1 // number of SIMD groups in a thread group
-#define N_SIMDWIDTH 16 // SIMD group size
-#elif defined (ADRENO_GPU)
-#define N_DST 2
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_q3_K_f32_flat(
-        global uchar * src0_qs,
-        global uchar * src0_hm,
-        global uchar * src0_sc,
-        global half * src0_d,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_q3_K);
-    ulong row_stride = nb01/sizeof(block_q3_K);
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global float      * yy = (global float      *) (src1 + offset_src1);
-
-    float yl[32];
-
-    // Eight lanes cover one super-block.
-    int ix  = get_sub_group_local_id() % BLOCK_STRIDE;
-    int tid = get_sub_group_local_id() / BLOCK_STRIDE;
-
-    int ip = tid/4;         // 0 or 1
-    int il = 2*((tid%4)/2); // 0 or 2
-    int ir = tid%2;
-    int l0 = 8*ir;
-
-    // Shift the masks instead of indexing private arrays to avoid spills.
-    ushort4 hm = (ushort4)(0x0001, 0x0100, 0x0002, 0x0200) << (ushort4)(2*(2*ip + il/2));
-    int4    qm = (int4)   (0x0003, 0x0300, 0x000c, 0x0c00) << (int4)   (4*(il/2));
-
-    int shift = 2*il;
-
-    float v1 = il == 0 ? 4.f : 64.f;
-    float v2 = 4.f * v1;
-
-    int s_shift1 = 4*ip;
-    int s_shift2 = s_shift1 + il;
-
-    int q_offset = 32*ip + l0;
-    int y_offset = 128*ip + 32*il + l0;
-
-    global float * y1 = yy + ix*QK_K + y_offset;
-
-    float sumf1[N_DST] = {0.f};
-    float sumf2[N_DST] = {0.f};
-    float all_sum;
-
-    for (int i = ix; i < nb; i += BLOCK_STRIDE) {
-        for (int l = 0; l < 8; ++l) {
-            yl[l+ 0] = y1[l+ 0];
-            yl[l+ 8] = y1[l+16];
-            yl[l+16] = y1[l+32];
-            yl[l+24] = y1[l+48];
-        }
-
-        global ushort * q = (global ushort *)(src0_qs + 64*(offset_src0 + i) + q_offset);
-        global ushort * h = (global ushort *)(src0_hm + 32*(offset_src0 + i) + l0);
-        global ushort * a = (global ushort *)(src0_sc + 12*(offset_src0 + i));
-        global half   * dh = src0_d + offset_src0 + i;
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; ++row) {
-            float d_all = dh[0];
-
-            uint s32   = (uint)a[4] | ((uint)a[5] << 16);
-            uint aux32 = ((s32 >> s_shift2) << 4) & 0x30303030u;
-            s32 = (uint)a[il+0] | ((uint)a[il+1] << 16);
-            s32 = ((s32 >> s_shift1) & 0x0f0f0f0fu) | aux32;
-            char4 sc = as_char4(s32);
-
-            float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0;
-            for (int l = 0; l < 8; l += 2) {
-                int qs = q[l/2];
-                s1 += yl[l+0] * (qs & qm.s0);
-                s2 += yl[l+1] * (qs & qm.s1);
-                s3 += ((h[l/2] & hm.s0) ? 0.f : yl[l+0]) + ((h[l/2] & hm.s1) ? 0.f : yl[l+1]);
-                s4 += yl[l+16] * (qs & qm.s2);
-                s5 += yl[l+17] * (qs & qm.s3);
-                s6 += ((h[l/2] & hm.s2) ? 0.f : yl[l+16]) + ((h[l/2] & hm.s3) ? 0.f : yl[l+17]);
-            }
-            float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
-            float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
-            sumf1[row] += d1 * (sc.s0 - 32);
-            sumf2[row] += d2 * (sc.s2 - 32);
-
-            s1 = s2 = s3 = s4 = s5 = s6 = 0;
-            for (int l = 0; l < 8; l += 2) {
-                int qs = q[l/2+8];
-                s1 += yl[l+8] * (qs & qm.s0);
-                s2 += yl[l+9] * (qs & qm.s1);
-                s3 += ((h[l/2+8] & hm.s0) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm.s1) ? 0.f : yl[l+9]);
-                s4 += yl[l+24] * (qs & qm.s2);
-                s5 += yl[l+25] * (qs & qm.s3);
-                s6 += ((h[l/2+8] & hm.s2) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm.s3) ? 0.f : yl[l+25]);
-            }
-            d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
-            d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
-            sumf1[row] += d1 * (sc.s1 - 32);
-            sumf2[row] += d2 * (sc.s3 - 32);
-
-            q += row_stride*32;
-            h += row_stride*16;
-            a += row_stride*6;
-            dh += row_stride*1;
-        }
-
-        y1 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add((sumf1[row] + 0.25f * sumf2[row]) / (1 << shift));
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/top_k.cl

@@ -1,207 +0,0 @@
-#ifdef cl_khr_subgroups
-#pragma OPENCL EXTENSION cl_khr_subgroups : enable
-#endif
-
-uint top_k_key(float value) {
-    uint bits = as_uint(value);
-    if ((bits & 0x7fffffffu) == 0) {
-        bits = 0;
-    }
-    return bits & 0x80000000u ? ~bits : bits ^ 0x80000000u;
-}
-
-uint4 top_k_keys(global const char * src, int i, int n, ulong nb0) {
-    if (nb0 == sizeof(float) && i + 3 < n) {
-        const float4 v = vload4(0, (global const float *) src + i);
-        return (uint4) (top_k_key(v.s0), top_k_key(v.s1), top_k_key(v.s2), top_k_key(v.s3));
-    }
-    return (uint4) (
-        i     < n ? top_k_key(*(global const float *) (src + (ulong)  i     *nb0)) : 0,
-        i + 1 < n ? top_k_key(*(global const float *) (src + (ulong) (i + 1)*nb0)) : 0,
-        i + 2 < n ? top_k_key(*(global const float *) (src + (ulong) (i + 2)*nb0)) : 0,
-        i + 3 < n ? top_k_key(*(global const float *) (src + (ulong) (i + 3)*nb0)) : 0);
-}
-
-void top_k_store(global int * dst, global uint2 * candidates, global const char * src,
-        ulong nb0, int indexed, int base, int i, uint pos) {
-    const int index = indexed ? *(global const int *) (src + (ulong) i*nb0 + sizeof(float)) : base + i;
-    if (candidates) {
-        candidates[pos] = (uint2) (as_uint(*(global const float *) (src + (ulong) i*nb0)), index);
-    } else {
-        dst[pos] = index;
-    }
-}
-
-void top_k_select(
-        global const char * src,
-        global int * dst,
-        global uint2 * candidates,
-        ulong nb00,
-        int ne00,
-        int k,
-        int base,
-        int indexed,
-        local uint * hist
-) {
-    const int tid = get_local_id(0);
-    const int nth = get_local_size(0);
-    if (k == ne00) {
-        for (int i = tid; i < ne00; i += nth) {
-            top_k_store(dst, candidates, src, nb00, indexed, base, i, i);
-        }
-        return;
-    }
-
-    uint prefix = 0;
-    uint mask = 0;
-    uint rank = k;
-    for (int shift = 24; shift >= 0; shift -= 8) {
-        for (int i = tid; i < 256; i += nth) {
-            hist[i] = 0;
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-        for (int i = 4*tid; i < ne00; i += 4*nth) {
-            const uint4 v = top_k_keys(src, i, ne00, nb00);
-            const uint keys[4] = {v.s0, v.s1, v.s2, v.s3};
-            #pragma unroll
-            for (int j = 0; j < 4; ++j) {
-                const uint key = keys[j];
-                if (i + j < ne00 && (key & mask) == prefix) {
-                    atomic_inc(hist + ((key >> shift) & 255));
-                }
-            }
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-        if (tid == 0) {
-            uint above = 0;
-            for (int b = 255; b >= 0; --b) {
-                uint count = hist[b];
-                if (above + count >= rank) {
-                    hist[256] = b;
-                    hist[257] = rank - above;
-                    hist[258] = count;
-                    break;
-                }
-                above += count;
-            }
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-        prefix |= hist[256] << shift;
-        rank = hist[257];
-        const bool done = hist[258] == rank;
-        barrier(CLK_LOCAL_MEM_FENCE);
-        if (done) {
-            // The whole bucket is selected, so its low bits do not matter.
-            break;
-        }
-        mask |= 255u << shift;
-    }
-
-    uint greater = 0;
-    uint equal = 0;
-    for (int i = 4*tid; i < ne00; i += 4*nth) {
-        const uint4 v = top_k_keys(src, i, ne00, nb00);
-        const uint keys[4] = {v.s0, v.s1, v.s2, v.s3};
-        #pragma unroll
-        for (int j = 0; j < 4; ++j) {
-            greater += i + j < ne00 && keys[j] > prefix;
-            equal += i + j < ne00 && keys[j] == prefix;
-        }
-    }
-    uint pos_greater;
-    uint pos_equal;
-#ifdef cl_khr_subgroups
-    const uint sg = get_sub_group_id();
-    const uint nsg = get_num_sub_groups();
-    const uint total_greater = sub_group_reduce_add(greater);
-    const uint total_equal = sub_group_reduce_add(equal);
-    pos_greater = sub_group_scan_exclusive_add(greater);
-    pos_equal = sub_group_scan_exclusive_add(equal);
-    if (get_sub_group_local_id() == 0) {
-        hist[sg] = total_greater;
-        hist[nsg + sg] = total_equal;
-    }
-    barrier(CLK_LOCAL_MEM_FENCE);
-    for (uint i = 0; i < nsg; ++i) {
-        pos_equal += hist[i];
-        if (i < sg) {
-            pos_greater += hist[i];
-            pos_equal += hist[nsg + i];
-        }
-    }
-#else
-    hist[tid] = greater;
-    hist[nth + tid] = equal;
-    barrier(CLK_LOCAL_MEM_FENCE);
-    pos_greater = 0;
-    pos_equal = 0;
-    for (int i = 0; i < nth; ++i) {
-        pos_equal += hist[i];
-        if (i < tid) {
-            pos_greater += hist[i];
-            pos_equal += hist[nth + i];
-        }
-    }
-#endif
-    for (int i = 4*tid; i < ne00; i += 4*nth) {
-        const uint4 v = top_k_keys(src, i, ne00, nb00);
-        const uint keys[4] = {v.s0, v.s1, v.s2, v.s3};
-        #pragma unroll
-        for (int j = 0; j < 4; ++j) {
-            if (i + j < ne00) {
-                if (keys[j] > prefix) {
-                    top_k_store(dst, candidates, src, nb00, indexed, base, i + j, pos_greater++);
-                } else if (keys[j] == prefix && pos_equal < k) {
-                    top_k_store(dst, candidates, src, nb00, indexed, base, i + j, pos_equal++);
-                }
-            }
-        }
-    }
-}
-
-kernel void kernel_top_k_f32(
-        global const char * src,
-        ulong offset0,
-        global int * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        int ne02,
-        ulong nb00,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int k_max,
-        local uint * hist,
-        global uint2 * candidates,
-        int tile_width
-) {
-    const ulong row = get_group_id(1);
-    const int part = get_group_id(0);
-    const int parts = get_num_groups(0);
-    const int base = part*tile_width;
-    const int n = min(ne00 - base, tile_width);
-    const int k = min(k_max, n);
-    const int out_width = (parts - 1)*k_max + min(k_max, ne00 - (parts - 1)*tile_width);
-    src += offset0 + (row % ne01)*nb01 + (row / ne01 % ne02)*nb02 + (row / ne01 / ne02)*nb03 + (ulong) base*nb00;
-    if (candidates) {
-        candidates += row*out_width + part*k_max;
-    } else {
-        dst = (global int *) ((global char *) dst + offsetd) + row*out_width + part*k_max;
-    }
-    top_k_select(src, dst, candidates, nb00, n, k, base, 0, hist);
-}
-
-kernel void kernel_top_k_merge_f32(
-        global const uint2 * candidates,
-        global int * dst,
-        ulong offsetd,
-        int n_candidates,
-        int k,
-        local uint * hist
-) {
-    const ulong row = get_group_id(1);
-    candidates += row*n_candidates;
-    dst = (global int *) ((global char *) dst + offsetd) + row*k;
-    top_k_select((global const char *) candidates, dst, 0, sizeof(uint2), n_candidates, k, 0, 1, hist);
-}