hexagon: add support for Q1_0 and Q2_0 (#30122)

* hexagon: add support for q1_0 quant

* hexagon: enable Q1_0 weights in the 2D HMX MUL_MAT path

* hex-q1: cleanup gates, generalize tiling and remove in-loop divs

* hex-tiling: document tiled formats in developer.md and update htp-tensor.h

* hex-q2: add support for Q2_0

* hex-q2: remove stale asserts and speacial cases

* hex-mm: fix MUL_MAT_NX assumption about square tiling, Q1/Q2 have non-square tiles

* hex-q1/2: minor optimizations (unrolls/un-unrolls/...)

* hex-q1: optimize Q1_0 layout to reduce unpack/vrmpy overhead

* hex-q1/q2_0: further optimize vec_dots

---------

Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>

.flake8

@@ -5,6 +5,8 @@ exclude =
     # Do not traverse examples and tools
     examples,
     tools,
+    # Reference scripts for skills
+    skills,
     # Do not include package initializers
     __init__.py,
     # No need to traverse our git directory

.github/workflows/build-and-test-snapdragon.yml

@@ -104,11 +104,11 @@ jobs:
     name: Test on QDC Device (${{ matrix.device }})
     needs: [android-ndk-snapdragon, linux-iot-snapdragon]
     runs-on: ubuntu-24.04-arm
-    timeout-minutes: 90
+    timeout-minutes: 180
     strategy:
       fail-fast: false
       matrix:
-        device: [SM8750, SM8850, QCS9075M]
+        device: [SM8850, QCS9075M]
 
     permissions:
       actions: read
@@ -155,7 +155,7 @@ jobs:
               --pkg-dir    pkg-snapdragon/llama.cpp \
               --model-url  "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf" \
               --device     ${{ matrix.device }} \
-              ${{ startsWith(matrix.device, 'QCS') && '--retries 2 --retry-delay 300' || '' }}
+              ${{ startsWith(matrix.device, 'QCS') && '--job-timeout 5400 --retries 2 --retry-delay 300' || '' }}
         env:
           QDC_API_KEY: ${{ secrets.QDC_API_KEY }}
 

.github/workflows/make-release.yml

@@ -61,8 +61,17 @@ jobs:
           RELEASE_BRANCH: ${{ github.ref_name }}
           SKIP_APIABI_CHECK: ${{ github.event.inputs.skip_apiabi_check }}
           APIABI_COMPARE_TAG: ${{ github.event.inputs.apiabi_compare_tag }}
+          APIABI_REPORT_DIR: ${{ runner.temp }}/apiabi-reports
           REQUIRE_DOCKER: ${{ github.event.inputs.require_docker }}
 
+      - name: Upload API/ABI reports
+        if: ${{ always() && github.event.inputs.skip_apiabi_check != 'true' }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: apiabi-reports
+          path: ${{ runner.temp }}/apiabi-reports
+          if-no-files-found: ignore
+
       - name: Create release tag
         if: ${{ github.event.inputs.dry_run == 'false' }}
         run: |

AGENTS.md

@@ -246,3 +246,6 @@ Chat template and parser:
 - [PEG parser](docs/development/parsing.md) - alternative to regex that llama.cpp uses to parse model's output
 - [Auto parser](docs/autoparser.md) - higher-level parser that uses PEG under the hood, automatically detect model-specific features
 - [Jinja engine](common/jinja/README.md)
+
+Testing:
+- [Slice model](skills/slice-model/SKILL.md) - build a small slice (few layers/experts) of a big HF model without downloading the full weights, to test conversion and inference

common/chat.cpp

@@ -1104,6 +1104,13 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
         return common_chat_params_init_llm_jp_harmony;
     }
 
+    // Harmony XML tool calls, checked before the GPT-OSS JSON calls
+    if (src.find("<|channel|>") != std::string::npos && src.find("<|constrain|>xml") != std::string::npos &&
+        src.find("rfl:parameter") != std::string::npos) {
+        LOG_DBG("Using specialized template: Harmony XML\n");
+        return common_chat_params_init_harmony_xml;
+    }
+
     // GPT-OSS - has unique channel-based structure that needs dedicated handler
     if (src.find("<|channel|>") != std::string::npos) {
         LOG_DBG("Using specialized template: GPT-OSS\n");

common/common.cpp

@@ -1174,6 +1174,7 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
     { COMMON_DECISION_TYPE_VLLM_D3,        "vllm-d3"        },
     { COMMON_DECISION_TYPE_LFM2_D1,        "lfm2-d1"       },
     { COMMON_DECISION_TYPE_LFM2_D1_OMNI,   "lfm2-d1-omni"  },
+    { COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"},
     // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
 };
 

common/common.h

@@ -970,6 +970,7 @@ enum common_decision_type {
     COMMON_DECISION_TYPE_VLLM_D3, // same as pplx-decider, non-causal attention
     COMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question type
     COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout
+    COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch
     COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
 };
 

common/download.cpp

@@ -39,6 +39,9 @@
 // downloader
 //
 
+// the suffix of the file a download writes until it completes
+static const std::string PARTIAL_SUFFIX = ".downloadInProgress";
+
 static void write_etag(const std::string & path, const std::string & etag) {
     const std::string etag_path = path + ".etag";
     fs_write_atomic(std::filesystem::u8path(etag_path), etag);
@@ -186,6 +189,13 @@ static bool common_pull_file(httplib::Client & cli,
         [&](const httplib::Response &response) {
             if (p.downloaded > 0 && response.status != 206) {
                 LOG_WRN("%s: server did not respond with 206 Partial Content for a resume request. Status: %d\n", func, response.status);
+                if (response.status == 416 || response.status == 200) {
+                    // the kept partial cannot be resumed: 416 for a complete or stale one,
+                    // 200 from a server that ignores the range; drop it, so the next
+                    // attempt starts over instead of wedging on the same answer
+                    ofs.close();
+                    remove(path_tmp.c_str());
+                }
                 return false;
             }
             if (p.downloaded == 0 && response.status != 200) {
@@ -214,7 +224,7 @@ static bool common_pull_file(httplib::Client & cli,
             if (progress_step >= p.total / 1000 || p.downloaded == p.total) {
                 if (callback) {
                     callback->on_update(p);
-                    if (callback->is_cancelled()) {
+                    if (callback->is_cancelled() && (p.total == 0 || p.downloaded != p.total)) {
                         return false;
                     }
                 }
@@ -331,10 +341,24 @@ static int common_download_file_single_online(const std::string & url,
     }
 
     bool success = false;
-    const std::string path_temporary = path + ".downloadInProgress";
+    const std::string path_temporary = path + PARTIAL_SUFFIX;
     int delay = retry_delay_seconds;
 
+    if (!skip_etag && !etag.empty()) {
+        if (std::filesystem::exists(path_temporary) && read_etag(path_temporary) != etag) {
+            LOG_WRN("%s: partial file does not match the server etag, discarding it: %s\n", __func__, path_temporary.c_str());
+            if (remove(path_temporary.c_str()) != 0) {
+                LOG_ERR("%s: unable to delete file: %s\n", __func__, path_temporary.c_str());
+                return -1;
+            }
+        }
+        write_etag(path_temporary, etag);
+    }
+
     if (opts.callback) {
+        // report the resume offset, so a resumed file does not start at 0
+        p.downloaded = supports_ranges && std::filesystem::exists(path_temporary) ?
+                       std::filesystem::file_size(path_temporary) : 0;
         opts.callback->on_start(p);
     }
 
@@ -372,6 +396,7 @@ static int common_download_file_single_online(const std::string & url,
             }
             if (!etag.empty() && !skip_etag) {
                 write_etag(path, etag);
+                remove((path_temporary + ".etag").c_str());
             }
             success = true;
             break;
@@ -381,12 +406,7 @@ static int common_download_file_single_online(const std::string & url,
     if (opts.callback) {
         opts.callback->on_done(p, success);
     }
-    if (opts.callback && opts.callback->is_cancelled() &&
-        std::filesystem::exists(path_temporary)) {
-        if (remove(path_temporary.c_str()) != 0) {
-            LOG_ERR("%s: unable to delete temporary file: %s\n", __func__, path_temporary.c_str());
-        }
-    }
+    // a stopped download keeps path_temporary, so the next attempt resumes from it
     if (!success) {
         LOG_ERR("%s: download failed after %d attempts\n", __func__, max_attempts);
         return -1; // max attempts reached
@@ -594,33 +614,65 @@ static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files,
     return best;
 }
 
-static hf_cache::hf_file find_best_mmproj(const hf_cache::hf_files & files,
-                                          const std::string        & model) {
-    return find_best_sibling(files, model, "mmproj");
+static const std::vector<common_download_sidecar> sidecars = {
+    { "mmproj", "mmproj",  false, &common_download_opts::download_mmproj, &common_download_hf_plan::mmproj },
+    { "mtp",    "mtp-",    true,  &common_download_opts::download_mtp,    &common_download_hf_plan::mtp    },
+    { "dflash", "dflash-", true,  &common_download_opts::download_dflash, &common_download_hf_plan::dflash },
+    { "eagle3", "eagle3-", true,  &common_download_opts::download_eagle3, &common_download_hf_plan::eagle3 },
+    { "dspark", "dspark-", true,  &common_download_opts::download_dspark, &common_download_hf_plan::dspark },
+};
+
+const std::vector<common_download_sidecar> & common_download_sidecars() {
+    return sidecars;
 }
 
-static hf_cache::hf_file find_best_mtp(const hf_cache::hf_files & files,
-                                       const std::string        & model,
-                                       const std::string        & tag = "") {
-    return find_best_sibling(files, model, "mtp-", tag);
+bool common_download_opts_add_sidecar(common_download_opts & opts, const std::string & kind) {
+    for (const auto & s : sidecars) {
+        if (kind == s.name) {
+            opts.*s.opt = true;
+            return true;
+        }
+    }
+    return false;
 }
 
-static hf_cache::hf_file find_best_eagle3(const hf_cache::hf_files & files,
-                                          const std::string        & model,
-                                          const std::string        & tag = "") {
-    return find_best_sibling(files, model, "eagle3-", tag);
+std::vector<std::string> common_download_plan_sidecars(const common_download_hf_plan & plan) {
+    std::vector<std::string> out;
+    for (const auto & s : sidecars) {
+        if (!(plan.*s.file).path.empty()) {
+            out.emplace_back(s.name);
+        }
+    }
+    return out;
 }
 
-static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files,
-                                          const std::string        & model,
-                                          const std::string        & tag = "") {
-    return find_best_sibling(files, model, "dflash-", tag);
+std::vector<common_download_partial> common_download_plan_partials(const common_download_hf_plan & plan) {
+    std::vector<common_download_partial> out;
+
+    // a file in the cache resolves to its snapshot entry, a file still to download to its blob
+    auto add = [&](const std::string & kind, const hf_cache::hf_file & f) {
+        if (!f.local_path.empty() && f.local_path != f.final_path) {
+            out.push_back({ kind, f.local_path + PARTIAL_SUFFIX });
+        }
+    };
+
+    for (const auto & f : plan.model_files) {
+        add("", f);
+    }
+    for (const auto & s : sidecars) {
+        add(s.name, plan.*s.file);
+    }
+    return out;
 }
 
-static hf_cache::hf_file find_best_dspark(const hf_cache::hf_files & files,
-                                          const std::string        & model,
-                                          const std::string        & tag = "") {
-    return find_best_sibling(files, model, "dspark-", tag);
+// true when `name` carries the keyword of a sidecar kind
+static bool name_is_sidecar(const std::string & name) {
+    for (const auto & s : sidecars) {
+        if (name.find(s.keyword) != std::string::npos) {
+            return true;
+        }
+    }
+    return false;
 }
 
 static bool gguf_filename_is_model(const std::string & filepath) {
@@ -633,12 +685,7 @@ static bool gguf_filename_is_model(const std::string & filepath) {
         filename = filename.substr(pos + 1);
     }
 
-    return filename.find("mmproj")  == std::string::npos &&
-           filename.find("imatrix") == std::string::npos &&
-           filename.find("mtp-")    == std::string::npos &&
-           filename.find("eagle3-") == std::string::npos &&
-           filename.find("dflash-") == std::string::npos &&
-           filename.find("dspark-") == std::string::npos;
+    return filename.find("imatrix") == std::string::npos && !name_is_sidecar(filename);
 }
 
 static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files,
@@ -730,8 +777,11 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model &
         }
     } else {
         primary = find_best_model(all, tag);
-        // a requested sidecar can resolve on its own, without a full model of the same tag
-        if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3 && !opts.download_dspark) {
+        // a requested draft can resolve on its own, without a full model of the same tag
+        const bool want_draft = std::any_of(sidecars.begin(), sidecars.end(), [&](const common_download_sidecar & s) {
+            return s.draft && opts.*s.opt;
+        });
+        if (primary.path.empty() && !want_draft) {
             LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());
             list_available_gguf_files(all);
             return plan;
@@ -743,24 +793,22 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model &
         plan.model_files = get_split_files(all, primary);
     }
 
-    if (opts.download_mmproj && !primary.path.empty()) {
-        plan.mmproj = find_best_mmproj(all, primary.path);
-    }
-    if (opts.download_mtp) {
-        plan.mtp = find_best_mtp(all, primary.path, tag);
-    }
-    if (opts.download_dflash) {
-        plan.dflash = find_best_dflash(all, primary.path, tag);
-    }
-    if (opts.download_eagle3) {
-        plan.eagle3 = find_best_eagle3(all, primary.path, tag);
-    }
-    if (opts.download_dspark) {
-        plan.dspark = find_best_dspark(all, primary.path, tag);
+    // a draft resolves at the tag, the mmproj pairs with the model
+    for (const auto & s : sidecars) {
+        if (!(opts.*s.opt)) {
+            continue;
+        }
+        if (s.draft) {
+            plan.*s.file = find_best_sibling(all, primary.path, s.keyword, tag);
+        } else if (!primary.path.empty()) {
+            plan.*s.file = find_best_sibling(all, primary.path, s.keyword);
+        }
     }
 
-    if (primary.path.empty() &&
-        plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty() && plan.dspark.local_path.empty()) {
+    const bool has_draft = std::any_of(sidecars.begin(), sidecars.end(), [&](const common_download_sidecar & s) {
+        return s.draft && !(plan.*s.file).local_path.empty();
+    });
+    if (primary.path.empty() && !has_draft) {
         LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());
         list_available_gguf_files(all);
     }
@@ -926,6 +974,11 @@ std::string common_docker_resolve_model(const std::string & docker) {
     }
 }
 
+// a cached file that lists a model under its tag: the first split of a model, not a sidecar
+static bool is_listed_model(const gguf_split_info & split) {
+    return split.index == 1 && !split.tag.empty() && !name_is_sidecar(split.prefix);
+}
+
 std::vector<common_cached_model_info> common_list_cached_models() {
     std::unordered_set<std::string> seen;
     std::vector<common_cached_model_info> result;
@@ -934,12 +987,7 @@ std::vector<common_cached_model_info> common_list_cached_models() {
 
     for (const auto & f : files) {
         auto split = get_gguf_split_info(f.path);
-        if (split.index != 1 || split.tag.empty() ||
-            split.prefix.find("mmproj")  != std::string::npos ||
-            split.prefix.find("mtp-")    != std::string::npos ||
-            split.prefix.find("eagle3-") != std::string::npos ||
-            split.prefix.find("dflash-") != std::string::npos ||
-            split.prefix.find("dspark-") != std::string::npos) {
+        if (!is_listed_model(split)) {
             continue;
         }
         if (seen.insert(f.repo_id + ":" + split.tag).second) {
@@ -970,6 +1018,44 @@ std::string common_download_resolve_path(const std::string & hf_repo_with_tag, c
     return find_best_model(files, tag).local_path;
 }
 
+// the files -hf loads from the cache, the model with every sidecar, for each cached
+// tag of `repo_id` other than `tag`
+static std::unordered_set<std::string> files_in_use(const std::string & repo_id, const std::string & tag) {
+    common_download_opts opts;
+    opts.offline = true;
+    for (const auto & s : sidecars) {
+        opts.*s.opt = true;
+    }
+
+    std::unordered_set<std::string> tags;
+    std::unordered_set<std::string> used;
+
+    for (const auto & f : hf_cache::get_cached_files(repo_id)) {
+        auto split = get_gguf_split_info(f.path);
+        if (!is_listed_model(split) || split.tag == tag || !tags.insert(split.tag).second) {
+            continue;
+        }
+
+        common_params_model model;
+        model.hf_repo = repo_id + ":" + split.tag;
+
+        auto plan = common_download_get_hf_plan(model, opts);
+
+        for (const auto & m : plan.model_files) {
+            used.insert(m.local_path);
+        }
+        for (const auto & s : sidecars) {
+            const auto & sf = plan.*s.file;
+            if (!sf.local_path.empty()) {
+                used.insert(sf.local_path);
+            }
+        }
+    }
+    return used;
+}
+
+static bool remove_snapshot_entries(const std::string & repo_id, const std::vector<std::filesystem::path> & to_remove);
+
 bool common_download_remove(const std::string & hf_repo_with_tag) {
     namespace fs = std::filesystem;
 
@@ -989,11 +1075,13 @@ bool common_download_remove(const std::string & hf_repo_with_tag) {
         return false;
     }
 
-    // collect snapshot entries whose tag matches
+    // collect snapshot entries whose tag matches, but a file another tag loads stays
+    const auto used = files_in_use(repo_id, tag_upper);
+
     std::vector<fs::path> to_remove;
     for (const auto & f : files) {
         auto split = get_gguf_split_info(f.path);
-        if (split.tag == tag_upper) {
+        if (split.tag == tag_upper && used.find(f.local_path) == used.end()) {
             to_remove.emplace_back(f.local_path);
         }
     }
@@ -1002,6 +1090,13 @@ bool common_download_remove(const std::string & hf_repo_with_tag) {
         return false;
     }
 
+    return remove_snapshot_entries(repo_id, to_remove);
+}
+
+// remove snapshot entries of `repo_id`, then the blobs no remaining entry references
+static bool remove_snapshot_entries(const std::string & repo_id, const std::vector<std::filesystem::path> & to_remove) {
+    namespace fs = std::filesystem;
+
     // resolve blob paths from symlinks before deleting snapshot entries
     std::vector<fs::path> blobs_to_check;
     for (const auto & p : to_remove) {
@@ -1053,3 +1148,26 @@ bool common_download_remove(const std::string & hf_repo_with_tag) {
 
     return true;
 }
+
+bool common_download_remove_sidecars(const std::string & hf_repo_with_tag, const common_download_opts & opts) {
+    common_params_model model;
+    model.hf_repo = hf_repo_with_tag;
+
+    common_download_opts cached = opts;
+    cached.offline = true;
+
+    auto plan = common_download_get_hf_plan(model, cached);
+
+    std::vector<std::filesystem::path> to_remove;
+    for (const auto & s : sidecars) {
+        const auto & f = plan.*s.file;
+        if (!f.local_path.empty()) {
+            to_remove.emplace_back(std::filesystem::u8path(f.local_path));
+        }
+    }
+    if (to_remove.empty()) {
+        return false;
+    }
+
+    return remove_snapshot_entries(common_download_split_repo_tag(hf_repo_with_tag).first, to_remove);
+}

common/download.h

@@ -104,10 +104,17 @@ std::string common_docker_resolve_model(const std::string & docker);
 // Remove a cached model from disk
 // input format: "user/model" or "user/model:tag"
 // - if tag is omitted, removes the entire repo cache directory
-// - if tag is present, removes only files matching that tag (and orphaned blobs)
+// - if tag is present, removes only files matching that tag (and orphaned blobs),
+//   except the files another cached tag of the repo still loads
 // returns true if anything was removed
 bool common_download_remove(const std::string & hf_repo_with_tag);
 
+// remove the cached sidecar files `opts` selects for "user/model:tag", as the cached
+// plan resolves them, and leave the model files in place; a sidecar file another tag
+// shares goes for every tag that resolves it
+// returns true if anything was removed
+bool common_download_remove_sidecars(const std::string & hf_repo_with_tag, const common_download_opts & opts);
+
 struct common_download_hf_plan {
     hf_cache::hf_file primary;
     hf_cache::hf_files model_files;
@@ -119,3 +126,33 @@ struct common_download_hf_plan {
     hf_cache::hf_file preset; // if set, only this file is downloaded
 };
 common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts);
+
+// a sidecar kind a download can pull next to its model: mmproj, and the draft heads
+// of --spec-type draft-<name>
+struct common_download_sidecar {
+    const char * name;    // the kind, as the API names it
+    const char * keyword; // the part of a file name that marks a file of this kind
+    bool         draft;   // a draft head, resolved at the tag even without a model
+
+    bool              common_download_opts::*    opt;  // the option that requests it
+    hf_cache::hf_file common_download_hf_plan::* file; // the plan file it resolves to
+};
+
+// every sidecar kind, the single list the resolution, the listing and the removal follow
+const std::vector<common_download_sidecar> & common_download_sidecars();
+
+// select the sidecar `kind` in `opts`; returns false for an unknown kind
+bool common_download_opts_add_sidecar(common_download_opts & opts, const std::string & kind);
+
+// the sidecar kinds `plan` resolves a file for
+std::vector<std::string> common_download_plan_sidecars(const common_download_hf_plan & plan);
+
+// a file `plan` still has to download, at the path its partial takes while it is
+// incomplete; kind is the sidecar it belongs to, empty for the model files
+struct common_download_partial {
+    std::string kind;
+    std::string path;
+};
+
+// the partials the download of `plan` writes, for the files not in the cache yet
+std::vector<common_download_partial> common_download_plan_partials(const common_download_hf_plan & plan);

common/http.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "log.h"
+
 #include <cpp-httplib/httplib.h>
 
 #ifdef _WIN32
@@ -97,6 +99,118 @@ static common_http_url common_http_parse_url(const std::string & url) {
     return parts;
 }
 
+// match a loopback destination: a remote proxy can never reach the client machine's own
+// loopback, so these hosts are always contacted directly, before NO_PROXY is consulted
+// (similar to Go's net/http)
+static bool common_http_is_loopback(const std::string & host) {
+    auto lower = [](std::string s) {
+        for (auto & c : s) {
+            if (c >= 'A' && c <= 'Z') {
+                c += 'a' - 'A';
+            }
+        }
+        return s;
+    };
+
+    const std::string h = lower(host);
+
+    if (h == "localhost" || h == "::1" || h == "0.0.0.0") {
+        return true;
+    }
+
+    if (h.size() > 10 && h.compare(h.size() - 10, 10, ".localhost") == 0) {
+        return true;
+    }
+
+    // IPv4 literal: exactly four all-digit octets with values 0-255, loopback is 127.0.0.0/8
+    int octets[4] = { 0, 0, 0, 0 };
+    size_t pos = 0;
+    int n = 0;
+    bool complete = false;
+    while (n < 4 && pos <= h.size()) {
+        size_t dot = h.find('.', pos);
+        if (dot == std::string::npos) {
+            dot = h.size();
+        }
+        if (dot == pos || dot - pos > 3) {
+            return false;
+        }
+        int val = 0;
+        for (size_t i = pos; i < dot; i++) {
+            if (h[i] < '0' || h[i] > '9') {
+                return false;
+            }
+            val = val * 10 + (h[i] - '0');
+        }
+        if (val > 255) {
+            return false;
+        }
+        octets[n++] = val;
+        if (dot == h.size()) {
+            complete = (n == 4);
+            break;
+        }
+        pos = dot + 1;
+    }
+
+    return complete && octets[0] == 127;
+}
+
+// match a host against a NO_PROXY list: comma separated entries, "*" bypasses the proxy for every host,
+// an entry matches the host itself or any of its subdomains, a leading dot is optional, ports are ignored
+static bool common_http_proxy_bypass(const std::string & no_proxy, const std::string & host) {
+    auto lower = [](std::string s) {
+        for (auto & c : s) {
+            if (c >= 'A' && c <= 'Z') {
+                c += 'a' - 'A';
+            }
+        }
+        return s;
+    };
+
+    const std::string h = lower(host);
+
+    for (size_t pos = 0; pos < no_proxy.size(); ) {
+        size_t end = no_proxy.find(',', pos);
+        if (end == std::string::npos) {
+            end = no_proxy.size();
+        }
+
+        std::string entry = lower(no_proxy.substr(pos, end - pos));
+        pos = end + 1;
+
+        entry.erase(0, entry.find_first_not_of(" \t"));
+        auto last = entry.find_last_not_of(" \t");
+        if (last == std::string::npos) {
+            continue;
+        }
+        entry.erase(last + 1);
+
+        if (entry == "*") {
+            return true;
+        }
+
+        if (entry.front() == '.') {
+            entry.erase(0, 1);
+        }
+
+        if (entry.empty()) {
+            continue;
+        }
+
+        if (h == entry) {
+            return true;
+        }
+
+        if (h.size() > entry.size() && h[h.size() - entry.size() - 1] == '.' &&
+            h.compare(h.size() - entry.size(), entry.size(), entry) == 0) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
 static std::pair<httplib::Client, common_http_url> common_http_client(const std::string & url) {
     common_http_url parts = common_http_parse_url(url);
 
@@ -123,6 +237,33 @@ static std::pair<httplib::Client, common_http_url> common_http_client(const std:
 
     cli.set_follow_location(true);
 
+    // Honor HTTP_PROXY / HTTPS_PROXY environment variables (both lower- and upper-case).
+    auto getenv_s = [](const char * upper, const char * lower) -> std::string {
+        const char * val = std::getenv(upper);
+        if (!val) {
+            val = std::getenv(lower);
+        }
+        return val ? val : "";
+    };
+
+    const std::string proxy_url = (parts.scheme == "https")
+        ? getenv_s("HTTPS_PROXY", "https_proxy")
+        : getenv_s("HTTP_PROXY",  "http_proxy");
+
+    if (!proxy_url.empty() && !common_http_is_loopback(parts.host) &&
+        !common_http_proxy_bypass(getenv_s("NO_PROXY", "no_proxy"), parts.host)) {
+        try {
+            common_http_url proxy = common_http_parse_url(proxy_url);
+            cli.set_proxy(proxy.host, proxy.port);
+            if (!proxy.user.empty()) {
+                cli.set_proxy_basic_auth(proxy.user, proxy.password);
+            }
+        } catch (const std::exception & e) {
+            // fall back to a direct connection, the URL itself is not logged as it can carry credentials
+            LOG_WRN("%s: ignoring malformed proxy URL from the environment: %s\n", __func__, e.what());
+        }
+    }
+
     return { std::move(cli), std::move(parts) };
 }
 

common/jinja/value.cpp

@@ -1299,6 +1299,15 @@ const func_builtins & value_object_t::get_builtins() const {
             }
             return result;
         }},
+        {"list", [](const func_args & args) -> value {
+            args.ensure_vals<value_object>();
+            const auto & obj = args.get_pos(0)->as_ordered_object();
+            auto result = mk_val<value_array>();
+            for (const auto & pair : obj) {
+                result->push_back(pair.first);
+            }
+            return result;
+        }},
         {"values", [](const func_args & args) -> value {
             args.ensure_vals<value_object>();
             const auto & obj = args.get_pos(0)->as_ordered_object();

common/parsers/harmony-xml.cpp

@@ -0,0 +1,207 @@
+#include "parsers.h"
+
+// Harmony with XML tool calls: <|start|>assistant to=NAME<|channel|>analysis <|constrain|>xml<|message|><rfl:parameter name="KEY">\nVALUE\n</rfl:parameter>
+common_chat_params common_chat_params_init_harmony_xml(const common_chat_template &          tmpl,
+                                                       const autoparser::generation_params & inputs) {
+    common_chat_params data;
+
+    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);
+    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
+    data.message_delimiters = {
+        { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },
+        { COMMON_CHAT_ROLE_USER,      "<|start|>user"      },
+        { COMMON_CHAT_ROLE_SYSTEM,    "<|start|>developer" },
+        { COMMON_CHAT_ROLE_SYSTEM,    "<|start|>system"    },
+    };
+
+    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
+    data.supports_thinking = true;
+
+    data.thinking_start_tag = "<|channel|>analysis<|message|>";
+    data.thinking_end_tags  = {"<|end|>"};
+
+    // EOG text must reach the parser: an XML call has no closing tag of its own
+    static const std::vector<std::string> EOG = {
+        "<|call|>", "<|return|>", "<|endoftext|>", "<|im_end|>", "<|fim_pad|>", "<|repo_name|>", "<|file_sep|>",
+    };
+
+    data.preserved_tokens = {
+        "<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",
+    };
+    data.preserved_tokens.insert(data.preserved_tokens.end(), EOG.begin(), EOG.end());
+
+    if (inputs.has_continuation()) {
+        const auto & msg = inputs.continue_msg;
+
+        data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content;
+        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+            data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content();
+        }
+
+        data.prompt.append(data.generation_prompt);
+    }
+
+    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
+    auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
+    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
+    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
+
+    static const std::string PARAM_OPEN  = "<rfl:parameter name=\"";
+    static const std::string PARAM_CLOSE = "\n</rfl:parameter>";
+
+    data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+        auto start = p.rule("start", p.literal("<|start|>assistant"));
+
+        auto eog = p.choice();
+        for (const auto & tok : EOG) {
+            eog |= p.literal(tok);
+        }
+        auto with_eog = [&](std::vector<std::string> delims) {
+            delims.insert(delims.end(), EOG.begin(), EOG.end());
+            return delims;
+        };
+
+        auto analysis_open  = p.literal("<|channel|>analysis<|message|>");
+        auto analysis_body  = p.until_one_of(with_eog({ "<|end|>", "<|constrain|><|start|>assistant", "<|start|>assistant" }));
+        auto analysis_close = p.optional(p.literal("<|end|>") | p.literal("<|constrain|>") | eog);
+        if (extract_reasoning) {
+            p.rule("analysis", analysis_open + p.reasoning(analysis_body) + analysis_close);
+        } else {
+            p.rule("analysis", p.atomic(p.content(analysis_open)) + p.content(analysis_body + analysis_close));
+        }
+        auto analysis = p.ref("analysis");
+
+        auto final_msg = p.rule("final",
+            p.literal("<|channel|>final<|message|>") +
+            p.content(p.until_one_of(with_eog({ "<|end|>" }))) +
+            p.optional(p.literal("<|end|>") | eog));
+
+        if (has_response_format) {
+            auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1);
+            auto constraint     = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
+            auto response_format = p.rule("response-format",
+                p.literal("<|channel|>final") + constraint + p.literal("<|message|>") +
+                p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
+                p.optional(p.literal("<|end|>") | eog));
+
+            return start + p.zero_or_more(analysis + start) + response_format;
+        }
+
+        if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
+            // tolerate a missing closing tag
+            std::vector<std::string> value_ends = with_eog({ PARAM_OPEN, "<|end|>" });
+            std::vector<std::string> string_ends = { PARAM_CLOSE };
+            auto next_open = p.choice();
+            for (const auto & e : value_ends) {
+                string_ends.push_back("\n" + e);
+                next_open |= p.literal(e);
+            }
+            auto arg_close = p.tool_arg_close(p.gbnf(
+                p.literal(PARAM_CLOSE) | (p.literal("\n") + p.peek(p.optional(p.literal("\n")) + next_open)),
+                gbnf_format_literal("\n") + " " + gbnf_format_literal(PARAM_CLOSE.substr(1)) + "?"));
+            auto arg_string = p.rule("rfl-arg-string",
+                p.ac(p.tool_arg_string_value(p.until_one_of(string_ends)) + arg_close, PARAM_CLOSE));
+            auto arg_sep = p.optional(p.literal("\n"));
+
+            std::string call_close_gbnf = gbnf_format_literal("<|end|>");
+            for (const auto & tok : EOG) {
+                call_close_gbnf += " | " + gbnf_format_literal(tok);
+            }
+            auto call_close = p.gbnf(p.literal("<|end|>") | eog | p.peek(p.literal("<|start|>assistant")),
+                "(" + call_close_gbnf + ")?");
+
+            auto tool_choice = p.choice();
+            foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
+                const auto & function = tool.at("function");
+                std::string  name     = function.at("name");
+
+                std::vector<common_peg_parser> arg_rules;
+                foreach_parameter(function, [&](size_t param_index, const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
+                    auto rule_name = "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index);
+                    auto arg_open  = p.tool_arg_open(p.literal(PARAM_OPEN) + p.tool_arg_name(p.literal(param.name)) + p.literal("\">\n"));
+
+                    auto types     = param.schema->value_types();
+                    auto arg_value = p.eps();
+                    if (!types.has(common_chat_schema::TYPE_STRING) &&
+                        (types.has(common_chat_schema::TYPE_OBJECT) || types.has(common_chat_schema::TYPE_ARRAY))) {
+                        // the schema grammar fixes the key order, and the model writes keys in its own order
+                        arg_value = p.tool_arg_json_value(p.json()) + arg_close;
+                    } else if (!types.has(common_chat_schema::TYPE_STRING)) {
+                        arg_value = p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close;
+                    } else if (types.is_only(common_chat_schema::TYPE_STRING)) {
+                        arg_value = arg_string;
+                    } else {
+                        // grammar keeps only the string rule; the parser tries JSON first to type the value
+                        auto json_value = p.choice();
+                        if (types.has(common_chat_schema::TYPE_OBJECT)) {
+                            json_value |= p.json_object();
+                        }
+                        if (types.has(common_chat_schema::TYPE_ARRAY)) {
+                            json_value |= p.json_array();
+                        }
+                        if (types.has(common_chat_schema::TYPE_NUMBER) || types.has(common_chat_schema::TYPE_INTEGER)) {
+                            json_value |= p.json_number();
+                        }
+                        if (types.has(common_chat_schema::TYPE_BOOLEAN)) {
+                            json_value |= p.json_bool();
+                        }
+                        if (types.has(common_chat_schema::TYPE_NULL)) {
+                            json_value |= p.json_null();
+                        }
+                        arg_value = p.gbnf(p.atomic(p.tool_arg_json_value(json_value) + arg_close) | arg_string, "rfl-arg-string");
+                    }
+
+                    arg_rules.push_back(p.rule(rule_name, p.tool_arg(arg_open + arg_value)));
+                });
+
+                if (arg_rules.empty()) {
+                    auto any_name = p.chars("[^\"\\n<]", 1, -1);
+                    arg_rules.push_back(p.rule("tool-" + std::to_string(tool_index) + "-arg-any",
+                        p.tool_arg(p.tool_arg_open(p.literal(PARAM_OPEN) + p.tool_arg_name(any_name) + p.literal("\">\n")) + arg_string)));
+                }
+
+                auto args = p.zero_or_more(p.choice(arg_rules) + arg_sep);
+
+                auto call = p.tool(
+                    p.tool_open(p.literal(" to=") + p.tool_name(p.literal(name)) +
+                                p.literal("<|channel|>analysis <|constrain|>xml<|message|>")) +
+                    p.tool_args(args) +
+                    p.tool_close(call_close));
+
+                tool_choice |= p.rule("tool-" + std::to_string(tool_index), call);
+            });
+
+            auto tool_calls = inputs.parallel_tool_calls
+                ? tool_choice + p.zero_or_more(start + tool_choice)
+                : tool_choice;
+            auto tool_call = p.trigger_rule("tool-call", tool_calls);
+
+            if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
+                return start + p.zero_or_more(analysis + start) + tool_call;
+            }
+
+            return start + p.zero_or_more(analysis + start) + (tool_call | final_msg | p.content(p.rest()));
+        }
+
+        auto unsolicited = p.rule("unsolicited", p.atomic(p.literal(" to=") + p.rest()));
+
+        // output without a channel header is returned as content instead of failing the request
+        return start + p.zero_or_more(analysis + start) + (final_msg | unsolicited | p.content(p.rest()));
+    });
+
+    if (include_grammar) {
+        data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
+        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
+            data.parser.build_grammar(builder, data.grammar_lazy);
+        });
+
+        if (data.grammar_lazy) {
+            data.grammar_triggers = {
+                { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" },
+                { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" },
+            };
+        }
+    }
+
+    return data;
+}

common/parsers/parsers.h

@@ -59,6 +59,8 @@ common_chat_params common_chat_params_init_gigachat_v3(const common_chat_templat
 
 common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
 
+common_chat_params common_chat_params_init_harmony_xml(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
+
 common_chat_params common_chat_params_init_k2_horizon(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
 
 common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);

common/parsers/sources.cmake

@@ -9,6 +9,7 @@ set(LLAMA_CHAT_PARSERS_SOURCES
     ${CMAKE_CURRENT_LIST_DIR}/gemma4.cpp
     ${CMAKE_CURRENT_LIST_DIR}/gigachat-v3.cpp
     ${CMAKE_CURRENT_LIST_DIR}/gpt-oss.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/harmony-xml.cpp
     ${CMAKE_CURRENT_LIST_DIR}/k2-horizon.cpp
     ${CMAKE_CURRENT_LIST_DIR}/kimi-k2.cpp
     ${CMAKE_CURRENT_LIST_DIR}/kimi-k3.cpp

conversion/__init__.py

@@ -24,6 +24,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "ArcticForCausalLM": "arctic",
     "AudioFlamingo3ForConditionalGeneration": "qwen",
     "BaiChuanForCausalLM": "baichuan",
+    "BeamForCausalLM": "beam",
     "BaichuanForCausalLM": "baichuan",
     "BailingMoeForCausalLM": "bailingmoe",
     "BailingMoeV2ForCausalLM": "bailingmoe",
@@ -145,6 +146,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "JinaBertModel": "bert",
     "JinaEmbeddingsV5Model": "bert",
     "K2HorizonForCausalLM": "k2_horizon",
+    "Kolibri1ForCausalLM": "kolibri1",
     "KORMoForCausalLM": "qwen",
     "KimiK25ForConditionalGeneration": "deepseek",
     "KimiK3ForConditionalGeneration": "kimi_k3",

conversion/base.py

@@ -1872,6 +1872,9 @@ class TextModel(ModelBase):
         if chkhsh == "f5c965e4198da6e7525072d87d1ecf4c51d981fd486c7c4ec728ef9cec7e03fc":
             # ref: https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B
             res = "gpt-4o"
+        if chkhsh == "6e040dfe72e4b85855588c53acf4909ac4e98e55bc3a33cd7db499180dc42a78":
+            # ref: https://huggingface.co/Aleph-Alpha/Kolibri-1
+            res = "qwen2"
         if chkhsh == "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df":
             # ref: https://huggingface.co/aari1995/German_Semantic_V3
             res = "jina-v2-de"

conversion/beam.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+from typing import Callable, Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+    from torch import Tensor
+
+from .base import ModelBase, TextModel, gguf, logger
+
+
+@ModelBase.register("BeamForCausalLM")
+class BeamModel(TextModel):
+    model_arch = gguf.MODEL_ARCH.BEAM
+
+    # reference NoPE query-scale constants; not in config.json
+    NOPE_SCALE_POWER  = 0.1
+    NOPE_SCALE_LENGTH = 2048
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        hp = self.hparams
+        rope = hp.get("rope_parameters") or {}
+        if rope.get("full_attention") is not None:
+            raise ValueError("Beam: expected NoPE full-attention layers (rope_parameters.full_attention = null)")
+        if "sliding_attention" in rope:
+            swa_rope = rope["sliding_attention"] or {}
+        else:
+            # flat rope_theta / rope_scaling, already normalized by TextModel
+            swa_rope = {k: v for k, v in self.rope_parameters.items() if k not in ("sliding_attention", "full_attention", "type")}
+            swa_rope.update(self.rope_parameters.get("sliding_attention") or {})
+        if swa_rope.get("rope_type", "default") != "default":
+            raise ValueError(f"Beam: unsupported sliding-attention rope type {swa_rope.get('rope_type')!r}")
+        # full layers are NoPE, so mirror the sliding rope params for the base class
+        swa_rope = {k: v for k, v in swa_rope.items() if not (k == "rope_type" and v == "default")}
+        self.rope_parameters = {"sliding_attention": dict(swa_rope), "full_attention": dict(swa_rope)}
+
+        layer_types = hp["layer_types"]
+        mlp_types = hp["mlp_layer_types"]
+        n_layer = hp["num_hidden_layers"]
+        if len(layer_types) != n_layer or len(mlp_types) != n_layer:
+            raise ValueError("Beam: layer_types / mlp_layer_types must have num_hidden_layers entries")
+        if any(t not in ("sliding_attention", "full_attention") for t in layer_types):
+            raise ValueError(f"Beam: unknown layer type in {layer_types}")
+        if "sliding_attention" not in layer_types:
+            raise ValueError("Beam: expected at least one sliding_attention layer")
+        n_dense = 0
+        while n_dense < n_layer and mlp_types[n_dense] == "dense":
+            n_dense += 1
+        if any(t != "sparse" for t in mlp_types[n_dense:]):
+            raise ValueError("Beam: llama.cpp supports dense MLP layers only as a leading block")
+        self._n_dense_lead = n_dense
+        if hp.get("attention_bias", False):
+            raise ValueError("Beam: attention_bias=True is not supported")
+        self._experts: dict[tuple[int, str], dict[int, Tensor]] = {}
+
+    def set_vocab(self):
+        self._set_vocab_gpt2()
+        # otherwise EOT is guessed from a hash-ordered scan
+        from transformers import AutoTokenizer
+        tok = AutoTokenizer.from_pretrained(self.dir_model)
+        ret_id = tok.convert_tokens_to_ids("<|return|>")  # ty: ignore[unresolved-attribute]
+        if isinstance(ret_id, int) and ret_id >= 0:
+            self.gguf_writer.add_eot_token_id(ret_id)
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        hp = self.hparams
+
+        head_dim = hp["head_dim"]
+        self.gguf_writer.add_rope_dimension_count(head_dim)
+
+        self.gguf_writer.add_sliding_window(hp["sliding_window"])
+        self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hp["layer_types"]])
+
+        self.gguf_writer.add_attn_temperature_scale(self.NOPE_SCALE_POWER)
+        self.gguf_writer.add_attn_temperature_length(self.NOPE_SCALE_LENGTH)
+
+        self.gguf_writer.add_leading_dense_block_count(self._n_dense_lead)
+        self.gguf_writer.add_expert_feed_forward_length(hp["moe_intermediate_size"])
+        self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
+        self.gguf_writer.add_expert_weights_norm(bool(hp.get("norm_topk_prob", True)))
+        # equivalent to the reference's post-aggregation scaling
+        self.gguf_writer.add_expert_weights_scale(float(hp["routed_scaling_factor"]))
+
+        if (softcap := hp.get("final_logit_softcapping")) is not None:
+            self.gguf_writer.add_final_logit_softcapping(float(softcap))
+
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        name, gen = item
+        if name.endswith(".self_attn.gate_bias"):
+            name = name.replace(".self_attn.gate_bias", ".self_attn.gate_proj.bias")
+        return super().filter_tensors((name, gen))
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        hp = self.hparams
+        if name.endswith(".self_attn.gate_proj.bias"):
+            assert tuple(data_torch.shape) == (hp["num_attention_heads"],), f"{name}: one gate bias per head expected, got {tuple(data_torch.shape)}"
+        elif name.endswith(".self_attn.gate_proj.weight"):
+            assert tuple(data_torch.shape) == (hp["num_attention_heads"] * hp["head_dim"], hp["hidden_size"]), f"{name}: {tuple(data_torch.shape)}"
+        elif name.endswith(".mlp.gate.e_score_correction.bias"):
+            assert tuple(data_torch.shape) == (hp["num_experts"],), f"{name}: {tuple(data_torch.shape)}"
+        elif name.endswith((".self_attn.q_norm.weight", ".self_attn.k_norm.weight")):
+            assert tuple(data_torch.shape) == (hp["head_dim"],), f"{name}: per-head q/k norm expected, got {tuple(data_torch.shape)}"
+        elif bid is not None and bid < self._n_dense_lead and ".mlp.experts." in name:
+            raise ValueError(f"Beam: expert tensor {name} in a dense layer")
+        if ".mlp.experts." in name:
+            assert bid is not None
+            n_experts = self.hparams["num_experts"]
+            parts = name.split(".")
+            xid = int(parts[parts.index("experts") + 1])
+            proj = parts[parts.index("experts") + 2]
+            if proj not in ("gate_proj", "up_proj", "down_proj"):
+                raise ValueError(f"Beam: unexpected expert tensor {name}")
+            bucket = self._experts.setdefault((bid, proj), {})
+            if xid in bucket:
+                raise ValueError(f"Beam: duplicate expert tensor {name}")
+            bucket[xid] = data_torch
+            if len(bucket) < n_experts:
+                return
+            del self._experts[(bid, proj)]
+            data = torch.stack([bucket[e] for e in range(n_experts)], dim=0)
+            yield from super().modify_tensors(data, f"model.layers.{bid}.mlp.experts.{proj}.weight", bid)
+            return
+
+        yield from super().modify_tensors(data_torch, name, bid)
+
+    def prepare_tensors(self):
+        super().prepare_tensors()
+        if self._experts:
+            missing = sorted(f"layer {b} {p} ({len(v)} experts)" for (b, p), v in self._experts.items())
+            raise ValueError(f"Beam: unprocessed experts: {missing}")
+        logger.info("Beam: all routed experts stacked")

conversion/granite.py

@@ -8,7 +8,7 @@ import torch
 if TYPE_CHECKING:
     from torch import Tensor
 
-from .base import MmprojModel, ModelBase, gguf, logger
+from .base import MmprojModel, ModelBase, gguf, jinja_str_or_json, logger
 
 from .llama import LlamaModel
 from .mamba import Mamba2Model
@@ -284,10 +284,40 @@ class GraniteSwitchModel(GraniteMoeModel):
         self.gguf_writer.add_adapter_lora_rank(self._max_lora_rank)
         self.gguf_writer.add_adapter_token_ids_activate(self.hparams["adapter_token_ids"])
         self.gguf_writer.add_adapter_token_ids_substitute(self.hparams["adapter_substitute_token_ids"])
+        if self._has_decision_adapter():
+            self.gguf_writer.add_decision_type(gguf.DecisionType.GRANITE_SWITCH)
         router_gain = float(self.hparams.get("control_token_gain", 15.0))
         self.gguf_writer.add_adapter_router_gain(router_gain)
         logger.info("gguf: (graniteswitch) num_adapters=%s max_lora_rank=%s n_slots=%s router_gain=%s", self._n_adapters, self._max_lora_rank, self._n_slots, router_gain)
 
+    def _has_decision_adapter(self) -> bool:
+        return "decision" in self.hparams.get("adapter_names", [])
+
+    def set_vocab(self):
+        super().set_vocab()
+        if self._has_decision_adapter():
+            systemone_template = self._systemone_template()
+            logger.info("System One Template: %s", systemone_template)
+            self.gguf_writer.add_chat_template([{"name": "systemone", "template": systemone_template}])
+
+    def _systemone_template(self) -> str:
+        # prompt of the decision aLoRA: a state turn, then a question turn that opens with the control token
+        # (it replaces the first token of "### DECISION"), the answer is the label token that comes next
+        description = jinja_str_or_json("o.description")
+        option = (
+            "{% if type != 'noul' %}{{ o.label }}: {% if o.description %}" + description + "{% endif %}"
+            "{% elif o.key == 'true' %}A: {% if o.description %}" + description + "{% else %}The statement is true.{% endif %}"
+            "{% else %}B: {% if o.description %}" + description + "{% else %}The statement is false.{% endif %}{% endif %}"
+        )
+        return (
+            "<|start_of_role|>user<|end_of_role|>State:\n" + jinja_str_or_json("state") + "<|end_of_text|>\n"
+            "<|start_of_role|>user<|end_of_role|><|decision|> DECISION\nQuestion:\n" + jinja_str_or_json("instructions")
+            + "{% if type == 'score' %} Rate along the ordered levels below (lowest first).{% endif %}"
+            "{{ '\n\nOptions:\n' }}"
+            "{% for o in options %}" + option + "{{ '\n' }}{% endfor %}"
+            "{{ '\nReply with exactly one option label.<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>' }}"
+        )
+
     def _lora_a(self, data: Tensor) -> Tensor:
         # on-disk A: [n_adapters, 1, max_rank, in] -> [n_adapters+1, max_rank, in]
         a = data.squeeze(1)

conversion/kolibri1.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from .afmoe import AfmoeModel
+from .base import ModelBase, gguf
+
+
+@ModelBase.register("Kolibri1ForCausalLM")
+@ModelBase.example("Aleph-Alpha/Kolibri-1")
+class Kolibri1Model(AfmoeModel):
+    model_arch = gguf.MODEL_ARCH.KOLIBRI1
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # AfmoeModel reads these keys, this config stores the same values under other names
+        self.hparams["num_shared_experts"] = 1
+        self.hparams["route_norm"] = self.hparams["norm_topk_prob"]
+        self.hparams["intermediate_size"] = self.hparams["moe_intermediate_size"]
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        self.gguf_writer.add_expert_shared_feed_forward_length(self.hparams["shared_expert_intermediate_size"])
+        self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in self.hparams["layer_types"]])
+        self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)

convert_hf_to_gguf_update.py

@@ -191,6 +191,8 @@ pre_computed_hashes = [
     {"name": "grok-2",    "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/alvarobartt/grok-2-tokenizer", "chkhsh": "66b8d4e19ab16c3bfd89bce5d785fb7e0155e8648708a1f42077cb9fe002c273"},
     # muse-spark uses the l4_202k tokenizer, whose pre-tokenizer regex is gpt-4o's
     {"name": "gpt-4o",    "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "chkhsh": "f5c965e4198da6e7525072d87d1ecf4c51d981fd486c7c4ec728ef9cec7e03fc"},
+    # kolibri1 uses a 128k vocabulary with the qwen2 pre-tokenizer regex
+    {"name": "qwen2",     "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Aleph-Alpha/Kolibri-1", "chkhsh": "6e040dfe72e4b85855588c53acf4909ac4e98e55bc3a33cd7db499180dc42a78"},
     # jina-v2-de variants
     {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"},
     {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"},

docs/backend/snapdragon/developer.md

@@ -48,6 +48,157 @@ Alternatively, users can partition and split the model across multiple virtual s
 tensor-splitting, or row-splitting modes. For user-facing execution modes and examples, see the
 [Snapdragon user guide](README.md#multi-device-execution-modes).
 
+## Tiled Weight Formats
+
+Weight tensors in the Hexagon backend are repacked from flat row-major GGML layouts into 2D tiles in memory.
+Tiling groups 2D matrix blocks contiguously:
+- Eliminates strided row reads when fetching 2D tiles into VTCM via DMA.
+- Matches native HVX vector width (128 bytes) and HMX systolic array dimensions (32x32).
+- Repacking happens on the host during model load (`set_tensor()`) or via runtime repacking (`rt-repack`).
+Implementation: [`htp-repack.cpp`](../../../ggml/src/ggml-hexagon/htp-repack.cpp).
+
+### Tile Dimensions and Grid Layout
+
+Each tile represents a submatrix of `N_rows = 32` rows and `K_cols` columns:
+- Standard types (`Q4_0`, `Q4_1`, `Q8_0`, `IQ4_NL`, `MXFP4`, and K-quants): `K_cols = 32` (`HTP_TENSOR_TILE_N_COLS = 32`).
+- `Q2_0`: `K_cols = 64` (`HTP_TENSOR_TILE_K_Q2_0 = 64`).
+- `Q1_0`: `K_cols = 128` (`HTP_TENSOR_TILE_K_Q1_0 = 128`).
+
+For a weight matrix of logical shape `[ne0, ne1]` (where `ne0 = K`, `ne1 = N`):
+- `n_k_tiles   = ne0 / K_cols`
+- `n_col_tiles = (ne1 + 31) / 32`
+- Tile index for column-tile `kt` and row-tile `ct`: `tile_idx = ct * n_k_tiles + kt`
+- Byte offset in matrix buffer: `tile_offset = tile_idx * tile_size`
+
+### Block-32 Quantized Formats
+
+Block-32 formats use a tile size of 32 rows x 32 columns (`K_cols = 32`):
+
+- **Q4_0** (576 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q4_0`):
+  - **Quants** (bytes 0..511, 512 bytes):
+    Contains 32 rows x 32 columns of 4-bit quants.
+    Divided into 16 column pairs `cp = 0..15` covering columns `2*cp` and `2*cp + 1`.
+    For each column pair `cp`, 32 bytes are stored consecutively for rows `row = 0..31`:
+    `tile[cp * 32 + row]` holds `(quant[row][2*cp + 1] << 4) | quant[row][2*cp]`.
+    Low nibble holds the even column; high nibble holds the odd column.
+  - **Scales** (bytes 512..575, 64 bytes):
+    32 FP16 values (`ggml_half`), storing scale `d` for each row `row = 0..31`.
+
+- **Q8_0** (1088 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q8_0`):
+  - **Quants** (bytes 0..1023, 1024 bytes):
+    Contains 32 rows x 32 columns of signed 8-bit quants (`int8_t`).
+    Divided into 16 column pairs `cp = 0..15` covering columns `col0 = 2*cp` and `col1 = 2*cp + 1`.
+    For each column pair `cp`, 64 bytes store interleaved column elements for each row `row = 0..31`:
+    `tile[cp * 64 + 2 * row + 0] = quant[row][col0]`
+    `tile[cp * 64 + 2 * row + 1] = quant[row][col1]`
+  - **Scales** (bytes 1024..1087, 64 bytes):
+    32 FP16 values (`ggml_half`), storing scale `d` for each row `row = 0..31`.
+
+- **IQ4_NL** (576 bytes per tile, `HTP_TENSOR_TILE_SIZE_IQ4_NL`):
+  - **Quants** (bytes 0..511, 512 bytes):
+    Same nibble transposition and storage layout as `Q4_0`.
+    Contains 32 rows x 32 columns of non-linear 4-bit indices (values 0..15).
+    Divided into 16 column pairs `cp = 0..15`:
+    `tile[cp * 32 + row]` holds `(quant[row][2*cp + 1] << 4) | quant[row][2*cp]`.
+  - **Scales** (bytes 512..575, 64 bytes):
+    32 FP16 values (`ggml_half`), storing scale `d` for each row `row = 0..31`.
+  - **Dequantization**:
+    In HVX/HMX kernels, each 4-bit index is looked up in the 16-entry non-linear table (`iq4_nl_to_fp16_lut`) to produce an FP16 value, then multiplied by the row scale `d`.
+
+- **MXFP4** (544 bytes per tile, `HTP_TENSOR_TILE_SIZE_MXFP4`):
+  - **Quants** (bytes 0..511, 512 bytes):
+    Same nibble transposition and storage layout as `Q4_0`.
+    Contains 32 rows x 32 columns of 4-bit OCP Microscaling FP4 (E2M1) values.
+    Divided into 16 column pairs `cp = 0..15`:
+    `tile[cp * 32 + row]` holds `(quant[row][2*cp + 1] << 4) | quant[row][2*cp]`.
+  - **Scales** (bytes 512..543, 32 bytes):
+    32 unsigned 8-bit values (`uint8_t`), storing one 8-bit biased exponent (E8M0, bias 127) scale `e` per row `row = 0..31`.
+  - **Dequantization**:
+    Kernel converts E8M0 bias to IEEE FP16 bias via `vh = clamp(e - 112, 0, 30) << 10` to form a power-of-two FP16 scale.
+    Each 4-bit FP4 nibble is unpacked via `mxfp4_to_fp16_lut` and multiplied by this scale.
+
+- **Q4_1** (640 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q4_1`):
+  - **Quants** (bytes 0..511, 512 bytes):
+    Same 4-bit nibble transposition and storage layout as `Q4_0`.
+  - **Scales** (bytes 512..639, 128 bytes):
+    32 pairs of FP16 values (`ggml_half`): `scale[2 * row + 0] = d` (scale) and `scale[2 * row + 1] = m` (min) for each row `row = 0..31`.
+
+### Q1_0 Format (1-Bit Quantization)
+
+`Q1_0` packs 128 columns per tile (`K_cols = 128`, `HTP_TENSOR_TILE_K_Q1_0 = 128`):
+
+- **Tile Layout** (576 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q1_0`):
+  - **Quants** (bytes 0..511, 512 bytes):
+    Contains 32 rows x 128 columns of 1-bit quants (-1 or +1).
+    Divided into 4 quarters (`quarter = 0..3`), where each quarter covers 32 columns in K.
+    Each quarter occupies 128 bytes (1 HVX vector) in `vrmpy` byte order.
+    Within a quarter, columns `kq = 0..31` map into 8 groups `g = kq / 4` (0..7) of 4 consecutive columns:
+    `byte_offset = quarter * 128 + 4 * row + (kq % 4)`
+    `bit = g` (0..7)
+  - **Scales** (bytes 512..575, 64 bytes):
+    32 FP16 values (`ggml_half`), storing scale `d` for each row `row = 0..31`.
+    One scale per row applies to all 128 columns (all 4 quarters) of the tile.
+
+- **HMX Mapping (1-to-4 Tile Expansion)**:
+  - HMX hardware operates strictly on 32x32 FP16 tiles (`HTP_TENSOR_TILE_N_ROWS = 32`, `HTP_TENSOR_TILE_N_COLS = 32`).
+  - During kernel execution, HVX dequantization (`dequantize_tiled_weight_to_fp16_task_q1_0`) expands each 32x128 Q1_0 tile (576 bytes) into 4 contiguous 32x32 FP16 HMX tiles (4 x 2048 bytes = 8192 bytes total) in VTCM.
+  - HMX systolic array processes the 4 resulting 32x32 FP16 tiles over 4 compute steps along K.
+
+### Q2_0 Format (2-Bit Quantization)
+
+`Q2_0` packs 64 columns per tile (`K_cols = 64`, `HTP_TENSOR_TILE_K_Q2_0 = 64`):
+
+- **Tile Layout** (576 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q2_0`):
+  - **Quants** (bytes 0..511, 512 bytes):
+    Contains 32 rows x 64 columns of 2-bit quants (codes 0..3 representing signed values -1..2).
+    Divided into 16 column groups `group = 0..15` of 4 columns each.
+    Stored in `vrmpy` byte order across 4 HVX vectors (vec 0..3):
+    Vector `group >> 2` (0..3) holds `group & 3` at bit offset `2 * (group & 3)`.
+    Byte `4 * row + byte` holds row `row = 0..31` and column byte `0..3`.
+    Vec 0..1 holds the first 32 columns (half 0), vec 2..3 holds the next 32 columns (half 1).
+  - **Scales** (bytes 512..575, 64 bytes):
+    32 FP16 values (`ggml_half`), storing scale `d` for each row `row = 0..31` (at offset `512 + 2 * row`).
+    One scale per row applies to all 64 columns of the tile.
+
+- **HMX Mapping (1-to-2 Tile Expansion)**:
+  - HMX hardware operates strictly on 32x32 FP16 tiles (`HTP_TENSOR_TILE_N_ROWS = 32`, `HTP_TENSOR_TILE_N_COLS = 32`).
+  - During kernel execution, HVX dequantization (`dequantize_tiled_weight_to_fp16_task_q2_0`) expands each 32x64 Q2_0 tile (576 bytes) into 2 contiguous 32x32 FP16 HMX tiles (2 x 2048 bytes = 4096 bytes total) in VTCM.
+  - HMX systolic array processes the 2 resulting 32x32 FP16 tiles over 2 compute steps along K.
+
+- **HVX Vector Dot-Product**:
+  - `tiled_vec_dot_q2_0_32x1` and `tiled_vec_dot_q2_0_32x2` unpack 2-bit quants using `unpack_q3_k_low2`, subtract 1, and accumulate against Q8_0 activation tiles using `Q6_Vw_vrmpyacc_VwVbVb`.
+
+### K-Quants Formats
+
+In GGML, K-quants pack 256 elements per super-block.
+On Hexagon, each 256-element super-block splits along K into 8 tiles of 32 columns (`K_cols = 32`).
+
+Byte ordering: `Q2_K`, `Q3_K`, and `Q6_K` use `vrmpy` byte order within each 128B HVX vector:
+`k = 0..31` is split into 8 groups `g = k/4` of 4 `k` each; byte `4*r + (k%4)` holds row `r`.
+
+- **Q2_K** (336 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q2_K`, lossless tiling with same size as `block_q2_K`):
+  - `bytes   0..255` (vec 0..1): unsigned 2-bit quants; vec `m` holds groups `4m..4m+3` at bit offsets 0, 2, 4, 6.
+  - `bytes 256..319`: raw `scales[]` bytes (`min << 4 | sc`): `[0..31]` for k 0..15, `[32..63]` for k 16..31.
+  - `bytes 320..335`: 8 FP16 in tile `j` (0..7) of super-block: `d` (`j < 4`) or `-dmin` (`j >= 4`) for rows `8*(j%4)..8*(j%4)+7`.
+
+- **Q3_K** (512 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q3_K`):
+  - `bytes   0..255` (vec 0..1): low 2 bits; vec `m` holds groups `4m..4m+3` at bit offsets 0, 2, 4, 6.
+  - `bytes 256..383` (vec 2): sign mask, bit `g` set when `hmask` is 0 (quant = `low2 - 4`).
+  - `bytes 384..511` (vec 3): FP16 scales `d * (scales[] - 32)`: `fp16[0..31]` for k 0..15, `fp16[32..63]` for k 16..31.
+
+- **Q4_K** (640 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q4_K`, same layout as Q4_1 tile; each 256-k super-block splits into 8 tiles of 32 k):
+  - `bytes   0..511` (vec 0..3): 4-bit quants; byte `32*cp + r` holds `k = 2*cp` (low nibble) and `2*cp + 1` (high nibble).
+  - `bytes 512..639` (vec 4): 32x2 FP16 per row: `(d * sc, -dmin * m)`.
+
+- **Q5_K** (768 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q5_K`):
+  - `bytes   0..639` (vec 0..4): Q4_K tile (low 4 bits + FP16 scale/min pairs).
+  - `bytes 640..767` (vec 5): 5th-bit plane; byte `l` holds the 8 high bits for lane `l` across vec `i = 0..3` (`bit 2i = lo, 2i+1 = hi`).
+
+- **Q6_K** (896 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q6_K`):
+  - `bytes   0..511` (vec 0..3): low 4 bits; vec `i` holds group `2i` (low nibble) and `2i+1` (high nibble).
+  - `bytes 512..767` (vec 4..5): high 2 bits; vec `4+m` holds groups `4m..4m+3` at bit offsets 0, 2, 4, 6.
+  - `bytes 768..895` (vec 6): FP16 scales `d * scales[]`: `fp16[0..31]` for k 0..15, `fp16[32..63]` for k 16..31.
+
 ## Op and Kernel Development Guidelines
 
 Writing high-performance operators for Hexagon requires following specific guidelines.

ggml/src/ggml-cuda/fattn.cu

@@ -417,9 +417,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
 
 #define FATTN_VEC_CASE(D, type_K_case, type_V_case)                                                                                \
     if constexpr (GGML_CUDA_FA_##type_K_case##_##type_V_case) {                                                                    \
-        const bool type_K_okay = type_K == GGML_TYPE_##type_K_case || (type_K == GGML_TYPE_F32 && GGML_TYPE_##type_K_case == GGML_TYPE_F16); \
-        const bool type_V_okay = type_V == GGML_TYPE_##type_V_case || (type_V == GGML_TYPE_F32 && GGML_TYPE_##type_V_case == GGML_TYPE_F16); \
-        if (head_size == (D) && type_K_okay && type_V_okay) {                                                                      \
+        if (head_size == (D) && type_K == GGML_TYPE_##type_K_case && type_V == GGML_TYPE_##type_V_case) {                          \
             return ggml_cuda_flash_attn_ext_vec_case<D, GGML_TYPE_##type_K_case, GGML_TYPE_##type_V_case>;                         \
         }                                                                                                                          \
     }                                                                                                                              \
@@ -432,7 +430,9 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
 typedef void (* fattn_vec_case_t)(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
 
 // Vector kernel for the given head size and K/V types, nullptr if its template instance was not compiled:
-static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, const ggml_type type_K, const ggml_type type_V) {
+static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, const ggml_type type_K_in, const ggml_type type_V_in) {
+    const ggml_type type_K = type_K_in == GGML_TYPE_F32 ? GGML_TYPE_F16 : type_K_in;
+    const ggml_type type_V = type_V_in == GGML_TYPE_F32 ? GGML_TYPE_F16 : type_V_in;
     FATTN_VEC_CASES_ALL_D(F16,  F16)
     FATTN_VEC_CASES_ALL_D(Q4_0, F16)
     FATTN_VEC_CASES_ALL_D(Q4_1, F16)

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -3029,6 +3029,65 @@ static int ggml_cuda_try_ssm_scan_cache_fusion(
     return skip;
 }
 
+// expert group selection from build_moe_ffn (n_expert_groups > 1)
+// src0/src1 index into this table, -1 is the (biased) probs, -2 is not checked
+static const struct {
+    ggml_op op;
+    int     src0;
+    int     src1;
+} topk_moe_group_pattern[] = {
+    { GGML_OP_RESHAPE,  -1, -2 }, //  0: [n_exp_per_group, n_expert_groups, n_tokens]
+    { GGML_OP_RESHAPE,   0, -2 }, //  1: [1, n_exp_per_group, n_expert_groups, n_tokens]
+    { GGML_OP_ARGSORT,   0, -2 }, //  2
+    { GGML_OP_VIEW,      2, -2 }, //  3: top-2 per group
+    { GGML_OP_GET_ROWS,  1,  3 }, //  4
+    { GGML_OP_RESHAPE,   4, -2 }, //  5
+    { GGML_OP_SUM_ROWS,  5, -2 }, //  6: group scores
+    { GGML_OP_RESHAPE,   6, -2 }, //  7
+    { GGML_OP_ARGSORT,   7, -2 }, //  8
+    { GGML_OP_VIEW,      8, -2 }, //  9: top n_group_used groups
+    { GGML_OP_GET_ROWS,  0,  9 }, // 10
+    { GGML_OP_FILL,      0, -2 }, // 11
+    { GGML_OP_SET_ROWS, 10,  9 }, // 12: unused groups -> -inf
+    { GGML_OP_RESHAPE,  12, -2 }, // 13: [n_expert, n_tokens]
+};
+
+static bool ggml_cuda_topk_moe_group_fusion(const struct ggml_cgraph * cgraph, int node_idx, const ggml_tensor * selection, ggml_cuda_topk_moe_args & args) {
+#if defined(GGML_USE_HIP) && HIP_VERSION < 60300000
+    GGML_UNUSED_VARS(cgraph, node_idx, selection, args);
+    return false;
+#else
+    const int n_ops = (int) std::size(topk_moe_group_pattern);
+    if (node_idx + n_ops > cgraph->n_nodes) {
+        return false;
+    }
+
+    ggml_tensor ** nodes = cgraph->nodes + node_idx;
+    auto src_ok = [&](const ggml_tensor * src, int idx) {
+        return idx == -2 || src == (idx == -1 ? selection : nodes[idx]);
+    };
+
+    for (int i = 0; i < n_ops; i++) {
+        const auto & p = topk_moe_group_pattern[i];
+        if (nodes[i]->op != p.op || !src_ok(nodes[i]->src[0], p.src0) || !src_ok(nodes[i]->src[1], p.src1)) {
+            return false;
+        }
+    }
+
+    const int n_groups     = nodes[0]->ne[1];
+    const int n_group_used = nodes[9]->ne[0];
+
+    if (nodes[3]->ne[0] != 2 || nodes[12]->src[2] != nodes[11] || ggml_get_op_params_f32(nodes[11], 0) != -INFINITY ||
+            n_groups > WARP_SIZE) {
+        return false;
+    }
+
+    args.n_groups     = n_groups;
+    args.n_group_used = n_group_used;
+    return true;
+#endif // defined(GGML_USE_HIP) && HIP_VERSION < 60300000
+}
+
 static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int node_idx, ggml_cuda_topk_moe_args & args) {
     args.sigmoid         = false;
     args.sqrt_softplus   = false;
@@ -3036,6 +3095,8 @@ static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int nod
     args.delayed_softmax = false;
     args.prob_bias       = false;
     args.norm            = false;
+    args.n_groups        = 0;
+    args.n_group_used    = 0;
 
     const int      n_nodes = cgraph->n_nodes;
     ggml_tensor ** nodes   = cgraph->nodes;
@@ -3078,18 +3139,20 @@ static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int nod
         }
 
         // src of bias add is the unreshaped probs (-2 instead of -1)
-        if (nodes[node_idx]->op == GGML_OP_ADD && nodes[node_idx]->src[0] == nodes[node_idx - 2]) {
+        const ggml_tensor * selection = nodes[node_idx - 2];
+        if (nodes[node_idx]->op == GGML_OP_ADD && nodes[node_idx]->src[0] == selection) {
             args.prob_bias = true;
+            selection = nodes[node_idx];
             node_idx++;
         }
-        // RESHAPE/ADD -> ARGSORT
-        if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_ARGSORT) {
-            return false;
+
+        if (ggml_cuda_topk_moe_group_fusion(cgraph, node_idx, selection, args)) {
+            node_idx += (int) std::size(topk_moe_group_pattern);
+            selection = nodes[node_idx - 1];
         }
 
-        if (args.prob_bias && nodes[node_idx]->src[0] != nodes[node_idx - 1]) {
-            return false;
-        } else if (!args.prob_bias && nodes[node_idx]->src[0] != nodes[node_idx - 2]) {
+        // RESHAPE/ADD/group mask -> ARGSORT
+        if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_ARGSORT || nodes[node_idx]->src[0] != selection) {
             return false;
         }
 
@@ -3100,6 +3163,11 @@ static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int nod
                 nodes[node_idx]->src[0] != nodes[node_idx - 1]) {
             return false;
         }
+
+        // kept groups must hold all used experts
+        if (args.n_groups > 0 && nodes[node_idx]->ne[0] > args.n_group_used * (probs_reshaped->ne[1] / args.n_groups)) {
+            return false;
+        }
         node_idx++;
 
         if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_GET_ROWS) {
@@ -3754,17 +3822,18 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
                 } else {
                     ops.insert(ops.end(), { GGML_OP_SOFT_MAX });
                 }
-                const int i_probs = i + (int) ops.size() - 1;  // last node of the gating activation
-
+                ops.push_back(GGML_OP_RESHAPE);
                 if (args.prob_bias) {
-                    bias = cgraph->nodes[i_probs + 2]->src[1];
-                    ops.insert(ops.end(), { GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW,
-                                            GGML_OP_GET_ROWS });
-                    out_nodes[0] = i_probs + 4;
-                } else {
-                    ops.insert(ops.end(), { GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS });
-                    out_nodes[0] = i_probs + 3;
+                    ops.push_back(GGML_OP_ADD);
+                    bias = cgraph->nodes[i + ops.size() - 1]->src[1];
                 }
+                if (args.n_groups > 0) {
+                    for (const auto & p : topk_moe_group_pattern) {
+                        ops.push_back(p.op);
+                    }
+                }
+                ops.insert(ops.end(), { GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS });
+                out_nodes[0] = i + ops.size() - 2;
                 ids = cgraph->nodes[out_nodes[0]];
 
                 if (args.norm) {

ggml/src/ggml-cuda/mmq-load-tiles.cuh

@@ -46,6 +46,25 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
         for (int j = 0; j < 2; ++j) {
             const int q  = qxi[j];
 
+#if defined(GGML_USE_HIP)
+            const uint32_t q_bits = (uint16_t) q;
+
+            // Expand each group of four Q1_0 bits into byte selectors for v_perm_b32.
+            const uint32_t q0_bits = q_bits >>  0;
+            const uint32_t q1_bits = q_bits >>  4;
+            const uint32_t q2_bits = q_bits >>  8;
+            const uint32_t q3_bits = q_bits >> 12;
+
+            const uint32_t q0_indices = (q0_bits & 0x01) | ((q0_bits & 0x02) << 7) | ((q0_bits & 0x04) << 14) | ((q0_bits & 0x08) << 21);
+            const uint32_t q1_indices = (q1_bits & 0x01) | ((q1_bits & 0x02) << 7) | ((q1_bits & 0x04) << 14) | ((q1_bits & 0x08) << 21);
+            const uint32_t q2_indices = (q2_bits & 0x01) | ((q2_bits & 0x02) << 7) | ((q2_bits & 0x04) << 14) | ((q2_bits & 0x08) << 21);
+            const uint32_t q3_indices = (q3_bits & 0x01) | ((q3_bits & 0x02) << 7) | ((q3_bits & 0x04) << 14) | ((q3_bits & 0x08) << 21);
+
+            const int v0 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q0_indices);
+            const int v1 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q1_indices);
+            const int v2 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q2_indices);
+            const int v3 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q3_indices);
+#else
             // unpack crumbs into nibble indices
             const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0); // [0, 1, 4, 5] [ 8,  9, 12, 13]
             const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2); // [2, 3, 6, 7] [10, 11, 14, 15]
@@ -59,6 +78,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
             const int v1 = __byte_perm(s0, s1, 0x7632);
             const int v2 = __byte_perm(s2, s3, 0x5410);
             const int v3 = __byte_perm(s2, s3, 0x7632);
+#endif // defined(GGML_USE_HIP)
 
 #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
             x_qs[i*sram_stride           + dst_offset + j*4+0] = v0;

ggml/src/ggml-cuda/mmvq.cu

@@ -468,10 +468,12 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d
     } else if (table_id == MMVQ_PARAMETERS_GCN) {
         switch (ncols_dst) {
             case 1:
+                return 2;
             case 2:
             case 3:
+                return type == GGML_TYPE_Q3_K ? 1 : 2;
             case 4:
-                return 2;
+                return type == GGML_TYPE_Q6_K? 1 : 2;
             case 5:
             case 6:
             case 7:
@@ -576,11 +578,31 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d
     return 1;
 }
 
-static constexpr __host__ __device__ int calc_rows_per_block(int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) {
+// Low bit types load few weight bytes per thread, a second row per block shares the q8_1 loads and adds memory parallelism.
+static constexpr __host__ __device__ bool mmvq_two_rows_ncols1(ggml_type type) {
+    switch (type) {
+        case GGML_TYPE_Q1_0:
+        case GGML_TYPE_Q2_K:
+        case GGML_TYPE_Q3_K:
+        case GGML_TYPE_IQ1_S:
+        case GGML_TYPE_IQ1_M:
+            return true;
+        default:
+            return false;
+    }
+}
+
+static constexpr __host__ __device__ int calc_rows_per_block(ggml_type type, int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) {
     if (table_id == MMVQ_PARAMETERS_GENERIC || table_id == MMVQ_PARAMETERS_GCN || table_id == MMVQ_PARAMETERS_TURING || table_id == MMVQ_PARAMETERS_GB10) {
         switch (ncols_dst) {
             case 1:
-                return small_k ? nwarps : 1;
+                if (small_k) {
+                    return nwarps;
+                }
+                if (table_id == MMVQ_PARAMETERS_GB10) {
+                    return type == GGML_TYPE_IQ2_S || type == GGML_TYPE_IQ3_S ? 2 : 1;
+                }
+                return table_id == MMVQ_PARAMETERS_GENERIC && mmvq_two_rows_ncols1(type) ? 2 : 1;
             case 2:
             case 3:
             case 4:
@@ -615,7 +637,7 @@ static __global__ void mul_mat_vec_q(
     constexpr int vdr = get_vdr_mmvq(type);
     constexpr mmvq_parameter_table_id table_id = get_device_table_id();
     constexpr int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters);
-    constexpr int rows_per_cuda_block = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps);
+    constexpr int rows_per_cuda_block = calc_rows_per_block(type, ncols_dst, table_id, small_k, nwarps);
     constexpr int warp_size = ggml_cuda_get_physical_warp_size();
 
     constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type);
@@ -727,7 +749,8 @@ static __global__ void mul_mat_vec_q(
 #if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
         // start the next iterations' weight loads early
         if constexpr (mmvq_should_prefetch(type)) {
-            constexpr int pf_dist = 2; // loop iterations, not blocks
+            // loop iterations, not blocks; q3_K and q6_K keep the distance in blocks they had with vdr 1
+            constexpr int pf_dist = type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q6_K ? 1 : 2;
             const int kbx_pf = kbx + pf_dist*blocks_per_iter;
             if (kbx_pf < blocks_per_row_x) {
 #pragma unroll
@@ -1013,7 +1036,7 @@ static std::pair<dim3, dim3> calc_launch_params(
         const int ncols_dst, const int nrows_x, const int nchannels_dst, const int nsamples_or_ntokens,
         const int warp_size, const mmvq_parameter_table_id table_id, const bool small_k = false, const bool halve_iters = false) {
     const int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters);
-    const int rpb = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps);
+    const int rpb = calc_rows_per_block(type, ncols_dst, table_id, small_k, nwarps);
     const int64_t nblocks = (nrows_x + rpb - 1) / rpb;
     const dim3 block_nums(nblocks, nchannels_dst, nsamples_or_ntokens);
     const dim3 block_dims(warp_size, nwarps, 1);

ggml/src/ggml-cuda/topk-moe.cu

@@ -11,6 +11,8 @@ struct topk_moe_config {
     bool use_sqrt_softplus;
     bool with_norm;
     bool delayed_softmax;
+    int  n_groups;
+    int  n_group_used;
 };
 
 // Warp-local softmax used for both the pre-top-k logits and the post-top-k delayed path.
@@ -78,6 +80,53 @@ __device__ void sqrt_softplus_warp_inplace(float (&vals)[experts_per_thread], co
     }
 }
 
+#if !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000
+// Set -inf for experts outside the top n_group_used groups. Group score is the sum of its top-2 experts (DeepSeek V3).
+template <int n_experts, int experts_per_thread>
+__device__ void group_mask_warp_inplace(float (&vals)[experts_per_thread], const int n_groups, const int n_group_used, const int lane) {
+    const int n_per_group = n_experts / n_groups;
+
+    float score = -INFINITY; // lane g holds the score of group g
+    for (int g = 0; g < n_groups; g++) {
+        float top1 = -INFINITY;
+        float top2 = -INFINITY;
+#pragma unroll
+        for (int i = 0; i < experts_per_thread; i++) {
+            if ((i * WARP_SIZE + lane) / n_per_group == g) {
+                top2 = fmaxf(top2, fminf(top1, vals[i]));
+                top1 = fmaxf(top1, vals[i]);
+            }
+        }
+#pragma unroll
+        for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) {
+            const float o1 = __shfl_xor_sync(0xFFFFFFFF, top1, offset, WARP_SIZE);
+            const float o2 = __shfl_xor_sync(0xFFFFFFFF, top2, offset, WARP_SIZE);
+            top2 = fmaxf(fminf(top1, o1), fmaxf(top2, o2));
+            top1 = fmaxf(top1, o1);
+        }
+        if (lane == g) {
+            score = top1 + top2;
+        }
+    }
+
+    // rank of each group, lower index wins ties
+    int rank = 0;
+    for (int g = 0; g < n_groups; g++) {
+        const float other = __shfl_sync(0xFFFFFFFF, score, g, WARP_SIZE);
+        rank += other > score || (other == score && g < lane);
+    }
+    const auto keep = __ballot_sync(0xFFFFFFFFULL, lane < n_groups && rank < n_group_used);
+
+#pragma unroll
+    for (int i = 0; i < experts_per_thread; i++) {
+        const int g = (i * WARP_SIZE + lane) / n_per_group;
+        if (g >= n_groups || !((keep >> g) & 1)) {
+            vals[i] = -INFINITY;
+        }
+    }
+}
+#endif // !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000
+
 /*
     This kernel does the following:
     1. optionally softmax over the logits per token [n_experts, n_tokens]
@@ -171,6 +220,17 @@ __global__ void topk_moe_cuda(const float *         logits,
         }
     }
 
+#if !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000
+    // without bias, masking wt is safe: the matcher ensures the kept groups hold all used experts
+    if (config.n_groups > 0) {
+        if constexpr (has_bias) {
+            group_mask_warp_inplace<n_experts>(selection_wt, config.n_groups, config.n_group_used, threadIdx.x);
+        } else {
+            group_mask_warp_inplace<n_experts>(wt, config.n_groups, config.n_group_used, threadIdx.x);
+        }
+    }
+#endif // !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000
+
     //at this point, each thread holds either a portion of the softmax distribution
     //or the raw logits. We do the argmax reduce over n_expert_used, each time marking
     //the expert weight as -inf to exclude from the next iteration
@@ -297,6 +357,10 @@ static void launch_topk_moe_cuda(ggml_backend_cuda_context & ctx,
     cudaStream_t stream = ctx.stream();
     const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream);
 
+#if defined(GGML_USE_HIP) && HIP_VERSION < 60300000
+    GGML_ASSERT(config.n_groups == 0 && "expert groups fusion is not supported on this HIP version");
+#endif
+
     switch (n_expert) {
         case 1:
             ggml_cuda_kernel_launch(topk_moe_cuda<1, has_bias>, launch_params,
@@ -389,6 +453,8 @@ void ggml_cuda_op_topk_moe(ggml_backend_cuda_context &     ctx,
     config.use_sqrt_softplus = args.sqrt_softplus;
     config.with_norm         = with_norm;
     config.delayed_softmax   = args.delayed_softmax;
+    config.n_groups          = args.n_groups;
+    config.n_group_used      = args.n_group_used;
 
     if (bias) {
         launch_topk_moe_cuda<true>(ctx, logits_d, weights_d, ids_d, bias_d, n_rows, n_experts, n_expert_used, clamp_val,

ggml/src/ggml-cuda/topk-moe.cuh

@@ -14,6 +14,8 @@ struct ggml_cuda_topk_moe_args {
     bool prob_bias{};
     bool norm{};
     bool scale{};
+    int  n_groups{};     // > 0: pick experts only from the top n_group_used expert groups
+    int  n_group_used{};
 };
 
 void ggml_cuda_op_topk_moe(ggml_backend_cuda_context &     ctx,

ggml/src/ggml-cuda/vecdotq.cuh

@@ -360,12 +360,12 @@ static __device__ __forceinline__ float vec_dot_nvfp4_q8_1(
 
     return sum;
 }
-#define VDR_Q2_K_Q8_1_MMVQ 1
+#define VDR_Q2_K_Q8_1_MMVQ 2
 #define VDR_Q2_K_Q8_1_MMQ  4
 
 // contiguous v/x values
 static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
-    const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
+    const int * __restrict__ v, const int * __restrict__ u, const int * __restrict__ scales,
     const half2 & dm2, const float * __restrict__ d8) {
 
     float sumf_d = 0.0f;
@@ -373,17 +373,20 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
 
 #pragma unroll
     for (int i = 0; i < QR2_K; ++i) {
-        const int sc = scales[2*i];
+        const int sc = scales[i/2] >> (16*(i%2));
 
-        const int vi = (v >> (2*i)) & 0x03030303;
+        int sumi_d = 0;
+        int sumi_m = 0;
+#pragma unroll
+        for (int j = 0; j < VDR_Q2_K_Q8_1_MMVQ; ++j) {
+            const int vi = (v[j] >> (2*i)) & 0x03030303;
 
-        sumf_d += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product
+            sumi_d = ggml_cuda_dp4a(vi,         u[VDR_Q2_K_Q8_1_MMVQ*i + j], sumi_d); // SIMD dot product
+            sumi_m = ggml_cuda_dp4a(0x01010101, u[VDR_Q2_K_Q8_1_MMVQ*i + j], sumi_m); // sum of q8_1 values
+        }
 
-        // fill int with 4x m
-        int m = sc >> 4;
-        m |= m <<  8;
-        m |= m << 16;
-        sumf_m += d8[i] * ggml_cuda_dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values
+        sumf_d += d8[i] * (sumi_d * (sc & 0xF));
+        sumf_m += d8[i] * (sumi_m * ((sc >> 4) & 0xF)); // multiply constant q2_K part with sum of q8_1 values
     }
 
     const float2 dm2f = __half22float2(dm2);
@@ -443,37 +446,35 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(
     return sumf + d8*sumf_d8;
 }
 
-#define VDR_Q3_K_Q8_1_MMVQ 1
+#define VDR_Q3_K_Q8_1_MMVQ 2
 #define VDR_Q3_K_Q8_1_MMQ  2
 
 // contiguous v/x values
 static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(
-    const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,
-    const int & scale_offset, const float & d3, const float * __restrict__ d8) {
+    const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const int * __restrict__ scales,
+    const float & d3, const float * __restrict__ d8) {
 
     float sumf = 0.0f;
 
 #pragma unroll
     for (int i = 0; i < QR3_K; ++i) {
-        const int isc = scale_offset + 2*i;
+        // 4 low bits from scales[0..7], 2 high bits from scales[8..11]
+        const int sc_low  = (scales[i/2] >> (16*(i%2))) & 0x0F;
+        const int sc_high = (scales[2] >> (16*(i%2) + 2*(i/2))) & 0x03;
+        const int sc = (sc_low | (sc_high << 4)) - 32;
 
-        const int isc_low = isc % (QK_K/32);
-        const int sc_shift_low = 4 * (isc / (QK_K/32));
-        const int sc_low  = (scales[isc_low] >> sc_shift_low) & 0xF;
+        int sumi = 0;
+#pragma unroll
+        for (int j = 0; j < VDR_Q3_K_Q8_1_MMVQ; ++j) {
+            const int vil = (vl[j] >> (2*i)) & 0x03030303;
+            const int vih = ((vh[j] >> i) << 2) & 0x04040404;
 
-        const int isc_high = isc % (QK_K/64);
-        const int sc_shift_high = 2 * (isc / (QK_K/64));
-        const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;
+            // (vil | vih) - 4 for each byte, the - 4 is done by the second dp4a
+            sumi = ggml_cuda_dp4a(vil | vih,  u[VDR_Q3_K_Q8_1_MMVQ*i + j], sumi); // SIMD dot product
+            sumi = ggml_cuda_dp4a(0xFCFCFCFC, u[VDR_Q3_K_Q8_1_MMVQ*i + j], sumi);
+        }
 
-        const int sc = (sc_low | sc_high) - 32;
-
-        const int vil = (vl >> (2*i)) & 0x03030303;
-
-        const int vih = ((vh >> i) << 2) & 0x04040404;
-
-        const int vi = __vsub4(vil, vih);
-
-        sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product
+        sumf += d8[i] * (sumi * sc);
     }
 
     return d3 * sumf;
@@ -620,12 +621,12 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(
     return dm4f.x*sumf_d - dm4f.y*sumf_m;
 }
 
-#define VDR_Q6_K_Q8_1_MMVQ 1
+#define VDR_Q6_K_Q8_1_MMVQ 2
 #define VDR_Q6_K_Q8_1_MMQ  8
 
 // contiguous v/x values
 static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
-    const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,
+    const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const int8_t * __restrict__ scales,
     const float & d, const float * __restrict__ d8) {
 
     float sumf = 0.0f;
@@ -634,13 +635,18 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
     for (int i = 0; i < QR6_K; ++i) {
         const int sc = scales[4*i];
 
-        const int vil = (vl >> (4*i)) & 0x0F0F0F0F;
+        int sumi = 0;
+#pragma unroll
+        for (int j = 0; j < VDR_Q6_K_Q8_1_MMVQ; ++j) {
+            const int vil = (vl[j] >> (4*i)) & 0x0F0F0F0F;
+            const int vih = ((vh[j] >> (4*i)) << 4) & 0x30303030;
 
-        const int vih = ((vh >> (4*i)) << 4) & 0x30303030;
+            // (vil | vih) - 32 for each byte, the - 32 is done by the second dp4a
+            sumi = ggml_cuda_dp4a(vil | vih,  u[VDR_Q6_K_Q8_1_MMVQ*i + j], sumi); // SIMD dot product
+            sumi = ggml_cuda_dp4a(0xE0E0E0E0, u[VDR_Q6_K_Q8_1_MMVQ*i + j], sumi);
+        }
 
-        const int vi = __vsub4((vil | vih), 0x20202020); // vi = (vil | vih) - 32
-
-        sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product
+        sumf += d8[i] * (sumi * sc);
     }
 
     return d*sumf;
@@ -700,6 +706,25 @@ static __device__ __forceinline__ float vec_dot_q1_0_q8_1(
         const int u2 = get_int_b4(bq8_1_chunk->qs, j*4+2);
         const int u3 = get_int_b4(bq8_1_chunk->qs, j*4+3);
 
+#if defined(GGML_USE_HIP)
+        const uint32_t q_bits = (uint16_t) q;
+
+        // Expand each group of four Q1_0 bits into byte selectors for v_perm_b32.
+        const uint32_t q0_bits = q_bits >>  0;
+        const uint32_t q1_bits = q_bits >>  4;
+        const uint32_t q2_bits = q_bits >>  8;
+        const uint32_t q3_bits = q_bits >> 12;
+
+        const uint32_t q0_indices = (q0_bits & 0x01) | ((q0_bits & 0x02) << 7) | ((q0_bits & 0x04) << 14) | ((q0_bits & 0x08) << 21);
+        const uint32_t q1_indices = (q1_bits & 0x01) | ((q1_bits & 0x02) << 7) | ((q1_bits & 0x04) << 14) | ((q1_bits & 0x08) << 21);
+        const uint32_t q2_indices = (q2_bits & 0x01) | ((q2_bits & 0x02) << 7) | ((q2_bits & 0x04) << 14) | ((q2_bits & 0x08) << 21);
+        const uint32_t q3_indices = (q3_bits & 0x01) | ((q3_bits & 0x02) << 7) | ((q3_bits & 0x04) << 14) | ((q3_bits & 0x08) << 21);
+
+        const int v0 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q0_indices);
+        const int v1 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q1_indices);
+        const int v2 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q2_indices);
+        const int v3 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q3_indices);
+#else
         // unpack crumbs into nibble indices
         const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0); // [0, 1, 4, 5] [ 8,  9, 12, 13]
         const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2); // [2, 3, 6, 7] [10, 11, 14, 15]
@@ -713,6 +738,7 @@ static __device__ __forceinline__ float vec_dot_q1_0_q8_1(
         const int v1 = __byte_perm(s0, s1, 0x7632);
         const int v2 = __byte_perm(s2, s3, 0x5410);
         const int v3 = __byte_perm(s2, s3, 0x7632);
+#endif // defined(GGML_USE_HIP)
 
         sumi = ggml_cuda_dp4a(v0, u0, sumi);
         sumi = ggml_cuda_dp4a(v1, u1, sumi);
@@ -870,18 +896,31 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1(
 
     const block_q2_K * bq2_K = (const block_q2_K *) vbq + kbx;
 
+    // iqs is in 0,2..14, each 32 bytes of qs hold 4 q8_1 blocks (one per 2 bit shift)
     const int bq8_offset = QR2_K * (iqs / QI8_1);
-    const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
+    const int iqs8 = iqs % QI8_1;
 
-    const uint8_t * scales = bq2_K->scales + scale_offset;
+    int v[VDR_Q2_K_Q8_1_MMVQ];
+#pragma unroll
+    for (int j = 0; j < VDR_Q2_K_Q8_1_MMVQ; ++j) {
+        v[j] = get_int_b4(bq2_K->qs, iqs + j);
+    }
 
-    const int v = get_int_b4(bq2_K->qs, iqs);
-    int    u[QR2_K];
+    // scale of shift i is byte 2*i + (iqs8 >= QI8_1/2) of the 8 bytes for this half
+    const int sc_shift = 8 * (iqs8 / (QI8_1/2));
+    int scales[2];
+    scales[0] = get_int_b4(bq2_K->scales, 2*(iqs / QI8_1) + 0) >> sc_shift;
+    scales[1] = get_int_b4(bq2_K->scales, 2*(iqs / QI8_1) + 1) >> sc_shift;
+
+    int    u[VDR_Q2_K_Q8_1_MMVQ*QR2_K];
     float d8[QR2_K];
 
 #pragma unroll
-    for (int i = 0; i < QR2_K; ++ i) {
-        u[i]  = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
+    for (int i = 0; i < QR2_K; ++i) {
+#pragma unroll
+        for (int j = 0; j < VDR_Q2_K_Q8_1_MMVQ; ++j) {
+            u[VDR_Q2_K_Q8_1_MMVQ*i + j] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs8 + j);
+        }
         d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
     }
 
@@ -893,26 +932,42 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1(
 
     const block_q3_K * bq3_K = (const block_q3_K *) vbq + kbx;
 
+    // iqs is in 0,2..14, each 32 bytes of qs hold 4 q8_1 blocks (one per 2 bit shift)
     const int bq8_offset = QR3_K * (iqs / (QI3_K/2));
-    const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
+    const int iqs8 = iqs % QI8_1;
 
     const float d = bq3_K->d;
 
-    const int vl = get_int_b2(bq3_K->qs, iqs);
+    int vl[VDR_Q3_K_Q8_1_MMVQ];
+    int vh[VDR_Q3_K_Q8_1_MMVQ];
+#pragma unroll
+    for (int j = 0; j < VDR_Q3_K_Q8_1_MMVQ; ++j) {
+        vl[j] = get_int_b2(bq3_K->qs, iqs + j);
+        vh[j] = get_int_b2(bq3_K->hmask, iqs8 + j) >> bq8_offset;
+    }
 
-    // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
-    const int vh = ~get_int_b2(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;
+    // sub-block 8*(iqs/8) + 2*i + (iqs8 >= QI8_1/2) for shift i:
+    // 4 low bits in nibble iqs/8 of bytes, 2 high bits at bit 4*(iqs/8) + 2*(i/2) of bytes 8..11
+    const int sc_shift = 8 * (iqs8 / (QI8_1/2)) + 4 * (iqs / (QI3_K/2));
+    int scales[3];
+#pragma unroll
+    for (int l = 0; l < 3; ++l) {
+        scales[l] = get_int_b2(bq3_K->scales, l) >> sc_shift;
+    }
 
-    int    u[QR3_K];
+    int    u[VDR_Q3_K_Q8_1_MMVQ*QR3_K];
     float d8[QR3_K];
 
 #pragma unroll
     for (int i = 0; i < QR3_K; ++i) {
-        u[i]  = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
+#pragma unroll
+        for (int j = 0; j < VDR_Q3_K_Q8_1_MMVQ; ++j) {
+            u[VDR_Q3_K_Q8_1_MMVQ*i + j] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs8 + j);
+        }
         d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
     }
 
-    return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);
+    return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, scales, d, d8);
 }
 
 static __device__ __forceinline__ float vec_dot_q4_K_q8_1(
@@ -1025,17 +1080,25 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1(
     const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);
     const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));
 
-    const int vl = get_int_b2(bq6_K->ql, iqs);
-    const int vh = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;
+    int vl[VDR_Q6_K_Q8_1_MMVQ];
+    int vh[VDR_Q6_K_Q8_1_MMVQ];
+#pragma unroll
+    for (int j = 0; j < VDR_Q6_K_Q8_1_MMVQ; ++j) {
+        vl[j] = get_int_b2(bq6_K->ql, iqs + j);
+        vh[j] = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4) + j) >> vh_shift;
+    }
 
     const int8_t * scales = bq6_K->scales + scale_offset;
 
-    int    u[QR6_K];
+    int    u[VDR_Q6_K_Q8_1_MMVQ*QR6_K];
     float d8[QR6_K];
 
 #pragma unroll
     for (int i = 0; i < QR6_K; ++i) {
-        u[i]  = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1);
+#pragma unroll
+        for (int j = 0; j < VDR_Q6_K_Q8_1_MMVQ; ++j) {
+            u[VDR_Q6_K_Q8_1_MMVQ*i + j] = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1 + j);
+        }
         d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds);
     }
 

ggml/src/ggml-hexagon/ggml-hexagon.cpp

@@ -956,7 +956,6 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,
         return;
     }
 
-    GGML_ASSERT(offset == 0);
     htp_unpack_tensor_tiled(data, tensor, offset, size);
 }
 
@@ -1045,7 +1044,6 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff
 
     GGML_ASSERT((slice_offset % row_size_bytes) == 0 && "offset must be aligned to row boundary");
     GGML_ASSERT((temp_size % row_size_bytes)    == 0 && "temp_size must be a multiple of row size");
-    GGML_ASSERT((slice_offset / row_size_bytes) % 32 == 0 && "offset must be aligned to tile size (32 rows)");
     GGML_ASSERT((offset + temp_size) <= ggml_nbytes(tensor));
 
     std::vector<uint8_t> temp_buf(temp_size);
@@ -3769,6 +3767,8 @@ static bool ggml_hexagon_matmul_is_hmx_eligible(
     bool is_matmul_id,
     bool is_batched
 ) {
+    GGML_UNUSED(dst);
+
     if (src1->type != GGML_TYPE_F32 && (src1->type != GGML_TYPE_F16 || is_matmul_id)) {
         return false;
     }
@@ -3780,7 +3780,7 @@ static bool ggml_hexagon_matmul_is_hmx_eligible(
     const int wtype = src0->type;
 
     // HMX weight tiles accept non-32-aligned N for non-matmul_id.
-    if (ne01 % 32 != 0 && is_matmul_id) {
+    if (ne01 % HTP_TENSOR_TILE_N_ROWS != 0 && is_matmul_id) {
         return false;
     }
 
@@ -3789,8 +3789,8 @@ static bool ggml_hexagon_matmul_is_hmx_eligible(
         return false;
     }
 
-    // HMX paths require K aligned to 32.
-    if (ne00 % 32 != 0) {
+    // HMX paths require K aligned to HMX tile cols (32).
+    if (ne00 % HTP_TENSOR_TILE_N_COLS != 0) {
         return false;
     }
 
@@ -3952,7 +3952,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(
         kparams->tile_size = htp_tensor_get_tile_size(wtype);
         kparams->aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
 
-        const bool k_align = (ne10 % 32 == 0);
+        const bool k_align = (ne10 % htp_tensor_get_tile_k(wtype) == 0);
 
         if (is_matmul_id) {
             kparams->kernel_type  = (src1_nrows < (int) sess->n_threads)
@@ -4124,7 +4124,7 @@ static void ggml_hexagon_precompute_matmul_params_impl(
 
     const bool is_tiled = ggml_hexagon_tensor_is_tiled(src0);
     const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;
-    const int ne00_padded = is_tiled ? hex_round_up(ne00, 32) : ne00;
+    const int ne00_padded = is_tiled ? hex_round_up(ne00, htp_tensor_get_tile_k(wtype)) : ne00;
     const int ne11_padded = hex_round_up(ne11, 32);
     // VTCM has to hold whole 32-row weight tiles, so size for the rounded-up N
     // even when the tensor itself is ragged.
@@ -5372,6 +5372,8 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s
     }
 
     switch (src0->type) {
+        case GGML_TYPE_Q1_0:
+        case GGML_TYPE_Q2_0:
         case GGML_TYPE_Q4_0:
         case GGML_TYPE_Q4_1:
         case GGML_TYPE_Q8_0:
@@ -5381,13 +5383,13 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s
         case GGML_TYPE_Q5_K:
         case GGML_TYPE_Q6_K:
         case GGML_TYPE_Q3_K:
-        case GGML_TYPE_Q2_K:
+        case GGML_TYPE_Q2_K: {
             if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_view(src0)) {
                 return false;
             }
 
-            if (src0->ne[0] % ((src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q5_K || src0->type == GGML_TYPE_Q4_K ||
-                                src0->type == GGML_TYPE_Q3_K || src0->type == GGML_TYPE_Q2_K) ? QK_K : 32)) {
+            const int64_t qk = ggml_blck_size(src0->type);
+            if (src0->ne[0] % qk) {
                 return false;
             }
 
@@ -5398,6 +5400,7 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s
                 return false;
             }
             break;
+        }
 
         case GGML_TYPE_F16:
             if (src0->nb[1] < src0->nb[0]) {
@@ -5467,6 +5470,8 @@ static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session
     }
 
     switch (src0->type) {
+        case GGML_TYPE_Q1_0:
+        case GGML_TYPE_Q2_0:
         case GGML_TYPE_Q4_0:
         case GGML_TYPE_Q4_1:
         case GGML_TYPE_Q8_0:
@@ -5481,8 +5486,7 @@ static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session
                 return false;
             }
 
-            if (src0->ne[0] % ((src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q5_K || src0->type == GGML_TYPE_Q4_K ||
-                                src0->type == GGML_TYPE_Q3_K || src0->type == GGML_TYPE_Q2_K) ? QK_K : 32)) {
+            if (src0->ne[0] % ggml_blck_size(src0->type) != 0) {
                 return false;
             }
             break;
@@ -8317,6 +8321,10 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
                   "please update hexagon_type to match ggml_type");
     static_assert((unsigned int) HTP_TYPE_Q2_K == (unsigned int) GGML_TYPE_Q2_K,
                   "please update hexagon_type to match ggml_type");
+    static_assert((unsigned int) HTP_TYPE_Q1_0 == (unsigned int) GGML_TYPE_Q1_0,
+                  "please update hexagon_type to match ggml_type");
+    static_assert((unsigned int) HTP_TYPE_Q2_0 == (unsigned int) GGML_TYPE_Q2_0,
+                  "please update hexagon_type to match ggml_type");
     static_assert((unsigned int) HTP_TYPE_BF16 == (unsigned int) GGML_TYPE_BF16,
                   "please update hexagon_type to match ggml_type");
 

ggml/src/ggml-hexagon/htp-repack.cpp

@@ -289,6 +289,80 @@ static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t offset,
 
 // --- Float Types (F16, F32, BF16 -> f16_tiled HMX layout) ---
 
+static void repack_q2_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
+    GGML_ASSERT(offset == 0);
+    GGML_ASSERT(t->ne[0] % 64 == 0);
+    GGML_ASSERT(size == ggml_nbytes(t));
+
+    const block_q2_0 * src = (const block_q2_0 *) data;
+    const int64_t blocks_per_row = t->ne[0] / 64;
+    const int64_t n_col_tiles = hex_round_up(t->ne[1], 32) / 32;
+    const int64_t n_weight_tiles = blocks_per_row;
+    const size_t matrix_size = (size_t) n_col_tiles * n_weight_tiles * HTP_TENSOR_TILE_SIZE_Q2_0;
+
+    for (int64_t slice = 0; slice < t->ne[2] * t->ne[3]; ++slice) {
+        const block_q2_0 * src_slice = src + slice * t->ne[1] * blocks_per_row;
+        uint8_t * dst_slice = (uint8_t *) t->data + slice * matrix_size;
+        memset(dst_slice, 0, matrix_size);
+
+        for (int64_t row_index = 0; row_index < t->ne[1]; ++row_index) {
+            const int64_t ct = row_index / 32;
+            const int row = (int) (row_index % 32);
+            for (int64_t kt = 0; kt < n_weight_tiles; ++kt) {
+                const block_q2_0 * block = src_slice + row_index * blocks_per_row + kt;
+                uint8_t * tile = dst_slice + (ct * n_weight_tiles + kt) * HTP_TENSOR_TILE_SIZE_Q2_0;
+                for (int group = 0; group < 16; ++group) {
+                    for (int byte = 0; byte < 4; ++byte) {
+                        const int k = 4 * group + byte;
+                        const uint8_t code = (block->qs[k / 4] >> (2 * (k % 4))) & 3;
+                        tile[(group >> 2) * 128 + 4 * row + byte] |= code << (2 * (group & 3));
+                    }
+                }
+                memcpy(tile + 512 + 2 * row, &block->d, sizeof(block->d));
+            }
+        }
+    }
+}
+
+static void repack_tiled_q2_0(void * data, const ggml_tensor * t, size_t offset, size_t size) {
+    GGML_ASSERT(t->ne[0] % 64 == 0);
+    GGML_ASSERT(offset + size <= ggml_nbytes(t));
+
+    const size_t first_block = offset / sizeof(block_q2_0);
+    const size_t end_block = (offset + size + sizeof(block_q2_0) - 1) / sizeof(block_q2_0);
+    const size_t blocks_per_row = t->ne[0] / 64;
+    const size_t blocks_per_slice = t->ne[1] * blocks_per_row;
+    const size_t n_weight_tiles = blocks_per_row;
+    const size_t n_col_tiles = hex_round_up(t->ne[1], 32) / 32;
+    const size_t matrix_size = n_col_tiles * n_weight_tiles * HTP_TENSOR_TILE_SIZE_Q2_0;
+
+    for (size_t block_index = first_block; block_index < end_block; ++block_index) {
+        const size_t slice = block_index / blocks_per_slice;
+        const size_t in_slice = block_index % blocks_per_slice;
+        const size_t row_index = in_slice / blocks_per_row;
+        const size_t kt = in_slice % blocks_per_row;
+        const size_t ct = row_index / 32;
+        const size_t row = row_index % 32;
+        const uint8_t * tile = (const uint8_t *) t->data + slice * matrix_size +
+            (ct * n_weight_tiles + kt) * HTP_TENSOR_TILE_SIZE_Q2_0;
+
+        block_q2_0 block = {};
+        memcpy(&block.d, tile + 512 + 2 * row, sizeof(block.d));
+        for (int group = 0; group < 16; ++group) {
+            for (int byte = 0; byte < 4; ++byte) {
+                const uint8_t packed = tile[(group >> 2) * 128 + 4 * row + byte];
+                const uint8_t code = (packed >> (2 * (group & 3))) & 3;
+                block.qs[group] |= code << (2 * byte);
+            }
+        }
+
+        const size_t block_begin = block_index * sizeof(block_q2_0);
+        const size_t copy_begin = (std::max)(block_begin, offset);
+        const size_t copy_end = (std::min)(block_begin + sizeof(block_q2_0), offset + size);
+        memcpy((uint8_t *) data + copy_begin - offset, (const uint8_t *) &block + copy_begin - block_begin, copy_end - copy_begin);
+    }
+}
+
 template <typename SrcT, typename ToFp16Fn>
 static void repack_float_to_f16_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size, ToFp16Fn && to_fp16) {
     repack_to_tiled_grid_32<SrcT, 32>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_F16,
@@ -900,10 +974,122 @@ static void repack_tiled_q5_K(void * data, const ggml_tensor * t, size_t offset,
         });
 }
 
+static inline void q1_0_tiled_bit_location(int k, int row, int & byte_offset, int & bit) {
+    const int quarter = k / 32;
+    const int kq      = k % 32;
+
+    byte_offset = quarter * 128 + 4 * row + (kq % 4);
+    bit         = kq / 4;
+}
+
+static void repack_q1_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
+    const block_q1_0 * src_matrix = (const block_q1_0 *) data;
+    const int64_t ne0 = t->ne[0];
+    const int64_t ne1 = t->ne[1];
+    const int64_t ne2 = t->ne[2];
+    const int64_t ne3 = t->ne[3];
+    const int n_col_tiles = hex_round_up(ne1, 32) / 32;
+    const int n_k_tiles = ne0 / QK1_0;
+    const size_t tile_size = HTP_TENSOR_TILE_SIZE_Q1_0;
+    const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;
+    const size_t slice_size = ne1 * ggml_row_size(t->type, ne0);
+    const int64_t start_slice = offset / slice_size;
+    const int64_t end_slice = (std::min)((int64_t) ((offset + size + slice_size - 1) / slice_size), ne2 * ne3);
+
+    GGML_ASSERT(ne0 % QK1_0 == 0);
+
+    for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
+        const block_q1_0 * src_slice = src_matrix + (slice_idx - start_slice) * ne1 * n_k_tiles;
+        uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size;
+
+        for (int ct = 0; ct < n_col_tiles; ct++) {
+            for (int kt = 0; kt < n_k_tiles; kt++) {
+                uint8_t * tile_dst = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size;
+                ggml_half * scale_dst = (ggml_half *) (tile_dst + 512);
+                memset(tile_dst, 0, tile_size);
+
+                for (int row = 0; row < 32; row++) {
+                    const int64_t r = ct * 32 + row;
+                    if (r >= ne1) {
+                        continue;
+                    }
+
+                    const block_q1_0 & block = src_slice[r * n_k_tiles + kt];
+                    scale_dst[row] = block.d;
+                    for (int k = 0; k < QK1_0; k++) {
+                        int byte_offset;
+                        int bit;
+                        q1_0_tiled_bit_location(k, row, byte_offset, bit);
+                        if (block.qs[k / 8] & (1u << (k % 8))) {
+                            tile_dst[byte_offset] |= (uint8_t) (1u << bit);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+static void repack_tiled_q1_0(void * data, const ggml_tensor * t, size_t offset, size_t size) {
+    block_q1_0 * dst_matrix = (block_q1_0 *) data;
+    const int64_t ne0 = t->ne[0];
+    const int64_t ne1 = t->ne[1];
+    const int64_t ne2 = t->ne[2];
+    const int64_t ne3 = t->ne[3];
+    const int n_col_tiles = hex_round_up(ne1, 32) / 32;
+    const int n_k_tiles = ne0 / QK1_0;
+    const size_t tile_size = HTP_TENSOR_TILE_SIZE_Q1_0;
+    const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;
+    const size_t row_size_bytes = ggml_row_size(t->type, ne0);
+    const size_t slice_size = ne1 * row_size_bytes;
+    const int64_t start_slice = offset / slice_size;
+    const int64_t end_slice = (std::min)((int64_t) ((offset + size + slice_size - 1) / slice_size), ne2 * ne3);
+
+    GGML_ASSERT(ne0 % QK1_0 == 0);
+
+    for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
+        const size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size);
+        const size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size);
+        const int64_t start_row = (cur_start_byte - (size_t) slice_idx * slice_size) / row_size_bytes;
+        const int64_t end_row = (std::min)((int64_t) ((cur_end_byte - (size_t) slice_idx * slice_size + row_size_bytes - 1) / row_size_bytes), ne1);
+        block_q1_0 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_q1_0);
+        const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size;
+
+        for (int ct = start_row / 32; ct < (end_row + 31) / 32; ct++) {
+            for (int kt = 0; kt < n_k_tiles; kt++) {
+                const uint8_t * tile_src = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size;
+                const ggml_half * scale_src = (const ggml_half *) (tile_src + 512);
+
+                for (int row = 0; row < 32; row++) {
+                    const int64_t r = ct * 32 + row;
+                    if (r < start_row || r >= end_row) {
+                        continue;
+                    }
+
+                    block_q1_0 & block = dst_slice[(r - start_row) * n_k_tiles + kt];
+                    memset(block.qs, 0, sizeof(block.qs));
+                    block.d = scale_src[row];
+                    for (int k = 0; k < QK1_0; k++) {
+                        int byte_offset;
+                        int bit;
+                        q1_0_tiled_bit_location(k, row, byte_offset, bit);
+                        if (tile_src[byte_offset] & (1u << bit)) {
+                            block.qs[k / 8] |= (uint8_t) (1u << (k % 8));
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
 // --- Top-level Repack / Unpack Dispatchers ---
 
 void htp_repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t size) {
     switch (tensor->type) {
+        case GGML_TYPE_Q1_0:
+            repack_q1_0_tiled(tensor, data, 0, size);
+            break;
         case GGML_TYPE_Q4_0:
         case GGML_TYPE_IQ4_NL:
             repack_q4_0_tiled(tensor, data, 0, size);
@@ -917,6 +1103,9 @@ void htp_repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t siz
         case GGML_TYPE_MXFP4:
             repack_mxfp4_tiled(tensor, data, 0, size);
             break;
+        case GGML_TYPE_Q2_0:
+            repack_q2_0_tiled(tensor, data, 0, size);
+            break;
         case GGML_TYPE_Q2_K:
             repack_q2_K_tiled(tensor, data, 0, size);
             break;
@@ -953,6 +1142,9 @@ void htp_unpack_tensor_tiled(void * data, const ggml_tensor * tensor, size_t off
     GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
 
     switch (tensor->type) {
+        case GGML_TYPE_Q1_0:
+            repack_tiled_q1_0(data, tensor, offset, size);
+            break;
         case GGML_TYPE_Q4_0:
         case GGML_TYPE_IQ4_NL:
             repack_tiled_q4_0(data, tensor, offset, size);
@@ -966,6 +1158,9 @@ void htp_unpack_tensor_tiled(void * data, const ggml_tensor * tensor, size_t off
         case GGML_TYPE_MXFP4:
             repack_tiled_mxfp4(data, tensor, offset, size);
             break;
+        case GGML_TYPE_Q2_0:
+            repack_tiled_q2_0(data, tensor, offset, size);
+            break;
         case GGML_TYPE_Q2_K:
             repack_tiled_q2_K(data, tensor, offset, size);
             break;

ggml/src/ggml-hexagon/htp-repack.h

@@ -10,9 +10,11 @@
 static inline bool htp_is_tiled_weight_type(enum ggml_type type) {
     return type == GGML_TYPE_Q4_0  || type == GGML_TYPE_Q4_1   ||
            type == GGML_TYPE_Q8_0  || type == GGML_TYPE_IQ4_NL ||
-           type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q6_K   ||
+           type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q1_0   ||
+           type == GGML_TYPE_Q6_K   ||
            type == GGML_TYPE_Q4_K  || type == GGML_TYPE_Q5_K   ||
            type == GGML_TYPE_Q3_K  || type == GGML_TYPE_Q2_K   ||
+           type == GGML_TYPE_Q2_0  ||
            type == GGML_TYPE_F16   || type == GGML_TYPE_F32    ||
            type == GGML_TYPE_BF16;
 }
@@ -42,6 +44,9 @@ static inline size_t htp_tiled_row_size(enum ggml_type type, int64_t ne0) {
     if (type == GGML_TYPE_Q6_K) {
         return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q6_K / 32);
     }
+    if (type == GGML_TYPE_Q2_0) {
+        return (size_t) (ne0 / 64) * 18;
+    }
     if (type == GGML_TYPE_Q4_K) {
         return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q4_K / 32);
     }

ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h

@@ -26,6 +26,7 @@ struct tiled_dequantize_state {
 
     struct fastdiv_values     n_k_tiles_div;
     uint32_t                  n_k_tiles;
+    uint32_t                  n_weight_tiles;
     uint32_t                  n_tot_tiles;
     uint32_t                  n_tiles_per_task;
     uint32_t                  tile_size;
@@ -37,6 +38,39 @@ struct tiled_dequantize_state {
     uint32_t                  weight_type;
 };
 
+static void dequantize_tiled_weight_to_fp16_task_q1_0(
+        const tiled_dequantize_state_t *state,
+        uint32_t start_tile, uint32_t end_tile) {
+
+    const HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1);
+    const HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1);
+
+    for (uint32_t t = start_tile; t < end_tile; t++) {
+        const uint8_t * tile_src = state->src + t * state->aligned_tile_size;
+        __fp16 * dst_ptr = state->dst + 4 * t * HTP_MM_HMX_TILE_N_ELMS;
+
+        const HVX_Vector v_sc = hvx_vmem(tile_src + 512);
+        const HVX_Vector v_scale = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2));
+
+        for (uint32_t q = 0; q < 4; q++) {
+            const HVX_Vector v_plane = hvx_vmem(tile_src + q * 128);
+            __fp16 * quarter_dst = dst_ptr + q * HTP_MM_HMX_TILE_N_ELMS;
+
+            #pragma unroll
+            for (int g = 0; g < 8; g++) {
+                const HVX_Vector     v_q  = unpack_q1_0_group(v_plane, g, v_neg_one, v_pos_one);
+                const HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q);
+                const HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4);
+
+                hvx_vmem(quarter_dst + (2 * g + 0) * 64) =
+                    Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale));
+                hvx_vmem(quarter_dst + (2 * g + 1) * 64) =
+                    Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale));
+            }
+        }
+    }
+}
+
 // Dequantize a single tile from tiled weight data (already in VTCM) to tile-major FP16.
 static void dequantize_tiled_weight_to_fp16_task_q4_0(
         const tiled_dequantize_state_t *state,
@@ -751,6 +785,40 @@ static void dequantize_tiled_weight_to_fp16_task_q2_k(
     }
 }
 
+// Q2_0 stores codes 0..3 (value code - 1) and one fp16 scale per row, see HTP_MM_WEIGHT_TILE_SIZE_Q2_0.
+// One packed 64-k tile expands into two consecutive 32-k fp16 tiles.
+static void dequantize_tiled_weight_to_fp16_task_q2_0(
+        const tiled_dequantize_state_t *state,
+        uint32_t start_tile, uint32_t end_tile) {
+
+    const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
+    const HVX_Vector i1      = Q6_Vb_vsplat_R(1);
+
+    for (uint32_t t = start_tile; t < end_tile; t++) {
+        const HVX_Vector * vptr = (const HVX_Vector *) (state->src + t * state->aligned_tile_size);
+        __fp16 * dst_base = state->dst + 2 * t * HTP_MM_HMX_TILE_N_ELMS;
+
+        HVX_Vector v_sc    = vptr[4];
+        HVX_Vector v_scale = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2));
+
+        for (int half = 0; half < 2; half++) {
+            __fp16 * dst_ptr = dst_base + half * HTP_MM_HMX_TILE_N_ELMS;
+
+            #pragma unroll
+            for (int g = 0; g < 8; g++) {
+                HVX_Vector     v_q  = Q6_Vb_vsub_VbVb(unpack_q3_k_low2(vptr + 2 * half, g, mask_03), i1);
+                HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q);
+                HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4);
+
+                hvx_vmem(dst_ptr + (2 * g + 0) * 64) =
+                    Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale));
+                hvx_vmem(dst_ptr + (2 * g + 1) * 64) =
+                    Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale));
+            }
+        }
+    }
+}
+
 static __attribute__((noinline))
 void convert_f16_weight_to_fp16_tiles_task(
         const tiled_dequantize_state_t *state,

ggml/src/ggml-hexagon/htp/htp-ops.h

@@ -32,6 +32,8 @@ enum htp_data_type {
     HTP_TYPE_I64    = 27,
     HTP_TYPE_BF16   = 30,
     HTP_TYPE_MXFP4  = 39,
+    HTP_TYPE_Q1_0   = 41,
+    HTP_TYPE_Q2_0   = 42,
 
     HTP_TYPE_INVALID
 };

ggml/src/ggml-hexagon/htp/htp-tensor.h

@@ -25,32 +25,13 @@
 #define HTP_TENSOR_TILE_SIZE_Q6_K   896
 #define HTP_TENSOR_TILE_SIZE_Q3_K   512
 #define HTP_TENSOR_TILE_SIZE_Q2_K   336
+#define HTP_TENSOR_TILE_K_Q1_0      128
+#define HTP_TENSOR_TILE_SIZE_Q1_0   576
+#define HTP_TENSOR_TILE_K_Q2_0      64
+#define HTP_TENSOR_TILE_SIZE_Q2_0   576
 
-// Q2_K, Q3_K, Q6_K use vrmpy byte order within each 128B HVX vector:
-//   k = 0..31 is split into 8 groups g = k/4 of 4 k each; byte 4*r + (k%4) holds row r
-//
-// Q2_K (336B, lossless tiling with same size as block_q2_K):
-//   bytes   0..255 (vec 0..1): unsigned 2-bit quants, vec m holds groups 4m..4m+3 at bit offsets 0,2,4,6
-//   bytes 256..319:            raw scales[] bytes (min << 4 | sc): [0..31] for k 0..15, [32..63] for k 16..31
-//   bytes 320..335:            8 fp16 in tile j (0..7) of super-block: d (j < 4) or -dmin (j >= 4) for rows 8*(j%4)..8*(j%4)+7
-//
-// Q3_K (512B):
-//   bytes   0..255 (vec 0..1): low 2 bits, vec m holds groups 4m..4m+3 at bit offsets 0,2,4,6
-//   bytes 256..383 (vec 2):    sign mask, bit g set when hmask is 0 (quant = low2 - 4)
-//   bytes 384..511 (vec 3):    fp16 scales d * (scales[] - 32): fp16[0..31] for k 0..15, fp16[32..63] for k 16..31
-//
-// Q4_K (640B, same layout as Q4_1 tile; each 256-k super-block splits into 8 tiles of 32 k):
-//   bytes   0..511 (vec 0..3): 4-bit quants, byte 32*cp + r holds k = 2*cp (low nibble) and 2*cp + 1 (high nibble)
-//   bytes 512..639 (vec 4):    32x2 fp16 per row: (d * sc, -dmin * m)
-//
-// Q5_K (768B):
-//   bytes   0..639 (vec 0..4): Q4_K tile (low 4 bits + fp16 scale/min pairs)
-//   bytes 640..767 (vec 5):    5th-bit plane; byte l holds the 8 high bits for lane l across vec i = 0..3 (bit 2i = lo, 2i+1 = hi)
-//
-// Q6_K (896B):
-//   bytes   0..511 (vec 0..3): low 4 bits, vec i holds group 2i (low nibble) and 2i+1 (high nibble)
-//   bytes 512..767 (vec 4..5): high 2 bits, vec 4+m holds groups 4m..4m+3 at bit offsets 0,2,4,6
-//   bytes 768..895 (vec 6):    fp16 scales d * scales[]: fp16[0..31] for k 0..15, fp16[32..63] for k 16..31
+// Tiled weight tensor layouts (Q4_0, Q8_0, IQ4_NL, MXFP4, Q1_0, Q2_0, K-quants):
+// See docs/backend/snapdragon/developer.md ("Tiled Weight Formats").
 
 struct htp_tensor_mdev_range {
     uint32_t start;
@@ -168,12 +149,22 @@ static inline uint32_t htp_tensor_get_tile_size(int type) {
         case HTP_TYPE_Q6_K:   return HTP_TENSOR_TILE_SIZE_Q6_K;
         case HTP_TYPE_Q3_K:   return HTP_TENSOR_TILE_SIZE_Q3_K;
         case HTP_TYPE_Q2_K:   return HTP_TENSOR_TILE_SIZE_Q2_K;
+        case HTP_TYPE_Q1_0:   return HTP_TENSOR_TILE_SIZE_Q1_0;
+        case HTP_TYPE_Q2_0:   return HTP_TENSOR_TILE_SIZE_Q2_0;
         case HTP_TYPE_MXFP4:  return HTP_TENSOR_TILE_SIZE_MXFP4;
         case HTP_TYPE_F16:    return HTP_TENSOR_TILE_SIZE_F16;
         default:              return 0;
     }
 }
 
+static inline uint32_t htp_tensor_get_tile_k(int type) {
+    switch (type) {
+        case HTP_TYPE_Q1_0: return HTP_TENSOR_TILE_K_Q1_0;
+        case HTP_TYPE_Q2_0: return HTP_TENSOR_TILE_K_Q2_0;
+        default:            return 32;
+    }
+}
+
 static inline uint32_t htp_tensor_get_tile_padded_size(int type) {
     return (htp_tensor_get_tile_size(type) + 127u) & ~127u;
 }
@@ -193,10 +184,12 @@ static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) {
         case HTP_TYPE_Q4_1:   return (ne00 / 32) * 20;
         case HTP_TYPE_MXFP4:  return (ne00 / 32) * 17;
         case HTP_TYPE_Q2_K:   return (ne00 / 256) * 84;
+        case HTP_TYPE_Q2_0:   return (ne00 / 64) * 18;
         case HTP_TYPE_Q3_K:   return (ne00 / 256) * 110;
         case HTP_TYPE_Q4_K:   return (ne00 / 256) * 144;
         case HTP_TYPE_Q5_K:   return (ne00 / 256) * 176;
         case HTP_TYPE_Q6_K:   return (ne00 / 256) * 210;
+        case HTP_TYPE_Q1_0:   return (ne00 / HTP_TENSOR_TILE_K_Q1_0) * 18;
         case HTP_TYPE_I32:    return ne00 * 4;
         case HTP_TYPE_I64:    return ne00 * 8;
         default:              return 0;

ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h

@@ -395,6 +395,36 @@ static inline HVX_VectorPair unpack_and_interleave_4bit_x2(HVX_Vector v_src, HVX
     return Q6_W_vcombine_VV(v_W1, v_W0);
 }
 
+static inline HVX_Vector unpack_q1_0_group(
+    HVX_Vector v_plane,
+    int g,
+    HVX_Vector v_neg_one,
+    HVX_Vector v_pos_one
+) {
+    HVX_VectorPred q = Q6_Q_vand_VR(v_plane, 0x01010101u << g);
+    return Q6_V_vmux_QVV(q, v_pos_one, v_neg_one);
+}
+
+static inline HVX_Vector accum_q1_0_32x1(
+    HVX_Vector v_plane,
+    const HVX_Vector * restrict v_act,
+    HVX_Vector v_neg_one,
+    HVX_Vector v_pos_one
+) {
+    HVX_Vector v_sum_lo = Q6_V_vzero();
+    HVX_Vector v_sum_hi = Q6_V_vzero();
+
+    #pragma unroll
+    for (int g = 0; g < 4; g++) {
+        HVX_Vector v_W_lo = unpack_q1_0_group(v_plane, g,     v_neg_one, v_pos_one);
+        HVX_Vector v_W_hi = unpack_q1_0_group(v_plane, g + 4, v_neg_one, v_pos_one);
+        v_sum_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum_lo, v_W_lo, v_act[g]);
+        v_sum_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum_hi, v_W_hi, v_act[g + 4]);
+    }
+
+    return Q6_Vw_vadd_VwVw(v_sum_lo, v_sum_hi);
+}
+
 static inline HVX_Vector accum_4bit_32x1(
     const HVX_Vector * restrict vptr,
     const HVX_Vector * restrict v_act,
@@ -437,6 +467,33 @@ static inline HVX_Vector accum_4bit_32x1_lut(
     return Q6_Vw_vadd_VwVw(v_sum0, v_sum1);
 }
 
+static inline HVX_VectorPair accum_q1_0_32x2(
+    HVX_Vector v_plane,
+    const HVX_Vector * restrict v_act0,
+    const HVX_Vector * restrict v_act1,
+    HVX_Vector v_neg_one,
+    HVX_Vector v_pos_one
+) {
+    HVX_Vector v_sum0_lo = Q6_V_vzero();
+    HVX_Vector v_sum0_hi = Q6_V_vzero();
+    HVX_Vector v_sum1_lo = Q6_V_vzero();
+    HVX_Vector v_sum1_hi = Q6_V_vzero();
+
+    #pragma unroll
+    for (int g = 0; g < 4; g++) {
+        HVX_Vector v_W_lo = unpack_q1_0_group(v_plane, g,     v_neg_one, v_pos_one);
+        HVX_Vector v_W_hi = unpack_q1_0_group(v_plane, g + 4, v_neg_one, v_pos_one);
+        v_sum0_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_lo, v_W_lo, v_act0[g]);
+        v_sum0_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_hi, v_W_hi, v_act0[g + 4]);
+        v_sum1_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_lo, v_W_lo, v_act1[g]);
+        v_sum1_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_hi, v_W_hi, v_act1[g + 4]);
+    }
+
+    return Q6_W_vcombine_VV(
+        Q6_Vw_vadd_VwVw(v_sum1_lo, v_sum1_hi),
+        Q6_Vw_vadd_VwVw(v_sum0_lo, v_sum0_hi));
+}
+
 static inline HVX_VectorPair accum_4bit_32x2(
     const HVX_Vector * restrict vptr,
     const HVX_Vector * restrict v_act0,
@@ -771,6 +828,53 @@ static inline void accum_q2_k_32x2(
     *v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo);
 }
 
+// Q2_0: unsigned codes 0..3; the -1 offset is applied via the Q8_1 block sum (v_act[9])
+static inline HVX_Vector accum_q2_0_32x1(
+    const HVX_Vector * restrict vptr,
+    const HVX_Vector * restrict v_act
+) {
+    HVX_Vector v_sum_lo = Q6_V_vzero();
+    HVX_Vector v_sum_hi = Q6_V_vzero();
+    HVX_Vector mask_03  = Q6_Vb_vsplat_R(0x03);
+
+    #pragma unroll
+    for (int g = 0; g < 4; g++) {
+        HVX_Vector v_W_lo = unpack_q3_k_low2(vptr, g,     mask_03);
+        HVX_Vector v_W_hi = unpack_q3_k_low2(vptr, g + 4, mask_03);
+        v_sum_lo = Q6_Vw_vrmpyacc_VwVubVb(v_sum_lo, v_W_lo, v_act[g]);
+        v_sum_hi = Q6_Vw_vrmpyacc_VwVubVb(v_sum_hi, v_W_hi, v_act[g + 4]);
+    }
+
+    return Q6_Vw_vadd_VwVw(v_sum_lo, v_sum_hi);
+}
+
+static inline void accum_q2_0_32x2(
+    const HVX_Vector * restrict vptr,
+    const HVX_Vector * restrict v_act0,
+    const HVX_Vector * restrict v_act1,
+    HVX_Vector * v_sum0,
+    HVX_Vector * v_sum1
+) {
+    HVX_Vector v_sum0_lo = Q6_V_vzero();
+    HVX_Vector v_sum0_hi = Q6_V_vzero();
+    HVX_Vector v_sum1_lo = Q6_V_vzero();
+    HVX_Vector v_sum1_hi = Q6_V_vzero();
+    HVX_Vector mask_03   = Q6_Vb_vsplat_R(0x03);
+
+    #pragma unroll
+    for (int g = 0; g < 4; g++) {
+        HVX_Vector v_W_lo = unpack_q3_k_low2(vptr, g,     mask_03);
+        HVX_Vector v_W_hi = unpack_q3_k_low2(vptr, g + 4, mask_03);
+        v_sum0_lo = Q6_Vw_vrmpyacc_VwVubVb(v_sum0_lo, v_W_lo, v_act0[g]);
+        v_sum0_hi = Q6_Vw_vrmpyacc_VwVubVb(v_sum0_hi, v_W_hi, v_act0[g + 4]);
+        v_sum1_lo = Q6_Vw_vrmpyacc_VwVubVb(v_sum1_lo, v_W_lo, v_act1[g]);
+        v_sum1_hi = Q6_Vw_vrmpyacc_VwVubVb(v_sum1_hi, v_W_hi, v_act1[g + 4]);
+    }
+
+    *v_sum0 = Q6_Vw_vadd_VwVw(v_sum0_lo, v_sum0_hi);
+    *v_sum1 = Q6_Vw_vadd_VwVw(v_sum1_lo, v_sum1_hi);
+}
+
 // Q2_K: fp16 d (*v_d) and -dmin (*v_m) of the 32 rows of a super-block (8 tiles, VTCM stride 384), each in both vector halves
 static inline void q2_k_load_dmin(const uint8_t * restrict sb, HVX_Vector * restrict v_d, HVX_Vector * restrict v_m) {
     HVX_Vector v_dd = Q6_V_vzero();  // lanes 0..31: d, lanes 32..63: -dmin
@@ -824,6 +928,86 @@ static inline HVX_Vector q2_k_int_scale(HVX_Vector v_isum, HVX_Vector v_msum, HV
     return hvx_vec_mul_f32_f32(v_t, v_da32);
 }
 
+static void tiled_vec_dot_q1_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
+    const uint8_t * restrict tile_ptr = vx;
+    const uint8_t * restrict y_q = vy;
+
+    HVX_Vector v_sum_float = Q6_V_vzero();
+    HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1);
+    HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1);
+
+    const uint32_t n_k_tiles = n / HTP_TENSOR_TILE_K_Q1_0;
+    for (uint32_t kt = 0; kt < n_k_tiles; kt++) {
+        const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * htp_tensor_get_tile_padded_size(HTP_TYPE_Q1_0));
+        HVX_Vector v_tile_qf = Q6_V_vzero();
+
+        #pragma unroll
+        for (int quarter = 0; quarter < 4; quarter++) {
+            const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + (4 * kt + quarter) * HTP_MM_ACT_TILE_SIZE_Q8_0);
+            HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(accum_q1_0_32x1(vptr[quarter], v_act, v_neg_one, v_pos_one));
+            HVX_Vector v_da_sf  = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act[8]));
+            HVX_Vector v_q_qf   = Q6_Vqf32_vmpy_VsfVsf(v_sum_sf, v_da_sf);
+            v_tile_qf = (quarter == 0) ? v_q_qf : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf, v_q_qf);
+        }
+
+        HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4]));
+        v_sum_float = hvx_vec_add_f32_f32(v_sum_float, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf));
+    }
+
+    if (sz) {
+        hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz)));
+    } else {
+        hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float);
+    }
+}
+
+static void tiled_vec_dot_q1_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) {
+    const uint8_t * restrict tile_ptr = vx;
+    const uint8_t * restrict y0_q = vy0;
+    const uint8_t * restrict y1_q = vy1;
+
+    HVX_Vector v_sum_float_c0 = Q6_V_vzero();
+    HVX_Vector v_sum_float_c1 = Q6_V_vzero();
+    HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1);
+    HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1);
+
+    const uint32_t n_k_tiles = n / HTP_TENSOR_TILE_K_Q1_0;
+    for (uint32_t kt = 0; kt < n_k_tiles; kt++) {
+        const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * htp_tensor_get_tile_padded_size(HTP_TYPE_Q1_0));
+        HVX_Vector v_tile_c0 = Q6_V_vzero();
+        HVX_Vector v_tile_c1 = Q6_V_vzero();
+
+        for (int quarter = 0; quarter < 4; quarter++) {
+            const uint32_t act_tile = 4 * kt + quarter;
+            const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + act_tile * HTP_MM_ACT_TILE_SIZE_Q8_0);
+            const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + act_tile * HTP_MM_ACT_TILE_SIZE_Q8_0);
+            HVX_VectorPair v_sums = accum_q1_0_32x2(vptr[quarter], v_act0, v_act1, v_neg_one, v_pos_one);
+            HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(Q6_V_lo_W(v_sums));
+            HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(Q6_V_hi_W(v_sums));
+            HVX_Vector v_da_sf_c0  = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act0[8]));
+            HVX_Vector v_da_sf_c1  = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act1[8]));
+
+            v_tile_c0 = hvx_vec_add_f32_f32(v_tile_c0, hvx_vec_mul_f32_f32(v_sum_sf_c0, v_da_sf_c0));
+            v_tile_c1 = hvx_vec_add_f32_f32(v_tile_c1, hvx_vec_mul_f32_f32(v_sum_sf_c1, v_da_sf_c1));
+        }
+
+        HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4]));
+        v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vec_mul_f32_f32(v_tile_c0, v_dw_sf));
+        v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vec_mul_f32_f32(v_tile_c1, v_dw_sf));
+    }
+
+    if (sz0) {
+        hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0)));
+    } else {
+        hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0);
+    }
+    if (sz1) {
+        hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1)));
+    } else {
+        hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1);
+    }
+}
+
 static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
     const uint8_t * restrict tile_ptr = vx;
     const uint8_t * restrict y_q = vy;
@@ -1362,6 +1546,93 @@ static void tiled_vec_dot_q2_k_32x2(const uint32_t n, float * restrict s0, float
     }
 }
 
+// One Q2_0 weight tile covers 64 k: act tiles 2 * kt and 2 * kt + 1, each 32 k wide (Q8_1)
+static void tiled_vec_dot_q2_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
+    const uint8_t * restrict tile_ptr = vx;
+    const uint8_t * restrict y_q = vy;
+
+    HVX_Vector v_sum_float = Q6_V_vzero();
+    HVX_Vector v_one_f16   = hvx_vec_splat_f16(1.0);
+
+    uint32_t n_weight_tiles = n / HTP_MM_WEIGHT_TILE_K_Q2_0;
+    for (uint32_t kt = 0; kt < n_weight_tiles; kt++) {
+        const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0);
+        HVX_Vector v_tile_qf = Q6_V_vzero();
+
+        #pragma unroll
+        for (int half = 0; half < 2; half++) {
+            const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + (2 * kt + half) * HTP_MM_ACT_TILE_SIZE_Q8_1);
+
+            HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(accum_q2_0_32x1(vptr + 2 * half, v_act));
+            HVX_Vector v_da_sf  = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act[8]));
+            HVX_Vector v_sa_qf  = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(v_act[9], v_one_f16));
+            HVX_Vector v_h_qf   = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(v_sum_sf, v_da_sf), v_sa_qf);
+            v_tile_qf = (half == 0) ? v_h_qf : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf, v_h_qf);
+        }
+
+        HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4]));
+        v_sum_float = hvx_vec_add_f32_f32(v_sum_float, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf));
+    }
+
+    if (sz) {
+        hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz)));
+    } else {
+        hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float);
+    }
+}
+
+static void tiled_vec_dot_q2_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) {
+    const uint8_t * restrict tile_ptr = vx;
+    const uint8_t * restrict y0_q = vy0;
+    const uint8_t * restrict y1_q = vy1;
+
+    HVX_Vector v_sum_float_c0 = Q6_V_vzero();
+    HVX_Vector v_sum_float_c1 = Q6_V_vzero();
+    HVX_Vector v_one_f16      = hvx_vec_splat_f16(1.0);
+
+    uint32_t n_weight_tiles = n / HTP_MM_WEIGHT_TILE_K_Q2_0;
+    for (uint32_t kt = 0; kt < n_weight_tiles; kt++) {
+        const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0);
+        HVX_Vector v_tile_qf_c0 = Q6_V_vzero();
+        HVX_Vector v_tile_qf_c1 = Q6_V_vzero();
+
+        #pragma unroll
+        for (int half = 0; half < 2; half++) {
+            const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + (2 * kt + half) * HTP_MM_ACT_TILE_SIZE_Q8_1);
+            const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + (2 * kt + half) * HTP_MM_ACT_TILE_SIZE_Q8_1);
+
+            HVX_Vector v_sum_c0, v_sum_c1;
+            accum_q2_0_32x2(vptr + 2 * half, v_act0, v_act1, &v_sum_c0, &v_sum_c1);
+
+            HVX_Vector v_da_sf_c0 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act0[8]));
+            HVX_Vector v_da_sf_c1 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act1[8]));
+            HVX_Vector v_sa_qf_c0 = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(v_act0[9], v_one_f16));
+            HVX_Vector v_sa_qf_c1 = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(v_act1[9], v_one_f16));
+
+            HVX_Vector v_h_qf_c0 = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vw(v_sum_c0), v_da_sf_c0), v_sa_qf_c0);
+            HVX_Vector v_h_qf_c1 = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vw(v_sum_c1), v_da_sf_c1), v_sa_qf_c1);
+
+            v_tile_qf_c0 = (half == 0) ? v_h_qf_c0 : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c0, v_h_qf_c0);
+            v_tile_qf_c1 = (half == 0) ? v_h_qf_c1 : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c1, v_h_qf_c1);
+        }
+
+        HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4]));
+        v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf_c0), v_dw_sf));
+        v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf_c1), v_dw_sf));
+    }
+
+    if (sz0) {
+        hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0)));
+    } else {
+        hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0);
+    }
+    if (sz1) {
+        hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1)));
+    } else {
+        hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1);
+    }
+}
+
 static void tiled_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
     const uint8_t * restrict tile_ptr = vx;
     const uint8_t * restrict y_q = vy;

ggml/src/ggml-hexagon/htp/matmul-ops.c

@@ -277,10 +277,9 @@ static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
     const uint32_t tile_size = htp_tensor_get_tile_size(src0->type);
     const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type);
 
-    uint32_t n_k_tiles_w = ne00 / 32;
-    uint32_t n_k_tiles_a = ne10 / 32;
+    uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type);
     uint32_t tile_row_stride = n_k_tiles_w * tile_size;
-    uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
+    uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
 
     uint32_t ct_start = src0_start_row / 32;
     uint32_t ct_end   = (src0_end_row + 31) / 32;
@@ -289,7 +288,7 @@ static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
     if (src0_start_row < src0_end_row) {
         for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
             dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned,
-                           src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                           src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
         }
     }
 
@@ -341,7 +340,7 @@ static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
 
         if (push_ct < ct_end) {
             dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride),
-                           aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                           aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
             push_ct++;
         }
     }
@@ -379,10 +378,9 @@ static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
     const uint32_t tile_size = htp_tensor_get_tile_size(src0->type);
     const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type);
 
-    uint32_t n_k_tiles_w = ne00 / 32;
-    uint32_t n_k_tiles_a = ne10 / 32;
+    uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type);
     uint32_t tile_row_stride = n_k_tiles_w * tile_size;
-    uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
+    uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
 
     uint32_t ct_start = src0_start_row / 32;
     uint32_t ct_end   = (src0_end_row + 31) / 32;
@@ -401,7 +399,7 @@ static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
         }
         for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
             dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned,
-                           src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                           src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
         }
     }
 
@@ -422,7 +420,7 @@ static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
 
         if (push_ct < ct_end) {
             dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride),
-                           aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                           aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
             push_ct++;
         }
     }
@@ -460,8 +458,6 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data)
     const uint32_t n_prefetch = kparams->n_prefetch;
     assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0);
 
-    uint32_t n_k_tiles_a = ne10 / 32;
-
     for (uint32_t widx = 0; widx < n_weights; widx++) {
         const struct htp_tensor * restrict src_w = octx->src[widx];
         const struct htp_tensor * restrict dst   = octx->dsts[widx];
@@ -470,15 +466,14 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data)
 
         const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type);
         const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type);
-        const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
-
         const uint32_t ne00 = src_w->ne[0];
         const uint32_t ne01 = src_w->ne[1];
         const size_t dst_row_size = dst->nb[1];
         const dma_addr_t src_w_row = src_w->data;
 
-        uint32_t n_k_tiles_w = ne00 / 32;
-        uint32_t tile_row_stride = n_k_tiles_w * tile_size;
+        const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src_w->type);
+        const uint32_t tile_row_stride = n_k_tiles_w * tile_size;
+        const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
 
         uint32_t src0_start_row = 0;
         uint32_t src0_end_row   = ne01;
@@ -504,7 +499,7 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data)
         uint32_t push_ct = ct_start;
         for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
             dma_queue_push(dma_q, dma_make_data(vtcm_weight_ptr + d * tile_row_transfer_size_aligned,
-                           src_w_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                            src_w_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
         }
 
         for (uint32_t ct = ct_start; ct < ct_end; ct++) {
@@ -536,7 +531,7 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data)
 
             if (push_ct < ct_end) {
                 dma_queue_push(dma_q, dma_make_data(w_tile, src_w_row + push_ct * tile_row_stride),
-                               aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                                aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
                 push_ct++;
             }
         }
@@ -840,10 +835,9 @@ static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) {
     const uint32_t tile_size = htp_tensor_get_tile_size(src0->type);
     const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type);
 
-    const uint32_t n_k_tiles_w = ne00 / 32;
-    const uint32_t n_k_tiles_a = ne10 / 32;
+    const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type);
     const uint32_t tile_row_stride = n_k_tiles_w * tile_size;
-    const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
+    const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
     const uint32_t src0_slice_stride = ((ne01 + 31) / 32) * tile_row_stride;
 
     const uint32_t ct_start = src0_start_row / 32;
@@ -884,7 +878,7 @@ static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) {
         for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
             dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned,
                            src0_slice + (size_t) push_ct * tile_row_stride),
-                           aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                           aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
         }
 
         for (uint32_t ct = ct_start; ct < ct_end; ct++) {
@@ -929,7 +923,7 @@ static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) {
 
             if (push_ct < ct_end) {
                 dma_queue_push(dma_q, dma_make_data(w_tile, src0_slice + (size_t) push_ct * tile_row_stride),
-                               aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                               aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
                 push_ct++;
             }
         }
@@ -1343,10 +1337,9 @@ static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) {
 
         const uint32_t tile_size = htp_tensor_get_tile_size(src0->type);
         const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type);
-        const uint32_t n_k_tiles_w = ne00 / 32;
-        const uint32_t n_k_tiles_a = ne10 / 32;
+        const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type);
         const uint32_t tile_row_stride = n_k_tiles_w * tile_size;
-        const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
+        const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
 
         const uint32_t ct_start = src0_start_row / 32;
         const uint32_t ct_end   = (src0_end_row + 31) / 32;
@@ -1354,7 +1347,7 @@ static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) {
         uint32_t push_ct = ct_start;
         for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
             dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride),
-                           aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                           aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
         }
 
         for (uint32_t ct = ct_start; ct < ct_end; ct++) {
@@ -1379,7 +1372,7 @@ static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) {
 
             if (push_ct < ct_end) {
                 dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride),
-                               aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                               aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
                 push_ct++;
             }
         }
@@ -1431,10 +1424,9 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) {
 
         const uint32_t tile_size = htp_tensor_get_tile_size(src0->type);
         const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type);
-        const uint32_t n_k_tiles_w = ne00 / 32;
-        const uint32_t n_k_tiles_a = ne10 / 32;
+        const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type);
         const uint32_t tile_row_stride = n_k_tiles_w * tile_size;
-        const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
+        const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
 
         const uint32_t ct_start = src0_start_row / 32;
         const uint32_t ct_end   = (src0_end_row + 31) / 32;
@@ -1442,7 +1434,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) {
         uint32_t push_ct = ct_start;
         for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
             dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride),
-                           aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                           aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
         }
 
         for (uint32_t ct = ct_start; ct < ct_end; ct++) {
@@ -1457,7 +1449,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) {
 
             if (push_ct < ct_end) {
                 dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride),
-                               aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                               aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
                 push_ct++;
             }
         }
@@ -1520,10 +1512,9 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
 
             const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type);
             const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type);
-            const uint32_t n_k_tiles_w = src_w->ne[0] / 32;
-            const uint32_t n_k_tiles_a = act->ne[0] / 32;
+            const uint32_t n_k_tiles_w = src_w->ne[0] / htp_tensor_get_tile_k(src_w->type);
             const uint32_t tile_row_stride = n_k_tiles_w * tile_size;
-            const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
+            const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
 
             const uint32_t ct_start = src0_start_row / 32;
             const uint32_t ct_end   = (src0_end_row + 31) / 32;
@@ -1531,7 +1522,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
             uint32_t push_ct = ct_start;
             for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
                 dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride),
-                               aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                               aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
             }
 
             for (uint32_t ct = ct_start; ct < ct_end; ct++) {
@@ -1546,7 +1537,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
 
                 if (push_ct < ct_end) {
                     dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride),
-                                   aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                                    aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
                     push_ct++;
                 }
             }
@@ -1611,10 +1602,9 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
 
             const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type);
             const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type);
-            const uint32_t n_k_tiles_w = src_w->ne[0] / 32;
-            const uint32_t n_k_tiles_a = act->ne[0] / 32;
+            const uint32_t n_k_tiles_w = src_w->ne[0] / htp_tensor_get_tile_k(src_w->type);
             const uint32_t tile_row_stride = n_k_tiles_w * tile_size;
-            const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
+            const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
 
             const uint32_t ct_start = src0_start_row / 32;
             const uint32_t ct_end   = (src0_end_row + 31) / 32;
@@ -1622,7 +1612,7 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
             uint32_t push_ct = ct_start;
             for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
                 dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride),
-                               aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                               aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
             }
 
             for (uint32_t ct = ct_start; ct < ct_end; ct++) {
@@ -1647,7 +1637,7 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
 
                 if (push_ct < ct_end) {
                     dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride),
-                                   aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                                    aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
                     push_ct++;
                 }
             }
@@ -1693,6 +1683,11 @@ static int hvx_mm_init_tiled_vec_dot(struct htp_mm_context * mmctx, enum htp_dat
             mmctx->vec_dot_32x1 = tiled_vec_dot_q2_k_32x1;
             mmctx->vec_dot_32x2 = tiled_vec_dot_q2_k_32x2;
             return 0;
+        case HTP_TYPE_Q2_0:
+            mmctx->type         = "q2_0_tiled-f32";
+            mmctx->vec_dot_32x1 = tiled_vec_dot_q2_0_32x1;
+            mmctx->vec_dot_32x2 = tiled_vec_dot_q2_0_32x2;
+            return 0;
         case HTP_TYPE_IQ4_NL:
             mmctx->type         = "iq4nl_tiled-f32";
             mmctx->vec_dot_32x1 = tiled_vec_dot_iq4nl_32x1;
@@ -1703,6 +1698,11 @@ static int hvx_mm_init_tiled_vec_dot(struct htp_mm_context * mmctx, enum htp_dat
             mmctx->vec_dot_32x1 = tiled_vec_dot_mxfp4_32x1;
             mmctx->vec_dot_32x2 = tiled_vec_dot_mxfp4_32x2;
             return 0;
+        case HTP_TYPE_Q1_0:
+            mmctx->type         = "q1_0_tiled-f32";
+            mmctx->vec_dot_32x1 = tiled_vec_dot_q1_0_32x1;
+            mmctx->vec_dot_32x2 = tiled_vec_dot_q1_0_32x2;
+            return 0;
         case HTP_TYPE_F16:
             mmctx->type         = "f16_tiled-f16";
             mmctx->vec_dot_32x1 = tiled_vec_dot_f16_32x1;
@@ -2325,6 +2325,8 @@ static inline tiled_dequant_task_fn_t hmx_get_dequant_worker_fn(int weight_type,
         case HTP_TYPE_Q6_K:   return dequantize_tiled_weight_to_fp16_task_q6_k;
         case HTP_TYPE_Q3_K:   return dequantize_tiled_weight_to_fp16_task_q3_k;
         case HTP_TYPE_Q2_K:   return dequantize_tiled_weight_to_fp16_task_q2_k;
+        case HTP_TYPE_Q1_0:   return dequantize_tiled_weight_to_fp16_task_q1_0;
+        case HTP_TYPE_Q2_0:   return dequantize_tiled_weight_to_fp16_task_q2_0;
         case HTP_TYPE_F16:    return is_tiled ? NULL : convert_f16_weight_to_fp16_tiles_task;
         case HTP_TYPE_BF16:   return convert_bf16_weight_to_fp16_tiles_task;
         case HTP_TYPE_F32:    return quantize_f32_weight_to_fp16_tiles_task;
@@ -2346,8 +2348,9 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles(
 
     assert(k_block % HTP_MM_HMX_TILE_N_COLS == 0);
 
-    size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS);
-    size_t n_tot_tiles = n_col_tiles * n_k_tiles;
+    const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS);
+    const uint32_t n_weight_tiles = k_block / htp_tensor_get_tile_k(weight_type);
+    const size_t n_tot_tiles = n_col_tiles * n_weight_tiles;
 
     size_t n_tiles_per_task = (n_threads == 1) ? n_tot_tiles : hmx_ceil_div(n_tot_tiles, n_threads);
 
@@ -2363,6 +2366,7 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles(
     state.row_stride       = row_stride;
     state.weight_type      = weight_type;
     state.n_k_tiles        = n_k_tiles;
+    state.n_weight_tiles   = n_weight_tiles;
     state.n_k_tiles_div    = n_k_tiles_div;
     state.traces           = ctx->trace;
     state.ctx              = ctx;
@@ -2666,6 +2670,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
     tiled_dequant_task_fn_t dequant_worker_fn = hmx_get_dequant_worker_fn(weight_type, is_tiled);
 
     const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS;
+    const int n_weight_tiles = k / htp_tensor_get_tile_k(weight_type);
     const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles);
 
     const size_t vec_dot_size = k * sizeof(__fp16);
@@ -2679,8 +2684,6 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
     size_t n_chunk_n_cols = n_chunk;
     size_t vtcm_used      = vtcm_size;
 
-    const size_t qweight_row_stride = is_tiled ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0;
-
     struct htp_mm_hmx_vtcm_layout L;
     htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_threads, aligned_tile_size, src2_bytes);
 
@@ -2759,13 +2762,13 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
 
             // Prologue: push A0 and optionally A1 (if n_chunk_cnt > 1)
             const size_t   n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols);
-            const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_k_tiles : n_cols_A0;
+            const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_weight_tiles : n_cols_A0;
             dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight),
                            dma_dst_stride, dma_src_stride, dma_width_bytes, height_A0);
 
             if (1 < n_chunk_cnt) {
                 const size_t   n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols);
-                const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_k_tiles : n_cols_A1;
+                const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_weight_tiles : n_cols_A1;
                 dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[1], weight + n_chunk_n_cols * weight_stride),
                                dma_dst_stride, dma_src_stride, dma_width_bytes, height_A1);
             }
@@ -2789,7 +2792,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
 
                 // 3. push A_{i+2} (if i+2 < n_chunk_cnt)
                 if (i + 2 < n_chunk_cnt) {
-                    const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_k_tiles : n_cols_p2;
+                    const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_weight_tiles : n_cols_p2;
                     dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 2) % 4], weight + nc_p2 * weight_stride),
                                    dma_dst_stride, dma_src_stride, dma_width_bytes, height_p2);
                 }
@@ -2855,7 +2858,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
             // A0: Pre-fetch the first weight chunk (nc = 0)
             if (n > 0) {
                 const size_t n_cols = hex_smin(n, n_chunk_n_cols);
-                const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_k_tiles : n_cols;
+                const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_weight_tiles : n_cols;
                 dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height);
             }
 
@@ -2878,7 +2881,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
                 const size_t nc_next = nc + n_chunk_n_cols;
                 if (nc_next < n) {
                     const size_t n_cols_next = hex_smin(n - nc_next, n_chunk_n_cols);
-                    const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_k_tiles : n_cols_next;
+                    const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_weight_tiles : n_cols_next;
                     dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 1) % 2], weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next);
                 }
 
@@ -2930,6 +2933,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
     }
 
     const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS;
+    const int n_weight_tiles = k / htp_tensor_get_tile_k(weight_type);
     const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles);
 
     const int tile_size       = kparams->tile_size;
@@ -3057,13 +3061,13 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
                 const uint32_t dma_src_stride = is_tiled ? tile_size : weight_stride;
 
                 const size_t   n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols);
-                const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_k_tiles : n_cols_A0;
+                const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_weight_tiles : n_cols_A0;
                 dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight),
                                dma_dst_stride, dma_src_stride, dma_width_bytes, height_A0);
 
                 if (1 < n_chunk_cnt) {
                     const size_t   n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols);
-                    const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_k_tiles : n_cols_A1;
+                    const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_weight_tiles : n_cols_A1;
                     dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[1], weight + n_chunk_n_cols * weight_stride),
                                    dma_dst_stride, dma_src_stride, dma_width_bytes, height_A1);
                 }
@@ -3083,7 +3087,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
                         n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads);
 
                     if (i + 2 < n_chunk_cnt) {
-                        const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_k_tiles : n_cols_p2;
+                        const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_weight_tiles : n_cols_p2;
                         dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 2) % 4], weight + nc_p2 * weight_stride),
                                        dma_dst_stride, dma_src_stride, dma_width_bytes, height_p2);
                     }
@@ -3160,7 +3164,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
 
                 if (n > 0) {
                     const size_t n_cols = hex_smin(n, n_chunk_n_cols);
-                    const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_k_tiles : n_cols;
+                    const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_weight_tiles : n_cols;
                     dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height);
                 }
 
@@ -3180,7 +3184,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
                     const size_t nc_next = nc + n_chunk_n_cols;
                     if (nc_next < n) {
                         const size_t n_cols_next = hex_smin(n - nc_next, n_chunk_n_cols);
-                        const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_k_tiles : n_cols_next;
+                        const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_weight_tiles : n_cols_next;
                         dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 1) % 2], weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next);
                     }
 
@@ -3544,6 +3548,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
     }
 
     const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS;
+    const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(weight_type);
     const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles);
 
     const size_t vec_dot_size = k * sizeof(__fp16);
@@ -3557,7 +3562,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
     const uint32_t dma_src_stride  = is_tiled ? tile_size : weight_stride;
     const uint32_t dma_width_bytes = is_tiled ? tile_size : row_stride;
 
-    const size_t qweight_row_stride = is_tiled ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0;
+    const size_t qweight_row_stride = is_tiled ? (size_t)(n_weight_tiles * aligned_tile_size) / 32 : 0;
     const size_t weight_row_stride = is_tiled ? qweight_row_stride : row_stride;
 
     size_t size_per_n = 0, size_per_m = 0, size_per_mn = 0;
@@ -3613,7 +3618,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
         // A0: Pre-fetch the first weight chunk (nc = 0)
         if (n > 0) {
             const size_t n_cols = hex_smin((size_t) n, n_chunk_n_cols);
-            const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_k_tiles : n_cols;
+            const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_weight_tiles : n_cols;
             dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight),
                            dma_dst_stride, dma_src_stride, dma_width_bytes, height);
         }
@@ -3637,7 +3642,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
             const size_t nc_next = nc + n_chunk_n_cols;
             if (nc_next < (size_t) n) {
                 const size_t n_cols_next = hex_smin((size_t) n - nc_next, n_chunk_n_cols);
-                const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_k_tiles : n_cols_next;
+                const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_weight_tiles : n_cols_next;
                 dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 1) % 2], weight + nc_next * weight_stride),
                                dma_dst_stride, dma_src_stride, dma_width_bytes, height_next);
             }

ggml/src/ggml-hexagon/htp/matmul-ops.h

@@ -20,6 +20,10 @@ extern "C" {
 #define HTP_MM_HMX_TILE_N_ELMS 1024
 #define HTP_MM_HMX_MIN_NROWS   4
 
+#define HTP_MM_WEIGHT_TILE_K_Q2_0             64
+#define HTP_MM_WEIGHT_TILE_SIZE_Q2_0          HTP_TENSOR_TILE_SIZE_Q2_0
+#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0  640
+
 // --- Activation Tiled Block Sizes (including padding) ---
 #define HTP_MM_ACT_TILE_SIZE_Q8_0      1152
 #define HTP_MM_ACT_TILE_SIZE_Q8_1      1280
@@ -183,7 +187,7 @@ next_nc:
 // (Q2_K: per-16 k sums, q8_1_s16)
 static inline bool htp_mm_weight_has_offset(int weight_type) {
     return weight_type == HTP_TYPE_Q4_1 || weight_type == HTP_TYPE_Q4_K || weight_type == HTP_TYPE_Q5_K ||
-           weight_type == HTP_TYPE_Q2_K;
+           weight_type == HTP_TYPE_Q2_K || weight_type == HTP_TYPE_Q2_0;
 }
 
 // --- Activation/Row Size Helpers ---
@@ -227,8 +231,8 @@ static inline void htp_mm_hmx_get_2d_chunk_costs(
     const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32 && wtype != HTP_TYPE_BF16);
     const size_t row_stride = htp_tensor_get_row_size(wtype, k);
     const size_t vec_dot_size = k * sizeof(uint16_t);
-    const uint32_t n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS;
-    const size_t qweight_row_stride = is_quant ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0;
+    const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(wtype);
+    const size_t qweight_row_stride = is_quant ? (size_t)(n_weight_tiles * aligned_tile_size) / 32 : 0;
 
     *size_per_n_out = (pipeline ? 2 : 1) * (is_quant ? qweight_row_stride : row_stride) +
                       (pipeline ? 2 * vec_dot_size : vec_dot_size);
@@ -359,11 +363,11 @@ static inline void htp_mm_hmx_vtcm_layout_build(
         const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32 && wtype != HTP_TYPE_BF16);
         const size_t row_stride = htp_tensor_get_row_size(wtype, k);
         const size_t vec_dot_size = k * sizeof(uint16_t);
-        const uint32_t n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS;
+        const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(wtype);
 
         const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128);
         const size_t weight_area_size = is_quant
-            ? hex_align_up((nc / 32) * n_k_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE)
+            ? hex_align_up((nc / 32) * n_weight_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE)
             : hex_align_up(nc * row_stride, HTP_MM_HMX_TILE_SIZE);
         const size_t act_area_size    = hex_align_up(mc * vec_dot_size, HTP_MM_HMX_TILE_SIZE);
         const size_t output_area_size = hex_align_up(mc * nc * sizeof(__fp16), HTP_MM_HMX_TILE_SIZE);
@@ -435,7 +439,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(
 
     if (is_fused_nx) {
         uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
-        uint32_t n_k_tiles = hex_round_up(ne10, 32) / 32;
+        uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype);
         uint32_t tile_row_size = n_k_tiles * aligned_tile_size;
         size_t weight_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128);
 
@@ -451,7 +455,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(
     } else if (is_matmul_id) {
         const size_t act_row_size_tiled = htp_mm_tiled_act_row_size(wtype, ne10);
         const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
-        const uint32_t n_k_tiles         = ne10 / 32;
+        const uint32_t n_k_tiles         = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype);
         const uint32_t tile_row_size     = n_k_tiles * aligned_tile_size;
         size_t src0_sz_per_thread        = htp_mm_round_up(n_prefetch * tile_row_size, 256);
 
@@ -489,8 +493,8 @@ static inline void htp_mm_hvx_vtcm_layout_build(
             case HTP_MM_KERNEL_HVX_QUANT_ROW: {
                 size_t q_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
                 uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
-                uint32_t n_k_tiles = (ne10 + 31) / 32;
-                uint32_t tile_row_size = n_k_tiles * aligned_tile_size;
+                uint32_t n_weight_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype);
+                uint32_t tile_row_size = n_weight_tiles * aligned_tile_size;
                 size_t tiled_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256);
 
                 src0_sz = tiled_vtcm_size * n_threads;

ggml/src/ggml-hexagon/htp/unary-ops.c

@@ -690,22 +690,48 @@ static void tri_f32(const void * restrict src,
     }
 }
 
+// softplus(x) = log(1 + exp(x))
+// Match CPU reference: return x for x > 20 to avoid overflow.
+static inline void hvx_softplus_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+    const HVX_Vector * restrict v_src = (const HVX_Vector *) src;
+    HVX_Vector * restrict v_dst       = (HVX_Vector *) dst;
+
+    const HVX_Vector twenty = hvx_vec_splat_f32(20.0f);
+    const HVX_Vector one    = hvx_vec_splat_f32(1.0f);
+
+    const uint32_t nvec = n / VLEN_FP32;
+    const uint32_t nloe = n % VLEN_FP32;
+
+    #pragma unroll(4)
+    for (uint32_t i = 0; i < nvec; i++) {
+        HVX_Vector x              = v_src[i];
+        HVX_VectorPred pred_gt20  = Q6_Q_vcmp_gt_VsfVsf(x, twenty);
+        HVX_Vector expx           = hvx_vec_exp_f32(x);
+        HVX_Vector logv           = hvx_vec_log_f32(hvx_vec_add_f32_f32(one, expx));
+        v_dst[i]                  = Q6_V_vmux_QVV(pred_gt20, x, logv);
+    }
+
+    if (nloe > 0) {
+        HVX_Vector x             = v_src[nvec];
+        HVX_VectorPred pred_gt20 = Q6_Q_vcmp_gt_VsfVsf(x, twenty);
+        HVX_Vector expx          = hvx_vec_exp_f32(x);
+        HVX_Vector logv          = hvx_vec_log_f32(hvx_vec_add_f32_f32(one, expx));
+        HVX_Vector sp            = Q6_V_vmux_QVV(pred_gt20, x, logv);
+        hvx_vec_store_a((void *) &v_dst[nvec], nloe * sizeof(float), sp);
+    }
+}
+
 static void softplus_f32(const void * restrict src,
                          void * restrict dst,
                          const uint32_t num_rows,
                          const struct htp_unary_context * uctx) {
     htp_unary_op_preamble;
-    // softplus(x) = log(1 + exp(x))
-    // Match CPU reference: ggml_compute_softplus_f32() in ggml-impl.h
-    for (uint32_t ir = 0; ir < num_rows; ir++) {
-        const float * restrict src_f = (const float *)((const uint8_t *)src + (ir * src0_row_size_aligned));
-        float * restrict dst_f       = (float *)((uint8_t *)dst + (ir * dst_row_size_aligned));
 
-        for (uint32_t i = 0; i < ne0; i++) {
-            float x = src_f[i];
-            // For x > 20: softplus(x) ~ x (avoids exp overflow)
-            dst_f[i] = (x > 20.0f) ? x : logf(1.0f + expf(x));
-        }
+    for (uint32_t ir = 0; ir < num_rows; ir++) {
+        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
+        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
+
+        hvx_softplus_f32_aa(dst_local, src_local, ne0);
     }
 }
 
@@ -866,12 +892,7 @@ static void tile_gelu_erf_f32(void * restrict dst, const void * restrict src, ui
 
 static void tile_softplus_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) {
     (void) uctx;
-    const float * restrict sf = (const float *) src;
-    float * restrict df       = (float *) dst;
-    for (uint32_t i = 0; i < tw; i++) {
-        float x = sf[i];
-        df[i] = (x > 20.0f) ? x : logf(1.0f + expf(x));
-    }
+    hvx_softplus_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw);
 }
 
 static void tile_tanh_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) {

ggml/src/ggml-opencl/CMakeLists.txt

@@ -132,6 +132,7 @@ set(GGML_OPENCL_KERNELS
     gemm_moe_q6_k_q8_1_dp4a
     gemm_moe_q8_1_dp4a
     moe_reorder_quant_a_q8_1
+    transpose_activation_q8
     gemm_noshuffle_q4_k_q8_1_dp4a
     gemm_noshuffle_q5_k_q8_1_dp4a
     gemm_noshuffle_q6_k_q8_1_dp4a

ggml/src/ggml-opencl/ggml-opencl.cpp

@@ -720,6 +720,10 @@ struct ggml_backend_opencl_context {
     ggml_cl_buffer prealloc_moe_qa;   // int8 quants  [tok_slots * ne00]
     ggml_cl_buffer prealloc_moe_da;   // per-block d  [tok_slots * ne00/32] (half)
     ggml_cl_buffer prealloc_moe_sa;   // per-block s  [tok_slots * ne00/32] (half)
+    // per-tile transposed qa/da and on-chip scratch for the ldck dp4a MoE GEMM
+    ggml_cl_buffer prealloc_moe_qa_trans;    // int8 quants  [tok_slots * ne00]
+    ggml_cl_buffer prealloc_moe_da_trans;    // per-block d  [tok_slots * ne00/32] (float)
+    ggml_cl_buffer prealloc_moe_qda_on_chip;
     // scratch copy of the router weights to avoid dst aliasing
     ggml_cl_buffer prealloc_moe_combine_w;
     ggml_cl_buffer prealloc_splitk_partial;  // [ksplit * M] partials for split-K GEMV
@@ -1016,6 +1020,7 @@ struct ggml_backend_opencl_context {
     cl_kernel kernel_gemv_moe_q4_k_f32_ns_wimg = nullptr;  // weight-as-texture MoE decode GEMV (opt-in)
     cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a = nullptr;    // dp4a (int8) prefill GEMM variant
     cl_kernel kernel_moe_reorder_quant_a_q8_1;   // fused reorder + q8_1 quant for the dp4a GEMM
+    cl_kernel kernel_transpose_activation_q8;    // per-tile transpose of q8_1 activations for the ldck dp4a GEMM
     cl_kernel kernel_gemm_moe_q8_1_dp4a_q80 = nullptr;   // generic dp4a MoE GEMM (MOE_QT=80), opt-in
     cl_kernel kernel_moe_expand_scale_q8_0 = nullptr;    // q8_0 per-block d -> uniform scale[16]
     cl_kernel kernel_gemm_moe_q8_1_dp4a_q50 = nullptr;   // generic dp4a MoE GEMM (MOE_QT=50, q5_0), opt-in
@@ -1031,6 +1036,7 @@ struct ggml_backend_opencl_context {
     cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a = nullptr;   // dp4a (int8) mxfp4 MoE prefill GEMM
     cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a = nullptr;    // dp4a (int8) q4_0 MoE prefill GEMM
     cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr;   // binary dp4a (int8) mxfp4 MoE prefill GEMM
+    cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = nullptr;   // binary dp4a (int8) mxfp4 MoE prefill GEMM using ldck
     cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr;    // binary dp4a (int8) q4_0 MoE prefill GEMM
     cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a_bin = nullptr;   // binary dp4a (int8) q4_k MoE prefill GEMM
     cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a_bin = nullptr;    // binary dp4a (int8) q6_k MoE prefill GEMM
@@ -4903,6 +4909,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
         }
     }
 
+    // gemm_moe_mxfp4_q8_1_dp4a_ldck_bin (dp4a prefill GEMM)
+    if (backend_ctx->has_integer_dot) {
+        size_t bin_size = 0;
+        backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = nullptr;
+
+        if (use_adreno_bin_kernels(backend_ctx)) {
+            const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_q8_1_dp4a_ila_ldck", &bin_size);
+            if (kernel_bin && bin_size > 0) {
+                cl_program prog =
+                    build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size);
+
+                CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a_ila_ldck", &err), err));
+                CL_CHECK(clReleaseProgram(prog));
+                GGML_LOG_CONT(".");
+            }
+        }
+    }
+
     // gemm_moe_q4_0_q8_1_dp4a (dp4a prefill GEMM)
     if (backend_ctx->has_integer_dot) {
 #ifdef GGML_OPENCL_EMBED_KERNELS
@@ -4984,6 +5008,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
         GGML_LOG_CONT(".");
     }
 
+    // transpose_activation_q8 (per-tile transpose of q8_1 activations for the ldck dp4a GEMM)
+    {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+        const std::string kernel_src {
+            #include "transpose_activation_q8.cl.h"
+        };
+#else
+        const std::string kernel_src = read_file("transpose_activation_q8.cl");
+#endif
+        cl_program prog =
+            build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts);
+
+        CL_CHECK((backend_ctx->kernel_transpose_activation_q8 = clCreateKernel(prog, "kernel_transpose_activation_q8", &err), err));
+        CL_CHECK(clReleaseProgram(prog));
+        GGML_LOG_CONT(".");
+    }
+
     // gemv_moe_q5_k_f32_ns
     {
 #ifdef GGML_OPENCL_EMBED_KERNELS
@@ -28258,11 +28299,19 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
                     static const char * mxfp4_moe_dp4a_env = getenv("GGML_OPENCL_MXFP4_MOE_DP4A");
                     bool use_moe_dp4a = mxfp4_moe_dp4a_env
                         ? (atoi(mxfp4_moe_dp4a_env) != 0)
-                        : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+                        : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E);
                     // dot prod has to be available
                     use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;
                     // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel
-                    if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr) {
+                    // ldck dp4a bin kernel (X1) has the same priority as normal bin kernel (X2/A8x), they are not expected to coexist.
+                    const bool use_moe_dp4a_ldck = backend_ctx->has_integer_dot &&
+                        (!mxfp4_moe_dp4a_env || atoi(mxfp4_moe_dp4a_env) != 0) &&
+                        backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr &&
+                        backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr &&
+                        backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin != nullptr;
+                    if (use_moe_dp4a_ldck) {
+                        use_moe_dp4a = true;
+                    } else if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr) {
                         use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr;
                     }
 
@@ -28351,6 +28400,12 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
                         backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));
                         backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
                         backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+                        if (use_moe_dp4a_ldck) {
+                            backend_ctx->prealloc_moe_qa_trans.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));
+                            backend_ctx->prealloc_moe_da_trans.allocate(backend_ctx->context, n_blocks * sizeof(cl_float));
+                            // matches max_constant_size of src1_a_cm in the ldck kernel
+                            backend_ctx->prealloc_moe_qda_on_chip.allocate(backend_ctx->context, 2304);
+                        }
 
                         // fused reorder + q8_1 quant straight from the original
                         // activations (no intermediate f32 reorder buffer). mxfp4 has no
@@ -28375,13 +28430,33 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
                         cl_kernel dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a;
                         if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin) {
                             dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin;
+                        } else if (use_moe_dp4a_ldck) {
+                            dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin;
+
+                            // per-tile transpose of qa/da for the ldck kernel
+                            cl_kernel tk = backend_ctx->kernel_transpose_activation_q8;
+                            CL_CHECK(clSetKernelArg(tk, 0, sizeof(cl_mem),  &backend_ctx->prealloc_moe_qa.buffer));
+                            CL_CHECK(clSetKernelArg(tk, 1, sizeof(cl_mem),  &backend_ctx->prealloc_moe_qa_trans.buffer));
+                            CL_CHECK(clSetKernelArg(tk, 2, sizeof(cl_mem),  &backend_ctx->prealloc_moe_da.buffer));
+                            CL_CHECK(clSetKernelArg(tk, 3, sizeof(cl_mem),  &backend_ctx->prealloc_moe_da_trans.buffer));
+                            CL_CHECK(clSetKernelArg(tk, 4, sizeof(cl_mem),  &(backend_ctx->prealloc_total_tiles.buffer)));
+                            CL_CHECK(clSetKernelArg(tk, 5, sizeof(cl_uint), &ne00));
+                            size_t tk_local[3]  = { 32, 8, 1 };
+                            size_t tk_global[3] = { 32, (size_t)(ne00 / 4), (size_t)max_post_router_tile };
+                            backend_ctx->enqueue_ndrange_kernel(tk, 3, tk_global, tk_local, dst);
                         }
 
                         int aidx = 0;
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->q_img));
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->e));
-                        CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
-                        CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+                        if (use_moe_dp4a_ldck) {
+                            CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa_trans.buffer));
+                            CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da_trans.buffer));
+                            CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qda_on_chip.buffer));
+                        } else {
+                            CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+                            CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+                        }
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2));
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap));
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image));

ggml/src/ggml-opencl/kernels/transpose_activation_q8.cl

@@ -0,0 +1,35 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+
+#define tile_size 32
+
+kernel void kernel_transpose_activation_q8(
+        __global uint * src1_qa,
+        __global uint * src1_qa_trans,
+        __global half * src1_sa,
+        __global float * src1_sa_trans,
+        __global const int * total_tiles,
+        uint  K
+) {
+    uint row_in_tile = get_global_id(0);
+    uint blk_k = get_group_id(1);
+    uint col_in_tile = get_local_id(1);
+    uint tile_n = get_global_id(2);
+
+    if (blk_k >= (K / tile_size) || tile_n >= (uint)total_tiles[0]) {
+        return;
+    }
+
+    uint src_idx = (tile_n * tile_size + row_in_tile) * K / 4 + blk_k * 8 + col_in_tile;
+    uint dst_idx = tile_n * tile_size * K / 4 + (blk_k * 8 + col_in_tile) * tile_size + row_in_tile;
+
+    src1_qa_trans[dst_idx] = src1_qa[src_idx];
+
+    if (col_in_tile == 0) {
+        src_idx = (tile_n * tile_size + row_in_tile) * K / 32 + blk_k;
+        dst_idx = tile_n * tile_size * K / 32 + blk_k * tile_size + row_in_tile;
+
+        src1_sa_trans[dst_idx] = (float)(src1_sa[src_idx]);
+    }
+
+}

ggml/src/ggml-sycl/convert.cpp

@@ -158,16 +158,16 @@ static void dequantize_row_q4_0_sycl_reorder(const void *vx, dst_t *y, const int
     dpct::has_capability_or_fail(stream->get_device(),
                                     {sycl::aspect::fp16});
 
-    int constexpr WARP_K = WARP_SIZE * QK4_0;
-    const int n_warp = (k + WARP_K - 1) / WARP_K;
-    GGML_ASSERT(k % 2 == 0);
-    stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) *
-        sycl::range<3>(1, 1, WARP_SIZE),
-        sycl::range<3>(1, 1, WARP_SIZE)),
-        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{
+    GGML_ASSERT(k % QK4_0 == 0);
+    const int64_t n_groups = ceil_div(k / 2, SYCL_DEQUANTIZE_BLOCK_SIZE);
+    // decrease global range when it exceeds the max int; the kernel loops to cover the rest
+    const int64_t wg_size  = downsample_sycl_global_range(n_groups, SYCL_DEQUANTIZE_BLOCK_SIZE);
+    stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_groups) *
+        sycl::range<3>(1, 1, wg_size),
+        sycl::range<3>(1, 1, wg_size)),
+        [=](sycl::nd_item<3> item_ct1) {
             dequantize_block_q4_0_reorder(vx, y, k, item_ct1);
         });
-
 }
 
 template <typename dst_t>
@@ -177,16 +177,16 @@ static void dequantize_row_q8_0_sycl_reorder(const void *vx, dst_t *y, const int
     dpct::has_capability_or_fail(stream->get_device(),
                                     {sycl::aspect::fp16});
 
-    int constexpr WARP_K = WARP_SIZE * QK8_0;
-    const int n_warp = (k + WARP_K - 1) / WARP_K;
     GGML_ASSERT(k % QK8_0 == 0);
-    stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) *
-        sycl::range<3>(1, 1, WARP_SIZE),
-        sycl::range<3>(1, 1, WARP_SIZE)),
-        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{
+    const int64_t n_groups = ceil_div(k / 2, SYCL_DEQUANTIZE_BLOCK_SIZE);
+    // decrease global range when it exceeds the max int; the kernel loops to cover the rest
+    const int64_t wg_size  = downsample_sycl_global_range(n_groups, SYCL_DEQUANTIZE_BLOCK_SIZE);
+    stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_groups) *
+        sycl::range<3>(1, 1, wg_size),
+        sycl::range<3>(1, 1, wg_size)),
+        [=](sycl::nd_item<3> item_ct1) {
             dequantize_block_q8_0_reorder(vx, y, k, item_ct1);
         });
-
 }
 
 template <typename dst_t>

ggml/src/ggml-sycl/dequantize.hpp

@@ -818,61 +818,43 @@ static void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restri
 }
 
 template<typename dst_t>
-static void dequantize_block_q4_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb32,
+static void dequantize_block_q4_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k,
                                   const sycl::nd_item<3> &item_ct1) {
+    // one thread per qs byte, like dequantize_block<dequantize_q4_0>; the reorder layout makes the byte reads contiguous
+    // grid-stride loop because the launcher may shrink the work-group to keep the global range below INT_MAX
+    // float math, not dequantize_q4_0_reorder: that helper computes in half with GGML_SYCL_F16, which changes f32 output
+    const sycl::half * d_ptr  = (const sycl::half *) ((const uint8_t *) vx + k/2);
+    const int64_t      stride = item_ct1.get_local_range(2) * item_ct1.get_group_range(2);
 
-    const int64_t i = item_ct1.get_group(2);
-    auto k=nb32;
-    // assume 32 threads
-    const int64_t tid = item_ct1.get_local_id(2);
-    const int lane_ib = i * WARP_SIZE + tid;
+    for (int64_t iq = item_ct1.get_global_id(2); iq < k/2; iq += stride) {
+        const int64_t ib  = iq / (QK4_0/2);
+        const int64_t iqs = iq % (QK4_0/2);
+        const int     vq  = ((const uint8_t *) vx)[iq];
+        const float   d   = float(d_ptr[ib]);
 
-    if (lane_ib >= k / QK4_0) {
-        return;
+        dst_t * y = yy + ib*QK4_0 + iqs;
+        y[0]       = d * ((vq & 0xF) - 8);
+        y[QK4_0/2] = d * ((vq >>  4) - 8);
     }
-
-    dst_t * y_ptr = yy + lane_ib * QK4_0;
-
-    auto qs = (const uint8_t*)vx + lane_ib * QK4_0 / 2;
-    auto s_ptr = (const sycl::half*)((const uint8_t*)vx + k / 2) + lane_ib;
-
-    const float d = float(*s_ptr);
-
-#pragma unroll
-    for (int l = 0; l < QK4_0 / 2; ++l) {
-        int vq = qs[l];
-        y_ptr[l + 0] = d * ((vq & 0xF) - 8);
-        y_ptr[l + 16] = d * ((vq >> 4) - 8);
-    }
-
 }
 
 // Dequantize Q8_0 from reorder layout: [all qs (k bytes)][all d values]
-// Each thread handles one block of QK8_0 elements.
 template<typename dst_t>
 static void dequantize_block_q8_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k,
                                   const sycl::nd_item<3> &item_ct1) {
+    // two values per thread, like dequantize_block<dequantize_q8_0>; the reorder layout keeps all qs contiguous
+    // grid-stride loop because the launcher may shrink the work-group to keep the global range below INT_MAX
+    // float math, not dequantize_q8_0_reorder: that helper computes in half with GGML_SYCL_F16, which changes f32 output
+    const int8_t *     qs     = (const int8_t *) vx;
+    const sycl::half * d_ptr  = (const sycl::half *) ((const uint8_t *) vx + k);
+    const int64_t      stride = 2 * item_ct1.get_local_range(2) * item_ct1.get_group_range(2);
 
-    const int64_t i = item_ct1.get_group(2);
-    const int64_t tid = item_ct1.get_local_id(2);
-    const int lane_ib = i * WARP_SIZE + tid;
+    for (int64_t i = 2 * item_ct1.get_global_id(2); i < k; i += stride) {
+        const float d = float(d_ptr[i / QK8_0]);
 
-    if (lane_ib >= k / QK8_0) {
-        return;
+        yy[i + 0] = d * qs[i + 0];
+        yy[i + 1] = d * qs[i + 1];
     }
-
-    dst_t * y_ptr = yy + lane_ib * QK8_0;
-
-    auto qs = (const int8_t*)vx + lane_ib * QK8_0;
-    auto s_ptr = (const sycl::half*)((const uint8_t*)vx + k) + lane_ib;
-
-    const float d = float(*s_ptr);
-
-#pragma unroll
-    for (int l = 0; l < QK8_0; ++l) {
-        y_ptr[l] = d * qs[l];
-    }
-
 }
 
 template<typename dst_t>

gguf-py/gguf/constants.py

@@ -573,6 +573,7 @@ class MODEL_ARCH(IntEnum):
     QWEN3MOE         = auto()
     QWEN3NEXT        = auto()
     HRM_TEXT         = auto()
+    BEAM             = auto()
     QWEN3VL          = auto()
     QWEN3VLMOE       = auto()
     QWEN35           = auto()
@@ -700,6 +701,7 @@ class MODEL_ARCH(IntEnum):
     QWEN3TTS         = auto()
     POCKETTTS        = auto()
     K2HORIZON        = auto()
+    KOLIBRI1         = auto()
 
 
 class VISION_PROJECTOR_TYPE(IntEnum):
@@ -1388,6 +1390,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.QWEN3MOE:         "qwen3moe",
     MODEL_ARCH.QWEN3NEXT:        "qwen3next",
     MODEL_ARCH.HRM_TEXT:         "hrm_text",
+    MODEL_ARCH.BEAM:             "beam",
     MODEL_ARCH.QWEN3VL:          "qwen3vl",
     MODEL_ARCH.QWEN3VLMOE:       "qwen3vlmoe",
     MODEL_ARCH.QWEN35:           "qwen35",
@@ -1516,6 +1519,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.QWEN3TTS:         "qwen3tts",
     MODEL_ARCH.POCKETTTS:        "pockettts",
     MODEL_ARCH.K2HORIZON:        "k2-horizon",
+    MODEL_ARCH.KOLIBRI1:         "kolibri1",
 }
 
 VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = {
@@ -2981,6 +2985,30 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
         MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
     ],
+    MODEL_ARCH.BEAM: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_POST_NORM,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.ATTN_Q_NORM,
+        MODEL_TENSOR.ATTN_K_NORM,
+        MODEL_TENSOR.ATTN_GATE,
+        MODEL_TENSOR.FFN_PRE_NORM,
+        MODEL_TENSOR.FFN_POST_NORM,
+        MODEL_TENSOR.FFN_GATE,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.FFN_GATE_INP,
+        MODEL_TENSOR.FFN_EXP_PROBS_B,
+        MODEL_TENSOR.FFN_GATE_EXP,
+        MODEL_TENSOR.FFN_DOWN_EXP,
+        MODEL_TENSOR.FFN_UP_EXP,
+    ],
     MODEL_ARCH.HRM_TEXT: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT,
@@ -6027,6 +6055,29 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_UP_SHEXP,
         MODEL_TENSOR.FFN_DOWN_SHEXP,
     ],
+    MODEL_ARCH.KOLIBRI1: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_POST_NORM,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.ATTN_Q_NORM,
+        MODEL_TENSOR.ATTN_K_NORM,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_POST_NORM,
+        MODEL_TENSOR.FFN_GATE_INP,
+        MODEL_TENSOR.FFN_EXP_PROBS_B,
+        MODEL_TENSOR.FFN_GATE_EXP,
+        MODEL_TENSOR.FFN_UP_EXP,
+        MODEL_TENSOR.FFN_DOWN_EXP,
+        MODEL_TENSOR.FFN_GATE_SHEXP,
+        MODEL_TENSOR.FFN_UP_SHEXP,
+        MODEL_TENSOR.FFN_DOWN_SHEXP,
+    ],
 }
 
 # tensors that will not be serialized
@@ -6283,6 +6334,7 @@ class DecisionType:
     VLLM_D3 = "vllm-d3"  # same as pplx-decider, non-causal attention
     LFM2_D1 = "lfm2-d1"  # same as openjev, the labels depend on the question type
     LFM2_D1_OMNI = "lfm2-d1-omni"  # same head as laya on a bidirectional LFM2 trunk, other prompt layout
+    GRANITE_SWITCH = "granite-switch"  # same as openjev, run by the decision aLoRA of granite switch
 
 
 class VisionProjectorType:

gguf-py/gguf/tensor_mapping.py

@@ -416,6 +416,7 @@ class TensorNameMap:
             "model.layers.{bid}.post_attention_layernorm",       # gemma2 olmo2    # ge
             "layers.{bid}.post_attention_layernorm",             # embeddinggemma
             "model.layers.{bid}.post_self_attn_layernorm",       # glm-4-0414
+            "model.layers.{bid}.post_attn_norm",                 # kolibri1
             "model.layers.layers.{bid}.post_mixer_norm.weight",  # plamo2
         ),
 
@@ -498,6 +499,7 @@ class TensorNameMap:
             "model.layers.{bid}.post_feedforward_layernorm",  # gemma2 olmo2
             "layers.{bid}.post_feedforward_layernorm",        # embeddinggemma
             "model.layers.{bid}.post_mlp_layernorm",          # glm-4-0414
+            "model.layers.{bid}.post_ffn_norm",               # kolibri1
             "model.layers.layers.{bid}.post_mlp_norm.weight", # plamo2
             "model.layers.{bid}.feed_forward.up_proj",
             "model.layers.{bid}.post_moe_norm",               # grok-2
@@ -542,6 +544,7 @@ class TensorNameMap:
             "model.layers.{bid}.mlp.moe_statics.e_score_correction",        # ernie4.5-moe
             "model.layers.{bid}.mlp.gate.expert_bias",                      # bailingmoe2
             "model.layers.{bid}.mlp.expert_bias",                           # afmoe
+            "model.layers.{bid}.moe.router.expert_bias",                    # kolibri1
             "model.layers.{bid}.feed_forward.expert_bias",                  # lfm2moe
             "model.layers.{bid}.block_sparse_moe.e_score_correction",       # minimax-m2
             "backbone.layers.{bid}.mixer.gate.e_score_correction",          # nemotron-h-moe

scripts/check-release-apiabi.sh

@@ -15,6 +15,7 @@
 # Environment:
 #   SKIP_APIABI_CHECK:    set to 1 or true to skip
 #   APIABI_COMPARE_TAG:   equivalent to --tag (used by CI)
+#   APIABI_REPORT_DIR:    preserve reports and dumps in this directory, even if the comparison fails
 
 set -euo pipefail
 
@@ -113,4 +114,15 @@ cmake --build "$BUILD_NEW" --parallel "$(nproc)"
 
 (cd "$WORKTREE_DIR" && "$SCRIPT_DIR/check-apiabi-compat.sh" --include-path ggml/include --generate "$BUILD_OLD" "${LIBS[@]}")
 (cd "$REPO_ROOT"    && "$SCRIPT_DIR/check-apiabi-compat.sh" --include-path ggml/include --generate "$BUILD_NEW" "${LIBS[@]}")
-(cd "$REPO_ROOT"    && "$SCRIPT_DIR/check-apiabi-compat.sh" "${CHECK_FLAGS[@]}" --check "$BUILD_OLD" "$BUILD_NEW")
+
+CHECK_RESULT=0
+(cd "$REPO_ROOT" && "$SCRIPT_DIR/check-apiabi-compat.sh" "${CHECK_FLAGS[@]}" --check "$BUILD_OLD" "$BUILD_NEW") || CHECK_RESULT=$?
+
+if [[ -n "${APIABI_REPORT_DIR:-}" ]]; then
+    mkdir -p "$APIABI_REPORT_DIR/old" "$APIABI_REPORT_DIR/new"
+    cp -R "$BUILD_OLD/apiabi/." "$APIABI_REPORT_DIR/old/"
+    cp -R "$BUILD_NEW/apiabi/." "$APIABI_REPORT_DIR/new/"
+    echo "API/ABI reports saved to $APIABI_REPORT_DIR"
+fi
+
+exit "$CHECK_RESULT"

skills/slice-model/SKILL.md

@@ -0,0 +1,26 @@
+---
+name: slice-model
+description: Build a small test "slice" of a HF model (fewer layers/experts) by downloading only the needed tensors via HTTP range requests. Use when the user needs a small version of a big model to test conversion or inference.
+---
+
+# Slice a HF model for testing
+
+[slice_model.py](slice_model.py) is a **reference** implementation. Read it first: its docstring and `--help` explain how it works and what it does. Its heuristics only cover common layouts, and new models often have tensor names, shapes or config keys it does not handle. Do not run it blindly. If the model needs special handling, copy it to `tmp/slice_<model>.py` and adapt the copy, or write a new script with the same approach.
+
+## Steps
+
+1. Ask the user for the repo, what the slice must exercise (e.g. a MoE layer, each layer type of a hybrid model, MTP, the vision encoder), the size budget and the output dir.
+2. Inspect the model: run the script with `--inspect`, read `config.json`, and read the HF modeling code if anything is unclear. Check the plan against the pitfalls below.
+3. Build the slice, then read the config patches in the log and check each one.
+4. Verify: load the slice in transformers (no missing/unexpected weight warnings, no NaN logits), then convert it with `convert_hf_to_gguf.py` and run it. For numerical checks, see `examples/model-conversion/`.
+
+## Pitfalls
+
+- Layer counts in nested or unknown config keys, or several towers with the same layer count.
+- Hybrid layer patterns (`layer_types`, `full_attention_interval`, ...): keep at least one layer of each type, and check the patched config still matches the kept layers.
+- Dense leading layers (`first_k_dense_replace`) and MTP/nextn layers numbered after the regular layers.
+- Every per-expert tensor must be sliced to the same expert count: experts, router weight and biases. Tensors with the expert dim not at dim 0 are not contiguous.
+- Routing constraints: top-k, expert groups, shared experts, zero-compute experts.
+- Quantized weights (FP8, MXFP4): slice the scale tensors in the same way as their weight.
+- Sizes derived in modeling code instead of stored in the config: follow the exact formula if you shrink them.
+- Slicing the vocab without the tokenizer breaks conversion.

skills/slice-model/slice_model.py

@@ -0,0 +1,558 @@
+#!/usr/bin/env python3
+"""
+REFERENCE ONLY - build a small test "slice" of a HF safetensors model without downloading the full weights.
+
+The heuristics below (layer/expert detection, config patching) cover common layouts only. New models often use tensor names, shapes or config keys that this script does not know about. Always run --inspect first, check the plan, and adapt the script to the model.
+
+How it works: each safetensors shard starts with an 8-byte header length + JSON header, which gives the exact byte span of every tensor. We fetch only the headers, then use HTTP range requests to fetch the tensors we keep:
+  1. keep only some layers (per numeric "family" in tensor names whose count matches a layer count in config.json), renumbered to 0..n-1
+  2. keep only the first E experts (per-expert tensors are dropped, stacked expert tensors and router weights are row-sliced along dim 0)
+  3. optionally keep only the first V rows of vocab-sized tensors (off by default, see --vocab-size)
+config.json is patched to match. Other small repo files are copied as-is.
+
+Usage:
+    python slice_model.py Qwen/Qwen3-0.6B --inspect
+    python slice_model.py Qwen/Qwen3-30B-A3B --num-layers 2 --num-experts 8 --dry-run
+    python slice_model.py Qwen/Qwen3-30B-A3B --keep-layers 0,3 --output ../models/qwen3moe-slice
+"""
+from __future__ import annotations
+
+import argparse
+import concurrent.futures as cf
+import fnmatch
+import json
+import logging
+import os
+import re
+import struct
+import sys
+import threading
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import requests
+from huggingface_hub import HfApi, get_token, hf_hub_url
+
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+logger = logging.getLogger("slice_model")
+
+INDEX_FILE = "model.safetensors.index.json"
+CONFIG_FILE = "config.json"
+
+# files not worth copying into a slice
+AUX_BLACKLIST = [
+    # weights in other formats (the slice replaces them)
+    "*.bin", "*.pth", "*.pt", "*.ckpt", "*.h5", "*.msgpack",
+    "*.onnx", "*.tflite", "*.mlmodel", "*.mlpackage/*", "*.gguf", "*.nemo",
+    "onnx/*", "openvino/*", "coreml/*", "original/*", "metal/*",
+    # media / demo assets
+    "*.png", "*.jpg", "*.jpeg", "*.gif", "*.webp", "*.avif", "*.svg",
+    "*.mp4", "*.webm", "*.mov", "*.wav", "*.mp3", "*.flac", "*.ogg", "*.pdf",
+    ".gitattributes",
+]
+AUX_MAX_SIZE = 50 * 1024 * 1024
+
+LAYER_CONTAINERS = {"layers", "layer", "blocks", "block", "h"}
+EXPERT_ROW_KEYWORDS = ("expert", "router", "gate", "score")
+
+LAYER_KEYS = {
+    "num_hidden_layers", "num_layers", "n_layer", "n_layers", "num_layer",
+    "num_decoder_layers", "encoder_layers", "decoder_layers", "depth",
+    "num_encoder_layers", "num_attention_layers", "layer_count",
+}
+VOCAB_KEYS = {"vocab_size", "padded_vocab_size"}
+NEXTN_KEYS = {"num_nextn_predict_layers", "num_mtp_layers"}
+TOPK_SUBSTR = ("per_tok", "topk", "top_k", "moe_k")
+EXPERT_KEY_EXCLUDE = ("per_tok", "topk", "top_k", "shared", "group", "intermediate", "size", "dim", "dtype")
+
+
+def parse_size(text: str) -> int:
+    m = re.fullmatch(r"(\d+(?:\.\d+)?)\s*([kKmMgGtT]?)[bB]?", text.strip())
+    if not m:
+        raise argparse.ArgumentTypeError(f"invalid size: {text!r}")
+    mult = {"": 1, "k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4}[m.group(2).lower()]
+    return int(float(m.group(1)) * mult)
+
+
+def human(n: float) -> str:
+    for unit in ("B", "KB", "MB", "GB"):
+        if n < 1024 or unit == "GB":
+            return f"{n:.1f}{unit}" if unit != "B" else f"{int(n)}B"
+        n /= 1024
+    return f"{n}GB"
+
+
+# --------------------------------------------------------------------------------------
+# HTTP
+# --------------------------------------------------------------------------------------
+
+_tls = threading.local()
+
+
+def _session() -> requests.Session:
+    s = getattr(_tls, "s", None)
+    if s is None:
+        s = requests.Session()
+        _tls.s = s
+    return s
+
+
+class HubClient:
+    def __init__(self, repo_id: str, revision: str, token: str | None):
+        self.repo_id = repo_id
+        self.revision = revision
+        self.headers = {"Authorization": f"Bearer {token}"} if token else {}
+
+    def url(self, filename: str) -> str:
+        return hf_hub_url(self.repo_id, filename, revision=self.revision)
+
+    def _request(self, filename: str, headers: dict, stream: bool = False, retries: int = 5):
+        last: Exception | None = None
+        for attempt in range(1, retries + 1):
+            try:
+                r = _session().get(self.url(filename), headers=headers, stream=stream, timeout=90, allow_redirects=True)
+                if r.status_code in (429, 500, 502, 503, 504):
+                    raise requests.HTTPError(f"HTTP {r.status_code}", response=r)
+                r.raise_for_status()
+                return r
+            except Exception as e:  # noqa: BLE001
+                last = e
+                if attempt == retries or "404" in str(e) or "401" in str(e) or "403" in str(e):
+                    break
+                time.sleep(2.0 * attempt)
+        raise RuntimeError(f"{self.repo_id}/{filename}: {last}") from last
+
+    def get_range(self, filename: str, start: int, end_inclusive: int) -> bytes:
+        h = dict(self.headers)
+        h["Range"] = f"bytes={start}-{end_inclusive}"
+        return self._request(filename, h).content
+
+    def get_json(self, filename: str) -> Any:
+        return json.loads(self._request(filename, dict(self.headers)).content)
+
+    def download_file(self, filename: str, dest: Path):
+        r = self._request(filename, dict(self.headers), stream=True)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        with open(dest, "wb") as f:
+            for chunk in r.iter_content(chunk_size=4 * 1024 * 1024):
+                if chunk:
+                    f.write(chunk)
+
+    def stream_range_into_fd(self, filename: str, start: int, end_inclusive: int, fd: int, write_offset: int):
+        h = dict(self.headers)
+        h["Range"] = f"bytes={start}-{end_inclusive}"
+        r = self._request(filename, h, stream=True)
+        pos = write_offset
+        for chunk in r.iter_content(chunk_size=8 * 1024 * 1024):
+            if chunk:
+                os.pwrite(fd, chunk, pos)
+                pos += len(chunk)
+        if pos - write_offset != end_inclusive - start + 1:
+            raise RuntimeError(f"short read: {filename} bytes={start}-{end_inclusive}")
+
+
+def fetch_shard_header(client: HubClient, filename: str) -> tuple[dict, int]:
+    (header_len,) = struct.unpack("<Q", client.get_range(filename, 0, 7))
+    header = json.loads(client.get_range(filename, 8, 8 + header_len - 1))
+    header.pop("__metadata__", None)
+    return header, 8 + header_len
+
+
+# --------------------------------------------------------------------------------------
+# config.json analysis / patching
+# --------------------------------------------------------------------------------------
+
+def walk_config(cfg: Any):
+    """Yield (container_dict, key, value) for every leaf in nested dicts."""
+    if isinstance(cfg, dict):
+        for k, v in cfg.items():
+            if isinstance(v, dict):
+                yield from walk_config(v)
+            else:
+                yield cfg, k, v
+
+
+@dataclass
+class ConfigInfo:
+    cfg: dict | None
+    layer_counts: set[int] = field(default_factory=set)
+    nextn_counts: set[int] = field(default_factory=set)
+    expert_counts: set[int] = field(default_factory=set)
+    vocab_sizes: set[int] = field(default_factory=set)
+    first_k_dense: int = 0
+
+
+def analyze_config(cfg: dict | None) -> ConfigInfo:
+    info = ConfigInfo(cfg=cfg)
+    if cfg is None:
+        return info
+    for _d, key, val in walk_config(cfg):
+        if not isinstance(val, int) or isinstance(val, bool) or val <= 0:
+            continue
+        if key in LAYER_KEYS:
+            info.layer_counts.add(val)
+        elif key in NEXTN_KEYS:
+            info.nextn_counts.add(val)
+        elif key in VOCAB_KEYS:
+            info.vocab_sizes.add(val)
+        elif key == "first_k_dense_replace":
+            info.first_k_dense = val
+        elif "expert" in key and val > 1 and not any(x in key for x in EXPERT_KEY_EXCLUDE):
+            info.expert_counts.add(val)
+    return info
+
+
+def patch_config(cfg: dict, families: dict[str, Family], new_vocab: int | None, orig_vocabs: set[int]) -> list[str]:
+    """Patch counts in-place to match the slice. Returns log lines."""
+    log = []
+    by_card: dict[int, Family] = {}
+    for fam in families.values():
+        if fam.kept is not None:
+            by_card.setdefault(fam.card, fam)
+            by_card.setdefault(fam.regular_total, fam)
+    max_new_experts = max((f.new_count for f in families.values() if f.kind == "expert" and f.kept is not None), default=None)
+
+    for d, key, val in list(walk_config(cfg)):
+        if isinstance(val, int) and not isinstance(val, bool):
+            fam = by_card.get(val)
+            if key in LAYER_KEYS and fam is not None and fam.kind == "layer":
+                d[key] = fam.new_regular
+            elif key == "first_k_dense_replace":
+                lfam = next((f for f in families.values() if f.kind == "layer"), None)
+                if lfam is None:
+                    continue
+                d[key] = sum(1 for i in lfam.kept if i < val)
+            elif "expert" in key and fam is not None and fam.kind == "expert" and not any(x in key for x in EXPERT_KEY_EXCLUDE):
+                d[key] = fam.new_count
+            elif "expert" in key and any(x in key for x in TOPK_SUBSTR) and max_new_experts is not None and val > max_new_experts:
+                d[key] = max_new_experts
+            elif key in VOCAB_KEYS and new_vocab is not None and val in orig_vocabs:
+                d[key] = new_vocab
+            if d[key] != val:
+                log.append(f"{key}: {val} -> {d[key]}")
+        elif isinstance(val, list) and val and all(not isinstance(x, (dict, list)) for x in val):
+            # per-layer lists, e.g. layer_types
+            fam = by_card.get(len(val))
+            if fam is not None and fam.kind == "layer" and len(val) in (fam.card, fam.regular_total):
+                d[key] = [val[i] for i in fam.kept if i < len(val)]
+                log.append(f"{key}: list[{len(val)}] -> list[{len(d[key])}]")
+    return log
+
+
+# --------------------------------------------------------------------------------------
+# tensor name analysis
+# --------------------------------------------------------------------------------------
+
+@dataclass
+class Family:
+    """One numeric slot in tensor names, e.g. 'model.layers.#'."""
+    prefix: str
+    kind: str            # "layer" | "expert" | "other"
+    indices: set[int] = field(default_factory=set)
+    kept: list[int] | None = None    # None = keep all, no renumbering
+    renumber: dict[int, int] = field(default_factory=dict)
+    regular_total: int = 0           # layer count excluding MTP tail
+    new_regular: int = 0
+
+    @property
+    def card(self) -> int:
+        return len(self.indices)
+
+    @property
+    def new_count(self) -> int:
+        return len(self.kept) if self.kept is not None else self.card
+
+
+def name_slots(name: str):
+    """Yield (slot_pos, family_prefix, index) for each numeric path component."""
+    tokens = name.split(".")
+    norm: list[str] = []
+    for i, tok in enumerate(tokens):
+        if tok.isdigit():
+            yield i, ".".join(norm), int(tok)
+            norm.append("#")
+        else:
+            norm.append(tok)
+
+
+def build_families(names: list[str], cfg_info: ConfigInfo) -> dict[str, Family]:
+    families: dict[str, Family] = {}
+    for name in names:
+        tokens = name.split(".")
+        for i, prefix, idx in name_slots(name):
+            container = tokens[i - 1] if i > 0 else ""
+            fam = families.get(prefix)
+            if fam is None:
+                fam = Family(prefix=prefix, kind="expert" if "expert" in container else "other")
+                families[prefix] = fam
+            fam.indices.add(idx)
+
+    acceptable_layer_cards: dict[int, int] = {}  # card -> regular layer count
+    for c in cfg_info.layer_counts:
+        acceptable_layer_cards[c] = c
+        for m in cfg_info.nextn_counts:
+            acceptable_layer_cards[c + m] = c
+
+    for fam in families.values():
+        if fam.kind == "expert":
+            continue
+        container = fam.prefix.rsplit(".", 1)[-1]
+        if cfg_info.cfg is not None:
+            if fam.card in acceptable_layer_cards:
+                fam.kind = "layer"
+                fam.regular_total = acceptable_layer_cards[fam.card]
+        elif fam.card >= 4 and container in LAYER_CONTAINERS:
+            fam.kind = "layer"
+            fam.regular_total = fam.card
+    return families
+
+
+def plan_families(families: dict[str, Family], num_layers: int, keep_layers: list[int] | None, num_experts: int, first_k_dense: int):
+    for fam in families.values():
+        if fam.kind == "layer":
+            if keep_layers is not None:
+                kept = [i for i in keep_layers if i < fam.regular_total]
+            else:
+                base = min(num_layers, fam.regular_total)
+                kept = list(range(base))
+                # keep one MoE layer for models whose first k layers are dense
+                if 0 < first_k_dense < fam.regular_total and first_k_dense >= base >= 1:
+                    kept = list(range(base - 1)) + [first_k_dense]
+            fam.new_regular = len(kept)
+            # keep the MTP tail (layer indices beyond the regular count)
+            kept += [i for i in sorted(fam.indices) if i >= fam.regular_total]
+            fam.kept = sorted(set(kept))
+        elif fam.kind == "expert":
+            fam.kept = sorted(fam.indices)[:num_experts]
+            fam.new_regular = len(fam.kept)
+        else:
+            fam.kept = None
+            continue
+        fam.renumber = {old: new for new, old in enumerate(fam.kept)}
+
+
+def slice_tensor_name(name: str, families: dict[str, Family]) -> str | None:
+    """Return the renumbered output name, or None if the tensor is dropped."""
+    tokens = name.split(".")
+    for i, prefix, idx in name_slots(name):
+        fam = families[prefix]
+        if fam.kept is None:
+            continue
+        if idx not in fam.renumber:
+            return None
+        tokens[i] = str(fam.renumber[idx])
+    return ".".join(tokens)
+
+
+# --------------------------------------------------------------------------------------
+# slicing
+# --------------------------------------------------------------------------------------
+
+@dataclass
+class TensorPlan:
+    src_name: str
+    out_name: str
+    dtype: str
+    shape: list[int]
+    orig_shape: list[int]
+    size: int
+    src_file: str
+    seg_start: int       # absolute byte range in src_file
+    seg_end: int         # inclusive
+
+
+def pick_weight_files(filenames: list[str]) -> tuple[list[str], bool]:
+    """Return (weight files, has_index). Preference: index -> model.safetensors -> any root safetensors."""
+    if INDEX_FILE in filenames:
+        return [], True
+    if "model.safetensors" in filenames:
+        return ["model.safetensors"], False
+    root_st = [f for f in filenames if f.endswith(".safetensors") and "/" not in f]
+    if "consolidated.safetensors" in root_st:
+        return ["consolidated.safetensors"], False
+    return root_st[:1], False
+
+
+def row_slice(shape: list[int], size: int, keep_rows: int) -> tuple[list[int], int] | None:
+    # only a prefix of dim 0 is a single contiguous byte range
+    if len(shape) < 1 or shape[0] <= keep_rows or size % shape[0] != 0:
+        return None
+    row_bytes = size // shape[0]
+    return [keep_rows] + shape[1:], keep_rows * row_bytes
+
+
+def build_plan(args, families, headers, weight_files, src_file_of, cfg_info) -> list[TensorPlan]:
+    plan_families(families, args.num_layers, args.keep_layers, args.num_experts, cfg_info.first_k_dense)
+    plans = []
+    for fname in weight_files:
+        header, data_start = headers[fname]
+        for name, meta in header.items():
+            out_name = slice_tensor_name(name, families)
+            if out_name is None:
+                continue
+            start, end = meta["data_offsets"]
+            shape, size = list(meta["shape"]), end - start
+            new_rows = None
+            if args.vocab_size and shape and shape[0] > args.vocab_size \
+                    and any(shape[0] >= vs and shape[0] - vs <= 1024 for vs in cfg_info.vocab_sizes):
+                new_rows = args.vocab_size
+            elif shape and shape[0] in cfg_info.expert_counts and shape[0] > args.num_experts \
+                    and any(k in name for k in EXPERT_ROW_KEYWORDS):
+                new_rows = args.num_experts
+            if new_rows is not None:
+                sliced = row_slice(shape, size, new_rows)
+                if sliced is not None:
+                    shape, size = sliced
+            plans.append(TensorPlan(name, out_name, meta["dtype"], shape, list(meta["shape"]), size, src_file_of[name],
+                                    data_start + start, data_start + start + size - 1))
+    return plans
+
+
+def print_inspect(cfg_info: ConfigInfo, families: dict[str, Family], plans: list[TensorPlan], all_names: list[str]):
+    logger.info("=== config analysis ===")
+    logger.info(f"layer counts: {sorted(cfg_info.layer_counts)}  nextn: {sorted(cfg_info.nextn_counts)}  "
+                f"experts: {sorted(cfg_info.expert_counts)}  vocab: {sorted(cfg_info.vocab_sizes)}  first_k_dense: {cfg_info.first_k_dense}")
+    logger.info("\n=== numeric families in tensor names ===")
+    for fam in sorted(families.values(), key=lambda f: f.prefix):
+        kept = "all" if fam.kept is None else f"{fam.kept[:16]}{'...' if len(fam.kept) > 16 else ''}"
+        logger.info(f"{fam.prefix:60s} kind={fam.kind:6s} count={fam.card:4d} kept={kept}")
+    logger.info("\n=== kept tensors (first index of each family only) ===")
+    seen = set()
+    for t in plans:
+        key = re.sub(r"\.\d+\.", ".#.", t.out_name)
+        if key in seen and t.shape == t.orig_shape:
+            continue
+        seen.add(key)
+        mark = f"  (sliced from {t.orig_shape})" if t.shape != t.orig_shape else ""
+        logger.info(f"{t.out_name:70s} {t.dtype:5s} {t.shape}{mark}")
+    kept_src = {t.src_name for t in plans}
+    dropped = [n for n in all_names if n not in kept_src]
+    logger.info(f"\n{len(plans)}/{len(all_names)} tensors kept, {len(dropped)} dropped")
+
+
+def write_slice(out_dir: Path, client: HubClient, plans: list[TensorPlan], jobs: int):
+    header_out: dict[str, Any] = {}
+    offset = 0
+    for t in plans:
+        header_out[t.out_name] = {"dtype": t.dtype, "shape": t.shape, "data_offsets": [offset, offset + t.size]}
+        offset += t.size
+    header_out["__metadata__"] = {"format": "pt"}
+    hbytes = json.dumps(header_out, separators=(",", ":")).encode()
+    hbytes += b" " * ((-(len(hbytes) + 8)) % 8)
+    data_start = 8 + len(hbytes)
+
+    shard_path = out_dir / "model.safetensors"
+    with open(shard_path, "wb") as f:
+        f.write(struct.pack("<Q", len(hbytes)))
+        f.write(hbytes)
+        f.truncate(data_start + offset)
+    fd = os.open(shard_path, os.O_WRONLY)
+    try:
+        def fetch(t: TensorPlan):
+            dst = data_start + header_out[t.out_name]["data_offsets"][0]
+            client.stream_range_into_fd(t.src_file, t.seg_start, t.seg_end, fd, dst)
+
+        with cf.ThreadPoolExecutor(max_workers=jobs) as ex:
+            list(ex.map(fetch, plans))
+    finally:
+        os.close(fd)
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("repo", help="HF repo id, e.g. Qwen/Qwen3-0.6B")
+    ap.add_argument("--revision", default=None)
+    ap.add_argument("--output", type=Path, default=None, help="output dir (default: ./tmp/hf_slices/{user}--{model})")
+    ap.add_argument("--inspect", action="store_true", help="print config analysis, tensor families and the plan, write nothing")
+    ap.add_argument("--dry-run", action="store_true", help="plan and estimate size only, write nothing")
+    ap.add_argument("--num-layers", type=int, default=2, help="keep the first N layers (default: 2)")
+    ap.add_argument("--keep-layers", default=None, help="explicit comma-separated layer indices to keep, overrides --num-layers")
+    ap.add_argument("--num-experts", type=int, default=8)
+    ap.add_argument("--vocab-size", type=int, default=0,
+                    help="row-slice vocab-sized tensors to this size (default: 0 = off; tokenizer is NOT sliced, so this breaks conversion for most tokenizers)")
+    ap.add_argument("--max-size", type=parse_size, default="1G", help="abort if the sliced weights exceed this (default: 1G)")
+    ap.add_argument("--hf-token", default=None, help="HF token (default: HF_TOKEN env var or 'hf auth login' token)")
+    ap.add_argument("--jobs", type=int, default=8, help="concurrent HTTP requests")
+    args = ap.parse_args()
+    args.keep_layers = sorted({int(x) for x in args.keep_layers.split(",")}) if args.keep_layers else None
+
+    token = args.hf_token or get_token()
+    api = HfApi(token=token)
+    info = api.model_info(args.repo, revision=args.revision, files_metadata=True)
+    client = HubClient(args.repo, info.sha or "main", token)
+    siblings = {s.rfilename: (s.size or 0) for s in info.siblings or []}
+    filenames = sorted(siblings)
+
+    weight_files, has_index = pick_weight_files(filenames)
+    weight_map: dict[str, str] = {}
+    if has_index:
+        weight_map = client.get_json(INDEX_FILE)["weight_map"]
+        weight_files = sorted(set(weight_map.values()))
+    elif not weight_files:
+        logger.error("no .safetensors found")
+        return 1
+
+    cfg = client.get_json(CONFIG_FILE) if CONFIG_FILE in filenames else None
+    cfg_info = analyze_config(cfg)
+
+    with cf.ThreadPoolExecutor(max_workers=args.jobs) as ex:
+        headers = dict(zip(weight_files, ex.map(lambda f: fetch_shard_header(client, f), weight_files)))
+    all_names = [n for fname in weight_files for n in headers[fname][0]]
+    src_file_of = weight_map if has_index else {n: weight_files[0] for n in all_names}
+
+    families = build_families(all_names, cfg_info)
+    plans = build_plan(args, families, headers, weight_files, src_file_of, cfg_info)
+    est = sum(t.size for t in plans)
+
+    if args.inspect:
+        print_inspect(cfg_info, families, plans, all_names)
+    logger.info(f"{args.repo}@{info.sha}: {len(plans)}/{len(all_names)} tensors, {human(est)}")
+    if not any(f.kind == "layer" for f in families.values()):
+        logger.warning("no layer family detected - check --inspect output and adapt the script")
+    if est > args.max_size:
+        logger.error(f"slice is {human(est)} > --max-size {human(args.max_size)}")
+        return 1
+    if args.inspect or args.dry_run:
+        return 0
+
+    out_dir = args.output or Path("tmp") / "hf_slices" / args.repo.replace("/", "--")
+    out_dir.mkdir(parents=True, exist_ok=True)
+    write_slice(out_dir, client, plans, args.jobs)
+
+    skipped_aux = []
+    for fname in filenames:
+        if fname in weight_files or fname in (INDEX_FILE, CONFIG_FILE) or fname.endswith(".safetensors"):
+            continue
+        if any(fnmatch.fnmatch(fname, p) or fnmatch.fnmatch(Path(fname).name, p) for p in AUX_BLACKLIST):
+            continue
+        if siblings[fname] > AUX_MAX_SIZE:
+            skipped_aux.append(f"{fname} ({human(siblings[fname])})")
+            continue
+        client.download_file(fname, out_dir / fname)
+    for f in skipped_aux:
+        logger.warning(f"skipped large aux file {f}")
+
+    patch_log = []
+    if cfg is not None:
+        new_vocab = args.vocab_size if args.vocab_size and any(vs > args.vocab_size for vs in cfg_info.vocab_sizes) else None
+        patch_log = patch_config(cfg, families, new_vocab, cfg_info.vocab_sizes)
+        with open(out_dir / CONFIG_FILE, "w") as f:
+            json.dump(cfg, f, indent=2)
+            f.write("\n")
+    for line in patch_log:
+        logger.info(f"config: {line}")
+
+    with open(out_dir / ".slice_meta.json", "w") as f:
+        json.dump({
+            "repo": args.repo, "revision": info.sha,
+            "tensors": len(plans), "total_tensors": len(all_names), "size": est,
+            "config_patches": patch_log, "skipped_aux": skipped_aux,
+        }, f, indent=2)
+    logger.info(f"wrote {out_dir}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/llama-arch.cpp

@@ -165,6 +165,8 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_QWEN3TTS,         "qwen3tts"         },
     { LLM_ARCH_POCKETTTS,        "pockettts"        },
     { LLM_ARCH_K2_HORIZON,       "k2-horizon"       },
+    { LLM_ARCH_BEAM,             "beam"             },
+    { LLM_ARCH_KOLIBRI1,         "kolibri1"         },
     { LLM_ARCH_UNKNOWN,          "(unknown)"        },
 };
 

src/llama-arch.h

@@ -170,6 +170,8 @@ enum llm_arch {
     LLM_ARCH_MINIMAX_01,
     LLM_ARCH_HRM_TEXT,
     LLM_ARCH_K2_HORIZON,
+    LLM_ARCH_BEAM,
+    LLM_ARCH_KOLIBRI1,
     LLM_ARCH_UNKNOWN,
 };
 

src/llama-graph.cpp

@@ -218,17 +218,31 @@ void llm_graph_input_attn_temp::set_input(const llama_ubatch * ubatch) {
         GGML_ASSERT(n_attn_temp_floor_scale != 0);
 
         std::vector<float> attn_scale_data(n_tokens, 0.0f);
-        for (int i = 0; i < n_tokens; ++i) {
-            const float pos = ubatch->pos[i];
-            attn_scale_data[i] = std::log(
-                std::floor((pos + f_attn_temp_offset) / n_attn_temp_floor_scale) + 1.0
-            ) * f_attn_temp_scale + 1.0;
+        if (attn_temp_pow) {
+            // (1 + (pos + offset) / length) ^ power
+            for (int i = 0; i < n_tokens; ++i) {
+                const double pos = ubatch->pos[i];
+                attn_scale_data[i] = (float) std::exp((double) f_attn_temp_scale *
+                    std::log1p((pos + f_attn_temp_offset) / (double) n_attn_temp_floor_scale));
+            }
+        } else {
+            for (int i = 0; i < n_tokens; ++i) {
+                const float pos = ubatch->pos[i];
+                attn_scale_data[i] = std::log(
+                    std::floor((pos + f_attn_temp_offset) / n_attn_temp_floor_scale) + 1.0
+                ) * f_attn_temp_scale + 1.0;
+            }
         }
 
         ggml_backend_tensor_set(attn_scale, attn_scale_data.data(), 0, n_tokens*ggml_element_size(attn_scale));
     }
 }
 
+bool llm_graph_input_attn_temp::can_reuse(const llm_graph_params & params) {
+    // only the power form opts in; the log form keeps no reuse
+    return attn_temp_pow && attn_scale != nullptr && attn_scale->ne[2] == params.ubatch.n_tokens;
+}
+
 void llm_graph_input_pos_bucket::set_input(const llama_ubatch * ubatch) {
     if (pos_bucket) {
         const int64_t n_tokens = ubatch->n_tokens;
@@ -2743,7 +2757,7 @@ ggml_tensor * llm_graph_context::build_inp_pos() const {
 }
 
 ggml_tensor * llm_graph_context::build_inp_attn_scale() const {
-    auto inp = std::make_unique<llm_graph_input_attn_temp>(hparams.n_attn_temp_floor_scale, hparams.f_attn_temp_scale, hparams.f_attn_temp_offset);
+    auto inp = std::make_unique<llm_graph_input_attn_temp>(hparams.n_attn_temp_floor_scale, hparams.f_attn_temp_scale, hparams.f_attn_temp_offset, hparams.attn_temp_pow);
 
     auto & cur = inp->attn_scale;
 

src/llama-graph.h

@@ -188,17 +188,20 @@ public:
 // temperature tuning, used by llama4
 class llm_graph_input_attn_temp : public llm_graph_input_i {
 public:
-    llm_graph_input_attn_temp(uint32_t n_attn_temp_floor_scale, float f_attn_temp_scale, float f_attn_temp_offset)
-        : n_attn_temp_floor_scale(n_attn_temp_floor_scale), f_attn_temp_scale(f_attn_temp_scale), f_attn_temp_offset(f_attn_temp_offset) {}
+    llm_graph_input_attn_temp(uint32_t n_attn_temp_floor_scale, float f_attn_temp_scale, float f_attn_temp_offset, bool attn_temp_pow = false)
+        : n_attn_temp_floor_scale(n_attn_temp_floor_scale), f_attn_temp_scale(f_attn_temp_scale), f_attn_temp_offset(f_attn_temp_offset), attn_temp_pow(attn_temp_pow) {}
     virtual ~llm_graph_input_attn_temp() = default;
 
     void set_input(const llama_ubatch * ubatch) override;
 
+    bool can_reuse(const llm_graph_params & params) override;
+
     ggml_tensor * attn_scale = nullptr; // F32 [n_batch]
 
     const uint32_t n_attn_temp_floor_scale;
     const float    f_attn_temp_scale;
     const float    f_attn_temp_offset;
+    const bool     attn_temp_pow;
 };
 
 class llm_graph_input_pos_bucket : public llm_graph_input_i {

src/llama-hparams.h

@@ -280,6 +280,8 @@ struct llama_hparams {
     uint32_t n_attn_temp_floor_scale = 0;
     float    f_attn_temp_scale       = 0.0f;
     float    f_attn_temp_offset      = 0.0f; // offset position index
+    // use (1 + (pos + offset) / floor_scale) ^ scale instead of the llama4 log form
+    bool     attn_temp_pow           = false;
 
     // realtime models: number of tokens the output lags behind the audio input
     uint32_t n_tokens_delay          = 0;

src/llama-model.cpp

@@ -322,6 +322,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_minimax_m3(params);
         case LLM_ARCH_HRM_TEXT:
             return new llama_model_hrm_text(params);
+        case LLM_ARCH_BEAM:
+            return new llama_model_beam(params);
         case LLM_ARCH_COGVLM:
             return new llama_model_cogvlm(params);
         case LLM_ARCH_PANGU_EMBED:
@@ -358,6 +360,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_spark2_5(params);
         case LLM_ARCH_K2_HORIZON:
             return new llama_model_k2_horizon(params);
+        case LLM_ARCH_KOLIBRI1:
+            return new llama_model_kolibri1(params);
         default:
             throw std::runtime_error(std::string("unsupported model architecture: '") + llm_arch_name(arch) + "'");
     }
@@ -405,6 +409,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
     static const std::regex pattern_idx_cache       ("cache_idx_(k|v)_l\\d*");
     static const std::regex pattern_dsv4_state      ("dsv4_(csa|hca|lid)_state_(kv|score)_l\\d*");
     static const std::regex pattern_attn_sinks      ("blk\\.\\d*\\.attn_sinks.weight");
+    static const std::regex pattern_attn_gate_bias  ("blk\\.\\d*\\.attn_gate\\.bias");
     static const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight");
     static const std::regex pattern_attn_out_bias   ("blk\\.\\d*\\.attn_output.bias");
     static const std::regex pattern_attn_out_a_weight("blk\\.\\d*\\.attn_output_a\\.weight");
@@ -550,7 +555,8 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
         if (std::regex_match(tensor_name, pattern_qk_norm)) {
             return get_tensor_config_impl(tensor->ne[1] == 1 ? GGML_BACKEND_SPLIT_AXIS_MIRRORED : GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight");
         }
-        if (std::regex_match(tensor_name, pattern_kv_cache) || std::regex_match(tensor_name, pattern_attn_sinks)) {
+        if (std::regex_match(tensor_name, pattern_kv_cache) || std::regex_match(tensor_name, pattern_attn_sinks) ||
+                std::regex_match(tensor_name, pattern_attn_gate_bias)) {
             return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight");
         }
         if (std::regex_match(tensor_name, pattern_attn_out_weight)) {
@@ -749,6 +755,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
 
             const int64_t granularity_q    = std::lcm(n_embd_q, blck_size_perf);
             const int64_t granularity_head = granularity_q / hparams.n_embd_head_k(il); // for tensors with one value per head
+            if (std::regex_match(tensor_name, pattern_attn_gate_bias)) {
+                GGML_ASSERT(segments.size() == 1);
+                return {granularity_head};
+            }
             if (std::regex_match(tensor_name, pattern_attn_sinks)) {
                 GGML_ASSERT(segments.size() == 1);
                 if (is_dsv4) {
@@ -994,6 +1004,7 @@ const char * llm_type_name(llm_type type) {
         case LLM_TYPE_10B_128x3_66B: return "10B+128x3.66B";
         case LLM_TYPE_57B_A14B:      return "57B.A14B";
         case LLM_TYPE_816B_A42B:     return "816B.A42B";
+        case LLM_TYPE_78B_A3_5B:     return "78B.A3.5B";
         case LLM_TYPE_17B_16E:       return "17Bx16E (Scout)";
         case LLM_TYPE_17B_128E:      return "17Bx128E (Maverick)";
         case LLM_TYPE_A13B:          return "A13B";
@@ -3606,6 +3617,8 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_MELLUM:
         case LLM_ARCH_MAPLE:
         case LLM_ARCH_HRM_TEXT:
+        case LLM_ARCH_BEAM:
+        case LLM_ARCH_KOLIBRI1:
             return LLAMA_ROPE_TYPE_NEOX;
 
         case LLM_ARCH_DFLASH:

src/llama-model.h

@@ -115,6 +115,7 @@ enum llm_type {
     LLM_TYPE_10B_128x3_66B,
     LLM_TYPE_57B_A14B,
     LLM_TYPE_816B_A42B,
+    LLM_TYPE_78B_A3_5B,  // Kolibri1
     LLM_TYPE_17B_16E, // llama4 Scout
     LLM_TYPE_17B_128E, // llama4 Maverick
     LLM_TYPE_A13B,
@@ -310,6 +311,7 @@ struct llama_layer {
     struct ggml_tensor * wv_enc    = nullptr;
     struct ggml_tensor * wo_enc    = nullptr;
     struct ggml_tensor * wqkv_gate = nullptr;
+    struct ggml_tensor * wqkv_gate_b = nullptr;
     // K2 Horizon MoVA
     struct ggml_tensor * attn_v_gate   = nullptr;
     struct ggml_tensor * attn_v_gate_b = nullptr;

src/llama-vocab.cpp

@@ -3142,9 +3142,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
             }
         }
 
-        // gemma4 and plamo have a normal </s> token, unlike paddleocr
+        // gemma4, plamo and beam have a normal </s> token, unlike paddleocr
         {
-            bool has_normal_s_marker = false;
+            // beam's tool call marker is not an EOG token, so it is looked up in the whole vocab
+            bool has_normal_s_marker = token_to_id.count("<rfl:parameter name=\"") > 0;
             bool has_s = false;
 
             llama_token s_id = LLAMA_TOKEN_NULL;

src/models/beam.cpp

@@ -0,0 +1,261 @@
+#include "models.h"
+
+void llama_model_beam::load_arch_hparams(llama_model_loader & ml) {
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,       hparams.f_norm_rms_eps);
+    ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,         hparams.n_layer_dense_lead);
+    ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
+    ml.get_key(LLM_KV_EXPERT_GATING_FUNC,                hparams.expert_gating_func, false);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,              hparams.expert_weights_scale);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,               hparams.expert_weights_norm);
+    hparams.f_final_logit_softcapping = 0.0f; // absent key = no softcap (default is 30)
+    ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING,           hparams.f_final_logit_softcapping, false);
+
+    if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {
+        hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;
+    }
+    if (hparams.n_layer_dense_lead > hparams.n_layer()) {
+        throw std::runtime_error("beam: leading_dense_block_count exceeds the layer count");
+    }
+
+    ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; // masked iff p1 - p0 >= n_swa
+    uint32_t n_pattern = 0;
+    ml.get_arr_n(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern);
+    if (hparams.n_swa == 0 || n_pattern != hparams.n_layer_all) {
+        throw std::runtime_error("beam: requires a positive sliding window and a sliding window pattern for each layer");
+    }
+    ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+    if (!hparams.is_swa_any()) {
+        throw std::runtime_error("beam: sliding window pattern has no sliding window layer");
+    }
+
+    // full layers are NoPE
+    hparams.rope_pattern = hparams.is_swa_impl;
+    hparams.rope_freq_base_train_swa  = hparams.rope_freq_base_train;
+    hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
+    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
+
+    // NoPE query scale uses pos + 1
+    hparams.f_attn_temp_scale = 0.1f;
+    hparams.attn_temp_length  = 2048;
+    ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_SCALE,  hparams.f_attn_temp_scale, false);
+    ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.attn_temp_length,  false);
+    if (hparams.f_attn_temp_scale == 0.0f) {
+        throw std::runtime_error("beam: attention.temperature_scale must be nonzero");
+    }
+    if (hparams.attn_temp_length == 0) {
+        throw std::runtime_error("beam: attention.temperature_length must be positive");
+    }
+    hparams.n_attn_temp_floor_scale = hparams.attn_temp_length;
+    hparams.f_attn_temp_offset      = 1.0f;
+    hparams.attn_temp_pow           = true;
+
+    switch (hparams.n_layer()) {
+        case 52: type = LLM_TYPE_UNKNOWN; break;
+        default: type = LLM_TYPE_UNKNOWN;
+    }
+}
+
+void llama_model_beam::load_arch_tensors(llama_model_loader &) {
+    LLAMA_LOAD_LOCALS;
+
+    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+    if (output == NULL) {
+        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+    }
+
+    for (int i = 0; i < n_layer; ++i) {
+        auto & layer = layers[i];
+
+        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM,      "weight", i), {n_embd}, 0);
+        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0);
+
+        create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
+        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
+
+        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
+        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
+
+        layer.wqkv_gate   = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
+        layer.wqkv_gate_b = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "bias",   i), {n_head}, 0);
+
+        layer.ffn_norm      = create_tensor(tn(LLM_TENSOR_FFN_NORM,      "weight", i), {n_embd}, 0);
+        layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0);
+
+        if ((uint32_t) i < hparams.n_layer_dense_lead) {
+            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
+            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
+            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd, n_ff}, 0);
+        } else {
+            const int64_t n_ff_exp = hparams.n_ff_exp(i);
+
+            layer.ffn_gate_inp    = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,    "weight", i), {n_embd, n_expert}, 0);
+            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias",   i), {n_expert}, 0);
+
+            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
+            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
+            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
+        }
+    }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_beam::build_arch_graph(const llm_graph_params & params) const {
+    return std::make_unique<graph>(*this, params);
+}
+
+static ggml_tensor * beam_residual_add(ggml_context * ctx, ggml_tensor * h, ggml_tensor * r, int i) {
+    const float h_scale = (float) (1.0 / std::sqrt((double) i));
+    const float r_scale = (float) std::sqrt((double) (i - 1) / (double) i);
+    return ggml_add(ctx, ggml_scale(ctx, h, h_scale), ggml_scale(ctx, r, r_scale));
+}
+
+llama_model_beam::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
+    const int64_t n_embd_head = hparams.n_embd_head_v();
+    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+
+    ggml_tensor * cur;
+    ggml_tensor * inpL;
+
+    inpL = build_inp_embd(model.tok_embd);
+
+    ggml_tensor * inp_pos = build_inp_pos();
+
+    bool has_nope = false;
+    for (int il = 0; il < n_layer; ++il) {
+        has_nope = has_nope || !hparams.is_swa(il);
+    }
+    ggml_tensor * inp_attn_scale = has_nope ? build_inp_attn_scale() : nullptr;
+
+    auto * inp_attn = build_attn_inp_kv_iswa();
+
+    ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+    const float kq_scale = 1.0f / sqrtf(float(n_embd_head));
+
+    for (int il = 0; il < n_layer; ++il) {
+        const auto & layer = model.layers[il];
+
+        const bool is_swa = hparams.is_swa(il);
+
+        ggml_tensor * inpSA = inpL;
+
+        cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "attn_norm", il);
+
+        {
+            ggml_tensor * attn_inp = cur;
+
+            auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il);
+
+            Qcur = build_norm(Qcur, layer.attn_q_norm, NULL, LLM_NORM_RMS, il);
+            Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il);
+            cb(Qcur, "Qcur_normed", il);
+            cb(Kcur, "Kcur_normed", il);
+
+            if (is_swa) {
+                const float freq_base_l  = model.get_rope_freq_base (cparams, il);
+                const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
+
+                Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,
+                        n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+                        ext_factor, attn_factor, beta_fast, beta_slow);
+                Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
+                        n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+                        ext_factor, attn_factor, beta_fast, beta_slow);
+                cb(Qcur, "Qcur_rope", il);
+                cb(Kcur, "Kcur_rope", il);
+            } else {
+                Qcur = ggml_mul(ctx0, Qcur, inp_attn_scale);
+                cb(Qcur, "Qcur_scaled", il);
+            }
+
+            cur = build_attn(inp_attn,
+                    NULL, NULL, NULL, // o_proj is applied after the gate
+                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
+            cb(cur, "attn_out", il);
+
+            ggml_tensor * gate = build_lora_mm(layer.wqkv_gate, attn_inp);
+            cb(gate, "attn_gate_proj", il);
+            gate = ggml_reshape_3d(ctx0, gate, n_embd_head, n_head, n_tokens);
+            gate = ggml_add(ctx0, gate, ggml_reshape_3d(ctx0, layer.wqkv_gate_b, 1, n_head, 1));
+            gate = ggml_sigmoid(ctx0, gate);
+            gate = ggml_reshape_2d(ctx0, gate, n_embd_head * n_head, n_tokens);
+            cb(gate, "attn_gate_sig", il);
+
+            cur = ggml_mul(ctx0, cur, gate);
+            cb(cur, "attn_gated", il);
+
+            cur = build_lora_mm(layer.wo, cur, layer.wo_s);
+            cb(cur, "attn_o_proj", il);
+        }
+
+        cur = build_norm(cur, layer.attn_post_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "attn_post_norm", il);
+
+        if (il == n_layer - 1 && inp_out_ids) {
+            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);
+            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+        }
+
+        const int i_attn = 2 * (il + 1);
+        ggml_tensor * ffn_inp = beam_residual_add(ctx0, cur, inpSA, i_attn);
+        cb(ffn_inp, "ffn_inp", il);
+
+        cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "ffn_norm", il);
+
+        if ((uint32_t) il < hparams.n_layer_dense_lead) {
+            cur = build_ffn(cur,
+                    layer.ffn_up,   NULL, NULL,
+                    layer.ffn_gate, NULL, NULL,
+                    layer.ffn_down, NULL, NULL,
+                    NULL,
+                    LLM_FFN_SILU, LLM_FFN_PAR, il);
+            cb(cur, "ffn_out", il);
+        } else {
+            cur = build_moe_ffn(cur,
+                    layer.ffn_gate_inp,
+                    layer.ffn_up_exps,
+                    layer.ffn_gate_exps,
+                    layer.ffn_down_exps,
+                    layer.ffn_exp_probs_b,
+                    n_expert, n_expert_used,
+                    LLM_FFN_SILU,
+                    hparams.expert_weights_norm,
+                    hparams.expert_weights_scale,
+                    (llama_expert_gating_func_type) hparams.expert_gating_func,
+                    il);
+            cb(cur, "ffn_moe_out", il);
+        }
+
+        cur = build_norm(cur, layer.ffn_post_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "ffn_post_norm", il);
+
+        cur = beam_residual_add(ctx0, cur, ffn_inp, i_attn + 1);
+
+        cur = build_cvec(cur, il);
+        cb(cur, "l_out", il);
+
+        inpL = cur;
+    }
+
+    cur = inpL;
+
+    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
+    cb(cur, "result_norm", -1);
+    res->t_embd = cur;
+
+    cur = build_lora_mm(model.output, cur, model.output_s);
+
+    if (hparams.f_final_logit_softcapping != 0.0f) {
+        cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping);
+        cur = ggml_tanh(ctx0, cur);
+        cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping);
+    }
+
+    cb(cur, "result_output", -1);
+    res->t_logits = cur;
+}

src/models/kolibri1.cpp

@@ -0,0 +1,187 @@
+#include "models.h"
+
+void llama_model_kolibri1::load_arch_hparams(llama_model_loader & ml) {
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,       hparams.f_norm_rms_eps);
+    ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
+    ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,               hparams.n_expert_shared);
+    ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,               hparams.expert_weights_norm);
+    ml.get_key(LLM_KV_EXPERT_GATING_FUNC,                hparams.expert_gating_func);
+    ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,          hparams.n_swa);
+
+    if (hparams.n_swa == 0 || hparams.n_expert_shared != 1 || hparams.n_ff_shexp == 0) {
+        throw std::runtime_error("Kolibri-1: invalid sliding window or shared expert configuration");
+    }
+
+    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
+    ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+
+    hparams.rope_freq_base_train_swa  = hparams.rope_freq_base_train;
+    hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
+
+    switch (hparams.n_layer()) {
+        case 50: type = LLM_TYPE_78B_A3_5B; break;
+        default: type = LLM_TYPE_UNKNOWN;
+    }
+}
+
+void llama_model_kolibri1::load_arch_tensors(llama_model_loader &) {
+    LLAMA_LOAD_LOCALS;
+
+    const int64_t n_ff_exp   = hparams.n_ff_exp();
+    const int64_t n_ff_shexp = hparams.n_ff_shexp;
+
+    tok_embd    = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD,  "weight"), {n_embd, n_vocab}, 0);
+    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, 0);
+
+    for (int i = 0; i < n_layer; ++i) {
+        auto & layer = layers[i];
+
+        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM,      "weight", i), {n_embd}, 0);
+        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0);
+
+        create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
+        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
+
+        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
+        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
+
+        layer.ffn_norm      = create_tensor(tn(LLM_TENSOR_FFN_NORM,      "weight", i), {n_embd}, 0);
+        layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0);
+
+        // Routing is sigmoid, the selection bias is added to the logits before top-k.
+        layer.ffn_gate_inp    = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,    "weight", i), {n_embd, n_expert}, 0);
+        layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias",   i), {n_expert}, 0);
+
+        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
+        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
+        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
+
+        layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0);
+        layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   "weight", i), {n_embd, n_ff_shexp}, 0);
+        layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, 0);
+    }
+}
+
+llama_model_kolibri1::graph::graph(const llama_model & model, const llm_graph_params & params)
+    : llm_graph_context(params) {
+    const int64_t n_embd_head = hparams.n_embd_head_v();
+    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+
+    ggml_tensor * cur;
+    ggml_tensor * inpL = build_inp_embd(model.tok_embd);
+    ggml_tensor * inp_pos = build_inp_pos();
+
+    auto * inp_attn = build_attn_inp_kv_iswa();
+
+    ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+    const float kq_scale = 1.0f / sqrtf(float(n_embd_head));
+
+    for (int il = 0; il < n_layer; ++il) {
+        const auto & layer = model.layers[il];
+
+        ggml_tensor * residual = inpL;
+
+        cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "attn_norm", il);
+
+        // QK-norm, then RoPE on the sliding layers only: the global layers are NoPE.
+        auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il);
+
+        Qcur = build_norm(Qcur, layer.attn_q_norm, NULL, LLM_NORM_RMS, il);
+        Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il);
+        cb(Qcur, "Qcur_normed", il);
+        cb(Kcur, "Kcur_normed", il);
+
+        if (hparams.is_swa(il)) {
+            const float freq_base_l  = model.get_rope_freq_base (cparams, il);
+            const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
+
+            Qcur = ggml_rope_ext(
+                    ctx0, Qcur, inp_pos, nullptr,
+                    n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+                    ext_factor, attn_factor, beta_fast, beta_slow);
+            cb(Qcur, "Qcur_rope", il);
+
+            Kcur = ggml_rope_ext(
+                    ctx0, Kcur, inp_pos, nullptr,
+                    n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+                    ext_factor, attn_factor, beta_fast, beta_slow);
+            cb(Kcur, "Kcur_rope", il);
+        }
+
+        cur = build_attn(inp_attn,
+                layer.wo, NULL, layer.wo_s,
+                Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
+        cb(cur, "attn_out", il);
+
+        cur = build_norm(cur, layer.attn_post_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "attn_post_norm", il);
+
+        if (il == n_layer - 1 && inp_out_ids) {
+            cur      = ggml_get_rows(ctx0, cur,      inp_out_ids);
+            residual = ggml_get_rows(ctx0, residual, inp_out_ids);
+        }
+
+        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, residual);
+        cb(ffn_inp, "ffn_inp", il);
+
+        cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "ffn_norm", il);
+
+        // The router runs in f32: the selection adds the bias to the logits, the weights
+        // are the sigmoid of the unbiased logits, so the same logits feed both.
+        ggml_tensor * logits = build_lora_mm(layer.ffn_gate_inp, cur);
+        ggml_prec_set_acc(logits, GGML_PREC_F32);
+        ggml_prec_set_src(logits, GGML_PREC_F32, 1);
+        cb(logits, "ffn_moe_logits", il);
+
+        ggml_tensor * selection = ggml_add(ctx0, logits, layer.ffn_exp_probs_b);
+        cb(selection, "ffn_moe_logits_biased", il);
+
+        ggml_tensor * selected = ggml_argsort_top_k(ctx0, selection, n_expert_used);
+        cb(selected, "ffn_moe_topk", il);
+
+        ggml_tensor * moe_out = build_moe_ffn(cur,
+                layer.ffn_gate_inp,
+                layer.ffn_up_exps,
+                layer.ffn_gate_exps,
+                layer.ffn_down_exps,
+                nullptr,
+                n_expert, n_expert_used,
+                LLM_FFN_SILU, hparams.expert_weights_norm, 1.0f,
+                (llama_expert_gating_func_type) hparams.expert_gating_func,
+                il,
+                logits,
+                nullptr, nullptr, nullptr, nullptr, selected);
+        cb(moe_out, "ffn_moe_out", il);
+
+        ggml_tensor * shared = build_ffn(cur,
+                layer.ffn_up_shexp,   nullptr, nullptr,
+                layer.ffn_gate_shexp, nullptr, nullptr,
+                layer.ffn_down_shexp, nullptr, nullptr, nullptr,
+                LLM_FFN_SILU, LLM_FFN_PAR, il);
+        cb(shared, "ffn_shexp_out", il);
+
+        cur = ggml_add(ctx0, moe_out, shared);
+        cur = build_norm(cur, layer.ffn_post_norm, NULL, LLM_NORM_RMS, il);
+        cur = ggml_add(ctx0, cur, ffn_inp);
+
+        inpL = build_cvec(cur, il);
+        cb(inpL, "l_out", il);
+    }
+
+    cur = build_norm(inpL, model.output_norm, NULL, LLM_NORM_RMS, -1);
+    cb(cur, "result_norm", -1);
+    res->t_embd = cur;
+
+    cur = build_lora_mm(model.output, cur, model.output_s);
+    cb(cur, "result_output", -1);
+    res->t_logits = cur;
+}
+
+std::unique_ptr<llm_graph_context> llama_model_kolibri1::build_arch_graph(const llm_graph_params & params) const {
+    return std::make_unique<graph>(*this, params);
+}

src/models/models.h

@@ -2041,6 +2041,19 @@ struct llama_model_afmoe : public llama_model_base {
 };
 
 
+struct llama_model_beam : public llama_model_base {
+    llama_model_beam(const struct llama_model_params & params) : llama_model_base(params) {}
+    void load_arch_hparams(llama_model_loader & ml) override;
+    void load_arch_tensors(llama_model_loader & ml) override;
+
+    struct graph : public llm_graph_context {
+        graph(const llama_model & model, const llm_graph_params & params);
+    };
+
+    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
 struct llama_model_laguna : public llama_model_base {
     llama_model_laguna(const struct llama_model_params & params) : llama_model_base(params) {}
     void load_arch_hparams(llama_model_loader & ml) override;
@@ -2961,3 +2974,16 @@ struct llama_model_k2_horizon : public llama_model_base {
 
     std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
 };
+
+
+struct llama_model_kolibri1 : public llama_model_base {
+    llama_model_kolibri1(const struct llama_model_params & params) : llama_model_base(params) {}
+    void load_arch_hparams(llama_model_loader & ml) override;
+    void load_arch_tensors(llama_model_loader & ml) override;
+
+    struct graph : public llm_graph_context {
+        graph(const llama_model & model, const llm_graph_params & params);
+    };
+
+    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};

tests/fusion/MTL.csv

@@ -139,6 +139,9 @@ kimi-linear         ,0   ,any     ,GATED_DELTA_NET+CPY         ,      1
 kimi-linear         ,0   ,any     ,MUL+ADD                     ,      2
 kimi-linear         ,0   ,any     ,RMS_NORM+MUL                ,      7
 kimi-linear         ,0   ,any     ,RMS_NORM+SCALE              ,      2
+kolibri1            ,0   ,any     ,MUL+ADD                     ,      4
+kolibri1            ,0   ,any     ,RMS_NORM+MUL                ,     10
+kolibri1            ,0   ,any     ,RMS_NORM+MUL+ADD            ,      3
 laguna              ,0   ,any     ,ADD+ADD                     ,      1
 laguna              ,0   ,any     ,MUL+ADD                     ,      2
 laguna              ,0   ,any     ,RMS_NORM+MUL                ,      9

tests/test-backend-ops.cpp

@@ -7328,6 +7328,7 @@ struct test_topk_moe : public test_case {
     const bool bias_probs;
     const MoeGatingFunc gating_func;
     const float scale_w;
+    const int n_expert_groups;
     ggml_tensor * weights {};
     ggml_tensor * selected_experts {};
 
@@ -7336,17 +7337,21 @@ struct test_topk_moe : public test_case {
                   bool                   with_norm       = false,
                   bool                   bias_probs      = false,
                   MoeGatingFunc          gating_func     = GATING_FUNC_SOFTMAX,
-                  float                  scale_w         = 0.0f) :
+                  float                  scale_w         = 0.0f,
+                  int                    n_expert_groups = 1) :
         ne(ne),
         n_expert_used(n_expert_used),
         with_norm(with_norm),
         bias_probs(bias_probs),
         gating_func(gating_func),
-        scale_w(scale_w) {
+        scale_w(scale_w),
+        n_expert_groups(n_expert_groups) {
         GGML_ASSERT(n_expert_used <= ne[0]);
+        GGML_ASSERT(ne[0] % n_expert_groups == 0);
+        GGML_ASSERT(n_expert_used <= ne[0] / n_expert_groups);
     }
 
-    std::string vars() override { return VARS_TO_STR6(ne, n_expert_used, with_norm, bias_probs, gating_func, scale_w); }
+    std::string vars() override { return VARS_TO_STR7(ne, n_expert_used, with_norm, bias_probs, gating_func, scale_w, n_expert_groups); }
 
     std::string op_desc(ggml_tensor * t) override {
         GGML_UNUSED(t);
@@ -7358,6 +7363,8 @@ struct test_topk_moe : public test_case {
     ggml_tensor * build_graph(ggml_context * ctx) override {
         const int n_expert = ne[0];
         const int n_tokens = ne[1];
+        const int n_exp_per_group = n_expert / n_expert_groups;
+        const int n_group_used = n_expert_groups / 2;
 
         ggml_tensor * logits = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data());
         ggml_tensor * probs            =
@@ -7374,6 +7381,24 @@ struct test_topk_moe : public test_case {
             ggml_set_name(selection_probs, "selection_probs");
         }
 
+        if (n_expert_groups > 1) {
+            ggml_tensor * selection_groups = ggml_reshape_3d(ctx, selection_probs, n_exp_per_group, n_expert_groups, n_tokens); // [n_exp_per_group, n_expert_groups, n_tokens]
+
+            ggml_tensor * group_scores = ggml_argsort_top_k(ctx, selection_groups, 2); // [2, n_expert_groups, n_tokens]
+            group_scores = ggml_get_rows(ctx, ggml_reshape_4d(ctx, selection_groups, 1, selection_groups->ne[0], selection_groups->ne[1], selection_groups->ne[2]), group_scores); // [1, 2, n_expert_groups, n_tokens]
+
+            group_scores = ggml_sum_rows(ctx, ggml_reshape_3d(ctx, group_scores, group_scores->ne[1], group_scores->ne[2], group_scores->ne[3])); // [1, n_expert_groups, n_tokens]
+            group_scores = ggml_reshape_2d(ctx, group_scores, group_scores->ne[1], group_scores->ne[2]); // [n_expert_groups, n_tokens]
+
+            ggml_tensor * expert_groups = ggml_argsort_top_k(ctx, group_scores, n_group_used); // [n_group_used, n_tokens]
+            ggml_set_name(expert_groups, "ffn_moe_group_topk");
+
+            selection_probs = ggml_get_rows(ctx, selection_groups, expert_groups); // [n_exp_per_group, n_group_used, n_tokens]
+            selection_probs = ggml_set_rows(ctx, ggml_fill(ctx, selection_groups, -INFINITY), selection_probs, expert_groups); // [n_exp_per_group, n_expert_groups, n_tokens]
+            selection_probs = ggml_reshape_2d(ctx, selection_probs, n_expert, n_tokens); // [n_expert, n_tokens]
+            ggml_set_name(selection_probs, "ffn_moe_probs_masked");
+        }
+
         selected_experts = ggml_argsort_top_k(ctx, selection_probs, n_expert_used); // [n_expert_used, n_tokens]
         ggml_set_name(selected_experts, "selected_experts");
 
@@ -11937,6 +11962,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
                     test_cases.emplace_back(new test_topk_moe({32, 8, 1, 1}, 4, with_norm, bias_probs, gate, scale_w));
                     test_cases.emplace_back(new test_topk_moe({32, 8, 1, 1}, 8, with_norm, bias_probs, gate, scale_w));
                     test_cases.emplace_back(new test_topk_moe({32, 9, 1, 1}, 8, with_norm, bias_probs, gate, scale_w));
+
+                    // grouped experts
+                    test_cases.emplace_back(new test_topk_moe({128, 1, 1, 1}, 8, with_norm, bias_probs, gate, scale_w, 16));
+                    test_cases.emplace_back(new test_topk_moe({256, 22, 1, 1}, 6, with_norm, bias_probs, gate, scale_w, 32));
+                    test_cases.emplace_back(new test_topk_moe({256, 7, 1, 1}, 8, with_norm, bias_probs, gate, scale_w, 8)); // DeepSeek V3
+                    test_cases.emplace_back(new test_topk_moe({128, 5, 1, 1}, 6, with_norm, bias_probs, gate, scale_w, 8));
+                    test_cases.emplace_back(new test_topk_moe({512, 3, 1, 1}, 8, with_norm, bias_probs, gate, scale_w, 4));
                 }
             }
         }

tests/test-jinja.cpp

@@ -2011,6 +2011,12 @@ static void test_object_methods(testing & t) {
         "a b "
     );
 
+    test_template(t, "object | list",
+        "{{ (obj | list) == ['b', 'a'] }} {{ obj | list | join(',') }}",
+        {{"obj", {{"b", 1}, {"a", 2}}}},
+        "True b,a"
+    );
+
     test_template(t, "object.values()",
         "{% for v in obj.values() %}{{ v }} {% endfor %}",
         {{"obj", {{"a", 1}, {"b", 2}}}},

tests/test-llama-archs.cpp

@@ -53,33 +53,66 @@ struct tensor_data_params {
     float  stdev;
 };
 
+// fills the values of one tensor, the name carries whatever the property needs
+typedef void (*tensor_filler)(std::vector<float> & dst, const char * name, const tensor_data_params & params);
+
+// a normal draw keyed on a name, so that tensors which share a draw ask for the same key
+static void fill_normal(std::vector<float> & dst, const char * key, const tensor_data_params & params) {
+    std::mt19937 gen(params.seed ^ std::hash<std::string>{}(key));
+    std::normal_distribution<float> dis(0.0f, params.stdev);
+    for (float & v : dst) {
+        v = dis(gen);
+    }
+}
+
+// Mamba A must be negative (state decay)
+static void fill_ssm_a(std::vector<float> & dst, const char * name, const tensor_data_params & params) {
+    fill_normal(dst, name, params);
+    for (float & v : dst) {
+        v = -fabsf(v);
+    }
+}
+
+// a norm preserving residual gate holds alpha^2 + beta^2 == 1, and the conversion resolves both from one raw weight
+// the alpha and the beta of a block share that weight, so the key drops the suffix and keeps the block
+static void fill_resid_gate(std::vector<float> & dst, const char * name, const tensor_data_params & params) {
+    const char * suffix = strstr(name, "_resid_");
+    const bool is_beta = strstr(suffix, "_resid_beta") != nullptr;
+    fill_normal(dst, std::string(name, suffix - name).c_str(), params);
+    for (float & v : dst) {
+        const float beta = 1.0f / (1.0f + expf(-v));
+        v = is_beta ? beta : sqrtf(1.0f - beta*beta);
+    }
+}
+
+// a tensor whose values hold a property of their own, matched on a substring of its name, first match wins
+static const struct { const char * match; tensor_filler fill; } TENSOR_FILLERS[] = {
+    { "ssm_a",   fill_ssm_a      },
+    { "_resid_", fill_resid_gate },
+};
+
 static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {
     const tensor_data_params & params = *(const tensor_data_params *) userdata;
-    size_t seed = params.seed;
-    std::hash<std::string> hasher;
-    seed ^= hasher(tensor->name);
-    std::mt19937 gen(seed);
-    std::normal_distribution<float> dis(0.0f, params.stdev);
 
-    // TODO: refactor per-tensor initialization logic in a cleaner way
-
-    // note: Mamba A must be negative (state decay)
-    const bool is_ssm_a = strstr(tensor->name, "ssm_a") != nullptr;
-    const int64_t ne = ggml_nelements(tensor);
-    if (tensor->type == GGML_TYPE_F32) {
-        std::vector<float> tmp(ne);
-        for (int64_t i = 0; i < ne; i++) {
-            float val = dis(gen);
-            tmp[i] = is_ssm_a ? -fabsf(val) : val;
+    tensor_filler fill = fill_normal;
+    for (const auto & filler : TENSOR_FILLERS) {
+        if (strstr(tensor->name, filler.match) != nullptr) {
+            fill = filler.fill;
+            break;
         }
+    }
+
+    std::vector<float> tmp(ggml_nelements(tensor));
+    fill(tmp, tensor->name, params);
+
+    if (tensor->type == GGML_TYPE_F32) {
         ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor));
     } else if (tensor->type == GGML_TYPE_F16) {
-        std::vector<ggml_fp16_t> tmp(ne);
-        for (int64_t i = 0; i < ne; i++) {
-            float val = dis(gen);
-            tmp[i] = ggml_fp32_to_fp16(is_ssm_a ? -fabsf(val) : val);
+        std::vector<ggml_fp16_t> tmp16(tmp.size());
+        for (size_t i = 0; i < tmp.size(); i++) {
+            tmp16[i] = ggml_fp32_to_fp16(tmp[i]);
         }
-        ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor));
+        ggml_backend_tensor_set(tensor, tmp16.data(), 0, ggml_nbytes(tensor));
     } else {
         GGML_ABORT("fatal error");
     }
@@ -185,9 +218,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
     uint32_t n_head_kv = n_head;
     if (arch == LLM_ARCH_QWEN3) {
         n_head_kv = 1; // MQA coverage
-    } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER) {
+    } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER ||
+            arch == LLM_ARCH_KOLIBRI1) {
         n_head_kv = 2; // GQA coverage
     }
+    if (arch == LLM_ARCH_BEAM) {
+        n_head    = 4;
+        n_head_kv = 2;
+        n_layer   = 4;
+    }
     const uint32_t n_embd_head = n_embd / n_head;
 
     ms.add_kv(LLM_KV_GENERAL_ARCHITECTURE,      llm_arch_name(arch));
@@ -294,6 +333,13 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
 
     if (arch == LLM_ARCH_MAION_CODER) {
         ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector<uint32_t>{1, 1, 0, 0});
+    } else if (arch == LLM_ARCH_BEAM) {
+        std::vector<uint32_t> pattern;
+        pattern.reserve(n_layer);
+        for (uint32_t il = 0; il < n_layer; il++) {
+            pattern.push_back(il % 2);
+        }
+        ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);
     } else if (arch == LLM_ARCH_GEMMA4) {
         ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER,      n_embd/2);
         ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS,      uint32_t(0));
@@ -304,7 +350,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5));
     } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 ||
             arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE ||
-            arch == LLM_ARCH_MAPLE) {
+            arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE) {
         std::vector<uint32_t> pattern;
         pattern.reserve(n_layer);
         for (uint32_t il = 0; il < n_layer; il++) {
@@ -324,6 +370,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM,     true);
     }
 
+    if (arch == LLM_ARCH_KOLIBRI1) {
+        ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, false);
+    }
+
     // MSA requires one indexer head per GQA (KV) head, unlike the DSA archs where the
     // indexer head count is independent of the main attention head count.
     if (arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_GLM5_NEXT) {
@@ -470,6 +520,14 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         }
     }
 
+    if (arch == LLM_ARCH_BEAM) {
+        ms.add_kv(LLM_KV_EXPERT_COUNT,            uint32_t(8));
+        ms.add_kv(LLM_KV_EXPERT_GROUP_COUNT,      uint32_t(4));
+        ms.add_kv(LLM_KV_EXPERT_GROUP_USED_COUNT, uint32_t(2));
+        ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE,    2.5f);
+        ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM,     true);
+    }
+
     if (arch == LLM_ARCH_MAION_CODER) {
         ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd / 2);
         ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, std::vector<uint32_t>{n_ff, n_ff / 2, n_ff, n_ff});
@@ -712,6 +770,7 @@ static bool moe_mandatory(const llm_arch arch) {
         case LLM_ARCH_BAILINGMOE3:
         case LLM_ARCH_DOTS1:
         case LLM_ARCH_AFMOE:
+        case LLM_ARCH_BEAM:
         case LLM_ARCH_ERNIE4_5:
         case LLM_ARCH_ERNIE4_5_MOE:
         case LLM_ARCH_HUNYUAN_MOE:
@@ -737,6 +796,7 @@ static bool moe_mandatory(const llm_arch arch) {
         case LLM_ARCH_LAGUNA:
         case LLM_ARCH_MAPLE:
         case LLM_ARCH_MUSE_SPARK:
+        case LLM_ARCH_KOLIBRI1:
             return true;
         default:
             return false;

tests/test-model-resolution.cpp

@@ -15,6 +15,7 @@
 #include <cstdio>
 #include <cstdlib>
 #include <filesystem>
+#include <fstream>
 #include <map>
 #include <thread>
 #include <string>
@@ -339,6 +340,124 @@ static void test_plan_resolution() {
     g_repos.clear();
 }
 
+// put a file in the cache the way a finished download leaves it, refs/main included
+static void cache_put(const std::string & repo_id, const std::string & path) {
+    namespace fs = std::filesystem;
+
+    const fs::path local = cached(repo_id, path);
+    fs::create_directories(local.parent_path());
+    { std::ofstream(local) << "gguf"; }
+
+    const fs::path ref = local.parent_path().parent_path().parent_path() / "refs" / "main";
+    fs::create_directories(ref.parent_path());
+    if (!fs::exists(ref)) {
+        std::ofstream(ref) << COMMIT << "\n";
+    }
+}
+
+//
+// partials of a download: the files of its plan not in the cache yet, next to
+// the blobs, named after the oids the listing serves
+//
+
+static void test_plan_partials() {
+    namespace fs = std::filesystem;
+
+    printf("test-model-resolution: plan partials\n");
+
+    g_context = "plan partials";
+    g_repos.clear();
+    g_repos["test/partial"] = { "model-Q4_0.gguf", "mmproj-model-F16.gguf", "mtp-model-Q4_0.gguf" };
+
+    // the mmproj is complete, the model and its mtp are still to download
+    cache_put("test/partial", "mmproj-model-F16.gguf");
+
+    common_download_opts opts;
+    opts.download_mmproj = true;
+    opts.download_mtp    = true;
+
+    const auto parts = common_download_plan_partials(common_download_get_hf_plan(model_ref("test/partial:Q4_0"), opts));
+
+    const fs::path blobs = cache_dir / "models--test--partial" / "blobs";
+    REQUIRE(parts.size() == 2);
+    REQUIRE(parts[0].kind.empty());
+    REQUIRE(fs::path(parts[0].path).lexically_normal() == (blobs / "0000000000000000000000000000000000000001.downloadInProgress").lexically_normal());
+    REQUIRE_EQ(parts[1].kind, "mtp");
+    REQUIRE(fs::path(parts[1].path).lexically_normal() == (blobs / "0000000000000000000000000000000000000003.downloadInProgress").lexically_normal());
+
+    fs::remove_all(cache_dir / "models--test--partial");
+    g_repos.clear();
+}
+
+//
+// removal of a sidecar alone: its snapshot entry goes, the model keeps its files
+//
+
+static void test_sidecar_removal() {
+    namespace fs = std::filesystem;
+
+    printf("test-model-resolution: sidecar removal\n");
+
+    g_context = "sidecar removal";
+    g_repos.clear();
+
+    cache_put("test/sidecar", "model-Q4_0.gguf");
+    cache_put("test/sidecar", "dflash-model-Q4_0.gguf");
+    cache_put("test/sidecar", "mtp-model-Q4_0.gguf");
+
+    common_download_opts opts;
+    opts.download_dflash = true;
+
+    // the selected sidecar goes, the other one and the model stay
+    REQUIRE(common_download_remove_sidecars("test/sidecar:Q4_0", opts));
+    REQUIRE(!fs::exists(cached("test/sidecar", "dflash-model-Q4_0.gguf")));
+    REQUIRE(fs::exists(cached("test/sidecar", "mtp-model-Q4_0.gguf")));
+    REQUIRE(fs::exists(cached("test/sidecar", "model-Q4_0.gguf")));
+    REQUIRE(!common_download_remove_sidecars("test/sidecar:Q4_0", opts));
+
+    fs::remove_all(cache_dir / "models--test--sidecar");
+}
+
+//
+// removal of a tag keeps the files another cached tag still loads: a sidecar
+// named after the removed tag that another quant resolves stays on disk
+//
+
+static void test_shared_removal() {
+    namespace fs = std::filesystem;
+
+    printf("test-model-resolution: shared removal\n");
+
+    g_context = "shared removal";
+    g_repos.clear();
+
+    for (const char * f : { "model-Q8_0.gguf", "model-BF16.gguf", "mmproj-model-BF16.gguf", "mtp-model-BF16.gguf" }) {
+        cache_put("test/shared", f);
+    }
+
+    // the BF16 model goes, the mmproj and the mtp Q8_0 resolves stay
+    REQUIRE(common_download_remove("test/shared:BF16"));
+    REQUIRE(!fs::exists(cached("test/shared", "model-BF16.gguf")));
+    REQUIRE(fs::exists(cached("test/shared", "mmproj-model-BF16.gguf")));
+    REQUIRE(fs::exists(cached("test/shared", "mtp-model-BF16.gguf")));
+    REQUIRE(fs::exists(cached("test/shared", "model-Q8_0.gguf")));
+
+    fs::remove_all(cache_dir / "models--test--shared");
+
+    // with no other tag, the files of the removed tag all go, sidecars included
+    for (const char * f : { "model-BF16.gguf", "mmproj-model-BF16.gguf", "mtp-model-BF16.gguf" }) {
+        cache_put("test/alone", f);
+    }
+
+    REQUIRE(common_download_remove("test/alone:BF16"));
+    REQUIRE(!fs::exists(cached("test/alone", "model-BF16.gguf")));
+    REQUIRE(!fs::exists(cached("test/alone", "mmproj-model-BF16.gguf")));
+    REQUIRE(!fs::exists(cached("test/alone", "mtp-model-BF16.gguf")));
+    REQUIRE(!common_download_remove("test/alone:BF16"));
+
+    fs::remove_all(cache_dir / "models--test--alone");
+}
+
 //
 // end-to-end assembly: real CLI parsing, real handler init resolving over the
 // loopback, downloads skipped by flipping offline before apply
@@ -509,6 +628,9 @@ int main(void) {
     common_set_env("MODEL_ENDPOINT", "http://127.0.0.1:" + std::to_string(port) + "/");
 
     test_plan_resolution();
+    test_plan_partials();
+    test_sidecar_removal();
+    test_shared_removal();
     test_task_assembly();
 
     server.stop();

tools/server/README.md

@@ -405,6 +405,10 @@ Recommended `--cors-origins` setting, depending on where the server runs:
 
 Related flags: `--cors-origins`, `--cors-methods`, `--cors-headers`, `--cors-credentials` / `--no-cors-credentials`. Background and rationale: [#25655](https://github.com/ggml-org/llama.cpp/pull/25655).
 
+### Running behind a proxy
+
+If your host reaches the internet through a corporate proxy, export `HTTP_PROXY` and `HTTPS_PROXY` before running commands that download models (e.g. `export HTTPS_PROXY=http://proxy.example.com:8080`). Hosts listed in `NO_PROXY` and loopback addresses are contacted directly.
+
 ## Build
 
 `llama-server` is built alongside everything else from the root of the project
@@ -1719,9 +1723,9 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo
 
 The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly.
 
-The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
+The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for granite-switch, laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
 
-For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available.
+For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.
 
 *Image input:*
 
@@ -2102,6 +2106,32 @@ Note: for "downloading" state, there can be multiple files be downloading in par
 }
 ```
 
+A download that never finished (paused or interrupted) is listed under a separate
+"incomplete" status, without an instance: posting the model to `/models` again resumes it
+from the files already on disk, and `DELETE /models` drops its partial files.
+
+```json
+"status": {
+  "value": "incomplete",
+  "args": []
+}
+```
+
+Cached models and incomplete downloads carry the sidecar kinds they have on disk
+(`mmproj`, `mtp`, `dflash`, `eagle3`, `dspark`), resolved the way `-hf` loads them:
+
+```json
+"sidecars": ["mmproj", "mtp"]
+```
+
+A cached model whose download of more sidecars never finished also lists the kinds not on
+disk yet: posting the model with them resumes them, and `DELETE /models` with `&sidecar={kind}`
+drops their partial files.
+
+```json
+"incomplete_sidecars": ["mtp"]
+```
+
 ### POST `/models/load`: Load a model
 
 Load a model
@@ -2221,7 +2251,7 @@ Example events:
 
 Trigger a new download (non-blocking), the progress can be tracked via SSE endpoint `/models/sse`
 
-To cancel model downloading, send an event to `/models/unload`
+To pause a download, send an event to `/models/unload`: it keeps its partial files and is listed as "incomplete" until it is posted again. To cancel it and drop its files, use `DELETE /models`.
 
 Download procedure:
 - Send POST request to `/models`
@@ -2234,9 +2264,14 @@ Payload:
 ```json
 {
   "model": "ggml-org/gemma-3-4b-it-GGUF:Q4_K_M",
+  "sidecars": ["mmproj", "mtp"]
 }
 ```
 
+`sidecars` lists the sidecar kinds pulled with the model, picked by the server the way `-hf` picks them: `mmproj`, and the draft heads `mtp`, `dflash`, `eagle3` and `dspark`. It defaults to `["mmproj"]`, as `-hf` does.
+
+Posting a model that is already in the cache and not running downloads the sidecars it lacks and keeps the files already on disk.
+
 Response (download is started in the background):
 
 ```json
@@ -2263,6 +2298,10 @@ IMPORTANT: only model stored in cache can be deleted. You cannot delete models i
 
 Model name must be passed via query param: `?model={name}`
 
+To remove one sidecar of a cached model and keep the model, add `&sidecar={kind}` (`mmproj`, `mtp`, `dflash`, `eagle3` or `dspark`); the model must not be running.
+
+Deleting a model keeps the files another cached quant of the repo still loads, such as an mmproj named after the deleted quant, and the partial files another download still writes. A sidecar deleted on its own goes for every quant that shares it.
+
 If delete success, it will send an SSE event of type `model_remove`
 
 Response:

tools/server/server-common.cpp

@@ -161,6 +161,8 @@ std::vector<std::string> server_model_output_modalities(common_decision_type dec
         case COMMON_DECISION_TYPE_LFM2_D1:
         case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
             return {"decisions"};
+        case COMMON_DECISION_TYPE_GRANITE_SWITCH:
+            return {"text", "decisions"};
         default:
             // fallback when there is no decision type or the metadata is bad
             return {"text"};

tools/server/server-decision.cpp

@@ -99,7 +99,7 @@ void server_decision_context::init(const llama_model * model) {
         }
         n_options_max   = labels.size();
         noul_true_first = true;
-    } else if (model_type == COMMON_DECISION_TYPE_LEV || model_type == COMMON_DECISION_TYPE_NIMBLE || model_type == COMMON_DECISION_TYPE_PPLX_DECIDER || model_type == COMMON_DECISION_TYPE_VLLM_D3) {
+    } else if (model_type == COMMON_DECISION_TYPE_LEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH || model_type == COMMON_DECISION_TYPE_NIMBLE || model_type == COMMON_DECISION_TYPE_PPLX_DECIDER || model_type == COMMON_DECISION_TYPE_VLLM_D3) {
         // label codes are A..Z then AA..ZZ, only the ones that are a single token are used
         std::vector<std::string> codes;
         for (char a = 'A'; a <= 'Z'; a++) {
@@ -121,6 +121,9 @@ void server_decision_context::init(const llama_model * model) {
         if (model_type == COMMON_DECISION_TYPE_VLLM_D3) {
             canonical_legend = true;
         }
+        if (model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) {
+            noul_true_first = true;
+        }
     } else if (model_type == COMMON_DECISION_TYPE_KEV) {
         // the hidden state of an option is read at the token that ends it
         const auto toks = common_tokenize(vocab, "<|box_end|>", false, true);
@@ -753,7 +756,7 @@ void server_decision_context::fill_task(
 
     const std::string prompt = render(state, questions, question, variant, files.size());
 
-    if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) {
+    if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_GRANITE_SWITCH || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) {
         // lev reads the ratings of a noul question at its first labels, not at the digits
         task.decision.labels.assign(labels.begin(), labels.begin() + n_outputs(question));
     }

tools/server/server-decision.h

@@ -39,6 +39,7 @@ struct server_decision_context {
     bool can_share_prompt() const {
         switch (type) {
             case COMMON_DECISION_TYPE_OPENJEV:
+            case COMMON_DECISION_TYPE_GRANITE_SWITCH:
             case COMMON_DECISION_TYPE_LEV:
             case COMMON_DECISION_TYPE_KEV:
             case COMMON_DECISION_TYPE_NIMBLE:

tools/server/server-models.cpp

@@ -26,6 +26,7 @@
 #include <chrono>
 #include <queue>
 #include <filesystem>
+#include <fstream>
 #include <random>
 #include <sstream>
 #include <cstring>
@@ -618,6 +619,7 @@ server_models::server_models(
         LOG_WRN("failed to get server executable path: %s\n", e.what());
         LOG_WRN("using original argv[0] as fallback: %s\n", argv[0]);
     }
+    unfinished_downloads.load();
     load_models();
     debug_fake_timing = !common_get_env("LLAMA_SERVER_DEBUG_FAKE_TIMING").empty();
 }
@@ -686,6 +688,233 @@ void server_models::add_model(server_model_meta && meta) {
     };
 }
 
+static bool has_sidecar(const std::vector<std::string> & sidecars, const std::string & kind) {
+    return std::find(sidecars.begin(), sidecars.end(), kind) != sidecars.end();
+}
+
+// reject a sidecar kind no download can pull
+static void check_sidecar(const std::string & kind) {
+    std::vector<std::string> kinds;
+    for (const auto & s : common_download_sidecars()) {
+        kinds.emplace_back(s.name);
+    }
+    if (!has_sidecar(kinds, kind)) {
+        throw std::invalid_argument("unknown sidecar '" + kind + "', expected one of " + string_join(kinds, ", "));
+    }
+}
+
+// the sidecars a download asks for; by default the ones -hf pulls without a draft type
+static std::vector<std::string> parse_sidecars(const json & body) {
+    std::vector<std::string> defaults;
+    for (const auto & s : common_download_sidecars()) {
+        if (!s.draft) {
+            defaults.emplace_back(s.name);
+        }
+    }
+    std::vector<std::string> sidecars = json_value(body, "sidecars", defaults);
+    for (const auto & kind : sidecars) {
+        check_sidecar(kind);
+    }
+    return sidecars;
+}
+
+// the download options of a model with its sidecars, the plan a removal purges partials from
+static common_download_opts sidecar_download_opts(const common_params & params, const std::vector<std::string> & sidecars) {
+    common_download_opts opts;
+    opts.bearer_token = params.hf_token;
+    for (const auto & kind : sidecars) {
+        common_download_opts_add_sidecar(opts, kind);
+    }
+    return opts;
+}
+
+// the HF repo a model resolves from: a preset names its own, a cached model is
+// listed under it, and a local file or a models dir entry has none
+static std::string model_hf_repo(const server_model_meta & meta) {
+    std::string hf_repo;
+    if (meta.preset.get_option("LLAMA_ARG_HF_REPO", hf_repo) && !hf_repo.empty()) {
+        return hf_repo;
+    }
+    return meta.source == SERVER_MODEL_SOURCE_CACHE ? meta.name : "";
+}
+
+// the sidecar kinds of a HF repo present on disk, as the offline plan resolves them
+static json cached_sidecars(const std::string & hf_repo) {
+    common_params_model model;
+    model.hf_repo = hf_repo;
+
+    common_download_opts opts;
+    opts.offline = true;
+    for (const auto & s : common_download_sidecars()) {
+        opts.*s.opt = true;
+    }
+
+    return common_download_plan_sidecars(common_download_get_hf_plan(model, opts));
+}
+
+// replace any inherited value of `key` in a child environment
+static void set_child_env(std::vector<std::string> & env, const std::string & key, const std::string & value) {
+    env.erase(std::remove_if(env.begin(), env.end(), [&](const std::string & e) {
+        return e.rfind(key + "=", 0) == 0;
+    }), env.end());
+    env.push_back(key + "=" + value);
+}
+
+// where the unfinished downloads are remembered, inside the model cache
+static std::string unfinished_downloads_path() {
+    return hf_cache::get_cache_path() + "/unfinished-downloads.json";
+}
+
+void server_unfinished_downloads::load() {
+    std::ifstream in(unfinished_downloads_path());
+    if (!in) {
+        return;
+    }
+    std::string raw((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
+    std::lock_guard<std::mutex> lk(mutex);
+    try {
+        json data = json::parse(raw);
+        for (const auto & d : data.value("downloads", json::array())) {
+            server_unfinished_download download;
+            download.model    = json_value(d, "model", std::string());
+            download.sidecars = json_value(d, "sidecars", std::vector<std::string>());
+            for (const auto & p : d.value("partials", json::array())) {
+                download.partials.push_back({ json_value(p, "kind", std::string()), json_value(p, "path", std::string()) });
+            }
+            if (!download.model.empty()) {
+                downloads.push_back(std::move(download));
+            }
+        }
+    } catch (const std::exception & e) {
+        LOG_WRN("failed to read unfinished downloads: %s\n", e.what());
+    }
+}
+
+void server_unfinished_downloads::save() {
+    json data;
+    data["downloads"] = json::array();
+    for (const auto & d : downloads) {
+        json partials = json::array();
+        for (const auto & p : d.partials) {
+            partials.push_back({ {"kind", p.kind}, {"path", p.path} });
+        }
+        data["downloads"].push_back({ {"model", d.model}, {"sidecars", d.sidecars}, {"partials", partials} });
+    }
+    try {
+        fs_write_atomic(std::filesystem::u8path(unfinished_downloads_path()), data.dump());
+    } catch (const std::exception & e) {
+        LOG_WRN("failed to record unfinished downloads: %s\n", e.what());
+    }
+}
+
+void server_unfinished_downloads::delete_partials(const std::vector<common_download_partial> & parts) {
+    for (const auto & p : parts) {
+        const bool shared = std::any_of(downloads.begin(), downloads.end(), [&](const server_unfinished_download & d) {
+            return std::any_of(d.partials.begin(), d.partials.end(), [&](const common_download_partial & q) {
+                return q.path == p.path;
+            });
+        });
+        if (shared) {
+            continue;
+        }
+        std::error_code ec;
+        std::filesystem::remove(std::filesystem::u8path(p.path), ec);
+    }
+}
+
+void server_unfinished_downloads::add(const server_unfinished_download & download) {
+    std::lock_guard<std::mutex> lk(mutex);
+    std::vector<common_download_partial> stale;
+    auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == download.model; });
+    if (it != downloads.end()) {
+        stale = std::move(it->partials);
+        *it = download;
+    } else {
+        downloads.push_back(download);
+    }
+    save();
+    delete_partials(stale);
+}
+
+void server_unfinished_downloads::forget(const std::string & model) {
+    std::lock_guard<std::mutex> lk(mutex);
+    auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == model; });
+    if (it != downloads.end()) {
+        downloads.erase(it);
+        save();
+    }
+}
+
+bool server_unfinished_downloads::remove(const std::string & model) {
+    std::lock_guard<std::mutex> lk(mutex);
+    auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == model; });
+    if (it == downloads.end()) {
+        return false;
+    }
+    auto partials = std::move(it->partials);
+    downloads.erase(it);
+    save();
+    delete_partials(partials);
+    return true;
+}
+
+bool server_unfinished_downloads::remove_sidecar(const std::string & model, const std::string & kind) {
+    std::lock_guard<std::mutex> lk(mutex);
+    auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == model; });
+    if (it == downloads.end() || !has_sidecar(it->sidecars, kind)) {
+        return false;
+    }
+    it->sidecars.erase(std::remove(it->sidecars.begin(), it->sidecars.end(), kind), it->sidecars.end());
+
+    std::vector<common_download_partial> dropped;
+    auto & partials = it->partials;
+    auto pos = std::stable_partition(partials.begin(), partials.end(), [&](const common_download_partial & p) {
+        return p.kind != kind;
+    });
+    dropped.assign(pos, partials.end());
+    partials.erase(pos, partials.end());
+
+    save();
+    delete_partials(dropped);
+    return true;
+}
+
+std::vector<server_unfinished_download> server_unfinished_downloads::get_all() {
+    std::lock_guard<std::mutex> lk(mutex);
+    return downloads;
+}
+
+std::vector<server_unfinished_download> server_models::unfinished_downloads_get() {
+    return unfinished_downloads.get_all();
+}
+
+void server_models::unfinished_downloads_remove(const std::string & name) {
+    // under the lock, so a download of the same tag cannot start while its files go
+    std::lock_guard<std::mutex> lk(mutex);
+    if (mapping.find(name) != mapping.end()) {
+        throw std::invalid_argument("model '" + name + "' is downloading");
+    }
+    // a name with no record is no model: its files belong to the cached tags of the repo
+    if (!unfinished_downloads.remove(name)) {
+        throw std::runtime_error("model name=" + name + " is not found");
+    }
+    bool ok = common_download_remove(name);
+    SRV_INF("removing unfinished download name=%s (%s)\n", name.c_str(), ok ? "succeeded" : "nothing cached");
+}
+
+bool server_models::remove_sidecar(const std::string & name, const std::string & kind) {
+    std::lock_guard<std::mutex> lk(mutex);
+    auto it = mapping.find(name);
+    if (it != mapping.end() && (it->second.meta.is_running() || it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING)) {
+        throw std::invalid_argument("model '" + name + "' is in use, unload it first");
+    }
+    // a sidecar that only ever landed as a partial is still something removed
+    bool ok = unfinished_downloads.remove_sidecar(name, kind);
+    ok = common_download_remove_sidecars(name, sidecar_download_opts(base_params, { kind })) || ok;
+    SRV_INF("removing sidecar %s of model name=%s (%s)\n", kind.c_str(), name.c_str(), ok ? "succeeded" : "nothing cached");
+    return ok;
+}
+
 void server_models::notify_sse(const std::string & event, const std::string & model_id, const json & data) {
     std::unique_ptr<server_task_result_router> result = std::make_unique<server_task_result_router>();
     result->data = {
@@ -1135,7 +1364,8 @@ void server_models::load(const std::string & name) {
 }
 
 void server_models::load(const std::string & name, const load_options & opts) {
-    if (debug_fake_timing) {
+    // the fake load time of a model; a download worker loads none
+    if (debug_fake_timing && opts.mode == SERVER_CHILD_MODE_NORMAL) {
         // do not hold the mutex here, other requests must keep making progress
         std::this_thread::sleep_for(std::chrono::seconds(2));
     }
@@ -1152,6 +1382,43 @@ void server_models::load(const std::string & name, const load_options & opts) {
     // against the freshest preset and a consistent mapping state
     cv.wait(lk, [this]() { return !is_reloading; });
 
+    // a download over an existing entry is decided under the lock, so a load of the
+    // same model cannot start in between: a paused download or an idle cached model
+    // is downloaded again with its files kept, a download on its way and a running
+    // model are refused, and a preset or local model is not a download
+    if (opts.mode == SERVER_CHILD_MODE_DOWNLOAD) {
+        auto it = mapping.find(name);
+        if (it != mapping.end()) {
+            const auto & cur = it->second.meta;
+            if (cur.status == SERVER_MODEL_STATUS_DOWNLOADING) {
+                throw std::invalid_argument("model '" + name + "' is already downloading");
+            }
+            if (cur.is_running()) {
+                throw std::invalid_argument("model '" + name + "' is running, unload it first");
+            }
+            if (cur.source != SERVER_MODEL_SOURCE_CACHE) {
+                throw std::invalid_argument("model '" + name + "' already exists");
+            }
+            SRV_INF("resuming download for model '%s'\n", name.c_str());
+        }
+
+        // two downloads writing the same partial would interleave their bytes
+        for (const auto & d : unfinished_downloads.get_all()) {
+            auto other = mapping.find(d.model);
+            if (d.model == name || other == mapping.end() || other->second.meta.status != SERVER_MODEL_STATUS_DOWNLOADING) {
+                continue;
+            }
+            for (const auto & p : d.partials) {
+                const bool shared = std::any_of(opts.partials.begin(), opts.partials.end(), [&](const common_download_partial & q) {
+                    return q.path == p.path;
+                });
+                if (shared) {
+                    throw std::invalid_argument("model '" + name + "' shares files with the download of '" + d.model + "' on its way, retry once it is done");
+                }
+            }
+        }
+    }
+
     auto meta = opts.custom_meta.has_value() ? *opts.custom_meta : mapping[name].meta;
     if (meta.status != SERVER_MODEL_STATUS_UNLOADED) {
         SRV_INF("model %s is not ready\n", name.c_str());
@@ -1201,6 +1468,23 @@ void server_models::load(const std::string & name, const load_options & opts) {
             inst.meta.status = SERVER_MODEL_STATUS_DOWNLOADING;
             child_env.push_back("LLAMA_SERVER_CHILD_MODE=download");
             child_env.push_back("LLAMA_ARG_HF_REPO=" + name);
+
+            // the sidecars as -hf would pull them: the draft types, and the mmproj switch
+            // for the one that is not a draft
+            bool        mmproj = false;
+            std::string spec_types;
+            for (const auto & s : common_download_sidecars()) {
+                if (!has_sidecar(opts.sidecars, s.name)) {
+                    continue;
+                }
+                if (s.draft) {
+                    spec_types += (spec_types.empty() ? "" : ",") + std::string("draft-") + s.name;
+                } else {
+                    mmproj = true;
+                }
+            }
+            set_child_env(child_env, "LLAMA_ARG_MMPROJ_AUTO", mmproj ? "1" : "0");
+            set_child_env(child_env, "LLAMA_ARG_SPEC_TYPE", spec_types.empty() ? "none" : spec_types);
         }
 
         SRV_INF("%s", "spawning server instance with args:\n");
@@ -1216,6 +1500,11 @@ void server_models::load(const std::string & name, const load_options & opts) {
         }
     }
 
+    // remember the download: the partial files carry a content hash, not the name
+    if (opts.mode == SERVER_CHILD_MODE_DOWNLOAD) {
+        unfinished_downloads.add({ name, opts.sidecars, opts.partials });
+    }
+
     // old process should have exited already, but just in case, we clean it up here
     {
         auto it = mapping.find(name);
@@ -1390,6 +1679,12 @@ void server_models::update_download_progress(const std::string & name, const com
                 // mark the instance to be erased on next load_models() call
                 it->second.meta.status = SERVER_MODEL_STATUS_DOWNLOADED;
                 need_reload = true;
+
+                // a download that failed keeps its tag: its partial files are still
+                // there and re-posting the tag resumes them
+                if (ok) {
+                    unfinished_downloads.forget(name);
+                }
             } else {
                 json & info = it->second.meta.loaded_info;
                 if (!info.contains("progress")) {
@@ -1445,24 +1740,14 @@ bool server_models::remove(const std::string & name) {
             || meta.status == SERVER_MODEL_STATUS_DOWNLOADED;
     });
 
-    // re-find after wait - load_models() may have erased the entry during the wait
-    it = mapping.find(name);
-    if (it == mapping.end()) {
-        // load_models() already erased the entry; we just need to clean up the cached files on disk
-        lk.unlock();
-        bool ok = common_download_remove(name);
-        SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "partial");
-        notify_sse("model_remove", name, {});
-        return true;
-    }
-
-    // remove from disk (best-effort: cancelled downloads may have no cached files)
-    bool ok = common_download_remove(name);
+    // load_models() may have erased the entry during the wait; the files go under the
+    // lock, so a download of the same tag cannot start meanwhile
     mapping.erase(name);
-    if (!ok) {
-        SRV_WRN("removing model name=%s from disk returned false (no cached files?)\n", name.c_str());
-    }
-    SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "partial");
+    unfinished_downloads.remove(name);
+
+    // best-effort: a cancelled download may have no cached files
+    bool ok = common_download_remove(name);
+    SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "nothing cached");
     notify_sse("model_remove", name, {});
     return true;
 }
@@ -2091,6 +2376,9 @@ void server_models_routes::init_routes() {
         json models_json = json::array();
         auto all_models = models.get_all_meta();
         std::time_t t = std::time(0);
+        // a download on its way has no files to resolve yet, only the kinds it pulls
+        const auto unfinished = models.unfinished_downloads_get();
+
         for (const auto & meta : all_models) {
             if (meta.hidden) {
                 continue; // cache model deduplicated by a preset
@@ -2132,6 +2420,35 @@ void server_models_routes::init_routes() {
                 model_info["context_length"] = meta.n_ctx_train;
             }
 
+            // the sidecars a model has on disk, resolved as -hf would load them, for a
+            // model that resolves from a HF repo. A download on its way reports the
+            // kinds it pulls, as an unfinished one does
+            const auto it_unfinished = std::find_if(unfinished.begin(), unfinished.end(),
+                [&](const server_unfinished_download & d) { return d.model == meta.name; });
+            const std::string hf_repo = model_hf_repo(meta);
+
+            if (it_unfinished != unfinished.end() && meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
+                model_info["sidecars"] = it_unfinished->sidecars;
+            } else if (!hf_repo.empty()) {
+                const json cached = cached_sidecars(hf_repo);
+                model_info["sidecars"] = cached;
+
+                // the sidecars of an unfinished download of this model that are not on
+                // disk yet: a POST with the kind resumes one, a DELETE with it drops it
+                if (it_unfinished != unfinished.end()) {
+                    std::vector<std::string> incomplete;
+                    for (const auto & p : it_unfinished->partials) {
+                        if (!p.kind.empty() && !has_sidecar(incomplete, p.kind) &&
+                            std::find(cached.begin(), cached.end(), p.kind) == cached.end()) {
+                            incomplete.push_back(p.kind);
+                        }
+                    }
+                    if (!incomplete.empty()) {
+                        model_info["incomplete_sidecars"] = incomplete;
+                    }
+                }
+            }
+
             // merge with loaded_info from the child process if available
             if (meta.is_running()) {
                 for (auto it = meta.loaded_info.begin(); it != meta.loaded_info.end(); ++it) {
@@ -2142,6 +2459,33 @@ void server_models_routes::init_routes() {
             }
             models_json.push_back(model_info);
         }
+
+        const json architecture_default = json {
+            {"input_modalities",  json::array({"text"})},
+            {"output_modalities", json::array({"text"})},
+        };
+
+        // a download that never finished has no instance: report it as incomplete so
+        // the UI can offer to resume it, without starting anything
+        for (const auto & d : unfinished) {
+            if (models.has_model(d.model)) {
+                continue; // complete, or downloading right now
+            }
+            models_json.push_back(json {
+                {"id",           d.model},
+                {"aliases",      json::array()},
+                {"tags",         json::array()},
+                {"object",       "model"},
+                {"owned_by",     "llamacpp"},
+                {"created",      t},
+                {"status",       json { {"value", server_model_status_to_string(SERVER_MODEL_STATUS_INCOMPLETE)}, {"args", json::array()} }},
+                {"architecture", architecture_default},
+                {"source",       server_model_source_to_string(SERVER_MODEL_SOURCE_CACHE)},
+                {"can_remove",   true},
+                {"sidecars",     d.sidecars},
+            });
+        }
+
         res_ok(res, {
             {"data", models_json},
             {"object", "list"},
@@ -2193,32 +2537,27 @@ void server_models_routes::init_routes() {
         if (name.empty()) {
             throw std::invalid_argument("model must be a non-empty string");
         }
+        const auto sidecars = parse_sidecars(body);
 
-        common_params p;
-        p.model.hf_repo  = name;
-        p.hf_token       = params.hf_token;
+        common_params_model model;
+        model.hf_repo = name;
 
-        // validate by fetching metadata
-        bool ok = false;
+        // validate by fetching metadata, through the plan the download follows
+        common_download_hf_plan plan;
         try {
-            common_models_handler_init(p, LLAMA_EXAMPLE_SERVER);
-            ok = true;
+            plan = common_download_get_hf_plan(model, sidecar_download_opts(params, sidecars));
         } catch (...) {
             SRV_ERR("unknown error while validating model '%s'\n", name.c_str());
             // other exceptions will be handled by the outer ex_wrapper()
             throw;
         }
 
-        if (!ok) {
-            throw std::invalid_argument("model validation failed, unable to download");
+        // a tag that resolves no file has nothing to download
+        if (plan.preset.path.empty() && plan.primary.path.empty() && common_download_plan_sidecars(plan).empty()) {
+            throw std::invalid_argument("model '" + name + "' resolves no file to download");
         }
 
-        // reject if model already exists
-        if (models.has_model(name)) {
-            throw std::invalid_argument("model '" + name + "' already exists");
-        }
-
-        // then, proceed with the actual download
+        // then, proceed with the actual download; load() decides over an existing entry
         SRV_INF("starting download for model '%s'\n", name.c_str());
         {
             server_models::load_options load_opts;
@@ -2226,6 +2565,8 @@ void server_models_routes::init_routes() {
             load_opts.custom_meta = server_model_meta{};
             load_opts.custom_meta->source = SERVER_MODEL_SOURCE_CACHE;
             load_opts.custom_meta->name   = name;
+            load_opts.sidecars            = sidecars;
+            load_opts.partials            = common_download_plan_partials(plan);
             models.load(name, load_opts);
         }
 
@@ -2241,6 +2582,22 @@ void server_models_routes::init_routes() {
             throw std::invalid_argument("model must be a non-empty string");
         }
 
+        // a sidecar alone goes, the model keeps its files
+        std::string sidecar = req.get_param("sidecar");
+        if (!sidecar.empty()) {
+            check_sidecar(sidecar);
+            res_ok(res, {{"success", models.remove_sidecar(name, sidecar)}});
+            return res;
+        }
+
+        // a download listed from an earlier run has no instance, only its tag and
+        // its partial files
+        if (!models.has_model(name)) {
+            models.unfinished_downloads_remove(name);
+            res_ok(res, {{"success", true}});
+            return res;
+        }
+
         models.remove(name); // throws on error
 
         res_ok(res, {{"success", true}});

tools/server/server-models.h

@@ -16,6 +16,7 @@
 #include <set>
 #include <string>
 #include <unordered_map>
+#include <vector>
 
 /**
  * state diagram:
@@ -31,6 +32,7 @@
 enum server_model_status {
     // TODO: also add downloading state when the logic is added
     SERVER_MODEL_STATUS_DOWNLOADING,
+    SERVER_MODEL_STATUS_INCOMPLETE, // a download that never finished, listed without an instance
     SERVER_MODEL_STATUS_DOWNLOADED,
     SERVER_MODEL_STATUS_UNLOADED,
     SERVER_MODEL_STATUS_LOADING,
@@ -52,6 +54,7 @@ enum server_child_mode {
 static std::string server_model_status_to_string(server_model_status status) {
     switch (status) {
         case SERVER_MODEL_STATUS_DOWNLOADING: return "downloading";
+        case SERVER_MODEL_STATUS_INCOMPLETE:  return "incomplete";
         case SERVER_MODEL_STATUS_DOWNLOADED:  return "downloaded";
         case SERVER_MODEL_STATUS_UNLOADED:    return "unloaded";
         case SERVER_MODEL_STATUS_LOADING:     return "loading";
@@ -108,6 +111,51 @@ struct server_model_meta {
     void update_caps(const common_params & base);
 };
 
+// a download that never finished: the tag it was asked under, the sidecar kinds it
+// pulls and the partial files its plan writes
+struct server_unfinished_download {
+    std::string                          model;
+    std::vector<std::string>             sidecars;
+    std::vector<common_download_partial> partials;
+};
+
+// the downloads that never finished, recorded next to the model cache. The cache names
+// partial files after their content hash, so this record is the only trace of what is
+// still incomplete: it lets a restarted router list, resume or remove them
+// a partial another recorded download also writes is never deleted
+// these functions are thread-safe
+struct server_unfinished_downloads {
+    // read the record an earlier run left
+    void load();
+
+    // record a download that starts, replacing an earlier record of the same model,
+    // and delete the partials of that record the new plan no longer writes
+    void add(const server_unfinished_download & download);
+
+    // forget a download whose files are complete
+    void forget(const std::string & model);
+
+    // forget a download and delete its partials; returns false when it is not recorded
+    bool remove(const std::string & model);
+
+    // drop the sidecar `kind` from a download and delete its partials; returns false
+    // when the download is not recorded with that kind
+    bool remove_sidecar(const std::string & model, const std::string & kind);
+
+    // every recorded download
+    std::vector<server_unfinished_download> get_all();
+
+private:
+    std::mutex mutex;
+    std::vector<server_unfinished_download> downloads;
+
+    // write the record back; callers hold the mutex
+    void save();
+
+    // delete the partials of `parts` no recorded download writes; callers hold the mutex
+    void delete_partials(const std::vector<common_download_partial> & parts);
+};
+
 struct server_models_routes;
 struct server_lru_sched; // defined in server-models.cpp
 struct server_monitor;   // defined in server-models.cpp
@@ -134,6 +182,9 @@ private:
     // models asked to stop, still counted as running until the monitor records their exit
     std::set<std::string> stopping_models;
 
+    // downloads that never finished, with the sidecar kinds they pull
+    server_unfinished_downloads unfinished_downloads;
+
     // set to true while load_models() is executing a reload; load() will wait until clear
     bool is_reloading = false;
 
@@ -252,6 +303,16 @@ public:
     // check if a model instance exists (thread-safe)
     bool has_model(const std::string & name);
 
+    // downloads that never finished, with their sidecar kinds (thread-safe)
+    std::vector<server_unfinished_download> unfinished_downloads_get();
+
+    // drop a download that will never be resumed, its record and its partial files;
+    // throws for a name that is not recorded (thread-safe)
+    void unfinished_downloads_remove(const std::string & name);
+
+    // remove the sidecar `kind` of a model that is not in use, its files and partials (thread-safe)
+    bool remove_sidecar(const std::string & name, const std::string & kind);
+
     // return a copy of model metadata (thread-safe)
     std::optional<server_model_meta> get_meta(const std::string & name);
 
@@ -262,6 +323,10 @@ public:
         server_child_mode mode = SERVER_CHILD_MODE_NORMAL;
         // used for spawning a downloading child process
         std::optional<server_model_meta> custom_meta = std::nullopt;
+        // download mode: the sidecar kinds pulled with the model, as -hf would, and
+        // the partial files the download plan writes
+        std::vector<std::string>             sidecars;
+        std::vector<common_download_partial> partials;
     };
 
     // load and unload model instances

tools/server/tests/unit/test_router.py

@@ -1,3 +1,4 @@
+import shutil
 import threading
 import pytest
 from utils import *
@@ -546,6 +547,15 @@ def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: i
     return False
 
 
+def _own_cache(name: str) -> str:
+    """An empty model cache for one test: the workers run in parallel, and a download
+    test must not see the files or the record another test writes in the shared cache."""
+    path = os.path.join(TMP_DIR, name)
+    shutil.rmtree(path, ignore_errors=True)
+    os.makedirs(path)
+    return path
+
+
 def test_router_download_model():
     """Case 1: download a model at the model limit, verify SSE events and GET /models."""
     global server
@@ -626,3 +636,230 @@ def test_router_delete_model():
     # Model should no longer appear in GET /models
     ids = _get_model_ids(is_reload=False)
     assert MODEL_DOWNLOAD_ID not in ids, f"{MODEL_DOWNLOAD_ID} still present after deletion"
+
+
+def test_router_download_sidecars():
+    """Case 3: sidecars are a parameter of the download, never models of their own."""
+    global server
+    server.llama_cache = _own_cache("cache_download_sidecars")
+    server.start()
+
+    server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")
+
+    # an unknown sidecar kind is rejected before anything starts
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": ["lora"]})
+    assert res.status_code == 400
+    assert MODEL_DOWNLOAD_ID not in _get_model_ids(is_reload=False)
+
+    sse_events: list = []
+    stop = threading.Event()
+    sse_ready = threading.Event()
+    threading.Thread(
+        target=_listen_sse, args=(server, sse_events, stop, sse_ready), daemon=True
+    ).start()
+    assert sse_ready.wait(10), "SSE client failed to connect"
+
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []})
+    assert res.status_code == 200
+    finished = _wait_for_sse_event(
+        sse_events, "download_finished", MODEL_DOWNLOAD_ID, MODEL_DOWNLOAD_TIMEOUT
+    )
+    stop.set()
+    assert finished, "Never received download_finished SSE event"
+
+    # the model lists the sidecars it has on disk, and no sidecar is listed as a model
+    res = server.make_request("GET", "/models")
+    assert res.status_code == 200
+    entries = {item["id"]: item for item in res.body.get("data", [])}
+    assert entries[MODEL_DOWNLOAD_ID]["sidecars"] == []
+    for kind in ("mmproj", "mtp", "dflash", "eagle3", "dspark"):
+        assert not any(i.endswith("-" + kind) for i in entries), f"a {kind} sidecar is listed as a model"
+
+    server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")
+
+
+def test_router_download_conflicts():
+    """A download is refused over one on its way, and over a model that is running."""
+    global server
+    server.models_max = 2
+    server.llama_cache = _own_cache("cache_download_conflicts")
+    server.start()
+
+    server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")
+
+    # a download on its way is not started again; the entry reports the state before
+    # the child fetches anything, so the probe lands inside the download
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID})
+    assert res.status_code == 200
+    _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloading"}, timeout=30)
+
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID})
+    assert res.status_code == 400
+    assert "is already downloading" in res.body["error"]["message"]
+
+    # a loaded model holds its files, so it goes only once it is unloaded; the
+    # reload that follows the download parks the entry as unloaded
+    _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT)
+    _load_model_and_wait(MODEL_DOWNLOAD_ID, timeout=120)
+
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID})
+    assert res.status_code == 400
+    assert "is running, unload it first" in res.body["error"]["message"]
+
+
+def test_router_delete_unknown_name():
+    """A name that is neither a model nor a recorded download is not found, and the
+    cached tags of its repo keep their files."""
+    global server
+    server.llama_cache = _own_cache("cache_delete_unknown_name")
+    server.start()
+
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []})
+    assert res.status_code == 200
+    _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT)
+
+    repo = MODEL_DOWNLOAD_ID.split(":")[0]
+    res = server.make_request("DELETE", f"/models?model={repo}")
+    assert res.status_code != 200
+    assert "is not found" in res.body["error"]["message"]
+    assert MODEL_DOWNLOAD_ID in _get_model_ids(is_reload=True)
+
+    server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")
+
+
+def test_router_incomplete_sidecars():
+    """A cached model lists the sidecars its unfinished download has not written yet, until
+    they are dropped."""
+    global server
+    cache_dir = _own_cache("cache_incomplete_sidecars")
+    server.llama_cache = cache_dir
+    server.start()
+
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []})
+    assert res.status_code == 200
+    _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT)
+    server.stop()
+
+    # a paused mtp of the cached model, as the record of an earlier run holds it
+    partial_path = os.path.abspath(os.path.join(cache_dir, "test-mtp.gguf.downloadInProgress"))
+    with open(partial_path, "w") as f:
+        f.write("partial")
+    with open(os.path.join(cache_dir, "unfinished-downloads.json"), "w") as f:
+        json.dump({"downloads": [{
+            "model":    MODEL_DOWNLOAD_ID,
+            "sidecars": ["mtp"],
+            "partials": [{"kind": "mtp", "path": partial_path}],
+        }]}, f)
+
+    server.start()
+
+    def entry() -> dict:
+        res = server.make_request("GET", "/models")
+        assert res.status_code == 200
+        return {item["id"]: item for item in res.body.get("data", [])}[MODEL_DOWNLOAD_ID]
+
+    assert entry()["sidecars"] == []
+    assert entry()["incomplete_sidecars"] == ["mtp"]
+
+    res = server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}&sidecar=mtp")
+    assert res.status_code == 200
+    assert res.body.get("success") is True
+    assert not os.path.exists(partial_path)
+    assert "incomplete_sidecars" not in entry()
+
+    server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")
+
+
+def test_router_preset_model_sidecars():
+    """A preset model reports the sidecars its files have on disk, like a cached one."""
+    global server
+
+    preset_path = os.path.join(TMP_DIR, "test_preset_sidecars.ini")
+
+    with open(preset_path, "w") as f:
+        f.write("[preset-sidecars]\nhf-repo = ggml-org/tinygemma3-GGUF:Q8_0\n")
+
+    server.models_preset = preset_path
+    server.start()
+
+    try:
+        # the mmproj of that repo lands next to the model, as -hf would pull it
+        res = server.make_request("POST", "/models", data={"model": MODEL_A, "sidecars": ["mmproj"]})
+        assert res.status_code == 200
+        _wait_for_model_status(MODEL_A, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT)
+
+        res = server.make_request("GET", "/models")
+        assert res.status_code == 200
+        entries = {item["id"]: item for item in res.body.get("data", [])}
+
+        assert entries["preset-sidecars"]["source"] == "preset"
+        assert entries["preset-sidecars"]["sidecars"] == ["mmproj"]
+    finally:
+        os.remove(preset_path)
+
+
+def test_router_local_preset_listing():
+    """A preset on a local file resolves from no HF repo: it lists without sidecars."""
+    global server
+
+    model_path = os.path.join(TMP_DIR, "test_local_preset.gguf")
+    preset_path = os.path.join(TMP_DIR, "test_local_preset.ini")
+
+    with open(model_path, "w") as f:
+        f.write("GGUF")
+    with open(preset_path, "w") as f:
+        f.write(f"[local]\nmodel = {model_path}\n")
+
+    server.models_preset = preset_path
+    server.start()
+
+    try:
+        res = server.make_request("GET", "/models")
+        assert res.status_code == 200
+        entries = {item["id"]: item for item in res.body.get("data", [])}
+
+        assert entries["local"]["source"] == "preset"
+        assert "sidecars" not in entries["local"]
+    finally:
+        os.remove(preset_path)
+        os.remove(model_path)
+
+
+def test_router_download_unresolved_tag():
+    """A tag that resolves no file of the repo is refused before anything starts."""
+    global server
+    server.start()
+
+    model = MODEL_DOWNLOAD_ID.split(":")[0] + ":Q9_Z"
+    res = server.make_request("POST", "/models", data={"model": model})
+    assert res.status_code == 400
+    assert "resolves no file to download" in res.body["error"]["message"]
+    assert model not in _get_model_ids(is_reload=False)
+
+
+def test_router_download_drops_stale_partials():
+    """Posting a recorded download again deletes the partials its new plan no longer writes."""
+    global server
+
+    cache_dir = _own_cache("cache_download_drops_stale_partials")
+    stale_path = os.path.abspath(os.path.join(cache_dir, "test-stale.gguf.downloadInProgress"))
+    record_path = os.path.join(cache_dir, "unfinished-downloads.json")
+
+    with open(stale_path, "w") as f:
+        f.write("stale")
+    with open(record_path, "w") as f:
+        json.dump({"downloads": [{
+            "model":    MODEL_DOWNLOAD_ID,
+            "sidecars": ["mtp"],
+            "partials": [{"kind": "mtp", "path": stale_path}],
+        }]}, f)
+
+    server.llama_cache = cache_dir
+    server.start()
+
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []})
+    assert res.status_code == 200
+    assert not os.path.exists(stale_path), "the stale partial survived the new record"
+
+    _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT)
+    server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")

tools/server/tests/utils.py

@@ -127,6 +127,7 @@ class ServerProcess:
     mcp_servers_config: str | None = None
     mcp_servers_json: str | None = None
     cors_origins: str | None = None
+    llama_cache: str | None = None
 
     # session variables
     process: subprocess.Popen | None = None
@@ -145,7 +146,9 @@ class ServerProcess:
             **os.environ,
             "LLAMA_SERVER_DEBUG_FAKE_TIMING": "1",
         }
-        if "LLAMA_CACHE" not in os.environ:
+        if self.llama_cache is not None:
+            env["LLAMA_CACHE"] = self.llama_cache
+        elif "LLAMA_CACHE" not in os.environ:
             env["LLAMA_CACHE"] = "tmp"
         if self.external_server:
             print(f"[external_server]: Assuming external server running on {self.server_host}:{self.server_port}")

tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte

@@ -7,6 +7,7 @@
 		getMdastNodeHash,
 		isAppendMode
 	} from './markdown-utils';
+	import { SAFE_HTML_CONFIG } from './safe-html-config';
 	import {
 		ActionIconCopyToClipboard,
 		CodeBlockActions,
@@ -44,6 +45,7 @@
 	import { detectIncompleteCodeBlock, highlightCode, type IncompleteCodeBlock } from '$lib/utils';
 	import { sanitizeSvg } from '$lib/utils/sanitize-svg';
 	import { mountSvgShadow } from '$lib/utils/svg-shadow';
+	import DOMPurify from 'dompurify';
 	import type { Root as HastRoot, RootContent as HastRootContent } from 'hast';
 	import githubLightCss from 'highlight.js/styles/github.css?inline';
 	import githubDarkCss from 'highlight.js/styles/github-dark.css?inline';
@@ -57,6 +59,8 @@
 		content: string;
 		class?: string;
 		disableMath?: boolean;
+		/** Render raw HTML found in the markdown (sanitized) instead of escaping it. */
+		allowHtml?: boolean;
 	}
 
 	interface MarkdownBlock {
@@ -65,7 +69,13 @@
 		contentHash?: string;
 	}
 
-	let { attachments, class: className = '', content, disableMath = false }: Props = $props();
+	let {
+		allowHtml = false,
+		attachments,
+		class: className = '',
+		content,
+		disableMath = false
+	}: Props = $props();
 
 	let containerRef = $state<HTMLDivElement>();
 	let renderedBlocks = $state<MarkdownBlock[]>([]);
@@ -121,6 +131,8 @@
 
 	// Per-instance transform cache, avoids re-transforming stable blocks during streaming
 	// Garbage collected when component is destroyed (on conversation change)
+	const CACHE_KEY_PLAIN = 'plain';
+	const CACHE_KEY_ALLOW_HTML = 'allow-html';
 	const transformCache = new SvelteMap<string, string>();
 	let previousContent = '';
 
@@ -183,7 +195,10 @@
 		index: number
 	): Promise<{ html: string; hash: string }> {
 		const hash = getMdastNodeHash(node, index);
-		const cached = transformCache.get(hash);
+		// the rendered HTML also depends on allowHtml (sanitized raw vs escaped),
+		// so the cache is keyed per mode
+		const cacheKey = `${allowHtml ? CACHE_KEY_ALLOW_HTML : CACHE_KEY_PLAIN}:${hash}`;
+		const cached = transformCache.get(cacheKey);
 
 		if (cached) {
 			return { hash, html: cached };
@@ -192,10 +207,13 @@
 		const singleNodeRoot = { children: [node], type: 'root' };
 		const transformedRoot = (await processorInstance.run(singleNodeRoot as MdastRoot)) as HastRoot;
 		const html = processorInstance.stringify(transformedRoot);
+		const safeHtml = allowHtml
+			? (DOMPurify.sanitize(html, SAFE_HTML_CONFIG) as unknown as string)
+			: html;
 
-		transformCache.set(hash, html);
+		transformCache.set(cacheKey, safeHtml);
 
-		return { hash, html };
+		return { hash, html: safeHtml };
 	}
 
 	/**
@@ -300,7 +318,7 @@
 
 			if (prefixMarkdown.trim()) {
 				const normalizedPrefix = preprocessLaTeX(prefixMarkdown);
-				const processorInstance = getMarkdownProcessor({ attachments, disableMath });
+				const processorInstance = getMarkdownProcessor({ allowHtml, attachments, disableMath });
 				const ast = processorInstance.parse(normalizedPrefix) as MdastRoot;
 				const mdastChildren = (ast as { children?: unknown[] }).children ?? [];
 				const nextBlocks: MarkdownBlock[] = [];
@@ -350,7 +368,7 @@
 		incompleteCodeBlock = null;
 
 		const normalized = preprocessLaTeX(markdown);
-		const processorInstance = getMarkdownProcessor({ attachments, disableMath });
+		const processorInstance = getMarkdownProcessor({ allowHtml, attachments, disableMath });
 		const ast = processorInstance.parse(normalized) as MdastRoot;
 		const mdastChildren = (ast as { children?: unknown[] }).children ?? [];
 		const stableCount = Math.max(mdastChildren.length - 1, 0);
@@ -394,6 +412,10 @@
 			)) as HastRoot;
 
 			unstableHtml = processorInstance.stringify(transformedRoot);
+
+			if (allowHtml) {
+				unstableHtml = DOMPurify.sanitize(unstableHtml, SAFE_HTML_CONFIG) as unknown as string;
+			}
 		}
 
 		renderedBlocks = nextBlocks;

tools/ui/src/lib/components/app/content/MarkdownContent/markdown-processor.ts

@@ -42,12 +42,15 @@ export interface MarkdownProcessor {
 export interface MarkdownProcessorOptions {
 	attachments?: DatabaseMessageExtra[];
 	disableMath?: boolean;
+	/** Render raw HTML found in the markdown instead of escaping it. */
+	allowHtml?: boolean;
 }
 
 const sharedPipelines = new Map<string, MarkdownProcessor>();
-const attachmentPipelines = new WeakMap<object, MarkdownProcessor>();
+const attachmentPipelines = new WeakMap<object, Map<string, MarkdownProcessor>>();
 
 function buildPipeline({
+	allowHtml = false,
 	attachments,
 	disableMath = false
 }: MarkdownProcessorOptions): MarkdownProcessor {
@@ -58,11 +61,14 @@ function buildPipeline({
 		proc = proc.use(remarkMath); // Parse $inline$ and $$block$$ math
 	}
 
-	proc = proc
-		.use(remarkBreaks) // Convert line breaks to <br>
+	proc = proc.use(remarkBreaks); // Convert line breaks to <br>
+
+	if (!allowHtml) {
 		// Treat raw HTML as literal text with preserved indentation
-		.use(remarkLiteralHtml)
-		.use(remarkRehype); // Convert Markdown AST to rehype
+		proc = proc.use(remarkLiteralHtml);
+	}
+
+	proc = proc.use(remarkRehype, allowHtml ? { allowDangerousHtml: true } : undefined);
 
 	if (!disableMath) {
 		proc = proc.use(rehypeKatex); // Render math using KaTeX
@@ -91,17 +97,26 @@ function buildPipeline({
 
 export function getMarkdownProcessor(options: MarkdownProcessorOptions): MarkdownProcessor {
 	if (options.attachments && options.attachments.length > 0) {
-		let cached = attachmentPipelines.get(options.attachments);
+		let byOptions = attachmentPipelines.get(options.attachments);
+
+		if (!byOptions) {
+			byOptions = new Map<string, MarkdownProcessor>();
+			attachmentPipelines.set(options.attachments, byOptions);
+		}
+
+		const key = `${Boolean(options.disableMath)}:${Boolean(options.allowHtml)}`;
+
+		let cached = byOptions.get(key);
 
 		if (!cached) {
 			cached = buildPipeline(options);
-			attachmentPipelines.set(options.attachments, cached);
+			byOptions.set(key, cached);
 		}
 
 		return cached;
 	}
 
-	const key = String(Boolean(options.disableMath));
+	const key = `${Boolean(options.disableMath)}:${Boolean(options.allowHtml)}`;
 
 	let cached = sharedPipelines.get(key);
 

tools/ui/src/lib/components/app/content/MarkdownContent/safe-html-config.ts

@@ -0,0 +1,138 @@
+// Sanitization rules for raw HTML in model cards (MarkdownContent's
+// allowHtml mode). Kept as its own module so the adversarial test in
+// tests/client can import the exact shipped configuration.
+import DOMPurify from 'dompurify';
+
+// style is allowed for the KaTeX output, which lays its spans out with inline
+// styles; a third party card can use the same attribute to draw a fixed
+// overlay over the app, so positioned values are scrubbed
+DOMPurify.addHook('afterSanitizeAttributes', (node) => {
+	if (typeof (node as Element).getAttribute !== 'function') return;
+
+	const style = (node as Element).getAttribute('style');
+
+	if (!style || !/position\s*:\s*(fixed|absolute)/i.test(style)) return;
+
+	(node as Element).setAttribute(
+		'style',
+		style.replace(/position\s*:\s*(fixed|absolute)\s*;?/gi, '')
+	);
+});
+
+/**
+ * Explicit DOMPurify allow list: the tags and attributes model cards may use
+ * plus the KaTeX output (spans with inline styles and MathML). Everything
+ * else - scripts, event handlers, forms, iframes, style tags - is stripped.
+ */
+export const SAFE_HTML_CONFIG = {
+	ALLOWED_ATTR: [
+		'class',
+		'style',
+		'id',
+		'title',
+		'role',
+		'aria-hidden',
+		'aria-label',
+		'aria-describedby',
+		'aria-expanded',
+		'href',
+		'target',
+		'rel',
+		'src',
+		'srcset',
+		'alt',
+		'width',
+		'height',
+		'align',
+		'valign',
+		'colspan',
+		'rowspan',
+		'controls',
+		'type',
+		// MathML
+		'xmlns',
+		'display',
+		'encoding',
+		'mathvariant'
+	],
+	ALLOWED_TAGS: [
+		// text and structure
+		'a',
+		'b',
+		'i',
+		'u',
+		's',
+		'em',
+		'strong',
+		'code',
+		'pre',
+		'kbd',
+		'sub',
+		'sup',
+		'br',
+		'hr',
+		'p',
+		'span',
+		'div',
+		'blockquote',
+		'center',
+		'ul',
+		'ol',
+		'li',
+		'dl',
+		'dt',
+		'dd',
+		'h1',
+		'h2',
+		'h3',
+		'h4',
+		'h5',
+		'h6',
+		'table',
+		'thead',
+		'tbody',
+		'tr',
+		'th',
+		'td',
+		'colgroup',
+		'col',
+		'caption',
+		// media
+		'img',
+		'picture',
+		'source',
+		'figure',
+		'figcaption',
+		'video',
+		'audio',
+		// MathML, for the KaTeX output
+		'math',
+		'mrow',
+		'mi',
+		'mo',
+		'mn',
+		'ms',
+		'mtext',
+		'mfrac',
+		'mroot',
+		'msqrt',
+		'msub',
+		'msup',
+		'msubsup',
+		'munder',
+		'mover',
+		'mmultiscripts',
+		'mtable',
+		'mtr',
+		'mtd',
+		'mspace',
+		'mglyph',
+		'maligngroup',
+		'malignmark',
+		'mpadded',
+		'mphantom',
+		'mstyle',
+		'semantics',
+		'annotation'
+	]
+};

tools/ui/src/lib/components/app/dialogs/DialogManageModels.svelte

@@ -1,7 +1,18 @@
 <script lang="ts">
+	import { ArrowLeft } from '@lucide/svelte';
+	import ModelsDiscover from '$lib/components/app/models/discover/ModelsDiscover.svelte';
 	import ModelsManager from '$lib/components/app/models/ModelsManager/ModelsManager.svelte';
+	import { Button } from '$lib/components/ui/button';
 	import * as Dialog from '$lib/components/ui/dialog';
-	import { MODEL_ICON } from '$lib/constants';
+	import {
+		DISCOVER_ICON,
+		MODEL_ICON,
+		MODELS_DIALOG_VIEW_FADE_MS,
+		SETTINGS_KEYS
+	} from '$lib/constants';
+	import { ModelsDialogView } from '$lib/enums';
+	import { modelsStore, settingsStore, uiStore } from '$lib/stores';
+	import { untrack } from 'svelte';
 
 	interface Props {
 		open?: boolean;
@@ -10,6 +21,85 @@
 
 	let { onOpenChange, open = $bindable(false) }: Props = $props();
 
+	// what the user asked for, and what is actually rendered: a view change fades the
+	// current one out, swaps, then fades the next one in
+	let view = $state<ModelsDialogView>(ModelsDialogView.MANAGE);
+	let shownView = $state<ModelsDialogView>(ModelsDialogView.MANAGE);
+	let isSwapping = $state(false);
+	/** Set while an entry opens the dialog on an explicit view, so the open reset keeps hands off. */
+	let viewRequested = false;
+
+	// the discover view exists only while its setting is on
+	let discoverEnabled = $derived(
+		Boolean(settingsStore.getConfig(SETTINGS_KEYS.ENABLE_DISCOVER_MODELS))
+	);
+
+	$effect(() => {
+		const next = view;
+
+		if (next === shownView) return;
+
+		// nothing to fade while the dialog is closed, so the view is set before it opens
+		if (!open) {
+			untrack(() => (shownView = next));
+
+			return;
+		}
+
+		untrack(() => (isSwapping = true));
+
+		const timer = setTimeout(() => {
+			untrack(() => {
+				shownView = next;
+				isSwapping = false;
+			});
+		}, MODELS_DIALOG_VIEW_FADE_MS);
+
+		return () => clearTimeout(timer);
+	});
+
+	let title = $derived(view === ModelsDialogView.DISCOVER ? 'Discover' : 'Models');
+
+	// the sidebar's Discover entry opens this dialog on its Discover view
+	$effect(() => {
+		if (!uiStore.discoverModelsOpen) return;
+
+		untrack(() => {
+			uiStore.discoverModelsOpen = false;
+			view = discoverEnabled ? ModelsDialogView.DISCOVER : ModelsDialogView.MANAGE;
+
+			// the dialog may already be open, so no open transition will follow
+			if (open) return;
+
+			viewRequested = true;
+			handleOpenChange(true);
+		});
+	});
+
+	// opening always lands on the manager, unless an entry requested the discover view
+	$effect(() => {
+		if (!open) return;
+
+		untrack(() => {
+			if (viewRequested) {
+				viewRequested = false;
+			} else {
+				view = ModelsDialogView.MANAGE;
+			}
+
+			// sync point: the table opens on cached state and corrects itself when
+			// the list fetch lands; a no-op outside router mode
+			void modelsStore.fetchRouterModels();
+		});
+	});
+
+	// the discover view disappears with its setting; fall back to the manager
+	$effect(() => {
+		if (!discoverEnabled && view === ModelsDialogView.DISCOVER) {
+			untrack(() => (view = ModelsDialogView.MANAGE));
+		}
+	});
+
 	function handleOpenChange(value: boolean) {
 		open = value;
 		onOpenChange?.(value);
@@ -21,11 +111,49 @@
 		class="max-md:h-[100dvh]! max-md:w-screen! max-md:max-w-none! max-md:rounded-none! max-md:border-0! md:h-[calc(100vh-4rem)]! md:max-h-240! md:w-[calc(100vw-4rem)]! md:max-w-380! flex flex-col p-4 pb-0"
 		onOpenAutoFocus={(event) => event.preventDefault()}
 	>
-		<Dialog.Header class="flex flex-row items-center pr-8">
-			<Dialog.Title class="flex min-h-7 items-center gap-2">
-				<MODEL_ICON class="h-5 w-5 shrink-0" />
+		<Dialog.Header class="relative flex flex-row items-center justify-between p-2 pr-8">
+			<!--
+				The arrow is out of the flow and the title slides, so entering a sub-view
+				moves the title with a transform instead of reflowing the header. min-h
+				keeps the header's height fixed either way.
+			-->
+			<Button
+				aria-label="Back to models"
+				class="absolute top-1/2 left-1 h-7 w-7 -translate-y-1/2 transition-[opacity,visibility] duration-200 ease-[cubic-bezier(0.23,1,0.32,1)] {view ===
+				ModelsDialogView.MANAGE
+					? 'invisible opacity-0'
+					: 'visible opacity-100'}"
+				onclick={() => (view = ModelsDialogView.MANAGE)}
+				size="icon"
+				variant="ghost"
+			>
+				<ArrowLeft class="h-4 w-4" />
+			</Button>
 
-				<span>Models</span>
+			<Dialog.Title
+				class="flex min-h-7 items-center gap-2 transition-transform duration-200 ease-[cubic-bezier(0.23,1,0.32,1)] {view ===
+				ModelsDialogView.MANAGE
+					? 'translate-x-0'
+					: 'translate-x-8'}"
+			>
+				<!-- the marks cross-fade in one grid cell, so the title never shifts -->
+				<span class="grid h-5 w-5 shrink-0 place-items-center">
+					<MODEL_ICON
+						class="col-start-1 row-start-1 h-5 w-5 transition-opacity duration-150 {view ===
+						ModelsDialogView.MANAGE
+							? 'opacity-100'
+							: 'opacity-0'}"
+					/>
+
+					<DISCOVER_ICON
+						class="col-start-1 row-start-1 h-5 w-5 transition-opacity duration-150 {view ===
+						ModelsDialogView.DISCOVER
+							? 'opacity-100'
+							: 'opacity-0'}"
+					/>
+				</span>
+
+				<span>{title}</span>
 			</Dialog.Title>
 
 			<Dialog.Description class="sr-only">
@@ -33,8 +161,47 @@
 			</Dialog.Description>
 		</Dialog.Header>
 
-		<div class="min-h-0 flex-1">
-			<ModelsManager class="h-full" />
+		<div class="dialog-view min-h-0 flex-1" data-visible={!isSwapping}>
+			{#if shownView === ModelsDialogView.MANAGE}
+				<ModelsManager class="h-full">
+					{#snippet toolbarEnd()}
+						{#if discoverEnabled}
+							<Button
+								class="gap-1.5"
+								onclick={() => (view = ModelsDialogView.DISCOVER)}
+								size="sm"
+								variant="tertiary"
+							>
+								<DISCOVER_ICON class="h-3.5 w-3.5" />
+
+								Discover Models
+							</Button>
+						{/if}
+					{/snippet}
+				</ModelsManager>
+			{:else if shownView === ModelsDialogView.DISCOVER}
+				<div class="grid h-full overflow-hidden" style="grid-template-columns: auto 1fr;">
+					<ModelsDiscover />
+				</div>
+			{/if}
 		</div>
 	</Dialog.Content>
 </Dialog.Root>
+
+<style>
+	/* a view change reads as one surface swapping, not as content teleporting */
+	.dialog-view {
+		opacity: 0;
+		transition: opacity 150ms cubic-bezier(0.23, 1, 0.32, 1);
+	}
+
+	.dialog-view[data-visible='true'] {
+		opacity: 1;
+	}
+
+	@media (prefers-reduced-motion: reduce) {
+		.dialog-view {
+			transition-duration: 100ms;
+		}
+	}
+</style>

tools/ui/src/lib/components/app/models/ModelDraftSidecars.svelte

@@ -31,8 +31,6 @@
 </script>
 
 {#each badges as badge (badge.kind + (badge.quant ?? ''))}
-	<span aria-hidden="true" class="text-muted-foreground">+</span>
-
 	<span
 		class={MODEL_VARIANT_BADGE_CLASS}
 		title="{badge.kind.toUpperCase()} draft model{badge.quant ? ` at ${badge.quant}` : ''}"

tools/ui/src/lib/components/app/models/ModelDraftSubline.svelte

@@ -0,0 +1,53 @@
+<script lang="ts">
+	import { Check } from '@lucide/svelte';
+	import { MODEL_BADGE_CLASS, MODEL_VARIANT_BADGE_CLASS } from '$lib/constants';
+	import { ModelRowDownloadState } from '$lib/enums';
+
+	interface Props {
+		/**
+		 * Left inset of the tree connector, tuned per surface so the elbow hangs
+		 * under the main line's avatar and "Draft model" aligns with its id.
+		 */
+		connectorClass?: string;
+		/** Draft sidecar kind, e.g. dflash. */
+		kind: string;
+		/** Quant of the draft, when its tag carries one. */
+		quant?: string | null;
+		/** Live state of the draft download. */
+		state: ModelRowDownloadState;
+		/** Rounded download percent, when the feed reports progress. */
+		percent?: number | null;
+	}
+
+	let { connectorClass = 'ml-6', kind, percent = null, quant = null, state }: Props = $props();
+</script>
+
+<!-- the compact draft line lives inside its main item, under the id line; the
+     main model's controls pause, resume and cancel its quant-matched drafts, so
+     the line only reports its state -->
+<div class="flex items-center gap-2 py-1 pr-2 text-sm">
+	<span
+		aria-hidden="true"
+		class="h-2.5 w-3.5 shrink-0 self-center rounded-bl-sm border-b border-l border-border/70 {connectorClass}"
+	></span>
+
+	<span class="shrink-0 text-xs text-muted-foreground">Draft model</span>
+
+	<span class={MODEL_VARIANT_BADGE_CLASS}>{kind}</span>
+
+	{#if quant}
+		<span class={MODEL_BADGE_CLASS}>{quant}</span>
+	{/if}
+
+	{#if state === ModelRowDownloadState.DOWNLOADED}
+		<Check class="ml-auto h-4 w-4 shrink-0 text-green-500" />
+	{:else}
+		<span class="ml-auto flex shrink-0 items-center">
+			{#if percent !== null}
+				<span class="font-mono text-xs tabular-nums text-muted-foreground">{percent}%</span>
+			{:else if state === ModelRowDownloadState.PAUSED}
+				<span class="text-xs text-muted-foreground">Paused</span>
+			{/if}
+		</span>
+	{/if}
+</div>

tools/ui/src/lib/components/app/models/ModelId.svelte

@@ -5,6 +5,7 @@
 	import { TruncatedText } from '$lib/components/app';
 	import * as Tooltip from '$lib/components/ui/tooltip';
 	import { MODEL_BADGE_CLASS, MODEL_VARIANT_BADGE_CLASS } from '$lib/constants';
+	import type { ModelDraftSidecar } from '$lib/enums';
 	import { HuggingFaceService } from '$lib/services';
 	import { ModelsService } from '$lib/services/models.service';
 	import { settingsStore } from '$lib/stores';
@@ -39,6 +40,8 @@
 		sizeRange?: { min: number; max: number } | null;
 		/** Draft sidecars available for the model, badged with their own quant. */
 		draftSidecars?: ModelSidecarBadge[];
+		/** Draft sidecar kinds the repo offers, one badge per kind and no quant. */
+		draftKinds?: ModelDraftSidecar[];
 		/** Give the id a row of its own, with its badges and icons together under it. */
 		stackId?: boolean;
 		/** Allow badges to wrap onto new lines instead of truncating. */
@@ -50,6 +53,7 @@
 		aliases,
 		class: className = '',
 		contextLength,
+		draftKinds = [],
 		draftSidecars = [],
 		hideCapabilities = false,
 		hideModalities = false,
@@ -93,12 +97,17 @@
 	let uniqueAliases = $derived([...new Set(aliases ?? [])]);
 	let uniqueTags = $derived([...new Set([...(parsed.tags ?? []), ...(tags ?? [])])]);
 	let uniqueDraftSidecars = $derived(draftSidecars.filter((badge) => !isAuxSidecar(badge.kind)));
+	// the model id can already name one of the kinds
+	let uniqueDraftKinds = $derived([
+		...new Set(draftKinds.filter((kind) => kind !== parsed.sidecar))
+	]);
 
 	let primaryAlias = $derived(uniqueAliases.length === 1 ? uniqueAliases[0] : null);
 	let displayName = $derived(primaryAlias ?? parsed.modelName ?? modelId);
 
 	let hasBadges = $derived(
 		parsed.sidecar ||
+			uniqueDraftKinds.length > 0 ||
 			uniqueDraftSidecars.length > 0 ||
 			(parsed.params && !hideParameters) ||
 			(parsed.quantization && !resolvedHideQuantization) ||
@@ -144,6 +153,12 @@
 					</span>
 				{/if}
 
+				{#each uniqueDraftKinds as kind (kind)}
+					<span class={variantBadgeClass} title="{kind.toUpperCase()} draft model">
+						{kind}
+					</span>
+				{/each}
+
 				<ModelDraftSidecars {draftSidecars} />
 
 				{#if primaryAlias}

tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte

@@ -32,9 +32,22 @@
 	let capabilityFilter = $state<ModelCapability[]>([]);
 	let selectedId = $state<string | null>(null);
 
+	// tracked downloads get their own section, listed like any other model
+	let downloads = $derived(
+		downloadGroups(modelsStore.status.getDownloadEntries(), modelsStore.models, (repo) =>
+			modelsStore.status.getDraftSidecars(repo)
+		)
+	);
+
 	let allModels = $derived(modelsStore.models);
 
-	let selected = $derived(allModels.find((option) => option.id === selectedId) ?? null);
+	// the download groups stand in for rows the listing does not carry yet, so a
+	// click on them still finds its option and opens the pane
+	let selected = $derived(
+		allModels.find((option) => option.id === selectedId) ??
+			downloads.find((group) => group.base.id === selectedId)?.base ??
+			null
+	);
 	// The pane is laid out before it is ever opened, so the first open only slides a
 	// finished panel in. It renders the selection, else the model it last showed.
 	let lastPicked = $state<ModelOption | null>(null);
@@ -42,7 +55,11 @@
 	let shownId = $state<string | null>(null);
 	let isSwapping = $state(false);
 	let fade = $state<'open' | 'swap'>('open');
-	let shownOption = $derived(allModels.find((option) => option.id === shownId) ?? null);
+	let shownOption = $derived(
+		allModels.find((option) => option.id === shownId) ??
+			downloads.find((group) => group.base.id === shownId)?.base ??
+			null
+	);
 
 	$effect(() => {
 		const id = selectedId;
@@ -144,10 +161,6 @@
 		rank.size === 0 ? list : [...list].sort((a, b) => rankOf(a) - rankOf(b));
 	// one entry per repo, so a model with several quants takes a single table row
 	let entries = $derived(byRecency(groupModelQuants(matching)));
-	// tracked downloads get their own section, listed like any other model
-	let downloads = $derived(
-		downloadGroups(modelsStore.status.getDownloadEntries(), modelsStore.models)
-	);
 	let groups = $derived.by(() => {
 		// a loaded quant is a model of its own: its repo keeps the quants left behind
 		const isLoaded = (option: ModelOption) => modelsStore.isModelLoaded(option.model);
@@ -216,13 +229,16 @@
 		return ordered;
 	});
 
-	// a caller can ask for one model to be revealed, the download rows do
+	// a caller can ask for one model to be revealed, the download rows do; a
+	// tracked download is not in the listing yet, so its own rows are searched too
 	$effect(() => {
 		const focus = uiStore.manageModelFocus;
 
 		if (!focus) return;
 
-		const option = allModels.find((model) => model.id === focus || model.model === focus);
+		const matches = (candidate: ModelOption) => candidate.id === focus || candidate.model === focus;
+		const option =
+			allModels.find(matches) ?? downloads.find((group) => matches(group.base))?.base ?? null;
 
 		if (option) selectedId = option.id;
 

tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerDownloadControl.svelte

@@ -38,10 +38,7 @@
 			class="h-5 w-5 hover:text-foreground"
 			icon={isPaused ? MODEL_DOWNLOAD_ICONS.resume : MODEL_DOWNLOAD_ICONS.pause}
 			iconSize="h-4 w-4"
-			onclick={() =>
-				void (isPaused
-					? modelsStore.status.downloadModel(option.model)
-					: modelsStore.status.pauseDownload(option.model))}
+			onclick={() => void modelsStore.status.toggleDownload(option.model)}
 			stopPropagationOnClick
 			tooltip={isPaused ? 'Resume downloading' : 'Pause downloading'}
 			tooltipAsTitle

tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte

@@ -1,4 +1,6 @@
 <script lang="ts">
+	import ModelDownloadProgressBar from '../../ModelDownloadProgressBar.svelte';
+	import { downloadProgressFor } from '../utils';
 	import ModelsManagerModelConfigurationActions from './ModelsManagerModelConfigurationActions.svelte';
 	import ModelsManagerModelConfigurationHeader from './ModelsManagerModelConfigurationHeader.svelte';
 	import ModelsManagerModelConfigurationInformation from './ModelsManagerModelConfigurationInformation.svelte';
@@ -7,7 +9,7 @@
 	import { Button } from '$lib/components/ui/button';
 	import { ModelDownloadConfirmAction, ServerModelStatus } from '$lib/enums';
 	import { HuggingFaceService } from '$lib/services';
-	import { deviceStore, modelsStore } from '$lib/stores';
+	import { deviceStore, modelsStore, uiStore } from '$lib/stores';
 	import type { HfModelDetailInfo } from '$lib/types/huggingface';
 	import type { ModelOption } from '$lib/types/models';
 	import { repoOf } from '$lib/utils';
@@ -34,6 +36,23 @@
 	let status = $derived(modelsStore.getModelStatus(option.model));
 	let isLoaded = $derived(modelsStore.isModelRunning(option.model));
 
+	// an in-flight download has no configuration: the pane shows its progress and
+	// the download actions, everything a loaded or downloaded model offers waits
+	let isDownload = $derived(
+		modelsStore.status.isDownloadInProgress(option.model) ||
+			modelsStore.status.isDownloadPaused(option.model)
+	);
+	let isDownloadPaused = $derived(modelsStore.status.isDownloadPaused(option.model));
+	let downloadProgress = $derived(downloadProgressFor(option.model));
+	let downloadPercent = $derived(
+		downloadProgress && downloadProgress.totalBytes > 0
+			? Math.round((downloadProgress.downloadedBytes / downloadProgress.totalBytes) * 100)
+			: null
+	);
+
+	// the same cancellation the table's rows confirm, owned by this pane
+	let cancelDownloadOpen = $state(false);
+
 	// The server reports the full metadata only once a model is loaded, which loads it.
 	// With the Hub enabled, it fills those gaps instead.
 	let hubDetails = $state<HfModelDetailInfo | null>(null);
@@ -63,8 +82,15 @@
 
 <div class="flex h-full min-h-0 flex-col">
 	<ModelsManagerModelConfigurationHeader
+		{isDownload}
+		{isDownloadPaused}
 		{isLoaded}
 		{onClose}
+		onRequestCancelDownload={() => (cancelDownloadOpen = true)}
+		onSeeInDiscover={isDownload
+			? () => uiStore.openModelsDiscover(repoOf(option.model))
+			: undefined}
+		onToggleDownload={() => void modelsStore.status.toggleDownload(option.model)}
 		{onToggleLoad}
 		{onUseInChat}
 		{onUseInNewChat}
@@ -73,27 +99,63 @@
 	/>
 
 	<div class="min-h-0 flex-1 overflow-y-auto py-4 pl-4 max-md:pr-4">
-		<ModelsManagerModelConfigurationInformation hub={hubDetails} {option} {serverProps} />
+		{#if isDownload}
+			<!-- nothing is on disk yet, so the pane keeps to the download itself -->
+			<div class="space-y-2 py-1">
+				<div class="flex items-baseline justify-between gap-3 text-sm">
+					<span class="text-muted-foreground">
+						{isDownloadPaused ? 'Paused at' : 'Downloaded'}
+					</span>
 
-		<div class="mt-4 border-t border-border/30 pt-3 pb-2">
-			<Button
-				class="w-full justify-start gap-2 text-destructive hover:bg-destructive/10 hover:text-destructive dark:hover:bg-destructive/20"
-				onclick={() => (deleteOpen = true)}
-				size="sm"
-				variant="ghost"
-			>
-				<Trash2 class="h-4 w-4" />
-				Delete this model from disk
-			</Button>
-		</div>
+					<span class="font-mono text-xs tabular-nums text-muted-foreground">
+						{#if downloadPercent !== null}
+							{HuggingFaceService.formatFileSize(downloadProgress?.downloadedBytes ?? 0)}
+							/
+							{HuggingFaceService.formatFileSize(downloadProgress?.totalBytes ?? 0)}
+							({downloadPercent}%)
+						{:else}
+							Waiting for progress...
+						{/if}
+					</span>
+				</div>
+
+				{#if downloadProgress && downloadProgress.totalBytes > 0}
+					<ModelDownloadProgressBar
+						downloadedBytes={downloadProgress.downloadedBytes}
+						totalBytes={downloadProgress.totalBytes}
+					/>
+				{/if}
+			</div>
+		{:else}
+			<ModelsManagerModelConfigurationInformation hub={hubDetails} {option} {serverProps} />
+
+			<div class="mt-4 border-t border-border/30 pt-3 pb-2">
+				<Button
+					class="w-full justify-start gap-2 text-destructive hover:bg-destructive/10 hover:text-destructive dark:hover:bg-destructive/20"
+					onclick={() => (deleteOpen = true)}
+					size="sm"
+					variant="ghost"
+				>
+					<Trash2 class="h-4 w-4" />
+					Delete this model from disk
+				</Button>
+			</div>
+		{/if}
 	</div>
 
 	<!-- a phone keeps the actions at the bottom of its screen, in reach of the thumb -->
 	{#if deviceStore.isMobile}
 		<div class="shrink-0 border-t border-border/40 px-4 py-3">
 			<ModelsManagerModelConfigurationActions
+				{isDownload}
+				{isDownloadPaused}
 				{isLoaded}
 				isLoading={status === ServerModelStatus.LOADING}
+				onRequestCancelDownload={() => (cancelDownloadOpen = true)}
+				onSeeInDiscover={isDownload
+					? () => uiStore.openModelsDiscover(repoOf(option.model))
+					: undefined}
+				onToggleDownload={() => void modelsStore.status.toggleDownload(option.model)}
 				{onToggleLoad}
 				{onUseInChat}
 				{onUseInNewChat}
@@ -107,4 +169,12 @@
 		open={deleteOpen}
 		repoWithTag={option.model}
 	/>
+
+	<DialogConfirmDownload
+		action={ModelDownloadConfirmAction.CANCEL}
+		onClose={() => (cancelDownloadOpen = false)}
+		onConfirm={() => void modelsStore.status.cancelDownload(option.model)}
+		open={cancelDownloadOpen}
+		repoWithTag={option.model}
+	/>
 </div>

tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte

@@ -1,13 +1,31 @@
 <script lang="ts">
-	import { Eject, Loader2, MessageSquare, Power, SquarePen } from '@lucide/svelte';
+	import {
+		Compass,
+		Eject,
+		Loader2,
+		MessageSquare,
+		Pause,
+		Play,
+		Power,
+		SquarePen,
+		Trash2
+	} from '@lucide/svelte';
 	import { Button } from '$lib/components/ui/button';
-	import { conversationsStore, deviceStore } from '$lib/stores';
+	import { SETTINGS_KEYS } from '$lib/constants';
+	import { conversationsStore, deviceStore, settingsStore } from '$lib/stores';
 
 	interface Props {
 		/** A provider that cannot load or unload has no button to offer. */
 		canToggleLoad?: boolean;
+		isDownload?: boolean;
+		isDownloadPaused?: boolean;
 		isLoaded: boolean;
 		isLoading: boolean;
+		/** Opens the manager dialog on the Discover view, focused on this model's repo. */
+		onSeeInDiscover?: () => void;
+		/** Confirm-cancel dialog on the pane, so nothing deletes without a click. */
+		onRequestCancelDownload?: () => void;
+		onToggleDownload?: () => void;
 		onToggleLoad: () => void;
 		onUseInChat: () => void;
 		onUseInNewChat: () => void;
@@ -17,14 +35,24 @@
 
 	let {
 		canToggleLoad = true,
+		isDownload = false,
+		isDownloadPaused = false,
 		isLoaded,
 		isLoading,
+		onRequestCancelDownload,
+		onSeeInDiscover,
+		onToggleDownload,
 		onToggleLoad,
 		onUseInChat,
 		onUseInNewChat,
 		size = 'sm'
 	}: Props = $props();
 
+	// the discover view exists only while its setting is on
+	let discoverEnabled = $derived(
+		Boolean(settingsStore.getConfig(SETTINGS_KEYS.ENABLE_DISCOVER_MODELS))
+	);
+
 	// a chat with anything in it takes the model, a new chat is not what its user asked for
 	let hasChat = $derived(
 		conversationsStore.activeConversation !== null ||
@@ -34,52 +62,91 @@
 </script>
 
 <div class="my-3 flex flex-col gap-2">
-	{#if isMobile}
-		<!-- a phone picks its model in the manager, so pointing this chat at it leads -->
-		<Button class="w-full gap-1.5" onclick={hasChat ? onUseInChat : onUseInNewChat} {size}>
-			{#if hasChat}
-				<MessageSquare class="h-3.5 w-3.5" />
+	{#if isDownload}
+		<!-- an in-flight download cannot load or chat: it pauses, resumes, cancels -->
+		<div class="flex gap-3">
+			<Button class="flex-1 gap-1.5" onclick={onToggleDownload} {size} variant="secondary">
+				{#if isDownloadPaused}
+					<Play class="h-3.5 w-3.5" />
 
-				Use in this chat
-			{:else}
-				<SquarePen class="h-3.5 w-3.5" />
+					Resume downloading
+				{:else}
+					<Pause class="h-3.5 w-3.5" />
 
-				Start a new chat
-			{/if}
-		</Button>
-	{/if}
+					Pause downloading
+				{/if}
+			</Button>
 
-	<div class="flex gap-3 mt-2">
-		{#if !isMobile}
-			<Button class="flex-1 gap-1.5" onclick={onUseInNewChat} {size}>
-				<SquarePen class="h-3.5 w-3.5" />
+			<Button class="flex-1 gap-1.5" onclick={onRequestCancelDownload} {size} variant="secondary">
+				<Trash2 class="h-3.5 w-3.5" />
 
-				Start a new chat
+				Cancel download
+			</Button>
+		</div>
+
+		{#if onSeeInDiscover && discoverEnabled}
+			<Button class="gap-1.5" onclick={onSeeInDiscover} {size} variant="tertiary">
+				<Compass class="h-3.5 w-3.5" />
+
+				See in Discover
 			</Button>
 		{/if}
+	{:else}
+		{#if isMobile}
+			<!-- a phone picks its model in the manager, so pointing this chat at it leads -->
+			<Button class="w-full gap-1.5" onclick={hasChat ? onUseInChat : onUseInNewChat} {size}>
+				{#if hasChat}
+					<MessageSquare class="h-3.5 w-3.5" />
 
-		{#if canToggleLoad}
-			<Button
-				class="flex-1 gap-1.5"
-				disabled={isLoading}
-				onclick={onToggleLoad}
-				{size}
-				variant="secondary"
-			>
-				{#if isLoading}
-					<Loader2 class="h-3.5 w-3.5 animate-spin" />
-
-					Loading...
-				{:else if isLoaded}
-					<Eject class="h-3.5 w-3.5" />
-
-					Unload model
+					Use in this chat
 				{:else}
-					<Power class="h-3.5 w-3.5" />
+					<SquarePen class="h-3.5 w-3.5" />
 
-					Load model
+					Start a new chat
 				{/if}
 			</Button>
 		{/if}
-	</div>
+
+		<div class="flex gap-3 mt-2">
+			{#if !isMobile}
+				<Button class="flex-1 gap-1.5" onclick={onUseInNewChat} {size}>
+					<SquarePen class="h-3.5 w-3.5" />
+
+					Start a new chat
+				</Button>
+			{/if}
+
+			{#if canToggleLoad}
+				<Button
+					class="flex-1 gap-1.5"
+					disabled={isLoading}
+					onclick={onToggleLoad}
+					{size}
+					variant="secondary"
+				>
+					{#if isLoading}
+						<Loader2 class="h-3.5 w-3.5 animate-spin" />
+
+						Loading...
+					{:else if isLoaded}
+						<Eject class="h-3.5 w-3.5" />
+
+						Unload model
+					{:else}
+						<Power class="h-3.5 w-3.5" />
+
+						Load model
+					{/if}
+				</Button>
+			{/if}
+
+			{#if onSeeInDiscover && discoverEnabled}
+				<Button class="flex-1 gap-1.5" onclick={onSeeInDiscover} {size} variant="tertiary">
+					<Compass class="h-3.5 w-3.5" />
+
+					See in Discover
+				</Button>
+			{/if}
+		</div>
+	{/if}
 </div>

tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationHeader.svelte

@@ -8,8 +8,14 @@
 	import type { ModelOption } from '$lib/types/models';
 
 	interface Props {
+		isDownload?: boolean;
+		isDownloadPaused?: boolean;
 		isLoaded: boolean;
 		onClose: () => void;
+		/** Opens the manager dialog on the Discover view, focused on this model's repo. */
+		onSeeInDiscover?: () => void;
+		onRequestCancelDownload?: () => void;
+		onToggleDownload?: () => void;
 		onToggleLoad: () => void;
 		onUseInChat: () => void;
 		onUseInNewChat: () => void;
@@ -18,13 +24,27 @@
 		status: ServerModelStatus | null;
 	}
 
-	let { isLoaded, onClose, onToggleLoad, onUseInChat, onUseInNewChat, option, status }: Props =
-		$props();
+	let {
+		isDownload = false,
+		isDownloadPaused = false,
+		isLoaded,
+		onClose,
+		onRequestCancelDownload,
+		onSeeInDiscover,
+		onToggleDownload,
+		onToggleLoad,
+		onUseInChat,
+		onUseInNewChat,
+		option,
+		status
+	}: Props = $props();
 
 	let isMobile = $derived(deviceStore.isMobile);
 	let isLoading = $derived(status === ServerModelStatus.LOADING);
 
 	let statusLabel = $derived.by(() => {
+		if (isDownload) return isDownloadPaused ? 'Paused' : 'Downloading';
+
 		if (status === ServerModelStatus.LOADING) return 'Loading';
 
 		if (status === ServerModelStatus.FAILED) return 'Failed to load';
@@ -37,6 +57,8 @@
 	});
 
 	let statusDot = $derived.by(() => {
+		if (isDownload) return isDownloadPaused ? 'bg-yellow-500' : 'bg-blue-500 animate-pulse';
+
 		if (status === ServerModelStatus.FAILED) return 'bg-red-500';
 
 		if (status === ServerModelStatus.LOADING) return 'bg-muted-foreground/50 animate-pulse';
@@ -100,8 +122,13 @@
 	<!-- a phone floats the actions at the bottom of the pane, where the thumb is -->
 	{#if !isMobile}
 		<ModelsManagerModelConfigurationActions
+			{isDownload}
+			{isDownloadPaused}
 			{isLoaded}
 			{isLoading}
+			{onRequestCancelDownload}
+			{onSeeInDiscover}
+			{onToggleDownload}
 			{onToggleLoad}
 			{onUseInChat}
 			{onUseInNewChat}

tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte

@@ -3,6 +3,7 @@
 	import ModelCapabilities from '../ModelCapabilities.svelte';
 	import ModelContext from '../ModelContext.svelte';
 	import ModelDownloadProgressBar from '../ModelDownloadProgressBar.svelte';
+	import ModelDraftSubline from '../ModelDraftSubline.svelte';
 	import ModelId from '../ModelId.svelte';
 	import ModelsManagerStatusCell from './ModelsManagerStatusCell.svelte';
 	import { modelRowActions } from './row-actions';
@@ -12,7 +13,7 @@
 	import { MODEL_ROW_GRID_CLASS, MODEL_ROW_TRAILING_CELL_CLASS } from '$lib/constants';
 	import { ModelRowDownloadState } from '$lib/enums';
 	import { deviceStore, modelsStore } from '$lib/stores';
-	import type { ModelOption } from '$lib/types/models';
+	import type { ModelOption, ModelSidecarBadge } from '$lib/types/models';
 
 	interface Props {
 		isFavorite: (option: ModelOption) => boolean;
@@ -20,11 +21,21 @@
 		onDelete: (option: ModelOption) => void;
 		onSelect: (option: ModelOption) => void;
 		selected: boolean;
+		/** Draft sidecars already on disk, when the row stands for a download. */
+		sidecarBadges?: ModelSidecarBadge[];
 		/** Left padding in px, from the nesting depth. */
 		indent?: number;
 	}
 
-	let { indent = 0, isFavorite, onDelete, onSelect, option, selected }: Props = $props();
+	let {
+		indent = 0,
+		isFavorite,
+		onDelete,
+		onSelect,
+		option,
+		selected,
+		sidecarBadges
+	}: Props = $props();
 
 	let favorite = $derived(isFavorite(option));
 	let isHidden = $derived(modelsStore.isHidden(option.id));
@@ -38,6 +49,10 @@
 				? ModelRowDownloadState.DOWNLOADING
 				: null
 	);
+
+	function activate(): void {
+		onSelect(option);
+	}
 </script>
 
 <div
@@ -48,41 +63,58 @@
 		selected ? 'bg-accent text-accent-foreground' : 'hover:bg-muted/40'
 	]}
 >
-	<!-- the row holds a load control and an actions menu, so only the model itself is
+	<div class="flex min-w-0 flex-col">
+		<!-- the row holds a load control and an actions menu, so only the model itself is
 	     the button: a button nested in a role="button" row is invalid -->
-	<button
-		aria-pressed={selected}
-		class="flex min-w-0 cursor-pointer items-center gap-3 text-left max-md:gap-2 max-md:pr-9"
-		onclick={() => onSelect(option)}
-		style="padding-left: {indent}px"
-		type="button"
-	>
-		<ModelAvatar {option} size="size-7 md:size-9" />
+		<button
+			aria-pressed={download ? undefined : selected}
+			class="flex min-w-0 cursor-pointer items-center gap-3 text-left max-md:gap-2 max-md:pr-9"
+			onclick={activate}
+			style="padding-left: {indent}px"
+			type="button"
+		>
+			<ModelAvatar {option} size="size-7 md:size-9" />
 
-		<span class="flex min-w-0 items-center gap-1.25">
-			<ModelId
-				aliases={option.aliases}
-				class="min-w-0 flex-1 max-md:text-sm"
-				draftSidecars={option.draftSidecars}
-				hideCapabilities
-				hideModalities
-				modalities={option.modalities}
-				modelId={option.model}
-				tags={option.tags}
-				title={option.model}
-			/>
+			<span class="flex min-w-0 items-center gap-1.25">
+				<ModelId
+					aliases={option.aliases}
+					class="min-w-0 flex-1 max-md:text-sm"
+					draftSidecars={option.draftSidecars}
+					hideCapabilities
+					hideModalities
+					modalities={option.modalities}
+					modelId={option.model}
+					tags={option.tags}
+					title={option.model}
+				/>
 
-			<!-- a phone has no width for the modality icons, the id needs it more -->
-			<ModelCapabilities hideModalities={deviceStore.isMobile} {option} />
+				<!-- a phone has no width for the modality icons, the id needs it more -->
+				<ModelCapabilities hideModalities={deviceStore.isMobile} {option} />
 
-			{#if favorite}
-				<!-- the heart is decorative, the button's text carries the state -->
-				<Heart aria-hidden="true" class="h-3.5 w-3.5 shrink-0 text-rose-500" />
+				{#if favorite}
+					<!-- the heart is decorative, the button's text carries the state -->
+					<Heart aria-hidden="true" class="h-3.5 w-3.5 shrink-0 text-rose-500" />
 
-				<span class="sr-only">favorited</span>
-			{/if}
-		</span>
-	</button>
+					<span class="sr-only">favorited</span>
+				{/if}
+			</span>
+		</button>
+
+		{#if sidecarBadges?.length}
+			<!-- w-fit keeps the trailing state beside the badges instead of floating
+			     to the end of the wide model column -->
+			<div class="flex w-fit flex-col pb-1">
+				{#each sidecarBadges as badge (badge.repo + ':' + badge.kind + ':' + (badge.quant ?? ''))}
+					<ModelDraftSubline
+						connectorClass="ml-8"
+						kind={badge.kind}
+						quant={badge.quant}
+						state={ModelRowDownloadState.DOWNLOADED}
+					/>
+				{/each}
+			</div>
+		{/if}
+	</div>
 
 	<ModelContext
 		class="justify-self-end max-md:hidden"

tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte

@@ -88,6 +88,7 @@
 		[GroupedListUnit.GROUPS]: 'families'
 	};
 	let pendingDelete = $state('');
+	let pendingAction = $state(ModelDownloadConfirmAction.DELETE);
 	let deleteOpen = $state(false);
 	/** Repos whose quants are folded away; the rest show them. */
 	const collapsedQuants = new SvelteSet<string>();
@@ -129,6 +130,10 @@
 	// cancel is confirmed once for the whole list, so one dialog serves every row
 	function requestDelete(option: ModelOption): void {
 		pendingDelete = option.model;
+		// an in-flight row cancels (and drops the partials), a paused one deletes
+		pendingAction = modelsStore.status.isDownloadInProgress(option.model)
+			? ModelDownloadConfirmAction.CANCEL
+			: ModelDownloadConfirmAction.DELETE;
 		deleteOpen = true;
 	}
 
@@ -248,6 +253,7 @@
 			{onSelect}
 			option={entry.base}
 			selected={selectedId === entry.base.id}
+			sidecarBadges={entry.sidecarBadges}
 		/>
 	{/if}
 {/snippet}
@@ -328,7 +334,7 @@
 {/snippet}
 
 <DialogConfirmDownload
-	action={ModelDownloadConfirmAction.DELETE}
+	action={pendingAction}
 	onClose={() => (deleteOpen = false)}
 	open={deleteOpen}
 	repoWithTag={pendingDelete}

tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts

@@ -1,10 +1,10 @@
-import { Eye, EyeOff, Heart, HeartOff, Trash2 } from '@lucide/svelte';
+import { Compass, Eye, EyeOff, Heart, HeartOff, Trash2 } from '@lucide/svelte';
 import { MODEL_DOWNLOAD_ICONS } from '$lib/constants';
 import { ModelRowDownloadState } from '$lib/enums';
-import { modelsStore } from '$lib/stores';
+import { modelsStore, uiStore } from '$lib/stores';
 import type { ModelOption } from '$lib/types/models';
+import { repoOf } from '$lib/utils';
 
-/** Row actions follow the app's dropdown pattern: icon, label, separators, variants. */
 export function modelRowActions(
 	option: ModelOption,
 	favorite: boolean,
@@ -13,9 +13,17 @@ export function modelRowActions(
 	/** Download state, when the row stands for a tracked download. */
 	download?: ModelRowDownloadState | null
 ) {
+	const viewInDiscover = {
+		icon: Compass,
+		label: 'See in Discover',
+		// the details pane is keyed by repo, not by `<repo>:<quant>`
+		onclick: () => uiStore.openModelsDiscover(repoOf(option.model)),
+		separator: true
+	};
+
 	// a tracked download is paused, resumed or dropped: it is not loaded, hidden or
-	// drafted. Deleting it stops the download and removes what is on disk, which is
-	// what the same wording offers on a cached model
+	// drafted. An in-flight row cancels, which stops the download and removes the
+	// partial files; a paused one deletes, which removes the partials from disk
 	if (download) {
 		const isPaused = download === ModelRowDownloadState.PAUSED;
 
@@ -23,18 +31,16 @@ export function modelRowActions(
 			{
 				icon: isPaused ? MODEL_DOWNLOAD_ICONS.resume : MODEL_DOWNLOAD_ICONS.pause,
 				label: isPaused ? 'Resume downloading' : 'Pause downloading',
-				onclick: () =>
-					void (isPaused
-						? modelsStore.status.downloadModel(option.model)
-						: modelsStore.status.pauseDownload(option.model))
+				onclick: () => void modelsStore.status.toggleDownload(option.model)
 			},
 			{
 				icon: Trash2,
-				label: 'Delete from disk',
+				label: isPaused ? 'Delete from disk' : 'Cancel download',
 				onclick: () => onDelete(option),
 				separator: true,
 				variant: 'destructive' as const
-			}
+			},
+			viewInDiscover
 		];
 	}
 
@@ -56,6 +62,7 @@ export function modelRowActions(
 			label: isHidden ? 'Unhide model' : 'Hide model',
 			onclick: () => modelsStore.toggleHidden(option.id),
 			separator: true
-		}
+		},
+		viewInDiscover
 	];
 }

tools/ui/src/lib/components/app/models/ModelsManager/utils.ts

@@ -2,7 +2,12 @@ import { LOCAL_BACKEND_ID, type ModalityKey } from '$lib/constants';
 import { ModelCapability, ModelGroupKind, ModelsTableGroupKind } from '$lib/enums';
 import { HuggingFaceService, ModelsService } from '$lib/services';
 import { modelsStore } from '$lib/stores';
-import type { ModelDownloadEntry, ModelDownloadProgress, ModelOption } from '$lib/types/models';
+import type {
+	ModelDownloadEntry,
+	ModelDownloadProgress,
+	ModelOption,
+	ModelSidecarBadge
+} from '$lib/types/models';
 import { detectThinkingSupport, detectToolUseSupport, repoOf } from '$lib/utils';
 
 /** One repo of the table, with the rows it ships as. */
@@ -11,6 +16,8 @@ export interface ModelQuantGroup {
 	key: string;
 	kind: ModelGroupKind;
 	quants: ModelOption[];
+	/** Draft sidecars already on disk, shown as checked sub-lines. */
+	sidecarBadges?: ModelSidecarBadge[];
 }
 
 /** True when any of the manager's filters narrows the table. */
@@ -31,28 +38,36 @@ export function downloadProgressFor(repoWithTag: string): ModelDownloadProgress
 }
 
 /**
- * One table row per tracked download, so the table lists them the way it lists
- * any other model. A paused download is already in the router's listing, so its
- * own option is reused; a fresh one stands for its tag alone.
+ * One table row per tracked download, listed like any other model. A paused download
+ * is already in the router's listing, so its own option is reused; a fresh one is
+ * synthesized for its tag alone.
  */
 export function downloadGroups(
 	entries: ModelDownloadEntry[],
-	models: ModelOption[]
+	models: ModelOption[],
+	sidecarsFor: (repo: string) => ModelSidecarBadge[]
 ): ModelQuantGroup[] {
 	return entries.map((entry) => {
-		const option = models.find((candidate) => candidate.model === entry.repoWithTag) ?? {
+		const listed = models.find((candidate) => candidate.model === entry.repoWithTag);
+		// the synthesized option carries backendId, which ModelOption drops on
+		// assignment, so the callers spread it instead of typing this as one
+		const option = listed ?? {
 			backendId: LOCAL_BACKEND_ID,
 			capabilities: [],
 			id: entry.repoWithTag,
 			model: entry.repoWithTag,
 			name: entry.repoWithTag
 		};
+		// a draft still coming down badges the id, an on-disk one keeps its sub-line
+		const sidecars = sidecarsFor(repoOf(entry.repoWithTag));
+		const pending = sidecars.filter((badge) => badge.pending);
 
 		return {
-			base: option,
+			base: { ...option, draftSidecars: pending },
 			key: `download::${entry.repoWithTag}`,
 			kind: ModelGroupKind.QUANTS,
-			quants: [option]
+			quants: [{ ...option, draftSidecars: pending }],
+			sidecarBadges: sidecars.filter((badge) => !badge.pending)
 		};
 	});
 }

tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte

@@ -1,22 +1,38 @@
 <script lang="ts">
 	import ModelDownloadProgressBar from '../ModelDownloadProgressBar.svelte';
+	import ModelDraftSubline from '../ModelDraftSubline.svelte';
 	import ModelOrgAvatar from '../ModelOrgAvatar.svelte';
-	import { Loader2, Pause, Play, X } from '@lucide/svelte';
+	import { Info, Loader2, Pause, Play, X } from '@lucide/svelte';
 	import { ActionIcon, ModelId } from '$lib/components/app';
+	import { ModelRowDownloadState } from '$lib/enums';
 	import { HuggingFaceService, ModelsService } from '$lib/services';
-	import { modelsStore } from '$lib/stores';
-	import type { ModelDownloadEntry } from '$lib/types';
+	import { deviceStore, modelsStore, uiStore } from '$lib/stores';
+	import type { ModelDownloadEntry, ModelOption, ModelSidecarBadge } from '$lib/types';
 
 	interface Props {
 		/** One entry from the status feed: an in-flight or paused download. */
 		entry: ModelDownloadEntry;
+		/** Draft sidecars already on disk, rendered as checked sub-lines. */
+		sidecars?: ModelSidecarBadge[];
 		/** Ask the list to confirm cancelling; the list owns the single dialog. */
 		onRequestCancel?: (repoWithTag: string) => void;
 		/** Show the organization name in the repo id, as the other selector rows do. */
 		showOrgName?: boolean;
 	}
 
-	let { entry, onRequestCancel, showOrgName = true }: Props = $props();
+	let { entry, onRequestCancel, showOrgName = true, sidecars = [] }: Props = $props();
+
+	/** A draft still coming down badges the id, an on-disk one keeps its sub-line. */
+	let pendingDrafts = $derived(sidecars.filter((badge) => badge.pending));
+	let onDiskDrafts = $derived(sidecars.filter((badge) => !badge.pending));
+
+	/** The download stands for its model, so the row's info action can open on it. */
+	let option = $derived<ModelOption>({
+		capabilities: [],
+		id: entry.repoWithTag,
+		model: entry.repoWithTag,
+		name: modelsStore.toDisplayName(entry.repoWithTag)
+	});
 
 	let percent = $derived(
 		entry.progress && entry.progress.totalBytes > 0
@@ -53,48 +69,60 @@
 	});
 </script>
 
-<!-- One in-flight download; same actions as the discover quant chips: the row
-     itself pauses / resumes, the trailing X cancels (stops and discards the
-     partial files). Both affordances fade in on hover, the slots are reserved
-     so the list never reflows. -->
-<div
-	class="group relative flex items-center gap-2 rounded-sm p-2 text-left text-sm transition-colors hover:bg-accent/50"
->
-	<button
-		aria-label={actionText}
-		class="flex min-w-0 flex-1 cursor-pointer items-center gap-2 text-left"
-		onclick={() => {
-			if (entry.isPaused) {
-				void modelsStore.status
-					.downloadModel(entry.repoWithTag)
-					.catch((error) => console.warn('Failed to resume the download:', error));
-			} else {
-				void modelsStore.status.pauseDownload(entry.repoWithTag);
-			}
-		}}
-		type="button"
+<!-- One in-flight download: the row opens the model, the status slot beside it
+     pauses / resumes and the trailing X cancels (stops and discards the partial
+     files). The status slot is always there, the affordances fade in on hover,
+     so the list never reflows. The repo's on-disk sidecars hang off the item as
+     compact sub-lines instead of rows of their own. -->
+<div class="group relative rounded-sm transition-colors hover:bg-accent/50">
+	<div
+		class="flex items-center gap-2 p-2 text-left text-sm max-md:gap-2.5 max-md:px-2.5 max-md:py-2.5"
 	>
-		{#if orgName}
-			<ModelOrgAvatar
-				org={fetchedBaseModelOrg ?? orgName}
-				quantOrg={orgName}
-				quantPositionClass="-bottom-1 -right-1"
-				quantSize="h-3 w-3"
-				size="h-6 w-6"
+		<button
+			aria-label="Manage model"
+			class="flex min-w-0 flex-1 cursor-pointer items-center gap-2 text-left"
+			onclick={() =>
+				// a phone has no manager: its information dialog takes the row
+				deviceStore.isMobile
+					? uiStore.openModelInformation(option)
+					: uiStore.openModelsManager(entry.repoWithTag)}
+			type="button"
+		>
+			{#if orgName}
+				<ModelOrgAvatar
+					org={fetchedBaseModelOrg ?? orgName}
+					quantOrg={orgName}
+					quantPositionClass="-bottom-1 -right-1"
+					quantSize="h-3 w-3"
+					size="h-6 w-6"
+				/>
+			{/if}
+
+			<ModelId
+				class="flex-1"
+				draftSidecars={pendingDrafts}
+				hideOrgName={!showOrgName}
+				modelId={entry.repoWithTag}
+				showRawTooltip
 			/>
-		{/if}
 
-		<ModelId class="flex-1" hideOrgName={!showOrgName} modelId={entry.repoWithTag} showRawTooltip />
-
-		{#if percent !== null}
-			<span class="shrink-0 font-mono text-xs tabular-nums text-muted-foreground">{percent}%</span>
-		{:else if entry.isPaused}
-			<span class="shrink-0 text-xs text-muted-foreground">Paused</span>
-		{/if}
+			{#if percent !== null}
+				<span class="ml-auto shrink-0 font-mono text-xs tabular-nums text-muted-foreground">
+					{percent}%
+				</span>
+			{:else if entry.isPaused}
+				<span class="ml-auto shrink-0 text-xs text-muted-foreground">Paused</span>
+			{/if}
+		</button>
 
 		<!-- status action: spinner -> pause on hover while in flight, play on hover
-				 when paused; opacity only for the spinner, the spin owns the transform -->
-		<span class="relative inline-flex h-5 w-5 shrink-0 items-center justify-center">
+		     when paused; opacity only for the spinner, the spin owns the transform -->
+		<button
+			aria-label={actionText}
+			class="relative inline-flex h-5 w-5 shrink-0 cursor-pointer items-center justify-center"
+			onclick={() => void modelsStore.status.toggleDownload(entry.repoWithTag)}
+			type="button"
+		>
 			{#if entry.isPaused}
 				<Play
 					class="absolute h-4 w-4 scale-75 opacity-0 transition-[opacity,transform] duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:scale-100 group-hover:opacity-100 [@media(pointer:coarse)]:scale-100 [@media(pointer:coarse)]:opacity-100"
@@ -108,17 +136,45 @@
 					class="absolute h-4 w-4 opacity-0 transition-opacity duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:opacity-100 [@media(pointer:coarse)]:opacity-100"
 				/>
 			{/if}
-		</span>
-	</button>
+		</button>
 
-	<!-- the row actions keep the sizing and reveal of the other selector rows -->
-	<ActionIcon
-		ariaLabel="Cancel downloading"
-		class="pointer-events-none h-5 w-5 shrink-0 opacity-0 transition-opacity group-hover:pointer-events-auto group-hover:opacity-100 hover:text-destructive [@media(pointer:coarse)]:pointer-events-auto [@media(pointer:coarse)]:opacity-100"
-		icon={X}
-		iconSize="h-4 w-4"
-		onclick={() => onRequestCancel?.(entry.repoWithTag)}
-	/>
+		<!-- the row actions keep the sizing and reveal of the other selector rows -->
+		<ActionIcon
+			class="pointer-events-none h-5 w-5 shrink-0 opacity-0 transition-opacity group-hover:pointer-events-auto group-hover:opacity-100 hover:text-foreground [@media(pointer:coarse)]:pointer-events-auto [@media(pointer:coarse)]:opacity-100"
+			icon={Info}
+			iconSize="h-4 w-4"
+			onclick={() =>
+				// a phone has no manager: its information dialog takes the icon
+				deviceStore.isMobile
+					? uiStore.openModelInformation(option)
+					: uiStore.openModelsManager(entry.repoWithTag)}
+			tooltip="Manage model"
+			tooltipAsTitle
+		/>
+
+		<ActionIcon
+			ariaLabel="Cancel downloading"
+			class="pointer-events-none h-5 w-5 shrink-0 opacity-0 transition-opacity group-hover:pointer-events-auto group-hover:opacity-100 hover:text-destructive [@media(pointer:coarse)]:pointer-events-auto [@media(pointer:coarse)]:opacity-100"
+			icon={X}
+			iconSize="h-4 w-4"
+			onclick={() => onRequestCancel?.(entry.repoWithTag)}
+		/>
+	</div>
+
+	<!-- only with sub-lines: an empty strip would make the row taller than the
+	     plain ones, and the progress bar overlays the row's own bottom edge -->
+	{#if onDiskDrafts.length}
+		<div class="flex flex-col px-2 pb-1.5">
+			{#each onDiskDrafts as badge (badge.repo + ':' + badge.kind + ':' + (badge.quant ?? ''))}
+				<ModelDraftSubline
+					connectorClass="ml-4"
+					kind={badge.kind}
+					quant={badge.quant}
+					state={ModelRowDownloadState.DOWNLOADED}
+				/>
+			{/each}
+		</div>
+	{/if}
 
 	{#if entry.progress && entry.progress.totalBytes > 0}
 		<ModelDownloadProgressBar

tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorList.svelte

@@ -22,6 +22,7 @@
 		groupModelFamilies,
 		type ModelFamilyGroup,
 		type ModelItem,
+		repoOf,
 		windowLocalGroups
 	} from '$lib/utils';
 
@@ -202,7 +203,12 @@
 	<p class={headerClass} style={headerStyle}>Downloading</p>
 
 	{#each downloadEntries as entry (entry.repoWithTag)}
-		<ModelsSelectorDownloadItem {entry} onRequestCancel={requestCancel} {showOrgName} />
+		<ModelsSelectorDownloadItem
+			{entry}
+			onRequestCancel={requestCancel}
+			{showOrgName}
+			sidecars={modelsStore.status.getDraftSidecars(repoOf(entry.repoWithTag))}
+		/>
 	{/each}
 {/if}
 
@@ -230,6 +236,7 @@
 <DialogConfirmDownload
 	action={ModelDownloadConfirmAction.CANCEL}
 	onClose={() => (cancelOpen = false)}
+	onConfirm={() => void modelsStore.status.cancelDownload(pendingCancel)}
 	open={cancelOpen}
 	repoWithTag={pendingCancel}
 />

tools/ui/src/lib/components/app/models/discover/ModelsDiscover.svelte

@@ -0,0 +1,210 @@
+<script lang="ts">
+	import ModelsDiscoverListItem from './ModelsDiscoverList/ModelsDiscoverListItem.svelte';
+	import { Download } from '@lucide/svelte';
+	import { ModelsSection } from '$lib/components/app';
+	import { DialogConfirmDownload } from '$lib/components/app/dialogs';
+	import {
+		ModelsDiscoverDetailsContainer,
+		ModelsDiscoverList,
+		ModelsDiscoverListSearch
+	} from '$lib/components/app/models/discover';
+	import { Button } from '$lib/components/ui/button';
+	import { ModelDownloadConfirmAction } from '$lib/enums';
+	import { HuggingFaceService } from '$lib/services';
+	import { modelsDiscoverStore, modelsStore, uiStore } from '$lib/stores';
+	import type { HfModelInfo } from '$lib/types';
+	import { repoOf } from '$lib/utils';
+	import { untrack } from 'svelte';
+
+	let selectedId = $state<string | null>(null);
+	let searchQuery = $state('');
+	// running and paused downloads both lead the list; a search owns the pane, so the
+	// section steps aside while a query is active
+	let downloadEntries = $derived(modelsStore.status.getDownloadEntries());
+
+	let isSearching = $derived(searchQuery.trim().length > 0);
+
+	/** Hub record for a repo the list does not carry, so its row still renders. */
+	function hubRecord(id: string): HfModelInfo {
+		return {
+			_id: id,
+			downloads: 0,
+			id,
+			library_name: null,
+			likes: 0,
+			modelId: id,
+			pipeline_tag: null,
+			private: false,
+			tags: []
+		};
+	}
+
+	// Hub records of repos the list does not carry, e.g. a download started from a
+	// search hit: fetched once so the row shows the same badges and icons as a result.
+	let fetchedRecords = $state<Record<string, HfModelInfo>>({});
+
+	/** Record of a repo: what the list fetched, else its own details, else a bare one. */
+	function recordOf(repo: string): HfModelInfo {
+		return (
+			modelsDiscoverStore.models.find((model) => model.id === repo) ??
+			fetchedRecords[repo] ??
+			hubRecord(repo)
+		);
+	}
+
+	$effect(() => {
+		for (const entry of downloadEntries) {
+			const repo = repoOf(entry.repoWithTag);
+
+			if (fetchedRecords[repo] || modelsDiscoverStore.models.some((m) => m.id === repo)) continue;
+
+			void HuggingFaceService.getDetails(repo).then((details) => {
+				if (details) fetchedRecords = { ...fetchedRecords, [repo]: details };
+			});
+		}
+	});
+
+	// cancel is confirmed once, so a single dialog serves every download row; the
+	// target is kept while the dialog closes so its copy stays rendered
+	let pendingCancel = $state('');
+	let cancelOpen = $state(false);
+
+	function requestCancel(repoWithTag: string) {
+		pendingCancel = repoWithTag;
+		cancelOpen = true;
+	}
+
+	let searchTimeout: ReturnType<typeof setTimeout> | null = null;
+
+	// load the list on mount; untrack because fetch() toggles its loading flag
+	// synchronously, and tracking it would re-run the effect on every toggle - an
+	// endless retry while a repo stays failed
+	$effect(() => {
+		untrack(() => {
+			void modelsDiscoverStore.fetch();
+			void modelsDiscoverStore.search('');
+		});
+	});
+
+	$effect(() => {
+		const first = modelsDiscoverStore.firstModel;
+
+		if (!selectedId && first) {
+			selectedId = first.id;
+		}
+	});
+
+	// a caller can ask for one repo to be opened, the manager's rows do
+	$effect(() => {
+		const focus = uiStore.discoverModelFocus;
+
+		if (!focus) return;
+
+		selectedId = focus;
+		uiStore.discoverModelFocus = null;
+	});
+
+	function handleSearchInput(value: string) {
+		searchQuery = value;
+
+		if (searchTimeout) clearTimeout(searchTimeout);
+
+		searchTimeout = setTimeout(() => {
+			void modelsDiscoverStore.search(value);
+		}, 300);
+	}
+</script>
+
+{#snippet catalog()}
+	<ModelsDiscoverList
+		activeId={selectedId}
+		loading={modelsDiscoverStore.loading || modelsDiscoverStore.searching}
+		models={modelsDiscoverStore.models}
+		onSelect={(id) => (selectedId = id)}
+		showBaseModelAvatar
+	/>
+{/snippet}
+
+<aside
+	class="w-md shrink-0 self-start border-r border-border/40 bg-background overflow-y-auto md:pr-4 h-full space-y-4"
+>
+	<ModelsDiscoverListSearch bind:value={searchQuery} onSearch={handleSearchInput} />
+
+	{#if downloadEntries.length > 0 && !isSearching}
+		<ModelsSection
+			count={downloadEntries.length}
+			label="Downloading"
+			persistKey="discover-downloads"
+			sticky
+			stickyClass="sticky z-10 bg-background"
+		>
+			{#snippet icon()}
+				<Download class="h-3.5 w-3.5 shrink-0" />
+			{/snippet}
+
+			{#each downloadEntries as entry (entry.repoWithTag)}
+				{@const repo = repoOf(entry.repoWithTag)}
+
+				<!-- the row is the list's own, so a download opens the same details pane a
+				     result does; only the state line and the trailing control differ -->
+				<ModelsDiscoverListItem
+					active={repo === selectedId}
+					download={{
+						isPaused: entry.isPaused,
+						onRequestCancel: requestCancel,
+						progress: entry.progress,
+						repoWithTag: entry.repoWithTag
+					}}
+					model={recordOf(repo)}
+					onSelect={(id) => (selectedId = id)}
+					showBaseModelAvatar
+					sidecarBadges={modelsStore.status.getDraftSidecars(repo)}
+				/>
+			{/each}
+		</ModelsSection>
+	{/if}
+
+	<!-- One list instance, so the rows keep their state across search round trips;
+		 skeleton rows replace them while the initial catalog or a query loads. -->
+	<div>
+		{#if modelsDiscoverStore.error}
+			<div class="flex flex-col items-start gap-2 p-4">
+				<p class="text-sm text-destructive">{modelsDiscoverStore.error}</p>
+
+				<Button onclick={() => void modelsDiscoverStore.fetch()} size="sm" variant="outline">
+					Retry
+				</Button>
+			</div>
+		{:else if !modelsDiscoverStore.loading && !modelsDiscoverStore.searching && modelsDiscoverStore.models.length === 0}
+			<p class="p-4 text-sm text-muted-foreground">No models found</p>
+		{:else}
+			{#if isSearching}
+				{@render catalog()}
+			{:else}
+				<ModelsSection
+					count={modelsDiscoverStore.models.length}
+					label="Suggested models"
+					persistKey="discover-suggested"
+					sticky
+					stickyClass="sticky z-10 bg-background"
+				>
+					{@render catalog()}
+				</ModelsSection>
+			{/if}
+		{/if}
+	</div>
+</aside>
+
+<main class="overflow-y-auto">
+	{#if selectedId}
+		<ModelsDiscoverDetailsContainer modelId={selectedId} />
+	{/if}
+</main>
+
+<DialogConfirmDownload
+	action={ModelDownloadConfirmAction.CANCEL}
+	onClose={() => (cancelOpen = false)}
+	onConfirm={() => void modelsStore.status.cancelDownload(pendingCancel)}
+	open={cancelOpen}
+	repoWithTag={pendingCancel}
+/>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverChatTemplateDialog.svelte

@@ -0,0 +1,75 @@
+<script lang="ts">
+	import { Check, Copy, X } from '@lucide/svelte';
+	import { SyntaxHighlightedCode } from '$lib/components/app';
+	import { Button } from '$lib/components/ui/button';
+	import * as Dialog from '$lib/components/ui/dialog';
+	import { COPY_FEEDBACK_RESET_MS } from '$lib/constants';
+	import { copyToClipboard } from '$lib/utils';
+
+	interface Props {
+		open?: boolean;
+		chatTemplate: string;
+		onOpenChange?: (open: boolean) => void;
+	}
+
+	let { chatTemplate, onOpenChange, open = $bindable(false) }: Props = $props();
+
+	function handleOpenChange(value: boolean) {
+		open = value;
+		onOpenChange?.(value);
+	}
+
+	let copied = $state(false);
+
+	async function copy() {
+		await copyToClipboard(chatTemplate);
+		copied = true;
+		setTimeout(() => (copied = false), COPY_FEEDBACK_RESET_MS);
+	}
+</script>
+
+<Dialog.Root onOpenChange={handleOpenChange} {open}>
+	<Dialog.Content
+		class="flex max-h-[calc(100vh-4rem)] flex-col gap-0 p-0 md:w-[calc(100vw-4rem)]! md:max-w-4xl!"
+	>
+		<!-- The header's corner-pinned close X never lines up with a padded flex row,
+		     so it is replaced by one inside the row, aligned with the title -->
+		<Dialog.Header
+			class="flex-row items-center gap-2 border-b border-border/40 p-4"
+			showCloseButton={false}
+		>
+			<Dialog.Title class="text-sm font-semibold">Chat template</Dialog.Title>
+
+			<Button
+				aria-label="Copy chat template"
+				class="h-auto gap-1.5 px-2 py-1 text-xs"
+				onclick={() => void copy()}
+				size="sm"
+				variant="tertiary"
+			>
+				{#if copied}
+					<Check class="size-3.5 text-green-500" />
+				{:else}
+					<Copy class="size-3.5" />
+				{/if}
+
+				Copy
+			</Button>
+
+			<Dialog.Close
+				aria-label="Close"
+				class="ml-auto inline-flex cursor-pointer items-center justify-center rounded-md p-1.5 text-muted-foreground/70 transition-colors hover:bg-muted-foreground/10 hover:text-foreground"
+				type="button"
+			>
+				<X class="h-4 w-4" />
+			</Dialog.Close>
+		</Dialog.Header>
+
+		<SyntaxHighlightedCode
+			class="h-full rounded-none border-0 shadow-none"
+			code={chatTemplate}
+			language="jinja"
+			maxHeight="100%"
+		/>
+	</Dialog.Content>
+</Dialog.Root>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetails.svelte

@@ -0,0 +1,96 @@
+<script lang="ts">
+	import { ModelsDiscoverDetailsDownloadOptions } from './ModelsDiscoverDetailsDownloadOptions';
+	import ModelsDiscoverDetailsHeader from './ModelsDiscoverDetailsHeader.svelte';
+	import ModelsDiscoverDetailsReadme from './ModelsDiscoverDetailsReadme.svelte';
+	import ModelsDiscoverDetailsSkeleton from './ModelsDiscoverDetailsSkeleton.svelte';
+	import { OTHER_BIT_DEPTH } from '$lib/constants';
+	import { ModelAuxSidecar } from '$lib/enums';
+	import { HuggingFaceService } from '$lib/services';
+	import type { HfModelDetailInfo, HfModelSibling, ModelBitDepthRow } from '$lib/types';
+	import { detectThinkingSupport, detectToolUseSupport } from '$lib/utils';
+	import { SvelteMap } from 'svelte/reactivity';
+
+	interface Props {
+		/** Full HuggingFace model id, e.g. `ggml-org/gemma-3-4b-it-GGUF`. */
+		modelId: string;
+		/** Model details from `/api/models/{id}?full=true`; null while loading. */
+		details: HfModelDetailInfo | null;
+		/** GGUF files of the repo, shards collapsed, sorted by size desc. */
+		files: HfModelSibling[];
+		/** README.md content, frontmatter stripped; null when unavailable. */
+		readme: string | null;
+		/** True while the model data is being fetched. */
+		loading?: boolean;
+		/** Error message when loading failed. */
+		error?: string | null;
+	}
+
+	let { details, error = null, files, loading = false, modelId, readme }: Props = $props();
+
+	let gguf = $derived(details?.gguf);
+	let baseModels = $derived(HuggingFaceService.getBaseModels(details));
+	let licenseTag = $derived.by(() => {
+		const tags = details?.tags ?? [];
+
+		return tags.find((t) => t.startsWith('license:'))?.replace('license:', '') ?? null;
+	});
+
+	// Capabilities derived from HF metadata. Vision comes from an mmproj sidecar
+	// or a multimodal pipeline tag; tool use / reasoning from the chat template.
+	let hasMmproj = $derived(
+		files.some(
+			(f) => HuggingFaceService.extractQuantMeta(f.path)?.sidecar === ModelAuxSidecar.MMPROJ
+		)
+	);
+	let hasVision = $derived(hasMmproj || details?.pipeline_tag === 'image-text-to-text');
+	let hasTools = $derived(detectToolUseSupport(gguf?.chat_template ?? ''));
+	let hasReasoning = $derived(detectThinkingSupport(gguf?.chat_template ?? ''));
+
+	let bitDepthRows = $derived.by<ModelBitDepthRow[]>(() => {
+		const rows = new SvelteMap<number, HfModelSibling[]>();
+
+		for (const file of files) {
+			const meta = HuggingFaceService.extractQuantMeta(file.path);
+
+			// mmproj sidecars are already conveyed by the Vision capability badge;
+			// imatrix ships as a normal chip with its own badge.
+			if (meta?.sidecar === ModelAuxSidecar.MMPROJ) continue;
+
+			const depth = meta?.quant ? HuggingFaceService.getBitDepth(meta.quant) : null;
+			const bucket = depth ?? OTHER_BIT_DEPTH;
+			const list = rows.get(bucket) ?? [];
+
+			list.push(file);
+			rows.set(bucket, list);
+		}
+
+		return Array.from(rows.entries())
+			.map(([bitDepth, rowFiles]) => ({ bitDepth, files: rowFiles }))
+			.sort((a, b) => a.bitDepth - b.bitDepth);
+	});
+</script>
+
+{#if loading}
+	<ModelsDiscoverDetailsSkeleton />
+{:else if error}
+	<div class="flex h-full items-center justify-center py-20">
+		<p class="text-sm text-destructive">{error}</p>
+	</div>
+{:else if details}
+	<div class="@container space-y-6 p-6 pr-0">
+		<ModelsDiscoverDetailsHeader
+			{baseModels}
+			{details}
+			{gguf}
+			{hasReasoning}
+			{hasTools}
+			{hasVision}
+			{licenseTag}
+			{modelId}
+		/>
+
+		<ModelsDiscoverDetailsDownloadOptions {bitDepthRows} {modelId} />
+
+		<ModelsDiscoverDetailsReadme {readme} />
+	</div>
+{/if}

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsContainer.svelte

@@ -0,0 +1,66 @@
+<script lang="ts">
+	import ModelsDiscoverDetails from './ModelsDiscoverDetails.svelte';
+	import { HuggingFaceService } from '$lib/services';
+	import type { HfModelDetailInfo, HfModelSibling } from '$lib/types/huggingface';
+
+	interface Props {
+		/** Full HuggingFace model id, e.g. `ggml-org/gemma-3-4b-it-GGUF`. */
+		modelId: string;
+	}
+
+	let { modelId }: Props = $props();
+
+	let details = $state<HfModelDetailInfo | null>(null);
+	let files = $state<HfModelSibling[]>([]);
+	let readme = $state<string | null>(null);
+	let loading = $state(false);
+	let error = $state<string | null>(null);
+
+	$effect(() => {
+		const id = modelId;
+
+		if (!id) return;
+
+		let cancelled = false;
+
+		loading = true;
+		error = null;
+
+		void (async () => {
+			try {
+				const [info, tree, readmeText] = await Promise.all([
+					HuggingFaceService.getDetails(id),
+					HuggingFaceService.getTree(id),
+					HuggingFaceService.getReadme(id)
+				]);
+
+				if (cancelled) return;
+
+				if (!info) {
+					error = 'Model not found';
+
+					return;
+				}
+
+				details = info;
+				files = HuggingFaceService.filterByExtension(
+					HuggingFaceService.collapseGgufShards(tree),
+					'.gguf'
+				);
+				readme = readmeText;
+			} catch (err) {
+				if (cancelled) return;
+
+				error = err instanceof Error ? err.message : 'Failed to load model';
+			} finally {
+				if (!cancelled) loading = false;
+			}
+		})();
+
+		return () => {
+			cancelled = true;
+		};
+	});
+</script>
+
+<ModelsDiscoverDetails {details} {error} {files} {loading} {modelId} {readme} />

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptions.svelte

@@ -0,0 +1,258 @@
+<script lang="ts">
+	import { classify, draftBadgeOf, sortDraftKinds } from './download-options.utils';
+	import ModelsDiscoverDetailsDownloadOptionsRow from './ModelsDiscoverDetailsDownloadOptionsRow.svelte';
+	import { DialogConfirmDownload } from '$lib/components/app/dialogs';
+	import { Checkbox } from '$lib/components/ui/checkbox';
+	import * as Select from '$lib/components/ui/select';
+	import {
+		ModelDownloadConfirmAction,
+		ModelDraftSidecar,
+		ModelSelectableFileKind
+	} from '$lib/enums';
+	import { HuggingFaceService, ModelsService } from '$lib/services';
+	import { modelsStore } from '$lib/stores';
+	import type {
+		ModelBitDepthRow,
+		ModelCompanionDraft,
+		ModelDownloadEntryState,
+		ModelSelectableFile
+	} from '$lib/types';
+	import { directoryOf, isServerSidecarFile } from '$lib/utils';
+
+	interface Props {
+		/** Full HuggingFace repo id, e.g. `ggml-org/gemma-3-4b-it-GGUF`. */
+		modelId: string;
+		/** GGUF files grouped by bit depth. */
+		bitDepthRows: ModelBitDepthRow[];
+	}
+
+	let { bitDepthRows, modelId }: Props = $props();
+
+	// one shared dialog confirms the panel's destructive chip actions; the target
+	// is kept after closing so its copy stays rendered through the close transition
+	let pending: { action: ModelDownloadConfirmAction; repoWithTag: string } = $state({
+		action: ModelDownloadConfirmAction.CANCEL,
+		repoWithTag: ''
+	});
+	let confirmOpen = $state(false);
+
+	function requestCancel(repoWithTag: string) {
+		pending = { action: ModelDownloadConfirmAction.CANCEL, repoWithTag };
+		confirmOpen = true;
+	}
+
+	function requestDelete(repoWithTag: string) {
+		pending = { action: ModelDownloadConfirmAction.DELETE, repoWithTag };
+		confirmOpen = true;
+	}
+
+	/**
+	 * Download state of a main quant's chip, from the status feed: the quant alone,
+	 * since a draft it is missing gets a chip of its own to add it.
+	 */
+	function stateFor(repoWithTag: string): ModelDownloadEntryState {
+		const isDownloading = modelsStore.status.isDownloadInProgress(repoWithTag);
+		const isPaused = modelsStore.status.isDownloadPaused(repoWithTag);
+
+		return {
+			isDownloaded: !isDownloading && modelsStore.status.isModelDownloaded(repoWithTag),
+			isDownloading,
+			isFailed: modelsStore.status.hasFailedDownload(repoWithTag),
+			isPaused,
+			// live progress while downloading, else the frozen snapshot of the pause
+			progress:
+				modelsStore.status.getDownloadProgress(repoWithTag) ??
+				modelsStore.status.getPausedDownloadProgress(repoWithTag),
+			repoWithTag
+		};
+	}
+
+	/**
+	 * Every GGUF of the panel with its role, in row order.
+	 * Single source of truth for the chip rows and the command selects.
+	 */
+	let selectableFiles = $derived.by(() =>
+		bitDepthRows.flatMap((row) => row.files.map((file) => ({ ...file, kind: classify(file.path) })))
+	);
+
+	/** Sidecar kinds the repo reports on disk, so a chip tells a pulled draft from a missing one. */
+	let onDiskDrafts = $derived(modelsStore.status.getDraftSidecars(modelId));
+
+	/** Draft kinds the repo ships, best first and once each; one rides with every chip. */
+	let draftKinds = $derived(
+		sortDraftKinds(
+			selectableFiles
+				.flatMap((file) =>
+					file.kind === ModelSelectableFileKind.DRAFT ? [draftBadgeOf(file)] : []
+				)
+				.filter((kind): kind is ModelDraftSidecar => kind !== null)
+				.filter((kind, index, all) => all.indexOf(kind) === index)
+		)
+	);
+
+	// the operator's draft pick; the best available kind rides along until they change it
+	let draftPick = $state<ModelDraftSidecar | null>(null);
+	let draftKind = $derived(draftPick ?? draftKinds[0] ?? null);
+
+	/** The drafts are opt-in: unchecked, a chip downloads the quant alone. */
+	let withDraft = $state(true);
+
+	/** One kind leaves the pick nothing to choose: it reads as the value it holds. */
+	let draftKindReadonly = $derived(draftKinds.length < 2);
+
+	let draftKindLabel = $derived(draftKind?.toUpperCase() ?? '');
+
+	/** Bits of a file's quant, 0 when its name carries none, as the server reads it. */
+	function bitsOf(path: string): number {
+		const quant = HuggingFaceService.extractQuantMeta(path)?.quant;
+
+		return (quant ? HuggingFaceService.getBitDepth(quant) : null) ?? 0;
+	}
+
+	/**
+	 * Draft that rides with a quant: the file of the picked kind the server would
+	 * pull. Its own directory, then the exact quant, then the closest bit depth -
+	 * a 4-bit model pairs with a 16-bit draft head when the repo ships no 4-bit
+	 * one, the way -hf settles it.
+	 */
+	function companionFor(
+		main: Pick<ModelSelectableFile, 'path'>,
+		quant: string | null
+	): ModelCompanionDraft | null {
+		if (!draftKind) return null;
+
+		const directory = directoryOf(main.path);
+		const candidates = selectableFiles.filter(
+			(candidate) =>
+				candidate.kind === ModelSelectableFileKind.DRAFT &&
+				draftBadgeOf(candidate) === draftKind &&
+				isServerSidecarFile(candidate.path, draftKind) &&
+				directoryOf(candidate.path) === directory
+		);
+
+		if (candidates.length === 0) return null;
+
+		const quantOf = (candidate: ModelSelectableFile) =>
+			HuggingFaceService.extractQuantMeta(candidate.path)?.quant ?? null;
+		const mainBits = quant ? bitsOf(main.path) : 0;
+		const exact = quant ? candidates.find((candidate) => quantOf(candidate) === quant) : undefined;
+		// the closest bit depth wins when the exact quant is missing, ties on file order
+		const closest = [...candidates].sort(
+			(a, b) => Math.abs(bitsOf(a.path) - mainBits) - Math.abs(bitsOf(b.path) - mainBits)
+		)[0];
+		const picked = exact ?? closest;
+
+		if (!picked) return null;
+
+		const pickedQuant = quantOf(picked);
+
+		return {
+			kind: draftKind,
+			// on disk when the model's own sidecar option reports it, or when a loaded
+			// model's args name that same draft file
+			onDisk:
+				modelsStore.status.isModelDownloaded(
+					ModelsService.buildDownloadTag(modelId, quant, draftKind)
+				) || onDiskDrafts.some((badge) => badge.kind === draftKind && badge.quant === pickedQuant),
+			path: picked.path,
+			quant: pickedQuant,
+			size: picked.size ?? 0
+		};
+	}
+
+	let rows = $derived.by(() =>
+		bitDepthRows.map((row) => ({
+			bitDepth: row.bitDepth,
+			files: row.files.flatMap((file) => {
+				if (classify(file.path) !== ModelSelectableFileKind.MAIN) return [];
+
+				const quant = HuggingFaceService.extractQuantMeta(file.path)?.quant ?? null;
+				const companion = withDraft ? companionFor(file, quant) : null;
+
+				return [
+					{
+						...file,
+						companion,
+						kind: ModelSelectableFileKind.MAIN,
+						quant,
+						// the store adds the model's mmproj and the sidecars it already pulls
+						sidecars: companion ? [companion.kind] : [],
+						state: stateFor(ModelsService.buildDownloadTag(modelId, quant, null))
+					}
+				];
+			})
+		}))
+	);
+</script>
+
+{#if bitDepthRows.length}
+	<section class="rounded-3xl border border-border/30 bg-muted/60 shadow-sm dark:border-border/20">
+		<!-- One chip per main quant, each an independent download action. The draft
+			 of that quant rides with it and badges the chip; the server picks its file,
+			 the way -hf does. -->
+		<div class="flex w-full flex-col divide-y divide-border/50 px-4 pb-1 dark:divide-border/35">
+			{#each rows as row (row.bitDepth)}
+				<ModelsDiscoverDetailsDownloadOptionsRow
+					bitDepth={row.bitDepth}
+					draftKind={withDraft ? draftKind : null}
+					{draftKinds}
+					files={row.files}
+					{modelId}
+					onDraftKindChange={(kind) => (draftPick = kind)}
+					onRequestCancel={requestCancel}
+					onRequestDelete={requestDelete}
+				/>
+			{/each}
+		</div>
+
+		<!-- The drafts are opt-in, and the pick decides which flavour every chip
+		     above pulls. The pick sits between the two label halves, so its own click
+		     cannot toggle the checkbox. A repo shipping a single kind still shows it,
+		     readonly. -->
+		{#if draftKinds.length}
+			<div
+				class="flex flex-wrap items-center justify-start gap-x-1.5 gap-y-1 border-t border-border/50 px-4 py-2.5 text-xs text-muted-foreground dark:border-border/35"
+			>
+				<label class="flex cursor-pointer items-center gap-2" for="discover-draft-sidecars">
+					<Checkbox bind:checked={withDraft} id="discover-draft-sidecars" />
+
+					Download model with a
+				</label>
+
+				<Select.Root
+					disabled={!withDraft || draftKindReadonly}
+					onValueChange={(value) => value && (draftPick = value as ModelDraftSidecar)}
+					type="single"
+					value={draftKind}
+				>
+					<Select.Trigger
+						aria-label="Draft type"
+						class={draftKindReadonly
+							? 'disabled:cursor-default disabled:opacity-100 [&_svg]:hidden'
+							: ''}
+						size="xs"
+					>
+						{draftKindLabel}
+					</Select.Trigger>
+
+					<Select.Content class="font-mono text-xs">
+						{#each draftKinds as kind (kind)}
+							<Select.Item class="text-xs" label={kind.toUpperCase()} value={kind}>
+								{kind.toUpperCase()}
+							</Select.Item>
+						{/each}
+					</Select.Content>
+				</Select.Root>
+
+				<label class="cursor-pointer" for="discover-draft-sidecars">draft sidecar</label>
+			</div>
+		{/if}
+	</section>
+{/if}
+
+<DialogConfirmDownload
+	action={pending.action}
+	onClose={() => (confirmOpen = false)}
+	open={confirmOpen}
+	repoWithTag={pending.repoWithTag}
+/>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDownloadCommand.svelte

@@ -0,0 +1,131 @@
+<script lang="ts">
+	import { Check, Copy } from '@lucide/svelte';
+	import * as Select from '$lib/components/ui/select';
+	import { COPY_FEEDBACK_RESET_MS, MODEL_ID, SERVE_COMMAND, SPEC_TYPE } from '$lib/constants';
+	import type { ModelDraftSidecar } from '$lib/enums';
+	import { copyToClipboard } from '$lib/utils';
+
+	interface Props {
+		/** Full HuggingFace repo id, e.g. `ggml-org/gemma-3-4b-it-GGUF`. */
+		modelId: string;
+		/** Quant the command pins on `-hf`; null when the row's file carries none. */
+		quant?: string | null;
+		/**
+		 * Draft kind the panel downloads with the model, null when the sidecars are
+		 * off. The server resolves the draft file itself, so the command only names
+		 * its type.
+		 */
+		draftKind?: ModelDraftSidecar | null;
+		/** Draft kinds the repo ships; the command offers them in its own pick. */
+		draftKinds?: ModelDraftSidecar[];
+		/** Fires when the command's pick changes, so the panel's own pick follows. */
+		onDraftKindChange?: (kind: ModelDraftSidecar) => void;
+	}
+
+	let {
+		draftKind = null,
+		draftKinds = [],
+		modelId,
+		onDraftKindChange,
+		quant = null
+	}: Props = $props();
+
+	/** The llama serve command: the row's quant and, when it rides along, the draft. */
+	let command = $derived.by(() => {
+		const parts = [
+			SERVE_COMMAND.BIN,
+			SERVE_COMMAND.SUBCOMMAND,
+			SERVE_COMMAND.MODEL_FLAG,
+			quant ? `${modelId}${MODEL_ID.QUANTIZATION_SEPARATOR}${quant}` : modelId
+		];
+
+		if (draftKind) {
+			parts.push(SERVE_COMMAND.SPEC_TYPE_FLAG, SPEC_TYPE[draftKind]);
+		}
+
+		return parts.join(' ');
+	});
+
+	let copied = $state(false);
+
+	async function copy() {
+		await copyToClipboard(command);
+		copied = true;
+		setTimeout(() => (copied = false), COPY_FEEDBACK_RESET_MS);
+	}
+</script>
+
+<div
+	class="relative flex items-center gap-2 overflow-hidden rounded-lg border border-border/40 bg-background py-2.5 pl-4 pr-10 shadow-xs dark:border-border/35 dark:bg-background/50"
+>
+	<!-- Single line: long commands scroll horizontally instead of wrapping. -->
+	<div
+		class="flex min-w-0 flex-1 items-center gap-x-2 overflow-x-auto py-0.5 font-mono text-xs whitespace-nowrap text-foreground/90"
+	>
+		<span class="shrink-0">{SERVE_COMMAND.BIN}</span>
+
+		<span class="shrink-0">{SERVE_COMMAND.SUBCOMMAND}</span>
+
+		<span class="shrink-0">{SERVE_COMMAND.MODEL_FLAG}</span>
+
+		<span class="shrink-0">
+			{modelId}{quant ? MODEL_ID.QUANTIZATION_SEPARATOR : ''}
+		</span>
+
+		<!-- the row's own quant, so no pick is needed here -->
+		{#if quant}
+			<span class="shrink-0 rounded-md border border-primary/15 bg-primary/[0.07] px-1.5">
+				{quant}
+			</span>
+		{/if}
+
+		<!-- The draft rides with the model, so the flag only names its type; the
+		     checkbox below turns it on, the server resolves the file. The pick is
+		     the panel's own, so changing it here changes it there. -->
+		{#if draftKind}
+			<span class="shrink-0">{SERVE_COMMAND.SPEC_TYPE_FLAG}</span>
+
+			<!-- the select only earns its chrome when there is a real choice to make -->
+			{#if draftKinds.length > 1}
+				<Select.Root
+					onValueChange={(value) => value && onDraftKindChange?.(value as ModelDraftSidecar)}
+					type="single"
+					value={draftKind}
+				>
+					<Select.Trigger
+						aria-label="Draft type"
+						class="border-primary/15 bg-primary/[0.07] font-mono text-foreground hover:bg-primary/15 focus-visible:border-primary/40 focus-visible:ring-0"
+						size="xs"
+					>
+						{SPEC_TYPE[draftKind]}
+					</Select.Trigger>
+
+					<Select.Content class="font-mono text-xs">
+						{#each draftKinds as kind (kind)}
+							<Select.Item class="text-xs" label={SPEC_TYPE[kind]} value={kind}>
+								{SPEC_TYPE[kind]}
+							</Select.Item>
+						{/each}
+					</Select.Content>
+				</Select.Root>
+			{:else}
+				<span class="shrink-0 rounded-md border border-primary/15 bg-primary/[0.07] px-1.5">
+					{SPEC_TYPE[draftKind]}
+				</span>
+			{/if}
+		{/if}
+	</div>
+
+	<button
+		aria-label="Copy command"
+		class="absolute top-1/2 right-2 -translate-y-1/2 cursor-pointer rounded-md p-1.5 text-muted-foreground/70 transition-colors hover:bg-primary/10 hover:text-foreground"
+		onclick={copy}
+		type="button"
+	>
+		{#if copied}
+			<Check class="h-3.5 w-3.5 text-green-500" />
+		{:else}
+			<Copy class="h-3.5 w-3.5" />
+		{/if}
+	</button>
+</div>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton.svelte

@@ -0,0 +1,61 @@
+<script lang="ts">
+	import { Download } from '@lucide/svelte';
+	import * as Tooltip from '$lib/components/ui/tooltip';
+	import { HuggingFaceService } from '$lib/services';
+	import { modelsStore } from '$lib/stores';
+	import type { ModelCompanionDraft } from '$lib/types';
+
+	interface Props {
+		/** The draft its model is missing, with its own file, quant and size. */
+		companion: ModelCompanionDraft;
+		/** Tag of the model the draft belongs to, the one the server pulls it with. */
+		repoWithTag: string;
+	}
+
+	let { companion, repoWithTag }: Props = $props();
+
+	// one source, so the chip matches the draft badge of a quant chip
+	const SIDECAR_BADGE_CLASS =
+		'rounded-md bg-primary px-1 py-0.5 text-[10px] font-semibold tracking-wide text-primary-foreground uppercase';
+
+	let tooltipText = $derived(`Add the ${companion.kind} draft for ${repoWithTag}`);
+
+	// a draft is pulled with its model, so the model is posted again with the kind
+	function addDraft() {
+		void modelsStore.status.downloadModel(repoWithTag, [companion.kind]).catch(() => {
+			// the store already toasted the failure
+		});
+	}
+</script>
+
+<!-- A downloaded model missing its draft: the chip adds the draft alone, and
+     the model keeps the files it already has. -->
+<Tooltip.Root>
+	<Tooltip.Trigger>
+		{#snippet child({ props })}
+			<button
+				{...props}
+				aria-label={tooltipText}
+				class="group relative inline-flex h-auto cursor-pointer items-center gap-1 rounded-md border border-dashed border-border/60 px-2 py-1 text-left font-mono text-xs shadow-sm transition-[background-color,border-color,transform] duration-200 ease-[cubic-bezier(0.23,1,0.32,1)] hover:bg-muted-foreground/10 hover:border-border active:scale-[0.97] dark:border-border/40 dark:hover:bg-muted-foreground/15"
+				onclick={addDraft}
+				type="button"
+			>
+				<span class="{SIDECAR_BADGE_CLASS} inline-flex items-center gap-0.5">
+					{companion.kind}{companion.quant ? `:${companion.quant}` : ''}
+				</span>
+
+				<span class="-my-1 mx-0.75 w-px self-stretch bg-border"></span>
+
+				<span>{HuggingFaceService.formatFileSize(companion.size)}</span>
+
+				<Download
+					class="h-3.5 w-3.5 text-muted-foreground transition-colors duration-150 group-hover:text-foreground"
+				/>
+			</button>
+		{/snippet}
+	</Tooltip.Trigger>
+
+	<Tooltip.Content>
+		<p>{tooltipText}</p>
+	</Tooltip.Content>
+</Tooltip.Root>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton.svelte

@@ -0,0 +1,272 @@
+<script lang="ts">
+	import ModelDownloadProgressBar from '../../../ModelDownloadProgressBar.svelte';
+	import { labelFor } from './download-options.utils';
+	import { Check, Download, Loader2, Pause, Play, RotateCw, X } from '@lucide/svelte';
+	import * as Tooltip from '$lib/components/ui/tooltip';
+	import { HuggingFaceService } from '$lib/services';
+	import { modelsStore } from '$lib/stores';
+	import type { ModelQuantChip } from '$lib/types';
+	import type { Snippet } from 'svelte';
+
+	interface Props {
+		/** The main quant chip: its file, its download state and its joined drafts. */
+		file: ModelQuantChip;
+		/** Ask the parent to confirm deleting a downloaded model; the chip owns no dialog. */
+		onRequestDelete?: (repoWithTag: string) => void;
+		/** Ask the parent to confirm cancelling an in-flight download. */
+		onRequestCancel?: (repoWithTag: string) => void;
+	}
+
+	let { file, onRequestCancel, onRequestDelete }: Props = $props();
+
+	let entry = $derived(file.state);
+
+	// one source, so the idle, in-flight and downloaded chip states stay identical
+	const SIDECAR_BADGE_CLASS =
+		'rounded-md bg-primary px-1 py-0.5 text-[10px] font-semibold tracking-wide text-primary-foreground uppercase';
+
+	/** Queue the download; a failed attempt leaves partial files, drop them first. */
+	async function startDownload() {
+		try {
+			if (entry.isFailed) await modelsStore.status.cancelDownload(entry.repoWithTag);
+
+			// the joined drafts and the mmproj ride along: the server picks their files
+			await modelsStore.status.downloadModel(entry.repoWithTag, file.sidecars);
+		} catch {
+			// the store already toasted the failure
+		}
+	}
+
+	let label = $derived(labelFor(file.path));
+
+	let percent = $derived(
+		entry.progress && entry.progress.totalBytes > 0
+			? Math.round((entry.progress.downloadedBytes / entry.progress.totalBytes) * 100)
+			: null
+	);
+
+	/** Draft the chip pulls with its quant, e.g. `MTP`; null when none matches. */
+	let companion = $derived(file.companion);
+
+	/**
+	 * The joined draft reads once it rides along or lands: a downloaded model
+	 * missing its draft keeps to its own weights, and the draft gets its own chip.
+	 */
+	let showsCompanion = $derived(companion ? companion.onDisk || !entry.isDownloaded : false);
+
+	/** What the chip's actions name: the quant file and, when it rides along, its draft. */
+	let downloadLabel = $derived(
+		showsCompanion && companion ? `${file.path} + ${companion.path}` : file.path
+	);
+
+	let tooltipText = $derived(
+		entry.isDownloading
+			? 'Pause downloading'
+			: entry.isPaused
+				? 'Resume downloading'
+				: entry.isDownloaded
+					? `Delete ${downloadLabel}`
+					: entry.isFailed
+						? `Retry download: ${downloadLabel}`
+						: `Download ${downloadLabel}`
+	);
+
+	let downloadBytes = $derived(
+		(file.size ?? 0) + (showsCompanion && companion ? companion.size : 0)
+	);
+</script>
+
+{#snippet chipBody(dividerClass: string)}
+	<!-- label, joined draft and divider: identical on every chip state. The draft
+	     rides with this quant, so it reads as part of the same chip:
+	     `Q4_K_M + [MTP:Q8_0] | 20.0 GB`, the size covering both files. -->
+	<span class="font-medium">{label}</span>
+
+	{#if showsCompanion && companion}
+		<span class="text-muted-foreground/60">+</span>
+
+		<span class="{SIDECAR_BADGE_CLASS} inline-flex items-center gap-0.5">
+			<!-- the draft's own quant, so a 4-bit model paired with a 16-bit draft head reads at a glance -->
+			{companion.kind}{companion.quant ? `:${companion.quant}` : ''}
+
+			{#if companion.onDisk}
+				<Check class="h-2.5 w-2.5" />
+			{/if}
+		</span>
+	{/if}
+
+	<span class={dividerClass}></span>
+{/snippet}
+
+<!-- every chip is a tooltip trigger; the button renders as the trigger's child
+     so no nested button element is created -->
+{#snippet tooltipTrigger(tip: string, button: Snippet<[Record<string, unknown>]>)}
+	<Tooltip.Root>
+		<Tooltip.Trigger>
+			{#snippet child({ props })}
+				{@render button(props)}
+			{/snippet}
+		</Tooltip.Trigger>
+
+		<Tooltip.Content>
+			<p>{tip}</p>
+		</Tooltip.Content>
+	</Tooltip.Root>
+{/snippet}
+
+{#snippet deleteChip(props: Record<string, unknown>)}
+	<!-- downloaded chips are delete actions: green check by default, red X on hover -->
+	<button
+		{...props}
+		aria-label={tooltipText}
+		class="group relative inline-flex h-auto cursor-pointer items-center gap-1 rounded-md! border px-2 py-1 text-left font-mono text-xs shadow-sm transition-[background-color,border-color,transform] duration-200 ease-[cubic-bezier(0.23,1,0.32,1)] active:scale-[0.97]
+					border-green-600/25 bg-green-500/5 hover:border-destructive/50 hover:bg-destructive/10 dark:border-green-500/30 dark:bg-green-500/10 dark:hover:border-destructive/50 dark:hover:bg-destructive/15"
+		onclick={() => onRequestDelete?.(entry.repoWithTag)}
+		type="button"
+	>
+		{@render chipBody(
+			'-my-1 mx-0.75 w-px self-stretch bg-green-600/25 transition-colors duration-200 group-hover:bg-destructive/30 dark:bg-green-600/30 dark:group-hover:bg-destructive/30'
+		)}
+
+		<span>{HuggingFaceService.formatFileSize(downloadBytes)}</span>
+
+		<!-- icon slot: crossfade check -> x; touch devices show the delete affordance directly -->
+		<span class="relative inline-flex h-3.5 w-3.5 shrink-0 items-center justify-center">
+			<Check
+				class="absolute h-3.5 w-3.5 text-green-500 transition-[opacity,transform] duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:scale-75 group-hover:opacity-0 [@media(pointer:coarse)]:hidden"
+			/>
+
+			<X
+				class="absolute h-3.5 w-3.5 scale-75 text-destructive opacity-0 transition-[opacity,transform] duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:scale-100 group-hover:opacity-100 [@media(pointer:coarse)]:scale-100 [@media(pointer:coarse)]:opacity-100"
+			/>
+		</span>
+	</button>
+{/snippet}
+
+{#snippet pauseResumeChip(props: Record<string, unknown>)}
+	<button
+		{...props}
+		aria-label={tooltipText}
+		class="flex h-auto min-w-0 flex-1 cursor-pointer items-center gap-1 text-left"
+		onclick={() => {
+			if (entry.isDownloading) void modelsStore.status.pauseDownload(entry.repoWithTag);
+			else void modelsStore.status.downloadModel(entry.repoWithTag).catch(() => {});
+		}}
+		type="button"
+	>
+		{@render chipBody('-my-1 mx-0.75 w-px self-stretch bg-border')}
+
+		{#if percent !== null}
+			<span class="mr-1 tabular-nums">{percent}%</span>
+		{:else if entry.isPaused}
+			<span>Paused</span>
+		{:else}
+			<span>{HuggingFaceService.formatFileSize(downloadBytes)}</span>
+		{/if}
+
+		{#if entry.isDownloading}
+			<!-- spinner fades into the pause affordance on hover; opacity only, the
+			     spin keyframes own the transform so scale would fight them -->
+			<span class="relative inline-flex h-3.5 w-3.5 shrink-0 items-center justify-center">
+				<Loader2
+					class="absolute h-3.5 w-3.5 animate-spin text-muted-foreground transition-opacity duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:opacity-0 [@media(pointer:coarse)]:hidden"
+				/>
+
+				<Pause
+					class="absolute h-3.5 w-3.5 opacity-0 transition-opacity duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:opacity-100 [@media(pointer:coarse)]:opacity-100"
+				/>
+			</span>
+		{:else}
+			<!-- paused: the state at rest, the resume affordance taking its place on
+			     hover, the way the spinner gives way to the pause action above -->
+			<span class="relative inline-flex h-3.5 w-3.5 shrink-0 items-center justify-center">
+				<Pause
+					class="absolute h-3.5 w-3.5 text-muted-foreground transition-opacity duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:opacity-0 [@media(pointer:coarse)]:hidden"
+				/>
+
+				<Play
+					class="absolute h-3.5 w-3.5 opacity-0 transition-opacity duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:opacity-100 [@media(pointer:coarse)]:opacity-100"
+				/>
+			</span>
+		{/if}
+	</button>
+{/snippet}
+
+{#snippet cancelChip(props: Record<string, unknown>)}
+	<!-- cancel slot: same fixed slot and fade as the pause / play affordances
+	     so the two icons line up exactly; the X turns destructive on its hover -->
+	<button
+		{...props}
+		aria-label="Cancel downloading"
+		class="relative grid h-3.5 w-3.5 shrink-0 cursor-pointer items-center justify-center text-muted-foreground/70"
+		onclick={() => onRequestCancel?.(entry.repoWithTag)}
+		type="button"
+	>
+		<X
+			class="h-3.5 w-3.5 transition-[opacity,transform,color] duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] text-destructive [@media(pointer:coarse)]:scale-100 [@media(pointer:coarse)]:opacity-100"
+		/>
+	</button>
+{/snippet}
+
+{#snippet downloadChip(props: Record<string, unknown>)}
+	<!-- idle chips download on click (retry when the last attempt failed) -->
+	<button
+		{...props}
+		aria-label={tooltipText}
+		class="group relative inline-flex h-auto cursor-pointer items-center gap-1 overflow-hidden rounded-md! border px-2 py-1 text-left font-mono text-xs shadow-sm transition-[background-color,border-color,transform] duration-200 ease-[cubic-bezier(0.23,1,0.32,1)] active:scale-[0.97]
+				border-border/30 bg-background hover:bg-muted-foreground/10 dark:border-border/20 dark:bg-muted-foreground/15 dark:text-secondary-foreground dark:hover:bg-muted-foreground/25
+				{entry.isFailed ? 'border-destructive!' : ''}"
+		onclick={() => void startDownload()}
+		type="button"
+	>
+		{#if entry.isFailed}
+			<span
+				class="rounded-md bg-destructive px-1 py-0.5 text-[10px] font-semibold tracking-wide text-destructive-foreground uppercase"
+			>
+				Failed
+			</span>
+		{/if}
+
+		{@render chipBody('-my-1 mx-0.75 w-px self-stretch bg-border')}
+
+		<span>{HuggingFaceService.formatFileSize(downloadBytes)}</span>
+
+		<span class="inline-flex h-3.5 w-3.5 shrink-0 items-center justify-center">
+			{#if entry.isFailed}
+				<RotateCw class="h-3.5 w-3.5 text-destructive" />
+			{:else}
+				<Download
+					class="h-3.5 w-3.5 text-muted-foreground transition-colors duration-150 group-hover:text-foreground"
+				/>
+			{/if}
+		</span>
+	</button>
+{/snippet}
+
+{#if entry.isDownloaded}
+	{@render tooltipTrigger(tooltipText, deleteChip)}
+{:else if entry.isDownloading || entry.isPaused}
+	<!-- in-flight / paused chips: the chip body pauses / resumes on click, the X
+	     inside the chip cancels the download. The X slot is reserved, so the chip
+	     never reflows when the affordance fades in -->
+	<div
+		class="group relative inline-flex h-auto items-center gap-1 overflow-hidden rounded-md! border px-2 py-1 text-left font-mono text-xs shadow-sm transition-[background-color,border-color,transform] duration-200 ease-[cubic-bezier(0.23,1,0.32,1)] active:scale-[0.97]
+			{entry.isPaused
+			? 'border-yellow-600/40 bg-yellow-500/10 hover:bg-yellow-500/20 dark:border-yellow-500/30 dark:bg-yellow-500/10'
+			: 'border-border/30 bg-background hover:bg-muted-foreground/10 dark:border-border/20 dark:bg-muted-foreground/15'}"
+	>
+		{@render tooltipTrigger(tooltipText, pauseResumeChip)}
+
+		{@render tooltipTrigger('Cancel downloading', cancelChip)}
+
+		{#if percent !== null}
+			<ModelDownloadProgressBar
+				downloadedBytes={entry.progress?.downloadedBytes ?? 0}
+				overlay
+				totalBytes={entry.progress?.totalBytes ?? 0}
+			/>
+		{/if}
+	</div>
+{:else}
+	{@render tooltipTrigger(tooltipText, downloadChip)}
+{/if}

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsRow.svelte

@@ -0,0 +1,134 @@
+<script lang="ts">
+	import ModelsDiscoverDetailsDownloadOptionsDownloadCommand from './ModelsDiscoverDetailsDownloadOptionsDownloadCommand.svelte';
+	import ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton from './ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton.svelte';
+	import ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton from './ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton.svelte';
+	import { ChevronDown } from '@lucide/svelte';
+	import {
+		BIT_DEPTH_LABEL_SUFFIX,
+		GIGABYTE_LABEL,
+		OTHER_BIT_DEPTH,
+		OTHER_BIT_DEPTH_LABEL
+	} from '$lib/constants';
+	import type { ModelDraftSidecar } from '$lib/enums';
+	import type { ModelQuantChip } from '$lib/types';
+	import { minMemoryTierGb } from '$lib/utils';
+	import { cubicOut } from 'svelte/easing';
+	import { slide } from 'svelte/transition';
+
+	interface Props {
+		/** Bit depth of the row; `99` renders as "Other". */
+		bitDepth: number;
+		/** The row's main quant chips. */
+		files: ModelQuantChip[];
+		/** Full HuggingFace repo id, for the row's command preview. */
+		modelId: string;
+		/** Draft kind the panel pulls with the model, null when the sidecars are off. */
+		draftKind?: ModelDraftSidecar | null;
+		/** Draft kinds the repo ships, for the command's own pick. */
+		draftKinds?: ModelDraftSidecar[];
+		/** Forwarded to the command: the panel follows its draft pick. */
+		onDraftKindChange?: (kind: ModelDraftSidecar) => void;
+		/** Forwarded to each chip: ask the parent to confirm a cancel. */
+		onRequestCancel?: (repoWithTag: string) => void;
+		/** Forwarded to each chip: ask the parent to confirm a delete. */
+		onRequestDelete?: (repoWithTag: string) => void;
+	}
+
+	let {
+		bitDepth,
+		draftKind = null,
+		draftKinds = [],
+		files,
+		modelId,
+		onDraftKindChange,
+		onRequestCancel,
+		onRequestDelete
+	}: Props = $props();
+
+	// collapsed until asked for; the panel remounts per model
+	let commandOpen = $state(false);
+
+	let mainFile = $derived(files[0] ?? null);
+	let mainMemGb = $derived(mainFile ? minMemoryTierGb(mainFile.size ?? 0) : null);
+	// the row's weights plus, when one rides along, the draft's own tier
+	let totalMemGb = $derived.by(() => {
+		if (mainMemGb === null) return null;
+
+		const draftGb = mainFile?.companion?.size ? minMemoryTierGb(mainFile.companion.size) : 0;
+
+		return mainMemGb + (draftGb ?? 0);
+	});
+
+	/** Drafts a downloaded quant is missing, each getting a chip of its own to add. */
+	let missingDrafts = $derived(
+		files.flatMap((file) =>
+			file.state.isDownloaded && file.companion && !file.companion.onDisk
+				? [{ companion: file.companion, repoWithTag: file.state.repoWithTag }]
+				: []
+		)
+	);
+</script>
+
+<div class="py-3">
+	<div class="grid grid-cols-[5rem_1fr] items-center gap-3 @max-lg:grid-cols-1 @max-lg:gap-1.5">
+		<!-- the row's own label opens its command: it pins a quant of this row, so
+		     the copy matches what the chips download -->
+		<button
+			aria-expanded={commandOpen}
+			class="flex w-fit cursor-pointer flex-col items-start gap-0.5 text-left text-sm tabular-nums text-muted-foreground transition-colors hover:text-foreground @max-lg:flex-row @max-lg:items-baseline @max-lg:gap-2"
+			onclick={() => (commandOpen = !commandOpen)}
+			type="button"
+		>
+			<span class="flex items-center gap-1">
+				{#if bitDepth === OTHER_BIT_DEPTH}
+					{OTHER_BIT_DEPTH_LABEL}
+				{:else}
+					{bitDepth}{BIT_DEPTH_LABEL_SUFFIX}
+				{/if}
+
+				<ChevronDown
+					class="h-3 w-3 shrink-0 transition-transform duration-200 ease-[cubic-bezier(0.23,1,0.32,1)] {commandOpen
+						? 'rotate-180'
+						: ''}"
+				/>
+			</span>
+
+			{#if totalMemGb}
+				<span
+					class="text-[10px] whitespace-nowrap text-muted-foreground/60 @max-lg:whitespace-normal"
+				>
+					needs at least {totalMemGb}{GIGABYTE_LABEL} memory
+				</span>
+			{/if}
+		</button>
+
+		<div class="flex flex-wrap justify-end gap-1.5 @max-lg:justify-start">
+			{#each files as file (file.path)}
+				<ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton
+					{file}
+					{onRequestCancel}
+					{onRequestDelete}
+				/>
+			{/each}
+
+			{#each missingDrafts as draft (draft.repoWithTag + ':' + draft.companion.kind)}
+				<ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton
+					companion={draft.companion}
+					repoWithTag={draft.repoWithTag}
+				/>
+			{/each}
+		</div>
+	</div>
+
+	{#if commandOpen}
+		<div transition:slide={{ duration: 200, easing: cubicOut }} class="pt-2">
+			<ModelsDiscoverDetailsDownloadOptionsDownloadCommand
+				{draftKind}
+				{draftKinds}
+				{modelId}
+				{onDraftKindChange}
+				quant={mainFile?.quant ?? null}
+			/>
+		</div>
+	{/if}
+</div>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/download-options.utils.ts

@@ -0,0 +1,57 @@
+import { DRAFT_FILE_LABEL, MODEL_ID, PATH_SEPARATOR } from '$lib/constants';
+import { ModelDraftSidecar, ModelSelectableFileKind } from '$lib/enums';
+import { HuggingFaceService } from '$lib/services';
+import type { ModelSelectableFile } from '$lib/types';
+import { isAuxSidecar, isDraftSidecar } from '$lib/utils';
+
+/**
+ * Draft kinds in the order a chip lists them. DFlash wins: on the models that
+ * ship both it drafts a block at a time and stays accurate further out than
+ * MTP's few tokens; a kind not listed here sorts last.
+ */
+export const DRAFT_PREFERENCE_ORDER: readonly ModelDraftSidecar[] = [
+	ModelDraftSidecar.DFLASH,
+	ModelDraftSidecar.MTP
+];
+
+/** Draft sidecar of a file, typed for files already known to be drafts; null when it carries none. */
+export function draftBadgeOf(file: Pick<ModelSelectableFile, 'path'>): ModelDraftSidecar | null {
+	const sidecar = HuggingFaceService.extractQuantMeta(file.path)?.sidecar;
+
+	return sidecar && isDraftSidecar(sidecar) ? sidecar : null;
+}
+
+/** Kind of a file path: the main weights, a draft sidecar, or an aux sidecar (mmproj). */
+export function classify(path: string): ModelSelectableFileKind {
+	const sidecar = HuggingFaceService.extractQuantMeta(path)?.sidecar;
+
+	if (!sidecar) return ModelSelectableFileKind.MAIN;
+
+	return isAuxSidecar(sidecar) ? ModelSelectableFileKind.AUX : ModelSelectableFileKind.DRAFT;
+}
+
+/** Draft kinds best first, so a chip lists its joined badges in the pull order. */
+export function sortDraftKinds(kinds: readonly ModelDraftSidecar[]): ModelDraftSidecar[] {
+	const rank = (kind: ModelDraftSidecar) => {
+		const index = DRAFT_PREFERENCE_ORDER.indexOf(kind);
+
+		return index === -1 ? DRAFT_PREFERENCE_ORDER.length : index;
+	};
+
+	return [...kinds].sort((a, b) => rank(a) - rank(b));
+}
+
+/** Display label of a file: its quant, else the file name without the extension. */
+export function labelFor(path: string): string {
+	const meta = HuggingFaceService.extractQuantMeta(path);
+
+	if (meta?.quant) return meta.quant;
+
+	// Quantless sidecar files: the chip badge already carries the sidecar type,
+	// so the label only marks draft files; aux sidecars badge alone.
+	if (meta?.sidecar) return isDraftSidecar(meta.sidecar) ? DRAFT_FILE_LABEL : '';
+
+	const basename = path.split(PATH_SEPARATOR).pop() ?? path;
+
+	return basename.replace(MODEL_ID.WEIGHT_EXTENSION_REGEX, '');
+}