ggml-cpu: vectorize ggml_v_silu and ggml_v_expf (#30791)

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

.pi/gg/APPEND_SYSTEM.md

@@ -0,0 +1,6 @@
+
+Project-specific instructions:
+
+## Coding
+
+- In `test-backend-ops.cpp`, do not mention specific backends (e.g. Metal, CUDA) in comments

.pi/gg/SYSTEM.md

@@ -1,44 +0,0 @@
-You are a coding agent. Here are some very important rules that you must follow:
-
-General:
-- Be very precise and concise when writing code, comments, explanations, etc.
-- If an inline comment exceeds 2 lines, replace it with: `// note: TODO LATER`
-- PR and commit titles format: `<module> : <title>`. Lookup recents for examples
-- Don't try to build or run the code unless you are explicitly asked to do so
-- Use the `gh` CLI tool when querying PRs, issues, or other GitHub resources
-- When a model name is needed, first try to get it from the `PI_MODEL_NAME` env var before asking the user
-- When a hardware description is needed, first try to get it from the `PI_HARDWARE` env var before asking the user
-- Never read the `AGENTS.md` file
-
-Coding:
-- Never write inline comments in the code unless the user tells you to
-- When in doubt, always refer to the CONTRIBUTING.md file of the project
-- In `test-backend-ops.cpp`, do not mention specific backends (e.g. Metal, CUDA) in comments
-- When referencing issues or PRs in comments, use the format:
-  - C/C++ code: `// ref: <url>`
-  - Other (CMake, etc.): `# ref: <url>`
-
-Pull requests (PRs):
-- New branch names are prefixed with "gg/"
-- Before opening a pull request, ask the user to confirm the description
-- Don't explicitly wrap lines in the PR description (each paragraph and bullet is a single line)
-- When creating a pull request, look for the repository's PR template and follow it
-- For the AI usage disclosure section, write "YES", then these 4 bullets:
-  - Harness: `pi:llama.cpp`
-  - Hardware: `<hardware>`
-  - Model: `<model>`, linked to its Hugging Face page - the link text is the plain model name
-  - Stats: `<stats>`
-- Run `~/.pi/stats.sh "$PI_SESSION_FILE"` to get `<stats>` and write it as
-  `◷ <time> · ↑ <input tokens> · ↓ <output tokens> · ≡ <messages> · ⌨ <user messages>`,
-  e.g. `◷ 12.3m · ↑ 120k · ↓ 45k · ≡ 87 · ⌨ 6`
-- Write these symbols as literal unicode in the PR description, never `:shortcodes:`, and keep
-  commit trailers ASCII
-- If `PI_MODEL_NAME` or `PI_HARDWARE` is not set, ask the user and use their answer for `<model>` or `<hardware>`
-- Always create the pull requests in draft mode
-- Never reply to review comments or post comments on issues/PRs without explicit permission from the user
-
-Commits:
-- On every commit that you make, include an `Assisted-by: pi:llama.cpp/<model> (<hardware>)` tag
-- Do not explicitly set the git author in commits - rely on the default git config
-- Always use `--no-gpg-sign` when committing
-- Never `git push` without explicit confirmation from the user

common/download.cpp

@@ -39,6 +39,9 @@
 // downloader
 //
 
+// the suffix of the file a download writes until it completes
+static const std::string PARTIAL_SUFFIX = ".downloadInProgress";
+
 static void write_etag(const std::string & path, const std::string & etag) {
     const std::string etag_path = path + ".etag";
     fs_write_atomic(std::filesystem::u8path(etag_path), etag);
@@ -186,6 +189,13 @@ static bool common_pull_file(httplib::Client & cli,
         [&](const httplib::Response &response) {
             if (p.downloaded > 0 && response.status != 206) {
                 LOG_WRN("%s: server did not respond with 206 Partial Content for a resume request. Status: %d\n", func, response.status);
+                if (response.status == 416 || response.status == 200) {
+                    // the kept partial cannot be resumed: 416 for a complete or stale one,
+                    // 200 from a server that ignores the range; drop it, so the next
+                    // attempt starts over instead of wedging on the same answer
+                    ofs.close();
+                    remove(path_tmp.c_str());
+                }
                 return false;
             }
             if (p.downloaded == 0 && response.status != 200) {
@@ -214,7 +224,7 @@ static bool common_pull_file(httplib::Client & cli,
             if (progress_step >= p.total / 1000 || p.downloaded == p.total) {
                 if (callback) {
                     callback->on_update(p);
-                    if (callback->is_cancelled()) {
+                    if (callback->is_cancelled() && (p.total == 0 || p.downloaded != p.total)) {
                         return false;
                     }
                 }
@@ -331,10 +341,24 @@ static int common_download_file_single_online(const std::string & url,
     }
 
     bool success = false;
-    const std::string path_temporary = path + ".downloadInProgress";
+    const std::string path_temporary = path + PARTIAL_SUFFIX;
     int delay = retry_delay_seconds;
 
+    if (!skip_etag && !etag.empty()) {
+        if (std::filesystem::exists(path_temporary) && read_etag(path_temporary) != etag) {
+            LOG_WRN("%s: partial file does not match the server etag, discarding it: %s\n", __func__, path_temporary.c_str());
+            if (remove(path_temporary.c_str()) != 0) {
+                LOG_ERR("%s: unable to delete file: %s\n", __func__, path_temporary.c_str());
+                return -1;
+            }
+        }
+        write_etag(path_temporary, etag);
+    }
+
     if (opts.callback) {
+        // report the resume offset, so a resumed file does not start at 0
+        p.downloaded = supports_ranges && std::filesystem::exists(path_temporary) ?
+                       std::filesystem::file_size(path_temporary) : 0;
         opts.callback->on_start(p);
     }
 
@@ -372,6 +396,7 @@ static int common_download_file_single_online(const std::string & url,
             }
             if (!etag.empty() && !skip_etag) {
                 write_etag(path, etag);
+                remove((path_temporary + ".etag").c_str());
             }
             success = true;
             break;
@@ -381,12 +406,7 @@ static int common_download_file_single_online(const std::string & url,
     if (opts.callback) {
         opts.callback->on_done(p, success);
     }
-    if (opts.callback && opts.callback->is_cancelled() &&
-        std::filesystem::exists(path_temporary)) {
-        if (remove(path_temporary.c_str()) != 0) {
-            LOG_ERR("%s: unable to delete temporary file: %s\n", __func__, path_temporary.c_str());
-        }
-    }
+    // a stopped download keeps path_temporary, so the next attempt resumes from it
     if (!success) {
         LOG_ERR("%s: download failed after %d attempts\n", __func__, max_attempts);
         return -1; // max attempts reached
@@ -594,33 +614,65 @@ static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files,
     return best;
 }
 
-static hf_cache::hf_file find_best_mmproj(const hf_cache::hf_files & files,
-                                          const std::string        & model) {
-    return find_best_sibling(files, model, "mmproj");
+static const std::vector<common_download_sidecar> sidecars = {
+    { "mmproj", "mmproj",  false, &common_download_opts::download_mmproj, &common_download_hf_plan::mmproj },
+    { "mtp",    "mtp-",    true,  &common_download_opts::download_mtp,    &common_download_hf_plan::mtp    },
+    { "dflash", "dflash-", true,  &common_download_opts::download_dflash, &common_download_hf_plan::dflash },
+    { "eagle3", "eagle3-", true,  &common_download_opts::download_eagle3, &common_download_hf_plan::eagle3 },
+    { "dspark", "dspark-", true,  &common_download_opts::download_dspark, &common_download_hf_plan::dspark },
+};
+
+const std::vector<common_download_sidecar> & common_download_sidecars() {
+    return sidecars;
 }
 
-static hf_cache::hf_file find_best_mtp(const hf_cache::hf_files & files,
-                                       const std::string        & model,
-                                       const std::string        & tag = "") {
-    return find_best_sibling(files, model, "mtp-", tag);
+bool common_download_opts_add_sidecar(common_download_opts & opts, const std::string & kind) {
+    for (const auto & s : sidecars) {
+        if (kind == s.name) {
+            opts.*s.opt = true;
+            return true;
+        }
+    }
+    return false;
 }
 
-static hf_cache::hf_file find_best_eagle3(const hf_cache::hf_files & files,
-                                          const std::string        & model,
-                                          const std::string        & tag = "") {
-    return find_best_sibling(files, model, "eagle3-", tag);
+std::vector<std::string> common_download_plan_sidecars(const common_download_hf_plan & plan) {
+    std::vector<std::string> out;
+    for (const auto & s : sidecars) {
+        if (!(plan.*s.file).path.empty()) {
+            out.emplace_back(s.name);
+        }
+    }
+    return out;
 }
 
-static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files,
-                                          const std::string        & model,
-                                          const std::string        & tag = "") {
-    return find_best_sibling(files, model, "dflash-", tag);
+std::vector<common_download_partial> common_download_plan_partials(const common_download_hf_plan & plan) {
+    std::vector<common_download_partial> out;
+
+    // a file in the cache resolves to its snapshot entry, a file still to download to its blob
+    auto add = [&](const std::string & kind, const hf_cache::hf_file & f) {
+        if (!f.local_path.empty() && f.local_path != f.final_path) {
+            out.push_back({ kind, f.local_path + PARTIAL_SUFFIX });
+        }
+    };
+
+    for (const auto & f : plan.model_files) {
+        add("", f);
+    }
+    for (const auto & s : sidecars) {
+        add(s.name, plan.*s.file);
+    }
+    return out;
 }
 
-static hf_cache::hf_file find_best_dspark(const hf_cache::hf_files & files,
-                                          const std::string        & model,
-                                          const std::string        & tag = "") {
-    return find_best_sibling(files, model, "dspark-", tag);
+// true when `name` carries the keyword of a sidecar kind
+static bool name_is_sidecar(const std::string & name) {
+    for (const auto & s : sidecars) {
+        if (name.find(s.keyword) != std::string::npos) {
+            return true;
+        }
+    }
+    return false;
 }
 
 static bool gguf_filename_is_model(const std::string & filepath) {
@@ -633,12 +685,7 @@ static bool gguf_filename_is_model(const std::string & filepath) {
         filename = filename.substr(pos + 1);
     }
 
-    return filename.find("mmproj")  == std::string::npos &&
-           filename.find("imatrix") == std::string::npos &&
-           filename.find("mtp-")    == std::string::npos &&
-           filename.find("eagle3-") == std::string::npos &&
-           filename.find("dflash-") == std::string::npos &&
-           filename.find("dspark-") == std::string::npos;
+    return filename.find("imatrix") == std::string::npos && !name_is_sidecar(filename);
 }
 
 static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files,
@@ -730,8 +777,11 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model &
         }
     } else {
         primary = find_best_model(all, tag);
-        // a requested sidecar can resolve on its own, without a full model of the same tag
-        if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3 && !opts.download_dspark) {
+        // a requested draft can resolve on its own, without a full model of the same tag
+        const bool want_draft = std::any_of(sidecars.begin(), sidecars.end(), [&](const common_download_sidecar & s) {
+            return s.draft && opts.*s.opt;
+        });
+        if (primary.path.empty() && !want_draft) {
             LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());
             list_available_gguf_files(all);
             return plan;
@@ -743,24 +793,22 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model &
         plan.model_files = get_split_files(all, primary);
     }
 
-    if (opts.download_mmproj && !primary.path.empty()) {
-        plan.mmproj = find_best_mmproj(all, primary.path);
-    }
-    if (opts.download_mtp) {
-        plan.mtp = find_best_mtp(all, primary.path, tag);
-    }
-    if (opts.download_dflash) {
-        plan.dflash = find_best_dflash(all, primary.path, tag);
-    }
-    if (opts.download_eagle3) {
-        plan.eagle3 = find_best_eagle3(all, primary.path, tag);
-    }
-    if (opts.download_dspark) {
-        plan.dspark = find_best_dspark(all, primary.path, tag);
+    // a draft resolves at the tag, the mmproj pairs with the model
+    for (const auto & s : sidecars) {
+        if (!(opts.*s.opt)) {
+            continue;
+        }
+        if (s.draft) {
+            plan.*s.file = find_best_sibling(all, primary.path, s.keyword, tag);
+        } else if (!primary.path.empty()) {
+            plan.*s.file = find_best_sibling(all, primary.path, s.keyword);
+        }
     }
 
-    if (primary.path.empty() &&
-        plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty() && plan.dspark.local_path.empty()) {
+    const bool has_draft = std::any_of(sidecars.begin(), sidecars.end(), [&](const common_download_sidecar & s) {
+        return s.draft && !(plan.*s.file).local_path.empty();
+    });
+    if (primary.path.empty() && !has_draft) {
         LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());
         list_available_gguf_files(all);
     }
@@ -926,6 +974,11 @@ std::string common_docker_resolve_model(const std::string & docker) {
     }
 }
 
+// a cached file that lists a model under its tag: the first split of a model, not a sidecar
+static bool is_listed_model(const gguf_split_info & split) {
+    return split.index == 1 && !split.tag.empty() && !name_is_sidecar(split.prefix);
+}
+
 std::vector<common_cached_model_info> common_list_cached_models() {
     std::unordered_set<std::string> seen;
     std::vector<common_cached_model_info> result;
@@ -934,12 +987,7 @@ std::vector<common_cached_model_info> common_list_cached_models() {
 
     for (const auto & f : files) {
         auto split = get_gguf_split_info(f.path);
-        if (split.index != 1 || split.tag.empty() ||
-            split.prefix.find("mmproj")  != std::string::npos ||
-            split.prefix.find("mtp-")    != std::string::npos ||
-            split.prefix.find("eagle3-") != std::string::npos ||
-            split.prefix.find("dflash-") != std::string::npos ||
-            split.prefix.find("dspark-") != std::string::npos) {
+        if (!is_listed_model(split)) {
             continue;
         }
         if (seen.insert(f.repo_id + ":" + split.tag).second) {
@@ -970,6 +1018,44 @@ std::string common_download_resolve_path(const std::string & hf_repo_with_tag, c
     return find_best_model(files, tag).local_path;
 }
 
+// the files -hf loads from the cache, the model with every sidecar, for each cached
+// tag of `repo_id` other than `tag`
+static std::unordered_set<std::string> files_in_use(const std::string & repo_id, const std::string & tag) {
+    common_download_opts opts;
+    opts.offline = true;
+    for (const auto & s : sidecars) {
+        opts.*s.opt = true;
+    }
+
+    std::unordered_set<std::string> tags;
+    std::unordered_set<std::string> used;
+
+    for (const auto & f : hf_cache::get_cached_files(repo_id)) {
+        auto split = get_gguf_split_info(f.path);
+        if (!is_listed_model(split) || split.tag == tag || !tags.insert(split.tag).second) {
+            continue;
+        }
+
+        common_params_model model;
+        model.hf_repo = repo_id + ":" + split.tag;
+
+        auto plan = common_download_get_hf_plan(model, opts);
+
+        for (const auto & m : plan.model_files) {
+            used.insert(m.local_path);
+        }
+        for (const auto & s : sidecars) {
+            const auto & sf = plan.*s.file;
+            if (!sf.local_path.empty()) {
+                used.insert(sf.local_path);
+            }
+        }
+    }
+    return used;
+}
+
+static bool remove_snapshot_entries(const std::string & repo_id, const std::vector<std::filesystem::path> & to_remove);
+
 bool common_download_remove(const std::string & hf_repo_with_tag) {
     namespace fs = std::filesystem;
 
@@ -989,11 +1075,13 @@ bool common_download_remove(const std::string & hf_repo_with_tag) {
         return false;
     }
 
-    // collect snapshot entries whose tag matches
+    // collect snapshot entries whose tag matches, but a file another tag loads stays
+    const auto used = files_in_use(repo_id, tag_upper);
+
     std::vector<fs::path> to_remove;
     for (const auto & f : files) {
         auto split = get_gguf_split_info(f.path);
-        if (split.tag == tag_upper) {
+        if (split.tag == tag_upper && used.find(f.local_path) == used.end()) {
             to_remove.emplace_back(f.local_path);
         }
     }
@@ -1002,6 +1090,13 @@ bool common_download_remove(const std::string & hf_repo_with_tag) {
         return false;
     }
 
+    return remove_snapshot_entries(repo_id, to_remove);
+}
+
+// remove snapshot entries of `repo_id`, then the blobs no remaining entry references
+static bool remove_snapshot_entries(const std::string & repo_id, const std::vector<std::filesystem::path> & to_remove) {
+    namespace fs = std::filesystem;
+
     // resolve blob paths from symlinks before deleting snapshot entries
     std::vector<fs::path> blobs_to_check;
     for (const auto & p : to_remove) {
@@ -1053,3 +1148,26 @@ bool common_download_remove(const std::string & hf_repo_with_tag) {
 
     return true;
 }
+
+bool common_download_remove_sidecars(const std::string & hf_repo_with_tag, const common_download_opts & opts) {
+    common_params_model model;
+    model.hf_repo = hf_repo_with_tag;
+
+    common_download_opts cached = opts;
+    cached.offline = true;
+
+    auto plan = common_download_get_hf_plan(model, cached);
+
+    std::vector<std::filesystem::path> to_remove;
+    for (const auto & s : sidecars) {
+        const auto & f = plan.*s.file;
+        if (!f.local_path.empty()) {
+            to_remove.emplace_back(std::filesystem::u8path(f.local_path));
+        }
+    }
+    if (to_remove.empty()) {
+        return false;
+    }
+
+    return remove_snapshot_entries(common_download_split_repo_tag(hf_repo_with_tag).first, to_remove);
+}

common/download.h

@@ -104,10 +104,17 @@ std::string common_docker_resolve_model(const std::string & docker);
 // Remove a cached model from disk
 // input format: "user/model" or "user/model:tag"
 // - if tag is omitted, removes the entire repo cache directory
-// - if tag is present, removes only files matching that tag (and orphaned blobs)
+// - if tag is present, removes only files matching that tag (and orphaned blobs),
+//   except the files another cached tag of the repo still loads
 // returns true if anything was removed
 bool common_download_remove(const std::string & hf_repo_with_tag);
 
+// remove the cached sidecar files `opts` selects for "user/model:tag", as the cached
+// plan resolves them, and leave the model files in place; a sidecar file another tag
+// shares goes for every tag that resolves it
+// returns true if anything was removed
+bool common_download_remove_sidecars(const std::string & hf_repo_with_tag, const common_download_opts & opts);
+
 struct common_download_hf_plan {
     hf_cache::hf_file primary;
     hf_cache::hf_files model_files;
@@ -119,3 +126,33 @@ struct common_download_hf_plan {
     hf_cache::hf_file preset; // if set, only this file is downloaded
 };
 common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts);
+
+// a sidecar kind a download can pull next to its model: mmproj, and the draft heads
+// of --spec-type draft-<name>
+struct common_download_sidecar {
+    const char * name;    // the kind, as the API names it
+    const char * keyword; // the part of a file name that marks a file of this kind
+    bool         draft;   // a draft head, resolved at the tag even without a model
+
+    bool              common_download_opts::*    opt;  // the option that requests it
+    hf_cache::hf_file common_download_hf_plan::* file; // the plan file it resolves to
+};
+
+// every sidecar kind, the single list the resolution, the listing and the removal follow
+const std::vector<common_download_sidecar> & common_download_sidecars();
+
+// select the sidecar `kind` in `opts`; returns false for an unknown kind
+bool common_download_opts_add_sidecar(common_download_opts & opts, const std::string & kind);
+
+// the sidecar kinds `plan` resolves a file for
+std::vector<std::string> common_download_plan_sidecars(const common_download_hf_plan & plan);
+
+// a file `plan` still has to download, at the path its partial takes while it is
+// incomplete; kind is the sidecar it belongs to, empty for the model files
+struct common_download_partial {
+    std::string kind;
+    std::string path;
+};
+
+// the partials the download of `plan` writes, for the files not in the cache yet
+std::vector<common_download_partial> common_download_plan_partials(const common_download_hf_plan & plan);

conversion/base.py

@@ -1625,7 +1625,7 @@ class TextModel(ModelBase):
             self.gguf_writer.add_head_count_kv(n_head_kv)
             logger.info(f"gguf: key-value head count = {n_head_kv}")
 
-        if self.hparams.get("is_causal") is False:
+        if self.hparams.get("is_causal") is False or self.hparams.get("use_bidirectional_attention") is True:
             self.gguf_writer.add_causal_attention(False)
             logger.info("gguf: causal attention = False")
 

conversion/qwen.py

@@ -253,6 +253,11 @@ class Qwen3Model(Qwen2Model):
                     yield from super().modify_tensors(data_torch, name, bid)
                 return
 
+        # output embedding projection (voyage-4-nano)
+        if name == "linear.weight":
+            self.gguf_writer.add_embedding_length_out(data_torch.shape[0])
+            name = "dense_2_out.weight"
+
         yield from super().modify_tensors(data_torch, name, bid)
 
 

docs/backend/snapdragon/developer.md

@@ -48,6 +48,157 @@ Alternatively, users can partition and split the model across multiple virtual s
 tensor-splitting, or row-splitting modes. For user-facing execution modes and examples, see the
 [Snapdragon user guide](README.md#multi-device-execution-modes).
 
+## Tiled Weight Formats
+
+Weight tensors in the Hexagon backend are repacked from flat row-major GGML layouts into 2D tiles in memory.
+Tiling groups 2D matrix blocks contiguously:
+- Eliminates strided row reads when fetching 2D tiles into VTCM via DMA.
+- Matches native HVX vector width (128 bytes) and HMX systolic array dimensions (32x32).
+- Repacking happens on the host during model load (`set_tensor()`) or via runtime repacking (`rt-repack`).
+Implementation: [`htp-repack.cpp`](../../../ggml/src/ggml-hexagon/htp-repack.cpp).
+
+### Tile Dimensions and Grid Layout
+
+Each tile represents a submatrix of `N_rows = 32` rows and `K_cols` columns:
+- Standard types (`Q4_0`, `Q4_1`, `Q8_0`, `IQ4_NL`, `MXFP4`, and K-quants): `K_cols = 32` (`HTP_TENSOR_TILE_N_COLS = 32`).
+- `Q2_0`: `K_cols = 64` (`HTP_TENSOR_TILE_K_Q2_0 = 64`).
+- `Q1_0`: `K_cols = 128` (`HTP_TENSOR_TILE_K_Q1_0 = 128`).
+
+For a weight matrix of logical shape `[ne0, ne1]` (where `ne0 = K`, `ne1 = N`):
+- `n_k_tiles   = ne0 / K_cols`
+- `n_col_tiles = (ne1 + 31) / 32`
+- Tile index for column-tile `kt` and row-tile `ct`: `tile_idx = ct * n_k_tiles + kt`
+- Byte offset in matrix buffer: `tile_offset = tile_idx * tile_size`
+
+### Block-32 Quantized Formats
+
+Block-32 formats use a tile size of 32 rows x 32 columns (`K_cols = 32`):
+
+- **Q4_0** (576 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q4_0`):
+  - **Quants** (bytes 0..511, 512 bytes):
+    Contains 32 rows x 32 columns of 4-bit quants.
+    Divided into 16 column pairs `cp = 0..15` covering columns `2*cp` and `2*cp + 1`.
+    For each column pair `cp`, 32 bytes are stored consecutively for rows `row = 0..31`:
+    `tile[cp * 32 + row]` holds `(quant[row][2*cp + 1] << 4) | quant[row][2*cp]`.
+    Low nibble holds the even column; high nibble holds the odd column.
+  - **Scales** (bytes 512..575, 64 bytes):
+    32 FP16 values (`ggml_half`), storing scale `d` for each row `row = 0..31`.
+
+- **Q8_0** (1088 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q8_0`):
+  - **Quants** (bytes 0..1023, 1024 bytes):
+    Contains 32 rows x 32 columns of signed 8-bit quants (`int8_t`).
+    Divided into 16 column pairs `cp = 0..15` covering columns `col0 = 2*cp` and `col1 = 2*cp + 1`.
+    For each column pair `cp`, 64 bytes store interleaved column elements for each row `row = 0..31`:
+    `tile[cp * 64 + 2 * row + 0] = quant[row][col0]`
+    `tile[cp * 64 + 2 * row + 1] = quant[row][col1]`
+  - **Scales** (bytes 1024..1087, 64 bytes):
+    32 FP16 values (`ggml_half`), storing scale `d` for each row `row = 0..31`.
+
+- **IQ4_NL** (576 bytes per tile, `HTP_TENSOR_TILE_SIZE_IQ4_NL`):
+  - **Quants** (bytes 0..511, 512 bytes):
+    Same nibble transposition and storage layout as `Q4_0`.
+    Contains 32 rows x 32 columns of non-linear 4-bit indices (values 0..15).
+    Divided into 16 column pairs `cp = 0..15`:
+    `tile[cp * 32 + row]` holds `(quant[row][2*cp + 1] << 4) | quant[row][2*cp]`.
+  - **Scales** (bytes 512..575, 64 bytes):
+    32 FP16 values (`ggml_half`), storing scale `d` for each row `row = 0..31`.
+  - **Dequantization**:
+    In HVX/HMX kernels, each 4-bit index is looked up in the 16-entry non-linear table (`iq4_nl_to_fp16_lut`) to produce an FP16 value, then multiplied by the row scale `d`.
+
+- **MXFP4** (544 bytes per tile, `HTP_TENSOR_TILE_SIZE_MXFP4`):
+  - **Quants** (bytes 0..511, 512 bytes):
+    Same nibble transposition and storage layout as `Q4_0`.
+    Contains 32 rows x 32 columns of 4-bit OCP Microscaling FP4 (E2M1) values.
+    Divided into 16 column pairs `cp = 0..15`:
+    `tile[cp * 32 + row]` holds `(quant[row][2*cp + 1] << 4) | quant[row][2*cp]`.
+  - **Scales** (bytes 512..543, 32 bytes):
+    32 unsigned 8-bit values (`uint8_t`), storing one 8-bit biased exponent (E8M0, bias 127) scale `e` per row `row = 0..31`.
+  - **Dequantization**:
+    Kernel converts E8M0 bias to IEEE FP16 bias via `vh = clamp(e - 112, 0, 30) << 10` to form a power-of-two FP16 scale.
+    Each 4-bit FP4 nibble is unpacked via `mxfp4_to_fp16_lut` and multiplied by this scale.
+
+- **Q4_1** (640 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q4_1`):
+  - **Quants** (bytes 0..511, 512 bytes):
+    Same 4-bit nibble transposition and storage layout as `Q4_0`.
+  - **Scales** (bytes 512..639, 128 bytes):
+    32 pairs of FP16 values (`ggml_half`): `scale[2 * row + 0] = d` (scale) and `scale[2 * row + 1] = m` (min) for each row `row = 0..31`.
+
+### Q1_0 Format (1-Bit Quantization)
+
+`Q1_0` packs 128 columns per tile (`K_cols = 128`, `HTP_TENSOR_TILE_K_Q1_0 = 128`):
+
+- **Tile Layout** (576 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q1_0`):
+  - **Quants** (bytes 0..511, 512 bytes):
+    Contains 32 rows x 128 columns of 1-bit quants (-1 or +1).
+    Divided into 4 quarters (`quarter = 0..3`), where each quarter covers 32 columns in K.
+    Each quarter occupies 128 bytes (1 HVX vector) in `vrmpy` byte order.
+    Within a quarter, columns `kq = 0..31` map into 8 groups `g = kq / 4` (0..7) of 4 consecutive columns:
+    `byte_offset = quarter * 128 + 4 * row + (kq % 4)`
+    `bit = g` (0..7)
+  - **Scales** (bytes 512..575, 64 bytes):
+    32 FP16 values (`ggml_half`), storing scale `d` for each row `row = 0..31`.
+    One scale per row applies to all 128 columns (all 4 quarters) of the tile.
+
+- **HMX Mapping (1-to-4 Tile Expansion)**:
+  - HMX hardware operates strictly on 32x32 FP16 tiles (`HTP_TENSOR_TILE_N_ROWS = 32`, `HTP_TENSOR_TILE_N_COLS = 32`).
+  - During kernel execution, HVX dequantization (`dequantize_tiled_weight_to_fp16_task_q1_0`) expands each 32x128 Q1_0 tile (576 bytes) into 4 contiguous 32x32 FP16 HMX tiles (4 x 2048 bytes = 8192 bytes total) in VTCM.
+  - HMX systolic array processes the 4 resulting 32x32 FP16 tiles over 4 compute steps along K.
+
+### Q2_0 Format (2-Bit Quantization)
+
+`Q2_0` packs 64 columns per tile (`K_cols = 64`, `HTP_TENSOR_TILE_K_Q2_0 = 64`):
+
+- **Tile Layout** (576 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q2_0`):
+  - **Quants** (bytes 0..511, 512 bytes):
+    Contains 32 rows x 64 columns of 2-bit quants (codes 0..3 representing signed values -1..2).
+    Divided into 16 column groups `group = 0..15` of 4 columns each.
+    Stored in `vrmpy` byte order across 4 HVX vectors (vec 0..3):
+    Vector `group >> 2` (0..3) holds `group & 3` at bit offset `2 * (group & 3)`.
+    Byte `4 * row + byte` holds row `row = 0..31` and column byte `0..3`.
+    Vec 0..1 holds the first 32 columns (half 0), vec 2..3 holds the next 32 columns (half 1).
+  - **Scales** (bytes 512..575, 64 bytes):
+    32 FP16 values (`ggml_half`), storing scale `d` for each row `row = 0..31` (at offset `512 + 2 * row`).
+    One scale per row applies to all 64 columns of the tile.
+
+- **HMX Mapping (1-to-2 Tile Expansion)**:
+  - HMX hardware operates strictly on 32x32 FP16 tiles (`HTP_TENSOR_TILE_N_ROWS = 32`, `HTP_TENSOR_TILE_N_COLS = 32`).
+  - During kernel execution, HVX dequantization (`dequantize_tiled_weight_to_fp16_task_q2_0`) expands each 32x64 Q2_0 tile (576 bytes) into 2 contiguous 32x32 FP16 HMX tiles (2 x 2048 bytes = 4096 bytes total) in VTCM.
+  - HMX systolic array processes the 2 resulting 32x32 FP16 tiles over 2 compute steps along K.
+
+- **HVX Vector Dot-Product**:
+  - `tiled_vec_dot_q2_0_32x1` and `tiled_vec_dot_q2_0_32x2` unpack 2-bit quants using `unpack_q3_k_low2`, subtract 1, and accumulate against Q8_0 activation tiles using `Q6_Vw_vrmpyacc_VwVbVb`.
+
+### K-Quants Formats
+
+In GGML, K-quants pack 256 elements per super-block.
+On Hexagon, each 256-element super-block splits along K into 8 tiles of 32 columns (`K_cols = 32`).
+
+Byte ordering: `Q2_K`, `Q3_K`, and `Q6_K` use `vrmpy` byte order within each 128B HVX vector:
+`k = 0..31` is split into 8 groups `g = k/4` of 4 `k` each; byte `4*r + (k%4)` holds row `r`.
+
+- **Q2_K** (336 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q2_K`, lossless tiling with same size as `block_q2_K`):
+  - `bytes   0..255` (vec 0..1): unsigned 2-bit quants; vec `m` holds groups `4m..4m+3` at bit offsets 0, 2, 4, 6.
+  - `bytes 256..319`: raw `scales[]` bytes (`min << 4 | sc`): `[0..31]` for k 0..15, `[32..63]` for k 16..31.
+  - `bytes 320..335`: 8 FP16 in tile `j` (0..7) of super-block: `d` (`j < 4`) or `-dmin` (`j >= 4`) for rows `8*(j%4)..8*(j%4)+7`.
+
+- **Q3_K** (512 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q3_K`):
+  - `bytes   0..255` (vec 0..1): low 2 bits; vec `m` holds groups `4m..4m+3` at bit offsets 0, 2, 4, 6.
+  - `bytes 256..383` (vec 2): sign mask, bit `g` set when `hmask` is 0 (quant = `low2 - 4`).
+  - `bytes 384..511` (vec 3): FP16 scales `d * (scales[] - 32)`: `fp16[0..31]` for k 0..15, `fp16[32..63]` for k 16..31.
+
+- **Q4_K** (640 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q4_K`, same layout as Q4_1 tile; each 256-k super-block splits into 8 tiles of 32 k):
+  - `bytes   0..511` (vec 0..3): 4-bit quants; byte `32*cp + r` holds `k = 2*cp` (low nibble) and `2*cp + 1` (high nibble).
+  - `bytes 512..639` (vec 4): 32x2 FP16 per row: `(d * sc, -dmin * m)`.
+
+- **Q5_K** (768 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q5_K`):
+  - `bytes   0..639` (vec 0..4): Q4_K tile (low 4 bits + FP16 scale/min pairs).
+  - `bytes 640..767` (vec 5): 5th-bit plane; byte `l` holds the 8 high bits for lane `l` across vec `i = 0..3` (`bit 2i = lo, 2i+1 = hi`).
+
+- **Q6_K** (896 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q6_K`):
+  - `bytes   0..511` (vec 0..3): low 4 bits; vec `i` holds group `2i` (low nibble) and `2i+1` (high nibble).
+  - `bytes 512..767` (vec 4..5): high 2 bits; vec `4+m` holds groups `4m..4m+3` at bit offsets 0, 2, 4, 6.
+  - `bytes 768..895` (vec 6): FP16 scales `d * scales[]`: `fp16[0..31]` for k 0..15, `fp16[32..63]` for k 16..31.
+
 ## Op and Kernel Development Guidelines
 
 Writing high-performance operators for Hexagon requires following specific guidelines.

ggml/src/ggml-cpu/ops.cpp

@@ -1292,24 +1292,37 @@ static void ggml_compute_forward_sum_f32(
     }
 
     assert(ggml_is_scalar(dst));
-    assert(src0->nb[0] == sizeof(float));
 
     GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
     GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb)
 
-    ggml_float sum     = 0;
-    ggml_float row_sum = 0;
+    ggml_float sum = 0;
 
-    for (int64_t i03 = 0; i03 < ne03; i03++) {
-        for (int64_t i02 = 0; i02 < ne02; i02++) {
-            for (int64_t i01 = 0; i01 < ne01; i01++) {
-                ggml_vec_sum_f32_ggf(ne00,
-                        &row_sum,
-                        (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));
-                sum += row_sum;
+    if (nb00 == sizeof(float)) {
+        ggml_float row_sum = 0;
+        for (int64_t i03 = 0; i03 < ne03; i03++) {
+            for (int64_t i02 = 0; i02 < ne02; i02++) {
+                for (int64_t i01 = 0; i01 < ne01; i01++) {
+                    ggml_vec_sum_f32_ggf(ne00,
+                            &row_sum,
+                            (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));
+                    sum += row_sum;
+                }
+            }
+        }
+    } else {
+        for (int64_t i03 = 0; i03 < ne03; i03++) {
+            for (int64_t i02 = 0; i02 < ne02; i02++) {
+                for (int64_t i01 = 0; i01 < ne01; i01++) {
+                    const char * row = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
+                    for (int64_t i00 = 0; i00 < ne00; i00++) {
+                        sum += (ggml_float) *(const float *) (row + i00*nb00);
+                    }
+                }
             }
         }
     }
+
     ((float *) dst->data)[0] = sum;
 }
 

ggml/src/ggml-cpu/vec.cpp

@@ -401,6 +401,10 @@ void ggml_vec_silu_f32(const int n, float * y, const float * x) {
     for (; i + 3 < n; i += 4) {
         vst1q_f32(y + i, ggml_v_silu(vld1q_f32(x + i)));
     }
+#elif defined(__VXE__) || defined(__VXE2__)
+    for (; i + 3 < n; i += 4) {
+        vec_xst(ggml_v_silu(vec_xl(0, x + i)), 0, y + i);
+    }
 #elif defined(__riscv_v_intrinsic)
     for (int vl; i < n; i += vl) {
         vl = __riscv_vsetvl_e32m2(n - i);
@@ -438,6 +442,10 @@ void ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float *
     for (; i + 3 < n; i += 4) {
         vst1q_f32(y + i, vmulq_f32(ggml_v_silu(vld1q_f32(x + i)), vld1q_f32(g + i)));
     }
+#elif defined(__VXE__) || defined(__VXE2__)
+    for (; i + 3 < n; i += 4) {
+        vec_xst(vec_mul(ggml_v_silu(vec_xl(0, x + i)), vec_xl(0, g + i)), 0, y + i);
+    }
 #elif defined(__riscv_v_intrinsic)
     for (int vl; i < n; i += vl) {
         vl = __riscv_vsetvl_e32m2(n - i);
@@ -581,6 +589,12 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float
         vst1q_f32(y + i, val);
         sum += (ggml_float)vaddvq_f32(val);
     }
+#elif defined(__VXE__) || defined(__VXE2__)
+    for (; i + 3 < n; i += 4) {
+        float32x4_t v_val = ggml_v_expf(vec_sub(vec_xl(0, x + i), vec_splats(max)));
+        vec_xst(v_val, 0, y + i);
+        sum += (ggml_float)vec_hsum_f32x4(v_val);
+    }
 #elif defined(__riscv_v_intrinsic)
     vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1);
     for (int avl; i < n; i += avl) {

ggml/src/ggml-cpu/vec.h

@@ -1334,6 +1334,46 @@ inline static __m128 ggml_v_silu(__m128 x) {
     return _mm_div_ps(x, one_plus_exp_neg_x);
 }
 
+#elif defined(__VXE__) || defined(__VXE2__)
+
+// adapted from arm limited optimized routine
+// the maximum error is 1.45358 plus 0.5 ulps
+// numbers above 88.38 will flush to infinity
+// numbers beneath -103.97 will flush to zero
+inline static float32x4_t ggml_v_expf(float32x4_t v_x) {
+    const float32x4_t v_r = vec_splats(0x1.8p23f);
+    const float32x4_t v_z = vec_madd(v_x, vec_splats(0x1.715476p+0f), v_r);
+    const float32x4_t v_n = vec_sub(v_z, v_r);
+    const float32x4_t v_b = vec_nmsub(v_n, vec_splats(0x1.7f7d1cp-20f),
+                                      vec_nmsub(v_n, vec_splats(0x1.62e4p-1f), v_x));
+    const uint32x4_t  v_e = vec_sl((uint32x4_t)v_z, 23);
+    const float32x4_t v_k = (float32x4_t)vec_add(v_e, (uint32x4_t)vec_splats(1.0f));
+    const uint32x4_t  v_c = (uint32x4_t)vec_cmpgt(vec_abs(v_n), vec_splats(126.0f));
+    const float32x4_t v_u = vec_mul(v_b, v_b);
+    const float32x4_t v_j = vec_madd(
+        vec_madd(vec_madd(vec_splats(0x1.0e4020p-7f), v_b, vec_splats(0x1.573e2ep-5f)), v_u,
+                 vec_madd(vec_splats(0x1.555e66p-3f), v_b, vec_splats(0x1.fffdb6p-2f))),
+        v_u, vec_mul(vec_splats(0x1.ffffecp-1f), v_b));
+    if (vec_all_eq(v_c, vec_splats(UINT32_C(0)))) {
+        return vec_madd(v_j, v_k, v_k);
+    }
+    const uint32x4_t  v_d  = vec_and((uint32x4_t)vec_cmple(v_n, vec_splats(0.0f)), vec_splats(UINT32_C(0x82000000)));
+    const float32x4_t v_s1 = (float32x4_t)vec_add(v_d, vec_splats(UINT32_C(0x7f000000)));
+    const float32x4_t v_s2 = (float32x4_t)vec_sub(v_e, v_d);
+    return vec_sel(vec_sel(vec_madd(v_k, v_j, v_k), vec_mul(vec_madd(v_s2, v_j, v_s2), v_s1), v_c),
+                   vec_mul(v_s1, v_s1), (uint32x4_t)vec_cmpgt(vec_abs(v_n), vec_splats(192.0f)));
+}
+
+// computes silu x/(1+exp(-x)) in single precision vector
+inline static float32x4_t ggml_v_silu(float32x4_t v_x) {
+    const float32x4_t v_one = vec_splats(1.0f);
+    const float32x4_t v_zero = vec_splats(0.0f);
+    const float32x4_t v_neg_x = vec_sub(v_zero, v_x);
+    const float32x4_t v_exp_neg_x = ggml_v_expf(v_neg_x);
+    const float32x4_t v_one_plus_exp_neg_x = vec_add(v_one, v_exp_neg_x);
+    return vec_div(v_x, v_one_plus_exp_neg_x);
+}
+
 #elif defined(__riscv_v_intrinsic)
 
 // adapted from arm limited optimized routine
@@ -1386,7 +1426,7 @@ inline static vfloat32m2_t ggml_v_silu_m2(vfloat32m2_t x, int vl) {
     return __riscv_vfdiv_vv_f32m2(x, one_plus_exp_neg_x, vl);
 }
 
-#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __riscv_v_intrinsic
+#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __VXE__ / __riscv_v_intrinsic
 
 inline static void ggml_vec_silu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
     for (int i = 0; i < n; ++i) {

ggml/src/ggml-cuda/dsv4-hc.cu

@@ -145,7 +145,8 @@ static __global__ void dsv4_hc_pre_f32(
     dst[i0*sd0 + it*sd1] = scale * sum;
 }
 
-template <bool has_comb>
+// gated: post = gate_scale_out*sigmoid(gate_scale_in*post), the fused SCALE -> SIGMOID -> SCALE chain of qwen4exp
+template <bool has_comb, bool gated>
 static __global__ void dsv4_hc_post_f32(
         const float * x,
         const float * residual,
@@ -167,7 +168,9 @@ static __global__ void dsv4_hc_post_f32(
         int64_t sc2,
         int64_t sd0,
         int64_t sd1,
-        int64_t sd2) {
+        int64_t sd2,
+        float gate_scale_in,
+        float gate_scale_out) {
     ggml_cuda_pdl_lc();
     const int64_t ir = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
     const int64_t nr = n_embd * hc * n_tokens;
@@ -182,7 +185,12 @@ static __global__ void dsv4_hc_post_f32(
     const int64_t idst = (ir / n_embd) % hc;
     const int64_t it   = ir / (n_embd * hc);
 
-    float sum = x[i0*sx0 + it*sx1] * post[idst*sp0 + it*sp1];
+    float p = post[idst*sp0 + it*sp1];
+    if constexpr (gated) {
+        p = gate_scale_out * (1.0f / (1.0f + expf(-(gate_scale_in * p))));
+    }
+
+    float sum = x[i0*sx0 + it*sx1] * p;
     if constexpr (has_comb) {
         for (int64_t isrc = 0; isrc < hc; ++isrc) {
             sum += residual[i0*sr0 + isrc*sr1 + it*sr2] * comb[idst*sc0 + isrc*sc1 + it*sc2];
@@ -272,12 +280,19 @@ void ggml_cuda_op_dsv4_hc_pre(ggml_backend_cuda_context & ctx, ggml_tensor * dst
             scale);
 }
 
-void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * dst,
+        const ggml_tensor * gate_scale_in, const ggml_tensor * gate_scale_out) {
     const ggml_tensor * x        = dst->src[0];
     const ggml_tensor * residual = dst->src[1];
-    const ggml_tensor * post     = dst->src[2];
     const ggml_tensor * comb     = dst->src[3];
 
+    // when fused, read the raw post weights from before the SCALE -> SIGMOID -> SCALE chain
+    const bool          gated = gate_scale_in != nullptr;
+    const ggml_tensor * post  = gated ? gate_scale_in->src[0] : dst->src[2];
+
+    const float gate_in  = gated ? ggml_get_op_params_f32(gate_scale_in,  0) : 1.0f;
+    const float gate_out = gated ? ggml_get_op_params_f32(gate_scale_out, 0) : 1.0f;
+
     GGML_ASSERT(x->type == GGML_TYPE_F32);
     GGML_ASSERT(residual->type == GGML_TYPE_F32);
     GGML_ASSERT(post->type == GGML_TYPE_F32);
@@ -303,7 +318,8 @@ void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * ds
     const dim3 grid_dims((nr + block_size - 1) / block_size, 1, 1);
     const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, ctx.stream());
 
-    auto kernel = comb ? dsv4_hc_post_f32<true> : dsv4_hc_post_f32<false>;
+    auto kernel = comb ? (gated ? dsv4_hc_post_f32<true,  true> : dsv4_hc_post_f32<true,  false>)
+                       : (gated ? dsv4_hc_post_f32<false, true> : dsv4_hc_post_f32<false, false>);
     ggml_cuda_kernel_launch(kernel, launch_params,
             (const float *) x->data, (const float *) residual->data,
             (const float *) post->data, comb ? (const float *) comb->data : nullptr, (float *) dst->data,
@@ -312,5 +328,6 @@ void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * ds
             nbr0 / sizeof(float), nbr1 / sizeof(float), nbr2 / sizeof(float),
             nbp0 / sizeof(float), nbp1 / sizeof(float),
             nbc0 / sizeof(float), nbc1 / sizeof(float), nbc2 / sizeof(float),
-            nbd0 / sizeof(float), nbd1 / sizeof(float), nbd2 / sizeof(float));
+            nbd0 / sizeof(float), nbd1 / sizeof(float), nbd2 / sizeof(float),
+            gate_in, gate_out);
 }

ggml/src/ggml-cuda/dsv4-hc.cuh

@@ -3,4 +3,6 @@
 
 void ggml_cuda_op_dsv4_hc_comb(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
 void ggml_cuda_op_dsv4_hc_pre(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
-void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+// gate_scale_in/gate_scale_out: the SCALE nodes of a fused SCALE -> SIGMOID -> SCALE chain feeding src[2]
+void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * dst,
+        const ggml_tensor * gate_scale_in = nullptr, const ggml_tensor * gate_scale_out = nullptr);

ggml/src/ggml-cuda/fattn-tile.cuh

@@ -955,7 +955,7 @@ static __global__ void flash_attn_tile(
     if (ncols2 == 1) {
         // Branch with out-of-bounds checks.
         int k_VKQ_0 = blockIdx.y*nbatch_fa;
-        while (k_VKQ_0 < k_VKQ_max - nbatch_fa) {
+        while (k_VKQ_0 <= k_VKQ_max - nbatch_fa) {
             constexpr bool oob_check = false;
             flash_attn_tile_iter<warp_size, nwarps, ncols1, ncols2, DKQ, DV, nbatch_fa, nbatch_K, use_logit_softcap, oob_check>
                 (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp,

ggml/src/ggml-cuda/fattn-vec.cuh

@@ -317,7 +317,7 @@ static __global__ void flash_attn_ext_vec(
 #endif // V_DOT2_F32_F16_AVAILABLE
         }
 
-        ggml_cuda_syncwarp();
+        ggml_cuda_syncwarp(); // Needed for KQ synchronization.
 
 #pragma unroll
         for (int k0 = 0; k0 < WARP_SIZE; k0 += V_cols_per_iter) {
@@ -374,6 +374,8 @@ static __global__ void flash_attn_ext_vec(
             }
 #endif // V_DOT2_F32_F16_AVAILABLE
         }
+
+        ggml_cuda_syncwarp(); // Needed for KQ synchronization.
     }
 
     if (sinks && blockIdx.y == 0) {

ggml/src/ggml-cuda/fattn.cu

@@ -417,9 +417,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
 
 #define FATTN_VEC_CASE(D, type_K_case, type_V_case)                                                                                \
     if constexpr (GGML_CUDA_FA_##type_K_case##_##type_V_case) {                                                                    \
-        const bool type_K_okay = type_K == GGML_TYPE_##type_K_case || (type_K == GGML_TYPE_F32 && GGML_TYPE_##type_K_case == GGML_TYPE_F16); \
-        const bool type_V_okay = type_V == GGML_TYPE_##type_V_case || (type_V == GGML_TYPE_F32 && GGML_TYPE_##type_V_case == GGML_TYPE_F16); \
-        if (head_size == (D) && type_K_okay && type_V_okay) {                                                                      \
+        if (head_size == (D) && type_K == GGML_TYPE_##type_K_case && type_V == GGML_TYPE_##type_V_case) {                          \
             return ggml_cuda_flash_attn_ext_vec_case<D, GGML_TYPE_##type_K_case, GGML_TYPE_##type_V_case>;                         \
         }                                                                                                                          \
     }                                                                                                                              \
@@ -432,7 +430,9 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
 typedef void (* fattn_vec_case_t)(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
 
 // Vector kernel for the given head size and K/V types, nullptr if its template instance was not compiled:
-static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, const ggml_type type_K, const ggml_type type_V) {
+static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, const ggml_type type_K_in, const ggml_type type_V_in) {
+    const ggml_type type_K = type_K_in == GGML_TYPE_F32 ? GGML_TYPE_F16 : type_K_in;
+    const ggml_type type_V = type_V_in == GGML_TYPE_F32 ? GGML_TYPE_F16 : type_V_in;
     FATTN_VEC_CASES_ALL_D(F16,  F16)
     FATTN_VEC_CASES_ALL_D(Q4_0, F16)
     FATTN_VEC_CASES_ALL_D(Q4_1, F16)

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -3029,6 +3029,65 @@ static int ggml_cuda_try_ssm_scan_cache_fusion(
     return skip;
 }
 
+// expert group selection from build_moe_ffn (n_expert_groups > 1)
+// src0/src1 index into this table, -1 is the (biased) probs, -2 is not checked
+static const struct {
+    ggml_op op;
+    int     src0;
+    int     src1;
+} topk_moe_group_pattern[] = {
+    { GGML_OP_RESHAPE,  -1, -2 }, //  0: [n_exp_per_group, n_expert_groups, n_tokens]
+    { GGML_OP_RESHAPE,   0, -2 }, //  1: [1, n_exp_per_group, n_expert_groups, n_tokens]
+    { GGML_OP_ARGSORT,   0, -2 }, //  2
+    { GGML_OP_VIEW,      2, -2 }, //  3: top-2 per group
+    { GGML_OP_GET_ROWS,  1,  3 }, //  4
+    { GGML_OP_RESHAPE,   4, -2 }, //  5
+    { GGML_OP_SUM_ROWS,  5, -2 }, //  6: group scores
+    { GGML_OP_RESHAPE,   6, -2 }, //  7
+    { GGML_OP_ARGSORT,   7, -2 }, //  8
+    { GGML_OP_VIEW,      8, -2 }, //  9: top n_group_used groups
+    { GGML_OP_GET_ROWS,  0,  9 }, // 10
+    { GGML_OP_FILL,      0, -2 }, // 11
+    { GGML_OP_SET_ROWS, 10,  9 }, // 12: unused groups -> -inf
+    { GGML_OP_RESHAPE,  12, -2 }, // 13: [n_expert, n_tokens]
+};
+
+static bool ggml_cuda_topk_moe_group_fusion(const struct ggml_cgraph * cgraph, int node_idx, const ggml_tensor * selection, ggml_cuda_topk_moe_args & args) {
+#if defined(GGML_USE_HIP) && HIP_VERSION < 60300000
+    GGML_UNUSED_VARS(cgraph, node_idx, selection, args);
+    return false;
+#else
+    const int n_ops = (int) std::size(topk_moe_group_pattern);
+    if (node_idx + n_ops > cgraph->n_nodes) {
+        return false;
+    }
+
+    ggml_tensor ** nodes = cgraph->nodes + node_idx;
+    auto src_ok = [&](const ggml_tensor * src, int idx) {
+        return idx == -2 || src == (idx == -1 ? selection : nodes[idx]);
+    };
+
+    for (int i = 0; i < n_ops; i++) {
+        const auto & p = topk_moe_group_pattern[i];
+        if (nodes[i]->op != p.op || !src_ok(nodes[i]->src[0], p.src0) || !src_ok(nodes[i]->src[1], p.src1)) {
+            return false;
+        }
+    }
+
+    const int n_groups     = nodes[0]->ne[1];
+    const int n_group_used = nodes[9]->ne[0];
+
+    if (nodes[3]->ne[0] != 2 || nodes[12]->src[2] != nodes[11] || ggml_get_op_params_f32(nodes[11], 0) != -INFINITY ||
+            n_groups > WARP_SIZE) {
+        return false;
+    }
+
+    args.n_groups     = n_groups;
+    args.n_group_used = n_group_used;
+    return true;
+#endif // defined(GGML_USE_HIP) && HIP_VERSION < 60300000
+}
+
 static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int node_idx, ggml_cuda_topk_moe_args & args) {
     args.sigmoid         = false;
     args.sqrt_softplus   = false;
@@ -3036,6 +3095,8 @@ static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int nod
     args.delayed_softmax = false;
     args.prob_bias       = false;
     args.norm            = false;
+    args.n_groups        = 0;
+    args.n_group_used    = 0;
 
     const int      n_nodes = cgraph->n_nodes;
     ggml_tensor ** nodes   = cgraph->nodes;
@@ -3078,18 +3139,20 @@ static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int nod
         }
 
         // src of bias add is the unreshaped probs (-2 instead of -1)
-        if (nodes[node_idx]->op == GGML_OP_ADD && nodes[node_idx]->src[0] == nodes[node_idx - 2]) {
+        const ggml_tensor * selection = nodes[node_idx - 2];
+        if (nodes[node_idx]->op == GGML_OP_ADD && nodes[node_idx]->src[0] == selection) {
             args.prob_bias = true;
+            selection = nodes[node_idx];
             node_idx++;
         }
-        // RESHAPE/ADD -> ARGSORT
-        if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_ARGSORT) {
-            return false;
+
+        if (ggml_cuda_topk_moe_group_fusion(cgraph, node_idx, selection, args)) {
+            node_idx += (int) std::size(topk_moe_group_pattern);
+            selection = nodes[node_idx - 1];
         }
 
-        if (args.prob_bias && nodes[node_idx]->src[0] != nodes[node_idx - 1]) {
-            return false;
-        } else if (!args.prob_bias && nodes[node_idx]->src[0] != nodes[node_idx - 2]) {
+        // RESHAPE/ADD/group mask -> ARGSORT
+        if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_ARGSORT || nodes[node_idx]->src[0] != selection) {
             return false;
         }
 
@@ -3100,6 +3163,11 @@ static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int nod
                 nodes[node_idx]->src[0] != nodes[node_idx - 1]) {
             return false;
         }
+
+        // kept groups must hold all used experts
+        if (args.n_groups > 0 && nodes[node_idx]->ne[0] > args.n_group_used * (probs_reshaped->ne[1] / args.n_groups)) {
+            return false;
+        }
         node_idx++;
 
         if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_GET_ROWS) {
@@ -3482,6 +3550,26 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph *                cgraph,
         }
     }
 
+    // scale_out*sigmoid(scale_in*post) as the hc_post weights (qwen4exp hc_combine)
+    std::initializer_list<enum ggml_op> hc_post_gate_ops = { GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_SCALE, GGML_OP_DSV4_HC_POST };
+
+    if (is_equal(hc_post_gate_ops, ops) && ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx + 3 })) {
+        const ggml_tensor * scale_in  = cgraph->nodes[node_idx];
+        const ggml_tensor * sigmoid   = cgraph->nodes[node_idx + 1];
+        const ggml_tensor * scale_out = cgraph->nodes[node_idx + 2];
+
+        // the chain must feed hc_post->src[2]; a bias on either scale is not handled
+        if (ggml_check_edges(cgraph, node_idx, {{1, 0, 0}, {2, 0, 1}, {3, 2, 2}}) &&
+            ggml_get_unary_op(sigmoid) == GGML_UNARY_OP_SIGMOID &&
+            ggml_get_op_params_f32(scale_in, 1) == 0.0f && ggml_get_op_params_f32(scale_out, 1) == 0.0f &&
+            scale_in->src[0]->type == GGML_TYPE_F32 && scale_out->type == GGML_TYPE_F32 &&
+            ggml_are_same_shape(scale_in->src[0], scale_out)) {
+            int out_nodes[] = { node_idx + 3 };
+            return ggml_cuda_check_fusion_memory_ranges(cgraph, node_idx, (int)ops.size(), out_nodes, 1);
+        }
+        return false;
+    }
+
     if (!ggml_can_fuse(cgraph, node_idx, ops)) {
         return false;
     }
@@ -3754,17 +3842,18 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
                 } else {
                     ops.insert(ops.end(), { GGML_OP_SOFT_MAX });
                 }
-                const int i_probs = i + (int) ops.size() - 1;  // last node of the gating activation
-
+                ops.push_back(GGML_OP_RESHAPE);
                 if (args.prob_bias) {
-                    bias = cgraph->nodes[i_probs + 2]->src[1];
-                    ops.insert(ops.end(), { GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW,
-                                            GGML_OP_GET_ROWS });
-                    out_nodes[0] = i_probs + 4;
-                } else {
-                    ops.insert(ops.end(), { GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS });
-                    out_nodes[0] = i_probs + 3;
+                    ops.push_back(GGML_OP_ADD);
+                    bias = cgraph->nodes[i + ops.size() - 1]->src[1];
                 }
+                if (args.n_groups > 0) {
+                    for (const auto & p : topk_moe_group_pattern) {
+                        ops.push_back(p.op);
+                    }
+                }
+                ops.insert(ops.end(), { GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS });
+                out_nodes[0] = i + ops.size() - 2;
                 ids = cgraph->nodes[out_nodes[0]];
 
                 if (args.norm) {
@@ -4444,6 +4533,11 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
         return 1;
     }
 
+    if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_SCALE, GGML_OP_DSV4_HC_POST }, { GGML_UNARY_OP_SIGMOID })) {
+        ggml_cuda_op_dsv4_hc_post(*cuda_ctx, cgraph->nodes[i + 3], node, cgraph->nodes[i + 2]);
+        return 3;
+    }
+
     if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_SCALE }, { GGML_UNARY_OP_TANH })) {
         ggml_cuda_op_softcap(*cuda_ctx, cgraph->nodes[i + 2], node);
         return 2;
@@ -5828,7 +5922,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
             //return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]);
             return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);
         case GGML_OP_SUM:
-            return ggml_is_contiguous_rows(op->src[0]);
+            return ggml_is_contiguously_allocated(op->src[0]);
         case GGML_OP_TOP_K:
             return op->src[0]->ne[0] <= INT_MAX;
         case GGML_OP_ARGSORT:

ggml/src/ggml-cuda/mmq-load-tiles.cuh

@@ -46,6 +46,25 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
         for (int j = 0; j < 2; ++j) {
             const int q  = qxi[j];
 
+#if defined(GGML_USE_HIP)
+            const uint32_t q_bits = (uint16_t) q;
+
+            // Expand each group of four Q1_0 bits into byte selectors for v_perm_b32.
+            const uint32_t q0_bits = q_bits >>  0;
+            const uint32_t q1_bits = q_bits >>  4;
+            const uint32_t q2_bits = q_bits >>  8;
+            const uint32_t q3_bits = q_bits >> 12;
+
+            const uint32_t q0_indices = (q0_bits & 0x01) | ((q0_bits & 0x02) << 7) | ((q0_bits & 0x04) << 14) | ((q0_bits & 0x08) << 21);
+            const uint32_t q1_indices = (q1_bits & 0x01) | ((q1_bits & 0x02) << 7) | ((q1_bits & 0x04) << 14) | ((q1_bits & 0x08) << 21);
+            const uint32_t q2_indices = (q2_bits & 0x01) | ((q2_bits & 0x02) << 7) | ((q2_bits & 0x04) << 14) | ((q2_bits & 0x08) << 21);
+            const uint32_t q3_indices = (q3_bits & 0x01) | ((q3_bits & 0x02) << 7) | ((q3_bits & 0x04) << 14) | ((q3_bits & 0x08) << 21);
+
+            const int v0 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q0_indices);
+            const int v1 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q1_indices);
+            const int v2 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q2_indices);
+            const int v3 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q3_indices);
+#else
             // unpack crumbs into nibble indices
             const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0); // [0, 1, 4, 5] [ 8,  9, 12, 13]
             const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2); // [2, 3, 6, 7] [10, 11, 14, 15]
@@ -59,6 +78,7 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
             const int v1 = __byte_perm(s0, s1, 0x7632);
             const int v2 = __byte_perm(s2, s3, 0x5410);
             const int v3 = __byte_perm(s2, s3, 0x7632);
+#endif // defined(GGML_USE_HIP)
 
 #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
             x_qs[i*sram_stride           + dst_offset + j*4+0] = v0;

ggml/src/ggml-cuda/mmq.cuh

@@ -1441,10 +1441,12 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
     const int ntiles_dst = ntx * nty * ntzw;
     const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm;
     const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves);
-    const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : nsm, 1, 1);
 
     GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow.
 
+    const int nwork = ntiles_dst * ((args.ncols_x + config.K_vram - 1) / config.K_vram); // Maximum number of CUDA blocks that can do useful work.
+    const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : std::min(nsm, nwork), 1, 1);
+
     const bool fixup_needed = ntiles_dst % block_nums_stream_k.x != 0;
 
     ggml_cuda_pool & pool = ctx.pool(id);

ggml/src/ggml-cuda/mmvq.cu

@@ -468,10 +468,12 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d
     } else if (table_id == MMVQ_PARAMETERS_GCN) {
         switch (ncols_dst) {
             case 1:
+                return 2;
             case 2:
             case 3:
+                return type == GGML_TYPE_Q3_K ? 1 : 2;
             case 4:
-                return 2;
+                return type == GGML_TYPE_Q6_K? 1 : 2;
             case 5:
             case 6:
             case 7:
@@ -576,11 +578,31 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d
     return 1;
 }
 
-static constexpr __host__ __device__ int calc_rows_per_block(int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) {
+// Low bit types load few weight bytes per thread, a second row per block shares the q8_1 loads and adds memory parallelism.
+static constexpr __host__ __device__ bool mmvq_two_rows_ncols1(ggml_type type) {
+    switch (type) {
+        case GGML_TYPE_Q1_0:
+        case GGML_TYPE_Q2_K:
+        case GGML_TYPE_Q3_K:
+        case GGML_TYPE_IQ1_S:
+        case GGML_TYPE_IQ1_M:
+            return true;
+        default:
+            return false;
+    }
+}
+
+static constexpr __host__ __device__ int calc_rows_per_block(ggml_type type, int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) {
     if (table_id == MMVQ_PARAMETERS_GENERIC || table_id == MMVQ_PARAMETERS_GCN || table_id == MMVQ_PARAMETERS_TURING || table_id == MMVQ_PARAMETERS_GB10) {
         switch (ncols_dst) {
             case 1:
-                return small_k ? nwarps : 1;
+                if (small_k) {
+                    return nwarps;
+                }
+                if (table_id == MMVQ_PARAMETERS_GB10) {
+                    return type == GGML_TYPE_IQ2_S || type == GGML_TYPE_IQ3_S ? 2 : 1;
+                }
+                return table_id == MMVQ_PARAMETERS_GENERIC && mmvq_two_rows_ncols1(type) ? 2 : 1;
             case 2:
             case 3:
             case 4:
@@ -615,7 +637,7 @@ static __global__ void mul_mat_vec_q(
     constexpr int vdr = get_vdr_mmvq(type);
     constexpr mmvq_parameter_table_id table_id = get_device_table_id();
     constexpr int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters);
-    constexpr int rows_per_cuda_block = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps);
+    constexpr int rows_per_cuda_block = calc_rows_per_block(type, ncols_dst, table_id, small_k, nwarps);
     constexpr int warp_size = ggml_cuda_get_physical_warp_size();
 
     constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type);
@@ -727,7 +749,8 @@ static __global__ void mul_mat_vec_q(
 #if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
         // start the next iterations' weight loads early
         if constexpr (mmvq_should_prefetch(type)) {
-            constexpr int pf_dist = 2; // loop iterations, not blocks
+            // loop iterations, not blocks; q3_K and q6_K keep the distance in blocks they had with vdr 1
+            constexpr int pf_dist = type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q6_K ? 1 : 2;
             const int kbx_pf = kbx + pf_dist*blocks_per_iter;
             if (kbx_pf < blocks_per_row_x) {
 #pragma unroll
@@ -1013,7 +1036,7 @@ static std::pair<dim3, dim3> calc_launch_params(
         const int ncols_dst, const int nrows_x, const int nchannels_dst, const int nsamples_or_ntokens,
         const int warp_size, const mmvq_parameter_table_id table_id, const bool small_k = false, const bool halve_iters = false) {
     const int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters);
-    const int rpb = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps);
+    const int rpb = calc_rows_per_block(type, ncols_dst, table_id, small_k, nwarps);
     const int64_t nblocks = (nrows_x + rpb - 1) / rpb;
     const dim3 block_nums(nblocks, nchannels_dst, nsamples_or_ntokens);
     const dim3 block_dims(warp_size, nwarps, 1);

ggml/src/ggml-cuda/topk-moe.cu

@@ -11,6 +11,8 @@ struct topk_moe_config {
     bool use_sqrt_softplus;
     bool with_norm;
     bool delayed_softmax;
+    int  n_groups;
+    int  n_group_used;
 };
 
 // Warp-local softmax used for both the pre-top-k logits and the post-top-k delayed path.
@@ -78,6 +80,53 @@ __device__ void sqrt_softplus_warp_inplace(float (&vals)[experts_per_thread], co
     }
 }
 
+#if !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000
+// Set -inf for experts outside the top n_group_used groups. Group score is the sum of its top-2 experts (DeepSeek V3).
+template <int n_experts, int experts_per_thread>
+__device__ void group_mask_warp_inplace(float (&vals)[experts_per_thread], const int n_groups, const int n_group_used, const int lane) {
+    const int n_per_group = n_experts / n_groups;
+
+    float score = -INFINITY; // lane g holds the score of group g
+    for (int g = 0; g < n_groups; g++) {
+        float top1 = -INFINITY;
+        float top2 = -INFINITY;
+#pragma unroll
+        for (int i = 0; i < experts_per_thread; i++) {
+            if ((i * WARP_SIZE + lane) / n_per_group == g) {
+                top2 = fmaxf(top2, fminf(top1, vals[i]));
+                top1 = fmaxf(top1, vals[i]);
+            }
+        }
+#pragma unroll
+        for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) {
+            const float o1 = __shfl_xor_sync(0xFFFFFFFF, top1, offset, WARP_SIZE);
+            const float o2 = __shfl_xor_sync(0xFFFFFFFF, top2, offset, WARP_SIZE);
+            top2 = fmaxf(fminf(top1, o1), fmaxf(top2, o2));
+            top1 = fmaxf(top1, o1);
+        }
+        if (lane == g) {
+            score = top1 + top2;
+        }
+    }
+
+    // rank of each group, lower index wins ties
+    int rank = 0;
+    for (int g = 0; g < n_groups; g++) {
+        const float other = __shfl_sync(0xFFFFFFFF, score, g, WARP_SIZE);
+        rank += other > score || (other == score && g < lane);
+    }
+    const auto keep = __ballot_sync(0xFFFFFFFFULL, lane < n_groups && rank < n_group_used);
+
+#pragma unroll
+    for (int i = 0; i < experts_per_thread; i++) {
+        const int g = (i * WARP_SIZE + lane) / n_per_group;
+        if (g >= n_groups || !((keep >> g) & 1)) {
+            vals[i] = -INFINITY;
+        }
+    }
+}
+#endif // !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000
+
 /*
     This kernel does the following:
     1. optionally softmax over the logits per token [n_experts, n_tokens]
@@ -171,6 +220,17 @@ __global__ void topk_moe_cuda(const float *         logits,
         }
     }
 
+#if !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000
+    // without bias, masking wt is safe: the matcher ensures the kept groups hold all used experts
+    if (config.n_groups > 0) {
+        if constexpr (has_bias) {
+            group_mask_warp_inplace<n_experts>(selection_wt, config.n_groups, config.n_group_used, threadIdx.x);
+        } else {
+            group_mask_warp_inplace<n_experts>(wt, config.n_groups, config.n_group_used, threadIdx.x);
+        }
+    }
+#endif // !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000
+
     //at this point, each thread holds either a portion of the softmax distribution
     //or the raw logits. We do the argmax reduce over n_expert_used, each time marking
     //the expert weight as -inf to exclude from the next iteration
@@ -297,6 +357,10 @@ static void launch_topk_moe_cuda(ggml_backend_cuda_context & ctx,
     cudaStream_t stream = ctx.stream();
     const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream);
 
+#if defined(GGML_USE_HIP) && HIP_VERSION < 60300000
+    GGML_ASSERT(config.n_groups == 0 && "expert groups fusion is not supported on this HIP version");
+#endif
+
     switch (n_expert) {
         case 1:
             ggml_cuda_kernel_launch(topk_moe_cuda<1, has_bias>, launch_params,
@@ -389,6 +453,8 @@ void ggml_cuda_op_topk_moe(ggml_backend_cuda_context &     ctx,
     config.use_sqrt_softplus = args.sqrt_softplus;
     config.with_norm         = with_norm;
     config.delayed_softmax   = args.delayed_softmax;
+    config.n_groups          = args.n_groups;
+    config.n_group_used      = args.n_group_used;
 
     if (bias) {
         launch_topk_moe_cuda<true>(ctx, logits_d, weights_d, ids_d, bias_d, n_rows, n_experts, n_expert_used, clamp_val,

ggml/src/ggml-cuda/topk-moe.cuh

@@ -14,6 +14,8 @@ struct ggml_cuda_topk_moe_args {
     bool prob_bias{};
     bool norm{};
     bool scale{};
+    int  n_groups{};     // > 0: pick experts only from the top n_group_used expert groups
+    int  n_group_used{};
 };
 
 void ggml_cuda_op_topk_moe(ggml_backend_cuda_context &     ctx,

ggml/src/ggml-cuda/vecdotq.cuh

@@ -360,12 +360,12 @@ static __device__ __forceinline__ float vec_dot_nvfp4_q8_1(
 
     return sum;
 }
-#define VDR_Q2_K_Q8_1_MMVQ 1
+#define VDR_Q2_K_Q8_1_MMVQ 2
 #define VDR_Q2_K_Q8_1_MMQ  4
 
 // contiguous v/x values
 static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
-    const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
+    const int * __restrict__ v, const int * __restrict__ u, const int * __restrict__ scales,
     const half2 & dm2, const float * __restrict__ d8) {
 
     float sumf_d = 0.0f;
@@ -373,17 +373,20 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
 
 #pragma unroll
     for (int i = 0; i < QR2_K; ++i) {
-        const int sc = scales[2*i];
+        const int sc = scales[i/2] >> (16*(i%2));
 
-        const int vi = (v >> (2*i)) & 0x03030303;
+        int sumi_d = 0;
+        int sumi_m = 0;
+#pragma unroll
+        for (int j = 0; j < VDR_Q2_K_Q8_1_MMVQ; ++j) {
+            const int vi = (v[j] >> (2*i)) & 0x03030303;
 
-        sumf_d += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product
+            sumi_d = ggml_cuda_dp4a(vi,         u[VDR_Q2_K_Q8_1_MMVQ*i + j], sumi_d); // SIMD dot product
+            sumi_m = ggml_cuda_dp4a(0x01010101, u[VDR_Q2_K_Q8_1_MMVQ*i + j], sumi_m); // sum of q8_1 values
+        }
 
-        // fill int with 4x m
-        int m = sc >> 4;
-        m |= m <<  8;
-        m |= m << 16;
-        sumf_m += d8[i] * ggml_cuda_dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values
+        sumf_d += d8[i] * (sumi_d * (sc & 0xF));
+        sumf_m += d8[i] * (sumi_m * ((sc >> 4) & 0xF)); // multiply constant q2_K part with sum of q8_1 values
     }
 
     const float2 dm2f = __half22float2(dm2);
@@ -443,37 +446,35 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(
     return sumf + d8*sumf_d8;
 }
 
-#define VDR_Q3_K_Q8_1_MMVQ 1
+#define VDR_Q3_K_Q8_1_MMVQ 2
 #define VDR_Q3_K_Q8_1_MMQ  2
 
 // contiguous v/x values
 static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(
-    const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,
-    const int & scale_offset, const float & d3, const float * __restrict__ d8) {
+    const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const int * __restrict__ scales,
+    const float & d3, const float * __restrict__ d8) {
 
     float sumf = 0.0f;
 
 #pragma unroll
     for (int i = 0; i < QR3_K; ++i) {
-        const int isc = scale_offset + 2*i;
+        // 4 low bits from scales[0..7], 2 high bits from scales[8..11]
+        const int sc_low  = (scales[i/2] >> (16*(i%2))) & 0x0F;
+        const int sc_high = (scales[2] >> (16*(i%2) + 2*(i/2))) & 0x03;
+        const int sc = (sc_low | (sc_high << 4)) - 32;
 
-        const int isc_low = isc % (QK_K/32);
-        const int sc_shift_low = 4 * (isc / (QK_K/32));
-        const int sc_low  = (scales[isc_low] >> sc_shift_low) & 0xF;
+        int sumi = 0;
+#pragma unroll
+        for (int j = 0; j < VDR_Q3_K_Q8_1_MMVQ; ++j) {
+            const int vil = (vl[j] >> (2*i)) & 0x03030303;
+            const int vih = ((vh[j] >> i) << 2) & 0x04040404;
 
-        const int isc_high = isc % (QK_K/64);
-        const int sc_shift_high = 2 * (isc / (QK_K/64));
-        const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;
+            // (vil | vih) - 4 for each byte, the - 4 is done by the second dp4a
+            sumi = ggml_cuda_dp4a(vil | vih,  u[VDR_Q3_K_Q8_1_MMVQ*i + j], sumi); // SIMD dot product
+            sumi = ggml_cuda_dp4a(0xFCFCFCFC, u[VDR_Q3_K_Q8_1_MMVQ*i + j], sumi);
+        }
 
-        const int sc = (sc_low | sc_high) - 32;
-
-        const int vil = (vl >> (2*i)) & 0x03030303;
-
-        const int vih = ((vh >> i) << 2) & 0x04040404;
-
-        const int vi = __vsub4(vil, vih);
-
-        sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product
+        sumf += d8[i] * (sumi * sc);
     }
 
     return d3 * sumf;
@@ -620,12 +621,12 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(
     return dm4f.x*sumf_d - dm4f.y*sumf_m;
 }
 
-#define VDR_Q6_K_Q8_1_MMVQ 1
+#define VDR_Q6_K_Q8_1_MMVQ 2
 #define VDR_Q6_K_Q8_1_MMQ  8
 
 // contiguous v/x values
 static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
-    const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,
+    const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const int8_t * __restrict__ scales,
     const float & d, const float * __restrict__ d8) {
 
     float sumf = 0.0f;
@@ -634,13 +635,18 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
     for (int i = 0; i < QR6_K; ++i) {
         const int sc = scales[4*i];
 
-        const int vil = (vl >> (4*i)) & 0x0F0F0F0F;
+        int sumi = 0;
+#pragma unroll
+        for (int j = 0; j < VDR_Q6_K_Q8_1_MMVQ; ++j) {
+            const int vil = (vl[j] >> (4*i)) & 0x0F0F0F0F;
+            const int vih = ((vh[j] >> (4*i)) << 4) & 0x30303030;
 
-        const int vih = ((vh >> (4*i)) << 4) & 0x30303030;
+            // (vil | vih) - 32 for each byte, the - 32 is done by the second dp4a
+            sumi = ggml_cuda_dp4a(vil | vih,  u[VDR_Q6_K_Q8_1_MMVQ*i + j], sumi); // SIMD dot product
+            sumi = ggml_cuda_dp4a(0xE0E0E0E0, u[VDR_Q6_K_Q8_1_MMVQ*i + j], sumi);
+        }
 
-        const int vi = __vsub4((vil | vih), 0x20202020); // vi = (vil | vih) - 32
-
-        sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product
+        sumf += d8[i] * (sumi * sc);
     }
 
     return d*sumf;
@@ -700,6 +706,25 @@ static __device__ __forceinline__ float vec_dot_q1_0_q8_1(
         const int u2 = get_int_b4(bq8_1_chunk->qs, j*4+2);
         const int u3 = get_int_b4(bq8_1_chunk->qs, j*4+3);
 
+#if defined(GGML_USE_HIP)
+        const uint32_t q_bits = (uint16_t) q;
+
+        // Expand each group of four Q1_0 bits into byte selectors for v_perm_b32.
+        const uint32_t q0_bits = q_bits >>  0;
+        const uint32_t q1_bits = q_bits >>  4;
+        const uint32_t q2_bits = q_bits >>  8;
+        const uint32_t q3_bits = q_bits >> 12;
+
+        const uint32_t q0_indices = (q0_bits & 0x01) | ((q0_bits & 0x02) << 7) | ((q0_bits & 0x04) << 14) | ((q0_bits & 0x08) << 21);
+        const uint32_t q1_indices = (q1_bits & 0x01) | ((q1_bits & 0x02) << 7) | ((q1_bits & 0x04) << 14) | ((q1_bits & 0x08) << 21);
+        const uint32_t q2_indices = (q2_bits & 0x01) | ((q2_bits & 0x02) << 7) | ((q2_bits & 0x04) << 14) | ((q2_bits & 0x08) << 21);
+        const uint32_t q3_indices = (q3_bits & 0x01) | ((q3_bits & 0x02) << 7) | ((q3_bits & 0x04) << 14) | ((q3_bits & 0x08) << 21);
+
+        const int v0 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q0_indices);
+        const int v1 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q1_indices);
+        const int v2 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q2_indices);
+        const int v3 = __builtin_amdgcn_perm(0x000001FF, 0x000001FF, q3_indices);
+#else
         // unpack crumbs into nibble indices
         const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0); // [0, 1, 4, 5] [ 8,  9, 12, 13]
         const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2); // [2, 3, 6, 7] [10, 11, 14, 15]
@@ -713,6 +738,7 @@ static __device__ __forceinline__ float vec_dot_q1_0_q8_1(
         const int v1 = __byte_perm(s0, s1, 0x7632);
         const int v2 = __byte_perm(s2, s3, 0x5410);
         const int v3 = __byte_perm(s2, s3, 0x7632);
+#endif // defined(GGML_USE_HIP)
 
         sumi = ggml_cuda_dp4a(v0, u0, sumi);
         sumi = ggml_cuda_dp4a(v1, u1, sumi);
@@ -870,18 +896,31 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1(
 
     const block_q2_K * bq2_K = (const block_q2_K *) vbq + kbx;
 
+    // iqs is in 0,2..14, each 32 bytes of qs hold 4 q8_1 blocks (one per 2 bit shift)
     const int bq8_offset = QR2_K * (iqs / QI8_1);
-    const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
+    const int iqs8 = iqs % QI8_1;
 
-    const uint8_t * scales = bq2_K->scales + scale_offset;
+    int v[VDR_Q2_K_Q8_1_MMVQ];
+#pragma unroll
+    for (int j = 0; j < VDR_Q2_K_Q8_1_MMVQ; ++j) {
+        v[j] = get_int_b4(bq2_K->qs, iqs + j);
+    }
 
-    const int v = get_int_b4(bq2_K->qs, iqs);
-    int    u[QR2_K];
+    // scale of shift i is byte 2*i + (iqs8 >= QI8_1/2) of the 8 bytes for this half
+    const int sc_shift = 8 * (iqs8 / (QI8_1/2));
+    int scales[2];
+    scales[0] = get_int_b4(bq2_K->scales, 2*(iqs / QI8_1) + 0) >> sc_shift;
+    scales[1] = get_int_b4(bq2_K->scales, 2*(iqs / QI8_1) + 1) >> sc_shift;
+
+    int    u[VDR_Q2_K_Q8_1_MMVQ*QR2_K];
     float d8[QR2_K];
 
 #pragma unroll
-    for (int i = 0; i < QR2_K; ++ i) {
-        u[i]  = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
+    for (int i = 0; i < QR2_K; ++i) {
+#pragma unroll
+        for (int j = 0; j < VDR_Q2_K_Q8_1_MMVQ; ++j) {
+            u[VDR_Q2_K_Q8_1_MMVQ*i + j] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs8 + j);
+        }
         d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
     }
 
@@ -893,26 +932,42 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1(
 
     const block_q3_K * bq3_K = (const block_q3_K *) vbq + kbx;
 
+    // iqs is in 0,2..14, each 32 bytes of qs hold 4 q8_1 blocks (one per 2 bit shift)
     const int bq8_offset = QR3_K * (iqs / (QI3_K/2));
-    const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
+    const int iqs8 = iqs % QI8_1;
 
     const float d = bq3_K->d;
 
-    const int vl = get_int_b2(bq3_K->qs, iqs);
+    int vl[VDR_Q3_K_Q8_1_MMVQ];
+    int vh[VDR_Q3_K_Q8_1_MMVQ];
+#pragma unroll
+    for (int j = 0; j < VDR_Q3_K_Q8_1_MMVQ; ++j) {
+        vl[j] = get_int_b2(bq3_K->qs, iqs + j);
+        vh[j] = get_int_b2(bq3_K->hmask, iqs8 + j) >> bq8_offset;
+    }
 
-    // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
-    const int vh = ~get_int_b2(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;
+    // sub-block 8*(iqs/8) + 2*i + (iqs8 >= QI8_1/2) for shift i:
+    // 4 low bits in nibble iqs/8 of bytes, 2 high bits at bit 4*(iqs/8) + 2*(i/2) of bytes 8..11
+    const int sc_shift = 8 * (iqs8 / (QI8_1/2)) + 4 * (iqs / (QI3_K/2));
+    int scales[3];
+#pragma unroll
+    for (int l = 0; l < 3; ++l) {
+        scales[l] = get_int_b2(bq3_K->scales, l) >> sc_shift;
+    }
 
-    int    u[QR3_K];
+    int    u[VDR_Q3_K_Q8_1_MMVQ*QR3_K];
     float d8[QR3_K];
 
 #pragma unroll
     for (int i = 0; i < QR3_K; ++i) {
-        u[i]  = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
+#pragma unroll
+        for (int j = 0; j < VDR_Q3_K_Q8_1_MMVQ; ++j) {
+            u[VDR_Q3_K_Q8_1_MMVQ*i + j] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs8 + j);
+        }
         d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
     }
 
-    return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);
+    return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, scales, d, d8);
 }
 
 static __device__ __forceinline__ float vec_dot_q4_K_q8_1(
@@ -1025,17 +1080,25 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1(
     const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);
     const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));
 
-    const int vl = get_int_b2(bq6_K->ql, iqs);
-    const int vh = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;
+    int vl[VDR_Q6_K_Q8_1_MMVQ];
+    int vh[VDR_Q6_K_Q8_1_MMVQ];
+#pragma unroll
+    for (int j = 0; j < VDR_Q6_K_Q8_1_MMVQ; ++j) {
+        vl[j] = get_int_b2(bq6_K->ql, iqs + j);
+        vh[j] = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4) + j) >> vh_shift;
+    }
 
     const int8_t * scales = bq6_K->scales + scale_offset;
 
-    int    u[QR6_K];
+    int    u[VDR_Q6_K_Q8_1_MMVQ*QR6_K];
     float d8[QR6_K];
 
 #pragma unroll
     for (int i = 0; i < QR6_K; ++i) {
-        u[i]  = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1);
+#pragma unroll
+        for (int j = 0; j < VDR_Q6_K_Q8_1_MMVQ; ++j) {
+            u[VDR_Q6_K_Q8_1_MMVQ*i + j] = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1 + j);
+        }
         d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds);
     }
 

ggml/src/ggml-hexagon/ggml-hexagon.cpp

@@ -956,7 +956,6 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,
         return;
     }
 
-    GGML_ASSERT(offset == 0);
     htp_unpack_tensor_tiled(data, tensor, offset, size);
 }
 
@@ -1045,7 +1044,6 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff
 
     GGML_ASSERT((slice_offset % row_size_bytes) == 0 && "offset must be aligned to row boundary");
     GGML_ASSERT((temp_size % row_size_bytes)    == 0 && "temp_size must be a multiple of row size");
-    GGML_ASSERT((slice_offset / row_size_bytes) % 32 == 0 && "offset must be aligned to tile size (32 rows)");
     GGML_ASSERT((offset + temp_size) <= ggml_nbytes(tensor));
 
     std::vector<uint8_t> temp_buf(temp_size);
@@ -3474,15 +3472,16 @@ static bool ggml_hexagon_flash_attn_is_hmx_eligible(
         return true;
     }
 
-    // Compute-bound head dim
-    if (DK >= 256) {
-        return true;
-    }
-
     const uint32_t n_head = q->ne[2];
     const uint32_t n_kv_heads = k->ne[2];
     const uint32_t G = n_kv_heads > 0 ? n_head / n_kv_heads : 1;
     const uint32_t S = k->ne[1];
+    const uint32_t total_threads = sess->n_threads * sess->mdev.count;
+
+    // Compute-bound head dim (for DK == 256 decode, HVX is faster when KV heads span all threads)
+    if (DK > 256 || (DK == 256 && (DV > 256 || n_kv_heads < total_threads))) {
+        return true;
+    }
 
     // Tile alignment for 32-row HMX tiles
     const bool is_tile_aligned = (G > 0 && (32 % G == 0));
@@ -3495,7 +3494,9 @@ static bool ggml_hexagon_flash_attn_is_hmx_eligible(
 
     // Context depth crossover
     uint32_t s_cross = 512;
-    if (DK <= 64) {
+    if (DK == 256) {
+        s_cross = 4096;
+    } else if (DK <= 64) {
         s_cross = (sess->n_threads >= 8) ? 2048 : ((sess->n_threads >= 6) ? 768 : 512);
     } else {
         s_cross = (sess->n_threads >= 8) ? 1024 : 512;
@@ -3667,10 +3668,6 @@ static bool ggml_hexagon_supported_flash_attn_ext(const struct ggml_hexagon_sess
         return false;
     }
 
-    if (dst->ne[3] != 1) {
-        return false;
-    }
-
     struct htp_fa_kernel_params kparams;
     if (!ggml_hexagon_precompute_flash_attn_params(sess, op, &kparams)) {
         return false;
@@ -3769,6 +3766,8 @@ static bool ggml_hexagon_matmul_is_hmx_eligible(
     bool is_matmul_id,
     bool is_batched
 ) {
+    GGML_UNUSED(dst);
+
     if (src1->type != GGML_TYPE_F32 && (src1->type != GGML_TYPE_F16 || is_matmul_id)) {
         return false;
     }
@@ -3780,7 +3779,7 @@ static bool ggml_hexagon_matmul_is_hmx_eligible(
     const int wtype = src0->type;
 
     // HMX weight tiles accept non-32-aligned N for non-matmul_id.
-    if (ne01 % 32 != 0 && is_matmul_id) {
+    if (ne01 % HTP_TENSOR_TILE_N_ROWS != 0 && is_matmul_id) {
         return false;
     }
 
@@ -3789,8 +3788,8 @@ static bool ggml_hexagon_matmul_is_hmx_eligible(
         return false;
     }
 
-    // HMX paths require K aligned to 32.
-    if (ne00 % 32 != 0) {
+    // HMX paths require K aligned to HMX tile cols (32).
+    if (ne00 % HTP_TENSOR_TILE_N_COLS != 0) {
         return false;
     }
 
@@ -3952,7 +3951,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(
         kparams->tile_size = htp_tensor_get_tile_size(wtype);
         kparams->aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
 
-        const bool k_align = (ne10 % 32 == 0);
+        const bool k_align = (ne10 % htp_tensor_get_tile_k(wtype) == 0);
 
         if (is_matmul_id) {
             kparams->kernel_type  = (src1_nrows < (int) sess->n_threads)
@@ -4124,7 +4123,7 @@ static void ggml_hexagon_precompute_matmul_params_impl(
 
     const bool is_tiled = ggml_hexagon_tensor_is_tiled(src0);
     const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type;
-    const int ne00_padded = is_tiled ? hex_round_up(ne00, 32) : ne00;
+    const int ne00_padded = is_tiled ? hex_round_up(ne00, htp_tensor_get_tile_k(wtype)) : ne00;
     const int ne11_padded = hex_round_up(ne11, 32);
     // VTCM has to hold whole 32-row weight tiles, so size for the rounded-up N
     // even when the tensor itself is ragged.
@@ -5359,10 +5358,38 @@ static void ggml_hexagon_precompute_fused_mmidnx_params(
     kparams->n_weights = n_weights;
 }
 
+static bool ggml_hexagon_op_mul_mat_use_fwht(const struct ggml_tensor * dst) {
+    if (dst->op != GGML_OP_MUL_MAT || ggml_get_op_params_i32(dst, 1) != GGML_HINT_SRC0_IS_HADAMARD) {
+        return false;
+    }
+
+    const struct ggml_tensor * src0 = dst->src[0];
+    const struct ggml_tensor * src1 = dst->src[1];
+
+    const int64_t n = src1->ne[0];
+    if (n < 32 || (n & (n - 1)) != 0) {
+        return false;
+    }
+
+    return src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 &&
+           (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16) &&
+           ggml_is_contiguous_rows(src1) && ggml_is_contiguous(dst) &&
+           ggml_are_same_shape(src1, dst);
+}
+
 static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) {
     const struct ggml_tensor * src0 = dst->src[0];
     const struct ggml_tensor * src1 = dst->src[1];
 
+    if (ggml_get_op_params_i32(dst, 1) == GGML_HINT_SRC0_IS_HADAMARD) {
+        if (!ggml_hexagon_op_mul_mat_use_fwht(dst)) {
+            return false;
+        }
+        struct htp_unary_kernel_params ukparams;
+        ggml_hexagon_precompute_unary_params(sess, HTP_OP_HADAMARD, src1, nullptr, dst, &ukparams);
+        return ukparams.vtcm_row_per_thread > 0 && (size_t) ukparams.vtcm_size <= sess->vtcm_size;
+    }
+
     if (dst->type != GGML_TYPE_F32) {
         return false;
     }
@@ -5372,6 +5399,8 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s
     }
 
     switch (src0->type) {
+        case GGML_TYPE_Q1_0:
+        case GGML_TYPE_Q2_0:
         case GGML_TYPE_Q4_0:
         case GGML_TYPE_Q4_1:
         case GGML_TYPE_Q8_0:
@@ -5381,13 +5410,13 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s
         case GGML_TYPE_Q5_K:
         case GGML_TYPE_Q6_K:
         case GGML_TYPE_Q3_K:
-        case GGML_TYPE_Q2_K:
+        case GGML_TYPE_Q2_K: {
             if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_view(src0)) {
                 return false;
             }
 
-            if (src0->ne[0] % ((src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q5_K || src0->type == GGML_TYPE_Q4_K ||
-                                src0->type == GGML_TYPE_Q3_K || src0->type == GGML_TYPE_Q2_K) ? QK_K : 32)) {
+            const int64_t qk = ggml_blck_size(src0->type);
+            if (src0->ne[0] % qk) {
                 return false;
             }
 
@@ -5398,6 +5427,7 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s
                 return false;
             }
             break;
+        }
 
         case GGML_TYPE_F16:
             if (src0->nb[1] < src0->nb[0]) {
@@ -5467,6 +5497,8 @@ static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session
     }
 
     switch (src0->type) {
+        case GGML_TYPE_Q1_0:
+        case GGML_TYPE_Q2_0:
         case GGML_TYPE_Q4_0:
         case GGML_TYPE_Q4_1:
         case GGML_TYPE_Q8_0:
@@ -5481,8 +5513,7 @@ static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session
                 return false;
             }
 
-            if (src0->ne[0] % ((src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q5_K || src0->type == GGML_TYPE_Q4_K ||
-                                src0->type == GGML_TYPE_Q3_K || src0->type == GGML_TYPE_Q2_K) ? QK_K : 32)) {
+            if (src0->ne[0] % ggml_blck_size(src0->type) != 0) {
                 return false;
             }
             break;
@@ -5917,8 +5948,9 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *
     const struct ggml_tensor * src1 = op->src[1]; // indices
     const struct ggml_tensor * dst  = op;
 
-    if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16 &&
-        src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) {
+    if (src0->type != GGML_TYPE_F32  && src0->type != GGML_TYPE_F16 && src0->type  != GGML_TYPE_BF16 && src0->type != GGML_TYPE_I32  &&
+        src0->type != GGML_TYPE_Q1_0 && src0->type != GGML_TYPE_Q2_0 && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 &&
+        src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K ) {
         return false;
     }
 
@@ -6662,7 +6694,11 @@ static void ggml_backend_hexagon_free(ggml_backend_t backend) {
 static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
     switch (t->op) {
         case GGML_OP_FLASH_ATTN_EXT:  return HTP_OP_FLASH_ATTN_EXT;
-        case GGML_OP_MUL_MAT:         return HTP_OP_MUL_MAT;
+        case GGML_OP_MUL_MAT:
+            if (ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD) {
+                return HTP_OP_HADAMARD;
+            }
+            return HTP_OP_MUL_MAT;
         case GGML_OP_MUL_MAT_ID:      return HTP_OP_MUL_MAT_ID;
         case GGML_OP_MUL:             return HTP_OP_MUL;
         case GGML_OP_ADD:             return HTP_OP_ADD;
@@ -6791,6 +6827,7 @@ static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const st
 
 static bool is_mergeable_mul_mat(const ggml_tensor * t) {
     if (t->op != GGML_OP_MUL_MAT) return false;
+    if (ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD) return false;
 
     const ggml_tensor * src0 = t->src[0];
     const ggml_tensor * src1 = t->src[1];
@@ -6893,7 +6930,9 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
 
             if (graph->nodes[i]->op == GGML_OP_RMS_NORM && ggml_can_fuse(graph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {
                 extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE;
-            } else if (graph->nodes[i]->op == GGML_OP_MUL_MAT || graph->nodes[i]->op == GGML_OP_MUL_MAT_ID) {
+            } else if ((graph->nodes[i]->op == GGML_OP_MUL_MAT &&
+                        ggml_get_op_params_i32(graph->nodes[i], 1) != GGML_HINT_SRC0_IS_HADAMARD) ||
+                       graph->nodes[i]->op == GGML_OP_MUL_MAT_ID) {
                 if ((i + 1 < graph->n_nodes && graph->nodes[i + 1]->op == GGML_OP_ADD && ggml_can_fuse(graph, i, { graph->nodes[i]->op, GGML_OP_ADD })) ||
                     ggml_node_has_n_uses(graph, i, 1)) {
                     extra->flags |= GGML_HEXAGON_TENSOR_FUSEABLE;
@@ -8317,6 +8356,10 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
                   "please update hexagon_type to match ggml_type");
     static_assert((unsigned int) HTP_TYPE_Q2_K == (unsigned int) GGML_TYPE_Q2_K,
                   "please update hexagon_type to match ggml_type");
+    static_assert((unsigned int) HTP_TYPE_Q1_0 == (unsigned int) GGML_TYPE_Q1_0,
+                  "please update hexagon_type to match ggml_type");
+    static_assert((unsigned int) HTP_TYPE_Q2_0 == (unsigned int) GGML_TYPE_Q2_0,
+                  "please update hexagon_type to match ggml_type");
     static_assert((unsigned int) HTP_TYPE_BF16 == (unsigned int) GGML_TYPE_BF16,
                   "please update hexagon_type to match ggml_type");
 

ggml/src/ggml-hexagon/htp-opnode.h

@@ -47,13 +47,19 @@ struct htp_opnode {
     void init(ggml_tensor * node) {
         this->node = node;
         if (this->node) {
-            this->name = ggml_op_desc(this->node);
+            if (this->node->op == GGML_OP_MUL_MAT &&
+                ggml_get_op_params_i32(this->node, 1) == GGML_HINT_SRC0_IS_HADAMARD) {
+                this->name = "MUL_MAT_HADAMARD";
+                this->inputs.push_back(this->node->src[1]);
+            } else {
+                this->name = ggml_op_desc(this->node);
 
-            // Build inputs (preserving optional nullptrs)
-            int n_inputs = n_active_src(this->node);
-            this->inputs.resize(n_inputs, nullptr);
-            for (int i = 0; i < n_inputs; i++) {
-                this->inputs[i] = this->node->src[i];
+                // Build inputs (preserving optional nullptrs)
+                int n_inputs = n_active_src(this->node);
+                this->inputs.resize(n_inputs, nullptr);
+                for (int i = 0; i < n_inputs; i++) {
+                    this->inputs[i] = this->node->src[i];
+                }
             }
 
             // Build outputs
@@ -124,6 +130,10 @@ struct htp_opnode {
     bool stackable() const {
         switch (this->op()) {
             case GGML_OP_MUL_MAT:
+                if (ggml_get_op_params_i32(this->node, 1) == GGML_HINT_SRC0_IS_HADAMARD) {
+                    return false;
+                }
+                return ggml_is_quantized(this->src0()->type);
             case GGML_OP_MUL_MAT_ID:
                 return ggml_is_quantized(this->src0()->type);
             default:

ggml/src/ggml-hexagon/htp-repack.cpp

@@ -289,6 +289,80 @@ static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t offset,
 
 // --- Float Types (F16, F32, BF16 -> f16_tiled HMX layout) ---
 
+static void repack_q2_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
+    GGML_ASSERT(offset == 0);
+    GGML_ASSERT(t->ne[0] % 64 == 0);
+    GGML_ASSERT(size == ggml_nbytes(t));
+
+    const block_q2_0 * src = (const block_q2_0 *) data;
+    const int64_t blocks_per_row = t->ne[0] / 64;
+    const int64_t n_col_tiles = hex_round_up(t->ne[1], 32) / 32;
+    const int64_t n_weight_tiles = blocks_per_row;
+    const size_t matrix_size = (size_t) n_col_tiles * n_weight_tiles * HTP_TENSOR_TILE_SIZE_Q2_0;
+
+    for (int64_t slice = 0; slice < t->ne[2] * t->ne[3]; ++slice) {
+        const block_q2_0 * src_slice = src + slice * t->ne[1] * blocks_per_row;
+        uint8_t * dst_slice = (uint8_t *) t->data + slice * matrix_size;
+        memset(dst_slice, 0, matrix_size);
+
+        for (int64_t row_index = 0; row_index < t->ne[1]; ++row_index) {
+            const int64_t ct = row_index / 32;
+            const int row = (int) (row_index % 32);
+            for (int64_t kt = 0; kt < n_weight_tiles; ++kt) {
+                const block_q2_0 * block = src_slice + row_index * blocks_per_row + kt;
+                uint8_t * tile = dst_slice + (ct * n_weight_tiles + kt) * HTP_TENSOR_TILE_SIZE_Q2_0;
+                for (int group = 0; group < 16; ++group) {
+                    for (int byte = 0; byte < 4; ++byte) {
+                        const int k = 4 * group + byte;
+                        const uint8_t code = (block->qs[k / 4] >> (2 * (k % 4))) & 3;
+                        tile[(group >> 2) * 128 + 4 * row + byte] |= code << (2 * (group & 3));
+                    }
+                }
+                memcpy(tile + 512 + 2 * row, &block->d, sizeof(block->d));
+            }
+        }
+    }
+}
+
+static void repack_tiled_q2_0(void * data, const ggml_tensor * t, size_t offset, size_t size) {
+    GGML_ASSERT(t->ne[0] % 64 == 0);
+    GGML_ASSERT(offset + size <= ggml_nbytes(t));
+
+    const size_t first_block = offset / sizeof(block_q2_0);
+    const size_t end_block = (offset + size + sizeof(block_q2_0) - 1) / sizeof(block_q2_0);
+    const size_t blocks_per_row = t->ne[0] / 64;
+    const size_t blocks_per_slice = t->ne[1] * blocks_per_row;
+    const size_t n_weight_tiles = blocks_per_row;
+    const size_t n_col_tiles = hex_round_up(t->ne[1], 32) / 32;
+    const size_t matrix_size = n_col_tiles * n_weight_tiles * HTP_TENSOR_TILE_SIZE_Q2_0;
+
+    for (size_t block_index = first_block; block_index < end_block; ++block_index) {
+        const size_t slice = block_index / blocks_per_slice;
+        const size_t in_slice = block_index % blocks_per_slice;
+        const size_t row_index = in_slice / blocks_per_row;
+        const size_t kt = in_slice % blocks_per_row;
+        const size_t ct = row_index / 32;
+        const size_t row = row_index % 32;
+        const uint8_t * tile = (const uint8_t *) t->data + slice * matrix_size +
+            (ct * n_weight_tiles + kt) * HTP_TENSOR_TILE_SIZE_Q2_0;
+
+        block_q2_0 block = {};
+        memcpy(&block.d, tile + 512 + 2 * row, sizeof(block.d));
+        for (int group = 0; group < 16; ++group) {
+            for (int byte = 0; byte < 4; ++byte) {
+                const uint8_t packed = tile[(group >> 2) * 128 + 4 * row + byte];
+                const uint8_t code = (packed >> (2 * (group & 3))) & 3;
+                block.qs[group] |= code << (2 * byte);
+            }
+        }
+
+        const size_t block_begin = block_index * sizeof(block_q2_0);
+        const size_t copy_begin = (std::max)(block_begin, offset);
+        const size_t copy_end = (std::min)(block_begin + sizeof(block_q2_0), offset + size);
+        memcpy((uint8_t *) data + copy_begin - offset, (const uint8_t *) &block + copy_begin - block_begin, copy_end - copy_begin);
+    }
+}
+
 template <typename SrcT, typename ToFp16Fn>
 static void repack_float_to_f16_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size, ToFp16Fn && to_fp16) {
     repack_to_tiled_grid_32<SrcT, 32>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_F16,
@@ -900,10 +974,122 @@ static void repack_tiled_q5_K(void * data, const ggml_tensor * t, size_t offset,
         });
 }
 
+static inline void q1_0_tiled_bit_location(int k, int row, int & byte_offset, int & bit) {
+    const int quarter = k / 32;
+    const int kq      = k % 32;
+
+    byte_offset = quarter * 128 + 4 * row + (kq % 4);
+    bit         = kq / 4;
+}
+
+static void repack_q1_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
+    const block_q1_0 * src_matrix = (const block_q1_0 *) data;
+    const int64_t ne0 = t->ne[0];
+    const int64_t ne1 = t->ne[1];
+    const int64_t ne2 = t->ne[2];
+    const int64_t ne3 = t->ne[3];
+    const int n_col_tiles = hex_round_up(ne1, 32) / 32;
+    const int n_k_tiles = ne0 / QK1_0;
+    const size_t tile_size = HTP_TENSOR_TILE_SIZE_Q1_0;
+    const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;
+    const size_t slice_size = ne1 * ggml_row_size(t->type, ne0);
+    const int64_t start_slice = offset / slice_size;
+    const int64_t end_slice = (std::min)((int64_t) ((offset + size + slice_size - 1) / slice_size), ne2 * ne3);
+
+    GGML_ASSERT(ne0 % QK1_0 == 0);
+
+    for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
+        const block_q1_0 * src_slice = src_matrix + (slice_idx - start_slice) * ne1 * n_k_tiles;
+        uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size;
+
+        for (int ct = 0; ct < n_col_tiles; ct++) {
+            for (int kt = 0; kt < n_k_tiles; kt++) {
+                uint8_t * tile_dst = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size;
+                ggml_half * scale_dst = (ggml_half *) (tile_dst + 512);
+                memset(tile_dst, 0, tile_size);
+
+                for (int row = 0; row < 32; row++) {
+                    const int64_t r = ct * 32 + row;
+                    if (r >= ne1) {
+                        continue;
+                    }
+
+                    const block_q1_0 & block = src_slice[r * n_k_tiles + kt];
+                    scale_dst[row] = block.d;
+                    for (int k = 0; k < QK1_0; k++) {
+                        int byte_offset;
+                        int bit;
+                        q1_0_tiled_bit_location(k, row, byte_offset, bit);
+                        if (block.qs[k / 8] & (1u << (k % 8))) {
+                            tile_dst[byte_offset] |= (uint8_t) (1u << bit);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+static void repack_tiled_q1_0(void * data, const ggml_tensor * t, size_t offset, size_t size) {
+    block_q1_0 * dst_matrix = (block_q1_0 *) data;
+    const int64_t ne0 = t->ne[0];
+    const int64_t ne1 = t->ne[1];
+    const int64_t ne2 = t->ne[2];
+    const int64_t ne3 = t->ne[3];
+    const int n_col_tiles = hex_round_up(ne1, 32) / 32;
+    const int n_k_tiles = ne0 / QK1_0;
+    const size_t tile_size = HTP_TENSOR_TILE_SIZE_Q1_0;
+    const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;
+    const size_t row_size_bytes = ggml_row_size(t->type, ne0);
+    const size_t slice_size = ne1 * row_size_bytes;
+    const int64_t start_slice = offset / slice_size;
+    const int64_t end_slice = (std::min)((int64_t) ((offset + size + slice_size - 1) / slice_size), ne2 * ne3);
+
+    GGML_ASSERT(ne0 % QK1_0 == 0);
+
+    for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
+        const size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size);
+        const size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size);
+        const int64_t start_row = (cur_start_byte - (size_t) slice_idx * slice_size) / row_size_bytes;
+        const int64_t end_row = (std::min)((int64_t) ((cur_end_byte - (size_t) slice_idx * slice_size + row_size_bytes - 1) / row_size_bytes), ne1);
+        block_q1_0 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_q1_0);
+        const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size;
+
+        for (int ct = start_row / 32; ct < (end_row + 31) / 32; ct++) {
+            for (int kt = 0; kt < n_k_tiles; kt++) {
+                const uint8_t * tile_src = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size;
+                const ggml_half * scale_src = (const ggml_half *) (tile_src + 512);
+
+                for (int row = 0; row < 32; row++) {
+                    const int64_t r = ct * 32 + row;
+                    if (r < start_row || r >= end_row) {
+                        continue;
+                    }
+
+                    block_q1_0 & block = dst_slice[(r - start_row) * n_k_tiles + kt];
+                    memset(block.qs, 0, sizeof(block.qs));
+                    block.d = scale_src[row];
+                    for (int k = 0; k < QK1_0; k++) {
+                        int byte_offset;
+                        int bit;
+                        q1_0_tiled_bit_location(k, row, byte_offset, bit);
+                        if (tile_src[byte_offset] & (1u << bit)) {
+                            block.qs[k / 8] |= (uint8_t) (1u << (k % 8));
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
 // --- Top-level Repack / Unpack Dispatchers ---
 
 void htp_repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t size) {
     switch (tensor->type) {
+        case GGML_TYPE_Q1_0:
+            repack_q1_0_tiled(tensor, data, 0, size);
+            break;
         case GGML_TYPE_Q4_0:
         case GGML_TYPE_IQ4_NL:
             repack_q4_0_tiled(tensor, data, 0, size);
@@ -917,6 +1103,9 @@ void htp_repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t siz
         case GGML_TYPE_MXFP4:
             repack_mxfp4_tiled(tensor, data, 0, size);
             break;
+        case GGML_TYPE_Q2_0:
+            repack_q2_0_tiled(tensor, data, 0, size);
+            break;
         case GGML_TYPE_Q2_K:
             repack_q2_K_tiled(tensor, data, 0, size);
             break;
@@ -953,6 +1142,9 @@ void htp_unpack_tensor_tiled(void * data, const ggml_tensor * tensor, size_t off
     GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
 
     switch (tensor->type) {
+        case GGML_TYPE_Q1_0:
+            repack_tiled_q1_0(data, tensor, offset, size);
+            break;
         case GGML_TYPE_Q4_0:
         case GGML_TYPE_IQ4_NL:
             repack_tiled_q4_0(data, tensor, offset, size);
@@ -966,6 +1158,9 @@ void htp_unpack_tensor_tiled(void * data, const ggml_tensor * tensor, size_t off
         case GGML_TYPE_MXFP4:
             repack_tiled_mxfp4(data, tensor, offset, size);
             break;
+        case GGML_TYPE_Q2_0:
+            repack_tiled_q2_0(data, tensor, offset, size);
+            break;
         case GGML_TYPE_Q2_K:
             repack_tiled_q2_K(data, tensor, offset, size);
             break;

ggml/src/ggml-hexagon/htp-repack.h

@@ -10,9 +10,11 @@
 static inline bool htp_is_tiled_weight_type(enum ggml_type type) {
     return type == GGML_TYPE_Q4_0  || type == GGML_TYPE_Q4_1   ||
            type == GGML_TYPE_Q8_0  || type == GGML_TYPE_IQ4_NL ||
-           type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q6_K   ||
+           type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q1_0   ||
+           type == GGML_TYPE_Q6_K   ||
            type == GGML_TYPE_Q4_K  || type == GGML_TYPE_Q5_K   ||
            type == GGML_TYPE_Q3_K  || type == GGML_TYPE_Q2_K   ||
+           type == GGML_TYPE_Q2_0  ||
            type == GGML_TYPE_F16   || type == GGML_TYPE_F32    ||
            type == GGML_TYPE_BF16;
 }
@@ -42,6 +44,9 @@ static inline size_t htp_tiled_row_size(enum ggml_type type, int64_t ne0) {
     if (type == GGML_TYPE_Q6_K) {
         return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q6_K / 32);
     }
+    if (type == GGML_TYPE_Q2_0) {
+        return (size_t) (ne0 / 64) * 18;
+    }
     if (type == GGML_TYPE_Q4_K) {
         return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q4_K / 32);
     }

ggml/src/ggml-hexagon/htp/flash-attn-ops.c

@@ -64,6 +64,8 @@ struct htp_fa_context {
     __fp16 slopes[512];
 
     uint32_t n_blocks;
+    uint32_t n_active_blocks;
+    bool     shared_active_blocks;
 
     size_t size_q_row_padded;
     size_t size_k_row_padded;
@@ -82,12 +84,13 @@ struct htp_fa_context {
     size_t size_q_block;
     size_t size_vkq_acc;
 
-    uint8_t * spad_q;
-    uint8_t * spad_k;
-    uint8_t * spad_v;
-    uint8_t * spad_m;
-    uint8_t * spad_a;
-    float *   spad_sinks;
+    uint8_t *  spad_q;
+    uint8_t *  spad_k;
+    uint8_t *  spad_v;
+    uint8_t *  spad_m;
+    uint8_t *  spad_a;
+    float *    spad_sinks;
+    uint16_t   active_blocks[FA_MAX_KV_BLOCKS];
 
     const struct htp_tensor * k;
     const struct htp_tensor * v;
@@ -156,6 +159,7 @@ struct hmx_fa_context {
     size_t       row_buf_stride;       // HVX vectors per row buffer (Bc/64)
     size_t       mask_buf_row_stride;  // elements (__fp16) per row in mask buffer
     size_t       q_tile_bytes;
+    size_t       k_tile_bytes;
     size_t       o_tile_bytes;
     size_t       col_vec_bytes;
     size_t       d_tile_bytes;
@@ -163,6 +167,77 @@ struct hmx_fa_context {
     dma_cache_fa m_cache;
 };
 
+// Scan mask rows [q_start, q_start + n_rows_q) and build compact list of KV blocks with any entry != -inf.
+static uint32_t fa_scan_active_blocks(
+        dma_queue *               dma_q,
+        const struct htp_tensor * mask,
+        uint32_t                  q_start,
+        uint32_t                  n_rows_q,
+        uint32_t                  im2,
+        uint32_t                  im3,
+        uint32_t                  nek1,
+        uint32_t                  Bc,
+        uint32_t                  n_kv_blocks,
+        void *                    vtcm_scratch,
+        size_t                    scratch_bytes,
+        uint16_t *                active_blocks) {
+    const size_t row_bytes  = (size_t) nek1 * sizeof(__fp16);
+    const size_t row_stride = hex_round_up(row_bytes, 128);
+
+    if (!mask || n_kv_blocks <= 1 || n_kv_blocks > FA_MAX_KV_BLOCKS ||
+        (size_t) n_rows_q * row_stride > scratch_bytes) {
+        #pragma clang loop vectorize(disable)
+        for (uint32_t b = 0; b < n_kv_blocks; ++b) {
+            active_blocks[b] = (uint16_t) b;
+        }
+        return n_kv_blocks;
+    }
+
+    const dma_addr_t m_src = mask->data + q_start * mask->nb[1] + im2 * mask->nb[2] + im3 * mask->nb[3];
+    dma_queue_push(dma_q, dma_make_data(vtcm_scratch, m_src), row_stride, mask->nb[1], row_bytes, n_rows_q);
+    const uint8_t * m_buf = (const uint8_t *) dma_queue_pop(dma_q).dst;
+
+    const HVX_Vector v_ninf   = Q6_Vh_vsplat_R(0xFC00);
+    uint32_t         n_active = 0;
+
+    for (uint32_t b = 0; b < n_kv_blocks; ++b) {
+        const uint32_t col0 = b * Bc;
+        const uint32_t cols = MIN(Bc, nek1 - col0);
+
+        for (uint32_t r = 0; r < n_rows_q; ++r) {
+            const __fp16 * row_ptr = (const __fp16 *) (m_buf + r * row_stride) + col0;
+            HVX_Vector v_any = Q6_V_vzero();
+            uint32_t c = 0;
+            for (; c + 64 <= cols; c += 64) {
+                const HVX_Vector v_m = *(const HVX_Vector *) (row_ptr + c);
+                v_any = Q6_V_vor_VV(v_any, Q6_V_vxor_VV(v_m, v_ninf));
+            }
+            if (c < cols) {
+                const uint32_t rem = cols - c;
+                const HVX_VectorPred q_rem = Q6_Q_vsetq2_R(rem * sizeof(__fp16));
+                const HVX_Vector v_m = *(const HVX_Vector *) (row_ptr + c);
+                const HVX_Vector v_diff = Q6_V_vand_QV(q_rem, Q6_V_vxor_VV(v_m, v_ninf));
+                v_any = Q6_V_vor_VV(v_any, v_diff);
+            }
+            v_any = Q6_V_vor_VV(v_any, Q6_V_vror_VR(v_any, 64));
+            v_any = Q6_V_vor_VV(v_any, Q6_V_vror_VR(v_any, 32));
+            v_any = Q6_V_vor_VV(v_any, Q6_V_vror_VR(v_any, 16));
+            v_any = Q6_V_vor_VV(v_any, Q6_V_vror_VR(v_any, 8));
+            v_any = Q6_V_vor_VV(v_any, Q6_V_vror_VR(v_any, 4));
+            if (Q6_R_vextract_VR(v_any, 0) != 0) {
+                active_blocks[n_active++] = (uint16_t) b;
+                break;
+            }
+        }
+    }
+
+    if (n_active == 0 && n_kv_blocks > 0) {
+        active_blocks[0] = 0;
+        n_active = 1;
+    }
+    return n_active;
+}
+
 static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * data) {
     struct htp_fa_context * factx = (struct htp_fa_context *) data;
     const struct htp_ops_context * octx = factx->octx;
@@ -236,6 +311,18 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
     dma_cache_dm m_cache;
     dma_cache_dm_init(&m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE);
 
+    uint16_t thread_active_blocks[FA_MAX_KV_BLOCKS];
+    const uint16_t * active_blocks = factx->shared_active_blocks ? factx->active_blocks : thread_active_blocks;
+    uint32_t n_active_blocks       = factx->n_active_blocks;
+    dma_addr_t prev_mp_base        = 0;
+
+    if (!factx->shared_active_blocks && !mask) {
+        #pragma clang loop vectorize(disable)
+        for (uint32_t b = 0; b < factx->n_blocks; ++b) {
+            thread_active_blocks[b] = (uint16_t) b;
+        }
+    }
+
     const size_t size_vkq_acc_single = hex_round_up(DV * sizeof(float), 128);
 
     uint32_t ir = ir0;
@@ -271,6 +358,19 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
             }
         }
 
+        dma_addr_t mp_base = 0;
+        if (mask) {
+            const uint32_t im2 = fastmodulo(iq2, mask->ne[2], &factx->src3_div2);
+            const uint32_t im3 = fastmodulo(iq3, mask->ne[3], &factx->src3_div3);
+            mp_base = mask->data + iq1*mask->nb[1] + im2*mask->nb[2] + im3*mask->nb[3];
+            if (!factx->shared_active_blocks && mp_base != prev_mp_base) {
+                n_active_blocks = fa_scan_active_blocks(
+                    dma_q, mask, iq1, 1, im2, im3, nek1, FLASH_ATTN_BLOCK_SIZE,
+                    factx->n_blocks, spad_k, factx->size_k_block * 2, thread_active_blocks);
+                prev_mp_base = mp_base;
+            }
+        }
+
         uint32_t heads[FA_HVX_G_MAX];
         HVX_Vector slope_vecs[FA_HVX_G_MAX] __attribute__((aligned(128)));
         HVX_Vector S_vec[FA_HVX_G_MAX]      __attribute__((aligned(128)));
@@ -303,26 +403,20 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
             dma_queue_push(dma_q, dma_make_data(q_dst, q_row_ptr), factx->size_q_row_padded, nbq1, size_q_row, 1);
         }
 
-        dma_addr_t mp_base = 0;
-        if (mask) {
-            const uint32_t im2 = fastmodulo(iq2, mask->ne[2], &factx->src3_div2);
-            const uint32_t im3 = fastmodulo(iq3, mask->ne[3], &factx->src3_div3);
-            mp_base = mask->data + iq1*mask->nb[1] + im2*mask->nb[2] + im3*mask->nb[3];
-        }
-
-        // Prefetch first two blocks
-        for (uint32_t ib = 0; ib < MIN(factx->n_blocks, 2); ++ib) {
+        // Prefetch first two active blocks
+        for (uint32_t step = 0; step < MIN(n_active_blocks, 2); ++step) {
+            const uint32_t ib = active_blocks[step];
             const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE;
             const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start);
 
             // K
             const dma_addr_t k_src = k->data + ic_start*nbk1 + ik2*nbk2 + ik3*nbk3;
-            uint8_t * k_dst = spad_k + (ib % 2) * factx->size_k_block;
+            uint8_t * k_dst = spad_k + (step % 2) * factx->size_k_block;
             dma_queue_push(dma_q, dma_make_data(k_dst, k_src), factx->size_k_row_padded, nbk1, size_k_row, current_block_size);
 
             // V
             const dma_addr_t v_src = v->data + ic_start*nbv1 + iv2*nbv2 + iv3*nbv3;
-            uint8_t * v_dst = spad_v + (ib % 2) * factx->size_v_block;
+            uint8_t * v_dst = spad_v + (step % 2) * factx->size_v_block;
             dma_queue_push(dma_q, dma_make_data(v_dst, v_src), factx->size_v_row_padded, nbv1, size_v_row, current_block_size);
 
             // Mask
@@ -348,7 +442,8 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
         const HVX_Vector v_log2e   = hvx_vec_splat_f16(EXP_LOG2E_F);
         const uint32_t stride_v2   = factx->size_v_row_padded * 2;
 
-        for (uint32_t ib = 0; ib < factx->n_blocks; ++ib) {
+        for (uint32_t step = 0; step < n_active_blocks; ++step) {
+            const uint32_t ib = active_blocks[step];
             const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE;
             const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start);
 
@@ -411,9 +506,9 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
                 HVX_Vector v_max     = Q6_V_lo_W(hvx_vec_f16_to_f32(v_max_f16));
                 htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_QK, head_ir);
 
-                // prefetch K for block ib + 2 after last head finished QK
-                if (g + 1 == G_local && ib + 2 < factx->n_blocks) {
-                    const uint32_t next_ib = ib + 2;
+                // prefetch K for step + 2 after last head finished QK
+                if (g + 1 == G_local && step + 2 < n_active_blocks) {
+                    const uint32_t next_ib = active_blocks[step + 2];
                     const uint32_t next_ic_start = next_ib * FLASH_ATTN_BLOCK_SIZE;
                     const uint32_t next_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - next_ic_start);
 
@@ -504,6 +599,56 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
 
                         ((HVX_VectorPair *) VKQ32)[0] = vkq0;
                         ((HVX_VectorPair *) VKQ32)[1] = vkq1;
+                    } else if (DV == 256) {
+                        HVX_VectorPair vkq0 = ((const HVX_VectorPair *) VKQ32)[0];
+                        HVX_VectorPair vkq1 = ((const HVX_VectorPair *) VKQ32)[1];
+                        HVX_VectorPair vkq2 = ((const HVX_VectorPair *) VKQ32)[2];
+                        HVX_VectorPair vkq3 = ((const HVX_VectorPair *) VKQ32)[3];
+                        vkq0 = Q6_W_vcombine_VV(
+                            HVX_OP_MUL_F32(Q6_V_hi_W(vkq0), ms_vec),
+                            HVX_OP_MUL_F32(Q6_V_lo_W(vkq0), ms_vec)
+                        );
+                        vkq1 = Q6_W_vcombine_VV(
+                            HVX_OP_MUL_F32(Q6_V_hi_W(vkq1), ms_vec),
+                            HVX_OP_MUL_F32(Q6_V_lo_W(vkq1), ms_vec)
+                        );
+                        vkq2 = Q6_W_vcombine_VV(
+                            HVX_OP_MUL_F32(Q6_V_hi_W(vkq2), ms_vec),
+                            HVX_OP_MUL_F32(Q6_V_lo_W(vkq2), ms_vec)
+                        );
+                        vkq3 = Q6_W_vcombine_VV(
+                            HVX_OP_MUL_F32(Q6_V_hi_W(vkq3), ms_vec),
+                            HVX_OP_MUL_F32(Q6_V_lo_W(vkq3), ms_vec)
+                        );
+
+                        for (uint32_t j = 0; j < current_block_size; j += 2) {
+                            HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2));
+                            const HVX_Vector * vx0 = (const HVX_Vector *) v_ptr;
+                            if (j + 1 == current_block_size) {
+                                vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx0[0]), S0);
+                                vkq1 = hvx_vec_mpyacc_f32_f16(vkq1, Q6_Vh_vshuff_Vh(vx0[1]), S0);
+                                vkq2 = hvx_vec_mpyacc_f32_f16(vkq2, Q6_Vh_vshuff_Vh(vx0[2]), S0);
+                                vkq3 = hvx_vec_mpyacc_f32_f16(vkq3, Q6_Vh_vshuff_Vh(vx0[3]), S0);
+                                break;
+                            }
+
+                            HVX_Vector S1 = hvx_vec_repl_f16(Q6_V_vror_VR(P, (j + 1) * 2));
+                            const HVX_Vector * vx1 = (const HVX_Vector *) (v_ptr + factx->size_v_row_padded);
+                            vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx0[0]), S0);
+                            vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx1[0]), S1);
+                            vkq1 = hvx_vec_mpyacc_f32_f16(vkq1, Q6_Vh_vshuff_Vh(vx0[1]), S0);
+                            vkq1 = hvx_vec_mpyacc_f32_f16(vkq1, Q6_Vh_vshuff_Vh(vx1[1]), S1);
+                            vkq2 = hvx_vec_mpyacc_f32_f16(vkq2, Q6_Vh_vshuff_Vh(vx0[2]), S0);
+                            vkq2 = hvx_vec_mpyacc_f32_f16(vkq2, Q6_Vh_vshuff_Vh(vx1[2]), S1);
+                            vkq3 = hvx_vec_mpyacc_f32_f16(vkq3, Q6_Vh_vshuff_Vh(vx0[3]), S0);
+                            vkq3 = hvx_vec_mpyacc_f32_f16(vkq3, Q6_Vh_vshuff_Vh(vx1[3]), S1);
+                            v_ptr += stride_v2;
+                        }
+
+                        ((HVX_VectorPair *) VKQ32)[0] = vkq0;
+                        ((HVX_VectorPair *) VKQ32)[1] = vkq1;
+                        ((HVX_VectorPair *) VKQ32)[2] = vkq2;
+                        ((HVX_VectorPair *) VKQ32)[3] = vkq3;
                     } else {
                         hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec);
 
@@ -524,9 +669,9 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
                 }
                 htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, head_ir);
 
-                // prefetch V and mask for block ib + 2 after last head finished V accumulation
-                if (g + 1 == G_local && ib + 2 < factx->n_blocks) {
-                    const uint32_t next_ib = ib + 2;
+                // prefetch V and mask for step + 2 after last head finished V accumulation
+                if (g + 1 == G_local && step + 2 < n_active_blocks) {
+                    const uint32_t next_ib = active_blocks[step + 2];
                     const uint32_t next_ic_start = next_ib * FLASH_ATTN_BLOCK_SIZE;
                     const uint32_t next_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - next_ic_start);
 
@@ -541,7 +686,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
                     }
                 }
             } // end for g
-        } // end for ib
+        } // end for step
 
         for (uint32_t g = 0; g < G_local; ++g) {
             const uint32_t head_ir = ir + g;
@@ -1997,6 +2142,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
     factx.vtcm_mask_buf       = VTCM_LAYOUT_PTR(__fp16, base, L.off_mask_buf);
     factx.mask_buf_row_stride = L.mask_buf_row_stride;
     factx.q_tile_bytes        = L.q_tile_bytes;
+    factx.k_tile_bytes        = L.k_tile_bytes;
     factx.o_tile_bytes        = L.o_tile_bytes;
     factx.col_vec_bytes       = L.col_vec_bytes;
     factx.d_tile_bytes        = L.d_tile_bytes;
@@ -2049,7 +2195,6 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
     const size_t q_src_stride               = q_transposed ? q->nb[2] : q->nb[1];
     const size_t q_row_bytes_untransposed   = factx.G * factx.DK * qo_element_size;
     const size_t q_row_bytes_trans_factor   = factx.DK * qo_element_size;
-    const uint32_t kv_rows0                 = hex_smin(Bc, nek1);
 
     // ======== Reusable job descriptors for pipeline ========
     hmx_fa_qk_job_t       qk_job;
@@ -2065,34 +2210,51 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
             const size_t   g_br_actual = hex_align_up(n_rows_g, HMX_FP16_TILE_N_ROWS);
             const size_t   n_row_tiles = g_br_actual / HMX_FP16_TILE_N_ROWS;
 
+            uint16_t active_blocks[FA_MAX_KV_BLOCKS];
+            uint32_t n_active_blocks = factx.n_kv_blocks;
+            if (mask && factx.mask_broadcast && factx.n_kv_blocks > 1) {
+                const size_t scratch_bytes = factx.k_tile_bytes * (factx.pipeline ? 2 : 1);
+                n_active_blocks = fa_scan_active_blocks(
+                    dma_q, mask, q_start, n_rows_q, 0, im3, nek1, Bc,
+                    factx.n_kv_blocks, factx.vtcm_k_tiles[0], scratch_bytes, active_blocks);
+            } else {
+                #pragma clang loop vectorize(disable)
+                for (uint32_t b = 0; b < factx.n_kv_blocks; ++b) {
+                    active_blocks[b] = (uint16_t) b;
+                }
+            }
+            const uint32_t b0       = (n_active_blocks > 0) ? active_blocks[0] : 0;
+            const uint32_t b0_start = b0 * Bc;
+            const uint32_t b0_rows  = hex_smin(Bc, nek1 - b0_start);
+
             for (uint32_t kv_head = kv_head_min; kv_head < kv_head_max; ++kv_head) {
                 const uint32_t ik2 = kv_head;
                 const uint32_t ik3 = fastdiv(ib3, &kparams->broadcast_rk3);
                 const uint32_t iv2 = kv_head;
                 const uint32_t iv3 = fastdiv(ib3, &kparams->broadcast_rv3);
 
-                // 1. Push Q and KV DMAs for the very first iteration.
-                // Subsequent iterations are enqueued early at the end of the previous iteration.
-                if (ib3 == 0 && q_start == q_start_min && kv_head == kv_head_min) {
+                // 1. Push Q and KV DMAs for the first kv_head of this q_start block.
+                // Subsequent kv_head iterations are enqueued early at the end of the previous iteration.
+                if (kv_head == kv_head_min) {
                     const dma_addr_t q_ptr = q->data + q_start * q->nb[1] +
                                             (kv_head * factx.G) * q->nb[2] + ib3 * q->nb[3];
                     const size_t q_row_bytes = q_transposed ? n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed;
                     const size_t n_rows      = q_transposed ? factx.G : n_rows_q;
                     dma_queue_push(dma_q, dma_make_data(factx.vtcm_q_dma, q_ptr), q_row_bytes, hex_smax(q_src_stride, q_row_bytes), q_row_bytes, n_rows);
 
-                    if (factx.n_kv_blocks > 0) {
-                        const dma_addr_t k_src = k->data + ik2 * k->nb[2] + ik3 * k->nb[3];
-                        dma_queue_push(dma_q, dma_make_data(factx.vtcm_k_fp16[0], k_src), size_k_row_padded, k->nb[1], size_k_row, kv_rows0);
+                    if (n_active_blocks > 0) {
+                        const dma_addr_t k_src = k->data + b0_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3];
+                        dma_queue_push(dma_q, dma_make_data(factx.vtcm_k_fp16[0], k_src), size_k_row_padded, k->nb[1], size_k_row, b0_rows);
 
-                        const dma_addr_t v_src = v->data + iv2 * v->nb[2] + iv3 * v->nb[3];
-                        dma_queue_push(dma_q, dma_make_data(factx.vtcm_v_fp16[0], v_src), size_v_row_padded, v->nb[1], size_v_row, kv_rows0);
+                        const dma_addr_t v_src = v->data + b0_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3];
+                        dma_queue_push(dma_q, dma_make_data(factx.vtcm_v_fp16[0], v_src), size_v_row_padded, v->nb[1], size_v_row, b0_rows);
 
                         if (factx.pipeline && mask) {
                             if (__builtin_expect(factx.mask_broadcast, true)) {
-                                const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + 0;
-                                dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), n_rows_q);
+                                const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + b0_start * sizeof(__fp16);
+                                dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], b0_rows * sizeof(__fp16), n_rows_q);
                             } else {
-                                fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, 0, kv_head, G, m_line_bytes, kv_rows0, n_rows_q, &factx);
+                                fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, b0_start, kv_head, G, m_line_bytes, b0_rows, n_rows_q, &factx);
                             }
                         }
                     }
@@ -2124,27 +2286,28 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
                     hmx_fa_qk_job_t qk_job[2];
                     hmx_fa_o_update_job_t ou_job[2];
 
-                    // Prefetch block 1 early if there are multiple blocks
-                    if (factx.n_kv_blocks > 1) {
-                        fa_prefetch_block(dma_q, k, v, mask, 1, Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row,
+                    // Prefetch second active block early if there are multiple active blocks
+                    if (n_active_blocks > 1) {
+                        fa_prefetch_block(dma_q, k, v, mask, active_blocks[1], Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row,
                                           ik2, ik3, iv2, iv3, q_start, im3, kv_head, G, m_line_bytes, n_rows_q, nek1, 1, &factx);
                     }
 
                     // Prep and start QK-dot(0)
                     void * curr_k0 = (void *) dma_queue_pop(dma_q).dst;
-                    fa_phase_k_interleave(&factx, kv_rows0, k_src_stride, curr_k0, 0, 0);
+                    fa_phase_k_interleave(&factx, b0_rows, k_src_stride, curr_k0, b0_start, 0);
 
                     qk_job[0].q_tiles        = factx.vtcm_q_tiles;
                     qk_job[0].k_tiles        = factx.vtcm_k_tiles[0];
                     qk_job[0].s_tiles        = factx.vtcm_s_tiles[0];
                     qk_job[0].n_row_tiles    = n_row_tiles;
-                    qk_job[0].n_col_tiles    = hmx_ceil_div(kv_rows0, HMX_FP16_TILE_N_COLS);
+                    qk_job[0].n_col_tiles    = hmx_ceil_div(b0_rows, HMX_FP16_TILE_N_COLS);
                     qk_job[0].n_dot_tiles    = DK_pad / 32;
                     qk_job[0].n_tiles_per_bc = n_tiles_per_bc;
                     qk_job[0].hmx_scales     = factx.vtcm_hmx_scales_qk;
                     hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job[0]));
 
-                    for (uint32_t kv_blk = 0; kv_blk < factx.n_kv_blocks; ++kv_blk) {
+                    for (uint32_t step = 0; step < n_active_blocks; ++step) {
+                        const uint32_t kv_blk      = active_blocks[step];
                         const uint32_t kv_start    = kv_blk * Bc;
                         const uint32_t kv_rows     = hex_smin(Bc, nek1 - kv_start);
                         const size_t   n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS);
@@ -2164,13 +2327,14 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
                             }
                         }
 
-                        // ---- 3. Start HMX O update for block kv_blk - 1 (reads P[1 - buf_idx], V[1 - buf_idx], D) ----
-                        // O update relys on the previous block's P and V tiles.
+                        // ---- 3. Start HMX O update for step - 1 (reads P[1 - buf_idx], V[1 - buf_idx], D) ----
+                        // O update relies on the previous block's P and V tiles.
                         // O update MUST be pushed before the next block's QK-dot: hmx_queue_pop() retires the
                         // oldest descriptor, so push order alone decides which pop waits for which job.
                         // If OU went in after QK(i+1), the pop below would retire QK(i+1) and leave
                         // OU(i-1) in flight into the next iteration, where V-prep overwrites V[prev_buf].
-                        if (kv_blk > 0) {
+                        if (step > 0) {
+                            const uint32_t prev_blk      = active_blocks[step - 1];
                             const size_t prev_buf        = 1 - buf_idx;
                             ou_job[prev_buf].o_curr      = o_tile_curr;
                             ou_job[prev_buf].o_prev      = o_tile_prev;
@@ -2180,16 +2344,17 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
                             ou_job[prev_buf].hmx_scales  = factx.vtcm_hmx_scales_id;
                             ou_job[prev_buf].n_row_tiles = n_row_tiles;
                             ou_job[prev_buf].n_col_tiles =
-                                hmx_ceil_div(hex_smin(Bc, nek1 - (kv_blk - 1) * Bc), HMX_FP16_TILE_N_COLS);
+                                hmx_ceil_div(hex_smin(Bc, nek1 - prev_blk * Bc), HMX_FP16_TILE_N_COLS);
                             ou_job[prev_buf].n_row_tiles_g_br = n_row_tiles_g_br;
                             ou_job[prev_buf].n_tiles_per_bc   = n_tiles_per_bc;
                             ou_job[prev_buf].DV               = DV_pad;
                             hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job[prev_buf]));
                         }
 
-                        // ---- 4. Pop and run K-prep for next block & push next QK-dot ----
-                        if (kv_blk + 1 < factx.n_kv_blocks) {
-                            const uint32_t next_start = (kv_blk + 1) * Bc;
+                        // ---- 4. Pop and run K-prep for next active block & push next QK-dot ----
+                        if (step + 1 < n_active_blocks) {
+                            const uint32_t next_blk   = active_blocks[step + 1];
+                            const uint32_t next_start = next_blk * Bc;
                             const uint32_t next_rows  = hex_smin(Bc, nek1 - next_start);
                             const size_t   next_buf   = 1 - buf_idx;
 
@@ -2236,15 +2401,15 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
                         // Run Softmax on HVX (blocking call)
                         fa_phase_softmax_and_build_d(&factx, &sargs, n_row_tiles, n_row_tiles_g_br);
 
-                        // Wait for HMX O update for block kv_blk - 1 to finish
-                        if (kv_blk > 0) {
+                        // Wait for HMX O update for step - 1 to finish
+                        if (step > 0) {
                             hmx_queue_pop(hmx_q);
                             hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev);
                         }
 
-                        // Prefetch block kv_blk + 2
-                        if (kv_blk + 2 < factx.n_kv_blocks) {
-                            fa_prefetch_block(dma_q, k, v, mask, kv_blk + 2, Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row,
+                        // Prefetch block step + 2
+                        if (step + 2 < n_active_blocks) {
+                            fa_prefetch_block(dma_q, k, v, mask, active_blocks[step + 2], Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row,
                                               ik2, ik3, iv2, iv3, q_start, im3, kv_head, G, m_line_bytes, n_rows_q, nek1, buf_idx, &factx);
                         }
 
@@ -2252,8 +2417,8 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
                     }
 
                     // Epilogue
-                    if (factx.n_kv_blocks > 0) {
-                        const uint32_t last_blk = factx.n_kv_blocks - 1;
+                    if (n_active_blocks > 0) {
+                        const uint32_t last_blk = active_blocks[n_active_blocks - 1];
                         const size_t last_cols  = hmx_ceil_div(hex_smin(Bc, nek1 - last_blk * Bc), HMX_FP16_TILE_N_COLS);
                         ou_job[0].o_curr           = o_tile_curr;
                         ou_job[0].o_prev           = o_tile_prev;
@@ -2279,7 +2444,8 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
 
                 } else {
                     // Fallback path
-                    for (uint32_t kv_blk = 0; kv_blk < factx.n_kv_blocks; ++kv_blk) {
+                    for (uint32_t step = 0; step < n_active_blocks; ++step) {
+                        const uint32_t kv_blk      = active_blocks[step];
                         const uint32_t kv_start    = kv_blk * Bc;
                         const uint32_t kv_rows     = hex_smin(Bc, nek1 - kv_start);
                         const size_t   n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS);
@@ -2293,8 +2459,9 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
                             }
                         }
 
-                        if (kv_blk + 1 < factx.n_kv_blocks) {
-                            const uint32_t  prefetch_start = (kv_blk + 1) * Bc;
+                        if (step + 1 < n_active_blocks) {
+                            const uint32_t  prefetch_blk   = active_blocks[step + 1];
+                            const uint32_t  prefetch_start = prefetch_blk * Bc;
                             const uint32_t  prefetch_rows  = hex_smin(Bc, nek1 - prefetch_start);
                             const size_t    prefetch_buf   = 1 - buf_idx;
                             const dma_addr_t k_prefetch_src = k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3];
@@ -2372,7 +2539,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
                             ou_job.DV               = DV_pad;
 
                             hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job));
-                            if (kv_blk + 1 == factx.n_kv_blocks) {
+                            if (step + 1 == n_active_blocks) {
                                 // Overlapped: run HVX build diag inv L while HMX is busy executing the update
                                 htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start);
                                 fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br, kv_head, n_rows_g);
@@ -2387,53 +2554,32 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
                     }
                 }
 
-                // Enqueue DMAs for the next iteration early so they overlap with O-PROC
-                uint32_t next_kv_head = kv_head + 1;
-                uint32_t next_q_start = q_start;
-                uint32_t next_ib3     = ib3;
-                if (next_kv_head >= kv_head_max) {
-                    next_kv_head = kv_head_min;
-                    next_q_start = q_start + Br;
-                    if (next_q_start >= q_start_max) {
-                        next_q_start = q_start_min;
-                        next_ib3     = ib3 + 1;
-                    }
-                }
-                bool has_next = (next_ib3 < neq3);
+                // Enqueue DMAs for the next kv_head early so they overlap with O-PROC
+                const uint32_t next_kv_head = kv_head + 1;
+                const bool has_next = (next_kv_head < kv_head_max);
 
                 if (has_next) {
-                    const uint32_t next_n_rows_q = hex_smin(Br, neq1 - next_q_start);
-                    const dma_addr_t next_q_ptr = q->data + next_q_start * q->nb[1] + (next_kv_head * factx.G) * q->nb[2] + next_ib3 * q->nb[3];
-                    const size_t next_q_row_bytes = q_transposed ? next_n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed;
-                    const size_t next_n_rows      = q_transposed ? factx.G : next_n_rows_q;
+                    const dma_addr_t next_q_ptr = q->data + q_start * q->nb[1] + (next_kv_head * factx.G) * q->nb[2] + ib3 * q->nb[3];
+                    const size_t next_q_row_bytes = q_transposed ? n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed;
+                    const size_t next_n_rows      = q_transposed ? factx.G : n_rows_q;
                     dma_queue_push(dma_q, dma_make_data(factx.vtcm_q_dma, next_q_ptr), next_q_row_bytes, hex_smax(q_src_stride, next_q_row_bytes), next_q_row_bytes, next_n_rows);
 
-                    if (factx.n_kv_blocks > 0) {
+                    if (n_active_blocks > 0) {
                         const uint32_t next_ik2 = next_kv_head;
                         const uint32_t next_iv2 = next_kv_head;
-                        uint32_t next_ik3 = ik3;
-                        uint32_t next_iv3 = iv3;
-                        if (next_ib3 != ib3) {
-                            next_ik3 = fastdiv(next_ib3, &kparams->broadcast_rk3);
-                            next_iv3 = fastdiv(next_ib3, &kparams->broadcast_rv3);
-                        }
 
-                        const dma_addr_t next_k_src = k->data + next_ik2 * k->nb[2] + next_ik3 * k->nb[3];
-                        dma_queue_push(dma_q, dma_make_data(factx.vtcm_k_fp16[0], next_k_src), size_k_row_padded, k->nb[1], size_k_row, kv_rows0);
+                        const dma_addr_t next_k_src = k->data + b0_start * k->nb[1] + next_ik2 * k->nb[2] + ik3 * k->nb[3];
+                        dma_queue_push(dma_q, dma_make_data(factx.vtcm_k_fp16[0], next_k_src), size_k_row_padded, k->nb[1], size_k_row, b0_rows);
 
-                        const dma_addr_t next_v_src = v->data + next_iv2 * v->nb[2] + next_iv3 * v->nb[3];
-                        dma_queue_push(dma_q, dma_make_data(factx.vtcm_v_fp16[0], next_v_src), size_v_row_padded, v->nb[1], size_v_row, kv_rows0);
+                        const dma_addr_t next_v_src = v->data + b0_start * v->nb[1] + next_iv2 * v->nb[2] + iv3 * v->nb[3];
+                        dma_queue_push(dma_q, dma_make_data(factx.vtcm_v_fp16[0], next_v_src), size_v_row_padded, v->nb[1], size_v_row, b0_rows);
 
                         if (factx.pipeline && mask) {
-                            uint32_t next_im3 = im3;
-                            if (next_ib3 != ib3) {
-                                next_im3 = fastmodulo(next_ib3, mask->ne[3], &factx.src3_div3);
-                            }
                             if (__builtin_expect(factx.mask_broadcast, true)) {
-                                const dma_addr_t ms_src = mask->data + next_q_start * mask->nb[1] + next_im3 * mask->nb[3] + 0;
-                                dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), next_n_rows_q);
+                                const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + b0_start * sizeof(__fp16);
+                                dma_cache_fa_push(dma_q, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], b0_rows * sizeof(__fp16), n_rows_q);
                             } else {
-                                fa_push_mask_dma_gqa(dma_q, mask, next_q_start, next_im3, 0, next_kv_head, G, m_line_bytes, kv_rows0, next_n_rows_q, &factx);
+                                fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, b0_start, next_kv_head, G, m_line_bytes, b0_rows, n_rows_q, &factx);
                             }
                         }
                     }
@@ -2481,7 +2627,7 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
 
     const struct htp_fa_kernel_params * kparams = (const struct htp_fa_kernel_params *) octx->kernel_params;
 
-    if (kparams->kernel_type == HTP_FA_KERNEL_UNSUPPORTED) {
+    if (kparams->kernel_type == HTP_FA_KERNEL_UNSUPPORTED || kparams->n_kv_blocks > FA_MAX_KV_BLOCKS) {
         return HTP_STATUS_NO_SUPPORT;
     }
 
@@ -2596,12 +2742,21 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
 
     uint8_t * const base = octx->ctx->vtcm_base;
 
-    factx.spad_q     = VTCM_LAYOUT_PTR(uint8_t, base, L.off_q);
-    factx.spad_k     = VTCM_LAYOUT_PTR(uint8_t, base, L.off_k);
-    factx.spad_v     = VTCM_LAYOUT_PTR(uint8_t, base, L.off_v);
-    factx.spad_m     = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, L.off_m, has_mask);
-    factx.spad_a     = VTCM_LAYOUT_PTR(uint8_t, base, L.off_a);
-    factx.spad_sinks = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_sinks, has_sinks);
+    factx.spad_q               = VTCM_LAYOUT_PTR(uint8_t, base, L.off_q);
+    factx.spad_k               = VTCM_LAYOUT_PTR(uint8_t, base, L.off_k);
+    factx.spad_v               = VTCM_LAYOUT_PTR(uint8_t, base, L.off_v);
+    factx.spad_m               = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, L.off_m, has_mask);
+    factx.spad_a               = VTCM_LAYOUT_PTR(uint8_t, base, L.off_a);
+    factx.spad_sinks           = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_sinks, has_sinks);
+    factx.n_active_blocks      = factx.n_blocks;
+    factx.shared_active_blocks = false;
+
+    if (has_mask && neq1 == 1 && neq3 == 1 && mask->ne[2] == 1 && factx.n_blocks > 1 && factx.n_blocks <= FA_MAX_KV_BLOCKS) {
+        factx.n_active_blocks = fa_scan_active_blocks(
+            octx->ctx->dma[0], mask, 0, 1, 0, 0, k->ne[1], FLASH_ATTN_BLOCK_SIZE,
+            factx.n_blocks, factx.spad_k, factx.size_k_block * 2, factx.active_blocks);
+        factx.shared_active_blocks = true;
+    }
 
     if (has_sinks) {
         const struct htp_tensor * sinks = octx->src[4];

ggml/src/ggml-hexagon/htp/flash-attn-ops.h

@@ -126,6 +126,7 @@ struct hmx_fa_vtcm_layout {
     // Region byte sizes reused by the device at runtime (not just for allocation).
     size_t q_tile_bytes;
     size_t o_tile_bytes;
+    size_t k_tile_bytes;
     size_t s_tile_bytes;       // S and P tiles (same size)
     size_t d_tile_bytes;       // d_tiles[0..1] + d_inv_l, allocated back to back
     size_t m_line_bytes;       // one mask row
@@ -141,6 +142,8 @@ struct hmx_fa_vtcm_layout {
     size_t total_bytes;
 };
 
+#define FA_MAX_KV_BLOCKS 1024
+
 // Build the VTCM layout.
 
 static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L,
@@ -224,6 +227,7 @@ static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L,
 
     L->q_tile_bytes        = q_tile_size;
     L->o_tile_bytes        = o_tile_size;
+    L->k_tile_bytes        = k_tile_size;
     L->col_vec_bytes       = col_vec_size;
     L->s_tile_bytes        = s_tile_size;
     // Measured from the actual offsets rather than assumed to be N * d_tile_size, so
@@ -332,8 +336,12 @@ static inline int hmx_fa_find_chunk_size(size_t * Br_out,
 
     // Pipeline constraint: cap Bc so n_kv_blocks >= FA_MIN_KV_BLOCKS.
     // Only relax when kv_len is too short to form enough blocks.
-    const size_t Bc_limit     = can_pipeline ? hex_align_down(kv_len / FA_MIN_KV_BLOCKS, bc_unit) :
-                                               (kv_len >= bc_unit ? hex_align_down(kv_len, bc_unit) : bc_unit);
+    size_t Bc_limit     = can_pipeline ? hex_align_down(kv_len / FA_MIN_KV_BLOCKS, bc_unit) :
+                                         (kv_len >= bc_unit ? hex_align_down(kv_len, bc_unit) : bc_unit);
+    // For small-Q SWA-sized KV caches (qo_len <= 16, kv_len <= 2048), cap Bc at 256 for block-skipping granularity
+    if (qo_len <= 16 && kv_len <= 2048 && Bc_limit > 256) {
+        Bc_limit = 256;
+    }
     // Cost coefficients calibrated from profiling
     const size_t c_q_fixed    = 800;   // per-Q-block: q_load + epilogue o_update + o_norm + o_store
     const size_t c_iter_base  = 200;   // per-KV-iter base (HMX dot/update + DMA)

ggml/src/ggml-hexagon/htp/get-rows-ops.c

@@ -288,7 +288,7 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat
     }
 
     const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);
-    const uint32_t n_k_tiles = ne00 / HTP_TENSOR_TILE_N_COLS;
+    const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(octx->src[0]->type);
     const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout;
     uint8_t * src_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread;
     uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst)  + ith * vtcm_layout->dst_bytes_per_thread;
@@ -450,7 +450,9 @@ int op_get_rows(struct htp_ops_context * octx) {
             break;
         case HTP_GET_ROWS_KERNEL_TILED:
             switch (octx->src[0]->type) {
-                case HTP_TYPE_F16:  grctx.compute_tiled = hvx_dequantize_row_tiled_f16_f32; break;
+                case HTP_TYPE_F16:  grctx.compute_tiled = hvx_dequantize_row_tiled_f16_f32;  break;
+                case HTP_TYPE_Q1_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q1_0_f32; break;
+                case HTP_TYPE_Q2_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q2_0_f32; break;
                 case HTP_TYPE_Q8_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q8_0_f32; break;
                 case HTP_TYPE_Q4_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_0_f32; break;
                 case HTP_TYPE_Q4_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_k_f32; break;
@@ -466,6 +468,8 @@ int op_get_rows(struct htp_ops_context * octx) {
             switch (octx->src[0]->type) {
                 case HTP_TYPE_F16:  grctx.compute_flat = hvx_dequantize_row_flat_f16_f32;  break;
                 case HTP_TYPE_BF16: grctx.compute_flat = hvx_dequantize_row_flat_bf16_f32; break;
+                case HTP_TYPE_Q1_0: grctx.compute_flat = hvx_dequantize_row_flat_q1_0_f32; break;
+                case HTP_TYPE_Q2_0: grctx.compute_flat = hvx_dequantize_row_flat_q2_0_f32; break;
                 case HTP_TYPE_Q8_0: grctx.compute_flat = hvx_dequantize_row_flat_q8_0_f32; break;
                 case HTP_TYPE_Q4_0: grctx.compute_flat = hvx_dequantize_row_flat_q4_0_f32; break;
                 case HTP_TYPE_Q4_K: grctx.compute_flat = hvx_dequantize_row_flat_q4_k_f32; break;

ggml/src/ggml-hexagon/htp/get-rows-ops.h

@@ -64,7 +64,7 @@ static inline void htp_get_rows_vtcm_layout_build(
         size_t src0_row_size = 0;
         if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) {
             const size_t tile_stride = htp_tensor_get_tile_padded_size(type);
-            const uint32_t n_k_tiles = ne00 / HTP_TENSOR_TILE_N_COLS;
+            const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(type);
             src0_row_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride;
         } else {
             src0_row_size = htp_tensor_get_row_size(type, ne00);

ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h

@@ -26,6 +26,7 @@ struct tiled_dequantize_state {
 
     struct fastdiv_values     n_k_tiles_div;
     uint32_t                  n_k_tiles;
+    uint32_t                  n_weight_tiles;
     uint32_t                  n_tot_tiles;
     uint32_t                  n_tiles_per_task;
     uint32_t                  tile_size;
@@ -37,6 +38,39 @@ struct tiled_dequantize_state {
     uint32_t                  weight_type;
 };
 
+static void dequantize_tiled_weight_to_fp16_task_q1_0(
+        const tiled_dequantize_state_t *state,
+        uint32_t start_tile, uint32_t end_tile) {
+
+    const HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1);
+    const HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1);
+
+    for (uint32_t t = start_tile; t < end_tile; t++) {
+        const uint8_t * tile_src = state->src + t * state->aligned_tile_size;
+        __fp16 * dst_ptr = state->dst + 4 * t * HTP_MM_HMX_TILE_N_ELMS;
+
+        const HVX_Vector v_sc = hvx_vmem(tile_src + 512);
+        const HVX_Vector v_scale = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2));
+
+        for (uint32_t q = 0; q < 4; q++) {
+            const HVX_Vector v_plane = hvx_vmem(tile_src + q * 128);
+            __fp16 * quarter_dst = dst_ptr + q * HTP_MM_HMX_TILE_N_ELMS;
+
+            #pragma unroll
+            for (int g = 0; g < 8; g++) {
+                const HVX_Vector     v_q  = unpack_q1_0_group(v_plane, g, v_neg_one, v_pos_one);
+                const HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q);
+                const HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4);
+
+                hvx_vmem(quarter_dst + (2 * g + 0) * 64) =
+                    Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale));
+                hvx_vmem(quarter_dst + (2 * g + 1) * 64) =
+                    Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale));
+            }
+        }
+    }
+}
+
 // Dequantize a single tile from tiled weight data (already in VTCM) to tile-major FP16.
 static void dequantize_tiled_weight_to_fp16_task_q4_0(
         const tiled_dequantize_state_t *state,
@@ -751,6 +785,40 @@ static void dequantize_tiled_weight_to_fp16_task_q2_k(
     }
 }
 
+// Q2_0 stores codes 0..3 (value code - 1) and one fp16 scale per row, see HTP_MM_WEIGHT_TILE_SIZE_Q2_0.
+// One packed 64-k tile expands into two consecutive 32-k fp16 tiles.
+static void dequantize_tiled_weight_to_fp16_task_q2_0(
+        const tiled_dequantize_state_t *state,
+        uint32_t start_tile, uint32_t end_tile) {
+
+    const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
+    const HVX_Vector i1      = Q6_Vb_vsplat_R(1);
+
+    for (uint32_t t = start_tile; t < end_tile; t++) {
+        const HVX_Vector * vptr = (const HVX_Vector *) (state->src + t * state->aligned_tile_size);
+        __fp16 * dst_base = state->dst + 2 * t * HTP_MM_HMX_TILE_N_ELMS;
+
+        HVX_Vector v_sc    = vptr[4];
+        HVX_Vector v_scale = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2));
+
+        for (int half = 0; half < 2; half++) {
+            __fp16 * dst_ptr = dst_base + half * HTP_MM_HMX_TILE_N_ELMS;
+
+            #pragma unroll
+            for (int g = 0; g < 8; g++) {
+                HVX_Vector     v_q  = Q6_Vb_vsub_VbVb(unpack_q3_k_low2(vptr + 2 * half, g, mask_03), i1);
+                HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q);
+                HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4);
+
+                hvx_vmem(dst_ptr + (2 * g + 0) * 64) =
+                    Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale));
+                hvx_vmem(dst_ptr + (2 * g + 1) * 64) =
+                    Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale));
+            }
+        }
+    }
+}
+
 static __attribute__((noinline))
 void convert_f16_weight_to_fp16_tiles_task(
         const tiled_dequantize_state_t *state,

ggml/src/ggml-hexagon/htp/htp-ops.h

@@ -32,6 +32,8 @@ enum htp_data_type {
     HTP_TYPE_I64    = 27,
     HTP_TYPE_BF16   = 30,
     HTP_TYPE_MXFP4  = 39,
+    HTP_TYPE_Q1_0   = 41,
+    HTP_TYPE_Q2_0   = 42,
 
     HTP_TYPE_INVALID
 };
@@ -112,6 +114,7 @@ enum htp_op_code {
     HTP_OP_POOL_1D,
     HTP_OP_UNARY_GELU_QUICK,
     HTP_OP_IM2COL_3D,
+    HTP_OP_HADAMARD,
 
     HTP_OP_INVALID
 };

ggml/src/ggml-hexagon/htp/htp-tensor.h

@@ -25,32 +25,13 @@
 #define HTP_TENSOR_TILE_SIZE_Q6_K   896
 #define HTP_TENSOR_TILE_SIZE_Q3_K   512
 #define HTP_TENSOR_TILE_SIZE_Q2_K   336
+#define HTP_TENSOR_TILE_K_Q1_0      128
+#define HTP_TENSOR_TILE_SIZE_Q1_0   576
+#define HTP_TENSOR_TILE_K_Q2_0      64
+#define HTP_TENSOR_TILE_SIZE_Q2_0   576
 
-// Q2_K, Q3_K, Q6_K use vrmpy byte order within each 128B HVX vector:
-//   k = 0..31 is split into 8 groups g = k/4 of 4 k each; byte 4*r + (k%4) holds row r
-//
-// Q2_K (336B, lossless tiling with same size as block_q2_K):
-//   bytes   0..255 (vec 0..1): unsigned 2-bit quants, vec m holds groups 4m..4m+3 at bit offsets 0,2,4,6
-//   bytes 256..319:            raw scales[] bytes (min << 4 | sc): [0..31] for k 0..15, [32..63] for k 16..31
-//   bytes 320..335:            8 fp16 in tile j (0..7) of super-block: d (j < 4) or -dmin (j >= 4) for rows 8*(j%4)..8*(j%4)+7
-//
-// Q3_K (512B):
-//   bytes   0..255 (vec 0..1): low 2 bits, vec m holds groups 4m..4m+3 at bit offsets 0,2,4,6
-//   bytes 256..383 (vec 2):    sign mask, bit g set when hmask is 0 (quant = low2 - 4)
-//   bytes 384..511 (vec 3):    fp16 scales d * (scales[] - 32): fp16[0..31] for k 0..15, fp16[32..63] for k 16..31
-//
-// Q4_K (640B, same layout as Q4_1 tile; each 256-k super-block splits into 8 tiles of 32 k):
-//   bytes   0..511 (vec 0..3): 4-bit quants, byte 32*cp + r holds k = 2*cp (low nibble) and 2*cp + 1 (high nibble)
-//   bytes 512..639 (vec 4):    32x2 fp16 per row: (d * sc, -dmin * m)
-//
-// Q5_K (768B):
-//   bytes   0..639 (vec 0..4): Q4_K tile (low 4 bits + fp16 scale/min pairs)
-//   bytes 640..767 (vec 5):    5th-bit plane; byte l holds the 8 high bits for lane l across vec i = 0..3 (bit 2i = lo, 2i+1 = hi)
-//
-// Q6_K (896B):
-//   bytes   0..511 (vec 0..3): low 4 bits, vec i holds group 2i (low nibble) and 2i+1 (high nibble)
-//   bytes 512..767 (vec 4..5): high 2 bits, vec 4+m holds groups 4m..4m+3 at bit offsets 0,2,4,6
-//   bytes 768..895 (vec 6):    fp16 scales d * scales[]: fp16[0..31] for k 0..15, fp16[32..63] for k 16..31
+// Tiled weight tensor layouts (Q4_0, Q8_0, IQ4_NL, MXFP4, Q1_0, Q2_0, K-quants):
+// See docs/backend/snapdragon/developer.md ("Tiled Weight Formats").
 
 struct htp_tensor_mdev_range {
     uint32_t start;
@@ -168,12 +149,22 @@ static inline uint32_t htp_tensor_get_tile_size(int type) {
         case HTP_TYPE_Q6_K:   return HTP_TENSOR_TILE_SIZE_Q6_K;
         case HTP_TYPE_Q3_K:   return HTP_TENSOR_TILE_SIZE_Q3_K;
         case HTP_TYPE_Q2_K:   return HTP_TENSOR_TILE_SIZE_Q2_K;
+        case HTP_TYPE_Q1_0:   return HTP_TENSOR_TILE_SIZE_Q1_0;
+        case HTP_TYPE_Q2_0:   return HTP_TENSOR_TILE_SIZE_Q2_0;
         case HTP_TYPE_MXFP4:  return HTP_TENSOR_TILE_SIZE_MXFP4;
         case HTP_TYPE_F16:    return HTP_TENSOR_TILE_SIZE_F16;
         default:              return 0;
     }
 }
 
+static inline uint32_t htp_tensor_get_tile_k(int type) {
+    switch (type) {
+        case HTP_TYPE_Q1_0: return HTP_TENSOR_TILE_K_Q1_0;
+        case HTP_TYPE_Q2_0: return HTP_TENSOR_TILE_K_Q2_0;
+        default:            return 32;
+    }
+}
+
 static inline uint32_t htp_tensor_get_tile_padded_size(int type) {
     return (htp_tensor_get_tile_size(type) + 127u) & ~127u;
 }
@@ -193,10 +184,12 @@ static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) {
         case HTP_TYPE_Q4_1:   return (ne00 / 32) * 20;
         case HTP_TYPE_MXFP4:  return (ne00 / 32) * 17;
         case HTP_TYPE_Q2_K:   return (ne00 / 256) * 84;
+        case HTP_TYPE_Q2_0:   return (ne00 / 64) * 18;
         case HTP_TYPE_Q3_K:   return (ne00 / 256) * 110;
         case HTP_TYPE_Q4_K:   return (ne00 / 256) * 144;
         case HTP_TYPE_Q5_K:   return (ne00 / 256) * 176;
         case HTP_TYPE_Q6_K:   return (ne00 / 256) * 210;
+        case HTP_TYPE_Q1_0:   return (ne00 / HTP_TENSOR_TILE_K_Q1_0) * 18;
         case HTP_TYPE_I32:    return ne00 * 4;
         case HTP_TYPE_I64:    return ne00 * 8;
         default:              return 0;

ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h

@@ -29,29 +29,27 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_
 
     HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero);
     HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero);
-    HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero);
-    HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero);
-
     HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero);
     HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero);
+
+    HVX_Vector vmax01_hf   = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf)));
+    HVX_Vector vx01_hf     = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf)));
+    HVX_Vector vd01_qf16   = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008));  // 1.0 / 127.0
+    HVX_Vector vd01_hf     = Q6_Vhf_equals_Vqf16(vd01_qf16);
+    HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf);
+    vx01_hf                = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf));
+
+    HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero);
+    HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero);
     HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero);
     HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero);
 
-    HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf)));
-    HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf)));
-
-    HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf)));
-    HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf)));
-
-    HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008));  // 1.0 / 127.0
-    HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008));  // 1.0 / 127.0
-    HVX_Vector vd01_hf   = Q6_Vhf_equals_Vqf16(vd01_qf16);
-    HVX_Vector vd23_hf   = Q6_Vhf_equals_Vqf16(vd23_qf16);
-
-    HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf);
+    HVX_Vector vmax23_hf   = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf)));
+    HVX_Vector vx23_hf     = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf)));
+    HVX_Vector vd23_qf16   = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008));  // 1.0 / 127.0
+    HVX_Vector vd23_hf     = Q6_Vhf_equals_Vqf16(vd23_qf16);
     HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf);
-    vx01_hf              = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf));
-    vx23_hf              = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf));
+    vx23_hf                = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf));
 
     HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf);
     HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf);
@@ -81,25 +79,10 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_
     HVX_Vector voff01_hf = hvx_vec_f32_to_f16(voff0_sf, voff1_sf);
     HVX_Vector voff23_hf = hvx_vec_f32_to_f16(voff2_sf, voff3_sf);
 
-    HVX_Vector r_scale[4] = {
-        hvx_vec_repl_f16(vd01_hf),
-        hvx_vec_repl_f16(Q6_V_vror_VR(vd01_hf, 64)),
-        hvx_vec_repl_f16(vd23_hf),
-        hvx_vec_repl_f16(Q6_V_vror_VR(vd23_hf, 64)),
-    };
-    HVX_Vector r_offset[4] = {
-        hvx_vec_repl_f16(voff01_hf),
-        hvx_vec_repl_f16(Q6_V_vror_VR(voff01_hf, 64)),
-        hvx_vec_repl_f16(voff23_hf),
-        hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 64)),
-    };
+    int32_t __attribute__((aligned(128))) ws[32];
     if (sums16) {
         // |sum of 16 int8| <= 2032 fits a halfword
-        int32_t __attribute__((aligned(128))) ws[32];
         *(HVX_Vector *) ws = v_sums;
-        for (int b = 0; b < 4; b++) {
-            r_offset[b] = Q6_V_vsplat_R(((uint32_t) ws[8 * b + 4] << 16) | ((uint32_t) ws[8 * b] & 0xFFFF));
-        }
     }
 
     static const uint8_t __attribute__((aligned(128))) repl[128] = {
@@ -114,8 +97,15 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_
     };
     HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl;
 
+    #pragma unroll
     for (int b = 0; b < 4; b++) {
         HVX_Vector v_act = Q6_V_vror_VR(vx_i8, b * 32);
+        HVX_Vector vd_b   = (b < 2) ? vd01_hf   : vd23_hf;
+        HVX_Vector voff_b = (b < 2) ? voff01_hf : voff23_hf;
+        HVX_Vector r_scale  = hvx_vec_repl_f16((b & 1) ? Q6_V_vror_VR(vd_b, 64)   : vd_b);
+        HVX_Vector r_offset = sums16
+            ? Q6_V_vsplat_R(((uint32_t) ws[8 * b + 4] << 16) | ((uint32_t) ws[8 * b] & 0xFFFF))
+            : hvx_vec_repl_f16((b & 1) ? Q6_V_vror_VR(voff_b, 64) : voff_b);
 
         HVX_Vector r0 = Q6_V_vdelta_VV(v_act, v_repl_ctrl);
         HVX_Vector r1 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 4),  v_repl_ctrl);
@@ -135,8 +125,8 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_
         dst[5] = r5;
         dst[6] = r6;
         dst[7] = r7;
-        dst[8] = r_scale[b];
-        dst[9] = r_offset[b];
+        dst[8] = r_scale;
+        dst[9] = r_offset;
     }
 }
 
@@ -395,6 +385,36 @@ static inline HVX_VectorPair unpack_and_interleave_4bit_x2(HVX_Vector v_src, HVX
     return Q6_W_vcombine_VV(v_W1, v_W0);
 }
 
+static inline HVX_Vector unpack_q1_0_group(
+    HVX_Vector v_plane,
+    int g,
+    HVX_Vector v_neg_one,
+    HVX_Vector v_pos_one
+) {
+    HVX_VectorPred q = Q6_Q_vand_VR(v_plane, 0x01010101u << g);
+    return Q6_V_vmux_QVV(q, v_pos_one, v_neg_one);
+}
+
+static inline HVX_Vector accum_q1_0_32x1(
+    HVX_Vector v_plane,
+    const HVX_Vector * restrict v_act,
+    HVX_Vector v_neg_one,
+    HVX_Vector v_pos_one
+) {
+    HVX_Vector v_sum_lo = Q6_V_vzero();
+    HVX_Vector v_sum_hi = Q6_V_vzero();
+
+    #pragma unroll
+    for (int g = 0; g < 4; g++) {
+        HVX_Vector v_W_lo = unpack_q1_0_group(v_plane, g,     v_neg_one, v_pos_one);
+        HVX_Vector v_W_hi = unpack_q1_0_group(v_plane, g + 4, v_neg_one, v_pos_one);
+        v_sum_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum_lo, v_W_lo, v_act[g]);
+        v_sum_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum_hi, v_W_hi, v_act[g + 4]);
+    }
+
+    return Q6_Vw_vadd_VwVw(v_sum_lo, v_sum_hi);
+}
+
 static inline HVX_Vector accum_4bit_32x1(
     const HVX_Vector * restrict vptr,
     const HVX_Vector * restrict v_act,
@@ -437,6 +457,33 @@ static inline HVX_Vector accum_4bit_32x1_lut(
     return Q6_Vw_vadd_VwVw(v_sum0, v_sum1);
 }
 
+static inline HVX_VectorPair accum_q1_0_32x2(
+    HVX_Vector v_plane,
+    const HVX_Vector * restrict v_act0,
+    const HVX_Vector * restrict v_act1,
+    HVX_Vector v_neg_one,
+    HVX_Vector v_pos_one
+) {
+    HVX_Vector v_sum0_lo = Q6_V_vzero();
+    HVX_Vector v_sum0_hi = Q6_V_vzero();
+    HVX_Vector v_sum1_lo = Q6_V_vzero();
+    HVX_Vector v_sum1_hi = Q6_V_vzero();
+
+    #pragma unroll
+    for (int g = 0; g < 4; g++) {
+        HVX_Vector v_W_lo = unpack_q1_0_group(v_plane, g,     v_neg_one, v_pos_one);
+        HVX_Vector v_W_hi = unpack_q1_0_group(v_plane, g + 4, v_neg_one, v_pos_one);
+        v_sum0_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_lo, v_W_lo, v_act0[g]);
+        v_sum0_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_hi, v_W_hi, v_act0[g + 4]);
+        v_sum1_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_lo, v_W_lo, v_act1[g]);
+        v_sum1_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_hi, v_W_hi, v_act1[g + 4]);
+    }
+
+    return Q6_W_vcombine_VV(
+        Q6_Vw_vadd_VwVw(v_sum1_lo, v_sum1_hi),
+        Q6_Vw_vadd_VwVw(v_sum0_lo, v_sum0_hi));
+}
+
 static inline HVX_VectorPair accum_4bit_32x2(
     const HVX_Vector * restrict vptr,
     const HVX_Vector * restrict v_act0,
@@ -771,6 +818,53 @@ static inline void accum_q2_k_32x2(
     *v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo);
 }
 
+// Q2_0: unsigned codes 0..3; the -1 offset is applied via the Q8_1 block sum (v_act[9])
+static inline HVX_Vector accum_q2_0_32x1(
+    const HVX_Vector * restrict vptr,
+    const HVX_Vector * restrict v_act
+) {
+    HVX_Vector v_sum_lo = Q6_V_vzero();
+    HVX_Vector v_sum_hi = Q6_V_vzero();
+    HVX_Vector mask_03  = Q6_Vb_vsplat_R(0x03);
+
+    #pragma unroll
+    for (int g = 0; g < 4; g++) {
+        HVX_Vector v_W_lo = unpack_q3_k_low2(vptr, g,     mask_03);
+        HVX_Vector v_W_hi = unpack_q3_k_low2(vptr, g + 4, mask_03);
+        v_sum_lo = Q6_Vw_vrmpyacc_VwVubVb(v_sum_lo, v_W_lo, v_act[g]);
+        v_sum_hi = Q6_Vw_vrmpyacc_VwVubVb(v_sum_hi, v_W_hi, v_act[g + 4]);
+    }
+
+    return Q6_Vw_vadd_VwVw(v_sum_lo, v_sum_hi);
+}
+
+static inline void accum_q2_0_32x2(
+    const HVX_Vector * restrict vptr,
+    const HVX_Vector * restrict v_act0,
+    const HVX_Vector * restrict v_act1,
+    HVX_Vector * v_sum0,
+    HVX_Vector * v_sum1
+) {
+    HVX_Vector v_sum0_lo = Q6_V_vzero();
+    HVX_Vector v_sum0_hi = Q6_V_vzero();
+    HVX_Vector v_sum1_lo = Q6_V_vzero();
+    HVX_Vector v_sum1_hi = Q6_V_vzero();
+    HVX_Vector mask_03   = Q6_Vb_vsplat_R(0x03);
+
+    #pragma unroll
+    for (int g = 0; g < 4; g++) {
+        HVX_Vector v_W_lo = unpack_q3_k_low2(vptr, g,     mask_03);
+        HVX_Vector v_W_hi = unpack_q3_k_low2(vptr, g + 4, mask_03);
+        v_sum0_lo = Q6_Vw_vrmpyacc_VwVubVb(v_sum0_lo, v_W_lo, v_act0[g]);
+        v_sum0_hi = Q6_Vw_vrmpyacc_VwVubVb(v_sum0_hi, v_W_hi, v_act0[g + 4]);
+        v_sum1_lo = Q6_Vw_vrmpyacc_VwVubVb(v_sum1_lo, v_W_lo, v_act1[g]);
+        v_sum1_hi = Q6_Vw_vrmpyacc_VwVubVb(v_sum1_hi, v_W_hi, v_act1[g + 4]);
+    }
+
+    *v_sum0 = Q6_Vw_vadd_VwVw(v_sum0_lo, v_sum0_hi);
+    *v_sum1 = Q6_Vw_vadd_VwVw(v_sum1_lo, v_sum1_hi);
+}
+
 // Q2_K: fp16 d (*v_d) and -dmin (*v_m) of the 32 rows of a super-block (8 tiles, VTCM stride 384), each in both vector halves
 static inline void q2_k_load_dmin(const uint8_t * restrict sb, HVX_Vector * restrict v_d, HVX_Vector * restrict v_m) {
     HVX_Vector v_dd = Q6_V_vzero();  // lanes 0..31: d, lanes 32..63: -dmin
@@ -824,6 +918,89 @@ static inline HVX_Vector q2_k_int_scale(HVX_Vector v_isum, HVX_Vector v_msum, HV
     return hvx_vec_mul_f32_f32(v_t, v_da32);
 }
 
+static void tiled_vec_dot_q1_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
+    const uint8_t * restrict tile_ptr = vx;
+    const uint8_t * restrict y_q = vy;
+
+    HVX_Vector v_sum_qf  = Q6_V_vzero();
+    HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1);
+    HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1);
+
+    const uint32_t n_k_tiles = n / HTP_TENSOR_TILE_K_Q1_0;
+    for (uint32_t kt = 0; kt < n_k_tiles; kt++) {
+        const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * htp_tensor_get_tile_padded_size(HTP_TYPE_Q1_0));
+        HVX_Vector v_tile_qf = Q6_V_vzero();
+
+        #pragma unroll
+        for (int quarter = 0; quarter < 4; quarter++) {
+            const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + (4 * kt + quarter) * HTP_MM_ACT_TILE_SIZE_Q8_0);
+            HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(accum_q1_0_32x1(vptr[quarter], v_act, v_neg_one, v_pos_one));
+            HVX_Vector v_da_sf  = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act[8]));
+            HVX_Vector v_q_qf   = Q6_Vqf32_vmpy_VsfVsf(v_sum_sf, v_da_sf);
+            v_tile_qf = (quarter == 0) ? v_q_qf : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf, v_q_qf);
+        }
+
+        HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4]));
+        v_sum_qf = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf));
+    }
+
+    HVX_Vector v_sum_float = Q6_Vsf_equals_Vqf32(v_sum_qf);
+    if (sz) {
+        hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz)));
+    } else {
+        hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float);
+    }
+}
+
+static void tiled_vec_dot_q1_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) {
+    const uint8_t * restrict tile_ptr = vx;
+    const uint8_t * restrict y0_q = vy0;
+    const uint8_t * restrict y1_q = vy1;
+
+    HVX_Vector v_sum_qf_c0 = Q6_V_vzero();
+    HVX_Vector v_sum_qf_c1 = Q6_V_vzero();
+    HVX_Vector v_neg_one   = Q6_Vb_vsplat_R(-1);
+    HVX_Vector v_pos_one   = Q6_Vb_vsplat_R(1);
+
+    const uint32_t n_k_tiles = n / HTP_TENSOR_TILE_K_Q1_0;
+    for (uint32_t kt = 0; kt < n_k_tiles; kt++) {
+        const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * htp_tensor_get_tile_padded_size(HTP_TYPE_Q1_0));
+        HVX_Vector v_tile_qf_c0 = Q6_V_vzero();
+        HVX_Vector v_tile_qf_c1 = Q6_V_vzero();
+
+        for (int quarter = 0; quarter < 4; quarter++) {
+            const uint32_t act_tile = 4 * kt + quarter;
+            const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + act_tile * HTP_MM_ACT_TILE_SIZE_Q8_0);
+            const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + act_tile * HTP_MM_ACT_TILE_SIZE_Q8_0);
+            HVX_VectorPair v_sums = accum_q1_0_32x2(vptr[quarter], v_act0, v_act1, v_neg_one, v_pos_one);
+            HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(Q6_V_lo_W(v_sums));
+            HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(Q6_V_hi_W(v_sums));
+            HVX_Vector v_da_sf_c0  = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act0[8]));
+            HVX_Vector v_da_sf_c1  = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act1[8]));
+
+            v_tile_qf_c0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c0, Q6_Vqf32_vmpy_VsfVsf(v_sum_sf_c0, v_da_sf_c0));
+            v_tile_qf_c1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c1, Q6_Vqf32_vmpy_VsfVsf(v_sum_sf_c1, v_da_sf_c1));
+        }
+
+        HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4]));
+        v_sum_qf_c0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf_c0, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf_c0), v_dw_sf));
+        v_sum_qf_c1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf_c1, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf_c1), v_dw_sf));
+    }
+
+    HVX_Vector v_sum_float_c0 = Q6_Vsf_equals_Vqf32(v_sum_qf_c0);
+    HVX_Vector v_sum_float_c1 = Q6_Vsf_equals_Vqf32(v_sum_qf_c1);
+    if (sz0) {
+        hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0)));
+    } else {
+        hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0);
+    }
+    if (sz1) {
+        hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1)));
+    } else {
+        hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1);
+    }
+}
+
 static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
     const uint8_t * restrict tile_ptr = vx;
     const uint8_t * restrict y_q = vy;
@@ -1362,6 +1539,93 @@ static void tiled_vec_dot_q2_k_32x2(const uint32_t n, float * restrict s0, float
     }
 }
 
+// One Q2_0 weight tile covers 64 k: act tiles 2 * kt and 2 * kt + 1, each 32 k wide (Q8_1)
+static void tiled_vec_dot_q2_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
+    const uint8_t * restrict tile_ptr = vx;
+    const uint8_t * restrict y_q = vy;
+
+    HVX_Vector v_sum_float = Q6_V_vzero();
+    HVX_Vector v_one_f16   = hvx_vec_splat_f16(1.0);
+
+    uint32_t n_weight_tiles = n / HTP_MM_WEIGHT_TILE_K_Q2_0;
+    for (uint32_t kt = 0; kt < n_weight_tiles; kt++) {
+        const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0);
+        HVX_Vector v_tile_qf = Q6_V_vzero();
+
+        #pragma unroll
+        for (int half = 0; half < 2; half++) {
+            const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + (2 * kt + half) * HTP_MM_ACT_TILE_SIZE_Q8_1);
+
+            HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(accum_q2_0_32x1(vptr + 2 * half, v_act));
+            HVX_Vector v_da_sf  = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act[8]));
+            HVX_Vector v_sa_qf  = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(v_act[9], v_one_f16));
+            HVX_Vector v_h_qf   = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(v_sum_sf, v_da_sf), v_sa_qf);
+            v_tile_qf = (half == 0) ? v_h_qf : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf, v_h_qf);
+        }
+
+        HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4]));
+        v_sum_float = hvx_vec_add_f32_f32(v_sum_float, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf));
+    }
+
+    if (sz) {
+        hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz)));
+    } else {
+        hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float);
+    }
+}
+
+static void tiled_vec_dot_q2_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) {
+    const uint8_t * restrict tile_ptr = vx;
+    const uint8_t * restrict y0_q = vy0;
+    const uint8_t * restrict y1_q = vy1;
+
+    HVX_Vector v_sum_float_c0 = Q6_V_vzero();
+    HVX_Vector v_sum_float_c1 = Q6_V_vzero();
+    HVX_Vector v_one_f16      = hvx_vec_splat_f16(1.0);
+
+    uint32_t n_weight_tiles = n / HTP_MM_WEIGHT_TILE_K_Q2_0;
+    for (uint32_t kt = 0; kt < n_weight_tiles; kt++) {
+        const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0);
+        HVX_Vector v_tile_qf_c0 = Q6_V_vzero();
+        HVX_Vector v_tile_qf_c1 = Q6_V_vzero();
+
+        #pragma unroll
+        for (int half = 0; half < 2; half++) {
+            const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + (2 * kt + half) * HTP_MM_ACT_TILE_SIZE_Q8_1);
+            const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + (2 * kt + half) * HTP_MM_ACT_TILE_SIZE_Q8_1);
+
+            HVX_Vector v_sum_c0, v_sum_c1;
+            accum_q2_0_32x2(vptr + 2 * half, v_act0, v_act1, &v_sum_c0, &v_sum_c1);
+
+            HVX_Vector v_da_sf_c0 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act0[8]));
+            HVX_Vector v_da_sf_c1 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act1[8]));
+            HVX_Vector v_sa_qf_c0 = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(v_act0[9], v_one_f16));
+            HVX_Vector v_sa_qf_c1 = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(v_act1[9], v_one_f16));
+
+            HVX_Vector v_h_qf_c0 = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vw(v_sum_c0), v_da_sf_c0), v_sa_qf_c0);
+            HVX_Vector v_h_qf_c1 = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vw(v_sum_c1), v_da_sf_c1), v_sa_qf_c1);
+
+            v_tile_qf_c0 = (half == 0) ? v_h_qf_c0 : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c0, v_h_qf_c0);
+            v_tile_qf_c1 = (half == 0) ? v_h_qf_c1 : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c1, v_h_qf_c1);
+        }
+
+        HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4]));
+        v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf_c0), v_dw_sf));
+        v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf_c1), v_dw_sf));
+    }
+
+    if (sz0) {
+        hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0)));
+    } else {
+        hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0);
+    }
+    if (sz1) {
+        hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1)));
+    } else {
+        hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1);
+    }
+}
+
 static void tiled_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
     const uint8_t * restrict tile_ptr = vx;
     const uint8_t * restrict y_q = vy;

ggml/src/ggml-hexagon/htp/hvx-quant.h

@@ -191,6 +191,64 @@ static inline void hvx_dequantize_row_flat_bf16_f32(float * restrict dst_ptr, co
     }
 }
 
+static inline void hvx_dequantize_row_flat_q1_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
+    static const uint32_t __attribute__((aligned(128))) bit_mask[32] = {
+        1u <<  0, 1u <<  1, 1u <<  2, 1u <<  3, 1u <<  4, 1u <<  5, 1u <<  6, 1u <<  7,
+        1u <<  8, 1u <<  9, 1u << 10, 1u << 11, 1u << 12, 1u << 13, 1u << 14, 1u << 15,
+        1u << 16, 1u << 17, 1u << 18, 1u << 19, 1u << 20, 1u << 21, 1u << 22, 1u << 23,
+        1u << 24, 1u << 25, 1u << 26, 1u << 27, 1u << 28, 1u << 29, 1u << 30, 1u << 31,
+    };
+    const int nb = n / QK1_0;
+    const block_q1_0 * src = (const block_q1_0 *) src_ptr;
+    const HVX_Vector v_bit_mask = *(const HVX_Vector *) bit_mask;
+    const HVX_Vector v_zero     = Q6_V_vzero();
+    const HVX_Vector v_sign_bit = Q6_V_vsplat_R(0x80000000);
+
+    for (int i = 0; i < nb; i++) {
+        HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
+        HVX_Vector vd     = Q6_V_lo_W(hvx_vec_f16_to_f32(vd_f16));
+        HVX_Vector neg_vd = Q6_V_vxor_VV(vd, v_sign_bit);
+        HVX_Vector v_qs   = *(const HVX_UVector *) src[i].qs;
+
+        float * block_dst = dst_ptr + i * QK1_0;
+        #pragma unroll
+        for (int q = 0; q < 4; q++) {
+            HVX_Vector v_bits = hvx_vec_repl_u32(q == 0 ? v_qs : Q6_V_vror_VR(v_qs, 4 * q));
+            HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(v_bits, v_bit_mask), v_zero);
+            hvx_vmem(block_dst + q * 32) = Q6_V_vmux_QVV(q_zero, neg_vd, vd);
+        }
+    }
+}
+
+static inline void hvx_dequantize_row_flat_q2_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
+    const int nb = n / QK2_0;
+    const block_q2_0 * src = (const block_q2_0 *) src_ptr;
+    const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
+    const HVX_Vector v_one   = Q6_Vb_vsplat_R(1);
+
+    for (int i = 0; i < nb; i++) {
+        HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
+        HVX_Vector vd     = Q6_V_lo_W(hvx_vec_f16_to_f32(vd_f16));
+
+        HVX_Vector vq  = *(const HVX_UVector *) src[i].qs;
+        HVX_Vector v0  = Q6_V_vand_VV(vq, mask_03);
+        HVX_Vector v1  = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vq, 2), mask_03);
+        HVX_Vector v2  = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vq, 4), mask_03);
+        HVX_Vector v3  = Q6_Vub_vlsr_VubR(vq, 6);
+
+        HVX_Vector v01 = Q6_V_lo_W(Q6_W_vshuff_VVR(v1, v0, -1));
+        HVX_Vector v23 = Q6_V_lo_W(Q6_W_vshuff_VVR(v3, v2, -1));
+        HVX_Vector v64 = Q6_Vb_vsub_VbVb(Q6_V_lo_W(Q6_W_vshuff_VVR(v23, v01, -2)), v_one);
+
+        HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(v64);
+        HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
+
+        float * block_dst = dst_ptr + i * QK2_0;
+        hvx_vmem(block_dst + 0)  = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), vd);
+        hvx_vmem(block_dst + 32) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(p32)), vd);
+    }
+}
+
 static inline void hvx_dequantize_row_flat_q4_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
     const int nb = n / QK4_0;
     const block_q4_0 * src = (const block_q4_0 *) src_ptr;
@@ -603,4 +661,66 @@ static inline void hvx_dequantize_row_tiled_f16_f32(float * dst_spad, const uint
     }
 }
 
+static inline void hvx_dequantize_row_tiled_q1_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
+    static const uint32_t __attribute__((aligned(128))) bit_mask[32] = {
+        1u <<  0, 1u <<  8, 1u << 16, 1u << 24,
+        1u <<  1, 1u <<  9, 1u << 17, 1u << 25,
+        1u <<  2, 1u << 10, 1u << 18, 1u << 26,
+        1u <<  3, 1u << 11, 1u << 19, 1u << 27,
+        1u <<  4, 1u << 12, 1u << 20, 1u << 28,
+        1u <<  5, 1u << 13, 1u << 21, 1u << 29,
+        1u <<  6, 1u << 14, 1u << 22, 1u << 30,
+        1u <<  7, 1u << 15, 1u << 23, 1u << 31,
+    };
+    const HVX_Vector v_bit_mask = *(const HVX_Vector *) bit_mask;
+    const HVX_Vector v_zero     = Q6_V_vzero();
+    const HVX_Vector v_sign_bit = Q6_V_vsplat_R(0x80000000);
+
+    for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
+        const uint8_t * tile = src_spad + k_tile * tile_stride;
+        float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_K_Q1_0;
+
+        const HVX_Vector scales   = hvx_vmem(tile + 512);
+        const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
+        const HVX_Vector vd       = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
+        const HVX_Vector neg_vd   = Q6_V_vxor_VV(vd, v_sign_bit);
+
+        #pragma unroll
+        for (int q = 0; q < 4; ++q) {
+            const HVX_Vector v_bits = hvx_vec_repl_u32(Q6_V_vror_VR(hvx_vmem(tile + q * VLEN), 4 * row));
+            const HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(v_bits, v_bit_mask), v_zero);
+            hvx_vmem(dst + q * 32) = Q6_V_vmux_QVV(q_zero, neg_vd, vd);
+        }
+    }
+}
+
+static inline void hvx_dequantize_row_tiled_q2_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
+    const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
+    const HVX_Vector v_one   = Q6_Vb_vsplat_R(1);
+
+    for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
+        const uint8_t * tile = src_spad + k_tile * tile_stride;
+        float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_K_Q2_0;
+
+        const HVX_Vector v_packed = hvx_dequantize_tiled_f16_gather4(tile, 4 * row);
+        const HVX_Vector p0 = Q6_V_vand_VV(v_packed, mask_03);
+        const HVX_Vector p1 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_packed, 2), mask_03);
+        const HVX_Vector p2 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_packed, 4), mask_03);
+        const HVX_Vector p3 = Q6_Vub_vlsr_VubR(v_packed, 6);
+
+        const HVX_Vector p01 = Q6_V_lo_W(Q6_W_vshuff_VVR(p1, p0, -4));
+        const HVX_Vector p23 = Q6_V_lo_W(Q6_W_vshuff_VVR(p3, p2, -4));
+        const HVX_Vector vq  = Q6_Vb_vsub_VbVb(Q6_V_lo_W(Q6_W_vshuff_VVR(p23, p01, -8)), v_one);
+
+        const HVX_Vector scales   = hvx_vmem(tile + 512);
+        const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
+        const HVX_Vector scale    = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
+
+        const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);
+        const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
+        hvx_vmem(dst + 0)  = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
+        hvx_vmem(dst + 32) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(p32)), scale);
+    }
+}
+
 #endif // HVX_QUANT_H

ggml/src/ggml-hexagon/htp/main.c

@@ -940,6 +940,7 @@ static int execute_op(struct htp_ops_context * octx) {
         case HTP_OP_UNARY_RELU:
         case HTP_OP_UNARY_STEP:
         case HTP_OP_L2_NORM:
+        case HTP_OP_HADAMARD:
             return op_unary(octx);
 
         case HTP_OP_GLU_SWIGLU:

ggml/src/ggml-hexagon/htp/matmul-ops.c

@@ -277,10 +277,9 @@ static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
     const uint32_t tile_size = htp_tensor_get_tile_size(src0->type);
     const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type);
 
-    uint32_t n_k_tiles_w = ne00 / 32;
-    uint32_t n_k_tiles_a = ne10 / 32;
+    uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type);
     uint32_t tile_row_stride = n_k_tiles_w * tile_size;
-    uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
+    uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
 
     uint32_t ct_start = src0_start_row / 32;
     uint32_t ct_end   = (src0_end_row + 31) / 32;
@@ -289,7 +288,7 @@ static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
     if (src0_start_row < src0_end_row) {
         for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
             dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned,
-                           src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                           src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
         }
     }
 
@@ -341,7 +340,7 @@ static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
 
         if (push_ct < ct_end) {
             dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride),
-                           aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                           aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
             push_ct++;
         }
     }
@@ -379,10 +378,9 @@ static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
     const uint32_t tile_size = htp_tensor_get_tile_size(src0->type);
     const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type);
 
-    uint32_t n_k_tiles_w = ne00 / 32;
-    uint32_t n_k_tiles_a = ne10 / 32;
+    uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type);
     uint32_t tile_row_stride = n_k_tiles_w * tile_size;
-    uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
+    uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
 
     uint32_t ct_start = src0_start_row / 32;
     uint32_t ct_end   = (src0_end_row + 31) / 32;
@@ -401,7 +399,7 @@ static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
         }
         for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
             dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned,
-                           src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                           src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
         }
     }
 
@@ -422,7 +420,7 @@ static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) {
 
         if (push_ct < ct_end) {
             dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride),
-                           aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                           aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
             push_ct++;
         }
     }
@@ -460,8 +458,6 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data)
     const uint32_t n_prefetch = kparams->n_prefetch;
     assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0);
 
-    uint32_t n_k_tiles_a = ne10 / 32;
-
     for (uint32_t widx = 0; widx < n_weights; widx++) {
         const struct htp_tensor * restrict src_w = octx->src[widx];
         const struct htp_tensor * restrict dst   = octx->dsts[widx];
@@ -470,15 +466,14 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data)
 
         const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type);
         const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type);
-        const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
-
         const uint32_t ne00 = src_w->ne[0];
         const uint32_t ne01 = src_w->ne[1];
         const size_t dst_row_size = dst->nb[1];
         const dma_addr_t src_w_row = src_w->data;
 
-        uint32_t n_k_tiles_w = ne00 / 32;
-        uint32_t tile_row_stride = n_k_tiles_w * tile_size;
+        const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src_w->type);
+        const uint32_t tile_row_stride = n_k_tiles_w * tile_size;
+        const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
 
         uint32_t src0_start_row = 0;
         uint32_t src0_end_row   = ne01;
@@ -504,7 +499,7 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data)
         uint32_t push_ct = ct_start;
         for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
             dma_queue_push(dma_q, dma_make_data(vtcm_weight_ptr + d * tile_row_transfer_size_aligned,
-                           src_w_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                            src_w_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
         }
 
         for (uint32_t ct = ct_start; ct < ct_end; ct++) {
@@ -536,7 +531,7 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data)
 
             if (push_ct < ct_end) {
                 dma_queue_push(dma_q, dma_make_data(w_tile, src_w_row + push_ct * tile_row_stride),
-                               aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                                aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
                 push_ct++;
             }
         }
@@ -840,10 +835,9 @@ static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) {
     const uint32_t tile_size = htp_tensor_get_tile_size(src0->type);
     const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type);
 
-    const uint32_t n_k_tiles_w = ne00 / 32;
-    const uint32_t n_k_tiles_a = ne10 / 32;
+    const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type);
     const uint32_t tile_row_stride = n_k_tiles_w * tile_size;
-    const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
+    const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
     const uint32_t src0_slice_stride = ((ne01 + 31) / 32) * tile_row_stride;
 
     const uint32_t ct_start = src0_start_row / 32;
@@ -884,7 +878,7 @@ static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) {
         for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
             dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned,
                            src0_slice + (size_t) push_ct * tile_row_stride),
-                           aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                           aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
         }
 
         for (uint32_t ct = ct_start; ct < ct_end; ct++) {
@@ -929,7 +923,7 @@ static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) {
 
             if (push_ct < ct_end) {
                 dma_queue_push(dma_q, dma_make_data(w_tile, src0_slice + (size_t) push_ct * tile_row_stride),
-                               aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                               aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
                 push_ct++;
             }
         }
@@ -1343,10 +1337,9 @@ static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) {
 
         const uint32_t tile_size = htp_tensor_get_tile_size(src0->type);
         const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type);
-        const uint32_t n_k_tiles_w = ne00 / 32;
-        const uint32_t n_k_tiles_a = ne10 / 32;
+        const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type);
         const uint32_t tile_row_stride = n_k_tiles_w * tile_size;
-        const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
+        const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
 
         const uint32_t ct_start = src0_start_row / 32;
         const uint32_t ct_end   = (src0_end_row + 31) / 32;
@@ -1354,7 +1347,7 @@ static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) {
         uint32_t push_ct = ct_start;
         for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
             dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride),
-                           aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                           aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
         }
 
         for (uint32_t ct = ct_start; ct < ct_end; ct++) {
@@ -1379,7 +1372,7 @@ static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) {
 
             if (push_ct < ct_end) {
                 dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride),
-                               aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                               aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
                 push_ct++;
             }
         }
@@ -1431,10 +1424,9 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) {
 
         const uint32_t tile_size = htp_tensor_get_tile_size(src0->type);
         const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type);
-        const uint32_t n_k_tiles_w = ne00 / 32;
-        const uint32_t n_k_tiles_a = ne10 / 32;
+        const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type);
         const uint32_t tile_row_stride = n_k_tiles_w * tile_size;
-        const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
+        const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
 
         const uint32_t ct_start = src0_start_row / 32;
         const uint32_t ct_end   = (src0_end_row + 31) / 32;
@@ -1442,7 +1434,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) {
         uint32_t push_ct = ct_start;
         for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
             dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride),
-                           aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                           aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
         }
 
         for (uint32_t ct = ct_start; ct < ct_end; ct++) {
@@ -1457,7 +1449,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) {
 
             if (push_ct < ct_end) {
                 dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride),
-                               aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                               aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
                 push_ct++;
             }
         }
@@ -1520,10 +1512,9 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
 
             const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type);
             const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type);
-            const uint32_t n_k_tiles_w = src_w->ne[0] / 32;
-            const uint32_t n_k_tiles_a = act->ne[0] / 32;
+            const uint32_t n_k_tiles_w = src_w->ne[0] / htp_tensor_get_tile_k(src_w->type);
             const uint32_t tile_row_stride = n_k_tiles_w * tile_size;
-            const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
+            const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
 
             const uint32_t ct_start = src0_start_row / 32;
             const uint32_t ct_end   = (src0_end_row + 31) / 32;
@@ -1531,7 +1522,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
             uint32_t push_ct = ct_start;
             for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
                 dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride),
-                               aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                               aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
             }
 
             for (uint32_t ct = ct_start; ct < ct_end; ct++) {
@@ -1546,7 +1537,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
 
                 if (push_ct < ct_end) {
                     dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride),
-                                   aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                                    aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
                     push_ct++;
                 }
             }
@@ -1611,10 +1602,9 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
 
             const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type);
             const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type);
-            const uint32_t n_k_tiles_w = src_w->ne[0] / 32;
-            const uint32_t n_k_tiles_a = act->ne[0] / 32;
+            const uint32_t n_k_tiles_w = src_w->ne[0] / htp_tensor_get_tile_k(src_w->type);
             const uint32_t tile_row_stride = n_k_tiles_w * tile_size;
-            const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size;
+            const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size;
 
             const uint32_t ct_start = src0_start_row / 32;
             const uint32_t ct_end   = (src0_end_row + 31) / 32;
@@ -1622,7 +1612,7 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
             uint32_t push_ct = ct_start;
             for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) {
                 dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride),
-                               aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                               aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
             }
 
             for (uint32_t ct = ct_start; ct < ct_end; ct++) {
@@ -1647,7 +1637,7 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data)
 
                 if (push_ct < ct_end) {
                     dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride),
-                                   aligned_tile_size, tile_size, tile_size, n_k_tiles_a);
+                                    aligned_tile_size, tile_size, tile_size, n_k_tiles_w);
                     push_ct++;
                 }
             }
@@ -1693,6 +1683,11 @@ static int hvx_mm_init_tiled_vec_dot(struct htp_mm_context * mmctx, enum htp_dat
             mmctx->vec_dot_32x1 = tiled_vec_dot_q2_k_32x1;
             mmctx->vec_dot_32x2 = tiled_vec_dot_q2_k_32x2;
             return 0;
+        case HTP_TYPE_Q2_0:
+            mmctx->type         = "q2_0_tiled-f32";
+            mmctx->vec_dot_32x1 = tiled_vec_dot_q2_0_32x1;
+            mmctx->vec_dot_32x2 = tiled_vec_dot_q2_0_32x2;
+            return 0;
         case HTP_TYPE_IQ4_NL:
             mmctx->type         = "iq4nl_tiled-f32";
             mmctx->vec_dot_32x1 = tiled_vec_dot_iq4nl_32x1;
@@ -1703,6 +1698,11 @@ static int hvx_mm_init_tiled_vec_dot(struct htp_mm_context * mmctx, enum htp_dat
             mmctx->vec_dot_32x1 = tiled_vec_dot_mxfp4_32x1;
             mmctx->vec_dot_32x2 = tiled_vec_dot_mxfp4_32x2;
             return 0;
+        case HTP_TYPE_Q1_0:
+            mmctx->type         = "q1_0_tiled-f32";
+            mmctx->vec_dot_32x1 = tiled_vec_dot_q1_0_32x1;
+            mmctx->vec_dot_32x2 = tiled_vec_dot_q1_0_32x2;
+            return 0;
         case HTP_TYPE_F16:
             mmctx->type         = "f16_tiled-f16";
             mmctx->vec_dot_32x1 = tiled_vec_dot_f16_32x1;
@@ -2325,6 +2325,8 @@ static inline tiled_dequant_task_fn_t hmx_get_dequant_worker_fn(int weight_type,
         case HTP_TYPE_Q6_K:   return dequantize_tiled_weight_to_fp16_task_q6_k;
         case HTP_TYPE_Q3_K:   return dequantize_tiled_weight_to_fp16_task_q3_k;
         case HTP_TYPE_Q2_K:   return dequantize_tiled_weight_to_fp16_task_q2_k;
+        case HTP_TYPE_Q1_0:   return dequantize_tiled_weight_to_fp16_task_q1_0;
+        case HTP_TYPE_Q2_0:   return dequantize_tiled_weight_to_fp16_task_q2_0;
         case HTP_TYPE_F16:    return is_tiled ? NULL : convert_f16_weight_to_fp16_tiles_task;
         case HTP_TYPE_BF16:   return convert_bf16_weight_to_fp16_tiles_task;
         case HTP_TYPE_F32:    return quantize_f32_weight_to_fp16_tiles_task;
@@ -2346,8 +2348,9 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles(
 
     assert(k_block % HTP_MM_HMX_TILE_N_COLS == 0);
 
-    size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS);
-    size_t n_tot_tiles = n_col_tiles * n_k_tiles;
+    const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS);
+    const uint32_t n_weight_tiles = k_block / htp_tensor_get_tile_k(weight_type);
+    const size_t n_tot_tiles = n_col_tiles * n_weight_tiles;
 
     size_t n_tiles_per_task = (n_threads == 1) ? n_tot_tiles : hmx_ceil_div(n_tot_tiles, n_threads);
 
@@ -2363,6 +2366,7 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles(
     state.row_stride       = row_stride;
     state.weight_type      = weight_type;
     state.n_k_tiles        = n_k_tiles;
+    state.n_weight_tiles   = n_weight_tiles;
     state.n_k_tiles_div    = n_k_tiles_div;
     state.traces           = ctx->trace;
     state.ctx              = ctx;
@@ -2666,6 +2670,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
     tiled_dequant_task_fn_t dequant_worker_fn = hmx_get_dequant_worker_fn(weight_type, is_tiled);
 
     const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS;
+    const int n_weight_tiles = k / htp_tensor_get_tile_k(weight_type);
     const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles);
 
     const size_t vec_dot_size = k * sizeof(__fp16);
@@ -2679,8 +2684,6 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
     size_t n_chunk_n_cols = n_chunk;
     size_t vtcm_used      = vtcm_size;
 
-    const size_t qweight_row_stride = is_tiled ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0;
-
     struct htp_mm_hmx_vtcm_layout L;
     htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_threads, aligned_tile_size, src2_bytes);
 
@@ -2759,13 +2762,13 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
 
             // Prologue: push A0 and optionally A1 (if n_chunk_cnt > 1)
             const size_t   n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols);
-            const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_k_tiles : n_cols_A0;
+            const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_weight_tiles : n_cols_A0;
             dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight),
                            dma_dst_stride, dma_src_stride, dma_width_bytes, height_A0);
 
             if (1 < n_chunk_cnt) {
                 const size_t   n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols);
-                const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_k_tiles : n_cols_A1;
+                const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_weight_tiles : n_cols_A1;
                 dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[1], weight + n_chunk_n_cols * weight_stride),
                                dma_dst_stride, dma_src_stride, dma_width_bytes, height_A1);
             }
@@ -2789,7 +2792,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
 
                 // 3. push A_{i+2} (if i+2 < n_chunk_cnt)
                 if (i + 2 < n_chunk_cnt) {
-                    const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_k_tiles : n_cols_p2;
+                    const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_weight_tiles : n_cols_p2;
                     dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 2) % 4], weight + nc_p2 * weight_stride),
                                    dma_dst_stride, dma_src_stride, dma_width_bytes, height_p2);
                 }
@@ -2855,7 +2858,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
             // A0: Pre-fetch the first weight chunk (nc = 0)
             if (n > 0) {
                 const size_t n_cols = hex_smin(n, n_chunk_n_cols);
-                const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_k_tiles : n_cols;
+                const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_weight_tiles : n_cols;
                 dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height);
             }
 
@@ -2878,7 +2881,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
                 const size_t nc_next = nc + n_chunk_n_cols;
                 if (nc_next < n) {
                     const size_t n_cols_next = hex_smin(n - nc_next, n_chunk_n_cols);
-                    const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_k_tiles : n_cols_next;
+                    const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_weight_tiles : n_cols_next;
                     dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 1) % 2], weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next);
                 }
 
@@ -2930,6 +2933,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
     }
 
     const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS;
+    const int n_weight_tiles = k / htp_tensor_get_tile_k(weight_type);
     const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles);
 
     const int tile_size       = kparams->tile_size;
@@ -3057,13 +3061,13 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
                 const uint32_t dma_src_stride = is_tiled ? tile_size : weight_stride;
 
                 const size_t   n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols);
-                const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_k_tiles : n_cols_A0;
+                const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_weight_tiles : n_cols_A0;
                 dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight),
                                dma_dst_stride, dma_src_stride, dma_width_bytes, height_A0);
 
                 if (1 < n_chunk_cnt) {
                     const size_t   n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols);
-                    const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_k_tiles : n_cols_A1;
+                    const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_weight_tiles : n_cols_A1;
                     dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[1], weight + n_chunk_n_cols * weight_stride),
                                    dma_dst_stride, dma_src_stride, dma_width_bytes, height_A1);
                 }
@@ -3083,7 +3087,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
                         n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads);
 
                     if (i + 2 < n_chunk_cnt) {
-                        const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_k_tiles : n_cols_p2;
+                        const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_weight_tiles : n_cols_p2;
                         dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 2) % 4], weight + nc_p2 * weight_stride),
                                        dma_dst_stride, dma_src_stride, dma_width_bytes, height_p2);
                     }
@@ -3160,7 +3164,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
 
                 if (n > 0) {
                     const size_t n_cols = hex_smin(n, n_chunk_n_cols);
-                    const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_k_tiles : n_cols;
+                    const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_weight_tiles : n_cols;
                     dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height);
                 }
 
@@ -3180,7 +3184,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
                     const size_t nc_next = nc + n_chunk_n_cols;
                     if (nc_next < n) {
                         const size_t n_cols_next = hex_smin(n - nc_next, n_chunk_n_cols);
-                        const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_k_tiles : n_cols_next;
+                        const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_weight_tiles : n_cols_next;
                         dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 1) % 2], weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next);
                     }
 
@@ -3544,6 +3548,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
     }
 
     const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS;
+    const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(weight_type);
     const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles);
 
     const size_t vec_dot_size = k * sizeof(__fp16);
@@ -3557,7 +3562,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
     const uint32_t dma_src_stride  = is_tiled ? tile_size : weight_stride;
     const uint32_t dma_width_bytes = is_tiled ? tile_size : row_stride;
 
-    const size_t qweight_row_stride = is_tiled ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0;
+    const size_t qweight_row_stride = is_tiled ? (size_t)(n_weight_tiles * aligned_tile_size) / 32 : 0;
     const size_t weight_row_stride = is_tiled ? qweight_row_stride : row_stride;
 
     size_t size_per_n = 0, size_per_m = 0, size_per_mn = 0;
@@ -3613,7 +3618,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
         // A0: Pre-fetch the first weight chunk (nc = 0)
         if (n > 0) {
             const size_t n_cols = hex_smin((size_t) n, n_chunk_n_cols);
-            const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_k_tiles : n_cols;
+            const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_weight_tiles : n_cols;
             dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight),
                            dma_dst_stride, dma_src_stride, dma_width_bytes, height);
         }
@@ -3637,7 +3642,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
             const size_t nc_next = nc + n_chunk_n_cols;
             if (nc_next < (size_t) n) {
                 const size_t n_cols_next = hex_smin((size_t) n - nc_next, n_chunk_n_cols);
-                const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_k_tiles : n_cols_next;
+                const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_weight_tiles : n_cols_next;
                 dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 1) % 2], weight + nc_next * weight_stride),
                                dma_dst_stride, dma_src_stride, dma_width_bytes, height_next);
             }

ggml/src/ggml-hexagon/htp/matmul-ops.h

@@ -20,6 +20,10 @@ extern "C" {
 #define HTP_MM_HMX_TILE_N_ELMS 1024
 #define HTP_MM_HMX_MIN_NROWS   4
 
+#define HTP_MM_WEIGHT_TILE_K_Q2_0             64
+#define HTP_MM_WEIGHT_TILE_SIZE_Q2_0          HTP_TENSOR_TILE_SIZE_Q2_0
+#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0  640
+
 // --- Activation Tiled Block Sizes (including padding) ---
 #define HTP_MM_ACT_TILE_SIZE_Q8_0      1152
 #define HTP_MM_ACT_TILE_SIZE_Q8_1      1280
@@ -183,7 +187,7 @@ next_nc:
 // (Q2_K: per-16 k sums, q8_1_s16)
 static inline bool htp_mm_weight_has_offset(int weight_type) {
     return weight_type == HTP_TYPE_Q4_1 || weight_type == HTP_TYPE_Q4_K || weight_type == HTP_TYPE_Q5_K ||
-           weight_type == HTP_TYPE_Q2_K;
+           weight_type == HTP_TYPE_Q2_K || weight_type == HTP_TYPE_Q2_0;
 }
 
 // --- Activation/Row Size Helpers ---
@@ -227,8 +231,8 @@ static inline void htp_mm_hmx_get_2d_chunk_costs(
     const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32 && wtype != HTP_TYPE_BF16);
     const size_t row_stride = htp_tensor_get_row_size(wtype, k);
     const size_t vec_dot_size = k * sizeof(uint16_t);
-    const uint32_t n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS;
-    const size_t qweight_row_stride = is_quant ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0;
+    const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(wtype);
+    const size_t qweight_row_stride = is_quant ? (size_t)(n_weight_tiles * aligned_tile_size) / 32 : 0;
 
     *size_per_n_out = (pipeline ? 2 : 1) * (is_quant ? qweight_row_stride : row_stride) +
                       (pipeline ? 2 * vec_dot_size : vec_dot_size);
@@ -359,11 +363,11 @@ static inline void htp_mm_hmx_vtcm_layout_build(
         const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32 && wtype != HTP_TYPE_BF16);
         const size_t row_stride = htp_tensor_get_row_size(wtype, k);
         const size_t vec_dot_size = k * sizeof(uint16_t);
-        const uint32_t n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS;
+        const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(wtype);
 
         const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128);
         const size_t weight_area_size = is_quant
-            ? hex_align_up((nc / 32) * n_k_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE)
+            ? hex_align_up((nc / 32) * n_weight_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE)
             : hex_align_up(nc * row_stride, HTP_MM_HMX_TILE_SIZE);
         const size_t act_area_size    = hex_align_up(mc * vec_dot_size, HTP_MM_HMX_TILE_SIZE);
         const size_t output_area_size = hex_align_up(mc * nc * sizeof(__fp16), HTP_MM_HMX_TILE_SIZE);
@@ -435,7 +439,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(
 
     if (is_fused_nx) {
         uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
-        uint32_t n_k_tiles = hex_round_up(ne10, 32) / 32;
+        uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype);
         uint32_t tile_row_size = n_k_tiles * aligned_tile_size;
         size_t weight_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128);
 
@@ -451,7 +455,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(
     } else if (is_matmul_id) {
         const size_t act_row_size_tiled = htp_mm_tiled_act_row_size(wtype, ne10);
         const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
-        const uint32_t n_k_tiles         = ne10 / 32;
+        const uint32_t n_k_tiles         = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype);
         const uint32_t tile_row_size     = n_k_tiles * aligned_tile_size;
         size_t src0_sz_per_thread        = htp_mm_round_up(n_prefetch * tile_row_size, 256);
 
@@ -489,8 +493,8 @@ static inline void htp_mm_hvx_vtcm_layout_build(
             case HTP_MM_KERNEL_HVX_QUANT_ROW: {
                 size_t q_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
                 uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
-                uint32_t n_k_tiles = (ne10 + 31) / 32;
-                uint32_t tile_row_size = n_k_tiles * aligned_tile_size;
+                uint32_t n_weight_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype);
+                uint32_t tile_row_size = n_weight_tiles * aligned_tile_size;
                 size_t tiled_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256);
 
                 src0_sz = tiled_vtcm_size * n_threads;

ggml/src/ggml-hexagon/htp/unary-ops.c

@@ -751,6 +751,143 @@ static void l2_norm_f32(const void * restrict src,
     }
 }
 
+static inline HVX_VectorPair hvx_fwht_stage_pair(HVX_Vector v0, HVX_Vector v1, int32_t b) {
+    HVX_VectorPair p = Q6_W_vshuff_VVR(v1, v0, b);
+    HVX_Vector s = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(p), Q6_V_hi_W(p));
+    HVX_Vector d = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_V_lo_W(p), Q6_V_hi_W(p));
+    return Q6_W_vshuff_VVR(d, s, b);
+}
+
+static inline HVX_VectorPair hvx_fwht32_pair(HVX_Vector v0_sf, HVX_Vector v1_sf, HVX_Vector v_scale) {
+    HVX_Vector v0 = Q6_Vqf32_vmpy_VsfVsf(v0_sf, v_scale);
+    HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(v1_sf, v_scale);
+    HVX_VectorPair p;
+    p = hvx_fwht_stage_pair(v0, v1, 4);
+    p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 8);
+    p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 16);
+    p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 32);
+    p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 64);
+    return p;
+}
+
+static inline void hvx_fwht_store32(HVX_Vector * restrict v_dst, HVX_VectorPair p) {
+    v_dst[0] = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p));
+}
+
+static inline void hvx_fwht_store64(HVX_Vector * restrict v_dst, HVX_VectorPair p) {
+    HVX_Vector u = Q6_V_lo_W(p);
+    HVX_Vector v = Q6_V_hi_W(p);
+    v_dst[0] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(u, v));
+    v_dst[1] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(u, v));
+}
+
+static inline void hvx_fwht_tail_f32(HVX_Vector * restrict v_dst, uint32_t nvec) {
+    uint32_t len_vec = 2;
+    for (; len_vec * 2 < nvec; len_vec <<= 2) {
+        const uint32_t step = len_vec * 4;
+        for (uint32_t i = 0; i < nvec; i += step) {
+            for (uint32_t j = 0; j < len_vec; j++) {
+                HVX_Vector a0 = v_dst[i + j];
+                HVX_Vector a1 = v_dst[i + j + len_vec];
+                HVX_Vector a2 = v_dst[i + j + 2 * len_vec];
+                HVX_Vector a3 = v_dst[i + j + 3 * len_vec];
+
+                HVX_Vector t0 = Q6_Vqf32_vadd_VsfVsf(a0, a1);
+                HVX_Vector t1 = Q6_Vqf32_vsub_VsfVsf(a0, a1);
+                HVX_Vector t2 = Q6_Vqf32_vadd_VsfVsf(a2, a3);
+                HVX_Vector t3 = Q6_Vqf32_vsub_VsfVsf(a2, a3);
+
+                v_dst[i + j]               = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(t0, t2));
+                v_dst[i + j + len_vec]     = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(t1, t3));
+                v_dst[i + j + 2 * len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(t0, t2));
+                v_dst[i + j + 3 * len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(t1, t3));
+            }
+        }
+    }
+
+    if (len_vec < nvec) {
+        for (uint32_t i = 0; i < nvec; i += 2 * len_vec) {
+            for (uint32_t j = 0; j < len_vec; j++) {
+                HVX_Vector u = v_dst[i + j];
+                HVX_Vector v = v_dst[i + len_vec + j];
+                v_dst[i + j]           = hvx_vec_add_f32_f32(u, v);
+                v_dst[i + len_vec + j] = hvx_vec_sub_f32_f32(u, v);
+            }
+        }
+    }
+}
+
+static inline void hvx_fwht_row_f32(const uint8_t * restrict src, uint8_t * restrict dst, uint32_t n, HVX_Vector v_scale) {
+    const HVX_Vector * restrict v_src = (const HVX_Vector *) src;
+    HVX_Vector * restrict v_dst       = (HVX_Vector *) dst;
+    const uint32_t nvec               = n / VLEN_FP32;
+
+    if (nvec == 1) {
+        HVX_VectorPair p = hvx_fwht32_pair(v_src[0], v_src[0], v_scale);
+        hvx_fwht_store32(v_dst, p);
+        return;
+    }
+
+    for (uint32_t i = 0; i < nvec; i += 2) {
+        HVX_VectorPair p = hvx_fwht32_pair(v_src[i], v_src[i + 1], v_scale);
+        hvx_fwht_store64(&v_dst[i], p);
+    }
+
+    hvx_fwht_tail_f32(v_dst, nvec);
+}
+
+static inline void hvx_fwht_row_f16(const uint8_t * restrict src, uint8_t * restrict dst, uint32_t n, HVX_Vector v_scale) {
+    const HVX_Vector * restrict v_src = (const HVX_Vector *) src;
+    HVX_Vector * restrict v_dst       = (HVX_Vector *) dst;
+    const uint32_t nvec               = n / VLEN_FP32;
+
+    if (nvec == 1) {
+        HVX_Vector v0 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_src[0]));
+        HVX_VectorPair p = hvx_fwht32_pair(v0, v0, v_scale);
+        hvx_fwht_store32(v_dst, p);
+        return;
+    }
+
+    const uint32_t nvec_f16 = n / VLEN_FP16;
+    for (uint32_t i = 0; i < nvec_f16; i++) {
+        HVX_VectorPair f32p = hvx_vec_f16_to_f32(v_src[i]);
+        HVX_VectorPair p    = hvx_fwht32_pair(Q6_V_lo_W(f32p), Q6_V_hi_W(f32p), v_scale);
+        hvx_fwht_store64(&v_dst[2 * i], p);
+    }
+
+    hvx_fwht_tail_f32(v_dst, nvec);
+}
+
+static void hadamard_f32(const void * restrict src,
+                         void * restrict dst,
+                         const uint32_t num_rows,
+                         const struct htp_unary_context * uctx) {
+    htp_unary_op_preamble;
+    const HVX_Vector v_scale = hvx_vec_splat_f32(1.0f / sqrtf((float) ne0));
+
+    for (uint32_t ir = 0; ir < num_rows; ir++) {
+        const uint8_t * restrict src_row = (const uint8_t *) src + (ir * src0_row_size_aligned);
+        uint8_t * restrict dst_row       = (uint8_t *) dst + (ir * dst_row_size_aligned);
+
+        hvx_fwht_row_f32(src_row, dst_row, ne0, v_scale);
+    }
+}
+
+static void hadamard_f16(const void * restrict src,
+                         void * restrict dst,
+                         const uint32_t num_rows,
+                         const struct htp_unary_context * uctx) {
+    htp_unary_op_preamble;
+    const HVX_Vector v_scale = hvx_vec_splat_f32(1.0f / sqrtf((float) ne0));
+
+    for (uint32_t ir = 0; ir < num_rows; ir++) {
+        const uint8_t * restrict src_row = (const uint8_t *) src + (ir * src0_row_size_aligned);
+        uint8_t * restrict dst_row       = (uint8_t *) dst + (ir * dst_row_size_aligned);
+
+        hvx_fwht_row_f16(src_row, dst_row, ne0, v_scale);
+    }
+}
+
 static void tanh_f32(const void * restrict src,
                      void * restrict dst,
                      const uint32_t num_rows,
@@ -1644,6 +1781,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
         case HTP_OP_UNARY_STEP:      op_type = is_f16 ? "step-f16"     : "step-f32";         break;
         case HTP_OP_L2_NORM:         op_type = is_f16 ? "l2norm-f16"   : "l2norm-f32";       break;
         case HTP_OP_TRI:             op_type = "tri-f32";                                    break;
+        case HTP_OP_HADAMARD:        op_type = is_f16 ? "hadamard-f16" : "hadamard-f32";     break;
         default:
             FARF(ERROR, "Unsupported unary Op %u\n", octx->op);
             return HTP_STATUS_NO_SUPPORT;
@@ -1663,6 +1801,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
             case HTP_OP_UNARY_ABS:
             case HTP_OP_UNARY_LOG:
             case HTP_OP_UNARY_STEP:
+            case HTP_OP_HADAMARD:
                 break;
             default:
                 FARF(ERROR, "unary-%s: not supported for F16\n", op_type);
@@ -1677,16 +1816,17 @@ static int execute_op_unary(struct htp_ops_context * octx) {
     }
 
     const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
-    const size_t elem_size = is_f16 ? sizeof(_Float16) : sizeof(float);
-    const size_t src0_data_row_size = src0->ne[0] * elem_size;
-    const size_t dst_data_row_size  = dst->ne[0]  * elem_size;
+    const size_t src0_elem_size = is_f16 ? sizeof(_Float16) : sizeof(float);
+    const size_t dst_elem_size  = (dst->type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float);
+    const size_t src0_data_row_size = src0->ne[0] * src0_elem_size;
+    const size_t dst_data_row_size  = dst->ne[0]  * dst_elem_size;
 
     uint32_t row_start = 0;
     uint32_t nrows     = src0_nrows;
 
     if (octx->ctx->mdev.count > 1) {
         uint32_t rows_per_chunk = 0;
-        htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk);
+        htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) dst_elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk);
         const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
         row_start = range.start;
         nrows     = range.count;
@@ -1813,6 +1953,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
             case HTP_OP_UNARY_ABS:       compute_func = (void *) abs_f16;                 break;
             case HTP_OP_UNARY_LOG:       compute_func = (void *) log_f16;                 break;
             case HTP_OP_UNARY_STEP:      compute_func = (void *) step_f16;                break;
+            case HTP_OP_HADAMARD:        compute_func = (void *) hadamard_f16;            break;
             default:                     break;
         }
     } else {
@@ -1843,6 +1984,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
             case HTP_OP_UNARY_RELU:      compute_func = (void *) relu_f32;                break;
             case HTP_OP_UNARY_STEP:      compute_func = (void *) step_f32;                break;
             case HTP_OP_L2_NORM:         compute_func = (void *) l2_norm_f32;             break;
+            case HTP_OP_HADAMARD:        compute_func = (void *) hadamard_f32;            break;
             case HTP_OP_TRI:
                 task_func    = unary_thread_tri_f32;
                 compute_func = (void *) tri_f32;

ggml/src/ggml-hexagon/htp/unary-ops.h

@@ -64,6 +64,7 @@ static inline bool htp_op_is_unary(uint32_t opcode) {
         case HTP_OP_UNARY_STEP:
         case HTP_OP_L2_NORM:
         case HTP_OP_TRI:
+        case HTP_OP_HADAMARD:
             return true;
         default:
             return false;
@@ -94,8 +95,9 @@ static inline void htp_unary_vtcm_layout_build(
     uint32_t * out_col_tile,
     uint32_t * out_vtcm_row_per_thread
 ) {
+    const size_t dst_elem_size      = (op == HTP_OP_HADAMARD) ? sizeof(float) : elem_size;
     const size_t src0_data_row_size = ne00 * elem_size;
-    const size_t dst_data_row_size  = ne10 * elem_size;
+    const size_t dst_data_row_size  = ne10 * dst_elem_size;
 
     const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128);
     const size_t dst_row_size_aligned  = hex_round_up(dst_data_row_size,  128);
@@ -131,7 +133,8 @@ static inline void htp_unary_vtcm_layout_build(
     }
 
     const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM ||
-                               op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM);
+                               op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM ||
+                               op == HTP_OP_HADAMARD);
     // The tiled fallback path below only has F32 task functions (unary_task_f32_tiled_*);
     // F16 has no tiled kernels, so it must stay on the row-block path like reduction ops.
     // NOTE: if F16 ends up with vtcm_row_per_thread == 0 here (row too large for the VTCM

ggml/src/ggml-sycl/common.hpp

@@ -37,6 +37,10 @@ namespace syclexp = sycl::ext::oneapi::experimental;
     #endif
 #endif
 
+#if defined(__INTEL_LLVM_COMPILER)
+    #define GGML_SYCL_DMMV_HAS_ESIMD
+#endif
+
 #if GGML_SYCL_DNNL
 #include "dnnl.hpp"
 #include "dnnl_sycl.hpp"

ggml/src/ggml-sycl/dmmv.cpp

@@ -10,7 +10,6 @@
     #endif
     #include <sycl/ext/intel/esimd.hpp>
     #include "esimd.hpp"
-    #define GGML_SYCL_DMMV_HAS_ESIMD
 #endif
 
 static void convert_f16(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
@@ -1873,11 +1872,67 @@ static void dequantize_mul_mat_vec_q6_K_sycl(const void *vx, const float *y,
 #ifdef GGML_SYCL_DMMV_HAS_ESIMD
 using ggml_sycl_esimd::GGML_SYCL_DMMV_ESIMD_WG_SIZE;
 
-// generic reordered dequantize-matvec: each work-group owns a pair of
-// consecutive output rows and updates one 32-wide accumulator per row
-template <ggml_type T>
+// policies for dequantize_mul_mat_vec_reorder_esimd: row mapping and what to do with the two sums
+// plain mat-vec: a work-group owns two consecutive rows of one matrix
+struct esimd_dmmv_row_pair_policy {
+    const void * vx;
+    float *      dst;
+
+    ESIMD_INLINE const void * w_a() const { return vx; }
+    ESIMD_INLINE const void * w_b() const { return vx; }
+
+    ESIMD_INLINE int  row_a(int g) const { return g * 2; }
+    ESIMD_INLINE int  row_b(int g) const { return g * 2 + 1; }
+    ESIMD_INLINE bool has_b(int g, int nrows) const { return row_b(g) < nrows; }
+
+    ESIMD_INLINE void epilogue(int g, int nrows, float sum_a, float sum_b) const {
+        dst[row_a(g)] = sum_a;
+        if (has_b(g, nrows)) {
+            dst[row_b(g)] = sum_b;
+        }
+    }
+};
+
+// fused gate+up+GLU: a work-group owns one row of both matrices; a is the gate, b the up
+struct esimd_dmmv_glu_policy {
+    const void * vgate;
+    const void * vup;
+    float *      dst;
+    ggml_glu_op  glu_op;
+
+    ESIMD_INLINE const void * w_a() const { return vgate; }
+    ESIMD_INLINE const void * w_b() const { return vup; }
+
+    ESIMD_INLINE int  row_a(int g) const { return g; }
+    ESIMD_INLINE int  row_b(int g) const { return g; }
+    ESIMD_INLINE bool has_b(int, int) const { return true; } // up always contributes
+
+    ESIMD_INLINE void epilogue(int g, int, float gate, float up) const {
+        using namespace sycl::ext::intel::esimd;
+
+        // SwiGLU: silu(gate)*up = gate*sigmoid(gate)*up. GEGLU's tanh form reduces
+        // to the same shape via tanh(y) = 2*sigmoid(2y) - 1, so only the sigmoid
+        // argument differs. sigmoid via exp2 (width-1 simd is a degenerate case for
+        // the EM pipe, so broadcast to 16 lanes); lane 0 holds the result.
+        const float GELU_COEF_A    = 0.044715f;
+        const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;
+        const float NEG_LOG2E      = -1.44269504088896341f;
+
+        const float arg = (glu_op == GGML_GLU_OP_GEGLU) ?
+            2.0f * SQRT_2_OVER_PI * gate * (1.0f + GELU_COEF_A * gate * gate) :
+            gate;
+
+        simd<float, 16> t = arg * NEG_LOG2E;
+        simd<float, 16> e = exp2(t);
+        dst[g] = gate / (1.0f + e[0]) * up;
+    }
+};
+
+// generic reordered dequantize-matvec: each work-group owns a pair of rows chosen by the policy
+// and updates one 32-wide accumulator per row
+template <ggml_type T, typename Policy>
 ESIMD_INLINE void dequantize_mul_mat_vec_reorder_esimd(
-        const void * vx, const float * y, float * dst,
+        const Policy & policy, const float * y,
         const int ncols, const int nrows,
         sycl::local_accessor<float, 1> lmem,
         const sycl::nd_item<1> & it) {
@@ -1886,41 +1941,41 @@ ESIMD_INLINE void dequantize_mul_mat_vec_reorder_esimd(
 
     const int    num_blocks_per_row = ncols / QK_K;
     const size_t nb = (size_t) nrows * num_blocks_per_row;
-    const auto   ps = traits::make_ptrs(vx, nb);
+    const auto   pa = traits::make_ptrs(policy.w_a(), nb);
+    const auto   pb = traits::make_ptrs(policy.w_b(), nb);
 
-    const int  tid      = it.get_local_id(0);
-    const int  row_pair = it.get_group(0);
-    const int  row0     = row_pair * 2; // two consecutive output rows
-    const bool has_row1 = row0 + 1 < nrows;
+    const int  tid   = it.get_local_id(0);
+    const int  g     = it.get_group(0);
+    const int  row_a = policy.row_a(g);
+    const int  row_b = policy.row_b(g);
+    const bool has_b = policy.has_b(g, nrows);
 
-    // one 32-wide accumulator per output row (small footprint, no spill)
-    simd<float, 32> acc0 = 0.0f;
-    simd<float, 32> acc1 = 0.0f;
+    // one 32-wide accumulator per row (small footprint, no spill)
+    simd<float, 32> acc_a = 0.0f;
+    simd<float, 32> acc_b = 0.0f;
 
     for (int ib = tid; ib < num_blocks_per_row; ib += GGML_SYCL_DMMV_ESIMD_WG_SIZE) {
         simd<float, 256> y_vec = block_load<float, 256>(y + (size_t) ib * QK_K);
 
-        const size_t bi0 = (size_t) (row0 + 0) * num_blocks_per_row + ib;
-        const size_t bi1 = (size_t) (row0 + 1) * num_blocks_per_row + ib;
+        const size_t bi_a = (size_t) row_a * num_blocks_per_row + ib;
+        const size_t bi_b = (size_t) row_b * num_blocks_per_row + ib;
 
-        traits::mac_pair(ps, bi0, ps, bi1, has_row1, y_vec, acc0, acc1);
+        traits::mac_pair(pa, bi_a, pb, bi_b, has_b, y_vec, acc_a, acc_b);
     }
 
-    lmem[tid * 2 + 0] = reduce<float>(acc0, std::plus<>{});
-    lmem[tid * 2 + 1] = reduce<float>(acc1, std::plus<>{});
+    lmem[tid * 2 + 0] = reduce<float>(acc_a, std::plus<>{});
+    lmem[tid * 2 + 1] = reduce<float>(acc_b, std::plus<>{});
     it.barrier(sycl::access::fence_space::local_space);
 
     if (tid == 0) {
-        float sum0 = 0.0f;
-        float sum1 = 0.0f;
+        float sum_a = 0.0f;
+        float sum_b = 0.0f;
         for (int p = 0; p < GGML_SYCL_DMMV_ESIMD_WG_SIZE; ++p) {
-            sum0 += lmem[p * 2 + 0];
-            sum1 += lmem[p * 2 + 1];
-        }
-        dst[row0 + 0] = sum0;
-        if (has_row1) {
-            dst[row0 + 1] = sum1;
+            sum_a += lmem[p * 2 + 0];
+            sum_b += lmem[p * 2 + 1];
         }
+
+        policy.epilogue(g, nrows, sum_a, sum_b);
     }
 }
 
@@ -1936,7 +1991,7 @@ static void dequantize_mul_mat_vec_q2_K_sycl_reorder_esimd(const void *vx, const
             sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
             [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
                 dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q2_K>(
-                    vx, y, dst, ncols, nrows, lmem, it);
+                    esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);
             });
     });
 }
@@ -1953,7 +2008,7 @@ static void dequantize_mul_mat_vec_q3_K_sycl_reorder_esimd(const void *vx, const
             sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
             [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
                 dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q3_K>(
-                    vx, y, dst, ncols, nrows, lmem, it);
+                    esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);
             });
     });
 }
@@ -1970,7 +2025,7 @@ static void dequantize_mul_mat_vec_q4_K_sycl_reorder_esimd(const void *vx, const
             sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
             [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
                 dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q4_K>(
-                    vx, y, dst, ncols, nrows, lmem, it);
+                    esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);
             });
     });
 }
@@ -1987,7 +2042,7 @@ static void dequantize_mul_mat_vec_q5_K_sycl_reorder_esimd(const void *vx, const
             sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
             [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
                 dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q5_K>(
-                    vx, y, dst, ncols, nrows, lmem, it);
+                    esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);
             });
     });
 }
@@ -2004,7 +2059,7 @@ static void dequantize_mul_mat_vec_q6_K_sycl_reorder_esimd(const void *vx, const
             sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
             [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
                 dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q6_K>(
-                    vx, y, dst, ncols, nrows, lmem, it);
+                    esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);
             });
     });
 }
@@ -2045,9 +2100,9 @@ ESIMD_INLINE void q8_0_mac_stripe(
     }
 }
 
-template <int WG>
+template <int WG, typename Policy>
 ESIMD_INLINE void dequantize_mul_mat_vec_q8_0_reorder_esimd(
-        const void * vx, const float * y, float * dst,
+        const Policy & policy, const float * y,
         const int ncols, const int nrows,
         sycl::local_accessor<float, 1> lmem,
         const sycl::nd_item<1> & it) {
@@ -2057,84 +2112,137 @@ ESIMD_INLINE void dequantize_mul_mat_vec_q8_0_reorder_esimd(
 
     const int          nblk_row = ncols / QK8_0;
     const size_t       nb       = (size_t) nrows * nblk_row;
-    const int8_t *     qs       = (const int8_t *) vx;
-    const sycl::half * d        = (const sycl::half *) (qs + nb * QK8_0);
+    const int8_t *     qs_a     = (const int8_t *) policy.w_a();
+    const int8_t *     qs_b     = (const int8_t *) policy.w_b();
+    const sycl::half * d_a      = (const sycl::half *) (qs_a + nb * QK8_0);
+    const sycl::half * d_b      = (const sycl::half *) (qs_b + nb * QK8_0);
 
-    const int  tid      = it.get_local_id(0);
-    const int  row_pair = it.get_group(0);
-    const int  row0     = row_pair * 2;
-    const bool has_row1 = row0 + 1 < nrows;
+    const int  tid   = it.get_local_id(0);
+    const int  g     = it.get_group(0);
+    const bool has_b = policy.has_b(g, nrows);
 
-    const size_t base0 = (size_t) row0 * nblk_row;
-    const size_t base1 = has_row1 ? (size_t) (row0 + 1) * nblk_row : base0;
+    const size_t base_a = (size_t) policy.row_a(g) * nblk_row;
+    const size_t base_b = (size_t) policy.row_b(g) * nblk_row;
 
-    simd<float, 32> acc0 = 0.0f;
-    simd<float, 32> acc1 = 0.0f;
+    simd<float, 32> acc_a = 0.0f;
+    simd<float, 32> acc_b = 0.0f;
 
     // Each thread processes one contiguous stripe.
     int ib = 0;
     for (; ib + WG * STRIPE <= nblk_row; ib += WG * STRIPE) {
         const int b = ib + tid * STRIPE;
         simd<float, 256> y_vec = block_load<float, 256>(y + (size_t) b * QK8_0);
-        q8_0_mac_stripe<STRIPE>(qs + (base0 + b) * QK8_0, qs + (base1 + b) * QK8_0,
-                                d + base0 + b, d + base1 + b, has_row1, y_vec, acc0, acc1);
+        q8_0_mac_stripe<STRIPE>(qs_a + (base_a + b) * QK8_0, qs_b + (base_b + b) * QK8_0,
+                                d_a + base_a + b, d_b + base_b + b, has_b, y_vec, acc_a, acc_b);
     }
 
     // Distribute remaining blocks across the work-group.
     for (int b = ib + tid; b < nblk_row; b += WG) {
         simd<float, 32> y_vec = block_load<float, 32>(y + (size_t) b * QK8_0);
-        q8_0_mac_stripe<1>(qs + (base0 + b) * QK8_0, qs + (base1 + b) * QK8_0,
-                           d + base0 + b, d + base1 + b, has_row1, y_vec, acc0, acc1);
+        q8_0_mac_stripe<1>(qs_a + (base_a + b) * QK8_0, qs_b + (base_b + b) * QK8_0,
+                           d_a + base_a + b, d_b + base_b + b, has_b, y_vec, acc_a, acc_b);
     }
 
-    lmem[tid * 2 + 0] = reduce<float>(acc0, std::plus<>{});
-    lmem[tid * 2 + 1] = reduce<float>(acc1, std::plus<>{});
+    lmem[tid * 2 + 0] = reduce<float>(acc_a, std::plus<>{});
+    lmem[tid * 2 + 1] = reduce<float>(acc_b, std::plus<>{});
     it.barrier(sycl::access::fence_space::local_space);
 
     if (tid == 0) {
-        float sum0 = 0.0f;
-        float sum1 = 0.0f;
+        float sum_a = 0.0f;
+        float sum_b = 0.0f;
         for (int p = 0; p < WG; ++p) {
-            sum0 += lmem[p * 2 + 0];
-            sum1 += lmem[p * 2 + 1];
-        }
-        dst[row0 + 0] = sum0;
-        if (has_row1) {
-            dst[row0 + 1] = sum1;
+            sum_a += lmem[p * 2 + 0];
+            sum_b += lmem[p * 2 + 1];
         }
+
+        policy.epilogue(g, nrows, sum_a, sum_b);
     }
 }
 
-template <int WG>
-static void q8_0_esimd_launch(const void * vx, const float * y, float * dst, const int ncols,
-                              const int nrows, dpct::queue_ptr stream) {
-    const int workgroups = (nrows + 1) / 2;
+template <int WG, typename Policy>
+static void q8_0_esimd_launch(const Policy & policy, const float * y, const int ncols, const int nrows,
+                              const int workgroups, dpct::queue_ptr stream) {
     stream->submit([&](sycl::handler & h) {
         sycl::local_accessor<float, 1> lmem(sycl::range<1>(WG * 2), h);
         h.parallel_for(
             sycl::nd_range<1>(sycl::range<1>((size_t) workgroups * WG), sycl::range<1>(WG)),
             [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
-                dequantize_mul_mat_vec_q8_0_reorder_esimd<WG>(vx, y, dst, ncols, nrows, lmem, it);
+                dequantize_mul_mat_vec_q8_0_reorder_esimd<WG>(policy, y, ncols, nrows, lmem, it);
             });
     });
 }
 
+// Scale the work-group with the number of blocks per row.
+template <typename Policy>
+static void q8_0_esimd_launch_scaled(const Policy & policy, const float * y, const int ncols, const int nrows,
+                                     const int workgroups, dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK8_0 == 0);
+
+    const int nblk_row = ncols / QK8_0;
+    if (nblk_row >= 64) {
+        q8_0_esimd_launch<8>(policy, y, ncols, nrows, workgroups, stream);
+    } else if (nblk_row >= 32) {
+        q8_0_esimd_launch<4>(policy, y, ncols, nrows, workgroups, stream);
+    } else if (nblk_row >= 16) {
+        q8_0_esimd_launch<2>(policy, y, ncols, nrows, workgroups, stream);
+    } else {
+        q8_0_esimd_launch<1>(policy, y, ncols, nrows, workgroups, stream);
+    }
+}
+
 static void dequantize_mul_mat_vec_q8_0_sycl_reorder_esimd(const void *vx, const float *y,
                                                            float *dst, const int ncols,
                                                            const int nrows,
                                                            dpct::queue_ptr stream) {
-    GGML_ASSERT(ncols % QK8_0 == 0);
+    q8_0_esimd_launch_scaled(esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, (nrows + 1) / 2, stream);
+}
 
-    // Scale the work-group with the number of blocks per row.
-    const int nblk_row = ncols / QK8_0;
-    if (nblk_row >= 64) {
-        q8_0_esimd_launch<8>(vx, y, dst, ncols, nrows, stream);
-    } else if (nblk_row >= 32) {
-        q8_0_esimd_launch<4>(vx, y, dst, ncols, nrows, stream);
-    } else if (nblk_row >= 16) {
-        q8_0_esimd_launch<2>(vx, y, dst, ncols, nrows, stream);
-    } else {
-        q8_0_esimd_launch<1>(vx, y, dst, ncols, nrows, stream);
+// ESIMD fused gate+up+GLU on reordered K-quant SOA weights: one output row (gate
+// and up) per work-group, two independent 32-wide accumulators per super-block.
+// Reuses the DMMV kernels' dequant+MAC primitive (esimd_reorder_q_traits<T>::mac_pair).
+template <ggml_type T>
+static void dequantize_mul_mat_vec_glu_reorder_esimd(const void * vx, const void * vgate,
+                                                     const float * y, float * dst,
+                                                     const int ncols, const int nrows,
+                                                     const ggml_glu_op glu_op,
+                                                     dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int workgroups = nrows; // one output row (gate+up) per work-group
+    stream->submit([&](sycl::handler & h) {
+        sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h);
+        h.parallel_for(
+            sycl::nd_range<1>(sycl::range<1>((size_t) workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
+            [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
+                dequantize_mul_mat_vec_reorder_esimd<T>(
+                    esimd_dmmv_glu_policy{ vgate, vx, dst, glu_op }, y, ncols, nrows, lmem, it);
+            });
+    });
+}
+
+bool ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(enum ggml_type src0_type, enum ggml_glu_op glu_op,
+                                                        const void * vx, const void * vgate, const float * y,
+                                                        float * dst, int ncols, int nrows, dpct::queue_ptr stream) {
+    switch (src0_type) {
+        case GGML_TYPE_Q2_K:
+            dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q2_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);
+            return true;
+        case GGML_TYPE_Q3_K:
+            dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q3_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);
+            return true;
+        case GGML_TYPE_Q4_K:
+            dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q4_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);
+            return true;
+        case GGML_TYPE_Q5_K:
+            dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q5_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);
+            return true;
+        case GGML_TYPE_Q6_K:
+            dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q6_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);
+            return true;
+        case GGML_TYPE_Q8_0:
+            q8_0_esimd_launch_scaled(esimd_dmmv_glu_policy{ vgate, vx, dst, glu_op }, y, ncols, nrows, nrows, stream);
+            return true;
+        default:
+            return false;
     }
 }
 

ggml/src/ggml-sycl/dmmv.hpp

@@ -24,4 +24,13 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
     const int64_t src1_ncols, const int64_t src1_padded_row_size,
     const dpct::queue_ptr &stream);
 
+// Fused dense-FFN mat-vec: writes glu(gate . y, up . y) for same-type reordered
+// weights (Q2_K..Q6_K, Q8_0) and a single F32 activation column, using the ESIMD DMMV kernels.
+// vx is the up weight, vgate the gate weight. Only defined when GGML_SYCL_DMMV_HAS_ESIMD;
+// returns false if the type is unhandled.
+bool ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(
+    enum ggml_type src0_type, enum ggml_glu_op glu_op,
+    const void * vx, const void * vgate, const float * y,
+    float * dst, int ncols, int nrows, dpct::queue_ptr stream);
+
 #endif // GGML_SYCL_DMMV_HPP

ggml/src/ggml-sycl/fusion.cpp

@@ -4,7 +4,7 @@
 #include <algorithm>
 
 // mul_mat(gate) + mul_mat(up) + GLU: graph shape and tensor properties only. Backend state
-// (weight layout, split buffers, DMMV) is checked by ggml_sycl_mul_mat_glu_mmvq_fused().
+// (weight layout, split buffers, DMMV) is checked by ggml_sycl_op_mul_mat_glu_fused().
 static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const ggml_tensor * up,
                                               const ggml_tensor * glu) {
     // the fused epilogue implements these two; the rest fall back to the standalone GLU kernels
@@ -32,12 +32,22 @@ static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const gg
         return false;
     }
 
-    // fused GEMVs walk whole QK_K super-blocks: the reorder kernel covers same-type
+    // fused K-quant GEMVs walk whole QK_K super-blocks: the reorder kernel covers same-type
     // q4_K, the plain-layout kernel covers q5_K / iq4_xs pairs incl. mixed gate/up types
     const bool reorder_pair = wu->type == GGML_TYPE_Q4_K && wg->type == GGML_TYPE_Q4_K;
     const bool plain_pair   = (wu->type == GGML_TYPE_Q5_K || wu->type == GGML_TYPE_IQ4_XS) &&
                             (wg->type == GGML_TYPE_Q5_K || wg->type == GGML_TYPE_IQ4_XS);
-    if ((!reorder_pair && !plain_pair) || wu->ne[0] % QK_K != 0) {
+    // the ESIMD kernel covers same-type q2_K..q6_K and q8_0, single column only (as unfused DMMV)
+#ifdef GGML_SYCL_DMMV_HAS_ESIMD
+    const bool esimd_pair   = g_ggml_sycl_enable_esimd && wu->type == wg->type && act->ne[1] == 1 &&
+                            (wu->type == GGML_TYPE_Q2_K || wu->type == GGML_TYPE_Q3_K || wu->type == GGML_TYPE_Q4_K ||
+                             wu->type == GGML_TYPE_Q5_K || wu->type == GGML_TYPE_Q6_K || wu->type == GGML_TYPE_Q8_0);
+#else
+    const bool esimd_pair   = false;
+#endif
+    // q8_0 rows are walked in 32-wide blocks, so they only need to be a multiple of QK8_0
+    const int64_t row_align = esimd_pair && wu->type == GGML_TYPE_Q8_0 ? QK8_0 : QK_K;
+    if ((!reorder_pair && !plain_pair && !esimd_pair) || wu->ne[0] % row_align != 0) {
         return false;
     }
 

ggml/src/ggml-sycl/ggml-sycl.cpp

@@ -46,9 +46,6 @@
 #    include <sycl/ext/oneapi/virtual_mem/virtual_mem.hpp>
 #    define GGML_SYCL_SUPPORT_VMM
 #endif
-#if defined(__INTEL_LLVM_COMPILER)
-    #define GGML_SYCL_DMMV_HAS_ESIMD
-#endif
 #include <sycl/half_type.hpp>
 
 #include "ggml.h"
@@ -5287,6 +5284,12 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, gg
         return false;
     }
 
+    // the plain kernel only covers q5_K / iq4_xs pairs: bail before quantizing the activation
+    const auto plain_type = [](ggml_type t) { return t == GGML_TYPE_Q5_K || t == GGML_TYPE_IQ4_XS; };
+    if (!plain_type(wu->type) || !plain_type(wg->type)) {
+        return false;
+    }
+
     // log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises
     scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (plain layout)");
 
@@ -5311,9 +5314,42 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, gg
     GGML_UNUSED(gate);
 }
 
+#ifdef GGML_SYCL_DMMV_HAS_ESIMD
+// {mul_mat(gate), mul_mat(up), GLU} on the reorder layout via the ESIMD DMMV kernel, for
+// same-type K-quant or Q8_0 pairs and a single activation column. It reads the F32 activation
+// directly, so one launch replaces three and no q8_1 quantization is needed.
+static bool ggml_sycl_mul_mat_glu_dmmv_esimd(ggml_backend_sycl_context & ctx, ggml_tensor * glu,
+                                             ggml_tensor * gate, ggml_tensor * up, const ggml_tensor * wu,
+                                             const ggml_tensor * wg, const ggml_tensor * act) {
+    // the kernel reads the activation while writing the output; glu may reuse the dead activation's memory
+    const char * act_begin = (const char *) act->data;
+    const char * glu_begin = (const char *) glu->data;
+    if (act_begin < glu_begin + ggml_nbytes(glu) && glu_begin < act_begin + ggml_nbytes(act)) {
+        return false;
+    }
+
+    // install the reorder (SoA) layout the kernel needs, as the unfused DMMV path would
+    opt_for_reorder(&ctx, wu, act, up, mul_mat_algo::DMMV);
+    opt_for_reorder(&ctx, wg, act, gate, mul_mat_algo::DMMV);
+
+    const auto * extra_u = static_cast<const ggml_tensor_extra_gpu *>(wu->extra);
+    const auto * extra_g = static_cast<const ggml_tensor_extra_gpu *>(wg->extra);
+    if (!extra_u || !extra_g || !extra_u->optimized_feature.reorder || !extra_g->optimized_feature.reorder) {
+        return false;
+    }
+
+    // log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises
+    scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (ESIMD)");
+
+    return ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(wu->type, ggml_get_glu_op(glu), wu->data, wg->data,
+                                                              (const float *) act->data, (float *) glu->data,
+                                                              (int) wu->ne[0], (int) wu->ne[1], ctx.stream());
+}
+#endif
+
 // Fused dense-FFN mat-vec for the {mul_mat(gate), mul_mat(up), GLU} subgraph at node_idx.
 // Returns false if it declined, in which case the caller runs the three nodes normally.
-static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) {
+static bool ggml_sycl_mul_mat_glu_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) {
     if (!ggml_sycl_can_fuse(cgraph, node_idx, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }, {})) {
         return false;
     }
@@ -5331,6 +5367,16 @@ static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, gg
         return false;
     }
 
+#ifdef GGML_SYCL_DMMV_HAS_ESIMD
+    // single-column same-type K-quant or Q8_0 pairs take the ESIMD kernel, as the unfused mul-mats
+    // would via DMMV; if the reorder layout could not be installed, fall through to mmvq
+    if (g_ggml_sycl_enable_esimd && wu->type == wg->type && act->ne[1] == 1 &&
+        ggml_sycl_supports_reorder_esimd(wu->type) &&
+        ggml_sycl_mul_mat_glu_dmmv_esimd(ctx, glu, gate, up, wu, wg, act)) {
+        return true;
+    }
+#endif
+
     // with DMMV prioritised the unfused path would not have gone through mmvq at all
     if (g_ggml_sycl_prioritize_dmmv) {
         return false;
@@ -6594,7 +6640,7 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc
             continue;
         }
 
-        if (node->op == GGML_OP_MUL_MAT && ggml_sycl_mul_mat_glu_mmvq_fused(*sycl_ctx, cgraph, i)) {
+        if (node->op == GGML_OP_MUL_MAT && ggml_sycl_mul_mat_glu_fused(*sycl_ctx, cgraph, i)) {
             i += 2;
             continue;
         }

ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -15872,6 +15872,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
             return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32
                 && ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32;
         case GGML_OP_SUM:
+            return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguously_allocated(op->src[0]);
         case GGML_OP_SUM_ROWS:
         case GGML_OP_MEAN:
             return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]);

ggml/src/ggml-webgpu/ggml-webgpu.cpp

@@ -4781,6 +4781,8 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
             supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type;
             break;
         case GGML_OP_SUM:
+            supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type && ggml_is_contiguously_allocated(src0);
+            break;
         case GGML_OP_SUM_ROWS:
             supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type && ggml_is_contiguous_rows(src0);
             break;

gguf-py/gguf/constants.py

@@ -2929,6 +2929,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_GATE,
         MODEL_TENSOR.FFN_DOWN,
         MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.DENSE_2_OUT, # voyage-4-nano
     ],
     MODEL_ARCH.QWEN3MOE: [
         MODEL_TENSOR.TOKEN_EMBD,

scripts/pr2wt.sh

@@ -72,19 +72,12 @@ dir=$(basename $(pwd))
 git branch -D pr/$PR 2> /dev/null
 git worktree add -b pr/$PR ../$dir-pr-$PR pr/$PR/$head_ref 2> /dev/null
 
-og_path=$(pwd)
 wt_path=$(cd ../$dir-pr-$PR && pwd)
 
 echo "git worktree created in $wt_path"
 
 cd $wt_path
 
-# pi agent setup in the worktree
-if [[ -f "$og_path/.pi/SYSTEM.md" && ! -f ".pi/SYSTEM.md" ]]; then
-    mkdir -p .pi
-    ln -sfn "$og_path/.pi/SYSTEM.md" .pi/SYSTEM.md
-fi
-
 git branch --set-upstream-to=pr/$PR/$head_ref
 git pull   --ff-only || {
     echo "error: failed to pull pr/$PR"

scripts/wc2wt.sh

@@ -39,19 +39,12 @@ dir_suffix=$(echo "$BRANCH" | tr '/' '-')
 
 git worktree add "../$dir-$dir_suffix" "$BRANCH" || git worktree add -b "$BRANCH" "../$dir-$dir_suffix" HEAD
 
-og_path=$(pwd)
 wt_path=$(cd "../$dir-$dir_suffix" && pwd)
 
 echo "git worktree created in $wt_path"
 
 cd "$wt_path"
 
-# pi agent setup in the worktree
-if [[ -f "$og_path/.pi/SYSTEM.md" && ! -f ".pi/SYSTEM.md" ]]; then
-    mkdir -p .pi
-    ln -sfn "$og_path/.pi/SYSTEM.md" .pi/SYSTEM.md
-fi
-
 if [[ $# -eq 2 ]]; then
     echo "executing: $2"
     eval "$2"

skills/update-fusion-table/SKILL.md

@@ -0,0 +1,39 @@
+---
+name: update-fusion-table
+description: Regenerate a per-device test-fusion baseline (tests/fusion/MTL.csv, ...) after fusion patterns or the dummy-model set change. Use when test-fusion --check fails.
+---
+
+# Update a fusion baseline
+
+`tests/fusion/<BACKEND>.csv` is the `test-fusion` golden baseline: `arch,moe,mode,label,count`,
+where `count` is how many times a fusion label matched per graph run. See `tests/fusion/README.md`.
+
+Re-record only when the counts moved for the reason the contributor intended. A row failing on
+`nmse_fus` (> 1e-4) is a fused-vs-unfused correctness bug, not baseline drift - fix it, never
+record it away.
+
+## Steps
+
+```sh
+cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_METAL=ON       # enable the target backend
+cmake --build build --config Release --target test-llama-archs --target test-fusion -j
+
+rm -rf build-ci-models && mkdir -p build-ci-models              # always regenerate
+./build/bin/test-llama-archs -o build-ci-models
+
+D=MTL0; B=tests/fusion/MTL.csv; T=/tmp/$(basename $B)   # keep the target backend/device in sync
+./build/bin/test-fusion --models build-ci-models --device $D --check $B   # read the FAIL rows
+./build/bin/test-fusion --models build-ci-models --device $D --record $T
+diff $B $T
+cp $T $B
+./build/bin/test-fusion --models build-ci-models --device $D --check $B   # 0 failed, exit 0
+```
+
+- Read the diff before applying it. Every added, removed or changed row must be explainable by the
+  change under review; an unexplained row means the run picked up something unintended.
+- `expected 0` in a FAIL row = no baseline row for that label (new arch, or a fusion newly
+  firing); non-zero `expected` = count changed; `fusion ... is no longer matched` = row stopped
+  firing.
+- `--record` rewrites the whole file, so never record a partial run (`--model FILE`, a filtered
+  models dir) - it drops the rows of every architecture that was not run.
+- `--record` marks every row `ok`; only the `--check` run validates.

src/models/qwen3.cpp

@@ -28,6 +28,10 @@ void llama_model_qwen3::load_arch_tensors(llama_model_loader &) {
     // output rerank head
     cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, "weight"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED);
 
+    // output embedding projection
+    dense_2_out_layers   = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "weight"), {n_embd, hparams.n_embd_out()}, TENSOR_NOT_REQUIRED);
+    dense_2_out_layers_b = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "bias"),   {hparams.n_embd_out()        }, TENSOR_NOT_REQUIRED);
+
     for (int i = 0; i < n_layer; ++i) {
         auto & layer = layers[i];
 

tests/fusion/MTL.csv

@@ -20,6 +20,8 @@ bailingmoe3         ,1   ,any     ,GATED_DELTA_NET+CPY         ,      1
 bailingmoe3         ,1   ,any     ,MUL+ADD                     ,      2
 bailingmoe3         ,1   ,any     ,RMS_NORM+MUL                ,      8
 bailingmoe3         ,1   ,any     ,RMS_NORM+SCALE              ,      2
+beam                ,0   ,any     ,MUL+ADD                     ,      6
+beam                ,0   ,any     ,RMS_NORM+MUL                ,     25
 bloom               ,0   ,any     ,NORM+MUL+ADD                ,      6
 chatglm             ,0   ,any     ,RMS_NORM+MUL                ,      5
 codeshell           ,0   ,any     ,NORM+MUL+ADD                ,      5

tests/test-backend-ops.cpp

@@ -7328,6 +7328,7 @@ struct test_topk_moe : public test_case {
     const bool bias_probs;
     const MoeGatingFunc gating_func;
     const float scale_w;
+    const int n_expert_groups;
     ggml_tensor * weights {};
     ggml_tensor * selected_experts {};
 
@@ -7336,17 +7337,21 @@ struct test_topk_moe : public test_case {
                   bool                   with_norm       = false,
                   bool                   bias_probs      = false,
                   MoeGatingFunc          gating_func     = GATING_FUNC_SOFTMAX,
-                  float                  scale_w         = 0.0f) :
+                  float                  scale_w         = 0.0f,
+                  int                    n_expert_groups = 1) :
         ne(ne),
         n_expert_used(n_expert_used),
         with_norm(with_norm),
         bias_probs(bias_probs),
         gating_func(gating_func),
-        scale_w(scale_w) {
+        scale_w(scale_w),
+        n_expert_groups(n_expert_groups) {
         GGML_ASSERT(n_expert_used <= ne[0]);
+        GGML_ASSERT(ne[0] % n_expert_groups == 0);
+        GGML_ASSERT(n_expert_used <= ne[0] / n_expert_groups);
     }
 
-    std::string vars() override { return VARS_TO_STR6(ne, n_expert_used, with_norm, bias_probs, gating_func, scale_w); }
+    std::string vars() override { return VARS_TO_STR7(ne, n_expert_used, with_norm, bias_probs, gating_func, scale_w, n_expert_groups); }
 
     std::string op_desc(ggml_tensor * t) override {
         GGML_UNUSED(t);
@@ -7358,6 +7363,8 @@ struct test_topk_moe : public test_case {
     ggml_tensor * build_graph(ggml_context * ctx) override {
         const int n_expert = ne[0];
         const int n_tokens = ne[1];
+        const int n_exp_per_group = n_expert / n_expert_groups;
+        const int n_group_used = n_expert_groups / 2;
 
         ggml_tensor * logits = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data());
         ggml_tensor * probs            =
@@ -7374,6 +7381,24 @@ struct test_topk_moe : public test_case {
             ggml_set_name(selection_probs, "selection_probs");
         }
 
+        if (n_expert_groups > 1) {
+            ggml_tensor * selection_groups = ggml_reshape_3d(ctx, selection_probs, n_exp_per_group, n_expert_groups, n_tokens); // [n_exp_per_group, n_expert_groups, n_tokens]
+
+            ggml_tensor * group_scores = ggml_argsort_top_k(ctx, selection_groups, 2); // [2, n_expert_groups, n_tokens]
+            group_scores = ggml_get_rows(ctx, ggml_reshape_4d(ctx, selection_groups, 1, selection_groups->ne[0], selection_groups->ne[1], selection_groups->ne[2]), group_scores); // [1, 2, n_expert_groups, n_tokens]
+
+            group_scores = ggml_sum_rows(ctx, ggml_reshape_3d(ctx, group_scores, group_scores->ne[1], group_scores->ne[2], group_scores->ne[3])); // [1, n_expert_groups, n_tokens]
+            group_scores = ggml_reshape_2d(ctx, group_scores, group_scores->ne[1], group_scores->ne[2]); // [n_expert_groups, n_tokens]
+
+            ggml_tensor * expert_groups = ggml_argsort_top_k(ctx, group_scores, n_group_used); // [n_group_used, n_tokens]
+            ggml_set_name(expert_groups, "ffn_moe_group_topk");
+
+            selection_probs = ggml_get_rows(ctx, selection_groups, expert_groups); // [n_exp_per_group, n_group_used, n_tokens]
+            selection_probs = ggml_set_rows(ctx, ggml_fill(ctx, selection_groups, -INFINITY), selection_probs, expert_groups); // [n_exp_per_group, n_expert_groups, n_tokens]
+            selection_probs = ggml_reshape_2d(ctx, selection_probs, n_expert, n_tokens); // [n_expert, n_tokens]
+            ggml_set_name(selection_probs, "ffn_moe_probs_masked");
+        }
+
         selected_experts = ggml_argsort_top_k(ctx, selection_probs, n_expert_used); // [n_expert_used, n_tokens]
         ggml_set_name(selected_experts, "selected_experts");
 
@@ -7779,24 +7804,43 @@ struct test_sum : public test_case {
     const std::array<int64_t, 4> ne;
     const std::array<int64_t, 4> permute;
     bool _use_permute;
+    const bool slice;
 
     std::string vars() override {
         std::string v = VARS_TO_STR2(type, ne);
         if (_use_permute) v += "," + VAR_TO_STR(permute);
+        if (slice) v += "," + VAR_TO_STR(slice);
         return v;
     }
 
     test_sum(ggml_type type = GGML_TYPE_F32,
             std::array<int64_t, 4> ne = {10, 5, 4, 3},
-            std::array<int64_t, 4> permute = {0, 0, 0, 0})
+            std::array<int64_t, 4> permute = {0, 0, 0, 0},
+            bool slice = false)
         : type(type), ne(ne), permute(permute),
-            _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0) {}
+            _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0),
+            slice(slice) {}
 
     ggml_tensor * build_graph(ggml_context * ctx) override {
         ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data());
-        ggml_set_param(a);
+
+        // the backward pass of a sliced view scatters the gradient via GGML_OP_ACC,
+        // whose CPU implementation requires contiguous rows; a row-breaking permute
+        // (permute[0] != 0) on top of a slice violates that, so skip grad there
+        const bool grad_supported = !(slice && _use_permute && permute[0] != 0);
+        if (grad_supported) {
+            ggml_set_param(a);
+        }
+
         ggml_set_name(a, "a");
 
+        if (slice) {
+            a = ggml_view_4d(ctx, a,
+                             ne[0], ne[1], ne[2] / 2, ne[3] - 1,
+                             a->nb[1], a->nb[2] * 2, a->nb[3], /*offset=*/a->nb[3]);
+            ggml_set_name(a, "a_sliced");
+        }
+
         if (_use_permute) {
             a = ggml_permute(ctx, a, permute[0], permute[1], permute[2], permute[3]);
             ggml_set_name(a, "a_permuted");
@@ -11515,6 +11559,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
     test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 1024, 1, 1 }));
     test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 }));
     test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 }, { 1, 0, 2, 3 })); // sum dst not-contiguous
+    test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 10,  5,  4, 3 }, { 1, 0, 2, 3 })); // non-contiguous rows, 4D
+    test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 11,  5,  6, 3 }, { 2, 0, 1, 3 })); // rotated dims, nb[0] != sizeof(float)
+    test_cases.emplace_back(new test_sum(GGML_TYPE_F32, {11, 5, 6, 3}, {0, 0, 0, 0}, true)); // sliced view (gaps)
+    test_cases.emplace_back(new test_sum(GGML_TYPE_F32, {11, 5, 6, 3}, {1, 0, 2, 3}, true)); // sliced + non-contiguous rows
     test_cases.emplace_back(new test_sum_rows());
     test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, false));
     test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 11, 5, 6, 3 }, false, true));
@@ -11902,6 +11950,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
             true, 16, 8, b, false, true, false));
     }
 
+    for (int64_t k : {96, 896, 1024, 2880}) {
+        for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) {
+            test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q8_0, glu_op, 1, 32, k,
+                false, 16, 8, false, false, true, false, {1, 1}));
+        }
+    }
+
     // Fused row-pair coverage: minimum rows, an even pair, and an odd tail.
     // TODO: the max_nmse_err() for these cases is not estimated correctly causing sporadic false failures.
     //for (ggml_glu_op glu_op : { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU }) {
@@ -11937,6 +11992,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
                     test_cases.emplace_back(new test_topk_moe({32, 8, 1, 1}, 4, with_norm, bias_probs, gate, scale_w));
                     test_cases.emplace_back(new test_topk_moe({32, 8, 1, 1}, 8, with_norm, bias_probs, gate, scale_w));
                     test_cases.emplace_back(new test_topk_moe({32, 9, 1, 1}, 8, with_norm, bias_probs, gate, scale_w));
+
+                    // grouped experts
+                    test_cases.emplace_back(new test_topk_moe({128, 1, 1, 1}, 8, with_norm, bias_probs, gate, scale_w, 16));
+                    test_cases.emplace_back(new test_topk_moe({256, 22, 1, 1}, 6, with_norm, bias_probs, gate, scale_w, 32));
+                    test_cases.emplace_back(new test_topk_moe({256, 7, 1, 1}, 8, with_norm, bias_probs, gate, scale_w, 8)); // DeepSeek V3
+                    test_cases.emplace_back(new test_topk_moe({128, 5, 1, 1}, 6, with_norm, bias_probs, gate, scale_w, 8));
+                    test_cases.emplace_back(new test_topk_moe({512, 3, 1, 1}, 8, with_norm, bias_probs, gate, scale_w, 4));
                 }
             }
         }

tests/test-llama-archs.cpp

@@ -53,33 +53,66 @@ struct tensor_data_params {
     float  stdev;
 };
 
+// fills the values of one tensor, the name carries whatever the property needs
+typedef void (*tensor_filler)(std::vector<float> & dst, const char * name, const tensor_data_params & params);
+
+// a normal draw keyed on a name, so that tensors which share a draw ask for the same key
+static void fill_normal(std::vector<float> & dst, const char * key, const tensor_data_params & params) {
+    std::mt19937 gen(params.seed ^ std::hash<std::string>{}(key));
+    std::normal_distribution<float> dis(0.0f, params.stdev);
+    for (float & v : dst) {
+        v = dis(gen);
+    }
+}
+
+// Mamba A must be negative (state decay)
+static void fill_ssm_a(std::vector<float> & dst, const char * name, const tensor_data_params & params) {
+    fill_normal(dst, name, params);
+    for (float & v : dst) {
+        v = -fabsf(v);
+    }
+}
+
+// a norm preserving residual gate holds alpha^2 + beta^2 == 1, and the conversion resolves both from one raw weight
+// the alpha and the beta of a block share that weight, so the key drops the suffix and keeps the block
+static void fill_resid_gate(std::vector<float> & dst, const char * name, const tensor_data_params & params) {
+    const char * suffix = strstr(name, "_resid_");
+    const bool is_beta = strstr(suffix, "_resid_beta") != nullptr;
+    fill_normal(dst, std::string(name, suffix - name).c_str(), params);
+    for (float & v : dst) {
+        const float beta = 1.0f / (1.0f + expf(-v));
+        v = is_beta ? beta : sqrtf(1.0f - beta*beta);
+    }
+}
+
+// a tensor whose values hold a property of their own, matched on a substring of its name, first match wins
+static const struct { const char * match; tensor_filler fill; } TENSOR_FILLERS[] = {
+    { "ssm_a",   fill_ssm_a      },
+    { "_resid_", fill_resid_gate },
+};
+
 static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {
     const tensor_data_params & params = *(const tensor_data_params *) userdata;
-    size_t seed = params.seed;
-    std::hash<std::string> hasher;
-    seed ^= hasher(tensor->name);
-    std::mt19937 gen(seed);
-    std::normal_distribution<float> dis(0.0f, params.stdev);
 
-    // TODO: refactor per-tensor initialization logic in a cleaner way
-
-    // note: Mamba A must be negative (state decay)
-    const bool is_ssm_a = strstr(tensor->name, "ssm_a") != nullptr;
-    const int64_t ne = ggml_nelements(tensor);
-    if (tensor->type == GGML_TYPE_F32) {
-        std::vector<float> tmp(ne);
-        for (int64_t i = 0; i < ne; i++) {
-            float val = dis(gen);
-            tmp[i] = is_ssm_a ? -fabsf(val) : val;
+    tensor_filler fill = fill_normal;
+    for (const auto & filler : TENSOR_FILLERS) {
+        if (strstr(tensor->name, filler.match) != nullptr) {
+            fill = filler.fill;
+            break;
         }
+    }
+
+    std::vector<float> tmp(ggml_nelements(tensor));
+    fill(tmp, tensor->name, params);
+
+    if (tensor->type == GGML_TYPE_F32) {
         ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor));
     } else if (tensor->type == GGML_TYPE_F16) {
-        std::vector<ggml_fp16_t> tmp(ne);
-        for (int64_t i = 0; i < ne; i++) {
-            float val = dis(gen);
-            tmp[i] = ggml_fp32_to_fp16(is_ssm_a ? -fabsf(val) : val);
+        std::vector<ggml_fp16_t> tmp16(tmp.size());
+        for (size_t i = 0; i < tmp.size(); i++) {
+            tmp16[i] = ggml_fp32_to_fp16(tmp[i]);
         }
-        ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor));
+        ggml_backend_tensor_set(tensor, tmp16.data(), 0, ggml_nbytes(tensor));
     } else {
         GGML_ABORT("fatal error");
     }

tests/test-model-resolution.cpp

@@ -15,6 +15,7 @@
 #include <cstdio>
 #include <cstdlib>
 #include <filesystem>
+#include <fstream>
 #include <map>
 #include <thread>
 #include <string>
@@ -339,6 +340,124 @@ static void test_plan_resolution() {
     g_repos.clear();
 }
 
+// put a file in the cache the way a finished download leaves it, refs/main included
+static void cache_put(const std::string & repo_id, const std::string & path) {
+    namespace fs = std::filesystem;
+
+    const fs::path local = cached(repo_id, path);
+    fs::create_directories(local.parent_path());
+    { std::ofstream(local) << "gguf"; }
+
+    const fs::path ref = local.parent_path().parent_path().parent_path() / "refs" / "main";
+    fs::create_directories(ref.parent_path());
+    if (!fs::exists(ref)) {
+        std::ofstream(ref) << COMMIT << "\n";
+    }
+}
+
+//
+// partials of a download: the files of its plan not in the cache yet, next to
+// the blobs, named after the oids the listing serves
+//
+
+static void test_plan_partials() {
+    namespace fs = std::filesystem;
+
+    printf("test-model-resolution: plan partials\n");
+
+    g_context = "plan partials";
+    g_repos.clear();
+    g_repos["test/partial"] = { "model-Q4_0.gguf", "mmproj-model-F16.gguf", "mtp-model-Q4_0.gguf" };
+
+    // the mmproj is complete, the model and its mtp are still to download
+    cache_put("test/partial", "mmproj-model-F16.gguf");
+
+    common_download_opts opts;
+    opts.download_mmproj = true;
+    opts.download_mtp    = true;
+
+    const auto parts = common_download_plan_partials(common_download_get_hf_plan(model_ref("test/partial:Q4_0"), opts));
+
+    const fs::path blobs = cache_dir / "models--test--partial" / "blobs";
+    REQUIRE(parts.size() == 2);
+    REQUIRE(parts[0].kind.empty());
+    REQUIRE(fs::path(parts[0].path).lexically_normal() == (blobs / "0000000000000000000000000000000000000001.downloadInProgress").lexically_normal());
+    REQUIRE_EQ(parts[1].kind, "mtp");
+    REQUIRE(fs::path(parts[1].path).lexically_normal() == (blobs / "0000000000000000000000000000000000000003.downloadInProgress").lexically_normal());
+
+    fs::remove_all(cache_dir / "models--test--partial");
+    g_repos.clear();
+}
+
+//
+// removal of a sidecar alone: its snapshot entry goes, the model keeps its files
+//
+
+static void test_sidecar_removal() {
+    namespace fs = std::filesystem;
+
+    printf("test-model-resolution: sidecar removal\n");
+
+    g_context = "sidecar removal";
+    g_repos.clear();
+
+    cache_put("test/sidecar", "model-Q4_0.gguf");
+    cache_put("test/sidecar", "dflash-model-Q4_0.gguf");
+    cache_put("test/sidecar", "mtp-model-Q4_0.gguf");
+
+    common_download_opts opts;
+    opts.download_dflash = true;
+
+    // the selected sidecar goes, the other one and the model stay
+    REQUIRE(common_download_remove_sidecars("test/sidecar:Q4_0", opts));
+    REQUIRE(!fs::exists(cached("test/sidecar", "dflash-model-Q4_0.gguf")));
+    REQUIRE(fs::exists(cached("test/sidecar", "mtp-model-Q4_0.gguf")));
+    REQUIRE(fs::exists(cached("test/sidecar", "model-Q4_0.gguf")));
+    REQUIRE(!common_download_remove_sidecars("test/sidecar:Q4_0", opts));
+
+    fs::remove_all(cache_dir / "models--test--sidecar");
+}
+
+//
+// removal of a tag keeps the files another cached tag still loads: a sidecar
+// named after the removed tag that another quant resolves stays on disk
+//
+
+static void test_shared_removal() {
+    namespace fs = std::filesystem;
+
+    printf("test-model-resolution: shared removal\n");
+
+    g_context = "shared removal";
+    g_repos.clear();
+
+    for (const char * f : { "model-Q8_0.gguf", "model-BF16.gguf", "mmproj-model-BF16.gguf", "mtp-model-BF16.gguf" }) {
+        cache_put("test/shared", f);
+    }
+
+    // the BF16 model goes, the mmproj and the mtp Q8_0 resolves stay
+    REQUIRE(common_download_remove("test/shared:BF16"));
+    REQUIRE(!fs::exists(cached("test/shared", "model-BF16.gguf")));
+    REQUIRE(fs::exists(cached("test/shared", "mmproj-model-BF16.gguf")));
+    REQUIRE(fs::exists(cached("test/shared", "mtp-model-BF16.gguf")));
+    REQUIRE(fs::exists(cached("test/shared", "model-Q8_0.gguf")));
+
+    fs::remove_all(cache_dir / "models--test--shared");
+
+    // with no other tag, the files of the removed tag all go, sidecars included
+    for (const char * f : { "model-BF16.gguf", "mmproj-model-BF16.gguf", "mtp-model-BF16.gguf" }) {
+        cache_put("test/alone", f);
+    }
+
+    REQUIRE(common_download_remove("test/alone:BF16"));
+    REQUIRE(!fs::exists(cached("test/alone", "model-BF16.gguf")));
+    REQUIRE(!fs::exists(cached("test/alone", "mmproj-model-BF16.gguf")));
+    REQUIRE(!fs::exists(cached("test/alone", "mtp-model-BF16.gguf")));
+    REQUIRE(!common_download_remove("test/alone:BF16"));
+
+    fs::remove_all(cache_dir / "models--test--alone");
+}
+
 //
 // end-to-end assembly: real CLI parsing, real handler init resolving over the
 // loopback, downloads skipped by flipping offline before apply
@@ -509,6 +628,9 @@ int main(void) {
     common_set_env("MODEL_ENDPOINT", "http://127.0.0.1:" + std::to_string(port) + "/");
 
     test_plan_resolution();
+    test_plan_partials();
+    test_sidecar_removal();
+    test_shared_removal();
     test_task_assembly();
 
     server.stop();

tools/export-lora/export-lora.cpp

@@ -104,6 +104,9 @@ struct file_input {
         auto offset = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i_tensor_in);
         f_in.seekg(offset);
         f_in.read((char* )buf.data(), len);
+        if (!f_in) {
+            throw std::runtime_error("failed to read data for tensor " + name + ", input file may be truncated");
+        }
     }
 
     ~file_input() {

tools/gguf-split/gguf-split.cpp

@@ -362,6 +362,10 @@ struct split_strategy {
         }
         f_in.seekg(in_offset);
         f_in.read((char *)read_buf.data(), len);
+        if (!f_in) {
+            fprintf(stderr, "%s: failed to read %zu bytes at offset %zu, input file may be truncated\n", __func__, len, in_offset);
+            exit(EXIT_FAILURE);
+        }
         f_out.write((const char *)read_buf.data(), len);
     }
 };

tools/server/README.md

@@ -2106,6 +2106,32 @@ Note: for "downloading" state, there can be multiple files be downloading in par
 }
 ```
 
+A download that never finished (paused or interrupted) is listed under a separate
+"incomplete" status, without an instance: posting the model to `/models` again resumes it
+from the files already on disk, and `DELETE /models` drops its partial files.
+
+```json
+"status": {
+  "value": "incomplete",
+  "args": []
+}
+```
+
+Cached models and incomplete downloads carry the sidecar kinds they have on disk
+(`mmproj`, `mtp`, `dflash`, `eagle3`, `dspark`), resolved the way `-hf` loads them:
+
+```json
+"sidecars": ["mmproj", "mtp"]
+```
+
+A cached model whose download of more sidecars never finished also lists the kinds not on
+disk yet: posting the model with them resumes them, and `DELETE /models` with `&sidecar={kind}`
+drops their partial files.
+
+```json
+"incomplete_sidecars": ["mtp"]
+```
+
 ### POST `/models/load`: Load a model
 
 Load a model
@@ -2225,7 +2251,7 @@ Example events:
 
 Trigger a new download (non-blocking), the progress can be tracked via SSE endpoint `/models/sse`
 
-To cancel model downloading, send an event to `/models/unload`
+To pause a download, send an event to `/models/unload`: it keeps its partial files and is listed as "incomplete" until it is posted again. To cancel it and drop its files, use `DELETE /models`.
 
 Download procedure:
 - Send POST request to `/models`
@@ -2238,9 +2264,14 @@ Payload:
 ```json
 {
   "model": "ggml-org/gemma-3-4b-it-GGUF:Q4_K_M",
+  "sidecars": ["mmproj", "mtp"]
 }
 ```
 
+`sidecars` lists the sidecar kinds pulled with the model, picked by the server the way `-hf` picks them: `mmproj`, and the draft heads `mtp`, `dflash`, `eagle3` and `dspark`. It defaults to `["mmproj"]`, as `-hf` does.
+
+Posting a model that is already in the cache and not running downloads the sidecars it lacks and keeps the files already on disk.
+
 Response (download is started in the background):
 
 ```json
@@ -2267,6 +2298,10 @@ IMPORTANT: only model stored in cache can be deleted. You cannot delete models i
 
 Model name must be passed via query param: `?model={name}`
 
+To remove one sidecar of a cached model and keep the model, add `&sidecar={kind}` (`mmproj`, `mtp`, `dflash`, `eagle3` or `dspark`); the model must not be running.
+
+Deleting a model keeps the files another cached quant of the repo still loads, such as an mmproj named after the deleted quant, and the partial files another download still writes. A sidecar deleted on its own goes for every quant that shares it.
+
 If delete success, it will send an SSE event of type `model_remove`
 
 Response:

tools/server/server-models.cpp

@@ -26,6 +26,7 @@
 #include <chrono>
 #include <queue>
 #include <filesystem>
+#include <fstream>
 #include <random>
 #include <sstream>
 #include <cstring>
@@ -618,6 +619,7 @@ server_models::server_models(
         LOG_WRN("failed to get server executable path: %s\n", e.what());
         LOG_WRN("using original argv[0] as fallback: %s\n", argv[0]);
     }
+    unfinished_downloads.load();
     load_models();
     debug_fake_timing = !common_get_env("LLAMA_SERVER_DEBUG_FAKE_TIMING").empty();
 }
@@ -686,6 +688,233 @@ void server_models::add_model(server_model_meta && meta) {
     };
 }
 
+static bool has_sidecar(const std::vector<std::string> & sidecars, const std::string & kind) {
+    return std::find(sidecars.begin(), sidecars.end(), kind) != sidecars.end();
+}
+
+// reject a sidecar kind no download can pull
+static void check_sidecar(const std::string & kind) {
+    std::vector<std::string> kinds;
+    for (const auto & s : common_download_sidecars()) {
+        kinds.emplace_back(s.name);
+    }
+    if (!has_sidecar(kinds, kind)) {
+        throw std::invalid_argument("unknown sidecar '" + kind + "', expected one of " + string_join(kinds, ", "));
+    }
+}
+
+// the sidecars a download asks for; by default the ones -hf pulls without a draft type
+static std::vector<std::string> parse_sidecars(const json & body) {
+    std::vector<std::string> defaults;
+    for (const auto & s : common_download_sidecars()) {
+        if (!s.draft) {
+            defaults.emplace_back(s.name);
+        }
+    }
+    std::vector<std::string> sidecars = json_value(body, "sidecars", defaults);
+    for (const auto & kind : sidecars) {
+        check_sidecar(kind);
+    }
+    return sidecars;
+}
+
+// the download options of a model with its sidecars, the plan a removal purges partials from
+static common_download_opts sidecar_download_opts(const common_params & params, const std::vector<std::string> & sidecars) {
+    common_download_opts opts;
+    opts.bearer_token = params.hf_token;
+    for (const auto & kind : sidecars) {
+        common_download_opts_add_sidecar(opts, kind);
+    }
+    return opts;
+}
+
+// the HF repo a model resolves from: a preset names its own, a cached model is
+// listed under it, and a local file or a models dir entry has none
+static std::string model_hf_repo(const server_model_meta & meta) {
+    std::string hf_repo;
+    if (meta.preset.get_option("LLAMA_ARG_HF_REPO", hf_repo) && !hf_repo.empty()) {
+        return hf_repo;
+    }
+    return meta.source == SERVER_MODEL_SOURCE_CACHE ? meta.name : "";
+}
+
+// the sidecar kinds of a HF repo present on disk, as the offline plan resolves them
+static json cached_sidecars(const std::string & hf_repo) {
+    common_params_model model;
+    model.hf_repo = hf_repo;
+
+    common_download_opts opts;
+    opts.offline = true;
+    for (const auto & s : common_download_sidecars()) {
+        opts.*s.opt = true;
+    }
+
+    return common_download_plan_sidecars(common_download_get_hf_plan(model, opts));
+}
+
+// replace any inherited value of `key` in a child environment
+static void set_child_env(std::vector<std::string> & env, const std::string & key, const std::string & value) {
+    env.erase(std::remove_if(env.begin(), env.end(), [&](const std::string & e) {
+        return e.rfind(key + "=", 0) == 0;
+    }), env.end());
+    env.push_back(key + "=" + value);
+}
+
+// where the unfinished downloads are remembered, inside the model cache
+static std::string unfinished_downloads_path() {
+    return hf_cache::get_cache_path() + "/unfinished-downloads.json";
+}
+
+void server_unfinished_downloads::load() {
+    std::ifstream in(unfinished_downloads_path());
+    if (!in) {
+        return;
+    }
+    std::string raw((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
+    std::lock_guard<std::mutex> lk(mutex);
+    try {
+        json data = json::parse(raw);
+        for (const auto & d : data.value("downloads", json::array())) {
+            server_unfinished_download download;
+            download.model    = json_value(d, "model", std::string());
+            download.sidecars = json_value(d, "sidecars", std::vector<std::string>());
+            for (const auto & p : d.value("partials", json::array())) {
+                download.partials.push_back({ json_value(p, "kind", std::string()), json_value(p, "path", std::string()) });
+            }
+            if (!download.model.empty()) {
+                downloads.push_back(std::move(download));
+            }
+        }
+    } catch (const std::exception & e) {
+        LOG_WRN("failed to read unfinished downloads: %s\n", e.what());
+    }
+}
+
+void server_unfinished_downloads::save() {
+    json data;
+    data["downloads"] = json::array();
+    for (const auto & d : downloads) {
+        json partials = json::array();
+        for (const auto & p : d.partials) {
+            partials.push_back({ {"kind", p.kind}, {"path", p.path} });
+        }
+        data["downloads"].push_back({ {"model", d.model}, {"sidecars", d.sidecars}, {"partials", partials} });
+    }
+    try {
+        fs_write_atomic(std::filesystem::u8path(unfinished_downloads_path()), data.dump());
+    } catch (const std::exception & e) {
+        LOG_WRN("failed to record unfinished downloads: %s\n", e.what());
+    }
+}
+
+void server_unfinished_downloads::delete_partials(const std::vector<common_download_partial> & parts) {
+    for (const auto & p : parts) {
+        const bool shared = std::any_of(downloads.begin(), downloads.end(), [&](const server_unfinished_download & d) {
+            return std::any_of(d.partials.begin(), d.partials.end(), [&](const common_download_partial & q) {
+                return q.path == p.path;
+            });
+        });
+        if (shared) {
+            continue;
+        }
+        std::error_code ec;
+        std::filesystem::remove(std::filesystem::u8path(p.path), ec);
+    }
+}
+
+void server_unfinished_downloads::add(const server_unfinished_download & download) {
+    std::lock_guard<std::mutex> lk(mutex);
+    std::vector<common_download_partial> stale;
+    auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == download.model; });
+    if (it != downloads.end()) {
+        stale = std::move(it->partials);
+        *it = download;
+    } else {
+        downloads.push_back(download);
+    }
+    save();
+    delete_partials(stale);
+}
+
+void server_unfinished_downloads::forget(const std::string & model) {
+    std::lock_guard<std::mutex> lk(mutex);
+    auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == model; });
+    if (it != downloads.end()) {
+        downloads.erase(it);
+        save();
+    }
+}
+
+bool server_unfinished_downloads::remove(const std::string & model) {
+    std::lock_guard<std::mutex> lk(mutex);
+    auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == model; });
+    if (it == downloads.end()) {
+        return false;
+    }
+    auto partials = std::move(it->partials);
+    downloads.erase(it);
+    save();
+    delete_partials(partials);
+    return true;
+}
+
+bool server_unfinished_downloads::remove_sidecar(const std::string & model, const std::string & kind) {
+    std::lock_guard<std::mutex> lk(mutex);
+    auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == model; });
+    if (it == downloads.end() || !has_sidecar(it->sidecars, kind)) {
+        return false;
+    }
+    it->sidecars.erase(std::remove(it->sidecars.begin(), it->sidecars.end(), kind), it->sidecars.end());
+
+    std::vector<common_download_partial> dropped;
+    auto & partials = it->partials;
+    auto pos = std::stable_partition(partials.begin(), partials.end(), [&](const common_download_partial & p) {
+        return p.kind != kind;
+    });
+    dropped.assign(pos, partials.end());
+    partials.erase(pos, partials.end());
+
+    save();
+    delete_partials(dropped);
+    return true;
+}
+
+std::vector<server_unfinished_download> server_unfinished_downloads::get_all() {
+    std::lock_guard<std::mutex> lk(mutex);
+    return downloads;
+}
+
+std::vector<server_unfinished_download> server_models::unfinished_downloads_get() {
+    return unfinished_downloads.get_all();
+}
+
+void server_models::unfinished_downloads_remove(const std::string & name) {
+    // under the lock, so a download of the same tag cannot start while its files go
+    std::lock_guard<std::mutex> lk(mutex);
+    if (mapping.find(name) != mapping.end()) {
+        throw std::invalid_argument("model '" + name + "' is downloading");
+    }
+    // a name with no record is no model: its files belong to the cached tags of the repo
+    if (!unfinished_downloads.remove(name)) {
+        throw std::runtime_error("model name=" + name + " is not found");
+    }
+    bool ok = common_download_remove(name);
+    SRV_INF("removing unfinished download name=%s (%s)\n", name.c_str(), ok ? "succeeded" : "nothing cached");
+}
+
+bool server_models::remove_sidecar(const std::string & name, const std::string & kind) {
+    std::lock_guard<std::mutex> lk(mutex);
+    auto it = mapping.find(name);
+    if (it != mapping.end() && (it->second.meta.is_running() || it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING)) {
+        throw std::invalid_argument("model '" + name + "' is in use, unload it first");
+    }
+    // a sidecar that only ever landed as a partial is still something removed
+    bool ok = unfinished_downloads.remove_sidecar(name, kind);
+    ok = common_download_remove_sidecars(name, sidecar_download_opts(base_params, { kind })) || ok;
+    SRV_INF("removing sidecar %s of model name=%s (%s)\n", kind.c_str(), name.c_str(), ok ? "succeeded" : "nothing cached");
+    return ok;
+}
+
 void server_models::notify_sse(const std::string & event, const std::string & model_id, const json & data) {
     std::unique_ptr<server_task_result_router> result = std::make_unique<server_task_result_router>();
     result->data = {
@@ -1135,7 +1364,8 @@ void server_models::load(const std::string & name) {
 }
 
 void server_models::load(const std::string & name, const load_options & opts) {
-    if (debug_fake_timing) {
+    // the fake load time of a model; a download worker loads none
+    if (debug_fake_timing && opts.mode == SERVER_CHILD_MODE_NORMAL) {
         // do not hold the mutex here, other requests must keep making progress
         std::this_thread::sleep_for(std::chrono::seconds(2));
     }
@@ -1152,6 +1382,43 @@ void server_models::load(const std::string & name, const load_options & opts) {
     // against the freshest preset and a consistent mapping state
     cv.wait(lk, [this]() { return !is_reloading; });
 
+    // a download over an existing entry is decided under the lock, so a load of the
+    // same model cannot start in between: a paused download or an idle cached model
+    // is downloaded again with its files kept, a download on its way and a running
+    // model are refused, and a preset or local model is not a download
+    if (opts.mode == SERVER_CHILD_MODE_DOWNLOAD) {
+        auto it = mapping.find(name);
+        if (it != mapping.end()) {
+            const auto & cur = it->second.meta;
+            if (cur.status == SERVER_MODEL_STATUS_DOWNLOADING) {
+                throw std::invalid_argument("model '" + name + "' is already downloading");
+            }
+            if (cur.is_running()) {
+                throw std::invalid_argument("model '" + name + "' is running, unload it first");
+            }
+            if (cur.source != SERVER_MODEL_SOURCE_CACHE) {
+                throw std::invalid_argument("model '" + name + "' already exists");
+            }
+            SRV_INF("resuming download for model '%s'\n", name.c_str());
+        }
+
+        // two downloads writing the same partial would interleave their bytes
+        for (const auto & d : unfinished_downloads.get_all()) {
+            auto other = mapping.find(d.model);
+            if (d.model == name || other == mapping.end() || other->second.meta.status != SERVER_MODEL_STATUS_DOWNLOADING) {
+                continue;
+            }
+            for (const auto & p : d.partials) {
+                const bool shared = std::any_of(opts.partials.begin(), opts.partials.end(), [&](const common_download_partial & q) {
+                    return q.path == p.path;
+                });
+                if (shared) {
+                    throw std::invalid_argument("model '" + name + "' shares files with the download of '" + d.model + "' on its way, retry once it is done");
+                }
+            }
+        }
+    }
+
     auto meta = opts.custom_meta.has_value() ? *opts.custom_meta : mapping[name].meta;
     if (meta.status != SERVER_MODEL_STATUS_UNLOADED) {
         SRV_INF("model %s is not ready\n", name.c_str());
@@ -1201,6 +1468,23 @@ void server_models::load(const std::string & name, const load_options & opts) {
             inst.meta.status = SERVER_MODEL_STATUS_DOWNLOADING;
             child_env.push_back("LLAMA_SERVER_CHILD_MODE=download");
             child_env.push_back("LLAMA_ARG_HF_REPO=" + name);
+
+            // the sidecars as -hf would pull them: the draft types, and the mmproj switch
+            // for the one that is not a draft
+            bool        mmproj = false;
+            std::string spec_types;
+            for (const auto & s : common_download_sidecars()) {
+                if (!has_sidecar(opts.sidecars, s.name)) {
+                    continue;
+                }
+                if (s.draft) {
+                    spec_types += (spec_types.empty() ? "" : ",") + std::string("draft-") + s.name;
+                } else {
+                    mmproj = true;
+                }
+            }
+            set_child_env(child_env, "LLAMA_ARG_MMPROJ_AUTO", mmproj ? "1" : "0");
+            set_child_env(child_env, "LLAMA_ARG_SPEC_TYPE", spec_types.empty() ? "none" : spec_types);
         }
 
         SRV_INF("%s", "spawning server instance with args:\n");
@@ -1216,6 +1500,11 @@ void server_models::load(const std::string & name, const load_options & opts) {
         }
     }
 
+    // remember the download: the partial files carry a content hash, not the name
+    if (opts.mode == SERVER_CHILD_MODE_DOWNLOAD) {
+        unfinished_downloads.add({ name, opts.sidecars, opts.partials });
+    }
+
     // old process should have exited already, but just in case, we clean it up here
     {
         auto it = mapping.find(name);
@@ -1390,6 +1679,12 @@ void server_models::update_download_progress(const std::string & name, const com
                 // mark the instance to be erased on next load_models() call
                 it->second.meta.status = SERVER_MODEL_STATUS_DOWNLOADED;
                 need_reload = true;
+
+                // a download that failed keeps its tag: its partial files are still
+                // there and re-posting the tag resumes them
+                if (ok) {
+                    unfinished_downloads.forget(name);
+                }
             } else {
                 json & info = it->second.meta.loaded_info;
                 if (!info.contains("progress")) {
@@ -1445,24 +1740,14 @@ bool server_models::remove(const std::string & name) {
             || meta.status == SERVER_MODEL_STATUS_DOWNLOADED;
     });
 
-    // re-find after wait - load_models() may have erased the entry during the wait
-    it = mapping.find(name);
-    if (it == mapping.end()) {
-        // load_models() already erased the entry; we just need to clean up the cached files on disk
-        lk.unlock();
-        bool ok = common_download_remove(name);
-        SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "partial");
-        notify_sse("model_remove", name, {});
-        return true;
-    }
-
-    // remove from disk (best-effort: cancelled downloads may have no cached files)
-    bool ok = common_download_remove(name);
+    // load_models() may have erased the entry during the wait; the files go under the
+    // lock, so a download of the same tag cannot start meanwhile
     mapping.erase(name);
-    if (!ok) {
-        SRV_WRN("removing model name=%s from disk returned false (no cached files?)\n", name.c_str());
-    }
-    SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "partial");
+    unfinished_downloads.remove(name);
+
+    // best-effort: a cancelled download may have no cached files
+    bool ok = common_download_remove(name);
+    SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "nothing cached");
     notify_sse("model_remove", name, {});
     return true;
 }
@@ -2091,6 +2376,9 @@ void server_models_routes::init_routes() {
         json models_json = json::array();
         auto all_models = models.get_all_meta();
         std::time_t t = std::time(0);
+        // a download on its way has no files to resolve yet, only the kinds it pulls
+        const auto unfinished = models.unfinished_downloads_get();
+
         for (const auto & meta : all_models) {
             if (meta.hidden) {
                 continue; // cache model deduplicated by a preset
@@ -2132,6 +2420,35 @@ void server_models_routes::init_routes() {
                 model_info["context_length"] = meta.n_ctx_train;
             }
 
+            // the sidecars a model has on disk, resolved as -hf would load them, for a
+            // model that resolves from a HF repo. A download on its way reports the
+            // kinds it pulls, as an unfinished one does
+            const auto it_unfinished = std::find_if(unfinished.begin(), unfinished.end(),
+                [&](const server_unfinished_download & d) { return d.model == meta.name; });
+            const std::string hf_repo = model_hf_repo(meta);
+
+            if (it_unfinished != unfinished.end() && meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
+                model_info["sidecars"] = it_unfinished->sidecars;
+            } else if (!hf_repo.empty()) {
+                const json cached = cached_sidecars(hf_repo);
+                model_info["sidecars"] = cached;
+
+                // the sidecars of an unfinished download of this model that are not on
+                // disk yet: a POST with the kind resumes one, a DELETE with it drops it
+                if (it_unfinished != unfinished.end()) {
+                    std::vector<std::string> incomplete;
+                    for (const auto & p : it_unfinished->partials) {
+                        if (!p.kind.empty() && !has_sidecar(incomplete, p.kind) &&
+                            std::find(cached.begin(), cached.end(), p.kind) == cached.end()) {
+                            incomplete.push_back(p.kind);
+                        }
+                    }
+                    if (!incomplete.empty()) {
+                        model_info["incomplete_sidecars"] = incomplete;
+                    }
+                }
+            }
+
             // merge with loaded_info from the child process if available
             if (meta.is_running()) {
                 for (auto it = meta.loaded_info.begin(); it != meta.loaded_info.end(); ++it) {
@@ -2142,6 +2459,33 @@ void server_models_routes::init_routes() {
             }
             models_json.push_back(model_info);
         }
+
+        const json architecture_default = json {
+            {"input_modalities",  json::array({"text"})},
+            {"output_modalities", json::array({"text"})},
+        };
+
+        // a download that never finished has no instance: report it as incomplete so
+        // the UI can offer to resume it, without starting anything
+        for (const auto & d : unfinished) {
+            if (models.has_model(d.model)) {
+                continue; // complete, or downloading right now
+            }
+            models_json.push_back(json {
+                {"id",           d.model},
+                {"aliases",      json::array()},
+                {"tags",         json::array()},
+                {"object",       "model"},
+                {"owned_by",     "llamacpp"},
+                {"created",      t},
+                {"status",       json { {"value", server_model_status_to_string(SERVER_MODEL_STATUS_INCOMPLETE)}, {"args", json::array()} }},
+                {"architecture", architecture_default},
+                {"source",       server_model_source_to_string(SERVER_MODEL_SOURCE_CACHE)},
+                {"can_remove",   true},
+                {"sidecars",     d.sidecars},
+            });
+        }
+
         res_ok(res, {
             {"data", models_json},
             {"object", "list"},
@@ -2193,32 +2537,27 @@ void server_models_routes::init_routes() {
         if (name.empty()) {
             throw std::invalid_argument("model must be a non-empty string");
         }
+        const auto sidecars = parse_sidecars(body);
 
-        common_params p;
-        p.model.hf_repo  = name;
-        p.hf_token       = params.hf_token;
+        common_params_model model;
+        model.hf_repo = name;
 
-        // validate by fetching metadata
-        bool ok = false;
+        // validate by fetching metadata, through the plan the download follows
+        common_download_hf_plan plan;
         try {
-            common_models_handler_init(p, LLAMA_EXAMPLE_SERVER);
-            ok = true;
+            plan = common_download_get_hf_plan(model, sidecar_download_opts(params, sidecars));
         } catch (...) {
             SRV_ERR("unknown error while validating model '%s'\n", name.c_str());
             // other exceptions will be handled by the outer ex_wrapper()
             throw;
         }
 
-        if (!ok) {
-            throw std::invalid_argument("model validation failed, unable to download");
+        // a tag that resolves no file has nothing to download
+        if (plan.preset.path.empty() && plan.primary.path.empty() && common_download_plan_sidecars(plan).empty()) {
+            throw std::invalid_argument("model '" + name + "' resolves no file to download");
         }
 
-        // reject if model already exists
-        if (models.has_model(name)) {
-            throw std::invalid_argument("model '" + name + "' already exists");
-        }
-
-        // then, proceed with the actual download
+        // then, proceed with the actual download; load() decides over an existing entry
         SRV_INF("starting download for model '%s'\n", name.c_str());
         {
             server_models::load_options load_opts;
@@ -2226,6 +2565,8 @@ void server_models_routes::init_routes() {
             load_opts.custom_meta = server_model_meta{};
             load_opts.custom_meta->source = SERVER_MODEL_SOURCE_CACHE;
             load_opts.custom_meta->name   = name;
+            load_opts.sidecars            = sidecars;
+            load_opts.partials            = common_download_plan_partials(plan);
             models.load(name, load_opts);
         }
 
@@ -2241,6 +2582,22 @@ void server_models_routes::init_routes() {
             throw std::invalid_argument("model must be a non-empty string");
         }
 
+        // a sidecar alone goes, the model keeps its files
+        std::string sidecar = req.get_param("sidecar");
+        if (!sidecar.empty()) {
+            check_sidecar(sidecar);
+            res_ok(res, {{"success", models.remove_sidecar(name, sidecar)}});
+            return res;
+        }
+
+        // a download listed from an earlier run has no instance, only its tag and
+        // its partial files
+        if (!models.has_model(name)) {
+            models.unfinished_downloads_remove(name);
+            res_ok(res, {{"success", true}});
+            return res;
+        }
+
         models.remove(name); // throws on error
 
         res_ok(res, {{"success", true}});

tools/server/server-models.h

@@ -16,6 +16,7 @@
 #include <set>
 #include <string>
 #include <unordered_map>
+#include <vector>
 
 /**
  * state diagram:
@@ -31,6 +32,7 @@
 enum server_model_status {
     // TODO: also add downloading state when the logic is added
     SERVER_MODEL_STATUS_DOWNLOADING,
+    SERVER_MODEL_STATUS_INCOMPLETE, // a download that never finished, listed without an instance
     SERVER_MODEL_STATUS_DOWNLOADED,
     SERVER_MODEL_STATUS_UNLOADED,
     SERVER_MODEL_STATUS_LOADING,
@@ -52,6 +54,7 @@ enum server_child_mode {
 static std::string server_model_status_to_string(server_model_status status) {
     switch (status) {
         case SERVER_MODEL_STATUS_DOWNLOADING: return "downloading";
+        case SERVER_MODEL_STATUS_INCOMPLETE:  return "incomplete";
         case SERVER_MODEL_STATUS_DOWNLOADED:  return "downloaded";
         case SERVER_MODEL_STATUS_UNLOADED:    return "unloaded";
         case SERVER_MODEL_STATUS_LOADING:     return "loading";
@@ -108,6 +111,51 @@ struct server_model_meta {
     void update_caps(const common_params & base);
 };
 
+// a download that never finished: the tag it was asked under, the sidecar kinds it
+// pulls and the partial files its plan writes
+struct server_unfinished_download {
+    std::string                          model;
+    std::vector<std::string>             sidecars;
+    std::vector<common_download_partial> partials;
+};
+
+// the downloads that never finished, recorded next to the model cache. The cache names
+// partial files after their content hash, so this record is the only trace of what is
+// still incomplete: it lets a restarted router list, resume or remove them
+// a partial another recorded download also writes is never deleted
+// these functions are thread-safe
+struct server_unfinished_downloads {
+    // read the record an earlier run left
+    void load();
+
+    // record a download that starts, replacing an earlier record of the same model,
+    // and delete the partials of that record the new plan no longer writes
+    void add(const server_unfinished_download & download);
+
+    // forget a download whose files are complete
+    void forget(const std::string & model);
+
+    // forget a download and delete its partials; returns false when it is not recorded
+    bool remove(const std::string & model);
+
+    // drop the sidecar `kind` from a download and delete its partials; returns false
+    // when the download is not recorded with that kind
+    bool remove_sidecar(const std::string & model, const std::string & kind);
+
+    // every recorded download
+    std::vector<server_unfinished_download> get_all();
+
+private:
+    std::mutex mutex;
+    std::vector<server_unfinished_download> downloads;
+
+    // write the record back; callers hold the mutex
+    void save();
+
+    // delete the partials of `parts` no recorded download writes; callers hold the mutex
+    void delete_partials(const std::vector<common_download_partial> & parts);
+};
+
 struct server_models_routes;
 struct server_lru_sched; // defined in server-models.cpp
 struct server_monitor;   // defined in server-models.cpp
@@ -134,6 +182,9 @@ private:
     // models asked to stop, still counted as running until the monitor records their exit
     std::set<std::string> stopping_models;
 
+    // downloads that never finished, with the sidecar kinds they pull
+    server_unfinished_downloads unfinished_downloads;
+
     // set to true while load_models() is executing a reload; load() will wait until clear
     bool is_reloading = false;
 
@@ -252,6 +303,16 @@ public:
     // check if a model instance exists (thread-safe)
     bool has_model(const std::string & name);
 
+    // downloads that never finished, with their sidecar kinds (thread-safe)
+    std::vector<server_unfinished_download> unfinished_downloads_get();
+
+    // drop a download that will never be resumed, its record and its partial files;
+    // throws for a name that is not recorded (thread-safe)
+    void unfinished_downloads_remove(const std::string & name);
+
+    // remove the sidecar `kind` of a model that is not in use, its files and partials (thread-safe)
+    bool remove_sidecar(const std::string & name, const std::string & kind);
+
     // return a copy of model metadata (thread-safe)
     std::optional<server_model_meta> get_meta(const std::string & name);
 
@@ -262,6 +323,10 @@ public:
         server_child_mode mode = SERVER_CHILD_MODE_NORMAL;
         // used for spawning a downloading child process
         std::optional<server_model_meta> custom_meta = std::nullopt;
+        // download mode: the sidecar kinds pulled with the model, as -hf would, and
+        // the partial files the download plan writes
+        std::vector<std::string>             sidecars;
+        std::vector<common_download_partial> partials;
     };
 
     // load and unload model instances

tools/server/tests/unit/test_router.py

@@ -1,3 +1,4 @@
+import shutil
 import threading
 import pytest
 from utils import *
@@ -546,6 +547,15 @@ def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: i
     return False
 
 
+def _own_cache(name: str) -> str:
+    """An empty model cache for one test: the workers run in parallel, and a download
+    test must not see the files or the record another test writes in the shared cache."""
+    path = os.path.join(TMP_DIR, name)
+    shutil.rmtree(path, ignore_errors=True)
+    os.makedirs(path)
+    return path
+
+
 def test_router_download_model():
     """Case 1: download a model at the model limit, verify SSE events and GET /models."""
     global server
@@ -626,3 +636,230 @@ def test_router_delete_model():
     # Model should no longer appear in GET /models
     ids = _get_model_ids(is_reload=False)
     assert MODEL_DOWNLOAD_ID not in ids, f"{MODEL_DOWNLOAD_ID} still present after deletion"
+
+
+def test_router_download_sidecars():
+    """Case 3: sidecars are a parameter of the download, never models of their own."""
+    global server
+    server.llama_cache = _own_cache("cache_download_sidecars")
+    server.start()
+
+    server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")
+
+    # an unknown sidecar kind is rejected before anything starts
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": ["lora"]})
+    assert res.status_code == 400
+    assert MODEL_DOWNLOAD_ID not in _get_model_ids(is_reload=False)
+
+    sse_events: list = []
+    stop = threading.Event()
+    sse_ready = threading.Event()
+    threading.Thread(
+        target=_listen_sse, args=(server, sse_events, stop, sse_ready), daemon=True
+    ).start()
+    assert sse_ready.wait(10), "SSE client failed to connect"
+
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []})
+    assert res.status_code == 200
+    finished = _wait_for_sse_event(
+        sse_events, "download_finished", MODEL_DOWNLOAD_ID, MODEL_DOWNLOAD_TIMEOUT
+    )
+    stop.set()
+    assert finished, "Never received download_finished SSE event"
+
+    # the model lists the sidecars it has on disk, and no sidecar is listed as a model
+    res = server.make_request("GET", "/models")
+    assert res.status_code == 200
+    entries = {item["id"]: item for item in res.body.get("data", [])}
+    assert entries[MODEL_DOWNLOAD_ID]["sidecars"] == []
+    for kind in ("mmproj", "mtp", "dflash", "eagle3", "dspark"):
+        assert not any(i.endswith("-" + kind) for i in entries), f"a {kind} sidecar is listed as a model"
+
+    server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")
+
+
+def test_router_download_conflicts():
+    """A download is refused over one on its way, and over a model that is running."""
+    global server
+    server.models_max = 2
+    server.llama_cache = _own_cache("cache_download_conflicts")
+    server.start()
+
+    server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")
+
+    # a download on its way is not started again; the entry reports the state before
+    # the child fetches anything, so the probe lands inside the download
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID})
+    assert res.status_code == 200
+    _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloading"}, timeout=30)
+
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID})
+    assert res.status_code == 400
+    assert "is already downloading" in res.body["error"]["message"]
+
+    # a loaded model holds its files, so it goes only once it is unloaded; the
+    # reload that follows the download parks the entry as unloaded
+    _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT)
+    _load_model_and_wait(MODEL_DOWNLOAD_ID, timeout=120)
+
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID})
+    assert res.status_code == 400
+    assert "is running, unload it first" in res.body["error"]["message"]
+
+
+def test_router_delete_unknown_name():
+    """A name that is neither a model nor a recorded download is not found, and the
+    cached tags of its repo keep their files."""
+    global server
+    server.llama_cache = _own_cache("cache_delete_unknown_name")
+    server.start()
+
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []})
+    assert res.status_code == 200
+    _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT)
+
+    repo = MODEL_DOWNLOAD_ID.split(":")[0]
+    res = server.make_request("DELETE", f"/models?model={repo}")
+    assert res.status_code != 200
+    assert "is not found" in res.body["error"]["message"]
+    assert MODEL_DOWNLOAD_ID in _get_model_ids(is_reload=True)
+
+    server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")
+
+
+def test_router_incomplete_sidecars():
+    """A cached model lists the sidecars its unfinished download has not written yet, until
+    they are dropped."""
+    global server
+    cache_dir = _own_cache("cache_incomplete_sidecars")
+    server.llama_cache = cache_dir
+    server.start()
+
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []})
+    assert res.status_code == 200
+    _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT)
+    server.stop()
+
+    # a paused mtp of the cached model, as the record of an earlier run holds it
+    partial_path = os.path.abspath(os.path.join(cache_dir, "test-mtp.gguf.downloadInProgress"))
+    with open(partial_path, "w") as f:
+        f.write("partial")
+    with open(os.path.join(cache_dir, "unfinished-downloads.json"), "w") as f:
+        json.dump({"downloads": [{
+            "model":    MODEL_DOWNLOAD_ID,
+            "sidecars": ["mtp"],
+            "partials": [{"kind": "mtp", "path": partial_path}],
+        }]}, f)
+
+    server.start()
+
+    def entry() -> dict:
+        res = server.make_request("GET", "/models")
+        assert res.status_code == 200
+        return {item["id"]: item for item in res.body.get("data", [])}[MODEL_DOWNLOAD_ID]
+
+    assert entry()["sidecars"] == []
+    assert entry()["incomplete_sidecars"] == ["mtp"]
+
+    res = server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}&sidecar=mtp")
+    assert res.status_code == 200
+    assert res.body.get("success") is True
+    assert not os.path.exists(partial_path)
+    assert "incomplete_sidecars" not in entry()
+
+    server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")
+
+
+def test_router_preset_model_sidecars():
+    """A preset model reports the sidecars its files have on disk, like a cached one."""
+    global server
+
+    preset_path = os.path.join(TMP_DIR, "test_preset_sidecars.ini")
+
+    with open(preset_path, "w") as f:
+        f.write("[preset-sidecars]\nhf-repo = ggml-org/tinygemma3-GGUF:Q8_0\n")
+
+    server.models_preset = preset_path
+    server.start()
+
+    try:
+        # the mmproj of that repo lands next to the model, as -hf would pull it
+        res = server.make_request("POST", "/models", data={"model": MODEL_A, "sidecars": ["mmproj"]})
+        assert res.status_code == 200
+        _wait_for_model_status(MODEL_A, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT)
+
+        res = server.make_request("GET", "/models")
+        assert res.status_code == 200
+        entries = {item["id"]: item for item in res.body.get("data", [])}
+
+        assert entries["preset-sidecars"]["source"] == "preset"
+        assert entries["preset-sidecars"]["sidecars"] == ["mmproj"]
+    finally:
+        os.remove(preset_path)
+
+
+def test_router_local_preset_listing():
+    """A preset on a local file resolves from no HF repo: it lists without sidecars."""
+    global server
+
+    model_path = os.path.join(TMP_DIR, "test_local_preset.gguf")
+    preset_path = os.path.join(TMP_DIR, "test_local_preset.ini")
+
+    with open(model_path, "w") as f:
+        f.write("GGUF")
+    with open(preset_path, "w") as f:
+        f.write(f"[local]\nmodel = {model_path}\n")
+
+    server.models_preset = preset_path
+    server.start()
+
+    try:
+        res = server.make_request("GET", "/models")
+        assert res.status_code == 200
+        entries = {item["id"]: item for item in res.body.get("data", [])}
+
+        assert entries["local"]["source"] == "preset"
+        assert "sidecars" not in entries["local"]
+    finally:
+        os.remove(preset_path)
+        os.remove(model_path)
+
+
+def test_router_download_unresolved_tag():
+    """A tag that resolves no file of the repo is refused before anything starts."""
+    global server
+    server.start()
+
+    model = MODEL_DOWNLOAD_ID.split(":")[0] + ":Q9_Z"
+    res = server.make_request("POST", "/models", data={"model": model})
+    assert res.status_code == 400
+    assert "resolves no file to download" in res.body["error"]["message"]
+    assert model not in _get_model_ids(is_reload=False)
+
+
+def test_router_download_drops_stale_partials():
+    """Posting a recorded download again deletes the partials its new plan no longer writes."""
+    global server
+
+    cache_dir = _own_cache("cache_download_drops_stale_partials")
+    stale_path = os.path.abspath(os.path.join(cache_dir, "test-stale.gguf.downloadInProgress"))
+    record_path = os.path.join(cache_dir, "unfinished-downloads.json")
+
+    with open(stale_path, "w") as f:
+        f.write("stale")
+    with open(record_path, "w") as f:
+        json.dump({"downloads": [{
+            "model":    MODEL_DOWNLOAD_ID,
+            "sidecars": ["mtp"],
+            "partials": [{"kind": "mtp", "path": stale_path}],
+        }]}, f)
+
+    server.llama_cache = cache_dir
+    server.start()
+
+    res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []})
+    assert res.status_code == 200
+    assert not os.path.exists(stale_path), "the stale partial survived the new record"
+
+    _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT)
+    server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}")

tools/server/tests/utils.py

@@ -127,6 +127,7 @@ class ServerProcess:
     mcp_servers_config: str | None = None
     mcp_servers_json: str | None = None
     cors_origins: str | None = None
+    llama_cache: str | None = None
 
     # session variables
     process: subprocess.Popen | None = None
@@ -145,7 +146,9 @@ class ServerProcess:
             **os.environ,
             "LLAMA_SERVER_DEBUG_FAKE_TIMING": "1",
         }
-        if "LLAMA_CACHE" not in os.environ:
+        if self.llama_cache is not None:
+            env["LLAMA_CACHE"] = self.llama_cache
+        elif "LLAMA_CACHE" not in os.environ:
             env["LLAMA_CACHE"] = "tmp"
         if self.external_server:
             print(f"[external_server]: Assuming external server running on {self.server_host}:{self.server_port}")

tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte

@@ -7,6 +7,7 @@
 		getMdastNodeHash,
 		isAppendMode
 	} from './markdown-utils';
+	import { SAFE_HTML_CONFIG } from './safe-html-config';
 	import {
 		ActionIconCopyToClipboard,
 		CodeBlockActions,
@@ -44,6 +45,7 @@
 	import { detectIncompleteCodeBlock, highlightCode, type IncompleteCodeBlock } from '$lib/utils';
 	import { sanitizeSvg } from '$lib/utils/sanitize-svg';
 	import { mountSvgShadow } from '$lib/utils/svg-shadow';
+	import DOMPurify from 'dompurify';
 	import type { Root as HastRoot, RootContent as HastRootContent } from 'hast';
 	import githubLightCss from 'highlight.js/styles/github.css?inline';
 	import githubDarkCss from 'highlight.js/styles/github-dark.css?inline';
@@ -57,6 +59,8 @@
 		content: string;
 		class?: string;
 		disableMath?: boolean;
+		/** Render raw HTML found in the markdown (sanitized) instead of escaping it. */
+		allowHtml?: boolean;
 	}
 
 	interface MarkdownBlock {
@@ -65,7 +69,13 @@
 		contentHash?: string;
 	}
 
-	let { attachments, class: className = '', content, disableMath = false }: Props = $props();
+	let {
+		allowHtml = false,
+		attachments,
+		class: className = '',
+		content,
+		disableMath = false
+	}: Props = $props();
 
 	let containerRef = $state<HTMLDivElement>();
 	let renderedBlocks = $state<MarkdownBlock[]>([]);
@@ -121,6 +131,8 @@
 
 	// Per-instance transform cache, avoids re-transforming stable blocks during streaming
 	// Garbage collected when component is destroyed (on conversation change)
+	const CACHE_KEY_PLAIN = 'plain';
+	const CACHE_KEY_ALLOW_HTML = 'allow-html';
 	const transformCache = new SvelteMap<string, string>();
 	let previousContent = '';
 
@@ -183,7 +195,10 @@
 		index: number
 	): Promise<{ html: string; hash: string }> {
 		const hash = getMdastNodeHash(node, index);
-		const cached = transformCache.get(hash);
+		// the rendered HTML also depends on allowHtml (sanitized raw vs escaped),
+		// so the cache is keyed per mode
+		const cacheKey = `${allowHtml ? CACHE_KEY_ALLOW_HTML : CACHE_KEY_PLAIN}:${hash}`;
+		const cached = transformCache.get(cacheKey);
 
 		if (cached) {
 			return { hash, html: cached };
@@ -192,10 +207,13 @@
 		const singleNodeRoot = { children: [node], type: 'root' };
 		const transformedRoot = (await processorInstance.run(singleNodeRoot as MdastRoot)) as HastRoot;
 		const html = processorInstance.stringify(transformedRoot);
+		const safeHtml = allowHtml
+			? (DOMPurify.sanitize(html, SAFE_HTML_CONFIG) as unknown as string)
+			: html;
 
-		transformCache.set(hash, html);
+		transformCache.set(cacheKey, safeHtml);
 
-		return { hash, html };
+		return { hash, html: safeHtml };
 	}
 
 	/**
@@ -300,7 +318,7 @@
 
 			if (prefixMarkdown.trim()) {
 				const normalizedPrefix = preprocessLaTeX(prefixMarkdown);
-				const processorInstance = getMarkdownProcessor({ attachments, disableMath });
+				const processorInstance = getMarkdownProcessor({ allowHtml, attachments, disableMath });
 				const ast = processorInstance.parse(normalizedPrefix) as MdastRoot;
 				const mdastChildren = (ast as { children?: unknown[] }).children ?? [];
 				const nextBlocks: MarkdownBlock[] = [];
@@ -350,7 +368,7 @@
 		incompleteCodeBlock = null;
 
 		const normalized = preprocessLaTeX(markdown);
-		const processorInstance = getMarkdownProcessor({ attachments, disableMath });
+		const processorInstance = getMarkdownProcessor({ allowHtml, attachments, disableMath });
 		const ast = processorInstance.parse(normalized) as MdastRoot;
 		const mdastChildren = (ast as { children?: unknown[] }).children ?? [];
 		const stableCount = Math.max(mdastChildren.length - 1, 0);
@@ -394,6 +412,10 @@
 			)) as HastRoot;
 
 			unstableHtml = processorInstance.stringify(transformedRoot);
+
+			if (allowHtml) {
+				unstableHtml = DOMPurify.sanitize(unstableHtml, SAFE_HTML_CONFIG) as unknown as string;
+			}
 		}
 
 		renderedBlocks = nextBlocks;

tools/ui/src/lib/components/app/content/MarkdownContent/markdown-content.css

@@ -139,13 +139,13 @@
 /* Lists */
 .markdown-content :global(ul) {
 	list-style-type: disc;
-	margin-inline-start: 1.5rem;
+	margin-inline-start: 2rem;
 	margin-bottom: 1rem;
 }
 
 .markdown-content :global(ol) {
 	list-style-type: decimal;
-	margin-inline-start: 1.5rem;
+	margin-inline-start: 2rem;
 	margin-bottom: 1rem;
 }
 
@@ -500,6 +500,8 @@ div.markdown-user-content :global(.table-wrapper) {
 	margin: 1.5rem 0;
 	border-radius: 0.5rem;
 	border: 1px solid var(--border);
+	/* a wide table scrolls here, it does not widen the column */
+	max-width: 100%;
 }
 
 .markdown-content :global(.table-wrapper table) {
@@ -530,11 +532,9 @@ div.markdown-user-content :global(.table-wrapper) {
 		padding: 0.375rem 0.5rem;
 	}
 
+	/* no full-bleed: it hid the outer columns */
 	.markdown-content :global(.table-wrapper) {
-		margin: 0.5rem -1rem;
-		border-radius: 0;
-		border-left: none;
-		border-right: none;
+		margin: 1rem 0;
 	}
 }
 

tools/ui/src/lib/components/app/content/MarkdownContent/markdown-processor.ts

@@ -42,12 +42,15 @@ export interface MarkdownProcessor {
 export interface MarkdownProcessorOptions {
 	attachments?: DatabaseMessageExtra[];
 	disableMath?: boolean;
+	/** Render raw HTML found in the markdown instead of escaping it. */
+	allowHtml?: boolean;
 }
 
 const sharedPipelines = new Map<string, MarkdownProcessor>();
-const attachmentPipelines = new WeakMap<object, MarkdownProcessor>();
+const attachmentPipelines = new WeakMap<object, Map<string, MarkdownProcessor>>();
 
 function buildPipeline({
+	allowHtml = false,
 	attachments,
 	disableMath = false
 }: MarkdownProcessorOptions): MarkdownProcessor {
@@ -58,11 +61,14 @@ function buildPipeline({
 		proc = proc.use(remarkMath); // Parse $inline$ and $$block$$ math
 	}
 
-	proc = proc
-		.use(remarkBreaks) // Convert line breaks to <br>
+	proc = proc.use(remarkBreaks); // Convert line breaks to <br>
+
+	if (!allowHtml) {
 		// Treat raw HTML as literal text with preserved indentation
-		.use(remarkLiteralHtml)
-		.use(remarkRehype); // Convert Markdown AST to rehype
+		proc = proc.use(remarkLiteralHtml);
+	}
+
+	proc = proc.use(remarkRehype, allowHtml ? { allowDangerousHtml: true } : undefined);
 
 	if (!disableMath) {
 		proc = proc.use(rehypeKatex); // Render math using KaTeX
@@ -91,17 +97,26 @@ function buildPipeline({
 
 export function getMarkdownProcessor(options: MarkdownProcessorOptions): MarkdownProcessor {
 	if (options.attachments && options.attachments.length > 0) {
-		let cached = attachmentPipelines.get(options.attachments);
+		let byOptions = attachmentPipelines.get(options.attachments);
+
+		if (!byOptions) {
+			byOptions = new Map<string, MarkdownProcessor>();
+			attachmentPipelines.set(options.attachments, byOptions);
+		}
+
+		const key = `${Boolean(options.disableMath)}:${Boolean(options.allowHtml)}`;
+
+		let cached = byOptions.get(key);
 
 		if (!cached) {
 			cached = buildPipeline(options);
-			attachmentPipelines.set(options.attachments, cached);
+			byOptions.set(key, cached);
 		}
 
 		return cached;
 	}
 
-	const key = String(Boolean(options.disableMath));
+	const key = `${Boolean(options.disableMath)}:${Boolean(options.allowHtml)}`;
 
 	let cached = sharedPipelines.get(key);
 

tools/ui/src/lib/components/app/content/MarkdownContent/safe-html-config.ts

@@ -0,0 +1,138 @@
+// Sanitization rules for raw HTML in model cards (MarkdownContent's
+// allowHtml mode). Kept as its own module so the adversarial test in
+// tests/client can import the exact shipped configuration.
+import DOMPurify from 'dompurify';
+
+// style is allowed for the KaTeX output, which lays its spans out with inline
+// styles; a third party card can use the same attribute to draw a fixed
+// overlay over the app, so positioned values are scrubbed
+DOMPurify.addHook('afterSanitizeAttributes', (node) => {
+	if (typeof (node as Element).getAttribute !== 'function') return;
+
+	const style = (node as Element).getAttribute('style');
+
+	if (!style || !/position\s*:\s*(fixed|absolute)/i.test(style)) return;
+
+	(node as Element).setAttribute(
+		'style',
+		style.replace(/position\s*:\s*(fixed|absolute)\s*;?/gi, '')
+	);
+});
+
+/**
+ * Explicit DOMPurify allow list: the tags and attributes model cards may use
+ * plus the KaTeX output (spans with inline styles and MathML). Everything
+ * else - scripts, event handlers, forms, iframes, style tags - is stripped.
+ */
+export const SAFE_HTML_CONFIG = {
+	ALLOWED_ATTR: [
+		'class',
+		'style',
+		'id',
+		'title',
+		'role',
+		'aria-hidden',
+		'aria-label',
+		'aria-describedby',
+		'aria-expanded',
+		'href',
+		'target',
+		'rel',
+		'src',
+		'srcset',
+		'alt',
+		'width',
+		'height',
+		'align',
+		'valign',
+		'colspan',
+		'rowspan',
+		'controls',
+		'type',
+		// MathML
+		'xmlns',
+		'display',
+		'encoding',
+		'mathvariant'
+	],
+	ALLOWED_TAGS: [
+		// text and structure
+		'a',
+		'b',
+		'i',
+		'u',
+		's',
+		'em',
+		'strong',
+		'code',
+		'pre',
+		'kbd',
+		'sub',
+		'sup',
+		'br',
+		'hr',
+		'p',
+		'span',
+		'div',
+		'blockquote',
+		'center',
+		'ul',
+		'ol',
+		'li',
+		'dl',
+		'dt',
+		'dd',
+		'h1',
+		'h2',
+		'h3',
+		'h4',
+		'h5',
+		'h6',
+		'table',
+		'thead',
+		'tbody',
+		'tr',
+		'th',
+		'td',
+		'colgroup',
+		'col',
+		'caption',
+		// media
+		'img',
+		'picture',
+		'source',
+		'figure',
+		'figcaption',
+		'video',
+		'audio',
+		// MathML, for the KaTeX output
+		'math',
+		'mrow',
+		'mi',
+		'mo',
+		'mn',
+		'ms',
+		'mtext',
+		'mfrac',
+		'mroot',
+		'msqrt',
+		'msub',
+		'msup',
+		'msubsup',
+		'munder',
+		'mover',
+		'mmultiscripts',
+		'mtable',
+		'mtr',
+		'mtd',
+		'mspace',
+		'mglyph',
+		'maligngroup',
+		'malignmark',
+		'mpadded',
+		'mphantom',
+		'mstyle',
+		'semantics',
+		'annotation'
+	]
+};

tools/ui/src/lib/components/app/dialogs/DialogManageModels.svelte

@@ -1,7 +1,18 @@
 <script lang="ts">
+	import { ArrowLeft } from '@lucide/svelte';
+	import ModelsDiscover from '$lib/components/app/models/discover/ModelsDiscover.svelte';
 	import ModelsManager from '$lib/components/app/models/ModelsManager/ModelsManager.svelte';
+	import { Button } from '$lib/components/ui/button';
 	import * as Dialog from '$lib/components/ui/dialog';
-	import { MODEL_ICON } from '$lib/constants';
+	import {
+		DISCOVER_ICON,
+		MODEL_ICON,
+		MODELS_DIALOG_VIEW_FADE_MS,
+		SETTINGS_KEYS
+	} from '$lib/constants';
+	import { ModelsDialogView } from '$lib/enums';
+	import { modelsStore, settingsStore, uiStore } from '$lib/stores';
+	import { untrack } from 'svelte';
 
 	interface Props {
 		open?: boolean;
@@ -10,6 +21,85 @@
 
 	let { onOpenChange, open = $bindable(false) }: Props = $props();
 
+	// what the user asked for, and what is actually rendered: a view change fades the
+	// current one out, swaps, then fades the next one in
+	let view = $state<ModelsDialogView>(ModelsDialogView.MANAGE);
+	let shownView = $state<ModelsDialogView>(ModelsDialogView.MANAGE);
+	let isSwapping = $state(false);
+	/** Set while an entry opens the dialog on an explicit view, so the open reset keeps hands off. */
+	let viewRequested = false;
+
+	// the discover view exists only while its setting is on
+	let discoverEnabled = $derived(
+		Boolean(settingsStore.getConfig(SETTINGS_KEYS.ENABLE_DISCOVER_MODELS))
+	);
+
+	$effect(() => {
+		const next = view;
+
+		if (next === shownView) return;
+
+		// nothing to fade while the dialog is closed, so the view is set before it opens
+		if (!open) {
+			untrack(() => (shownView = next));
+
+			return;
+		}
+
+		untrack(() => (isSwapping = true));
+
+		const timer = setTimeout(() => {
+			untrack(() => {
+				shownView = next;
+				isSwapping = false;
+			});
+		}, MODELS_DIALOG_VIEW_FADE_MS);
+
+		return () => clearTimeout(timer);
+	});
+
+	let title = $derived(view === ModelsDialogView.DISCOVER ? 'Discover' : 'Models');
+
+	// the sidebar's Discover entry opens this dialog on its Discover view
+	$effect(() => {
+		if (!uiStore.discoverModelsOpen) return;
+
+		untrack(() => {
+			uiStore.discoverModelsOpen = false;
+			view = discoverEnabled ? ModelsDialogView.DISCOVER : ModelsDialogView.MANAGE;
+
+			// the dialog may already be open, so no open transition will follow
+			if (open) return;
+
+			viewRequested = true;
+			handleOpenChange(true);
+		});
+	});
+
+	// opening always lands on the manager, unless an entry requested the discover view
+	$effect(() => {
+		if (!open) return;
+
+		untrack(() => {
+			if (viewRequested) {
+				viewRequested = false;
+			} else {
+				view = ModelsDialogView.MANAGE;
+			}
+
+			// sync point: the table opens on cached state and corrects itself when
+			// the list fetch lands; a no-op outside router mode
+			void modelsStore.fetchRouterModels();
+		});
+	});
+
+	// the discover view disappears with its setting; fall back to the manager
+	$effect(() => {
+		if (!discoverEnabled && view === ModelsDialogView.DISCOVER) {
+			untrack(() => (view = ModelsDialogView.MANAGE));
+		}
+	});
+
 	function handleOpenChange(value: boolean) {
 		open = value;
 		onOpenChange?.(value);
@@ -21,11 +111,49 @@
 		class="max-md:h-[100dvh]! max-md:w-screen! max-md:max-w-none! max-md:rounded-none! max-md:border-0! md:h-[calc(100vh-4rem)]! md:max-h-240! md:w-[calc(100vw-4rem)]! md:max-w-380! flex flex-col p-4 pb-0"
 		onOpenAutoFocus={(event) => event.preventDefault()}
 	>
-		<Dialog.Header class="flex flex-row items-center pr-8">
-			<Dialog.Title class="flex min-h-7 items-center gap-2">
-				<MODEL_ICON class="h-5 w-5 shrink-0" />
+		<Dialog.Header class="relative flex flex-row items-center justify-between p-2 pr-8">
+			<!--
+				The arrow is out of the flow and the title slides, so entering a sub-view
+				moves the title with a transform instead of reflowing the header. min-h
+				keeps the header's height fixed either way.
+			-->
+			<Button
+				aria-label="Back to models"
+				class="absolute top-1/2 left-1 h-7 w-7 -translate-y-1/2 transition-[opacity,visibility] duration-200 ease-[cubic-bezier(0.23,1,0.32,1)] {view ===
+				ModelsDialogView.MANAGE
+					? 'invisible opacity-0'
+					: 'visible opacity-100'}"
+				onclick={() => (view = ModelsDialogView.MANAGE)}
+				size="icon"
+				variant="ghost"
+			>
+				<ArrowLeft class="h-4 w-4" />
+			</Button>
 
-				<span>Models</span>
+			<Dialog.Title
+				class="flex min-h-7 items-center gap-2 transition-transform duration-200 ease-[cubic-bezier(0.23,1,0.32,1)] {view ===
+				ModelsDialogView.MANAGE
+					? 'translate-x-0'
+					: 'translate-x-8'}"
+			>
+				<!-- the marks cross-fade in one grid cell, so the title never shifts -->
+				<span class="grid h-5 w-5 shrink-0 place-items-center">
+					<MODEL_ICON
+						class="col-start-1 row-start-1 h-5 w-5 transition-opacity duration-150 {view ===
+						ModelsDialogView.MANAGE
+							? 'opacity-100'
+							: 'opacity-0'}"
+					/>
+
+					<DISCOVER_ICON
+						class="col-start-1 row-start-1 h-5 w-5 transition-opacity duration-150 {view ===
+						ModelsDialogView.DISCOVER
+							? 'opacity-100'
+							: 'opacity-0'}"
+					/>
+				</span>
+
+				<span>{title}</span>
 			</Dialog.Title>
 
 			<Dialog.Description class="sr-only">
@@ -33,8 +161,47 @@
 			</Dialog.Description>
 		</Dialog.Header>
 
-		<div class="min-h-0 flex-1">
-			<ModelsManager class="h-full" />
+		<div class="dialog-view min-h-0 flex-1" data-visible={!isSwapping}>
+			{#if shownView === ModelsDialogView.MANAGE}
+				<ModelsManager class="h-full">
+					{#snippet toolbarEnd()}
+						{#if discoverEnabled}
+							<Button
+								class="gap-1.5"
+								onclick={() => (view = ModelsDialogView.DISCOVER)}
+								size="sm"
+								variant="tertiary"
+							>
+								<DISCOVER_ICON class="h-3.5 w-3.5" />
+
+								Discover Models
+							</Button>
+						{/if}
+					{/snippet}
+				</ModelsManager>
+			{:else if shownView === ModelsDialogView.DISCOVER}
+				<div class="grid h-full overflow-hidden" style="grid-template-columns: auto 1fr;">
+					<ModelsDiscover />
+				</div>
+			{/if}
 		</div>
 	</Dialog.Content>
 </Dialog.Root>
+
+<style>
+	/* a view change reads as one surface swapping, not as content teleporting */
+	.dialog-view {
+		opacity: 0;
+		transition: opacity 150ms cubic-bezier(0.23, 1, 0.32, 1);
+	}
+
+	.dialog-view[data-visible='true'] {
+		opacity: 1;
+	}
+
+	@media (prefers-reduced-motion: reduce) {
+		.dialog-view {
+			transition-duration: 100ms;
+		}
+	}
+</style>

tools/ui/src/lib/components/app/models/ModelDraftSidecars.svelte

@@ -31,8 +31,6 @@
 </script>
 
 {#each badges as badge (badge.kind + (badge.quant ?? ''))}
-	<span aria-hidden="true" class="text-muted-foreground">+</span>
-
 	<span
 		class={MODEL_VARIANT_BADGE_CLASS}
 		title="{badge.kind.toUpperCase()} draft model{badge.quant ? ` at ${badge.quant}` : ''}"

tools/ui/src/lib/components/app/models/ModelDraftSubline.svelte

@@ -0,0 +1,53 @@
+<script lang="ts">
+	import { Check } from '@lucide/svelte';
+	import { MODEL_BADGE_CLASS, MODEL_VARIANT_BADGE_CLASS } from '$lib/constants';
+	import { ModelRowDownloadState } from '$lib/enums';
+
+	interface Props {
+		/**
+		 * Left inset of the tree connector, tuned per surface so the elbow hangs
+		 * under the main line's avatar and "Draft model" aligns with its id.
+		 */
+		connectorClass?: string;
+		/** Draft sidecar kind, e.g. dflash. */
+		kind: string;
+		/** Quant of the draft, when its tag carries one. */
+		quant?: string | null;
+		/** Live state of the draft download. */
+		state: ModelRowDownloadState;
+		/** Rounded download percent, when the feed reports progress. */
+		percent?: number | null;
+	}
+
+	let { connectorClass = 'ml-6', kind, percent = null, quant = null, state }: Props = $props();
+</script>
+
+<!-- the compact draft line lives inside its main item, under the id line; the
+     main model's controls pause, resume and cancel its quant-matched drafts, so
+     the line only reports its state -->
+<div class="flex items-center gap-2 py-1 pr-2 text-sm">
+	<span
+		aria-hidden="true"
+		class="h-2.5 w-3.5 shrink-0 self-center rounded-bl-sm border-b border-l border-border/70 {connectorClass}"
+	></span>
+
+	<span class="shrink-0 text-xs text-muted-foreground">Draft model</span>
+
+	<span class={MODEL_VARIANT_BADGE_CLASS}>{kind}</span>
+
+	{#if quant}
+		<span class={MODEL_BADGE_CLASS}>{quant}</span>
+	{/if}
+
+	{#if state === ModelRowDownloadState.DOWNLOADED}
+		<Check class="ml-auto h-4 w-4 shrink-0 text-green-500" />
+	{:else}
+		<span class="ml-auto flex shrink-0 items-center">
+			{#if percent !== null}
+				<span class="font-mono text-xs tabular-nums text-muted-foreground">{percent}%</span>
+			{:else if state === ModelRowDownloadState.PAUSED}
+				<span class="text-xs text-muted-foreground">Paused</span>
+			{/if}
+		</span>
+	{/if}
+</div>

tools/ui/src/lib/components/app/models/ModelId.svelte

@@ -5,6 +5,7 @@
 	import { TruncatedText } from '$lib/components/app';
 	import * as Tooltip from '$lib/components/ui/tooltip';
 	import { MODEL_BADGE_CLASS, MODEL_VARIANT_BADGE_CLASS } from '$lib/constants';
+	import type { ModelDraftSidecar } from '$lib/enums';
 	import { HuggingFaceService } from '$lib/services';
 	import { ModelsService } from '$lib/services/models.service';
 	import { settingsStore } from '$lib/stores';
@@ -39,6 +40,8 @@
 		sizeRange?: { min: number; max: number } | null;
 		/** Draft sidecars available for the model, badged with their own quant. */
 		draftSidecars?: ModelSidecarBadge[];
+		/** Draft sidecar kinds the repo offers, one badge per kind and no quant. */
+		draftKinds?: ModelDraftSidecar[];
 		/** Give the id a row of its own, with its badges and icons together under it. */
 		stackId?: boolean;
 		/** Allow badges to wrap onto new lines instead of truncating. */
@@ -50,6 +53,7 @@
 		aliases,
 		class: className = '',
 		contextLength,
+		draftKinds = [],
 		draftSidecars = [],
 		hideCapabilities = false,
 		hideModalities = false,
@@ -93,12 +97,17 @@
 	let uniqueAliases = $derived([...new Set(aliases ?? [])]);
 	let uniqueTags = $derived([...new Set([...(parsed.tags ?? []), ...(tags ?? [])])]);
 	let uniqueDraftSidecars = $derived(draftSidecars.filter((badge) => !isAuxSidecar(badge.kind)));
+	// the model id can already name one of the kinds
+	let uniqueDraftKinds = $derived([
+		...new Set(draftKinds.filter((kind) => kind !== parsed.sidecar))
+	]);
 
 	let primaryAlias = $derived(uniqueAliases.length === 1 ? uniqueAliases[0] : null);
 	let displayName = $derived(primaryAlias ?? parsed.modelName ?? modelId);
 
 	let hasBadges = $derived(
 		parsed.sidecar ||
+			uniqueDraftKinds.length > 0 ||
 			uniqueDraftSidecars.length > 0 ||
 			(parsed.params && !hideParameters) ||
 			(parsed.quantization && !resolvedHideQuantization) ||
@@ -144,6 +153,12 @@
 					</span>
 				{/if}
 
+				{#each uniqueDraftKinds as kind (kind)}
+					<span class={variantBadgeClass} title="{kind.toUpperCase()} draft model">
+						{kind}
+					</span>
+				{/each}
+
 				<ModelDraftSidecars {draftSidecars} />
 
 				{#if primaryAlias}

tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte

@@ -32,9 +32,22 @@
 	let capabilityFilter = $state<ModelCapability[]>([]);
 	let selectedId = $state<string | null>(null);
 
+	// tracked downloads get their own section, listed like any other model
+	let downloads = $derived(
+		downloadGroups(modelsStore.status.getDownloadEntries(), modelsStore.models, (repo) =>
+			modelsStore.status.getDraftSidecars(repo)
+		)
+	);
+
 	let allModels = $derived(modelsStore.models);
 
-	let selected = $derived(allModels.find((option) => option.id === selectedId) ?? null);
+	// the download groups stand in for rows the listing does not carry yet, so a
+	// click on them still finds its option and opens the pane
+	let selected = $derived(
+		allModels.find((option) => option.id === selectedId) ??
+			downloads.find((group) => group.base.id === selectedId)?.base ??
+			null
+	);
 	// The pane is laid out before it is ever opened, so the first open only slides a
 	// finished panel in. It renders the selection, else the model it last showed.
 	let lastPicked = $state<ModelOption | null>(null);
@@ -42,7 +55,11 @@
 	let shownId = $state<string | null>(null);
 	let isSwapping = $state(false);
 	let fade = $state<'open' | 'swap'>('open');
-	let shownOption = $derived(allModels.find((option) => option.id === shownId) ?? null);
+	let shownOption = $derived(
+		allModels.find((option) => option.id === shownId) ??
+			downloads.find((group) => group.base.id === shownId)?.base ??
+			null
+	);
 
 	$effect(() => {
 		const id = selectedId;
@@ -144,10 +161,6 @@
 		rank.size === 0 ? list : [...list].sort((a, b) => rankOf(a) - rankOf(b));
 	// one entry per repo, so a model with several quants takes a single table row
 	let entries = $derived(byRecency(groupModelQuants(matching)));
-	// tracked downloads get their own section, listed like any other model
-	let downloads = $derived(
-		downloadGroups(modelsStore.status.getDownloadEntries(), modelsStore.models)
-	);
 	let groups = $derived.by(() => {
 		// a loaded quant is a model of its own: its repo keeps the quants left behind
 		const isLoaded = (option: ModelOption) => modelsStore.isModelLoaded(option.model);
@@ -216,13 +229,16 @@
 		return ordered;
 	});
 
-	// a caller can ask for one model to be revealed, the download rows do
+	// a caller can ask for one model to be revealed, the download rows do; a
+	// tracked download is not in the listing yet, so its own rows are searched too
 	$effect(() => {
 		const focus = uiStore.manageModelFocus;
 
 		if (!focus) return;
 
-		const option = allModels.find((model) => model.id === focus || model.model === focus);
+		const matches = (candidate: ModelOption) => candidate.id === focus || candidate.model === focus;
+		const option =
+			allModels.find(matches) ?? downloads.find((group) => matches(group.base))?.base ?? null;
 
 		if (option) selectedId = option.id;
 

tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerDownloadControl.svelte

@@ -38,10 +38,7 @@
 			class="h-5 w-5 hover:text-foreground"
 			icon={isPaused ? MODEL_DOWNLOAD_ICONS.resume : MODEL_DOWNLOAD_ICONS.pause}
 			iconSize="h-4 w-4"
-			onclick={() =>
-				void (isPaused
-					? modelsStore.status.downloadModel(option.model)
-					: modelsStore.status.pauseDownload(option.model))}
+			onclick={() => void modelsStore.status.toggleDownload(option.model)}
 			stopPropagationOnClick
 			tooltip={isPaused ? 'Resume downloading' : 'Pause downloading'}
 			tooltipAsTitle

tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte

@@ -1,4 +1,6 @@
 <script lang="ts">
+	import ModelDownloadProgressBar from '../../ModelDownloadProgressBar.svelte';
+	import { downloadProgressFor } from '../utils';
 	import ModelsManagerModelConfigurationActions from './ModelsManagerModelConfigurationActions.svelte';
 	import ModelsManagerModelConfigurationHeader from './ModelsManagerModelConfigurationHeader.svelte';
 	import ModelsManagerModelConfigurationInformation from './ModelsManagerModelConfigurationInformation.svelte';
@@ -7,7 +9,7 @@
 	import { Button } from '$lib/components/ui/button';
 	import { ModelDownloadConfirmAction, ServerModelStatus } from '$lib/enums';
 	import { HuggingFaceService } from '$lib/services';
-	import { deviceStore, modelsStore } from '$lib/stores';
+	import { deviceStore, modelsStore, uiStore } from '$lib/stores';
 	import type { HfModelDetailInfo } from '$lib/types/huggingface';
 	import type { ModelOption } from '$lib/types/models';
 	import { repoOf } from '$lib/utils';
@@ -34,6 +36,23 @@
 	let status = $derived(modelsStore.getModelStatus(option.model));
 	let isLoaded = $derived(modelsStore.isModelRunning(option.model));
 
+	// an in-flight download has no configuration: the pane shows its progress and
+	// the download actions, everything a loaded or downloaded model offers waits
+	let isDownload = $derived(
+		modelsStore.status.isDownloadInProgress(option.model) ||
+			modelsStore.status.isDownloadPaused(option.model)
+	);
+	let isDownloadPaused = $derived(modelsStore.status.isDownloadPaused(option.model));
+	let downloadProgress = $derived(downloadProgressFor(option.model));
+	let downloadPercent = $derived(
+		downloadProgress && downloadProgress.totalBytes > 0
+			? Math.round((downloadProgress.downloadedBytes / downloadProgress.totalBytes) * 100)
+			: null
+	);
+
+	// the same cancellation the table's rows confirm, owned by this pane
+	let cancelDownloadOpen = $state(false);
+
 	// The server reports the full metadata only once a model is loaded, which loads it.
 	// With the Hub enabled, it fills those gaps instead.
 	let hubDetails = $state<HfModelDetailInfo | null>(null);
@@ -63,8 +82,15 @@
 
 <div class="flex h-full min-h-0 flex-col">
 	<ModelsManagerModelConfigurationHeader
+		{isDownload}
+		{isDownloadPaused}
 		{isLoaded}
 		{onClose}
+		onRequestCancelDownload={() => (cancelDownloadOpen = true)}
+		onSeeInDiscover={isDownload
+			? () => uiStore.openModelsDiscover(repoOf(option.model))
+			: undefined}
+		onToggleDownload={() => void modelsStore.status.toggleDownload(option.model)}
 		{onToggleLoad}
 		{onUseInChat}
 		{onUseInNewChat}
@@ -73,27 +99,63 @@
 	/>
 
 	<div class="min-h-0 flex-1 overflow-y-auto py-4 pl-4 max-md:pr-4">
-		<ModelsManagerModelConfigurationInformation hub={hubDetails} {option} {serverProps} />
+		{#if isDownload}
+			<!-- nothing is on disk yet, so the pane keeps to the download itself -->
+			<div class="space-y-2 py-1">
+				<div class="flex items-baseline justify-between gap-3 text-sm">
+					<span class="text-muted-foreground">
+						{isDownloadPaused ? 'Paused at' : 'Downloaded'}
+					</span>
 
-		<div class="mt-4 border-t border-border/30 pt-3 pb-2">
-			<Button
-				class="w-full justify-start gap-2 text-destructive hover:bg-destructive/10 hover:text-destructive dark:hover:bg-destructive/20"
-				onclick={() => (deleteOpen = true)}
-				size="sm"
-				variant="ghost"
-			>
-				<Trash2 class="h-4 w-4" />
-				Delete this model from disk
-			</Button>
-		</div>
+					<span class="font-mono text-xs tabular-nums text-muted-foreground">
+						{#if downloadPercent !== null}
+							{HuggingFaceService.formatFileSize(downloadProgress?.downloadedBytes ?? 0)}
+							/
+							{HuggingFaceService.formatFileSize(downloadProgress?.totalBytes ?? 0)}
+							({downloadPercent}%)
+						{:else}
+							Waiting for progress...
+						{/if}
+					</span>
+				</div>
+
+				{#if downloadProgress && downloadProgress.totalBytes > 0}
+					<ModelDownloadProgressBar
+						downloadedBytes={downloadProgress.downloadedBytes}
+						totalBytes={downloadProgress.totalBytes}
+					/>
+				{/if}
+			</div>
+		{:else}
+			<ModelsManagerModelConfigurationInformation hub={hubDetails} {option} {serverProps} />
+
+			<div class="mt-4 border-t border-border/30 pt-3 pb-2">
+				<Button
+					class="w-full justify-start gap-2 text-destructive hover:bg-destructive/10 hover:text-destructive dark:hover:bg-destructive/20"
+					onclick={() => (deleteOpen = true)}
+					size="sm"
+					variant="ghost"
+				>
+					<Trash2 class="h-4 w-4" />
+					Delete this model from disk
+				</Button>
+			</div>
+		{/if}
 	</div>
 
 	<!-- a phone keeps the actions at the bottom of its screen, in reach of the thumb -->
 	{#if deviceStore.isMobile}
 		<div class="shrink-0 border-t border-border/40 px-4 py-3">
 			<ModelsManagerModelConfigurationActions
+				{isDownload}
+				{isDownloadPaused}
 				{isLoaded}
 				isLoading={status === ServerModelStatus.LOADING}
+				onRequestCancelDownload={() => (cancelDownloadOpen = true)}
+				onSeeInDiscover={isDownload
+					? () => uiStore.openModelsDiscover(repoOf(option.model))
+					: undefined}
+				onToggleDownload={() => void modelsStore.status.toggleDownload(option.model)}
 				{onToggleLoad}
 				{onUseInChat}
 				{onUseInNewChat}
@@ -107,4 +169,12 @@
 		open={deleteOpen}
 		repoWithTag={option.model}
 	/>
+
+	<DialogConfirmDownload
+		action={ModelDownloadConfirmAction.CANCEL}
+		onClose={() => (cancelDownloadOpen = false)}
+		onConfirm={() => void modelsStore.status.cancelDownload(option.model)}
+		open={cancelDownloadOpen}
+		repoWithTag={option.model}
+	/>
 </div>

tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte

@@ -1,13 +1,31 @@
 <script lang="ts">
-	import { Eject, Loader2, MessageSquare, Power, SquarePen } from '@lucide/svelte';
+	import {
+		Compass,
+		Eject,
+		Loader2,
+		MessageSquare,
+		Pause,
+		Play,
+		Power,
+		SquarePen,
+		Trash2
+	} from '@lucide/svelte';
 	import { Button } from '$lib/components/ui/button';
-	import { conversationsStore, deviceStore } from '$lib/stores';
+	import { SETTINGS_KEYS } from '$lib/constants';
+	import { conversationsStore, deviceStore, settingsStore } from '$lib/stores';
 
 	interface Props {
 		/** A provider that cannot load or unload has no button to offer. */
 		canToggleLoad?: boolean;
+		isDownload?: boolean;
+		isDownloadPaused?: boolean;
 		isLoaded: boolean;
 		isLoading: boolean;
+		/** Opens the manager dialog on the Discover view, focused on this model's repo. */
+		onSeeInDiscover?: () => void;
+		/** Confirm-cancel dialog on the pane, so nothing deletes without a click. */
+		onRequestCancelDownload?: () => void;
+		onToggleDownload?: () => void;
 		onToggleLoad: () => void;
 		onUseInChat: () => void;
 		onUseInNewChat: () => void;
@@ -17,14 +35,24 @@
 
 	let {
 		canToggleLoad = true,
+		isDownload = false,
+		isDownloadPaused = false,
 		isLoaded,
 		isLoading,
+		onRequestCancelDownload,
+		onSeeInDiscover,
+		onToggleDownload,
 		onToggleLoad,
 		onUseInChat,
 		onUseInNewChat,
 		size = 'sm'
 	}: Props = $props();
 
+	// the discover view exists only while its setting is on
+	let discoverEnabled = $derived(
+		Boolean(settingsStore.getConfig(SETTINGS_KEYS.ENABLE_DISCOVER_MODELS))
+	);
+
 	// a chat with anything in it takes the model, a new chat is not what its user asked for
 	let hasChat = $derived(
 		conversationsStore.activeConversation !== null ||
@@ -34,52 +62,91 @@
 </script>
 
 <div class="my-3 flex flex-col gap-2">
-	{#if isMobile}
-		<!-- a phone picks its model in the manager, so pointing this chat at it leads -->
-		<Button class="w-full gap-1.5" onclick={hasChat ? onUseInChat : onUseInNewChat} {size}>
-			{#if hasChat}
-				<MessageSquare class="h-3.5 w-3.5" />
+	{#if isDownload}
+		<!-- an in-flight download cannot load or chat: it pauses, resumes, cancels -->
+		<div class="flex gap-3">
+			<Button class="flex-1 gap-1.5" onclick={onToggleDownload} {size} variant="secondary">
+				{#if isDownloadPaused}
+					<Play class="h-3.5 w-3.5" />
 
-				Use in this chat
-			{:else}
-				<SquarePen class="h-3.5 w-3.5" />
+					Resume downloading
+				{:else}
+					<Pause class="h-3.5 w-3.5" />
 
-				Start a new chat
-			{/if}
-		</Button>
-	{/if}
+					Pause downloading
+				{/if}
+			</Button>
 
-	<div class="flex gap-3 mt-2">
-		{#if !isMobile}
-			<Button class="flex-1 gap-1.5" onclick={onUseInNewChat} {size}>
-				<SquarePen class="h-3.5 w-3.5" />
+			<Button class="flex-1 gap-1.5" onclick={onRequestCancelDownload} {size} variant="secondary">
+				<Trash2 class="h-3.5 w-3.5" />
 
-				Start a new chat
+				Cancel download
+			</Button>
+		</div>
+
+		{#if onSeeInDiscover && discoverEnabled}
+			<Button class="gap-1.5" onclick={onSeeInDiscover} {size} variant="tertiary">
+				<Compass class="h-3.5 w-3.5" />
+
+				See in Discover
 			</Button>
 		{/if}
+	{:else}
+		{#if isMobile}
+			<!-- a phone picks its model in the manager, so pointing this chat at it leads -->
+			<Button class="w-full gap-1.5" onclick={hasChat ? onUseInChat : onUseInNewChat} {size}>
+				{#if hasChat}
+					<MessageSquare class="h-3.5 w-3.5" />
 
-		{#if canToggleLoad}
-			<Button
-				class="flex-1 gap-1.5"
-				disabled={isLoading}
-				onclick={onToggleLoad}
-				{size}
-				variant="secondary"
-			>
-				{#if isLoading}
-					<Loader2 class="h-3.5 w-3.5 animate-spin" />
-
-					Loading...
-				{:else if isLoaded}
-					<Eject class="h-3.5 w-3.5" />
-
-					Unload model
+					Use in this chat
 				{:else}
-					<Power class="h-3.5 w-3.5" />
+					<SquarePen class="h-3.5 w-3.5" />
 
-					Load model
+					Start a new chat
 				{/if}
 			</Button>
 		{/if}
-	</div>
+
+		<div class="flex gap-3 mt-2">
+			{#if !isMobile}
+				<Button class="flex-1 gap-1.5" onclick={onUseInNewChat} {size}>
+					<SquarePen class="h-3.5 w-3.5" />
+
+					Start a new chat
+				</Button>
+			{/if}
+
+			{#if canToggleLoad}
+				<Button
+					class="flex-1 gap-1.5"
+					disabled={isLoading}
+					onclick={onToggleLoad}
+					{size}
+					variant="secondary"
+				>
+					{#if isLoading}
+						<Loader2 class="h-3.5 w-3.5 animate-spin" />
+
+						Loading...
+					{:else if isLoaded}
+						<Eject class="h-3.5 w-3.5" />
+
+						Unload model
+					{:else}
+						<Power class="h-3.5 w-3.5" />
+
+						Load model
+					{/if}
+				</Button>
+			{/if}
+
+			{#if onSeeInDiscover && discoverEnabled}
+				<Button class="flex-1 gap-1.5" onclick={onSeeInDiscover} {size} variant="tertiary">
+					<Compass class="h-3.5 w-3.5" />
+
+					See in Discover
+				</Button>
+			{/if}
+		</div>
+	{/if}
 </div>

tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationHeader.svelte

@@ -8,8 +8,14 @@
 	import type { ModelOption } from '$lib/types/models';
 
 	interface Props {
+		isDownload?: boolean;
+		isDownloadPaused?: boolean;
 		isLoaded: boolean;
 		onClose: () => void;
+		/** Opens the manager dialog on the Discover view, focused on this model's repo. */
+		onSeeInDiscover?: () => void;
+		onRequestCancelDownload?: () => void;
+		onToggleDownload?: () => void;
 		onToggleLoad: () => void;
 		onUseInChat: () => void;
 		onUseInNewChat: () => void;
@@ -18,13 +24,27 @@
 		status: ServerModelStatus | null;
 	}
 
-	let { isLoaded, onClose, onToggleLoad, onUseInChat, onUseInNewChat, option, status }: Props =
-		$props();
+	let {
+		isDownload = false,
+		isDownloadPaused = false,
+		isLoaded,
+		onClose,
+		onRequestCancelDownload,
+		onSeeInDiscover,
+		onToggleDownload,
+		onToggleLoad,
+		onUseInChat,
+		onUseInNewChat,
+		option,
+		status
+	}: Props = $props();
 
 	let isMobile = $derived(deviceStore.isMobile);
 	let isLoading = $derived(status === ServerModelStatus.LOADING);
 
 	let statusLabel = $derived.by(() => {
+		if (isDownload) return isDownloadPaused ? 'Paused' : 'Downloading';
+
 		if (status === ServerModelStatus.LOADING) return 'Loading';
 
 		if (status === ServerModelStatus.FAILED) return 'Failed to load';
@@ -37,6 +57,8 @@
 	});
 
 	let statusDot = $derived.by(() => {
+		if (isDownload) return isDownloadPaused ? 'bg-yellow-500' : 'bg-blue-500 animate-pulse';
+
 		if (status === ServerModelStatus.FAILED) return 'bg-red-500';
 
 		if (status === ServerModelStatus.LOADING) return 'bg-muted-foreground/50 animate-pulse';
@@ -100,8 +122,13 @@
 	<!-- a phone floats the actions at the bottom of the pane, where the thumb is -->
 	{#if !isMobile}
 		<ModelsManagerModelConfigurationActions
+			{isDownload}
+			{isDownloadPaused}
 			{isLoaded}
 			{isLoading}
+			{onRequestCancelDownload}
+			{onSeeInDiscover}
+			{onToggleDownload}
 			{onToggleLoad}
 			{onUseInChat}
 			{onUseInNewChat}

tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte

@@ -3,6 +3,7 @@
 	import ModelCapabilities from '../ModelCapabilities.svelte';
 	import ModelContext from '../ModelContext.svelte';
 	import ModelDownloadProgressBar from '../ModelDownloadProgressBar.svelte';
+	import ModelDraftSubline from '../ModelDraftSubline.svelte';
 	import ModelId from '../ModelId.svelte';
 	import ModelsManagerStatusCell from './ModelsManagerStatusCell.svelte';
 	import { modelRowActions } from './row-actions';
@@ -12,7 +13,7 @@
 	import { MODEL_ROW_GRID_CLASS, MODEL_ROW_TRAILING_CELL_CLASS } from '$lib/constants';
 	import { ModelRowDownloadState } from '$lib/enums';
 	import { deviceStore, modelsStore } from '$lib/stores';
-	import type { ModelOption } from '$lib/types/models';
+	import type { ModelOption, ModelSidecarBadge } from '$lib/types/models';
 
 	interface Props {
 		isFavorite: (option: ModelOption) => boolean;
@@ -20,11 +21,21 @@
 		onDelete: (option: ModelOption) => void;
 		onSelect: (option: ModelOption) => void;
 		selected: boolean;
+		/** Draft sidecars already on disk, when the row stands for a download. */
+		sidecarBadges?: ModelSidecarBadge[];
 		/** Left padding in px, from the nesting depth. */
 		indent?: number;
 	}
 
-	let { indent = 0, isFavorite, onDelete, onSelect, option, selected }: Props = $props();
+	let {
+		indent = 0,
+		isFavorite,
+		onDelete,
+		onSelect,
+		option,
+		selected,
+		sidecarBadges
+	}: Props = $props();
 
 	let favorite = $derived(isFavorite(option));
 	let isHidden = $derived(modelsStore.isHidden(option.id));
@@ -38,6 +49,10 @@
 				? ModelRowDownloadState.DOWNLOADING
 				: null
 	);
+
+	function activate(): void {
+		onSelect(option);
+	}
 </script>
 
 <div
@@ -48,41 +63,58 @@
 		selected ? 'bg-accent text-accent-foreground' : 'hover:bg-muted/40'
 	]}
 >
-	<!-- the row holds a load control and an actions menu, so only the model itself is
+	<div class="flex min-w-0 flex-col">
+		<!-- the row holds a load control and an actions menu, so only the model itself is
 	     the button: a button nested in a role="button" row is invalid -->
-	<button
-		aria-pressed={selected}
-		class="flex min-w-0 cursor-pointer items-center gap-3 text-left max-md:gap-2 max-md:pr-9"
-		onclick={() => onSelect(option)}
-		style="padding-left: {indent}px"
-		type="button"
-	>
-		<ModelAvatar {option} size="size-7 md:size-9" />
+		<button
+			aria-pressed={download ? undefined : selected}
+			class="flex min-w-0 cursor-pointer items-center gap-3 text-left max-md:gap-2 max-md:pr-9"
+			onclick={activate}
+			style="padding-left: {indent}px"
+			type="button"
+		>
+			<ModelAvatar {option} size="size-7 md:size-9" />
 
-		<span class="flex min-w-0 items-center gap-1.25">
-			<ModelId
-				aliases={option.aliases}
-				class="min-w-0 flex-1 max-md:text-sm"
-				draftSidecars={option.draftSidecars}
-				hideCapabilities
-				hideModalities
-				modalities={option.modalities}
-				modelId={option.model}
-				tags={option.tags}
-				title={option.model}
-			/>
+			<span class="flex min-w-0 items-center gap-1.25">
+				<ModelId
+					aliases={option.aliases}
+					class="min-w-0 flex-1 max-md:text-sm"
+					draftSidecars={option.draftSidecars}
+					hideCapabilities
+					hideModalities
+					modalities={option.modalities}
+					modelId={option.model}
+					tags={option.tags}
+					title={option.model}
+				/>
 
-			<!-- a phone has no width for the modality icons, the id needs it more -->
-			<ModelCapabilities hideModalities={deviceStore.isMobile} {option} />
+				<!-- a phone has no width for the modality icons, the id needs it more -->
+				<ModelCapabilities hideModalities={deviceStore.isMobile} {option} />
 
-			{#if favorite}
-				<!-- the heart is decorative, the button's text carries the state -->
-				<Heart aria-hidden="true" class="h-3.5 w-3.5 shrink-0 text-rose-500" />
+				{#if favorite}
+					<!-- the heart is decorative, the button's text carries the state -->
+					<Heart aria-hidden="true" class="h-3.5 w-3.5 shrink-0 text-rose-500" />
 
-				<span class="sr-only">favorited</span>
-			{/if}
-		</span>
-	</button>
+					<span class="sr-only">favorited</span>
+				{/if}
+			</span>
+		</button>
+
+		{#if sidecarBadges?.length}
+			<!-- w-fit keeps the trailing state beside the badges instead of floating
+			     to the end of the wide model column -->
+			<div class="flex w-fit flex-col pb-1">
+				{#each sidecarBadges as badge (badge.repo + ':' + badge.kind + ':' + (badge.quant ?? ''))}
+					<ModelDraftSubline
+						connectorClass="ml-8"
+						kind={badge.kind}
+						quant={badge.quant}
+						state={ModelRowDownloadState.DOWNLOADED}
+					/>
+				{/each}
+			</div>
+		{/if}
+	</div>
 
 	<ModelContext
 		class="justify-self-end max-md:hidden"

tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte

@@ -88,6 +88,7 @@
 		[GroupedListUnit.GROUPS]: 'families'
 	};
 	let pendingDelete = $state('');
+	let pendingAction = $state(ModelDownloadConfirmAction.DELETE);
 	let deleteOpen = $state(false);
 	/** Repos whose quants are folded away; the rest show them. */
 	const collapsedQuants = new SvelteSet<string>();
@@ -129,6 +130,10 @@
 	// cancel is confirmed once for the whole list, so one dialog serves every row
 	function requestDelete(option: ModelOption): void {
 		pendingDelete = option.model;
+		// an in-flight row cancels (and drops the partials), a paused one deletes
+		pendingAction = modelsStore.status.isDownloadInProgress(option.model)
+			? ModelDownloadConfirmAction.CANCEL
+			: ModelDownloadConfirmAction.DELETE;
 		deleteOpen = true;
 	}
 
@@ -248,6 +253,7 @@
 			{onSelect}
 			option={entry.base}
 			selected={selectedId === entry.base.id}
+			sidecarBadges={entry.sidecarBadges}
 		/>
 	{/if}
 {/snippet}
@@ -328,7 +334,7 @@
 {/snippet}
 
 <DialogConfirmDownload
-	action={ModelDownloadConfirmAction.DELETE}
+	action={pendingAction}
 	onClose={() => (deleteOpen = false)}
 	open={deleteOpen}
 	repoWithTag={pendingDelete}

tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts

@@ -1,10 +1,10 @@
-import { Eye, EyeOff, Heart, HeartOff, Trash2 } from '@lucide/svelte';
+import { Compass, Eye, EyeOff, Heart, HeartOff, Trash2 } from '@lucide/svelte';
 import { MODEL_DOWNLOAD_ICONS } from '$lib/constants';
 import { ModelRowDownloadState } from '$lib/enums';
-import { modelsStore } from '$lib/stores';
+import { modelsStore, uiStore } from '$lib/stores';
 import type { ModelOption } from '$lib/types/models';
+import { repoOf } from '$lib/utils';
 
-/** Row actions follow the app's dropdown pattern: icon, label, separators, variants. */
 export function modelRowActions(
 	option: ModelOption,
 	favorite: boolean,
@@ -13,9 +13,17 @@ export function modelRowActions(
 	/** Download state, when the row stands for a tracked download. */
 	download?: ModelRowDownloadState | null
 ) {
+	const viewInDiscover = {
+		icon: Compass,
+		label: 'See in Discover',
+		// the details pane is keyed by repo, not by `<repo>:<quant>`
+		onclick: () => uiStore.openModelsDiscover(repoOf(option.model)),
+		separator: true
+	};
+
 	// a tracked download is paused, resumed or dropped: it is not loaded, hidden or
-	// drafted. Deleting it stops the download and removes what is on disk, which is
-	// what the same wording offers on a cached model
+	// drafted. An in-flight row cancels, which stops the download and removes the
+	// partial files; a paused one deletes, which removes the partials from disk
 	if (download) {
 		const isPaused = download === ModelRowDownloadState.PAUSED;
 
@@ -23,18 +31,16 @@ export function modelRowActions(
 			{
 				icon: isPaused ? MODEL_DOWNLOAD_ICONS.resume : MODEL_DOWNLOAD_ICONS.pause,
 				label: isPaused ? 'Resume downloading' : 'Pause downloading',
-				onclick: () =>
-					void (isPaused
-						? modelsStore.status.downloadModel(option.model)
-						: modelsStore.status.pauseDownload(option.model))
+				onclick: () => void modelsStore.status.toggleDownload(option.model)
 			},
 			{
 				icon: Trash2,
-				label: 'Delete from disk',
+				label: isPaused ? 'Delete from disk' : 'Cancel download',
 				onclick: () => onDelete(option),
 				separator: true,
 				variant: 'destructive' as const
-			}
+			},
+			viewInDiscover
 		];
 	}
 
@@ -56,6 +62,7 @@ export function modelRowActions(
 			label: isHidden ? 'Unhide model' : 'Hide model',
 			onclick: () => modelsStore.toggleHidden(option.id),
 			separator: true
-		}
+		},
+		viewInDiscover
 	];
 }

tools/ui/src/lib/components/app/models/ModelsManager/utils.ts

@@ -2,7 +2,12 @@ import { LOCAL_BACKEND_ID, type ModalityKey } from '$lib/constants';
 import { ModelCapability, ModelGroupKind, ModelsTableGroupKind } from '$lib/enums';
 import { HuggingFaceService, ModelsService } from '$lib/services';
 import { modelsStore } from '$lib/stores';
-import type { ModelDownloadEntry, ModelDownloadProgress, ModelOption } from '$lib/types/models';
+import type {
+	ModelDownloadEntry,
+	ModelDownloadProgress,
+	ModelOption,
+	ModelSidecarBadge
+} from '$lib/types/models';
 import { detectThinkingSupport, detectToolUseSupport, repoOf } from '$lib/utils';
 
 /** One repo of the table, with the rows it ships as. */
@@ -11,6 +16,8 @@ export interface ModelQuantGroup {
 	key: string;
 	kind: ModelGroupKind;
 	quants: ModelOption[];
+	/** Draft sidecars already on disk, shown as checked sub-lines. */
+	sidecarBadges?: ModelSidecarBadge[];
 }
 
 /** True when any of the manager's filters narrows the table. */
@@ -31,28 +38,36 @@ export function downloadProgressFor(repoWithTag: string): ModelDownloadProgress
 }
 
 /**
- * One table row per tracked download, so the table lists them the way it lists
- * any other model. A paused download is already in the router's listing, so its
- * own option is reused; a fresh one stands for its tag alone.
+ * One table row per tracked download, listed like any other model. A paused download
+ * is already in the router's listing, so its own option is reused; a fresh one is
+ * synthesized for its tag alone.
  */
 export function downloadGroups(
 	entries: ModelDownloadEntry[],
-	models: ModelOption[]
+	models: ModelOption[],
+	sidecarsFor: (repo: string) => ModelSidecarBadge[]
 ): ModelQuantGroup[] {
 	return entries.map((entry) => {
-		const option = models.find((candidate) => candidate.model === entry.repoWithTag) ?? {
+		const listed = models.find((candidate) => candidate.model === entry.repoWithTag);
+		// the synthesized option carries backendId, which ModelOption drops on
+		// assignment, so the callers spread it instead of typing this as one
+		const option = listed ?? {
 			backendId: LOCAL_BACKEND_ID,
 			capabilities: [],
 			id: entry.repoWithTag,
 			model: entry.repoWithTag,
 			name: entry.repoWithTag
 		};
+		// a draft still coming down badges the id, an on-disk one keeps its sub-line
+		const sidecars = sidecarsFor(repoOf(entry.repoWithTag));
+		const pending = sidecars.filter((badge) => badge.pending);
 
 		return {
-			base: option,
+			base: { ...option, draftSidecars: pending },
 			key: `download::${entry.repoWithTag}`,
 			kind: ModelGroupKind.QUANTS,
-			quants: [option]
+			quants: [{ ...option, draftSidecars: pending }],
+			sidecarBadges: sidecars.filter((badge) => !badge.pending)
 		};
 	});
 }

tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte

@@ -1,22 +1,38 @@
 <script lang="ts">
 	import ModelDownloadProgressBar from '../ModelDownloadProgressBar.svelte';
+	import ModelDraftSubline from '../ModelDraftSubline.svelte';
 	import ModelOrgAvatar from '../ModelOrgAvatar.svelte';
-	import { Loader2, Pause, Play, X } from '@lucide/svelte';
+	import { Info, Loader2, Pause, Play, X } from '@lucide/svelte';
 	import { ActionIcon, ModelId } from '$lib/components/app';
+	import { ModelRowDownloadState } from '$lib/enums';
 	import { HuggingFaceService, ModelsService } from '$lib/services';
-	import { modelsStore } from '$lib/stores';
-	import type { ModelDownloadEntry } from '$lib/types';
+	import { deviceStore, modelsStore, uiStore } from '$lib/stores';
+	import type { ModelDownloadEntry, ModelOption, ModelSidecarBadge } from '$lib/types';
 
 	interface Props {
 		/** One entry from the status feed: an in-flight or paused download. */
 		entry: ModelDownloadEntry;
+		/** Draft sidecars already on disk, rendered as checked sub-lines. */
+		sidecars?: ModelSidecarBadge[];
 		/** Ask the list to confirm cancelling; the list owns the single dialog. */
 		onRequestCancel?: (repoWithTag: string) => void;
 		/** Show the organization name in the repo id, as the other selector rows do. */
 		showOrgName?: boolean;
 	}
 
-	let { entry, onRequestCancel, showOrgName = true }: Props = $props();
+	let { entry, onRequestCancel, showOrgName = true, sidecars = [] }: Props = $props();
+
+	/** A draft still coming down badges the id, an on-disk one keeps its sub-line. */
+	let pendingDrafts = $derived(sidecars.filter((badge) => badge.pending));
+	let onDiskDrafts = $derived(sidecars.filter((badge) => !badge.pending));
+
+	/** The download stands for its model, so the row's info action can open on it. */
+	let option = $derived<ModelOption>({
+		capabilities: [],
+		id: entry.repoWithTag,
+		model: entry.repoWithTag,
+		name: modelsStore.toDisplayName(entry.repoWithTag)
+	});
 
 	let percent = $derived(
 		entry.progress && entry.progress.totalBytes > 0
@@ -53,48 +69,60 @@
 	});
 </script>
 
-<!-- One in-flight download; same actions as the discover quant chips: the row
-     itself pauses / resumes, the trailing X cancels (stops and discards the
-     partial files). Both affordances fade in on hover, the slots are reserved
-     so the list never reflows. -->
-<div
-	class="group relative flex items-center gap-2 rounded-sm p-2 text-left text-sm transition-colors hover:bg-accent/50"
->
-	<button
-		aria-label={actionText}
-		class="flex min-w-0 flex-1 cursor-pointer items-center gap-2 text-left"
-		onclick={() => {
-			if (entry.isPaused) {
-				void modelsStore.status
-					.downloadModel(entry.repoWithTag)
-					.catch((error) => console.warn('Failed to resume the download:', error));
-			} else {
-				void modelsStore.status.pauseDownload(entry.repoWithTag);
-			}
-		}}
-		type="button"
+<!-- One in-flight download: the row opens the model, the status slot beside it
+     pauses / resumes and the trailing X cancels (stops and discards the partial
+     files). The status slot is always there, the affordances fade in on hover,
+     so the list never reflows. The repo's on-disk sidecars hang off the item as
+     compact sub-lines instead of rows of their own. -->
+<div class="group relative rounded-sm transition-colors hover:bg-accent/50">
+	<div
+		class="flex items-center gap-2 p-2 text-left text-sm max-md:gap-2.5 max-md:px-2.5 max-md:py-2.5"
 	>
-		{#if orgName}
-			<ModelOrgAvatar
-				org={fetchedBaseModelOrg ?? orgName}
-				quantOrg={orgName}
-				quantPositionClass="-bottom-1 -right-1"
-				quantSize="h-3 w-3"
-				size="h-6 w-6"
+		<button
+			aria-label="Manage model"
+			class="flex min-w-0 flex-1 cursor-pointer items-center gap-2 text-left"
+			onclick={() =>
+				// a phone has no manager: its information dialog takes the row
+				deviceStore.isMobile
+					? uiStore.openModelInformation(option)
+					: uiStore.openModelsManager(entry.repoWithTag)}
+			type="button"
+		>
+			{#if orgName}
+				<ModelOrgAvatar
+					org={fetchedBaseModelOrg ?? orgName}
+					quantOrg={orgName}
+					quantPositionClass="-bottom-1 -right-1"
+					quantSize="h-3 w-3"
+					size="h-6 w-6"
+				/>
+			{/if}
+
+			<ModelId
+				class="flex-1"
+				draftSidecars={pendingDrafts}
+				hideOrgName={!showOrgName}
+				modelId={entry.repoWithTag}
+				showRawTooltip
 			/>
-		{/if}
 
-		<ModelId class="flex-1" hideOrgName={!showOrgName} modelId={entry.repoWithTag} showRawTooltip />
-
-		{#if percent !== null}
-			<span class="shrink-0 font-mono text-xs tabular-nums text-muted-foreground">{percent}%</span>
-		{:else if entry.isPaused}
-			<span class="shrink-0 text-xs text-muted-foreground">Paused</span>
-		{/if}
+			{#if percent !== null}
+				<span class="ml-auto shrink-0 font-mono text-xs tabular-nums text-muted-foreground">
+					{percent}%
+				</span>
+			{:else if entry.isPaused}
+				<span class="ml-auto shrink-0 text-xs text-muted-foreground">Paused</span>
+			{/if}
+		</button>
 
 		<!-- status action: spinner -> pause on hover while in flight, play on hover
-				 when paused; opacity only for the spinner, the spin owns the transform -->
-		<span class="relative inline-flex h-5 w-5 shrink-0 items-center justify-center">
+		     when paused; opacity only for the spinner, the spin owns the transform -->
+		<button
+			aria-label={actionText}
+			class="relative inline-flex h-5 w-5 shrink-0 cursor-pointer items-center justify-center"
+			onclick={() => void modelsStore.status.toggleDownload(entry.repoWithTag)}
+			type="button"
+		>
 			{#if entry.isPaused}
 				<Play
 					class="absolute h-4 w-4 scale-75 opacity-0 transition-[opacity,transform] duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:scale-100 group-hover:opacity-100 [@media(pointer:coarse)]:scale-100 [@media(pointer:coarse)]:opacity-100"
@@ -108,17 +136,45 @@
 					class="absolute h-4 w-4 opacity-0 transition-opacity duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:opacity-100 [@media(pointer:coarse)]:opacity-100"
 				/>
 			{/if}
-		</span>
-	</button>
+		</button>
 
-	<!-- the row actions keep the sizing and reveal of the other selector rows -->
-	<ActionIcon
-		ariaLabel="Cancel downloading"
-		class="pointer-events-none h-5 w-5 shrink-0 opacity-0 transition-opacity group-hover:pointer-events-auto group-hover:opacity-100 hover:text-destructive [@media(pointer:coarse)]:pointer-events-auto [@media(pointer:coarse)]:opacity-100"
-		icon={X}
-		iconSize="h-4 w-4"
-		onclick={() => onRequestCancel?.(entry.repoWithTag)}
-	/>
+		<!-- the row actions keep the sizing and reveal of the other selector rows -->
+		<ActionIcon
+			class="pointer-events-none h-5 w-5 shrink-0 opacity-0 transition-opacity group-hover:pointer-events-auto group-hover:opacity-100 hover:text-foreground [@media(pointer:coarse)]:pointer-events-auto [@media(pointer:coarse)]:opacity-100"
+			icon={Info}
+			iconSize="h-4 w-4"
+			onclick={() =>
+				// a phone has no manager: its information dialog takes the icon
+				deviceStore.isMobile
+					? uiStore.openModelInformation(option)
+					: uiStore.openModelsManager(entry.repoWithTag)}
+			tooltip="Manage model"
+			tooltipAsTitle
+		/>
+
+		<ActionIcon
+			ariaLabel="Cancel downloading"
+			class="pointer-events-none h-5 w-5 shrink-0 opacity-0 transition-opacity group-hover:pointer-events-auto group-hover:opacity-100 hover:text-destructive [@media(pointer:coarse)]:pointer-events-auto [@media(pointer:coarse)]:opacity-100"
+			icon={X}
+			iconSize="h-4 w-4"
+			onclick={() => onRequestCancel?.(entry.repoWithTag)}
+		/>
+	</div>
+
+	<!-- only with sub-lines: an empty strip would make the row taller than the
+	     plain ones, and the progress bar overlays the row's own bottom edge -->
+	{#if onDiskDrafts.length}
+		<div class="flex flex-col px-2 pb-1.5">
+			{#each onDiskDrafts as badge (badge.repo + ':' + badge.kind + ':' + (badge.quant ?? ''))}
+				<ModelDraftSubline
+					connectorClass="ml-4"
+					kind={badge.kind}
+					quant={badge.quant}
+					state={ModelRowDownloadState.DOWNLOADED}
+				/>
+			{/each}
+		</div>
+	{/if}
 
 	{#if entry.progress && entry.progress.totalBytes > 0}
 		<ModelDownloadProgressBar

tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorList.svelte

@@ -22,6 +22,7 @@
 		groupModelFamilies,
 		type ModelFamilyGroup,
 		type ModelItem,
+		repoOf,
 		windowLocalGroups
 	} from '$lib/utils';
 
@@ -202,7 +203,12 @@
 	<p class={headerClass} style={headerStyle}>Downloading</p>
 
 	{#each downloadEntries as entry (entry.repoWithTag)}
-		<ModelsSelectorDownloadItem {entry} onRequestCancel={requestCancel} {showOrgName} />
+		<ModelsSelectorDownloadItem
+			{entry}
+			onRequestCancel={requestCancel}
+			{showOrgName}
+			sidecars={modelsStore.status.getDraftSidecars(repoOf(entry.repoWithTag))}
+		/>
 	{/each}
 {/if}
 
@@ -230,6 +236,7 @@
 <DialogConfirmDownload
 	action={ModelDownloadConfirmAction.CANCEL}
 	onClose={() => (cancelOpen = false)}
+	onConfirm={() => void modelsStore.status.cancelDownload(pendingCancel)}
 	open={cancelOpen}
 	repoWithTag={pendingCancel}
 />

tools/ui/src/lib/components/app/models/discover/ModelsDiscover.svelte

@@ -0,0 +1,210 @@
+<script lang="ts">
+	import ModelsDiscoverListItem from './ModelsDiscoverList/ModelsDiscoverListItem.svelte';
+	import { Download } from '@lucide/svelte';
+	import { ModelsSection } from '$lib/components/app';
+	import { DialogConfirmDownload } from '$lib/components/app/dialogs';
+	import {
+		ModelsDiscoverDetailsContainer,
+		ModelsDiscoverList,
+		ModelsDiscoverListSearch
+	} from '$lib/components/app/models/discover';
+	import { Button } from '$lib/components/ui/button';
+	import { ModelDownloadConfirmAction } from '$lib/enums';
+	import { HuggingFaceService } from '$lib/services';
+	import { modelsDiscoverStore, modelsStore, uiStore } from '$lib/stores';
+	import type { HfModelInfo } from '$lib/types';
+	import { repoOf } from '$lib/utils';
+	import { untrack } from 'svelte';
+
+	let selectedId = $state<string | null>(null);
+	let searchQuery = $state('');
+	// running and paused downloads both lead the list; a search owns the pane, so the
+	// section steps aside while a query is active
+	let downloadEntries = $derived(modelsStore.status.getDownloadEntries());
+
+	let isSearching = $derived(searchQuery.trim().length > 0);
+
+	/** Hub record for a repo the list does not carry, so its row still renders. */
+	function hubRecord(id: string): HfModelInfo {
+		return {
+			_id: id,
+			downloads: 0,
+			id,
+			library_name: null,
+			likes: 0,
+			modelId: id,
+			pipeline_tag: null,
+			private: false,
+			tags: []
+		};
+	}
+
+	// Hub records of repos the list does not carry, e.g. a download started from a
+	// search hit: fetched once so the row shows the same badges and icons as a result.
+	let fetchedRecords = $state<Record<string, HfModelInfo>>({});
+
+	/** Record of a repo: what the list fetched, else its own details, else a bare one. */
+	function recordOf(repo: string): HfModelInfo {
+		return (
+			modelsDiscoverStore.models.find((model) => model.id === repo) ??
+			fetchedRecords[repo] ??
+			hubRecord(repo)
+		);
+	}
+
+	$effect(() => {
+		for (const entry of downloadEntries) {
+			const repo = repoOf(entry.repoWithTag);
+
+			if (fetchedRecords[repo] || modelsDiscoverStore.models.some((m) => m.id === repo)) continue;
+
+			void HuggingFaceService.getDetails(repo).then((details) => {
+				if (details) fetchedRecords = { ...fetchedRecords, [repo]: details };
+			});
+		}
+	});
+
+	// cancel is confirmed once, so a single dialog serves every download row; the
+	// target is kept while the dialog closes so its copy stays rendered
+	let pendingCancel = $state('');
+	let cancelOpen = $state(false);
+
+	function requestCancel(repoWithTag: string) {
+		pendingCancel = repoWithTag;
+		cancelOpen = true;
+	}
+
+	let searchTimeout: ReturnType<typeof setTimeout> | null = null;
+
+	// load the list on mount; untrack because fetch() toggles its loading flag
+	// synchronously, and tracking it would re-run the effect on every toggle - an
+	// endless retry while a repo stays failed
+	$effect(() => {
+		untrack(() => {
+			void modelsDiscoverStore.fetch();
+			void modelsDiscoverStore.search('');
+		});
+	});
+
+	$effect(() => {
+		const first = modelsDiscoverStore.firstModel;
+
+		if (!selectedId && first) {
+			selectedId = first.id;
+		}
+	});
+
+	// a caller can ask for one repo to be opened, the manager's rows do
+	$effect(() => {
+		const focus = uiStore.discoverModelFocus;
+
+		if (!focus) return;
+
+		selectedId = focus;
+		uiStore.discoverModelFocus = null;
+	});
+
+	function handleSearchInput(value: string) {
+		searchQuery = value;
+
+		if (searchTimeout) clearTimeout(searchTimeout);
+
+		searchTimeout = setTimeout(() => {
+			void modelsDiscoverStore.search(value);
+		}, 300);
+	}
+</script>
+
+{#snippet catalog()}
+	<ModelsDiscoverList
+		activeId={selectedId}
+		loading={modelsDiscoverStore.loading || modelsDiscoverStore.searching}
+		models={modelsDiscoverStore.models}
+		onSelect={(id) => (selectedId = id)}
+		showBaseModelAvatar
+	/>
+{/snippet}
+
+<aside
+	class="w-md shrink-0 self-start border-r border-border/40 bg-background overflow-y-auto md:pr-4 h-full space-y-4"
+>
+	<ModelsDiscoverListSearch bind:value={searchQuery} onSearch={handleSearchInput} />
+
+	{#if downloadEntries.length > 0 && !isSearching}
+		<ModelsSection
+			count={downloadEntries.length}
+			label="Downloading"
+			persistKey="discover-downloads"
+			sticky
+			stickyClass="sticky z-10 bg-background"
+		>
+			{#snippet icon()}
+				<Download class="h-3.5 w-3.5 shrink-0" />
+			{/snippet}
+
+			{#each downloadEntries as entry (entry.repoWithTag)}
+				{@const repo = repoOf(entry.repoWithTag)}
+
+				<!-- the row is the list's own, so a download opens the same details pane a
+				     result does; only the state line and the trailing control differ -->
+				<ModelsDiscoverListItem
+					active={repo === selectedId}
+					download={{
+						isPaused: entry.isPaused,
+						onRequestCancel: requestCancel,
+						progress: entry.progress,
+						repoWithTag: entry.repoWithTag
+					}}
+					model={recordOf(repo)}
+					onSelect={(id) => (selectedId = id)}
+					showBaseModelAvatar
+					sidecarBadges={modelsStore.status.getDraftSidecars(repo)}
+				/>
+			{/each}
+		</ModelsSection>
+	{/if}
+
+	<!-- One list instance, so the rows keep their state across search round trips;
+		 skeleton rows replace them while the initial catalog or a query loads. -->
+	<div>
+		{#if modelsDiscoverStore.error}
+			<div class="flex flex-col items-start gap-2 p-4">
+				<p class="text-sm text-destructive">{modelsDiscoverStore.error}</p>
+
+				<Button onclick={() => void modelsDiscoverStore.fetch()} size="sm" variant="outline">
+					Retry
+				</Button>
+			</div>
+		{:else if !modelsDiscoverStore.loading && !modelsDiscoverStore.searching && modelsDiscoverStore.models.length === 0}
+			<p class="p-4 text-sm text-muted-foreground">No models found</p>
+		{:else}
+			{#if isSearching}
+				{@render catalog()}
+			{:else}
+				<ModelsSection
+					count={modelsDiscoverStore.models.length}
+					label="Suggested models"
+					persistKey="discover-suggested"
+					sticky
+					stickyClass="sticky z-10 bg-background"
+				>
+					{@render catalog()}
+				</ModelsSection>
+			{/if}
+		{/if}
+	</div>
+</aside>
+
+<main class="overflow-y-auto">
+	{#if selectedId}
+		<ModelsDiscoverDetailsContainer modelId={selectedId} />
+	{/if}
+</main>
+
+<DialogConfirmDownload
+	action={ModelDownloadConfirmAction.CANCEL}
+	onClose={() => (cancelOpen = false)}
+	onConfirm={() => void modelsStore.status.cancelDownload(pendingCancel)}
+	open={cancelOpen}
+	repoWithTag={pendingCancel}
+/>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverChatTemplateDialog.svelte

@@ -0,0 +1,75 @@
+<script lang="ts">
+	import { Check, Copy, X } from '@lucide/svelte';
+	import { SyntaxHighlightedCode } from '$lib/components/app';
+	import { Button } from '$lib/components/ui/button';
+	import * as Dialog from '$lib/components/ui/dialog';
+	import { COPY_FEEDBACK_RESET_MS } from '$lib/constants';
+	import { copyToClipboard } from '$lib/utils';
+
+	interface Props {
+		open?: boolean;
+		chatTemplate: string;
+		onOpenChange?: (open: boolean) => void;
+	}
+
+	let { chatTemplate, onOpenChange, open = $bindable(false) }: Props = $props();
+
+	function handleOpenChange(value: boolean) {
+		open = value;
+		onOpenChange?.(value);
+	}
+
+	let copied = $state(false);
+
+	async function copy() {
+		await copyToClipboard(chatTemplate);
+		copied = true;
+		setTimeout(() => (copied = false), COPY_FEEDBACK_RESET_MS);
+	}
+</script>
+
+<Dialog.Root onOpenChange={handleOpenChange} {open}>
+	<Dialog.Content
+		class="flex max-h-[calc(100vh-4rem)] flex-col gap-0 p-0 md:w-[calc(100vw-4rem)]! md:max-w-4xl!"
+	>
+		<!-- The header's corner-pinned close X never lines up with a padded flex row,
+		     so it is replaced by one inside the row, aligned with the title -->
+		<Dialog.Header
+			class="flex-row items-center gap-2 border-b border-border/40 p-4"
+			showCloseButton={false}
+		>
+			<Dialog.Title class="text-sm font-semibold">Chat template</Dialog.Title>
+
+			<Button
+				aria-label="Copy chat template"
+				class="h-auto gap-1.5 px-2 py-1 text-xs"
+				onclick={() => void copy()}
+				size="sm"
+				variant="tertiary"
+			>
+				{#if copied}
+					<Check class="size-3.5 text-green-500" />
+				{:else}
+					<Copy class="size-3.5" />
+				{/if}
+
+				Copy
+			</Button>
+
+			<Dialog.Close
+				aria-label="Close"
+				class="ml-auto inline-flex cursor-pointer items-center justify-center rounded-md p-1.5 text-muted-foreground/70 transition-colors hover:bg-muted-foreground/10 hover:text-foreground"
+				type="button"
+			>
+				<X class="h-4 w-4" />
+			</Dialog.Close>
+		</Dialog.Header>
+
+		<SyntaxHighlightedCode
+			class="h-full rounded-none border-0 shadow-none"
+			code={chatTemplate}
+			language="jinja"
+			maxHeight="100%"
+		/>
+	</Dialog.Content>
+</Dialog.Root>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetails.svelte

@@ -0,0 +1,96 @@
+<script lang="ts">
+	import { ModelsDiscoverDetailsDownloadOptions } from './ModelsDiscoverDetailsDownloadOptions';
+	import ModelsDiscoverDetailsHeader from './ModelsDiscoverDetailsHeader.svelte';
+	import ModelsDiscoverDetailsReadme from './ModelsDiscoverDetailsReadme.svelte';
+	import ModelsDiscoverDetailsSkeleton from './ModelsDiscoverDetailsSkeleton.svelte';
+	import { OTHER_BIT_DEPTH } from '$lib/constants';
+	import { ModelAuxSidecar } from '$lib/enums';
+	import { HuggingFaceService } from '$lib/services';
+	import type { HfModelDetailInfo, HfModelSibling, ModelBitDepthRow } from '$lib/types';
+	import { detectThinkingSupport, detectToolUseSupport } from '$lib/utils';
+	import { SvelteMap } from 'svelte/reactivity';
+
+	interface Props {
+		/** Full HuggingFace model id, e.g. `ggml-org/gemma-3-4b-it-GGUF`. */
+		modelId: string;
+		/** Model details from `/api/models/{id}?full=true`; null while loading. */
+		details: HfModelDetailInfo | null;
+		/** GGUF files of the repo, shards collapsed, sorted by size desc. */
+		files: HfModelSibling[];
+		/** README.md content, frontmatter stripped; null when unavailable. */
+		readme: string | null;
+		/** True while the model data is being fetched. */
+		loading?: boolean;
+		/** Error message when loading failed. */
+		error?: string | null;
+	}
+
+	let { details, error = null, files, loading = false, modelId, readme }: Props = $props();
+
+	let gguf = $derived(details?.gguf);
+	let baseModels = $derived(HuggingFaceService.getBaseModels(details));
+	let licenseTag = $derived.by(() => {
+		const tags = details?.tags ?? [];
+
+		return tags.find((t) => t.startsWith('license:'))?.replace('license:', '') ?? null;
+	});
+
+	// Capabilities derived from HF metadata. Vision comes from an mmproj sidecar
+	// or a multimodal pipeline tag; tool use / reasoning from the chat template.
+	let hasMmproj = $derived(
+		files.some(
+			(f) => HuggingFaceService.extractQuantMeta(f.path)?.sidecar === ModelAuxSidecar.MMPROJ
+		)
+	);
+	let hasVision = $derived(hasMmproj || details?.pipeline_tag === 'image-text-to-text');
+	let hasTools = $derived(detectToolUseSupport(gguf?.chat_template ?? ''));
+	let hasReasoning = $derived(detectThinkingSupport(gguf?.chat_template ?? ''));
+
+	let bitDepthRows = $derived.by<ModelBitDepthRow[]>(() => {
+		const rows = new SvelteMap<number, HfModelSibling[]>();
+
+		for (const file of files) {
+			const meta = HuggingFaceService.extractQuantMeta(file.path);
+
+			// mmproj sidecars are already conveyed by the Vision capability badge;
+			// imatrix ships as a normal chip with its own badge.
+			if (meta?.sidecar === ModelAuxSidecar.MMPROJ) continue;
+
+			const depth = meta?.quant ? HuggingFaceService.getBitDepth(meta.quant) : null;
+			const bucket = depth ?? OTHER_BIT_DEPTH;
+			const list = rows.get(bucket) ?? [];
+
+			list.push(file);
+			rows.set(bucket, list);
+		}
+
+		return Array.from(rows.entries())
+			.map(([bitDepth, rowFiles]) => ({ bitDepth, files: rowFiles }))
+			.sort((a, b) => a.bitDepth - b.bitDepth);
+	});
+</script>
+
+{#if loading}
+	<ModelsDiscoverDetailsSkeleton />
+{:else if error}
+	<div class="flex h-full items-center justify-center py-20">
+		<p class="text-sm text-destructive">{error}</p>
+	</div>
+{:else if details}
+	<div class="@container space-y-6 p-6 pr-0">
+		<ModelsDiscoverDetailsHeader
+			{baseModels}
+			{details}
+			{gguf}
+			{hasReasoning}
+			{hasTools}
+			{hasVision}
+			{licenseTag}
+			{modelId}
+		/>
+
+		<ModelsDiscoverDetailsDownloadOptions {bitDepthRows} {modelId} />
+
+		<ModelsDiscoverDetailsReadme {readme} />
+	</div>
+{/if}

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsContainer.svelte

@@ -0,0 +1,66 @@
+<script lang="ts">
+	import ModelsDiscoverDetails from './ModelsDiscoverDetails.svelte';
+	import { HuggingFaceService } from '$lib/services';
+	import type { HfModelDetailInfo, HfModelSibling } from '$lib/types/huggingface';
+
+	interface Props {
+		/** Full HuggingFace model id, e.g. `ggml-org/gemma-3-4b-it-GGUF`. */
+		modelId: string;
+	}
+
+	let { modelId }: Props = $props();
+
+	let details = $state<HfModelDetailInfo | null>(null);
+	let files = $state<HfModelSibling[]>([]);
+	let readme = $state<string | null>(null);
+	let loading = $state(false);
+	let error = $state<string | null>(null);
+
+	$effect(() => {
+		const id = modelId;
+
+		if (!id) return;
+
+		let cancelled = false;
+
+		loading = true;
+		error = null;
+
+		void (async () => {
+			try {
+				const [info, tree, readmeText] = await Promise.all([
+					HuggingFaceService.getDetails(id),
+					HuggingFaceService.getTree(id),
+					HuggingFaceService.getReadme(id)
+				]);
+
+				if (cancelled) return;
+
+				if (!info) {
+					error = 'Model not found';
+
+					return;
+				}
+
+				details = info;
+				files = HuggingFaceService.filterByExtension(
+					HuggingFaceService.collapseGgufShards(tree),
+					'.gguf'
+				);
+				readme = readmeText;
+			} catch (err) {
+				if (cancelled) return;
+
+				error = err instanceof Error ? err.message : 'Failed to load model';
+			} finally {
+				if (!cancelled) loading = false;
+			}
+		})();
+
+		return () => {
+			cancelled = true;
+		};
+	});
+</script>
+
+<ModelsDiscoverDetails {details} {error} {files} {loading} {modelId} {readme} />

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptions.svelte

@@ -0,0 +1,258 @@
+<script lang="ts">
+	import { classify, draftBadgeOf, sortDraftKinds } from './download-options.utils';
+	import ModelsDiscoverDetailsDownloadOptionsRow from './ModelsDiscoverDetailsDownloadOptionsRow.svelte';
+	import { DialogConfirmDownload } from '$lib/components/app/dialogs';
+	import { Checkbox } from '$lib/components/ui/checkbox';
+	import * as Select from '$lib/components/ui/select';
+	import {
+		ModelDownloadConfirmAction,
+		ModelDraftSidecar,
+		ModelSelectableFileKind
+	} from '$lib/enums';
+	import { HuggingFaceService, ModelsService } from '$lib/services';
+	import { modelsStore } from '$lib/stores';
+	import type {
+		ModelBitDepthRow,
+		ModelCompanionDraft,
+		ModelDownloadEntryState,
+		ModelSelectableFile
+	} from '$lib/types';
+	import { directoryOf, isServerSidecarFile } from '$lib/utils';
+
+	interface Props {
+		/** Full HuggingFace repo id, e.g. `ggml-org/gemma-3-4b-it-GGUF`. */
+		modelId: string;
+		/** GGUF files grouped by bit depth. */
+		bitDepthRows: ModelBitDepthRow[];
+	}
+
+	let { bitDepthRows, modelId }: Props = $props();
+
+	// one shared dialog confirms the panel's destructive chip actions; the target
+	// is kept after closing so its copy stays rendered through the close transition
+	let pending: { action: ModelDownloadConfirmAction; repoWithTag: string } = $state({
+		action: ModelDownloadConfirmAction.CANCEL,
+		repoWithTag: ''
+	});
+	let confirmOpen = $state(false);
+
+	function requestCancel(repoWithTag: string) {
+		pending = { action: ModelDownloadConfirmAction.CANCEL, repoWithTag };
+		confirmOpen = true;
+	}
+
+	function requestDelete(repoWithTag: string) {
+		pending = { action: ModelDownloadConfirmAction.DELETE, repoWithTag };
+		confirmOpen = true;
+	}
+
+	/**
+	 * Download state of a main quant's chip, from the status feed: the quant alone,
+	 * since a draft it is missing gets a chip of its own to add it.
+	 */
+	function stateFor(repoWithTag: string): ModelDownloadEntryState {
+		const isDownloading = modelsStore.status.isDownloadInProgress(repoWithTag);
+		const isPaused = modelsStore.status.isDownloadPaused(repoWithTag);
+
+		return {
+			isDownloaded: !isDownloading && modelsStore.status.isModelDownloaded(repoWithTag),
+			isDownloading,
+			isFailed: modelsStore.status.hasFailedDownload(repoWithTag),
+			isPaused,
+			// live progress while downloading, else the frozen snapshot of the pause
+			progress:
+				modelsStore.status.getDownloadProgress(repoWithTag) ??
+				modelsStore.status.getPausedDownloadProgress(repoWithTag),
+			repoWithTag
+		};
+	}
+
+	/**
+	 * Every GGUF of the panel with its role, in row order.
+	 * Single source of truth for the chip rows and the command selects.
+	 */
+	let selectableFiles = $derived.by(() =>
+		bitDepthRows.flatMap((row) => row.files.map((file) => ({ ...file, kind: classify(file.path) })))
+	);
+
+	/** Sidecar kinds the repo reports on disk, so a chip tells a pulled draft from a missing one. */
+	let onDiskDrafts = $derived(modelsStore.status.getDraftSidecars(modelId));
+
+	/** Draft kinds the repo ships, best first and once each; one rides with every chip. */
+	let draftKinds = $derived(
+		sortDraftKinds(
+			selectableFiles
+				.flatMap((file) =>
+					file.kind === ModelSelectableFileKind.DRAFT ? [draftBadgeOf(file)] : []
+				)
+				.filter((kind): kind is ModelDraftSidecar => kind !== null)
+				.filter((kind, index, all) => all.indexOf(kind) === index)
+		)
+	);
+
+	// the operator's draft pick; the best available kind rides along until they change it
+	let draftPick = $state<ModelDraftSidecar | null>(null);
+	let draftKind = $derived(draftPick ?? draftKinds[0] ?? null);
+
+	/** The drafts are opt-in: unchecked, a chip downloads the quant alone. */
+	let withDraft = $state(true);
+
+	/** One kind leaves the pick nothing to choose: it reads as the value it holds. */
+	let draftKindReadonly = $derived(draftKinds.length < 2);
+
+	let draftKindLabel = $derived(draftKind?.toUpperCase() ?? '');
+
+	/** Bits of a file's quant, 0 when its name carries none, as the server reads it. */
+	function bitsOf(path: string): number {
+		const quant = HuggingFaceService.extractQuantMeta(path)?.quant;
+
+		return (quant ? HuggingFaceService.getBitDepth(quant) : null) ?? 0;
+	}
+
+	/**
+	 * Draft that rides with a quant: the file of the picked kind the server would
+	 * pull. Its own directory, then the exact quant, then the closest bit depth -
+	 * a 4-bit model pairs with a 16-bit draft head when the repo ships no 4-bit
+	 * one, the way -hf settles it.
+	 */
+	function companionFor(
+		main: Pick<ModelSelectableFile, 'path'>,
+		quant: string | null
+	): ModelCompanionDraft | null {
+		if (!draftKind) return null;
+
+		const directory = directoryOf(main.path);
+		const candidates = selectableFiles.filter(
+			(candidate) =>
+				candidate.kind === ModelSelectableFileKind.DRAFT &&
+				draftBadgeOf(candidate) === draftKind &&
+				isServerSidecarFile(candidate.path, draftKind) &&
+				directoryOf(candidate.path) === directory
+		);
+
+		if (candidates.length === 0) return null;
+
+		const quantOf = (candidate: ModelSelectableFile) =>
+			HuggingFaceService.extractQuantMeta(candidate.path)?.quant ?? null;
+		const mainBits = quant ? bitsOf(main.path) : 0;
+		const exact = quant ? candidates.find((candidate) => quantOf(candidate) === quant) : undefined;
+		// the closest bit depth wins when the exact quant is missing, ties on file order
+		const closest = [...candidates].sort(
+			(a, b) => Math.abs(bitsOf(a.path) - mainBits) - Math.abs(bitsOf(b.path) - mainBits)
+		)[0];
+		const picked = exact ?? closest;
+
+		if (!picked) return null;
+
+		const pickedQuant = quantOf(picked);
+
+		return {
+			kind: draftKind,
+			// on disk when the model's own sidecar option reports it, or when a loaded
+			// model's args name that same draft file
+			onDisk:
+				modelsStore.status.isModelDownloaded(
+					ModelsService.buildDownloadTag(modelId, quant, draftKind)
+				) || onDiskDrafts.some((badge) => badge.kind === draftKind && badge.quant === pickedQuant),
+			path: picked.path,
+			quant: pickedQuant,
+			size: picked.size ?? 0
+		};
+	}
+
+	let rows = $derived.by(() =>
+		bitDepthRows.map((row) => ({
+			bitDepth: row.bitDepth,
+			files: row.files.flatMap((file) => {
+				if (classify(file.path) !== ModelSelectableFileKind.MAIN) return [];
+
+				const quant = HuggingFaceService.extractQuantMeta(file.path)?.quant ?? null;
+				const companion = withDraft ? companionFor(file, quant) : null;
+
+				return [
+					{
+						...file,
+						companion,
+						kind: ModelSelectableFileKind.MAIN,
+						quant,
+						// the store adds the model's mmproj and the sidecars it already pulls
+						sidecars: companion ? [companion.kind] : [],
+						state: stateFor(ModelsService.buildDownloadTag(modelId, quant, null))
+					}
+				];
+			})
+		}))
+	);
+</script>
+
+{#if bitDepthRows.length}
+	<section class="rounded-3xl border border-border/30 bg-muted/60 shadow-sm dark:border-border/20">
+		<!-- One chip per main quant, each an independent download action. The draft
+			 of that quant rides with it and badges the chip; the server picks its file,
+			 the way -hf does. -->
+		<div class="flex w-full flex-col divide-y divide-border/50 px-4 pb-1 dark:divide-border/35">
+			{#each rows as row (row.bitDepth)}
+				<ModelsDiscoverDetailsDownloadOptionsRow
+					bitDepth={row.bitDepth}
+					draftKind={withDraft ? draftKind : null}
+					{draftKinds}
+					files={row.files}
+					{modelId}
+					onDraftKindChange={(kind) => (draftPick = kind)}
+					onRequestCancel={requestCancel}
+					onRequestDelete={requestDelete}
+				/>
+			{/each}
+		</div>
+
+		<!-- The drafts are opt-in, and the pick decides which flavour every chip
+		     above pulls. The pick sits between the two label halves, so its own click
+		     cannot toggle the checkbox. A repo shipping a single kind still shows it,
+		     readonly. -->
+		{#if draftKinds.length}
+			<div
+				class="flex flex-wrap items-center justify-start gap-x-1.5 gap-y-1 border-t border-border/50 px-4 py-2.5 text-xs text-muted-foreground dark:border-border/35"
+			>
+				<label class="flex cursor-pointer items-center gap-2" for="discover-draft-sidecars">
+					<Checkbox bind:checked={withDraft} id="discover-draft-sidecars" />
+
+					Download model with a
+				</label>
+
+				<Select.Root
+					disabled={!withDraft || draftKindReadonly}
+					onValueChange={(value) => value && (draftPick = value as ModelDraftSidecar)}
+					type="single"
+					value={draftKind}
+				>
+					<Select.Trigger
+						aria-label="Draft type"
+						class={draftKindReadonly
+							? 'disabled:cursor-default disabled:opacity-100 [&_svg]:hidden'
+							: ''}
+						size="xs"
+					>
+						{draftKindLabel}
+					</Select.Trigger>
+
+					<Select.Content class="font-mono text-xs">
+						{#each draftKinds as kind (kind)}
+							<Select.Item class="text-xs" label={kind.toUpperCase()} value={kind}>
+								{kind.toUpperCase()}
+							</Select.Item>
+						{/each}
+					</Select.Content>
+				</Select.Root>
+
+				<label class="cursor-pointer" for="discover-draft-sidecars">draft sidecar</label>
+			</div>
+		{/if}
+	</section>
+{/if}
+
+<DialogConfirmDownload
+	action={pending.action}
+	onClose={() => (confirmOpen = false)}
+	open={confirmOpen}
+	repoWithTag={pending.repoWithTag}
+/>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDownloadCommand.svelte

@@ -0,0 +1,131 @@
+<script lang="ts">
+	import { Check, Copy } from '@lucide/svelte';
+	import * as Select from '$lib/components/ui/select';
+	import { COPY_FEEDBACK_RESET_MS, MODEL_ID, SERVE_COMMAND, SPEC_TYPE } from '$lib/constants';
+	import type { ModelDraftSidecar } from '$lib/enums';
+	import { copyToClipboard } from '$lib/utils';
+
+	interface Props {
+		/** Full HuggingFace repo id, e.g. `ggml-org/gemma-3-4b-it-GGUF`. */
+		modelId: string;
+		/** Quant the command pins on `-hf`; null when the row's file carries none. */
+		quant?: string | null;
+		/**
+		 * Draft kind the panel downloads with the model, null when the sidecars are
+		 * off. The server resolves the draft file itself, so the command only names
+		 * its type.
+		 */
+		draftKind?: ModelDraftSidecar | null;
+		/** Draft kinds the repo ships; the command offers them in its own pick. */
+		draftKinds?: ModelDraftSidecar[];
+		/** Fires when the command's pick changes, so the panel's own pick follows. */
+		onDraftKindChange?: (kind: ModelDraftSidecar) => void;
+	}
+
+	let {
+		draftKind = null,
+		draftKinds = [],
+		modelId,
+		onDraftKindChange,
+		quant = null
+	}: Props = $props();
+
+	/** The llama serve command: the row's quant and, when it rides along, the draft. */
+	let command = $derived.by(() => {
+		const parts = [
+			SERVE_COMMAND.BIN,
+			SERVE_COMMAND.SUBCOMMAND,
+			SERVE_COMMAND.MODEL_FLAG,
+			quant ? `${modelId}${MODEL_ID.QUANTIZATION_SEPARATOR}${quant}` : modelId
+		];
+
+		if (draftKind) {
+			parts.push(SERVE_COMMAND.SPEC_TYPE_FLAG, SPEC_TYPE[draftKind]);
+		}
+
+		return parts.join(' ');
+	});
+
+	let copied = $state(false);
+
+	async function copy() {
+		await copyToClipboard(command);
+		copied = true;
+		setTimeout(() => (copied = false), COPY_FEEDBACK_RESET_MS);
+	}
+</script>
+
+<div
+	class="relative flex items-center gap-2 overflow-hidden rounded-lg border border-border/40 bg-background py-2.5 pl-4 pr-10 shadow-xs dark:border-border/35 dark:bg-background/50"
+>
+	<!-- Single line: long commands scroll horizontally instead of wrapping. -->
+	<div
+		class="flex min-w-0 flex-1 items-center gap-x-2 overflow-x-auto py-0.5 font-mono text-xs whitespace-nowrap text-foreground/90"
+	>
+		<span class="shrink-0">{SERVE_COMMAND.BIN}</span>
+
+		<span class="shrink-0">{SERVE_COMMAND.SUBCOMMAND}</span>
+
+		<span class="shrink-0">{SERVE_COMMAND.MODEL_FLAG}</span>
+
+		<span class="shrink-0">
+			{modelId}{quant ? MODEL_ID.QUANTIZATION_SEPARATOR : ''}
+		</span>
+
+		<!-- the row's own quant, so no pick is needed here -->
+		{#if quant}
+			<span class="shrink-0 rounded-md border border-primary/15 bg-primary/[0.07] px-1.5">
+				{quant}
+			</span>
+		{/if}
+
+		<!-- The draft rides with the model, so the flag only names its type; the
+		     checkbox below turns it on, the server resolves the file. The pick is
+		     the panel's own, so changing it here changes it there. -->
+		{#if draftKind}
+			<span class="shrink-0">{SERVE_COMMAND.SPEC_TYPE_FLAG}</span>
+
+			<!-- the select only earns its chrome when there is a real choice to make -->
+			{#if draftKinds.length > 1}
+				<Select.Root
+					onValueChange={(value) => value && onDraftKindChange?.(value as ModelDraftSidecar)}
+					type="single"
+					value={draftKind}
+				>
+					<Select.Trigger
+						aria-label="Draft type"
+						class="border-primary/15 bg-primary/[0.07] font-mono text-foreground hover:bg-primary/15 focus-visible:border-primary/40 focus-visible:ring-0"
+						size="xs"
+					>
+						{SPEC_TYPE[draftKind]}
+					</Select.Trigger>
+
+					<Select.Content class="font-mono text-xs">
+						{#each draftKinds as kind (kind)}
+							<Select.Item class="text-xs" label={SPEC_TYPE[kind]} value={kind}>
+								{SPEC_TYPE[kind]}
+							</Select.Item>
+						{/each}
+					</Select.Content>
+				</Select.Root>
+			{:else}
+				<span class="shrink-0 rounded-md border border-primary/15 bg-primary/[0.07] px-1.5">
+					{SPEC_TYPE[draftKind]}
+				</span>
+			{/if}
+		{/if}
+	</div>
+
+	<button
+		aria-label="Copy command"
+		class="absolute top-1/2 right-2 -translate-y-1/2 cursor-pointer rounded-md p-1.5 text-muted-foreground/70 transition-colors hover:bg-primary/10 hover:text-foreground"
+		onclick={copy}
+		type="button"
+	>
+		{#if copied}
+			<Check class="h-3.5 w-3.5 text-green-500" />
+		{:else}
+			<Copy class="h-3.5 w-3.5" />
+		{/if}
+	</button>
+</div>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton.svelte

@@ -0,0 +1,61 @@
+<script lang="ts">
+	import { Download } from '@lucide/svelte';
+	import * as Tooltip from '$lib/components/ui/tooltip';
+	import { HuggingFaceService } from '$lib/services';
+	import { modelsStore } from '$lib/stores';
+	import type { ModelCompanionDraft } from '$lib/types';
+
+	interface Props {
+		/** The draft its model is missing, with its own file, quant and size. */
+		companion: ModelCompanionDraft;
+		/** Tag of the model the draft belongs to, the one the server pulls it with. */
+		repoWithTag: string;
+	}
+
+	let { companion, repoWithTag }: Props = $props();
+
+	// one source, so the chip matches the draft badge of a quant chip
+	const SIDECAR_BADGE_CLASS =
+		'rounded-md bg-primary px-1 py-0.5 text-[10px] font-semibold tracking-wide text-primary-foreground uppercase';
+
+	let tooltipText = $derived(`Add the ${companion.kind} draft for ${repoWithTag}`);
+
+	// a draft is pulled with its model, so the model is posted again with the kind
+	function addDraft() {
+		void modelsStore.status.downloadModel(repoWithTag, [companion.kind]).catch(() => {
+			// the store already toasted the failure
+		});
+	}
+</script>
+
+<!-- A downloaded model missing its draft: the chip adds the draft alone, and
+     the model keeps the files it already has. -->
+<Tooltip.Root>
+	<Tooltip.Trigger>
+		{#snippet child({ props })}
+			<button
+				{...props}
+				aria-label={tooltipText}
+				class="group relative inline-flex h-auto cursor-pointer items-center gap-1 rounded-md border border-dashed border-border/60 px-2 py-1 text-left font-mono text-xs shadow-sm transition-[background-color,border-color,transform] duration-200 ease-[cubic-bezier(0.23,1,0.32,1)] hover:bg-muted-foreground/10 hover:border-border active:scale-[0.97] dark:border-border/40 dark:hover:bg-muted-foreground/15"
+				onclick={addDraft}
+				type="button"
+			>
+				<span class="{SIDECAR_BADGE_CLASS} inline-flex items-center gap-0.5">
+					{companion.kind}{companion.quant ? `:${companion.quant}` : ''}
+				</span>
+
+				<span class="-my-1 mx-0.75 w-px self-stretch bg-border"></span>
+
+				<span>{HuggingFaceService.formatFileSize(companion.size)}</span>
+
+				<Download
+					class="h-3.5 w-3.5 text-muted-foreground transition-colors duration-150 group-hover:text-foreground"
+				/>
+			</button>
+		{/snippet}
+	</Tooltip.Trigger>
+
+	<Tooltip.Content>
+		<p>{tooltipText}</p>
+	</Tooltip.Content>
+</Tooltip.Root>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton.svelte

@@ -0,0 +1,272 @@
+<script lang="ts">
+	import ModelDownloadProgressBar from '../../../ModelDownloadProgressBar.svelte';
+	import { labelFor } from './download-options.utils';
+	import { Check, Download, Loader2, Pause, Play, RotateCw, X } from '@lucide/svelte';
+	import * as Tooltip from '$lib/components/ui/tooltip';
+	import { HuggingFaceService } from '$lib/services';
+	import { modelsStore } from '$lib/stores';
+	import type { ModelQuantChip } from '$lib/types';
+	import type { Snippet } from 'svelte';
+
+	interface Props {
+		/** The main quant chip: its file, its download state and its joined drafts. */
+		file: ModelQuantChip;
+		/** Ask the parent to confirm deleting a downloaded model; the chip owns no dialog. */
+		onRequestDelete?: (repoWithTag: string) => void;
+		/** Ask the parent to confirm cancelling an in-flight download. */
+		onRequestCancel?: (repoWithTag: string) => void;
+	}
+
+	let { file, onRequestCancel, onRequestDelete }: Props = $props();
+
+	let entry = $derived(file.state);
+
+	// one source, so the idle, in-flight and downloaded chip states stay identical
+	const SIDECAR_BADGE_CLASS =
+		'rounded-md bg-primary px-1 py-0.5 text-[10px] font-semibold tracking-wide text-primary-foreground uppercase';
+
+	/** Queue the download; a failed attempt leaves partial files, drop them first. */
+	async function startDownload() {
+		try {
+			if (entry.isFailed) await modelsStore.status.cancelDownload(entry.repoWithTag);
+
+			// the joined drafts and the mmproj ride along: the server picks their files
+			await modelsStore.status.downloadModel(entry.repoWithTag, file.sidecars);
+		} catch {
+			// the store already toasted the failure
+		}
+	}
+
+	let label = $derived(labelFor(file.path));
+
+	let percent = $derived(
+		entry.progress && entry.progress.totalBytes > 0
+			? Math.round((entry.progress.downloadedBytes / entry.progress.totalBytes) * 100)
+			: null
+	);
+
+	/** Draft the chip pulls with its quant, e.g. `MTP`; null when none matches. */
+	let companion = $derived(file.companion);
+
+	/**
+	 * The joined draft reads once it rides along or lands: a downloaded model
+	 * missing its draft keeps to its own weights, and the draft gets its own chip.
+	 */
+	let showsCompanion = $derived(companion ? companion.onDisk || !entry.isDownloaded : false);
+
+	/** What the chip's actions name: the quant file and, when it rides along, its draft. */
+	let downloadLabel = $derived(
+		showsCompanion && companion ? `${file.path} + ${companion.path}` : file.path
+	);
+
+	let tooltipText = $derived(
+		entry.isDownloading
+			? 'Pause downloading'
+			: entry.isPaused
+				? 'Resume downloading'
+				: entry.isDownloaded
+					? `Delete ${downloadLabel}`
+					: entry.isFailed
+						? `Retry download: ${downloadLabel}`
+						: `Download ${downloadLabel}`
+	);
+
+	let downloadBytes = $derived(
+		(file.size ?? 0) + (showsCompanion && companion ? companion.size : 0)
+	);
+</script>
+
+{#snippet chipBody(dividerClass: string)}
+	<!-- label, joined draft and divider: identical on every chip state. The draft
+	     rides with this quant, so it reads as part of the same chip:
+	     `Q4_K_M + [MTP:Q8_0] | 20.0 GB`, the size covering both files. -->
+	<span class="font-medium">{label}</span>
+
+	{#if showsCompanion && companion}
+		<span class="text-muted-foreground/60">+</span>
+
+		<span class="{SIDECAR_BADGE_CLASS} inline-flex items-center gap-0.5">
+			<!-- the draft's own quant, so a 4-bit model paired with a 16-bit draft head reads at a glance -->
+			{companion.kind}{companion.quant ? `:${companion.quant}` : ''}
+
+			{#if companion.onDisk}
+				<Check class="h-2.5 w-2.5" />
+			{/if}
+		</span>
+	{/if}
+
+	<span class={dividerClass}></span>
+{/snippet}
+
+<!-- every chip is a tooltip trigger; the button renders as the trigger's child
+     so no nested button element is created -->
+{#snippet tooltipTrigger(tip: string, button: Snippet<[Record<string, unknown>]>)}
+	<Tooltip.Root>
+		<Tooltip.Trigger>
+			{#snippet child({ props })}
+				{@render button(props)}
+			{/snippet}
+		</Tooltip.Trigger>
+
+		<Tooltip.Content>
+			<p>{tip}</p>
+		</Tooltip.Content>
+	</Tooltip.Root>
+{/snippet}
+
+{#snippet deleteChip(props: Record<string, unknown>)}
+	<!-- downloaded chips are delete actions: green check by default, red X on hover -->
+	<button
+		{...props}
+		aria-label={tooltipText}
+		class="group relative inline-flex h-auto cursor-pointer items-center gap-1 rounded-md! border px-2 py-1 text-left font-mono text-xs shadow-sm transition-[background-color,border-color,transform] duration-200 ease-[cubic-bezier(0.23,1,0.32,1)] active:scale-[0.97]
+					border-green-600/25 bg-green-500/5 hover:border-destructive/50 hover:bg-destructive/10 dark:border-green-500/30 dark:bg-green-500/10 dark:hover:border-destructive/50 dark:hover:bg-destructive/15"
+		onclick={() => onRequestDelete?.(entry.repoWithTag)}
+		type="button"
+	>
+		{@render chipBody(
+			'-my-1 mx-0.75 w-px self-stretch bg-green-600/25 transition-colors duration-200 group-hover:bg-destructive/30 dark:bg-green-600/30 dark:group-hover:bg-destructive/30'
+		)}
+
+		<span>{HuggingFaceService.formatFileSize(downloadBytes)}</span>
+
+		<!-- icon slot: crossfade check -> x; touch devices show the delete affordance directly -->
+		<span class="relative inline-flex h-3.5 w-3.5 shrink-0 items-center justify-center">
+			<Check
+				class="absolute h-3.5 w-3.5 text-green-500 transition-[opacity,transform] duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:scale-75 group-hover:opacity-0 [@media(pointer:coarse)]:hidden"
+			/>
+
+			<X
+				class="absolute h-3.5 w-3.5 scale-75 text-destructive opacity-0 transition-[opacity,transform] duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:scale-100 group-hover:opacity-100 [@media(pointer:coarse)]:scale-100 [@media(pointer:coarse)]:opacity-100"
+			/>
+		</span>
+	</button>
+{/snippet}
+
+{#snippet pauseResumeChip(props: Record<string, unknown>)}
+	<button
+		{...props}
+		aria-label={tooltipText}
+		class="flex h-auto min-w-0 flex-1 cursor-pointer items-center gap-1 text-left"
+		onclick={() => {
+			if (entry.isDownloading) void modelsStore.status.pauseDownload(entry.repoWithTag);
+			else void modelsStore.status.downloadModel(entry.repoWithTag).catch(() => {});
+		}}
+		type="button"
+	>
+		{@render chipBody('-my-1 mx-0.75 w-px self-stretch bg-border')}
+
+		{#if percent !== null}
+			<span class="mr-1 tabular-nums">{percent}%</span>
+		{:else if entry.isPaused}
+			<span>Paused</span>
+		{:else}
+			<span>{HuggingFaceService.formatFileSize(downloadBytes)}</span>
+		{/if}
+
+		{#if entry.isDownloading}
+			<!-- spinner fades into the pause affordance on hover; opacity only, the
+			     spin keyframes own the transform so scale would fight them -->
+			<span class="relative inline-flex h-3.5 w-3.5 shrink-0 items-center justify-center">
+				<Loader2
+					class="absolute h-3.5 w-3.5 animate-spin text-muted-foreground transition-opacity duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:opacity-0 [@media(pointer:coarse)]:hidden"
+				/>
+
+				<Pause
+					class="absolute h-3.5 w-3.5 opacity-0 transition-opacity duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:opacity-100 [@media(pointer:coarse)]:opacity-100"
+				/>
+			</span>
+		{:else}
+			<!-- paused: the state at rest, the resume affordance taking its place on
+			     hover, the way the spinner gives way to the pause action above -->
+			<span class="relative inline-flex h-3.5 w-3.5 shrink-0 items-center justify-center">
+				<Pause
+					class="absolute h-3.5 w-3.5 text-muted-foreground transition-opacity duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:opacity-0 [@media(pointer:coarse)]:hidden"
+				/>
+
+				<Play
+					class="absolute h-3.5 w-3.5 opacity-0 transition-opacity duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] group-hover:opacity-100 [@media(pointer:coarse)]:opacity-100"
+				/>
+			</span>
+		{/if}
+	</button>
+{/snippet}
+
+{#snippet cancelChip(props: Record<string, unknown>)}
+	<!-- cancel slot: same fixed slot and fade as the pause / play affordances
+	     so the two icons line up exactly; the X turns destructive on its hover -->
+	<button
+		{...props}
+		aria-label="Cancel downloading"
+		class="relative grid h-3.5 w-3.5 shrink-0 cursor-pointer items-center justify-center text-muted-foreground/70"
+		onclick={() => onRequestCancel?.(entry.repoWithTag)}
+		type="button"
+	>
+		<X
+			class="h-3.5 w-3.5 transition-[opacity,transform,color] duration-150 ease-[cubic-bezier(0.23,1,0.32,1)] text-destructive [@media(pointer:coarse)]:scale-100 [@media(pointer:coarse)]:opacity-100"
+		/>
+	</button>
+{/snippet}
+
+{#snippet downloadChip(props: Record<string, unknown>)}
+	<!-- idle chips download on click (retry when the last attempt failed) -->
+	<button
+		{...props}
+		aria-label={tooltipText}
+		class="group relative inline-flex h-auto cursor-pointer items-center gap-1 overflow-hidden rounded-md! border px-2 py-1 text-left font-mono text-xs shadow-sm transition-[background-color,border-color,transform] duration-200 ease-[cubic-bezier(0.23,1,0.32,1)] active:scale-[0.97]
+				border-border/30 bg-background hover:bg-muted-foreground/10 dark:border-border/20 dark:bg-muted-foreground/15 dark:text-secondary-foreground dark:hover:bg-muted-foreground/25
+				{entry.isFailed ? 'border-destructive!' : ''}"
+		onclick={() => void startDownload()}
+		type="button"
+	>
+		{#if entry.isFailed}
+			<span
+				class="rounded-md bg-destructive px-1 py-0.5 text-[10px] font-semibold tracking-wide text-destructive-foreground uppercase"
+			>
+				Failed
+			</span>
+		{/if}
+
+		{@render chipBody('-my-1 mx-0.75 w-px self-stretch bg-border')}
+
+		<span>{HuggingFaceService.formatFileSize(downloadBytes)}</span>
+
+		<span class="inline-flex h-3.5 w-3.5 shrink-0 items-center justify-center">
+			{#if entry.isFailed}
+				<RotateCw class="h-3.5 w-3.5 text-destructive" />
+			{:else}
+				<Download
+					class="h-3.5 w-3.5 text-muted-foreground transition-colors duration-150 group-hover:text-foreground"
+				/>
+			{/if}
+		</span>
+	</button>
+{/snippet}
+
+{#if entry.isDownloaded}
+	{@render tooltipTrigger(tooltipText, deleteChip)}
+{:else if entry.isDownloading || entry.isPaused}
+	<!-- in-flight / paused chips: the chip body pauses / resumes on click, the X
+	     inside the chip cancels the download. The X slot is reserved, so the chip
+	     never reflows when the affordance fades in -->
+	<div
+		class="group relative inline-flex h-auto items-center gap-1 overflow-hidden rounded-md! border px-2 py-1 text-left font-mono text-xs shadow-sm transition-[background-color,border-color,transform] duration-200 ease-[cubic-bezier(0.23,1,0.32,1)] active:scale-[0.97]
+			{entry.isPaused
+			? 'border-yellow-600/40 bg-yellow-500/10 hover:bg-yellow-500/20 dark:border-yellow-500/30 dark:bg-yellow-500/10'
+			: 'border-border/30 bg-background hover:bg-muted-foreground/10 dark:border-border/20 dark:bg-muted-foreground/15'}"
+	>
+		{@render tooltipTrigger(tooltipText, pauseResumeChip)}
+
+		{@render tooltipTrigger('Cancel downloading', cancelChip)}
+
+		{#if percent !== null}
+			<ModelDownloadProgressBar
+				downloadedBytes={entry.progress?.downloadedBytes ?? 0}
+				overlay
+				totalBytes={entry.progress?.totalBytes ?? 0}
+			/>
+		{/if}
+	</div>
+{:else}
+	{@render tooltipTrigger(tooltipText, downloadChip)}
+{/if}

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsRow.svelte

@@ -0,0 +1,134 @@
+<script lang="ts">
+	import ModelsDiscoverDetailsDownloadOptionsDownloadCommand from './ModelsDiscoverDetailsDownloadOptionsDownloadCommand.svelte';
+	import ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton from './ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton.svelte';
+	import ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton from './ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton.svelte';
+	import { ChevronDown } from '@lucide/svelte';
+	import {
+		BIT_DEPTH_LABEL_SUFFIX,
+		GIGABYTE_LABEL,
+		OTHER_BIT_DEPTH,
+		OTHER_BIT_DEPTH_LABEL
+	} from '$lib/constants';
+	import type { ModelDraftSidecar } from '$lib/enums';
+	import type { ModelQuantChip } from '$lib/types';
+	import { minMemoryTierGb } from '$lib/utils';
+	import { cubicOut } from 'svelte/easing';
+	import { slide } from 'svelte/transition';
+
+	interface Props {
+		/** Bit depth of the row; `99` renders as "Other". */
+		bitDepth: number;
+		/** The row's main quant chips. */
+		files: ModelQuantChip[];
+		/** Full HuggingFace repo id, for the row's command preview. */
+		modelId: string;
+		/** Draft kind the panel pulls with the model, null when the sidecars are off. */
+		draftKind?: ModelDraftSidecar | null;
+		/** Draft kinds the repo ships, for the command's own pick. */
+		draftKinds?: ModelDraftSidecar[];
+		/** Forwarded to the command: the panel follows its draft pick. */
+		onDraftKindChange?: (kind: ModelDraftSidecar) => void;
+		/** Forwarded to each chip: ask the parent to confirm a cancel. */
+		onRequestCancel?: (repoWithTag: string) => void;
+		/** Forwarded to each chip: ask the parent to confirm a delete. */
+		onRequestDelete?: (repoWithTag: string) => void;
+	}
+
+	let {
+		bitDepth,
+		draftKind = null,
+		draftKinds = [],
+		files,
+		modelId,
+		onDraftKindChange,
+		onRequestCancel,
+		onRequestDelete
+	}: Props = $props();
+
+	// collapsed until asked for; the panel remounts per model
+	let commandOpen = $state(false);
+
+	let mainFile = $derived(files[0] ?? null);
+	let mainMemGb = $derived(mainFile ? minMemoryTierGb(mainFile.size ?? 0) : null);
+	// the row's weights plus, when one rides along, the draft's own tier
+	let totalMemGb = $derived.by(() => {
+		if (mainMemGb === null) return null;
+
+		const draftGb = mainFile?.companion?.size ? minMemoryTierGb(mainFile.companion.size) : 0;
+
+		return mainMemGb + (draftGb ?? 0);
+	});
+
+	/** Drafts a downloaded quant is missing, each getting a chip of its own to add. */
+	let missingDrafts = $derived(
+		files.flatMap((file) =>
+			file.state.isDownloaded && file.companion && !file.companion.onDisk
+				? [{ companion: file.companion, repoWithTag: file.state.repoWithTag }]
+				: []
+		)
+	);
+</script>
+
+<div class="py-3">
+	<div class="grid grid-cols-[5rem_1fr] items-center gap-3 @max-lg:grid-cols-1 @max-lg:gap-1.5">
+		<!-- the row's own label opens its command: it pins a quant of this row, so
+		     the copy matches what the chips download -->
+		<button
+			aria-expanded={commandOpen}
+			class="flex w-fit cursor-pointer flex-col items-start gap-0.5 text-left text-sm tabular-nums text-muted-foreground transition-colors hover:text-foreground @max-lg:flex-row @max-lg:items-baseline @max-lg:gap-2"
+			onclick={() => (commandOpen = !commandOpen)}
+			type="button"
+		>
+			<span class="flex items-center gap-1">
+				{#if bitDepth === OTHER_BIT_DEPTH}
+					{OTHER_BIT_DEPTH_LABEL}
+				{:else}
+					{bitDepth}{BIT_DEPTH_LABEL_SUFFIX}
+				{/if}
+
+				<ChevronDown
+					class="h-3 w-3 shrink-0 transition-transform duration-200 ease-[cubic-bezier(0.23,1,0.32,1)] {commandOpen
+						? 'rotate-180'
+						: ''}"
+				/>
+			</span>
+
+			{#if totalMemGb}
+				<span
+					class="text-[10px] whitespace-nowrap text-muted-foreground/60 @max-lg:whitespace-normal"
+				>
+					needs at least {totalMemGb}{GIGABYTE_LABEL} memory
+				</span>
+			{/if}
+		</button>
+
+		<div class="flex flex-wrap justify-end gap-1.5 @max-lg:justify-start">
+			{#each files as file (file.path)}
+				<ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton
+					{file}
+					{onRequestCancel}
+					{onRequestDelete}
+				/>
+			{/each}
+
+			{#each missingDrafts as draft (draft.repoWithTag + ':' + draft.companion.kind)}
+				<ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton
+					companion={draft.companion}
+					repoWithTag={draft.repoWithTag}
+				/>
+			{/each}
+		</div>
+	</div>
+
+	{#if commandOpen}
+		<div transition:slide={{ duration: 200, easing: cubicOut }} class="pt-2">
+			<ModelsDiscoverDetailsDownloadOptionsDownloadCommand
+				{draftKind}
+				{draftKinds}
+				{modelId}
+				{onDraftKindChange}
+				quant={mainFile?.quant ?? null}
+			/>
+		</div>
+	{/if}
+</div>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/download-options.utils.ts

@@ -0,0 +1,57 @@
+import { DRAFT_FILE_LABEL, MODEL_ID, PATH_SEPARATOR } from '$lib/constants';
+import { ModelDraftSidecar, ModelSelectableFileKind } from '$lib/enums';
+import { HuggingFaceService } from '$lib/services';
+import type { ModelSelectableFile } from '$lib/types';
+import { isAuxSidecar, isDraftSidecar } from '$lib/utils';
+
+/**
+ * Draft kinds in the order a chip lists them. DFlash wins: on the models that
+ * ship both it drafts a block at a time and stays accurate further out than
+ * MTP's few tokens; a kind not listed here sorts last.
+ */
+export const DRAFT_PREFERENCE_ORDER: readonly ModelDraftSidecar[] = [
+	ModelDraftSidecar.DFLASH,
+	ModelDraftSidecar.MTP
+];
+
+/** Draft sidecar of a file, typed for files already known to be drafts; null when it carries none. */
+export function draftBadgeOf(file: Pick<ModelSelectableFile, 'path'>): ModelDraftSidecar | null {
+	const sidecar = HuggingFaceService.extractQuantMeta(file.path)?.sidecar;
+
+	return sidecar && isDraftSidecar(sidecar) ? sidecar : null;
+}
+
+/** Kind of a file path: the main weights, a draft sidecar, or an aux sidecar (mmproj). */
+export function classify(path: string): ModelSelectableFileKind {
+	const sidecar = HuggingFaceService.extractQuantMeta(path)?.sidecar;
+
+	if (!sidecar) return ModelSelectableFileKind.MAIN;
+
+	return isAuxSidecar(sidecar) ? ModelSelectableFileKind.AUX : ModelSelectableFileKind.DRAFT;
+}
+
+/** Draft kinds best first, so a chip lists its joined badges in the pull order. */
+export function sortDraftKinds(kinds: readonly ModelDraftSidecar[]): ModelDraftSidecar[] {
+	const rank = (kind: ModelDraftSidecar) => {
+		const index = DRAFT_PREFERENCE_ORDER.indexOf(kind);
+
+		return index === -1 ? DRAFT_PREFERENCE_ORDER.length : index;
+	};
+
+	return [...kinds].sort((a, b) => rank(a) - rank(b));
+}
+
+/** Display label of a file: its quant, else the file name without the extension. */
+export function labelFor(path: string): string {
+	const meta = HuggingFaceService.extractQuantMeta(path);
+
+	if (meta?.quant) return meta.quant;
+
+	// Quantless sidecar files: the chip badge already carries the sidecar type,
+	// so the label only marks draft files; aux sidecars badge alone.
+	if (meta?.sidecar) return isDraftSidecar(meta.sidecar) ? DRAFT_FILE_LABEL : '';
+
+	const basename = path.split(PATH_SEPARATOR).pop() ?? path;
+
+	return basename.replace(MODEL_ID.WEIGHT_EXTENSION_REGEX, '');
+}

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/index.ts

@@ -0,0 +1,24 @@
+/**
+ *
+ * MODELS DISCOVER - DETAILS - DOWNLOAD OPTIONS
+ *
+ * The download area of the detail pane: one chip per main quant, the drafts it rides with
+ * badged on it, a draft checkbox and type pick at the bottom, and a terminal command
+ * preview inside every row.
+ *
+ */
+
+/** One download action per main quant, the sidecars it pulls, and the row's command. */
+export { default as ModelsDiscoverDetailsDownloadOptions } from './ModelsDiscoverDetailsDownloadOptions.svelte';
+
+/** One bit-depth row: the depth label opens its command, the quants of that depth download beside it. */
+export { default as ModelsDiscoverDetailsDownloadOptionsRow } from './ModelsDiscoverDetailsDownloadOptionsRow.svelte';
+
+/** A single GGUF file as an independent action chip. */
+export { default as ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton } from './ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton.svelte';
+
+/** Chip for a draft a downloaded model is missing; posts the model with that kind, so the server keeps what is on disk and pulls the draft. */
+export { default as ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton } from './ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton.svelte';
+
+/** The `llama serve -hf ...` command box of one row, with a draft type pick and a copy button. */
+export { default as ModelsDiscoverDetailsDownloadOptionsDownloadCommand } from './ModelsDiscoverDetailsDownloadOptionsDownloadCommand.svelte';

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHeader.svelte

@@ -0,0 +1,96 @@
+<script lang="ts">
+	import ModelCapabilityIcons from '../../ModelCapabilityIcons.svelte';
+	import ModelOrgAvatar from '../../ModelOrgAvatar.svelte';
+	import ModelsDiscoverDetailsHfHubStats from './ModelsDiscoverDetailsHfHubStats.svelte';
+	import ModelsDiscoverDetailsMetadata from './ModelsDiscoverDetailsMetadata.svelte';
+	import { ExternalLink } from '@lucide/svelte';
+	import { base } from '$app/paths';
+	import { Button } from '$lib/components/ui/button';
+	import { HuggingFaceService } from '$lib/services';
+	import type { HfModelDetailInfo, HfModelGguf } from '$lib/types/huggingface';
+	import { orgOf } from '$lib/utils';
+
+	interface Props {
+		modelId: string;
+		details: HfModelDetailInfo;
+		gguf?: HfModelGguf;
+		baseModels: string[];
+		licenseTag: string | null;
+		hasVision: boolean;
+		hasTools: boolean;
+		hasReasoning: boolean;
+	}
+
+	let { baseModels, details, gguf, hasReasoning, hasTools, hasVision, licenseTag, modelId }: Props =
+		$props();
+
+	// Avatar shows the base model's org (e.g. the Qwen logo for a ggml-org GGUF)
+	// with the quant org as a corner badge when they differ.
+	let repoOrg = $derived(orgOf(details.id) || orgOf(modelId));
+	let baseOrg = $derived(orgOf(baseModels[0]));
+	let avatarOrg = $derived(baseOrg || repoOrg);
+	let quantOrg = $derived(baseOrg && baseOrg !== repoOrg ? repoOrg : undefined);
+</script>
+
+<header class="space-y-3">
+	<div class="flex flex-wrap items-start justify-between gap-3">
+		<div class="flex min-w-0 items-center gap-2">
+			<ModelOrgAvatar
+				org={avatarOrg}
+				{quantOrg}
+				quantPositionClass="-bottom-1.5 -right-1.5"
+				quantSize="h-6 w-6"
+				size="h-12 w-12"
+			/>
+
+			<div class="min-w-0">
+				<div class="flex items-center gap-2 @max-lg:flex-col @max-lg:items-start">
+					<h1 class="truncate text-lg font-semibold @max-lg:w-full">{details.id ?? modelId}</h1>
+
+					<ModelCapabilityIcons
+						gapClass="gap-2"
+						iconSize="h-4 w-4"
+						modalities={{ audio: false, video: false, vision: hasVision }}
+						supportsThinking={hasReasoning}
+						supportsToolUse={hasTools}
+					/>
+				</div>
+
+				{#if baseModels.length}
+					<div class="flex items-center gap-1">
+						<span class="truncate text-xs text-muted-foreground">{baseModels.join(', ')}</span>
+
+						<a
+							aria-label="View base model on HuggingFace"
+							class="shrink-0 text-muted-foreground transition-colors hover:text-foreground"
+							href={HuggingFaceService.getModelUrl(baseModels[0])}
+							rel="noopener noreferrer"
+							target="_blank"
+						>
+							<ExternalLink class="h-3 w-3" />
+						</a>
+					</div>
+				{/if}
+			</div>
+		</div>
+
+		<Button
+			class="h-auto gap-1.5 px-2.5 py-1.5 text-xs"
+			href={HuggingFaceService.getModelUrl(modelId)}
+			rel="noopener noreferrer"
+			size="sm"
+			target="_blank"
+			variant="tertiary"
+		>
+			<img alt="" class="h-3.5 w-3.5" src="{base}/recommended-mcp/huggingface.ico" />
+
+			View on Hugging Face
+
+			<ExternalLink class="size-3.5" />
+		</Button>
+	</div>
+
+	<ModelsDiscoverDetailsHfHubStats {details} />
+
+	<ModelsDiscoverDetailsMetadata {details} {gguf} {licenseTag} {modelId} />
+</header>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHfHubStats.svelte

@@ -0,0 +1,31 @@
+<script lang="ts">
+	import { Download, Heart } from '@lucide/svelte';
+	import { HuggingFaceService } from '$lib/services';
+	import type { HfModelDetailInfo } from '$lib/types/huggingface';
+
+	interface Props {
+		details: HfModelDetailInfo;
+	}
+
+	let { details }: Props = $props();
+</script>
+
+<div class="flex flex-wrap items-center gap-x-4 gap-y-1 text-sm text-muted-foreground">
+	{#if typeof details.downloads === 'number'}
+		<span class="inline-flex items-center gap-1.5">
+			<Download class="h-3.5 w-3.5" />
+			{HuggingFaceService.formatDownloads(details.downloads)}
+		</span>
+	{/if}
+
+	{#if typeof details.likes === 'number'}
+		<span class="inline-flex items-center gap-1.5">
+			<Heart class="h-3.5 w-3.5" />
+			{HuggingFaceService.formatLikes(details.likes)}
+		</span>
+	{/if}
+
+	{#if details.lastModified}
+		<span>Updated {HuggingFaceService.formatRelativeTime(details.lastModified)}</span>
+	{/if}
+</div>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadata.svelte

@@ -0,0 +1,82 @@
+<script lang="ts">
+	import ModelsDiscoverChatTemplateDialog from './ModelsDiscoverChatTemplateDialog.svelte';
+	import ModelsDiscoverDetailsMetadataItem from './ModelsDiscoverDetailsMetadataItem.svelte';
+	import { MessageSquareCode } from '@lucide/svelte';
+	import { Button } from '$lib/components/ui/button';
+	import { modelsDiscoverStore } from '$lib/stores';
+	import type { HfModelDetailInfo, HfModelGguf } from '$lib/types/huggingface';
+	import { formatParameters } from '$lib/utils';
+
+	interface Props {
+		/** Full HuggingFace model id, e.g. `ggml-org/gemma-3-4b-it-GGUF`. */
+		modelId: string;
+		details: HfModelDetailInfo;
+		gguf?: HfModelGguf;
+		licenseTag: string | null;
+	}
+
+	let { details, gguf, licenseTag, modelId }: Props = $props();
+
+	// Catalog family description when curated, else the HF card description.
+	let description = $derived(
+		modelsDiscoverStore.descriptionFor(modelId) ?? details.cardData?.description
+	);
+
+	let chatTemplateOpen = $state(false);
+</script>
+
+{#if description}
+	<p class="text-sm text-muted-foreground">{description}</p>
+{/if}
+
+<!-- Metadata chips: label | value pairs, matching the HF model page style -->
+<div class="flex flex-wrap items-center gap-1.5">
+	{#if gguf?.total}
+		<ModelsDiscoverDetailsMetadataItem
+			label="Model size"
+			value="{formatParameters(gguf.total)} params"
+		/>
+	{/if}
+
+	{#if gguf?.context_length}
+		<ModelsDiscoverDetailsMetadataItem
+			label="Context"
+			value={gguf.context_length.toLocaleString()}
+		/>
+	{/if}
+
+	{#if gguf?.architecture}
+		<ModelsDiscoverDetailsMetadataItem label="Architecture" value={gguf.architecture} />
+	{/if}
+
+	{#if gguf?.chat_template}
+		<Button
+			class="h-auto gap-1.5 px-2.5 py-1 text-xs"
+			onclick={() => (chatTemplateOpen = true)}
+			size="sm"
+			variant="tertiary"
+		>
+			<MessageSquareCode class="size-3" />
+			Chat template
+		</Button>
+	{/if}
+
+	{#if licenseTag}
+		<ModelsDiscoverDetailsMetadataItem label="License" value={licenseTag} />
+	{/if}
+
+	{#if details.gated === true}
+		<span
+			class="rounded bg-yellow-500/10 px-2 py-0.5 text-xs font-medium text-yellow-600 dark:text-yellow-400"
+		>
+			gated
+		</span>
+	{/if}
+</div>
+
+{#if gguf?.chat_template}
+	<ModelsDiscoverChatTemplateDialog
+		bind:open={chatTemplateOpen}
+		chatTemplate={gguf.chat_template}
+	/>
+{/if}

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadataItem.svelte

@@ -0,0 +1,18 @@
+<script lang="ts">
+	interface Props {
+		label: string;
+		value: string;
+	}
+
+	let { label, value }: Props = $props();
+</script>
+
+<!-- Metadata chip: label | value pair, matching the HF model page style, on the
+     raised surface the tertiary button variant uses -->
+<span
+	class="inline-flex items-center divide-x divide-border rounded-md border border-border/30 bg-muted/60 text-xs shadow-sm dark:border-border/20 dark:bg-muted/75"
+>
+	<span class="px-2.5 py-1 text-muted-foreground">{label}</span>
+
+	<span class="px-2.5 py-1 font-medium">{value}</span>
+</span>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsReadme.svelte

@@ -0,0 +1,17 @@
+<script lang="ts">
+	import { MarkdownContent } from '$lib/components/app';
+
+	interface Props {
+		readme: string | null;
+	}
+
+	let { readme }: Props = $props();
+</script>
+
+{#if readme}
+	<section
+		class="p-3 rounded-3xl border border-border/30 bg-muted/60 shadow-sm dark:border-border/20"
+	>
+		<MarkdownContent allowHtml class="prose-sm max-w-none" content={readme} />
+	</section>
+{/if}

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsSkeleton.svelte

@@ -0,0 +1,98 @@
+<script lang="ts">
+	import { Skeleton } from '$lib/components/ui/skeleton';
+</script>
+
+<!-- Static skeleton of ModelsDiscoverDetails: header, metadata chips, download options and readme lines. -->
+<div class="@container space-y-6 p-6 pr-0" data-slot="model-details-skeleton">
+	<!-- Header: avatar, name with capability icons, base model line, HF link -->
+	<div class="space-y-3">
+		<div class="flex items-start justify-between gap-3">
+			<div class="flex min-w-0 items-center gap-2">
+				<Skeleton class="h-12 w-12 rounded-md" />
+
+				<div class="min-w-0 space-y-1.5">
+					<Skeleton class="h-5 w-56 max-w-full" />
+
+					<Skeleton class="h-3 w-40 max-w-full" />
+				</div>
+			</div>
+
+			<Skeleton class="h-7.5 w-44 shrink-0 rounded-md" />
+		</div>
+
+		<!-- downloads / likes / last updated -->
+		<div class="flex items-center gap-4">
+			<Skeleton class="h-3.5 w-16" />
+
+			<Skeleton class="h-3.5 w-12" />
+
+			<Skeleton class="h-3.5 w-24" />
+		</div>
+
+		<!-- catalog family description, else the HF card description -->
+		<Skeleton class="h-3.5 my-4 w-full" />
+
+		<!-- metadata chips -->
+		<div class="space-y-3">
+			<div class="flex flex-wrap items-center gap-1.5">
+				<Skeleton class="h-7 w-36 rounded-md" />
+
+				<Skeleton class="h-7 w-28 rounded-md" />
+
+				<Skeleton class="h-7 w-32 rounded-md" />
+
+				<Skeleton class="h-7 w-24 rounded-md" />
+			</div>
+		</div>
+	</div>
+
+	<!-- Download options: quant rows and the draft opt-in line -->
+	<div class="rounded-3xl border border-border/30 bg-muted/60 shadow-sm dark:border-border/20">
+		<div class="flex w-full flex-col divide-y divide-border/50 px-4 pb-1 dark:divide-border/35">
+			{#each [0, 1, 2] as _, index (index)}
+				<div
+					class="grid grid-cols-[5rem_1fr] items-center gap-3 py-3 @max-lg:grid-cols-1 @max-lg:gap-1.5"
+				>
+					<div class="space-y-1.5">
+						<Skeleton class="h-4 w-14" />
+
+						<Skeleton class="h-2.5 w-24" />
+					</div>
+
+					<div class="flex flex-wrap items-center justify-end gap-1.5">
+						<Skeleton class="h-6 w-28 rounded-md" />
+
+						<Skeleton class="h-6 w-20 rounded-md" />
+					</div>
+				</div>
+			{/each}
+		</div>
+
+		<div class="space-y-2.5 border-t border-border/50 px-4 pt-3.5 dark:border-border/35">
+			<Skeleton class="h-9 w-full rounded-md" />
+
+			<div class="flex items-center gap-3">
+				<span class="h-px flex-1 bg-border/50"></span>
+
+				<Skeleton class="h-3 w-44" />
+
+				<span class="h-px flex-1 bg-border/50"></span>
+			</div>
+
+			<Skeleton class="h-11 w-full rounded-md" />
+		</div>
+	</div>
+
+	<!-- Readme -->
+	<div class="p-3 rounded-3xl border border-border/30 bg-muted/60 shadow-sm dark:border-border/20">
+		<div class="space-y-2.5">
+			<Skeleton class="h-3.5 w-full" />
+
+			<Skeleton class="h-3.5 w-full" />
+
+			<Skeleton class="h-3.5 w-5/6" />
+
+			<Skeleton class="h-3.5 w-2/3" />
+		</div>
+	</div>
+</div>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/index.ts

@@ -0,0 +1,37 @@
+/**
+ *
+ * MODELS DISCOVER - DETAILS
+ *
+ * The right-hand detail pane of the discover view: header (avatar, name, stats, metadata
+ * chips, capability badges), the download options area and the model-card README.
+ *
+ */
+
+/** Detail pane of the selected model; fetches its own details and GGUF file list. */
+export { default as ModelsDiscoverDetails } from './ModelsDiscoverDetails.svelte';
+
+/** Fetches one repo's details, files and readme for hosts outside Discover. */
+export { default as ModelsDiscoverDetailsContainer } from './ModelsDiscoverDetailsContainer.svelte';
+
+/** Header of the detail pane. */
+export { default as ModelsDiscoverDetailsHeader } from './ModelsDiscoverDetailsHeader.svelte';
+
+/** Downloads, likes and last updated, formatted the way the Hub shows them. */
+export { default as ModelsDiscoverDetailsHfHubStats } from './ModelsDiscoverDetailsHfHubStats.svelte';
+
+/** Static placeholder matching the detail layout. */
+export { default as ModelsDiscoverDetailsSkeleton } from './ModelsDiscoverDetailsSkeleton.svelte';
+
+/** Model card README as markdown. */
+export { default as ModelsDiscoverDetailsReadme } from './ModelsDiscoverDetailsReadme.svelte';
+
+/** Metadata chips: size, context, architecture, license, chat template. */
+export { default as ModelsDiscoverDetailsMetadata } from './ModelsDiscoverDetailsMetadata.svelte';
+
+/** One label | value chip of the metadata row. */
+export { default as ModelsDiscoverDetailsMetadataItem } from './ModelsDiscoverDetailsMetadataItem.svelte';
+
+/** Chat template in a scrollable dialog with a copy button. */
+export { default as ModelsDiscoverChatTemplateDialog } from './ModelsDiscoverChatTemplateDialog.svelte';
+
+export * from './ModelsDiscoverDetailsDownloadOptions';