webgpu: address review @yomaytk, name the mat-vec row threshold

The flash attention vec path and the MMVQ path append a scratch area
after dst, but get_alloc_size only accounted for it when the current
batch takes that path. The size was therefore not monotonic in the
batch length: a graph reserved at n_ubatch (tiled path, no scratch)
or at one token did not cover batches of 2 to 19 query rows, and
GGML_SCHED_NO_REALLOC builds aborted on an unexpected reallocation.

Pass the row count to both path predicates and size the scratch with
the row count clamped to the longest batch the short path serves.

README.md

@@ -13,7 +13,7 @@
 [![Docker](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/docker.yml?label=Docker)](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
 [![Winget](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/winget.yml?label=Winget)](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
 
-[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Asanmai%20OR%20author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Athelittlefireman%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3AmasterFoad%20OR%20author%3Aallozaur%20OR%20author%3AAndreas-Krebbel%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Achaxu01%20OR%20author%3Alhez%20OR%20author%3AAlekseiNikiforovIBM%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming%20OR%20author%3Aynankani)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
+[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Asanmai%20OR%20author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Athelittlefireman%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3AmasterFoad%20OR%20author%3Aallozaur%20OR%20author%3AAndreas-Krebbel%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Achaxu01%20OR%20author%3Alhez%20OR%20author%3AAlekseiNikiforovIBM%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming%20OR%20author%3Aynankani)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
 
 </div>
 
@@ -79,7 +79,7 @@ The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-or
 | --- | --- |
 | [BLAS](docs/build.md#blas-build) | All |
 | [BLIS](docs/backend/BLIS.md) | All |
-| [CANN](docs/build.md#cann) | Huawei Ascend NPU |
+| [CANN](docs/build.md#cann) | Ascend NPU |
 | [CUDA](docs/build.md#cuda) | Nvidia GPU |
 | [HIP](docs/build.md#hip) | AMD GPU |
 | [Hexagon](docs/backend/snapdragon/README.md) | Snapdragon |

common/arg.cpp

@@ -12,6 +12,16 @@
 #include "speculative.h"
 #include "preset.h"
 
+// fix problem with std::min and std::max
+#if defined(_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#   define NOMINMAX
+#endif
+#include <windows.h>
+#include <shellapi.h>
+#endif
+
 #include <algorithm>
 #include <cinttypes>
 #include <climits>
@@ -28,6 +38,20 @@
 #include <thread> // for hardware_concurrency
 #include <vector>
 
+#ifndef __EMSCRIPTEN__
+#ifdef __linux__
+#include <linux/limits.h>
+#elif defined(_WIN32)
+#   if !defined(PATH_MAX)
+#   define PATH_MAX MAX_PATH
+#   endif
+#elif defined(_AIX)
+#include <sys/limits.h>
+#else
+#include <sys/syslimits.h>
+#endif
+#endif
+
 #define LLAMA_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 2083
 
 using json = common_json;
@@ -327,7 +351,7 @@ static bool parse_bool_value(const std::string & value) {
 static std::string get_default_local_path(const std::string & url) {
     auto f = string_split<std::string>(url, '#').front();
     f = string_split<std::string>(f, '?').front();
-    return fs_get_cache_file(string_split<std::string>(f, '/').back()).string();
+    return fs_path_to_utf8(fs_get_cache_file(string_split<std::string>(f, '/').back()));
 }
 
 static bool spec_types_is_default(const common_params & params) {
@@ -738,7 +762,7 @@ static void common_params_apply_system_config(common_params & params, llama_exam
     common_preset_context ctx(ex);
     ctx.ignore_unknown_keys = true; // the same config file is shared by all programs
     for (const auto & path : found) {
-        LOG_INF("using config file: %s\n", path.string().c_str());
+        LOG_INF("using config file: %s\n", fs_path_to_utf8(path).c_str());
         common_preset global;
         common_presets presets = ctx.load_from_ini(path, global);
         global.apply_to_params(params);
@@ -1235,7 +1259,44 @@ bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map<com
     return true;
 }
 
+#ifdef _WIN32
+struct utf8_argv {
+    std::vector<std::string> buf;
+    std::vector<char*> ptrs;
+};
+
+static utf8_argv make_utf8_argv() {
+    utf8_argv out;
+    int wargc = 0;
+    LPWSTR* wargv = CommandLineToArgvW(GetCommandLineW(), &wargc);
+    if (!wargv) return out;
+
+    out.buf.reserve(wargc);
+    for (int i = 0; i < wargc; ++i) {
+        int n = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wargv[i], -1, nullptr, 0, nullptr, nullptr);
+        if (n <= 0) { out.buf.emplace_back(); continue; }
+        auto& s = out.buf.emplace_back();
+        s.resize(static_cast<size_t>(n - 1));
+        (void)WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, s.data(), n, nullptr, nullptr);
+    }
+    LocalFree(wargv);
+
+    out.ptrs.reserve(out.buf.size() + 1);
+    for (auto& s : out.buf) out.ptrs.push_back(s.data());
+    out.ptrs.push_back(nullptr);
+    return out;
+}
+#endif
+
 bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **)) {
+#ifdef _WIN32
+    auto utf8 = make_utf8_argv();
+    // repair argv only when it matches the process command line
+    if (static_cast<int>(utf8.buf.size()) == argc) {
+        argv = utf8.ptrs.data();
+    }
+#endif
+
     auto ctx_arg = common_params_parser_init(params, ex, print_usage);
     const common_params params_org = ctx_arg.params; // the example can modify the default params
 

common/common.cpp

@@ -934,6 +934,12 @@ std::string wstring_to_utf8(const std::wstring & str) {
 }
 #endif
 
+// returns the path as a UTF-8 string, preserving its separators
+std::string fs_path_to_utf8(const std::filesystem::path & path) {
+    const auto value = path.u8string();
+    return std::string(value.begin(), value.end());
+}
+
 void fs_write_atomic(const std::filesystem::path & path, const std::string & data) {
     // a random suffix gives every writer its own temporary file, across processes and containers
     std::random_device rd;
@@ -957,7 +963,7 @@ void fs_write_atomic(const std::filesystem::path & path, const std::string & dat
 
     if (file.fail() || ec) {
         std::filesystem::remove(path_tmp, ec);
-        throw std::runtime_error("failed to write file: " + path.string());
+        throw std::runtime_error("failed to write file: " + fs_path_to_utf8(path));
     }
 }
 
@@ -1051,7 +1057,7 @@ std::filesystem::path fs_get_cache_file(const std::string & filename) {
     std::error_code ec;
     common_create_directories(cache_directory, ec);
     if (ec) {
-        throw std::runtime_error("failed to create cache directory: " + cache_directory.string());
+        throw std::runtime_error("failed to create cache directory: " + fs_path_to_utf8(cache_directory));
     }
     return cache_directory / std::filesystem::u8path(filename);
 }
@@ -2275,16 +2281,6 @@ int32_t common_batch::add_embd(llama_embd embd, const llama_pos * pos, llama_seq
     return size() - 1;
 }
 
-int32_t common_batch::add_embd_row(int32_t row, const llama_pos * pos, llama_seq_id seq_id, bool output) {
-    token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, { nullptr, 0, 0 }, { nullptr, 0, 0 }, {} };
-    t.embd_row = row;
-    for (int32_t j = 0; j < n_pos; ++j) {
-        t.pos[j] = pos[j];
-    }
-    tokens.push_back(t);
-    return size() - 1;
-}
-
 llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
     GGML_ASSERT(batch && "common_batch was not initialized with a context");
     GGML_ASSERT(off >= 0 && n >= 0 && off + n <= size());
@@ -2305,12 +2301,6 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
             if (t.embd.data && !llama_batch_ext_set_embd_token(res, idx, t.embd)) {
                 GGML_ABORT("%s: failed to set the embedding of token %d at index %d\n", __func__, t.id, i);
             }
-        } else if (t.embd_row >= 0) {
-            idx = llama_batch_ext_add_embd_row(res, t.seq_id, t.embd_row);
-            if (idx < 0) {
-                GGML_ABORT("%s: failed to add embd row %d at index %d (error %d, n = %d)\n", __func__, t.embd_row, i, idx, n);
-            }
-            llama_batch_ext_set_pos(res, idx, t.pos.data());
         } else {
             idx = llama_batch_ext_add_embd(res, t.seq_id, t.embd);
             if (idx < 0) {

common/common.h

@@ -910,6 +910,9 @@ std::wstring utf8_to_wstring(const std::string & str);
 std::string  wstring_to_utf8(const std::wstring & str);
 #endif
 
+// returns the path as a UTF-8 string, preserving its separators
+std::string fs_path_to_utf8(const std::filesystem::path & path);
+
 //
 // Environment utils
 //
@@ -1096,7 +1099,6 @@ struct common_batch {
         llama_embd   state; // non-owning view of the data passed to set_embd_state(), data == NULL if none
         std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
         int32_t      decision_order = 0; // see llama_batch_ext_set_decision_order()
-        int32_t      embd_row = -1; // see add_embd_row(), -1 if none
     };
 
     std::vector<token> tokens; // mirror of the entries, tokens[i] describes batch index i
@@ -1139,10 +1141,6 @@ struct common_batch {
     // pos points to n_pos positions
     int32_t add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output);
 
-    // add an entry whose embedding is row `row` of the layer input device buffer of the context's ctx_other
-    // pos points to n_pos positions
-    int32_t add_embd_row(int32_t row, const llama_pos * pos, llama_seq_id seq_id, bool output);
-
     int32_t size() const { return (int32_t) tokens.size(); }
 };
 

common/download.cpp

@@ -1005,7 +1005,7 @@ std::string common_docker_resolve_model(const std::string & docker) {
         std::string model_filename = repo;
         std::replace(model_filename.begin(), model_filename.end(), '/', '_');
         model_filename += "_" + tag + ".gguf";
-        std::string local_path = fs_get_cache_file(model_filename).string();
+        std::string local_path = fs_path_to_utf8(fs_get_cache_file(model_filename));
 
         const std::string blob_url = url_prefix + "/blobs/" + gguf_digest;
         common_download_opts opts;

common/hf-cache.cpp

@@ -63,7 +63,7 @@ static fs::path get_cache_directory() {
 }
 
 std::string get_cache_path() {
-    return get_cache_directory().string();
+    return fs_path_to_utf8(get_cache_directory());
 }
 
 static std::string folder_name_to_repo(const std::string & folder) {
@@ -340,11 +340,11 @@ hf_files get_repo_files(const std::string & repo_id,
             file.url = endpoint + repo_id + "/resolve/" + commit + "/" + file.path;
 
             fs::path final_path = commit_path / subpath;
-            file.final_path = final_path.string();
+            file.final_path = fs_path_to_utf8(final_path);
 
             if (!file.oid.empty() && !fs::exists(final_path)) {
                 fs::path local_path = blobs_path / file.oid;
-                file.local_path = local_path.string();
+                file.local_path = fs_path_to_utf8(local_path);
             } else {
                 file.local_path = file.final_path;
             }
@@ -411,7 +411,7 @@ hf_files get_cached_files(const std::string & repo_id) {
         if (!fs::exists(snapshots_path)) {
             continue;
         }
-        std::string _repo_id = folder_name_to_repo(repo.path().filename().string());
+        std::string _repo_id = folder_name_to_repo(fs_path_to_utf8(repo.path().filename()));
 
         if (!is_valid_repo_id(_repo_id)) {
             continue;
@@ -436,7 +436,7 @@ hf_files get_cached_files(const std::string & repo_id) {
                 file.repo_id = _repo_id;
                 const auto generic_path = path.generic_u8string();
                 file.path = std::string(generic_path.begin(), generic_path.end());
-                file.local_path = entry.path().string();
+                file.local_path = fs_path_to_utf8(entry.path());
                 file.final_path = file.local_path;
                 files.push_back(std::move(file));
             }
@@ -516,7 +516,7 @@ bool remove_cached_repo(const std::string & repo_id) {
     std::error_code ec;
     auto removed = fs::remove_all(repo_path, ec);
     if (ec) {
-        LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str());
+        LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, fs_path_to_utf8(repo_path).c_str(), ec.message().c_str());
         return false;
     }
     return removed > 0;

common/jinja/runtime.cpp

@@ -307,13 +307,58 @@ value binary_expression::execute_impl(context & ctx) const {
     throw std::runtime_error("Unknown operator \"" + op.value + "\" between " + left_val->type() + " and " + right_val->type());
 }
 
+static value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false) {
+    JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str());
+    if (ctx.is_get_stats) {
+        value_t::stats_t::mark_used(input);
+        input->stats.ops.insert(name);
+    }
+    auto builtins = input->get_builtins();
+    auto it = builtins.find(name);
+    if (it != builtins.end()) {
+        JJ_DEBUG("Binding built-in '%s'", name.c_str());
+        return mk_val<value_func>(name, it->second, input);
+    }
+    if (undef_on_missing) {
+        return mk_val<value_undefined>(name);
+    }
+    throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type());
+}
+
 static value apply_filter(context & ctx, const statement_ptr & filter, value input) {
     JJ_DEBUG("Applying filter to %s", input->type().c_str());
 
+    auto set_filter_alias = [](auto & filter_id) {
+        if (filter_id == "count") {
+            filter_id = "length";
+        } else if (filter_id == "d") {
+            filter_id = "default";
+        } else if (filter_id == "e") {
+            filter_id = "escape";
+        } else if (filter_id == "trim") {
+            filter_id = "strip";
+        }
+    };
+
     if (is_stmt<identifier>(filter)) {
         auto filter_id = cast_stmt<identifier>(filter)->val;
+
+        set_filter_alias(filter_id);
         JJ_DEBUG("Applying filter '%s' to %s", filter_id.c_str(), input->type().c_str());
-        return apply_filter_by_name(ctx, filter_id, input, func_args(ctx));
+        // TODO: Refactor filters so this coercion can be done automatically
+        if (!input->is_undefined() && !is_val<value_string>(input) && (
+            filter_id == "capitalize" ||
+            filter_id == "lower" ||
+            filter_id == "replace" ||
+            filter_id == "strip" ||
+            filter_id == "title" ||
+            filter_id == "upper" ||
+            filter_id == "wordcount"
+        )) {
+            JJ_DEBUG("Coercing %s to String for '%s' filter", input->type().c_str(), filter_id.c_str());
+            input = mk_val<value_string>(input->as_string());
+        }
+        return try_builtin_func(ctx, filter_id, input)->invoke(func_args(ctx));
 
     } else if (is_stmt<call_expression>(filter)) {
         auto call = cast_stmt<call_expression>(filter);
@@ -322,13 +367,14 @@ static value apply_filter(context & ctx, const statement_ptr & filter, value inp
         }
         auto filter_id = cast_stmt<identifier>(call->callee)->val;
 
+        set_filter_alias(filter_id);
         JJ_DEBUG("Applying filter '%s' with arguments to %s", filter_id.c_str(), input->type().c_str());
         func_args args(ctx);
         for (const auto & arg_expr : call->args) {
             args.push_back(arg_expr->execute(ctx));
         }
 
-        return apply_filter_by_name(ctx, filter_id, input, args);
+        return try_builtin_func(ctx, filter_id, input)->invoke(args);
 
     } else {
         throw std::runtime_error("Invalid filter expression");

common/jinja/value.cpp

@@ -72,17 +72,6 @@ const std::vector<value> & func_args::get_args() const {
     return args;
 }
 
-void func_args::ensure_kwargs(std::initializer_list<const char *> names) const {
-    for (const auto & arg : args) {
-        if (is_val<value_kwarg>(arg)) {
-            auto * kwarg = cast_val<value_kwarg>(arg);
-            if (std::find(names.begin(), names.end(), kwarg->key) == names.end()) {
-                throw raised_exception("Function '" + func_name + "' got an unexpected keyword argument '" + kwarg->key + "'");
-            }
-        }
-    }
-}
-
 /**
  * Function that mimics Python's array slicing.
  */
@@ -145,14 +134,6 @@ static value empty_value_fn(const func_args &) {
         return mk_val<T>();
     }
 }
-
-template<typename T>
-static value empty_value_no_args_fn(const func_args & args) {
-    args.ensure_kwargs();
-    args.ensure_count(1, 1);
-    return empty_value_fn<T>(args);
-}
-
 template<typename T>
 static value test_type_fn(const func_args & args) {
     args.ensure_count(1);
@@ -613,52 +594,6 @@ const func_builtins & global_builtins() {
     return builtins;
 }
 
-value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing) {
-    JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str());
-    if (ctx.is_get_stats) {
-        value_t::stats_t::mark_used(input);
-        input->stats.ops.insert(name);
-    }
-    auto builtins = input->get_builtins();
-    auto it = builtins.find(name);
-    if (it != builtins.end()) {
-        JJ_DEBUG("Binding built-in '%s'", name.c_str());
-        return mk_val<value_func>(name, it->second, input);
-    }
-    if (undef_on_missing) {
-        return mk_val<value_undefined>(name);
-    }
-    throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type());
-}
-
-value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args) {
-    if (filter_id == "count") {
-        filter_id = "length";
-    } else if (filter_id == "d") {
-        filter_id = "default";
-    } else if (filter_id == "e") {
-        filter_id = "escape";
-    } else if (filter_id == "trim") {
-        filter_id = "strip";
-    }
-
-    // TODO: Refactor filters so this coercion can be done automatically
-    if (!input->is_undefined() && !is_val<value_string>(input) && (
-        filter_id == "capitalize" ||
-        filter_id == "lower" ||
-        filter_id == "replace" ||
-        filter_id == "strip" ||
-        filter_id == "title" ||
-        filter_id == "upper" ||
-        filter_id == "wordcount"
-    )) {
-        JJ_DEBUG("Coercing %s to String for '%s' filter", input->type().c_str(), filter_id.c_str());
-        input = mk_val<value_string>(input->as_string());
-    }
-
-    return try_builtin_func(ctx, filter_id, input)->invoke(args);
-}
-
 
 const func_builtins & value_int_t::get_builtins() const {
     static const func_builtins builtins = {
@@ -732,22 +667,16 @@ const func_builtins & value_string_t::get_builtins() const {
     static const func_builtins builtins = {
         {"default", default_value},
         {"upper", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().uppercase();
             return mk_val<value_string>(str);
         }},
         {"lower", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().lowercase();
             return mk_val<value_string>(str);
         }},
         {"strip", [](const func_args & args) -> value {
-            args.ensure_kwargs({"chars"});
-            args.ensure_count(1, 2);
             value val_input = args.get_pos(0);
             if (!is_val<value_string>(val_input)) {
                 throw raised_exception("strip() first argument must be a string");
@@ -778,15 +707,11 @@ const func_builtins & value_string_t::get_builtins() const {
             }
         }},
         {"title", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().titlecase();
             return mk_val<value_string>(str);
         }},
         {"capitalize", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().capitalize();
             return mk_val<value_string>(str);
@@ -1088,10 +1013,6 @@ const func_builtins & value_bool_t::get_builtins() const {
     return builtins;
 }
 
-[[noreturn]] static value array_sum_not_implemented(const func_args &) {
-    throw not_implemented_exception("Array sum builtin not implemented");
-}
-
 const func_builtins & value_array_t::get_builtins() const {
     static const func_builtins builtins = {
         {"default", default_value},
@@ -1158,9 +1079,7 @@ const func_builtins & value_array_t::get_builtins() const {
         {"select", select_or_reject<false, false>},
         {"rejectattr", select_or_reject<true, true>},
         {"reject", select_or_reject<true, false>},
-        {"sum", array_sum_not_implemented},
         {"join", [](const func_args & args) -> value {
-            args.ensure_kwargs({"d", "attribute"});
             args.ensure_count(1, 3);
             if (!is_val<value_array>(args.get_pos(0))) {
                 throw raised_exception("join() first argument must be an array");
@@ -1198,28 +1117,13 @@ const func_builtins & value_array_t::get_builtins() const {
         {"tojson", tojson},
         {"map", [](const func_args & args) -> value {
             args.ensure_count(2);
-            value val = args.get_pos(0);
-            if (!is_val<value_array>(val)) {
+            if (!is_val<value_array>(args.get_pos(0))) {
                 throw raised_exception("map: first argument must be an array");
             }
             if (!is_val<value_kwarg>(args.get_args().at(1))) {
-                value filter = args.get_pos(1);
-                if (!is_val<value_string>(filter)) {
-                    throw raised_exception("map: filter name must be a string");
-                }
-                auto out = mk_val<value_array>();
-                func_args filter_args(args.ctx);
-                const auto & all_args = args.get_args();
-                for (size_t i = 2; i < all_args.size(); ++i) {
-                    filter_args.push_back(all_args[i]);
-                }
-                const std::string filter_id = filter->as_string().str();
-                auto arr = val->as_array();
-                for (const auto & item : arr) {
-                    out->push_back(apply_filter_by_name(args.ctx, filter_id, item, filter_args));
-                }
-                return is_val<value_tuple>(val) ? mk_val<value_tuple>(out->as_array()) : out;
+                throw not_implemented_exception("map: filter-mapping not implemented");
             }
+            value val       = args.get_pos(0);
             value attribute = args.get_kwarg_or_pos("attribute", 1);
             value default_val = args.get_kwarg("default", mk_val<value_undefined>());
             auto out = mk_val<value_array>();
@@ -1228,7 +1132,7 @@ const func_builtins & value_array_t::get_builtins() const {
                 value attr_val = get_attribute(item, attribute, default_val);
                 out->push_back(attr_val);
             }
-            return is_val<value_tuple>(val) ? mk_val<value_tuple>(out->as_array()) : out;
+            return is_val<value_tuple>(val) ? mk_val<value_tuple>(std::move(out->as_array())) : out;
         }},
         {"append", [](const func_args & args) -> value {
             args.ensure_count(2);
@@ -1501,18 +1405,14 @@ const func_builtins & value_none_t::get_builtins() const {
 const func_builtins & value_undefined_t::get_builtins() const {
     static const func_builtins builtins = {
         {"default", default_value},
-        {"capitalize", empty_value_no_args_fn<value_string>},
+        {"capitalize", empty_value_fn<value_string>},
         {"first", empty_value_fn<value_undefined>},
         {"items", empty_value_fn<value_array>},
-        {"join", [](const func_args & args) -> value {
-            args.ensure_kwargs({"d", "attribute"});
-            args.ensure_count(1, 3);
-            return empty_value_fn<value_string>(args);
-        }},
+        {"join", empty_value_fn<value_string>},
         {"last", empty_value_fn<value_undefined>},
         {"length", empty_value_fn<value_int>},
         {"list", empty_value_fn<value_array>},
-        {"lower", empty_value_no_args_fn<value_string>},
+        {"lower", empty_value_fn<value_string>},
         {"map", empty_value_fn<value_array>},
         {"max", empty_value_fn<value_undefined>},
         {"min", empty_value_fn<value_undefined>},
@@ -1525,16 +1425,12 @@ const func_builtins & value_undefined_t::get_builtins() const {
         {"selectattr", empty_value_fn<value_array>},
         {"sort", empty_value_fn<value_array>},
         {"string", empty_value_fn<value_string>},
-        {"strip", [](const func_args & args) -> value {
-            args.ensure_kwargs({"chars"});
-            args.ensure_count(1, 2);
-            return empty_value_fn<value_string>(args);
-        }},
+        {"strip", empty_value_fn<value_string>},
         {"sum", empty_value_fn<value_int>},
-        {"title", empty_value_no_args_fn<value_string>},
+        {"title", empty_value_fn<value_string>},
         {"truncate", empty_value_fn<value_string>},
         {"unique", empty_value_fn<value_array>},
-        {"upper", empty_value_no_args_fn<value_string>},
+        {"upper", empty_value_fn<value_string>},
         {"wordcount", empty_value_fn<value_int>},
     };
     return builtins;

common/jinja/value.h

@@ -7,7 +7,6 @@
 #include <cmath>
 #include <cstdint>
 #include <functional>
-#include <initializer_list>
 #include <map>
 #include <memory>
 #include <set>
@@ -177,8 +176,6 @@ protected:
 //
 
 const func_builtins & global_builtins();
-value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false);
-value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args);
 
 std::string value_to_json(const value & val, int indent = -1, const std::string_view item_sep = ", ", const std::string_view key_sep = ": ");
 
@@ -661,7 +658,6 @@ public:
     size_t count() const { return args.size(); }
     void push_back(const value & val);
     void push_front(const value & val);
-    void ensure_kwargs(std::initializer_list<const char *> names = {}) const;
     void ensure_count(size_t min, size_t max = 999) const {
         size_t n = args.size();
         if (n < min || n > max) {

common/parsers/gpt-oss.cpp

@@ -71,7 +71,6 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
         auto content         = p.rule("message-content", p.until("<|end|>"));
         auto channel         = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis"));
         auto constrain_type  = p.chars("[A-Za-z0-9_-]", 1, -1);
-        auto constraint      = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
 
         // Occasionally, gpt-oss-20b will prefix channels with this commentary
         auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant")));
@@ -85,7 +84,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
 
         auto analysis = p.ref("analysis");
         auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end);
-        auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final") + constraint + p.literal("<|message|>") + p.content(content));
+        auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final<|message|>") + p.content(content));
 
         // Consume any unsolicited tool calls, e.g. builtin functions
         auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end));
@@ -93,6 +92,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
         auto any = p.rule("any", preamble | analysis);
 
         if (has_response_format) {
+            auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
             auto response_format = p.rule("response-format",
                 p.literal("<|channel|>final") + constraint + p.literal("<|message|>") +
                 p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));
@@ -109,6 +109,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
                 const auto   params   = common_chat_tool_parameters(function);
 
                 auto func_name  = p.literal(" to=functions.") + p.tool_name(p.literal(name));
+                auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
                 auto args       = p.tool_args(p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-schema", params));
 
                 // recipient in role header

common/parsers/qwen3-coder.cpp

@@ -74,24 +74,16 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
     data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
         auto generation_prompt = p.literal(GEN_PREFIX);
 
-        // The template strips the newline before </think>, so leave it out of the reasoning
-        auto think_body  = p.until_one_of({ "\n</think>", "</think>", "<tool_call>" });
-        auto think_close = p.literal("\n</think>") | p.literal("</think>") | p.peek(p.literal("<tool_call>"));
-
-        auto reasoning_extracted = p.optional("<think>" + p.space() + p.reasoning(think_body) + think_close);
-        auto reasoning_skipped   = p.optional("<think>" + p.space() + think_body + think_close);
-
         auto reasoning = p.eps();
         if (supports_reasoning && extract_reasoning) {
-            reasoning = reasoning_extracted;
+            // The template strips the newline before </think>, so leave it out of the reasoning
+            reasoning = p.optional("<think>" + p.space() +
+                                   p.reasoning(p.until_one_of({ "\n</think>", "</think>", "<tool_call>" })) +
+                                   (p.literal("\n</think>") | p.literal("</think>") | p.peek(p.literal("<tool_call>"))));
         }
 
         // Response format parser
         if (has_response_format) {
-            // The answer must be bare JSON, so the think block is consumed even when it is not extracted
-            if (supports_reasoning && !extract_reasoning) {
-                reasoning = reasoning_skipped;
-            }
             return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));
         }
 

common/preset.cpp

@@ -171,12 +171,12 @@ static std::map<std::string, std::map<std::string, std::string>> parse_ini_from_
     std::map<std::string, std::map<std::string, std::string>> parsed;
 
     if (!std::filesystem::exists(path)) {
-        throw std::runtime_error("preset file does not exist: " + path.string());
+        throw std::runtime_error("preset file does not exist: " + fs_path_to_utf8(path));
     }
 
     std::ifstream file(path);
     if (!file.good()) {
-        throw std::runtime_error("failed to open server preset file: " + path.string());
+        throw std::runtime_error("failed to open server preset file: " + fs_path_to_utf8(path));
     }
 
     std::string contents((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
@@ -225,7 +225,7 @@ static std::map<std::string, std::map<std::string, std::string>> parse_ini_from_
     common_peg_parse_context ctx(contents);
     const auto result = parser.parse(ctx);
     if (!result.success()) {
-        throw std::runtime_error("failed to parse server config file: " + path.string());
+        throw std::runtime_error("failed to parse server config file: " + fs_path_to_utf8(path));
     }
 
     std::string current_section = COMMON_PRESET_DEFAULT_NAME;
@@ -323,7 +323,7 @@ common_presets common_preset_context::load_from_ini(const std::filesystem::path
                 }
                 LOG_DBG("accepted option: %s = %s\n", key.c_str(), preset.options[opt].c_str());
             } else if (ignore_unknown_keys) {
-                LOG_WRN("ignoring option '%s' from %s: not supported by this program\n", key.c_str(), path.string().c_str());
+                LOG_WRN("ignoring option '%s' from %s: not supported by this program\n", key.c_str(), fs_path_to_utf8(path).c_str());
             } else {
                 throw std::runtime_error(string_format(
                     "option '%s' not recognized in preset '%s'",
@@ -401,7 +401,7 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
             if (!entry.is_regular_file(ec)) {
                 continue;
             }
-            const std::string fname = entry.path().filename().string();
+            const std::string fname = fs_path_to_utf8(entry.path().filename());
             if (!string_ends_with(fname, ".gguf")) {
                 continue;
             }
@@ -422,9 +422,9 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
         if (!path.empty()) {
             models.push_back({
                 /* name        */ name,
-                /* path        */ path.string(),
-                /* path_mmproj */ mmproj_file.string(), // can be empty
-                /* path_draft  */ draft_file.string()   // can be empty
+                /* path        */ fs_path_to_utf8(path),
+                /* path_mmproj */ fs_path_to_utf8(mmproj_file), // can be empty
+                /* path_draft  */ fs_path_to_utf8(draft_file)    // can be empty
             });
         }
     };
@@ -432,13 +432,13 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
     for (const auto & entry : std::filesystem::directory_iterator(dir)) {
         std::error_code ec;
         if (entry.is_directory(ec)) {
-            scan_subdir(entry.path(), entry.path().filename().string());
+            scan_subdir(entry.path(), fs_path_to_utf8(entry.path().filename()));
             continue;
         }
         if (!entry.is_regular_file(ec)) {
             continue;
         }
-        const std::string fname = entry.path().filename().string();
+        const std::string fname = fs_path_to_utf8(entry.path().filename());
         if (!string_ends_with(fname, ".gguf")) {
             continue;
         }
@@ -450,7 +450,7 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
         string_replace_all(name, ".gguf", "");
         models.push_back({
             /* name        */ name,
-            /* path        */ entry.path().string(),
+            /* path        */ fs_path_to_utf8(entry.path()),
             /* path_mmproj */ "",
             /* path_draft  */ ""
         });

common/speculative.cpp

@@ -1027,9 +1027,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
     const int32_t * target_layer_ids   = nullptr; // model_dft's extract layer indices
     uint32_t        target_layer_ids_n = 0;
 
-    // inject target features from the target's device buffer instead of host copies (env LLAMA_DFLASH_DEV=0 disables)
-    bool use_dev_features = false;
-
     common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq,
             common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)
         : common_speculative_impl(type, n_seq, params.draft.n_max)
@@ -1144,23 +1141,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
             llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true);
         }
 
-        // the device buffer keeps layers in ascending id order, the fc input expects target_layer_ids order
-        {
-            const char * env = getenv("LLAMA_DFLASH_DEV");
-            const bool ascending = std::adjacent_find(target_layer_ids, target_layer_ids + target_layer_ids_n,
-                    [](int32_t a, int32_t b) { return a >= b; }) == target_layer_ids + target_layer_ids_n;
-
-            use_dev_features = !is_lilicorr && ascending && (env == nullptr || atoi(env) != 0);
-            if (use_dev_features) {
-                llama_set_embeddings_layer_inp_dev(ctx_tgt, true);
-                use_dev_features = llama_supports_embd_row(ctx_dft);
-                if (!use_dev_features) {
-                    llama_set_embeddings_layer_inp_dev(ctx_tgt, false);
-                }
-            }
-            LOG_INF("%s: DFlash target features via %s\n", __func__, use_dev_features ? "device buffer" : "host copies");
-        }
-
         // DFlash2 reads its selector lattice from h_nextn and never consumes raw logits.
         llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ !is_dflash2 && !is_lilicorr);
         llama_set_causal_attn(ctx_dft, causal_attn); // DFlash needs non-causal attention unless the model says otherwise
@@ -1253,25 +1233,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
             for (int32_t offset = 0; offset < n_rows; offset += n_ubatch) {
                 const int32_t n_chunk = std::min(n_ubatch, n_rows - offset);
 
-                if (use_dev_features) {
-                    // the features are the target batch rows in the target's device buffer
-                    batch_inject.clear();
-                    for (int32_t i = 0; i < n_chunk; ++i) {
-                        const int32_t   row        = i_batch_beg[seq_id] + offset + i;
-                        const llama_pos p          = batch_in.tokens[row].pos[0];
-                        const llama_pos pos_arr[4] = { p, p, p, 0 };
-                        batch_inject.add_embd_row(row, pos_arr, seq_id, false);
-                    }
-
-                    const int32_t rc = llama_process(ctx_dft, LLAMA_PROCESS_TYPE_DECODE, batch_inject.get());
-                    if (rc != 0) {
-                        LOG_ERR("%s: llama_process(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n",
-                                __func__, rc, (int) n_chunk, (int) offset);
-                        return false;
-                    }
-                    continue;
-                }
-
                 // gather target features per extract layer; the fused decode encodes and
                 // injects them into the K/V cache at the target positions
                 features_buf.resize((size_t) n_chunk * n_embd_enc);

conversion/__init__.py

@@ -46,7 +46,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "ChatGLMForConditionalGeneration": "chatglm",
     "ChatGLMModel": "chatglm",
     "ClefModel": "clef",
-    "CloverMoeModelLM": "clover",
     "CodeShellForCausalLM": "codeshell",
     "CogVLMForCausalLM": "cogvlm",
     "Cohere2MoeForCausalLM": "command_r",

conversion/clover.py

@@ -1,41 +0,0 @@
-from __future__ import annotations
-
-from typing import Callable, Iterable, TYPE_CHECKING
-
-from .base import ModelBase, gguf
-from .qwen import _LinearAttentionVReorderBase
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-
-@ModelBase.register("CloverMoeModelLM")
-@ModelBase.example("Diffuspace/Clover-35B-A3B-Preview")
-class CloverMoeModel(_LinearAttentionVReorderBase):
-    model_arch = gguf.MODEL_ARCH.CLOVER_MOE
-    supports_mtp_export = False
-
-    def __init__(self, *args, **kwargs):
-        self.no_mtp = True
-        super().__init__(*args, **kwargs)
-
-    def set_vocab(self):
-        self._set_vocab_gpt2()
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        self.gguf_writer.add_expert_shared_count(self.hparams.get("num_shared_experts", 1))
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, _ = item
-        if name.endswith((".conv_right_weight", ".bd_softmax_logit_scale", ".bd_softmax_history_bias", ".bd_softmax_history_c")):
-            return None
-
-        return super().filter_tensors(item)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.endswith(".A_log") or (name.endswith("norm.weight") and not name.endswith("linear_attn.norm.weight")):
-            data_torch = data_torch.float()
-
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/pockettts.py

@@ -5,7 +5,6 @@ from pathlib import Path
 from typing import Any, Iterable, TYPE_CHECKING
 
 import torch
-import torch.nn.functional as F
 
 if TYPE_CHECKING:
     from torch import Tensor
@@ -46,102 +45,6 @@ def _tensor_shapes(dir_model: Path) -> dict[str, tuple[int, ...]]:
         return {name: tuple(part[name].shape) for name in part.keys()}
 
 
-def _has_voice_encoder(dir_model: Path) -> bool:
-    # without voice cloning, the mimi encoder weights are all zero
-    part_names = ModelBase.get_model_part_names(dir_model, "model", ".safetensors")
-    with gguf.utility.SafetensorsLocal(dir_model / part_names[0]) as part:
-        nonzero = {bool(part[name].mmap_bytes().any()) for name in part.keys() if name.startswith("mimi.encoder")}
-    if len(nonzero) != 1:
-        raise ValueError(f"{dir_model}: the mimi encoder is partly zeroed out")
-    return nonzero.pop()
-
-
-def _load_safetensors(path: Path, prefixes: tuple[str, ...]) -> dict[str, Tensor]:
-    dtypes = {"F32": torch.float32, "BF16": torch.bfloat16, "I64": torch.int64}
-    with gguf.utility.SafetensorsLocal(path) as part:
-        return {name: torch.from_numpy(part[name].mmap_bytes()).view(dtypes[part[name].dtype]).reshape(part[name].shape)
-                for name in part.keys() if name.startswith(prefixes)}
-
-
-def _rope_pairs(x: Tensor, n_head: int, rope_theta: float) -> Tensor:
-    # ggml NORM mode rope, x: [n_pos, n_embd]
-    n_pos, n_embd = x.shape
-    head_dim = n_embd // n_head
-    angle = torch.arange(n_pos, dtype=x.dtype)[:, None, None] * \
-        torch.pow(rope_theta, -torch.arange(0, head_dim, 2, dtype=x.dtype) / head_dim)
-    x = x.view(n_pos, n_head, head_dim // 2, 2)
-    return torch.stack([x[..., 0] * angle.cos() - x[..., 1] * angle.sin(),
-                        x[..., 0] * angle.sin() + x[..., 1] * angle.cos()], -1).reshape(n_pos, n_embd)
-
-
-def _voice_prompt_embd(weights: dict[str, Tensor], kv: list[Tensor], n_head: int, rope_theta: float) -> tuple[Tensor, float]:
-    # the voice state only has K/V of each layer (kv[il] is [k/v, pos, n_embd]): layer 0 V gives each frame up to its scale, layer 1 V gives the scale
-    # returns the frames without bos_before_voice, and the max relative K/V error when the frames are prompted again
-    w = {name: t.double() for name, t in weights.items()}
-    kv = [t.double() for t in kv]
-    k0, v0, v1 = kv[0][0], kv[0][1], kv[1][1]
-    n_pos, n_embd = v0.shape
-    head_dim = n_embd // n_head
-
-    def p(il: int, name: str) -> Tensor:
-        return w[f"flow_lm.transformer.layers.{il}.{name}"]
-
-    def norm(x: Tensor, il: int, name: str) -> Tensor:
-        return F.layer_norm(x, (n_embd,), p(il, f"{name}.weight"), p(il, f"{name}.bias"), 1e-5)
-
-    def heads(t: Tensor) -> Tensor:
-        return t.reshape(n_pos, n_head, head_dim).transpose(0, 1)
-
-    def rows(t: Tensor) -> Tensor:
-        return t.transpose(0, 1).reshape(n_pos, n_embd)
-
-    def attn(il: int, h: Tensor, k: Tensor, v: Tensor) -> Tensor:
-        q = _rope_pairs(h @ p(il, "self_attn.in_proj.weight").chunk(3)[0].T, n_head, rope_theta)
-        return rows(F.scaled_dot_product_attention(heads(q), heads(k), heads(v), is_causal=True)) @ p(il, "self_attn.out_proj.weight").T
-
-    def ffn(il: int, x: Tensor) -> Tensor:
-        return F.gelu(norm(x, il, "norm2") @ p(il, "linear1.weight").T) @ p(il, "linear2.weight").T
-
-    wv = p(0, "self_attn.in_proj.weight").chunk(3)[2]
-    h = torch.linalg.solve(wv, v0.T).T                   # norm1(x)
-    z = (h - p(0, "norm1.bias")) / p(0, "norm1.weight")  # (x - mean(x)) / std(x)
-
-    # fit z on [speaker_proj, 1] to get the latent direction back
-    proj = w["flow_lm.speaker_proj_weight"]
-    fit = torch.linalg.lstsq(torch.cat([proj, torch.ones(n_embd, 1, dtype=proj.dtype)], 1), z[1:].T).solution
-    frames = (proj @ fit[:-1]).T
-
-    a0 = attn(0, h, k0, v0)[1:]
-    wv1 = p(1, "self_attn.in_proj.weight").chunk(3)[2]
-
-    def err(scale: Tensor) -> Tensor:  # per frame relative error of layer 1 V
-        x = scale[:, None] * frames + a0
-        x = x + ffn(0, x)
-        return (norm(x, 1, "norm1") @ wv1.T - v1[1:]).norm(dim=1) / v1[1:].norm(dim=1)
-
-    grid = torch.logspace(-3, 3, 49, dtype=torch.float64)
-    best = torch.stack([err(s.expand(n_pos - 1)) for s in grid]).argmin(0)
-    lo, hi = grid[(best - 1).clamp(min=0)], grid[(best + 1).clamp(max=len(grid) - 1)]
-    for _ in range(40):  # golden section search, all frames at once
-        m1, m2 = hi - 0.618 * (hi - lo), lo + 0.618 * (hi - lo)
-        left = err(m1) < err(m2)
-        lo, hi = torch.where(left, lo, m1), torch.where(left, m2, hi)
-    frames = (lo + hi)[:, None] / 2 * frames
-
-    # prompt the frames again and compare K/V with the shipped state
-    x = torch.cat([w["flow_lm.bos_before_voice"].reshape(1, n_embd), frames])
-    drift = 0.0
-    for il, (k_ref, v_ref) in enumerate(kv):
-        h = norm(x, il, "norm1")
-        _, wk, wv = p(il, "self_attn.in_proj.weight").chunk(3)
-        k, v = _rope_pairs(h @ wk.T, n_head, rope_theta), h @ wv.T
-        for a, b in ((k, k_ref), (v, v_ref)):
-            drift = max(drift, ((a - b).norm(dim=1) / b.norm(dim=1)).median().item())
-        x = x + attn(il, h, k, v)
-        x = x + ffn(il, x)
-    return frames, drift
-
-
 @ModelBase.register_hparams_loader(lambda dir_model: "flow_lm.bos_emb" in _tensor_shapes(dir_model))
 def _load_hparams(dir_model: Path) -> dict[str, Any]:
     logger.info("gguf: detected pocket-tts checkpoint, deriving hparams from tensor shapes")
@@ -291,27 +194,22 @@ class PocketTTSMmprojModel(MmprojModel):
         (gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V),
     )
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.has_audio_encoder = _has_voice_encoder(self.dir_model)
-
     def set_gguf_parameters(self):
         self.gguf_writer.add_file_type(self.ftype)
         assert self.hparams_audio is not None
 
         # voice-prompt encoder: mimi encoder + speaker_proj
-        if self.has_audio_encoder:
-            self.gguf_writer.add_clip_has_audio_encoder(True)
-            # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models
-            self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC)
-            self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
-            self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"])
-            self.gguf_writer.add_audio_embedding_length(self.hparams_audio["hidden_size"])
-            self.gguf_writer.add_audio_feed_forward_length(self.hparams_audio["intermediate_size"])
-            self.gguf_writer.add_audio_head_count(self.hparams_audio["num_attention_heads"])
-            self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
-            # mimi convolves the waveform directly, it is passed around as a 1-row "mel"
-            self.gguf_writer.add_audio_num_mel_bins(1)
+        self.gguf_writer.add_clip_has_audio_encoder(True)
+        # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models
+        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC)
+        self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
+        self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"])
+        self.gguf_writer.add_audio_embedding_length(self.hparams_audio["hidden_size"])
+        self.gguf_writer.add_audio_feed_forward_length(self.hparams_audio["intermediate_size"])
+        self.gguf_writer.add_audio_head_count(self.hparams_audio["num_attention_heads"])
+        self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
+        # mimi convolves the waveform directly, it is passed around as a 1-row "mel"
+        self.gguf_writer.add_audio_num_mel_bins(1)
 
         # generation: flow-matching decoder + mimi decoder
         # the SEANet and flow net hparams are constant across the family, clip.cpp holds them
@@ -331,57 +229,12 @@ class PocketTTSMmprojModel(MmprojModel):
         # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path
         if ".seanet." in new_name or new_name in ("a.downsample.conv.weight", "a.gen.wav.upsample.weight"):
             return gguf.GGMLQuantizationType.F16
-        # voices are read on the host
-        if new_name.startswith("voice."):
-            return gguf.GGMLQuantizationType.F32
         return False
 
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        # embeddings/<name>.safetensors hold the K/V state after the voice prompt (see get_state_for_audio_prompt() in the reference), turn it back into prompt embeddings
-        paths = sorted((self.dir_model / "embeddings").glob("*.safetensors"))
-        if not paths:
-            return
-        weights = _load_safetensors(self.dir_model / "model.safetensors",
-                                    ("flow_lm.transformer.layers.", "flow_lm.speaker_proj_weight", "flow_lm.bos_before_voice"))
-        n_layer = self.global_config["num_hidden_layers"]
-        for path in paths:
-            state = _load_safetensors(path, ("transformer.layers.",))
-            kv = []
-            for il in range(n_layer):
-                prefix = f"transformer.layers.{il}.self_attn/"
-                if any(prefix + k not in state for k in ("cache", "offset")):
-                    break
-                n_pos = int(state[prefix + "offset"].item())
-                # older packs have no pad entry
-                if prefix + "pad" in state and int(state[prefix + "pad"].item()) != 0:
-                    break
-                kv.append(state[prefix + "cache"][:, 0, :n_pos].flatten(2))  # [k/v, pos, n_embd]
-            if len(kv) != n_layer or any(t.shape != kv[0].shape for t in kv) or kv[0].shape[1] < 2 or \
-                    kv[0].shape[2] != self.global_config["hidden_size"]:
-                logger.warning(f"voice {path.stem}: skipped, not an unpadded voice state of this backbone")
-                continue
-            embd, drift = _voice_prompt_embd(weights, kv, self.global_config["num_attention_heads"],
-                                             self.global_config["rope_theta"])
-            # some packs ship a state made with other weights, the reference still uses it: keep up to 50% error, warn above 5%
-            if drift > 0.5:
-                logger.warning(f"voice {path.stem}: skipped, its state does not belong to these weights (error {drift:.1e})")
-                continue
-            log = logger.warning if drift > 0.05 else logger.info
-            log(f"voice {path.stem}: {embd.shape[0]} frames, prompting them again gives the shipped state within {drift:.1e}")
-            yield (f"voice.{path.stem}", embd.float())
-
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         del bid  # the block index of the mimi transformers is parsed here, not by the base class
         T = gguf.MODEL_TENSOR
 
-        if name.startswith("voice."):
-            yield (name, data_torch)
-            return
-
-        if not self.has_audio_encoder and (name.startswith(("mimi.encoder", "mimi.downsample."))
-                                           or name == "flow_lm.speaker_proj_weight"):
-            return  # voice-prompt encoder
-
         if name in ("flow_lm.bos_emb", "flow_lm.bos_before_voice", "flow_lm.conditioner.embed.weight"):
             return  # folded into the backbone embedding table
         if name.startswith("flow_lm.transformer.") or name.startswith("flow_lm.out_norm."):

docs/backend/CANN.md

@@ -161,7 +161,7 @@ lspci -n | grep -Eo '19e5:d[0-9a-f]{3}' | cut -d: -f2
 | FP16                   | Support | Support |
 | Q8_0                   | Support | Partial |
 | Q4_0                   | Support | Partial |
-| BF16                   | Support |    No   |
+| BF16                   | Support |         |
 
 > **310P note**
 > - `Q8_0`: data transform / buffer path is implemented, and `GET_ROWS` is supported, but quantized `MUL_MAT` / `MUL_MAT_ID` are not supported.
@@ -274,9 +274,7 @@ cmake --build build --config release
 
     **Notes**:
 
-      - On 910B the CANN backend supports FP16, BF16 and Q4_0/Q8_0 models. On 310P, BF16 is not
-        supported and quantized `MUL_MAT` / `MUL_MAT_ID` are not implemented, so only FP16 models
-        are supported (see [DataType Supports](#datatype-supports)).
+      - CANN backend only supports FP16/Q4_0/Q8_0 models currently.
 
 2. **Launch inference**
 
@@ -357,7 +355,3 @@ Enable ACL graph execution during the prefill stage, default is false. This opti
 ### GGML_CANN_OPERATOR_FUSION
 
 Enable operator fusion during computation, default is false. This option fuses compatible operators (e.g., ADD + RMS_NORM) to reduce overhead and improve performance.
-
-### GGML_CANN_NO_PINNED
-
-Disable pinned (host-locked) host buffers. When it is set (any value), `aclrtMallocHost` is not called: the CANN host buffer type allocates regular CPU memory instead and the backend reports no `host_buffer` capability, so host-side staging uses pageable memory. Use it when pinned host allocation fails, or when the pinned memory is needed by other processes on the host.

docs/build-s390x.md

@@ -249,7 +249,7 @@ IBM VXE/VXE2 SIMD acceleration depends on the BLAS implementation. It is strongl
 | MXFP4      | ✅           | ❓    | ❓     |
 | Q5_0       | ✅           | ❓    | ❓     |
 | Q5_1       | ✅           | ❓    | ❓     |
-| Q8_0       | ✅           | ✅    | ❓     |
+| Q8_0       | ✅           | ❓    | ❓     |
 | Q2_K       | 🚫           | ❓    | ❓     |
 | Q3_K       | ✅           | ❓    | ❓     |
 | Q4_K       | ✅           | ❓    | ❓     |
@@ -266,11 +266,11 @@ IBM VXE/VXE2 SIMD acceleration depends on the BLAS implementation. It is strongl
 | IQ1_M      | 🚫           | ❓    | ❓     |
 | IQ4_NL     | ✅           | ❓    | ❓     |
 | IQ4_XS     | ✅           | ❓    | ❓     |
-| FP32->FP16 | ✅           | ❓    | ❓     |
+| FP32->FP16 | 🚫           | ❓    | ❓     |
 | FP16->FP32 | 🚫           | ❓    | ❓     |
 
 -   ✅ - acceleration available
 -   🚫 - acceleration unavailable, will still run using scalar implementation
 -   ❓ - acceleration unknown, please contribute if you can test it yourself
 
-Last Updated by **Aaron Teo (aaron.teo1@ibm.com)** on Nov 3, 2026.
+Last Updated by **Aaron Teo (aaron.teo1@ibm.com)** on Sep 8, 2026.

docs/build.md

@@ -481,7 +481,7 @@ cmake --build build --config Release
 Now you can load the model in conversation mode using `Vulkan`
 
 ```sh
-build/bin/Release/llama-cli -m "[PATH TO MODEL]" -ngl 100 -c 16384 -t 10 -n -2
+build/bin/Release/llama-cli -m "[PATH TO MODEL]" -ngl 100 -c 16384 -t 10 -n -2 -cnv
 ```
 
 **MSYS2**

docs/ops.md

@@ -19,7 +19,7 @@ Legend:
 |                              ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             ADD1 | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 |                           ADD_ID | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                           ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                           ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                           ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          ARGSORT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
@@ -74,7 +74,7 @@ Legend:
 |                             MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ | ❌ |
 |                              MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 |
-|                 MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 |
+|                 MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | 🟡 |
 |                       MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ❌ |
 |                  MUL_MAT_ID_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                  MUL_MAT_ID_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |

docs/ops/OpenCL.csv

@@ -9242,31 +9242,31 @@
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=128,n=32,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=128,n=4,k=128,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=256,n=512,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=64,n=1,k=64,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=512,n=1,k=512,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=32,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=512,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=64,n=1,k=64,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=512,n=1,k=512,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=32,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=512,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=32,n=1,k=32,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=16384,n=1,k=16384,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=64,n=1,k=64,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=512,n=1,k=512,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=32,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=4,k=128,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=512,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=64,n=1,k=64,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=512,n=1,k=512,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=32,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=4,k=128,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=512,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=1024,n=1,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=2048,n=1,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=4096,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=8192,n=1,k=8192,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=1024,n=7,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=1024,n=1,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=2048,n=1,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=4096,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=8192,n=1,k=8192,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=1024,n=7,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=1024,n=1,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=2048,n=1,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=4096,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=8192,n=1,k=8192,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=1024,n=7,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
 "OpenCL","MUL_MAT_W4A8","type_a=nvfp4,type_b=f32,m=32,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
 "OpenCL","MUL_MAT_W4A8","type_a=nvfp4,type_b=f32,m=32,n=32,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
 "OpenCL","MUL_MAT_W4A8","type_a=nvfp4,type_b=f32,m=64,n=16,k=512,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"

docs/ops/WebGPU.csv

@@ -12326,16 +12326,8 @@
 "WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[512,34,2,1],pad_0=10,pad_1=9","support","0","no","WebGPU"
 "WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[3000,384,4,1],pad_0=10,pad_1=9","support","0","no","WebGPU"
 "WebGPU: WebGPU","ROLL","shift0=3,shift1=-2,shift3=1,shift4=-1","support","0","no","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=10.000000,step=1.000000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=3.000000,stop=3.500000,step=1.000000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-10.000000,stop=10.000000,step=0.300000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=62.750000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=63.000000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=63.250000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=254.750000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=255.000000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=255.250000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=1048576.000000,step=1.000000","support","1","yes","WebGPU"
+"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=10.000000,step=1.000000","support","0","no","WebGPU"
+"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=1048576.000000,step=1.000000","support","0","no","WebGPU"
 "WebGPU: WebGPU","TIMESTEP_EMBEDDING","type=f32,ne_a=[2,1,1,1],dim=320,max_period=10000","support","0","no","WebGPU"
 "WebGPU: WebGPU","LEAKY_RELU","type=f32,ne_a=[10,5,4,3],negative_slope=0.100000","support","0","no","WebGPU"
 "WebGPU: WebGPU","CUMSUM","type=f32,ne=[10,5,4,3]","support","1","yes","WebGPU"

ggml/include/ggml.h

@@ -2516,24 +2516,6 @@ extern "C" {
             float                 max_bias,
             float                 logit_softcap);
 
-    // same as ggml_flash_attn_ext, but each q slice attends to its own list of K/V rows
-    // q:       [n_embd_k, n_batch, n_head,    ne3]
-    // k:       [n_embd_k, n_rows,  n_head_kv, 1  ] shared by all q slices
-    // v:       [n_embd_v, n_rows,  n_head_kv, 1  ]
-    // mask:    [n_kv,     n_batch, ne32,      ne3]
-    // kv_rows: [n_kv,     ne3] I32, mask column i of slice i3 uses row kv_rows[i, i3] of k and v
-    // negative rows are padding, their mask entries must be -inf
-    GGML_API struct ggml_tensor * ggml_flash_attn_ext_rows(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * q,
-            struct ggml_tensor  * k,
-            struct ggml_tensor  * v,
-            struct ggml_tensor  * mask,
-            struct ggml_tensor  * kv_rows,
-            float                 scale,
-            float                 max_bias,
-            float                 logit_softcap);
-
     GGML_DEPRECATED(GGML_API void ggml_flash_attn_ext_set_prec(
             struct ggml_tensor * a,
             enum ggml_prec       prec),

ggml/src/ggml-cann/ggml-cann.cpp

@@ -2670,10 +2670,6 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
             return true;
         case GGML_OP_FLASH_ATTN_EXT:
             {
-                // K/V row lists (ggml_flash_attn_ext_rows) are not supported
-                if (op->src[5] != nullptr) {
-                    return false;
-                }
 #ifdef ASCEND_310P
                 // FA not support on 310p device
                 return false;

ggml/src/ggml-cpu/ggml-cpu-impl.h

@@ -30,7 +30,7 @@ struct ggml_compute_params {
 };
 
 
-#if defined(_MSC_VER) && !defined(__clang__)
+#if defined(_MSC_VER)
 
 #define m512bh(p) p
 #define m512i(p) p

ggml/src/ggml-cpu/ops.cpp

@@ -8844,7 +8844,6 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
     const ggml_tensor * v     = dst->src[2];
     const ggml_tensor * mask  = dst->src[3];
     const ggml_tensor * sinks = dst->src[4];
-    const ggml_tensor * kv_rows = dst->src[5];
 
     GGML_TENSOR_LOCALS(int64_t, neq, q,   ne)
     GGML_TENSOR_LOCALS(size_t,  nbq, q,   nb)
@@ -8952,8 +8951,6 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
         const float * pq = (const float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3));
         q_to_vec_dot(pq, Q_q, DK);
 
-        const int32_t * rows = kv_rows ? (const int32_t *) ((const char *) kv_rows->data + iq3*kv_rows->nb[1]) : nullptr;
-
         // online softmax / attention
         // loop over n_kv and n_head_kv
         // ref: https://arxiv.org/pdf/2112.05682.pdf
@@ -8964,14 +8961,9 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
                 continue;
             }
 
-            const int64_t ik1 = rows ? rows[ic] : ic;
-            if (ik1 < 0) {
-                continue;
-            }
-
             float s; // KQ value
 
-            const char * k_data = (const char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3);
+            const char * k_data = (const char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3);
             kq_vec_dot(DK, &s, 0, k_data, 0, Q_q, 0, 1);
 
             s = s*scale; // scale KQ value
@@ -8987,7 +8979,7 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
             float ms = 1.0f; // upon new higher max val, scale VKQ and KQ sum with this value
             float vs = 1.0f; // post-softmax KQ value, expf(s - M)
 
-            const char * v_data = ((const char *) v->data + (ik1*nbv1 + iv2*nbv2 + iv3*nbv3));
+            const char * v_data = ((const char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3));
 
             if (v->type == GGML_TYPE_F16) {
                 if (s > M) {
@@ -9154,10 +9146,6 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
     static constexpr int Q_TILE_SZ  = ggml_fa_tile_config::Q;
     static constexpr int KV_TILE_SZ = ggml_fa_tile_config::KV;
 
-    // with kv_rows the loop runs over the mask columns and reads K/V rows through kv_rows
-    const ggml_tensor * kv_rows = dst->src[5];
-    const int64_t n_kv_cols = kv_rows ? kv_rows->ne[0] : nek1;
-
     int ir = ir0;
     while (ir < ir1) {
         // q indices for the start of this tile
@@ -9165,10 +9153,6 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
         const int iq2 = (ir - iq3*neq2*neq1)/neq1;
         const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);
 
-        // padding rows are masked, read row 0 for them
-        const int32_t * rows = kv_rows ? (const int32_t *) ((const char *) kv_rows->data + iq3*kv_rows->nb[1]) : nullptr;
-        auto kv_row = [rows](int64_t i) -> int64_t { return rows ? std::max<int32_t>(rows[i], 0) : i; };
-
         // Number of valid rows in this tile:
         // - limited by tile size (Q_TILE_SZ)
         // - limited by chunk boundary (ir1 - ir)
@@ -9228,8 +9212,8 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
         memset(K_f32, 0, DK * KV_TILE_SZ * sizeof(float));
         memset(V32,   0, KV_TILE_SZ * DV * sizeof(float));
 
-        for (int64_t ic = 0; ic < n_kv_cols; ic += KV_TILE_SZ) {
-            const int kv_tile = (int)std::min((int64_t)KV_TILE_SZ, n_kv_cols - ic);
+        for (int64_t ic = 0; ic < nek1; ic += KV_TILE_SZ) {
+            const int kv_tile = (int)std::min((int64_t)KV_TILE_SZ, nek1 - ic);
 
             // skip the tile entirely if all the masks are -inf
             if (mask) {
@@ -9256,7 +9240,7 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
             // Pack K tile transposed: K_f32[dk][kv] so KV_TILE is contiguous (SIMD dim)
             // Zero-pad the last tile so the GEMM always operates on KV_TILE_SZ columns
             for (int tk = 0; tk < kv_tile; tk++) {
-                const char * k_data = (const char *)k->data + kv_row(ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3;
+                const char * k_data = (const char *)k->data + (ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3;
                 if (kv_type == GGML_TYPE_F16) {
                     const ggml_fp16_t * k_f16 = (const ggml_fp16_t *)k_data;
                     for (int64_t dk = 0; dk < DK; dk++) {
@@ -9321,7 +9305,7 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
             // V accumulation: VKQ32 += softmax(KQ) * V
             // Pack V tile to contiguous F32, zero-padded
             for (int tk = 0; tk < kv_tile; tk++) {
-                const char * v_data = (const char *)v->data + kv_row(ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;
+                const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;
                 if (kv_type == GGML_TYPE_F16) {
                     ggml_cpu_fp16_to_fp32((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);
                 } else {
@@ -9493,12 +9477,8 @@ static void ggml_compute_forward_flash_attn_ext_f16(
     // When use_ref is set, force the vec-only reference implementation (no tiling, no KV-chunking)
     const bool use_ref = params->use_ref;
 
-    // with kv_rows the loops run over the mask columns, the split KV path does not support it
-    const ggml_tensor * kv_rows = dst->src[5];
-    const int64_t n_kv_cols = kv_rows ? kv_rows->ne[0] : nek1;
-
     const bool kv_is_f32_or_f16 = (k->type == GGML_TYPE_F32 || k->type == GGML_TYPE_F16);
-    const bool use_split_kv_path = !use_ref && !kv_rows && (neq1 == 1 && neq3 == 1) && kv_is_f32_or_f16 && (k->type == v->type) && q->type == GGML_TYPE_F32 && nek1 >= 512;
+    const bool use_split_kv_path = !use_ref && (neq1 == 1 && neq3 == 1) && kv_is_f32_or_f16 && (k->type == v->type) && q->type == GGML_TYPE_F32 && nek1 >= 512;
 
     if (use_split_kv_path) {
         const int64_t chunk_size = (nek1 + nth - 1) / nth;
@@ -9577,7 +9557,7 @@ static void ggml_compute_forward_flash_attn_ext_f16(
             if (use_tiled) {
                 ggml_compute_forward_flash_attn_ext_tiled(params, dst, ir0, ir1);
             } else {
-                ggml_compute_forward_flash_attn_ext_f16_one_chunk(params, dst, ir0, ir1, 0, n_kv_cols, nullptr, 0);
+                ggml_compute_forward_flash_attn_ext_f16_one_chunk(params, dst, ir0, ir1, 0, nek1, nullptr, 0);
             }
 
             current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);

ggml/src/ggml-cpu/spacemit/ime.cpp

@@ -1046,10 +1046,6 @@ class tensor_traits_common : public tensor_traits_base {
                         return true;
                 }
             case GGML_OP_FLASH_ATTN_EXT:
-                // K/V row lists are handled by the generic CPU path
-                if (op->src[5] != nullptr) {
-                    return false;
-                }
                 forward_flash_attn_ext_f16(params, op);
                 return true;
             case GGML_OP_CONT:

ggml/src/ggml-cuda/common.cuh

@@ -1469,21 +1469,16 @@ struct ggml_backend_cuda_context {
     // when the computation is split across CPU/GPU (e.g., with --n-cpu-moe)
     std::unordered_map<const void *, std::unique_ptr<ggml_cuda_graph>> cuda_graphs;
 
-    int64_t next_graph_eviction_time = 0;
-    int64_t last_graph_use_time = 0;
+    int64_t last_graph_eviction_sweep = 0;
 
-    ggml_cuda_graph * cuda_graph(const void * first_node_ptr, int64_t time_now = ggml_time_us()) {
-        // Give all cached graphs 10s to be reused after an idle period, without updating each graph.
-        if (!cuda_graphs.empty() && time_now - last_graph_use_time >= 10'000'000) {
-            next_graph_eviction_time = time_now + 10'000'000;
-        }
-        last_graph_use_time = time_now;
+    ggml_cuda_graph * cuda_graph(const void * first_node_ptr) {
+        const int64_t time_now = ggml_time_us();
 
-        // Sweep every 5s during active use, evicting graphs unused for at least 10s.
-        if (time_now >= next_graph_eviction_time) {
-            next_graph_eviction_time = time_now + 5'000'000;
+        // sweep every 5s, evicting cuda graphs unused for >=10s
+        if (time_now - last_graph_eviction_sweep >= 5'000'000) {
+            last_graph_eviction_sweep = time_now;
             for (auto it = cuda_graphs.begin(); it != cuda_graphs.end(); ) {
-                if (it->first != first_node_ptr && time_now - it->second->last_used_time >= 10'000'000) {
+                if (time_now - it->second->last_used_time >= 10'000'000) {
                     it = cuda_graphs.erase(it);
                 } else {
                     ++it;

ggml/src/ggml-cuda/fattn-common.cuh

@@ -25,7 +25,6 @@ typedef void (* fattn_kernel_t)(
         const char * __restrict__ mask,
         const char * __restrict__ sinks,
         const int  * __restrict__ KV_max,
-        const int  * __restrict__ KV_rows,
         float      * __restrict__ dst,
         float2     * __restrict__ dst_meta,
         const float scale,
@@ -992,13 +991,8 @@ void launch_fattn(
 
     const bool V_is_K_view = V->view_src && (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs));
 
-    const ggml_tensor * mask    = dst->src[3];
-    const ggml_tensor * sinks   = dst->src[4];
-    const ggml_tensor * kv_rows = dst->src[5];
-
-    // with kv_rows the kernel iterates over the mask columns and reads K/V rows through kv_rows
-    const int64_t n_kv_cols = kv_rows ? kv_rows->ne[0] : K->ne[1];
-    GGML_ASSERT(!kv_rows || !use_sparse);
+    const ggml_tensor * mask  = dst->src[3];
+    const ggml_tensor * sinks = dst->src[4];
 
     ggml_tensor * KQV = dst;
 
@@ -1119,8 +1113,8 @@ void launch_fattn(
 
     // Optional optimization where the mask is scanned to determine whether part of the calculation can be skipped.
     // Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or
-    //     multiple sequences of possibly different lengths, or kv_rows lists that end with padding.
-    const bool scan_mask = !use_sparse && mask && n_kv_cols % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1 || kv_rows);
+    //     multiple sequences of possibly different lengths.
+    const bool scan_mask = !use_sparse && mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1);
     if (scan_mask) {
         const int64_t s31 = mask->nb[1] / sizeof(half2);
         const int64_t s33 = mask->nb[3] / sizeof(half2);
@@ -1129,7 +1123,7 @@ void launch_fattn(
         const dim3 block_dim_KV_max(FATTN_KQ_STRIDE/2, 1, 1);
 
         const int ne_KV_max = blocks_num_KV_max.x*blocks_num_KV_max.y;
-        const int iter_k = n_kv_cols / FATTN_KQ_STRIDE;
+        const int iter_k = K->ne[1] / FATTN_KQ_STRIDE;
 
         KV_max.alloc(ne_KV_max);
         ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks_num_KV_max, block_dim_KV_max, 0, main_stream);
@@ -1138,19 +1132,13 @@ void launch_fattn(
         CUDA_CHECK(cudaGetLastError());
     }
 
-    // K/V are shared by all slices when read through kv_rows
-    if (kv_rows) {
-        nb13 = 0;
-        nb23 = 0;
-    }
-
     const dim3 block_dim(warp_size, nwarps, 1);
     int max_blocks_per_sm = 1; // Max. number of active blocks limited by occupancy.
     CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks_per_sm, fattn_kernel, block_dim.x * block_dim.y * block_dim.z, nbytes_shared));
     GGML_ASSERT(max_blocks_per_sm > 0);
     int parallel_blocks = max_blocks_per_sm;
 
-    const int64_t n_kv = use_sparse ? n_kv_max : n_kv_cols;
+    const int64_t n_kv = use_sparse ? n_kv_max : K->ne[1];
     const int ntiles_KV = (n_kv + nbatch_fa - 1) / nbatch_fa; // Max. number of parallel blocks limited by KV cache length.
 
     dim3 blocks_num;
@@ -1267,7 +1255,6 @@ void launch_fattn(
         mask ? ((const char *) mask->data) : nullptr,
         sinks ? ((const char *) sinks->data) : nullptr,
         KV_max.ptr,
-        kv_rows ? (const int *) kv_rows->data : nullptr,
         !stream_k && parallel_blocks > 1 ? dst_tmp.ptr : (float *) KQV->data, dst_tmp_meta.ptr,
         scale, max_bias, m0, m1, n_head_log2, logit_softcap,
         Q->ne[0], ne01,     Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3],

ggml/src/ggml-cuda/fattn-mma-f16.cuh

@@ -420,9 +420,6 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile(
                     // padded slots gather row 0, the -inf mask removes their contribution
                     const int32_t index = i < i_sup ? indices[k_VKQ_0 + i] : 0;
                     i_KV = index >= 0 ? index : 0;
-                } else if (indices) {
-                    // K/V rows given per mask column, negative rows are padding
-                    i_KV = max(indices[k_VKQ_0 + i], 0);
                 } else {
                     i_KV = k_VKQ_0 + i;
                 }
@@ -470,11 +467,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile(
                     if constexpr (use_sparse) {
                         const int32_t index = i < i_sup ? indices[k_VKQ_0 + i] : -1;
                         src = index >= 0 ? KV + int64_t(index)*stride_KV + k*h2_per_chunk : zero;
-                    } else if (!oob_check || i < i_sup) {
-                        const int64_t i_KV = indices ? max(indices[k_VKQ_0 + i], 0) : k_VKQ_0 + i;
-                        src = KV + i_KV*stride_KV + k*h2_per_chunk;
                     } else {
-                        src = zero;
+                        src = !oob_check || i < i_sup ? KV + int64_t(k_VKQ_0 + i)*stride_KV + k*h2_per_chunk : zero;
                     }
                     ggml_cuda_memcpy_1<16>(swizzle<stride_tile>(tile_KV, i*stride_tile + k*h2_per_chunk, i), src);
                 }
@@ -640,7 +634,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
         cp_async_wait_all();
         __syncthreads();
         flash_attn_ext_f16_load_tile<stride_tile_V, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
-            (V_h2, tile_V, nbatch_V2, stride_V, k_VKQ_0, k_VKQ_sup, indices);
+            (V_h2, tile_V, nbatch_V2, stride_V, k_VKQ_0, k_VKQ_sup, nullptr);
     } else {
         // the sparse mask values are gathered per element, always load them synchronously
         constexpr bool use_cp_async = nstages == 1 && !use_sparse;
@@ -997,7 +991,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
                     (mask_h, tile_mask, stride_mask, k_VKQ_0 + nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);
             }
             flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
-                (K_h2, tile_K, nbatch_K2, stride_K, k_VKQ_0 + nbatch_fa, k_VKQ_sup, indices);
+                (K_h2, tile_K, nbatch_K2, stride_K, k_VKQ_0 + nbatch_fa, k_VKQ_sup, nullptr);
         }
     }
 
@@ -1349,7 +1343,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
                 (mask_h, tile_mask, stride_mask, kb0*nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);
         }
         flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
-            (K_h2, tile_K, nbatch_K2, stride_K, kb0*nbatch_fa, k_VKQ_sup, indices);
+            (K_h2, tile_K, nbatch_K2, stride_K, kb0*nbatch_fa, k_VKQ_sup, nullptr);
     }
 
     // kb0_start is always < kb0_stop so the last iter can be executed unconditionally.
@@ -1808,7 +1802,6 @@ static __global__ void flash_attn_ext_f16(
         const char * mask_ptr,
         const char * sinks_ptr,
         const int  * KV_max_ptr,
-        const int  * KV_rows_ptr,
         float      * dst_ptr,
         float2     * dst_meta_ptr,
         const float scale,
@@ -1931,8 +1924,7 @@ static __global__ void flash_attn_ext_f16(
 
         const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);
         const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr;
-        const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 :
-                                  KV_rows_ptr ? KV_rows_ptr + int64_t(sequence)*ne11 : nullptr;
+        const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 : nullptr;
 
         const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f;
 
@@ -1981,8 +1973,7 @@ static __global__ void flash_attn_ext_f16(
 
     const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);
     const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr;
-    const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 :
-                              KV_rows_ptr ? KV_rows_ptr + int64_t(sequence)*ne11 : nullptr;
+    const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 : nullptr;
 
     const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f;
 
@@ -1998,7 +1989,7 @@ static __global__ void flash_attn_ext_f16(
         (Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
          ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
 #else
-    GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, KV_rows_ptr, dst_ptr, dst_meta_ptr, scale,
+    GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, dst_ptr, dst_meta_ptr, scale,
         max_bias, m0, m1, n_head_log2, logit_softcap,
         ne00, ne01, ne02, ne03,
               nb01, nb02, nb03,

ggml/src/ggml-cuda/fattn-tile.cuh

@@ -797,7 +797,6 @@ static __global__ void flash_attn_tile(
         const char * mask_ptr,
         const char * sinks_ptr,
         const int  * KV_max_ptr,
-        const int  * KV_rows_ptr,
         float      * dst_ptr,
         float2     * dst_meta_ptr,
         const float scale,
@@ -822,7 +821,6 @@ static __global__ void flash_attn_tile(
     const int  * GGML_CUDA_RESTRICT KV_max   = KV_max_ptr;
     float      * GGML_CUDA_RESTRICT dst      = dst_ptr;
     float2     * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr;
-    GGML_UNUSED(KV_rows_ptr);
 
     // Skip unused kernel variants for faster compilation:
 
@@ -1134,7 +1132,7 @@ static __global__ void flash_attn_tile(
         }
     }
 #else
-    GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, KV_rows_ptr, dst_ptr, dst_meta_ptr, scale,
+    GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, dst_ptr, dst_meta_ptr, scale,
         max_bias, m0, m1, n_head_log2, logit_softcap,
         ne00, ne01, ne02, ne03,
               nb01, nb02, nb03,

ggml/src/ggml-cuda/fattn-vec.cuh

@@ -25,7 +25,6 @@ static __global__ void flash_attn_ext_vec(
         const char * mask_ptr,
         const char * sinks_ptr,
         const int  * KV_max_ptr,
-        const int  * KV_rows_ptr,
         float      * dst_ptr,
         float2     * dst_meta_ptr,
         const float scale,
@@ -51,7 +50,6 @@ static __global__ void flash_attn_ext_vec(
     const int  * GGML_CUDA_RESTRICT KV_max   = KV_max_ptr;
     float      * GGML_CUDA_RESTRICT dst      = dst_ptr;
     float2     * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr;
-    GGML_UNUSED(KV_rows_ptr);
 
     // Skip unused kernel variants for faster compilation:
     if (use_logit_softcap && !(D == 128 || D == 256)) {
@@ -516,7 +514,7 @@ static __global__ void flash_attn_ext_vec(
         dst_meta[((sequence*int(ne01.z) + ic0 + tid)*ne02 + head)*gridDim.y + blockIdx.y] = make_float2(KQ_max[tid], KQ_sum[tid]);
     }
 #else
-    GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, KV_rows_ptr, dst_ptr, dst_meta_ptr, scale,
+    GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, dst_ptr, dst_meta_ptr, scale,
         max_bias, m0, m1, n_head_log2, logit_softcap,
         ne00, ne01, ne02, ne03,
               nb01, nb02, nb03,

ggml/src/ggml-cuda/fattn.cu

@@ -161,7 +161,7 @@ bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(const int cc, const ggml_
     // the dense kernel handles up to 64/ncols2 queries per K/V pass, the single-query gather has to beat that
     const int64_t n_gather = (ncols1 == 1 ? std::min<int64_t>(Q->ne[1], 64/ncols2) : ncols1) * (int64_t) n_kv_max;
 
-    return ((GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc)) || amd_mfma_available(cc)) && dst->src[5] == nullptr &&
+    return ((GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc)) || amd_mfma_available(cc)) &&
         mask != nullptr && n_kv_max > 0 && max_bias == 0.0f && logit_softcap == 0.0f &&
         mask->ne[0] == K->ne[1] && mask->ne[1] >= Q->ne[1] && mask->ne[2] == 1 &&
         K->ne[1] >= std::max<int64_t>(4096, GGML_CUDA_CC_IS_NVIDIA(cc) ? 2*n_gather : 2LL*n_kv_max);
@@ -216,15 +216,12 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con
     const ggml_tensor * V    = dst->src[2];
     const ggml_tensor * mask = dst->src[3];
 
-    // with kv_rows the kernel iterates over the mask columns
-    const int64_t n_kv = dst->src[5] ? dst->src[5]->ne[0] : K->ne[1];
-
     float max_bias = 0.0f;
     memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float));
 
     // Edge cases like no mask, ALiBi, unpadded K/V, or misaligned addresses for large data transfers
     //     are put into the template specialization without GQA optimizations.
-    bool use_gqa_opt = mask && max_bias == 0.0f && n_kv % FATTN_KQ_STRIDE == 0;
+    bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0;
     for (const ggml_tensor * t : {Q, K, V, mask}) {
         if (t == nullptr || ggml_is_quantized(t->type)) {
             continue;
@@ -570,10 +567,6 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
     const ggml_tensor * K     = dst->src[1];
     const ggml_tensor * V     = dst->src[2];
     const ggml_tensor * mask  = dst->src[3];
-    const ggml_tensor * kv_rows = dst->src[5];
-
-    // with kv_rows the kernel iterates over the mask columns
-    const int64_t n_kv = kv_rows ? kv_rows->ne[0] : K->ne[1];
 
     const int gqa_ratio = Q->ne[2] / K->ne[2];
     GGML_ASSERT(Q->ne[2] % K->ne[2] == 0);
@@ -583,7 +576,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
 
     // The effective batch size for the kernel can be increased by gqa_ratio.
     // The kernel versions without this optimization are also used for ALiBi, if there is no mask, or if the KV cache is not padded,
-    bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && n_kv % FATTN_KQ_STRIDE == 0;
+    bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0;
     for (const ggml_tensor * t : {Q, K, V, mask}) {
         if (t == nullptr || ggml_is_quantized(t->type)) {
             continue;
@@ -655,12 +648,6 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
         return BEST_FATTN_KERNEL_NONE;
     }
 
-    // only the MMA kernel reads K/V through kv_rows
-    if (kv_rows) {
-        return GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) && mask && Q->ne[0] != 40 && Q->ne[0] != 72 ?
-            BEST_FATTN_KERNEL_MMA_F16 : BEST_FATTN_KERNEL_NONE;
-    }
-
     // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes:
     // 192 satisfies % 64 == 0 but has no vec instance (DKQ != DV); force it onto the MMA path.
     const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % 64 == 0 && Q->ne[0] != 192 && K->ne[1] % FATTN_KQ_STRIDE == 0;

ggml/src/ggml-et/ggml-et.cpp

@@ -1273,7 +1273,7 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm
         case GGML_OP_FLASH_ATTN_EXT:
             if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] &&
                 (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) && op->src[2] &&
-                (op->src[2]->type == GGML_TYPE_F32 || op->src[2]->type == GGML_TYPE_F16) && op->src[4] == nullptr && op->src[5] == nullptr &&
+                (op->src[2]->type == GGML_TYPE_F32 || op->src[2]->type == GGML_TYPE_F16) && op->src[4] == nullptr &&
                 ggml_is_contiguous_rows(op) && ggml_is_contiguous_rows(op->src[0])) {
                 float max_bias      = 0.0f;
                 float logit_softcap = 0.0f;

ggml/src/ggml-hexagon/htp-kparams.cpp

@@ -348,11 +348,6 @@ static bool htp_kparams_solve_flash_attn(
         return false;
     }
 
-    // K/V row lists (ggml_flash_attn_ext_rows) are not supported
-    if (op->src[5] != nullptr) {
-        return false;
-    }
-
     memset(kparams, 0, sizeof(*kparams));
 
     const struct ggml_tensor * q     = op->src[0];
@@ -2660,129 +2655,6 @@ static bool htp_kparams_solve_concat(
     return false;
 }
 
-// Permuted copy with strided dim 0 (e.g. channel-first <-> channel-last)
-static bool htp_kparams_solve_cpy_transpose(
-    const struct ggml_hexagon_session * sess,
-    const struct ggml_tensor * src0,
-    const struct ggml_tensor * dst,
-    uint32_t type_size,
-    struct htp_copy_kernel_params * kparams) {
-
-    // View contiguous dst with src shape to support CPY with reshape
-    const bool dst_cont = ggml_is_contiguous(dst);
-    if (!dst_cont && !ggml_are_same_shape(src0, dst)) {
-        return false;
-    }
-
-    // Drop size-1 dims and merge dims adjacent in both tensors
-    size_t ne[GGML_MAX_DIMS];
-    size_t nbs[GGML_MAX_DIMS];
-    size_t nbd[GGML_MAX_DIMS];
-    size_t dst_stride = type_size;
-    int    n          = 0;
-    for (int d = 0; d < GGML_MAX_DIMS; d++) {
-        const size_t sd = dst_cont ? dst_stride : dst->nb[d];
-        dst_stride *= (size_t) src0->ne[d];
-        if (src0->ne[d] == 1) {
-            continue;
-        }
-        if (n > 0 && src0->nb[d] == nbs[n - 1] * ne[n - 1] && sd == nbd[n - 1] * ne[n - 1]) {
-            ne[n - 1] *= (size_t) src0->ne[d];
-            continue;
-        }
-        ne[n]  = (size_t) src0->ne[d];
-        nbs[n] = src0->nb[d];
-        nbd[n] = sd;
-        n++;
-    }
-
-    // Fold leading dense dim into element block
-    size_t es = type_size;
-    for (int i = 0; i < n; i++) {
-        if (nbs[i] == type_size && nbd[i] == type_size) {
-            es *= ne[i];
-            for (int j = i; j + 1 < n; j++) {
-                ne[j]  = ne[j + 1];
-                nbs[j] = nbs[j + 1];
-                nbd[j] = nbd[j + 1];
-            }
-            n--;
-            break;
-        }
-    }
-    if (es > 128 || (es & (es - 1)) != 0) {
-        return false;
-    }
-
-    // Transpose axes: dim a dense in dst, dim k dense in src
-    int a = -1;
-    int k = -1;
-    for (int i = 0; i < n; i++) {
-        if (a < 0 && nbd[i] == es) {
-            a = i;
-        }
-        if (k < 0 && nbs[i] == es) {
-            k = i;
-        }
-    }
-    if (a < 0 || k < 0 || a == k) {
-        return false;
-    }
-
-    struct htp_copy_transpose_params tp = {};
-    tp.elem_size = (uint8_t) es;
-    tp.ne[0]     = (uint32_t) ne[a];
-    tp.ne[1]     = (uint32_t) ne[k];
-    tp.ne[2]     = 1;
-    tp.src_nb[0] = (uint32_t) nbs[a];
-    tp.dst_nb[0] = (uint32_t) nbd[k];
-    tp.packed    = nbd[k] == ne[a] * es && ne[a] * es < 128 && 128 % (ne[a] * es) == 0;
-
-    size_t ne3 = 1;
-    int    m   = 0;
-    for (int i = 0; i < n; i++) {
-        if (i == a || i == k) {
-            continue;
-        }
-        if (m == 0) {
-            tp.ne[2] = (uint32_t) ne[i];
-        } else {
-            ne3 = ne[i];
-        }
-        tp.src_nb[1 + m] = (uint32_t) nbs[i];
-        tp.dst_nb[1 + m] = (uint32_t) nbd[i];
-        m++;
-    }
-
-    const uint32_t n_threads = htp_session_n_threads(sess);
-    const size_t   vtcm_size = htp_session_vtcm_size(sess);
-    uint64_t       vtcm      = 0;
-    for (uint32_t blk_bytes = HTP_COPY_TRANSPOSE_BLK_BYTES; ; blk_bytes /= 2) {
-        if (!htp_copy_transpose_layout_build(&tp, blk_bytes)) {
-            return false;
-        }
-        vtcm = (uint64_t) n_threads * 2 * (tp.in_buf_size + tp.out_buf_size);
-        if (vtcm <= vtcm_size) {
-            break;
-        }
-    }
-
-    const uint64_t total_tiles = (uint64_t) tp.n0b * tp.nkb * tp.ne[2] * ne3;
-    if (total_tiles > UINT32_MAX) {
-        return false;
-    }
-    tp.total_tiles = (uint32_t) total_tiles;
-    tp.div_nkb     = init_fastdiv_values(tp.nkb);
-    tp.div_n0b     = init_fastdiv_values(tp.n0b);
-    tp.div_ne2     = init_fastdiv_values(tp.ne[2]);
-
-    kparams->kernel_type = HTP_COPY_KERNEL_TRANSPOSE;
-    kparams->n_threads   = (uint8_t) n_threads;
-    kparams->vtcm_size   = (uint32_t) vtcm;
-    kparams->u.transpose = tp;
-    return true;
-}
-
 static bool htp_kparams_solve_cpy(
     const struct ggml_hexagon_session * sess,
     const struct ggml_tensor * op,
@@ -2880,10 +2752,6 @@ static bool htp_kparams_solve_cpy(
             }
         }
 
-        if (htp_kparams_solve_cpy_transpose(sess, src0, dst, src_type_size, kparams)) {
-            return true;
-        }
-
         if (sameshape) {
             kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE;
             kparams->total_rows  = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);

ggml/src/ggml-hexagon/htp/cpy-ops.c

@@ -35,10 +35,6 @@ struct htp_copy_context {
     struct htp_copy_convert_vtcm_layout layout;
     uint8_t * vtcm_src0;
     uint8_t * vtcm_dst;
-
-    uint32_t tile_start;
-    uint32_t ntiles;
-    uint32_t tiles_per_thread;
 };
 
 #define cpy_preamble                              \
@@ -166,171 +162,6 @@ static inline void cpy_cursor_advance(struct cpy_row_cursor * c, uint32_t n, uin
     }
 }
 
-struct cpy_tile_cursor {
-    uint32_t ibk;
-    uint32_t ib0;
-    uint32_t i2;
-    uint32_t i3;
-};
-
-static inline void cpy_tile_cursor_advance(struct cpy_tile_cursor * c, const struct htp_copy_transpose_params * tp) {
-    if (++c->ibk == tp->nkb) {
-        c->ibk = 0;
-        if (++c->ib0 == tp->n0b) {
-            c->ib0 = 0;
-            if (++c->i2 == tp->ne[2]) {
-                c->i2 = 0;
-                c->i3++;
-            }
-        }
-    }
-}
-
-// Gather offsets for element e = l >> log2_m at (e % r, e / r)
-static inline HVX_Vector cpy_transpose_offsets(const struct htp_copy_transpose_params * tp) {
-    const uint32_t r_mask = (1u << tp->log2_r) - 1;
-    if (tp->elem_size != 2) {
-        const uint32_t m_mask = (1u << tp->log2_m) - 1;
-        int32_t offsets[32] __attribute__((aligned(128)));
-        for (uint32_t l = 0; l < 32; l++) {
-            const uint32_t e = l >> tp->log2_m;
-            offsets[l] = (e & r_mask) * tp->in_stride + (e >> tp->log2_r) * tp->elem_size + (l & m_mask) * 4;
-        }
-        return *(HVX_Vector *) offsets;
-    }
-    int16_t offsets[64] __attribute__((aligned(128)));
-    for (uint32_t l = 0; l < 64; l++) {
-        offsets[l] = (l & r_mask) * tp->in_stride + (l >> tp->log2_r) * 2;
-    }
-    return *(HVX_Vector *) offsets;
-}
-
-static inline void cpy_transpose_push_in(dma_queue * q, const struct htp_copy_transpose_params * tp, dma_addr_t src,
-                                         const struct cpy_tile_cursor * c, uint8_t * buf) {
-    const uint32_t a0  = c->ib0 * tp->blk0;
-    const uint32_t ak  = c->ibk * tp->blkk;
-    const uint32_t off = a0 * tp->src_nb[0] + ak * tp->elem_size + c->i2 * tp->src_nb[1] + c->i3 * tp->src_nb[2];
-    dma_queue_push(q, dma_make_data(buf, src + off), tp->in_stride, tp->src_nb[0],
-                   MIN(tp->blkk, tp->ne[1] - ak) * tp->elem_size, MIN(tp->blk0, tp->ne[0] - a0));
-}
-
-// Pipelined tile worker: DMA in -> vgather transpose -> DMA out
-static void cpy_thread_transpose(unsigned int nth, unsigned int ith, void * data) {
-    struct htp_copy_context * ct = (struct htp_copy_context *) data;
-    struct htp_ops_context * octx = ct->octx;
-    const struct htp_copy_transpose_params * tp = &ct->kparams->u.transpose;
-
-    const uint32_t t0 = ct->tile_start + ith * ct->tiles_per_thread;
-    const uint32_t t1 = MIN(t0 + ct->tiles_per_thread, ct->tile_start + ct->ntiles);
-    if (t0 >= t1) {
-        return;
-    }
-
-    dma_queue * dma_q = octx->ctx->dma[ith];
-    struct htp_thread_trace * tr = &octx->ctx->trace[ith];
-
-    const dma_addr_t src = octx->src[0]->data;
-    const dma_addr_t dst = octx->dst->data;
-
-    const uint32_t es         = tp->elem_size;
-    const uint32_t ne0        = tp->ne[0];
-    const uint32_t nek        = tp->ne[1];
-    const uint32_t in_stride  = tp->in_stride;
-    const uint32_t out_stride = tp->out_stride;
-
-    const HVX_Vector offsets   = cpy_transpose_offsets(tp);
-    const uint32_t packed_mu   = (ne0 - 1) * in_stride + 127;
-    const uint32_t packed_step = 128 >> tp->log2_r;
-    const uint32_t epv         = 1u << tp->log2_r;
-
-    uint8_t * in_buf[2];
-    uint8_t * out_buf[2];
-    in_buf[0]  = ct->vtcm_src0 + ith * 2 * tp->in_buf_size;
-    in_buf[1]  = in_buf[0] + tp->in_buf_size;
-    out_buf[0] = ct->vtcm_dst + ith * 2 * tp->out_buf_size;
-    out_buf[1] = out_buf[0] + tp->out_buf_size;
-
-    struct cpy_tile_cursor cur;
-    {
-        const uint32_t q0 = fastdiv(t0, &tp->div_nkb);
-        const uint32_t q1 = fastdiv(q0, &tp->div_n0b);
-        cur.ibk = t0 - q0 * tp->nkb;
-        cur.ib0 = q0 - q1 * tp->n0b;
-        cur.i3  = fastdiv(q1, &tp->div_ne2);
-        cur.i2  = q1 - cur.i3 * tp->ne[2];
-    }
-
-    struct cpy_tile_cursor nxt = cur;
-
-    cpy_transpose_push_in(dma_q, tp, src, &cur, in_buf[0]);
-
-    for (uint32_t t = t0, i = 0; t < t1; t++, i++) {
-        while (dma_queue_depth(dma_q) > (i > 0 ? 1u : 0u)) {
-            dma_queue_pop(dma_q);
-        }
-
-        uint8_t * in  = in_buf[i & 1];
-        uint8_t * out = out_buf[i & 1];
-
-        const uint32_t a0 = cur.ib0 * tp->blk0;
-        const uint32_t ak = cur.ibk * tp->blkk;
-        const uint32_t n0 = MIN(tp->blk0, ne0 - a0);
-        const uint32_t nk = MIN(tp->blkk, nek - ak);
-
-        if (t + 1 < t1) {
-            cpy_tile_cursor_advance(&nxt, tp);
-            cpy_transpose_push_in(dma_q, tp, src, &nxt, in_buf[(i + 1) & 1]);
-        }
-
-        htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) t);
-        if (tp->packed) {
-            const uint32_t nvec = (nk * ne0 * es + 127) >> 7;
-            if (es != 2) {
-                #pragma unroll(4)
-                for (uint32_t v = 0; v < nvec; v++) {
-                    Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) (in + v * packed_step), packed_mu, offsets);
-                }
-            } else {
-                #pragma unroll(4)
-                for (uint32_t v = 0; v < nvec; v++) {
-                    Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) (in + v * packed_step), packed_mu, offsets);
-                }
-            }
-        } else {
-            for (uint32_t rb = 0; rb < n0; rb += epv) {
-                const uint32_t  mu = (MIN(epv, n0 - rb) - 1) * in_stride + es - 1;
-                uint8_t *       d  = out + rb * es;
-                const uint8_t * s  = in + rb * in_stride;
-                if (es != 2) {
-                    #pragma unroll(4)
-                    for (uint32_t ik = 0; ik < nk; ik++) {
-                        Q6_vgather_ARMVw((HVX_Vector *) (d + ik * out_stride), (size_t) (s + ik * es), mu, offsets);
-                    }
-                } else {
-                    #pragma unroll(4)
-                    for (uint32_t ik = 0; ik < nk; ik++) {
-                        Q6_vgather_ARMVh((HVX_Vector *) (d + ik * out_stride), (size_t) (s + ik * 2), mu, offsets);
-                    }
-                }
-            }
-        }
-        hvx_gather_sync(out);
-        htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) t);
-
-        const uint32_t doff = a0 * es + ak * tp->dst_nb[0] + cur.i2 * tp->dst_nb[1] + cur.i3 * tp->dst_nb[2];
-        if (tp->packed) {
-            const uint32_t n = nk * ne0 * es;
-            dma_queue_push(dma_q, dma_make_data(dst + doff, out), n, n, n, 1);
-        } else {
-            dma_queue_push(dma_q, dma_make_data(dst + doff, out), tp->dst_nb[0], out_stride, n0 * es, nk);
-        }
-
-        cur = nxt;
-    }
-
-    dma_queue_flush(dma_q);
-}
-
 #define DEFINE_CPY_CONVERT_SAMESHAPE(NAME, CONV_FUNC)                                        \
 static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \
     struct htp_copy_context * ct = (struct htp_copy_context *) data;                         \
@@ -608,48 +439,6 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern
     return HTP_STATUS_OK;
 }
 
-static int cpy_transpose(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) {
-    if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
-        return HTP_STATUS_INVAL_PARAMS;
-    }
-
-    const struct htp_copy_transpose_params * tp = &kparams->u.transpose;
-
-    uint32_t tile_start = 0;
-    uint32_t ntiles     = tp->total_tiles;
-
-    if (octx->ctx->mdev.count > 1) {
-        const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
-            tp->total_tiles, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
-        tile_start = range.start;
-        ntiles     = range.count;
-    }
-
-    if (ntiles == 0) {
-        return HTP_STATUS_OK;
-    }
-
-    if (kparams->vtcm_size > octx->ctx->vtcm_size) {
-        return HTP_STATUS_VTCM_TOO_SMALL;
-    }
-
-    const uint32_t n_threads = octx->n_threads;
-
-    struct htp_copy_context ct;
-    ct.octx             = octx;
-    ct.kparams          = kparams;
-    ct.tile_start       = tile_start;
-    ct.ntiles           = ntiles;
-    ct.tiles_per_thread = fastdiv(ntiles + n_threads - 1, &octx->n_threads_div);
-
-    uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
-    ct.vtcm_src0 = vtcm_base;
-    ct.vtcm_dst  = vtcm_base + (size_t) n_threads * 2 * tp->in_buf_size;
-
-    work_queue_run(octx->ctx->work_queue, cpy_thread_transpose, &ct, n_threads);
-    return HTP_STATUS_OK;
-}
-
 int op_cpy(struct htp_ops_context * octx) {
     const struct htp_copy_kernel_params * kparams = (const struct htp_copy_kernel_params *) octx->kernel_params;
     int status = HTP_STATUS_OK;
@@ -670,9 +459,6 @@ int op_cpy(struct htp_ops_context * octx) {
         case HTP_COPY_KERNEL_RESHAPE:
             status = cpy_reshape(octx, kparams);
             break;
-        case HTP_COPY_KERNEL_TRANSPOSE:
-            status = cpy_transpose(octx, kparams);
-            break;
         case HTP_COPY_KERNEL_GATHER_ROWS:
             status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_threads);
             break;

ggml/src/ggml-hexagon/htp/cpy-ops.h

@@ -15,7 +15,6 @@ enum htp_copy_kernel_type {
     HTP_COPY_KERNEL_RESHAPE            = 4,
     HTP_COPY_KERNEL_SCALAR             = 5,
     HTP_COPY_KERNEL_GATHER_ROWS        = 6,
-    HTP_COPY_KERNEL_TRANSPOSE          = 7,
 };
 
 struct htp_copy_convert_params {
@@ -33,29 +32,6 @@ struct htp_copy_reshape_params {
     struct fastdiv_values div_ne02_ne01_ne00;
 };
 
-// Batched 2D transpose of elem_size elements (dim 0 dense in dst, dim 1 dense in src)
-struct htp_copy_transpose_params {
-    uint32_t ne[3];          // dims 0..2
-    uint32_t src_nb[3];      // src strides for dims 0, 2, 3
-    uint32_t dst_nb[3];      // dst strides for dims 1, 2, 3
-    struct fastdiv_values div_nkb;
-    struct fastdiv_values div_n0b;
-    struct fastdiv_values div_ne2;
-    uint32_t blk0;           // dim-0 elements per tile
-    uint32_t blkk;           // dim-1 elements per tile
-    uint32_t n0b;            // tiles along dim 0
-    uint32_t nkb;            // tiles along dim 1
-    uint32_t in_stride;      // VTCM row pitch of staged src tile
-    uint32_t out_stride;     // VTCM row pitch of transposed tile
-    uint32_t in_buf_size;
-    uint32_t out_buf_size;
-    uint32_t total_tiles;
-    uint8_t  elem_size;
-    uint8_t  packed;         // gather straight to dst order
-    uint8_t  log2_m;         // log2(gather lanes per element)
-    uint8_t  log2_r;         // log2(dim-0 elements per vector)
-};
-
 struct htp_copy_kernel_params {
     uint8_t  kernel_type;
     uint8_t  src0_type_size;
@@ -70,7 +46,6 @@ struct htp_copy_kernel_params {
         struct htp_copy_convert_params convert;
         struct htp_copy_reshape_params reshape;
         struct htp_gather_rows_params  gather;
-        struct htp_copy_transpose_params transpose;
     } u;
 };
 
@@ -141,76 +116,6 @@ static inline void htp_copy_convert_solve_layout(
     htp_copy_convert_vtcm_layout_build(layout, ne00, src_type_size, dst_type_size, blk_rows, n_threads);
 }
 
-#define HTP_COPY_TRANSPOSE_BLK_BYTES (64 * 1024)
-
-static inline bool htp_copy_transpose_layout_build(struct htp_copy_transpose_params * tp, uint32_t blk_bytes) {
-    const uint32_t ne0  = tp->ne[0];
-    const uint32_t nek  = tp->ne[1];
-    const uint32_t es   = tp->elem_size;
-    const uint32_t epv  = 128 / es;
-
-    uint32_t blk0;
-    uint32_t blkk;
-    if (tp->packed) {
-        // Packed: vector holds epv / ne0 whole dst rows
-        blk0 = ne0;
-        blkk = blk_bytes / (ne0 * es);
-        if (es == 2 && blkk > 16384 / ne0 - 64) {
-            blkk = 16384 / ne0 - 64;
-        }
-        if (blkk > nek) {
-            blkk = nek;
-        }
-        if (blkk == 0) {
-            return false;
-        }
-    } else if (nek <= epv) {
-        // Stage short contiguous dim and expand dim 0
-        blkk = nek;
-        blk0 = blk_bytes / (nek * es);
-        blk0 -= blk0 % epv;
-        if (blk0 > hex_round_up(ne0, epv)) {
-            blk0 = hex_round_up(ne0, epv);
-        }
-        if (blk0 < epv) {
-            return false;
-        }
-    } else {
-        blk0 = hex_round_up(ne0, epv);
-        if (blk0 > 256) {
-            blk0 = 256;
-        }
-        blkk = blk_bytes / (blk0 * es);
-        blkk -= blkk % epv;
-        // Cap f16 blkk to 256 for 16-bit gather offsets
-        const uint32_t max_blkk = (es == 2 ? 512 : 4096) / es;
-        if (blkk > max_blkk) {
-            blkk = max_blkk;
-        }
-        if (blkk > hex_round_up(nek, epv)) {
-            blkk = hex_round_up(nek, epv);
-        }
-        if (blkk < epv) {
-            return false;
-        }
-    }
-
-    tp->blk0         = blk0;
-    tp->blkk         = blkk;
-    tp->n0b          = (ne0 + blk0 - 1) / blk0;
-    tp->nkb          = (nek + blkk - 1) / blkk;
-    // Pad row pitch by 1 element to avoid VTCM bank conflicts
-    const uint32_t pad = (es == 2) ? 2 : 4;
-    tp->in_stride    = blkk * es + pad;
-    tp->out_stride   = blk0 * es;
-    // Extra vector for packed gather overhang
-    tp->in_buf_size  = hex_round_up(blk0 * tp->in_stride + (tp->packed ? 128 : 0), 128);
-    tp->out_buf_size = hex_round_up(blkk * tp->out_stride, 128);
-    tp->log2_m       = es == 2 ? 0 : __builtin_ctz(es) - 2;
-    tp->log2_r       = __builtin_ctz(tp->packed ? ne0 : epv);
-    return true;
-}
-
 #if defined(__cplusplus)
 static_assert(sizeof(struct htp_copy_kernel_params) <= 128, "htp_copy_kernel_params is too large for kernel_params blob");
 #else

ggml/src/ggml-hexagon/htp/dma-queue.c

@@ -141,9 +141,43 @@ bool dma_queue_push_fallback_2d(dma_queue * q, dma_data ddata, size_t dst_stride
     dma_ring_flush(r1);
     r0->tail = r1->tail;
 
+    return dma_ring_push_single_2d(r0, ddata, 0, 0, 0, /*nrows=*/ 0);
+}
+
+bool dma_queue_push_fallback_contig(dma_queue * q, dma_data ddata, size_t total) {
+    dma_ring * r0 = q->ring0;
+    dma_ring * r1 = q->ring1;
+
+    if (((r0->push_idx + 1) & r0->idx_mask) == r0->pop_idx) {
+        return false;
+    }
+
+    r1->tail = r0->tail;
+
+    size_t rem_bytes   = total;
+    dma_addr_t cur_dst = ddata.dst;
+    dma_addr_t cur_src = ddata.src;
+
+    while (rem_bytes > 0) {
+        const uint32_t cur_bytes = MIN(rem_bytes, DMA_SAFE_CHUNK_SIZE);
+        dma_data cur_data = dma_make_data(cur_dst, cur_src);
+        if (!dma_ring_push_single_contig(r1, cur_data, cur_bytes)) {
+            dma_ring_flush(r1);
+            dma_ring_push_single_contig(r1, cur_data, cur_bytes);
+        }
+        cur_dst   += cur_bytes;
+        cur_src   += cur_bytes;
+        rem_bytes -= cur_bytes;
+    }
+
+    dma_ring_flush(r1);
+    r0->tail = r1->tail;
+
     return dma_ring_push_single_1d(r0, ddata, /*size=*/ 0);
 }
 
+#if __HVX_ARCH__ < 75
+
 bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
     dma_ring * r0 = q->ring0;
     dma_ring * r1 = q->ring1;
@@ -154,28 +188,19 @@ bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride
 
     r1->tail = r0->tail;
 
-    dma_addr_t row_dst = ddata.dst;
-    dma_addr_t row_src = ddata.src;
+    size_t rem_rows    = nrows;
+    dma_addr_t cur_dst = ddata.dst;
+    dma_addr_t cur_src = ddata.src;
 
-    for (size_t i = 0; i < nrows; i++) {
-        size_t     rem_bytes = row_size;
-        dma_addr_t cur_dst   = row_dst;
-        dma_addr_t cur_src   = row_src;
-
-        while (rem_bytes > 0) {
-            const uint32_t cur_bytes = MIN(rem_bytes, DMA_SAFE_CHUNK_SIZE);
-            dma_data cur_data = dma_make_data(cur_dst, cur_src);
-            if (!dma_ring_push_single_contig(r1, cur_data, cur_bytes)) {
-                dma_ring_flush(r1);
-                dma_ring_push_single_contig(r1, cur_data, cur_bytes);
-            }
-            cur_dst   += cur_bytes;
-            cur_src   += cur_bytes;
-            rem_bytes -= cur_bytes;
+    while (rem_rows > 0) {
+        dma_data cur_data = dma_make_data(cur_dst, cur_src);
+        if (!dma_ring_push_single_1d(r1, cur_data, row_size)) {
+            dma_ring_flush(r1);
+            dma_ring_push_single_1d(r1, cur_data, row_size);
         }
-
-        row_dst += dst_stride;
-        row_src += src_stride;
+        cur_dst  += dst_stride;
+        cur_src  += src_stride;
+        rem_rows -= 1;
     }
 
     dma_ring_flush(r1);
@@ -183,3 +208,5 @@ bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride
 
     return dma_ring_push_single_1d(r0, ddata, /*size=*/ 0);
 }
+
+#endif

ggml/src/ggml-hexagon/htp/dma-queue.h

@@ -148,7 +148,10 @@ dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q);
 void        dma_queue_alias_free(dma_queue_t q);
 
 bool        dma_queue_push_fallback_2d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows);
+bool        dma_queue_push_fallback_contig(dma_queue * q, dma_data ddata, size_t total);
+#if __HVX_ARCH__ < 75
 bool        dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows);
+#endif
 
 // TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead
 // but those do not seem to always compiler properly.
@@ -397,7 +400,7 @@ static inline bool dma_queue_push(dma_queue * q, dma_data ddata, size_t dst_stri
         if (__builtin_expect(total <= DMA_MAX_SIZE_24B, 1)) {
             return dma_ring_push_single_contig(q->ring0, ddata, total);
         }
-        return dma_queue_push_fallback_1d(q, ddata, 0, 0, total, 1);
+        return dma_queue_push_fallback_contig(q, ddata, total);
     }
 
     // 2. Hot path: Standard strided 2D (10-20% of calls)
@@ -407,10 +410,14 @@ static inline bool dma_queue_push(dma_queue * q, dma_data ddata, size_t dst_stri
     }
 
     // 3. Cold path: Descriptor chunking fallbacks (< 0.1%)
-    if ((row_size | src_stride | dst_stride) <= DMA_MAX_2D_ROW_SIZE) {
+#if __HVX_ARCH__ < 75
+    if (row_size <= DMA_MAX_SIZE_16B && (src_stride | dst_stride) <= DMA_MAX_STRIDE_16B) {
         return dma_queue_push_fallback_2d(q, ddata, dst_stride, src_stride, row_size, nrows);
     }
     return dma_queue_push_fallback_1d(q, ddata, dst_stride, src_stride, row_size, nrows);
+#else
+    return dma_queue_push_fallback_2d(q, ddata, dst_stride, src_stride, row_size, nrows);
+#endif
 }
 
 static inline void dma_sync_read(dma_queue * dma_q, void * dst, dma_addr_t src, size_t bytes) {

ggml/src/ggml-metal/ggml-metal-device.cpp

@@ -1987,23 +1987,6 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext(
     return res;
 }
 
-ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_rows(
-        ggml_metal_library_t lib,
-        const ggml_tensor * op) {
-    assert(op->op == GGML_OP_FLASH_ATTN_EXT);
-
-    const char * base = "kernel_flash_attn_ext_rows";
-
-    ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, base);
-    if (!res.pipeline) {
-        res = ggml_metal_library_compile_pipeline(lib, base, base, nullptr);
-    }
-
-    GGML_UNUSED(op);
-
-    return res;
-}
-
 ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_idx(
         ggml_metal_library_t lib,
         const ggml_tensor * op) {
@@ -2033,7 +2016,6 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v
         bool    has_scap,
         bool    has_kvpad,
         bool    has_sparse,
-        bool    has_rows,
         int32_t nqpsg,
         int32_t ne,
         int32_t nsg,
@@ -2063,7 +2045,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v
             dv,
             qne_suffix);
 
-    snprintf(name, 256, "%s_mask=%d_sink=%d_bias=%d_scap=%d_kvpad=%d_sparse=%d_rows=%d_ns10=%d_ns20=%d_nsg=%d_nwg=%d",
+    snprintf(name, 256, "%s_mask=%d_sink=%d_bias=%d_scap=%d_kvpad=%d_sparse=%d_ns10=%d_ns20=%d_nsg=%d_nwg=%d",
             base,
             has_mask,
             has_sinks,
@@ -2071,7 +2053,6 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v
             has_scap,
             has_kvpad,
             has_sparse,
-            has_rows,
             ns10,
             ns20,
             nsg, nwg);
@@ -2086,7 +2067,6 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v
         ggml_metal_cv_set_bool(cv, has_scap,  FC_FLASH_ATTN_EXT_VEC + 3);
         ggml_metal_cv_set_bool(cv, has_kvpad,  FC_FLASH_ATTN_EXT_VEC + 4);
         ggml_metal_cv_set_bool(cv, has_sparse, FC_FLASH_ATTN_EXT_VEC + 5);
-        ggml_metal_cv_set_bool(cv, has_rows,   FC_FLASH_ATTN_EXT_VEC + 6);
 
         ggml_metal_cv_set_int32(cv, ns10, FC_FLASH_ATTN_EXT_VEC + 20);
         ggml_metal_cv_set_int32(cv, ns20, FC_FLASH_ATTN_EXT_VEC + 21);

ggml/src/ggml-metal/ggml-metal-device.h

@@ -219,10 +219,6 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att
         int32_t ns10,
         int32_t ns20);
 
-struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_rows(
-        ggml_metal_library_t lib,
-        const struct ggml_tensor * op);
-
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_idx(
         ggml_metal_library_t lib,
         const struct ggml_tensor * op);
@@ -236,7 +232,6 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att
         bool    has_scap,
         bool    has_kvpad,
         bool    has_sparse,
-        bool    has_rows,
         int32_t nqpsg,
         int32_t ne,
         int32_t nsg,

ggml/src/ggml-metal/ggml-metal-device.m

@@ -1727,39 +1727,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
         case GGML_OP_ROLL:
             return ggml_is_contiguous(op->src[0]);
         case GGML_OP_FLASH_ATTN_EXT:
-            // K/V row lists (ggml_flash_attn_ext_rows): one list of cache rows per q slice
-            if (op->src[5] != NULL) {
-                const struct ggml_tensor * kv_rows = op->src[5];
-
-                if (kv_rows->type != GGML_TYPE_I32 ||
-                    !ggml_is_contiguous(kv_rows) ||
-                    kv_rows->ne[2] != 1 || kv_rows->ne[3] != 1 ||
-                    kv_rows->ne[1] != op->src[0]->ne[3] ||
-                    !op->src[3] || op->src[1]->ne[3] != 1 || op->src[2]->ne[3] != 1) {
-                    return false;
-                }
-
-                // the non-vec kernels attend over a contiguous F16 copy of the referenced rows:
-                // - F16 K/V is gathered directly
-                // - quantized K/V is first dequantized with kernel_flash_attn_ext_kv_*_f16
-                // - other types keep attending to the whole cache (dense mask)
-                switch (op->src[1]->type) {
-                    case GGML_TYPE_F16:
-                    case GGML_TYPE_Q4_0:
-                    case GGML_TYPE_Q4_1:
-                    case GGML_TYPE_Q5_0:
-                    case GGML_TYPE_Q5_1:
-                    case GGML_TYPE_Q8_0:
-                        break;
-                    default:
-                        return false;
-                }
-
-                // the row gather copies F16 values in groups of 4
-                if (op->src[2]->ne[0] % 4 != 0) {
-                    return false;
-                }
-            }
             // for new head sizes, add checks here
             if (op->src[0]->ne[0] != 32 &&
                 op->src[0]->ne[0] != 40 &&
@@ -2135,11 +2102,19 @@ static void ggml_metal_buffer_rset_free(ggml_metal_buffer_t buf) {
 static void * ggml_metal_host_malloc(size_t n) {
     void * data = NULL;
 
+#if TARGET_OS_OSX
     kern_return_t err = vm_allocate((vm_map_t) mach_task_self(), (void *) &data, n, VM_FLAGS_ANYWHERE);
     if (err != KERN_SUCCESS) {
         GGML_LOG_ERROR("%s: error: vm_allocate failed\n", __func__);
         return NULL;
     }
+#else
+    const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n);
+    if (result != 0) {
+        GGML_LOG_ERROR("%s: error: posix_memalign failed\n", __func__);
+        return NULL;
+    }
+#endif
 
     return data;
 }
@@ -2321,7 +2296,11 @@ void ggml_metal_buffer_free(ggml_metal_buffer_t buf) {
     }
 
     if (buf->is_shared && buf->owned) {
+#if TARGET_OS_OSX
         vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)buf->all_data, buf->all_size);
+#else
+        free(buf->all_data);
+#endif
     }
 
     free(buf);

ggml/src/ggml-metal/ggml-metal-impl.h

@@ -385,19 +385,6 @@ typedef struct {
     int32_t  nblocks;
 } ggml_metal_kargs_flash_attn_ext_kv_f16;
 
-// gather the K/V cache rows referenced by the kv_rows lists into contiguous F16 buffers
-typedef struct {
-    int32_t  ne11;      // number of rows per list (n_kv_rows)
-    int32_t  ne_12_2;   // number of K/V heads
-    int32_t  ne10;      // DK
-    int32_t  ne20;      // DV, set to 0 to skip gathering V (V is a view of K)
-    uint64_t v_off;     // byte offset of the gathered V region in dst
-    uint64_t nb11;      // K source strides (ne13 = 1: the cache is shared by all streams)
-    uint64_t nb12;
-    uint64_t nb21;      // V source strides (ne23 = 1)
-    uint64_t nb22;
-} ggml_metal_kargs_flash_attn_ext_rows;
-
 typedef struct {
     int32_t  ne11;
     int32_t  ne_12_2; // assume K and V are same shape

ggml/src/ggml-metal/ggml-metal-ops.cpp

@@ -3261,10 +3261,7 @@ static bool ggml_metal_op_flash_attn_ext_use_tensor(const ggml_tensor * op, bool
         return false;
     }
 
-    // with kv_rows, the attention runs over the compacted row lists (padded to 256), not over the cache
-    const int64_t ne11_attn = op->src[5] ? op->src[5]->ne[0] : op->src[1]->ne[1];
-
-    return ne11_attn % OP_FLASH_ATTN_EXT_TENSOR_NCPSG == 0;
+    return op->src[1]->ne[1] % OP_FLASH_ATTN_EXT_TENSOR_NCPSG == 0;
 }
 
 // returns the n_kv_max hint if the sparse path is available for this op, or 0 otherwise
@@ -3273,11 +3270,6 @@ static bool ggml_metal_op_flash_attn_ext_use_tensor(const ggml_tensor * op, bool
 static int ggml_metal_op_flash_attn_ext_n_kv_max_sparse(const ggml_tensor * op) {
     assert(op->op == GGML_OP_FLASH_ATTN_EXT);
 
-    // K/V row lists are handled by the dense kernels, not by the sparse path
-    if (op->src[5] != nullptr) {
-        return 0;
-    }
-
     int32_t n_kv_max = 0;
     memcpy(&n_kv_max, ((const int32_t *) op->op_params) + 4, sizeof(n_kv_max));
 
@@ -3521,32 +3513,6 @@ size_t ggml_metal_op_flash_attn_ext_extra_idx(const ggml_tensor * op) {
     return GGML_PAD(sizeof(int32_t)*(size_t) n_kv_max_padded*ne31*ne32*ne33, 16);
 }
 
-// size of the contiguous F16 buffers holding the K/V rows referenced by the kv_rows lists
-// the attention over these buffers is a standard dense op (no per-tile gather in the main kernel)
-size_t ggml_metal_op_flash_attn_ext_extra_rows(const ggml_tensor * op) {
-    assert(op->op == GGML_OP_FLASH_ATTN_EXT);
-
-    const ggml_tensor * kv_rows = op->src[5];
-
-    if (!kv_rows) {
-        return 0;
-    }
-
-    GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);
-    GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne);
-
-    const int64_t n_rows   = kv_rows->ne[0];
-    const int64_t n_slices = kv_rows->ne[1];
-
-    // note: when V is a view of K, the gathered V is read from the gathered K, so only K is gathered
-    const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op);
-
-    const size_t k_size = GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne10*n_rows*ne12*n_slices, 16);
-    const size_t v_size = v_is_view_of_k ? 0 : GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne20*n_rows*ne22*n_slices, 16);
-
-    return k_size + v_size;
-}
-
 int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
     ggml_tensor * op = ctx->node(idx);
 
@@ -3579,22 +3545,6 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
     GGML_ASSERT(!op->src[3] || op->src[3]->ne[1] >= op->src[0]->ne[1] &&
             "the Flash-Attention Metal kernel requires the mask to be at least n_queries big");
 
-    // K/V row lists (ggml_flash_attn_ext_rows): src[5] holds one list of cache rows per q slice
-    // the attention loops then run over the mask columns and read K/V rows through the lists
-    const ggml_tensor * kv_rows = op->src[5];
-    const bool has_rows = kv_rows != nullptr;
-
-    GGML_ASSERT(!has_rows || (kv_rows->type == GGML_TYPE_I32 && ggml_is_contiguous(kv_rows)));
-    GGML_ASSERT(!has_rows || (kv_rows->ne[1] == ne03 && kv_rows->ne[2] == 1 && kv_rows->ne[3] == 1));
-    GGML_ASSERT(!has_rows || op->src[3]);
-
-    // the loop bound over the mask columns / K/V row lists
-    const int64_t ne11_attn = has_rows ? kv_rows->ne[0] : ne11;
-
-    // the row lists are padded by the KV cache to a multiple of 256, so the pad path never triggers
-    GGML_ASSERT(!has_rows || ne11_attn % OP_FLASH_ATTN_EXT_NCPSG == 0);
-    GGML_ASSERT(!has_rows || ne11_attn % OP_FLASH_ATTN_EXT_VEC_NCPSG == 0);
-
     float scale;
     float max_bias;
     float logit_softcap;
@@ -3609,8 +3559,6 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
 
     const bool has_mask  = op->src[3] != NULL;
     const bool has_sinks = op->src[4] != NULL;
-
-    GGML_ASSERT(!has_rows || has_mask);
     const bool has_bias  = max_bias != 0.0f;
     const bool has_scap  = logit_softcap != 0.0f;
 
@@ -3647,25 +3595,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
     const bool use_sparse = n_kv_max_sparse > 0;
     const int n_kv_max_padded = use_sparse ? GGML_PAD(n_kv_max_sparse, OP_FLASH_ATTN_EXT_VEC_NCPSG) : 0;
 
-    const bool use_vec = ggml_metal_op_flash_attn_ext_use_vec(op);
-
-    // with kv_rows, the non-vec path gathers the referenced rows into a contiguous F16 buffer and
-    // attends over it with the F16 kernel - quantized KV is first dequantized with the F16 pass
-    const bool rows_gather = has_rows && !use_vec;
-
     // the vec kernels dequantize the KV inline; no need for the F16 dequant pass in the sparse path
-    const bool dequant_kv = !use_sparse && (rows_gather ?
-        ggml_is_quantized(op->src[1]->type) : ggml_metal_op_flash_attn_ext_use_kv_f16(op));
-
-    const bool use_kv_f16 = !use_sparse && (rows_gather ? true : dequant_kv);
+    const bool use_kv_f16 = !use_sparse && ggml_metal_op_flash_attn_ext_use_kv_f16(op);
 
     ggml_metal_buffer_id bid_idx = bid_kv_f16;
     bid_idx.offs += ggml_metal_op_flash_attn_ext_extra_kv_f16(op);
 
-    // contiguous K/V buffers holding the rows referenced by the kv_rows lists
-    ggml_metal_buffer_id bid_rows = bid_idx;
-    bid_rows.offs += ggml_metal_op_flash_attn_ext_extra_idx(op);
-
     ggml_metal_buffer_id bid_k = bid_src1;
     ggml_metal_buffer_id bid_v = bid_src2;
 
@@ -3678,7 +3613,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
     uint64_t nb22_attn = nb22;
     uint64_t nb23_attn = nb23;
 
-    if (dequant_kv) {
+    if (use_kv_f16) {
         assert(ggml_metal_op_flash_attn_ext_extra_kv_f16(op) != 0);
 
         const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op);
@@ -3766,70 +3701,6 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
         }
     }
 
-    // with kv_rows, gather the cache rows referenced by the lists into contiguous F16 buffers
-    // the attention then runs as a standard dense op over the gathered rows (tensor or half8x8 kernel)
-    if (rows_gather) {
-        assert(ggml_metal_op_flash_attn_ext_extra_rows(op) != 0);
-
-        const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op);
-
-        GGML_ASSERT(ne20 % 4 == 0);
-
-        const size_t k_size = GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne10*ne11_attn*ne12*ne03, 16);
-
-        ggml_metal_kargs_flash_attn_ext_rows args0 = {
-            /*.ne11    =*/ (int32_t) ne11_attn,
-            /*.ne_12_2 =*/ ne12,
-            /*.ne10    =*/ ne10,
-            /*.ne20    =*/ v_is_view_of_k ? 0 : ne20, // the gathered V is a view of the gathered K
-            /*.v_off   =*/ k_size,
-            /*.nb11    =*/ nb11_attn,
-            /*.nb12    =*/ nb12_attn,
-            /*.nb21    =*/ nb21_attn,
-            /*.nb22    =*/ nb22_attn,
-        };
-
-        auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_rows(lib, op);
-        const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline0), 64);
-
-        ggml_metal_encoder_set_pipeline(enc, pipeline0);
-        ggml_metal_encoder_set_bytes   (enc, &args0, sizeof(args0), 0);
-        ggml_metal_encoder_set_buffer  (enc, bid_k, 1);
-        ggml_metal_encoder_set_buffer  (enc, bid_v, 2);
-        ggml_metal_encoder_set_buffer  (enc, ggml_metal_get_buffer_id(kv_rows), 3);
-        ggml_metal_encoder_set_buffer  (enc, bid_rows, 4);
-
-        ggml_metal_encoder_dispatch_threadgroups(enc, ne11_attn, ne03, 1, nth, 1, 1);
-
-        ggml_metal_op_concurrency_reset(ctx);
-
-        // contiguous F16 layout of the gathered rows: [DK, n_rows, n_head_kv, n_streams]
-        bid_k = bid_rows;
-
-        nb10_attn = sizeof(ggml_fp16_t);
-        nb11_attn = nb10_attn*ne10;
-        nb12_attn = nb11_attn*ne11_attn;
-        nb13_attn = nb12_attn*ne12;
-
-        // if V is a view of K, the gathered V is read from the gathered K with K's strides
-        if (v_is_view_of_k) {
-            bid_v = bid_k;
-
-            nb20_attn = nb10_attn;
-            nb21_attn = nb11_attn;
-            nb22_attn = nb12_attn;
-            nb23_attn = nb13_attn;
-        } else {
-            bid_v = bid_rows;
-            bid_v.offs += k_size;
-
-            nb20_attn = sizeof(ggml_fp16_t);
-            nb21_attn = nb20_attn*ne20;
-            nb22_attn = nb21_attn*ne11_attn;
-            nb23_attn = nb22_attn*ne22;
-        }
-    }
-
     if (!use_sparse && ggml_metal_op_flash_attn_ext_use_tensor(op, props_dev->has_tensor)) {
         // tensor API kernel
         const int nqptg = ne00 >= 512 ? OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE : OP_FLASH_ATTN_EXT_TENSOR_NQPSG; // queries per threadgroup
@@ -3875,9 +3746,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
             /*.nb01          =*/ nb01,
             /*.nb02          =*/ nb02,
             /*.nb03          =*/ nb03,
-            /*.ne11          =*/ (int32_t) ne11_attn,
+            /*.ne11          =*/ ne11,
             /*.ne_12_2       =*/ ne12,
-            /*.ne_12_3       =*/ has_rows ? ne03 : ne13, // the gathered K/V holds one copy per stream
+            /*.ne_12_3       =*/ ne13,
             /*.ns10          =*/ ns10,
             /*.nb11          =*/ nb11_attn,
             /*.nb12          =*/ nb12_attn,
@@ -3935,7 +3806,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
 
         bool need_sync = false;
 
-        const bool has_kvpad = ne11_attn % ncpsg != 0;
+        const bool has_kvpad = ne11 % ncpsg != 0;
 
         if (has_kvpad) {
             assert(ggml_metal_op_flash_attn_ext_extra_pad(op) != 0);
@@ -4035,9 +3906,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
             /*.nb01          =*/ nb01,
             /*.nb02          =*/ nb02,
             /*.nb03          =*/ nb03,
-            /*.ne11          =*/ (int32_t) ne11_attn,
+            /*.ne11          =*/ ne11,
             /*.ne_12_2       =*/ ne12,
-            /*.ne_12_3       =*/ has_rows ? ne03 : ne13, // the gathered K/V holds one copy per stream
+            /*.ne_12_3       =*/ ne13,
             /*.ns10          =*/ ns10,
             /*.nb11          =*/ nb11_attn,
             /*.nb12          =*/ nb12_attn,
@@ -4103,7 +3974,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
 
         bool need_sync = false;
 
-        const bool has_kvpad = !use_sparse && ne11_attn % ncpsg != 0;
+        const bool has_kvpad = !use_sparse && ne11 % ncpsg != 0;
 
         if (use_sparse) {
             assert(ggml_metal_op_flash_attn_ext_extra_idx(op) != 0);
@@ -4186,13 +4057,13 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
         //   queries:      Q*NSG*PAD2(ne00, 128)
         //   attn + mask:  NSG*4*Q*C
         //   results:      2*NSG*Q*PAD2(ne20, 128)
-        //   sparse idx:   NSG*C ints (when use_sparse, or when gathering K/V rows)
+        //   sparse idx:   NSG*C ints (only when use_sparse)
         const int64_t dk_pad = GGML_PAD(ne00, 128);
         const int64_t dv_pad = GGML_PAD(ne20, 128);
 
         auto fa_vec_smem = [&](int64_t nsg, int32_t nqptg) -> size_t {
             const size_t smem_half = (size_t) (dk_pad + 4*ncpsg + 2*dv_pad)*nqptg*nsg;
-            return GGML_PAD(smem_half*sizeof(ggml_fp16_t) + ((use_sparse || has_rows) ? (size_t) nsg*ncpsg*sizeof(int) : 0), 16);
+            return GGML_PAD(smem_half*sizeof(ggml_fp16_t) + (use_sparse ? (size_t) nsg*ncpsg*sizeof(int) : 0), 16);
         };
 
         int64_t nsg = 1;
@@ -4223,7 +4094,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
         } else {
             nwg = 32;
             nsg = 1;
-            while (2*nwg*nsg*ncpsg < ne11_attn && nsg < 4) {
+            while (2*nwg*nsg*ncpsg < ne11 && nsg < 4) {
                 nsg *= 2;
             }
         }
@@ -4244,7 +4115,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
             /*.nb01          =*/ nb01,
             /*.nb02          =*/ nb02,
             /*.nb03          =*/ nb03,
-            /*.ne11          =*/ use_sparse ? n_kv_max_padded : (int32_t) ne11_attn,
+            /*.ne11          =*/ use_sparse ? n_kv_max_padded : ne11,
             /*.ne_12_2       =*/ ne12,
             /*.ne_12_3       =*/ ne13,
             /*.ns10          =*/ ns10,
@@ -4273,7 +4144,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
             /*.n_kv_max_padded =*/ n_kv_max_padded,
         };
 
-        auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, use_sparse, has_rows, nqptg, cfg.NE, nsg, nwg, use_kv_f16, ns10, ns20);
+        auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, use_sparse, nqptg, cfg.NE, nsg, nwg, use_kv_f16, ns10, ns20);
 
         GGML_ASSERT(nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));
 
@@ -4284,7 +4155,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
         ggml_metal_encoder_set_buffer  (enc, bid_v,    3);
         ggml_metal_encoder_set_buffer  (enc, bid_src3, 4);
         ggml_metal_encoder_set_buffer  (enc, bid_src4, 5);
-        ggml_metal_encoder_set_buffer  (enc, use_sparse ? bid_idx : (has_rows ? ggml_metal_get_buffer_id(kv_rows) : bid_src0), 8);
+        ggml_metal_encoder_set_buffer  (enc, use_sparse ? bid_idx : bid_src0, 8);
 
         const size_t smem = fa_vec_smem(nsg, nqptg);
 

ggml/src/ggml-metal/ggml-metal-ops.h

@@ -45,7 +45,6 @@ size_t ggml_metal_op_flash_attn_ext_extra_blk(const struct ggml_tensor * op);
 size_t ggml_metal_op_flash_attn_ext_extra_tmp(const struct ggml_tensor * op);
 size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const struct ggml_tensor * op);
 size_t ggml_metal_op_flash_attn_ext_extra_idx(const struct ggml_tensor * op);
-size_t ggml_metal_op_flash_attn_ext_extra_rows(const struct ggml_tensor * op);
 
 size_t ggml_metal_op_argmax_extra(const struct ggml_tensor * op);
 

ggml/src/ggml-metal/ggml-metal.cpp

@@ -300,7 +300,6 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
                 res += ggml_metal_op_flash_attn_ext_extra_tmp(tensor);
                 res += ggml_metal_op_flash_attn_ext_extra_kv_f16(tensor);
                 res += ggml_metal_op_flash_attn_ext_extra_idx(tensor);
-                res += ggml_metal_op_flash_attn_ext_extra_rows(tensor);
             } break;
         case GGML_OP_CUMSUM:
         case GGML_OP_ARGSORT:

ggml/src/ggml-metal/kernels/fa_aux.metal

@@ -48,80 +48,6 @@ template [[host_name("kernel_flash_attn_ext_kv_q5_0_f16")]] kernel kernel_flash_
 template [[host_name("kernel_flash_attn_ext_kv_q5_1_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16<block_q5_1, 32, dequantize_q5_1>;
 template [[host_name("kernel_flash_attn_ext_kv_q8_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16<block_q8_0, 32, dequantize_q8_0>;
 
-// gather the K/V cache rows referenced by the kv_rows lists into contiguous F16 buffers
-// - one threadgroup per (row, stream); the attention is then run as a standard dense op over the
-//   gathered rows, which avoids the per-tile K/V gather in the main kernels
-// - negative rows are padding: filled with zeros, their mask entries are -INF so they contribute nothing
-kernel void kernel_flash_attn_ext_rows(
-        constant ggml_metal_kargs_flash_attn_ext_rows & args,
-        device const char * k_src,
-        device const char * v_src,
-        device const int  * rows,
-        device       char * dst,
-        uint3   tgpig[[threadgroup_position_in_grid]],
-        ushort  tiitg[[thread_index_in_threadgroup]],
-        ushort3   ntg[[threads_per_threadgroup]]) {
-    const int64_t i1 = tgpig[0]; // row index
-    const int64_t i3 = tgpig[1]; // stream
-
-    const int64_t row = rows[i1 + i3*args.ne11];
-
-    // gathered layout: [DK, n_rows, n_head_kv, n_streams] and [DV, n_rows, n_head_kv, n_streams] F16
-    device half4 * k_dst = (device half4 *) (dst + i3*args.ne_12_2*args.ne11*args.ne10*sizeof(half));
-    device half4 * v_dst = (device half4 *) (dst + args.v_off + i3*args.ne_12_2*args.ne11*args.ne20*sizeof(half));
-
-    // note: the K/V cache is shared by all streams (ne13 = ne23 = 1), the row list selects the rows
-    // each thread copies up to 4 half4 values (32 bytes) to keep the memory transactions large
-
-    // K: all heads of this row
-    {
-        const int64_t per_head = args.ne10/4;
-
-        for (int64_t i = 4*tiitg; i < args.ne_12_2*per_head; i += 4*ntg.x) {
-            const int64_t i2 = i/per_head;
-            const int64_t i0 = i%per_head;
-            const int      n  = (int) min((int64_t) 4, per_head - i0);
-
-            device half4 * dst4 = k_dst + (i2*args.ne11 + i1)*per_head + i0;
-
-            if (row >= 0) {
-                device const half4 * src4 = (device const half4 *) (k_src + row*args.nb11 + i2*args.nb12) + i0;
-                for (int j = 0; j < n; ++j) {
-                    dst4[j] = src4[j];
-                }
-            } else {
-                for (int j = 0; j < n; ++j) {
-                    dst4[j] = half4(0.0h);
-                }
-            }
-        }
-    }
-
-    // V: all heads of this row
-    {
-        const int64_t per_head = args.ne20/4;
-
-        for (int64_t i = 4*tiitg; i < args.ne_12_2*per_head; i += 4*ntg.x) {
-            const int64_t i2 = i/per_head;
-            const int64_t i0 = i%per_head;
-            const int      n  = (int) min((int64_t) 4, per_head - i0);
-
-            device half4 * dst4 = v_dst + (i2*args.ne11 + i1)*per_head + i0;
-
-            if (row >= 0) {
-                device const half4 * src4 = (device const half4 *) (v_src + row*args.nb21 + i2*args.nb22) + i0;
-                for (int j = 0; j < n; ++j) {
-                    dst4[j] = src4[j];
-                }
-            } else {
-                for (int j = 0; j < n; ++j) {
-                    dst4[j] = half4(0.0h);
-                }
-            }
-        }
-    }
-}
-
 constant bool FC_flash_attn_ext_pad_has_mask [[function_constant(FC_FLASH_ATTN_EXT_PAD + 0)]];
 
 constant int32_t FC_flash_attn_ext_pad_ncpsg [[function_constant(FC_FLASH_ATTN_EXT_PAD + 25)]];

ggml/src/ggml-metal/kernels/fa_vec_common.metal

@@ -13,7 +13,6 @@ constant int32_t FC_flash_attn_ext_vec_ns20 [[function_constant(FC_FLASH_ATTN_EX
 constant int32_t FC_flash_attn_ext_vec_nsg  [[function_constant(FC_FLASH_ATTN_EXT_VEC + 22)]];
 constant int32_t FC_flash_attn_ext_vec_nwg  [[function_constant(FC_FLASH_ATTN_EXT_VEC + 23)]];
 constant bool    FC_flash_attn_ext_vec_has_sparse [[function_constant(FC_FLASH_ATTN_EXT_VEC + 5)]];
-constant bool    FC_flash_attn_ext_vec_has_rows   [[function_constant(FC_FLASH_ATTN_EXT_VEC + 6)]];
 template<
     typename q4_t,  // query types in shared memory
     typename k4_t,  // key types in shared memory
@@ -91,8 +90,7 @@ kernel void kernel_flash_attn_ext_vec(
     threadgroup o4_t  * so4 = (threadgroup o4_t  *) (shmem_f16 + SMEM_Q + SMEM_S + 2*sgitg*Q*PV); // scratch buffer for the results
 
     // sparse indices for the current block
-    // note: with kv_rows, the buffer holds the row list of the current slice instead
-    threadgroup int * spidx = (FC_flash_attn_ext_vec_has_sparse || FC_flash_attn_ext_vec_has_rows)
+    threadgroup int * spidx = FC_flash_attn_ext_vec_has_sparse
         ? (threadgroup int *) (shmem_f16 + SMEM) + sgitg*C
         : nullptr;
 
@@ -167,12 +165,6 @@ kernel void kernel_flash_attn_ext_vec(
                 ((int64_t)(iq3%args.ne33)*args.ne32 + (iq2%args.ne32))*args.ne31*args.n_kv_max_padded + (iq1%args.ne31)*args.n_kv_max_padded;
         }
 
-        // K/V row list of the current slice (ggml_flash_attn_ext_rows)
-        // with kv_rows, the loop below runs over the mask columns (args.ne11) and the K/V cache rows
-        // are read indirectly through the list. negative entries are padding, their mask is -INF
-        device const int * rows_list = FC_flash_attn_ext_vec_has_rows ?
-            (device const int *) idx + (int64_t)(iq3%args.ne33)*args.ne11 : nullptr;
-
         float slope = 1.0f;
 
         // ALiBi
@@ -230,12 +222,12 @@ kernel void kernel_flash_attn_ext_vec(
                 ic = 0;
             }
 
-            // load the KV row indices for the current block into shared memory
-            if (FC_flash_attn_ext_vec_has_sparse || FC_flash_attn_ext_vec_has_rows) {
+            // load the sparse KV indices for the current block into shared memory
+            if (FC_flash_attn_ext_vec_has_sparse) {
                 FOR_UNROLL (short ii = 0; ii < C/NW; ++ii) {
                     const short i = ii*NW + tiisg;
 
-                    spidx[i] = FC_flash_attn_ext_vec_has_rows ? rows_list[ic + i] : pidx[ic + i];
+                    spidx[i] = pidx[ic + i];
                 }
                 simdgroup_barrier(mem_flags::mem_threadgroup);
             }
@@ -250,15 +242,6 @@ kernel void kernel_flash_attn_ext_vec(
                             sm[qq*C + tiisg] = -MAXHALF;
                         }
                     }
-                } else if (FC_flash_attn_ext_vec_has_rows) {
-                    // the mask is compact: column ic maps to cache row spidx[tiisg]
-                    FOR_UNROLL (short qq = 0; qq < Q; ++qq) {
-                        if ((iq1*Q + qq) < args.ne01) {
-                            sm[qq*C + tiisg] = pm[qq][ic + tiisg];
-                        } else {
-                            sm[qq*C + tiisg] = -MAXHALF;
-                        }
-                    }
                 } else {
                     FOR_UNROLL (short qq = 0; qq < Q; ++qq) {
                         if ((iq1*Q + qq) < args.ne01) {
@@ -293,7 +276,7 @@ kernel void kernel_flash_attn_ext_vec(
             {
                 device      const k4_t * pk4 = nullptr;
 
-                if (!FC_flash_attn_ext_vec_has_sparse && !FC_flash_attn_ext_vec_has_rows) {
+                if (!FC_flash_attn_ext_vec_has_sparse) {
                     pk4 = (device const k4_t *) (k + ic*args.nb11);
 
                     pk4 += ty*NS10/4 + tx;
@@ -308,7 +291,7 @@ kernel void kernel_flash_attn_ext_vec(
 
                 // each simdgroup processes Q queries and NE (NW/NL) cache elements
                 FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) {
-                    if (FC_flash_attn_ext_vec_has_sparse || FC_flash_attn_ext_vec_has_rows) {
+                    if (FC_flash_attn_ext_vec_has_sparse) {
                         // the KV rows are gathered from the index list; -1 entries are padding
                         const int i11 = spidx[NE*cc + ty];
                         if (i11 >= 0) {
@@ -451,7 +434,7 @@ kernel void kernel_flash_attn_ext_vec(
                     }
                 }
 
-                if (FC_flash_attn_ext_vec_has_sparse || FC_flash_attn_ext_vec_has_rows) {
+                if (FC_flash_attn_ext_vec_has_sparse) {
                     FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) {
                         // the KV rows are gathered from the index list; -1 entries are padding
                         const int i11 = spidx[NE*cc + ty];

ggml/src/ggml-metal/kernels/quantize.h

@@ -20,7 +20,6 @@ void quantize_q1_0(device const float * src, device block_q1_0 & dst) {
 }
 
 void quantize_q2_0(device const float * src, device block_q2_0 & dst) {
-#pragma METAL fp math_mode(safe)
     float amax = 0.0f;
     for (int j = 0; j < QK2_0; j++) {
         float a = fabs(src[j]);

ggml/src/ggml-opencl/CMakeLists.txt

@@ -67,11 +67,9 @@ set(GGML_OPENCL_KERNELS
     argmax
     argsort
     top_k
-    top_k_mask
     lightning_indexer
     tri
     fill
-    fwht
     clamp
     cpy
     cvt
@@ -119,21 +117,6 @@ set(GGML_OPENCL_KERNELS
     gemm_noshuffle_q2_0_f32
     gemm_noshuffle_q2_0_q8_1_dp4a
     gemv_noshuffle_q2_0_f32
-    gemm_noshuffle_iq1_m_f32
-    gemm_noshuffle_iq1_m_q8_1_dp4a
-    gemv_noshuffle_iq1_m_f32
-    gemm_noshuffle_iq1_s_f32
-    gemm_noshuffle_iq1_s_q8_1_dp4a
-    gemv_noshuffle_iq1_s_f32
-    gemm_noshuffle_iq3_s_f32
-    gemm_noshuffle_iq3_s_q8_1_dp4a
-    gemv_noshuffle_iq3_s_f32
-    gemm_noshuffle_iq3_xxs_f32
-    gemm_noshuffle_iq3_xxs_q8_1_dp4a
-    gemv_noshuffle_iq3_xxs_f32
-    gemm_noshuffle_iq4_xs_f32
-    gemm_noshuffle_iq4_xs_q8_1_dp4a
-    gemv_noshuffle_iq4_xs_f32
     gemm_noshuffle_q3_k_q8_1_dp4a
     gemv_noshuffle_q3_k_f32
     mul_mv_q2_k_f32
@@ -153,25 +136,6 @@ set(GGML_OPENCL_KERNELS
     mul_mv_q2_0_f32
     mul_mv_q2_0_f32_flat
     mul_mm_q2_0_f32_l4_lm
-    mul_mv_iq1_m_f32
-    mul_mv_iq1_m_f32_flat
-    mul_mm_iq1_m_f32_l4_lm
-    iq1_s_grid
-    mul_mv_iq1_s_f32
-    mul_mv_iq1_s_f32_flat
-    mul_mm_iq1_s_f32_l4_lm
-    iq3_xxs_grid
-    mul_mv_iq3_xxs_f32
-    mul_mv_iq3_xxs_f32_flat
-    mul_mm_iq3_xxs_f32_l4_lm
-    iq3_s_grid
-    mul_mv_iq3_s_f32
-    mul_mv_iq3_s_f32_flat
-    mul_mm_iq3_s_f32_l4_lm
-    iq4_xs_grid
-    mul_mv_iq4_xs_f32
-    mul_mv_iq4_xs_f32_flat
-    mul_mm_iq4_xs_f32_l4_lm
     mul_mv_q3_k_f32_flat
     mul_mm_q3_k_f32_l4_lm
     mul_mv_q2_k_f32_flat

ggml/src/ggml-opencl/kernels/cvt.cl

@@ -3088,293 +3088,3 @@ kernel void kernel_restore_block_q2_0(
         dst[i].qs[j] = src_qs[16*i + j];
     }
 }
-
-struct block_iq1_s {
-    half d;
-    uchar qs[QK_K/8];
-    ushort qh[QK_K/32];
-};
-
-kernel void kernel_convert_block_iq1_s(
-    global struct block_iq1_s * src0,
-    global uchar * dst_qs,
-    global ushort * dst_qh,
-    global half * dst_d,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst_d[i] = src0[i].d;
-    for (int j = 0; j < 32; ++j) {
-        dst_qs[32*i + j] = src0[i].qs[j];
-    }
-    for (int j = 0; j < 8; ++j) {
-        dst_qh[8*i + j] = src0[i].qh[j];
-    }
-}
-
-kernel void kernel_restore_block_iq1_s(
-    global uchar * src_qs,
-    global ushort * src_qh,
-    global half * src_d,
-    global struct block_iq1_s * dst,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst[i].d = src_d[i];
-    for (int j = 0; j < 32; ++j) {
-        dst[i].qs[j] = src_qs[32*i + j];
-    }
-    for (int j = 0; j < 8; ++j) {
-        dst[i].qh[j] = src_qh[8*i + j];
-    }
-}
-
-struct block_iq1_m {
-    uchar qs[QK_K/8];
-    uchar qh[QK_K/16];
-    uchar scales[QK_K/32];
-};
-
-kernel void kernel_convert_block_iq1_m(
-    global struct block_iq1_m * src0,
-    global uchar * dst_qs,
-    global uchar * dst_qh,
-    global ushort * dst_sc,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    for (int j = 0; j < 32; ++j) {
-        dst_qs[32*i + j] = src0[i].qs[j];
-    }
-    for (int j = 0; j < 16; ++j) {
-        dst_qh[16*i + j] = src0[i].qh[j];
-    }
-    for (int j = 0; j < 4; ++j) {
-        dst_sc[4*i + j] = (ushort)src0[i].scales[2*j] | ((ushort)src0[i].scales[2*j + 1] << 8);
-    }
-}
-
-kernel void kernel_restore_block_iq1_m(
-    global uchar * src_qs,
-    global uchar * src_qh,
-    global ushort * src_sc,
-    global struct block_iq1_m * dst,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    for (int j = 0; j < 32; ++j) {
-        dst[i].qs[j] = src_qs[32*i + j];
-    }
-    for (int j = 0; j < 16; ++j) {
-        dst[i].qh[j] = src_qh[16*i + j];
-    }
-    for (int j = 0; j < 4; ++j) {
-        ushort sc = src_sc[4*i + j];
-        dst[i].scales[2*j] = (uchar)sc;
-        dst[i].scales[2*j + 1] = (uchar)(sc >> 8);
-    }
-}
-
-struct block_iq3_s {
-    half d;
-    uchar qs[QK_K/4];
-    uchar qh[QK_K/32];
-    uchar signs[QK_K/8];
-    uchar scales[QK_K/64];
-};
-
-kernel void kernel_convert_block_iq3_s(
-    global struct block_iq3_s * src0,
-    global uchar * dst_qs,
-    global uchar * dst_sg,
-    global uchar * dst_qh,
-    global uchar * dst_sc,
-    global half * dst_d,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst_d[i] = src0[i].d;
-    for (int j = 0; j < 64; ++j) {
-        dst_qs[64*i + j] = src0[i].qs[j];
-    }
-    for (int j = 0; j < 32; ++j) {
-        dst_sg[32*i + j] = src0[i].signs[j];
-    }
-    for (int j = 0; j < 8; ++j) {
-        dst_qh[8*i + j] = src0[i].qh[j];
-    }
-    for (int j = 0; j < 4; ++j) {
-        dst_sc[4*i + j] = src0[i].scales[j];
-    }
-}
-
-kernel void kernel_restore_block_iq3_s(
-    global uchar * src_qs,
-    global uchar * src_sg,
-    global uchar * src_qh,
-    global uchar * src_sc,
-    global half * src_d,
-    global struct block_iq3_s * dst,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst[i].d = src_d[i];
-    for (int j = 0; j < 64; ++j) {
-        dst[i].qs[j] = src_qs[64*i + j];
-    }
-    for (int j = 0; j < 32; ++j) {
-        dst[i].signs[j] = src_sg[32*i + j];
-    }
-    for (int j = 0; j < 8; ++j) {
-        dst[i].qh[j] = src_qh[8*i + j];
-    }
-    for (int j = 0; j < 4; ++j) {
-        dst[i].scales[j] = src_sc[4*i + j];
-    }
-}
-
-struct block_iq3_xxs {
-    half d;
-    uchar qs[3*QK_K/8];
-};
-
-kernel void kernel_convert_block_iq3_xxs(
-    global struct block_iq3_xxs * src0,
-    global uchar * dst_qs,
-    global uint * dst_sas,
-    global half * dst_d,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst_d[i] = src0[i].d;
-    for (int j = 0; j < 64; ++j) {
-        dst_qs[64*i + j] = src0[i].qs[j];
-    }
-    for (int j = 0; j < 8; ++j) {
-        global ushort * q = (global ushort *)(src0[i].qs + 64 + 4*j);
-        dst_sas[8*i + j] = (uint)q[0] | ((uint)q[1] << 16);
-    }
-}
-
-kernel void kernel_restore_block_iq3_xxs(
-    global uchar * src_qs,
-    global uint * src_sas,
-    global half * src_d,
-    global struct block_iq3_xxs * dst,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst[i].d = src_d[i];
-    for (int j = 0; j < 64; ++j) {
-        dst[i].qs[j] = src_qs[64*i + j];
-    }
-    for (int j = 0; j < 8; ++j) {
-        global ushort * q = (global ushort *)(dst[i].qs + 64 + 4*j);
-        uint sas = src_sas[8*i + j];
-        q[0] = (ushort)sas;
-        q[1] = (ushort)(sas >> 16);
-    }
-}
-
-struct block_iq4_xs {
-    half d;
-    ushort scales_h;
-    uchar scales_l[QK_K/64];
-    uchar qs[QK_K/2];
-};
-
-kernel void kernel_convert_block_iq4_xs(
-    global struct block_iq4_xs * src0,
-    global uchar * dst_qs,
-    global uchar * dst_sl,
-    global ushort * dst_sh,
-    global half * dst_d,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst_d[i] = src0[i].d;
-    dst_sh[i] = src0[i].scales_h;
-    for (int j = 0; j < 128; ++j) {
-        dst_qs[128*i + j] = src0[i].qs[j];
-    }
-    for (int j = 0; j < 4; ++j) {
-        dst_sl[4*i + j] = src0[i].scales_l[j];
-    }
-}
-
-kernel void kernel_restore_block_iq4_xs(
-    global uchar * src_qs,
-    global uchar * src_sl,
-    global ushort * src_sh,
-    global half * src_d,
-    global struct block_iq4_xs * dst,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst[i].d = src_d[i];
-    dst[i].scales_h = src_sh[i];
-    for (int j = 0; j < 128; ++j) {
-        dst[i].qs[j] = src_qs[128*i + j];
-    }
-    for (int j = 0; j < 4; ++j) {
-        dst[i].scales_l[j] = src_sl[4*i + j];
-    }
-}
-
-kernel void kernel_convert_block_iq4_xs_noshuffle(
-    global struct block_iq4_xs * src0,
-    global ushort * dst_qs,
-    global uint * dst_sl,
-    global ushort * dst_sh,
-    global half * dst_d,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst_d[i] = src0[i].d;
-    dst_sh[i] = src0[i].scales_h;
-    dst_sl[i] = as_uint(vload4(0, src0[i].scales_l));
-    for (int sb = 0; sb < 8; ++sb) {
-        for (int u = 0; u < 8; ++u) {
-            ushort q = 0;
-            for (int j = 0; j < 4; ++j) {
-                uchar v = src0[i].qs[16*sb + 4*(u & 3) + j];
-                q |= (ushort)(((v >> (4*(u >> 2))) & 15) << (4*j));
-            }
-            dst_qs[64*i + 8*sb + u] = q;
-        }
-    }
-}
-
-kernel void kernel_restore_block_iq4_xs_noshuffle(
-    global ushort * src_qs,
-    global uint * src_sl,
-    global ushort * src_sh,
-    global half * src_d,
-    global struct block_iq4_xs * dst,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst[i].d = src_d[i];
-    dst[i].scales_h = src_sh[i];
-    vstore4(as_uchar4(src_sl[i]), 0, dst[i].scales_l);
-    for (int sb = 0; sb < 8; ++sb) {
-        for (int j = 0; j < 16; ++j) {
-            const int shift = 4*(j & 3);
-            const ushort lo = src_qs[64*i + 8*sb + j/4];
-            const ushort hi = src_qs[64*i + 8*sb + j/4 + 4];
-            dst[i].qs[16*sb + j] = (uchar)(((lo >> shift) & 15) | (((hi >> shift) & 15) << 4));
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/fwht.cl

@@ -1,44 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-kernel void kernel_fwht(
-        global const char * src,
-        ulong offset_src,
-        global char * dst,
-        ulong offset_dst,
-        int n,
-        ulong ne1,
-        ulong ne2,
-        ulong nb1,
-        ulong nb2,
-        ulong nb3,
-        float scale,
-        local float * work) {
-    const ulong row = get_group_id(1);
-    const ulong i1 = row % ne1;
-    const ulong i2 = row / ne1 % ne2;
-    const ulong i3 = row / ne1 / ne2;
-    global const SRC_TYPE * x = (global const SRC_TYPE *)(src + offset_src + i1*nb1 + i2*nb2 + i3*nb3);
-    global float * y = (global float *)(dst + offset_dst) + row*n;
-    const int tid = get_local_id(0);
-    const int nth = get_local_size(0);
-
-    for (int i = tid; i < n; i += nth) {
-        work[i] = (float)x[i] * scale;
-    }
-    barrier(CLK_LOCAL_MEM_FENCE);
-
-    for (int h = 1; h < n; h *= 2) {
-        for (int i = tid; i < n/2; i += nth) {
-            const int j = 2*i - (i & (h - 1));
-            const float a = work[j];
-            const float b = work[j + h];
-            work[j] = a + b;
-            work[j + h] = a - b;
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    for (int i = tid; i < n; i += nth) {
-        y[i] = work[i];
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_m_f32.cl

@@ -1,93 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_qcom_reqd_sub_group_size
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-__attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-kernel void kernel_gemm_noshuffle_iq1_m_f32(
-    global const uchar * src0_qs,
-    global const uchar * src0_qh,
-    global const ushort * src0_sc,
-    read_only image1d_buffer_t src1,
-    global float * dst,
-    ulong offsetd,
-    int m,
-    int n,
-    int k,
-    int n_no_padding
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-    const int gy = get_global_id(0);
-    const int row = get_global_id(1) * 4;
-    const int n_4 = n / 4;
-    if (row >= m) {
-        return;
-    }
-
-    half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
-    for (int i = 0; i < k; i += 16) {
-        const uint4 w0 = convert_uint4(vload4(0, src0_sc + row + (4*(i / 256) + 0) * m));
-        const uint4 w1 = convert_uint4(vload4(0, src0_sc + row + (4*(i / 256) + 1) * m));
-        const uint4 w2 = convert_uint4(vload4(0, src0_sc + row + (4*(i / 256) + 2) * m));
-        const uint4 w3 = convert_uint4(vload4(0, src0_sc + row + (4*(i / 256) + 3) * m));
-        const ushort4 bits = convert_ushort4((w0 >> (uint4)(12)) | ((w1 >> (uint4)(8)) & (uint4)(0x00f0))
-                                           | ((w2 >> (uint4)(4)) & (uint4)(0x0f00)) | (w3 & (uint4)(0xf000)));
-        const half4 d = as_half4(bits);
-        const uint4 sc = convert_uint4(vload4(0, src0_sc + row + (i / 64) * m));
-        const half4 scale = d * convert_half4(2*((sc >> (uint4)(3*((i / 16) & 3))) & (uint4)(7)) + 1);
-        const uint4 qh = convert_uint4(vload4(0, src0_qh + row + (i / 16) * m));
-
-        for (int l = 0; l < 16; l += 4) {
-            const int ki = i + l;
-            const uint4 gi = convert_uint4(vload4(0, src0_qs + row + (ki / 8) * m))
-                           | (((qh >> (uint4)(4*((ki / 8) & 1))) & (uint4)(7)) << (uint4)(8));
-            const uint4 grid = (uint4)(iq1_s_grid[gi.s0], iq1_s_grid[gi.s1], iq1_s_grid[gi.s2], iq1_s_grid[gi.s3]);
-            const float4 delta = select((float4)(-0.875f), (float4)(-1.125f), (qh & (uint4)(0x08 << (4*((ki / 8) & 1)))) != 0);
-            #pragma unroll
-            for (int j = 0; j < 4; ++j) {
-                half8 b;
-                b.s0123 = read_imageh(src1, gy * 2     + (ki + j) * n_4);
-                b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
-                const float4 magnitude = convert_float4((grid >> (uint4)(8*j + (ki & 4))) & (uint4)(15));
-                const half4 w = convert_half4(magnitude + delta) * scale;
-                c0 += b * w.s0;
-                c1 += b * w.s1;
-                c2 += b * w.s2;
-                c3 += b * w.s3;
-            }
-        }
-    }
-
-    int idx = gy * 8 * m + row;
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_m_q8_1_dp4a.cl

@@ -1,144 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#pragma OPENCL EXTENSION cl_khr_subgroups : enable
-#ifdef cl_khr_integer_dot_product
-#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
-#endif
-
-#define QK_K 256
-
-#ifndef TILESIZE_N
-#define TILESIZE_N 32
-#endif
-
-inline uint iq1_m_pack(uint g, uint shift, uint negative_delta) {
-    // 8*(grid + delta) is an exact signed integer in [-9, 9].
-    const int bias = negative_delta ? -9 : -7;
-    const int4 values = 8*convert_int4(as_uchar4((g >> shift) & 0x0F0F0F0Fu)) + bias;
-    return as_uint(convert_char4(values));
-}
-
-// Pass staged operands by value; local dp4a operands miscompile on X2.
-inline int dot4_q8a_v(uint4 qw, uint4 a) {
-    int r = 0;
-    r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r);
-    return r;
-}
-
-__attribute__((qcom_wave_pair_mode(1)))
-kernel void kernel_gemm_noshuffle_iq1_m_q8_1_dp4a(
-        __global const uchar  * src0_qs,
-        __global const uchar  * src0_qh,
-        __global const ushort * src0_sc,
-        __global const uint   * src1_qa,
-        __global const half   * src1_da,
-        __global       float  * dst,
-        ulong  offsetd,
-        int    m,
-        int    n_no_padding,
-        int    k
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-
-    const uint lid = get_local_id(0);
-    const uint block_id_m = get_global_id(1);
-    const uint block_id_n = get_global_id(2);
-
-    const uint row      = block_id_m * 64 + lid;
-    const uint col_base = block_id_n * TILESIZE_N;
-    const bool row_valid = row < (uint)m;
-    const uint rrow     = row_valid ? row : 0;
-
-    const uint k_u = (uint)k >> 2;
-    const uint k_b = (uint)k >> 5;
-
-    __local uint4 sh_qa4[TILESIZE_N][2];
-    __local half sh_d[TILESIZE_N];
-
-#define IQ1_M_GRID(i) iq1_s_grid[(i)]
-
-#define NGROUPS (TILESIZE_N / 4)
-    float4 acc[NGROUPS];
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
-
-    float dsuper = 0.f;
-    for (uint step = 0; step < (uint)k; step += 32) {
-        const uint sub = step >> 5;
-        const uint ib  = sub >> 3;
-
-        if ((sub & 7u) == 0u) {
-            const uint sb = rrow + 4u*ib*(uint)m;
-            const uint w0 = src0_sc[sb + 0u*(uint)m];
-            const uint w1 = src0_sc[sb + 1u*(uint)m];
-            const uint w2 = src0_sc[sb + 2u*(uint)m];
-            const uint w3 = src0_sc[sb + 3u*(uint)m];
-            const ushort bits = (ushort)((w0 >> 12) | ((w1 >> 8) & 0x00f0u)
-                                       | ((w2 >> 4) & 0x0f00u) | (w3 & 0xf000u));
-            dsuper = (float)as_half(bits);
-        }
-        const uint sc = src0_sc[rrow + (4u*ib + (sub & 7u)/2u)*(uint)m];
-        const uint sh = 6u*(sub & 1u);
-        const float dl0 = dsuper * (float)(2u*((sc >> sh) & 7u) + 1u) * 0.125f;
-        const float dl1 = dsuper * (float)(2u*((sc >> (sh + 3u)) & 7u) + 1u) * 0.125f;
-        const uint h0 = src0_qh[rrow + (2u*sub + 0u)*(uint)m];
-        const uint h1 = src0_qh[rrow + (2u*sub + 1u)*(uint)m];
-        const uint gb = rrow + 4u*sub*(uint)m;
-        const uint g0 = IQ1_M_GRID((uint)src0_qs[gb + 0u*(uint)m] | (( h0       & 7u) << 8));
-        const uint g1 = IQ1_M_GRID((uint)src0_qs[gb + 1u*(uint)m] | (((h0 >> 4) & 7u) << 8));
-        const uint g2 = IQ1_M_GRID((uint)src0_qs[gb + 2u*(uint)m] | (( h1       & 7u) << 8));
-        const uint g3 = IQ1_M_GRID((uint)src0_qs[gb + 3u*(uint)m] | (((h1 >> 4) & 7u) << 8));
-        const uint4 qlo = (uint4)(iq1_m_pack(g0, 0, h0 & 0x08u), iq1_m_pack(g0, 4, h0 & 0x08u),
-                                 iq1_m_pack(g1, 0, h0 & 0x80u), iq1_m_pack(g1, 4, h0 & 0x80u));
-        const uint4 qhi = (uint4)(iq1_m_pack(g2, 0, h1 & 0x08u), iq1_m_pack(g2, 4, h1 & 0x08u),
-                                 iq1_m_pack(g3, 0, h1 & 0x80u), iq1_m_pack(g3, 4, h1 & 0x80u));
-
-        for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
-            const uint t = idx >> 1;
-            const uint v = idx & 1;
-            const uint c = col_base + t;
-            sh_qa4[t][v] = (c < (uint)n_no_padding)
-                         ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
-                         : (uint4)(0u);
-        }
-        if (lid < TILESIZE_N) {
-            const uint c = col_base + lid;
-            sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
-#define IQ1_M_COL(b) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) \
-                           + dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1])))
-        #pragma unroll
-        for (int g = 0; g < NGROUPS; ++g) {
-            const int b = g * 4;
-            float4 rf;
-            rf.s0 = IQ1_M_COL(b+0);  rf.s1 = IQ1_M_COL(b+1);
-            rf.s2 = IQ1_M_COL(b+2);  rf.s3 = IQ1_M_COL(b+3);
-            acc[g] += LD4(sh_d, b) * rf;
-        }
-#undef IQ1_M_COL
-#undef LD4
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    if (!row_valid) {
-        return;
-    }
-
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) {
-        const uint b = (uint)(g * 4);
-        const float4 a = acc[g];
-        const uint c0 = col_base + b;
-        if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
-        if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
-        if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
-        if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
-    }
-#undef NGROUPS
-#undef IQ1_M_GRID
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_s_f32.cl

@@ -1,86 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_qcom_reqd_sub_group_size
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-__attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-kernel void kernel_gemm_noshuffle_iq1_s_f32(
-    global const uchar * src0_qs,
-    global const ushort * src0_qh,
-    global const half  * src0_d,
-    read_only image1d_buffer_t src1,
-    global float * dst,
-    ulong offsetd,
-    int m,
-    int n,
-    int k,
-    int n_no_padding
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-    const int gy = get_global_id(0);
-    const int row = get_global_id(1) * 4;
-    const int n_4 = n / 4;
-    if (row >= m) {
-        return;
-    }
-
-    half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
-    for (int i = 0; i < k; i += 16) {
-        const uint4 qh = convert_uint4(vload4(0, src0_qh + row + (i / 32) * m));
-        const half4 d = vload4(0, src0_d + row + (i / 256) * m);
-        const half4 scale = d * convert_half4(2*((qh >> (uint4)(12)) & (uint4)(7)) + 1);
-        const float4 delta = select((float4)(-0.875f), (float4)(-1.125f), (qh & (uint4)(0x8000)) != 0);
-
-        for (int l = 0; l < 16; l += 4) {
-            const int ki = i + l;
-            const uint4 gi = convert_uint4(vload4(0, src0_qs + row + (ki / 8) * m))
-                           | (((qh >> (uint4)(3*((ki / 8) & 3))) & (uint4)(7)) << (uint4)(8));
-            const uint4 grid = (uint4)(iq1_s_grid[gi.s0], iq1_s_grid[gi.s1], iq1_s_grid[gi.s2], iq1_s_grid[gi.s3]);
-            #pragma unroll
-            for (int j = 0; j < 4; ++j) {
-                half8 b;
-                b.s0123 = read_imageh(src1, gy * 2     + (ki + j) * n_4);
-                b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
-                const float4 magnitude = convert_float4((grid >> (uint4)(8*j + (ki & 4))) & (uint4)(15));
-                const half4 w = convert_half4(magnitude + delta) * scale;
-                c0 += b * w.s0;
-                c1 += b * w.s1;
-                c2 += b * w.s2;
-                c3 += b * w.s3;
-            }
-        }
-    }
-
-    int idx = gy * 8 * m + row;
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_s_q8_1_dp4a.cl

@@ -1,138 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#pragma OPENCL EXTENSION cl_khr_subgroups : enable
-#ifdef cl_khr_integer_dot_product
-#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
-#endif
-
-#define QK_K 256
-
-#ifndef TILESIZE_N
-#define TILESIZE_N 32
-#endif
-
-// Pass staged operands by value; local dp4a operands miscompile on X2.
-inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
-    int r = 0;
-    r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
-    return r;
-}
-
-__attribute__((qcom_wave_pair_mode(1)))
-kernel void kernel_gemm_noshuffle_iq1_s_q8_1_dp4a(
-        __global const uchar  * src0_qs,
-        __global const ushort * src0_qh,
-        __global const half   * src0_d,
-        __global const uint   * src1_qa,
-        __global const half   * src1_da,
-        __global const half   * src1_sa,
-        __global       float  * dst,
-        ulong  offsetd,
-        int    m,
-        int    n_no_padding,
-        int    k
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-
-    const uint lid = get_local_id(0);
-    const uint block_id_m = get_global_id(1);
-    const uint block_id_n = get_global_id(2);
-
-    const uint row      = block_id_m * 64 + lid;
-    const uint col_base = block_id_n * TILESIZE_N;
-    const bool row_valid = row < (uint)m;
-    const uint rrow     = row_valid ? row : 0;
-
-    const uint k_u = (uint)k >> 2;
-    const uint k_b = (uint)k >> 5;
-
-    __local uint4 sh_qa4[TILESIZE_N][2];
-    __local half sh_da[TILESIZE_N];
-    __local half sh_sa[TILESIZE_N];
-
-#define IQ1_S_GRID(i) iq1_s_grid[(i)]
-
-#define NGROUPS (TILESIZE_N / 4)
-    float4 acc[NGROUPS];
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
-
-    for (uint step = 0; step < (uint)k; step += 32) {
-        const uint sub = step >> 5;
-        const uint ib  = sub >> 3;
-
-        const uint  qhv = (uint)src0_qh[rrow + sub * (uint)m];
-        const float dl  = (float)src0_d[rrow + ib * (uint)m]
-                        * (float)(2u * ((qhv >> 12) & 7u) + 1u);
-
-        // The codebook stores grid + 1; sa includes the Q8_1 activation scale.
-        const float dlt = ((qhv & 0x8000u) ? -0.125f : 0.125f) - 1.0f;
-
-        const uint gb = rrow + (step >> 3) * (uint)m;
-
-        uint8 qw;
-        {
-            const uint g0 = IQ1_S_GRID((uint)src0_qs[gb + 0u * (uint)m] | ((( qhv       & 7u) << 8)));
-            const uint g1 = IQ1_S_GRID((uint)src0_qs[gb + 1u * (uint)m] | ((((qhv >> 3) & 7u) << 8)));
-            const uint g2 = IQ1_S_GRID((uint)src0_qs[gb + 2u * (uint)m] | ((((qhv >> 6) & 7u) << 8)));
-            const uint g3 = IQ1_S_GRID((uint)src0_qs[gb + 3u * (uint)m] | ((((qhv >> 9) & 7u) << 8)));
-            qw.s0 =  g0        & 0x0F0F0F0Fu;   qw.s1 = (g0 >> 4) & 0x0F0F0F0Fu;
-            qw.s2 =  g1        & 0x0F0F0F0Fu;   qw.s3 = (g1 >> 4) & 0x0F0F0F0Fu;
-            qw.s4 =  g2        & 0x0F0F0F0Fu;   qw.s5 = (g2 >> 4) & 0x0F0F0F0Fu;
-            qw.s6 =  g3        & 0x0F0F0F0Fu;   qw.s7 = (g3 >> 4) & 0x0F0F0F0Fu;
-        }
-
-        for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
-            const uint t = idx >> 1;
-            const uint v = idx & 1;
-            const uint c = col_base + t;
-            sh_qa4[t][v] = (c < (uint)n_no_padding)
-                         ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
-                         : (uint4)(0u);
-        }
-        if (lid < TILESIZE_N) {
-            const uint c = col_base + lid;
-            const bool ok = c < (uint)n_no_padding;
-            sh_da[lid] = ok ? src1_da[c * k_b + sub] : (half)0;
-            sh_sa[lid] = ok ? src1_sa[c * k_b + sub] : (half)0;
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
-        #pragma unroll
-        for (int g = 0; g < NGROUPS; ++g) {
-            const int b = g * 4;
-            float4 rf;
-#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1]))
-            rf.s0 = (float)TDOT4(b+0);  rf.s1 = (float)TDOT4(b+1);
-            rf.s2 = (float)TDOT4(b+2);  rf.s3 = (float)TDOT4(b+3);
-            acc[g] += dl * (LD4(sh_da, b) * rf) + (dl * dlt) * LD4(sh_sa, b);
-        }
-#undef TDOT4
-#undef LD4
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    if (!row_valid) {
-        return;
-    }
-
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) {
-        const uint b = (uint)(g * 4);
-        const float4 a = acc[g];
-        const uint c0 = col_base + b;
-        if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
-        if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
-        if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
-        if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
-    }
-#undef NGROUPS
-#undef IQ1_S_GRID
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_f32.cl

@@ -1,92 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_qcom_reqd_sub_group_size
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-__attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-kernel void kernel_gemm_noshuffle_iq3_s_f32(
-    global const uchar * src0_qs,
-    global const uchar * src0_sg,
-    global const uchar * src0_qh,
-    global const uchar * src0_sc,
-    global const half  * src0_d,
-    read_only image1d_buffer_t src1,
-    global float * dst,
-    ulong offsetd,
-    int m,
-    int n,
-    int k,
-    int n_no_padding
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-    const int gy = get_global_id(0);
-    const int row = get_global_id(1) * 4;
-    const int n_4 = n / 4;
-    if (row >= m) {
-        return;
-    }
-
-    half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
-    for (int i = 0; i < k; i += 16) {
-        const uchar4 sc = vload4(0, src0_sc + row + (i / 64) * m);
-        const half4 d = vload4(0, src0_d + row + (i / 256) * m);
-        const float4 scale_f = convert_float4(d) * (1.0f + 2.0f*convert_float4((sc >> (uchar4)(4*((i / 32) & 1))) & (uchar4)(15)));
-        const half4 scale = convert_half4(scale_f);
-
-        for (int l = 0; l < 16; l += 4) {
-            const int ki = i + l;
-            const uint4 qs = convert_uint4(vload4(0, src0_qs + row + (ki / 4) * m));
-            const uint4 qh = convert_uint4(vload4(0, src0_qh + row + (ki / 32) * m));
-            const uchar4 sg = vload4(0, src0_sg + row + (ki / 8) * m);
-            const uint4 gi = qs | (((qh >> (uint4)(((ki / 4) & 7))) & (uint4)(1)) << (uint4)(8));
-            const uint4 grid = (uint4)(iq3_s_grid[gi.s0], iq3_s_grid[gi.s1],
-                                      iq3_s_grid[gi.s2], iq3_s_grid[gi.s3]);
-            #pragma unroll
-            for (int j = 0; j < 4; ++j) {
-                half8 b;
-                b.s0123 = read_imageh(src1, gy * 2     + (ki + j) * n_4);
-                b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
-                const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255));
-                const int4 signs = convert_int4((sg >> (uchar4)((ki & 7) + j)) & (uchar4)(1));
-                const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale;
-                c0 += b * w.s0;
-                c1 += b * w.s1;
-                c2 += b * w.s2;
-                c3 += b * w.s3;
-            }
-        }
-    }
-
-    int idx = gy * 8 * m + row;
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_q8_1_dp4a.cl

@@ -1,154 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#pragma OPENCL EXTENSION cl_khr_subgroups : enable
-#ifdef cl_khr_integer_dot_product
-#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
-#endif
-
-#define QK_K 256
-
-#ifndef TILESIZE_N
-#define TILESIZE_N 32
-#endif
-
-inline uint iq3_s_pack(uint gv, uint sg, uint base) {
-    int v0 = (int)((gv >>  0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; }
-    int v1 = (int)((gv >>  8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; }
-    int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; }
-    int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; }
-    return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) <<  8)
-         | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24);
-}
-
-#define IQ3_S_GEMM_GRID(i) iq3_s_grid[(i)]
-
-inline uint iq3_s_load(__global const uchar * qs, uint qsb, uint qhb, uint m,
-                      uint u, uint sgv, uint base) {
-    const uint gidx = (uint)qs[qsb + u * m] | (((qhb >> u) & 1u) << 8);
-    return iq3_s_pack(IQ3_S_GEMM_GRID(gidx), sgv, base);
-}
-
-// Pass staged operands by value; local dp4a operands miscompile on X2.
-inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
-    int r = 0;
-    r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
-    return r;
-}
-
-__attribute__((qcom_wave_pair_mode(1)))
-kernel void kernel_gemm_noshuffle_iq3_s_q8_1_dp4a(
-        __global const uchar  * src0_qs,
-        __global const uchar  * src0_sg,
-        __global const uchar  * src0_qh,
-        __global const uchar  * src0_sc,
-        __global const half   * src0_d,
-        __global const uint   * src1_qa,
-        __global const half   * src1_da,
-        __global       float  * dst,
-        ulong  offsetd,
-        int    m,
-        int    n_no_padding,
-        int    k
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-
-    const uint lid = get_local_id(0);
-    const uint block_id_m = get_global_id(1);
-    const uint block_id_n = get_global_id(2);
-
-    const uint row      = block_id_m * 64 + lid;
-    const uint col_base = block_id_n * TILESIZE_N;
-    const bool row_valid = row < (uint)m;
-    const uint rrow     = row_valid ? row : 0;
-
-    const uint k_u = (uint)k >> 2;
-    const uint k_b = (uint)k >> 5;
-
-    __local uint4 sh_qa4[TILESIZE_N][2];
-    __local half sh_d[TILESIZE_N];
-
-#define NGROUPS (TILESIZE_N / 4)
-    float4 acc[NGROUPS];
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
-
-    for (uint step = 0; step < (uint)k; step += 32) {
-        const uint sub = step >> 5;
-
-        const uint ib = sub >> 3;
-        const uint sb = sub & 7u;
-
-        const uint  scb = (uint)src0_sc[rrow + (ib*(QK_K/64) + (sb >> 1)) * (uint)m];
-        const uint  nib = (sb & 1u) ? (scb >> 4) : (scb & 0xFu);
-        const float d_w = (float)src0_d[rrow + ib * (uint)m] * (float)(1 + 2*nib);
-
-        const uint qhb  = (uint)src0_qh[rrow + (ib*(QK_K/32) + sb) * (uint)m];
-        const uint qsb  = rrow + (ib*(QK_K/4) + 8*sb) * (uint)m;
-        const uint sgb  = rrow + (ib*(QK_K/8) + 4*sb) * (uint)m;
-
-        const uint sg0 = (uint)src0_sg[sgb + 0 * (uint)m];
-        const uint sg1 = (uint)src0_sg[sgb + 1 * (uint)m];
-        const uint sg2 = (uint)src0_sg[sgb + 2 * (uint)m];
-        const uint sg3 = (uint)src0_sg[sgb + 3 * (uint)m];
-
-        uint8 qw;
-        qw.s0 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 0u, sg0, 0u);
-        qw.s1 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 1u, sg0, 4u);
-        qw.s2 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 2u, sg1, 0u);
-        qw.s3 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 3u, sg1, 4u);
-        qw.s4 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 4u, sg2, 0u);
-        qw.s5 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 5u, sg2, 4u);
-        qw.s6 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 6u, sg3, 0u);
-        qw.s7 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 7u, sg3, 4u);
-
-        for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
-            const uint t = idx >> 1;
-            const uint v = idx & 1;
-            const uint c = col_base + t;
-            sh_qa4[t][v] = (c < (uint)n_no_padding)
-                         ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
-                         : (uint4)(0u);
-        }
-        if (lid < TILESIZE_N) {
-            const uint c = col_base + lid;
-            sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
-        #pragma unroll
-        for (int g = 0; g < NGROUPS; ++g) {
-            const int b = g * 4;
-            float4 rf;
-#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1]))
-            rf.s0 = (float)TDOT4(b+0);  rf.s1 = (float)TDOT4(b+1);
-            rf.s2 = (float)TDOT4(b+2);  rf.s3 = (float)TDOT4(b+3);
-            acc[g] += d_w * LD4(sh_d, b) * rf;
-        }
-#undef TDOT4
-#undef LD4
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    if (!row_valid) {
-        return;
-    }
-
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) {
-        const uint b = (uint)(g * 4);
-        const float4 a = acc[g];
-        const uint c0 = col_base + b;
-        if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
-        if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
-        if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
-        if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
-    }
-#undef NGROUPS
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_f32.cl

@@ -1,89 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_qcom_reqd_sub_group_size
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-__attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-kernel void kernel_gemm_noshuffle_iq3_xxs_f32(
-    global const uchar * src0_qs,
-    global const uint * src0_sas,
-    global const half  * src0_d,
-    read_only image1d_buffer_t src1,
-    global float * dst,
-    ulong offsetd,
-    int m,
-    int n,
-    int k,
-    int n_no_padding
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-    const int gy = get_global_id(0);
-    const int row = get_global_id(1) * 4;
-    const int n_4 = n / 4;
-    if (row >= m) {
-        return;
-    }
-
-    half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
-    for (int i = 0; i < k; i += 16) {
-        const uint4 sas = vload4(0, src0_sas + row + (i / 32) * m);
-        const half4 d = vload4(0, src0_d + row + (i / 256) * m);
-        const float4 scale_f = convert_float4(d) * 0.5f * (0.5f + convert_float4(sas >> (uint4)(28)));
-        const half4 scale = convert_half4(scale_f);
-
-        for (int l = 0; l < 16; l += 4) {
-            const int ki = i + l;
-            const uint4 gi = convert_uint4(vload4(0, src0_qs + row + (ki / 4) * m));
-            const uint4 code = (sas >> (uint4)(7*((ki / 8) & 3))) & (uint4)(127);
-            const uint4 sg = code | ((popcount(code) & (uint4)(1)) << (uint4)(7));
-            const uint4 grid = (uint4)(iq3_xxs_grid[gi.s0], iq3_xxs_grid[gi.s1],
-                                      iq3_xxs_grid[gi.s2], iq3_xxs_grid[gi.s3]);
-            #pragma unroll
-            for (int j = 0; j < 4; ++j) {
-                half8 b;
-                b.s0123 = read_imageh(src1, gy * 2     + (ki + j) * n_4);
-                b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
-                const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255));
-                const int4 signs = convert_int4((sg >> (uint4)((ki & 7) + j)) & (uint4)(1));
-                const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale;
-                c0 += b * w.s0;
-                c1 += b * w.s1;
-                c2 += b * w.s2;
-                c3 += b * w.s3;
-            }
-        }
-    }
-
-    int idx = gy * 8 * m + row;
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_q8_1_dp4a.cl

@@ -1,137 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#pragma OPENCL EXTENSION cl_khr_subgroups : enable
-#ifdef cl_khr_integer_dot_product
-#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
-#endif
-
-#define QK_K 256
-
-#ifndef TILESIZE_N
-#define TILESIZE_N 32
-#endif
-
-inline uint iq3_xxs_pack(uint gv, uint sg, uint base) {
-    int v0 = (int)((gv >>  0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; }
-    int v1 = (int)((gv >>  8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; }
-    int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; }
-    int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; }
-    return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) <<  8)
-         | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24);
-}
-
-// Pass staged operands by value; local dp4a operands miscompile on X2.
-inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
-    int r = 0;
-    r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
-    return r;
-}
-
-__attribute__((qcom_wave_pair_mode(1)))
-kernel void kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a(
-        __global const uchar  * src0_qs,
-        __global const uint   * src0_sas,
-        __global const half   * src0_d,
-        __global const uint   * src1_qa,
-        __global const half   * src1_da,
-        __global       float  * dst,
-        ulong  offsetd,
-        int    m,
-        int    n_no_padding,
-        int    k
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-
-    const uint lid = get_local_id(0);
-    const uint block_id_m = get_global_id(1);
-    const uint block_id_n = get_global_id(2);
-
-    const uint row      = block_id_m * 64 + lid;
-    const uint col_base = block_id_n * TILESIZE_N;
-    const bool row_valid = row < (uint)m;
-    const uint rrow     = row_valid ? row : 0;
-
-    const uint k_u = (uint)k >> 2;
-    const uint k_b = (uint)k >> 5;
-
-    __local uint4 sh_qa4[TILESIZE_N][2];
-    __local half sh_d[TILESIZE_N];
-
-#define NGROUPS (TILESIZE_N / 4)
-    float4 acc[NGROUPS];
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
-
-    for (uint step = 0; step < (uint)k; step += 32) {
-        const uint sub = step >> 5;
-
-        const uint ib = sub >> 3;
-        const uint aux = src0_sas[rrow + sub*(uint)m];
-        const float d_w = (float)src0_d[rrow + ib*(uint)m] * (0.5f + (float)(aux >> 28)) * 0.5f;
-        const uint sg0 = iq3_xxs_signs( aux        & 127u);
-        const uint sg1 = iq3_xxs_signs((aux >>  7) & 127u);
-        const uint sg2 = iq3_xxs_signs((aux >> 14) & 127u);
-        const uint sg3 = iq3_xxs_signs((aux >> 21) & 127u);
-        const uint qsb = rrow + sub*8u*(uint)m;
-
-        uint8 qw;
-        qw.s0 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 0u*(uint)m]], sg0, 0u);
-        qw.s1 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 1u*(uint)m]], sg0, 4u);
-        qw.s2 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 2u*(uint)m]], sg1, 0u);
-        qw.s3 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 3u*(uint)m]], sg1, 4u);
-        qw.s4 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 4u*(uint)m]], sg2, 0u);
-        qw.s5 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 5u*(uint)m]], sg2, 4u);
-        qw.s6 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 6u*(uint)m]], sg3, 0u);
-        qw.s7 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 7u*(uint)m]], sg3, 4u);
-
-        for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
-            const uint t = idx >> 1;
-            const uint v = idx & 1;
-            const uint c = col_base + t;
-            sh_qa4[t][v] = (c < (uint)n_no_padding)
-                         ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
-                         : (uint4)(0u);
-        }
-        if (lid < TILESIZE_N) {
-            const uint c = col_base + lid;
-            sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
-        #pragma unroll
-        for (int g = 0; g < NGROUPS; ++g) {
-            const int b = g * 4;
-            float4 rf;
-#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1]))
-            rf.s0 = (float)TDOT4(b+0);  rf.s1 = (float)TDOT4(b+1);
-            rf.s2 = (float)TDOT4(b+2);  rf.s3 = (float)TDOT4(b+3);
-            acc[g] += d_w * LD4(sh_d, b) * rf;
-        }
-#undef TDOT4
-#undef LD4
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    if (!row_valid) {
-        return;
-    }
-
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) {
-        const uint b = (uint)(g * 4);
-        const float4 a = acc[g];
-        const uint c0 = col_base + b;
-        if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
-        if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
-        if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
-        if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
-    }
-#undef NGROUPS
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_f32.cl

@@ -1,88 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_qcom_reqd_sub_group_size
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-__attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-kernel void kernel_gemm_noshuffle_iq4_xs_f32(
-    global const ushort * src0_qs,
-    global const uint * src0_sl,
-    global const ushort * src0_sh,
-    global const half  * src0_d,
-    read_only image1d_buffer_t src1,
-    global float * dst,
-    ulong offsetd,
-    int m,
-    int n,
-    int k,
-    int n_no_padding
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-    const int gy = get_global_id(0);
-    const int row = get_global_id(1) * 4;
-    const int n_4 = n / 4;
-    if (row >= m) {
-        return;
-    }
-
-    half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
-    for (int i = 0; i < k; i += 16) {
-        const uint4 sl = vload4(0, src0_sl + row + (i / 256) * m);
-        const uint4 sh = convert_uint4(vload4(0, src0_sh + row + (i / 256) * m));
-        const half4 d = vload4(0, src0_d + row + (i / 256) * m);
-        const int sb = (i / 32) & 7;
-        const int4 ls = convert_int4(((sl >> (uint4)(4*sb)) & (uint4)(15)) | (((sh >> (uint4)(2*sb)) & (uint4)(3)) << (uint4)(4))) - (int4)(32);
-        const half4 scale = convert_half4(convert_float4(d) * convert_float4(ls));
-
-        for (int l = 0; l < 16; l += 4) {
-            const int ki = i + l;
-            const ushort4 q = vload4(0, src0_qs + row + (ki / 4) * m);
-            #pragma unroll
-            for (int j = 0; j < 4; ++j) {
-                half8 b;
-                b.s0123 = read_imageh(src1, gy * 2     + (ki + j) * n_4);
-                b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
-                const ushort4 indices = (q >> (ushort4)(4*j)) & (ushort4)(15);
-                const half4 w = (half4)(iq4_xs_values[indices.s0], iq4_xs_values[indices.s1],
-                                       iq4_xs_values[indices.s2], iq4_xs_values[indices.s3]) * scale;
-                c0 += b * w.s0;
-                c1 += b * w.s1;
-                c2 += b * w.s2;
-                c3 += b * w.s3;
-            }
-        }
-    }
-
-    int idx = gy * 8 * m + row;
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_q8_1_dp4a.cl

@@ -1,135 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#pragma OPENCL EXTENSION cl_khr_subgroups : enable
-#ifdef cl_khr_integer_dot_product
-#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
-#endif
-
-#define QK_K 256
-
-#ifndef TILESIZE_N
-#define TILESIZE_N 32
-#endif
-
-inline uint iq4_xs_pack(ushort q) {
-    return iq4_xs_code(q & 15)
-         | (iq4_xs_code((q >> 4) & 15) << 8)
-         | (iq4_xs_code((q >> 8) & 15) << 16)
-         | (iq4_xs_code(q >> 12) << 24);
-}
-
-// Pass staged operands by value; local dp4a operands miscompile on X2.
-inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
-    int r = 0;
-    r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
-    return r;
-}
-
-__attribute__((qcom_wave_pair_mode(1)))
-kernel void kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a(
-        __global const ushort * src0_qs,
-        __global const uint   * src0_sl,
-        __global const ushort * src0_sh,
-        __global const half   * src0_d,
-        __global const uint   * src1_qa,
-        __global const half   * src1_da,
-        __global       float  * dst,
-        ulong  offsetd,
-        int    m,
-        int    n_no_padding,
-        int    k
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-
-    const uint lid = get_local_id(0);
-    const uint block_id_m = get_global_id(1);
-    const uint block_id_n = get_global_id(2);
-
-    const uint row      = block_id_m * 64 + lid;
-    const uint col_base = block_id_n * TILESIZE_N;
-    const bool row_valid = row < (uint)m;
-    const uint rrow     = row_valid ? row : 0;
-
-    const uint k_u = (uint)k >> 2;
-    const uint k_b = (uint)k >> 5;
-
-    __local uint4 sh_qa4[TILESIZE_N][2];
-    __local half sh_d[TILESIZE_N];
-
-#define NGROUPS (TILESIZE_N / 4)
-    float4 acc[NGROUPS];
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
-
-    for (uint step = 0; step < (uint)k; step += 32) {
-        const uint sub = step >> 5;
-
-        const uint ib = sub >> 3;
-        const uint sb = sub & 7u;
-        const uint sl = src0_sl[rrow + ib*(uint)m];
-        const uint sh = src0_sh[rrow + ib*(uint)m];
-        const int ls = (int)(((sl >> (4*sb)) & 15u) | (((sh >> (2*sb)) & 3u) << 4)) - 32;
-        const float d_w = (float)src0_d[rrow + ib*(uint)m] * (float)ls;
-        const uint qsb = rrow + sub*8u*(uint)m;
-
-        uint8 qw;
-        qw.s0 = iq4_xs_pack(src0_qs[qsb + 0u*(uint)m]);
-        qw.s1 = iq4_xs_pack(src0_qs[qsb + 1u*(uint)m]);
-        qw.s2 = iq4_xs_pack(src0_qs[qsb + 2u*(uint)m]);
-        qw.s3 = iq4_xs_pack(src0_qs[qsb + 3u*(uint)m]);
-        qw.s4 = iq4_xs_pack(src0_qs[qsb + 4u*(uint)m]);
-        qw.s5 = iq4_xs_pack(src0_qs[qsb + 5u*(uint)m]);
-        qw.s6 = iq4_xs_pack(src0_qs[qsb + 6u*(uint)m]);
-        qw.s7 = iq4_xs_pack(src0_qs[qsb + 7u*(uint)m]);
-
-        for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
-            const uint t = idx >> 1;
-            const uint v = idx & 1;
-            const uint c = col_base + t;
-            sh_qa4[t][v] = (c < (uint)n_no_padding)
-                         ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
-                         : (uint4)(0u);
-        }
-        if (lid < TILESIZE_N) {
-            const uint c = col_base + lid;
-            sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
-        #pragma unroll
-        for (int g = 0; g < NGROUPS; ++g) {
-            const int b = g * 4;
-            float4 rf;
-#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1]))
-            rf.s0 = (float)TDOT4(b+0);  rf.s1 = (float)TDOT4(b+1);
-            rf.s2 = (float)TDOT4(b+2);  rf.s3 = (float)TDOT4(b+3);
-            acc[g] += d_w * LD4(sh_d, b) * rf;
-        }
-#undef TDOT4
-#undef LD4
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    if (!row_valid) {
-        return;
-    }
-
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) {
-        const uint b = (uint)(g * 4);
-        const float4 a = acc[g];
-        const uint c0 = col_base + b;
-        if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
-        if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
-        if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
-        if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
-    }
-#undef NGROUPS
-}

ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq1_m_f32.cl

@@ -1,134 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#define QK_K 256
-#define IQ1_M_DELTA 0.125f
-
-#ifndef IQ1_M_MV_NSG
-#define IQ1_M_MV_NSG 4
-#endif
-
-inline float4 iq1_m_vals(uint g) {
-    return (float4)((float)( g        & 0xFu), (float)((g >>  8) & 0xFu),
-                    (float)((g >> 16) & 0xFu), (float)((g >> 24) & 0xFu));
-}
-
-inline float iq1_m_super(uint w0, uint w1, uint w2, uint w3) {
-    const ushort u16 = (ushort)(((w0 >> 12)) | ((w1 >> 8) & 0x00f0u)
-                              | ((w2 >> 4) & 0x0f00u) | (w3 & 0xf000u));
-    return (float)as_half(u16);
-}
-
-kernel void kernel_gemv_noshuffle_iq1_m_f32(
-        __read_only image1d_buffer_t grid_img,
-        global const uchar  * src0_qs,
-        global const uchar  * src0_qh,
-        global const ushort * src0_sc,
-        global const float  * src1,
-        ulong offset1,
-        global float * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        int ne10,
-        int ne0
-) {
-    src1 = (global const float *)((global const char *)src1 + offset1);
-    dst  = (global float       *)((global char       *)dst  + offsetd);
-
-    const uint m   = (uint)ne01;
-    const uint K   = (uint)ne00;
-    const uint nsb = K / QK_K;
-
-    const uint lid = get_local_id(0);
-    const uint sgi = get_local_id(1);
-    const uint col = get_group_id(1);
-
-    global const float * y = src1 + (ulong)col * (uint)ne10;
-
-#define IQ1_M_GRID(i) (read_imageui(grid_img, (int)(i)).x)
-
-    const uint mh  = m >> 1;
-    const uint j   = get_group_id(0) * 64u + lid;
-    const uint row = j << 1;
-
-    float sumf = 0.f, sumf1 = 0.f;
-
-    if (j < mh) {
-        global const ushort * qsu = (global const ushort *)src0_qs;
-        global const ushort * qhu = (global const ushort *)src0_qh;
-        global const uint   * scu = (global const uint   *)src0_sc;
-
-        for (uint ib = sgi; ib < nsb; ib += IQ1_M_MV_NSG) {
-            const uint sb = j + (ib * 4u) * mh;
-            const uint w0 = scu[sb + 0u * mh];
-            const uint w1 = scu[sb + 1u * mh];
-            const uint w2 = scu[sb + 2u * mh];
-            const uint w3 = scu[sb + 3u * mh];
-            const float d0 = iq1_m_super( w0        & 0xFFFFu,  w1        & 0xFFFFu,
-                                         w2        & 0xFFFFu,  w3        & 0xFFFFu);
-            const float d1 = iq1_m_super((w0 >> 16) & 0xFFFFu, (w1 >> 16) & 0xFFFFu,
-                                        (w2 >> 16) & 0xFFFFu, (w3 >> 16) & 0xFFFFu);
-
-            float acc0 = 0.f, acc1 = 0.f;
-
-            for (uint sub_l = 0; sub_l < 8u; ++sub_l) {
-                const uint sub = ib * 8u + sub_l;
-                const uint scw = scu[j + (ib * 4u + (sub_l >> 1)) * mh];
-                const uint grp = sub * 8u;
-                const uint shb = 3u * (2u * (sub_l & 1u));
-
-                for (uint l = 0; l < 4u; ++l) {
-                    const uint sh  = shb + 3u * (l >> 1);
-                    const float s0 = (float)(2u * (((scw       ) >> sh) & 7u) + 1u);
-                    const float s1 = (float)(2u * (((scw >> 16) >> sh) & 7u) + 1u);
-
-                    const uint qhv = (uint)qhu[j + (sub * 2u + (l >> 1)) * mh];
-                    const uint h0  =  qhv       & 0xFFu;
-                    const uint h1  = (qhv >> 8) & 0xFFu;
-                    const uint qsv = (uint)qsu[j + (sub * 4u + l) * mh];
-
-                    const uint msk = 0x08u << (4u * (l & 1u));
-                    const float t0 = (h0 & msk) ? (-1.f - IQ1_M_DELTA) : (-1.f + IQ1_M_DELTA);
-                    const float t1 = (h1 & msk) ? (-1.f - IQ1_M_DELTA) : (-1.f + IQ1_M_DELTA);
-
-                    const uint g0 = IQ1_M_GRID(( qsv       & 0xFFu)
-                                              | ((((h0 >> (4u*(l & 1u))) & 7u) << 8)));
-                    const uint g1 = IQ1_M_GRID(((qsv >> 8) & 0xFFu)
-                                              | ((((h1 >> (4u*(l & 1u))) & 7u) << 8)));
-
-                    const float4 y0 = vload4(grp + 2u*l + 0u, y);
-                    const float4 y1 = vload4(grp + 2u*l + 1u, y);
-
-                    const float4 ys = y0 + y1;
-                    const float  as = ys.s0 + ys.s1 + ys.s2 + ys.s3;
-
-                    const float a0 = dot(y0, iq1_m_vals(g0)) + dot(y1, iq1_m_vals(g0 >> 4));
-                    const float a1 = dot(y0, iq1_m_vals(g1)) + dot(y1, iq1_m_vals(g1 >> 4));
-                    acc0 += s0 * (a0 + t0 * as);
-                    acc1 += s1 * (a1 + t1 * as);
-                }
-            }
-            sumf  += d0 * acc0;
-            sumf1 += d1 * acc1;
-        }
-    }
-
-#if IQ1_M_MV_NSG > 1
-    __local float2 part[IQ1_M_MV_NSG][64];
-    part[sgi][lid] = (float2)(sumf, sumf1);
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < IQ1_M_MV_NSG; ++s) {
-        const float2 p = part[s][lid];
-        sumf  += p.s0;
-        sumf1 += p.s1;
-    }
-#endif
-
-    if (j < mh) {
-        vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
-    }
-#undef IQ1_M_GRID
-}

ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq1_s_f32.cl

@@ -1,117 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#define QK_K 256
-
-#ifndef IQ1_S_MV_NSG
-#define IQ1_S_MV_NSG 4
-#endif
-
-inline float4 iq1_s_vals(uint g) {
-    return (float4)((float)( g        & 0xFu), (float)((g >>  8) & 0xFu),
-                    (float)((g >> 16) & 0xFu), (float)((g >> 24) & 0xFu));
-}
-
-kernel void kernel_gemv_noshuffle_iq1_s_f32(
-        __read_only image1d_buffer_t grid_img,
-        global const uchar  * src0_qs,
-        global const ushort * src0_qh,
-        global const half   * src0_d,
-        global const float  * src1,
-        ulong offset1,
-        global float * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        int ne10,
-        int ne0
-) {
-    src1 = (global const float *)((global const char *)src1 + offset1);
-    dst  = (global float       *)((global char       *)dst  + offsetd);
-
-    const uint m   = (uint)ne01;
-    const uint K   = (uint)ne00;
-    const uint nsb = K / QK_K;
-
-    const uint lid = get_local_id(0);
-    const uint sgi = get_local_id(1);
-    const uint col = get_group_id(1);
-
-    global const float * y = src1 + (ulong)col * (uint)ne10;
-
-#define IQ1_S_GRID(i) (read_imageui(grid_img, (int)(i)).x)
-
-    const uint mh  = m >> 1;
-    const uint j   = get_group_id(0) * 64u + lid;
-    const uint row = j << 1;
-
-    float sumf = 0.f, sumf1 = 0.f;
-
-    if (j < mh) {
-        global const ushort * qsu = (global const ushort *)src0_qs;
-
-        for (uint ib = sgi; ib < nsb; ib += IQ1_S_MV_NSG) {
-            const half2 dh = vload2(j + ib * mh, src0_d);
-
-            float acc0 = 0.f, acc1 = 0.f;
-            for (uint sb = 0; sb < 8u; ++sb) {
-                const uint  sub = ib * 8u + sb;
-                const ushort2 qhv = vload2(j + sub * mh, src0_qh);
-                const uint  h0  = (uint)qhv.s0, h1 = (uint)qhv.s1;
-
-                const float s0  = (float)(2u * ((h0 >> 12) & 7u) + 1u);
-                const float s1  = (float)(2u * ((h1 >> 12) & 7u) + 1u);
-                const float t0  = ((h0 & 0x8000u) ? -0.125f : 0.125f) - 1.0f;
-                const float t1  = ((h1 & 0x8000u) ? -0.125f : 0.125f) - 1.0f;
-
-                const uint gb  = j + (sub * 4u) * mh;
-                const uint grp = sub * 8u;
-
-                float a0 = 0.f, a1 = 0.f;
-                float4 as = (float4)(0.f);
-                for (uint l = 0; l < 4u; ++l) {
-                    const uint qsv = (uint)qsu[gb + l * mh];
-                    const uint g0  = IQ1_S_GRID(( qsv       & 0xFFu) | ((((h0 >> (3u*l)) & 7u) << 8)));
-                    const uint g1  = IQ1_S_GRID(((qsv >> 8) & 0xFFu) | ((((h1 >> (3u*l)) & 7u) << 8)));
-
-                    const float4 y0 = vload4(grp + 2u*l + 0u, y);
-                    const float4 y1 = vload4(grp + 2u*l + 1u, y);
-                    as += y0 + y1;
-                    a0 += dot(y0, iq1_s_vals(g0)) + dot(y1, iq1_s_vals(g0 >> 4));
-                    a1 += dot(y0, iq1_s_vals(g1)) + dot(y1, iq1_s_vals(g1 >> 4));
-                }
-
-                const float asum = as.s0 + as.s1 + as.s2 + as.s3;
-                acc0 += s0 * (a0 + t0 * asum);
-                acc1 += s1 * (a1 + t1 * asum);
-            }
-            sumf  += (float)dh.s0 * acc0;
-            sumf1 += (float)dh.s1 * acc1;
-        }
-    }
-
-#if IQ1_S_MV_NSG > 1
-    __local float2 part[IQ1_S_MV_NSG][64];
-    part[sgi][lid] = (float2)(sumf, sumf1);
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < IQ1_S_MV_NSG; ++s) {
-        const float2 p = part[s][lid];
-        sumf  += p.s0;
-        sumf1 += p.s1;
-    }
-#endif
-
-    if (j < mh) {
-        vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
-    }
-#undef IQ1_S_GRID
-}
-
-kernel void kernel_iq1_s_grid_export(global uint * out) {
-    const uint i = get_global_id(0);
-    if (i < 2048u) {
-        out[i] = iq1_s_grid[i];
-    }
-}

ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_s_f32.cl

@@ -1,128 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#define QK_K 256
-
-#ifndef IQ3_S_MV_NSG
-#define IQ3_S_MV_NSG 4
-#endif
-
-inline float4 iq3_s_vals(uint gv, uint sg, uint base) {
-
-    const uint  s   = sg >> base;
-    const uint4 sgn = (uint4)(s << 31, s << 30, s << 29, s << 28) & 0x80000000u;
-    return as_float4(as_uint4(convert_float4(as_uchar4(gv))) ^ sgn);
-}
-
-kernel void kernel_gemv_noshuffle_iq3_s_f32(
-        __read_only image1d_buffer_t grid_img,
-        global const uchar * src0_qs,
-        global const uchar * src0_sg,
-        global const uchar * src0_qh,
-        global const uchar * src0_sc,
-        global const half  * src0_d,
-        global const float * src1,
-        ulong offset1,
-        global float * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        int ne10,
-        int ne0
-) {
-    src1 = (global const float *)((global const char *)src1 + offset1);
-    dst  = (global float       *)((global char       *)dst  + offsetd);
-
-    const uint m   = (uint)ne01;
-    const uint K   = (uint)ne00;
-    const uint nsb = K / QK_K;
-
-    const uint lid = get_local_id(0);
-    const uint sgi = get_local_id(1);
-    const uint col = get_group_id(1);
-
-    global const float * y = src1 + (ulong)col * (uint)ne10;
-
-#define IQ3_S_GRID(i) (read_imageui(grid_img, (int)(i)).x)
-
-    const uint mh  = m >> 1;
-    const uint j   = get_group_id(0) * 64u + lid;
-    const uint row = j << 1;
-
-    float sumf  = 0.f;
-    float sumf1 = 0.f;
-
-    if (j < mh) {
-        global const ushort * qsu = (global const ushort *)src0_qs;
-        global const ushort * qhu = (global const ushort *)src0_qh;
-        global const ushort * sgu = (global const ushort *)src0_sg;
-        global const ushort * scu = (global const ushort *)src0_sc;
-
-        for (uint ib = sgi; ib < nsb; ib += IQ3_S_MV_NSG) {
-            const half2 dh = vload2(j + ib * mh, src0_d);
-            const float d0 = (float)dh.s0;
-            const float d1 = (float)dh.s1;
-
-            const uint scbase = j + ib * 4u * mh;
-            ushort sc4[4];
-            sc4[0] = scu[scbase + 0u * mh];
-            sc4[1] = scu[scbase + 1u * mh];
-            sc4[2] = scu[scbase + 2u * mh];
-            sc4[3] = scu[scbase + 3u * mh];
-
-            float acc0 = 0.f, acc1 = 0.f;
-            for (uint sb = 0; sb < 8u; ++sb) {
-                const uint scv  = (uint)sc4[sb >> 1];
-                const uint nib0 = (sb & 1u) ? ((scv >>  4) & 0xFu) : ( scv        & 0xFu);
-                const uint nib1 = (sb & 1u) ? ((scv >> 12) & 0xFu) : ((scv >>  8) & 0xFu);
-
-                const uint qhv = (uint)qhu[j + (ib * 8u + sb) * mh];
-                const uint qh0 = qhv & 0xFFu;
-                const uint qh1 = qhv >> 8;
-
-                const uint grp = ib * 64u + sb * 8u;
-                const uint qsb = j + grp * mh;
-                const uint sgb = j + (ib * 32u + sb * 4u) * mh;
-
-                float a0 = 0.f, a1 = 0.f;
-                for (uint u = 0; u < 8u; ++u) {
-                    const uint qsv = (uint)qsu[qsb + u * mh];
-                    const uint sgv = (uint)sgu[sgb + (u >> 1) * mh];
-                    const uint g0  = ( qsv       & 0xFFu) | (((qh0 >> u) & 1u) << 8);
-                    const uint g1  = ((qsv >> 8) & 0xFFu) | (((qh1 >> u) & 1u) << 8);
-                    const uint base = (u & 1u) * 4u;
-                    const float4 yv = vload4(grp + u, y);
-                    const uint gw0 = IQ3_S_GRID(g0);
-                    const uint gw1 = IQ3_S_GRID(g1);
-                    a0 += dot(yv, iq3_s_vals(gw0,  sgv       & 0xFFu, base));
-                    a1 += dot(yv, iq3_s_vals(gw1, (sgv >> 8) & 0xFFu, base));
-                }
-                acc0 += (float)(1u + 2u * nib0) * a0;
-                acc1 += (float)(1u + 2u * nib1) * a1;
-            }
-            sumf  += d0 * acc0;
-            sumf1 += d1 * acc1;
-        }
-    }
-
-#if IQ3_S_MV_NSG > 1
-    __local float2 part[IQ3_S_MV_NSG][64];
-    part[sgi][lid] = (float2)(sumf, sumf1);
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < IQ3_S_MV_NSG; ++s) {
-        const float2 p = part[s][lid];
-        sumf  += p.s0;
-        sumf1 += p.s1;
-    }
-#endif
-
-    if (j < mh) {
-        vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
-    }
-}
-
-kernel void kernel_iq3_s_grid_export(global uint * out) {
-    out[get_global_id(0)] = iq3_s_grid[get_global_id(0)];
-}

ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_xxs_f32.cl

@@ -1,120 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#define QK_K 256
-
-#ifndef IQ3_XXS_MV_NSG
-#define IQ3_XXS_MV_NSG 4
-#endif
-
-inline float4 iq3_xxs_vals(uint gv, uint sgv, uint base) {
-    const uint  s   = sgv >> base;
-    const uint4 sgn = (uint4)(s << 31, s << 30, s << 29, s << 28) & 0x80000000u;
-    return as_float4(as_uint4(convert_float4(as_uchar4(gv))) ^ sgn);
-}
-
-kernel void kernel_gemv_noshuffle_iq3_xxs_f32(
-        __read_only image1d_buffer_t grid_img,
-        global const uchar * src0_qs,
-        global const uint  * src0_sas,
-        global const half  * src0_d,
-        global const float * src1,
-        ulong offset1,
-        global float * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        int ne10,
-        int ne0
-) {
-    src1 = (global const float *)((global const char *)src1 + offset1);
-    dst  = (global float       *)((global char       *)dst  + offsetd);
-
-    const uint m   = (uint)ne01;
-    const uint K   = (uint)ne00;
-    const uint nsb = K / QK_K;
-
-    const uint lid = get_local_id(0);
-    const uint sgi = get_local_id(1);
-    const uint col = get_group_id(1);
-
-    global const float * y = src1 + (ulong)col * (uint)ne10;
-
-#define IQ3_XXS_GRID(i) (read_imageui(grid_img, (int)(i)).x)
-
-    const uint mh  = m >> 1;
-    const uint j   = get_group_id(0) * 64u + lid;
-    const uint row = j << 1;
-
-    float sumf  = 0.f;
-    float sumf1 = 0.f;
-
-    if (j < mh) {
-        global const ushort * qsu = (global const ushort *)src0_qs;
-
-        for (uint ib = sgi; ib < nsb; ib += IQ3_XXS_MV_NSG) {
-            const half2 dh = vload2(j + ib * mh, src0_d);
-            const float d0 = (float)dh.s0;
-            const float d1 = (float)dh.s1;
-
-            float acc0 = 0.f, acc1 = 0.f;
-            for (uint sb = 0; sb < 8u; ++sb) {
-                const uint  sub = ib * 8u + sb;
-                const uint2 aux = vload2(j + sub * mh, src0_sas);
-
-                const uint grp = sub * 8u;
-                const uint qsb = j + grp * mh;
-
-                float a0 = 0.f, a1 = 0.f;
-                for (uint u = 0; u < 4u; ++u) {
-                    const uint qsv0 = (uint)qsu[qsb + (2u*u)*mh];
-                    const uint qsv1 = (uint)qsu[qsb + (2u*u + 1u)*mh];
-                    const uint sg0 = iq3_xxs_signs((aux.s0 >> (7u * u)) & 127u);
-                    const uint sg1 = iq3_xxs_signs((aux.s1 >> (7u * u)) & 127u);
-
-                    const uint g00 =  qsv0       & 0xFFu;
-                    const uint g01 =  qsv1       & 0xFFu;
-                    const uint g10 = (qsv0 >> 8) & 0xFFu;
-                    const uint g11 = (qsv1 >> 8) & 0xFFu;
-
-                    const float4 yl = vload4(sub * 8u + u * 2u + 0u, y);
-                    const float4 yh = vload4(sub * 8u + u * 2u + 1u, y);
-
-                    a0 += dot(yl, iq3_xxs_vals(IQ3_XXS_GRID(g00), sg0, 0u))
-                        + dot(yh, iq3_xxs_vals(IQ3_XXS_GRID(g01), sg0, 4u));
-                    a1 += dot(yl, iq3_xxs_vals(IQ3_XXS_GRID(g10), sg1, 0u))
-                        + dot(yh, iq3_xxs_vals(IQ3_XXS_GRID(g11), sg1, 4u));
-                }
-                acc0 += (0.5f + (float)(aux.s0 >> 28)) * a0;
-                acc1 += (0.5f + (float)(aux.s1 >> 28)) * a1;
-            }
-            sumf  += d0 * 0.5f * acc0;
-            sumf1 += d1 * 0.5f * acc1;
-        }
-    }
-
-#if IQ3_XXS_MV_NSG > 1
-    __local float2 part[IQ3_XXS_MV_NSG][64];
-    part[sgi][lid] = (float2)(sumf, sumf1);
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < IQ3_XXS_MV_NSG; ++s) {
-        const float2 p = part[s][lid];
-        sumf  += p.s0;
-        sumf1 += p.s1;
-    }
-#endif
-
-    if (j < mh) {
-        vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
-    }
-#undef IQ3_XXS_GRID
-}
-
-kernel void kernel_iq3_xxs_grid_export(global uint * out) {
-    const uint i = get_global_id(0);
-    if (i < 256u) {
-        out[i] = iq3_xxs_grid[i];
-    }
-}

ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_xs_f32.cl

@@ -1,85 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#define QK_K 256
-
-#ifndef IQ4_XS_MV_NSG
-#define IQ4_XS_MV_NSG 8
-#endif
-
-inline float4 iq4_xs_vals(ushort q) {
-    return (float4)((char)iq4_xs_code(q & 15), (char)iq4_xs_code((q >> 4) & 15),
-                    (char)iq4_xs_code((q >> 8) & 15), (char)iq4_xs_code(q >> 12));
-}
-
-kernel void kernel_gemv_noshuffle_iq4_xs_f32(
-        global const ushort * src0_qs,
-        global const uint * src0_sl,
-        global const ushort * src0_sh,
-        global const half * src0_d,
-        global const float * src1,
-        ulong offset1,
-        global float * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        int ne10,
-        int ne0
-) {
-    src1 = (global const float *)((global const char *)src1 + offset1);
-    dst = (global float *)((global char *)dst + offsetd);
-
-    const uint m = (uint)ne01;
-    const uint mh = m >> 1;
-    const uint lid = get_local_id(0);
-    const uint sgi = get_local_id(1);
-    const uint col = get_group_id(1);
-    const uint j = get_group_id(0)*64u + lid;
-    const uint row = j << 1;
-    global const float * y = src1 + (ulong)col * (uint)ne10;
-
-    float sum0 = 0.0f, sum1 = 0.0f;
-    if (j < mh) {
-        global const uint * qs = (global const uint *)src0_qs;
-        global const uint * sh = (global const uint *)src0_sh;
-        for (uint ib = sgi; ib < (uint)ne00/QK_K; ib += IQ4_XS_MV_NSG) {
-            const uint base = j + ib*mh;
-            const uint2 sl = vload2(base, src0_sl);
-            const uint shp = sh[base];
-            const half2 d = vload2(base, src0_d);
-            float acc0 = 0.0f, acc1 = 0.0f;
-            for (uint sb = 0; sb < 8; ++sb) {
-                const int ls0 = (int)(((sl.s0 >> (4*sb)) & 15u) | (((shp >> (2*sb)) & 3u) << 4)) - 32;
-                const int ls1 = (int)(((sl.s1 >> (4*sb)) & 15u) | (((shp >> (16 + 2*sb)) & 3u) << 4)) - 32;
-                float a0 = 0.0f, a1 = 0.0f;
-                for (uint u = 0; u < 8; ++u) {
-                    const uint group = ib*64 + sb*8 + u;
-                    const uint q = qs[j + group*mh];
-                    const float4 yv = vload4(group, y);
-                    a0 += dot(yv, iq4_xs_vals((ushort)q));
-                    a1 += dot(yv, iq4_xs_vals((ushort)(q >> 16)));
-                }
-                acc0 += (float)ls0*a0;
-                acc1 += (float)ls1*a1;
-            }
-            sum0 += (float)d.s0*acc0;
-            sum1 += (float)d.s1*acc1;
-        }
-    }
-
-#if IQ4_XS_MV_NSG > 1
-    local float2 part[IQ4_XS_MV_NSG][64];
-    part[sgi][lid] = (float2)(sum0, sum1);
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < IQ4_XS_MV_NSG; ++s) {
-        const float2 v = part[s][lid];
-        sum0 += v.s0;
-        sum1 += v.s1;
-    }
-#endif
-    if (j < mh) {
-        vstore2((float2)(sum0, sum1), 0, dst + (ulong)col*(uint)ne0 + row);
-    }
-}

ggml/src/ggml-opencl/kernels/iq1_s_grid.cl

@@ -1,259 +0,0 @@
-// IQ1 codebook: each byte packs grid[j] + 1 and grid[j + 4] + 1.
-constant uint iq1_s_grid[2048] = {
-    0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000,
-    0x00020002, 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101,
-    0x02000000, 0x02000002, 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200,
-    0x02020202, 0x00000110, 0x00000111, 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212,
-    0x00020111, 0x01000011, 0x01000112, 0x01000211, 0x01010012, 0x01010111, 0x01010212, 0x01020011,
-    0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011, 0x02010110, 0x02010112, 0x02020111,
-    0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020, 0x00020022, 0x00020220,
-    0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020, 0x02000022,
-    0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220,
-    0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101,
-    0x01011202, 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110,
-    0x00001111, 0x00001112, 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111,
-    0x01001212, 0x01011010, 0x01011011, 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010,
-    0x01021012, 0x01021111, 0x01021210, 0x01021212, 0x02001011, 0x02011011, 0x02011111, 0x02011210,
-    0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112, 0x02021211, 0x00011120, 0x00011221,
-    0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220, 0x01021020, 0x01021021,
-    0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000, 0x00002002,
-    0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101,
-    0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101,
-    0x02022000, 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211,
-    0x00022110, 0x00022111, 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110,
-    0x01022211, 0x02012011, 0x02012110, 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022,
-    0x00002220, 0x00002222, 0x00012121, 0x00022020, 0x00022022, 0x00022220, 0x00022222, 0x01002121,
-    0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020, 0x02002022, 0x02002121, 0x02002220,
-    0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222, 0x00110000, 0x00110001,
-    0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000, 0x01110101,
-    0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102,
-    0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012,
-    0x00110111, 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010,
-    0x01110011, 0x01110012, 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111,
-    0x02100110, 0x02110012, 0x02110111, 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122,
-    0x00120121, 0x01100020, 0x01100122, 0x01100221, 0x01110022, 0x01110121, 0x01110220, 0x01110222,
-    0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120, 0x02110122, 0x02120121, 0x00101001,
-    0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201, 0x00121001, 0x00121102,
-    0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100, 0x01111101,
-    0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000,
-    0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101,
-    0x02121201, 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112,
-    0x00111211, 0x00121010, 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110,
-    0x01101111, 0x01101112, 0x01111011, 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211,
-    0x01111212, 0x01121011, 0x01121110, 0x01121111, 0x01121112, 0x01121211, 0x02101010, 0x02101012,
-    0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010, 0x02111011, 0x02111110, 0x02111111,
-    0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111, 0x00101021, 0x00101120,
-    0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021, 0x00121122,
-    0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121,
-    0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221,
-    0x01121222, 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001,
-    0x00112102, 0x00122101, 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101,
-    0x01112200, 0x01112202, 0x01122000, 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101,
-    0x02112001, 0x02112100, 0x02122101, 0x00112010, 0x00112012, 0x00112111, 0x00112212, 0x00122011,
-    0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210, 0x01112011, 0x01112110, 0x01112111,
-    0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212, 0x02102211, 0x02112011,
-    0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221, 0x00112122,
-    0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121,
-    0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222,
-    0x00200000, 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101,
-    0x00220200, 0x00220202, 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000,
-    0x02200002, 0x02200200, 0x02200202, 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200,
-    0x02220202, 0x00200111, 0x00210011, 0x00210110, 0x00210211, 0x00220111, 0x01200012, 0x01200110,
-    0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011, 0x01220110, 0x01220111, 0x01220112,
-    0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021, 0x00200220, 0x00200222,
-    0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121, 0x01210021,
-    0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121,
-    0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201,
-    0x00221101, 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200,
-    0x01211202, 0x01221102, 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101,
-    0x00201211, 0x00211111, 0x00221011, 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011,
-    0x01211110, 0x01211111, 0x01211211, 0x01221012, 0x01221111, 0x01221210, 0x02201211, 0x02211010,
-    0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011, 0x02221110, 0x02221112, 0x02221211,
-    0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021, 0x01201221, 0x01211121,
-    0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222, 0x00202000,
-    0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202,
-    0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202,
-    0x02222000, 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211,
-    0x00222111, 0x01202112, 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112,
-    0x01222211, 0x02202111, 0x02212010, 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020,
-    0x00202022, 0x00202220, 0x00202222, 0x00222020, 0x00222022, 0x00222220, 0x00222222, 0x01202121,
-    0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020, 0x02202022, 0x02202220, 0x02202222,
-    0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101, 0x10010001, 0x10010102,
-    0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001, 0x11020100,
-    0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110,
-    0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011,
-    0x10020112, 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111,
-    0x11010112, 0x11010211, 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110,
-    0x12000112, 0x12010010, 0x12010012, 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121,
-    0x10010021, 0x10010120, 0x10010122, 0x10020121, 0x11000021, 0x11010022, 0x11010121, 0x11010222,
-    0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121, 0x10001001, 0x10011101, 0x10011201,
-    0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100, 0x11011101, 0x11011102,
-    0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102, 0x12001201,
-    0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012,
-    0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010,
-    0x10021111, 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010,
-    0x11011011, 0x11011110, 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110,
-    0x11021111, 0x11021112, 0x11021211, 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011,
-    0x12011110, 0x12011111, 0x12011112, 0x12011211, 0x12011212, 0x12021111, 0x12021210, 0x12021212,
-    0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121, 0x10011220, 0x10011222, 0x10021021,
-    0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220, 0x11011020, 0x11011021,
-    0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220, 0x12001021,
-    0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101,
-    0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101,
-    0x11012200, 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100,
-    0x12012102, 0x12012201, 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010,
-    0x10012110, 0x10012111, 0x10012210, 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111,
-    0x11002212, 0x11012011, 0x11012012, 0x11012110, 0x11012111, 0x11012112, 0x11012211, 0x11022010,
-    0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112, 0x12002211, 0x12012012, 0x12012111,
-    0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211, 0x10012122, 0x11002120,
-    0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221, 0x12012120,
-    0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101,
-    0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001,
-    0x11110100, 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201,
-    0x12110101, 0x12110200, 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210,
-    0x10100211, 0x10100212, 0x10110011, 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211,
-    0x10120010, 0x10120111, 0x10120112, 0x10120210, 0x10120212, 0x11100011, 0x11100110, 0x11100111,
-    0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012, 0x11110110, 0x11110111, 0x11110112,
-    0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111, 0x11120112, 0x11120211,
-    0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, 0x12120010,
-    0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021,
-    0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122,
-    0x11110221, 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221,
-    0x12110222, 0x12120120, 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102,
-    0x10111200, 0x10111201, 0x10121001, 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100,
-    0x11101101, 0x11101102, 0x11101201, 0x11101202, 0x11111000, 0x11111001, 0x11111100, 0x11111101,
-    0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001, 0x11121002, 0x11121100, 0x11121101,
-    0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001, 0x12111100, 0x12111101,
-    0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011, 0x10101012,
-    0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110,
-    0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112,
-    0x10121211, 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210,
-    0x11101211, 0x11111010, 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210,
-    0x11111211, 0x11111212, 0x11121010, 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210,
-    0x11121211, 0x11121212, 0x12101011, 0x12101110, 0x12101111, 0x12101211, 0x12101212, 0x12111010,
-    0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210, 0x12111211, 0x12121011, 0x12121110,
-    0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022, 0x10101120, 0x10101122,
-    0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221, 0x10121020,
-    0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021,
-    0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022,
-    0x11111120, 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120,
-    0x11121121, 0x11121221, 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222,
-    0x12111021, 0x12111121, 0x12111222, 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221,
-    0x10102100, 0x10102101, 0x10102102, 0x10102201, 0x10112000, 0x10112101, 0x10112200, 0x10122001,
-    0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001, 0x11112100, 0x11112101, 0x11112102,
-    0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101, 0x12102002, 0x12102201,
-    0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011, 0x10102012,
-    0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111,
-    0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012,
-    0x11112110, 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110,
-    0x11122111, 0x11122112, 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110,
-    0x12112111, 0x12112112, 0x12112210, 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121,
-    0x10112222, 0x10122020, 0x10122121, 0x10122122, 0x10122221, 0x11102121, 0x11102220, 0x11102221,
-    0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221, 0x11122022, 0x11122121, 0x11122220,
-    0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122, 0x12112220, 0x12112222,
-    0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100, 0x11210000,
-    0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201,
-    0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012,
-    0x10210111, 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011,
-    0x11210111, 0x11210112, 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212,
-    0x12210012, 0x12210111, 0x12220011, 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221,
-    0x11200020, 0x11200021, 0x11200122, 0x11210121, 0x11210122, 0x11210220, 0x11220020, 0x12200121,
-    0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002, 0x10211101, 0x10211102, 0x10211202,
-    0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101, 0x11201200, 0x11201202,
-    0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000, 0x11221002,
-    0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101,
-    0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210,
-    0x10201212, 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112,
-    0x11201211, 0x11211010, 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011,
-    0x11221110, 0x11221111, 0x11221112, 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011,
-    0x12211111, 0x12211112, 0x12211211, 0x12211212, 0x12221012, 0x12221111, 0x12221112, 0x12221210,
-    0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122, 0x10221220, 0x10221221, 0x11201020,
-    0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121, 0x11211122, 0x11211220,
-    0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121, 0x12201222,
-    0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222,
-    0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001,
-    0x11222201, 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010,
-    0x10212111, 0x10222011, 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111,
-    0x11202112, 0x11202210, 0x11212011, 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010,
-    0x11222111, 0x11222212, 0x12202012, 0x12202110, 0x12202212, 0x12212111, 0x12222011, 0x12222110,
-    0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220, 0x11202021, 0x11202120, 0x11202221,
-    0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121, 0x11222221, 0x12202122,
-    0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200, 0x20000202,
-    0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100,
-    0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101,
-    0x22020000, 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112,
-    0x20010211, 0x20020111, 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111,
-    0x21010112, 0x21010210, 0x21010211, 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211,
-    0x22010110, 0x22010112, 0x22010211, 0x22020111, 0x20000020, 0x20000022, 0x20000220, 0x20000222,
-    0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222, 0x21010021, 0x21010120, 0x21010221,
-    0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121, 0x22020020, 0x22020022,
-    0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001, 0x21011101,
-    0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211,
-    0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111,
-    0x21001210, 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111,
-    0x21021112, 0x21021210, 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010,
-    0x22011012, 0x22011111, 0x22011210, 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121,
-    0x21001021, 0x21001120, 0x21001221, 0x21001222, 0x21011020, 0x21011121, 0x21011221, 0x21011222,
-    0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021, 0x22011222, 0x22021120, 0x20002000,
-    0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002, 0x20022200, 0x20022202,
-    0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201, 0x22002000,
-    0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202,
-    0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110,
-    0x21002112, 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110,
-    0x22002111, 0x22012112, 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222,
-    0x20012121, 0x20022020, 0x20022022, 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120,
-    0x21012122, 0x22002020, 0x22002022, 0x22002220, 0x22002222, 0x22012121, 0x22022020, 0x22022022,
-    0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102, 0x20110200, 0x20110201, 0x20120101,
-    0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202, 0x21120201, 0x21120202,
-    0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011, 0x20100110,
-    0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110,
-    0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111,
-    0x21110112, 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111,
-    0x22110210, 0x22120011, 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120,
-    0x20110221, 0x20120121, 0x21100120, 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121,
-    0x21110220, 0x21120122, 0x21120221, 0x22100121, 0x22110120, 0x22110122, 0x22120221, 0x20101001,
-    0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200, 0x20121102, 0x21101000, 0x21101202,
-    0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201, 0x21121000, 0x21121001,
-    0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101, 0x22111200,
-    0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011,
-    0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212,
-    0x21101011, 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012,
-    0x21111110, 0x21111111, 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110,
-    0x21121111, 0x21121112, 0x21121211, 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012,
-    0x22111110, 0x22111111, 0x22111112, 0x22111211, 0x22111212, 0x22121010, 0x22121012, 0x22121111,
-    0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020, 0x20111121, 0x20111221, 0x20121020,
-    0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021, 0x21111022, 0x21111121,
-    0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221, 0x22101222,
-    0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102,
-    0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102,
-    0x21112202, 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101,
-    0x20102110, 0x20102112, 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212,
-    0x20122010, 0x20122011, 0x20122110, 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210,
-    0x21102212, 0x21112011, 0x21112110, 0x21112111, 0x21112112, 0x21112211, 0x21122012, 0x21122111,
-    0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010, 0x22112012, 0x22112111, 0x22112212,
-    0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120, 0x21102122, 0x21102221,
-    0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120, 0x22112121,
-    0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002,
-    0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000,
-    0x22200002, 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202,
-    0x20200111, 0x20200211, 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112,
-    0x21200211, 0x21210011, 0x21210111, 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111,
-    0x22210010, 0x22210012, 0x22210112, 0x22210211, 0x20200022, 0x20200220, 0x20200222, 0x20210020,
-    0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121, 0x21210021, 0x21210122, 0x21210221,
-    0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121, 0x22220020, 0x22220022,
-    0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000, 0x21211100,
-    0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201,
-    0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112,
-    0x20221211, 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211,
-    0x21221111, 0x21221212, 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012,
-    0x22211111, 0x22211210, 0x20201121, 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121,
-    0x21201120, 0x21201122, 0x21201222, 0x21211022, 0x21211121, 0x21211122, 0x21211220, 0x21221020,
-    0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122, 0x22211221, 0x22221021, 0x22221120,
-    0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000, 0x20222002, 0x20222200,
-    0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002, 0x22202200,
-    0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110,
-    0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011,
-    0x21222112, 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222,
-    0x20222020, 0x20222022, 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020,
-    0x22202022, 0x22202220, 0x22202222, 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222
-};

ggml/src/ggml-opencl/kernels/iq3_s_grid.cl

@@ -1,66 +0,0 @@
-constant uint iq3_s_grid[512] = {
-    0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305,
-    0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905,
-    0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09,
-    0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b,
-    0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b,
-    0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d,
-    0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03,
-    0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505,
-    0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03,
-    0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901,
-    0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d,
-    0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303,
-    0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501,
-    0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105,
-    0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505,
-    0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101,
-    0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707,
-    0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b,
-    0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01,
-    0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f,
-    0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305,
-    0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103,
-    0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509,
-    0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503,
-    0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b,
-    0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f,
-    0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f,
-    0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f,
-    0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109,
-    0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f,
-    0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509,
-    0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501,
-    0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303,
-    0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f,
-    0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907,
-    0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703,
-    0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03,
-    0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01,
-    0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01,
-    0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903,
-    0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505,
-    0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b,
-    0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107,
-    0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509,
-    0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303,
-    0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103,
-    0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05,
-    0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b,
-    0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f,
-    0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701,
-    0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909,
-    0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305,
-    0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d,
-    0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b,
-    0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d,
-    0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307,
-    0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09,
-    0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309,
-    0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709,
-    0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f,
-    0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303,
-    0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503,
-    0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b,
-    0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101
-};

ggml/src/ggml-opencl/kernels/iq3_xxs_grid.cl

@@ -1,38 +0,0 @@
-constant uint iq3_xxs_grid[256] = {
-    0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,
-    0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,
-    0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,
-    0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,
-    0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,
-    0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,
-    0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,
-    0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,
-    0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,
-    0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,
-    0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,
-    0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,
-    0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,
-    0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,
-    0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,
-    0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,
-    0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,
-    0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,
-    0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,
-    0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,
-    0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,
-    0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,
-    0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,
-    0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,
-    0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,
-    0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,
-    0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,
-    0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,
-    0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,
-    0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,
-    0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,
-    0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04
-};
-
-inline uint iq3_xxs_signs(uint code) {
-    return code | ((popcount(code) & 1u) << 7);
-}

ggml/src/ggml-opencl/kernels/iq4_xs_grid.cl

@@ -1,10 +0,0 @@
-constant char iq4_xs_values[16] = {
-    -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113,
-};
-
-inline uint iq4_xs_code(uint n) {
-    const uint lo = (n & 4u) ? 0xF6EADDCFu : 0xBFAD9881u;
-    const uint hi = (n & 4u) ? 0x71594535u : 0x26190D01u;
-    const uint t = (n & 8u) ? hi : lo;
-    return (t >> (8*(n & 3u))) & 255u;
-}

ggml/src/ggml-opencl/kernels/mul_mm_iq1_m_f32_l4_lm.cl

@@ -1,191 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#define INTEL_GPU 1
-#endif
-
-#define QK_K 256
-
-struct block_iq1_m {
-    uchar qs[QK_K/8];
-    uchar qh[QK_K/16];
-    uchar scales[QK_K/32];
-};
-
-#define LOAD_VEC_A 4
-#define LOAD_VEC_B 4
-
-#define BM 64
-#define BN 64
-#define BK 32
-#ifdef INTEL_GPU
-#define TM 8
-#define TN 8
-#else
-#define TM 4
-#define TN 8
-#endif
-
-kernel void kernel_mul_mm_iq1_m_f32_l4_lm(
-    global uchar * src0_qs,
-    global uchar * src0_qh,
-    global ushort * src0_sc,
-    ulong offset0,
-    global float4 * src1,
-    ulong offset1,
-    global float  * dst,
-    ulong offsetd,
-
-    int ne00,
-    int ne01,
-    int ne02,
-    int ne11,
-    int ne12,
-
-    int stride_a,
-    int stride_b,
-    int stride_d,
-
-    int batch_stride_a,
-    int batch_stride_b,
-    int batch_stride_d,
-
-    int r2,
-    int r3
-) {
-    src1 = (global float4*)((global char*)src1 + offset1);
-    dst  = (global float *)((global char*)dst  + offsetd);
-
-    local float buf_a[BM * BK];
-    local float buf_b[BN * BK];
-
-    const int batch_idx = get_global_id(2);
-
-    const int i13 = batch_idx / ne12;
-    const int i12 = batch_idx % ne12;
-
-    const int i03 = i13 / r3;
-    const int i02 = i12 / r2;
-
-    const int batch_idx_a = i03 * ne02 + i02;
-
-    const int ir = get_group_id(0);
-    const int ic = get_group_id(1);
-
-    const int tid = get_local_id(0);
-    const int th_r  = tid % (BM / TM);
-    const int th_c  = tid / (BM / TM);
-
-    const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
-    const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
-    const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
-    const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
-
-    const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
-    const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
-
-    // pos_a counts elements, not blocks - one iq1_m super block holds QK_K of them
-    int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
-    int pos_b = (batch_idx   * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
-
-    float sums[TM * TN];
-    float cache_a[TM];
-    float cache_b[TN];
-
-    for (int i = 0; i < TM * TN; i++) {
-        sums[i] = 0.0f;
-    }
-
-    for (int block = 0; block < ne00; block += BK) {
-        for (int l = 0; l < BM; l += loadstride_a) {
-            if (ir*BM + loadc_a + l < ne01) {
-                int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
-                int ib  = idx / QK_K;
-                ib += offset0/sizeof(struct block_iq1_m);
-                int e   = idx % QK_K;
-
-                int ib32 = e >> 5;
-                int rem  = e & 31;
-                int lg    = rem >> 3;
-                int j0   = rem & 7;
-
-                global ushort * sc = src0_sc + 4*ib;
-                ushort u16 = (ushort)((sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0)
-                                    | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000));
-                float dl = (float)as_half(u16) * (float)(2*((sc[ib32/2] >> (6*(ib32%2) + 3*(lg/2))) & 7) + 1);
-                uchar qhb = src0_qh[16*ib + 2*ib32 + lg/2];
-                float dlt = (qhb & (0x08 << (4*(lg%2)))) ? -1.125f : -0.875f;
-                uint gi = (uint)src0_qs[32*ib + 4*ib32 + lg] | ((((uint)qhb >> (4*(lg%2))) & 7) << 8);
-                uint g = iq1_s_grid[gi];
-                int sh = (j0 >> 2) * 4;
-
-                float4 v1;
-                v1.s0 = dl * ((float)((g >> ( 0 + sh)) & 0xF) + dlt);
-                v1.s1 = dl * ((float)((g >> ( 8 + sh)) & 0xF) + dlt);
-                v1.s2 = dl * ((float)((g >> (16 + sh)) & 0xF) + dlt);
-                v1.s3 = dl * ((float)((g >> (24 + sh)) & 0xF) + dlt);
-
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
-            } else {
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
-            }
-        }
-
-        for (int l = 0; l < BN; l += loadstride_b) {
-            if (ic*BN + loadc_b + l < ne11) {
-                int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
-            } else {
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
-            }
-        }
-
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-        pos_a += BK;
-        pos_b += BK / LOAD_VEC_B;
-
-        for (int i = 0; i < BK; i++) {
-            for (int j = 0; j < TM; j++) {
-                cache_a[j] = buf_a[(i) * BM + th_r * TM + j];
-            }
-
-            for (int j = 0; j < TN; j++) {
-                cache_b[j] = buf_b[(i) * BN + th_c * TN + j];
-            }
-
-            for (int cc = 0; cc < TN; cc++) {
-                for (int cr = 0; cr < TM; cr++) {
-                    const int sums_idx = cc*TM + cr;
-                    sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);
-                }
-            }
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    const int dr = ir * BM + th_r * TM;
-    const int dc = ic * BN + th_c * TN;
-
-    const ulong offsets = batch_idx * batch_stride_d;
-
-    for (int cc = 0; cc < TN; cc++) {
-        for (int cr = 0; cr < TM; cr++) {
-            if (dr + cr < ne01 && dc + cc < ne11) {
-                dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mm_iq1_s_f32_l4_lm.cl

@@ -1,188 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#define INTEL_GPU 1
-#endif
-
-#define QK_K 256
-
-struct block_iq1_s {
-    half d;
-    uchar qs[QK_K/8];
-    ushort qh[QK_K/32];
-};
-
-#define LOAD_VEC_A 4
-#define LOAD_VEC_B 4
-
-#define BM 64
-#define BN 64
-#define BK 32
-#ifdef INTEL_GPU
-#define TM 8
-#define TN 8
-#else
-#define TM 4
-#define TN 8
-#endif
-
-kernel void kernel_mul_mm_iq1_s_f32_l4_lm(
-    global uchar * src0_qs,
-    global ushort * src0_qh,
-    global half * src0_d,
-    ulong offset0,
-    global float4 * src1,
-    ulong offset1,
-    global float  * dst,
-    ulong offsetd,
-
-    int ne00,
-    int ne01,
-    int ne02,
-    int ne11,
-    int ne12,
-
-    int stride_a,
-    int stride_b,
-    int stride_d,
-
-    int batch_stride_a,
-    int batch_stride_b,
-    int batch_stride_d,
-
-    int r2,
-    int r3
-) {
-    src1 = (global float4*)((global char*)src1 + offset1);
-    dst  = (global float *)((global char*)dst  + offsetd);
-
-    local float buf_a[BM * BK];
-    local float buf_b[BN * BK];
-
-    const int batch_idx = get_global_id(2);
-
-    const int i13 = batch_idx / ne12;
-    const int i12 = batch_idx % ne12;
-
-    const int i03 = i13 / r3;
-    const int i02 = i12 / r2;
-
-    const int batch_idx_a = i03 * ne02 + i02;
-
-    const int ir = get_group_id(0);
-    const int ic = get_group_id(1);
-
-    const int tid = get_local_id(0);
-    const int th_r  = tid % (BM / TM);
-    const int th_c  = tid / (BM / TM);
-
-    const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
-    const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
-    const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
-    const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
-
-    const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
-    const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
-
-    // pos_a counts elements, not blocks - one iq1_s super block holds QK_K of them
-    int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
-    int pos_b = (batch_idx   * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
-
-    float sums[TM * TN];
-    float cache_a[TM];
-    float cache_b[TN];
-
-    for (int i = 0; i < TM * TN; i++) {
-        sums[i] = 0.0f;
-    }
-
-    for (int block = 0; block < ne00; block += BK) {
-        for (int l = 0; l < BM; l += loadstride_a) {
-            if (ir*BM + loadc_a + l < ne01) {
-                int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
-                int ib  = idx / QK_K;
-                ib += offset0/sizeof(struct block_iq1_s);
-                int e   = idx % QK_K;
-
-                int ib32 = e >> 5;
-                int rem  = e & 31;
-                int lg    = rem >> 3;
-                int j0   = rem & 7;
-
-                ushort qhb = src0_qh[8*ib + ib32];
-                float dl = (float)src0_d[ib] * (float)(2*((qhb >> 12) & 7) + 1);
-                float dlt = (qhb & 0x8000) ? -1.125f : -0.875f;
-                uint gi = (uint)src0_qs[32*ib + 4*ib32 + lg] | ((((uint)qhb >> (3*lg)) & 7) << 8);
-                uint g = iq1_s_grid[gi];
-                int sh = (j0 >> 2) * 4;
-
-                float4 v1;
-                v1.s0 = dl * ((float)((g >> ( 0 + sh)) & 0xF) + dlt);
-                v1.s1 = dl * ((float)((g >> ( 8 + sh)) & 0xF) + dlt);
-                v1.s2 = dl * ((float)((g >> (16 + sh)) & 0xF) + dlt);
-                v1.s3 = dl * ((float)((g >> (24 + sh)) & 0xF) + dlt);
-
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
-            } else {
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
-            }
-        }
-
-        for (int l = 0; l < BN; l += loadstride_b) {
-            if (ic*BN + loadc_b + l < ne11) {
-                int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
-            } else {
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
-            }
-        }
-
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-        pos_a += BK;
-        pos_b += BK / LOAD_VEC_B;
-
-        for (int i = 0; i < BK; i++) {
-            for (int j = 0; j < TM; j++) {
-                cache_a[j] = buf_a[(i) * BM + th_r * TM + j];
-            }
-
-            for (int j = 0; j < TN; j++) {
-                cache_b[j] = buf_b[(i) * BN + th_c * TN + j];
-            }
-
-            for (int cc = 0; cc < TN; cc++) {
-                for (int cr = 0; cr < TM; cr++) {
-                    const int sums_idx = cc*TM + cr;
-                    sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);
-                }
-            }
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    const int dr = ir * BM + th_r * TM;
-    const int dc = ic * BN + th_c * TN;
-
-    const ulong offsets = batch_idx * batch_stride_d;
-
-    for (int cc = 0; cc < TN; cc++) {
-        for (int cr = 0; cr < TM; cr++) {
-            if (dr + cr < ne01 && dc + cc < ne11) {
-                dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mm_iq3_s_f32_l4_lm.cl

@@ -1,193 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#define INTEL_GPU 1
-#endif
-
-#define QK_K 256
-
-struct block_iq3_s {
-    half d;
-    uchar qs[QK_K/4];
-    uchar qh[QK_K/32];
-    uchar signs[QK_K/8];
-    uchar scales[QK_K/64];
-};
-
-#define LOAD_VEC_A 4
-#define LOAD_VEC_B 4
-
-#define BM 64
-#define BN 64
-#define BK 32
-#ifdef INTEL_GPU
-#define TM 8
-#define TN 8
-#else
-#define TM 4
-#define TN 8
-#endif
-
-kernel void kernel_mul_mm_iq3_s_f32_l4_lm(
-    global uchar * src0_qs,
-    global uchar * src0_sg,
-    global uchar * src0_qh,
-    global uchar * src0_sc,
-    global half * src0_d,
-    ulong offset0,
-    global float4 * src1,
-    ulong offset1,
-    global float  * dst,
-    ulong offsetd,
-
-    int ne00,
-    int ne01,
-    int ne02,
-    int ne11,
-    int ne12,
-
-    int stride_a,
-    int stride_b,
-    int stride_d,
-
-    int batch_stride_a,
-    int batch_stride_b,
-    int batch_stride_d,
-
-    int r2,
-    int r3
-) {
-    src1 = (global float4*)((global char*)src1 + offset1);
-    dst  = (global float *)((global char*)dst  + offsetd);
-
-    local float buf_a[BM * BK];
-    local float buf_b[BN * BK];
-
-    const int batch_idx = get_global_id(2);
-
-    const int i13 = batch_idx / ne12;
-    const int i12 = batch_idx % ne12;
-
-    const int i03 = i13 / r3;
-    const int i02 = i12 / r2;
-
-    const int batch_idx_a = i03 * ne02 + i02;
-
-    const int ir = get_group_id(0);
-    const int ic = get_group_id(1);
-
-    const int tid = get_local_id(0);
-    const int th_r  = tid % (BM / TM);
-    const int th_c  = tid / (BM / TM);
-
-    const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
-    const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
-    const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
-    const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
-
-    const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
-    const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
-
-    // pos_a counts elements, not blocks - one iq3_s super block holds QK_K of them
-    int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
-    int pos_b = (batch_idx   * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
-
-    float sums[TM * TN];
-    float cache_a[TM];
-    float cache_b[TN];
-
-    for (int i = 0; i < TM * TN; i++) {
-        sums[i] = 0.0f;
-    }
-
-    for (int block = 0; block < ne00; block += BK) {
-        for (int l = 0; l < BM; l += loadstride_a) {
-            if (ir*BM + loadc_a + l < ne01) {
-                int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
-                int ib  = idx / QK_K;
-                ib += offset0/sizeof(struct block_iq3_s);
-                int e   = idx % QK_K;
-
-                int ib32 = e >> 5;
-                int rem  = e & 31;
-                int lg    = rem >> 3;
-                int j0   = rem & 7;
-
-                uchar scb = src0_sc[4*ib + ib32/2];
-                float db = (float)src0_d[ib] * (float)(1 + 2*((scb >> (4*(ib32 & 1))) & 15));
-                int which = j0 >> 2;
-                uint gi = (uint)src0_qs[64*ib + 8*ib32 + 2*lg + which]
-                        | (((uint)src0_qh[8*ib + ib32] << (8 - 2*lg - which)) & 256);
-                uchar sg = src0_sg[32*ib + 4*ib32 + lg];
-                uint g = iq3_s_grid[gi];
-
-                float4 v1;
-                v1.s0 = db * (float)((g >>  0) & 0xFF) * ((sg & (1 << (j0+0))) ? -1.f : 1.f);
-                v1.s1 = db * (float)((g >>  8) & 0xFF) * ((sg & (1 << (j0+1))) ? -1.f : 1.f);
-                v1.s2 = db * (float)((g >> 16) & 0xFF) * ((sg & (1 << (j0+2))) ? -1.f : 1.f);
-                v1.s3 = db * (float)((g >> 24) & 0xFF) * ((sg & (1 << (j0+3))) ? -1.f : 1.f);
-
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
-            } else {
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
-            }
-        }
-
-        for (int l = 0; l < BN; l += loadstride_b) {
-            if (ic*BN + loadc_b + l < ne11) {
-                int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
-            } else {
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
-            }
-        }
-
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-        pos_a += BK;
-        pos_b += BK / LOAD_VEC_B;
-
-        for (int i = 0; i < BK; i++) {
-            for (int j = 0; j < TM; j++) {
-                cache_a[j] = buf_a[(i) * BM + th_r * TM + j];
-            }
-
-            for (int j = 0; j < TN; j++) {
-                cache_b[j] = buf_b[(i) * BN + th_c * TN + j];
-            }
-
-            for (int cc = 0; cc < TN; cc++) {
-                for (int cr = 0; cr < TM; cr++) {
-                    const int sums_idx = cc*TM + cr;
-                    sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);
-                }
-            }
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    const int dr = ir * BM + th_r * TM;
-    const int dc = ic * BN + th_c * TN;
-
-    const ulong offsets = batch_idx * batch_stride_d;
-
-    for (int cc = 0; cc < TN; cc++) {
-        for (int cr = 0; cr < TM; cr++) {
-            if (dr + cr < ne01 && dc + cc < ne11) {
-                dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mm_iq3_xxs_f32_l4_lm.cl

@@ -1,187 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#define INTEL_GPU 1
-#endif
-
-#define QK_K 256
-
-struct block_iq3_xxs {
-    half d;
-    uchar qs[3*QK_K/8];
-};
-
-#define LOAD_VEC_A 4
-#define LOAD_VEC_B 4
-
-#define BM 64
-#define BN 64
-#define BK 32
-#ifdef INTEL_GPU
-#define TM 8
-#define TN 8
-#else
-#define TM 4
-#define TN 8
-#endif
-
-kernel void kernel_mul_mm_iq3_xxs_f32_l4_lm(
-    global uchar * src0_qs,
-    global uint * src0_sas,
-    global half * src0_d,
-    ulong offset0,
-    global float4 * src1,
-    ulong offset1,
-    global float  * dst,
-    ulong offsetd,
-
-    int ne00,
-    int ne01,
-    int ne02,
-    int ne11,
-    int ne12,
-
-    int stride_a,
-    int stride_b,
-    int stride_d,
-
-    int batch_stride_a,
-    int batch_stride_b,
-    int batch_stride_d,
-
-    int r2,
-    int r3
-) {
-    src1 = (global float4*)((global char*)src1 + offset1);
-    dst  = (global float *)((global char*)dst  + offsetd);
-
-    local float buf_a[BM * BK];
-    local float buf_b[BN * BK];
-
-    const int batch_idx = get_global_id(2);
-
-    const int i13 = batch_idx / ne12;
-    const int i12 = batch_idx % ne12;
-
-    const int i03 = i13 / r3;
-    const int i02 = i12 / r2;
-
-    const int batch_idx_a = i03 * ne02 + i02;
-
-    const int ir = get_group_id(0);
-    const int ic = get_group_id(1);
-
-    const int tid = get_local_id(0);
-    const int th_r  = tid % (BM / TM);
-    const int th_c  = tid / (BM / TM);
-
-    const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
-    const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
-    const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
-    const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
-
-    const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
-    const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
-
-    // pos_a counts elements, not blocks - one iq3_xxs super block holds QK_K of them
-    int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
-    int pos_b = (batch_idx   * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
-
-    float sums[TM * TN];
-    float cache_a[TM];
-    float cache_b[TN];
-
-    for (int i = 0; i < TM * TN; i++) {
-        sums[i] = 0.0f;
-    }
-
-    for (int block = 0; block < ne00; block += BK) {
-        for (int l = 0; l < BM; l += loadstride_a) {
-            if (ir*BM + loadc_a + l < ne01) {
-                int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
-                int ib  = idx / QK_K;
-                ib += offset0/sizeof(struct block_iq3_xxs);
-                int e   = idx % QK_K;
-
-                int ib32 = e >> 5;
-                int rem  = e & 31;
-                int lg    = rem >> 3;
-                int j0   = rem & 7;
-
-                uint aux = src0_sas[8*ib + ib32];
-                float db = (float)src0_d[ib] * (0.5f + (float)(aux >> 28)) * 0.5f;
-
-                uint gi = src0_qs[64*ib + 8*ib32 + 2*lg + (j0 >> 2)];
-                uint sg = iq3_xxs_signs((aux >> (7*lg)) & 127u);
-                uint  g  = iq3_xxs_grid[gi];
-
-                float4 v1;
-                v1.s0 = db * (float)((g >>  0) & 0xFF) * ((sg & (1 << (j0+0))) ? -1.f : 1.f);
-                v1.s1 = db * (float)((g >>  8) & 0xFF) * ((sg & (1 << (j0+1))) ? -1.f : 1.f);
-                v1.s2 = db * (float)((g >> 16) & 0xFF) * ((sg & (1 << (j0+2))) ? -1.f : 1.f);
-                v1.s3 = db * (float)((g >> 24) & 0xFF) * ((sg & (1 << (j0+3))) ? -1.f : 1.f);
-
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
-            } else {
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
-            }
-        }
-
-        for (int l = 0; l < BN; l += loadstride_b) {
-            if (ic*BN + loadc_b + l < ne11) {
-                int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
-            } else {
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
-            }
-        }
-
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-        pos_a += BK;
-        pos_b += BK / LOAD_VEC_B;
-
-        for (int i = 0; i < BK; i++) {
-            for (int j = 0; j < TM; j++) {
-                cache_a[j] = buf_a[(i) * BM + th_r * TM + j];
-            }
-
-            for (int j = 0; j < TN; j++) {
-                cache_b[j] = buf_b[(i) * BN + th_c * TN + j];
-            }
-
-            for (int cc = 0; cc < TN; cc++) {
-                for (int cr = 0; cr < TM; cr++) {
-                    const int sums_idx = cc*TM + cr;
-                    sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);
-                }
-            }
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    const int dr = ir * BM + th_r * TM;
-    const int dc = ic * BN + th_c * TN;
-
-    const ulong offsets = batch_idx * batch_stride_d;
-
-    for (int cc = 0; cc < TN; cc++) {
-        for (int cr = 0; cr < TM; cr++) {
-            if (dr + cr < ne01 && dc + cc < ne11) {
-                dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mm_iq4_xs_f32_l4_lm.cl

@@ -1,186 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#define INTEL_GPU 1
-#endif
-
-#define QK_K 256
-
-struct block_iq4_xs {
-    half d;
-    ushort scales_h;
-    uchar scales_l[QK_K/64];
-    uchar qs[QK_K/2];
-};
-
-#define LOAD_VEC_A 4
-#define LOAD_VEC_B 4
-
-#define BM 64
-#define BN 64
-#define BK 32
-#ifdef INTEL_GPU
-#define TM 8
-#define TN 8
-#else
-#define TM 4
-#define TN 8
-#endif
-
-kernel void kernel_mul_mm_iq4_xs_f32_l4_lm(
-    global uchar * src0_qs,
-    global uchar * src0_sl,
-    global ushort * src0_sh,
-    global half * src0_d,
-    ulong offset0,
-    global float4 * src1,
-    ulong offset1,
-    global float  * dst,
-    ulong offsetd,
-
-    int ne00,
-    int ne01,
-    int ne02,
-    int ne11,
-    int ne12,
-
-    int stride_a,
-    int stride_b,
-    int stride_d,
-
-    int batch_stride_a,
-    int batch_stride_b,
-    int batch_stride_d,
-
-    int r2,
-    int r3
-) {
-    src1 = (global float4*)((global char*)src1 + offset1);
-    dst  = (global float *)((global char*)dst  + offsetd);
-
-    local float buf_a[BM * BK];
-    local float buf_b[BN * BK];
-
-    const int batch_idx = get_global_id(2);
-
-    const int i13 = batch_idx / ne12;
-    const int i12 = batch_idx % ne12;
-
-    const int i03 = i13 / r3;
-    const int i02 = i12 / r2;
-
-    const int batch_idx_a = i03 * ne02 + i02;
-
-    const int ir = get_group_id(0);
-    const int ic = get_group_id(1);
-
-    const int tid = get_local_id(0);
-    const int th_r  = tid % (BM / TM);
-    const int th_c  = tid / (BM / TM);
-
-    const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
-    const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
-    const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
-    const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
-
-    const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
-    const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
-
-    // pos_a counts elements, not blocks - one iq4_xs super block holds QK_K of them
-    int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
-    int pos_b = (batch_idx   * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
-
-    float sums[TM * TN];
-    float cache_a[TM];
-    float cache_b[TN];
-
-    for (int i = 0; i < TM * TN; i++) {
-        sums[i] = 0.0f;
-    }
-
-    for (int block = 0; block < ne00; block += BK) {
-        for (int l = 0; l < BM; l += loadstride_a) {
-            if (ir*BM + loadc_a + l < ne01) {
-                int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
-                int ib  = idx / QK_K;
-                ib += offset0/sizeof(struct block_iq4_xs);
-                int e   = idx % QK_K;
-
-                int ib32 = e >> 5;
-                int j = e & 15;
-                int shift = (e & 16) >> 2;
-                int ls = ((src0_sl[4*ib + ib32/2] >> (4*(ib32 & 1))) & 15) | (((src0_sh[ib] >> (2*ib32)) & 3) << 4);
-                float db = (float)src0_d[ib] * (float)(ls - 32);
-                uchar4 q = vload4(0, src0_qs + 128*ib + 16*ib32 + j);
-                q = (q >> (uchar4)(shift)) & (uchar4)(15);
-
-                float4 v1;
-                v1.s0 = db * (float)iq4_xs_values[q.s0];
-                v1.s1 = db * (float)iq4_xs_values[q.s1];
-                v1.s2 = db * (float)iq4_xs_values[q.s2];
-                v1.s3 = db * (float)iq4_xs_values[q.s3];
-
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
-            } else {
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
-            }
-        }
-
-        for (int l = 0; l < BN; l += loadstride_b) {
-            if (ic*BN + loadc_b + l < ne11) {
-                int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
-            } else {
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
-            }
-        }
-
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-        pos_a += BK;
-        pos_b += BK / LOAD_VEC_B;
-
-        for (int i = 0; i < BK; i++) {
-            for (int j = 0; j < TM; j++) {
-                cache_a[j] = buf_a[(i) * BM + th_r * TM + j];
-            }
-
-            for (int j = 0; j < TN; j++) {
-                cache_b[j] = buf_b[(i) * BN + th_c * TN + j];
-            }
-
-            for (int cc = 0; cc < TN; cc++) {
-                for (int cr = 0; cr < TM; cr++) {
-                    const int sums_idx = cc*TM + cr;
-                    sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);
-                }
-            }
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    const int dr = ir * BM + th_r * TM;
-    const int dc = ic * BN + th_c * TN;
-
-    const ulong offsets = batch_idx * batch_stride_d;
-
-    for (int cc = 0; cc < TN; cc++) {
-        for (int cr = 0; cr < TM; cr++) {
-            if (dr + cr < ne01 && dc + cc < ne11) {
-                dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq1_m_f32.cl

@@ -1,149 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-#define IQ1_M_DELTA 0.125f
-
-// no d field: the super block scale is 4 nibbles spread over scales[]
-typedef struct {
-    uchar qs[QK_K/8];
-    uchar qh[QK_K/16];
-    uchar scales[QK_K/32];
-} block_iq1_m;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq1_m_f32(
-        global char * src0,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src0 = src0 + offset0;
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global block_iq1_m * x = (global block_iq1_m *) (src0 + offset_src0);
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        global char * xrow = (global char *)(x + ib);
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            global block_iq1_m * xb = (global block_iq1_m *)(xrow + row*nb01);
-
-            global ushort * sc = (global ushort *)xb->scales;
-            ushort u16 = (ushort)(((uint)sc[0] >> 12) | (((uint)sc[1] >> 8) & 0x00f0)
-                                | (((uint)sc[2] >> 4) & 0x0f00) | ((uint)sc[3] & 0xf000));
-            float dsuper = (float)as_half(u16);
-
-            float acc = 0.f;
-            for (int il = 0; il < 4; ++il) {
-                int   ib16 = 2*it + il/2;
-                float dl   = dsuper * (float)(2*(((uint)sc[ib16/4] >> (3*(ib16%4))) & 7) + 1);
-
-                uchar qhb = xb->qh[2*it + il/2];
-                float dlt = (qhb & (0x08 << (4*(il%2)))) ? (-1.f - IQ1_M_DELTA) : (-1.f + IQ1_M_DELTA);
-
-                uint gi = (uint)xb->qs[4*it+il] | ((((uint)qhb >> (4*(il%2))) & 7) << 8);
-                uint g  = iq1_s_grid[gi];
-
-                float a = 0.f;
-                for (int j = 0; j < 4; ++j) {
-                    a += yl[8*il+j+0] * ((float)((g >> (8*j+0)) & 0xF) + dlt);
-                    a += yl[8*il+j+4] * ((float)((g >> (8*j+4)) & 0xF) + dlt);
-                }
-                acc += dl * a;
-            }
-
-            sumf[row] += acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq1_m_f32_flat.cl

@@ -1,148 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-#define IQ1_M_DELTA 0.125f
-
-// no d field: the super block scale is 4 nibbles spread over scales[]
-typedef struct {
-    uchar qs[QK_K/8];
-    uchar qh[QK_K/16];
-    uchar scales[QK_K/32];
-} block_iq1_m;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq1_m_f32_flat(
-        global uchar * src0_qs,
-        global uchar * src0_qh,
-        global ushort * src0_sc,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq1_m);
-    ulong row_stride = nb01/sizeof(block_iq1_m);
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            const ulong b = offset_src0 + ib + row*row_stride;
-
-            global ushort * sc = src0_sc + 4*b;
-            ushort u16 = (ushort)(((uint)sc[0] >> 12) | (((uint)sc[1] >> 8) & 0x00f0)
-                                | (((uint)sc[2] >> 4) & 0x0f00) | ((uint)sc[3] & 0xf000));
-            float dsuper = (float)as_half(u16);
-
-            float acc = 0.f;
-            for (int il = 0; il < 4; ++il) {
-                int   ib16 = 2*it + il/2;
-                float dl   = dsuper * (float)(2*(((uint)sc[ib16/4] >> (3*(ib16%4))) & 7) + 1);
-
-                uchar qhb = src0_qh[16*b + 2*it + il/2];
-                float dlt = (qhb & (0x08 << (4*(il%2)))) ? (-1.f - IQ1_M_DELTA) : (-1.f + IQ1_M_DELTA);
-
-                uint gi = (uint)src0_qs[32*b + 4*it+il] | ((((uint)qhb >> (4*(il%2))) & 7) << 8);
-                uint g  = iq1_s_grid[gi];
-
-                float a = 0.f;
-                for (int j = 0; j < 4; ++j) {
-                    a += yl[8*il+j+0] * ((float)((g >> (8*j+0)) & 0xF) + dlt);
-                    a += yl[8*il+j+4] * ((float)((g >> (8*j+4)) & 0xF) + dlt);
-                }
-                acc += dl * a;
-            }
-
-            sumf[row] += acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq1_s_f32.cl

@@ -1,140 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-#define IQ1_S_DELTA 0.125f
-
-typedef struct {
-    half   d;
-    uchar  qs[QK_K/8];
-    ushort qh[QK_K/32];
-} block_iq1_s;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq1_s_f32(
-        global char * src0,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src0 = src0 + offset0;
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global block_iq1_s * x = (global block_iq1_s *) (src0 + offset_src0);
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        global char * xrow = (global char *)(x + ib);
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            global block_iq1_s * xb = (global block_iq1_s *)(xrow + row*nb01);
-
-            ushort qhb = xb->qh[it];
-            float  dl  = (float)xb->d * (float)(2*((qhb >> 12) & 7) + 1);
-            float  dlt = (qhb & 0x8000) ? (-1.f - IQ1_S_DELTA) : (-1.f + IQ1_S_DELTA);
-
-            global uchar * qsb = xb->qs + 4*it;
-
-            float acc = 0.f;
-            for (int l = 0; l < 4; ++l) {
-                uint gi = (uint)qsb[l] | ((((uint)qhb >> (3*l)) & 7) << 8);
-                uint g  = iq1_s_grid[gi];
-                for (int j = 0; j < 4; ++j) {
-                    acc += yl[8*l+j+0] * ((float)((g >> (8*j+0)) & 0xF) + dlt);
-                    acc += yl[8*l+j+4] * ((float)((g >> (8*j+4)) & 0xF) + dlt);
-                }
-            }
-
-            sumf[row] += dl * acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq1_s_f32_flat.cl

@@ -1,139 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-#define IQ1_S_DELTA 0.125f
-
-typedef struct {
-    half   d;
-    uchar  qs[QK_K/8];
-    ushort qh[QK_K/32];
-} block_iq1_s;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq1_s_f32_flat(
-        global uchar * src0_qs,
-        global ushort * src0_qh,
-        global half * src0_d,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq1_s);
-    ulong row_stride = nb01/sizeof(block_iq1_s);
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            const ulong b = offset_src0 + ib + row*row_stride;
-
-            ushort qhb = src0_qh[8*b + it];
-            float  dl  = (float)src0_d[b] * (float)(2*((qhb >> 12) & 7) + 1);
-            float  dlt = (qhb & 0x8000) ? (-1.f - IQ1_S_DELTA) : (-1.f + IQ1_S_DELTA);
-
-            global uchar * qsb = src0_qs + 32*b + 4*it;
-
-            float acc = 0.f;
-            for (int l = 0; l < 4; ++l) {
-                uint gi = (uint)qsb[l] | ((((uint)qhb >> (3*l)) & 7) << 8);
-                uint g  = iq1_s_grid[gi];
-                for (int j = 0; j < 4; ++j) {
-                    acc += yl[8*l+j+0] * ((float)((g >> (8*j+0)) & 0xF) + dlt);
-                    acc += yl[8*l+j+4] * ((float)((g >> (8*j+4)) & 0xF) + dlt);
-                }
-            }
-
-            sumf[row] += dl * acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32.cl

@@ -1,146 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-#define IQ3_S_N_SCALE (QK_K/64)
-
-typedef struct {
-    half  d;
-    uchar qs[QK_K/4];
-    uchar qh[QK_K/32];
-    uchar signs[QK_K/8];
-    uchar scales[IQ3_S_N_SCALE];
-} block_iq3_s;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq3_s_f32(
-        global char * src0,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src0 = src0 + offset0;
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global block_iq3_s * x = (global block_iq3_s *) (src0 + offset_src0);
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        global char * xrow = (global char *)(x + ib);
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            global block_iq3_s * xb = (global block_iq3_s *)(xrow + row*nb01);
-
-            uchar nib = (it & 1) ? (xb->scales[it >> 1] >> 4) : (xb->scales[it >> 1] & 0xf);
-            float db  = (float)xb->d * (float)(1 + 2*nib);
-
-            uchar qhb = xb->qh[it];
-            global uchar * qsb = xb->qs + 8*it;
-            global uchar * sgb = xb->signs + 4*it;
-
-            float acc = 0.f;
-            for (int l = 0; l < 4; ++l) {
-                uint i1 = (uint)qsb[2*l+0] | (((uint)qhb << (8-2*l)) & 256);
-                uint i2 = (uint)qsb[2*l+1] | (((uint)qhb << (7-2*l)) & 256);
-                uint g1 = iq3_s_grid[i1];
-                uint g2 = iq3_s_grid[i2];
-                uchar sg = sgb[l];
-                for (int j = 0; j < 4; ++j) {
-                    acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * ((sg & (1 << j))     ? -1.f : 1.f);
-                    acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f);
-                }
-            }
-
-            sumf[row] += db * acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32_flat.cl

@@ -1,147 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-#define IQ3_S_N_SCALE (QK_K/64)
-
-typedef struct {
-    half  d;
-    uchar qs[QK_K/4];
-    uchar qh[QK_K/32];
-    uchar signs[QK_K/8];
-    uchar scales[IQ3_S_N_SCALE];
-} block_iq3_s;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq3_s_f32_flat(
-        global uchar * src0_qs,
-        global uchar * src0_sg,
-        global uchar * src0_qh,
-        global uchar * src0_sc,
-        global half * src0_d,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq3_s);
-    ulong row_stride = nb01/sizeof(block_iq3_s);
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            const ulong b = offset_src0 + ib + row*row_stride;
-
-            uchar nib = (it & 1) ? (src0_sc[4*b + (it >> 1)] >> 4) : (src0_sc[4*b + (it >> 1)] & 0xf);
-            float db  = (float)src0_d[b] * (float)(1 + 2*nib);
-
-            uchar qhb = src0_qh[8*b + it];
-            global uchar * qsb = src0_qs + 64*b + 8*it;
-            global uchar * sgb = src0_sg + 32*b + 4*it;
-
-            float acc = 0.f;
-            for (int l = 0; l < 4; ++l) {
-                uint i1 = (uint)qsb[2*l+0] | (((uint)qhb << (8-2*l)) & 256);
-                uint i2 = (uint)qsb[2*l+1] | (((uint)qhb << (7-2*l)) & 256);
-                uint g1 = iq3_s_grid[i1];
-                uint g2 = iq3_s_grid[i2];
-                uchar sg = sgb[l];
-                for (int j = 0; j < 4; ++j) {
-                    acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * ((sg & (1 << j))     ? -1.f : 1.f);
-                    acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f);
-                }
-            }
-
-            sumf[row] += db * acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32.cl

@@ -1,145 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-
-// d, then 64 grid index bytes, then 8 uint32 of packed scale+signs
-typedef struct {
-    half  d;
-    uchar qs[3*QK_K/8];
-} block_iq3_xxs;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq3_xxs_f32(
-        global char * src0,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src0 = src0 + offset0;
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global block_iq3_xxs * x = (global block_iq3_xxs *) (src0 + offset_src0);
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        global char * xrow = (global char *)(x + ib);
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            global block_iq3_xxs * xb = (global block_iq3_xxs *)(xrow + row*nb01);
-
-            global uchar * qs = xb->qs + 8*it;
-
-            // Blocks are 98 bytes, so packed signs are only 2-byte aligned.
-            global ushort * sw = (global ushort *)(xb->qs + 64 + 4*it);
-            uint aux32 = (uint)sw[0] | ((uint)sw[1] << 16);
-
-            float db = (float)xb->d * (0.5f + (float)(aux32 >> 28)) * 0.5f;
-
-            float acc = 0.f;
-            for (int l = 0; l < 4; ++l) {
-                uchar signs = iq3_xxs_signs((aux32 >> (7*l)) & 127u);
-                uint  g1 = iq3_xxs_grid[qs[2*l+0]];
-                uint  g2 = iq3_xxs_grid[qs[2*l+1]];
-
-                for (int j = 0; j < 4; ++j) {
-                    float s1 = (signs & (1 << j))       ? -1.f : 1.f;
-                    float s2 = (signs & (1 << (j+4)))   ? -1.f : 1.f;
-                    acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * s1;
-                    acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * s2;
-                }
-            }
-
-            sumf[row] += db * acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32_flat.cl

@@ -1,142 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-
-// d, then 64 grid index bytes, then 8 uint32 of packed scale+signs
-typedef struct {
-    half  d;
-    uchar qs[3*QK_K/8];
-} block_iq3_xxs;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq3_xxs_f32_flat(
-        global uchar * src0_qs,
-        global uint * src0_sas,
-        global half * src0_d,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq3_xxs);
-    ulong row_stride = nb01/sizeof(block_iq3_xxs);
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            const ulong b = offset_src0 + ib + row*row_stride;
-
-            global uchar * qs = src0_qs + 64*b + 8*it;
-
-            uint aux32 = src0_sas[8*b + it];
-
-            float db = (float)src0_d[b] * (0.5f + (float)(aux32 >> 28)) * 0.5f;
-
-            float acc = 0.f;
-            for (int l = 0; l < 4; ++l) {
-                uchar signs = iq3_xxs_signs((aux32 >> (7*l)) & 127u);
-                uint  g1 = iq3_xxs_grid[qs[2*l+0]];
-                uint  g2 = iq3_xxs_grid[qs[2*l+1]];
-
-                for (int j = 0; j < 4; ++j) {
-                    float s1 = (signs & (1 << j))       ? -1.f : 1.f;
-                    float s2 = (signs & (1 << (j+4)))   ? -1.f : 1.f;
-                    acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * s1;
-                    acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * s2;
-                }
-            }
-
-            sumf[row] += db * acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32.cl

@@ -1,134 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-
-typedef struct {
-    half d;
-    ushort scales_h;
-    uchar scales_l[QK_K/64];
-    uchar qs[QK_K/2];
-} block_iq4_xs;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq4_xs_f32(
-        global char * src0,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src0 = src0 + offset0;
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global block_iq4_xs * x = (global block_iq4_xs *) (src0 + offset_src0);
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        global char * xrow = (global char *)(x + ib);
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            global block_iq4_xs * xb = (global block_iq4_xs *)(xrow + row*nb01);
-
-            int ls = ((xb->scales_l[it/2] >> (4*(it%2))) & 15) | (((xb->scales_h >> (2*it)) & 3) << 4);
-            float db = (float)xb->d * (float)(ls - 32);
-            global uchar * qs = xb->qs + 16*it;
-
-            float acc = 0.f;
-            for (int j = 0; j < 16; ++j) {
-                acc += yl[j]    * (float)iq4_xs_values[qs[j] & 15];
-                acc += yl[j+16] * (float)iq4_xs_values[qs[j] >>  4];
-            }
-
-            sumf[row] += db * acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32_flat.cl

@@ -1,134 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-
-typedef struct {
-    half d;
-    ushort scales_h;
-    uchar scales_l[QK_K/64];
-    uchar qs[QK_K/2];
-} block_iq4_xs;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq4_xs_f32_flat(
-        global uchar * src0_qs,
-        global uchar * src0_sl,
-        global ushort * src0_sh,
-        global half * src0_d,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq4_xs);
-    ulong row_stride = nb01/sizeof(block_iq4_xs);
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            const ulong b = offset_src0 + ib + row*row_stride;
-
-            int ls = ((src0_sl[4*b + it/2] >> (4*(it%2))) & 15) | (((src0_sh[b] >> (2*it)) & 3) << 4);
-            float db = (float)src0_d[b] * (float)(ls - 32);
-            global uchar * qs = src0_qs + 128*b + 16*it;
-
-            float acc = 0.f;
-            for (int j = 0; j < 16; ++j) {
-                acc += yl[j]    * (float)iq4_xs_values[qs[j] & 15];
-                acc += yl[j+16] * (float)iq4_xs_values[qs[j] >>  4];
-            }
-
-            sumf[row] += db * acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/top_k_mask.cl

@@ -1,129 +0,0 @@
-// Integer storage preserves the mask bits without FP16 conversion.
-#if MASK_F16
-#define MASK_T ushort
-#define MASK_V ushort8
-#define MASK_NV 8
-#define MASK_NINF 0xfc00
-#define MASK_STORE vstore8
-#else
-#define MASK_T uint
-#define MASK_V uint4
-#define MASK_NV 4
-#define MASK_NINF 0xff800000u
-#define MASK_STORE vstore4
-#endif
-
-kernel void kernel_top_k_mask_fill(
-        global char * dst,
-        ulong         offsetd,
-        ulong         n
-) {
-    global MASK_T * out = (global MASK_T *) (dst + offsetd);
-    const ulong c = get_global_id(0)*MASK_NV;
-    if (c + MASK_NV <= n) {
-        MASK_STORE((MASK_V) (MASK_NINF), 0, out + c);
-    } else {
-        for (ulong j = c; j < n; ++j) {
-            out[j] = MASK_NINF;
-        }
-    }
-}
-
-kernel void kernel_top_k_mask(
-        global const char * mask,
-        ulong               offsetm,
-        global const char * top_k,
-        ulong               offsetk,
-        global const char * score,
-        ulong               offsets,
-        global const char * blocks,
-        ulong               offsetb,
-        global const char * extra,
-        ulong               offsete,
-        global       char * dst,
-        ulong               offsetd,
-        ulong nem0,
-        ulong nem1,
-        ulong nem2,
-        ulong nem3,
-        ulong nbm1,
-        ulong nbm2,
-        ulong nbm3,
-        ulong nek0,
-        ulong nbk0,
-        ulong nbk1,
-        ulong nbk2,
-        ulong nbk3,
-        ulong nes0,
-        ulong nbs0,
-        ulong nbs1,
-        ulong nbs2,
-        ulong nbs3,
-        ulong neb0,
-        ulong neb1,
-        ulong nbb0,
-        ulong nbb1,
-        ulong nee0,
-        ulong nbe0,
-        ulong nbe1,
-        ulong nbe2,
-        ulong nbe3,
-        ulong ned1,
-        ulong ned2,
-        ulong nbd1,
-        ulong nbd2,
-        ulong nbd3,
-        int   fill
-) {
-    const ulong row = get_group_id(1);
-    const ulong i1 = row % ned1;
-    const ulong i2 = (row / ned1) % ned2;
-    const ulong i3 = row / (ned1*ned2);
-    const ulong tid = get_local_id(0);
-    const ulong nth = get_local_size(0);
-    const ulong n_pick = blocks ? neb1 : score ? nes0 : nem0;
-
-    global const MASK_T * mask_row = (global const MASK_T *) (mask + offsetm
-        + (i1 % nem1)*nbm1 + (i2 % nem2)*nbm2 + (i3 % nem3)*nbm3);
-    global MASK_T * dst_row = (global MASK_T *) (dst + offsetd + i1*nbd1 + i2*nbd2 + i3*nbd3);
-    global const char * top_k_row = top_k + offsetk + i1*nbk1 + i2*nbk2 + i3*nbk3;
-
-    if (fill) {
-        for (ulong c = tid; c < nem0/MASK_NV; c += nth) {
-            MASK_STORE((MASK_V) (MASK_NINF), c, dst_row);
-        }
-        for (ulong c = nem0/MASK_NV*MASK_NV + tid; c < nem0; c += nth) {
-            dst_row[c] = MASK_NINF;
-        }
-        barrier(CLK_GLOBAL_MEM_FENCE);
-    }
-
-    // Repeated cells copy the same mask value, as in the CUDA and Metal kernels.
-    for (ulong j = tid; j < nek0 + nee0; j += nth) {
-        if (j >= nek0) {
-            const int c = *(global const int *) (extra + offsete + (j - nek0)*nbe0 + i1*nbe1 + i2*nbe2 + i3*nbe3);
-            if (c >= 0 && (ulong) c < nem0) {
-                dst_row[c] = mask_row[c];
-            }
-            continue;
-        }
-
-        const int p = *(global const int *) (top_k_row + j*nbk0);
-        if (p < 0 || (ulong) p >= n_pick) {
-            continue;
-        }
-        if (score && !(*(global const float *) (score + offsets + p*nbs0 + i1*nbs1 + i2*nbs2 + i3*nbs3) > -INFINITY)) {
-            continue;
-        }
-        if (blocks) {
-            for (ulong k = 0; k < neb0; ++k) {
-                const int c = *(global const int *) (blocks + offsetb + k*nbb0 + p*nbb1);
-                if (c >= 0 && (ulong) c < nem0) {
-                    dst_row[c] = mask_row[c];
-                }
-            }
-        } else if ((ulong) p < nem0) {
-            dst_row[p] = mask_row[p];
-        }
-    }
-}

ggml/src/ggml-openvino/ggml-openvino.cpp

@@ -1520,11 +1520,6 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
             return {false, "FLASH_ATTN_EXT gemma3n pattern on GPU is not supported"};
         }
 
-        // K/V row lists (ggml_flash_attn_ext_rows) are not supported
-        if (op->src[5] != nullptr) {
-            return {false, "FLASH_ATTN_EXT with K/V row lists is not supported"};
-        }
-
         if (op->src[4] != nullptr) {
             return {false, "FLASH_ATTN_EXT with sinks is not supported"};
         }

ggml/src/ggml-sycl/CMakeLists.txt

@@ -107,36 +107,24 @@ target_compile_options(ggml-sycl PRIVATE "-Wno-narrowing")
 
 message(STATUS "GGML_SYCL_SUPPORT_LEVEL_ZERO_API ${GGML_SYCL_SUPPORT_LEVEL_ZERO_API}")
 if (GGML_SYCL_SUPPORT_LEVEL_ZERO_API)
-    find_package(PkgConfig)
-    if (PkgConfig_FOUND)
-        pkg_check_modules(LEVEL_ZERO level-zero)
-        if (LEVEL_ZERO_FOUND)
-            target_include_directories(ggml-sycl PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS})
-            target_link_libraries(ggml-sycl PRIVATE ${LEVEL_ZERO_LINK_LIBRARIES})
-            target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API)
-            message(STATUS "Level Zero loader found: ${LEVEL_ZERO_LINK_LIBRARIES}")
-            message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_INCLUDE_DIRS}")
-        endif()
+    # Link against Level Zero loader for direct device memory allocation.
+    # Avoids sycl::malloc_device triggering DMA-buf/TTM system RAM staging
+    # in the xe kernel driver during multi-GPU inference.
+    find_path(LEVEL_ZERO_DEV_INCLUDE_DIR level_zero/ze_api.h HINTS ${ONEAPI_ROOT}/include ${LEVEL_ZERO_V1_SDK_PATH}/include)
+    find_library(ZE_LOADER_LIB ze_loader HINTS ${ONEAPI_ROOT}/lib ${LEVEL_ZERO_V1_SDK_LIB_PATH} ENV LD_LIBRARY_PATH)
+    if(ZE_LOADER_LIB AND LEVEL_ZERO_DEV_INCLUDE_DIR)
+        target_link_libraries(ggml-sycl PRIVATE ${ZE_LOADER_LIB})
+        target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API)
+        message(STATUS "Level Zero loader found: ${ZE_LOADER_LIB}")
+        message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_DEV_INCLUDE_DIR}")
     else()
-        # Link against Level Zero loader for direct device memory allocation.
-        # Avoids sycl::malloc_device triggering DMA-buf/TTM system RAM staging
-        # in the xe kernel driver during multi-GPU inference.
-        find_path(LEVEL_ZERO_DEV_INCLUDE_DIR level_zero/ze_api.h HINTS ${ONEAPI_ROOT}/include ${LEVEL_ZERO_V1_SDK_PATH}/include)
-        find_library(ZE_LOADER_LIB ze_loader HINTS ${ONEAPI_ROOT}/lib ${LEVEL_ZERO_V1_SDK_LIB_PATH} ENV LD_LIBRARY_PATH)
-        if(ZE_LOADER_LIB AND LEVEL_ZERO_DEV_INCLUDE_DIR)
-            target_link_libraries(ggml-sycl PRIVATE ${ZE_LOADER_LIB})
-            target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API)
-            message(STATUS "Level Zero loader found: ${ZE_LOADER_LIB}")
-            message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_DEV_INCLUDE_DIR}")
-        else()
-            message(WARNING "Level Zero loader or development headers not found, "
-                            "Level Zero API support disabled. "
-                            "Please install the Level Zero SDK/development package "
-                            "to support Level Zero API features. "
-                            "Level Zero API is not mandatory for SYCL backend, "
-                            "but it is required by the special features for better "
-                            "function & performance on Intel GPUs.")
-        endif()
+        message(WARNING "Level Zero loader or development headers not found, "
+                        "Level Zero API support disabled. "
+                        "Please install the Level Zero SDK/development package "
+                        "to support Level Zero API features. "
+                        "Level Zero API is not mandatory for SYCL backend, "
+                        "but it is required by the special features for better "
+                        "function & performance on Intel GPUs.")
     endif()
 endif()
 

ggml/src/ggml-sycl/cpy.hpp

@@ -70,22 +70,8 @@ inline void cpy_blck_f32_q1_0(const char * cxi, char * cdsti) {
     }
 }
 
-// exact roundf (half away from zero); x + 0.5f carries for x = 0.5 - 2^-25, and sycl::round is not exact under fast fp
-inline int round_half_away_int(float x) {
-    const float t = sycl::trunc(x);
-    return (int) (sycl::fabs(x - t) >= 0.5f ? t + sycl::copysign(1.0f, x) : t);
-}
-
-// correctly rounded a / b to match the CPU: fast fp gives an approximate reciprocal (~12% off by an ulp on B580),
-// so pick the approximation or a neighbour by smallest exact FMA residual
-inline float div_correctly_rounded(float a, float b) {
-    const float y  = a / b;
-    const float lo = sycl::nextafter(y, -INFINITY);
-    const float hi = sycl::nextafter(y, INFINITY);
-    const float e  = sycl::fabs(sycl::fma(-b, y, a));
-    const float el = sycl::fabs(sycl::fma(-b, lo, a));
-    const float eh = sycl::fabs(sycl::fma(-b, hi, a));
-    return el < e && el <= eh ? lo : (eh < e ? hi : y);
+inline int round_nearest_int(float x) {
+    return (int)(x >= 0.0f ? x + 0.5f : x - 0.5f);
 }
 
 inline void cpy_blck_f32_q2_0(const char * cxi, char * cdsti) {
@@ -99,7 +85,7 @@ inline void cpy_blck_f32_q2_0(const char * cxi, char * cdsti) {
     }
 
     const float d  = amax;
-    const float id = d > 0.0f ? div_correctly_rounded(1.0f, d) : 0.0f;
+    const float id = d > 0.0f ? 1.0f / d : 0.0f;
 
     dsti->d = d;
 
@@ -108,7 +94,7 @@ inline void cpy_blck_f32_q2_0(const char * cxi, char * cdsti) {
     }
 
     for (int j = 0; j < QK2_0; ++j) {
-        int q = round_half_away_int(xi[j] * id) + 1;
+        int q = round_nearest_int(xi[j] * id) + 1;
         q = dpct::max(0, dpct::min(3, q));
 
         const int byte_index = j / 4;
@@ -392,7 +378,7 @@ inline void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) {
     }
 
     const float d  = vmax / -8;
-    const float id = d ? div_correctly_rounded(1.0f, d) : 0.0f;
+    const float id = d ? 1.0f / d : 0.0f;
 
     dsti->d = d;
 
@@ -400,9 +386,8 @@ inline void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) {
         const float x0 = xi[0 + j] * id;
         const float x1 = xi[QK4_0 / 2 + j] * id;
 
-        // no-op fmin stops x * id + 8.5f being fused into an FMA, so x * id is rounded as on the CPU
-        const uint8_t xi0 = dpct::min(15, (int8_t) (sycl::fmin(x0, 6.5f) + 8.5f));
-        const uint8_t xi1 = dpct::min(15, (int8_t) (sycl::fmin(x1, 6.5f) + 8.5f));
+        const uint8_t xi0 = dpct::min(15, (int8_t) (x0 + 8.5f));
+        const uint8_t xi1 = dpct::min(15, (int8_t) (x1 + 8.5f));
 
         dsti->qs[j] = xi0;
         dsti->qs[j] |= xi1 << 4;
@@ -457,7 +442,7 @@ inline void cpy_blck_f32_q5_0(const char * cxi, char * cdsti) {
     }
 
     const float d  = vmax / -16;
-    const float id = d ? div_correctly_rounded(1.0f, d) : 0.0f;
+    const float id = d ? 1.0f / d : 0.0f;
 
     dsti->d = d;
 
@@ -466,9 +451,8 @@ inline void cpy_blck_f32_q5_0(const char * cxi, char * cdsti) {
         const float x0 = xi[0 + j] * id;
         const float x1 = xi[QK5_0 / 2 + j] * id;
 
-        // fmin: no FMA, see cpy_blck_f32_q4_0
-        const uint8_t xi0 = dpct::min(31, (int8_t) (sycl::fmin(x0, 14.5f) + 16.5f));
-        const uint8_t xi1 = dpct::min(31, (int8_t) (sycl::fmin(x1, 14.5f) + 16.5f));
+        const uint8_t xi0 = dpct::min(31, (int8_t) (x0 + 16.5f));
+        const uint8_t xi1 = dpct::min(31, (int8_t) (x1 + 16.5f));
 
         dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);
         qh |= ((xi0 & 0x10u) >> 4) << (j + 0);

ggml/src/ggml-sycl/dmmv.cpp

@@ -112,12 +112,11 @@ static void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat *
     }
 }
 
-template <int qk, int qr, int qbytes, dequantize_kernel_t_reorder dequantize_kernel_reorder>
+template <int qk, int qr, dequantize_kernel_t_reorder dequantize_kernel_reorder>
 static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const dfloat * __restrict__ y, float * __restrict__ dst, const int ncols, const int nrows,
                                    const sycl::nd_item<3> &item_ct1) {
     // qk = quantized weights per x block
     // qr = number of quantized weights per data value in x block
-    // qbytes = bytes of the quants region per block (not qk/qr for bit-packed types)
     const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
                     item_ct1.get_local_id(1);
 
@@ -128,7 +127,7 @@ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const d
     const int tid = item_ct1.get_local_id(2);
 
 
-    const int ncols_left = ncols % (qk*WARP_SIZE);
+    const int ncols_left = ncols % (QK4_0*WARP_SIZE);
     const int ncols_align = ncols - ncols_left;
     const int iter_stride = 8*2*GGML_SYCL_DMMV_X;
     const int vals_per_iter = iter_stride / WARP_SIZE; // num quantized vals per thread and i iter //64/16=4, 512/16/2= 16
@@ -140,7 +139,7 @@ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const d
 #else
     float tmp = 0.0f;
 #endif // GGML_SYCL_F16
-    const char *d_ptr = (const char*)vx + (size_t) ncols * nrows / qk * qbytes;
+    const char *d_ptr = (const char*)vx+ncols*nrows/2;
     int i=0;
     for (i = 0; i < ncols_align; i += iter_stride) {
         const int col = i + vals_per_iter*tid;
@@ -156,7 +155,7 @@ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const d
             // dequantize
             // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val
             dfloat2 v;
-            dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * (qk/qr) +iqs+j/qr, v);
+            dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * QK4_0 / 2 +iqs+j/qr, v);
 
             // matrix multiplication
             // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2
@@ -173,7 +172,7 @@ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const d
     }
 
     for (; i < ncols; i += iter_stride) {
-        if (tid>=ncols_left/qk) continue;
+        if (tid>=ncols_left/QK4_0) continue;
         const int col = i + vals_per_iter*tid;
         const int ib = (row*ncols + col)/qk; // x block index
         const int iqs = (col%qk)/qr; // x quant index
@@ -187,7 +186,7 @@ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const d
             // dequantize
             // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val
             dfloat2 v;
-            dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * (qk/qr) +iqs+j/qr, v);
+            dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * QK4_0 / 2 +iqs+j/qr, v);
 
             // matrix multiplication
             // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2
@@ -1488,10 +1487,10 @@ static void dequantize_mul_mat_vec_q6_k_reorder(const void * __restrict__ vx, co
     }
 }
 
-template <int qk, int qr, dequantize_kernel_t dequantize_kernel>
-static void launch_dequantize_mul_mat_vec(const void *vx, const dfloat *y, float *dst,
-                                          const int ncols, const int nrows,
-                                          dpct::queue_ptr stream) {
+static void dequantize_mul_mat_vec_q4_0_sycl_reorder(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
     GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
     const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
     // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead
@@ -1504,16 +1503,17 @@ static void launch_dequantize_mul_mat_vec(const void *vx, const dfloat *y, float
         stream->parallel_for(
             sycl::nd_range<3>(block_nums * block_dims, block_dims),
             [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-                dequantize_mul_mat_vec<qk, qr, dequantize_kernel>(
+                dequantize_mul_mat_vec_reorder<QK4_0, QR4_0, dequantize_q4_0_reorder>(
                     vx, y, dst, ncols, nrows, item_ct1);
             });
     }
 }
 
-template <int qk, int qr, int qbytes, dequantize_kernel_t_reorder dequantize_kernel>
-static void launch_dequantize_mul_mat_vec_reorder(const void *vx, const dfloat *y, float *dst,
-                                                  const int ncols, const int nrows,
-                                                  dpct::queue_ptr stream) {
+
+static void dequantize_mul_mat_vec_q4_0_sycl(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
     GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
     const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
     // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead
@@ -1526,24 +1526,346 @@ static void launch_dequantize_mul_mat_vec_reorder(const void *vx, const dfloat *
         stream->parallel_for(
             sycl::nd_range<3>(block_nums * block_dims, block_dims),
             [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-                dequantize_mul_mat_vec_reorder<qk, qr, qbytes, dequantize_kernel>(
+                dequantize_mul_mat_vec<QK4_0, QR4_0, dequantize_q4_0>(
                     vx, y, dst, ncols, nrows, item_ct1);
             });
     }
 }
 
-template <int ny, auto kernel>
-static void launch_dequantize_mul_mat_vec_k(const void *vx, const float *y, float *dst,
-                                            const int ncols, const int nrows,
-                                            dpct::queue_ptr stream) {
+static void dequantize_mul_mat_vec_q1_0_sycl_reorder(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
+    const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
+    // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
+    {
+        dpct::has_capability_or_fail(stream->get_device(),
+                                     {sycl::aspect::fp16});
+
+        stream->parallel_for(
+            sycl::nd_range<3>(block_nums * block_dims, block_dims),
+            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                dequantize_mul_mat_vec_reorder<QK1_0, QR1_0, dequantize_q1_0_reorder>(
+                    vx, y, dst, ncols, nrows, item_ct1);
+            });
+    }
+}
+
+static void dequantize_mul_mat_vec_q1_0_sycl(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
+    const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
+    // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
+    {
+        dpct::has_capability_or_fail(stream->get_device(),
+                                     {sycl::aspect::fp16});
+
+        stream->parallel_for(
+            sycl::nd_range<3>(block_nums * block_dims, block_dims),
+            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                dequantize_mul_mat_vec<QK1_0, QR1_0, dequantize_q1_0>(
+                    vx, y, dst, ncols, nrows, item_ct1);
+            });
+    }
+}
+
+static void dequantize_mul_mat_vec_q4_1_sycl(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
+    const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
+    {
+        dpct::has_capability_or_fail(stream->get_device(),
+                                     {sycl::aspect::fp16});
+
+        stream->parallel_for(
+            sycl::nd_range<3>(block_nums * block_dims, block_dims),
+            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                dequantize_mul_mat_vec<QK4_1, QR4_1, dequantize_q4_1>(
+                    vx, y, dst, ncols, nrows, item_ct1);
+            });
+    }
+}
+
+static void dequantize_mul_mat_vec_q5_0_sycl(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
+    const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
+    {
+        dpct::has_capability_or_fail(stream->get_device(),
+                                     {sycl::aspect::fp16});
+
+        stream->parallel_for(
+            sycl::nd_range<3>(block_nums * block_dims, block_dims),
+            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                dequantize_mul_mat_vec<QK5_0, QR5_0, dequantize_q5_0>(
+                    vx, y, dst, ncols, nrows, item_ct1);
+            });
+    }
+}
+
+static void dequantize_mul_mat_vec_q5_1_sycl(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
+    const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
+    {
+        dpct::has_capability_or_fail(stream->get_device(),
+                                     {sycl::aspect::fp16});
+
+        stream->parallel_for(
+            sycl::nd_range<3>(block_nums * block_dims, block_dims),
+            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                dequantize_mul_mat_vec<QK5_1, QR5_1, dequantize_q5_1>(
+                    vx, y, dst, ncols, nrows, item_ct1);
+            });
+    }
+}
+
+static void dequantize_mul_mat_vec_q8_0_sycl_reorder(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
+    const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
+    {
+        dpct::has_capability_or_fail(stream->get_device(),
+                                     {sycl::aspect::fp16});
+
+        stream->parallel_for(
+            sycl::nd_range<3>(block_nums * block_dims, block_dims),
+            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                // Q8_0 reorder layout: [all qs (ncols*nrows bytes)][all d values]
+                // Cannot reuse dequantize_mul_mat_vec_reorder template because it has
+                // Q4_0-specific constants hardcoded (d_ptr offset and qs stride).
+                const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
+                    item_ct1.get_local_id(1);
+                if (row >= nrows) return;
+
+                const int tid = item_ct1.get_local_id(2);
+                const int iter_stride = 8*2*GGML_SYCL_DMMV_X;
+                const int vals_per_iter = iter_stride / WARP_SIZE;
+                const int ncols_left = ncols % (QK8_0*WARP_SIZE);
+                const int ncols_align = ncols - ncols_left;
+
+#ifdef GGML_SYCL_F16
+                sycl::half2 tmp = {0.0f, 0.0f};
+#else
+                float tmp = 0.0f;
+#endif
+                const char *d_ptr = (const char*)vx + ncols*nrows;  // d after all qs
+
+                int i = 0;
+                for (i = 0; i < ncols_align; i += iter_stride) {
+                    const int col = i + vals_per_iter*tid;
+                    const int ib = (row*ncols + col)/QK8_0;
+                    const int iqs = col % QK8_0;
+
+#pragma unroll
+                    for (int j = 0; j < vals_per_iter; j += 2) {
+                        dfloat2 v;
+                        dequantize_q8_0_reorder((const void *)d_ptr, ib, (const void *)vx,
+                                                ib * QK8_0 + iqs + j, v);
+
+#ifdef GGML_SYCL_F16
+                        dfloat2 t1{y[col + j + 0], y[col + j + 1]};
+                        tmp += v * t1;
+#else
+                        tmp += v.x() * y[col + j + 0];
+                        tmp += v.y() * y[col + j + 1];
+#endif
+                    }
+                }
+
+                // handle remaining columns
+                for (; i < ncols; i += iter_stride) {
+                    if (tid >= ncols_left/QK8_0) continue;
+                    const int col = i + vals_per_iter*tid;
+                    const int ib = (row*ncols + col)/QK8_0;
+                    const int iqs = col % QK8_0;
+
+#pragma unroll
+                    for (int j = 0; j < vals_per_iter; j += 2) {
+                        dfloat2 v;
+                        dequantize_q8_0_reorder((const void *)d_ptr, ib, (const void *)vx,
+                                                ib * QK8_0 + iqs + j, v);
+
+#ifdef GGML_SYCL_F16
+                        dfloat2 t1{y[col + j + 0], y[col + j + 1]};
+                        tmp += v * t1;
+#else
+                        tmp += v.x() * y[col + j + 0];
+                        tmp += v.y() * y[col + j + 1];
+#endif
+                    }
+                }
+
+                // reduce
+                const int mask_start = ncols > GGML_SYCL_DMMV_X ? WARP_SIZE >> 1 : WARP_SIZE >> 2;
+                for (int mask = mask_start; mask > 0; mask >>= 1) {
+                    tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
+                }
+
+                if (tid == 0) {
+#ifdef GGML_SYCL_F16
+                    dst[row] = tmp.x() + tmp.y();
+#else
+                    dst[row] = tmp;
+#endif
+                }
+            });
+    }
+}
+
+static void dequantize_mul_mat_vec_q8_0_sycl(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
+    const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
+    {
+        dpct::has_capability_or_fail(stream->get_device(),
+                                     {sycl::aspect::fp16});
+
+        stream->parallel_for(
+            sycl::nd_range<3>(block_nums * block_dims, block_dims),
+            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                dequantize_mul_mat_vec<QK8_0, QR8_0, dequantize_q8_0>(
+                    vx, y, dst, ncols, nrows, item_ct1);
+            });
+    }
+}
+
+static void dequantize_mul_mat_vec_q2_K_sycl(const void *vx, const float *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
     GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2; // very slightly faster than 1 even when K_QUANTS_PER_ITERATION = 2
     const int block_num_y = (nrows + ny - 1) / ny;
     const sycl::range<3> block_nums(1, 1, block_num_y);
     const sycl::range<3> block_dims(1, ny, WARP_SIZE);
     stream->parallel_for(
         sycl::nd_range<3>(block_nums * block_dims, block_dims),
         [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-            kernel(vx, y, dst, ncols, nrows, item_ct1);
+            dequantize_mul_mat_vec_q2_k(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q2_K_sycl_reorder(const void *vx, const float *y,
+                                                     float *dst, const int ncols,
+                                                     const int nrows,
+                                                     dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q2_k_reorder(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q3_K_sycl(const void *vx, const float *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q3_k(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q3_K_sycl_reorder(const void *vx, const float *y,
+                                                     float *dst, const int ncols,
+                                                     const int nrows,
+                                                     dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q3_k_reorder(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q4_K_sycl(const void *vx, const float *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q4_k(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q5_K_sycl(const void *vx, const float *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q5_k(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q6_K_sycl(const void *vx, const float *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q6_k(vx, y, dst, ncols, nrows, item_ct1);
         });
 }
 
@@ -1926,6 +2248,54 @@ bool ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(enum ggml_type src0_type
 
 #endif // GGML_SYCL_DMMV_HAS_ESIMD
 
+static void dequantize_mul_mat_vec_q4_K_sycl_reorder(const void *vx, const float *y,
+                                                     float *dst, const int ncols,
+                                                     const int nrows,
+                                                     dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q4_k_reorder(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q5_K_sycl_reorder(const void *vx, const float *y,
+                                                     float *dst, const int ncols,
+                                                     const int nrows,
+                                                     dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q5_k_reorder(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q6_K_sycl_reorder(const void *vx, const float *y,
+                                                     float *dst, const int ncols,
+                                                     const int nrows,
+                                                     dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q6_k_reorder(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
 void ggml_sycl_op_dequantize_mul_mat_vec(
     ggml_backend_sycl_context & ctx,
     const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst,
@@ -1974,27 +2344,27 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
         case GGML_TYPE_Q1_0:
             if ((ggml_tensor_extra_gpu*)dst->src[0]->extra &&
                 ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) {
-                launch_dequantize_mul_mat_vec_reorder<QK1_0, QR1_0, QK1_0/8, dequantize_q1_0_reorder>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q1_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             } else {
-                launch_dequantize_mul_mat_vec<QK1_0, QR1_0, dequantize_q1_0>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q1_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_Q4_0:
             if ((ggml_tensor_extra_gpu*)dst->src[0]->extra &&
                 ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) {
-                launch_dequantize_mul_mat_vec_reorder<QK4_0, QR4_0, QK4_0/2, dequantize_q4_0_reorder>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q4_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             } else {
-                launch_dequantize_mul_mat_vec<QK4_0, QR4_0, dequantize_q4_0>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q4_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_Q4_1:
-            launch_dequantize_mul_mat_vec<QK4_1, QR4_1, dequantize_q4_1>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+            dequantize_mul_mat_vec_q4_1_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             break;
         case GGML_TYPE_Q5_0:
-            launch_dequantize_mul_mat_vec<QK5_0, QR5_0, dequantize_q5_0>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+            dequantize_mul_mat_vec_q5_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             break;
         case GGML_TYPE_Q5_1:
-            launch_dequantize_mul_mat_vec<QK5_1, QR5_1, dequantize_q5_1>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+            dequantize_mul_mat_vec_q5_1_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             break;
         case GGML_TYPE_Q8_0:
             if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
@@ -2005,10 +2375,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
                 } else
 #endif
                 {
-                    launch_dequantize_mul_mat_vec_reorder<QK8_0, QR8_0, QK8_0, dequantize_q8_0_reorder>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+                    dequantize_mul_mat_vec_q8_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
                 }
             } else {
-                launch_dequantize_mul_mat_vec<QK8_0, QR8_0, dequantize_q8_0>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q8_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_Q2_K:
@@ -2021,10 +2391,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
                 else
 #endif
                 {
-                    launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q2_k_reorder>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                    dequantize_mul_mat_vec_q2_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
                 }
             } else {
-                launch_dequantize_mul_mat_vec_k<2, dequantize_mul_mat_vec_q2_k>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q2_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_Q3_K:
@@ -2037,10 +2407,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
                 else
 #endif
                 {
-                    launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q3_k_reorder>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                    dequantize_mul_mat_vec_q3_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
                 }
             } else {
-                launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q3_k>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q3_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_Q4_K:
@@ -2053,10 +2423,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
                 else
 #endif
                 {
-                    launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q4_k_reorder>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                    dequantize_mul_mat_vec_q4_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
                 }
             } else {
-                launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q4_k>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q4_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_Q5_K:
@@ -2069,10 +2439,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
                 else
 #endif
                 {
-                    launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q5_k_reorder>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                    dequantize_mul_mat_vec_q5_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
                 }
             } else {
-                launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q5_k>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q5_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_Q6_K:
@@ -2085,10 +2455,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
                 else
 #endif
                 {
-                    launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q6_k_reorder>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                    dequantize_mul_mat_vec_q6_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
                 }
             } else {
-                launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q6_k>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q6_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_F16:

ggml/src/ggml-sycl/ggml-sycl.cpp

@@ -7653,8 +7653,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
         case GGML_OP_SOLVE_TRI:
             return op->src[0]->ne[0] <= SYCL_SOLVE_TRI_MAX_N && op->src[1]->ne[0] <= SYCL_SOLVE_TRI_MAX_K;
         case GGML_OP_FLASH_ATTN_EXT:
-            // K/V row lists (ggml_flash_attn_ext_rows) are not supported
-            return op->src[5] == nullptr && ggml_sycl_flash_attn_ext_supported(device, op);
+            return ggml_sycl_flash_attn_ext_supported(device, op);
         default:
             return false;
     }
@@ -7830,15 +7829,9 @@ struct ggml_backend_sycl_comm_context {
 
     // pinned host staging per device context, reused across calls (queues are in-order)
     uint8_t * host_out[2] = { nullptr, nullptr };
-    uint8_t * host_in[2]  = { nullptr, nullptr };  // only used by the 4-buffer path (separate SYCL contexts)
+    uint8_t * host_in[2]  = { nullptr, nullptr };
     size_t    host_bytes  = 0;
 
-    // SYCL_ENABLE_DEFAULT_CONTEXTS=1 sets comm_exchange_fast on Linux - PR #30956
-    using exchange_fn_t = void (*)(ggml_backend_sycl_comm_context * comm, queue_ptr q0, queue_ptr q1,
-                                   void * dst0, const void * src1, void * dst1, const void * src0,
-                                   size_t nbytes);
-    exchange_fn_t comm_exchange = nullptr; // Moves same context check out of hot path; tests only once
-
     void free_host() {
         for (int i = 0; i < 2; ++i) {
             const sycl::context sctx = ((ggml_backend_sycl_context *) backends[i]->context)->stream()->get_context();
@@ -7849,41 +7842,6 @@ struct ggml_backend_sycl_comm_context {
     }
 };
 
-// dst0 <- src1 and dst1 <- src0 through pinned host memory.
-static void ggml_sycl_comm_exchange_fast(ggml_backend_sycl_comm_context * comm, queue_ptr q0, queue_ptr q1,
-                                         void * dst0, const void * src1, void * dst1, const void * src0,
-                                         size_t nbytes) {
-
-    // Increases Linux tensor split prompt eval speeds on par with layer split speeds
-    // Reallocate host buffers if they are too small
-    if (comm->host_bytes < nbytes) {
-        q0->wait();
-        q1->wait();
-        comm->free_host();
-        comm->host_out[0] = sycl::malloc_host<uint8_t>(nbytes, q0->get_context());
-        comm->host_out[1] = sycl::malloc_host<uint8_t>(nbytes, q1->get_context());
-
-        GGML_ASSERT(comm->host_out[0] && comm->host_out[1]);
-        comm->host_bytes = nbytes;
-    }
-
-    // Step 1: Concurrently copy from GPUs into host buffers (Asynchronous)
-    sycl::event e_d2h_0 = q0->memcpy(comm->host_out[0], src0, nbytes);
-    sycl::event e_d2h_1 = q1->memcpy(comm->host_out[1], src1, nbytes);
-
-    // Step 2: Asynchronous Barrier Interlocking (NO HOST BLOCKING), wait() causes MTP conflicts
-    q0->ext_oneapi_submit_barrier({e_d2h_1}); // q0 waits for q1's device-to-host copy
-    q1->ext_oneapi_submit_barrier({e_d2h_0}); // q1 waits for q0's device-to-host copy
-
-    // Step 3: Cross-upload directly from the opposing host buffer.
-    sycl::event e_h2d_0 = q0->memcpy(dst0, comm->host_out[1], nbytes); // q0 pulls from host_out[1]
-    sycl::event e_h2d_1 = q1->memcpy(dst1, comm->host_out[0], nbytes); // q1 pulls from host_out[0]
-
-    // Step 4: Finalize the exchange before returning to host code
-    e_h2d_0.wait();
-    e_h2d_1.wait();
-}
-
 // dst0 <- src1 and dst1 <- src0 through pinned host memory, with a single host wait
 static void ggml_sycl_comm_exchange(ggml_backend_sycl_comm_context * comm, queue_ptr q0, queue_ptr q1,
                                     void * dst0, const void * src1, void * dst1, const void * src0,
@@ -7929,15 +7887,6 @@ void * ggml_backend_sycl_comm_init(ggml_backend_t * backends, size_t n_backends)
     auto * sctx1 = (ggml_backend_sycl_context *) backends[1]->context;
     ctx->buf0 = std::make_unique<ggml_sycl_pool_alloc<uint8_t>>(sctx0->pool());
     ctx->buf1 = std::make_unique<ggml_sycl_pool_alloc<uint8_t>>(sctx1->pool());
-
-#if defined(_WIN32) || defined(_WIN64) // Not supported on Windows
-    ctx->comm_exchange = ggml_sycl_comm_exchange;
-#else
-    // SYCL_ENABLE_DEFAULT_CONTEXTS=1 sets comm_exchange_fast on Linux - PR #30956
-    ctx->comm_exchange = (sctx0->stream()->get_context() == sctx1->stream()->get_context())
-                       ? ggml_sycl_comm_exchange_fast
-                       : ggml_sycl_comm_exchange;
-#endif
     return ctx;
 }
 catch (const sycl::exception &) { return nullptr; }
@@ -8021,7 +7970,7 @@ bool ggml_backend_sycl_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tens
         sycl::half * f16_tmp0 = (sycl::half *) buf0;
         sycl::half * f16_tmp1 = (sycl::half *) buf1;
 
-        comm_ctx->comm_exchange(comm_ctx, q0, q1, f16_tmp0, tensors[1]->data, f16_tmp1, tensors[0]->data, nbytes);
+        ggml_sycl_comm_exchange(comm_ctx, q0, q1, f16_tmp0, tensors[1]->data, f16_tmp1, tensors[0]->data, nbytes);
 
         q0->submit([&](sycl::handler & h) {
             h.parallel_for(sycl::range<1>(nelem), [=](sycl::id<1> i) {
@@ -8051,7 +8000,7 @@ bool ggml_backend_sycl_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tens
         float * tmp1 = (float *) buf1;
 
         // separate SYCL contexts: a raw peer-USM q->memcpy is a silent no-op, so stage through host memory
-        comm_ctx->comm_exchange(comm_ctx, q0, q1, tmp0, tensors[1]->data, tmp1, tensors[0]->data, nbytes);
+        ggml_sycl_comm_exchange(comm_ctx, q0, q1, tmp0, tensors[1]->data, tmp1, tensors[0]->data, nbytes);
 
         q0->submit([&](sycl::handler & h) {
             h.parallel_for(sycl::range<1>(nelem), [=](sycl::id<1> i) {
@@ -8085,7 +8034,7 @@ bool ggml_backend_sycl_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tens
 
     // Phase B: exchange the compressed bytes (in-order queues keep this after the compress kernels).
     const size_t bf16_bytes = nelem * sizeof(uint16_t);
-    comm_ctx->comm_exchange(comm_ctx, q0, q1, inbox0, outbox1, inbox1, outbox0, bf16_bytes);
+    ggml_sycl_comm_exchange(comm_ctx, q0, q1, inbox0, outbox1, inbox1, outbox0, bf16_bytes);
 
     // Phase C: decompress + add into local FP32 partial.
     q0->submit([&](sycl::handler & h) {

ggml/src/ggml-vulkan/CMakeLists.txt

@@ -66,7 +66,6 @@ if (Vulkan_FOUND)
                              ggml-vulkan-push-constants.h
                              ggml-vulkan-common.h
                              ggml-vulkan-buffers.cpp
-                             ggml-vulkan-comm.cpp
                              ggml-vulkan-debug.cpp
                             )
 

ggml/src/ggml-vulkan/ggml-vulkan-comm.cpp

@@ -1,591 +0,0 @@
-#include "ggml-vulkan-common.h"
-
-#if !defined(_WIN32)
-#include <fcntl.h>
-#include <unistd.h>
-#if defined(__linux__)
-#include <pthread.h>
-#endif
-
-static constexpr size_t ggml_vk_comm_f16_min_size = 1024 * 1024;
-
-// follows ops ggml-backend-meta may delay a reduction across
-// TODO: the required precision should come from the meta backend instead.
-static bool ggml_vk_comm_needs_f32(const ggml_tensor * t, int depth = 0) {
-    if (t == nullptr || depth > 64) {
-        return false;
-    }
-    switch (t->op) {
-        case GGML_OP_MUL_MAT:
-        case GGML_OP_MUL_MAT_ID:
-            return ggml_get_op_params_i32(t, 0) == GGML_PREC_F32;
-        case GGML_OP_ADD:
-            return ggml_vk_comm_needs_f32(t->src[0], depth + 1) || ggml_vk_comm_needs_f32(t->src[1], depth + 1);
-        case GGML_OP_MUL:
-        case GGML_OP_ADD_ID:
-            return ggml_vk_comm_needs_f32(t->src[0], depth + 1);
-        case GGML_OP_VIEW:
-            return ggml_vk_comm_needs_f32(t->src[0], depth + 1);
-        default:
-            return false;
-    }
-}
-
-// host_bridge: device i signals done[i], a host thread signals go[i] once all peers of i are done
-// sync_fd: a host thread exchanges SYNC_FD semaphores between peers, the wait resolves in the kernel
-enum class vk_comm_sync {
-    host_bridge,
-    sync_fd,
-};
-
-struct vk_comm_device {
-    ggml_backend_t                     backend  = nullptr;
-    ggml_backend_vk_context *          ctx      = nullptr;
-    void *                             host_ptr = nullptr;
-    std::vector<ggml_backend_buffer_t> host_buf; // host_ptr imported into each peer
-
-    struct {
-        PFN_vkGetSemaphoreFdKHR    get_fd    = nullptr;
-        PFN_vkImportSemaphoreFdKHR import_fd = nullptr;
-        vk::Semaphore              sem;
-        std::vector<vk::Semaphore> imp;      // [peer], pooled, reused only after ggml_vk_graph_cleanup
-    } sfd;
-
-    struct {
-        vk::Semaphore done;
-        vk::Semaphore go;
-    } bridge;
-};
-
-struct ggml_backend_vk_comm_context {
-    std::vector<vk_comm_device> devs;
-    vk_comm_sync                mode     = vk_comm_sync::host_bridge;
-    size_t                      align    = 4096;
-    size_t                      cap      = 0;
-    uint64_t                    round    = 0;
-    ggml_context *              tctx     = nullptr;
-    ggml_tensor *               slot     = nullptr;
-    bool                        disabled = false;
-
-    std::thread                 thread;
-    std::chrono::nanoseconds    spin{ 200'000 };
-    std::atomic<uint64_t>       submitted{ 0 };
-    std::atomic<uint64_t>       imported{ 0 };
-    std::atomic<bool>           sleeping{ false };
-    std::atomic<bool>           stop{ false };
-    std::mutex                  mutex;
-    std::condition_variable     cv;
-};
-
-static void ggml_vk_comm_free_buffers(ggml_backend_vk_comm_context * comm) {
-    for (vk_comm_device & d : comm->devs) {
-        for (ggml_backend_buffer_t & buf : d.host_buf) {
-            ggml_backend_buffer_free(buf);
-            buf = nullptr;
-        }
-        std::free(d.host_ptr);
-        d.host_ptr = nullptr;
-    }
-    comm->cap = 0;
-}
-
-static bool ggml_backend_vk_comm_ensure(ggml_backend_vk_comm_context * comm, size_t nbytes) {
-    if (comm->disabled) {
-        return false;
-    }
-    if (nbytes <= comm->cap) {
-        return true;
-    }
-    auto fail = [comm]() {
-        GGML_LOG_WARN("ggml_vulkan: shared host buffer allocation failed, using generic all-reduce\n");
-        comm->disabled = true;
-        return false;
-    };
-    for (vk_comm_device & d : comm->devs) {
-        ggml_backend_synchronize(d.backend);
-    }
-    ggml_vk_comm_free_buffers(comm);
-
-    const size_t n   = comm->devs.size();
-    const size_t cap = GGML_PAD(nbytes, comm->align);
-    for (size_t i = 0; i < n; i++) {
-        vk_comm_device & d = comm->devs[i];
-        if (posix_memalign(&d.host_ptr, comm->align, 2 * cap) != 0) {
-            d.host_ptr = nullptr;
-            return fail();
-        }
-        for (size_t j = 0; j < n; j++) {
-            d.host_buf[j] = ggml_backend_vk_device_buffer_from_host_ptr(
-                ggml_backend_get_device(comm->devs[j].backend), d.host_ptr, 2 * cap, 2 * cap);
-            if (!d.host_buf[j]) {
-                return fail();
-            }
-        }
-    }
-    comm->cap = cap;
-    return true;
-}
-
-static void ggml_vk_comm_set_slot(ggml_tensor * slot, const ggml_tensor * src, ggml_type type, ggml_backend_buffer_t buf, size_t offs) {
-    slot->type  = type;
-    slot->nb[0] = ggml_type_size(type);
-    for (int d = 0; d < GGML_MAX_DIMS; d++) {
-        slot->ne[d] = src->ne[d];
-        if (d > 0) {
-            slot->nb[d] = slot->nb[d - 1] * slot->ne[d - 1];
-        }
-    }
-    slot->buffer = buf;
-    slot->data   = (char *) ggml_backend_buffer_get_base(buf) + offs;
-}
-
-static void ggml_vk_comm_submit(ggml_backend_vk_context * ctx, vk_context & c, vk::Semaphore signal, uint64_t value) {
-    ggml_vk_ctx_end(c);
-    for (auto & cpy : c->in_memcpys) {
-        memcpy(cpy.dst, cpy.src, cpy.n);
-    }
-    for (auto & mset : c->memsets) {
-        memset(mset.dst, mset.val, mset.n);
-    }
-    c->in_memcpys.clear();
-    c->memsets.clear();
-    if (signal) {
-        c->seqs.back().back().signal_semaphores.push_back({ signal, value });
-    }
-    ggml_vk_submit(c, {});
-    ctx->submit_pending = true;
-    ctx->compute_ctx.reset();
-}
-
-static void ggml_vk_comm_notify_submitted(ggml_backend_vk_comm_context * comm, uint64_t round) {
-    comm->submitted.store(round);
-    if (comm->sleeping.load()) {
-        std::lock_guard<std::mutex> lock(comm->mutex);
-        comm->cv.notify_one();
-    }
-}
-
-// false on stop
-static bool ggml_vk_comm_wait_submitted(ggml_backend_vk_comm_context * comm, uint64_t r) {
-    using clock = std::chrono::steady_clock;
-    for (const auto t0 = clock::now(); comm->submitted.load() < r;) {
-        if (comm->stop.load()) {
-            return false;
-        }
-        if (clock::now() - t0 < comm->spin) {
-            YIELD();
-            continue;
-        }
-        comm->sleeping.store(true);
-        if (comm->submitted.load() < r) {
-            std::unique_lock<std::mutex> lock(comm->mutex);
-            comm->cv.wait(lock, [&] { return comm->submitted.load() >= r || comm->stop.load(); });
-        }
-        comm->sleeping.store(false);
-    }
-    return true;
-}
-
-// signals go[i] once all peers of i are done; false on stop
-static bool ggml_vk_comm_bridge_release(ggml_backend_vk_comm_context * comm, uint64_t r, std::vector<uint64_t> & seen, std::vector<bool> & sent) {
-    using clock = std::chrono::steady_clock;
-    const size_t n = comm->devs.size();
-    std::fill(sent.begin(), sent.end(), false);
-    size_t left = n;
-    for (const auto t0 = clock::now();;) {
-        size_t pending = n;
-        for (size_t i = 0; i < n; i++) {
-            if (sent[i]) {
-                continue;
-            }
-            bool ready = true;
-            for (size_t j = 0; j < n && ready; j++) {
-                if (j != i && seen[j] < r) {
-                    const vk_comm_device & p = comm->devs[j];
-                    seen[j] = p.ctx->device->device.getSemaphoreCounterValue(p.bridge.done);
-                    ready   = seen[j] >= r;
-                    pending = ready ? pending : j;
-                }
-            }
-            if (ready) {
-                const vk_comm_device & d = comm->devs[i];
-                d.ctx->device->device.signalSemaphore({ d.bridge.go, r });
-                sent[i] = true;
-                left--;
-            }
-        }
-        if (left == 0) {
-            return true;
-        }
-        if (clock::now() - t0 < comm->spin) {
-            YIELD();
-            continue;
-        }
-        const vk_comm_device &      p = comm->devs[pending];
-        const vk::SemaphoreWaitInfo wi{ {}, 1, &p.bridge.done, &r };
-        while (p.ctx->device->device.waitSemaphores(wi, 100'000'000) != vk::Result::eSuccess) {
-            if (comm->stop.load()) {
-                return false;
-            }
-        }
-    }
-}
-
-static void ggml_vk_comm_bridge_thread(ggml_backend_vk_comm_context * comm) {
-#if defined(__linux__)
-    pthread_setname_np(pthread_self(), "ggml-vk-comm");
-#endif
-    std::vector<uint64_t> seen(comm->devs.size(), 0);
-    std::vector<bool>     sent(comm->devs.size());
-    try {
-        for (uint64_t r = 1;; r++) {
-            if (!ggml_vk_comm_wait_submitted(comm, r) || !ggml_vk_comm_bridge_release(comm, r, seen, sent)) {
-                return;
-            }
-        }
-    } catch (const vk::SystemError & e) {
-        GGML_ABORT("ggml_vulkan: all-reduce bridge failed: %s", e.what());
-    }
-}
-
-static void ggml_vk_comm_bridge_init(ggml_backend_vk_comm_context * comm) {
-    vk::SemaphoreTypeCreateInfo stci{ vk::SemaphoreType::eTimeline, 0 };
-    vk::SemaphoreCreateInfo     sci{};
-    sci.pNext = &stci;
-    for (vk_comm_device & d : comm->devs) {
-        d.bridge.done = d.ctx->device->device.createSemaphore(sci);
-        d.bridge.go   = d.ctx->device->device.createSemaphore(sci);
-    }
-    comm->thread = std::thread(ggml_vk_comm_bridge_thread, comm);
-}
-
-static void ggml_vk_comm_bridge_free(vk_comm_device & d) {
-    if (d.bridge.done) {
-        d.ctx->device->device.destroySemaphore(d.bridge.done);
-    }
-    if (d.bridge.go) {
-        d.ctx->device->device.destroySemaphore(d.bridge.go);
-    }
-}
-
-static void ggml_vk_comm_barrier_to_host(vk_context & c) {
-    const vk::MemoryBarrier to_host{ vk::AccessFlagBits::eTransferWrite | vk::AccessFlagBits::eShaderWrite, vk::AccessFlagBits::eHostRead };
-    c->s->buffer->buf.pipelineBarrier(c->p->q->stage_flags, vk::PipelineStageFlagBits::eHost, {}, { to_host }, {}, {});
-}
-
-// the peers' slots were made available to the host before the wait resolved, the submit predates that
-static void ggml_vk_comm_barrier_from_host(vk_context & c) {
-    const vk::MemoryBarrier from_host{ vk::AccessFlagBits::eHostWrite, vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eTransferRead };
-    c->s->buffer->buf.pipelineBarrier(vk::PipelineStageFlagBits::eHost, c->p->q->stage_flags, {}, { from_host }, {}, {});
-}
-
-static void ggml_vk_comm_bridge_signal(ggml_backend_vk_comm_context * comm, size_t i, vk_context & c, uint64_t round) {
-    const vk_comm_device & d = comm->devs[i];
-    ggml_vk_comm_barrier_to_host(c);
-    ggml_vk_comm_submit(d.ctx, c, d.bridge.done, round);
-}
-
-static void ggml_vk_comm_bridge_wait(ggml_backend_vk_comm_context * comm, size_t i, vk_context & c, uint64_t round) {
-    c->s->wait_semaphores.push_back({ comm->devs[i].bridge.go, round });
-    ggml_vk_comm_barrier_from_host(c);
-}
-
-static bool ggml_vk_comm_sfd_supported(vk_comm_device & d) {
-    const vk_device & dev = d.ctx->device;
-    const bool mesa = dev->driver_id == vk::DriverId::eMesaRadv ||
-                      dev->driver_id == vk::DriverId::eIntelOpenSourceMESA ||
-                      dev->driver_id == vk::DriverId::eMesaNvk;
-    if (!mesa || !dev->external_semaphore) {
-        return false;
-    }
-    d.sfd.get_fd    = (PFN_vkGetSemaphoreFdKHR)    dev->device.getProcAddr("vkGetSemaphoreFdKHR");
-    d.sfd.import_fd = (PFN_vkImportSemaphoreFdKHR) dev->device.getProcAddr("vkImportSemaphoreFdKHR");
-    if (!d.sfd.get_fd || !d.sfd.import_fd) {
-        return false;
-    }
-    vk::PhysicalDeviceExternalSemaphoreInfo esi{ vk::ExternalSemaphoreHandleTypeFlagBits::eSyncFd };
-    vk::ExternalSemaphoreProperties esp = dev->physical_device.getExternalSemaphoreProperties(esi);
-    const vk::ExternalSemaphoreFeatureFlags need =
-        vk::ExternalSemaphoreFeatureFlagBits::eExportable | vk::ExternalSemaphoreFeatureFlagBits::eImportable;
-    return (esp.externalSemaphoreFeatures & need) == need &&
-           (esp.compatibleHandleTypes & vk::ExternalSemaphoreHandleTypeFlagBits::eSyncFd);
-}
-
-// Mesa moves a queue to its submit thread on the first wait-before-signal, taking the CS ioctls off this thread
-static void ggml_vk_comm_sfd_enable_submit_thread(const vk_device & dev) {
-    vk::SemaphoreTypeCreateInfo stci{ vk::SemaphoreType::eTimeline, 0 };
-    vk::SemaphoreCreateInfo     sci{};
-    sci.pNext = &stci;
-    const vk::Semaphore sem   = dev->device.createSemaphore(sci);
-    const vk::Fence     fence = dev->device.createFence({});
-    const uint64_t      one   = 1;
-    vk::TimelineSemaphoreSubmitInfo tsi{ 1, &one, 0, nullptr };
-    const vk::PipelineStageFlags stage = vk::PipelineStageFlagBits::eAllCommands;
-    vk::SubmitInfo si{ 1, &sem, &stage, 0, nullptr, 0, nullptr };
-    si.setPNext(&tsi);
-    dev->compute_queue->handle->submit(si, fence);
-    dev->device.signalSemaphore({ sem, 1 });
-    VK_CHECK(dev->device.waitForFences({ fence }, true, UINT64_MAX), "ggml_vk_comm_sfd_enable_submit_thread", dev);
-    dev->device.destroyFence(fence);
-    dev->device.destroySemaphore(sem);
-}
-
-// Requires a submitted signal on sem; the export resets it.
-static int ggml_vk_comm_sfd_export(vk_comm_device & d) {
-    int fd = -1;
-    VkSemaphoreGetFdInfoKHR gi{};
-    gi.sType      = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR;
-    gi.semaphore  = d.sfd.sem;
-    gi.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
-    const VkResult result = d.sfd.get_fd((VkDevice) d.ctx->device->device, &gi, &fd);
-    if (result != VK_SUCCESS) {
-        GGML_ABORT("ggml_vulkan: vkGetSemaphoreFdKHR failed: %d", (int) result);
-    }
-    return fd;
-}
-
-// fd -1 imports an already signalled payload
-static void ggml_vk_comm_sfd_import(vk_comm_device & d, vk::Semaphore sem, int fd) {
-    VkImportSemaphoreFdInfoKHR isi{};
-    isi.sType      = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR;
-    isi.semaphore  = sem;
-    isi.flags      = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT;
-    isi.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
-    isi.fd         = fd;
-    const VkResult result = d.sfd.import_fd((VkDevice) d.ctx->device->device, &isi);
-    if (result != VK_SUCCESS) {
-        GGML_ABORT("ggml_vulkan: vkImportSemaphoreFdKHR failed: %d", (int) result);
-    }
-}
-
-static void ggml_vk_comm_sfd_exchange(ggml_backend_vk_comm_context * comm, uint64_t r) {
-    const size_t n = comm->devs.size();
-    for (size_t j = 0; j < n; j++) {
-        const int    fd   = ggml_vk_comm_sfd_export(comm->devs[j]);
-        const size_t last = j == n - 1 ? n - 2 : n - 1;
-        for (size_t i = 0; i < n; i++) {
-            if (i == j) {
-                continue;
-            }
-            const int imp_fd = fd < 0 || i == last ? fd : fcntl(fd, F_DUPFD_CLOEXEC, 0);
-            GGML_ASSERT(imp_fd >= 0 || fd < 0);
-            ggml_vk_comm_sfd_import(comm->devs[i], comm->devs[i].sfd.imp[j], imp_fd);
-        }
-    }
-    comm->imported.store(r, std::memory_order_release);
-}
-
-static void ggml_vk_comm_sfd_thread(ggml_backend_vk_comm_context * comm) {
-#if defined(__linux__)
-    pthread_setname_np(pthread_self(), "ggml-vk-comm");
-#endif
-    for (uint64_t r = 1; ggml_vk_comm_wait_submitted(comm, r); r++) {
-        ggml_vk_comm_sfd_exchange(comm, r);
-    }
-}
-
-static void ggml_vk_comm_sfd_init(ggml_backend_vk_comm_context * comm) {
-    vk::ExportSemaphoreCreateInfo esci{ vk::ExternalSemaphoreHandleTypeFlagBits::eSyncFd };
-    vk::SemaphoreCreateInfo       sci{};
-    sci.pNext = &esci;
-    for (vk_comm_device & d : comm->devs) {
-        d.sfd.sem = d.ctx->device->device.createSemaphore(sci);
-        d.sfd.imp.assign(comm->devs.size(), {});
-        ggml_vk_comm_sfd_enable_submit_thread(d.ctx->device);
-    }
-    comm->thread = std::thread(ggml_vk_comm_sfd_thread, comm);
-}
-
-static void ggml_vk_comm_sfd_free(vk_comm_device & d) {
-    if (d.sfd.sem) {
-        d.ctx->device->device.destroySemaphore(d.sfd.sem);
-    }
-}
-
-static void ggml_vk_comm_sfd_signal(ggml_backend_vk_comm_context * comm, size_t i, vk_context & c) {
-    vk_comm_device & d = comm->devs[i];
-    ggml_vk_comm_barrier_to_host(c);
-    ggml_vk_comm_submit(d.ctx, c, d.sfd.sem, 0);
-    for (size_t j = 0; j < comm->devs.size(); j++) {
-        d.sfd.imp[j] = j == i ? vk::Semaphore{} : ggml_vk_create_binary_semaphore(d.ctx);
-    }
-}
-
-static void ggml_vk_comm_sfd_wait(ggml_backend_vk_comm_context * comm, size_t i, vk_context & c, uint64_t round) {
-    for (size_t j = 0; j < comm->devs.size(); j++) {
-        if (j != i) {
-            c->s->wait_semaphores.push_back({ comm->devs[i].sfd.imp[j], 0 });
-        }
-    }
-    c->s->ready       = &comm->imported;
-    c->s->ready_value = round;
-    ggml_vk_comm_barrier_from_host(c);
-}
-
-void ggml_backend_vk_comm_free(void * comm_ctx) {
-    ggml_backend_vk_comm_context * comm = static_cast<ggml_backend_vk_comm_context *>(comm_ctx);
-    // pending reductions need the thread
-    for (vk_comm_device & d : comm->devs) {
-        ggml_backend_synchronize(d.backend);
-    }
-    if (comm->thread.joinable()) {
-        {
-            std::lock_guard<std::mutex> lock(comm->mutex);
-            comm->stop.store(true);
-        }
-        comm->cv.notify_all();
-        comm->thread.join();
-    }
-    for (vk_comm_device & d : comm->devs) {
-        ggml_vk_comm_sfd_free(d);
-        ggml_vk_comm_bridge_free(d);
-    }
-    ggml_vk_comm_free_buffers(comm);
-    if (comm->tctx) {
-        ggml_free(comm->tctx);
-    }
-    delete comm;
-}
-
-void * ggml_backend_vk_comm_init(ggml_backend_t * backends, size_t n_backends) {
-    if (vk_instance.allreduce_disabled || n_backends < 2) {
-        return nullptr;
-    }
-    for (size_t i = 0; i < n_backends; i++) {
-        if (!ggml_backend_is_vk(backends[i])) {
-            return nullptr;
-        }
-    }
-    ggml_backend_vk_comm_context * comm = new ggml_backend_vk_comm_context;
-    comm->devs.resize(n_backends);
-    for (size_t i = 0; i < n_backends; i++) {
-        vk_comm_device & d = comm->devs[i];
-        d.backend = backends[i];
-        d.ctx     = (ggml_backend_vk_context *) backends[i]->context;
-        d.host_buf.assign(n_backends, nullptr);
-    }
-
-    for (size_t i = 0; i < n_backends; i++) {
-        const vk_device & dev = comm->devs[i].ctx->device;
-        comm->align = std::max(comm->align, (size_t) dev->min_imported_host_pointer_alignment);
-        if (!dev->external_memory_host) {
-            GGML_LOG_WARN("ggml_vulkan: device %zu: VK_EXT_external_memory_host missing, using generic all-reduce\n", i);
-            ggml_backend_vk_comm_free(comm);
-            return nullptr;
-        }
-        if (dev->serialize_submissions) {
-            GGML_LOG_WARN("ggml_vulkan: device %zu: GGML_VK_SERIALIZE_SUBMISSIONS set, using generic all-reduce\n", i);
-            ggml_backend_vk_comm_free(comm);
-            return nullptr;
-        }
-    }
-
-    if (std::all_of(comm->devs.begin(), comm->devs.end(), ggml_vk_comm_sfd_supported)) {
-        comm->mode = vk_comm_sync::sync_fd;
-    }
-
-    if (const char * spin = getenv("GGML_VK_ALLREDUCE_SPIN_US")) {
-        comm->spin = std::chrono::microseconds(std::max(0, atoi(spin)));
-    }
-    const ggml_init_params ip = { ggml_tensor_overhead(), nullptr, true };
-    comm->tctx = ggml_init(ip);
-    comm->slot = ggml_new_tensor_1d(comm->tctx, GGML_TYPE_F32, 1);
-
-    try {
-        if (comm->mode == vk_comm_sync::sync_fd) {
-            ggml_vk_comm_sfd_init(comm);
-        } else {
-            ggml_vk_comm_bridge_init(comm);
-        }
-    } catch (const vk::SystemError & e) {
-        GGML_LOG_WARN("ggml_vulkan: semaphore initialization failed: %s, using generic all-reduce\n", e.what());
-        ggml_backend_vk_comm_free(comm);
-        return nullptr;
-    }
-    return comm;
-}
-
-// A: partial to own host slot (round & 1), signal; B: wait on peers' A, add their slots; all A precede any B
-// slot reuse: A(k) follows own B(k-1), which waited on peers' A(k-1), submitted after their B(k-2), the last reader
-bool ggml_backend_vk_comm_allreduce_tensor(void * comm_ctx, ggml_tensor ** tensors) {
-    ggml_backend_vk_comm_context * comm = static_cast<ggml_backend_vk_comm_context *>(comm_ctx);
-    const size_t n = comm->devs.size();
-
-    const int64_t ne = ggml_nelements(tensors[0]);
-    for (size_t i = 0; i < n; i++) {
-        const ggml_tensor * t = tensors[i];
-        if (t->type != GGML_TYPE_F32 || ggml_nelements(t) != ne || !ggml_is_contiguous(t) || !ggml_backend_buffer_is_vk(t->buffer)) {
-            return false;
-        }
-        const ggml_backend_vk_context * ctx = comm->devs[i].ctx;
-        if (!ctx->compute_ctx.expired() && !ctx->compute_ctx.lock()->out_memcpys.empty()) {
-            return false;
-        }
-    }
-    if (ne == 0) {
-        return true;
-    }
-    const size_t nbytes = ggml_nbytes(tensors[0]);
-    if (!ggml_backend_vk_comm_ensure(comm, nbytes)) {
-        return false;
-    }
-
-    const bool      f32_wire  = nbytes < ggml_vk_comm_f16_min_size || ggml_vk_comm_needs_f32(tensors[0]);
-    const ggml_type wire      = f32_wire ? GGML_TYPE_F32 : GGML_TYPE_F16;
-    const uint64_t  round     = ++comm->round;
-    const size_t    slot_offs = (round & 1) * comm->cap;
-    const size_t    slot_size = ne * ggml_type_size(wire);
-
-    for (size_t i = 0; i < n; i++) {
-        const vk_comm_device &    d   = comm->devs[i];
-        ggml_backend_vk_context * ctx = d.ctx;
-        ggml_tensor *             t   = tensors[i];
-        vk_buffer & tbuf = ((ggml_backend_vk_buffer_context *) t->buffer->context)->dev_buffer;
-        vk_buffer & hbuf = ((ggml_backend_vk_buffer_context *) d.host_buf[i]->context)->dev_buffer;
-        const size_t toffs = vk_tensor_offset(t) + t->view_offs;
-
-        vk_context c = ggml_vk_get_compute_ctx(ctx);
-        ggml_vk_sync_buffers(ctx, c);
-        if (!(t->flags & GGML_TENSOR_FLAG_COMPUTE)) {
-            c->s->buffer->buf.fillBuffer(hbuf->buffer, slot_offs, GGML_PAD(slot_size, 4), 0);
-            c->s->buffer->buf.fillBuffer(tbuf->buffer, toffs, nbytes, 0);
-        } else if (wire == GGML_TYPE_F32) {
-            ggml_vk_buffer_copy_async(c, hbuf, slot_offs, tbuf, toffs, nbytes);
-        } else {
-            ggml_vk_comm_set_slot(comm->slot, t, wire, d.host_buf[i], slot_offs);
-            ggml_vk_cpy(ctx, c, t, comm->slot);
-        }
-        if (comm->mode == vk_comm_sync::sync_fd) {
-            ggml_vk_comm_sfd_signal(comm, i, c);
-        } else {
-            ggml_vk_comm_bridge_signal(comm, i, c, round);
-        }
-    }
-    ggml_vk_comm_notify_submitted(comm, round);
-
-    for (size_t i = 0; i < n; i++) {
-        ggml_backend_vk_context * ctx = comm->devs[i].ctx;
-        vk_context c = ggml_vk_get_compute_ctx(ctx);
-        ctx->do_add_rms_partials = false;
-        if (comm->mode == vk_comm_sync::sync_fd) {
-            ggml_vk_comm_sfd_wait(comm, i, c, round);
-        } else {
-            ggml_vk_comm_bridge_wait(comm, i, c, round);
-        }
-        ggml_vk_sync_buffers(ctx, c);
-        for (size_t j = 0; j < n; j++) {
-            if (j == i) {
-                continue;
-            }
-            ggml_vk_comm_set_slot(comm->slot, tensors[i], wire, comm->devs[j].host_buf[i], slot_offs);
-            ggml_vk_add(ctx, c, tensors[i], comm->slot, tensors[i]);
-            ggml_vk_sync_buffers(ctx, c);
-        }
-        // small reductions go out with the next graph's first submit
-        if (wire != GGML_TYPE_F32 || vk_perf_logger_enabled) {
-            ggml_vk_comm_submit(ctx, c, {}, 0);
-        }
-    }
-    return true;
-}
-#endif

ggml/src/ggml-vulkan/ggml-vulkan-common.h

@@ -86,7 +86,6 @@ void ggml_vk_cmd_label_begin(vk::CommandBuffer buf, const char * name);
 void ggml_vk_ctx_end(vk_context& ctx);
 void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx);
 vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx);
-vk::Semaphore ggml_vk_create_binary_semaphore(ggml_backend_vk_context * ctx);
 vk_context ggml_vk_get_transfer_ctx(ggml_backend_vk_context * ctx);
 bool ggml_vk_submit_transfer_ctx(ggml_backend_vk_context * ctx);
 size_t ggml_vk_align_size(size_t width, size_t align);
@@ -234,16 +233,8 @@ size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft
 size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft);
 size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor);
 void ggml_backend_vk_free(ggml_backend_t backend);
-ggml_backend_buffer_t ggml_backend_vk_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size);
 ggml_backend_reg_t ggml_backend_vk_reg();
 
-// comm
-#if !defined(_WIN32)
-void * ggml_backend_vk_comm_init(ggml_backend_t * backends, size_t n_backends);
-void ggml_backend_vk_comm_free(void * comm_ctx);
-bool ggml_backend_vk_comm_allreduce_tensor(void * comm_ctx, ggml_tensor ** tensors);
-#endif
-
 // debug
 int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op);
 #ifdef GGML_VULKAN_RUN_TESTS

ggml/src/ggml-vulkan/ggml-vulkan-types.h

@@ -679,7 +679,6 @@ struct vk_device_struct {
     uint64_t suballocation_block_size;
     uint64_t min_imported_host_pointer_alignment;
     bool external_memory_host {};
-    bool external_semaphore {};
     bool fp16;
     bool bf16;
     bool pipeline_robustness;
@@ -1088,9 +1087,6 @@ struct vk_submission {
     vk_command_buffer* buffer = nullptr;
     std::vector<vk_semaphore> wait_semaphores;
     std::vector<vk_semaphore> signal_semaphores;
-    // submit waits until *ready >= ready_value
-    const std::atomic<uint64_t> * ready = nullptr;
-    uint64_t ready_value = 0;
 };
 
 typedef std::vector<vk_submission> vk_sequence;
@@ -1229,7 +1225,7 @@ struct ggml_backend_vk_context {
 
     vk_device device;
 
-    size_t semaphore_idx, binary_semaphore_idx, event_idx;
+    size_t semaphore_idx, event_idx;
     ggml_vk_garbage_collector gc;
     size_t prealloc_size_x, prealloc_size_y, prealloc_size_split_k, prealloc_size_add_rms_partials, prealloc_size_add_rms_partials_offset;
     vk_buffer prealloc_x, prealloc_y, prealloc_split_k, prealloc_add_rms_partials, sync_staging;
@@ -1345,7 +1341,6 @@ struct vk_instance_t {
     uint32_t host_import_failed = 0;  // bitmask by device->idx, guarded by host_allocations_mutex
 
     bool multi_device = false;
-    bool allreduce_disabled = false;
 };
 
 typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);

ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -934,11 +934,6 @@ void ggml_vk_submit(vk_context& ctx, vk::Fence fence) {
 
     for (const auto& sequence : ctx->seqs) {
         for (const auto& submission : sequence) {
-            if (submission.ready) {
-                while (submission.ready->load(std::memory_order_acquire) < submission.ready_value) {
-                    YIELD();
-                }
-            }
             stage_flags.push_back({});
             idx++;
             tl_wait_vals.push_back({});
@@ -1087,16 +1082,14 @@ vk_context ggml_vk_create_temporary_context(vk_command_pool& p) {
     return result;
 }
 
-vk::Semaphore ggml_vk_create_binary_semaphore(ggml_backend_vk_context * ctx) {
-    VK_LOG_DEBUG("ggml_vk_create_binary_semaphore()");
-    if (ctx->binary_semaphore_idx >= ctx->gc.semaphores.size()) {
-        vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eBinary, 0 };
-        vk::SemaphoreCreateInfo ci{};
-        ci.setPNext(&tci);
-        vk::Semaphore semaphore = ctx->device->device.createSemaphore(ci);
-        ctx->gc.semaphores.push_back({ semaphore, 0 });
-    }
-    return ctx->gc.semaphores[ctx->binary_semaphore_idx++].s;
+static vk_semaphore * ggml_vk_create_binary_semaphore(ggml_backend_vk_context * ctx) {
+    VK_LOG_DEBUG("ggml_vk_create_timeline_semaphore()");
+    vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eBinary, 0 };
+    vk::SemaphoreCreateInfo ci{};
+    ci.setPNext(&tci);
+    vk::Semaphore semaphore = ctx->device->device.createSemaphore(ci);
+    ctx->gc.semaphores.push_back({ semaphore, 0 });
+    return &ctx->gc.semaphores[ctx->gc.semaphores.size() - 1];
 }
 
 static vk_semaphore * ggml_vk_create_timeline_semaphore(ggml_backend_vk_context * ctx) {
@@ -1457,8 +1450,8 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec
         lut_size = 4*16;
         break;
     case GGML_TYPE_NVFP4:
-        // Same kvalues budget as MXFP4 plus ue4m3_fp32_lut[256] (types.glsl, DATA_A_NVFP4).
-        lut_size = 4*16 + 256u * (uint32_t)sizeof(float);
+        // Same kvalues budget as MXFP4 plus ue4m3_fp32_lut[128] (types.glsl, DATA_A_NVFP4).
+        lut_size = 4*16 + 128u * (uint32_t)sizeof(float);
         break;
     default:
         break;
@@ -1592,7 +1585,7 @@ static bool ggml_vk_matmul_cm1_int_shmem_support(const vk_device& device, const
         total += 16u * (uint32_t)sizeof(int8_t);         // cm1_kvalues[16]
     }
     if (src0_type == GGML_TYPE_NVFP4 && !device->ocp_fp4) {
-        total += 256u * (uint32_t)sizeof(float);         // ue4m3_fp32_lut[256]
+        total += 128u * (uint32_t)sizeof(float);         // ue4m3_fp32_lut[128]
     }
     if (mul_mat_id) {
         total += BN * 2u * (uint32_t)sizeof(uint16_t);   // row_ids[BN] (u16vec2)
@@ -4135,8 +4128,6 @@ vk_device ggml_vk_get_device(size_t idx) {
                 device->memory_priority = true;
             } else if (strcmp("VK_EXT_external_memory_host", properties.extensionName) == 0) {
                 device->external_memory_host = true;
-            } else if (strcmp("VK_KHR_external_semaphore_fd", properties.extensionName) == 0) {
-                device->external_semaphore = true;
 #if defined(VK_EXT_shader_64bit_indexing)
             } else if (strcmp("VK_EXT_shader_64bit_indexing", properties.extensionName) == 0) {
                 device->shader_64b_indexing = true;
@@ -4484,9 +4475,6 @@ vk_device ggml_vk_get_device(size_t idx) {
         if (device->external_memory_host) {
             device_extensions.push_back("VK_EXT_external_memory_host");
         }
-        if (device->external_semaphore) {
-            device_extensions.push_back("VK_KHR_external_semaphore_fd");
-        }
 
 #if defined(VK_EXT_shader_64bit_indexing)
         VkPhysicalDeviceShader64BitIndexingFeaturesEXT shader_64bit_indexing_features {};
@@ -5299,7 +5287,6 @@ void ggml_vk_instance_init() {
     vk_perf_logger_concurrent = getenv("GGML_VK_PERF_LOGGER_CONCURRENT") != nullptr;
     vk_enable_sync_logger = getenv("GGML_VK_SYNC_LOGGER") != nullptr;
     vk_memory_logger_enabled = getenv("GGML_VK_MEMORY_LOGGER") != nullptr;
-    vk_instance.allreduce_disabled = getenv("GGML_VK_DISABLE_ALLREDUCE") != nullptr;
     const char* GGML_VK_PIPELINE_STATS = getenv("GGML_VK_PIPELINE_STATS");
     if (GGML_VK_PIPELINE_STATS != nullptr) {
         vk_pipeline_stats_filter = GGML_VK_PIPELINE_STATS;
@@ -5506,7 +5493,6 @@ void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) {
     ctx->device = ggml_vk_get_device(idx);
 
     ctx->semaphore_idx = 0;
-    ctx->binary_semaphore_idx = 0;
     ctx->event_idx = 0;
 
     ctx->prealloc_size_x = 0;
@@ -5934,18 +5920,8 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m,
         return 1;
     }
 
-    const bool rdna = ctx->device->architecture == AMD_RDNA3 || ctx->device->architecture == AMD_RDNA4;
     uint32_t split_k = 1;
-    if (rdna && ctx->device->shader_core_count != 0 && k >= 2048 && n <= 32) {
-        // Small n gives too few workgroups to hide latency, aim for 8 per CU. Small tiles are one subgroup each.
-        const uint32_t tiles = CEIL_DIV(m, pipeline->wg_denoms[0]) * CEIL_DIV(n, pipeline->wg_denoms[1]);
-        if (tiles < 4 * ctx->device->shader_core_count) {
-            split_k = std::min(8u, CEIL_DIV(8 * ctx->device->shader_core_count, tiles));
-        }
-    }
-
-    // On RDNA, also split when n <= 64 only partly fills one tile column.
-    if (split_k == 1 && ctx->device->shader_core_count != 0 && (n >= pipeline->wg_denoms[1] || (rdna && n <= 64))) {
+    if (ctx->device->shader_core_count != 0 && n >= pipeline->wg_denoms[1]) {
         // If k is 'large' and the SMs will fill less than halfway, use split_k.
         uint32_t m_tiles = CEIL_DIV(m, pipeline->wg_denoms[0]);
         uint32_t n_tiles = CEIL_DIV(n, pipeline->wg_denoms[1]);
@@ -5958,22 +5934,22 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m,
             }
             // Cap the split at 8x. Unless k is huge this is a lot of overhead.
             split_k = std::min(split_k, 8u);
-        }
-    }
 
-    // ggml_vk_matmul will align the splits to be a multiple of 256.
-    // If this rounded up size would cause the last split to be empty,
-    // then reduce the split count.
-    while (true) {
-        if (split_k == 1) {
-            break;
+            // ggml_vk_matmul will align the splits to be a multiple of 256.
+            // If this rounded up size would cause the last split to be empty,
+            // then reduce the split count.
+            while (true) {
+                if (split_k == 1) {
+                    break;
+                }
+                uint32_t k_split = CEIL_DIV(k, split_k);
+                k_split = ROUNDUP_POW2(k_split, 256);
+                if (k_split * (split_k - 1) < k) {
+                    break;
+                }
+                split_k--;
+            }
         }
-        uint32_t k_split = CEIL_DIV(k, split_k);
-        k_split = ROUNDUP_POW2(k_split, 256);
-        if (k_split * (split_k - 1) < k) {
-            break;
-        }
-        split_k--;
     }
 
     return split_k;
@@ -13027,7 +13003,10 @@ void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) {
         ggml_vk_command_pool_cleanup(ctx->device, ctx->transfer_cmd_pool);
     }
 
-    ctx->binary_semaphore_idx = 0;
+    for (size_t i = 0; i < ctx->gc.semaphores.size(); i++) {
+        ctx->device->device.destroySemaphore({ ctx->gc.semaphores[i].s });
+    }
+    ctx->gc.semaphores.clear();
 
     for (size_t i = 0; i < ctx->gc.tl_semaphores.size(); i++) {
         ctx->device->device.destroySemaphore({ ctx->gc.tl_semaphores[i].s });
@@ -13075,11 +13054,6 @@ void ggml_vk_cleanup(ggml_backend_vk_context * ctx) {
     }
     ctx->gc.events.clear();
 
-    for (auto & sem : ctx->gc.semaphores) {
-        ctx->device->device.destroySemaphore(sem.s);
-    }
-    ctx->gc.semaphores.clear();
-
     ctx->device->device.destroyFence(ctx->fence);
     ctx->device->device.destroyFence(ctx->almost_ready_fence);
 
@@ -14480,9 +14454,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
             flops_cap = 2'000'000'000ULL * ctx->device->shader_core_count;
         }
     }
-    // small graphs gain nothing from splitting, each submit costs CPU time
-    uint64_t flops_per_submit = ctx->last_total_flops < 1'000'000'000ULL ? 0 :
-        std::clamp(ctx->last_total_flops / 40u, std::min<uint64_t>(100'000'000ULL, flops_cap), flops_cap);
+    uint64_t flops_per_submit = std::min(flops_cap, ctx->last_total_flops / 40u);
 
     auto const submit_after = [&](int start, int end) {
         if (ctx->device->serialize_submissions) {
@@ -15682,10 +15654,6 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
             }
         case GGML_OP_FLASH_ATTN_EXT:
             {
-                // K/V row lists (ggml_flash_attn_ext_rows) are not supported
-                if (op->src[5] != nullptr) {
-                    return false;
-                }
                 bool coopmat2 = device->coopmat2;
                 uint32_t HSK = op->src[1]->ne[0];
                 uint32_t HSV = op->src[2]->ne[0];
@@ -16288,7 +16256,7 @@ static void ggml_backend_vk_device_event_synchronize(ggml_backend_dev_t dev, ggm
     }
 }
 
-ggml_backend_buffer_t ggml_backend_vk_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) {
+static ggml_backend_buffer_t ggml_backend_vk_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) {
     VK_LOG_DEBUG("ggml_backend_vk_device_buffer_from_host_ptr(backend=" << dev << ", ptr=" << ptr << ", size=" << size << ")");
     GGML_UNUSED(max_tensor_size);
 
@@ -16370,29 +16338,11 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg,
     return devices[device];
 }
 
-static void * ggml_backend_vk_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {
-    UNUSED(reg);
-#if !defined(_WIN32)
-    if (strcmp(name, "ggml_backend_comm_init") == 0) {
-        return (void *) ggml_backend_vk_comm_init;
-    }
-    if (strcmp(name, "ggml_backend_comm_free") == 0) {
-        return (void *) ggml_backend_vk_comm_free;
-    }
-    if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) {
-        return (void *) ggml_backend_vk_comm_allreduce_tensor;
-    }
-#else
-    UNUSED(name);
-#endif
-    return nullptr;
-}
-
 static const struct ggml_backend_reg_i ggml_backend_vk_reg_i = {
     /* .get_name         = */ ggml_backend_vk_reg_get_name,
     /* .get_device_count = */ ggml_backend_vk_reg_get_device_count,
     /* .get_device       = */ ggml_backend_vk_reg_get_device,
-    /* .get_proc_address = */ ggml_backend_vk_reg_get_proc_address,
+    /* .get_proc_address = */ NULL,
 };
 
 ggml_backend_reg_t ggml_backend_vk_reg() {

ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp

@@ -40,8 +40,6 @@ layout (binding = 2) writeonly buffer DST {float data_dst[];};
 shared uint vals[EXPERTS_PER_CHUNK];
 shared uint offsets[EXPERTS_PER_CHUNK];
 shared uint cursors[EXPERTS_PER_CHUNK];
-shared uint skipped_rows;
-shared uint skip_count;
 
 // data_d[0] is the tile count. Each tile has (expert, valid rows).
 // Packed (token << 16) | slot row ids start at p.row_ids_offset, with each expert on a tile boundary.
@@ -53,38 +51,23 @@ void write_tiles(uint expert, uint count, uint tile_begin, uint n_tiles) {
     }
 }
 
-// The matmul does not write skipped rows, so collect their offsets and zero them cooperatively.
+// the matmul writes only the rows that an expert owns, so zero the rows of the skipped slots
 void zero_skipped_rows() {
     const uint num_elements = p.ne00 * p.ne01;
     const uint tid = gl_LocalInvocationID.x;
 
-    // Reuse offsets[] as a queue, so each batch must fit in EXPERTS_PER_CHUNK.
-    for (uint base = 0; base < num_elements; base += EXPERTS_PER_CHUNK) {
-        const uint end = min(base + EXPERTS_PER_CHUNK, num_elements);
-        if (tid == 0) {
-            skip_count = 0;
-        }
-        barrier();
+    for (uint idx = 0; idx < num_elements; idx++) {
+        const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);
+        const uint i00 = idx - i01 * p.ne00;
 
-        for (uint idx = base + tid; idx < end; idx += BLOCK_SIZE) {
-            const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);
-            const uint i00 = idx - i01 * p.ne00;
-
-            if (int(data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00]) == -1) {
-                const uint slot = atomicAdd(skip_count, 1);
-                offsets[slot] = i00 * p.stride_dst + i01 * p.batch_stride_dst;
-            }
+        if (int(data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00]) != -1) {
+            continue;
         }
-        barrier();
 
-        const uint n_skipped = skip_count;
-        for (uint r = 0; r < n_skipped; ++r) {
-            const uint d_off = offsets[r];
-            for (uint i = tid; i < p.ne0_dst; i += BLOCK_SIZE) {
-                data_dst[d_off + i] = 0.0f;
-            }
+        const uint d_off = i00 * p.stride_dst + i01 * p.batch_stride_dst;
+        for (uint i = tid; i < p.ne0_dst; i += BLOCK_SIZE) {
+            data_dst[d_off + i] = 0.0f;
         }
-        barrier();
     }
 }
 
@@ -93,11 +76,6 @@ void main() {
     const uint tid = gl_LocalInvocationID.x;
     const uint tile_shift = findLSB(p.row_tile_size);
     uint total_tiles = 0;
-    uint local_skipped = 0;
-    if (tid == 0) {
-        skipped_rows = 0;
-    }
-    barrier();
 #ifdef USE_SUBGROUPS
     // Use the subgroup that contains invocation 0.
     const bool prefix_subgroup = subgroupAny(tid == 0);
@@ -157,12 +135,7 @@ void main() {
         for (uint idx = tid; idx < num_elements; idx += BLOCK_SIZE) {
             const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);
             const uint i00 = idx - i01 * p.ne00;
-            const uint id = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00];
-            // Count skipped slots only in the first expert chunk.
-            if (expert_base == 0 && int(id) == -1) {
-                local_skipped++;
-            }
-            const uint expert = id - expert_base;
+            const uint expert = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00] - expert_base;
             if (expert < n_experts) {
                 const uint row = atomicAdd(cursors[expert], 1);
                 data_d[p.row_ids_offset + offsets[expert] + row] = (i01 << 16) | i00;
@@ -171,17 +144,9 @@ void main() {
         // The next chunk clears vals. Scatter uses only offsets and cursors.
         expert_base += n_experts;
     }
-    if (local_skipped != 0) {
-        atomicAdd(skipped_rows, local_skipped);
-    }
-    // All scatter reads must finish before reusing offsets[] as a queue.
-    barrier();
-
     if (tid == 0) {
         data_d[0] = total_tiles;
     }
 
-    if (skipped_rows != 0) {
-        zero_skipped_rows();
-    }
+    zero_skipped_rows();
 }

ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp

@@ -173,7 +173,7 @@ void main() {
         cm1_kvalues[gl_LocalInvocationIndex] = kvalues_mxfp4_const[gl_LocalInvocationIndex];
     }
 #if !defined(USE_OCP_FP4)
-    for (uint i = gl_LocalInvocationIndex; i < 256u; i += BLOCK_SIZE) {
+    for (uint i = gl_LocalInvocationIndex; i < 128u; i += BLOCK_SIZE) {
         ue4m3_fp32_lut[i] = ue4m3_to_fp32_build(i);
     }
 #endif