webgpu: address review @yomaytk, name the mat-vec row threshold

The flash attention vec path and the MMVQ path append a scratch area
after dst, but get_alloc_size only accounted for it when the current
batch takes that path. The size was therefore not monotonic in the
batch length: a graph reserved at n_ubatch (tiled path, no scratch)
or at one token did not cover batches of 2 to 19 query rows, and
GGML_SCHED_NO_REALLOC builds aborted on an unexpected reallocation.

Pass the row count to both path predicates and size the scratch with
the row count clamped to the longest batch the short path serves.

README.md

@@ -79,7 +79,7 @@ The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-or
 | --- | --- |
 | [BLAS](docs/build.md#blas-build) | All |
 | [BLIS](docs/backend/BLIS.md) | All |
-| [CANN](docs/build.md#cann) | Huawei Ascend NPU |
+| [CANN](docs/build.md#cann) | Ascend NPU |
 | [CUDA](docs/build.md#cuda) | Nvidia GPU |
 | [HIP](docs/build.md#hip) | AMD GPU |
 | [Hexagon](docs/backend/snapdragon/README.md) | Snapdragon |

common/arg.cpp

@@ -12,6 +12,16 @@
 #include "speculative.h"
 #include "preset.h"
 
+// fix problem with std::min and std::max
+#if defined(_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#   define NOMINMAX
+#endif
+#include <windows.h>
+#include <shellapi.h>
+#endif
+
 #include <algorithm>
 #include <cinttypes>
 #include <climits>
@@ -28,6 +38,20 @@
 #include <thread> // for hardware_concurrency
 #include <vector>
 
+#ifndef __EMSCRIPTEN__
+#ifdef __linux__
+#include <linux/limits.h>
+#elif defined(_WIN32)
+#   if !defined(PATH_MAX)
+#   define PATH_MAX MAX_PATH
+#   endif
+#elif defined(_AIX)
+#include <sys/limits.h>
+#else
+#include <sys/syslimits.h>
+#endif
+#endif
+
 #define LLAMA_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 2083
 
 using json = common_json;
@@ -327,7 +351,7 @@ static bool parse_bool_value(const std::string & value) {
 static std::string get_default_local_path(const std::string & url) {
     auto f = string_split<std::string>(url, '#').front();
     f = string_split<std::string>(f, '?').front();
-    return fs_get_cache_file(string_split<std::string>(f, '/').back()).string();
+    return fs_path_to_utf8(fs_get_cache_file(string_split<std::string>(f, '/').back()));
 }
 
 static bool spec_types_is_default(const common_params & params) {
@@ -738,7 +762,7 @@ static void common_params_apply_system_config(common_params & params, llama_exam
     common_preset_context ctx(ex);
     ctx.ignore_unknown_keys = true; // the same config file is shared by all programs
     for (const auto & path : found) {
-        LOG_INF("using config file: %s\n", path.string().c_str());
+        LOG_INF("using config file: %s\n", fs_path_to_utf8(path).c_str());
         common_preset global;
         common_presets presets = ctx.load_from_ini(path, global);
         global.apply_to_params(params);
@@ -1235,7 +1259,44 @@ bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map<com
     return true;
 }
 
+#ifdef _WIN32
+struct utf8_argv {
+    std::vector<std::string> buf;
+    std::vector<char*> ptrs;
+};
+
+static utf8_argv make_utf8_argv() {
+    utf8_argv out;
+    int wargc = 0;
+    LPWSTR* wargv = CommandLineToArgvW(GetCommandLineW(), &wargc);
+    if (!wargv) return out;
+
+    out.buf.reserve(wargc);
+    for (int i = 0; i < wargc; ++i) {
+        int n = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wargv[i], -1, nullptr, 0, nullptr, nullptr);
+        if (n <= 0) { out.buf.emplace_back(); continue; }
+        auto& s = out.buf.emplace_back();
+        s.resize(static_cast<size_t>(n - 1));
+        (void)WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, s.data(), n, nullptr, nullptr);
+    }
+    LocalFree(wargv);
+
+    out.ptrs.reserve(out.buf.size() + 1);
+    for (auto& s : out.buf) out.ptrs.push_back(s.data());
+    out.ptrs.push_back(nullptr);
+    return out;
+}
+#endif
+
 bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **)) {
+#ifdef _WIN32
+    auto utf8 = make_utf8_argv();
+    // repair argv only when it matches the process command line
+    if (static_cast<int>(utf8.buf.size()) == argc) {
+        argv = utf8.ptrs.data();
+    }
+#endif
+
     auto ctx_arg = common_params_parser_init(params, ex, print_usage);
     const common_params params_org = ctx_arg.params; // the example can modify the default params
 

common/common.cpp

@@ -934,6 +934,12 @@ std::string wstring_to_utf8(const std::wstring & str) {
 }
 #endif
 
+// returns the path as a UTF-8 string, preserving its separators
+std::string fs_path_to_utf8(const std::filesystem::path & path) {
+    const auto value = path.u8string();
+    return std::string(value.begin(), value.end());
+}
+
 void fs_write_atomic(const std::filesystem::path & path, const std::string & data) {
     // a random suffix gives every writer its own temporary file, across processes and containers
     std::random_device rd;
@@ -957,7 +963,7 @@ void fs_write_atomic(const std::filesystem::path & path, const std::string & dat
 
     if (file.fail() || ec) {
         std::filesystem::remove(path_tmp, ec);
-        throw std::runtime_error("failed to write file: " + path.string());
+        throw std::runtime_error("failed to write file: " + fs_path_to_utf8(path));
     }
 }
 
@@ -1051,7 +1057,7 @@ std::filesystem::path fs_get_cache_file(const std::string & filename) {
     std::error_code ec;
     common_create_directories(cache_directory, ec);
     if (ec) {
-        throw std::runtime_error("failed to create cache directory: " + cache_directory.string());
+        throw std::runtime_error("failed to create cache directory: " + fs_path_to_utf8(cache_directory));
     }
     return cache_directory / std::filesystem::u8path(filename);
 }

common/common.h

@@ -910,6 +910,9 @@ std::wstring utf8_to_wstring(const std::string & str);
 std::string  wstring_to_utf8(const std::wstring & str);
 #endif
 
+// returns the path as a UTF-8 string, preserving its separators
+std::string fs_path_to_utf8(const std::filesystem::path & path);
+
 //
 // Environment utils
 //

common/download.cpp

@@ -1005,7 +1005,7 @@ std::string common_docker_resolve_model(const std::string & docker) {
         std::string model_filename = repo;
         std::replace(model_filename.begin(), model_filename.end(), '/', '_');
         model_filename += "_" + tag + ".gguf";
-        std::string local_path = fs_get_cache_file(model_filename).string();
+        std::string local_path = fs_path_to_utf8(fs_get_cache_file(model_filename));
 
         const std::string blob_url = url_prefix + "/blobs/" + gguf_digest;
         common_download_opts opts;

common/hf-cache.cpp

@@ -63,7 +63,7 @@ static fs::path get_cache_directory() {
 }
 
 std::string get_cache_path() {
-    return get_cache_directory().string();
+    return fs_path_to_utf8(get_cache_directory());
 }
 
 static std::string folder_name_to_repo(const std::string & folder) {
@@ -340,11 +340,11 @@ hf_files get_repo_files(const std::string & repo_id,
             file.url = endpoint + repo_id + "/resolve/" + commit + "/" + file.path;
 
             fs::path final_path = commit_path / subpath;
-            file.final_path = final_path.string();
+            file.final_path = fs_path_to_utf8(final_path);
 
             if (!file.oid.empty() && !fs::exists(final_path)) {
                 fs::path local_path = blobs_path / file.oid;
-                file.local_path = local_path.string();
+                file.local_path = fs_path_to_utf8(local_path);
             } else {
                 file.local_path = file.final_path;
             }
@@ -411,7 +411,7 @@ hf_files get_cached_files(const std::string & repo_id) {
         if (!fs::exists(snapshots_path)) {
             continue;
         }
-        std::string _repo_id = folder_name_to_repo(repo.path().filename().string());
+        std::string _repo_id = folder_name_to_repo(fs_path_to_utf8(repo.path().filename()));
 
         if (!is_valid_repo_id(_repo_id)) {
             continue;
@@ -436,7 +436,7 @@ hf_files get_cached_files(const std::string & repo_id) {
                 file.repo_id = _repo_id;
                 const auto generic_path = path.generic_u8string();
                 file.path = std::string(generic_path.begin(), generic_path.end());
-                file.local_path = entry.path().string();
+                file.local_path = fs_path_to_utf8(entry.path());
                 file.final_path = file.local_path;
                 files.push_back(std::move(file));
             }
@@ -516,7 +516,7 @@ bool remove_cached_repo(const std::string & repo_id) {
     std::error_code ec;
     auto removed = fs::remove_all(repo_path, ec);
     if (ec) {
-        LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str());
+        LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, fs_path_to_utf8(repo_path).c_str(), ec.message().c_str());
         return false;
     }
     return removed > 0;

common/jinja/runtime.cpp

@@ -307,13 +307,58 @@ value binary_expression::execute_impl(context & ctx) const {
     throw std::runtime_error("Unknown operator \"" + op.value + "\" between " + left_val->type() + " and " + right_val->type());
 }
 
+static value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false) {
+    JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str());
+    if (ctx.is_get_stats) {
+        value_t::stats_t::mark_used(input);
+        input->stats.ops.insert(name);
+    }
+    auto builtins = input->get_builtins();
+    auto it = builtins.find(name);
+    if (it != builtins.end()) {
+        JJ_DEBUG("Binding built-in '%s'", name.c_str());
+        return mk_val<value_func>(name, it->second, input);
+    }
+    if (undef_on_missing) {
+        return mk_val<value_undefined>(name);
+    }
+    throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type());
+}
+
 static value apply_filter(context & ctx, const statement_ptr & filter, value input) {
     JJ_DEBUG("Applying filter to %s", input->type().c_str());
 
+    auto set_filter_alias = [](auto & filter_id) {
+        if (filter_id == "count") {
+            filter_id = "length";
+        } else if (filter_id == "d") {
+            filter_id = "default";
+        } else if (filter_id == "e") {
+            filter_id = "escape";
+        } else if (filter_id == "trim") {
+            filter_id = "strip";
+        }
+    };
+
     if (is_stmt<identifier>(filter)) {
         auto filter_id = cast_stmt<identifier>(filter)->val;
+
+        set_filter_alias(filter_id);
         JJ_DEBUG("Applying filter '%s' to %s", filter_id.c_str(), input->type().c_str());
-        return apply_filter_by_name(ctx, filter_id, input, func_args(ctx));
+        // TODO: Refactor filters so this coercion can be done automatically
+        if (!input->is_undefined() && !is_val<value_string>(input) && (
+            filter_id == "capitalize" ||
+            filter_id == "lower" ||
+            filter_id == "replace" ||
+            filter_id == "strip" ||
+            filter_id == "title" ||
+            filter_id == "upper" ||
+            filter_id == "wordcount"
+        )) {
+            JJ_DEBUG("Coercing %s to String for '%s' filter", input->type().c_str(), filter_id.c_str());
+            input = mk_val<value_string>(input->as_string());
+        }
+        return try_builtin_func(ctx, filter_id, input)->invoke(func_args(ctx));
 
     } else if (is_stmt<call_expression>(filter)) {
         auto call = cast_stmt<call_expression>(filter);
@@ -322,13 +367,14 @@ static value apply_filter(context & ctx, const statement_ptr & filter, value inp
         }
         auto filter_id = cast_stmt<identifier>(call->callee)->val;
 
+        set_filter_alias(filter_id);
         JJ_DEBUG("Applying filter '%s' with arguments to %s", filter_id.c_str(), input->type().c_str());
         func_args args(ctx);
         for (const auto & arg_expr : call->args) {
             args.push_back(arg_expr->execute(ctx));
         }
 
-        return apply_filter_by_name(ctx, filter_id, input, args);
+        return try_builtin_func(ctx, filter_id, input)->invoke(args);
 
     } else {
         throw std::runtime_error("Invalid filter expression");

common/jinja/value.cpp

@@ -72,17 +72,6 @@ const std::vector<value> & func_args::get_args() const {
     return args;
 }
 
-void func_args::ensure_kwargs(std::initializer_list<const char *> names) const {
-    for (const auto & arg : args) {
-        if (is_val<value_kwarg>(arg)) {
-            auto * kwarg = cast_val<value_kwarg>(arg);
-            if (std::find(names.begin(), names.end(), kwarg->key) == names.end()) {
-                throw raised_exception("Function '" + func_name + "' got an unexpected keyword argument '" + kwarg->key + "'");
-            }
-        }
-    }
-}
-
 /**
  * Function that mimics Python's array slicing.
  */
@@ -145,14 +134,6 @@ static value empty_value_fn(const func_args &) {
         return mk_val<T>();
     }
 }
-
-template<typename T>
-static value empty_value_no_args_fn(const func_args & args) {
-    args.ensure_kwargs();
-    args.ensure_count(1, 1);
-    return empty_value_fn<T>(args);
-}
-
 template<typename T>
 static value test_type_fn(const func_args & args) {
     args.ensure_count(1);
@@ -613,52 +594,6 @@ const func_builtins & global_builtins() {
     return builtins;
 }
 
-value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing) {
-    JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str());
-    if (ctx.is_get_stats) {
-        value_t::stats_t::mark_used(input);
-        input->stats.ops.insert(name);
-    }
-    auto builtins = input->get_builtins();
-    auto it = builtins.find(name);
-    if (it != builtins.end()) {
-        JJ_DEBUG("Binding built-in '%s'", name.c_str());
-        return mk_val<value_func>(name, it->second, input);
-    }
-    if (undef_on_missing) {
-        return mk_val<value_undefined>(name);
-    }
-    throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type());
-}
-
-value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args) {
-    if (filter_id == "count") {
-        filter_id = "length";
-    } else if (filter_id == "d") {
-        filter_id = "default";
-    } else if (filter_id == "e") {
-        filter_id = "escape";
-    } else if (filter_id == "trim") {
-        filter_id = "strip";
-    }
-
-    // TODO: Refactor filters so this coercion can be done automatically
-    if (!input->is_undefined() && !is_val<value_string>(input) && (
-        filter_id == "capitalize" ||
-        filter_id == "lower" ||
-        filter_id == "replace" ||
-        filter_id == "strip" ||
-        filter_id == "title" ||
-        filter_id == "upper" ||
-        filter_id == "wordcount"
-    )) {
-        JJ_DEBUG("Coercing %s to String for '%s' filter", input->type().c_str(), filter_id.c_str());
-        input = mk_val<value_string>(input->as_string());
-    }
-
-    return try_builtin_func(ctx, filter_id, input)->invoke(args);
-}
-
 
 const func_builtins & value_int_t::get_builtins() const {
     static const func_builtins builtins = {
@@ -732,22 +667,16 @@ const func_builtins & value_string_t::get_builtins() const {
     static const func_builtins builtins = {
         {"default", default_value},
         {"upper", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().uppercase();
             return mk_val<value_string>(str);
         }},
         {"lower", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().lowercase();
             return mk_val<value_string>(str);
         }},
         {"strip", [](const func_args & args) -> value {
-            args.ensure_kwargs({"chars"});
-            args.ensure_count(1, 2);
             value val_input = args.get_pos(0);
             if (!is_val<value_string>(val_input)) {
                 throw raised_exception("strip() first argument must be a string");
@@ -778,15 +707,11 @@ const func_builtins & value_string_t::get_builtins() const {
             }
         }},
         {"title", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().titlecase();
             return mk_val<value_string>(str);
         }},
         {"capitalize", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().capitalize();
             return mk_val<value_string>(str);
@@ -1088,10 +1013,6 @@ const func_builtins & value_bool_t::get_builtins() const {
     return builtins;
 }
 
-[[noreturn]] static value array_sum_not_implemented(const func_args &) {
-    throw not_implemented_exception("Array sum builtin not implemented");
-}
-
 const func_builtins & value_array_t::get_builtins() const {
     static const func_builtins builtins = {
         {"default", default_value},
@@ -1158,9 +1079,7 @@ const func_builtins & value_array_t::get_builtins() const {
         {"select", select_or_reject<false, false>},
         {"rejectattr", select_or_reject<true, true>},
         {"reject", select_or_reject<true, false>},
-        {"sum", array_sum_not_implemented},
         {"join", [](const func_args & args) -> value {
-            args.ensure_kwargs({"d", "attribute"});
             args.ensure_count(1, 3);
             if (!is_val<value_array>(args.get_pos(0))) {
                 throw raised_exception("join() first argument must be an array");
@@ -1198,28 +1117,13 @@ const func_builtins & value_array_t::get_builtins() const {
         {"tojson", tojson},
         {"map", [](const func_args & args) -> value {
             args.ensure_count(2);
-            value val = args.get_pos(0);
-            if (!is_val<value_array>(val)) {
+            if (!is_val<value_array>(args.get_pos(0))) {
                 throw raised_exception("map: first argument must be an array");
             }
             if (!is_val<value_kwarg>(args.get_args().at(1))) {
-                value filter = args.get_pos(1);
-                if (!is_val<value_string>(filter)) {
-                    throw raised_exception("map: filter name must be a string");
-                }
-                auto out = mk_val<value_array>();
-                func_args filter_args(args.ctx);
-                const auto & all_args = args.get_args();
-                for (size_t i = 2; i < all_args.size(); ++i) {
-                    filter_args.push_back(all_args[i]);
-                }
-                const std::string filter_id = filter->as_string().str();
-                auto arr = val->as_array();
-                for (const auto & item : arr) {
-                    out->push_back(apply_filter_by_name(args.ctx, filter_id, item, filter_args));
-                }
-                return is_val<value_tuple>(val) ? mk_val<value_tuple>(out->as_array()) : out;
+                throw not_implemented_exception("map: filter-mapping not implemented");
             }
+            value val       = args.get_pos(0);
             value attribute = args.get_kwarg_or_pos("attribute", 1);
             value default_val = args.get_kwarg("default", mk_val<value_undefined>());
             auto out = mk_val<value_array>();
@@ -1228,7 +1132,7 @@ const func_builtins & value_array_t::get_builtins() const {
                 value attr_val = get_attribute(item, attribute, default_val);
                 out->push_back(attr_val);
             }
-            return is_val<value_tuple>(val) ? mk_val<value_tuple>(out->as_array()) : out;
+            return is_val<value_tuple>(val) ? mk_val<value_tuple>(std::move(out->as_array())) : out;
         }},
         {"append", [](const func_args & args) -> value {
             args.ensure_count(2);
@@ -1501,18 +1405,14 @@ const func_builtins & value_none_t::get_builtins() const {
 const func_builtins & value_undefined_t::get_builtins() const {
     static const func_builtins builtins = {
         {"default", default_value},
-        {"capitalize", empty_value_no_args_fn<value_string>},
+        {"capitalize", empty_value_fn<value_string>},
         {"first", empty_value_fn<value_undefined>},
         {"items", empty_value_fn<value_array>},
-        {"join", [](const func_args & args) -> value {
-            args.ensure_kwargs({"d", "attribute"});
-            args.ensure_count(1, 3);
-            return empty_value_fn<value_string>(args);
-        }},
+        {"join", empty_value_fn<value_string>},
         {"last", empty_value_fn<value_undefined>},
         {"length", empty_value_fn<value_int>},
         {"list", empty_value_fn<value_array>},
-        {"lower", empty_value_no_args_fn<value_string>},
+        {"lower", empty_value_fn<value_string>},
         {"map", empty_value_fn<value_array>},
         {"max", empty_value_fn<value_undefined>},
         {"min", empty_value_fn<value_undefined>},
@@ -1525,16 +1425,12 @@ const func_builtins & value_undefined_t::get_builtins() const {
         {"selectattr", empty_value_fn<value_array>},
         {"sort", empty_value_fn<value_array>},
         {"string", empty_value_fn<value_string>},
-        {"strip", [](const func_args & args) -> value {
-            args.ensure_kwargs({"chars"});
-            args.ensure_count(1, 2);
-            return empty_value_fn<value_string>(args);
-        }},
+        {"strip", empty_value_fn<value_string>},
         {"sum", empty_value_fn<value_int>},
-        {"title", empty_value_no_args_fn<value_string>},
+        {"title", empty_value_fn<value_string>},
         {"truncate", empty_value_fn<value_string>},
         {"unique", empty_value_fn<value_array>},
-        {"upper", empty_value_no_args_fn<value_string>},
+        {"upper", empty_value_fn<value_string>},
         {"wordcount", empty_value_fn<value_int>},
     };
     return builtins;

common/jinja/value.h

@@ -7,7 +7,6 @@
 #include <cmath>
 #include <cstdint>
 #include <functional>
-#include <initializer_list>
 #include <map>
 #include <memory>
 #include <set>
@@ -177,8 +176,6 @@ protected:
 //
 
 const func_builtins & global_builtins();
-value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false);
-value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args);
 
 std::string value_to_json(const value & val, int indent = -1, const std::string_view item_sep = ", ", const std::string_view key_sep = ": ");
 
@@ -661,7 +658,6 @@ public:
     size_t count() const { return args.size(); }
     void push_back(const value & val);
     void push_front(const value & val);
-    void ensure_kwargs(std::initializer_list<const char *> names = {}) const;
     void ensure_count(size_t min, size_t max = 999) const {
         size_t n = args.size();
         if (n < min || n > max) {

common/parsers/qwen3-coder.cpp

@@ -74,24 +74,16 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
     data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
         auto generation_prompt = p.literal(GEN_PREFIX);
 
-        // The template strips the newline before </think>, so leave it out of the reasoning
-        auto think_body  = p.until_one_of({ "\n</think>", "</think>", "<tool_call>" });
-        auto think_close = p.literal("\n</think>") | p.literal("</think>") | p.peek(p.literal("<tool_call>"));
-
-        auto reasoning_extracted = p.optional("<think>" + p.space() + p.reasoning(think_body) + think_close);
-        auto reasoning_skipped   = p.optional("<think>" + p.space() + think_body + think_close);
-
         auto reasoning = p.eps();
         if (supports_reasoning && extract_reasoning) {
-            reasoning = reasoning_extracted;
+            // The template strips the newline before </think>, so leave it out of the reasoning
+            reasoning = p.optional("<think>" + p.space() +
+                                   p.reasoning(p.until_one_of({ "\n</think>", "</think>", "<tool_call>" })) +
+                                   (p.literal("\n</think>") | p.literal("</think>") | p.peek(p.literal("<tool_call>"))));
         }
 
         // Response format parser
         if (has_response_format) {
-            // The answer must be bare JSON, so the think block is consumed even when it is not extracted
-            if (supports_reasoning && !extract_reasoning) {
-                reasoning = reasoning_skipped;
-            }
             return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));
         }
 

common/preset.cpp

@@ -171,12 +171,12 @@ static std::map<std::string, std::map<std::string, std::string>> parse_ini_from_
     std::map<std::string, std::map<std::string, std::string>> parsed;
 
     if (!std::filesystem::exists(path)) {
-        throw std::runtime_error("preset file does not exist: " + path.string());
+        throw std::runtime_error("preset file does not exist: " + fs_path_to_utf8(path));
     }
 
     std::ifstream file(path);
     if (!file.good()) {
-        throw std::runtime_error("failed to open server preset file: " + path.string());
+        throw std::runtime_error("failed to open server preset file: " + fs_path_to_utf8(path));
     }
 
     std::string contents((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
@@ -225,7 +225,7 @@ static std::map<std::string, std::map<std::string, std::string>> parse_ini_from_
     common_peg_parse_context ctx(contents);
     const auto result = parser.parse(ctx);
     if (!result.success()) {
-        throw std::runtime_error("failed to parse server config file: " + path.string());
+        throw std::runtime_error("failed to parse server config file: " + fs_path_to_utf8(path));
     }
 
     std::string current_section = COMMON_PRESET_DEFAULT_NAME;
@@ -323,7 +323,7 @@ common_presets common_preset_context::load_from_ini(const std::filesystem::path
                 }
                 LOG_DBG("accepted option: %s = %s\n", key.c_str(), preset.options[opt].c_str());
             } else if (ignore_unknown_keys) {
-                LOG_WRN("ignoring option '%s' from %s: not supported by this program\n", key.c_str(), path.string().c_str());
+                LOG_WRN("ignoring option '%s' from %s: not supported by this program\n", key.c_str(), fs_path_to_utf8(path).c_str());
             } else {
                 throw std::runtime_error(string_format(
                     "option '%s' not recognized in preset '%s'",
@@ -401,7 +401,7 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
             if (!entry.is_regular_file(ec)) {
                 continue;
             }
-            const std::string fname = entry.path().filename().string();
+            const std::string fname = fs_path_to_utf8(entry.path().filename());
             if (!string_ends_with(fname, ".gguf")) {
                 continue;
             }
@@ -422,9 +422,9 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
         if (!path.empty()) {
             models.push_back({
                 /* name        */ name,
-                /* path        */ path.string(),
-                /* path_mmproj */ mmproj_file.string(), // can be empty
-                /* path_draft  */ draft_file.string()   // can be empty
+                /* path        */ fs_path_to_utf8(path),
+                /* path_mmproj */ fs_path_to_utf8(mmproj_file), // can be empty
+                /* path_draft  */ fs_path_to_utf8(draft_file)    // can be empty
             });
         }
     };
@@ -432,13 +432,13 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
     for (const auto & entry : std::filesystem::directory_iterator(dir)) {
         std::error_code ec;
         if (entry.is_directory(ec)) {
-            scan_subdir(entry.path(), entry.path().filename().string());
+            scan_subdir(entry.path(), fs_path_to_utf8(entry.path().filename()));
             continue;
         }
         if (!entry.is_regular_file(ec)) {
             continue;
         }
-        const std::string fname = entry.path().filename().string();
+        const std::string fname = fs_path_to_utf8(entry.path().filename());
         if (!string_ends_with(fname, ".gguf")) {
             continue;
         }
@@ -450,7 +450,7 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
         string_replace_all(name, ".gguf", "");
         models.push_back({
             /* name        */ name,
-            /* path        */ entry.path().string(),
+            /* path        */ fs_path_to_utf8(entry.path()),
             /* path_mmproj */ "",
             /* path_draft  */ ""
         });

docs/build.md

@@ -481,7 +481,7 @@ cmake --build build --config Release
 Now you can load the model in conversation mode using `Vulkan`
 
 ```sh
-build/bin/Release/llama-cli -m "[PATH TO MODEL]" -ngl 100 -c 16384 -t 10 -n -2
+build/bin/Release/llama-cli -m "[PATH TO MODEL]" -ngl 100 -c 16384 -t 10 -n -2 -cnv
 ```
 
 **MSYS2**

docs/ops.md

@@ -74,7 +74,7 @@ Legend:
 |                             MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ | ❌ |
 |                              MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 |
-|                 MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 |
+|                 MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | 🟡 |
 |                       MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ❌ |
 |                  MUL_MAT_ID_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                  MUL_MAT_ID_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |

docs/ops/OpenCL.csv

@@ -9242,31 +9242,31 @@
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=128,n=32,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=128,n=4,k=128,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=256,n=512,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=64,n=1,k=64,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=512,n=1,k=512,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=32,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=512,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=64,n=1,k=64,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=512,n=1,k=512,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=32,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=512,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=32,n=1,k=32,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=16384,n=1,k=16384,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=64,n=1,k=64,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=512,n=1,k=512,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=32,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=4,k=128,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=512,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=64,n=1,k=64,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=1,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=512,n=1,k=512,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=32,k=128,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=128,n=4,k=128,bs=[2,3],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=256,n=512,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=1024,n=1,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=2048,n=1,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=4096,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=8192,n=1,k=8192,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
 "OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f32,m=1024,n=7,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=1024,n=1,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=2048,n=1,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=4096,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=8192,n=1,k=8192,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
-"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=1024,n=7,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","1","yes","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=1024,n=1,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=2048,n=1,k=2048,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=4096,n=1,k=4096,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=8192,n=1,k=8192,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
+"OpenCL","MUL_MAT_HADAMARD","type_a=f32,type_b=f16,m=1024,n=7,k=1024,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
 "OpenCL","MUL_MAT_W4A8","type_a=nvfp4,type_b=f32,m=32,n=1,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
 "OpenCL","MUL_MAT_W4A8","type_a=nvfp4,type_b=f32,m=32,n=32,k=256,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"
 "OpenCL","MUL_MAT_W4A8","type_a=nvfp4,type_b=f32,m=64,n=16,k=512,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1,src_overlap=0,m_v=0,pad=0","support","0","no","OpenCL"

ggml/src/ggml-cpu/ggml-cpu-impl.h

@@ -30,7 +30,7 @@ struct ggml_compute_params {
 };
 
 
-#if defined(_MSC_VER) && !defined(__clang__)
+#if defined(_MSC_VER)
 
 #define m512bh(p) p
 #define m512i(p) p

ggml/src/ggml-hexagon/htp/dma-queue.c

@@ -141,9 +141,43 @@ bool dma_queue_push_fallback_2d(dma_queue * q, dma_data ddata, size_t dst_stride
     dma_ring_flush(r1);
     r0->tail = r1->tail;
 
+    return dma_ring_push_single_2d(r0, ddata, 0, 0, 0, /*nrows=*/ 0);
+}
+
+bool dma_queue_push_fallback_contig(dma_queue * q, dma_data ddata, size_t total) {
+    dma_ring * r0 = q->ring0;
+    dma_ring * r1 = q->ring1;
+
+    if (((r0->push_idx + 1) & r0->idx_mask) == r0->pop_idx) {
+        return false;
+    }
+
+    r1->tail = r0->tail;
+
+    size_t rem_bytes   = total;
+    dma_addr_t cur_dst = ddata.dst;
+    dma_addr_t cur_src = ddata.src;
+
+    while (rem_bytes > 0) {
+        const uint32_t cur_bytes = MIN(rem_bytes, DMA_SAFE_CHUNK_SIZE);
+        dma_data cur_data = dma_make_data(cur_dst, cur_src);
+        if (!dma_ring_push_single_contig(r1, cur_data, cur_bytes)) {
+            dma_ring_flush(r1);
+            dma_ring_push_single_contig(r1, cur_data, cur_bytes);
+        }
+        cur_dst   += cur_bytes;
+        cur_src   += cur_bytes;
+        rem_bytes -= cur_bytes;
+    }
+
+    dma_ring_flush(r1);
+    r0->tail = r1->tail;
+
     return dma_ring_push_single_1d(r0, ddata, /*size=*/ 0);
 }
 
+#if __HVX_ARCH__ < 75
+
 bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
     dma_ring * r0 = q->ring0;
     dma_ring * r1 = q->ring1;
@@ -154,28 +188,19 @@ bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride
 
     r1->tail = r0->tail;
 
-    dma_addr_t row_dst = ddata.dst;
-    dma_addr_t row_src = ddata.src;
+    size_t rem_rows    = nrows;
+    dma_addr_t cur_dst = ddata.dst;
+    dma_addr_t cur_src = ddata.src;
 
-    for (size_t i = 0; i < nrows; i++) {
-        size_t     rem_bytes = row_size;
-        dma_addr_t cur_dst   = row_dst;
-        dma_addr_t cur_src   = row_src;
-
-        while (rem_bytes > 0) {
-            const uint32_t cur_bytes = MIN(rem_bytes, DMA_SAFE_CHUNK_SIZE);
-            dma_data cur_data = dma_make_data(cur_dst, cur_src);
-            if (!dma_ring_push_single_contig(r1, cur_data, cur_bytes)) {
-                dma_ring_flush(r1);
-                dma_ring_push_single_contig(r1, cur_data, cur_bytes);
-            }
-            cur_dst   += cur_bytes;
-            cur_src   += cur_bytes;
-            rem_bytes -= cur_bytes;
+    while (rem_rows > 0) {
+        dma_data cur_data = dma_make_data(cur_dst, cur_src);
+        if (!dma_ring_push_single_1d(r1, cur_data, row_size)) {
+            dma_ring_flush(r1);
+            dma_ring_push_single_1d(r1, cur_data, row_size);
         }
-
-        row_dst += dst_stride;
-        row_src += src_stride;
+        cur_dst  += dst_stride;
+        cur_src  += src_stride;
+        rem_rows -= 1;
     }
 
     dma_ring_flush(r1);
@@ -183,3 +208,5 @@ bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride
 
     return dma_ring_push_single_1d(r0, ddata, /*size=*/ 0);
 }
+
+#endif

ggml/src/ggml-hexagon/htp/dma-queue.h

@@ -148,7 +148,10 @@ dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q);
 void        dma_queue_alias_free(dma_queue_t q);
 
 bool        dma_queue_push_fallback_2d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows);
+bool        dma_queue_push_fallback_contig(dma_queue * q, dma_data ddata, size_t total);
+#if __HVX_ARCH__ < 75
 bool        dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows);
+#endif
 
 // TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead
 // but those do not seem to always compiler properly.
@@ -397,7 +400,7 @@ static inline bool dma_queue_push(dma_queue * q, dma_data ddata, size_t dst_stri
         if (__builtin_expect(total <= DMA_MAX_SIZE_24B, 1)) {
             return dma_ring_push_single_contig(q->ring0, ddata, total);
         }
-        return dma_queue_push_fallback_1d(q, ddata, 0, 0, total, 1);
+        return dma_queue_push_fallback_contig(q, ddata, total);
     }
 
     // 2. Hot path: Standard strided 2D (10-20% of calls)
@@ -407,10 +410,14 @@ static inline bool dma_queue_push(dma_queue * q, dma_data ddata, size_t dst_stri
     }
 
     // 3. Cold path: Descriptor chunking fallbacks (< 0.1%)
-    if ((row_size | src_stride | dst_stride) <= DMA_MAX_2D_ROW_SIZE) {
+#if __HVX_ARCH__ < 75
+    if (row_size <= DMA_MAX_SIZE_16B && (src_stride | dst_stride) <= DMA_MAX_STRIDE_16B) {
         return dma_queue_push_fallback_2d(q, ddata, dst_stride, src_stride, row_size, nrows);
     }
     return dma_queue_push_fallback_1d(q, ddata, dst_stride, src_stride, row_size, nrows);
+#else
+    return dma_queue_push_fallback_2d(q, ddata, dst_stride, src_stride, row_size, nrows);
+#endif
 }
 
 static inline void dma_sync_read(dma_queue * dma_q, void * dst, dma_addr_t src, size_t bytes) {

ggml/src/ggml-metal/kernels/quantize.h

@@ -20,7 +20,6 @@ void quantize_q1_0(device const float * src, device block_q1_0 & dst) {
 }
 
 void quantize_q2_0(device const float * src, device block_q2_0 & dst) {
-#pragma METAL fp math_mode(safe)
     float amax = 0.0f;
     for (int j = 0; j < QK2_0; j++) {
         float a = fabs(src[j]);

ggml/src/ggml-opencl/CMakeLists.txt

@@ -70,7 +70,6 @@ set(GGML_OPENCL_KERNELS
     lightning_indexer
     tri
     fill
-    fwht
     clamp
     cpy
     cvt
@@ -118,18 +117,6 @@ set(GGML_OPENCL_KERNELS
     gemm_noshuffle_q2_0_f32
     gemm_noshuffle_q2_0_q8_1_dp4a
     gemv_noshuffle_q2_0_f32
-    gemm_noshuffle_iq1_m_f32
-    gemm_noshuffle_iq1_m_q8_1_dp4a
-    gemv_noshuffle_iq1_m_f32
-    gemm_noshuffle_iq1_s_f32
-    gemm_noshuffle_iq1_s_q8_1_dp4a
-    gemv_noshuffle_iq1_s_f32
-    gemm_noshuffle_iq3_s_f32
-    gemm_noshuffle_iq3_s_q8_1_dp4a
-    gemv_noshuffle_iq3_s_f32
-    gemm_noshuffle_iq3_xxs_f32
-    gemm_noshuffle_iq3_xxs_q8_1_dp4a
-    gemv_noshuffle_iq3_xxs_f32
     gemm_noshuffle_q3_k_q8_1_dp4a
     gemv_noshuffle_q3_k_f32
     mul_mv_q2_k_f32
@@ -149,21 +136,6 @@ set(GGML_OPENCL_KERNELS
     mul_mv_q2_0_f32
     mul_mv_q2_0_f32_flat
     mul_mm_q2_0_f32_l4_lm
-    mul_mv_iq1_m_f32
-    mul_mv_iq1_m_f32_flat
-    mul_mm_iq1_m_f32_l4_lm
-    iq1_s_grid
-    mul_mv_iq1_s_f32
-    mul_mv_iq1_s_f32_flat
-    mul_mm_iq1_s_f32_l4_lm
-    iq3_xxs_grid
-    mul_mv_iq3_xxs_f32
-    mul_mv_iq3_xxs_f32_flat
-    mul_mm_iq3_xxs_f32_l4_lm
-    iq3_s_grid
-    mul_mv_iq3_s_f32
-    mul_mv_iq3_s_f32_flat
-    mul_mm_iq3_s_f32_l4_lm
     mul_mv_q3_k_f32_flat
     mul_mm_q3_k_f32_l4_lm
     mul_mv_q2_k_f32_flat

ggml/src/ggml-opencl/kernels/cvt.cl

@@ -3088,198 +3088,3 @@ kernel void kernel_restore_block_q2_0(
         dst[i].qs[j] = src_qs[16*i + j];
     }
 }
-
-struct block_iq1_s {
-    half d;
-    uchar qs[QK_K/8];
-    ushort qh[QK_K/32];
-};
-
-kernel void kernel_convert_block_iq1_s(
-    global struct block_iq1_s * src0,
-    global uchar * dst_qs,
-    global ushort * dst_qh,
-    global half * dst_d,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst_d[i] = src0[i].d;
-    for (int j = 0; j < 32; ++j) {
-        dst_qs[32*i + j] = src0[i].qs[j];
-    }
-    for (int j = 0; j < 8; ++j) {
-        dst_qh[8*i + j] = src0[i].qh[j];
-    }
-}
-
-kernel void kernel_restore_block_iq1_s(
-    global uchar * src_qs,
-    global ushort * src_qh,
-    global half * src_d,
-    global struct block_iq1_s * dst,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst[i].d = src_d[i];
-    for (int j = 0; j < 32; ++j) {
-        dst[i].qs[j] = src_qs[32*i + j];
-    }
-    for (int j = 0; j < 8; ++j) {
-        dst[i].qh[j] = src_qh[8*i + j];
-    }
-}
-
-struct block_iq1_m {
-    uchar qs[QK_K/8];
-    uchar qh[QK_K/16];
-    uchar scales[QK_K/32];
-};
-
-kernel void kernel_convert_block_iq1_m(
-    global struct block_iq1_m * src0,
-    global uchar * dst_qs,
-    global uchar * dst_qh,
-    global ushort * dst_sc,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    for (int j = 0; j < 32; ++j) {
-        dst_qs[32*i + j] = src0[i].qs[j];
-    }
-    for (int j = 0; j < 16; ++j) {
-        dst_qh[16*i + j] = src0[i].qh[j];
-    }
-    for (int j = 0; j < 4; ++j) {
-        dst_sc[4*i + j] = (ushort)src0[i].scales[2*j] | ((ushort)src0[i].scales[2*j + 1] << 8);
-    }
-}
-
-kernel void kernel_restore_block_iq1_m(
-    global uchar * src_qs,
-    global uchar * src_qh,
-    global ushort * src_sc,
-    global struct block_iq1_m * dst,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    for (int j = 0; j < 32; ++j) {
-        dst[i].qs[j] = src_qs[32*i + j];
-    }
-    for (int j = 0; j < 16; ++j) {
-        dst[i].qh[j] = src_qh[16*i + j];
-    }
-    for (int j = 0; j < 4; ++j) {
-        ushort sc = src_sc[4*i + j];
-        dst[i].scales[2*j] = (uchar)sc;
-        dst[i].scales[2*j + 1] = (uchar)(sc >> 8);
-    }
-}
-
-struct block_iq3_s {
-    half d;
-    uchar qs[QK_K/4];
-    uchar qh[QK_K/32];
-    uchar signs[QK_K/8];
-    uchar scales[QK_K/64];
-};
-
-kernel void kernel_convert_block_iq3_s(
-    global struct block_iq3_s * src0,
-    global uchar * dst_qs,
-    global uchar * dst_sg,
-    global uchar * dst_qh,
-    global uchar * dst_sc,
-    global half * dst_d,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst_d[i] = src0[i].d;
-    for (int j = 0; j < 64; ++j) {
-        dst_qs[64*i + j] = src0[i].qs[j];
-    }
-    for (int j = 0; j < 32; ++j) {
-        dst_sg[32*i + j] = src0[i].signs[j];
-    }
-    for (int j = 0; j < 8; ++j) {
-        dst_qh[8*i + j] = src0[i].qh[j];
-    }
-    for (int j = 0; j < 4; ++j) {
-        dst_sc[4*i + j] = src0[i].scales[j];
-    }
-}
-
-kernel void kernel_restore_block_iq3_s(
-    global uchar * src_qs,
-    global uchar * src_sg,
-    global uchar * src_qh,
-    global uchar * src_sc,
-    global half * src_d,
-    global struct block_iq3_s * dst,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst[i].d = src_d[i];
-    for (int j = 0; j < 64; ++j) {
-        dst[i].qs[j] = src_qs[64*i + j];
-    }
-    for (int j = 0; j < 32; ++j) {
-        dst[i].signs[j] = src_sg[32*i + j];
-    }
-    for (int j = 0; j < 8; ++j) {
-        dst[i].qh[j] = src_qh[8*i + j];
-    }
-    for (int j = 0; j < 4; ++j) {
-        dst[i].scales[j] = src_sc[4*i + j];
-    }
-}
-
-struct block_iq3_xxs {
-    half d;
-    uchar qs[3*QK_K/8];
-};
-
-kernel void kernel_convert_block_iq3_xxs(
-    global struct block_iq3_xxs * src0,
-    global uchar * dst_qs,
-    global uint * dst_sas,
-    global half * dst_d,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst_d[i] = src0[i].d;
-    for (int j = 0; j < 64; ++j) {
-        dst_qs[64*i + j] = src0[i].qs[j];
-    }
-    for (int j = 0; j < 8; ++j) {
-        global ushort * q = (global ushort *)(src0[i].qs + 64 + 4*j);
-        dst_sas[8*i + j] = (uint)q[0] | ((uint)q[1] << 16);
-    }
-}
-
-kernel void kernel_restore_block_iq3_xxs(
-    global uchar * src_qs,
-    global uint * src_sas,
-    global half * src_d,
-    global struct block_iq3_xxs * dst,
-    ulong n_blocks
-) {
-    const ulong i = get_global_id(0);
-    if (i >= n_blocks) { return; }
-    dst[i].d = src_d[i];
-    for (int j = 0; j < 64; ++j) {
-        dst[i].qs[j] = src_qs[64*i + j];
-    }
-    for (int j = 0; j < 8; ++j) {
-        global ushort * q = (global ushort *)(dst[i].qs + 64 + 4*j);
-        uint sas = src_sas[8*i + j];
-        q[0] = (ushort)sas;
-        q[1] = (ushort)(sas >> 16);
-    }
-}

ggml/src/ggml-opencl/kernels/fwht.cl

@@ -1,44 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-kernel void kernel_fwht(
-        global const char * src,
-        ulong offset_src,
-        global char * dst,
-        ulong offset_dst,
-        int n,
-        ulong ne1,
-        ulong ne2,
-        ulong nb1,
-        ulong nb2,
-        ulong nb3,
-        float scale,
-        local float * work) {
-    const ulong row = get_group_id(1);
-    const ulong i1 = row % ne1;
-    const ulong i2 = row / ne1 % ne2;
-    const ulong i3 = row / ne1 / ne2;
-    global const SRC_TYPE * x = (global const SRC_TYPE *)(src + offset_src + i1*nb1 + i2*nb2 + i3*nb3);
-    global float * y = (global float *)(dst + offset_dst) + row*n;
-    const int tid = get_local_id(0);
-    const int nth = get_local_size(0);
-
-    for (int i = tid; i < n; i += nth) {
-        work[i] = (float)x[i] * scale;
-    }
-    barrier(CLK_LOCAL_MEM_FENCE);
-
-    for (int h = 1; h < n; h *= 2) {
-        for (int i = tid; i < n/2; i += nth) {
-            const int j = 2*i - (i & (h - 1));
-            const float a = work[j];
-            const float b = work[j + h];
-            work[j] = a + b;
-            work[j + h] = a - b;
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    for (int i = tid; i < n; i += nth) {
-        y[i] = work[i];
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_m_f32.cl

@@ -1,93 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_qcom_reqd_sub_group_size
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-__attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-kernel void kernel_gemm_noshuffle_iq1_m_f32(
-    global const uchar * src0_qs,
-    global const uchar * src0_qh,
-    global const ushort * src0_sc,
-    read_only image1d_buffer_t src1,
-    global float * dst,
-    ulong offsetd,
-    int m,
-    int n,
-    int k,
-    int n_no_padding
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-    const int gy = get_global_id(0);
-    const int row = get_global_id(1) * 4;
-    const int n_4 = n / 4;
-    if (row >= m) {
-        return;
-    }
-
-    half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
-    for (int i = 0; i < k; i += 16) {
-        const uint4 w0 = convert_uint4(vload4(0, src0_sc + row + (4*(i / 256) + 0) * m));
-        const uint4 w1 = convert_uint4(vload4(0, src0_sc + row + (4*(i / 256) + 1) * m));
-        const uint4 w2 = convert_uint4(vload4(0, src0_sc + row + (4*(i / 256) + 2) * m));
-        const uint4 w3 = convert_uint4(vload4(0, src0_sc + row + (4*(i / 256) + 3) * m));
-        const ushort4 bits = convert_ushort4((w0 >> (uint4)(12)) | ((w1 >> (uint4)(8)) & (uint4)(0x00f0))
-                                           | ((w2 >> (uint4)(4)) & (uint4)(0x0f00)) | (w3 & (uint4)(0xf000)));
-        const half4 d = as_half4(bits);
-        const uint4 sc = convert_uint4(vload4(0, src0_sc + row + (i / 64) * m));
-        const half4 scale = d * convert_half4(2*((sc >> (uint4)(3*((i / 16) & 3))) & (uint4)(7)) + 1);
-        const uint4 qh = convert_uint4(vload4(0, src0_qh + row + (i / 16) * m));
-
-        for (int l = 0; l < 16; l += 4) {
-            const int ki = i + l;
-            const uint4 gi = convert_uint4(vload4(0, src0_qs + row + (ki / 8) * m))
-                           | (((qh >> (uint4)(4*((ki / 8) & 1))) & (uint4)(7)) << (uint4)(8));
-            const uint4 grid = (uint4)(iq1_s_grid[gi.s0], iq1_s_grid[gi.s1], iq1_s_grid[gi.s2], iq1_s_grid[gi.s3]);
-            const float4 delta = select((float4)(-0.875f), (float4)(-1.125f), (qh & (uint4)(0x08 << (4*((ki / 8) & 1)))) != 0);
-            #pragma unroll
-            for (int j = 0; j < 4; ++j) {
-                half8 b;
-                b.s0123 = read_imageh(src1, gy * 2     + (ki + j) * n_4);
-                b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
-                const float4 magnitude = convert_float4((grid >> (uint4)(8*j + (ki & 4))) & (uint4)(15));
-                const half4 w = convert_half4(magnitude + delta) * scale;
-                c0 += b * w.s0;
-                c1 += b * w.s1;
-                c2 += b * w.s2;
-                c3 += b * w.s3;
-            }
-        }
-    }
-
-    int idx = gy * 8 * m + row;
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_m_q8_1_dp4a.cl

@@ -1,144 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#pragma OPENCL EXTENSION cl_khr_subgroups : enable
-#ifdef cl_khr_integer_dot_product
-#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
-#endif
-
-#define QK_K 256
-
-#ifndef TILESIZE_N
-#define TILESIZE_N 32
-#endif
-
-inline uint iq1_m_pack(uint g, uint shift, uint negative_delta) {
-    // 8*(grid + delta) is an exact signed integer in [-9, 9].
-    const int bias = negative_delta ? -9 : -7;
-    const int4 values = 8*convert_int4(as_uchar4((g >> shift) & 0x0F0F0F0Fu)) + bias;
-    return as_uint(convert_char4(values));
-}
-
-// Pass staged operands by value; local dp4a operands miscompile on X2.
-inline int dot4_q8a_v(uint4 qw, uint4 a) {
-    int r = 0;
-    r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r);
-    return r;
-}
-
-__attribute__((qcom_wave_pair_mode(1)))
-kernel void kernel_gemm_noshuffle_iq1_m_q8_1_dp4a(
-        __global const uchar  * src0_qs,
-        __global const uchar  * src0_qh,
-        __global const ushort * src0_sc,
-        __global const uint   * src1_qa,
-        __global const half   * src1_da,
-        __global       float  * dst,
-        ulong  offsetd,
-        int    m,
-        int    n_no_padding,
-        int    k
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-
-    const uint lid = get_local_id(0);
-    const uint block_id_m = get_global_id(1);
-    const uint block_id_n = get_global_id(2);
-
-    const uint row      = block_id_m * 64 + lid;
-    const uint col_base = block_id_n * TILESIZE_N;
-    const bool row_valid = row < (uint)m;
-    const uint rrow     = row_valid ? row : 0;
-
-    const uint k_u = (uint)k >> 2;
-    const uint k_b = (uint)k >> 5;
-
-    __local uint4 sh_qa4[TILESIZE_N][2];
-    __local half sh_d[TILESIZE_N];
-
-#define IQ1_M_GRID(i) iq1_s_grid[(i)]
-
-#define NGROUPS (TILESIZE_N / 4)
-    float4 acc[NGROUPS];
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
-
-    float dsuper = 0.f;
-    for (uint step = 0; step < (uint)k; step += 32) {
-        const uint sub = step >> 5;
-        const uint ib  = sub >> 3;
-
-        if ((sub & 7u) == 0u) {
-            const uint sb = rrow + 4u*ib*(uint)m;
-            const uint w0 = src0_sc[sb + 0u*(uint)m];
-            const uint w1 = src0_sc[sb + 1u*(uint)m];
-            const uint w2 = src0_sc[sb + 2u*(uint)m];
-            const uint w3 = src0_sc[sb + 3u*(uint)m];
-            const ushort bits = (ushort)((w0 >> 12) | ((w1 >> 8) & 0x00f0u)
-                                       | ((w2 >> 4) & 0x0f00u) | (w3 & 0xf000u));
-            dsuper = (float)as_half(bits);
-        }
-        const uint sc = src0_sc[rrow + (4u*ib + (sub & 7u)/2u)*(uint)m];
-        const uint sh = 6u*(sub & 1u);
-        const float dl0 = dsuper * (float)(2u*((sc >> sh) & 7u) + 1u) * 0.125f;
-        const float dl1 = dsuper * (float)(2u*((sc >> (sh + 3u)) & 7u) + 1u) * 0.125f;
-        const uint h0 = src0_qh[rrow + (2u*sub + 0u)*(uint)m];
-        const uint h1 = src0_qh[rrow + (2u*sub + 1u)*(uint)m];
-        const uint gb = rrow + 4u*sub*(uint)m;
-        const uint g0 = IQ1_M_GRID((uint)src0_qs[gb + 0u*(uint)m] | (( h0       & 7u) << 8));
-        const uint g1 = IQ1_M_GRID((uint)src0_qs[gb + 1u*(uint)m] | (((h0 >> 4) & 7u) << 8));
-        const uint g2 = IQ1_M_GRID((uint)src0_qs[gb + 2u*(uint)m] | (( h1       & 7u) << 8));
-        const uint g3 = IQ1_M_GRID((uint)src0_qs[gb + 3u*(uint)m] | (((h1 >> 4) & 7u) << 8));
-        const uint4 qlo = (uint4)(iq1_m_pack(g0, 0, h0 & 0x08u), iq1_m_pack(g0, 4, h0 & 0x08u),
-                                 iq1_m_pack(g1, 0, h0 & 0x80u), iq1_m_pack(g1, 4, h0 & 0x80u));
-        const uint4 qhi = (uint4)(iq1_m_pack(g2, 0, h1 & 0x08u), iq1_m_pack(g2, 4, h1 & 0x08u),
-                                 iq1_m_pack(g3, 0, h1 & 0x80u), iq1_m_pack(g3, 4, h1 & 0x80u));
-
-        for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
-            const uint t = idx >> 1;
-            const uint v = idx & 1;
-            const uint c = col_base + t;
-            sh_qa4[t][v] = (c < (uint)n_no_padding)
-                         ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
-                         : (uint4)(0u);
-        }
-        if (lid < TILESIZE_N) {
-            const uint c = col_base + lid;
-            sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
-#define IQ1_M_COL(b) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) \
-                           + dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1])))
-        #pragma unroll
-        for (int g = 0; g < NGROUPS; ++g) {
-            const int b = g * 4;
-            float4 rf;
-            rf.s0 = IQ1_M_COL(b+0);  rf.s1 = IQ1_M_COL(b+1);
-            rf.s2 = IQ1_M_COL(b+2);  rf.s3 = IQ1_M_COL(b+3);
-            acc[g] += LD4(sh_d, b) * rf;
-        }
-#undef IQ1_M_COL
-#undef LD4
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    if (!row_valid) {
-        return;
-    }
-
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) {
-        const uint b = (uint)(g * 4);
-        const float4 a = acc[g];
-        const uint c0 = col_base + b;
-        if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
-        if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
-        if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
-        if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
-    }
-#undef NGROUPS
-#undef IQ1_M_GRID
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_s_f32.cl

@@ -1,86 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_qcom_reqd_sub_group_size
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-__attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-kernel void kernel_gemm_noshuffle_iq1_s_f32(
-    global const uchar * src0_qs,
-    global const ushort * src0_qh,
-    global const half  * src0_d,
-    read_only image1d_buffer_t src1,
-    global float * dst,
-    ulong offsetd,
-    int m,
-    int n,
-    int k,
-    int n_no_padding
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-    const int gy = get_global_id(0);
-    const int row = get_global_id(1) * 4;
-    const int n_4 = n / 4;
-    if (row >= m) {
-        return;
-    }
-
-    half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
-    for (int i = 0; i < k; i += 16) {
-        const uint4 qh = convert_uint4(vload4(0, src0_qh + row + (i / 32) * m));
-        const half4 d = vload4(0, src0_d + row + (i / 256) * m);
-        const half4 scale = d * convert_half4(2*((qh >> (uint4)(12)) & (uint4)(7)) + 1);
-        const float4 delta = select((float4)(-0.875f), (float4)(-1.125f), (qh & (uint4)(0x8000)) != 0);
-
-        for (int l = 0; l < 16; l += 4) {
-            const int ki = i + l;
-            const uint4 gi = convert_uint4(vload4(0, src0_qs + row + (ki / 8) * m))
-                           | (((qh >> (uint4)(3*((ki / 8) & 3))) & (uint4)(7)) << (uint4)(8));
-            const uint4 grid = (uint4)(iq1_s_grid[gi.s0], iq1_s_grid[gi.s1], iq1_s_grid[gi.s2], iq1_s_grid[gi.s3]);
-            #pragma unroll
-            for (int j = 0; j < 4; ++j) {
-                half8 b;
-                b.s0123 = read_imageh(src1, gy * 2     + (ki + j) * n_4);
-                b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
-                const float4 magnitude = convert_float4((grid >> (uint4)(8*j + (ki & 4))) & (uint4)(15));
-                const half4 w = convert_half4(magnitude + delta) * scale;
-                c0 += b * w.s0;
-                c1 += b * w.s1;
-                c2 += b * w.s2;
-                c3 += b * w.s3;
-            }
-        }
-    }
-
-    int idx = gy * 8 * m + row;
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq1_s_q8_1_dp4a.cl

@@ -1,138 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#pragma OPENCL EXTENSION cl_khr_subgroups : enable
-#ifdef cl_khr_integer_dot_product
-#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
-#endif
-
-#define QK_K 256
-
-#ifndef TILESIZE_N
-#define TILESIZE_N 32
-#endif
-
-// Pass staged operands by value; local dp4a operands miscompile on X2.
-inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
-    int r = 0;
-    r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
-    return r;
-}
-
-__attribute__((qcom_wave_pair_mode(1)))
-kernel void kernel_gemm_noshuffle_iq1_s_q8_1_dp4a(
-        __global const uchar  * src0_qs,
-        __global const ushort * src0_qh,
-        __global const half   * src0_d,
-        __global const uint   * src1_qa,
-        __global const half   * src1_da,
-        __global const half   * src1_sa,
-        __global       float  * dst,
-        ulong  offsetd,
-        int    m,
-        int    n_no_padding,
-        int    k
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-
-    const uint lid = get_local_id(0);
-    const uint block_id_m = get_global_id(1);
-    const uint block_id_n = get_global_id(2);
-
-    const uint row      = block_id_m * 64 + lid;
-    const uint col_base = block_id_n * TILESIZE_N;
-    const bool row_valid = row < (uint)m;
-    const uint rrow     = row_valid ? row : 0;
-
-    const uint k_u = (uint)k >> 2;
-    const uint k_b = (uint)k >> 5;
-
-    __local uint4 sh_qa4[TILESIZE_N][2];
-    __local half sh_da[TILESIZE_N];
-    __local half sh_sa[TILESIZE_N];
-
-#define IQ1_S_GRID(i) iq1_s_grid[(i)]
-
-#define NGROUPS (TILESIZE_N / 4)
-    float4 acc[NGROUPS];
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
-
-    for (uint step = 0; step < (uint)k; step += 32) {
-        const uint sub = step >> 5;
-        const uint ib  = sub >> 3;
-
-        const uint  qhv = (uint)src0_qh[rrow + sub * (uint)m];
-        const float dl  = (float)src0_d[rrow + ib * (uint)m]
-                        * (float)(2u * ((qhv >> 12) & 7u) + 1u);
-
-        // The codebook stores grid + 1; sa includes the Q8_1 activation scale.
-        const float dlt = ((qhv & 0x8000u) ? -0.125f : 0.125f) - 1.0f;
-
-        const uint gb = rrow + (step >> 3) * (uint)m;
-
-        uint8 qw;
-        {
-            const uint g0 = IQ1_S_GRID((uint)src0_qs[gb + 0u * (uint)m] | ((( qhv       & 7u) << 8)));
-            const uint g1 = IQ1_S_GRID((uint)src0_qs[gb + 1u * (uint)m] | ((((qhv >> 3) & 7u) << 8)));
-            const uint g2 = IQ1_S_GRID((uint)src0_qs[gb + 2u * (uint)m] | ((((qhv >> 6) & 7u) << 8)));
-            const uint g3 = IQ1_S_GRID((uint)src0_qs[gb + 3u * (uint)m] | ((((qhv >> 9) & 7u) << 8)));
-            qw.s0 =  g0        & 0x0F0F0F0Fu;   qw.s1 = (g0 >> 4) & 0x0F0F0F0Fu;
-            qw.s2 =  g1        & 0x0F0F0F0Fu;   qw.s3 = (g1 >> 4) & 0x0F0F0F0Fu;
-            qw.s4 =  g2        & 0x0F0F0F0Fu;   qw.s5 = (g2 >> 4) & 0x0F0F0F0Fu;
-            qw.s6 =  g3        & 0x0F0F0F0Fu;   qw.s7 = (g3 >> 4) & 0x0F0F0F0Fu;
-        }
-
-        for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
-            const uint t = idx >> 1;
-            const uint v = idx & 1;
-            const uint c = col_base + t;
-            sh_qa4[t][v] = (c < (uint)n_no_padding)
-                         ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
-                         : (uint4)(0u);
-        }
-        if (lid < TILESIZE_N) {
-            const uint c = col_base + lid;
-            const bool ok = c < (uint)n_no_padding;
-            sh_da[lid] = ok ? src1_da[c * k_b + sub] : (half)0;
-            sh_sa[lid] = ok ? src1_sa[c * k_b + sub] : (half)0;
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
-        #pragma unroll
-        for (int g = 0; g < NGROUPS; ++g) {
-            const int b = g * 4;
-            float4 rf;
-#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1]))
-            rf.s0 = (float)TDOT4(b+0);  rf.s1 = (float)TDOT4(b+1);
-            rf.s2 = (float)TDOT4(b+2);  rf.s3 = (float)TDOT4(b+3);
-            acc[g] += dl * (LD4(sh_da, b) * rf) + (dl * dlt) * LD4(sh_sa, b);
-        }
-#undef TDOT4
-#undef LD4
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    if (!row_valid) {
-        return;
-    }
-
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) {
-        const uint b = (uint)(g * 4);
-        const float4 a = acc[g];
-        const uint c0 = col_base + b;
-        if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
-        if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
-        if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
-        if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
-    }
-#undef NGROUPS
-#undef IQ1_S_GRID
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_f32.cl

@@ -1,92 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_qcom_reqd_sub_group_size
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-__attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-kernel void kernel_gemm_noshuffle_iq3_s_f32(
-    global const uchar * src0_qs,
-    global const uchar * src0_sg,
-    global const uchar * src0_qh,
-    global const uchar * src0_sc,
-    global const half  * src0_d,
-    read_only image1d_buffer_t src1,
-    global float * dst,
-    ulong offsetd,
-    int m,
-    int n,
-    int k,
-    int n_no_padding
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-    const int gy = get_global_id(0);
-    const int row = get_global_id(1) * 4;
-    const int n_4 = n / 4;
-    if (row >= m) {
-        return;
-    }
-
-    half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
-    for (int i = 0; i < k; i += 16) {
-        const uchar4 sc = vload4(0, src0_sc + row + (i / 64) * m);
-        const half4 d = vload4(0, src0_d + row + (i / 256) * m);
-        const float4 scale_f = convert_float4(d) * (1.0f + 2.0f*convert_float4((sc >> (uchar4)(4*((i / 32) & 1))) & (uchar4)(15)));
-        const half4 scale = convert_half4(scale_f);
-
-        for (int l = 0; l < 16; l += 4) {
-            const int ki = i + l;
-            const uint4 qs = convert_uint4(vload4(0, src0_qs + row + (ki / 4) * m));
-            const uint4 qh = convert_uint4(vload4(0, src0_qh + row + (ki / 32) * m));
-            const uchar4 sg = vload4(0, src0_sg + row + (ki / 8) * m);
-            const uint4 gi = qs | (((qh >> (uint4)(((ki / 4) & 7))) & (uint4)(1)) << (uint4)(8));
-            const uint4 grid = (uint4)(iq3_s_grid[gi.s0], iq3_s_grid[gi.s1],
-                                      iq3_s_grid[gi.s2], iq3_s_grid[gi.s3]);
-            #pragma unroll
-            for (int j = 0; j < 4; ++j) {
-                half8 b;
-                b.s0123 = read_imageh(src1, gy * 2     + (ki + j) * n_4);
-                b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
-                const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255));
-                const int4 signs = convert_int4((sg >> (uchar4)((ki & 7) + j)) & (uchar4)(1));
-                const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale;
-                c0 += b * w.s0;
-                c1 += b * w.s1;
-                c2 += b * w.s2;
-                c3 += b * w.s3;
-            }
-        }
-    }
-
-    int idx = gy * 8 * m + row;
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_s_q8_1_dp4a.cl

@@ -1,154 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#pragma OPENCL EXTENSION cl_khr_subgroups : enable
-#ifdef cl_khr_integer_dot_product
-#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
-#endif
-
-#define QK_K 256
-
-#ifndef TILESIZE_N
-#define TILESIZE_N 32
-#endif
-
-inline uint iq3_s_pack(uint gv, uint sg, uint base) {
-    int v0 = (int)((gv >>  0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; }
-    int v1 = (int)((gv >>  8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; }
-    int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; }
-    int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; }
-    return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) <<  8)
-         | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24);
-}
-
-#define IQ3_S_GEMM_GRID(i) iq3_s_grid[(i)]
-
-inline uint iq3_s_load(__global const uchar * qs, uint qsb, uint qhb, uint m,
-                      uint u, uint sgv, uint base) {
-    const uint gidx = (uint)qs[qsb + u * m] | (((qhb >> u) & 1u) << 8);
-    return iq3_s_pack(IQ3_S_GEMM_GRID(gidx), sgv, base);
-}
-
-// Pass staged operands by value; local dp4a operands miscompile on X2.
-inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
-    int r = 0;
-    r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
-    return r;
-}
-
-__attribute__((qcom_wave_pair_mode(1)))
-kernel void kernel_gemm_noshuffle_iq3_s_q8_1_dp4a(
-        __global const uchar  * src0_qs,
-        __global const uchar  * src0_sg,
-        __global const uchar  * src0_qh,
-        __global const uchar  * src0_sc,
-        __global const half   * src0_d,
-        __global const uint   * src1_qa,
-        __global const half   * src1_da,
-        __global       float  * dst,
-        ulong  offsetd,
-        int    m,
-        int    n_no_padding,
-        int    k
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-
-    const uint lid = get_local_id(0);
-    const uint block_id_m = get_global_id(1);
-    const uint block_id_n = get_global_id(2);
-
-    const uint row      = block_id_m * 64 + lid;
-    const uint col_base = block_id_n * TILESIZE_N;
-    const bool row_valid = row < (uint)m;
-    const uint rrow     = row_valid ? row : 0;
-
-    const uint k_u = (uint)k >> 2;
-    const uint k_b = (uint)k >> 5;
-
-    __local uint4 sh_qa4[TILESIZE_N][2];
-    __local half sh_d[TILESIZE_N];
-
-#define NGROUPS (TILESIZE_N / 4)
-    float4 acc[NGROUPS];
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
-
-    for (uint step = 0; step < (uint)k; step += 32) {
-        const uint sub = step >> 5;
-
-        const uint ib = sub >> 3;
-        const uint sb = sub & 7u;
-
-        const uint  scb = (uint)src0_sc[rrow + (ib*(QK_K/64) + (sb >> 1)) * (uint)m];
-        const uint  nib = (sb & 1u) ? (scb >> 4) : (scb & 0xFu);
-        const float d_w = (float)src0_d[rrow + ib * (uint)m] * (float)(1 + 2*nib);
-
-        const uint qhb  = (uint)src0_qh[rrow + (ib*(QK_K/32) + sb) * (uint)m];
-        const uint qsb  = rrow + (ib*(QK_K/4) + 8*sb) * (uint)m;
-        const uint sgb  = rrow + (ib*(QK_K/8) + 4*sb) * (uint)m;
-
-        const uint sg0 = (uint)src0_sg[sgb + 0 * (uint)m];
-        const uint sg1 = (uint)src0_sg[sgb + 1 * (uint)m];
-        const uint sg2 = (uint)src0_sg[sgb + 2 * (uint)m];
-        const uint sg3 = (uint)src0_sg[sgb + 3 * (uint)m];
-
-        uint8 qw;
-        qw.s0 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 0u, sg0, 0u);
-        qw.s1 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 1u, sg0, 4u);
-        qw.s2 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 2u, sg1, 0u);
-        qw.s3 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 3u, sg1, 4u);
-        qw.s4 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 4u, sg2, 0u);
-        qw.s5 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 5u, sg2, 4u);
-        qw.s6 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 6u, sg3, 0u);
-        qw.s7 = iq3_s_load(src0_qs, qsb, qhb, (uint)m, 7u, sg3, 4u);
-
-        for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
-            const uint t = idx >> 1;
-            const uint v = idx & 1;
-            const uint c = col_base + t;
-            sh_qa4[t][v] = (c < (uint)n_no_padding)
-                         ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
-                         : (uint4)(0u);
-        }
-        if (lid < TILESIZE_N) {
-            const uint c = col_base + lid;
-            sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
-        #pragma unroll
-        for (int g = 0; g < NGROUPS; ++g) {
-            const int b = g * 4;
-            float4 rf;
-#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1]))
-            rf.s0 = (float)TDOT4(b+0);  rf.s1 = (float)TDOT4(b+1);
-            rf.s2 = (float)TDOT4(b+2);  rf.s3 = (float)TDOT4(b+3);
-            acc[g] += d_w * LD4(sh_d, b) * rf;
-        }
-#undef TDOT4
-#undef LD4
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    if (!row_valid) {
-        return;
-    }
-
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) {
-        const uint b = (uint)(g * 4);
-        const float4 a = acc[g];
-        const uint c0 = col_base + b;
-        if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
-        if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
-        if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
-        if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
-    }
-#undef NGROUPS
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_f32.cl

@@ -1,89 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_qcom_reqd_sub_group_size
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-__attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-kernel void kernel_gemm_noshuffle_iq3_xxs_f32(
-    global const uchar * src0_qs,
-    global const uint * src0_sas,
-    global const half  * src0_d,
-    read_only image1d_buffer_t src1,
-    global float * dst,
-    ulong offsetd,
-    int m,
-    int n,
-    int k,
-    int n_no_padding
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-    const int gy = get_global_id(0);
-    const int row = get_global_id(1) * 4;
-    const int n_4 = n / 4;
-    if (row >= m) {
-        return;
-    }
-
-    half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
-    for (int i = 0; i < k; i += 16) {
-        const uint4 sas = vload4(0, src0_sas + row + (i / 32) * m);
-        const half4 d = vload4(0, src0_d + row + (i / 256) * m);
-        const float4 scale_f = convert_float4(d) * 0.5f * (0.5f + convert_float4(sas >> (uint4)(28)));
-        const half4 scale = convert_half4(scale_f);
-
-        for (int l = 0; l < 16; l += 4) {
-            const int ki = i + l;
-            const uint4 gi = convert_uint4(vload4(0, src0_qs + row + (ki / 4) * m));
-            const uint4 code = (sas >> (uint4)(7*((ki / 8) & 3))) & (uint4)(127);
-            const uint4 sg = code | ((popcount(code) & (uint4)(1)) << (uint4)(7));
-            const uint4 grid = (uint4)(iq3_xxs_grid[gi.s0], iq3_xxs_grid[gi.s1],
-                                      iq3_xxs_grid[gi.s2], iq3_xxs_grid[gi.s3]);
-            #pragma unroll
-            for (int j = 0; j < 4; ++j) {
-                half8 b;
-                b.s0123 = read_imageh(src1, gy * 2     + (ki + j) * n_4);
-                b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
-                const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255));
-                const int4 signs = convert_int4((sg >> (uint4)((ki & 7) + j)) & (uint4)(1));
-                const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale;
-                c0 += b * w.s0;
-                c1 += b * w.s1;
-                c2 += b * w.s2;
-                c3 += b * w.s3;
-            }
-        }
-    }
-
-    int idx = gy * 8 * m + row;
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
-        idx += m;
-    }
-    if (idx + 3 < m * n_no_padding) {
-        vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
-    }
-}

ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq3_xxs_q8_1_dp4a.cl

@@ -1,137 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#pragma OPENCL EXTENSION cl_khr_subgroups : enable
-#ifdef cl_khr_integer_dot_product
-#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
-#endif
-
-#define QK_K 256
-
-#ifndef TILESIZE_N
-#define TILESIZE_N 32
-#endif
-
-inline uint iq3_xxs_pack(uint gv, uint sg, uint base) {
-    int v0 = (int)((gv >>  0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; }
-    int v1 = (int)((gv >>  8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; }
-    int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; }
-    int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; }
-    return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) <<  8)
-         | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24);
-}
-
-// Pass staged operands by value; local dp4a operands miscompile on X2.
-inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
-    int r = 0;
-    r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
-    r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
-    return r;
-}
-
-__attribute__((qcom_wave_pair_mode(1)))
-kernel void kernel_gemm_noshuffle_iq3_xxs_q8_1_dp4a(
-        __global const uchar  * src0_qs,
-        __global const uint   * src0_sas,
-        __global const half   * src0_d,
-        __global const uint   * src1_qa,
-        __global const half   * src1_da,
-        __global       float  * dst,
-        ulong  offsetd,
-        int    m,
-        int    n_no_padding,
-        int    k
-) {
-    dst = (global float *)((global char *)dst + offsetd);
-
-    const uint lid = get_local_id(0);
-    const uint block_id_m = get_global_id(1);
-    const uint block_id_n = get_global_id(2);
-
-    const uint row      = block_id_m * 64 + lid;
-    const uint col_base = block_id_n * TILESIZE_N;
-    const bool row_valid = row < (uint)m;
-    const uint rrow     = row_valid ? row : 0;
-
-    const uint k_u = (uint)k >> 2;
-    const uint k_b = (uint)k >> 5;
-
-    __local uint4 sh_qa4[TILESIZE_N][2];
-    __local half sh_d[TILESIZE_N];
-
-#define NGROUPS (TILESIZE_N / 4)
-    float4 acc[NGROUPS];
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
-
-    for (uint step = 0; step < (uint)k; step += 32) {
-        const uint sub = step >> 5;
-
-        const uint ib = sub >> 3;
-        const uint aux = src0_sas[rrow + sub*(uint)m];
-        const float d_w = (float)src0_d[rrow + ib*(uint)m] * (0.5f + (float)(aux >> 28)) * 0.5f;
-        const uint sg0 = iq3_xxs_signs( aux        & 127u);
-        const uint sg1 = iq3_xxs_signs((aux >>  7) & 127u);
-        const uint sg2 = iq3_xxs_signs((aux >> 14) & 127u);
-        const uint sg3 = iq3_xxs_signs((aux >> 21) & 127u);
-        const uint qsb = rrow + sub*8u*(uint)m;
-
-        uint8 qw;
-        qw.s0 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 0u*(uint)m]], sg0, 0u);
-        qw.s1 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 1u*(uint)m]], sg0, 4u);
-        qw.s2 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 2u*(uint)m]], sg1, 0u);
-        qw.s3 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 3u*(uint)m]], sg1, 4u);
-        qw.s4 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 4u*(uint)m]], sg2, 0u);
-        qw.s5 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 5u*(uint)m]], sg2, 4u);
-        qw.s6 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 6u*(uint)m]], sg3, 0u);
-        qw.s7 = iq3_xxs_pack(iq3_xxs_grid[src0_qs[qsb + 7u*(uint)m]], sg3, 4u);
-
-        for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
-            const uint t = idx >> 1;
-            const uint v = idx & 1;
-            const uint c = col_base + t;
-            sh_qa4[t][v] = (c < (uint)n_no_padding)
-                         ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
-                         : (uint4)(0u);
-        }
-        if (lid < TILESIZE_N) {
-            const uint c = col_base + lid;
-            sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
-        #pragma unroll
-        for (int g = 0; g < NGROUPS; ++g) {
-            const int b = g * 4;
-            float4 rf;
-#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1]))
-            rf.s0 = (float)TDOT4(b+0);  rf.s1 = (float)TDOT4(b+1);
-            rf.s2 = (float)TDOT4(b+2);  rf.s3 = (float)TDOT4(b+3);
-            acc[g] += d_w * LD4(sh_d, b) * rf;
-        }
-#undef TDOT4
-#undef LD4
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    if (!row_valid) {
-        return;
-    }
-
-    #pragma unroll
-    for (int g = 0; g < NGROUPS; ++g) {
-        const uint b = (uint)(g * 4);
-        const float4 a = acc[g];
-        const uint c0 = col_base + b;
-        if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
-        if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
-        if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
-        if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
-    }
-#undef NGROUPS
-}

ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq1_m_f32.cl

@@ -1,134 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#define QK_K 256
-#define IQ1_M_DELTA 0.125f
-
-#ifndef IQ1_M_MV_NSG
-#define IQ1_M_MV_NSG 4
-#endif
-
-inline float4 iq1_m_vals(uint g) {
-    return (float4)((float)( g        & 0xFu), (float)((g >>  8) & 0xFu),
-                    (float)((g >> 16) & 0xFu), (float)((g >> 24) & 0xFu));
-}
-
-inline float iq1_m_super(uint w0, uint w1, uint w2, uint w3) {
-    const ushort u16 = (ushort)(((w0 >> 12)) | ((w1 >> 8) & 0x00f0u)
-                              | ((w2 >> 4) & 0x0f00u) | (w3 & 0xf000u));
-    return (float)as_half(u16);
-}
-
-kernel void kernel_gemv_noshuffle_iq1_m_f32(
-        __read_only image1d_buffer_t grid_img,
-        global const uchar  * src0_qs,
-        global const uchar  * src0_qh,
-        global const ushort * src0_sc,
-        global const float  * src1,
-        ulong offset1,
-        global float * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        int ne10,
-        int ne0
-) {
-    src1 = (global const float *)((global const char *)src1 + offset1);
-    dst  = (global float       *)((global char       *)dst  + offsetd);
-
-    const uint m   = (uint)ne01;
-    const uint K   = (uint)ne00;
-    const uint nsb = K / QK_K;
-
-    const uint lid = get_local_id(0);
-    const uint sgi = get_local_id(1);
-    const uint col = get_group_id(1);
-
-    global const float * y = src1 + (ulong)col * (uint)ne10;
-
-#define IQ1_M_GRID(i) (read_imageui(grid_img, (int)(i)).x)
-
-    const uint mh  = m >> 1;
-    const uint j   = get_group_id(0) * 64u + lid;
-    const uint row = j << 1;
-
-    float sumf = 0.f, sumf1 = 0.f;
-
-    if (j < mh) {
-        global const ushort * qsu = (global const ushort *)src0_qs;
-        global const ushort * qhu = (global const ushort *)src0_qh;
-        global const uint   * scu = (global const uint   *)src0_sc;
-
-        for (uint ib = sgi; ib < nsb; ib += IQ1_M_MV_NSG) {
-            const uint sb = j + (ib * 4u) * mh;
-            const uint w0 = scu[sb + 0u * mh];
-            const uint w1 = scu[sb + 1u * mh];
-            const uint w2 = scu[sb + 2u * mh];
-            const uint w3 = scu[sb + 3u * mh];
-            const float d0 = iq1_m_super( w0        & 0xFFFFu,  w1        & 0xFFFFu,
-                                         w2        & 0xFFFFu,  w3        & 0xFFFFu);
-            const float d1 = iq1_m_super((w0 >> 16) & 0xFFFFu, (w1 >> 16) & 0xFFFFu,
-                                        (w2 >> 16) & 0xFFFFu, (w3 >> 16) & 0xFFFFu);
-
-            float acc0 = 0.f, acc1 = 0.f;
-
-            for (uint sub_l = 0; sub_l < 8u; ++sub_l) {
-                const uint sub = ib * 8u + sub_l;
-                const uint scw = scu[j + (ib * 4u + (sub_l >> 1)) * mh];
-                const uint grp = sub * 8u;
-                const uint shb = 3u * (2u * (sub_l & 1u));
-
-                for (uint l = 0; l < 4u; ++l) {
-                    const uint sh  = shb + 3u * (l >> 1);
-                    const float s0 = (float)(2u * (((scw       ) >> sh) & 7u) + 1u);
-                    const float s1 = (float)(2u * (((scw >> 16) >> sh) & 7u) + 1u);
-
-                    const uint qhv = (uint)qhu[j + (sub * 2u + (l >> 1)) * mh];
-                    const uint h0  =  qhv       & 0xFFu;
-                    const uint h1  = (qhv >> 8) & 0xFFu;
-                    const uint qsv = (uint)qsu[j + (sub * 4u + l) * mh];
-
-                    const uint msk = 0x08u << (4u * (l & 1u));
-                    const float t0 = (h0 & msk) ? (-1.f - IQ1_M_DELTA) : (-1.f + IQ1_M_DELTA);
-                    const float t1 = (h1 & msk) ? (-1.f - IQ1_M_DELTA) : (-1.f + IQ1_M_DELTA);
-
-                    const uint g0 = IQ1_M_GRID(( qsv       & 0xFFu)
-                                              | ((((h0 >> (4u*(l & 1u))) & 7u) << 8)));
-                    const uint g1 = IQ1_M_GRID(((qsv >> 8) & 0xFFu)
-                                              | ((((h1 >> (4u*(l & 1u))) & 7u) << 8)));
-
-                    const float4 y0 = vload4(grp + 2u*l + 0u, y);
-                    const float4 y1 = vload4(grp + 2u*l + 1u, y);
-
-                    const float4 ys = y0 + y1;
-                    const float  as = ys.s0 + ys.s1 + ys.s2 + ys.s3;
-
-                    const float a0 = dot(y0, iq1_m_vals(g0)) + dot(y1, iq1_m_vals(g0 >> 4));
-                    const float a1 = dot(y0, iq1_m_vals(g1)) + dot(y1, iq1_m_vals(g1 >> 4));
-                    acc0 += s0 * (a0 + t0 * as);
-                    acc1 += s1 * (a1 + t1 * as);
-                }
-            }
-            sumf  += d0 * acc0;
-            sumf1 += d1 * acc1;
-        }
-    }
-
-#if IQ1_M_MV_NSG > 1
-    __local float2 part[IQ1_M_MV_NSG][64];
-    part[sgi][lid] = (float2)(sumf, sumf1);
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < IQ1_M_MV_NSG; ++s) {
-        const float2 p = part[s][lid];
-        sumf  += p.s0;
-        sumf1 += p.s1;
-    }
-#endif
-
-    if (j < mh) {
-        vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
-    }
-#undef IQ1_M_GRID
-}

ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq1_s_f32.cl

@@ -1,117 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#define QK_K 256
-
-#ifndef IQ1_S_MV_NSG
-#define IQ1_S_MV_NSG 4
-#endif
-
-inline float4 iq1_s_vals(uint g) {
-    return (float4)((float)( g        & 0xFu), (float)((g >>  8) & 0xFu),
-                    (float)((g >> 16) & 0xFu), (float)((g >> 24) & 0xFu));
-}
-
-kernel void kernel_gemv_noshuffle_iq1_s_f32(
-        __read_only image1d_buffer_t grid_img,
-        global const uchar  * src0_qs,
-        global const ushort * src0_qh,
-        global const half   * src0_d,
-        global const float  * src1,
-        ulong offset1,
-        global float * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        int ne10,
-        int ne0
-) {
-    src1 = (global const float *)((global const char *)src1 + offset1);
-    dst  = (global float       *)((global char       *)dst  + offsetd);
-
-    const uint m   = (uint)ne01;
-    const uint K   = (uint)ne00;
-    const uint nsb = K / QK_K;
-
-    const uint lid = get_local_id(0);
-    const uint sgi = get_local_id(1);
-    const uint col = get_group_id(1);
-
-    global const float * y = src1 + (ulong)col * (uint)ne10;
-
-#define IQ1_S_GRID(i) (read_imageui(grid_img, (int)(i)).x)
-
-    const uint mh  = m >> 1;
-    const uint j   = get_group_id(0) * 64u + lid;
-    const uint row = j << 1;
-
-    float sumf = 0.f, sumf1 = 0.f;
-
-    if (j < mh) {
-        global const ushort * qsu = (global const ushort *)src0_qs;
-
-        for (uint ib = sgi; ib < nsb; ib += IQ1_S_MV_NSG) {
-            const half2 dh = vload2(j + ib * mh, src0_d);
-
-            float acc0 = 0.f, acc1 = 0.f;
-            for (uint sb = 0; sb < 8u; ++sb) {
-                const uint  sub = ib * 8u + sb;
-                const ushort2 qhv = vload2(j + sub * mh, src0_qh);
-                const uint  h0  = (uint)qhv.s0, h1 = (uint)qhv.s1;
-
-                const float s0  = (float)(2u * ((h0 >> 12) & 7u) + 1u);
-                const float s1  = (float)(2u * ((h1 >> 12) & 7u) + 1u);
-                const float t0  = ((h0 & 0x8000u) ? -0.125f : 0.125f) - 1.0f;
-                const float t1  = ((h1 & 0x8000u) ? -0.125f : 0.125f) - 1.0f;
-
-                const uint gb  = j + (sub * 4u) * mh;
-                const uint grp = sub * 8u;
-
-                float a0 = 0.f, a1 = 0.f;
-                float4 as = (float4)(0.f);
-                for (uint l = 0; l < 4u; ++l) {
-                    const uint qsv = (uint)qsu[gb + l * mh];
-                    const uint g0  = IQ1_S_GRID(( qsv       & 0xFFu) | ((((h0 >> (3u*l)) & 7u) << 8)));
-                    const uint g1  = IQ1_S_GRID(((qsv >> 8) & 0xFFu) | ((((h1 >> (3u*l)) & 7u) << 8)));
-
-                    const float4 y0 = vload4(grp + 2u*l + 0u, y);
-                    const float4 y1 = vload4(grp + 2u*l + 1u, y);
-                    as += y0 + y1;
-                    a0 += dot(y0, iq1_s_vals(g0)) + dot(y1, iq1_s_vals(g0 >> 4));
-                    a1 += dot(y0, iq1_s_vals(g1)) + dot(y1, iq1_s_vals(g1 >> 4));
-                }
-
-                const float asum = as.s0 + as.s1 + as.s2 + as.s3;
-                acc0 += s0 * (a0 + t0 * asum);
-                acc1 += s1 * (a1 + t1 * asum);
-            }
-            sumf  += (float)dh.s0 * acc0;
-            sumf1 += (float)dh.s1 * acc1;
-        }
-    }
-
-#if IQ1_S_MV_NSG > 1
-    __local float2 part[IQ1_S_MV_NSG][64];
-    part[sgi][lid] = (float2)(sumf, sumf1);
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < IQ1_S_MV_NSG; ++s) {
-        const float2 p = part[s][lid];
-        sumf  += p.s0;
-        sumf1 += p.s1;
-    }
-#endif
-
-    if (j < mh) {
-        vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
-    }
-#undef IQ1_S_GRID
-}
-
-kernel void kernel_iq1_s_grid_export(global uint * out) {
-    const uint i = get_global_id(0);
-    if (i < 2048u) {
-        out[i] = iq1_s_grid[i];
-    }
-}

ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_s_f32.cl

@@ -1,128 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#define QK_K 256
-
-#ifndef IQ3_S_MV_NSG
-#define IQ3_S_MV_NSG 4
-#endif
-
-inline float4 iq3_s_vals(uint gv, uint sg, uint base) {
-
-    const uint  s   = sg >> base;
-    const uint4 sgn = (uint4)(s << 31, s << 30, s << 29, s << 28) & 0x80000000u;
-    return as_float4(as_uint4(convert_float4(as_uchar4(gv))) ^ sgn);
-}
-
-kernel void kernel_gemv_noshuffle_iq3_s_f32(
-        __read_only image1d_buffer_t grid_img,
-        global const uchar * src0_qs,
-        global const uchar * src0_sg,
-        global const uchar * src0_qh,
-        global const uchar * src0_sc,
-        global const half  * src0_d,
-        global const float * src1,
-        ulong offset1,
-        global float * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        int ne10,
-        int ne0
-) {
-    src1 = (global const float *)((global const char *)src1 + offset1);
-    dst  = (global float       *)((global char       *)dst  + offsetd);
-
-    const uint m   = (uint)ne01;
-    const uint K   = (uint)ne00;
-    const uint nsb = K / QK_K;
-
-    const uint lid = get_local_id(0);
-    const uint sgi = get_local_id(1);
-    const uint col = get_group_id(1);
-
-    global const float * y = src1 + (ulong)col * (uint)ne10;
-
-#define IQ3_S_GRID(i) (read_imageui(grid_img, (int)(i)).x)
-
-    const uint mh  = m >> 1;
-    const uint j   = get_group_id(0) * 64u + lid;
-    const uint row = j << 1;
-
-    float sumf  = 0.f;
-    float sumf1 = 0.f;
-
-    if (j < mh) {
-        global const ushort * qsu = (global const ushort *)src0_qs;
-        global const ushort * qhu = (global const ushort *)src0_qh;
-        global const ushort * sgu = (global const ushort *)src0_sg;
-        global const ushort * scu = (global const ushort *)src0_sc;
-
-        for (uint ib = sgi; ib < nsb; ib += IQ3_S_MV_NSG) {
-            const half2 dh = vload2(j + ib * mh, src0_d);
-            const float d0 = (float)dh.s0;
-            const float d1 = (float)dh.s1;
-
-            const uint scbase = j + ib * 4u * mh;
-            ushort sc4[4];
-            sc4[0] = scu[scbase + 0u * mh];
-            sc4[1] = scu[scbase + 1u * mh];
-            sc4[2] = scu[scbase + 2u * mh];
-            sc4[3] = scu[scbase + 3u * mh];
-
-            float acc0 = 0.f, acc1 = 0.f;
-            for (uint sb = 0; sb < 8u; ++sb) {
-                const uint scv  = (uint)sc4[sb >> 1];
-                const uint nib0 = (sb & 1u) ? ((scv >>  4) & 0xFu) : ( scv        & 0xFu);
-                const uint nib1 = (sb & 1u) ? ((scv >> 12) & 0xFu) : ((scv >>  8) & 0xFu);
-
-                const uint qhv = (uint)qhu[j + (ib * 8u + sb) * mh];
-                const uint qh0 = qhv & 0xFFu;
-                const uint qh1 = qhv >> 8;
-
-                const uint grp = ib * 64u + sb * 8u;
-                const uint qsb = j + grp * mh;
-                const uint sgb = j + (ib * 32u + sb * 4u) * mh;
-
-                float a0 = 0.f, a1 = 0.f;
-                for (uint u = 0; u < 8u; ++u) {
-                    const uint qsv = (uint)qsu[qsb + u * mh];
-                    const uint sgv = (uint)sgu[sgb + (u >> 1) * mh];
-                    const uint g0  = ( qsv       & 0xFFu) | (((qh0 >> u) & 1u) << 8);
-                    const uint g1  = ((qsv >> 8) & 0xFFu) | (((qh1 >> u) & 1u) << 8);
-                    const uint base = (u & 1u) * 4u;
-                    const float4 yv = vload4(grp + u, y);
-                    const uint gw0 = IQ3_S_GRID(g0);
-                    const uint gw1 = IQ3_S_GRID(g1);
-                    a0 += dot(yv, iq3_s_vals(gw0,  sgv       & 0xFFu, base));
-                    a1 += dot(yv, iq3_s_vals(gw1, (sgv >> 8) & 0xFFu, base));
-                }
-                acc0 += (float)(1u + 2u * nib0) * a0;
-                acc1 += (float)(1u + 2u * nib1) * a1;
-            }
-            sumf  += d0 * acc0;
-            sumf1 += d1 * acc1;
-        }
-    }
-
-#if IQ3_S_MV_NSG > 1
-    __local float2 part[IQ3_S_MV_NSG][64];
-    part[sgi][lid] = (float2)(sumf, sumf1);
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < IQ3_S_MV_NSG; ++s) {
-        const float2 p = part[s][lid];
-        sumf  += p.s0;
-        sumf1 += p.s1;
-    }
-#endif
-
-    if (j < mh) {
-        vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
-    }
-}
-
-kernel void kernel_iq3_s_grid_export(global uint * out) {
-    out[get_global_id(0)] = iq3_s_grid[get_global_id(0)];
-}

ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq3_xxs_f32.cl

@@ -1,120 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#define QK_K 256
-
-#ifndef IQ3_XXS_MV_NSG
-#define IQ3_XXS_MV_NSG 4
-#endif
-
-inline float4 iq3_xxs_vals(uint gv, uint sgv, uint base) {
-    const uint  s   = sgv >> base;
-    const uint4 sgn = (uint4)(s << 31, s << 30, s << 29, s << 28) & 0x80000000u;
-    return as_float4(as_uint4(convert_float4(as_uchar4(gv))) ^ sgn);
-}
-
-kernel void kernel_gemv_noshuffle_iq3_xxs_f32(
-        __read_only image1d_buffer_t grid_img,
-        global const uchar * src0_qs,
-        global const uint  * src0_sas,
-        global const half  * src0_d,
-        global const float * src1,
-        ulong offset1,
-        global float * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        int ne10,
-        int ne0
-) {
-    src1 = (global const float *)((global const char *)src1 + offset1);
-    dst  = (global float       *)((global char       *)dst  + offsetd);
-
-    const uint m   = (uint)ne01;
-    const uint K   = (uint)ne00;
-    const uint nsb = K / QK_K;
-
-    const uint lid = get_local_id(0);
-    const uint sgi = get_local_id(1);
-    const uint col = get_group_id(1);
-
-    global const float * y = src1 + (ulong)col * (uint)ne10;
-
-#define IQ3_XXS_GRID(i) (read_imageui(grid_img, (int)(i)).x)
-
-    const uint mh  = m >> 1;
-    const uint j   = get_group_id(0) * 64u + lid;
-    const uint row = j << 1;
-
-    float sumf  = 0.f;
-    float sumf1 = 0.f;
-
-    if (j < mh) {
-        global const ushort * qsu = (global const ushort *)src0_qs;
-
-        for (uint ib = sgi; ib < nsb; ib += IQ3_XXS_MV_NSG) {
-            const half2 dh = vload2(j + ib * mh, src0_d);
-            const float d0 = (float)dh.s0;
-            const float d1 = (float)dh.s1;
-
-            float acc0 = 0.f, acc1 = 0.f;
-            for (uint sb = 0; sb < 8u; ++sb) {
-                const uint  sub = ib * 8u + sb;
-                const uint2 aux = vload2(j + sub * mh, src0_sas);
-
-                const uint grp = sub * 8u;
-                const uint qsb = j + grp * mh;
-
-                float a0 = 0.f, a1 = 0.f;
-                for (uint u = 0; u < 4u; ++u) {
-                    const uint qsv0 = (uint)qsu[qsb + (2u*u)*mh];
-                    const uint qsv1 = (uint)qsu[qsb + (2u*u + 1u)*mh];
-                    const uint sg0 = iq3_xxs_signs((aux.s0 >> (7u * u)) & 127u);
-                    const uint sg1 = iq3_xxs_signs((aux.s1 >> (7u * u)) & 127u);
-
-                    const uint g00 =  qsv0       & 0xFFu;
-                    const uint g01 =  qsv1       & 0xFFu;
-                    const uint g10 = (qsv0 >> 8) & 0xFFu;
-                    const uint g11 = (qsv1 >> 8) & 0xFFu;
-
-                    const float4 yl = vload4(sub * 8u + u * 2u + 0u, y);
-                    const float4 yh = vload4(sub * 8u + u * 2u + 1u, y);
-
-                    a0 += dot(yl, iq3_xxs_vals(IQ3_XXS_GRID(g00), sg0, 0u))
-                        + dot(yh, iq3_xxs_vals(IQ3_XXS_GRID(g01), sg0, 4u));
-                    a1 += dot(yl, iq3_xxs_vals(IQ3_XXS_GRID(g10), sg1, 0u))
-                        + dot(yh, iq3_xxs_vals(IQ3_XXS_GRID(g11), sg1, 4u));
-                }
-                acc0 += (0.5f + (float)(aux.s0 >> 28)) * a0;
-                acc1 += (0.5f + (float)(aux.s1 >> 28)) * a1;
-            }
-            sumf  += d0 * 0.5f * acc0;
-            sumf1 += d1 * 0.5f * acc1;
-        }
-    }
-
-#if IQ3_XXS_MV_NSG > 1
-    __local float2 part[IQ3_XXS_MV_NSG][64];
-    part[sgi][lid] = (float2)(sumf, sumf1);
-    barrier(CLK_LOCAL_MEM_FENCE);
-    if (sgi != 0) {
-        return;
-    }
-    for (uint s = 1; s < IQ3_XXS_MV_NSG; ++s) {
-        const float2 p = part[s][lid];
-        sumf  += p.s0;
-        sumf1 += p.s1;
-    }
-#endif
-
-    if (j < mh) {
-        vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
-    }
-#undef IQ3_XXS_GRID
-}
-
-kernel void kernel_iq3_xxs_grid_export(global uint * out) {
-    const uint i = get_global_id(0);
-    if (i < 256u) {
-        out[i] = iq3_xxs_grid[i];
-    }
-}

ggml/src/ggml-opencl/kernels/iq1_s_grid.cl

@@ -1,259 +0,0 @@
-// IQ1 codebook: each byte packs grid[j] + 1 and grid[j + 4] + 1.
-constant uint iq1_s_grid[2048] = {
-    0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000,
-    0x00020002, 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101,
-    0x02000000, 0x02000002, 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200,
-    0x02020202, 0x00000110, 0x00000111, 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212,
-    0x00020111, 0x01000011, 0x01000112, 0x01000211, 0x01010012, 0x01010111, 0x01010212, 0x01020011,
-    0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011, 0x02010110, 0x02010112, 0x02020111,
-    0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020, 0x00020022, 0x00020220,
-    0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020, 0x02000022,
-    0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220,
-    0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101,
-    0x01011202, 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110,
-    0x00001111, 0x00001112, 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111,
-    0x01001212, 0x01011010, 0x01011011, 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010,
-    0x01021012, 0x01021111, 0x01021210, 0x01021212, 0x02001011, 0x02011011, 0x02011111, 0x02011210,
-    0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112, 0x02021211, 0x00011120, 0x00011221,
-    0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220, 0x01021020, 0x01021021,
-    0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000, 0x00002002,
-    0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101,
-    0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101,
-    0x02022000, 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211,
-    0x00022110, 0x00022111, 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110,
-    0x01022211, 0x02012011, 0x02012110, 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022,
-    0x00002220, 0x00002222, 0x00012121, 0x00022020, 0x00022022, 0x00022220, 0x00022222, 0x01002121,
-    0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020, 0x02002022, 0x02002121, 0x02002220,
-    0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222, 0x00110000, 0x00110001,
-    0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000, 0x01110101,
-    0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102,
-    0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012,
-    0x00110111, 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010,
-    0x01110011, 0x01110012, 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111,
-    0x02100110, 0x02110012, 0x02110111, 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122,
-    0x00120121, 0x01100020, 0x01100122, 0x01100221, 0x01110022, 0x01110121, 0x01110220, 0x01110222,
-    0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120, 0x02110122, 0x02120121, 0x00101001,
-    0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201, 0x00121001, 0x00121102,
-    0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100, 0x01111101,
-    0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000,
-    0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101,
-    0x02121201, 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112,
-    0x00111211, 0x00121010, 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110,
-    0x01101111, 0x01101112, 0x01111011, 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211,
-    0x01111212, 0x01121011, 0x01121110, 0x01121111, 0x01121112, 0x01121211, 0x02101010, 0x02101012,
-    0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010, 0x02111011, 0x02111110, 0x02111111,
-    0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111, 0x00101021, 0x00101120,
-    0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021, 0x00121122,
-    0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121,
-    0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221,
-    0x01121222, 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001,
-    0x00112102, 0x00122101, 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101,
-    0x01112200, 0x01112202, 0x01122000, 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101,
-    0x02112001, 0x02112100, 0x02122101, 0x00112010, 0x00112012, 0x00112111, 0x00112212, 0x00122011,
-    0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210, 0x01112011, 0x01112110, 0x01112111,
-    0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212, 0x02102211, 0x02112011,
-    0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221, 0x00112122,
-    0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121,
-    0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222,
-    0x00200000, 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101,
-    0x00220200, 0x00220202, 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000,
-    0x02200002, 0x02200200, 0x02200202, 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200,
-    0x02220202, 0x00200111, 0x00210011, 0x00210110, 0x00210211, 0x00220111, 0x01200012, 0x01200110,
-    0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011, 0x01220110, 0x01220111, 0x01220112,
-    0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021, 0x00200220, 0x00200222,
-    0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121, 0x01210021,
-    0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121,
-    0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201,
-    0x00221101, 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200,
-    0x01211202, 0x01221102, 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101,
-    0x00201211, 0x00211111, 0x00221011, 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011,
-    0x01211110, 0x01211111, 0x01211211, 0x01221012, 0x01221111, 0x01221210, 0x02201211, 0x02211010,
-    0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011, 0x02221110, 0x02221112, 0x02221211,
-    0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021, 0x01201221, 0x01211121,
-    0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222, 0x00202000,
-    0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202,
-    0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202,
-    0x02222000, 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211,
-    0x00222111, 0x01202112, 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112,
-    0x01222211, 0x02202111, 0x02212010, 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020,
-    0x00202022, 0x00202220, 0x00202222, 0x00222020, 0x00222022, 0x00222220, 0x00222222, 0x01202121,
-    0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020, 0x02202022, 0x02202220, 0x02202222,
-    0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101, 0x10010001, 0x10010102,
-    0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001, 0x11020100,
-    0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110,
-    0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011,
-    0x10020112, 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111,
-    0x11010112, 0x11010211, 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110,
-    0x12000112, 0x12010010, 0x12010012, 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121,
-    0x10010021, 0x10010120, 0x10010122, 0x10020121, 0x11000021, 0x11010022, 0x11010121, 0x11010222,
-    0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121, 0x10001001, 0x10011101, 0x10011201,
-    0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100, 0x11011101, 0x11011102,
-    0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102, 0x12001201,
-    0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012,
-    0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010,
-    0x10021111, 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010,
-    0x11011011, 0x11011110, 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110,
-    0x11021111, 0x11021112, 0x11021211, 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011,
-    0x12011110, 0x12011111, 0x12011112, 0x12011211, 0x12011212, 0x12021111, 0x12021210, 0x12021212,
-    0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121, 0x10011220, 0x10011222, 0x10021021,
-    0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220, 0x11011020, 0x11011021,
-    0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220, 0x12001021,
-    0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101,
-    0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101,
-    0x11012200, 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100,
-    0x12012102, 0x12012201, 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010,
-    0x10012110, 0x10012111, 0x10012210, 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111,
-    0x11002212, 0x11012011, 0x11012012, 0x11012110, 0x11012111, 0x11012112, 0x11012211, 0x11022010,
-    0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112, 0x12002211, 0x12012012, 0x12012111,
-    0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211, 0x10012122, 0x11002120,
-    0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221, 0x12012120,
-    0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101,
-    0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001,
-    0x11110100, 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201,
-    0x12110101, 0x12110200, 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210,
-    0x10100211, 0x10100212, 0x10110011, 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211,
-    0x10120010, 0x10120111, 0x10120112, 0x10120210, 0x10120212, 0x11100011, 0x11100110, 0x11100111,
-    0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012, 0x11110110, 0x11110111, 0x11110112,
-    0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111, 0x11120112, 0x11120211,
-    0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, 0x12120010,
-    0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021,
-    0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122,
-    0x11110221, 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221,
-    0x12110222, 0x12120120, 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102,
-    0x10111200, 0x10111201, 0x10121001, 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100,
-    0x11101101, 0x11101102, 0x11101201, 0x11101202, 0x11111000, 0x11111001, 0x11111100, 0x11111101,
-    0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001, 0x11121002, 0x11121100, 0x11121101,
-    0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001, 0x12111100, 0x12111101,
-    0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011, 0x10101012,
-    0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110,
-    0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112,
-    0x10121211, 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210,
-    0x11101211, 0x11111010, 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210,
-    0x11111211, 0x11111212, 0x11121010, 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210,
-    0x11121211, 0x11121212, 0x12101011, 0x12101110, 0x12101111, 0x12101211, 0x12101212, 0x12111010,
-    0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210, 0x12111211, 0x12121011, 0x12121110,
-    0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022, 0x10101120, 0x10101122,
-    0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221, 0x10121020,
-    0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021,
-    0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022,
-    0x11111120, 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120,
-    0x11121121, 0x11121221, 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222,
-    0x12111021, 0x12111121, 0x12111222, 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221,
-    0x10102100, 0x10102101, 0x10102102, 0x10102201, 0x10112000, 0x10112101, 0x10112200, 0x10122001,
-    0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001, 0x11112100, 0x11112101, 0x11112102,
-    0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101, 0x12102002, 0x12102201,
-    0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011, 0x10102012,
-    0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111,
-    0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012,
-    0x11112110, 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110,
-    0x11122111, 0x11122112, 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110,
-    0x12112111, 0x12112112, 0x12112210, 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121,
-    0x10112222, 0x10122020, 0x10122121, 0x10122122, 0x10122221, 0x11102121, 0x11102220, 0x11102221,
-    0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221, 0x11122022, 0x11122121, 0x11122220,
-    0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122, 0x12112220, 0x12112222,
-    0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100, 0x11210000,
-    0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201,
-    0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012,
-    0x10210111, 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011,
-    0x11210111, 0x11210112, 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212,
-    0x12210012, 0x12210111, 0x12220011, 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221,
-    0x11200020, 0x11200021, 0x11200122, 0x11210121, 0x11210122, 0x11210220, 0x11220020, 0x12200121,
-    0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002, 0x10211101, 0x10211102, 0x10211202,
-    0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101, 0x11201200, 0x11201202,
-    0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000, 0x11221002,
-    0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101,
-    0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210,
-    0x10201212, 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112,
-    0x11201211, 0x11211010, 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011,
-    0x11221110, 0x11221111, 0x11221112, 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011,
-    0x12211111, 0x12211112, 0x12211211, 0x12211212, 0x12221012, 0x12221111, 0x12221112, 0x12221210,
-    0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122, 0x10221220, 0x10221221, 0x11201020,
-    0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121, 0x11211122, 0x11211220,
-    0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121, 0x12201222,
-    0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222,
-    0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001,
-    0x11222201, 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010,
-    0x10212111, 0x10222011, 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111,
-    0x11202112, 0x11202210, 0x11212011, 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010,
-    0x11222111, 0x11222212, 0x12202012, 0x12202110, 0x12202212, 0x12212111, 0x12222011, 0x12222110,
-    0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220, 0x11202021, 0x11202120, 0x11202221,
-    0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121, 0x11222221, 0x12202122,
-    0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200, 0x20000202,
-    0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100,
-    0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101,
-    0x22020000, 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112,
-    0x20010211, 0x20020111, 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111,
-    0x21010112, 0x21010210, 0x21010211, 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211,
-    0x22010110, 0x22010112, 0x22010211, 0x22020111, 0x20000020, 0x20000022, 0x20000220, 0x20000222,
-    0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222, 0x21010021, 0x21010120, 0x21010221,
-    0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121, 0x22020020, 0x22020022,
-    0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001, 0x21011101,
-    0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211,
-    0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111,
-    0x21001210, 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111,
-    0x21021112, 0x21021210, 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010,
-    0x22011012, 0x22011111, 0x22011210, 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121,
-    0x21001021, 0x21001120, 0x21001221, 0x21001222, 0x21011020, 0x21011121, 0x21011221, 0x21011222,
-    0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021, 0x22011222, 0x22021120, 0x20002000,
-    0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002, 0x20022200, 0x20022202,
-    0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201, 0x22002000,
-    0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202,
-    0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110,
-    0x21002112, 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110,
-    0x22002111, 0x22012112, 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222,
-    0x20012121, 0x20022020, 0x20022022, 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120,
-    0x21012122, 0x22002020, 0x22002022, 0x22002220, 0x22002222, 0x22012121, 0x22022020, 0x22022022,
-    0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102, 0x20110200, 0x20110201, 0x20120101,
-    0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202, 0x21120201, 0x21120202,
-    0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011, 0x20100110,
-    0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110,
-    0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111,
-    0x21110112, 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111,
-    0x22110210, 0x22120011, 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120,
-    0x20110221, 0x20120121, 0x21100120, 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121,
-    0x21110220, 0x21120122, 0x21120221, 0x22100121, 0x22110120, 0x22110122, 0x22120221, 0x20101001,
-    0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200, 0x20121102, 0x21101000, 0x21101202,
-    0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201, 0x21121000, 0x21121001,
-    0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101, 0x22111200,
-    0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011,
-    0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212,
-    0x21101011, 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012,
-    0x21111110, 0x21111111, 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110,
-    0x21121111, 0x21121112, 0x21121211, 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012,
-    0x22111110, 0x22111111, 0x22111112, 0x22111211, 0x22111212, 0x22121010, 0x22121012, 0x22121111,
-    0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020, 0x20111121, 0x20111221, 0x20121020,
-    0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021, 0x21111022, 0x21111121,
-    0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221, 0x22101222,
-    0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102,
-    0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102,
-    0x21112202, 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101,
-    0x20102110, 0x20102112, 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212,
-    0x20122010, 0x20122011, 0x20122110, 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210,
-    0x21102212, 0x21112011, 0x21112110, 0x21112111, 0x21112112, 0x21112211, 0x21122012, 0x21122111,
-    0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010, 0x22112012, 0x22112111, 0x22112212,
-    0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120, 0x21102122, 0x21102221,
-    0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120, 0x22112121,
-    0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002,
-    0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000,
-    0x22200002, 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202,
-    0x20200111, 0x20200211, 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112,
-    0x21200211, 0x21210011, 0x21210111, 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111,
-    0x22210010, 0x22210012, 0x22210112, 0x22210211, 0x20200022, 0x20200220, 0x20200222, 0x20210020,
-    0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121, 0x21210021, 0x21210122, 0x21210221,
-    0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121, 0x22220020, 0x22220022,
-    0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000, 0x21211100,
-    0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201,
-    0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112,
-    0x20221211, 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211,
-    0x21221111, 0x21221212, 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012,
-    0x22211111, 0x22211210, 0x20201121, 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121,
-    0x21201120, 0x21201122, 0x21201222, 0x21211022, 0x21211121, 0x21211122, 0x21211220, 0x21221020,
-    0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122, 0x22211221, 0x22221021, 0x22221120,
-    0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000, 0x20222002, 0x20222200,
-    0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002, 0x22202200,
-    0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110,
-    0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011,
-    0x21222112, 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222,
-    0x20222020, 0x20222022, 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020,
-    0x22202022, 0x22202220, 0x22202222, 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222
-};

ggml/src/ggml-opencl/kernels/iq3_s_grid.cl

@@ -1,66 +0,0 @@
-constant uint iq3_s_grid[512] = {
-    0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305,
-    0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905,
-    0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09,
-    0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b,
-    0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b,
-    0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d,
-    0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03,
-    0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505,
-    0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03,
-    0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901,
-    0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d,
-    0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303,
-    0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501,
-    0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105,
-    0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505,
-    0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101,
-    0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707,
-    0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b,
-    0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01,
-    0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f,
-    0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305,
-    0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103,
-    0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509,
-    0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503,
-    0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b,
-    0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f,
-    0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f,
-    0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f,
-    0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109,
-    0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f,
-    0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509,
-    0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501,
-    0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303,
-    0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f,
-    0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907,
-    0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703,
-    0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03,
-    0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01,
-    0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01,
-    0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903,
-    0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505,
-    0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b,
-    0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107,
-    0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509,
-    0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303,
-    0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103,
-    0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05,
-    0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b,
-    0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f,
-    0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701,
-    0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909,
-    0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305,
-    0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d,
-    0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b,
-    0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d,
-    0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307,
-    0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09,
-    0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309,
-    0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709,
-    0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f,
-    0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303,
-    0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503,
-    0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b,
-    0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101
-};

ggml/src/ggml-opencl/kernels/iq3_xxs_grid.cl

@@ -1,38 +0,0 @@
-constant uint iq3_xxs_grid[256] = {
-    0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,
-    0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,
-    0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,
-    0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,
-    0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,
-    0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,
-    0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,
-    0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,
-    0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,
-    0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,
-    0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,
-    0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,
-    0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,
-    0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,
-    0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,
-    0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,
-    0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,
-    0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,
-    0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,
-    0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,
-    0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,
-    0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,
-    0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,
-    0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,
-    0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,
-    0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,
-    0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,
-    0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,
-    0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,
-    0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,
-    0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,
-    0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04
-};
-
-inline uint iq3_xxs_signs(uint code) {
-    return code | ((popcount(code) & 1u) << 7);
-}

ggml/src/ggml-opencl/kernels/mul_mm_iq1_m_f32_l4_lm.cl

@@ -1,191 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#define INTEL_GPU 1
-#endif
-
-#define QK_K 256
-
-struct block_iq1_m {
-    uchar qs[QK_K/8];
-    uchar qh[QK_K/16];
-    uchar scales[QK_K/32];
-};
-
-#define LOAD_VEC_A 4
-#define LOAD_VEC_B 4
-
-#define BM 64
-#define BN 64
-#define BK 32
-#ifdef INTEL_GPU
-#define TM 8
-#define TN 8
-#else
-#define TM 4
-#define TN 8
-#endif
-
-kernel void kernel_mul_mm_iq1_m_f32_l4_lm(
-    global uchar * src0_qs,
-    global uchar * src0_qh,
-    global ushort * src0_sc,
-    ulong offset0,
-    global float4 * src1,
-    ulong offset1,
-    global float  * dst,
-    ulong offsetd,
-
-    int ne00,
-    int ne01,
-    int ne02,
-    int ne11,
-    int ne12,
-
-    int stride_a,
-    int stride_b,
-    int stride_d,
-
-    int batch_stride_a,
-    int batch_stride_b,
-    int batch_stride_d,
-
-    int r2,
-    int r3
-) {
-    src1 = (global float4*)((global char*)src1 + offset1);
-    dst  = (global float *)((global char*)dst  + offsetd);
-
-    local float buf_a[BM * BK];
-    local float buf_b[BN * BK];
-
-    const int batch_idx = get_global_id(2);
-
-    const int i13 = batch_idx / ne12;
-    const int i12 = batch_idx % ne12;
-
-    const int i03 = i13 / r3;
-    const int i02 = i12 / r2;
-
-    const int batch_idx_a = i03 * ne02 + i02;
-
-    const int ir = get_group_id(0);
-    const int ic = get_group_id(1);
-
-    const int tid = get_local_id(0);
-    const int th_r  = tid % (BM / TM);
-    const int th_c  = tid / (BM / TM);
-
-    const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
-    const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
-    const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
-    const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
-
-    const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
-    const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
-
-    // pos_a counts elements, not blocks - one iq1_m super block holds QK_K of them
-    int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
-    int pos_b = (batch_idx   * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
-
-    float sums[TM * TN];
-    float cache_a[TM];
-    float cache_b[TN];
-
-    for (int i = 0; i < TM * TN; i++) {
-        sums[i] = 0.0f;
-    }
-
-    for (int block = 0; block < ne00; block += BK) {
-        for (int l = 0; l < BM; l += loadstride_a) {
-            if (ir*BM + loadc_a + l < ne01) {
-                int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
-                int ib  = idx / QK_K;
-                ib += offset0/sizeof(struct block_iq1_m);
-                int e   = idx % QK_K;
-
-                int ib32 = e >> 5;
-                int rem  = e & 31;
-                int lg    = rem >> 3;
-                int j0   = rem & 7;
-
-                global ushort * sc = src0_sc + 4*ib;
-                ushort u16 = (ushort)((sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0)
-                                    | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000));
-                float dl = (float)as_half(u16) * (float)(2*((sc[ib32/2] >> (6*(ib32%2) + 3*(lg/2))) & 7) + 1);
-                uchar qhb = src0_qh[16*ib + 2*ib32 + lg/2];
-                float dlt = (qhb & (0x08 << (4*(lg%2)))) ? -1.125f : -0.875f;
-                uint gi = (uint)src0_qs[32*ib + 4*ib32 + lg] | ((((uint)qhb >> (4*(lg%2))) & 7) << 8);
-                uint g = iq1_s_grid[gi];
-                int sh = (j0 >> 2) * 4;
-
-                float4 v1;
-                v1.s0 = dl * ((float)((g >> ( 0 + sh)) & 0xF) + dlt);
-                v1.s1 = dl * ((float)((g >> ( 8 + sh)) & 0xF) + dlt);
-                v1.s2 = dl * ((float)((g >> (16 + sh)) & 0xF) + dlt);
-                v1.s3 = dl * ((float)((g >> (24 + sh)) & 0xF) + dlt);
-
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
-            } else {
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
-            }
-        }
-
-        for (int l = 0; l < BN; l += loadstride_b) {
-            if (ic*BN + loadc_b + l < ne11) {
-                int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
-            } else {
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
-            }
-        }
-
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-        pos_a += BK;
-        pos_b += BK / LOAD_VEC_B;
-
-        for (int i = 0; i < BK; i++) {
-            for (int j = 0; j < TM; j++) {
-                cache_a[j] = buf_a[(i) * BM + th_r * TM + j];
-            }
-
-            for (int j = 0; j < TN; j++) {
-                cache_b[j] = buf_b[(i) * BN + th_c * TN + j];
-            }
-
-            for (int cc = 0; cc < TN; cc++) {
-                for (int cr = 0; cr < TM; cr++) {
-                    const int sums_idx = cc*TM + cr;
-                    sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);
-                }
-            }
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    const int dr = ir * BM + th_r * TM;
-    const int dc = ic * BN + th_c * TN;
-
-    const ulong offsets = batch_idx * batch_stride_d;
-
-    for (int cc = 0; cc < TN; cc++) {
-        for (int cr = 0; cr < TM; cr++) {
-            if (dr + cr < ne01 && dc + cc < ne11) {
-                dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mm_iq1_s_f32_l4_lm.cl

@@ -1,188 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#define INTEL_GPU 1
-#endif
-
-#define QK_K 256
-
-struct block_iq1_s {
-    half d;
-    uchar qs[QK_K/8];
-    ushort qh[QK_K/32];
-};
-
-#define LOAD_VEC_A 4
-#define LOAD_VEC_B 4
-
-#define BM 64
-#define BN 64
-#define BK 32
-#ifdef INTEL_GPU
-#define TM 8
-#define TN 8
-#else
-#define TM 4
-#define TN 8
-#endif
-
-kernel void kernel_mul_mm_iq1_s_f32_l4_lm(
-    global uchar * src0_qs,
-    global ushort * src0_qh,
-    global half * src0_d,
-    ulong offset0,
-    global float4 * src1,
-    ulong offset1,
-    global float  * dst,
-    ulong offsetd,
-
-    int ne00,
-    int ne01,
-    int ne02,
-    int ne11,
-    int ne12,
-
-    int stride_a,
-    int stride_b,
-    int stride_d,
-
-    int batch_stride_a,
-    int batch_stride_b,
-    int batch_stride_d,
-
-    int r2,
-    int r3
-) {
-    src1 = (global float4*)((global char*)src1 + offset1);
-    dst  = (global float *)((global char*)dst  + offsetd);
-
-    local float buf_a[BM * BK];
-    local float buf_b[BN * BK];
-
-    const int batch_idx = get_global_id(2);
-
-    const int i13 = batch_idx / ne12;
-    const int i12 = batch_idx % ne12;
-
-    const int i03 = i13 / r3;
-    const int i02 = i12 / r2;
-
-    const int batch_idx_a = i03 * ne02 + i02;
-
-    const int ir = get_group_id(0);
-    const int ic = get_group_id(1);
-
-    const int tid = get_local_id(0);
-    const int th_r  = tid % (BM / TM);
-    const int th_c  = tid / (BM / TM);
-
-    const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
-    const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
-    const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
-    const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
-
-    const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
-    const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
-
-    // pos_a counts elements, not blocks - one iq1_s super block holds QK_K of them
-    int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
-    int pos_b = (batch_idx   * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
-
-    float sums[TM * TN];
-    float cache_a[TM];
-    float cache_b[TN];
-
-    for (int i = 0; i < TM * TN; i++) {
-        sums[i] = 0.0f;
-    }
-
-    for (int block = 0; block < ne00; block += BK) {
-        for (int l = 0; l < BM; l += loadstride_a) {
-            if (ir*BM + loadc_a + l < ne01) {
-                int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
-                int ib  = idx / QK_K;
-                ib += offset0/sizeof(struct block_iq1_s);
-                int e   = idx % QK_K;
-
-                int ib32 = e >> 5;
-                int rem  = e & 31;
-                int lg    = rem >> 3;
-                int j0   = rem & 7;
-
-                ushort qhb = src0_qh[8*ib + ib32];
-                float dl = (float)src0_d[ib] * (float)(2*((qhb >> 12) & 7) + 1);
-                float dlt = (qhb & 0x8000) ? -1.125f : -0.875f;
-                uint gi = (uint)src0_qs[32*ib + 4*ib32 + lg] | ((((uint)qhb >> (3*lg)) & 7) << 8);
-                uint g = iq1_s_grid[gi];
-                int sh = (j0 >> 2) * 4;
-
-                float4 v1;
-                v1.s0 = dl * ((float)((g >> ( 0 + sh)) & 0xF) + dlt);
-                v1.s1 = dl * ((float)((g >> ( 8 + sh)) & 0xF) + dlt);
-                v1.s2 = dl * ((float)((g >> (16 + sh)) & 0xF) + dlt);
-                v1.s3 = dl * ((float)((g >> (24 + sh)) & 0xF) + dlt);
-
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
-            } else {
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
-            }
-        }
-
-        for (int l = 0; l < BN; l += loadstride_b) {
-            if (ic*BN + loadc_b + l < ne11) {
-                int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
-            } else {
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
-            }
-        }
-
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-        pos_a += BK;
-        pos_b += BK / LOAD_VEC_B;
-
-        for (int i = 0; i < BK; i++) {
-            for (int j = 0; j < TM; j++) {
-                cache_a[j] = buf_a[(i) * BM + th_r * TM + j];
-            }
-
-            for (int j = 0; j < TN; j++) {
-                cache_b[j] = buf_b[(i) * BN + th_c * TN + j];
-            }
-
-            for (int cc = 0; cc < TN; cc++) {
-                for (int cr = 0; cr < TM; cr++) {
-                    const int sums_idx = cc*TM + cr;
-                    sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);
-                }
-            }
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    const int dr = ir * BM + th_r * TM;
-    const int dc = ic * BN + th_c * TN;
-
-    const ulong offsets = batch_idx * batch_stride_d;
-
-    for (int cc = 0; cc < TN; cc++) {
-        for (int cr = 0; cr < TM; cr++) {
-            if (dr + cr < ne01 && dc + cc < ne11) {
-                dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mm_iq3_s_f32_l4_lm.cl

@@ -1,193 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#define INTEL_GPU 1
-#endif
-
-#define QK_K 256
-
-struct block_iq3_s {
-    half d;
-    uchar qs[QK_K/4];
-    uchar qh[QK_K/32];
-    uchar signs[QK_K/8];
-    uchar scales[QK_K/64];
-};
-
-#define LOAD_VEC_A 4
-#define LOAD_VEC_B 4
-
-#define BM 64
-#define BN 64
-#define BK 32
-#ifdef INTEL_GPU
-#define TM 8
-#define TN 8
-#else
-#define TM 4
-#define TN 8
-#endif
-
-kernel void kernel_mul_mm_iq3_s_f32_l4_lm(
-    global uchar * src0_qs,
-    global uchar * src0_sg,
-    global uchar * src0_qh,
-    global uchar * src0_sc,
-    global half * src0_d,
-    ulong offset0,
-    global float4 * src1,
-    ulong offset1,
-    global float  * dst,
-    ulong offsetd,
-
-    int ne00,
-    int ne01,
-    int ne02,
-    int ne11,
-    int ne12,
-
-    int stride_a,
-    int stride_b,
-    int stride_d,
-
-    int batch_stride_a,
-    int batch_stride_b,
-    int batch_stride_d,
-
-    int r2,
-    int r3
-) {
-    src1 = (global float4*)((global char*)src1 + offset1);
-    dst  = (global float *)((global char*)dst  + offsetd);
-
-    local float buf_a[BM * BK];
-    local float buf_b[BN * BK];
-
-    const int batch_idx = get_global_id(2);
-
-    const int i13 = batch_idx / ne12;
-    const int i12 = batch_idx % ne12;
-
-    const int i03 = i13 / r3;
-    const int i02 = i12 / r2;
-
-    const int batch_idx_a = i03 * ne02 + i02;
-
-    const int ir = get_group_id(0);
-    const int ic = get_group_id(1);
-
-    const int tid = get_local_id(0);
-    const int th_r  = tid % (BM / TM);
-    const int th_c  = tid / (BM / TM);
-
-    const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
-    const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
-    const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
-    const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
-
-    const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
-    const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
-
-    // pos_a counts elements, not blocks - one iq3_s super block holds QK_K of them
-    int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
-    int pos_b = (batch_idx   * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
-
-    float sums[TM * TN];
-    float cache_a[TM];
-    float cache_b[TN];
-
-    for (int i = 0; i < TM * TN; i++) {
-        sums[i] = 0.0f;
-    }
-
-    for (int block = 0; block < ne00; block += BK) {
-        for (int l = 0; l < BM; l += loadstride_a) {
-            if (ir*BM + loadc_a + l < ne01) {
-                int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
-                int ib  = idx / QK_K;
-                ib += offset0/sizeof(struct block_iq3_s);
-                int e   = idx % QK_K;
-
-                int ib32 = e >> 5;
-                int rem  = e & 31;
-                int lg    = rem >> 3;
-                int j0   = rem & 7;
-
-                uchar scb = src0_sc[4*ib + ib32/2];
-                float db = (float)src0_d[ib] * (float)(1 + 2*((scb >> (4*(ib32 & 1))) & 15));
-                int which = j0 >> 2;
-                uint gi = (uint)src0_qs[64*ib + 8*ib32 + 2*lg + which]
-                        | (((uint)src0_qh[8*ib + ib32] << (8 - 2*lg - which)) & 256);
-                uchar sg = src0_sg[32*ib + 4*ib32 + lg];
-                uint g = iq3_s_grid[gi];
-
-                float4 v1;
-                v1.s0 = db * (float)((g >>  0) & 0xFF) * ((sg & (1 << (j0+0))) ? -1.f : 1.f);
-                v1.s1 = db * (float)((g >>  8) & 0xFF) * ((sg & (1 << (j0+1))) ? -1.f : 1.f);
-                v1.s2 = db * (float)((g >> 16) & 0xFF) * ((sg & (1 << (j0+2))) ? -1.f : 1.f);
-                v1.s3 = db * (float)((g >> 24) & 0xFF) * ((sg & (1 << (j0+3))) ? -1.f : 1.f);
-
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
-            } else {
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
-            }
-        }
-
-        for (int l = 0; l < BN; l += loadstride_b) {
-            if (ic*BN + loadc_b + l < ne11) {
-                int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
-            } else {
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
-            }
-        }
-
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-        pos_a += BK;
-        pos_b += BK / LOAD_VEC_B;
-
-        for (int i = 0; i < BK; i++) {
-            for (int j = 0; j < TM; j++) {
-                cache_a[j] = buf_a[(i) * BM + th_r * TM + j];
-            }
-
-            for (int j = 0; j < TN; j++) {
-                cache_b[j] = buf_b[(i) * BN + th_c * TN + j];
-            }
-
-            for (int cc = 0; cc < TN; cc++) {
-                for (int cr = 0; cr < TM; cr++) {
-                    const int sums_idx = cc*TM + cr;
-                    sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);
-                }
-            }
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    const int dr = ir * BM + th_r * TM;
-    const int dc = ic * BN + th_c * TN;
-
-    const ulong offsets = batch_idx * batch_stride_d;
-
-    for (int cc = 0; cc < TN; cc++) {
-        for (int cr = 0; cr < TM; cr++) {
-            if (dr + cr < ne01 && dc + cc < ne11) {
-                dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mm_iq3_xxs_f32_l4_lm.cl

@@ -1,187 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#define INTEL_GPU 1
-#endif
-
-#define QK_K 256
-
-struct block_iq3_xxs {
-    half d;
-    uchar qs[3*QK_K/8];
-};
-
-#define LOAD_VEC_A 4
-#define LOAD_VEC_B 4
-
-#define BM 64
-#define BN 64
-#define BK 32
-#ifdef INTEL_GPU
-#define TM 8
-#define TN 8
-#else
-#define TM 4
-#define TN 8
-#endif
-
-kernel void kernel_mul_mm_iq3_xxs_f32_l4_lm(
-    global uchar * src0_qs,
-    global uint * src0_sas,
-    global half * src0_d,
-    ulong offset0,
-    global float4 * src1,
-    ulong offset1,
-    global float  * dst,
-    ulong offsetd,
-
-    int ne00,
-    int ne01,
-    int ne02,
-    int ne11,
-    int ne12,
-
-    int stride_a,
-    int stride_b,
-    int stride_d,
-
-    int batch_stride_a,
-    int batch_stride_b,
-    int batch_stride_d,
-
-    int r2,
-    int r3
-) {
-    src1 = (global float4*)((global char*)src1 + offset1);
-    dst  = (global float *)((global char*)dst  + offsetd);
-
-    local float buf_a[BM * BK];
-    local float buf_b[BN * BK];
-
-    const int batch_idx = get_global_id(2);
-
-    const int i13 = batch_idx / ne12;
-    const int i12 = batch_idx % ne12;
-
-    const int i03 = i13 / r3;
-    const int i02 = i12 / r2;
-
-    const int batch_idx_a = i03 * ne02 + i02;
-
-    const int ir = get_group_id(0);
-    const int ic = get_group_id(1);
-
-    const int tid = get_local_id(0);
-    const int th_r  = tid % (BM / TM);
-    const int th_c  = tid / (BM / TM);
-
-    const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
-    const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
-    const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
-    const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
-
-    const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
-    const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
-
-    // pos_a counts elements, not blocks - one iq3_xxs super block holds QK_K of them
-    int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
-    int pos_b = (batch_idx   * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
-
-    float sums[TM * TN];
-    float cache_a[TM];
-    float cache_b[TN];
-
-    for (int i = 0; i < TM * TN; i++) {
-        sums[i] = 0.0f;
-    }
-
-    for (int block = 0; block < ne00; block += BK) {
-        for (int l = 0; l < BM; l += loadstride_a) {
-            if (ir*BM + loadc_a + l < ne01) {
-                int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
-                int ib  = idx / QK_K;
-                ib += offset0/sizeof(struct block_iq3_xxs);
-                int e   = idx % QK_K;
-
-                int ib32 = e >> 5;
-                int rem  = e & 31;
-                int lg    = rem >> 3;
-                int j0   = rem & 7;
-
-                uint aux = src0_sas[8*ib + ib32];
-                float db = (float)src0_d[ib] * (0.5f + (float)(aux >> 28)) * 0.5f;
-
-                uint gi = src0_qs[64*ib + 8*ib32 + 2*lg + (j0 >> 2)];
-                uint sg = iq3_xxs_signs((aux >> (7*lg)) & 127u);
-                uint  g  = iq3_xxs_grid[gi];
-
-                float4 v1;
-                v1.s0 = db * (float)((g >>  0) & 0xFF) * ((sg & (1 << (j0+0))) ? -1.f : 1.f);
-                v1.s1 = db * (float)((g >>  8) & 0xFF) * ((sg & (1 << (j0+1))) ? -1.f : 1.f);
-                v1.s2 = db * (float)((g >> 16) & 0xFF) * ((sg & (1 << (j0+2))) ? -1.f : 1.f);
-                v1.s3 = db * (float)((g >> 24) & 0xFF) * ((sg & (1 << (j0+3))) ? -1.f : 1.f);
-
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
-            } else {
-                buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
-                buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
-            }
-        }
-
-        for (int l = 0; l < BN; l += loadstride_b) {
-            if (ic*BN + loadc_b + l < ne11) {
-                int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
-            } else {
-                buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
-                buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
-            }
-        }
-
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-        pos_a += BK;
-        pos_b += BK / LOAD_VEC_B;
-
-        for (int i = 0; i < BK; i++) {
-            for (int j = 0; j < TM; j++) {
-                cache_a[j] = buf_a[(i) * BM + th_r * TM + j];
-            }
-
-            for (int j = 0; j < TN; j++) {
-                cache_b[j] = buf_b[(i) * BN + th_c * TN + j];
-            }
-
-            for (int cc = 0; cc < TN; cc++) {
-                for (int cr = 0; cr < TM; cr++) {
-                    const int sums_idx = cc*TM + cr;
-                    sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);
-                }
-            }
-        }
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    const int dr = ir * BM + th_r * TM;
-    const int dc = ic * BN + th_c * TN;
-
-    const ulong offsets = batch_idx * batch_stride_d;
-
-    for (int cc = 0; cc < TN; cc++) {
-        for (int cr = 0; cr < TM; cr++) {
-            if (dr + cr < ne01 && dc + cc < ne11) {
-                dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq1_m_f32.cl

@@ -1,149 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-#define IQ1_M_DELTA 0.125f
-
-// no d field: the super block scale is 4 nibbles spread over scales[]
-typedef struct {
-    uchar qs[QK_K/8];
-    uchar qh[QK_K/16];
-    uchar scales[QK_K/32];
-} block_iq1_m;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq1_m_f32(
-        global char * src0,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src0 = src0 + offset0;
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global block_iq1_m * x = (global block_iq1_m *) (src0 + offset_src0);
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        global char * xrow = (global char *)(x + ib);
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            global block_iq1_m * xb = (global block_iq1_m *)(xrow + row*nb01);
-
-            global ushort * sc = (global ushort *)xb->scales;
-            ushort u16 = (ushort)(((uint)sc[0] >> 12) | (((uint)sc[1] >> 8) & 0x00f0)
-                                | (((uint)sc[2] >> 4) & 0x0f00) | ((uint)sc[3] & 0xf000));
-            float dsuper = (float)as_half(u16);
-
-            float acc = 0.f;
-            for (int il = 0; il < 4; ++il) {
-                int   ib16 = 2*it + il/2;
-                float dl   = dsuper * (float)(2*(((uint)sc[ib16/4] >> (3*(ib16%4))) & 7) + 1);
-
-                uchar qhb = xb->qh[2*it + il/2];
-                float dlt = (qhb & (0x08 << (4*(il%2)))) ? (-1.f - IQ1_M_DELTA) : (-1.f + IQ1_M_DELTA);
-
-                uint gi = (uint)xb->qs[4*it+il] | ((((uint)qhb >> (4*(il%2))) & 7) << 8);
-                uint g  = iq1_s_grid[gi];
-
-                float a = 0.f;
-                for (int j = 0; j < 4; ++j) {
-                    a += yl[8*il+j+0] * ((float)((g >> (8*j+0)) & 0xF) + dlt);
-                    a += yl[8*il+j+4] * ((float)((g >> (8*j+4)) & 0xF) + dlt);
-                }
-                acc += dl * a;
-            }
-
-            sumf[row] += acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq1_m_f32_flat.cl

@@ -1,148 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-#define IQ1_M_DELTA 0.125f
-
-// no d field: the super block scale is 4 nibbles spread over scales[]
-typedef struct {
-    uchar qs[QK_K/8];
-    uchar qh[QK_K/16];
-    uchar scales[QK_K/32];
-} block_iq1_m;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq1_m_f32_flat(
-        global uchar * src0_qs,
-        global uchar * src0_qh,
-        global ushort * src0_sc,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq1_m);
-    ulong row_stride = nb01/sizeof(block_iq1_m);
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            const ulong b = offset_src0 + ib + row*row_stride;
-
-            global ushort * sc = src0_sc + 4*b;
-            ushort u16 = (ushort)(((uint)sc[0] >> 12) | (((uint)sc[1] >> 8) & 0x00f0)
-                                | (((uint)sc[2] >> 4) & 0x0f00) | ((uint)sc[3] & 0xf000));
-            float dsuper = (float)as_half(u16);
-
-            float acc = 0.f;
-            for (int il = 0; il < 4; ++il) {
-                int   ib16 = 2*it + il/2;
-                float dl   = dsuper * (float)(2*(((uint)sc[ib16/4] >> (3*(ib16%4))) & 7) + 1);
-
-                uchar qhb = src0_qh[16*b + 2*it + il/2];
-                float dlt = (qhb & (0x08 << (4*(il%2)))) ? (-1.f - IQ1_M_DELTA) : (-1.f + IQ1_M_DELTA);
-
-                uint gi = (uint)src0_qs[32*b + 4*it+il] | ((((uint)qhb >> (4*(il%2))) & 7) << 8);
-                uint g  = iq1_s_grid[gi];
-
-                float a = 0.f;
-                for (int j = 0; j < 4; ++j) {
-                    a += yl[8*il+j+0] * ((float)((g >> (8*j+0)) & 0xF) + dlt);
-                    a += yl[8*il+j+4] * ((float)((g >> (8*j+4)) & 0xF) + dlt);
-                }
-                acc += dl * a;
-            }
-
-            sumf[row] += acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq1_s_f32.cl

@@ -1,140 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-#define IQ1_S_DELTA 0.125f
-
-typedef struct {
-    half   d;
-    uchar  qs[QK_K/8];
-    ushort qh[QK_K/32];
-} block_iq1_s;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq1_s_f32(
-        global char * src0,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src0 = src0 + offset0;
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global block_iq1_s * x = (global block_iq1_s *) (src0 + offset_src0);
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        global char * xrow = (global char *)(x + ib);
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            global block_iq1_s * xb = (global block_iq1_s *)(xrow + row*nb01);
-
-            ushort qhb = xb->qh[it];
-            float  dl  = (float)xb->d * (float)(2*((qhb >> 12) & 7) + 1);
-            float  dlt = (qhb & 0x8000) ? (-1.f - IQ1_S_DELTA) : (-1.f + IQ1_S_DELTA);
-
-            global uchar * qsb = xb->qs + 4*it;
-
-            float acc = 0.f;
-            for (int l = 0; l < 4; ++l) {
-                uint gi = (uint)qsb[l] | ((((uint)qhb >> (3*l)) & 7) << 8);
-                uint g  = iq1_s_grid[gi];
-                for (int j = 0; j < 4; ++j) {
-                    acc += yl[8*l+j+0] * ((float)((g >> (8*j+0)) & 0xF) + dlt);
-                    acc += yl[8*l+j+4] * ((float)((g >> (8*j+4)) & 0xF) + dlt);
-                }
-            }
-
-            sumf[row] += dl * acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq1_s_f32_flat.cl

@@ -1,139 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-#define IQ1_S_DELTA 0.125f
-
-typedef struct {
-    half   d;
-    uchar  qs[QK_K/8];
-    ushort qh[QK_K/32];
-} block_iq1_s;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq1_s_f32_flat(
-        global uchar * src0_qs,
-        global ushort * src0_qh,
-        global half * src0_d,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq1_s);
-    ulong row_stride = nb01/sizeof(block_iq1_s);
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            const ulong b = offset_src0 + ib + row*row_stride;
-
-            ushort qhb = src0_qh[8*b + it];
-            float  dl  = (float)src0_d[b] * (float)(2*((qhb >> 12) & 7) + 1);
-            float  dlt = (qhb & 0x8000) ? (-1.f - IQ1_S_DELTA) : (-1.f + IQ1_S_DELTA);
-
-            global uchar * qsb = src0_qs + 32*b + 4*it;
-
-            float acc = 0.f;
-            for (int l = 0; l < 4; ++l) {
-                uint gi = (uint)qsb[l] | ((((uint)qhb >> (3*l)) & 7) << 8);
-                uint g  = iq1_s_grid[gi];
-                for (int j = 0; j < 4; ++j) {
-                    acc += yl[8*l+j+0] * ((float)((g >> (8*j+0)) & 0xF) + dlt);
-                    acc += yl[8*l+j+4] * ((float)((g >> (8*j+4)) & 0xF) + dlt);
-                }
-            }
-
-            sumf[row] += dl * acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32.cl

@@ -1,146 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-#define IQ3_S_N_SCALE (QK_K/64)
-
-typedef struct {
-    half  d;
-    uchar qs[QK_K/4];
-    uchar qh[QK_K/32];
-    uchar signs[QK_K/8];
-    uchar scales[IQ3_S_N_SCALE];
-} block_iq3_s;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq3_s_f32(
-        global char * src0,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src0 = src0 + offset0;
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global block_iq3_s * x = (global block_iq3_s *) (src0 + offset_src0);
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        global char * xrow = (global char *)(x + ib);
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            global block_iq3_s * xb = (global block_iq3_s *)(xrow + row*nb01);
-
-            uchar nib = (it & 1) ? (xb->scales[it >> 1] >> 4) : (xb->scales[it >> 1] & 0xf);
-            float db  = (float)xb->d * (float)(1 + 2*nib);
-
-            uchar qhb = xb->qh[it];
-            global uchar * qsb = xb->qs + 8*it;
-            global uchar * sgb = xb->signs + 4*it;
-
-            float acc = 0.f;
-            for (int l = 0; l < 4; ++l) {
-                uint i1 = (uint)qsb[2*l+0] | (((uint)qhb << (8-2*l)) & 256);
-                uint i2 = (uint)qsb[2*l+1] | (((uint)qhb << (7-2*l)) & 256);
-                uint g1 = iq3_s_grid[i1];
-                uint g2 = iq3_s_grid[i2];
-                uchar sg = sgb[l];
-                for (int j = 0; j < 4; ++j) {
-                    acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * ((sg & (1 << j))     ? -1.f : 1.f);
-                    acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f);
-                }
-            }
-
-            sumf[row] += db * acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq3_s_f32_flat.cl

@@ -1,147 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-#define IQ3_S_N_SCALE (QK_K/64)
-
-typedef struct {
-    half  d;
-    uchar qs[QK_K/4];
-    uchar qh[QK_K/32];
-    uchar signs[QK_K/8];
-    uchar scales[IQ3_S_N_SCALE];
-} block_iq3_s;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq3_s_f32_flat(
-        global uchar * src0_qs,
-        global uchar * src0_sg,
-        global uchar * src0_qh,
-        global uchar * src0_sc,
-        global half * src0_d,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq3_s);
-    ulong row_stride = nb01/sizeof(block_iq3_s);
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            const ulong b = offset_src0 + ib + row*row_stride;
-
-            uchar nib = (it & 1) ? (src0_sc[4*b + (it >> 1)] >> 4) : (src0_sc[4*b + (it >> 1)] & 0xf);
-            float db  = (float)src0_d[b] * (float)(1 + 2*nib);
-
-            uchar qhb = src0_qh[8*b + it];
-            global uchar * qsb = src0_qs + 64*b + 8*it;
-            global uchar * sgb = src0_sg + 32*b + 4*it;
-
-            float acc = 0.f;
-            for (int l = 0; l < 4; ++l) {
-                uint i1 = (uint)qsb[2*l+0] | (((uint)qhb << (8-2*l)) & 256);
-                uint i2 = (uint)qsb[2*l+1] | (((uint)qhb << (7-2*l)) & 256);
-                uint g1 = iq3_s_grid[i1];
-                uint g2 = iq3_s_grid[i2];
-                uchar sg = sgb[l];
-                for (int j = 0; j < 4; ++j) {
-                    acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * ((sg & (1 << j))     ? -1.f : 1.f);
-                    acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f);
-                }
-            }
-
-            sumf[row] += db * acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32.cl

@@ -1,145 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-
-// d, then 64 grid index bytes, then 8 uint32 of packed scale+signs
-typedef struct {
-    half  d;
-    uchar qs[3*QK_K/8];
-} block_iq3_xxs;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq3_xxs_f32(
-        global char * src0,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src0 = src0 + offset0;
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global block_iq3_xxs * x = (global block_iq3_xxs *) (src0 + offset_src0);
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        global char * xrow = (global char *)(x + ib);
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            global block_iq3_xxs * xb = (global block_iq3_xxs *)(xrow + row*nb01);
-
-            global uchar * qs = xb->qs + 8*it;
-
-            // Blocks are 98 bytes, so packed signs are only 2-byte aligned.
-            global ushort * sw = (global ushort *)(xb->qs + 64 + 4*it);
-            uint aux32 = (uint)sw[0] | ((uint)sw[1] << 16);
-
-            float db = (float)xb->d * (0.5f + (float)(aux32 >> 28)) * 0.5f;
-
-            float acc = 0.f;
-            for (int l = 0; l < 4; ++l) {
-                uchar signs = iq3_xxs_signs((aux32 >> (7*l)) & 127u);
-                uint  g1 = iq3_xxs_grid[qs[2*l+0]];
-                uint  g2 = iq3_xxs_grid[qs[2*l+1]];
-
-                for (int j = 0; j < 4; ++j) {
-                    float s1 = (signs & (1 << j))       ? -1.f : 1.f;
-                    float s2 = (signs & (1 << (j+4)))   ? -1.f : 1.f;
-                    acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * s1;
-                    acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * s2;
-                }
-            }
-
-            sumf[row] += db * acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-opencl/kernels/mul_mv_iq3_xxs_f32_flat.cl

@@ -1,142 +0,0 @@
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#ifdef cl_intel_required_subgroup_size
-#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
-#define INTEL_GPU 1
-#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
-#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
-#elif defined(cl_qcom_reqd_sub_group_size)
-#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
-#define ADRENO_GPU 1
-#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
-#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
-#endif
-
-#define QK_K 256
-
-// d, then 64 grid index bytes, then 8 uint32 of packed scale+signs
-typedef struct {
-    half  d;
-    uchar qs[3*QK_K/8];
-} block_iq3_xxs;
-
-#undef N_DST
-#undef N_SIMDGROUP
-#undef N_SIMDWIDTH
-
-#ifdef INTEL_GPU
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 16
-#elif defined (ADRENO_GPU)
-#define N_DST 4
-#define N_SIMDGROUP 1
-#define N_SIMDWIDTH 64
-#endif
-
-#undef  BLOCK_STRIDE
-// 8 threads cover one super block, one 32 element sub block each
-#define BLOCK_STRIDE (N_SIMDWIDTH/8)
-
-#ifdef INTEL_GPU
-REQD_SUBGROUP_SIZE_16
-#elif defined (ADRENO_GPU)
-REQD_SUBGROUP_SIZE_64
-#endif
-kernel void kernel_mul_mv_iq3_xxs_f32_flat(
-        global uchar * src0_qs,
-        global uint * src0_sas,
-        global half * src0_d,
-        ulong offset0,
-        global char * src1,
-        ulong offset1,
-        global char * dst,
-        ulong offsetd,
-        int ne00,
-        int ne01,
-        ulong nb01,
-        ulong nb02,
-        ulong nb03,
-        int ne12,
-        ulong nb11,
-        ulong nb12,
-        ulong nb13,
-        int ne0,
-        int ne1,
-        int r2,
-        int r3
-) {
-    src1 = src1 + offset1;
-    dst  = dst  + offsetd;
-
-    int ix = get_sub_group_local_id()/8;  // super block index
-    int it = get_sub_group_local_id()%8;  // sub block inside the super block
-
-    int nb = ne00/QK_K;
-
-    int r0 = get_group_id(0);
-    int r1 = get_group_id(1);
-    int im = get_group_id(2);
-    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
-
-    int i12 = im%ne12;
-    int i13 = im/ne12;
-
-    ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq3_xxs);
-    ulong row_stride = nb01/sizeof(block_iq3_xxs);
-    ulong offset_src1 =        r1*nb11 + (i12   )*nb12 + (i13   )*nb13;
-
-    global float         * y = (global float         *) (src1 + offset_src1);
-
-    float yl[32];
-    float sumf[N_DST] = {0.f};
-    float all_sum;
-
-    global float * y4 = y + ix * QK_K + 32 * it;
-
-    for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
-        for (int i = 0; i < 32; ++i) {
-            yl[i] = y4[i];
-        }
-
-        // only ne01 output rows exist; reading past them can pull in an inf scale
-        for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
-            const ulong b = offset_src0 + ib + row*row_stride;
-
-            global uchar * qs = src0_qs + 64*b + 8*it;
-
-            uint aux32 = src0_sas[8*b + it];
-
-            float db = (float)src0_d[b] * (0.5f + (float)(aux32 >> 28)) * 0.5f;
-
-            float acc = 0.f;
-            for (int l = 0; l < 4; ++l) {
-                uchar signs = iq3_xxs_signs((aux32 >> (7*l)) & 127u);
-                uint  g1 = iq3_xxs_grid[qs[2*l+0]];
-                uint  g2 = iq3_xxs_grid[qs[2*l+1]];
-
-                for (int j = 0; j < 4; ++j) {
-                    float s1 = (signs & (1 << j))       ? -1.f : 1.f;
-                    float s2 = (signs & (1 << (j+4)))   ? -1.f : 1.f;
-                    acc += yl[8*l+j+0] * (float)((g1 >> (8*j)) & 0xFF) * s1;
-                    acc += yl[8*l+j+4] * (float)((g2 >> (8*j)) & 0xFF) * s2;
-                }
-            }
-
-            sumf[row] += db * acc;
-        }
-
-        y4 += BLOCK_STRIDE * QK_K;
-    }
-
-    global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
-
-    for (int row = 0; row < N_DST; ++row) {
-        all_sum = sub_group_reduce_add(sumf[row]);
-        if (first_row + row < ne01) {
-            if (get_sub_group_local_id() == 0) {
-                dst_f32[first_row + row] = all_sum;
-            }
-        }
-    }
-}

ggml/src/ggml-sycl/CMakeLists.txt

@@ -107,36 +107,24 @@ target_compile_options(ggml-sycl PRIVATE "-Wno-narrowing")
 
 message(STATUS "GGML_SYCL_SUPPORT_LEVEL_ZERO_API ${GGML_SYCL_SUPPORT_LEVEL_ZERO_API}")
 if (GGML_SYCL_SUPPORT_LEVEL_ZERO_API)
-    find_package(PkgConfig)
-    if (PkgConfig_FOUND)
-        pkg_check_modules(LEVEL_ZERO level-zero)
-        if (LEVEL_ZERO_FOUND)
-            target_include_directories(ggml-sycl PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS})
-            target_link_libraries(ggml-sycl PRIVATE ${LEVEL_ZERO_LINK_LIBRARIES})
-            target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API)
-            message(STATUS "Level Zero loader found: ${LEVEL_ZERO_LINK_LIBRARIES}")
-            message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_INCLUDE_DIRS}")
-        endif()
+    # Link against Level Zero loader for direct device memory allocation.
+    # Avoids sycl::malloc_device triggering DMA-buf/TTM system RAM staging
+    # in the xe kernel driver during multi-GPU inference.
+    find_path(LEVEL_ZERO_DEV_INCLUDE_DIR level_zero/ze_api.h HINTS ${ONEAPI_ROOT}/include ${LEVEL_ZERO_V1_SDK_PATH}/include)
+    find_library(ZE_LOADER_LIB ze_loader HINTS ${ONEAPI_ROOT}/lib ${LEVEL_ZERO_V1_SDK_LIB_PATH} ENV LD_LIBRARY_PATH)
+    if(ZE_LOADER_LIB AND LEVEL_ZERO_DEV_INCLUDE_DIR)
+        target_link_libraries(ggml-sycl PRIVATE ${ZE_LOADER_LIB})
+        target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API)
+        message(STATUS "Level Zero loader found: ${ZE_LOADER_LIB}")
+        message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_DEV_INCLUDE_DIR}")
     else()
-        # Link against Level Zero loader for direct device memory allocation.
-        # Avoids sycl::malloc_device triggering DMA-buf/TTM system RAM staging
-        # in the xe kernel driver during multi-GPU inference.
-        find_path(LEVEL_ZERO_DEV_INCLUDE_DIR level_zero/ze_api.h HINTS ${ONEAPI_ROOT}/include ${LEVEL_ZERO_V1_SDK_PATH}/include)
-        find_library(ZE_LOADER_LIB ze_loader HINTS ${ONEAPI_ROOT}/lib ${LEVEL_ZERO_V1_SDK_LIB_PATH} ENV LD_LIBRARY_PATH)
-        if(ZE_LOADER_LIB AND LEVEL_ZERO_DEV_INCLUDE_DIR)
-            target_link_libraries(ggml-sycl PRIVATE ${ZE_LOADER_LIB})
-            target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API)
-            message(STATUS "Level Zero loader found: ${ZE_LOADER_LIB}")
-            message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_DEV_INCLUDE_DIR}")
-        else()
-            message(WARNING "Level Zero loader or development headers not found, "
-                            "Level Zero API support disabled. "
-                            "Please install the Level Zero SDK/development package "
-                            "to support Level Zero API features. "
-                            "Level Zero API is not mandatory for SYCL backend, "
-                            "but it is required by the special features for better "
-                            "function & performance on Intel GPUs.")
-        endif()
+        message(WARNING "Level Zero loader or development headers not found, "
+                        "Level Zero API support disabled. "
+                        "Please install the Level Zero SDK/development package "
+                        "to support Level Zero API features. "
+                        "Level Zero API is not mandatory for SYCL backend, "
+                        "but it is required by the special features for better "
+                        "function & performance on Intel GPUs.")
     endif()
 endif()
 

ggml/src/ggml-sycl/cpy.hpp

@@ -70,22 +70,8 @@ inline void cpy_blck_f32_q1_0(const char * cxi, char * cdsti) {
     }
 }
 
-// exact roundf (half away from zero); x + 0.5f carries for x = 0.5 - 2^-25, and sycl::round is not exact under fast fp
-inline int round_half_away_int(float x) {
-    const float t = sycl::trunc(x);
-    return (int) (sycl::fabs(x - t) >= 0.5f ? t + sycl::copysign(1.0f, x) : t);
-}
-
-// correctly rounded a / b to match the CPU: fast fp gives an approximate reciprocal (~12% off by an ulp on B580),
-// so pick the approximation or a neighbour by smallest exact FMA residual
-inline float div_correctly_rounded(float a, float b) {
-    const float y  = a / b;
-    const float lo = sycl::nextafter(y, -INFINITY);
-    const float hi = sycl::nextafter(y, INFINITY);
-    const float e  = sycl::fabs(sycl::fma(-b, y, a));
-    const float el = sycl::fabs(sycl::fma(-b, lo, a));
-    const float eh = sycl::fabs(sycl::fma(-b, hi, a));
-    return el < e && el <= eh ? lo : (eh < e ? hi : y);
+inline int round_nearest_int(float x) {
+    return (int)(x >= 0.0f ? x + 0.5f : x - 0.5f);
 }
 
 inline void cpy_blck_f32_q2_0(const char * cxi, char * cdsti) {
@@ -99,7 +85,7 @@ inline void cpy_blck_f32_q2_0(const char * cxi, char * cdsti) {
     }
 
     const float d  = amax;
-    const float id = d > 0.0f ? div_correctly_rounded(1.0f, d) : 0.0f;
+    const float id = d > 0.0f ? 1.0f / d : 0.0f;
 
     dsti->d = d;
 
@@ -108,7 +94,7 @@ inline void cpy_blck_f32_q2_0(const char * cxi, char * cdsti) {
     }
 
     for (int j = 0; j < QK2_0; ++j) {
-        int q = round_half_away_int(xi[j] * id) + 1;
+        int q = round_nearest_int(xi[j] * id) + 1;
         q = dpct::max(0, dpct::min(3, q));
 
         const int byte_index = j / 4;
@@ -392,7 +378,7 @@ inline void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) {
     }
 
     const float d  = vmax / -8;
-    const float id = d ? div_correctly_rounded(1.0f, d) : 0.0f;
+    const float id = d ? 1.0f / d : 0.0f;
 
     dsti->d = d;
 
@@ -400,9 +386,8 @@ inline void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) {
         const float x0 = xi[0 + j] * id;
         const float x1 = xi[QK4_0 / 2 + j] * id;
 
-        // no-op fmin stops x * id + 8.5f being fused into an FMA, so x * id is rounded as on the CPU
-        const uint8_t xi0 = dpct::min(15, (int8_t) (sycl::fmin(x0, 6.5f) + 8.5f));
-        const uint8_t xi1 = dpct::min(15, (int8_t) (sycl::fmin(x1, 6.5f) + 8.5f));
+        const uint8_t xi0 = dpct::min(15, (int8_t) (x0 + 8.5f));
+        const uint8_t xi1 = dpct::min(15, (int8_t) (x1 + 8.5f));
 
         dsti->qs[j] = xi0;
         dsti->qs[j] |= xi1 << 4;
@@ -457,7 +442,7 @@ inline void cpy_blck_f32_q5_0(const char * cxi, char * cdsti) {
     }
 
     const float d  = vmax / -16;
-    const float id = d ? div_correctly_rounded(1.0f, d) : 0.0f;
+    const float id = d ? 1.0f / d : 0.0f;
 
     dsti->d = d;
 
@@ -466,9 +451,8 @@ inline void cpy_blck_f32_q5_0(const char * cxi, char * cdsti) {
         const float x0 = xi[0 + j] * id;
         const float x1 = xi[QK5_0 / 2 + j] * id;
 
-        // fmin: no FMA, see cpy_blck_f32_q4_0
-        const uint8_t xi0 = dpct::min(31, (int8_t) (sycl::fmin(x0, 14.5f) + 16.5f));
-        const uint8_t xi1 = dpct::min(31, (int8_t) (sycl::fmin(x1, 14.5f) + 16.5f));
+        const uint8_t xi0 = dpct::min(31, (int8_t) (x0 + 16.5f));
+        const uint8_t xi1 = dpct::min(31, (int8_t) (x1 + 16.5f));
 
         dsti->qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4);
         qh |= ((xi0 & 0x10u) >> 4) << (j + 0);

ggml/src/ggml-sycl/dmmv.cpp

@@ -112,12 +112,11 @@ static void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat *
     }
 }
 
-template <int qk, int qr, int qbytes, dequantize_kernel_t_reorder dequantize_kernel_reorder>
+template <int qk, int qr, dequantize_kernel_t_reorder dequantize_kernel_reorder>
 static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const dfloat * __restrict__ y, float * __restrict__ dst, const int ncols, const int nrows,
                                    const sycl::nd_item<3> &item_ct1) {
     // qk = quantized weights per x block
     // qr = number of quantized weights per data value in x block
-    // qbytes = bytes of the quants region per block (not qk/qr for bit-packed types)
     const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
                     item_ct1.get_local_id(1);
 
@@ -128,7 +127,7 @@ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const d
     const int tid = item_ct1.get_local_id(2);
 
 
-    const int ncols_left = ncols % (qk*WARP_SIZE);
+    const int ncols_left = ncols % (QK4_0*WARP_SIZE);
     const int ncols_align = ncols - ncols_left;
     const int iter_stride = 8*2*GGML_SYCL_DMMV_X;
     const int vals_per_iter = iter_stride / WARP_SIZE; // num quantized vals per thread and i iter //64/16=4, 512/16/2= 16
@@ -140,7 +139,7 @@ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const d
 #else
     float tmp = 0.0f;
 #endif // GGML_SYCL_F16
-    const char *d_ptr = (const char*)vx + (size_t) ncols * nrows / qk * qbytes;
+    const char *d_ptr = (const char*)vx+ncols*nrows/2;
     int i=0;
     for (i = 0; i < ncols_align; i += iter_stride) {
         const int col = i + vals_per_iter*tid;
@@ -156,7 +155,7 @@ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const d
             // dequantize
             // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val
             dfloat2 v;
-            dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * (qk/qr) +iqs+j/qr, v);
+            dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * QK4_0 / 2 +iqs+j/qr, v);
 
             // matrix multiplication
             // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2
@@ -173,7 +172,7 @@ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const d
     }
 
     for (; i < ncols; i += iter_stride) {
-        if (tid>=ncols_left/qk) continue;
+        if (tid>=ncols_left/QK4_0) continue;
         const int col = i + vals_per_iter*tid;
         const int ib = (row*ncols + col)/qk; // x block index
         const int iqs = (col%qk)/qr; // x quant index
@@ -187,7 +186,7 @@ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const d
             // dequantize
             // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val
             dfloat2 v;
-            dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * (qk/qr) +iqs+j/qr, v);
+            dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * QK4_0 / 2 +iqs+j/qr, v);
 
             // matrix multiplication
             // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2
@@ -1488,10 +1487,10 @@ static void dequantize_mul_mat_vec_q6_k_reorder(const void * __restrict__ vx, co
     }
 }
 
-template <int qk, int qr, dequantize_kernel_t dequantize_kernel>
-static void launch_dequantize_mul_mat_vec(const void *vx, const dfloat *y, float *dst,
-                                          const int ncols, const int nrows,
-                                          dpct::queue_ptr stream) {
+static void dequantize_mul_mat_vec_q4_0_sycl_reorder(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
     GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
     const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
     // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead
@@ -1504,16 +1503,17 @@ static void launch_dequantize_mul_mat_vec(const void *vx, const dfloat *y, float
         stream->parallel_for(
             sycl::nd_range<3>(block_nums * block_dims, block_dims),
             [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-                dequantize_mul_mat_vec<qk, qr, dequantize_kernel>(
+                dequantize_mul_mat_vec_reorder<QK4_0, QR4_0, dequantize_q4_0_reorder>(
                     vx, y, dst, ncols, nrows, item_ct1);
             });
     }
 }
 
-template <int qk, int qr, int qbytes, dequantize_kernel_t_reorder dequantize_kernel>
-static void launch_dequantize_mul_mat_vec_reorder(const void *vx, const dfloat *y, float *dst,
-                                                  const int ncols, const int nrows,
-                                                  dpct::queue_ptr stream) {
+
+static void dequantize_mul_mat_vec_q4_0_sycl(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
     GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
     const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
     // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead
@@ -1526,24 +1526,346 @@ static void launch_dequantize_mul_mat_vec_reorder(const void *vx, const dfloat *
         stream->parallel_for(
             sycl::nd_range<3>(block_nums * block_dims, block_dims),
             [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-                dequantize_mul_mat_vec_reorder<qk, qr, qbytes, dequantize_kernel>(
+                dequantize_mul_mat_vec<QK4_0, QR4_0, dequantize_q4_0>(
                     vx, y, dst, ncols, nrows, item_ct1);
             });
     }
 }
 
-template <int ny, auto kernel>
-static void launch_dequantize_mul_mat_vec_k(const void *vx, const float *y, float *dst,
-                                            const int ncols, const int nrows,
-                                            dpct::queue_ptr stream) {
+static void dequantize_mul_mat_vec_q1_0_sycl_reorder(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
+    const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
+    // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
+    {
+        dpct::has_capability_or_fail(stream->get_device(),
+                                     {sycl::aspect::fp16});
+
+        stream->parallel_for(
+            sycl::nd_range<3>(block_nums * block_dims, block_dims),
+            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                dequantize_mul_mat_vec_reorder<QK1_0, QR1_0, dequantize_q1_0_reorder>(
+                    vx, y, dst, ncols, nrows, item_ct1);
+            });
+    }
+}
+
+static void dequantize_mul_mat_vec_q1_0_sycl(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
+    const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
+    // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
+    {
+        dpct::has_capability_or_fail(stream->get_device(),
+                                     {sycl::aspect::fp16});
+
+        stream->parallel_for(
+            sycl::nd_range<3>(block_nums * block_dims, block_dims),
+            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                dequantize_mul_mat_vec<QK1_0, QR1_0, dequantize_q1_0>(
+                    vx, y, dst, ncols, nrows, item_ct1);
+            });
+    }
+}
+
+static void dequantize_mul_mat_vec_q4_1_sycl(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
+    const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
+    {
+        dpct::has_capability_or_fail(stream->get_device(),
+                                     {sycl::aspect::fp16});
+
+        stream->parallel_for(
+            sycl::nd_range<3>(block_nums * block_dims, block_dims),
+            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                dequantize_mul_mat_vec<QK4_1, QR4_1, dequantize_q4_1>(
+                    vx, y, dst, ncols, nrows, item_ct1);
+            });
+    }
+}
+
+static void dequantize_mul_mat_vec_q5_0_sycl(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
+    const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
+    {
+        dpct::has_capability_or_fail(stream->get_device(),
+                                     {sycl::aspect::fp16});
+
+        stream->parallel_for(
+            sycl::nd_range<3>(block_nums * block_dims, block_dims),
+            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                dequantize_mul_mat_vec<QK5_0, QR5_0, dequantize_q5_0>(
+                    vx, y, dst, ncols, nrows, item_ct1);
+            });
+    }
+}
+
+static void dequantize_mul_mat_vec_q5_1_sycl(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
+    const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
+    {
+        dpct::has_capability_or_fail(stream->get_device(),
+                                     {sycl::aspect::fp16});
+
+        stream->parallel_for(
+            sycl::nd_range<3>(block_nums * block_dims, block_dims),
+            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                dequantize_mul_mat_vec<QK5_1, QR5_1, dequantize_q5_1>(
+                    vx, y, dst, ncols, nrows, item_ct1);
+            });
+    }
+}
+
+static void dequantize_mul_mat_vec_q8_0_sycl_reorder(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
+    const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
+    {
+        dpct::has_capability_or_fail(stream->get_device(),
+                                     {sycl::aspect::fp16});
+
+        stream->parallel_for(
+            sycl::nd_range<3>(block_nums * block_dims, block_dims),
+            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                // Q8_0 reorder layout: [all qs (ncols*nrows bytes)][all d values]
+                // Cannot reuse dequantize_mul_mat_vec_reorder template because it has
+                // Q4_0-specific constants hardcoded (d_ptr offset and qs stride).
+                const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
+                    item_ct1.get_local_id(1);
+                if (row >= nrows) return;
+
+                const int tid = item_ct1.get_local_id(2);
+                const int iter_stride = 8*2*GGML_SYCL_DMMV_X;
+                const int vals_per_iter = iter_stride / WARP_SIZE;
+                const int ncols_left = ncols % (QK8_0*WARP_SIZE);
+                const int ncols_align = ncols - ncols_left;
+
+#ifdef GGML_SYCL_F16
+                sycl::half2 tmp = {0.0f, 0.0f};
+#else
+                float tmp = 0.0f;
+#endif
+                const char *d_ptr = (const char*)vx + ncols*nrows;  // d after all qs
+
+                int i = 0;
+                for (i = 0; i < ncols_align; i += iter_stride) {
+                    const int col = i + vals_per_iter*tid;
+                    const int ib = (row*ncols + col)/QK8_0;
+                    const int iqs = col % QK8_0;
+
+#pragma unroll
+                    for (int j = 0; j < vals_per_iter; j += 2) {
+                        dfloat2 v;
+                        dequantize_q8_0_reorder((const void *)d_ptr, ib, (const void *)vx,
+                                                ib * QK8_0 + iqs + j, v);
+
+#ifdef GGML_SYCL_F16
+                        dfloat2 t1{y[col + j + 0], y[col + j + 1]};
+                        tmp += v * t1;
+#else
+                        tmp += v.x() * y[col + j + 0];
+                        tmp += v.y() * y[col + j + 1];
+#endif
+                    }
+                }
+
+                // handle remaining columns
+                for (; i < ncols; i += iter_stride) {
+                    if (tid >= ncols_left/QK8_0) continue;
+                    const int col = i + vals_per_iter*tid;
+                    const int ib = (row*ncols + col)/QK8_0;
+                    const int iqs = col % QK8_0;
+
+#pragma unroll
+                    for (int j = 0; j < vals_per_iter; j += 2) {
+                        dfloat2 v;
+                        dequantize_q8_0_reorder((const void *)d_ptr, ib, (const void *)vx,
+                                                ib * QK8_0 + iqs + j, v);
+
+#ifdef GGML_SYCL_F16
+                        dfloat2 t1{y[col + j + 0], y[col + j + 1]};
+                        tmp += v * t1;
+#else
+                        tmp += v.x() * y[col + j + 0];
+                        tmp += v.y() * y[col + j + 1];
+#endif
+                    }
+                }
+
+                // reduce
+                const int mask_start = ncols > GGML_SYCL_DMMV_X ? WARP_SIZE >> 1 : WARP_SIZE >> 2;
+                for (int mask = mask_start; mask > 0; mask >>= 1) {
+                    tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
+                }
+
+                if (tid == 0) {
+#ifdef GGML_SYCL_F16
+                    dst[row] = tmp.x() + tmp.y();
+#else
+                    dst[row] = tmp;
+#endif
+                }
+            });
+    }
+}
+
+static void dequantize_mul_mat_vec_q8_0_sycl(const void *vx, const dfloat *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
+    const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
+    {
+        dpct::has_capability_or_fail(stream->get_device(),
+                                     {sycl::aspect::fp16});
+
+        stream->parallel_for(
+            sycl::nd_range<3>(block_nums * block_dims, block_dims),
+            [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+                dequantize_mul_mat_vec<QK8_0, QR8_0, dequantize_q8_0>(
+                    vx, y, dst, ncols, nrows, item_ct1);
+            });
+    }
+}
+
+static void dequantize_mul_mat_vec_q2_K_sycl(const void *vx, const float *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
     GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2; // very slightly faster than 1 even when K_QUANTS_PER_ITERATION = 2
     const int block_num_y = (nrows + ny - 1) / ny;
     const sycl::range<3> block_nums(1, 1, block_num_y);
     const sycl::range<3> block_dims(1, ny, WARP_SIZE);
     stream->parallel_for(
         sycl::nd_range<3>(block_nums * block_dims, block_dims),
         [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-            kernel(vx, y, dst, ncols, nrows, item_ct1);
+            dequantize_mul_mat_vec_q2_k(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q2_K_sycl_reorder(const void *vx, const float *y,
+                                                     float *dst, const int ncols,
+                                                     const int nrows,
+                                                     dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q2_k_reorder(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q3_K_sycl(const void *vx, const float *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q3_k(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q3_K_sycl_reorder(const void *vx, const float *y,
+                                                     float *dst, const int ncols,
+                                                     const int nrows,
+                                                     dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q3_k_reorder(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q4_K_sycl(const void *vx, const float *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q4_k(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q5_K_sycl(const void *vx, const float *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q5_k(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q6_K_sycl(const void *vx, const float *y,
+                                             float *dst, const int ncols,
+                                             const int nrows,
+                                             dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q6_k(vx, y, dst, ncols, nrows, item_ct1);
         });
 }
 
@@ -1926,6 +2248,54 @@ bool ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(enum ggml_type src0_type
 
 #endif // GGML_SYCL_DMMV_HAS_ESIMD
 
+static void dequantize_mul_mat_vec_q4_K_sycl_reorder(const void *vx, const float *y,
+                                                     float *dst, const int ncols,
+                                                     const int nrows,
+                                                     dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q4_k_reorder(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q5_K_sycl_reorder(const void *vx, const float *y,
+                                                     float *dst, const int ncols,
+                                                     const int nrows,
+                                                     dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q5_k_reorder(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
+static void dequantize_mul_mat_vec_q6_K_sycl_reorder(const void *vx, const float *y,
+                                                     float *dst, const int ncols,
+                                                     const int nrows,
+                                                     dpct::queue_ptr stream) {
+    GGML_ASSERT(ncols % QK_K == 0);
+    const int ny = 2 / K_QUANTS_PER_ITERATION;
+    const int block_num_y = (nrows + ny - 1) / ny;
+    const sycl::range<3> block_nums(1, 1, block_num_y);
+    const sycl::range<3> block_dims(1, ny, WARP_SIZE);
+    stream->parallel_for(
+        sycl::nd_range<3>(block_nums * block_dims, block_dims),
+        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
+            dequantize_mul_mat_vec_q6_k_reorder(vx, y, dst, ncols, nrows, item_ct1);
+        });
+}
+
 void ggml_sycl_op_dequantize_mul_mat_vec(
     ggml_backend_sycl_context & ctx,
     const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst,
@@ -1974,27 +2344,27 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
         case GGML_TYPE_Q1_0:
             if ((ggml_tensor_extra_gpu*)dst->src[0]->extra &&
                 ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) {
-                launch_dequantize_mul_mat_vec_reorder<QK1_0, QR1_0, QK1_0/8, dequantize_q1_0_reorder>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q1_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             } else {
-                launch_dequantize_mul_mat_vec<QK1_0, QR1_0, dequantize_q1_0>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q1_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_Q4_0:
             if ((ggml_tensor_extra_gpu*)dst->src[0]->extra &&
                 ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) {
-                launch_dequantize_mul_mat_vec_reorder<QK4_0, QR4_0, QK4_0/2, dequantize_q4_0_reorder>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q4_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             } else {
-                launch_dequantize_mul_mat_vec<QK4_0, QR4_0, dequantize_q4_0>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q4_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_Q4_1:
-            launch_dequantize_mul_mat_vec<QK4_1, QR4_1, dequantize_q4_1>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+            dequantize_mul_mat_vec_q4_1_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             break;
         case GGML_TYPE_Q5_0:
-            launch_dequantize_mul_mat_vec<QK5_0, QR5_0, dequantize_q5_0>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+            dequantize_mul_mat_vec_q5_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             break;
         case GGML_TYPE_Q5_1:
-            launch_dequantize_mul_mat_vec<QK5_1, QR5_1, dequantize_q5_1>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+            dequantize_mul_mat_vec_q5_1_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             break;
         case GGML_TYPE_Q8_0:
             if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
@@ -2005,10 +2375,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
                 } else
 #endif
                 {
-                    launch_dequantize_mul_mat_vec_reorder<QK8_0, QR8_0, QK8_0, dequantize_q8_0_reorder>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+                    dequantize_mul_mat_vec_q8_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
                 }
             } else {
-                launch_dequantize_mul_mat_vec<QK8_0, QR8_0, dequantize_q8_0>(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q8_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_Q2_K:
@@ -2021,10 +2391,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
                 else
 #endif
                 {
-                    launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q2_k_reorder>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                    dequantize_mul_mat_vec_q2_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
                 }
             } else {
-                launch_dequantize_mul_mat_vec_k<2, dequantize_mul_mat_vec_q2_k>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q2_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_Q3_K:
@@ -2037,10 +2407,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
                 else
 #endif
                 {
-                    launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q3_k_reorder>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                    dequantize_mul_mat_vec_q3_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
                 }
             } else {
-                launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q3_k>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q3_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_Q4_K:
@@ -2053,10 +2423,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
                 else
 #endif
                 {
-                    launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q4_k_reorder>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                    dequantize_mul_mat_vec_q4_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
                 }
             } else {
-                launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q4_k>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q4_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_Q5_K:
@@ -2069,10 +2439,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
                 else
 #endif
                 {
-                    launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q5_k_reorder>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                    dequantize_mul_mat_vec_q5_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
                 }
             } else {
-                launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q5_k>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q5_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_Q6_K:
@@ -2085,10 +2455,10 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
                 else
 #endif
                 {
-                    launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q6_k_reorder>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                    dequantize_mul_mat_vec_q6_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
                 }
             } else {
-                launch_dequantize_mul_mat_vec_k<2/K_QUANTS_PER_ITERATION, dequantize_mul_mat_vec_q6_k>(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
+                dequantize_mul_mat_vec_q6_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
             }
             break;
         case GGML_TYPE_F16:

ggml/src/ggml-sycl/ggml-sycl.cpp

@@ -7829,15 +7829,9 @@ struct ggml_backend_sycl_comm_context {
 
     // pinned host staging per device context, reused across calls (queues are in-order)
     uint8_t * host_out[2] = { nullptr, nullptr };
-    uint8_t * host_in[2]  = { nullptr, nullptr };  // only used by the 4-buffer path (separate SYCL contexts)
+    uint8_t * host_in[2]  = { nullptr, nullptr };
     size_t    host_bytes  = 0;
 
-    // SYCL_ENABLE_DEFAULT_CONTEXTS=1 sets comm_exchange_fast on Linux - PR #30956
-    using exchange_fn_t = void (*)(ggml_backend_sycl_comm_context * comm, queue_ptr q0, queue_ptr q1,
-                                   void * dst0, const void * src1, void * dst1, const void * src0,
-                                   size_t nbytes);
-    exchange_fn_t comm_exchange = nullptr; // Moves same context check out of hot path; tests only once
-
     void free_host() {
         for (int i = 0; i < 2; ++i) {
             const sycl::context sctx = ((ggml_backend_sycl_context *) backends[i]->context)->stream()->get_context();
@@ -7848,41 +7842,6 @@ struct ggml_backend_sycl_comm_context {
     }
 };
 
-// dst0 <- src1 and dst1 <- src0 through pinned host memory.
-static void ggml_sycl_comm_exchange_fast(ggml_backend_sycl_comm_context * comm, queue_ptr q0, queue_ptr q1,
-                                         void * dst0, const void * src1, void * dst1, const void * src0,
-                                         size_t nbytes) {
-
-    // Increases Linux tensor split prompt eval speeds on par with layer split speeds
-    // Reallocate host buffers if they are too small
-    if (comm->host_bytes < nbytes) {
-        q0->wait();
-        q1->wait();
-        comm->free_host();
-        comm->host_out[0] = sycl::malloc_host<uint8_t>(nbytes, q0->get_context());
-        comm->host_out[1] = sycl::malloc_host<uint8_t>(nbytes, q1->get_context());
-
-        GGML_ASSERT(comm->host_out[0] && comm->host_out[1]);
-        comm->host_bytes = nbytes;
-    }
-
-    // Step 1: Concurrently copy from GPUs into host buffers (Asynchronous)
-    sycl::event e_d2h_0 = q0->memcpy(comm->host_out[0], src0, nbytes);
-    sycl::event e_d2h_1 = q1->memcpy(comm->host_out[1], src1, nbytes);
-
-    // Step 2: Asynchronous Barrier Interlocking (NO HOST BLOCKING), wait() causes MTP conflicts
-    q0->ext_oneapi_submit_barrier({e_d2h_1}); // q0 waits for q1's device-to-host copy
-    q1->ext_oneapi_submit_barrier({e_d2h_0}); // q1 waits for q0's device-to-host copy
-
-    // Step 3: Cross-upload directly from the opposing host buffer.
-    sycl::event e_h2d_0 = q0->memcpy(dst0, comm->host_out[1], nbytes); // q0 pulls from host_out[1]
-    sycl::event e_h2d_1 = q1->memcpy(dst1, comm->host_out[0], nbytes); // q1 pulls from host_out[0]
-
-    // Step 4: Finalize the exchange before returning to host code
-    e_h2d_0.wait();
-    e_h2d_1.wait();
-}
-
 // dst0 <- src1 and dst1 <- src0 through pinned host memory, with a single host wait
 static void ggml_sycl_comm_exchange(ggml_backend_sycl_comm_context * comm, queue_ptr q0, queue_ptr q1,
                                     void * dst0, const void * src1, void * dst1, const void * src0,
@@ -7928,15 +7887,6 @@ void * ggml_backend_sycl_comm_init(ggml_backend_t * backends, size_t n_backends)
     auto * sctx1 = (ggml_backend_sycl_context *) backends[1]->context;
     ctx->buf0 = std::make_unique<ggml_sycl_pool_alloc<uint8_t>>(sctx0->pool());
     ctx->buf1 = std::make_unique<ggml_sycl_pool_alloc<uint8_t>>(sctx1->pool());
-
-#if defined(_WIN32) || defined(_WIN64) // Not supported on Windows
-    ctx->comm_exchange = ggml_sycl_comm_exchange;
-#else
-    // SYCL_ENABLE_DEFAULT_CONTEXTS=1 sets comm_exchange_fast on Linux - PR #30956
-    ctx->comm_exchange = (sctx0->stream()->get_context() == sctx1->stream()->get_context())
-                       ? ggml_sycl_comm_exchange_fast
-                       : ggml_sycl_comm_exchange;
-#endif
     return ctx;
 }
 catch (const sycl::exception &) { return nullptr; }
@@ -8020,7 +7970,7 @@ bool ggml_backend_sycl_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tens
         sycl::half * f16_tmp0 = (sycl::half *) buf0;
         sycl::half * f16_tmp1 = (sycl::half *) buf1;
 
-        comm_ctx->comm_exchange(comm_ctx, q0, q1, f16_tmp0, tensors[1]->data, f16_tmp1, tensors[0]->data, nbytes);
+        ggml_sycl_comm_exchange(comm_ctx, q0, q1, f16_tmp0, tensors[1]->data, f16_tmp1, tensors[0]->data, nbytes);
 
         q0->submit([&](sycl::handler & h) {
             h.parallel_for(sycl::range<1>(nelem), [=](sycl::id<1> i) {
@@ -8050,7 +8000,7 @@ bool ggml_backend_sycl_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tens
         float * tmp1 = (float *) buf1;
 
         // separate SYCL contexts: a raw peer-USM q->memcpy is a silent no-op, so stage through host memory
-        comm_ctx->comm_exchange(comm_ctx, q0, q1, tmp0, tensors[1]->data, tmp1, tensors[0]->data, nbytes);
+        ggml_sycl_comm_exchange(comm_ctx, q0, q1, tmp0, tensors[1]->data, tmp1, tensors[0]->data, nbytes);
 
         q0->submit([&](sycl::handler & h) {
             h.parallel_for(sycl::range<1>(nelem), [=](sycl::id<1> i) {
@@ -8084,7 +8034,7 @@ bool ggml_backend_sycl_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tens
 
     // Phase B: exchange the compressed bytes (in-order queues keep this after the compress kernels).
     const size_t bf16_bytes = nelem * sizeof(uint16_t);
-    comm_ctx->comm_exchange(comm_ctx, q0, q1, inbox0, outbox1, inbox1, outbox0, bf16_bytes);
+    ggml_sycl_comm_exchange(comm_ctx, q0, q1, inbox0, outbox1, inbox1, outbox0, bf16_bytes);
 
     // Phase C: decompress + add into local FP32 partial.
     q0->submit([&](sycl::handler & h) {

ggml/src/ggml.c

@@ -71,12 +71,12 @@ uint64_t ggml_graph_next_uid(void) {
 
 // Needed for ggml_fp32_to_bf16_row()
 #if defined(__AVX512BF16__)
-#if defined(_MSC_VER) && !defined(__clang__)
+#if defined(_MSC_VER)
 #define m512i(p) p
 #else
 #include <immintrin.h>
 #define m512i(p) (__m512i)(p)
-#endif // defined(_MSC_VER) && !defined(__clang__)
+#endif // defined(_MSC_VER)
 #endif // defined(__AVX512BF16__)
 
 #if defined(__linux__) || \

requirements/requirements-gguf_editor_gui.txt

@@ -1,4 +1,3 @@
-numpy~=2.2.6; python_version < "3.14"
-numpy~=2.4.6; python_version >= "3.14"
+numpy~=2.2.6
 PySide6~=6.9.0
 gguf>=0.17.0

requirements/requirements-server-bench.txt

@@ -1,6 +1,5 @@
 datasets~=4.8.0
 matplotlib~=3.10.0
-numpy~=2.2.6; python_version < "3.14"
-numpy~=2.4.6; python_version >= "3.14"
+numpy~=2.2.6
 requests~=2.32.3
 tqdm~=4.67.1

requirements/requirements-tool_bench.txt

@@ -1,9 +1,9 @@
+aiohttp~=3.9.3
 pytest~=8.3.3
 matplotlib~=3.10.0
-numpy~=2.2.6; python_version < "3.14"
-numpy~=2.4.6; python_version >= "3.14"
+numpy~=2.2.6
 openai~=2.14.0
-pandas~=2.3.3
+pandas~=2.2.3
 prometheus-client~=0.20.0
 requests~=2.32.3
 wget~=3.2

scripts/sync_vendor.py

@@ -21,7 +21,7 @@ vendor = {
 
     # not using latest tag to avoid this issue: https://github.com/ggml-org/llama.cpp/pull/17179#discussion_r2515877926
     # "https://github.com/mackron/miniaudio/raw/refs/tags/0.11.24/miniaudio.h": "vendor/miniaudio/miniaudio.h",
-    "https://github.com/mackron/miniaudio/raw/877ea4a7c2601c7b0f2a44e83dfac42e6470fea6/miniaudio.h": "vendor/miniaudio/miniaudio.h",
+    "https://github.com/mackron/miniaudio/raw/9634bedb5b5a2ca38c1ee7108a9358a4e233f14d/miniaudio.h": "vendor/miniaudio/miniaudio.h",
 
     f"https://raw.githubusercontent.com/yhirose/cpp-httplib/{HTTPLIB_VERSION}/httplib.h": "httplib.h",
     f"https://raw.githubusercontent.com/yhirose/cpp-httplib/{HTTPLIB_VERSION}/split.py":  "split.py",

src/llama-arch.cpp

@@ -1375,6 +1375,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
         case LLM_ARCH_NEMOTRON_H:
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_GRANITE_HYBRID:
+        case LLM_ARCH_LFM3:
         case LLM_ARCH_MINIMAX_01:
         case LLM_ARCH_MINIMAX_M2:
         case LLM_ARCH_MINIMAX_M3:

src/llama-hparams.cpp

@@ -319,7 +319,20 @@ uint32_t llama_hparams::n_embd_head_v_mla() const {
 }
 
 bool llama_hparams::has_kv(uint32_t il) const {
-    return kv_source(il) == il;
+    if (kv_source(il) != il) {
+        return false;
+    }
+
+    if (n_layer_kv_from_start >= 0) {
+        if (il < (uint32_t) n_layer_kv_from_start) {
+            return true;
+        }
+
+        return false;
+    }
+
+    // by default, all layers have kv
+    return true;
 }
 
 uint32_t llama_hparams::kv_source(uint32_t il) const {

src/llama-model-saver.cpp

@@ -199,9 +199,6 @@ void llama_model_saver::add_kv_from_model() {
     if (hparams.n_embd_out_impl > 0) {
         add_kv(LLM_KV_EMBEDDING_LENGTH_OUT,          hparams.n_embd_out_impl);
     }
-    if (hparams.n_embd_per_layer > 0) {
-        add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER,    hparams.n_embd_per_layer);
-    }
     add_kv(LLM_KV_BLOCK_COUNT,                       hparams.n_layer_all);
     add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT,         hparams.n_layer_dense_lead);
     add_kv(LLM_KV_FEED_FORWARD_LENGTH,               hparams.n_ff_arr, true);
@@ -310,14 +307,10 @@ void llama_model_saver::add_kv_from_model() {
     add_kv(LLM_KV_ATTENTION_INDEXER_KPOOL_SELECT_TAIL, hparams.indexer_kpool_select_tail);
     add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS,    hparams.indexer_local_blocks);
     add_kv(LLM_KV_ATTENTION_INDEXER_TYPES,           hparams.is_indexer_full_impl, true);
-    if (hparams.n_layer_kv_from_start >= 0) {
-        add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS,    uint32_t(hparams.n_layer_all - hparams.n_layer_kv_from_start));
-    } else {
-        for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
-            if (hparams.kv_source(il) != il) {
-                add_kv(LLM_KV_ATTENTION_KV_SOURCE_MAP, hparams.kv_source_impl, true);
-                break;
-            }
+    for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
+        if (hparams.kv_source(il) != il) {
+            add_kv(LLM_KV_ATTENTION_KV_SOURCE_MAP, hparams.kv_source_impl, true);
+            break;
         }
     }
     add_kv(LLM_KV_ATTENTION_RECURRENT_LAYERS,        hparams.is_recr_impl, true);
@@ -525,8 +518,6 @@ void llama_model_saver::add_tensors_from_model() {
     add_tensor(model->hc_head_base);
     add_tensor(model->hc_head_scale);
     add_tensor(model->per_layer_tok_embd);
-    add_tensor(model->per_layer_model_proj);
-    add_tensor(model->per_layer_proj_norm);
     add_tensor(model->value_embd);
     add_tensor(model->hc_head_norm);
     add_tensor(model->hc_head_down);

src/llama-model.cpp

@@ -486,9 +486,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
             GGML_ASSERT(length_prefix != std::string::npos);
             prefix = tensor_name.substr(0, length_prefix + 1);
             il = std::stoull(tensor_name.substr(4, length_prefix));
-            const bool is_attn = tensor_name.compare(prefix.size(), 5, "attn_") == 0;
-            // attention heads must be on the same device as the KV of the source layer
-            rotation = get_il_eff(is_attn ? hparams.kv_source(il) : il) % ud->n_devices;
+            rotation = get_il_eff(il) % ud->n_devices;
         } else if (tensor_name.substr(0, 6) == "cache_") {
             const size_t layer_index_start = tensor_name.find("_l", 6);
             GGML_ASSERT(layer_index_start != std::string::npos);
@@ -591,7 +589,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
             return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");
         }
         if (std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) {
-            if (ud->model->arch == LLM_ARCH_LFM2 || ud->model->arch == LLM_ARCH_LFM2MOE || ud->model->arch == LLM_ARCH_LFM3) {
+            if (ud->model->arch == LLM_ARCH_LFM2 || ud->model->arch == LLM_ARCH_LFM2MOE) {
                 // the LFM2 shortconv block runs fully mirrored, so its conv state must be mirrored too
                 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED, "");
             }
@@ -3282,8 +3280,14 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
                     llama_kv_cache::layer_share_cb share = nullptr;
 
                     if (arch == LLM_ARCH_GEMMA3N || arch == LLM_ARCH_GEMMA4) {
-                        reuse = [&](int32_t il) -> int32_t {
-                            return hparams.has_kv(il) ? -1 : (int32_t) hparams.kv_source(il);
+                        reuse = [&](uint32_t il) {
+                            GGML_ASSERT(hparams.n_layer_kv_from_start >= 2);
+
+                            if (il >= (uint32_t)hparams.n_layer_kv_from_start) {
+                                return hparams.n_layer_kv_from_start - (hparams.is_swa(il) ? 2 : 1);
+                            }
+
+                            return -1;
                         };
                     }
 

src/models/gemma3n.cpp

@@ -7,11 +7,6 @@ void llama_model_gemma3n::load_arch_hparams(llama_model_loader & ml) {
     hparams.n_layer_kv_from_start = 20;
     hparams.f_attention_scale     = 1.0f;
 
-    // shared layers read the KV of the last layer with own KV and the same SWA type
-    for (uint32_t il = hparams.n_layer_kv_from_start; il < hparams.n_layer_all; il++) {
-        hparams.kv_source_impl[il] = hparams.n_layer_kv_from_start - (hparams.is_swa(il) ? 2 : 1);
-    }
-
     ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA,          hparams.rope_freq_base_train_swa, false);
     ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,    hparams.n_swa);
     ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);

src/models/gemma4.cpp

@@ -11,12 +11,6 @@ void llama_model_gemma4::load_arch_hparams(llama_model_loader & ml) {
     hparams.n_layer_kv_from_start = hparams.n_layer_all - (int32_t)n_kv_shared_layers;
     hparams.f_attention_scale     = 1.0f; // Gemma4 uses self.scaling = 1.0 (no pre-attn scaling)
 
-    // shared layers read the KV of the last layer with own KV and the same SWA type
-    GGML_ASSERT(hparams.n_layer_kv_from_start >= 2);
-    for (uint32_t il = hparams.n_layer_kv_from_start; il < hparams.n_layer_all; il++) {
-        hparams.kv_source_impl[il] = hparams.n_layer_kv_from_start - (hparams.is_swa(il) ? 2 : 1);
-    }
-
     ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA,          hparams.rope_freq_base_train_swa, false);
     ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);
     ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,    hparams.n_swa);

src/models/lfm-base.cpp

@@ -22,10 +22,11 @@ ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_model & model,
     const uint32_t d_conv = hparams.n_shortconv_l_cache - 1;
     const int64_t  n_embd = hparams.n_embd;
 
+    // {n_embd, n_tokens} => {n_embd, n_seq_tokens, n_seqs}
+    cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs);
+
     auto * bcx = build_lora_mm(model.layers[il].shortconv.in_proj, cur);
     cb(bcx, "model.layers.{}.conv.in_proj", il);
-    // {3*n_embd, n_tokens} => {3*n_embd, n_seq_tokens, n_seqs}
-    bcx = ggml_reshape_3d(ctx0, bcx, bcx->ne[0], n_seq_tokens, n_seqs);
 
     constexpr auto n_chunks = 3;
     GGML_ASSERT(bcx->ne[0] % n_chunks == 0);
@@ -75,10 +76,10 @@ ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_model & model,
     cb(conv_out, "model.layers.{}.conv.conv", il);
 
     auto * y = ggml_mul(ctx0, c, conv_out);
+    y        = build_lora_mm(model.layers[il].shortconv.out_proj, y);
+    cb(y, "model.layers.{}.conv.out_proj", il);
     // {n_embd, n_seq_tokens, n_seqs} => {n_embd, n_tokens}
     y = ggml_reshape_2d(ctx0, y, y->ne[0], n_seq_tokens * n_seqs);
-    y = build_lora_mm(model.layers[il].shortconv.out_proj, y);
-    cb(y, "model.layers.{}.conv.out_proj", il);
 
     return y;
 }

tests/CMakeLists.txt

@@ -1,5 +1,3 @@
-find_package(Threads REQUIRED)
-
 llama_add_compile_flags()
 
 # Optional args:
@@ -353,7 +351,7 @@ if (GGML_METAL AND NOT GGML_BACKEND_DL)
     target_include_directories(test-metal-graph-optimize PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src ${PROJECT_SOURCE_DIR}/ggml/src/ggml-metal)
 endif()
 
-llama_build(test-backend-ops.cpp LINK ggml Threads::Threads)
+llama_build(test-backend-ops.cpp LINK ggml)
 
 llama_build_and_test(test-backend-tensor.cpp LINK ggml)
 

tests/fusion/MTL.csv

@@ -81,8 +81,6 @@ gemma2              ,0   ,any     ,RMS_NORM+MUL                ,      5
 gemma2              ,0   ,any     ,RMS_NORM+MUL+ADD            ,      4
 gemma3              ,0   ,any     ,RMS_NORM+MUL                ,      9
 gemma3              ,0   ,any     ,RMS_NORM+MUL+ADD            ,      4
-gemma4              ,0   ,any     ,RMS_NORM+MUL                ,     47
-gemma4              ,0   ,any     ,RMS_NORM+MUL+ADD            ,     20
 glm-dsa             ,0   ,any     ,ADD+ADD                     ,      1
 glm-dsa             ,0   ,any     ,MUL+ADD                     ,      2
 glm-dsa             ,0   ,any     ,NORM+MUL+ADD                ,      2

tests/gguf-model-data.cpp

@@ -584,7 +584,7 @@ static std::optional<gguf_remote_model> fetch_or_cached(
             auto result = gguf_parse_meta(cached);
             if (result.has_value()) {
                 if (verbose) {
-                    fprintf(stderr, "gguf_fetch: loaded from cache: %s\n", cache_path.string().c_str());
+                    fprintf(stderr, "gguf_fetch: loaded from cache: %s\n", fs_path_to_utf8(cache_path).c_str());
                 }
                 return result;
             }
@@ -685,7 +685,7 @@ gguf_context_ptr gguf_fetch_gguf_ctx(
     ggml_context_ptr ggml_ctx_ptr;
     ggml_context * ggml_ctx{};
     gguf_init_params params{true, &ggml_ctx};
-    gguf_context_ptr ctx{gguf_init_from_file(cache_path.string().c_str(), params)};
+    gguf_context_ptr ctx{gguf_init_from_file(fs_path_to_utf8(cache_path).c_str(), params)};
     ggml_ctx_ptr.reset(ggml_ctx);
 
     if (ctx == nullptr) {
@@ -723,7 +723,7 @@ gguf_context_ptr gguf_fetch_gguf_ctx(
             ggml_context_ptr shard_ggml_ctx_ptr;
             ggml_context * shard_ggml_ctx{};
             gguf_init_params shard_params{true, &shard_ggml_ctx};
-            gguf_context_ptr shard_ctx{gguf_init_from_file(shard_path.string().c_str(), shard_params)};
+            gguf_context_ptr shard_ctx{gguf_init_from_file(fs_path_to_utf8(shard_path).c_str(), shard_params)};
             shard_ggml_ctx_ptr.reset(shard_ggml_ctx);
 
             if (shard_ctx == nullptr) {

tests/test-backend-ops.cpp

@@ -10831,10 +10831,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
         }
     }
 
-    // source strides over 16 MB (24-bit DMA descriptor limit)
-    test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {512, 8193, 2, 1}, false, {2, 1, 0, 3}));
-    test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {256, 16385, 2, 1}, false, {0, 2, 1, 3}));
-
     auto add_test_bin_bcast = [&](ggml_type type, std::array<int64_t, 4> ne, std::array<int, 4> nr, bool perm1 = false, bool src_overlap = false) {
         for (auto op : {ggml_add, ggml_sub, ggml_mul, ggml_div}) {
             test_cases.emplace_back(new test_bin_bcast(op, type, ne, nr, 1, perm1, src_overlap));

tests/test-chat.cpp

@@ -5629,31 +5629,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
             .expect(message_assist_call_thoughts)
             .run();
 
-        const std::string answer_schema = R"({"type":"object","properties":{"answer":{"type":"integer"}},"required":["answer"]})";
-
-        tst.test("I'm\nthinking</think>{\"answer\":42}")
-            .enable_thinking(true)
-            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
-            .json_schema(answer_schema)
-            .expect_reasoning("I'm\nthinking")
-            .expect_content(R"({"answer":42})")
-            .run();
-
-        // the reasoning block of the generation prompt must still be accepted when it is not extracted
-        tst.test("{\"answer\":42}")
-            .enable_thinking(false)
-            .reasoning_format(COMMON_REASONING_FORMAT_NONE)
-            .json_schema(answer_schema)
-            .expect_content(R"({"answer":42})")
-            .run();
-
-        tst.test("I'm\nthinking</think>{\"answer\":42}")
-            .enable_thinking(true)
-            .reasoning_format(COMMON_REASONING_FORMAT_NONE)
-            .json_schema(answer_schema)
-            .expect_content(R"({"answer":42})")
-            .run();
-
         // a deferred tool is left out of <tools>, a tool_reference renders it, and its schema types the call
         const auto deferred_tools = common_chat_tools_parse_oaicompat(json::parse(R"([
             {"type": "function", "function": {"name": "tool_search", "description": "Search for tools",

tests/test-jinja.cpp

@@ -677,24 +677,6 @@ static void test_set_statement(testing & t) {
 }
 
 static void test_filters(testing & t) {
-    for (const std::string filter : {"upper", "lower", "title", "capitalize"}) {
-        for (const std::string input : {"'hello'", "missing"}) {
-            test_template_error(t, filter + " rejects unknown keyword on " + input,
-                "{{ " + input + "|" + filter + "(attribute='name') }}",
-                "unexpected keyword argument 'attribute'"
-            );
-            test_template_error(t, filter + " rejects extra positional argument on " + input,
-                "{{ " + input + "|" + filter + "('extra') }}",
-                "expected between 1 and 1 arguments, got 2"
-            );
-        }
-    }
-
-    test_template_error(t, "string method rejects unknown keyword",
-        "{{ 'hello'.upper(attribute='name') }}",
-        "unexpected keyword argument 'attribute'"
-    );
-
     test_template(t, "upper",
         "{{ 'hello'|upper }}",
         json::object(),
@@ -743,23 +725,6 @@ static void test_filters(testing & t) {
         "hello"
     );
 
-    test_template(t, "trim chars keyword",
-        "{{ 'xyxhelloxyx'|trim(chars='xy') }}|{{ missing|trim(chars='xy') }}",
-        json::object(),
-        "hello|"
-    );
-
-    for (const std::string input : {"'hello'", "missing"}) {
-        test_template_error(t, "trim rejects unknown keyword on " + input,
-            "{{ " + input + "|trim(attribute='name') }}",
-            "unexpected keyword argument 'attribute'"
-        );
-        test_template_error(t, "trim rejects extra positional argument on " + input,
-            "{{ " + input + "|trim('x', 'y') }}",
-            "expected between 1 and 2 arguments, got 3"
-        );
-    }
-
     test_template(t, "length string",
         "{{ 'hello'|length }}",
         json::object(),
@@ -2099,79 +2064,6 @@ static void test_array_methods(testing & t) {
         "{\"x\": 2}"
     );
 
-    test_template(t, "array|map with filter int materialized",
-        "{{ arr|map('int')|list|join(',') }}",
-        {{"arr", json::array({"1", "2", "3"})}},
-        "1,2,3"
-    );
-
-    test_template(t, "array|map with filter upper for schema union type",
-        "{{ item_value|map('upper')|list|join(',') }}",
-        {{"item_value", json::array({"string", "null"})}},
-        "STRING,NULL"
-    );
-
-    test_template_error(t, "array|map rejects unsupported filter attribute",
-        "{{ [{'name': 'alice'}]|map('upper', attribute='name')|join(',') }}",
-        "unexpected keyword argument 'attribute'"
-    );
-
-    test_template_error(t, "array|map rejects unknown keyword on undefined item",
-        "{{ [missing]|map('lower', attribute='name')|join(',') }}",
-        "unexpected keyword argument 'attribute'"
-    );
-
-    test_template(t, "array|map attribute then filter upper",
-        "{{ arr|map(attribute='name')|map('upper')|join(',') }}",
-        {{"arr", json::array({{{"name", "alice"}}, {{"name", "bob"}}})}},
-        "ALICE,BOB"
-    );
-
-    test_template(t, "array|map with filter arguments",
-        "{{ arr|map('replace', 'a', 'o')|join(',') }}",
-        {{"arr", json::array({"cat", "bat"})}},
-        "cot,bot"
-    );
-
-    test_template(t, "array|map with filter keyword arguments",
-        "{{ arr|map('join', '-', attribute='name')|join(';') }}",
-        {{"arr", json::array({
-            json::array({
-                json({{"name", "a"}}),
-                json({{"name", "b"}}),
-            }),
-            json::array({
-                json({{"name", "c"}}),
-            }),
-        })}},
-        "a-b;c"
-    );
-
-    test_template(t, "array|map with count alias",
-        "{{ arr|map('count')|join(',') }}",
-        {{"arr", json::array({"cat", "hello"})}},
-        "3,5"
-    );
-
-    test_template(t, "array|map with join keyword arguments",
-        "{{ [[{'name': 'a'}, {'name': 'b'}]]|map('join', d='-', attribute='name')|join(';') }}",
-        json::object(),
-        "a-b"
-    );
-
-    for (const std::string input : {"['a', 'b']", "[]", "missing"}) {
-        test_template_error(t, "join rejects unknown keyword on " + input,
-            "{{ " + input + "|join(typo='name') }}",
-            "unexpected keyword argument 'typo'"
-        );
-    }
-
-    test_template(t, "array|map with default alias and arguments",
-        "{{ arr|map('d', 'fallback', true)|join(',') }}",
-        {{"arr", json::array({"", "value"})}},
-        "fallback,value"
-    );
-
     // not used by any chat templates
     // test_template(t, "array.insert()",
     //     "{% set _ = arr.insert(1, 'x') %}{{ arr|join(',') }}",
@@ -2239,12 +2131,6 @@ static void test_array_methods(testing & t) {
         ""
     );
 
-    test_template(t, "undefined|join with keyword arguments",
-        "{{ arr|join(d='-', attribute='name') }}",
-        json::object(),
-        ""
-    );
-
     test_template(t, "undefined|sort",
         "{{ arr|sort|string }}",
         json::object(),

tests/test-llama-archs.cpp

@@ -243,9 +243,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         n_head = 4;
         n_layer = 4; // dense and MoE layers, each with local and full attention
     } else if (arch == LLM_ARCH_GEMMA4) {
-        // TODO: expand coverage to both GQA and MQA (see https://github.com/ggml-org/llama.cpp/pull/31401#pullrequestreview-5710165130)
-        n_head = 4;
-        n_layer = 10; // last 5 layers reuse KV, both SWA and full attention
+        n_embd = 128;
+        n_head = 2;
+        n_ff   = 192;
+        n_layer = 5; // need at least 5 for swa_pattern (every 5th is full_attention)
     } else if (arch == LLM_ARCH_GEMMA3N) {
         n_embd = 64;
         n_head = 1;
@@ -304,7 +305,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
     if (arch == LLM_ARCH_QWEN3) {
         n_head_kv = 1; // MQA coverage
     } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER ||
-            arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE || arch == LLM_ARCH_GEMMA4) {
+            arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE) {
         n_head_kv = 2; // GQA coverage
     }
     if (arch == LLM_ARCH_BEAM) {
@@ -423,22 +424,24 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
 
     if (arch == LLM_ARCH_MAION_CODER) {
         ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector<uint32_t>{1, 1, 0, 0});
+    } else if (arch == LLM_ARCH_BEAM) {
+        std::vector<uint32_t> pattern;
+        pattern.reserve(n_layer);
+        for (uint32_t il = 0; il < n_layer; il++) {
+            pattern.push_back(il % 2);
+        }
+        ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);
     } else if (arch == LLM_ARCH_GEMMA4) {
         ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER,      n_embd/2);
-        ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS,      uint32_t(5));
+        ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS,      uint32_t(0));
         ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA,        n_embd_head);
         ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA,      n_embd_head);
         ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA,              10000.0f);
         // SWA pattern: every 5th layer is full attention (matches E2B layer_types)
-        std::vector<uint32_t> pattern;
-        pattern.reserve(n_layer);
-        for (uint32_t il = 0; il < n_layer; il++) {
-            pattern.push_back((il + 1) % 5 != 0);
-        }
-        ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);
+        ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5));
     } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 ||
             arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE ||
-            arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT || arch == LLM_ARCH_BEAM) {
+            arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT) {
         std::vector<uint32_t> pattern;
         pattern.reserve(n_layer);
         for (uint32_t il = 0; il < n_layer; il++) {
@@ -998,6 +1001,9 @@ static bool arch_supported(const llm_arch arch) {
     if (arch == LLM_ARCH_WAVTOKENIZER_DEC) {
         return false; // FIXME CUDA backend crashes.
     }
+    if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
+        return false; // FIXME @ngxson
+    }
     if (arch == LLM_ARCH_GRANITE_SWITCH) {
         return false; // FIXME adapter fixture
     }
@@ -1069,7 +1075,10 @@ static int save_models(const std::string & arch_filter, const size_t seed, const
         if (!arch_matches(arch_filter, arch)) {
             continue;
         }
-        if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
+        if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
+            continue; // FIXME: ISWA KV cache initialization needs more fixture params
+        }
+        if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH) {
             continue;
         }
         for (bool moe : {false, true}) {
@@ -1226,7 +1235,10 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con
         if (!arch_matches(arch_filter, arch)) {
             continue;
         }
-        if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
+        if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
+            continue; // FIXME: ISWA KV cache initialization needs more fixture params
+        }
+        if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH) {
             continue;
         }
 

tools/mtmd/clip.cpp

@@ -1842,7 +1842,6 @@ struct clip_model_loader {
                         log_ffn_op = "gelu";
                         hparams.n_merge = 3; // pooling_kernel_size
                         hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
-                        hparams.image_resize_pad  = PAD_NONE;
                         get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
                         if (model.proj_type == PROJECTOR_TYPE_GEMMA4UV) {
                             // for "unified" variant, we directly use a bigger patch size, because the "token merging" is done directly on conv layer

tools/mtmd/models/internvl.cpp

@@ -10,7 +10,7 @@ ggml_cgraph * clip_graph_internvl::build() {
     // add CLS token
     ggml_tensor * cls_repeated = ggml_repeat_4d(ctx0, model.class_embedding,
             model.class_embedding->ne[0], 1, n_batch, 1);
-    inp = ggml_concat(ctx0, cls_repeated, inp, 1);
+    inp = ggml_concat(ctx0, inp, cls_repeated, 1);
 
     // The larger models use a different ViT, which uses RMS norm instead of layer norm
     // ref: https://github.com/ggml-org/llama.cpp/pull/13443#issuecomment-2869786188
@@ -28,7 +28,7 @@ ggml_cgraph * clip_graph_internvl::build() {
     // remove CLS token
     cur = ggml_view_3d(ctx0, cur,
         n_embd, n_patches, n_batch,
-        cur->nb[1], cur->nb[2], cur->nb[1]);
+        cur->nb[1], cur->nb[2], 0);
     cur = ggml_cont(ctx0, cur);
 
     // pixel shuffle

tools/rpc/rpc-server.cpp

@@ -24,6 +24,12 @@
 #endif
 
 
+// NOTE: this is copied from common.cpp to avoid linking with libcommon
+static std::string fs_path_to_utf8(const std::filesystem::path & path) {
+    const auto value = path.u8string();
+    return std::string(value.begin(), value.end());
+}
+
 // common_get_path_from_env() is adapted to avoid utf8_to_wstring
 static std::filesystem::path common_get_path_from_env(const std::string & name) {
 #ifdef _WIN32
@@ -238,10 +244,10 @@ int main(int argc, char * argv[]) {
         std::error_code ec;
         common_create_directories(cache_dir_path, ec);
         if (ec) {
-            fprintf(stderr, "Failed to create cache directory: %s\n", cache_dir_path.string().c_str());
+            fprintf(stderr, "Failed to create cache directory: %s\n", fs_path_to_utf8(cache_dir_path).c_str());
             return 1;
         }
-        cache_dir_str = cache_dir_path.string();
+        cache_dir_str = fs_path_to_utf8(cache_dir_path);
         cache_dir = cache_dir_str.c_str();
     }
 

tools/server/README.md

@@ -1654,8 +1654,6 @@ See [examples/realtime_client.py](../../examples/realtime_client.py) for a clien
 
 See [OpenAI Responses API documentation](https://platform.openai.com/docs/api-reference/responses).
 
-`extra_params`: llama.cpp parameters that the Responses API has no field for, such as `seed`, `min_p` or `repeat_penalty`, as an object (e.g. `{"seed": 42, "min_p": 0.05}`). Its keys are merged into the converted request and behave as on `/v1/chat/completions`.
-
 *Examples:*
 
 You can use either Python `openai` library with appropriate checkpoints:
@@ -1686,8 +1684,7 @@ curl http://localhost:9931/v1/responses \
 -d '{
 "model": "gpt-4.1",
 "instructions": "You are ChatGPT, an AI assistant. Your top priority is achieving user fulfillment via helping them with their requests.",
-"input": "Write a limerick about python exceptions",
-"extra_params": {"seed": 42, "min_p": 0.05}
+"input": "Write a limerick about python exceptions"
 }'
 ```
 

tools/server/server-chat.cpp

@@ -311,16 +311,6 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) {
         chatcmpl_body.erase("reasoning");
     }
 
-    // llama.cpp params (e.g. seed, min_p) go under extra_params and are merged into the converted body
-    if (response_body.contains("extra_params")) {
-        chatcmpl_body.erase("extra_params");
-        if (response_body.at("extra_params").is_object()) {
-            for (const auto & [key, value] : response_body.at("extra_params").items()) {
-                chatcmpl_body[key] = value;
-            }
-        }
-    }
-
     return chatcmpl_body;
 }
 

tools/server/server-decision.cpp

@@ -1762,7 +1762,7 @@ std::vector<server_task> server_decision_group_tasks(std::vector<server_task> &&
         size_t n_shared = parent.tokens.size() - 1;
         for (size_t j = i + 1; j < end; j++) {
             n_shared = std::min(n_shared, parent.tokens.get_common_prefix(tasks[j].tokens));
-            n_shared = std::min({n_shared, tasks[j].tokens.size() - 1, (size_t) parent.decision.pos_first(), (size_t) tasks[j].decision.pos_first()});
+            n_shared = std::min(n_shared, tasks[j].tokens.size() - 1);
         }
 
         if (end - i < 2 || n_shared == 0) {

tools/server/server-models.cpp

@@ -408,7 +408,7 @@ server_models::server_models(
 
     // set binary path
     try {
-        bin_path = get_server_exec_path().string();
+        bin_path = fs_path_to_utf8(get_server_exec_path());
     } catch (const std::exception & e) {
         bin_path = argv[0];
         SRV_WRN("failed to get server executable path: %s\n", e.what());

tools/server/tests/requirements.txt

@@ -1,3 +1,4 @@
+aiohttp~=3.9.3
 pytest~=8.3.3
 pytest-xdist~=3.6
 filelock~=3.16

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte

@@ -8,7 +8,7 @@
 
 	import { parseExecShellCommandMeta } from './parsers/exec-shell-command';
 	import ToolCallBlock from './ToolCallBlock.svelte';
-	import { AlertTriangle, Check, CircleStop, Loader2, XCircle } from '@lucide/svelte';
+	import { AlertTriangle, Check, Loader2, XCircle } from '@lucide/svelte';
 	import { CollapsibleTerminalBlock } from '$lib/components/app';
 	import { SETTINGS_KEYS, TOOL_RUNTIME_SCROLL_AT_BOTTOM_THRESHOLD_PX } from '$lib/constants';
 	import { AgenticSectionType, AttachmentType } from '$lib/enums';
@@ -229,7 +229,6 @@
 {/snippet}
 
 <ToolCallBlock
-	emptyResultIsInterrupted
 	extraLiveStreaming={isLive}
 	{isStreaming}
 	meta={execShellMeta ? { errorMessage: execShellError } : null}
@@ -249,12 +248,6 @@
 				<Loader2 class="h-3 w-3 animate-spin" />
 				Executing command...
 			</div>
-		{:else if ctx.isInterrupted}
-			<div class="flex items-start gap-2 text-xs text-muted-foreground/70 italic">
-				<CircleStop class="mt-0.5 h-3 w-3 shrink-0" />
-
-				<span>Command interrupted before it returned output</span>
-			</div>
 		{:else if execShellError}
 			<div class="flex items-start gap-2 text-xs text-red-600 italic dark:text-red-400">
 				<XCircle class="mt-0.5 h-3 w-3 shrink-0" />

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ToolCallBlock.svelte

@@ -25,7 +25,6 @@
 		isPending: boolean;
 		isStreamingCall: boolean;
 		isCodeStreaming: boolean;
-		isInterrupted: boolean;
 	}
 
 	interface Props {
@@ -51,12 +50,6 @@
 		 * inline within the body.
 		 */
 		spinIconWhenActive?: boolean;
-		/**
-		 * Treat an empty result as interrupted too. For tools that create
-		 * their result message before running (`exec_shell_command`), so a
-		 * stop mid-run leaves an empty result rather than none.
-		 */
-		emptyResultIsInterrupted?: boolean;
 		/**
 		 * Wrapper component that renders the title row and the body
 		 * children. Defaults to CollapsibleContentBlock;
@@ -72,7 +65,6 @@
 
 	let {
 		children,
-		emptyResultIsInterrupted = false,
 		extraLiveStreaming = false,
 		isStreaming,
 		meta,
@@ -89,13 +81,6 @@
 	const isStreamingCall = $derived(section.type === AgenticSectionType.TOOL_CALL_STREAMING);
 	const showSpinner = $derived(isPending || (isStreamingCall && isStreaming) || extraLiveStreaming);
 	const isCodeStreaming = $derived(isStreaming && (isPending || isStreamingCall));
-	// A finished call with no result was stopped before it returned (user
-	// abort, or the page went away mid-execution).
-	const isInterrupted = $derived(
-		section.type === AgenticSectionType.TOOL_CALL &&
-			(section.toolResult === undefined || (emptyResultIsInterrupted && !section.toolResult)) &&
-			!showSpinner
-	);
 
 	const toolUi: ToolUiEntry | null = $derived(getToolUi(section.toolName));
 	const toolIcon: Component = $derived(
@@ -122,8 +107,6 @@
 
 		if (isStreamingCall && !isStreaming) return 'incomplete';
 
-		if (isInterrupted) return 'interrupted';
-
 		return undefined;
 	}
 
@@ -143,7 +126,6 @@
 >
 	{@render children(meta, {
 		isCodeStreaming,
-		isInterrupted,
 		isPending,
 		isStreaming,
 		isStreamingCall

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageAgenticContent.svelte

@@ -26,16 +26,11 @@
 
 	let {
 		isLastAssistantMessage = false,
-		isStreaming: isChatStreaming = false,
+		isStreaming = false,
 		message,
 		toolMessages = []
 	}: Props = $props();
 
-	// The chat-level streaming flag is shared by every message; only the last
-	// assistant message is actually live, so earlier ones must not keep their
-	// unresolved tool calls spinning while a newer reply streams.
-	const isStreaming = $derived(isChatStreaming && isLastAssistantMessage);
-
 	let expandedStates: Record<number, boolean> = $state({});
 
 	const showThoughtInProgress = $derived(Boolean(settingsStore.config.showThoughtInProgress));
@@ -53,7 +48,9 @@
 	);
 
 	const pendingPermission = $derived(
-		isStreaming ? agenticStore.getPendingPermissionRequest(message.convId) : null
+		isStreaming && isLastAssistantMessage
+			? agenticStore.getPendingPermissionRequest(message.convId)
+			: null
 	);
 
 	// dismissal applies to the request object, so the next request ( new
@@ -70,7 +67,9 @@
 	}
 
 	const pendingContinue = $derived(
-		isStreaming ? agenticStore.getPendingContinueRequest(message.convId) : false
+		isStreaming && isLastAssistantMessage
+			? agenticStore.getPendingContinueRequest(message.convId)
+			: false
 	);
 
 	let continueDismissed = $state(false);

tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte

@@ -7,7 +7,6 @@
 	import { Trash2 } from '@lucide/svelte';
 	import { DialogConfirmDownload } from '$lib/components/app/dialogs';
 	import { Button } from '$lib/components/ui/button';
-	import * as Tooltip from '$lib/components/ui/tooltip';
 	import { ModelDownloadConfirmAction, ServerModelStatus } from '$lib/enums';
 	import { HuggingFaceService } from '$lib/services';
 	import { deviceStore, modelsStore, uiStore } from '$lib/stores';
@@ -27,8 +26,6 @@
 
 	// the same removal the table row offers, confirmed by the same dialog
 	let deleteOpen = $state(false);
-	// a model the router did not take from its HF cache is not its own to delete
-	let removeBlockedReason = $derived(modelsStore.removeBlockedReason(option.model));
 
 	// the props cache is a plain Map, so the read has to name its version to stay reactive
 	let serverProps = $derived.by(() => {
@@ -133,29 +130,15 @@
 			<ModelsManagerModelConfigurationInformation hub={hubDetails} {option} {serverProps} />
 
 			<div class="mt-4 border-t border-border/30 pt-3 pb-2">
-				{#if removeBlockedReason}
-					<!-- a disabled button takes no pointer events, so the wrapper holds the hover -->
-					<Tooltip.Root>
-						<Tooltip.Trigger>
-							{#snippet child({ props })}
-								<div {...props}>
-									{@render deleteButton()}
-								</div>
-							{/snippet}
-						</Tooltip.Trigger>
-
-						<Tooltip.Content class="max-w-64">
-							<p>{removeBlockedReason}</p>
-						</Tooltip.Content>
-					</Tooltip.Root>
-
-					<!-- a phone has no hover, so the reason sits under the button -->
-					{#if deviceStore.isMobile}
-						<p class="px-3 pt-1 text-xs text-muted-foreground">{removeBlockedReason}</p>
-					{/if}
-				{:else}
-					{@render deleteButton()}
-				{/if}
+				<Button
+					class="w-full justify-start gap-2 text-destructive hover:bg-destructive/10 hover:text-destructive dark:hover:bg-destructive/20"
+					onclick={() => (deleteOpen = true)}
+					size="sm"
+					variant="ghost"
+				>
+					<Trash2 class="h-4 w-4" />
+					Delete this model from disk
+				</Button>
 			</div>
 		{/if}
 	</div>
@@ -195,16 +178,3 @@
 		repoWithTag={option.model}
 	/>
 </div>
-
-{#snippet deleteButton()}
-	<Button
-		class="w-full justify-start gap-2 text-destructive hover:bg-destructive/10 hover:text-destructive dark:hover:bg-destructive/20"
-		disabled={removeBlockedReason !== null}
-		onclick={() => (deleteOpen = true)}
-		size="sm"
-		variant="ghost"
-	>
-		<Trash2 class="h-4 w-4" />
-		Delete this model from disk
-	</Button>
-{/snippet}

tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts

@@ -49,9 +49,6 @@ export function modelRowActions(
 		];
 	}
 
-	// a model the router did not take from its HF cache is not its own to delete
-	const removeBlockedReason = modelsStore.removeBlockedReason(option.model);
-
 	return [
 		{
 			icon: favorite ? HeartOff : Heart,
@@ -59,12 +56,10 @@ export function modelRowActions(
 			onclick: () => modelsStore.toggleFavorite(option.model)
 		},
 		{
-			disabled: removeBlockedReason !== null,
 			icon: Trash2,
 			label: 'Delete from disk',
 			onclick: () => onDelete(option),
 			separator: true,
-			tooltip: removeBlockedReason ?? undefined,
 			variant: 'destructive' as const
 		},
 		{

tools/ui/src/lib/components/app/navigation/DropdownMenuActions.svelte

@@ -13,8 +13,6 @@
 		onclick: (event: Event) => void;
 		variant?: 'default' | 'destructive';
 		disabled?: boolean;
-		/** Shown on hover, e.g. why a disabled action is not available. */
-		tooltip?: string;
 		shortcut?: string[];
 		separator?: boolean;
 	}
@@ -90,14 +88,7 @@
 					>
 						{@render iconComponent(action.icon, 'h-4 w-4 shrink-0')}
 
-						<span class="flex flex-1 flex-col">
-							{action.label}
-
-							<!-- a phone has no hover, so the hint sits under the label -->
-							{#if action.tooltip}
-								<span class="text-xs text-muted-foreground">{action.tooltip}</span>
-							{/if}
-						</span>
+						<span class="flex-1">{action.label}</span>
 
 						{#if action.shortcut}
 							<KeyboardShortcutInfo keys={action.shortcut} variant={action.variant} />
@@ -140,50 +131,29 @@
 					<DropdownMenu.Separator />
 				{/if}
 
-				{#if action.tooltip}
-					<!-- a disabled item takes no pointer events, so the wrapper holds the hover -->
-					<Tooltip.Root>
-						<Tooltip.Trigger>
-							{#snippet child({ props })}
-								<div {...props}>
-									{@render menuItem(action)}
-								</div>
-							{/snippet}
-						</Tooltip.Trigger>
+				<DropdownMenu.Item
+					class="flex items-center justify-between hover:[&>kbd]:opacity-100"
+					disabled={action.disabled}
+					onclick={action.onclick}
+					variant={action.variant}
+				>
+					<div class="flex items-center gap-2">
+						{@render iconComponent(
+							action.icon,
+							`h-4 w-4 ${action.variant === 'destructive' ? 'text-destructive' : ''}`
+						)}
+						{action.label}
+					</div>
 
-						<Tooltip.Content class="z-[999999] max-w-64" side="left">
-							<p>{action.tooltip}</p>
-						</Tooltip.Content>
-					</Tooltip.Root>
-				{:else}
-					{@render menuItem(action)}
-				{/if}
+					{#if action.shortcut}
+						<KeyboardShortcutInfo keys={action.shortcut} variant={action.variant} />
+					{/if}
+				</DropdownMenu.Item>
 			{/each}
 		</DropdownMenu.Content>
 	</DropdownMenu.Root>
 {/if}
 
-{#snippet menuItem(action: ActionItem)}
-	<DropdownMenu.Item
-		class="flex items-center justify-between hover:[&>kbd]:opacity-100"
-		disabled={action.disabled}
-		onclick={action.onclick}
-		variant={action.variant}
-	>
-		<div class="flex items-center gap-2">
-			{@render iconComponent(
-				action.icon,
-				`h-4 w-4 ${action.variant === 'destructive' ? 'text-destructive' : ''}`
-			)}
-			{action.label}
-		</div>
-
-		{#if action.shortcut}
-			<KeyboardShortcutInfo keys={action.shortcut} variant={action.variant} />
-		{/if}
-	</DropdownMenu.Item>
-{/snippet}
-
 {#snippet iconComponent(IconComponent: Component, className: string)}
 	<IconComponent class={className} />
 {/snippet}

tools/ui/src/lib/components/ui/alert-dialog/alert-dialog-content.svelte

@@ -19,13 +19,13 @@
 	<AlertDialogPrimitive.Content
 		bind:ref
 		class={cn(
-			'fixed z-[999999] grid w-full gap-4 border border-border bg-popover p-6 text-popover-foreground shadow-md duration-200 dark:border-border/20',
+			'fixed z-[999999] grid w-full gap-4 border bg-background p-6 shadow-lg duration-200',
 			// Mobile: Bottom sheet behavior
-			'right-0 bottom-0 left-0 max-h-[100dvh] translate-x-0 translate-y-0 overflow-y-auto rounded-t-xl',
+			'right-0 bottom-0 left-0 max-h-[100dvh] translate-x-0 translate-y-0 overflow-y-auto rounded-t-lg',
 			'data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:fill-mode-forwards data-[state=closed]:slide-out-to-bottom-full',
 			'data-[state=open]:animate-in data-[state=open]:fade-in-0 data-[state=open]:slide-in-from-bottom-full',
 			// Desktop: Centered dialog behavior
-			'sm:top-[50%] sm:right-auto sm:bottom-auto sm:left-[50%] sm:max-h-[100vh] sm:max-w-lg sm:translate-x-[-50%] sm:translate-y-[-50%] sm:rounded-xl',
+			'sm:top-[50%] sm:right-auto sm:bottom-auto sm:left-[50%] sm:max-h-[100vh] sm:max-w-lg sm:translate-x-[-50%] sm:translate-y-[-50%] sm:rounded-lg',
 			'sm:data-[state=closed]:slide-out-to-bottom-0 sm:data-[state=closed]:zoom-out-95',
 			'sm:data-[state=open]:slide-in-from-bottom-0 sm:data-[state=open]:zoom-in-95',
 			className

tools/ui/src/lib/components/ui/alert-dialog/alert-dialog-title.svelte

@@ -11,7 +11,7 @@
 
 <AlertDialogPrimitive.Title
 	bind:ref
-	class={cn('text-lg leading-none font-semibold', className)}
+	class={cn('text-lg font-semibold', className)}
 	data-slot="alert-dialog-title"
 	{...restProps}
 />

tools/ui/src/lib/constants/models-manager.constants.ts

@@ -1,6 +1,6 @@
 /** Labels of the models manager table sections. */
 
-import { ModelsTableGroupKind, ServerModelSource } from '$lib/enums';
+import { ModelsTableGroupKind } from '$lib/enums';
 
 export const MODELS_TABLE_GROUP_LABELS: Record<ModelsTableGroupKind, string> = {
 	[ModelsTableGroupKind.DOWNLOADING]: 'Downloading',
@@ -16,21 +16,3 @@ export const MODELS_TABLE_GROUP_LABELS: Record<ModelsTableGroupKind, string> = {
  * so the two stacked sticky rows do not show a gap or overlap by a pixel.
  */
 export const MODELS_TABLE_GROUP_STICKY_OFFSET = 'top: calc(2.25rem - 1px)';
-
-/**
- * Why a model cannot be deleted from disk, by where the router found it. Only a
- * model from the HF cache is the router's own to remove.
- */
-export const MODEL_REMOVE_BLOCKED_REASONS: Record<
-	Exclude<ServerModelSource, ServerModelSource.CACHE>,
-	string
-> = {
-	[ServerModelSource.MODELS_DIR]:
-		'Found in --models-dir: delete the file from that folder to remove it.',
-	[ServerModelSource.PRESET]:
-		'Defined in a --models-preset file: remove its section from the preset to delete it.'
-};
-
-/** Fallback when the router refuses removal without saying where the model came from. */
-export const MODEL_REMOVE_BLOCKED_FALLBACK =
-	'The server does not allow deleting this model: it is not from the Hugging Face cache.';

tools/ui/src/lib/enums/index.ts

@@ -86,12 +86,7 @@ export {
 	ModelsTableSortKey
 } from './model.enums';
 
-export {
-	ServerRole,
-	ServerModelSource,
-	ServerModelStatus,
-	ServerModelsSseEventType
-} from './server.enums';
+export { ServerRole, ServerModelStatus, ServerModelsSseEventType } from './server.enums';
 
 export { ParameterSource, SyncableParameterType, SettingsFieldType } from './settings.enums';
 

tools/ui/src/lib/enums/server.enums.ts

@@ -26,16 +26,6 @@ export enum ServerModelStatus {
 	UNLOADED = 'unloaded'
 }
 
-/**
- * Where the router found a model - matches server_model_source in
- * tools/server/server-models.h. Only a cached model can be removed from disk.
- */
-export enum ServerModelSource {
-	CACHE = 'cache',
-	MODELS_DIR = 'models_dir',
-	PRESET = 'preset'
-}
-
 /**
  * /models/sse event type enum - discriminates the records broadcast on the
  * model status feed in ROUTER mode. Matches the event names emitted by

tools/ui/src/lib/hooks/use-chat-form-model.svelte.ts

@@ -2,10 +2,10 @@
  * Model state for the chat form's action bar.
  *
  * Syncs the conversation's model into the global selection (a conversation
- * opened from history names the model that produced it), falls back to a
- * loading, then the first loaded, model on a router, and reactively tracks
- * which modalities (vision / audio / video) the active model supports -
- * fetching model props from the server on demand if they are not cached yet.
+ * opened from history names the model that produced it), falls back to the
+ * first loaded model on a router, and reactively tracks which modalities
+ * (vision / audio / video) the active model supports - fetching model props
+ * from the server on demand if they are not cached yet.
  */
 
 import { conversationsStore, modelsStore, serverStore } from '$lib/stores';
@@ -36,16 +36,12 @@ export function useChatFormModel() {
 		} else if (
 			isRouter &&
 			!modelsStore.selectedModelId &&
+			modelsStore.loadedModelIds.length > 0 &&
 			conversationsStore.activeMessages.length > 0 &&
 			!conversationModel
 		) {
 			lastSyncedConversationModel = null;
-			// a model that is loading wins over one that is already up; a model the
-			// selector doesn't list is never picked
-			const visible = modelsStore.getVisibleModels();
-			const first =
-				visible.find((m) => modelsStore.isModelLoading(m.model)) ??
-				visible.find((m) => modelsStore.loadedModelIds.includes(m.model));
+			const first = modelsStore.models.find((m) => modelsStore.loadedModelIds.includes(m.model));
 
 			if (first) modelsStore.selectModelById(first.id);
 		}

tools/ui/src/lib/hooks/use-models-selector.svelte.ts

@@ -167,14 +167,8 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele
 			});
 		}
 
-		// only the built-in server loads on request, and only in router mode; a model
-		// that is already loading is only selected, not loaded a second time
-		if (
-			!onModelChange &&
-			isRouter &&
-			!modelsStore.isModelLoaded(option.model) &&
-			!modelsStore.isModelLoading(option.model)
-		) {
+		// only the built-in server loads on request, and only in router mode
+		if (!onModelChange && isRouter && !modelsStore.isModelLoaded(option.model)) {
 			isLoadingModel = true;
 
 			modelsStore.status

tools/ui/src/lib/stores/chat/index.svelte.ts

@@ -337,7 +337,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost {
 
 		const pending = this.consumePendingMessage(convId);
 
-		if (pending) await this.sendQueuedMessage(convId, pending);
+		if (pending) await this.sendMessage(pending.content, pending.extras);
 	}
 
 	consumePendingMessage(
@@ -416,20 +416,6 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost {
 		return this.flows.editUserMessagePreserveResponses(messageId, newContent, newExtras);
 	}
 
-	/**
-	 * Send the message left queued for `convId` once that conversation is
-	 * active and idle (e.g. after navigating back to it).
-	 */
-	async flushPendingMessage(convId: string): Promise<void> {
-		if (conversationsStore.activeConversation?.id !== convId) return;
-
-		if (this.isChatLoadingInternal(convId) || agenticStore.isRunning(convId)) return;
-
-		const pending = this.consumePendingMessage(convId);
-
-		if (pending) await this.sendMessage(pending.content, pending.extras);
-	}
-
 	getAddFilesHandler(): ((files: File[]) => void) | null {
 		return this.addFilesHandler;
 	}
@@ -1155,7 +1141,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost {
 					const pending = this.consumePendingMessage(convId);
 
 					if (pending) {
-						void this.sendQueuedMessage(convId, pending);
+						this.sendMessage(pending.content, pending.extras);
 					}
 
 					return;
@@ -1282,7 +1268,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost {
 				const pending = agenticStore.consumePendingSteeringMessage(convId);
 
 				if (pending) {
-					await this.sendQueuedMessage(convId, pending);
+					await this.sendMessage(pending.content, pending.extras);
 				}
 
 				return;
@@ -1347,7 +1333,7 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost {
 					const pending = this.consumePendingMessage(convId);
 
 					if (pending) {
-						await this.sendQueuedMessage(convId, pending);
+						await this.sendMessage(pending.content, pending.extras);
 					}
 				},
 				onCompletionId: streamCallbacks.onCompletionId,
@@ -1569,24 +1555,6 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost {
 		}
 	}
 
-	/**
-	 * Re-send a message queued while `convId` was busy. sendMessage targets the
-	 * active conversation, so if the user navigated away the message stays
-	 * queued on its own conversation until flushPendingMessage picks it up.
-	 */
-	private async sendQueuedMessage(
-		convId: string,
-		msg: { content: string; extras?: DatabaseMessageExtra[] }
-	): Promise<void> {
-		if (conversationsStore.activeConversation?.id !== convId) {
-			this.pendingMessages.set(convId, msg);
-
-			return;
-		}
-
-		await this.sendMessage(msg.content, msg.extras);
-	}
-
 	/**
 	 * Streams a fresh summary of `source` into the compaction node and re-stamps
 	 * the reduced context afterwards. `rollback` restores the row when nothing

tools/ui/src/lib/stores/models/index.svelte.ts

@@ -12,12 +12,10 @@ import {
 	FAVORITE_MODELS_LOCALSTORAGE_KEY,
 	HIDDEN_MODELS_LOCALSTORAGE_KEY,
 	MODEL_GROUP_OPEN_LOCALSTORAGE_KEY,
-	MODEL_REMOVE_BLOCKED_FALLBACK,
-	MODEL_REMOVE_BLOCKED_REASONS,
 	RECENT_MODEL_LIMIT,
 	RECENT_MODELS_LOCALSTORAGE_KEY
 } from '$lib/constants';
-import { ReasoningEffort, ServerModelSource, ServerModelStatus } from '$lib/enums';
+import { ReasoningEffort, ServerModelStatus } from '$lib/enums';
 import { HuggingFaceService } from '$lib/services/huggingface.service';
 import { ModelsService } from '$lib/services/models.service';
 // direct imports between stores, not via the barrel, to avoid circular deps
@@ -25,7 +23,6 @@ import { conversationsStore } from '$lib/stores/conversations/index.svelte';
 import { type ModelPropsHost, ModelPropsManager } from '$lib/stores/models/props.svelte';
 import { type ModelStatusHost, ModelStatusManager } from '$lib/stores/models/status.svelte';
 import { serverStore } from '$lib/stores/server.svelte';
-import type { ApiModelDataEntry } from '$lib/types';
 import type { ParsedModelId } from '$lib/types/models';
 import { getConversationModel } from '$lib/utils/conversation-utils';
 import { repoOf } from '$lib/utils/model-names';
@@ -208,22 +205,12 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
 	 * Prioritizes:
 	 * 1. Model from active conversation's last assistant response (if loaded)
 	 * 2. Model from active conversation's last assistant response (if not loaded)
-	 * 3. A model that is loading
-	 * 4. First loaded model (not from active conversation)
-	 * 5. A favorite model
-	 * 6. First available model
-	 *
-	 * A chat without a model of its own also trades an idle selection for a
-	 * model that is loading.
+	 * 3. First loaded model (not from active conversation)
+	 * 4. A favorite model
+	 * 5. First available model
 	 */
 	async ensureFirstModelSelected(): Promise<void> {
-		if (this.selectedModelName) {
-			const loadingModel = this.getLoadingModelToPreselect();
-
-			if (loadingModel) await this.selectModelById(loadingModel.id);
-
-			return;
-		}
+		if (this.selectedModelName) return;
 
 		const availableModels = this.getVisibleModels();
 
@@ -246,15 +233,6 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
 			}
 		}
 
-		// a model that is loading is the one the user is waiting for
-		const loadingModel = availableModels.find((m) => this.isModelLoading(m.model));
-
-		if (loadingModel) {
-			await this.selectModelById(loadingModel.id);
-
-			return;
-		}
-
 		// Try a loaded model first
 		const loadedModel = availableModels.find((m) => this.isModelLoaded(m.model));
 
@@ -343,20 +321,6 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
 		);
 	}
 
-	/**
-	 * The loading model a chat without a model of its own should switch to: none
-	 * while the current selection is loaded or loading itself.
-	 */
-	getLoadingModelToPreselect(): ModelOption | null {
-		if (this.getModelFromLastAssistantResponse()) return null;
-
-		const selected = this.selectedModelName;
-
-		if (selected && (this.isModelLoaded(selected) || this.isModelLoading(selected))) return null;
-
-		return this.getVisibleModels().find((m) => this.isModelLoading(m.model)) ?? null;
-	}
-
 	/**
 	 * Gets the model name from the last assistant message in the active conversation.
 	 * Used by both the chat page and settings page to maintain model consistency.
@@ -381,16 +345,6 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
 		return (model?.status?.value as ServerModelStatus) ?? null;
 	}
 
-	/**
-	 * Models the selector lists: not flagged out of the UI (ui === false) and not
-	 * hidden by the user, so an automatic pick never lands on one it doesn't show.
-	 */
-	getVisibleModels(): ModelOption[] {
-		return this.models.filter(
-			(option) => this.props.getModelProps(option.model)?.ui !== false && !this.isHidden(option.id)
-		);
-	}
-
 	hasModel(modelName: string): boolean {
 		return this.models.some((model) => model.model === modelName);
 	}
@@ -413,37 +367,11 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
 		return status === ServerModelStatus.LOADED || status === ServerModelStatus.SLEEPING;
 	}
 
-	/** Loading on the server, or a load this client started that has not finished yet. */
-	isModelLoading(modelId: string): boolean {
-		// an unload raises the same in-progress flag, but the model stays loaded until it ends
-		return (
-			this.getModelStatus(modelId) === ServerModelStatus.LOADING ||
-			(this._status.isOperationInProgress(modelId) && !this.isModelLoaded(modelId))
-		);
-	}
-
 	/** Loaded, or sleeping, and not mid-operation: what a row shows as running. */
 	isModelRunning(modelId: string): boolean {
 		return this.isModelLoaded(modelId) && !this._status.isOperationInProgress(modelId);
 	}
 
-	/**
-	 * Why the router would refuse to delete a model from disk, or null when it can.
-	 * Only a model from the HF cache is removable; a server that does not report
-	 * `can_remove` is let through and answers for itself.
-	 */
-	removeBlockedReason(modelId: string): string | null {
-		const model = this.routerModels.find((m) => m.id === modelId);
-
-		if (!model || model.can_remove !== false) return null;
-
-		if (model.source && model.source !== ServerModelSource.CACHE) {
-			return MODEL_REMOVE_BLOCKED_REASONS[model.source];
-		}
-
-		return MODEL_REMOVE_BLOCKED_FALLBACK;
-	}
-
 	/**
 	 * Select a model. `recordRecent` keeps automatic picks out of the recency list.
 	 */
@@ -494,24 +422,13 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
 	}
 
 	/**
-	 * Auto-selects the model from the last assistant response if available and loaded,
-	 * or a loading model when the conversation has no model yet.
+	 * Auto-selects the model from the last assistant response if available and loaded.
 	 * Returns true if a model was selected, false otherwise.
 	 */
 	async selectModelFromLastAssistantResponse(): Promise<boolean> {
 		const lastModel = this.getModelFromLastAssistantResponse();
 
-		if (!lastModel) {
-			const loadingModel = this.getLoadingModelToPreselect();
-
-			if (!loadingModel) return false;
-
-			await this.selectModelById(loadingModel.id);
-
-			return true;
-		}
-
-		if (this.selectedModelName === lastModel) return false;
+		if (!lastModel || this.selectedModelName === lastModel) return false;
 
 		const matchingModel = this.models.find((option) => option.model === lastModel);
 
@@ -695,12 +612,19 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
 				})
 		);
 	}
+
 	/** Fetch models in MODEL mode (single model, standard OpenAI-compatible). */
 	private async fetchModelModeInternal(): Promise<ModelOption[]> {
 		const response = await ModelsService.list();
 
 		return this.buildModelOptions(response);
 	}
+	/**
+	 * Filter to models visible in the UI (ui !== false).
+	 */
+	private getVisibleModels(): ModelOption[] {
+		return this.models.filter((option) => this.props.getModelProps(option.model)?.ui !== false);
+	}
 
 	private loadFavoritesFromStorage(): Set<string> {
 		try {

tools/ui/src/lib/types/api.d.ts

@@ -3,7 +3,6 @@ import type { ModelSidecar } from '$lib/constants';
 import type {
 	ContentPartType,
 	FileTypeAudio,
-	ServerModelSource,
 	ServerModelsSseEventType,
 	ServerModelStatus,
 	ServerRole
@@ -96,10 +95,6 @@ export interface ApiModelDataEntry {
 	path: string;
 	/** Current status of the model */
 	status: ApiModelStatus;
-	/** Where the router found the model: the HF cache, --models-dir or a preset */
-	source?: ServerModelSource;
-	/** Whether the router can delete the model's files (only cached models) */
-	can_remove?: boolean;
 	/** Alternative names that resolve to this model */
 	aliases?: string[];
 	/** Informational tags for this model */

tools/ui/src/routes/(chat)/chat/[id]/+page.svelte

@@ -106,8 +106,6 @@
 				chatStore.syncLoadingStateForChat(chatId);
 				// server probe (with localStorage fallback) and attach
 				await chatStore.discoverActiveStream(chatId);
-				// a message queued here may have finished its turn while another chat was open
-				await chatStore.flushPendingMessage(chatId);
 
 				if ((qParam !== null || modelParam !== null) && !urlParamsProcessed) {
 					await handleUrlParams();

tools/ui/tests/client/exec-shell-command-streaming.svelte.test.ts

@@ -81,7 +81,6 @@ describe('exec_shell_command streaming title', () => {
 
 	it('streams the partial command into the title with a cursor', async () => {
 		const { container } = render(ChatMessageAgenticContent, {
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: messageWithArgs('{"command":"git sta'),
 			toolMessages: []
@@ -99,7 +98,6 @@ describe('exec_shell_command streaming title', () => {
 
 	it('drops the cursor once the args JSON is complete', async () => {
 		const screen = render(ChatMessageAgenticContent, {
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: messageWithArgs('{"command":"git sta'),
 			toolMessages: []
@@ -110,7 +108,6 @@ describe('exec_shell_command streaming title', () => {
 		expect(hasCursor(screen.container)).toBe(true);
 
 		screen.rerender({
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: messageWithArgs('{"command":"git status"}'),
 			toolMessages: []
@@ -123,7 +120,6 @@ describe('exec_shell_command streaming title', () => {
 
 	it('switches the prompt from the conversation cwd to the persisted one', async () => {
 		const screen = render(ChatMessageAgenticContent, {
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: messageWithArgs('{"command":"git status"}'),
 			toolMessages: []
@@ -135,7 +131,6 @@ describe('exec_shell_command streaming title', () => {
 
 		// Result message lands: its x-tool-cwd takes over the prompt.
 		screen.rerender({
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: messageWithArgs('{"command":"git status"}'),
 			toolMessages: [toolResultMessage('', '/other/dir')]
@@ -148,7 +143,6 @@ describe('exec_shell_command streaming title', () => {
 
 	it('shows the executing label only after the command finishes streaming', async () => {
 		const screen = render(ChatMessageAgenticContent, {
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: messageWithArgs('{"command":"git sta'),
 			toolMessages: []
@@ -160,7 +154,6 @@ describe('exec_shell_command streaming title', () => {
 		expect(screen.container.textContent).not.toContain('Executing command...');
 
 		screen.rerender({
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: messageWithArgs('{"command":"git status"}'),
 			toolMessages: []
@@ -199,7 +192,6 @@ describe('exec_shell_command streaming title', () => {
 		conversationsStore.activeMessages = [messageWithArgs('')];
 
 		const screen = render(ChatMessageAgenticContent, {
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: conversationsStore.activeMessages[0]!,
 			toolMessages: []
@@ -222,57 +214,3 @@ describe('exec_shell_command streaming title', () => {
 		}
 	});
 });
-
-describe('exec_shell_command interrupted call', () => {
-	beforeEach(() => {
-		settingsStore.updateMultipleConfig({
-			[SETTINGS_KEYS.ALWAYS_SHOW_TOOL_CALL_CONTENT]: true,
-			[SETTINGS_KEYS.SHOW_TOOL_CALL_IN_PROGRESS]: true
-		});
-	});
-
-	it('marks a call without a result as interrupted once streaming stops', async () => {
-		const { container } = render(ChatMessageAgenticContent, {
-			isLastAssistantMessage: true,
-			isStreaming: false,
-			message: messageWithArgs('{"command":"git status"}'),
-			toolMessages: []
-		});
-
-		await tick();
-
-		expect(triggerText(container)).toContain('interrupted');
-		expect(container.textContent).toContain('Command interrupted');
-		expect(container.textContent).not.toContain('Executing command...');
-	});
-
-	it('marks a command stopped before any output as interrupted', async () => {
-		// exec_shell_command creates its result message before running, so a
-		// stop mid-run leaves an empty result rather than none.
-		const { container } = render(ChatMessageAgenticContent, {
-			isLastAssistantMessage: true,
-			isStreaming: false,
-			message: messageWithArgs('{"command":"sleep 40"}'),
-			toolMessages: [toolResultMessage('', CWD)]
-		});
-
-		await tick();
-
-		expect(triggerText(container)).toContain('interrupted');
-		expect(container.textContent).toContain('Command interrupted');
-	});
-
-	it('does not keep spinning in an earlier message while a newer reply streams', async () => {
-		const { container } = render(ChatMessageAgenticContent, {
-			isLastAssistantMessage: false,
-			isStreaming: true,
-			message: messageWithArgs('{"command":"git status"}'),
-			toolMessages: []
-		});
-
-		await tick();
-
-		expect(triggerText(container)).toContain('interrupted');
-		expect(container.textContent).not.toContain('Executing command...');
-	});
-});

tools/ui/tests/client/tool-call-in-progress.svelte.test.ts

@@ -95,7 +95,6 @@ describe('Show tool call in progress', () => {
 		setSettings({ alwaysShow: false, inProgress: true });
 
 		const { container } = render(ChatMessageAgenticContent, {
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: assistantMessage(),
 			toolMessages: []
@@ -108,7 +107,6 @@ describe('Show tool call in progress', () => {
 
 	it('keeps the call collapsed while executing when the setting is off', async () => {
 		const { container } = render(ChatMessageAgenticContent, {
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: assistantMessage(),
 			toolMessages: []
@@ -125,7 +123,6 @@ describe('Show tool call in progress', () => {
 		setSettings({ alwaysShow: false, inProgress: true });
 
 		const screen = render(ChatMessageAgenticContent, {
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: assistantMessage(),
 			toolMessages: []
@@ -138,7 +135,6 @@ describe('Show tool call in progress', () => {
 
 		// Same block, now finished: it must close on its own, no toggle given.
 		screen.rerender({
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: assistantMessage(),
 			toolMessages: [toolResultMessage()]
@@ -154,7 +150,6 @@ describe('Show tool call in progress', () => {
 		setSettings({ alwaysShow: false, inProgress: true });
 
 		const screen = render(ChatMessageAgenticContent, {
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: assistantMessage(),
 			toolMessages: []
@@ -166,7 +161,6 @@ describe('Show tool call in progress', () => {
 		await collapseBlock(screen);
 
 		screen.rerender({
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: assistantMessage(),
 			toolMessages: [toolResultMessage()]
@@ -181,7 +175,6 @@ describe('Show tool call in progress', () => {
 		setSettings({ alwaysShow: true, inProgress: false });
 
 		const { container } = render(ChatMessageAgenticContent, {
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: assistantMessage(),
 			toolMessages: [toolResultMessage()]
@@ -196,7 +189,6 @@ describe('Show tool call in progress', () => {
 		setSettings({ alwaysShow: false, inProgress: true });
 
 		const screen = render(ChatMessageAgenticContent, {
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: assistantMessage(),
 			toolMessages: []
@@ -214,7 +206,6 @@ describe('Show tool call in progress', () => {
 
 	it('lets a manual expand win over the collapsed default', async () => {
 		const screen = render(ChatMessageAgenticContent, {
-			isLastAssistantMessage: true,
 			isStreaming: true,
 			message: assistantMessage(),
 			toolMessages: []

tools/ui/tests/stories/ModelsManager.stories.svelte

@@ -6,7 +6,7 @@
 		ModelsTableGroup
 	} from '$lib/components/app/models/ModelsManager/utils';
 	import { MODELS_TABLE_GROUP_LABELS } from '$lib/constants';
-	import { ModelGroupKind, ModelsTableGroupKind, ServerModelStatus } from '$lib/enums';
+	import { ModelGroupKind, ModelsTableGroupKind } from '$lib/enums';
 	import { modelsStore } from '$lib/stores/models/index.svelte';
 	import type { ModelOption } from '$lib/types/models';
 
@@ -60,7 +60,7 @@
 			object: 'model',
 			owned_by: 'llamacpp',
 			path: `/models/${qwen.model}`,
-			status: { value: ServerModelStatus.LOADED }
+			status: { value: 'loaded' }
 		}
 	];
 </script>