engine: separate engine and server

* feat(convert): Add decision.chat to support decision models that can also chat

Branch: GraniteSwitchDecision
AI-usage: full (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>

* feat(convert): Add conversion for Granite Switch as decision model

The template follows from the usage sample:
https://huggingface.co/ibm-granite/granite-switch-4.1-8b#guardian-and-decision-side-by-side

Branch: GraniteSwitchDecision
AI-usage: draft (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>

* feat(common): Add support for chat+decision models in common

Branch: GraniteSwitchDecision
AI-usage: full (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>

* feat(common): Add granite switch decision model enum

Branch: GraniteSwitchDecision
AI-usage: full (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>

* feat(server): Plumb through decision_chat bool to support decision+chat models

Branch: GraniteSwitchDecision
AI-usage: full (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>

* feat(server): Add Granite Switch decision support

Branch: GraniteSwitchDecision
AI-usage: draft (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>

* docs: Update server README for granite switch decisions / chat+decisions

Branch: GraniteSwitchDecision
AI-usage: draft (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>

* Revert "feat(convert): Add decision.chat to support decision models that can also chat"

This reverts commit 952ef56764.

* Revert "feat(common): Add support for chat+decision models in common"

This reverts commit 93ed782e3a.

* Revert "feat(server): Plumb through decision_chat bool to support decision+chat models"

This reverts commit 97e9af60dd.

* fix: If Granite Switch supports decisions, it always supports text too

Branch: GraniteSwitchDecision
AI-usage: none
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>

* docs: Remove reference to decision.chat in server README

Branch: GraniteSwitchDecision
AI-usage: none
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>

.devops/intel.Dockerfile

@@ -50,7 +50,7 @@ RUN if [ "${GGML_SYCL_F16}" = "ON" ]; then \
     fi && \
     echo "Building with dynamic libs" && \
     cmake -B build -DGGML_NATIVE=OFF -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DLLAMA_BUILD_TESTS=OFF ${OPT_SYCL_F16} && \
-    cmake --build build --config Release -j$(nproc)
+    cmake --build build --config Release -j$((($(nproc)+1)/2))
 
 RUN mkdir -p /app/lib && \
     find build -name "*.so*" -exec cp -P {} /app/lib \;

.github/labeler.yml

@@ -46,6 +46,7 @@ examples:
                 - examples/**
                 - tools/**
             - all-globs-to-all-files:
+                - '!tools/engine/**'
                 - '!tools/server/**'
                 - '!tools/mtmd/**'
                 - '!tools/ui/**'
@@ -76,6 +77,7 @@ server/ui:
 server:
     - changed-files:
         - any-glob-to-any-file:
+            - tools/engine/**
             - tools/server/**
 mtmd:
     - changed-files:

CODEOWNERS

@@ -101,6 +101,7 @@
 /tools/batched-bench/                   @ggerganov
 /tools/cli/                             @ngxson
 /tools/completion/                      @ggerganov
+/tools/engine/                          @ggml-org/llama-server
 /tools/mtmd/                            @ggml-org/llama-mtmd
 /tools/perplexity/                      @ggerganov
 /tools/parser/                          @pwilkin

common/common.cpp

@@ -1174,6 +1174,7 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
     { COMMON_DECISION_TYPE_VLLM_D3,        "vllm-d3"        },
     { COMMON_DECISION_TYPE_LFM2_D1,        "lfm2-d1"       },
     { COMMON_DECISION_TYPE_LFM2_D1_OMNI,   "lfm2-d1-omni"  },
+    { COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"},
     // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
 };
 

common/common.h

@@ -970,6 +970,7 @@ enum common_decision_type {
     COMMON_DECISION_TYPE_VLLM_D3, // same as pplx-decider, non-causal attention
     COMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question type
     COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout
+    COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch
     COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
 };
 

common/http.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "log.h"
+
 #include <cpp-httplib/httplib.h>
 
 #ifdef _WIN32
@@ -97,6 +99,118 @@ static common_http_url common_http_parse_url(const std::string & url) {
     return parts;
 }
 
+// match a loopback destination: a remote proxy can never reach the client machine's own
+// loopback, so these hosts are always contacted directly, before NO_PROXY is consulted
+// (similar to Go's net/http)
+static bool common_http_is_loopback(const std::string & host) {
+    auto lower = [](std::string s) {
+        for (auto & c : s) {
+            if (c >= 'A' && c <= 'Z') {
+                c += 'a' - 'A';
+            }
+        }
+        return s;
+    };
+
+    const std::string h = lower(host);
+
+    if (h == "localhost" || h == "::1" || h == "0.0.0.0") {
+        return true;
+    }
+
+    if (h.size() > 10 && h.compare(h.size() - 10, 10, ".localhost") == 0) {
+        return true;
+    }
+
+    // IPv4 literal: exactly four all-digit octets with values 0-255, loopback is 127.0.0.0/8
+    int octets[4] = { 0, 0, 0, 0 };
+    size_t pos = 0;
+    int n = 0;
+    bool complete = false;
+    while (n < 4 && pos <= h.size()) {
+        size_t dot = h.find('.', pos);
+        if (dot == std::string::npos) {
+            dot = h.size();
+        }
+        if (dot == pos || dot - pos > 3) {
+            return false;
+        }
+        int val = 0;
+        for (size_t i = pos; i < dot; i++) {
+            if (h[i] < '0' || h[i] > '9') {
+                return false;
+            }
+            val = val * 10 + (h[i] - '0');
+        }
+        if (val > 255) {
+            return false;
+        }
+        octets[n++] = val;
+        if (dot == h.size()) {
+            complete = (n == 4);
+            break;
+        }
+        pos = dot + 1;
+    }
+
+    return complete && octets[0] == 127;
+}
+
+// match a host against a NO_PROXY list: comma separated entries, "*" bypasses the proxy for every host,
+// an entry matches the host itself or any of its subdomains, a leading dot is optional, ports are ignored
+static bool common_http_proxy_bypass(const std::string & no_proxy, const std::string & host) {
+    auto lower = [](std::string s) {
+        for (auto & c : s) {
+            if (c >= 'A' && c <= 'Z') {
+                c += 'a' - 'A';
+            }
+        }
+        return s;
+    };
+
+    const std::string h = lower(host);
+
+    for (size_t pos = 0; pos < no_proxy.size(); ) {
+        size_t end = no_proxy.find(',', pos);
+        if (end == std::string::npos) {
+            end = no_proxy.size();
+        }
+
+        std::string entry = lower(no_proxy.substr(pos, end - pos));
+        pos = end + 1;
+
+        entry.erase(0, entry.find_first_not_of(" \t"));
+        auto last = entry.find_last_not_of(" \t");
+        if (last == std::string::npos) {
+            continue;
+        }
+        entry.erase(last + 1);
+
+        if (entry == "*") {
+            return true;
+        }
+
+        if (entry.front() == '.') {
+            entry.erase(0, 1);
+        }
+
+        if (entry.empty()) {
+            continue;
+        }
+
+        if (h == entry) {
+            return true;
+        }
+
+        if (h.size() > entry.size() && h[h.size() - entry.size() - 1] == '.' &&
+            h.compare(h.size() - entry.size(), entry.size(), entry) == 0) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
 static std::pair<httplib::Client, common_http_url> common_http_client(const std::string & url) {
     common_http_url parts = common_http_parse_url(url);
 
@@ -123,6 +237,33 @@ static std::pair<httplib::Client, common_http_url> common_http_client(const std:
 
     cli.set_follow_location(true);
 
+    // Honor HTTP_PROXY / HTTPS_PROXY environment variables (both lower- and upper-case).
+    auto getenv_s = [](const char * upper, const char * lower) -> std::string {
+        const char * val = std::getenv(upper);
+        if (!val) {
+            val = std::getenv(lower);
+        }
+        return val ? val : "";
+    };
+
+    const std::string proxy_url = (parts.scheme == "https")
+        ? getenv_s("HTTPS_PROXY", "https_proxy")
+        : getenv_s("HTTP_PROXY",  "http_proxy");
+
+    if (!proxy_url.empty() && !common_http_is_loopback(parts.host) &&
+        !common_http_proxy_bypass(getenv_s("NO_PROXY", "no_proxy"), parts.host)) {
+        try {
+            common_http_url proxy = common_http_parse_url(proxy_url);
+            cli.set_proxy(proxy.host, proxy.port);
+            if (!proxy.user.empty()) {
+                cli.set_proxy_basic_auth(proxy.user, proxy.password);
+            }
+        } catch (const std::exception & e) {
+            // fall back to a direct connection, the URL itself is not logged as it can carry credentials
+            LOG_WRN("%s: ignoring malformed proxy URL from the environment: %s\n", __func__, e.what());
+        }
+    }
+
     return { std::move(cli), std::move(parts) };
 }
 

common/ngram-cache.cpp

@@ -68,7 +68,7 @@ static llama_token try_draft(common_ngram_cache & nc_static, const common_ngram
     if (part_static_it == nc_static.end()) {
         return LLAMA_TOKEN_NULL;
     }
-    const common_ngram_cache_part part_static = part_static_it->second;
+    const common_ngram_cache_part & part_static = part_static_it->second;
 
     int max_count_static  = 0;
     int sum_count_static  = 0;
@@ -96,7 +96,7 @@ static llama_token try_draft(common_ngram_cache & nc_static, const common_ngram
 
 // Try to draft a token from primary cache (context/dynamic), validate with static cache:
 static llama_token try_draft(
-    common_ngram_cache & nc_primary, const std::vector<common_ngram> & ngrams_primary, common_ngram_cache_part & part_static,
+    common_ngram_cache & nc_primary, const std::vector<common_ngram> & ngrams_primary, const common_ngram_cache_part & part_static,
     const int * min_sample_size, const int * min_percent) {
 
     llama_token drafted_token = LLAMA_TOKEN_NULL;
@@ -108,7 +108,7 @@ static llama_token try_draft(
         if (part_primary_it == nc_primary.end()) {
             continue;
         }
-        const common_ngram_cache_part part_primary = part_primary_it->second;
+        const common_ngram_cache_part & part_primary = part_primary_it->second;
 
         int max_count_primary = 0;
         int max_count_static  = 0;
@@ -118,7 +118,7 @@ static llama_token try_draft(
         for (std::pair<llama_token, int> token_count_primary : part_primary) {
             const llama_token token = token_count_primary.first;
 
-            common_ngram_cache_part::iterator token_count_static_it = part_static.find(token);
+            common_ngram_cache_part::const_iterator token_count_static_it = part_static.find(token);
 
             const int32_t count_primary = token_count_primary.second;
             const int32_t count_static  = token_count_static_it != part_static.end() ? 100*token_count_static_it->second : 1;
@@ -162,11 +162,9 @@ void common_ngram_cache_draft(
         for (int j = ngram_start_static; j < ngram_start_static + LLAMA_NGRAM_STATIC; ++j) {
             ngram_static.tokens[j-ngram_start_static] = get_token(inp, draft, j);
         }
-        common_ngram_cache::iterator part_static_it = nc_static.find(ngram_static);
-        common_ngram_cache_part part_static;
-        if (part_static_it != nc_static.end()) {
-            part_static = part_static_it->second;
-        }
+        static const common_ngram_cache_part part_static_empty;
+        common_ngram_cache::const_iterator part_static_it = nc_static.find(ngram_static);
+        const common_ngram_cache_part & part_static = part_static_it != nc_static.end() ? part_static_it->second : part_static_empty;
 
         // cd = context + dynamic
         std::vector<common_ngram> ngrams_cd;

conversion/deepseek.py

@@ -243,7 +243,7 @@ class DeepseekV2Model(TextModel):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        hparams: dict = ModelBase.load_hparams(self.dir_model, is_mistral_format=False)
+        hparams: dict = ModelBase.load_hparams(self.dir_model, is_mistral_format=self.is_mistral_format)
         self.origin_hf_arch = hparams.get('architectures', [None])[0]
 
         # special handling for Deepseek OCR

conversion/granite.py

@@ -8,7 +8,7 @@ import torch
 if TYPE_CHECKING:
     from torch import Tensor
 
-from .base import MmprojModel, ModelBase, gguf, logger
+from .base import MmprojModel, ModelBase, gguf, jinja_str_or_json, logger
 
 from .llama import LlamaModel
 from .mamba import Mamba2Model
@@ -284,10 +284,39 @@ class GraniteSwitchModel(GraniteMoeModel):
         self.gguf_writer.add_adapter_lora_rank(self._max_lora_rank)
         self.gguf_writer.add_adapter_token_ids_activate(self.hparams["adapter_token_ids"])
         self.gguf_writer.add_adapter_token_ids_substitute(self.hparams["adapter_substitute_token_ids"])
+        if self._has_decision_adapter():
+            self.gguf_writer.add_decision_type(gguf.DecisionType.GRANITE_SWITCH)
         router_gain = float(self.hparams.get("control_token_gain", 15.0))
         self.gguf_writer.add_adapter_router_gain(router_gain)
         logger.info("gguf: (graniteswitch) num_adapters=%s max_lora_rank=%s n_slots=%s router_gain=%s", self._n_adapters, self._max_lora_rank, self._n_slots, router_gain)
 
+    def _has_decision_adapter(self) -> bool:
+        return "decision" in self.hparams.get("adapter_names", [])
+
+    def set_vocab(self):
+        super().set_vocab()
+        if self._has_decision_adapter():
+            self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])
+
+    def _systemone_template(self) -> str:
+        # prompt of the decision aLoRA: a state turn, then a question turn that opens with the control token
+        # (it replaces the first token of "### DECISION"), the answer is the label token that comes next
+        description = jinja_str_or_json("o.description")
+        option = (
+            "{% if type != 'noul' %}{{ letters[loop.index0] }}: {% if o.description %}" + description + "{% endif %}"
+            "{% elif o.key == 'true' %}A: {% if o.description %}" + description + "{% else %}The statement is true.{% endif %}"
+            "{% else %}B: {% if o.description %}" + description + "{% else %}The statement is false.{% endif %}{% endif %}"
+        )
+        return (
+            "{% set letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' %}"
+            "<|start_of_role|>user<|end_of_role|>State:\n" + jinja_str_or_json("state") + "<|end_of_text|>\n"
+            "<|start_of_role|>user<|end_of_role|><|decision|> DECISION\nQuestion:\n" + jinja_str_or_json("instructions")
+            + "{% if type == 'score' %} Rate along the ordered levels below (lowest first).{% endif %}"
+            "{{ '\n\nOptions:\n' }}"
+            "{% for o in options %}" + option + "{{ '\n' }}{% endfor %}"
+            "{{ '\nReply with exactly one option label.<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>' }}"
+        )
+
     def _lora_a(self, data: Tensor) -> Tensor:
         # on-disk A: [n_adapters, 1, max_rank, in] -> [n_adapters+1, max_rank, in]
         a = data.squeeze(1)

conversion/lfm2.py

@@ -188,7 +188,7 @@ def _load_d1_omni_hparams(dir_model: Path) -> dict[str, Any]:
 class D1OmniModel(LFM2Model):
     model_arch = gguf.MODEL_ARCH.LFM2
 
-    # the server cuts the text to these lengths, see server-decision.cpp
+    # the server cuts the text to these lengths, see engine-decision.cpp
     _MAX_LENGTH = 16384
     _IMAGE_TEXT_LENGTH = 896
     _AUDIO_TEXT_LENGTH = 15360

conversion/mellum.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Iterable, TYPE_CHECKING
+from typing import Callable, Iterable, TYPE_CHECKING
 
 import torch
 
@@ -14,6 +14,54 @@ from .base import ModelBase, TextModel, gguf, logger
 @ModelBase.example("JetBrains/Mellum2-12B-A2.5B-Base")
 class MellumModel(TextModel):
     model_arch = gguf.MODEL_ARCH.MELLUM
+    supports_mtp_export = True
+
+    _n_main_layers: int | None = None
+
+    _experts: list[dict[str, Tensor]] | None = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # number of exported MTP blocks, appended after the trunk layers
+        self.n_nextn = 0 if self.no_mtp else (self.find_hparam(["num_nextn_predict_layers", "num_mtp_layers"], optional=True) or 0)
+        if self.mtp_only and self.n_nextn == 0:
+            raise ValueError("--mtp requested, but the checkpoint has no MTP layers (num_nextn_predict_layers is 0)")
+        if self.n_nextn > 0:
+            if any(t != "sparse" for t in self.hparams.get("mtp_mlp_layer_types") or []):
+                raise ValueError("Mellum MTP layers with dense MLP are not supported")
+            self.block_count += self.n_nextn
+            self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+
+    def index_tensors(self, remote_hf_model_id: str | None = None):
+        type(self)._n_main_layers = self.hparams["num_hidden_layers"]
+        return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
+
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        if (titem := super().filter_tensors(item)) is None:
+            return None
+        name, gen = titem
+
+        if name.startswith("mtp."):
+            if cls.no_mtp:
+                return None
+            assert cls._n_main_layers is not None
+            # mtp.layers.{i}.* -> model.layers.{n_main + i}.*, mtp.{fc,norm,...} -> nextn names of the first MTP block
+            remapper = {
+                "fc":                    "eh_proj",
+                "pre_fc_norm_embedding": "enorm",
+                "pre_fc_norm_hidden":    "hnorm",
+                "norm":                  "shared_head.norm",
+            }
+            parts = name.split(".", 3)
+            if len(parts) == 4 and parts[1] == "layers" and parts[2].isdecimal():
+                name = f"model.layers.{cls._n_main_layers + int(parts[2])}.{parts[3]}"
+            elif len(parts) == 3 and parts[1] in remapper:
+                name = f"model.layers.{cls._n_main_layers}.{remapper[parts[1]]}.{parts[2]}"
+        elif cls.mtp_only and name not in ("model.embed_tokens.weight", "model.norm.weight", "lm_head.weight"):
+            return None
+
+        return name, gen
 
     def set_gguf_parameters(self):
         super().set_gguf_parameters()
@@ -26,13 +74,31 @@ class MellumModel(TextModel):
         if (use_sliding_window is True or use_sliding_window is None) and sliding_window is not None:
             self.gguf_writer.add_sliding_window(sliding_window)
             logger.info(f"gguf: sliding window = {sliding_window}")
-            self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in self.hparams["layer_types"]])
-            logger.info(f"gguf: sliding window pattern length = {len(self.hparams['layer_types'])}")
+            layer_types = list(self.hparams["layer_types"])
+            # MTP blocks default to sliding window attention, as in transformers
+            if self.n_nextn > 0:
+                layer_types += self.hparams.get("mtp_layer_types") or ["sliding_attention"] * self.n_nextn
+            self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in layer_types])
+            logger.info(f"gguf: sliding window pattern length = {len(layer_types)}")
 
         if (norm_topk_prob := self.hparams.get("norm_topk_prob")) is not None:
             self.gguf_writer.add_expert_weights_norm(norm_topk_prob)
 
-    _experts: list[dict[str, Tensor]] | None = None
+        if self.n_nextn > 0:
+            self.gguf_writer.add_nextn_predict_layers(self.n_nextn)
+
+    def prepare_metadata(self, vocab_only: bool):
+        from_dir = self.fname_out.is_dir()
+        super().prepare_metadata(vocab_only=vocab_only)
+
+        if not self.mtp_only or not from_dir:
+            return
+
+        output_type: str = self.ftype.name.partition("_")[2]
+        fname_default: str = gguf.naming_convention(
+            self.metadata.name, self.metadata.basename, self.metadata.finetune,
+            self.metadata.version, size_label=None, output_type=output_type, model_type=None)
+        self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
 
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         if name.find("experts") != -1:

conversion/pplx_decider.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import json
 
 from pathlib import Path
-from typing import Any, Callable, Iterable, TYPE_CHECKING
+from typing import Any, Iterable, TYPE_CHECKING
 
 import torch
 
@@ -35,6 +35,7 @@ def _load_pplx_decider_hparams(dir_model: Path) -> dict[str, Any]:
 
 @ModelBase.register("PplxDeciderModel")
 @ModelBase.example("perplexity-ai/pplx-decider-v1-27b")
+@ModelBase.example("FINAL-Bench/Darwin-27B-ZTC-v5")
 class PplxDeciderModel(Qwen3_5TextModel):
     model_arch = gguf.MODEL_ARCH.QWEN35
     no_mtp = True  # the checkpoint has no MTP head
@@ -77,14 +78,6 @@ class PplxDeciderModel(Qwen3_5TextModel):
         for name in ("choice", "score", "noul"):
             self.gguf_writer.add_decision_temperature(name, self.hparams["decision"]["temperature"])
 
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-        # the checkpoint is the bare backbone, its text tensors have no "model." prefix
-        if name.startswith("language_model."):
-            name = "model." + name
-        return super().filter_tensors((name, gen))
-
     def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
         yield from super().generate_extra_tensors()
         from safetensors.torch import load_file
@@ -92,7 +85,7 @@ class PplxDeciderModel(Qwen3_5TextModel):
         # the readout has one row per option label, store it as an LM head that is zero for the other tokens
         readout = load_file(self.dir_model / "readout.safetensors")["weight"]
         token_ids = self.hparams["decision"]["token_ids"]
-        n_vocab = self.hparams["text_config"]["vocab_size"]
+        n_vocab = self.hparams["vocab_size"]
         assert readout.shape[0] == len(token_ids) == len(set(token_ids))
         lm_head = torch.zeros(n_vocab, readout.shape[1], dtype=readout.dtype)
         lm_head[token_ids] = readout

docs/development/HOWTO-add-model.md

@@ -147,11 +147,11 @@ The conversion is the same as above, but a new model needs its own `DecisionType
 
 > [!IMPORTANT]
 >
-> Most of the logic is handled in `tools/server/server-decision.cpp`, to avoid too many changes to `libllama`.
+> Most of the logic is handled in `tools/engine/engine-decision.cpp`, to avoid too many changes to `libllama`.
 
 Note:
 - If a new public API is needed in `libllama`, add it to `llama-ext.h`.
-- Metadata with a single use case must be hard-coded in `server-decision.cpp` instead of being saved to the GGUF. This avoids bloating the conversion code.
+- Metadata with a single use case must be hard-coded in `engine-decision.cpp` instead of being saved to the GGUF. This avoids bloating the conversion code.
 - Most importantly, keep your change as small and as self-contained as possible. Reuse the existing infrastructure whenever you can.
 
 For more information, see [PR #29818](https://github.com/ggml-org/llama.cpp/pull/29818).

examples/llama-eval/README.md

@@ -24,3 +24,267 @@ python3 llama-eval.py \
   --dataset aime2025 --n_cases 240 \
   --grader-type regex
 ```
+
+### Authentication
+
+A server started with `--api-key` (or any OpenAI-compatible endpoint behind a
+token) needs one here too. `--api-key` is sent as `Authorization: Bearer` on
+every eval request; it falls back to `$LLAMA_API_KEY`, which keeps the secret
+out of your shell history and off the process list.
+
+```bash
+# One key for every server
+python3 llama-eval.py --server http://localhost:8033 --api-key sk-... ...
+
+# From the environment
+export LLAMA_API_KEY=sk-...
+python3 llama-eval.py --server http://localhost:8033 ...
+
+# One key per server, in --server order
+python3 llama-eval.py \
+  --server http://server1:8033,http://server2:8033 \
+  --api-key sk-one,sk-two --threads 16,16 ...
+```
+
+The LLM grader authenticates too. It reuses the key of the server it runs on,
+so `--grader-type llm` without `--grader-server` needs nothing extra; a grader
+on a separate endpoint takes `--grader-api-key`.
+
+Endpoints on an opencode host additionally get an `x-opencode-session` header,
+generated fresh for every task. An agentic episode sends the same id on all of
+its turns, so opencode sees one session per task instead of one per request;
+other servers never see the header.
+
+## Agentic suite
+
+`--dataset agentic` is a different shape from the other code suites. It runs
+**SACB** (Simple Agent Coding Benchmark), hosted at
+[`ilintar/SACB`](https://huggingface.co/datasets/ilintar/SACB) -- 60 tasks over
+three hand-written repositories, in Python and TypeScript. Rather than
+answering one prompt, the model is given a small but real repository, a
+deliberately narrow tool set, and a bug report, and drives its own multi-turn
+conversation until it declares itself finished. It is then graded by running a
+test suite it was never allowed to see.
+
+```bash
+python3 llama-eval.py \
+  --server http://localhost:8033 --model my-model \
+  --dataset agentic --threads 1 --temperature 0
+```
+
+### The tools
+
+| tool | purpose |
+|---|---|
+| `list_files` | enumerate the tree |
+| `read_file` | read, with 1-based line numbers and optional line-range narrowing |
+| `search` | regular-expression search across the repository |
+| `edit_lines` | replace an inclusive line range |
+| `edit_replace` | replace an exact snippet, which must be unique |
+| `write_file` | write a whole file |
+| `lint` | run the language's linter and type checker |
+| `finish` | declare the change complete |
+
+Three constraints are deliberate:
+
+**There is no shell and no way to run the tests.** The only feedback channel is
+`lint`. A syntax or type error can be driven out mechanically; a logic error has
+to be reasoned about. A harness that lets an agent iterate against the test
+suite measures something closer to search than to understanding.
+
+**There are two edit tools**, one line-addressed and one content-addressed.
+Which one a model reaches for, and whether it keeps line numbers straight after
+its own earlier edit, is itself a signal.
+
+**The tests never touch the working tree.** They are held outside it and copied
+into a throwaway copy only after the agent has stopped, so they can be neither
+read nor edited. Every tool path is resolved and confinement-checked against the
+repository root, so a symlink planted inside the tree cannot reach outside it.
+
+### Scoring
+
+A task is **resolved** when every test that was failing now passes *and* nothing
+that was already passing broke. Both sets are derived by running the tests twice
+when the corpus is built -- once against the defective tree and once against the
+reference fix -- never hand-written, so a task whose defect no test exercises is
+rejected rather than shipped.
+
+Because many tasks contain several independent defects, the fraction of target
+tests passed is reported alongside, which separates "fixed two of the three
+faults" from "changed nothing". Process metrics are reported too: turns, tool
+calls, rejected calls, edits, and the peak conversation size.
+
+### Dependencies
+
+Nothing here is installed unless you select this suite, and then only for the
+language you actually run. `--agentic-lang python` provisions ruff and mypy;
+`--agentic-lang typescript` provisions the TypeScript compiler and uses node's
+own test runner. Neither touches the other, and no other suite touches either.
+
+### Useful flags
+
+| flag | meaning |
+|---|---|
+| `--agentic-lang` | restrict to one language, and provision only its toolchain |
+| `--agentic-max-turns` | give up after this many assistant turns (default 50) |
+| `--agentic-max-tokens` | cap generation per turn (default: no cap); bounds a model that falls into a repetition loop, but also cuts reasoning off mid-thought. Truncated turns are counted as `truncated_turns` |
+| `--agentic-context-limit` | abandon an episode once its context exceeds this many tokens |
+| `--agentic-transcripts DIR` | save each episode's full message history to `DIR/<task_id>.json`, plus a `DIR/<task_id>.html` showing every reasoning block, reply, tool call and tool result; the main report links to it |
+
+
+## Code suites (HumanEval, ClassEval)
+
+`--dataset humaneval` runs the 164 OpenAI HumanEval problems; `--dataset
+classeval` runs the 100 ClassEval class-generation tasks. Unlike the other
+suites these are not graded by matching text: the model's reply is parsed for a
+```` ```python ```` block, that code is combined with the task's own unit tests,
+and the result is **executed**.
+
+ClassEval asks for a whole class rather than one function, so it is scored the
+way upstream does - **class level** (every one of the task's tests passes, which
+is what feeds pass@k) and **method level** (what fraction of all 2196 test
+methods passed). The second number separates "generated nothing usable" from
+"got most of the class right", and both are printed at the end of a run.
+
+Give it more room to generate than HumanEval - a class with ten methods does not
+fit in a short budget:
+
+```bash
+python3 llama-eval.py \
+  --server http://localhost:8033 --model my-model \
+  --dataset classeval --threads 8 --n_predict 4096 --temperature 0
+```
+
+```bash
+# pass@1, greedy
+python3 llama-eval.py \
+  --server http://localhost:8033 --model my-model \
+  --dataset humaneval --threads 8 --n_predict 1024 --temperature 0
+
+# pass@10 -- each extra multiple of 164 is another sample per problem
+python3 llama-eval.py \
+  --server http://localhost:8033 --model my-model \
+  --dataset humaneval --n_cases 1640 --temperature 0.8 --top-p 0.95
+```
+
+`--grader-type exec` is the default for code suites, so neither it nor
+`--dataset-source` normally needs to be passed.
+
+### Execution sandbox - no Docker required
+
+Generated code is untrusted, but these suites need far less than a container to
+run safely. **The package set is derived from the dataset's own imports at
+startup**, not hardcoded and not taken from the upstream `requirements.txt` -
+which for ClassEval lists `torch`, `openai` and `transformers` that its *tests*
+never touch. So each suite installs exactly what it needs:
+
+| suite | third-party packages |
+|---|---|
+| HumanEval | **none** - every problem imports only stdlib (`typing, math, random, copy, string, collections, hashlib, re`) |
+| ClassEval | 13: `beautifulsoup4, gensim, lxml, netifaces, nltk, numpy, openpyxl, pandas, pillow, pypdf2, python-docx, reportlab, scipy` |
+
+Two of ClassEval's are *declared* rather than derived, because no import
+statement mentions them: `lxml` is selected by string
+(`BeautifulSoup(html, 'lxml')`, ClassEval_44 - without it that task silently
+drops from 23/23 to 7/23), and `scipy` is a hard gensim requirement. Note also
+that `PyPDF2` must **not** be modernised to `pypdf`: ClassEval_69's test imports
+`PdfFileReader`, a name pypdf removed.
+
+The venv is cached under `~/.cache/llama-eval/` (override with
+`LLAMA_EVAL_CACHE`) and rebuilt only when the package set, interpreter version
+or provisioning steps change; force one with `--rebuild-sandbox`. `uv` is used
+when available, which makes even the ClassEval venv build in seconds.
+
+**Interpreter selection.** A suite can prefer a different Python than the one
+running the tool, and declares it: ClassEval asks for 3.13, because gensim
+publishes no wheel past cp313 and its generated C no longer compiles on 3.14
+(it still uses `PyLongObject.ob_digit`). That interpreter is found automatically,
+including via `uv python`; `--sandbox-python` overrides it.
+
+3.14 is fully supported anyway. ClassEval uses exactly two gensim APIs
+(`utils.decode_htmlentities`, `matutils.unitvec`), both pure Python upstream, so
+on 3.14+ the suite drops gensim from the install set and provisions a small shim
+instead. Both paths score **100/100**.
+
+**Offline provisioning.** A suite that would otherwise fetch data at test time
+declares a provisioning step that runs once at venv build time, while the
+network is still up. ClassEval uses this to pre-seed exactly three nltk corpora
+(`punkt_tab`, `averaged_perceptron_tagger_eng`, `wordnet`, ~33 MB), so the tests
+themselves run with no network at all. Those are the *current* names - the
+dataset asks for `punkt` and `averaged_perceptron_tagger`, which nltk has since
+renamed, and seeding the legacy names too is not harmless: with a network
+available nltk considers them satisfied and re-fetches the modern ones anyway.
+The corpora are written *inside* the venv, not `~/nltk_data`, because `$HOME` is
+a tmpfs under the strongest isolation tier.
+
+Each candidate program runs in a throwaway directory as its own process group,
+under the strongest isolation the host offers:
+
+| tier | requires | isolates |
+|---|---|---|
+| `bwrap` | bubblewrap installed | read-only root, tmpfs `/tmp` and `$HOME`, **no network** |
+| `unshare` | `unshare --map-current-user` | **no network** |
+| `unshare-root` | any `unshare -r` | no network, but **runs as uid 0** - see below |
+| `rlimit` | always available | cpu/memory/file-size/process caps only |
+
+**Candidate code must not run as root**, which is why `unshare -r` is a
+last-resort tier that prints a warning. Root bypasses file permission bits, so a
+test asserting that writing to a `chmod 0444` file *fails* instead sees it
+succeed - ClassEval_50 scores 14/16 under uid 0 and 16/16 under uid 1000, with
+nothing in the output to suggest the sandbox caused it. Install bubblewrap.
+
+All tiers additionally cap CPU time, address space (4 GiB), file size and core
+dumps, and kill the whole process group on timeout (`--exec-timeout`, default
+15 s - raise it for ClassEval, whose tests do real work). Writes to the
+throwaway working directory are allowed, which is all the file-handling tasks in
+ClassEval need.
+
+Note that the weakest tier does not contain filesystem access. If you are
+scoring output from a model you do not trust, check the tier printed at startup.
+
+### Reproducibility
+
+Several tasks draw their test inputs from the *global, unseeded* RNG, so a
+verdict is not reproducible run to run - which adds noise exactly when you are
+comparing two models or two quantisations. `--exec-seed` (default `0`) seeds the
+RNG before each test; pass `--exec-seed -1` for the stock upstream behaviour.
+This does not change what the model is asked to compute.
+
+It matters more than it sounds. HumanEval/38, /50 and /53 are affected, and
+ClassEval_58 (duplicate mine placement) fails about 30% of the time on its own
+*reference* solution - measured here at 21/30 passes unseeded versus **30/30
+seeded**.
+
+### Dataset fidelity
+
+The canonical dataset runs **unmodified on current Python** - all 164 reference
+solutions pass on CPython 3.14 with no patches, so no forked or re-hosted copy
+is needed. `--dataset-source` accepts a local `.jsonl` or a HuggingFace repo id
+if you do want to point at a modified copy.
+
+HumanEval does contain a handful of long-standing defects where the *prompt*
+contradicts the graded test (notably HumanEval/47, /116 and /148). Those
+penalise a model that reads the docstring carefully, but correcting them makes
+the benchmark marginally easier and the resulting scores no longer comparable
+with published HumanEval numbers, so nothing is patched by default.
+
+ClassEval is different: the upstream data has defects that cap the achievable
+score below 100% for reasons unrelated to the model, so `--dataset classeval`
+defaults to a **patched fork**,
+[`ilintar/ClassEval`](https://huggingface.co/datasets/ilintar/ClassEval).
+Six edits over five tasks lift the reference solutions from 96/100 classes and
+2182/2196 test methods to **100/100 and 2196/2196**. Three repair tests nothing
+could pass:
+
+- **ClassEval_17** was a time bomb - a hard-coded 2024 date checked against
+  `datetime.now()`, unpassable since January 2024.
+- **ClassEval_48** asserted `get_hostname('0.0.0.0') == 'LAPTOP-2CS86KUM'`, the
+  dataset author's own machine name.
+- **ClassEval_31** compared floats with `assertEqual`, failing on a 4-ULP numpy
+  difference.
+
+The other three fix only the *reference* solutions (NumPy 2.0 removals in
+ClassEval_51, a 30%-flaky mine generator in ClassEval_58) and cannot affect a
+model's score. No `skeleton` was touched, so what the model is asked to write is
+byte-identical to upstream. Pass `--dataset-source FudanSELab/ClassEval` for the
+unpatched original.

examples/llama-eval/agentic_eval.py

@@ -0,0 +1,918 @@
+#!/usr/bin/env python3
+"""Multi-turn agentic coding eval: tools over a real working tree.
+
+The existing code suites are single-shot -- one prompt, one reply, execute,
+grade. This one is not. The model is given a small but real repository and a
+deliberately narrow tool set, and has to find its way to the defect before it
+can fix it. That shape is what puts a run in the tens-of-thousands-of-tokens
+range without any artificial padding: the context grows because the model
+chose to read things.
+
+Three deliberate constraints:
+
+* No shell, and no way to run the tests. The only feedback channel is `lint`.
+  A syntax or type error can be driven out mechanically; a logic error has to
+  be reasoned about. Suites that let an agent iterate against the test suite
+  measure something closer to search than to understanding.
+* Two edit tools, one line-addressed and one content-addressed. Which one a
+  model reaches for, and whether it keeps line numbers straight after its own
+  earlier edit, is itself a signal.
+* Tests never touch the working tree. They are held outside it and copied in
+  after the agent has stopped, so they can be neither read nor edited.
+
+Everything here is imported lazily by llama-eval.py, and every dependency it
+needs is scoped to the language actually under test, so none of it is
+installed for anyone running the other suites.
+"""
+
+import html
+import json
+import os
+import re
+import shutil
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+MAX_READ_LINES = 400
+MAX_TOOL_CHARS = 12000
+MAX_SEARCH_HITS = 60
+
+
+@dataclass
+class LangSpec:
+    """Everything language-specific: how to lint, how to test, what to install.
+
+    Kept as data so adding a language is a corpus concern rather than a harness
+    change, and so a run only ever provisions the toolchain for the language it
+    is actually evaluating.
+    """
+    name: str
+    exts: Tuple[str, ...]
+    packages: Tuple[str, ...] = ()
+    provision: Tuple[str, ...] = ()
+    env: Dict[str, str] = field(default_factory=dict)
+    address_space_limit: Optional[int] = 4 * 1024 * 1024 * 1024
+
+    def source_files(self, root: Path) -> List[Path]:
+        return sorted(p for p in root.rglob("*")
+                      if p.is_file() and p.suffix in self.exts)
+
+
+PYTHON = LangSpec(
+    name="python",
+    exts=(".py",),
+    packages=("ruff==0.14.2", "mypy==1.18.2"),
+)
+
+TYPESCRIPT = LangSpec(
+    name="typescript",
+    exts=(".ts",),
+    address_space_limit=16 * 1024 * 1024 * 1024,
+    provision=(
+        "import os, subprocess, shutil, sys\n"
+        "npm = shutil.which('npm')\n"
+        "if not npm: sys.exit('npm not found on PATH; needed for the typescript suite')\n"
+        "dest = os.path.join(os.environ['SANDBOX_VENV'], 'node')\n"
+        "os.makedirs(dest, exist_ok=True)\n"
+        "r = subprocess.run([npm, 'install', '--silent', '--no-audit', '--no-fund',\n"
+        "                    '--prefix', dest, 'typescript@5.9.3'],\n"
+        "                   capture_output=True, text=True)\n"
+        "sys.exit(0 if r.returncode == 0 else (r.stderr or r.stdout)[-2000:])\n",
+    ),
+)
+
+LANGS: Dict[str, LangSpec] = {"python": PYTHON, "typescript": TYPESCRIPT}
+
+
+class PathEscape(Exception):
+    """A tool was handed a path outside the repository."""
+
+
+class Workspace:
+    """A private, writable copy of one task's repository.
+
+    Every tool path is resolved through here. Confinement is checked on the
+    fully-resolved path, so a symlink planted inside the tree cannot be used to
+    read or write outside it.
+    """
+
+    def __init__(self, root: Path):
+        self.root = root.resolve()
+        self.escape_attempts = 0
+
+    def resolve(self, rel: str) -> Path:
+        if rel is None or not str(rel).strip():
+            raise PathEscape("no path given")
+        raw = Path(str(rel))
+        base = raw if raw.is_absolute() else (self.root / raw)
+        candidate = base.resolve()
+        if candidate != self.root and self.root not in candidate.parents:
+            self.escape_attempts += 1
+            raise PathEscape(f"path {rel!r} is outside the repository")
+        return candidate
+
+    def rel(self, path: Path) -> str:
+        return str(path.relative_to(self.root))
+
+    def files(self) -> List[Path]:
+        return sorted(p for p in self.root.rglob("*")
+                      if p.is_file() and "__pycache__" not in p.parts)
+
+    def snapshot(self) -> Dict[str, str]:
+        out = {}
+        for p in self.files():
+            try:
+                out[self.rel(p)] = p.read_text()
+            except (UnicodeDecodeError, OSError):
+                pass
+        return out
+
+
+def _clip(text: str, limit: int = MAX_TOOL_CHARS) -> str:
+    if len(text) <= limit:
+        return text
+    return text[:limit] + f"\n... [truncated, {len(text) - limit} more characters]"
+
+
+def _numbered(lines: Sequence[str], start: int) -> str:
+    width = len(str(start + len(lines) - 1))
+    return "\n".join(f"{start + i:>{width}}\t{ln}" for i, ln in enumerate(lines))
+
+
+def tool_schemas() -> List[Dict[str, Any]]:
+    """OpenAI-format function schemas for the whole tool set."""
+    def fn(name, desc, props, required=()):
+        return {"type": "function", "function": {
+            "name": name, "description": desc,
+            "parameters": {"type": "object", "properties": props,
+                           "required": list(required)}}}
+
+    return [
+        fn("list_files", "List files in the repository.",
+           {"path": {"type": "string",
+                     "description": "Directory relative to the repository root. "
+                                    "Omit for the whole tree."}}),
+        fn("read_file",
+           "Read a file. Output is prefixed with 1-based line numbers. Give "
+           "start_line/end_line to read only part of a large file.",
+           {"path": {"type": "string", "description": "File path relative to the repository root."},
+            "start_line": {"type": "integer", "description": "First line to read, 1-based, inclusive."},
+            "end_line": {"type": "integer", "description": "Last line to read, 1-based, inclusive."}},
+           ["path"]),
+        fn("search",
+           "Search the repository with a regular expression. Returns matching "
+           "lines with their file and line number.",
+           {"pattern": {"type": "string", "description": "Python regular expression."},
+            "path": {"type": "string", "description": "Limit the search to this subdirectory."},
+            "glob": {"type": "string", "description": "Limit to files matching this glob, e.g. '*.py'."}},
+           ["pattern"]),
+        fn("edit_lines",
+           "Replace an inclusive 1-based line range with new text. Line numbers "
+           "refer to the file as it is now, so re-read it after any edit that "
+           "changed the number of lines.",
+           {"path": {"type": "string", "description": "File path relative to the repository root."},
+            "start_line": {"type": "integer", "description": "First line to replace, 1-based, inclusive."},
+            "end_line": {"type": "integer", "description": "Last line to replace, 1-based, inclusive."},
+            "new_text": {"type": "string", "description": "Replacement text. Use an empty string to delete the range."}},
+           ["path", "start_line", "end_line", "new_text"]),
+        fn("edit_replace",
+           "Replace an exact snippet of text. The snippet must occur exactly "
+           "once in the file unless replace_all is set.",
+           {"path": {"type": "string", "description": "File path relative to the repository root."},
+            "old_text": {"type": "string", "description": "Exact text to find, including indentation."},
+            "new_text": {"type": "string", "description": "Replacement text."},
+            "replace_all": {"type": "boolean", "description": "Replace every occurrence instead of requiring exactly one."}},
+           ["path", "old_text", "new_text"]),
+        fn("write_file",
+           "Write a whole file, creating or overwriting it. Prefer the edit "
+           "tools for changes to existing files.",
+           {"path": {"type": "string", "description": "File path relative to the repository root."},
+            "content": {"type": "string", "description": "Full new file contents."}},
+           ["path", "content"]),
+        fn("lint",
+           "Run the language's linter and type checker over the repository and "
+           "return the diagnostics. This is the only feedback available; the "
+           "test suite cannot be run.",
+           {"path": {"type": "string", "description": "Limit to this file or directory."}}),
+        fn("finish",
+           "Declare the task complete and end the session. This is the only "
+           "way to end it; call this once the change is made.",
+           {"summary": {"type": "string", "description": "Brief description of the change made."}}),
+    ]
+
+
+class ToolBox:
+    """Executes tool calls against one workspace. Returns plain text results.
+
+    Failures are reported as ordinary text rather than raised, because how a
+    model recovers from a rejected edit is part of what is being measured.
+    """
+
+    def __init__(self, ws: Workspace, linter=None):
+        self.ws = ws
+        self.linter = linter
+        self.calls = 0
+        self.errors = 0
+        self.edits = 0
+        self.finished = False
+        self.finish_summary: Optional[str] = None
+        self.per_tool: Dict[str, int] = {}
+
+    def dispatch(self, name: str, args: Dict[str, Any]) -> str:
+        self.calls += 1
+        self.per_tool[name] = self.per_tool.get(name, 0) + 1
+        handler = getattr(self, f"_t_{name}", None)
+        if handler is None:
+            result = f"Error: unknown tool {name!r}."
+        else:
+            try:
+                result = handler(args)
+            except PathEscape as e:
+                result = f"Error: {e}"
+            except KeyError as e:
+                result = f"Error: missing required argument {e}."
+            except Exception as e:
+                result = f"Error: {type(e).__name__}: {e}"
+        if result.startswith("Error:"):
+            self.errors += 1
+        return _clip(result)
+
+    def _t_list_files(self, a):
+        base = self.ws.resolve(a.get("path") or ".")
+        if not base.exists():
+            return f"Error: {a.get('path')!r} does not exist."
+        paths = [p for p in self.ws.files() if p == base or base in p.parents]
+        if not paths:
+            return "(no files)"
+        return "\n".join(f"{self.ws.rel(p)}\t{p.stat().st_size} bytes" for p in paths)
+
+    def _t_read_file(self, a):
+        path = self.ws.resolve(a["path"])
+        if not path.is_file():
+            return f"Error: {a['path']!r} is not a file."
+        lines = path.read_text().splitlines()
+        start = max(1, int(a.get("start_line") or 1))
+        end = int(a.get("end_line") or len(lines))
+        end = min(end, len(lines))
+        if start > len(lines):
+            return f"Error: {a['path']!r} has only {len(lines)} lines."
+        window = lines[start - 1:end]
+        note = ""
+        if len(window) > MAX_READ_LINES:
+            window = window[:MAX_READ_LINES]
+            note = (f"\n... [showing {MAX_READ_LINES} of {end - start + 1} requested "
+                    f"lines; re-read with start_line/end_line for the rest]")
+        header = f"{a['path']} ({len(lines)} lines total)\n"
+        return header + _numbered(window, start) + note
+
+    def _t_search(self, a):
+        try:
+            rx = re.compile(a["pattern"])
+        except re.error as e:
+            return f"Error: bad regular expression: {e}"
+        base = self.ws.resolve(a.get("path") or ".")
+        glob = a.get("glob")
+        hits, scanned = [], 0
+        for p in self.ws.files():
+            if not (p == base or base in p.parents):
+                continue
+            if glob and not p.match(glob):
+                continue
+            try:
+                text = p.read_text()
+            except (UnicodeDecodeError, OSError):
+                continue
+            scanned += 1
+            for i, line in enumerate(text.splitlines(), 1):
+                if rx.search(line):
+                    hits.append(f"{self.ws.rel(p)}:{i}:{line.strip()[:200]}")
+                    if len(hits) >= MAX_SEARCH_HITS:
+                        return ("\n".join(hits) +
+                                f"\n... [stopped at {MAX_SEARCH_HITS} matches; narrow the pattern]")
+        if not hits:
+            return f"No matches for {a['pattern']!r} in {scanned} file(s)."
+        return "\n".join(hits)
+
+    def _t_edit_lines(self, a):
+        path = self.ws.resolve(a["path"])
+        if not path.is_file():
+            return f"Error: {a['path']!r} is not a file."
+        lines = path.read_text().splitlines()
+        start, end = int(a["start_line"]), int(a["end_line"])
+        if start < 1 or start > len(lines):
+            return (f"Error: start_line {start} out of range; "
+                    f"{a['path']!r} has {len(lines)} lines.")
+        if end < start or end > len(lines):
+            return (f"Error: end_line {end} out of range; must be between "
+                    f"{start} and {len(lines)}.")
+        new = str(a["new_text"]).splitlines()
+        path.write_text("\n".join(lines[:start - 1] + new + lines[end:]) + "\n")
+        self.edits += 1
+        delta = len(new) - (end - start + 1)
+        return (f"Replaced lines {start}-{end} of {a['path']} "
+                f"({end - start + 1} -> {len(new)} lines, {delta:+d}). "
+                f"Later line numbers have shifted." if delta else
+                f"Replaced lines {start}-{end} of {a['path']}.")
+
+    def _t_edit_replace(self, a):
+        path = self.ws.resolve(a["path"])
+        if not path.is_file():
+            return f"Error: {a['path']!r} is not a file."
+        text = path.read_text()
+        old, new = str(a["old_text"]), str(a["new_text"])
+        if not old:
+            return "Error: old_text must not be empty."
+        n = text.count(old)
+        if n == 0:
+            return (f"Error: old_text not found in {a['path']}. It must match "
+                    f"exactly, including indentation.")
+        if n > 1 and not a.get("replace_all"):
+            return (f"Error: old_text occurs {n} times in {a['path']}. Include "
+                    f"more surrounding context to make it unique, or set "
+                    f"replace_all.")
+        path.write_text(text.replace(old, new))
+        self.edits += 1
+        return f"Replaced {n} occurrence(s) in {a['path']}."
+
+    def _t_write_file(self, a):
+        path = self.ws.resolve(a["path"])
+        path.parent.mkdir(parents=True, exist_ok=True)
+        existed = path.is_file()
+        path.write_text(str(a["content"]))
+        self.edits += 1
+        return f"{'Overwrote' if existed else 'Created'} {a['path']}."
+
+    def _t_lint(self, a):
+        if self.linter is None:
+            return "Error: no linter is configured for this task."
+        target = a.get("path")
+        if target:
+            self.ws.resolve(target)
+        return self.linter(target)
+
+    def _t_finish(self, a):
+        self.finished = True
+        self.finish_summary = a.get("summary")
+        return "Task marked complete."
+
+
+PY_TEST_RUNNER = r'''
+import json, os, random, sys, unittest
+sys.path.insert(0, os.getcwd())
+random.seed(0)
+
+class Collect(unittest.TestResult):
+    def __init__(self):
+        super().__init__()
+        self.outcomes = {}
+    def addSuccess(self, t): self.outcomes[t.id()] = "pass"
+    def addFailure(self, t, e): self.outcomes[t.id()] = "fail"
+    def addError(self, t, e): self.outcomes[t.id()] = "error"
+    def addSkip(self, t, r): self.outcomes[t.id()] = "skip"
+    def addExpectedFailure(self, t, e): self.outcomes[t.id()] = "pass"
+    def addUnexpectedSuccess(self, t): self.outcomes[t.id()] = "fail"
+
+def flatten(s):
+    for t in s:
+        if isinstance(t, unittest.TestSuite):
+            yield from flatten(t)
+        else:
+            yield t
+
+out = {"tests": {}}
+try:
+    suite = unittest.TestLoader().discover(os.environ.get("TEST_DIR", "tests"),
+                                           top_level_dir=".")
+    res = Collect()
+    for t in flatten(suite):
+        tid = t.id()
+        try:
+            t.run(res)
+        except Exception:
+            pass
+        res.outcomes.setdefault(tid, "error")
+    out["tests"] = res.outcomes
+except Exception as e:
+    out["error"] = f"{type(e).__name__}: {e}"
+
+print("###RESULT###")
+print(json.dumps(out))
+'''
+
+
+def _node_roots() -> List[Path]:
+    """Directories that must stay visible for node to start under bwrap."""
+    node = shutil.which("node")
+    if not node:
+        return []
+    real = Path(node).resolve()
+    return [Path(node).parent.parent, real.parent.parent]
+
+
+class Runner:
+    """Runs the linter and the hidden tests for one language, in the sandbox."""
+
+    def __init__(self, sandbox, lang: LangSpec, timeout: float = 90.0):
+        self.sandbox = sandbox
+        self.lang = lang
+        self.timeout = timeout
+
+    def _binds(self) -> List[Path]:
+        return _node_roots() if self.lang.name == "typescript" else []
+
+    def _env(self) -> Dict[str, str]:
+        if self.lang.name != "typescript":
+            return {}
+        node = shutil.which("node")
+        return {"PATH": f"{Path(node).parent}:/usr/bin:/bin"} if node else {}
+
+    def lint(self, workdir: Path, target: Optional[str] = None) -> str:
+        if self.lang.name == "python":
+            return self._lint_python(workdir, target)
+        return self._lint_typescript(workdir, target)
+
+    def _lint_python(self, workdir: Path, target: Optional[str]) -> str:
+        venv = self.sandbox.venv_dir
+        tgt = target or "."
+        chunks = []
+        r = self.sandbox.run_argv(
+            [str(venv / "bin" / "ruff"), "check", "--no-cache",
+             "--output-format", "concise", tgt],
+            workdir, timeout=self.timeout)
+        text = (r.stdout or r.stderr).strip()
+        if text and "All checks passed" not in text:
+            chunks.append(text)
+        r = self.sandbox.run_argv(
+            [str(venv / "bin" / "mypy"), "--no-color-output", "--no-error-summary",
+             "--cache-dir", "/dev/null", "--ignore-missing-imports", tgt],
+            workdir, timeout=self.timeout)
+        text = (r.stdout or r.stderr).strip()
+        if text and not text.startswith("Success:"):
+            chunks.append(text)
+        return "\n".join(chunks) if chunks else "No problems found."
+
+    def _lint_typescript(self, workdir: Path, target: Optional[str]) -> str:
+        tsc = self.sandbox.venv_dir / "node" / "node_modules" / "typescript" / "bin" / "tsc"
+        files = [str(p.relative_to(workdir)) for p in sorted(workdir.rglob("*.ts"))
+                 if "node_modules" not in p.parts]
+        if target:
+            t = (workdir / target).resolve()
+            files = [f for f in files if (workdir / f).resolve() == t
+                     or t in (workdir / f).resolve().parents]
+        if not files:
+            return "No TypeScript files to check."
+        r = self.sandbox.run_argv(
+            ["node", str(tsc), "--noEmit", "--strict", "--target", "es2022",
+             "--module", "esnext", "--moduleResolution", "bundler",
+             "--allowImportingTsExtensions", *files],
+            workdir, timeout=self.timeout, env=self._env(), ro_binds=self._binds())
+        out = (r.stdout or "") + (r.stderr or "")
+        return out.strip() or "No problems found."
+
+    def test(self, workdir: Path) -> Dict[str, Any]:
+        if self.lang.name == "python":
+            return self._test_python(workdir)
+        return self._test_typescript(workdir)
+
+    def _test_python(self, workdir: Path) -> Dict[str, Any]:
+        runner = workdir / "__run_tests.py"
+        runner.write_text(PY_TEST_RUNNER)
+        r = self.sandbox.run_argv(
+            [str(self.sandbox.python), "-I", "-B", str(runner)],
+            workdir, timeout=self.timeout, env={"TEST_DIR": "tests"})
+        return self._parse_json_result(r)
+
+    def _parse_json_result(self, r) -> Dict[str, Any]:
+        if r.status == "timeout":
+            return {"tests": {}, "error": "timeout"}
+        marker = "###RESULT###"
+        if marker not in (r.stdout or ""):
+            return {"tests": {},
+                    "error": f"no result: {(r.stderr or r.stdout or '')[-400:]}"}
+        try:
+            return json.loads(r.stdout.split(marker, 1)[1].strip())
+        except Exception as e:
+            return {"tests": {}, "error": f"unparseable result: {e}"}
+
+    def _test_typescript(self, workdir: Path) -> Dict[str, Any]:
+        tests = sorted(p for p in (workdir / "tests").rglob("*.ts")) \
+            if (workdir / "tests").is_dir() else []
+        if not tests:
+            return {"tests": {}, "error": "no test files"}
+        r = self.sandbox.run_argv(
+            ["node", "--test", "--experimental-strip-types",
+             "--test-reporter=tap",
+             *[str(p.relative_to(workdir)) for p in tests]],
+            workdir, timeout=self.timeout, env=self._env(), ro_binds=self._binds())
+        return self._parse_tap(r)
+
+    @staticmethod
+    def _parse_tap(r) -> Dict[str, Any]:
+        if r.status == "timeout":
+            return {"tests": {}, "error": "timeout"}
+        text = (r.stdout or "") + "\n" + (r.stderr or "")
+        outcomes: Dict[str, str] = {}
+        for line in text.splitlines():
+            m = re.match(r"\s*(not )?ok\s+\d+\s*-\s*(.+?)\s*$", line)
+            if not m:
+                continue
+            name = m.group(2).strip()
+            if name.endswith(" # SKIP") or " # SKIP" in name:
+                outcomes[name.split(" # ")[0]] = "skip"
+                continue
+            verdict = "fail" if m.group(1) else "pass"
+            if outcomes.get(name) != "fail":
+                outcomes[name] = verdict
+        if not outcomes:
+            return {"tests": {},
+                    "error": f"no TAP output: {text.strip()[-400:]}"}
+        return {"tests": outcomes}
+
+
+SYSTEM_PROMPT = """You are an expert {lang} engineer working in an existing repository.
+
+Resolve the user's request by editing files with the tools provided.
+
+How this session works:
+- You are working autonomously. There is no human watching; nobody will answer
+  questions, approve a plan, or read progress reports.
+- Every reply you send must contain at least one tool call until the task is
+  done. A reply that contains only text does nothing: no work happens and the
+  turn is wasted. Do not stop to describe what you are about to do; do it.
+- The session ends only when you call `finish`. Call it once the change is
+  made, and not before. Only the files as you leave them are evaluated.
+
+Important constraints:
+- You cannot run the code, the tests, or any shell command. The only feedback
+  available is the `lint` tool, which reports syntax and type problems.
+- The repository already has a test suite that will be run against your work
+  after you finish, but you cannot see it or run it. Do not create test files.
+- Read enough of the code to understand it before editing. A change that only
+  makes the symptom go away will usually fail the tests.
+- Make the smallest correct change. Do not reformat or refactor unrelated code.
+
+Keep calling tools until the change is complete, then call `finish`."""
+
+
+@dataclass
+class AgenticTask:
+    task_id: str
+    lang: str
+    category: str
+    difficulty: int
+    instruction: str
+    files: Dict[str, str]
+    tests: Dict[str, str]
+    gold: Dict[str, str] = field(default_factory=dict)
+    fail_to_pass: List[str] = field(default_factory=list)
+    pass_to_pass: List[str] = field(default_factory=list)
+
+    @classmethod
+    def from_record(cls, rec: Dict[str, Any]) -> "AgenticTask":
+        def as_map(v):
+            return v if isinstance(v, dict) else json.loads(v or "{}")
+        def as_list(v):
+            return v if isinstance(v, list) else json.loads(v or "[]")
+        return cls(
+            task_id=rec["task_id"], lang=rec["lang"], category=rec["category"],
+            difficulty=int(rec.get("difficulty", 3)),
+            instruction=rec["instruction"],
+            files=as_map(rec["files"]), tests=as_map(rec["tests"]),
+            gold=as_map(rec.get("gold")),
+            fail_to_pass=as_list(rec.get("fail_to_pass")),
+            pass_to_pass=as_list(rec.get("pass_to_pass")),
+        )
+
+    @property
+    def spec(self) -> LangSpec:
+        return LANGS[self.lang]
+
+
+def materialise(files: Dict[str, str], dest: Path) -> Path:
+    """Write a file map out as a real directory tree."""
+    dest.mkdir(parents=True, exist_ok=True)
+    for rel, content in files.items():
+        p = dest / rel
+        p.parent.mkdir(parents=True, exist_ok=True)
+        p.write_text(content)
+    return dest
+
+
+def repo_overview(files: Dict[str, str]) -> str:
+    """A compact tree, so a run is not lost merely to not finding the files."""
+    lines = []
+    for rel in sorted(files):
+        n = files[rel].count("\n") + 1
+        lines.append(f"  {rel} ({n} lines)")
+    return "\n".join(lines)
+
+
+def user_prompt(task: AgenticTask) -> str:
+    return (f"Repository files:\n{repo_overview(task.files)}\n\n"
+            f"Task:\n{task.instruction}")
+
+
+@dataclass
+class EpisodeResult:
+    stop_reason: str = "unknown"
+    turns: int = 0
+    tool_calls: int = 0
+    tool_errors: int = 0
+    edits: int = 0
+    escape_attempts: int = 0
+    finished: bool = False
+    bad_tool_json: int = 0
+    nudges: int = 0
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    peak_context: int = 0
+    truncated_turns: int = 0
+    per_tool: Dict[str, int] = field(default_factory=dict)
+    error: Optional[str] = None
+
+
+NUDGE = ("Your last reply contained no tool call, so nothing happened. There is "
+         "no human in this session to respond to text. If your change is "
+         "complete, call `finish`. Otherwise continue working by calling the "
+         "tools; do not reply with text alone.")
+
+
+def run_episode(task: AgenticTask, toolbox: ToolBox, chat,
+                max_turns: int = 50,
+                context_limit: Optional[int] = None,
+                max_nudges: int = 3) -> Tuple[EpisodeResult, List[Dict]]:
+    """Drive one multi-turn tool-calling episode.
+
+    `chat(messages, tools) -> response dict` is supplied by the caller so this
+    module stays free of any HTTP concerns.
+    """
+    tools = tool_schemas()
+    messages: List[Dict[str, Any]] = [
+        {"role": "system", "content": SYSTEM_PROMPT.format(lang=task.spec.name)},
+        {"role": "user", "content": user_prompt(task)},
+    ]
+    res = EpisodeResult()
+
+    for turn in range(max_turns):
+        res.turns = turn + 1
+        try:
+            reply = chat(messages, tools)
+        except Exception as e:
+            res.stop_reason, res.error = "request_failed", f"{type(e).__name__}: {e}"
+            break
+
+        usage = reply.get("usage") or {}
+        pt = int(usage.get("prompt_tokens") or 0)
+        ct = int(usage.get("completion_tokens") or 0)
+        res.prompt_tokens += pt
+        res.completion_tokens += ct
+        res.peak_context = max(res.peak_context, pt + ct)
+
+        choices = reply.get("choices") or []
+        if not choices:
+            res.stop_reason, res.error = "no_choices", json.dumps(reply)[:300]
+            break
+        msg = choices[0].get("message") or {}
+        if choices[0].get("finish_reason") == "length":
+            res.truncated_turns += 1
+        calls = msg.get("tool_calls") or []
+
+        reasoning = msg.get("reasoning_content")
+        messages.append({
+            "role": "assistant",
+            "content": msg.get("content") or "",
+            **({"reasoning_content": reasoning} if reasoning else {}),
+            **({"tool_calls": calls} if calls else {}),
+        })
+
+        if not calls:
+            if toolbox.finished:
+                res.stop_reason = "finished"
+                break
+            if res.nudges >= max_nudges:
+                res.stop_reason = "no_tool_calls"
+                break
+            res.nudges += 1
+            messages.append({"role": "user", "content": NUDGE})
+            continue
+
+        for call in calls:
+            fn = call.get("function") or {}
+            name = fn.get("name") or ""
+            raw = fn.get("arguments")
+            if isinstance(raw, dict):
+                args = raw
+            else:
+                try:
+                    args = json.loads(raw or "{}")
+                except (json.JSONDecodeError, TypeError):
+                    res.bad_tool_json += 1
+                    args = None
+            output = (toolbox.dispatch(name, args) if args is not None else
+                      f"Error: arguments for {name!r} were not valid JSON.")
+            messages.append({"role": "tool",
+                             "tool_call_id": call.get("id") or f"call_{res.tool_calls}",
+                             "content": output})
+
+        if toolbox.finished:
+            res.stop_reason = "finished"
+            break
+        if context_limit and res.peak_context > context_limit:
+            res.stop_reason = "context_limit"
+            break
+    else:
+        res.stop_reason = "max_turns"
+
+    res.tool_calls = toolbox.calls
+    res.tool_errors = toolbox.errors
+    res.edits = toolbox.edits
+    res.escape_attempts = toolbox.ws.escape_attempts
+    res.finished = toolbox.finished
+    res.per_tool = dict(toolbox.per_tool)
+    return res, messages
+
+
+def run_tests_against(files: Dict[str, str], task: AgenticTask, runner: Runner,
+                      workroot: Path, label: str) -> Dict[str, str]:
+    """Materialise a tree, drop the hidden tests in, and run them.
+
+    The tests are only ever written into this throwaway copy, never into the
+    tree the agent worked in, so there is no window in which they could have
+    been read or edited.
+    """
+    dest = workroot / f"grade-{label}"
+    shutil.rmtree(dest, ignore_errors=True)
+    materialise(files, dest)
+    materialise(task.tests, dest)
+    return runner.test(dest).get("tests", {})
+
+
+def score(task: AgenticTask, outcomes: Dict[str, str]) -> Dict[str, Any]:
+    """Resolved means every target test passes and nothing previously passing broke."""
+    def passing(tid):
+        return outcomes.get(tid) == "pass"
+
+    f2p_ok = [t for t in task.fail_to_pass if passing(t)]
+    p2p_ok = [t for t in task.pass_to_pass if passing(t)]
+    regressions = [t for t in task.pass_to_pass if not passing(t)]
+    resolved = (len(f2p_ok) == len(task.fail_to_pass)
+                and not regressions and bool(task.fail_to_pass))
+    return {
+        "resolved": resolved,
+        "f2p_passed": len(f2p_ok), "f2p_total": len(task.fail_to_pass),
+        "p2p_passed": len(p2p_ok), "p2p_total": len(task.pass_to_pass),
+        "regressions": regressions[:10],
+        "n_regressions": len(regressions),
+        "partial": (len(f2p_ok) / len(task.fail_to_pass)) if task.fail_to_pass else 0.0,
+    }
+
+
+def make_sandbox(lang_name: str, sandbox_cls, cache_root=None, base_python=None):
+    """A sandbox carrying only the toolchain for `lang_name`.
+
+    Keyed per language rather than per suite: running the Python tasks must
+    never provision node, and vice versa.
+    """
+    lang = LANGS[lang_name]
+    return sandbox_cls(
+        name=f"agentic-{lang_name}",
+        packages=list(lang.packages),
+        provision=list(lang.provision),
+        env=dict(lang.env),
+        address_space_limit=lang.address_space_limit,
+        cache_root=cache_root,
+        base_python=base_python,
+    )
+
+
+def run_task(task: AgenticTask, chat, sandbox, workroot: Optional[Path] = None,
+             max_turns: int = 50, context_limit: Optional[int] = None,
+             test_timeout: float = 90.0) -> Dict[str, Any]:
+    """One full attempt: fresh workspace, agent episode, then hidden tests."""
+    root = Path(tempfile.mkdtemp(prefix=f"agentic-{task.task_id}-",
+                                 dir=str(workroot) if workroot else None))
+    try:
+        ws_root = root / "repo"
+        materialise(task.files, ws_root)
+        ws = Workspace(ws_root)
+        runner = Runner(sandbox, task.spec, timeout=test_timeout)
+        toolbox = ToolBox(ws, linter=lambda t: runner.lint(ws_root, t))
+
+        episode, messages = run_episode(task, toolbox, chat,
+                                        max_turns=max_turns,
+                                        context_limit=context_limit)
+        final = ws.snapshot()
+        changed = sorted(k for k in final
+                         if final[k] != task.files.get(k)) + \
+                  sorted(k for k in task.files if k not in final)
+
+        outcomes = run_tests_against(final, task, runner, root, "final")
+        result = score(task, outcomes)
+        result.update({
+            "task_id": task.task_id, "lang": task.lang,
+            "category": task.category, "difficulty": task.difficulty,
+            "changed_files": changed,
+            "episode": {k: v for k, v in vars(episode).items()},
+            "messages": messages,
+        })
+        return result
+    finally:
+        shutil.rmtree(root, ignore_errors=True)
+
+
+TRANSCRIPT_CSS = """
+body { font-family: system-ui, sans-serif; margin: 0; padding: 16px; background: #fff; color: #222; max-width: 1100px; }
+h1 { font-size: 16px; margin: 0 0 4px; }
+.meta { font-size: 13px; color: #555; font-family: 'SF Mono', 'Menlo', 'Consolas', monospace; margin-bottom: 12px; white-space: pre-wrap; }
+.msg { border: 1px solid #e1e4e8; border-left-width: 4px; margin: 8px 0; padding: 6px 10px; background: #fafbfc; }
+.msg .role { font-size: 12px; font-weight: 600; color: #555; margin-bottom: 4px; }
+.system { border-left-color: #888; }
+.user { border-left-color: #0969da; }
+.nudge { border-left-color: #9a6700; }
+.assistant { border-left-color: #1a7f37; background: #fff; }
+.tool { border-left-color: #8250df; }
+.tool.err { border-left-color: #cf222e; }
+pre { font-family: 'SF Mono', 'Menlo', 'Consolas', monospace; font-size: 12px; background: #fff; border: 1px solid #e1e4e8; padding: 6px 8px; margin: 4px 0; white-space: pre-wrap; word-wrap: break-word; }
+.reasoning pre { color: #57606a; background: #f6f8fa; }
+.call { font-size: 12px; font-weight: 600; color: #8250df; margin-top: 6px; }
+details > summary { cursor: pointer; font-size: 12px; color: #555; }
+.empty { font-size: 12px; color: #888; font-style: italic; }
+"""
+
+
+def _pre(text: str) -> str:
+    return f"<pre>{html.escape(text)}</pre>"
+
+
+def _render_call(call: Dict[str, Any]) -> str:
+    fn = call.get("function") or {}
+    raw = fn.get("arguments")
+    try:
+        args = raw if isinstance(raw, dict) else json.loads(raw or "{}")
+        shown = json.dumps(args, indent=2, ensure_ascii=False)
+    except (json.JSONDecodeError, TypeError):
+        shown = f"(invalid JSON)\n{raw}"
+    return (f'<div class="call">&rarr; {html.escape(fn.get("name") or "?")} '
+            f'<span style="font-weight:400;color:#888">{html.escape(call.get("id") or "")}</span></div>'
+            + _pre(shown))
+
+
+def render_transcript_html(messages: List[Dict[str, Any]], title: str,
+                           meta: str = "") -> str:
+    """One page per episode: every turn, with reasoning, tool calls and results."""
+    names = {}
+    for m in messages:
+        for c in m.get("tool_calls") or []:
+            names[c.get("id")] = (c.get("function") or {}).get("name") or "?"
+
+    parts = []
+    turn = 0
+    for m in messages:
+        role = m.get("role") or "?"
+        content = m.get("content") or ""
+        if role == "assistant":
+            turn += 1
+            body = ""
+            if m.get("reasoning_content"):
+                body += ('<details class="reasoning" open><summary>reasoning '
+                         f'({len(m["reasoning_content"])} chars)</summary>'
+                         + _pre(m["reasoning_content"]) + "</details>")
+            body += _pre(content) if content.strip() else ""
+            body += "".join(_render_call(c) for c in m.get("tool_calls") or [])
+            if not body:
+                body = '<div class="empty">(empty reply)</div>'
+            parts.append(f'<div class="msg assistant"><div class="role">assistant '
+                         f'&middot; turn {turn}</div>{body}</div>')
+        elif role == "tool":
+            name = names.get(m.get("tool_call_id"), "?")
+            err = " err" if content.startswith("Error") else ""
+            parts.append(f'<div class="msg tool{err}"><div class="role">tool result '
+                         f'&middot; {html.escape(name)}</div>{_pre(content)}</div>')
+        elif role == "system":
+            parts.append('<div class="msg system"><details><summary>system prompt'
+                         f'</summary>{_pre(content)}</details></div>')
+        else:
+            cls = "nudge" if content == NUDGE else role
+            label = "user (nudge)" if content == NUDGE else role
+            parts.append(f'<div class="msg {html.escape(cls)}"><div class="role">'
+                         f'{html.escape(label)}</div>{_pre(content)}</div>')
+
+    return (f'<!DOCTYPE html>\n<html>\n<head>\n<meta charset="UTF-8">\n'
+            f'<title>{html.escape(title)}</title>\n<style>{TRANSCRIPT_CSS}</style>\n'
+            f'</head>\n<body>\n<h1>{html.escape(title)}</h1>\n'
+            f'<div class="meta">{html.escape(meta)}</div>\n'
+            + "\n".join(parts) + "\n</body>\n</html>\n")
+
+
+def write_transcript(messages: List[Dict[str, Any]], dest: Path, title: str,
+                     meta: str = "") -> Path:
+    """Write `<dest>.json` (the raw message history) and `<dest>.html`."""
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    json_path = dest.with_name(dest.name + ".json")
+    with open(json_path, "w") as f:
+        json.dump(messages, f, indent=2, ensure_ascii=False)
+    with open(dest.with_name(dest.name + ".html"), "w") as f:
+        f.write(render_transcript_html(messages, title, meta))
+    return json_path

examples/llama-eval/eval_sandbox.py

@@ -0,0 +1,488 @@
+#!/usr/bin/env python3
+"""Lightweight sandboxed execution for code-generation evals (HumanEval, ClassEval).
+
+Replaces the usual Docker-per-suite setup with a cached venv plus whatever
+process isolation the host can give us without root:
+
+    tier 1  bubblewrap   read-only root, tmpfs /tmp and $HOME, no network
+    tier 2  unshare -rn  no network (home still visible)
+    tier 3  rlimits only last resort, still capped on cpu/memory/procs/files
+
+Every tier also applies rlimits and runs the candidate program in a throwaway
+working directory as its own process group, so a runaway solution is killed
+along with anything it spawned.
+"""
+
+import json
+import os
+import re
+import resource
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import venv
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List, Optional, Sequence
+
+DEFAULT_CACHE_ROOT = Path(
+    os.environ.get("LLAMA_EVAL_CACHE", Path.home() / ".cache" / "llama-eval")
+)
+
+LIMIT_AS_BYTES = 4 * 1024 * 1024 * 1024
+LIMIT_CPU_SECONDS_SLACK = 5
+LIMIT_NPROC = 256
+LIMIT_FSIZE_BYTES = 64 * 1024 * 1024
+
+
+@dataclass
+class ExecResult:
+    passed: bool
+    status: str
+    stdout: str = ""
+    stderr: str = ""
+    returncode: Optional[int] = None
+    duration: float = 0.0
+
+    def summary(self, max_chars: int = 400) -> str:
+        """Short human-readable reason, for grader logs."""
+        if self.passed:
+            return "ok"
+        if self.status == "timeout":
+            return "timeout"
+        tail = (self.stderr or self.stdout).strip()
+        if not tail:
+            return f"{self.status} (rc={self.returncode})"
+        lines = [ln for ln in tail.splitlines() if ln.strip()]
+        reason = lines[-1] if lines else tail
+        return reason[:max_chars]
+
+
+def resolve_python(spec: Optional[str]) -> Optional[str]:
+    """Turn '3.13' or a path into a usable interpreter, or None if unavailable.
+
+    Suites can need an older interpreter than the one running this tool, because
+    a dependency has no wheel (or no working source build) for the newest
+    CPython. Accepts an explicit path, a bare version, or an executable name.
+    """
+    if not spec:
+        return None
+
+    candidate = Path(spec).expanduser()
+    if candidate.exists() and candidate.is_file():
+        return str(candidate)
+
+    uv = shutil.which("uv")
+    if uv and re.fullmatch(r"\d+\.\d+(\.\d+)?", spec):
+        try:
+            r = subprocess.run([uv, "python", "find", spec],
+                               capture_output=True, text=True, timeout=120)
+            path = r.stdout.strip()
+            if r.returncode == 0 and path and Path(path).exists():
+                return path
+        except Exception:
+            pass
+
+    return shutil.which(spec) or shutil.which(f"python{spec}")
+
+
+def _detect_isolation(probe_python: Optional[str] = None,
+                      ro_binds: Sequence[Path] = ()) -> str:
+    """Pick the strongest isolation tier this host supports, once.
+
+    The probe must run the *same* interpreter the suite will, with the same
+    binds. Probing a bare sys.executable silently downgrades the tier whenever
+    this tool is itself run from a venv under $HOME -- which is the usual case,
+    since it needs `datasets`. bwrap tmpfs-mounts the home directory, the
+    interpreter disappears, the probe fails, and the sandbox falls back to a
+    tier that does not hide $HOME at all. Nothing about the results looks wrong
+    when that happens, so it has to be got right here.
+    """
+    py = probe_python or sys.executable
+    probe = "import sys; sys.exit(0)"
+
+    if shutil.which("bwrap"):
+        try:
+            r = subprocess.run(
+                _bwrap_argv(Path.cwd(), [py, "-I", "-c", probe], ro_binds=ro_binds),
+                capture_output=True, timeout=20,
+            )
+            if r.returncode == 0:
+                return "bwrap"
+        except Exception:
+            pass
+
+    if shutil.which("unshare"):
+        for tier, argv in (
+            ("unshare", ["unshare", "--user", "--map-current-user", "--net"]),
+            ("unshare-root", ["unshare", "-rn"]),
+        ):
+            try:
+                r = subprocess.run(
+                    [*argv, py, "-I", "-c", probe],
+                    capture_output=True, timeout=20,
+                )
+                if r.returncode == 0:
+                    return tier
+            except Exception:
+                continue
+
+    return "rlimit"
+
+
+UNSHARE_ARGV = {
+    "unshare": ["unshare", "--user", "--map-current-user", "--net"],
+    "unshare-root": ["unshare", "-rn"],
+}
+
+
+def _under(path: Path, parent: str) -> bool:
+    return bool(parent) and (str(path) == parent
+                             or str(path).startswith(parent.rstrip("/") + os.sep))
+
+
+def _bwrap_argv(workdir: Path, inner: Sequence[str],
+                ro_binds: Sequence[Path] = ()) -> List[str]:
+    home = os.path.expanduser("~")
+    argv = [
+        "bwrap",
+        "--ro-bind", "/", "/",
+        "--dev", "/dev",
+        "--proc", "/proc",
+        "--tmpfs", "/tmp",
+    ]
+    if home and home != "/":
+        argv += ["--tmpfs", home]
+    for path in ro_binds:
+        if _under(path, home):
+            argv += ["--ro-bind", str(path), str(path)]
+    argv += [
+        "--bind", str(workdir), str(workdir),
+        "--chdir", str(workdir),
+        "--unshare-all",
+        "--die-with-parent",
+        "--new-session",
+        "--",
+    ]
+    return argv + list(inner)
+
+
+def _nproc_ceiling() -> int:
+    """A process cap that bounds a fork bomb without tripping on existing load.
+
+    RLIMIT_NPROC is enforced per real UID across the whole system, not per
+    process tree, so it has to sit above whatever the user is already running.
+    """
+    try:
+        current = len(os.listdir("/proc"))
+    except OSError:
+        current = 512
+    return max(current * 2, 1024)
+
+
+def _apply_rlimits(timeout: float, with_nproc: bool,
+                   as_bytes: Optional[int] = LIMIT_AS_BYTES):
+    def _preexec():
+        os.setsid()
+        cpu = int(timeout) + LIMIT_CPU_SECONDS_SLACK
+        limits = [
+            (resource.RLIMIT_CPU, (cpu, cpu)),
+            (resource.RLIMIT_FSIZE, (LIMIT_FSIZE_BYTES, LIMIT_FSIZE_BYTES)),
+            (resource.RLIMIT_CORE, (0, 0)),
+        ]
+        if as_bytes:
+            limits.append((resource.RLIMIT_AS, (as_bytes, as_bytes)))
+        if with_nproc:
+            ceiling = _nproc_ceiling()
+            limits.append((resource.RLIMIT_NPROC, (ceiling, ceiling)))
+        for res, lim in limits:
+            try:
+                resource.setrlimit(res, lim)
+            except (ValueError, OSError):
+                pass
+    return _preexec
+
+
+class Sandbox:
+    """A cached venv plus an isolated runner for untrusted candidate programs."""
+
+    def __init__(
+        self,
+        name: str,
+        packages: Optional[Sequence[str]] = None,
+        cache_root: Optional[Path] = None,
+        base_python: Optional[str] = None,
+        isolation: Optional[str] = None,
+        provision: Optional[Sequence[str]] = None,
+        env: Optional[Dict[str, str]] = None,
+        address_space_limit: Optional[int] = LIMIT_AS_BYTES,
+    ):
+        self.name = name
+        self.packages = sorted(set(packages or []))
+        self.provision = list(provision or [])
+        self.env = dict(env or {})
+        # None drops the RLIMIT_AS cap entirely -- see _apply_rlimits
+        self.address_space_limit = address_space_limit
+        self.cache_root = Path(cache_root or DEFAULT_CACHE_ROOT)
+        self.base_python = base_python or sys.executable
+        self.venv_dir = self.cache_root / f"{name}-venv"
+        self._python: Optional[Path] = None
+        self._isolation = isolation
+        self._base_prefix: List[Path] = []
+
+    @property
+    def python(self) -> Path:
+        if self._python is None:
+            raise RuntimeError("Sandbox.ensure() must be called first")
+        return self._python
+
+    @property
+    def isolation(self) -> str:
+        if self._isolation is None:
+            if self._python:
+                py = str(self._python)
+                roots = [self.venv_dir, *(self._base_prefix or [])]
+            else:
+                py = sys.executable
+                roots = self._detect_base_prefix(Path(py))
+            self._isolation = _detect_isolation(py, roots)
+        return self._isolation
+
+    def _stamp_path(self) -> Path:
+        return self.venv_dir / ".llama-eval-stamp.json"
+
+    def _stamp_matches(self) -> bool:
+        try:
+            stamp = json.loads(self._stamp_path().read_text())
+        except Exception:
+            return False
+        return (
+            stamp.get("packages") == self.packages
+            and stamp.get("py") == self._base_version()
+            and stamp.get("provision") == self.provision
+        )
+
+    def _resolved_env(self) -> Dict[str, str]:
+        return {k: v.replace("{venv}", str(self.venv_dir)) for k, v in self.env.items()}
+
+    def _base_version(self) -> str:
+        r = subprocess.run(
+            [self.base_python, "-c",
+             "import sys;print('.'.join(map(str,sys.version_info[:3])))"],
+            capture_output=True, text=True,
+        )
+        return r.stdout.strip()
+
+    def ensure(self, force: bool = False, quiet: bool = False) -> Path:
+        """Create/refresh the venv. Idempotent and cheap when already valid."""
+        py = self.venv_dir / "bin" / "python"
+        if force and self.venv_dir.exists():
+            shutil.rmtree(self.venv_dir)
+
+        if py.exists() and self._stamp_matches():
+            self._python = py
+            self._base_prefix = self._detect_base_prefix(py)
+            return py
+
+        if not quiet:
+            pkgs = ", ".join(self.packages) if self.packages else "stdlib only"
+            print(f"[sandbox] provisioning venv {self.venv_dir} ({pkgs})")
+
+        self.cache_root.mkdir(parents=True, exist_ok=True)
+        if self.venv_dir.exists():
+            shutil.rmtree(self.venv_dir)
+
+        uv = shutil.which("uv")
+        if uv:
+            subprocess.run(
+                [uv, "venv", "--python", self.base_python, str(self.venv_dir)],
+                check=True, capture_output=True,
+            )
+        elif Path(self.base_python).resolve() == Path(sys.executable).resolve():
+            venv.EnvBuilder(with_pip=bool(self.packages), clear=True).create(self.venv_dir)
+        else:
+            subprocess.run(
+                [self.base_python, "-m", "venv", str(self.venv_dir)],
+                check=True, capture_output=True,
+            )
+
+        if self.packages:
+            if uv:
+                cmd = [uv, "pip", "install", "--python", str(py), *self.packages]
+            else:
+                cmd = [str(py), "-m", "pip", "install", "--quiet",
+                       "--disable-pip-version-check", *self.packages]
+            r = subprocess.run(cmd, capture_output=True, text=True)
+            if r.returncode != 0:
+                raise RuntimeError(
+                    f"failed to install {len(self.packages)} package(s) into "
+                    f"{self.venv_dir}:\n{(r.stderr or r.stdout)[-3000:]}"
+                )
+
+        for i, snippet in enumerate(self.provision, 1):
+            if not quiet:
+                print(f"[sandbox] provisioning step {i}/{len(self.provision)}")
+            prov_env = dict(os.environ)
+            prov_env["SANDBOX_VENV"] = str(self.venv_dir)
+            prov_env.update(self._resolved_env())
+            r = subprocess.run([str(py), "-c", snippet], env=prov_env,
+                               capture_output=True, text=True, timeout=1800)
+            if r.returncode != 0:
+                raise RuntimeError(
+                    f"provisioning step {i} failed:\n{(r.stderr or r.stdout)[-3000:]}"
+                )
+
+        self._stamp_path().write_text(json.dumps({
+            "packages": self.packages,
+            "py": self._base_version(),
+            "provision": self.provision,
+        }))
+        self._python = py
+        self._base_prefix = self._detect_base_prefix(py)
+        if not quiet:
+            print(f"[sandbox] ready ({self.isolation} isolation)")
+        if self.isolation == "unshare-root":
+            print("Warning: this host's unshare cannot keep the caller's uid, so "
+                  "candidate code runs as root inside the namespace. Tests that "
+                  "assert a permission denial will not see one. Install "
+                  "bubblewrap (bwrap) for correct results.")
+        return py
+
+    @staticmethod
+    def _detect_base_prefix(py: Path) -> List[Path]:
+        """Every directory that must stay visible for the venv to start.
+
+        A venv is only symlinks into its base interpreter, so the whole install
+        tree has to survive the tmpfs that hides $HOME -- and binding just the
+        resolved tree is not enough. Version-managed interpreters (uv, pyenv)
+        are commonly reached through an unversioned alias symlink
+        (cpython-3.13-... -> cpython-3.13.14-...), and startup resolves through
+        the alias path, so both spellings need to be mounted. Walk the chain
+        and keep every install root it passes through.
+        """
+        roots: List[Path] = []
+
+        def add(path: Path):
+            if path.exists() and path not in roots:
+                roots.append(path)
+
+        cur, hops = py, 0
+        while hops < 16:
+            hops += 1
+            add(cur.parent.parent)
+            if cur.is_symlink():
+                target = Path(os.readlink(cur))
+                cur = target if target.is_absolute() else (cur.parent / target)
+            else:
+                break
+
+        try:
+            r = subprocess.run([str(py), "-c", "import sys; print(sys.base_prefix)"],
+                               capture_output=True, text=True, timeout=60)
+            if r.returncode == 0 and r.stdout.strip():
+                add(Path(r.stdout.strip()))
+        except Exception:
+            pass
+
+        return [p for p in roots if str(p) not in ("/", "/usr", "/usr/local")]
+
+    def _isolate(self, argv: Sequence[str], workdir: Path,
+                 ro_binds: Sequence[Path] = ()) -> List[str]:
+        """Wrap a command in the strongest isolation tier this host supports."""
+        if self.isolation == "bwrap":
+            return _bwrap_argv(
+                workdir, argv,
+                ro_binds=[self.venv_dir, *(self._base_prefix or []), *ro_binds],
+            )
+        if self.isolation in UNSHARE_ARGV:
+            return [*UNSHARE_ARGV[self.isolation], *argv]
+        return list(argv)
+
+    def _base_env(self, workdir: Path) -> Dict[str, str]:
+        env = {
+            "PATH": "/usr/bin:/bin",
+            "HOME": str(workdir),
+            "TMPDIR": str(workdir),
+            "LC_ALL": "C.UTF-8",
+            "PYTHONHASHSEED": "0",
+            "PYTHONDONTWRITEBYTECODE": "1",
+            "OPENBLAS_NUM_THREADS": "1",
+            "OMP_NUM_THREADS": "1",
+        }
+        env.update(self._resolved_env())
+        return env
+
+    def run_argv(self, argv: Sequence[str], workdir: Path,
+                 timeout: float = 15.0, env: Optional[Dict[str, str]] = None,
+                 ro_binds: Sequence[Path] = ()) -> ExecResult:
+        """Run an arbitrary command in `workdir`, which stays writable.
+
+        Unlike run(), the caller owns the directory and it survives the call.
+        That is what a multi-step agent needs: its edits have to persist across
+        many tool invocations before anything is graded. Taking an argv rather
+        than a source string is what lets a suite drive a linter or a test
+        runner for a language other than Python.
+        """
+        run_env = self._base_env(workdir)
+        if env:
+            run_env.update(env)
+
+        started = time.monotonic()
+        try:
+            proc = subprocess.Popen(
+                self._isolate(argv, workdir, ro_binds),
+                cwd=str(workdir),
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                errors="replace",
+                env=run_env,
+                preexec_fn=_apply_rlimits(timeout,
+                                          with_nproc=self.isolation == "rlimit",
+                                          as_bytes=self.address_space_limit),
+            )
+        except Exception as e:
+            return ExecResult(False, "error", "", f"{type(e).__name__}: {e}")
+
+        try:
+            out, err = proc.communicate(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            _kill_tree(proc)
+            out, err = proc.communicate()
+            return ExecResult(False, "timeout", out or "", err or "",
+                              None, time.monotonic() - started)
+
+        duration = time.monotonic() - started
+        if proc.returncode == 0:
+            return ExecResult(True, "ok", out, err, 0, duration)
+        return ExecResult(False, "failed", out, err, proc.returncode, duration)
+
+    def run(self, code: str, timeout: float = 15.0,
+            env: Optional[Dict[str, str]] = None) -> ExecResult:
+        """Run `code` to completion in the sandbox. Exit 0 == passed."""
+        workdir = Path(tempfile.mkdtemp(prefix=f"{self.name}-", dir="/tmp"))
+        try:
+            prog = workdir / "candidate.py"
+            prog.write_text(code)
+            return self.run_argv([str(self.python), "-I", "-B", str(prog)],
+                                 workdir, timeout=timeout, env=env)
+        except Exception as e:
+            return ExecResult(False, "error", "", f"{type(e).__name__}: {e}")
+        finally:
+            shutil.rmtree(workdir, ignore_errors=True)
+
+
+def _kill_tree(proc: subprocess.Popen):
+    """Kill the candidate and everything it spawned."""
+    for sig in (signal.SIGKILL,):
+        try:
+            os.killpg(os.getpgid(proc.pid), sig)
+        except (ProcessLookupError, PermissionError):
+            try:
+                proc.kill()
+            except ProcessLookupError:
+                pass

examples/llama-eval/llama-server-simulator.py

@@ -13,8 +13,6 @@ from typing import Dict, List, Optional
 from dataclasses import dataclass
 from pathlib import Path
 
-import datasets
-
 # Set cache directory for HuggingFace datasets
 cache_dir = Path.home() / ".cache" / "huggingface" / "datasets"
 cache_dir.mkdir(parents=True, exist_ok=True)
@@ -76,6 +74,8 @@ class AimeDataset:
         return question.get("problem", question.get("question", ""))
 
     def _load_dataset(self):
+        import datasets
+
         if self.dataset_type == "aime":
             print(f"Loading AIME dataset (split: {self.split})...")
             cache_path = Path.home() / ".cache" / "huggingface" / "datasets" / "AI-MO___aimo-validation-aime" / "default" / "0.0.0"
@@ -164,6 +164,58 @@ class AimeDataset:
             return str(normalized) if normalized is not None else answer
         return str(answer)
 
+class CodeDataset:
+    """Stand-in for a code model: replays canonical solutions as chat replies.
+
+    Handles both code suites: HumanEval keys off the function stub, ClassEval
+    off the class skeleton.
+    """
+
+    DEFAULT_SOURCE = {
+        "humaneval": "openai/openai_humaneval",
+        "classeval": "FudanSELab/ClassEval",
+    }
+
+    def __init__(self, dataset_type: str, source: Optional[str] = None):
+        self.dataset_type = dataset_type
+        self.source = source or self.DEFAULT_SOURCE[dataset_type]
+        self.questions: List[Dict] = []
+        self._load_dataset()
+
+    def _load_dataset(self):
+        print(f"Loading {self.dataset_type} dataset (source: {self.source})...")
+        local = Path(self.source)
+        if local.exists():
+            text = local.read_text()
+            if local.suffix == ".json":
+                rows = json.loads(text)
+            else:
+                rows = [json.loads(ln) for ln in text.splitlines() if ln.strip()]
+        else:
+            import datasets
+            rows = [dict(r) for r in datasets.load_dataset(self.source, split="test")]
+        self.questions = rows
+        print(f"{self.dataset_type} dataset loaded: {len(self.questions)} questions")
+
+    def _stub_field(self) -> str:
+        return "skeleton" if self.dataset_type == "classeval" else "prompt"
+
+    def _get_question_text(self, question: Dict) -> str:
+        return question[self._stub_field()]
+
+    def find_question(self, request_text: str) -> Optional[Dict]:
+        for question in self.questions:
+            stub = question[self._stub_field()].rstrip()
+            if stub and stub in request_text:
+                return question
+        return None
+
+    def get_answer(self, question: Dict) -> str:
+        if self.dataset_type == "classeval":
+            return question["solution_code"]
+        return question["prompt"] + question["canonical_solution"]
+
+
 class Simulator:
     def __init__(
         self,
@@ -171,12 +223,17 @@ class Simulator:
         host: str = "localhost",
         success_rate: float = 0.8,
         dataset_split: str = "train",
-        dataset_type: str = "aime"
+        dataset_type: str = "aime",
+        dataset_source: Optional[str] = None
     ):
         self.port = port
         self.host = host
         self.success_rate = success_rate
-        self.dataset = AimeDataset(dataset_split, dataset_type)
+        self.dataset_type = dataset_type
+        if dataset_type in ("humaneval", "classeval"):
+            self.dataset = CodeDataset(dataset_type, dataset_source)
+        else:
+            self.dataset = AimeDataset(dataset_split, dataset_type)
         self.eval_state = EvalState(
             id=dataset_type,
             tasks=[dataset_type],
@@ -196,6 +253,12 @@ class Simulator:
         else:
             response_text = self._generate_wrong_answer(question)
 
+        if self.dataset_type in ("humaneval", "classeval"):
+            response_text = (
+                f"Here is the implementation:\n\n```python\n{response_text}\n```\n\n"
+                "Let me know if you want it optimised further."
+            )
+
         comp_tokens = random.randint(10000, 60000)
         tps_gen = random.uniform(90.0, 110.0)
         t_gen_ms = comp_tokens / tps_gen * 1000
@@ -229,6 +292,12 @@ class Simulator:
     def _generate_wrong_answer(self, question: Dict) -> str:
         expected_answer = self.dataset.get_answer(question)
 
+        if self.dataset_type == "classeval":
+            return re.sub(r"(\n\s+def [^\n]+:\n)", r"\1        return None\n",
+                          question["skeleton"])
+        if self.dataset_type == "humaneval":
+            return question["prompt"] + "    return None\n"
+
         if expected_answer.isdigit():
             wrong_answer = str(int(expected_answer) + 1)
         else:
@@ -335,7 +404,7 @@ def main():
         "--dataset",
         type=str,
         default="aime",
-        choices=["aime", "aime2025"],
+        choices=["aime", "aime2025", "humaneval", "classeval"],
         help="Dataset type (default: aime)"
     )
     parser.add_argument(
@@ -344,6 +413,12 @@ def main():
         default="train",
         help="AIME dataset split to use (default: train)"
     )
+    parser.add_argument(
+        "--dataset-source",
+        type=str,
+        default=None,
+        help="Override the dataset location (local .jsonl path or HuggingFace repo id)"
+    )
 
     args = parser.parse_args()
 
@@ -353,7 +428,8 @@ def main():
         host=args.host,
         success_rate=args.success_rate,
         dataset_split=args.dataset_split,
-        dataset_type=args.dataset
+        dataset_type=args.dataset,
+        dataset_source=args.dataset_source
     )
 
     server = HTTPServer((args.host, args.port), RequestHandler)

ggml/src/ggml-cpu/arch/s390/quants.c

@@ -310,7 +310,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
         const int8x16_t v_yl = vec_xl(0      , y[ib].qs);
         const int8x16_t v_yh = vec_xl(QK8_1/2, y[ib].qs);
 
-        const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);
+        const int32x4_t v_xy_ = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh);
         const float32x4_t v_xy = vec_float(v_xy_);
 
         const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));
@@ -379,8 +379,8 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo
         const int8x16_t v_y1l = vec_xl(0,       y1->qs);
         const int8x16_t v_y1h = vec_xl(QK8_0/2, y1->qs);
 
-        const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0l), v_x0h, v_y0h);
-        const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y1l), v_x1h, v_y1h);
+        const int32x4_t v_xy0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0l, v_y0l), v_x0h, v_y0h);
+        const int32x4_t v_xy1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1l, v_y1l), v_x1h, v_y1h);
 
         const float32x4_t v_xy0f = vec_float(v_xy0);
         const float32x4_t v_xy1f = vec_float(v_xy1);
@@ -407,7 +407,7 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo
         const int8x16_t v_yl = vec_xl(0,       y0->qs);
         const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs);
 
-        const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);
+        const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh);
         const float32x4_t v_xyf = vec_float(v_xy);
 
         const float32x4_t v_d = vec_splats(GGML_E8M0_TO_FP32_HALF(x0->e) * GGML_CPU_FP16_TO_FP32(y0->d));
@@ -502,8 +502,8 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
         const int8x16_t v_y1l = vec_xl(0,       (const int8_t *)y1->qs);
         const int8x16_t v_y1h = vec_xl(QK8_0/2, (const int8_t *)y1->qs);
 
-        const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h);
-        const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h);
+        const int32x4_t v_xy0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h);
+        const int32x4_t v_xy1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h);
 
         const float32x4_t v_xy0f = vec_float(v_xy0);
         const float32x4_t v_xy1f = vec_float(v_xy1);
@@ -548,7 +548,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
         const int8x16_t v_yl = vec_xl(0,       (const int8_t *)y0->qs);
         const int8x16_t v_yh = vec_xl(QK8_0/2, (const int8_t *)y0->qs);
 
-        const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh);
+        const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh);
         const float32x4_t v_xyf = vec_float(v_xy);
 
         const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));
@@ -653,8 +653,8 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
         const int8x16_t v_y1l = vec_xl(0      , y1->qs);
         const int8x16_t v_y1h = vec_xl(QK8_1/2, y1->qs);
 
-        const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h);
-        const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h);
+        const int32x4_t v_xy0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h);
+        const int32x4_t v_xy1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h);
 
         const float32x4_t v_xy0f = vec_float(v_xy0);
         const float32x4_t v_xy1f = vec_float(v_xy1);
@@ -701,7 +701,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
         const int8x16_t v_yl = vec_xl(0      , y0->qs);
         const int8x16_t v_yh = vec_xl(QK8_1/2, y0->qs);
 
-        const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh);
+        const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh);
         const float32x4_t v_xyf = vec_float(v_xy);
 
         const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));
@@ -751,7 +751,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
         const int8x16_t v_yl = vec_xl(0      , y[ib].qs);
         const int8x16_t v_yh = vec_xl(QK8_0/2, y[ib].qs);
 
-        const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);
+        const int32x4_t v_xy_ = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh);
         const float32x4_t v_xy = vec_float(v_xy_);
         const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));
 
@@ -855,10 +855,10 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
             q3bytes[2] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 2), v_3m), (int8x16_t)q3h[2]);
             q3bytes[3] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 2), v_3m), (int8x16_t)q3h[3]);
 
-            isum0 = ggml_vec_dot(v_z, q3bytes[0], q8bytes[0]);
-            isum1 = ggml_vec_dot(v_z, q3bytes[1], q8bytes[1]);
-            isum2 = ggml_vec_dot(v_z, q3bytes[2], q8bytes[2]);
-            isum3 = ggml_vec_dot(v_z, q3bytes[3], q8bytes[3]);
+            isum0 = ggml_vec_dot_s32(v_z, q3bytes[0], q8bytes[0]);
+            isum1 = ggml_vec_dot_s32(v_z, q3bytes[1], q8bytes[1]);
+            isum2 = ggml_vec_dot_s32(v_z, q3bytes[2], q8bytes[2]);
+            isum3 = ggml_vec_dot_s32(v_z, q3bytes[3], q8bytes[3]);
 
             isum += (isum0[0] + isum0[1] + isum0[2] + isum0[3]) * scale[0];
             isum += (isum1[0] + isum1[1] + isum1[2] + isum1[3]) * scale[1];
@@ -877,10 +877,10 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
             q3bytes[2] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 6), v_3m), (int8x16_t)q3h[2]);
             q3bytes[3] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 6), v_3m), (int8x16_t)q3h[3]);
 
-            isum0 = ggml_vec_dot(v_z, q3bytes[0], q8bytes[4]);
-            isum1 = ggml_vec_dot(v_z, q3bytes[1], q8bytes[5]);
-            isum2 = ggml_vec_dot(v_z, q3bytes[2], q8bytes[6]);
-            isum3 = ggml_vec_dot(v_z, q3bytes[3], q8bytes[7]);
+            isum0 = ggml_vec_dot_s32(v_z, q3bytes[0], q8bytes[4]);
+            isum1 = ggml_vec_dot_s32(v_z, q3bytes[1], q8bytes[5]);
+            isum2 = ggml_vec_dot_s32(v_z, q3bytes[2], q8bytes[6]);
+            isum3 = ggml_vec_dot_s32(v_z, q3bytes[3], q8bytes[7]);
 
             isum += vec_hsum_i32x4(isum0) * scale[0];
             isum += vec_hsum_i32x4(isum1) * scale[1];
@@ -981,7 +981,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
             v_xl[0] = (int8x16_t)vec_and(v_x[0], v_lm);
             v_xl[1] = (int8x16_t)vec_and(v_x[1], v_lm);
 
-            const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]);
+            const int32x4_t p1 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]);
             sumi1 += vec_hsum_i32x4(p1) * scales[2*j+0];
 
             v_y[0] = vec_xl(0 , y0);
@@ -991,7 +991,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
             v_xl[0] = (int8x16_t)vec_sr(v_x[0], 4);
             v_xl[1] = (int8x16_t)vec_sr(v_x[1], 4);
 
-            const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]);
+            const int32x4_t p2 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]);
             sumi2 += vec_hsum_i32x4(p2) * scales[2*j+1];
         }
 
@@ -1107,8 +1107,8 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
             q5b[2] = (int8x16_t)vec_or(vec_sr(v_xl[0], 4), q5h[2]);
             q5b[3] = (int8x16_t)vec_or(vec_sr(v_xl[1], 4), q5h[3]);
 
-            int32x4_t sumi0 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[0], v_y[0]), q5b[1], v_y[1]);
-            int32x4_t sumi1 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[2], v_y[2]), q5b[3], v_y[3]);
+            int32x4_t sumi0 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, q5b[0], v_y[0]), q5b[1], v_y[1]);
+            int32x4_t sumi1 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, q5b[2], v_y[2]), q5b[3], v_y[3]);
 
             sumi += vec_hsum_i32x4(sumi0) * *scales++;
             sumi += vec_hsum_i32x4(sumi1) * *scales++;
@@ -1217,10 +1217,10 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
             q6b[2] = (int8x16_t)(vec_or(vec_and(v_xl[2], v_lm), q6h[2]));
             q6b[3] = (int8x16_t)(vec_or(vec_and(v_xl[3], v_lm), q6h[3]));
 
-            int32x4_t summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]);
-            int32x4_t summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]);
-            int32x4_t summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]);
-            int32x4_t summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]);
+            int32x4_t summs0 = ggml_vec_dot_s32(v_z, q6b[0], v_y[0]);
+            int32x4_t summs1 = ggml_vec_dot_s32(v_z, q6b[1], v_y[1]);
+            int32x4_t summs2 = ggml_vec_dot_s32(v_z, q6b[2], v_y[2]);
+            int32x4_t summs3 = ggml_vec_dot_s32(v_z, q6b[3], v_y[3]);
 
             isum += vec_hsum_i32x4(summs0) * scale[0] +
                     vec_hsum_i32x4(summs1) * scale[1] +
@@ -1251,10 +1251,10 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
             q6b[2] = (int8x16_t)(vec_or(vec_sr(v_xl[2], 4), q6h[2]));
             q6b[3] = (int8x16_t)(vec_or(vec_sr(v_xl[3], 4), q6h[3]));
 
-            summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]);
-            summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]);
-            summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]);
-            summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]);
+            summs0 = ggml_vec_dot_s32(v_z, q6b[0], v_y[0]);
+            summs1 = ggml_vec_dot_s32(v_z, q6b[1], v_y[1]);
+            summs2 = ggml_vec_dot_s32(v_z, q6b[2], v_y[2]);
+            summs3 = ggml_vec_dot_s32(v_z, q6b[3], v_y[3]);
 
             isum += vec_hsum_i32x4(summs0) * scale[0] +
                     vec_hsum_i32x4(summs1) * scale[1] +
@@ -1367,8 +1367,8 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
 //            q2u2 = vec_mul(q2u2, q2s2);
 //            q2u3 = vec_mul(q2u3, q2s3);
 
-//            const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u0, q8b0), q2u1, q8b1);
-//            const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u2, q8b2), q2u3, q8b3);
+//            const int32x4_t p1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splat_s32(0), q2u0, q8b0), q2u1, q8b1);
+//            const int32x4_t p2 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splat_s32(0), q2u2, q8b2), q2u3, q8b3);
 
 //            sumf1 += (p1[0] + p1[1] + p1[2] + p1[3]) * (0.5f + (aux32[1] >> 28));
 //            sumf2 += (p2[0] + p2[1] + p2[2] + p2[3]) * (0.5f + (aux32[3] >> 28));
@@ -1445,7 +1445,7 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v
 
         const int8x16_t v_yl = vec_xl(0      , y0->qs);
         const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs);
-        const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);
+        const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh);
 
         sumf += GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d) * vec_hsum_i32x4(v_xy);
     }
@@ -1508,8 +1508,8 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v
             const int8x16_t v_y3 = vec_xl(48, q8);
             q8 += 64;
 
-            int32x4_t vsumi0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0), v_x0h, v_y1);
-            int32x4_t vsumi1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y2), v_x1h, v_y3);
+            int32x4_t vsumi0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0l, v_y0), v_x0h, v_y1);
+            int32x4_t vsumi1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1l, v_y2), v_x1h, v_y3);
 
             int ls1 = ((x[ibl].scales_l[ib] & 0xF) | ((h << 4) & 0x30)) - 32;
             int ls2 = ((x[ibl].scales_l[ib] >>  4) | ((h << 2) & 0x30)) - 32;

ggml/src/ggml-cpu/arch/s390/repack.cpp

@@ -70,23 +70,6 @@ void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTR
 #endif
 }
 
-#if defined(__VXE__) || defined(__VXE2__)
-static inline int16x8_t vxe_dot_acc(const int8x16_t v_x, const int8x16_t v_y, const int16x8_t v_acc) {
-    return vec_meadd(v_x, v_y, vec_moadd(v_x, v_y, v_acc));
-}
-
-static inline int8x16_t vxe_splat_granule(const int8_t * qs) {
-    uint32_t g;
-    memcpy(&g, qs, sizeof(g));
-    return (int8x16_t)vec_splats(g);
-}
-
-static inline int32x4_t vxe_fold(const int16x8_t v_sumi) {
-    const int16x8_t v_ones = vec_splats((int16_t)1);
-    return vec_add(vec_mule(v_sumi, v_ones), vec_mulo(v_sumi, v_ones));
-}
-#endif
-
 void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
     const int qk = QK8_0;
     const int nb = n / qk;
@@ -131,21 +114,21 @@ void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo
 
             int16x8_t v_sumi = vec_splats((int16_t)0);
 
-            v_sumi = vxe_dot_acc(v_x0l, vxe_splat_granule(y_lo +  0), v_sumi);
-            v_sumi = vxe_dot_acc(v_x1l, vxe_splat_granule(y_lo +  4), v_sumi);
-            v_sumi = vxe_dot_acc(v_x2l, vxe_splat_granule(y_lo +  8), v_sumi);
-            v_sumi = vxe_dot_acc(v_x3l, vxe_splat_granule(y_lo + 12), v_sumi);
+            v_sumi = ggml_vec_dot_s16(v_sumi, v_x0l, ggml_vec_xl_s8x4_splat(y_lo +  0));
+            v_sumi = ggml_vec_dot_s16(v_sumi, v_x1l, ggml_vec_xl_s8x4_splat(y_lo +  4));
+            v_sumi = ggml_vec_dot_s16(v_sumi, v_x2l, ggml_vec_xl_s8x4_splat(y_lo +  8));
+            v_sumi = ggml_vec_dot_s16(v_sumi, v_x3l, ggml_vec_xl_s8x4_splat(y_lo + 12));
 
-            v_sumi = vxe_dot_acc(v_x0h, vxe_splat_granule(y_hi +  0), v_sumi);
-            v_sumi = vxe_dot_acc(v_x1h, vxe_splat_granule(y_hi +  4), v_sumi);
-            v_sumi = vxe_dot_acc(v_x2h, vxe_splat_granule(y_hi +  8), v_sumi);
-            v_sumi = vxe_dot_acc(v_x3h, vxe_splat_granule(y_hi + 12), v_sumi);
+            v_sumi = ggml_vec_dot_s16(v_sumi, v_x0h, ggml_vec_xl_s8x4_splat(y_hi +  0));
+            v_sumi = ggml_vec_dot_s16(v_sumi, v_x1h, ggml_vec_xl_s8x4_splat(y_hi +  4));
+            v_sumi = ggml_vec_dot_s16(v_sumi, v_x2h, ggml_vec_xl_s8x4_splat(y_hi +  8));
+            v_sumi = ggml_vec_dot_s16(v_sumi, v_x3h, ggml_vec_xl_s8x4_splat(y_hi + 12));
 
             const float32x4_t v_yd = vec_splats(GGML_CPU_FP16_TO_FP32(a_ptr[l].d));
-            const float32x4_t v_xd = __lzs_f16cx4_load(b_ptr[l].d);
+            const float32x4_t v_xd = ggml_vec_xl_f16x4_finite(b_ptr[l].d);
             const float32x4_t v_d  = vec_mul(v_yd, v_xd);
 
-            v_sumf = vec_madd(vec_float(vxe_fold(v_sumi)), v_d, v_sumf);
+            v_sumf = vec_madd(vec_float(vec_paddl_s16(v_sumi)), v_d, v_sumf);
         }
 
         vec_xst(v_sumf, 0, res_ptr + x * ncols_interleaved);
@@ -192,24 +175,24 @@ void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo
                     const int8_t * y_lo = a_ptr[l].qs + 16 * k;
                     const int8_t * y_hi = y_lo + qk / 2 * 4;
 
-                    v_sumi0 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo +  0), v_sumi0);
-                    v_sumi1 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo +  4), v_sumi1);
-                    v_sumi2 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo +  8), v_sumi2);
-                    v_sumi3 = vxe_dot_acc(v_xl, vxe_splat_granule(y_lo + 12), v_sumi3);
+                    v_sumi0 = ggml_vec_dot_s16(v_sumi0, v_xl, ggml_vec_xl_s8x4_splat(y_lo +  0));
+                    v_sumi1 = ggml_vec_dot_s16(v_sumi1, v_xl, ggml_vec_xl_s8x4_splat(y_lo +  4));
+                    v_sumi2 = ggml_vec_dot_s16(v_sumi2, v_xl, ggml_vec_xl_s8x4_splat(y_lo +  8));
+                    v_sumi3 = ggml_vec_dot_s16(v_sumi3, v_xl, ggml_vec_xl_s8x4_splat(y_lo + 12));
 
-                    v_sumi0 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi +  0), v_sumi0);
-                    v_sumi1 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi +  4), v_sumi1);
-                    v_sumi2 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi +  8), v_sumi2);
-                    v_sumi3 = vxe_dot_acc(v_xh, vxe_splat_granule(y_hi + 12), v_sumi3);
+                    v_sumi0 = ggml_vec_dot_s16(v_sumi0, v_xh, ggml_vec_xl_s8x4_splat(y_hi +  0));
+                    v_sumi1 = ggml_vec_dot_s16(v_sumi1, v_xh, ggml_vec_xl_s8x4_splat(y_hi +  4));
+                    v_sumi2 = ggml_vec_dot_s16(v_sumi2, v_xh, ggml_vec_xl_s8x4_splat(y_hi +  8));
+                    v_sumi3 = ggml_vec_dot_s16(v_sumi3, v_xh, ggml_vec_xl_s8x4_splat(y_hi + 12));
                 }
 
                 const float32x4_t v_yd = __lzs_f16cx4_load(a_ptr[l].d);
-                const float32x4_t v_xd = __lzs_f16cx4_load(b_ptr[l].d);
+                const float32x4_t v_xd = ggml_vec_xl_f16x4_finite(b_ptr[l].d);
 
-                v_sumf[0] = vec_madd(vec_float(vxe_fold(v_sumi0)), vec_mul(v_xd, vec_splat(v_yd, 0)), v_sumf[0]);
-                v_sumf[1] = vec_madd(vec_float(vxe_fold(v_sumi1)), vec_mul(v_xd, vec_splat(v_yd, 1)), v_sumf[1]);
-                v_sumf[2] = vec_madd(vec_float(vxe_fold(v_sumi2)), vec_mul(v_xd, vec_splat(v_yd, 2)), v_sumf[2]);
-                v_sumf[3] = vec_madd(vec_float(vxe_fold(v_sumi3)), vec_mul(v_xd, vec_splat(v_yd, 3)), v_sumf[3]);
+                v_sumf[0] = vec_madd(vec_float(vec_paddl_s16(v_sumi0)), vec_mul(v_xd, vec_splat(v_yd, 0)), v_sumf[0]);
+                v_sumf[1] = vec_madd(vec_float(vec_paddl_s16(v_sumi1)), vec_mul(v_xd, vec_splat(v_yd, 1)), v_sumf[1]);
+                v_sumf[2] = vec_madd(vec_float(vec_paddl_s16(v_sumi2)), vec_mul(v_xd, vec_splat(v_yd, 2)), v_sumf[2]);
+                v_sumf[3] = vec_madd(vec_float(vec_paddl_s16(v_sumi3)), vec_mul(v_xd, vec_splat(v_yd, 3)), v_sumf[3]);
             }
 
             for (int m = 0; m < 4; m++) {

ggml/src/ggml-cpu/ggml-cpu-impl.h

@@ -443,6 +443,13 @@ inline static ggml_int8x16x4_t ggml_vec_xl_s8x4(const int8_t * ptr) {
     return res;
 }
 
+// load 4 bytes and replicate them into all 4 word lanes
+inline static int8x16_t ggml_vec_xl_s8x4_splat(const int8_t * ptr) {
+    uint32_t tmp;
+    memcpy(&tmp, ptr, sizeof(tmp));
+    return (int8x16_t)vec_splats(tmp);
+}
+
 typedef struct ggml_int16x8x2_t {
     int16x8_t val[2];
 } ggml_int16x8x2_t;
@@ -456,31 +463,17 @@ inline static ggml_int16x8x2_t ggml_vec_xl_s16x2(const int16_t * ptr) {
     return res;
 }
 
-/*
-    ! WARNING: Very slow. Use vec_perm if possible. Refer to iq4_xs
-    !          or iq4_nl for example implementation.
-*/
-inline static int8x16_t ggml_vec_tbl(int8x16_t a, uint8x16_t b) {
-    int8x16_t res;
+// faster fp16 -> fp32 conversion using vector intrinsics
+// WARNING: fp16 inf converts into 65536.0f. use this only for finite values.
+inline static float32x4_t ggml_vec_xl_f16x4_finite(const ggml_fp16_t * x) {
+    uint64_t tmp;
+    memcpy(&tmp, x, sizeof(tmp));
 
-    res[ 0] = a[b[ 0]];
-    res[ 1] = a[b[ 1]];
-    res[ 2] = a[b[ 2]];
-    res[ 3] = a[b[ 3]];
-    res[ 4] = a[b[ 4]];
-    res[ 5] = a[b[ 5]];
-    res[ 6] = a[b[ 6]];
-    res[ 7] = a[b[ 7]];
-    res[ 8] = a[b[ 8]];
-    res[ 9] = a[b[ 9]];
-    res[10] = a[b[10]];
-    res[11] = a[b[11]];
-    res[12] = a[b[12]];
-    res[13] = a[b[13]];
-    res[14] = a[b[14]];
-    res[15] = a[b[15]];
-
-    return res;
+    const uint64x2_t  v_x    = { tmp, tmp };
+    const uint32x4_t  v_xh   = vec_unpackh((uint16x8_t)v_x);
+    const uint32x4_t  v_bits = vec_sl(vec_and(v_xh, vec_splats((uint32_t)0x7FFF)), 13);
+    const float32x4_t v_abs  = vec_mul((float32x4_t)v_bits, vec_splats(0x1.0p+112f));
+    return (float32x4_t)vec_or((uint32x4_t)v_abs, vec_sl(vec_and(v_xh, vec_splats((uint32_t)0x8000)), 16));
 }
 
 inline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) {
@@ -492,6 +485,11 @@ inline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) {
     return v_abo + v_abe;
 }
 
+inline static int32x4_t vec_paddl_s16(int16x8_t a) {
+    const int16x8_t v_ones = vec_splats((int16_t)1);
+    return vec_mule(a, v_ones) + vec_mulo(a, v_ones);
+}
+
 /**
  * @see https://github.com/ggml-org/llama.cpp/pull/14037
  */
@@ -505,7 +503,11 @@ inline static int32_t vec_hsum_i32x4(int32x4_t v) {
     return v_temp[0] + v_temp[1];
 }
 
-inline static int32x4_t ggml_vec_dot(int32x4_t acc, int8x16_t a, int8x16_t b) {
+inline static int16x8_t ggml_vec_dot_s16(int16x8_t acc, int8x16_t a, int8x16_t b) {
+    return vec_meadd(a, b, vec_moadd(a, b, acc));
+}
+
+inline static int32x4_t ggml_vec_dot_s32(int32x4_t acc, int8x16_t a, int8x16_t b) {
     const int16x8_t p = vec_mule(a, b) + vec_mulo(a, b);
     return acc + (vec_unpackh(p) + vec_unpackl(p));
 }

ggml/src/ggml-cpu/tiled/tiled-kernel.cpp

@@ -244,16 +244,6 @@ static void tiled_run_microtile_avx2(const tiled_tile_src0 & src0, const tiled_t
     const int bs_stride = (nkr == 1) ? TILED_TILE_ROWS : TILED_MICRO;
     const int bs_off = seff * TILED_MICRO;
 
-    // Horizontal sum helper: converts 8x32-bit int lane inside a YMM register to scalar int32
-    auto hsum256_epi32 = [](const __m256i v) -> int32_t {
-        __m128i low = _mm256_castsi256_si128(v);
-        __m128i high = _mm256_extracti128_si256(v, 1);
-        __m128i sum128 = _mm_add_epi32(low, high);
-        sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, _MM_SHUFFLE(2, 3, 0, 1)));
-        sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, _MM_SHUFFLE(1, 0, 3, 2)));
-        return _mm_cvtsi128_si32(sum128);
-    };
-
     for (int i = 0; i < TILED_MICRO; i++) {
         const int ar = i0 + i;
         const int d_off = seff * TILED_MICRO + ar;
@@ -362,15 +352,20 @@ static void tiled_run_microtile_avx2(const tiled_tile_src0 & src0, const tiled_t
                 }
             }
 
-            // OPTIMIZATION 2: Vectorized Epilogue with FMA3 gating
-            // Reduce accumulators into a 256-bit vector across 8 columns
-            alignas(32) int32_t s1_vals[GROUP];
-            #pragma GCC unroll 8
-            for (int t = 0; t < GROUP; t++) {
-                s1_vals[t] = hsum256_epi32(acc[t]);
-            }
-            __m256i s1_vec = _mm256_load_si256((const __m256i *) s1_vals);
+            // Reduce the eight accumulators to column sums in acc[0]..acc[7] order, with additions modulo 2^32.
+            const __m256i ab = _mm256_hadd_epi32(acc[0], acc[1]);
+            const __m256i cd = _mm256_hadd_epi32(acc[2], acc[3]);
+            const __m256i ef = _mm256_hadd_epi32(acc[4], acc[5]);
+            const __m256i gh = _mm256_hadd_epi32(acc[6], acc[7]);
 
+            // Each 128-bit half now holds four column sums over input lanes 0..3 or 4..7.
+            const __m256i abcd = _mm256_hadd_epi32(ab, cd);
+            const __m256i efgh = _mm256_hadd_epi32(ef, gh);
+            const __m256i lo = _mm256_permute2x128_si256(abcd, efgh, 0x20);
+            const __m256i hi = _mm256_permute2x128_si256(abcd, efgh, 0x31);
+            __m256i s1_vec = _mm256_add_epi32(lo, hi);
+
+            // OPTIMIZATION 2: Vectorized Epilogue with FMA3 gating
             if constexpr (BIAS != 0 && ACTBIAS) {
                 // act-bias correction: raw = desired + 128*sum_s scales[s]*w_bsum[s]; corr precomputed in repack_src0
                 s1_vec = _mm256_sub_epi32(s1_vec, _mm256_set1_epi32(mins_row[0]));

ggml/src/ggml-cuda/norm.cu

@@ -81,7 +81,7 @@ static __global__ void group_norm_f32(const float * x, float * dst, const int gr
     }
 }
 
-template <int block_size, bool do_multiply = false, bool do_add = false, bool do_scale = false>
+template <int block_size, int vpt, bool do_vec, bool do_multiply = false, bool do_add = false, bool do_scale = false>
 static __global__ void rms_norm_f32(const float * x,
                                     float *       dst,
                                     const int     ncols,
@@ -142,31 +142,82 @@ static __global__ void rms_norm_f32(const float * x,
                 addc = add + add_sample * add_stride_sample + add_channel * add_stride_channel + add_row * add_stride_row;
             }
 
-            float tmp = 0.0f; // partial sum for thread in warp
+            if constexpr (do_vec) {
+                const int      ncols4 = ncols / 4;
+                const float4 * x4     = (const float4 *) xc;
+                const float4 * mul4   = (const float4 *) mulc;
+                const float4 * add4   = (const float4 *) addc;
+                float4 *       dst4   = (float4 *) dstc;
 
-            for (int col = tid; col < ncols; col += block_size) {
-                const float xi = xc[col];
-                tmp += xi * xi;
-            }
+                float4 xs[vpt];
+                float  tmp = 0.0f;
 
-            // sum up partial sums
-            tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum);
+#pragma unroll
+                for (int k = 0; k < vpt; k++) {
+                    const int i = tid + k*block_size;
+                    if (i < ncols4) {
+                        xs[k] = x4[i];
+                        tmp += xs[k].x*xs[k].x + xs[k].y*xs[k].y + xs[k].z*xs[k].z + xs[k].w*xs[k].w;
+                    }
+                }
+                tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum);
+                const float scale = rsqrtf(tmp / ncols + eps);
 
-            const float mean = tmp / ncols;
-            const float scale = rsqrtf(mean + eps);
+#pragma unroll
+                for (int k = 0; k < vpt; k++) {
+                    const int i = tid + k*block_size;
+                    if (i < ncols4) {
+                        float4 r = make_float4(scale*xs[k].x, scale*xs[k].y, scale*xs[k].z, scale*xs[k].w);
+                        if constexpr (do_multiply) {
+                            const float4 m = mul4[i];
+                            r.x *= m.x;
+                            r.y *= m.y;
+                            r.z *= m.z;
+                            r.w *= m.w;
+                        }
+                        if constexpr (do_add) {
+                            const float4 a = add4[i];
+                            r.x += a.x;
+                            r.y += a.y;
+                            r.z += a.z;
+                            r.w += a.w;
+                        }
+                        if constexpr (do_scale) {
+                            r.x *= scale_out;
+                            r.y *= scale_out;
+                            r.z *= scale_out;
+                            r.w *= scale_out;
+                        }
+                        dst4[i] = r;
+                    }
+                }
+            } else {
+                float tmp = 0.0f; // partial sum for thread in warp
 
-            for (int col = tid; col < ncols; col += block_size) {
-                if constexpr (do_multiply && do_add) {
-                    const int mul_col = fastmodulo(col, mul_ncols_packed);
-                    const int add_col = fastmodulo(col, add_ncols_packed);
-                    dstc[col]         = scale * xc[col] * mulc[mul_col] + addc[add_col];
-                } else if constexpr (do_multiply) {
-                    const int mul_col = fastmodulo(col, mul_ncols_packed);
-                    dstc[col]         = scale * xc[col] * mulc[mul_col];
-                } else if constexpr (do_scale) {
-                    dstc[col] = scale_out * (scale * xc[col]);
-                } else {
-                    dstc[col] = scale * xc[col];
+                for (int col = tid; col < ncols; col += block_size) {
+                    const float xi = xc[col];
+                    tmp += xi * xi;
+                }
+
+                // sum up partial sums
+                tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum);
+
+                const float mean = tmp / ncols;
+                const float scale = rsqrtf(mean + eps);
+
+                for (int col = tid; col < ncols; col += block_size) {
+                    if constexpr (do_multiply && do_add) {
+                        const int mul_col = fastmodulo(col, mul_ncols_packed);
+                        const int add_col = fastmodulo(col, add_ncols_packed);
+                        dstc[col]         = scale * xc[col] * mulc[mul_col] + addc[add_col];
+                    } else if constexpr (do_multiply) {
+                        const int mul_col = fastmodulo(col, mul_ncols_packed);
+                        dstc[col]         = scale * xc[col] * mulc[mul_col];
+                    } else if constexpr (do_scale) {
+                        dstc[col] = scale_out * (scale * xc[col]);
+                    } else {
+                        dstc[col] = scale * xc[col];
+                    }
                 }
             }
 
@@ -332,34 +383,89 @@ static void group_norm_f32_cuda(
     }
 }
 
+#define RMS_NORM_LAUNCH(BLOCK_SIZE, VPT, VEC) \
+    do { \
+        const dim3 block_dims(BLOCK_SIZE, 1, 1); \
+        const ggml_cuda_kernel_launch_params launch_params{ blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float) : 0, stream }; \
+        ggml_cuda_kernel_launch(rms_norm_f32<BLOCK_SIZE, VPT, VEC, do_multiply, do_add, do_scale>, launch_params, \
+            x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, \
+            mul, mul_stride_row, mul_stride_channel, mul_stride_sample, \
+            mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, \
+            add, add_stride_row, add_stride_channel, add_stride_sample, \
+            add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, scale_out); \
+    } while (0)
+
+template <bool do_multiply, bool do_add, bool do_scale>
+static void rms_norm_f32_dispatch(
+        const dim3     blocks_num,
+        const ggml_tensor * x_t,   float * dst,   const int ncols, const int nchannels, const int nsamples,
+        const int64_t  stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps,
+        const ggml_tensor * mul_t, const int64_t mul_stride_row, const int64_t mul_stride_channel, const int64_t mul_stride_sample,
+        const uint3    mul_ncols_packed, const uint3 mul_nrows_packed, const uint3 mul_nchannels_packed, const uint3 mul_nsamples_packed,
+        const ggml_tensor * add_t, const int64_t add_stride_row, const int64_t add_stride_channel, const int64_t add_stride_sample,
+        const uint3    add_ncols_packed, const uint3 add_nrows_packed, const uint3 add_nchannels_packed, const uint3 add_nsamples_packed,
+        cudaStream_t   stream, const float scale_out = 1.0f) {
+    static_assert(!do_add || do_multiply, "fusing add is not supported without multiplying");
+    static_assert(!do_scale || !do_multiply, "fusing scale is not supported with multiplying");
+
+    const float * x   = (const float *) x_t->data;
+    const float * mul = mul_t ? (const float *) mul_t->data : nullptr;
+    const float * add = add_t ? (const float *) add_t->data : nullptr;
+
+    // dst is written at row * ncols, i.e. always at a multiple of ncols, so only its base needs checking here
+    bool vec = ncols % 4 == 0 && ncols > 0 && ((uintptr_t) dst & 15) == 0;
+    vec = vec && ggml_cuda_is_aligned(x_t, 16);
+    if constexpr (do_multiply) {
+        vec = vec && mul_ncols_packed.z == (uint32_t) ncols
+            && ggml_cuda_is_aligned(mul_t, 16);
+    }
+    if constexpr (do_add) {
+        vec = vec && add_ncols_packed.z == (uint32_t) ncols
+            && ggml_cuda_is_aligned(add_t, 16);
+    }
+
+    const int n4 = ncols / 4;
+    if (vec) {
+        if (n4 <= 32) {
+            RMS_NORM_LAUNCH(32,   1, true);
+        } else if (n4 <= 128) {
+            RMS_NORM_LAUNCH(128,  1, true);
+        } else if (n4 <= 256) {
+            RMS_NORM_LAUNCH(256,  1, true);
+        } else if (n4 <= 512) {
+            RMS_NORM_LAUNCH(512,  1, true);
+        } else if (n4 <= 1024) {
+            RMS_NORM_LAUNCH(1024, 1, true);
+        } else if (n4 <= 4096) {
+            RMS_NORM_LAUNCH(1024, 4, true);
+        } else {
+            RMS_NORM_LAUNCH(1024, 1, false);
+        }
+    } else if (ncols < 1024) {
+        RMS_NORM_LAUNCH(256,  1, false);
+    } else {
+        RMS_NORM_LAUNCH(1024, 1, false);
+    }
+}
+#undef RMS_NORM_LAUNCH
+
 template <bool do_scale = false>
 static void rms_norm_f32_cuda(
-        const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples,
+        const ggml_tensor * x_t, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples,
         const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream,
         const float scale_out = 1.0f) {
     const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX));
-    if (ncols < 1024) {
-        const dim3 block_dims(256, 1, 1);
-        const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream};
-        ggml_cuda_kernel_launch(rms_norm_f32<256, false, false, do_scale>, launch_params,
-            x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps,
-        // underlying cudaLaunchKernelEx does not support default params
+    rms_norm_f32_dispatch</*do_multiply*/false, /*do_add*/false, do_scale>(
+        blocks_num, x_t, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps,
         nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0),
-        nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), scale_out);
-    } else {
-        const dim3 block_dims(1024, 1, 1);
-        const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream};
-        ggml_cuda_kernel_launch(rms_norm_f32<1024, false, false, do_scale>, launch_params, x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps,
-        // underlying cudaLaunchKernelEx does not support default params
         nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0),
-        nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), scale_out);
-    }
+        stream, scale_out);
 }
 
-static void rms_norm_mul_f32_cuda(const float *  x,
-                                  const float *  mul,
-                                  const float *  add,
-                                  float *        dst,
+static void rms_norm_mul_f32_cuda(const ggml_tensor * x_t,
+                                  const ggml_tensor * mul_t,
+                                  const ggml_tensor * add_t,
+                                  float             * dst,
                                   const int      ncols,
                                   const int      nrows,
                                   const int      nchannels,
@@ -384,59 +490,33 @@ static void rms_norm_mul_f32_cuda(const float *  x,
                                   const float    eps,
                                   cudaStream_t   stream) {
     const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX));
-    if (mul == nullptr) {
-        rms_norm_f32_cuda(x, dst, ncols, nrows, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, stream);
+    if (mul_t == nullptr) {
+        rms_norm_f32_cuda(x_t, dst, ncols, nrows, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, stream);
         return;
     }
-    if (add == nullptr) {
-        const uint3 mul_ncols_packed     = init_fastdiv_values(mul_ncols);
-        const uint3 mul_nrows_packed     = init_fastdiv_values(mul_nrows);
-        const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels);
-        const uint3 mul_nsamples_packed  = init_fastdiv_values(mul_nsamples);
-        if (ncols < 1024) {
-            const dim3 block_dims(256, 1, 1);
-            const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream};
-            ggml_cuda_kernel_launch(rms_norm_f32<256, true>, launch_params,
-                x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel,
-                mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
-                // underlying cudaLaunchKernelEx does not support default params
-            nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), 1.0f);
-        } else {
-            const dim3 block_dims(1024, 1, 1);
-            const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream};
-            ggml_cuda_kernel_launch(rms_norm_f32<1024, true>, launch_params,
-                x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel,
-                mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
-                // underlying cudaLaunchKernelEx does not support default params
-            nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), 1.0f);
-        }
-    } else {
-        const uint3 mul_ncols_packed     = init_fastdiv_values(mul_ncols);
-        const uint3 mul_nrows_packed     = init_fastdiv_values(mul_nrows);
-        const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels);
-        const uint3 mul_nsamples_packed  = init_fastdiv_values(mul_nsamples);
+    const uint3 mul_ncols_packed     = init_fastdiv_values(mul_ncols);
+    const uint3 mul_nrows_packed     = init_fastdiv_values(mul_nrows);
+    const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels);
+    const uint3 mul_nsamples_packed  = init_fastdiv_values(mul_nsamples);
+    const uint3 zero_packed          = make_uint3(0, 0, 0);
 
+    if (add_t == nullptr) {
+        rms_norm_f32_dispatch</*do_multiply*/true, /*do_add*/false, /*do_scale*/false>(
+            blocks_num, x_t, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps,
+            mul_t, mul_stride_row, mul_stride_channel, mul_stride_sample,
+            mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
+            nullptr, 0, 0, 0, zero_packed, zero_packed, zero_packed, zero_packed, stream);
+    } else {
         const uint3 add_ncols_packed     = init_fastdiv_values(add_ncols);
         const uint3 add_nrows_packed     = init_fastdiv_values(add_nrows);
         const uint3 add_nchannels_packed = init_fastdiv_values(add_nchannels);
         const uint3 add_nsamples_packed  = init_fastdiv_values(add_nsamples);
-        if (ncols < 1024) {
-            const dim3 block_dims(256, 1, 1);
-            const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims,block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream};
-            ggml_cuda_kernel_launch(rms_norm_f32<256, true, true>, launch_params,
-                x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel,
-                mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, add,
-                add_stride_row, add_stride_channel, add_stride_sample, add_ncols_packed, add_nrows_packed,
-                add_nchannels_packed, add_nsamples_packed, 1.0f);
-        } else {
-            const dim3 block_dims(1024, 1, 1);
-            const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream};
-            ggml_cuda_kernel_launch(rms_norm_f32<1024, true, true>, launch_params,
-                x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel,
-                mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, add,
-                add_stride_row, add_stride_channel, add_stride_sample, add_ncols_packed, add_nrows_packed,
-                add_nchannels_packed, add_nsamples_packed, 1.0f);
-        }
+        rms_norm_f32_dispatch</*do_multiply*/true, /*do_add*/true, /*do_scale*/false>(
+            blocks_num, x_t, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps,
+            mul_t, mul_stride_row, mul_stride_channel, mul_stride_sample,
+            mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
+            add_t, add_stride_row, add_stride_channel, add_stride_sample,
+            add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, stream);
     }
 }
 
@@ -510,7 +590,6 @@ void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst)
 
 void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
     const ggml_tensor * src0 = dst->src[0];
-    const float * src0_d = (const float *) src0->data;
     float * dst_d = (float *) dst->data;
     cudaStream_t stream = ctx.stream();
 
@@ -529,12 +608,11 @@ void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
     const int64_t s02 = nb02 / ts0;
     const int64_t s03 = nb03 / ts0;
 
-    rms_norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream);
+    rms_norm_f32_cuda(src0, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream);
 }
 
 void ggml_cuda_op_rms_norm_scale_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * scale_tensor) {
     const ggml_tensor * src0   = dst->src[0];
-    const float *       src0_d = (const float *) src0->data;
     float *             dst_d  = (float *) scale_tensor->data;
     cudaStream_t        stream = ctx.stream();
 
@@ -556,7 +634,7 @@ void ggml_cuda_op_rms_norm_scale_fused(ggml_backend_cuda_context & ctx, ggml_ten
     const int64_t s02 = nb02 / ts0;
     const int64_t s03 = nb03 / ts0;
 
-    rms_norm_f32_cuda<true>(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream, scale);
+    rms_norm_f32_cuda<true>(src0, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream, scale);
 }
 
 void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor) {
@@ -565,15 +643,11 @@ void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor *
 
     memcpy(&eps, dst->op_params, sizeof(float));
 
-    const float * src0_d = (const float *) rms_norm_src->data;
-    const float * mul_d = nullptr;
     const ggml_tensor * mul_src = nullptr;
 
     if (mul_tensor->src[0] == dst) {
-        mul_d = (float *) mul_tensor->src[1]->data;
         mul_src = mul_tensor->src[1];
     } else if(mul_tensor->src[1] == dst) {
-        mul_d = (float *) mul_tensor->src[0]->data;
         mul_src = mul_tensor->src[0];
     } else {
         GGML_ASSERT(false);
@@ -609,7 +683,7 @@ void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor *
     const int mul_nchannels = mul_src->ne[2];
     const int mul_nsamples  = mul_src->ne[3];
 
-    rms_norm_mul_f32_cuda(src0_d, mul_d, nullptr, dst_d,
+    rms_norm_mul_f32_cuda(rms_norm_src, mul_src, nullptr, dst_d,
                           ne00, ne01, ne02, ne03,
                           /*s00*/ s01, s02, s03,
                           /*mul_s00*/ mul_s01, mul_s02, mul_s03,
@@ -628,28 +702,21 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx,
 
     memcpy(&eps, dst->op_params, sizeof(float));
 
-    const float *       src0_d  = (const float *) rms_norm_src->data;
-    const float *       mul_d   = nullptr;
     const ggml_tensor * mul_src = nullptr;
 
     if (mul_tensor->src[0] == dst) {
-        mul_d   = (float *) mul_tensor->src[1]->data;
         mul_src = mul_tensor->src[1];
     } else if (mul_tensor->src[1] == dst) {
-        mul_d   = (float *) mul_tensor->src[0]->data;
         mul_src = mul_tensor->src[0];
     } else {
         GGML_ASSERT(false);
     }
 
-    const float *       add_d   = nullptr;
     const ggml_tensor * add_src = nullptr;
 
     if (add_tensor->src[0] == mul_tensor) {
-        add_d   = (float *) add_tensor->src[1]->data;
         add_src = add_tensor->src[1];
     } else if (add_tensor->src[1] == mul_tensor) {
-        add_d   = (float *) add_tensor->src[0]->data;
         add_src = add_tensor->src[0];
     } else {
         GGML_ASSERT(false);
@@ -697,7 +764,7 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx,
     const int add_nchannels = add_src->ne[2];
     const int add_nsamples  = add_src->ne[3];
 
-    rms_norm_mul_f32_cuda(src0_d, mul_d,add_d,dst_d,
+    rms_norm_mul_f32_cuda(rms_norm_src, mul_src,add_src,dst_d,
                           ne00,ne01, ne02, ne03,
                           /*s00*/ s01, s02, s03,
                           /*mul_s00*/ mul_s01, mul_s02, mul_s03,

ggml/src/ggml-hexagon/htp/flash-attn-ops.c

@@ -755,40 +755,11 @@ static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) {
         const size_t m_start       = i * m_bytes_per_t;
         const size_t m_end         = hex_smin(m_start + m_bytes_per_t, col_vec_bytes);
 
-        if (factx->sinks) {
-            const float * sinks_data = factx->vtcm_sinks;
-            float *       m_vec      = (float *) factx->vtcm_m_vec;
-            const size_t  r_start    = l_start / sizeof(float);
-            const size_t  r_end      = l_end / sizeof(float);
-            const float   scale_factor = EXP_LOG2E_F;
-
-            const HVX_Vector v_scale = hvx_vec_splat_f32(scale_factor);
-
-            for (size_t r = r_start; r < r_end; r += 32) {
-                HVX_VectorAlias local_m;
-                for (size_t j = 0; j < 32; ++j) {
-                    size_t curr_r = r + j;
-                    if (curr_r < n_rows_g) {
-                        const size_t h_idx = fastmodulo(curr_r, G, &factx->div_G);
-                        const size_t head  = args->kv_head * G + h_idx;
-                        local_m.fp32[j] = sinks_data[head];
-                    } else {
-                        local_m.fp32[j] = HTP_FA_M_INITIAL_VAL;
-                    }
-                }
-                HVX_Vector v_scaled = HVX_OP_MUL_F32(local_m.v, v_scale);
-                *(HVX_Vector *) (m_vec + r) = v_scaled;
-            }
-            if (l_start < col_vec_bytes) {
-                hvx_splat_u8_a((char *) factx->vtcm_l_vec + l_start, 0, l_end - l_start);
-            }
-        } else {
-            if (l_start < col_vec_bytes) {
-                hvx_splat_u8_a((char *) factx->vtcm_l_vec + l_start, 0, l_end - l_start);
-            }
-            if (m_start < col_vec_bytes) {
-                hvx_splat_f32_a((char *) factx->vtcm_m_vec + m_start, HTP_FA_M_INITIAL_VAL, (m_end - m_start) / sizeof(float));
-            }
+        if (l_start < col_vec_bytes) {
+            hvx_splat_u8_a((char *) factx->vtcm_l_vec + l_start, 0, l_end - l_start);
+        }
+        if (m_start < col_vec_bytes) {
+            hvx_splat_f32_a((char *) factx->vtcm_m_vec + m_start, HTP_FA_M_INITIAL_VAL, (m_end - m_start) / sizeof(float));
         }
 
         // Zero the whole rescale region: vtcm_d_tiles[0], the optional vtcm_d_tiles[1]
@@ -1447,21 +1418,10 @@ static inline void fa_softmax_impl(
         HVX_Vector     v_rowsum_acc_f32_0 = Q6_V_lo_W(rowsum_acc_pair);
         HVX_Vector     v_rowsum_acc_f32_1 = Q6_V_hi_W(rowsum_acc_pair);
 
-        HVX_Vector v_l_curr0;
-        HVX_Vector v_l_curr1;
-        if (args->kv_start == 0 && factx->sinks != NULL) {
-            // First KV block with sinks: m_prev holds the seeded sink value (not -inf),
-            // so exp_m_diff = exp2(sink - m_curr) is the sink's contribution to the
-            // denominator. l_prev is 0 here, so add exp_m_diff directly instead of
-            // multiplying the (uninitialized) l_prev term.
-            v_l_curr0 = HVX_OP_ADD_F32(exp_m_diff0, v_rowsum_acc_f32_0);
-            v_l_curr1 = HVX_OP_ADD_F32(exp_m_diff1, v_rowsum_acc_f32_1);
-        } else {
-            HVX_Vector l_prev_v0 = factx->vtcm_l_vec[r_vec_idx * 2 + 0];
-            HVX_Vector l_prev_v1 = factx->vtcm_l_vec[r_vec_idx * 2 + 1];
-            v_l_curr0 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v0, exp_m_diff0), v_rowsum_acc_f32_0);
-            v_l_curr1 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v1, exp_m_diff1), v_rowsum_acc_f32_1);
-        }
+        HVX_Vector l_prev_v0 = factx->vtcm_l_vec[r_vec_idx * 2 + 0];
+        HVX_Vector l_prev_v1 = factx->vtcm_l_vec[r_vec_idx * 2 + 1];
+        HVX_Vector v_l_curr0 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v0, exp_m_diff0), v_rowsum_acc_f32_0);
+        HVX_Vector v_l_curr1 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v1, exp_m_diff1), v_rowsum_acc_f32_1);
 
         factx->vtcm_m_vec[r_vec_idx * 2 + 0] = v_m_curr0;
         factx->vtcm_m_vec[r_vec_idx * 2 + 1] = v_m_curr1;
@@ -1532,9 +1492,66 @@ static void fa_softmax_thread(unsigned int n, unsigned int i, void * data) {
     fa_softmax_impl(n, i, data, has_mask, mask_broadcast, is_g1, has_alibi, has_softcap);
 }
 
+static inline HVX_Vector hvx_fa_load_sinks(const float * sinks_data, size_t idx, size_t count) {
+    const HVX_Vector * v_src    = (const HVX_Vector *) sinks_data + (idx >> 5);
+    const size_t       lane_off = idx & 31;
+    const size_t       byte_off = lane_off * sizeof(float);
+    return (lane_off + count > 32)
+        ? Q6_V_valign_VVR(v_src[1], v_src[0], byte_off)
+        : Q6_V_vror_VR(v_src[0], byte_off);
+}
+
 static __attribute__((noinline)) void fa_build_d_diag_inv_l(struct hmx_fa_context * factx,
                                                             size_t                  n_row_tiles,
-                                                            size_t                  n_row_tiles_g_br) {
+                                                            size_t                  n_row_tiles_g_br,
+                                                            uint32_t                kv_head,
+                                                            size_t                  n_rows_g) {
+    (void) n_row_tiles_g_br;
+
+    // sinks: l += exp(sink - m * ln2) in f32, m and l are in the log2 domain
+    if (factx->sinks) {
+        const float *    sinks_data   = factx->vtcm_sinks;
+        const size_t     G            = factx->G;
+        const size_t     head_base    = (size_t) kv_head * G;
+        const HVX_Vector v_ln2        = Q6_V_vsplat_R(EXP_LOGN2);
+        const HVX_Vector v_head_sinks = (G <= 32) ? hvx_fa_load_sinks(sinks_data, head_base, G) : Q6_V_vzero();
+        HVX_Vector       v_sink       = Q6_V_vzero();
+        size_t           g_prev       = SIZE_MAX;
+
+        for (size_t i = 0; i < n_row_tiles; ++i) {
+            // the sink pattern of a tile only depends on (i * 32) % G, rebuild it when that changes
+            const size_t r0 = i * 32;
+            const size_t g0 = fastmodulo(r0, G, &factx->div_G);
+            if (g0 != g_prev) {
+                const size_t rem_g = G - g0;
+                if (G <= 32) {
+                    const HVX_Vector v_lo = Q6_V_vror_VR(v_head_sinks, g0 * sizeof(float));
+                    const HVX_Vector v_hi = Q6_V_vror_VR(v_head_sinks, 128 - rem_g * sizeof(float));
+                    v_sink = Q6_V_vmux_QVV(Q6_Q_vsetq2_R(rem_g * sizeof(float)), v_lo, v_hi);
+                    for (size_t len = G; len < 32; len <<= 1) {
+                        const HVX_VectorPred q_keep    = Q6_Q_vsetq_R(len * sizeof(float));
+                        const HVX_Vector     v_shifted = Q6_V_vror_VR(v_sink, 128 - len * sizeof(float));
+                        v_sink = Q6_V_vmux_QVV(q_keep, v_sink, v_shifted);
+                    }
+                } else if (rem_g >= 32) {
+                    v_sink = hvx_fa_load_sinks(sinks_data, head_base + g0, 32);
+                } else {
+                    const HVX_Vector v_lo = hvx_fa_load_sinks(sinks_data, head_base + g0, rem_g);
+                    const HVX_Vector v_hi = hvx_fa_load_sinks(sinks_data, head_base, 32 - rem_g);
+                    v_sink = Q6_V_vmux_QVV(Q6_Q_vsetq_R(rem_g * sizeof(float)), v_lo, Q6_V_vror_VR(v_hi, 128 - rem_g * sizeof(float)));
+                }
+                g_prev = g0;
+            }
+
+            HVX_Vector v_d = HVX_OP_SUB_F32(v_sink, HVX_OP_MUL_F32(factx->vtcm_m_vec[i], v_ln2));
+            if (r0 + 32 > n_rows_g) {
+                const HVX_VectorPred q_valid = Q6_Q_vsetq_R((n_rows_g - r0) * sizeof(float));
+                v_d = Q6_V_vmux_QVV(q_valid, v_d, Q6_V_vzero());
+            }
+            factx->vtcm_l_vec[i] = HVX_OP_ADD_F32(factx->vtcm_l_vec[i], hvx_vec_exp_f32(v_d));
+        }
+    }
+
     const HVX_Vector     v_offsets = *(const HVX_Vector *) d_tile_scatter_offsets;
     const HVX_VectorPred q_32_mask = Q6_Q_vsetq_R(32 * sizeof(__fp16));
     const HVX_Vector     one       = hvx_vec_splat_f32(1.0f);
@@ -2010,7 +2027,10 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
 
     // ======== Initialize HMX output scales ========
     hmx_init_column_scales(factx.vtcm_hmx_scales_id, Q6_V_vsplat_R(0x3c00)); // 1.0
-    hmx_init_column_scales(factx.vtcm_hmx_scales_qk, hvx_vec_splat_f16(factx.scale));
+    {
+        union { __fp16 f; uint16_t u; } sc = { .f = factx.scale };
+        hmx_init_column_scales(factx.vtcm_hmx_scales_qk, Q6_V_vsplat_R(sc.u));
+    }
 
     // ======== DMA setup ========
     dma_queue * const dma_q = ctx->dma[0];
@@ -2250,7 +2270,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
 
                         // Overlapped: run HVX build diag inv L while HMX is busy executing the update
                         htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start);
-                        fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br);
+                        fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br, kv_head, n_rows_g);
                         htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start);
                         hmx_queue_pop(hmx_q);
 
@@ -2355,7 +2375,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
                             if (kv_blk + 1 == factx.n_kv_blocks) {
                                 // Overlapped: run HVX build diag inv L while HMX is busy executing the update
                                 htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start);
-                                fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br);
+                                fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br, kv_head, n_rows_g);
                                 htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start);
                             }
                             hmx_queue_pop(ctx->hmx_queue);

ggml/src/ggml-sycl/common.cpp

@@ -115,7 +115,8 @@ void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type t
 #else
         ze_device_mem_alloc_desc_t alloc_desc = {ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, nullptr, 0, 0};
 #endif
-        ze_result_t r = zeMemAllocDevice(ze_ctx, &alloc_desc, size, 64, ze_dev, &ptr);
+        // request SYCL_BUFFER_ALIGNMENT so tensor offsets aligned to it stay aligned in absolute terms
+        ze_result_t r = zeMemAllocDevice(ze_ctx, &alloc_desc, size, SYCL_BUFFER_ALIGNMENT, ze_dev, &ptr);
         if (r == ZE_RESULT_SUCCESS && ptr) {
             ggml_sycl_memtrace_add(type, ptr, size);
             return ptr;

ggml/src/ggml-sycl/convert.cpp

@@ -212,20 +212,10 @@ template <typename dst_t>
 static void dequantize_row_q4_K_sycl(const void *vx, dst_t *y, const int64_t k,
                                      dpct::queue_ptr stream) {
     const int64_t nb = k / QK_K;
-    {
-        dpct::has_capability_or_fail(stream->get_device(),
-                                     {sycl::aspect::fp16});
-
-        stream->submit([&](sycl::handler &cgh) {
-            sycl::local_accessor<uint8_t, 1> scale_local_acc(sycl::range<1>(12), cgh);
-            cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *
-                                                   sycl::range<3>(1, 1, 32),
-                                               sycl::range<3>(1, 1, 32)),
-                             [=](sycl::nd_item<3> item_ct1) {
-                                 dequantize_block_q4_K(vx, y, get_pointer(scale_local_acc), item_ct1);
-                             });
-        });
-    }
+    dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 });
+    stream->parallel_for(sycl::nd_range<1>(sycl::range<1>((size_t) nb * SYCL_DEQK_WI_PER_BLOCK),
+                                           sycl::range<1>((size_t) SYCL_DEQK_WI_PER_BLOCK)),
+                         [=](sycl::nd_item<1> it) { dequantize_block_q4_K_wide(vx, y, nb, it); });
 }
 
 template <typename dst_t>
@@ -250,32 +240,10 @@ template <typename dst_t>
 static void dequantize_row_q5_K_sycl(const void *vx, dst_t *y, const int64_t k,
                                      dpct::queue_ptr stream) {
     const int64_t nb = k / QK_K;
-#if QK_K == 256
-    {
-        dpct::has_capability_or_fail(stream->get_device(),
-                                     {sycl::aspect::fp16});
-
-        stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *
-                                                   sycl::range<3>(1, 1, 64),
-                                               sycl::range<3>(1, 1, 64)),
-                             [=](sycl::nd_item<3> item_ct1) {
-                                 dequantize_block_q5_K(vx, y, item_ct1);
-                             });
-    }
-#else
-    {
-        dpct::has_capability_or_fail(stream->get_device(),
-                                     {sycl::aspect::fp16});
-
-        stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) *
-                                                   sycl::range<3>(1, 1, 32),
-                                               sycl::range<3>(1, 1, 32)),
-                             [=](sycl::nd_item<3> item_ct1) {
-                                 dequantize_block_q5_K(vx, y, item_ct1);
-                             });
-    }
-
-#endif
+    dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 });
+    stream->parallel_for(sycl::nd_range<1>(sycl::range<1>((size_t) nb * SYCL_DEQK_WI_PER_BLOCK),
+                                           sycl::range<1>((size_t) SYCL_DEQK_WI_PER_BLOCK)),
+                         [=](sycl::nd_item<1> it) { dequantize_block_q5_K_wide(vx, y, nb, it); });
 }
 
 template <typename dst_t>

ggml/src/ggml-sycl/cpy.cpp

@@ -1255,6 +1255,8 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co
     scope_op_debug_print scope_dbg_print(__func__, src1, /*num_src=*/0, debug_get_tensor_str("\tsrc0", src0));
     const int64_t ne = ggml_nelements(src0);
     GGML_ASSERT(ne == ggml_nelements(src1));
+    // the cpy kernels index in 32-bit
+    GGML_ASSERT(ne <= INT32_MAX);
 
     GGML_TENSOR_BINARY_OP_LOCALS01;
 

ggml/src/ggml-sycl/dequantize.hpp

@@ -1108,6 +1108,53 @@ static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8
 }
 #endif
 
+// Work-items that cover one 256-weight K-quant block: four 64-element chunks, 8 work-items
+// per chunk, 4 qs bytes per work-item.
+static constexpr int SYCL_DEQK_WI_PER_BLOCK = 32;
+
+template <typename dst_t>
+static inline void dequantize_q4_K_wide_one(dst_t * __restrict__ y, const uint8_t * __restrict__ qs,
+                                            const uint8_t * __restrict__ scales, const float dall,
+                                            const float dmin, const int tid) {
+    const int il = tid >> 3;        // 64-element chunk, 0..3
+    const int ir = (tid & 7) * 4;   // 4 qs bytes within the chunk
+
+    uint8_t sc, m;
+    get_scale_min_k4(2 * il + 0, scales, sc, m);
+    const float d1 = dall * sc;
+    const float m1 = dmin * m;
+    get_scale_min_k4(2 * il + 1, scales, sc, m);
+    const float d2 = dall * sc;
+    const float m2 = dmin * m;
+
+    const sycl::vec<uint8_t, 4> q = vec_aligned_load<uint8_t, 4>(qs + 32 * il + ir);
+
+    sycl::vec<dst_t, 4> lo, hi;
+#pragma unroll
+    for (int l = 0; l < 4; ++l) {
+        lo[l] = d1 * (q[l] & 0xF) - m1;
+        hi[l] = d2 * (q[l] >> 4)  - m2;
+    }
+    dst_t * yb = y + 64 * il + ir;
+    *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb +  0) = lo;
+    *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb + 32) = hi;
+}
+
+template <typename dst_t>
+static void dequantize_block_q4_K_wide(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb,
+                                       const sycl::nd_item<1> & it) {
+    static_assert(QK_K == 256, "wide K-quant dequant assumes QK_K == 256");
+    const int64_t g   = it.get_global_id(0);
+    const int64_t b   = g / SYCL_DEQK_WI_PER_BLOCK;
+    const int     tid = (int) (g % SYCL_DEQK_WI_PER_BLOCK);
+    if (b >= nb) {
+        return;
+    }
+    const block_q4_K * x = (const block_q4_K *) vx + b;
+    dequantize_q4_K_wide_one(yy + b * QK_K, x->qs, x->scales, (float) x->dm[0], (float) x->dm[1], tid);
+}
+
+
 template <typename dst_t>
 inline void dequantize_q4_K_common(dst_t * __restrict__ y, const uint8_t * __restrict__ qs_ptr, const float dall,
                                    const float dmin, uint8_t * __restrict__ scales_local, int il, int ir) {
@@ -1130,41 +1177,6 @@ inline void dequantize_q4_K_common(dst_t * __restrict__ y, const uint8_t * __res
     }
 }
 
-template<typename dst_t>
-static void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy,
-                                  uint8_t* scales_local, const sycl::nd_item<3> &item_ct1) {
-    const block_q4_K * x = (const block_q4_K *) vx;
-
-    const int64_t i = item_ct1.get_group(2);
-
-#if QK_K == 256
-    const int64_t tid = item_ct1.get_local_id(2);
-    const int64_t il  = tid / 8;
-    const int64_t ir  = tid % 8;
-
-    dst_t * y = yy + i * QK_K + 64 * il + 4 * ir;
-
-    const sycl::half2 dm = x[i].dm;
-    const float dall = dm[0];
-    const float dmin = dm[1];
-
-    if (tid < 12) {
-        scales_local[tid] = x[i].scales[tid];
-    }
-
-    item_ct1.barrier(sycl::access::fence_space::local_space);
-    dequantize_q4_K_common(y, x[i].qs, dall, dmin, scales_local, il, ir);
-#else
-    const int64_t tid = item_ct1.get_local_id(2);
-    const uint8_t * q = x[i].qs;
-    dst_t * y = yy + i*QK_K;
-    const float d = (float)x[i].dm[0];
-    const float m = (float)x[i].dm[1];
-    y[tid+ 0] = d * (x[i].scales[0] & 0xF) * (q[tid] & 0xF) - m * (x[i].scales[0] >> 4);
-    y[tid+32] = d * (x[i].scales[1] & 0xF) * (q[tid] >>  4) - m * (x[i].scales[1] >> 4);
-#endif
-}
-
 template <typename dst_t>
 static void dequantize_block_q4_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, uint8_t * scales_local,
                                           const sycl::nd_item<1> & item_ct1, int64_t nb) {
@@ -1195,52 +1207,51 @@ static void dequantize_block_q4_K_reorder(const void * __restrict__ vx, dst_t *
     dequantize_q4_K_common(y, qs_ptr, dall, dmin, scales_local, il, ir);
 }
 
-template<typename dst_t>
-static void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy,
-                                  const sycl::nd_item<3> &item_ct1) {
-    const block_q5_K * x = (const block_q5_K *) vx;
-
-    const int64_t i = item_ct1.get_group(2);
-
-#if QK_K == 256
-    // assume 64 threads - this is very slightly better than the one below
-    const int64_t tid = item_ct1.get_local_id(2);
-    const int64_t il  = tid/16;   // il is in 0...3
-    const int64_t ir  = tid%16;   // ir is in 0...15
-    const int64_t is  = 2*il;     // is is in 0...6
-
-    dst_t * y = yy + i*QK_K + 64*il + 2*ir;
-
-    const float dall = x[i].dm[0];
-    const float dmin = x[i].dm[1];
-
-    const uint8_t * ql = x[i].qs + 32*il + 2*ir;
-    const uint8_t * qh = x[i].qh + 2*ir;
+template <typename dst_t>
+static inline void dequantize_q5_K_wide_one(dst_t * __restrict__ y, const uint8_t * __restrict__ qs,
+                                            const uint8_t * __restrict__ qh_ptr,
+                                            const uint8_t * __restrict__ scales, const float dall,
+                                            const float dmin, const int tid) {
+    const int il = tid >> 3;        // 64-element chunk, 0..3
+    const int ir = (tid & 7) * 4;   // 4 qs bytes within the chunk; qh holds the 5th bit
 
     uint8_t sc, m;
-    get_scale_min_k4(is + 0, x[i].scales, sc, m);
-    const float d1 = dall * sc; const float m1 = dmin * m;
-    get_scale_min_k4(is + 1, x[i].scales, sc, m);
-    const float d2 = dall * sc; const float m2 = dmin * m;
+    get_scale_min_k4(2 * il + 0, scales, sc, m);
+    const float d1 = dall * sc;
+    const float m1 = dmin * m;
+    get_scale_min_k4(2 * il + 1, scales, sc, m);
+    const float d2 = dall * sc;
+    const float m2 = dmin * m;
 
-    uint8_t   hm  = 1 << (2*il);
-    y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1;
-    y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1;
-    hm <<= 1;
-    y[32] = d2 * ((ql[ 0] >>  4) + (qh[ 0] & hm ? 16 : 0)) - m2;
-    y[33] = d2 * ((ql[ 1] >>  4) + (qh[ 1] & hm ? 16 : 0)) - m2;
-#else
-    const int64_t tid = item_ct1.get_local_id(2);
-    const uint8_t q = x[i].qs[tid];
-    const int64_t im = tid/8;  // 0...3
-    const int64_t in = tid%8;  // 0...7
-    const int64_t is = tid/16; // 0 or 1
-    const uint8_t h = x[i].qh[in] >> im;
-    const float d = x[i].d;
-    dst_t * y = yy + i*QK_K + tid;
-    y[ 0] = d * x[i].scales[is+0] * ((q & 0xF) - ((h >> 0) & 1 ? 0 : 16));
-    y[32] = d * x[i].scales[is+2] * ((q >>  4) - ((h >> 4) & 1 ? 0 : 16));
-#endif
+    const sycl::vec<uint8_t, 4> q = vec_aligned_load<uint8_t, 4>(qs + 32 * il + ir);
+    const sycl::vec<uint8_t, 4> h = vec_aligned_load<uint8_t, 4>(qh_ptr + ir);
+
+    const uint8_t hm1 = 1 << (2 * il);
+    const uint8_t hm2 = hm1 << 1;
+
+    sycl::vec<dst_t, 4> lo, hi;
+#pragma unroll
+    for (int l = 0; l < 4; ++l) {
+        lo[l] = d1 * ((q[l] & 0xF) + ((h[l] & hm1) ? 16 : 0)) - m1;
+        hi[l] = d2 * ((q[l] >>  4) + ((h[l] & hm2) ? 16 : 0)) - m2;
+    }
+    dst_t * yb = y + 64 * il + ir;
+    *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb +  0) = lo;
+    *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb + 32) = hi;
+}
+
+template <typename dst_t>
+static void dequantize_block_q5_K_wide(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb,
+                                       const sycl::nd_item<1> & it) {
+    static_assert(QK_K == 256, "wide K-quant dequant assumes QK_K == 256");
+    const int64_t g   = it.get_global_id(0);
+    const int64_t b   = g / SYCL_DEQK_WI_PER_BLOCK;
+    const int     tid = (int) (g % SYCL_DEQK_WI_PER_BLOCK);
+    if (b >= nb) {
+        return;
+    }
+    const block_q5_K * x = (const block_q5_K *) vx + b;
+    dequantize_q5_K_wide_one(yy + b * QK_K, x->qs, x->qh, x->scales, (float) x->dm[0], (float) x->dm[1], tid);
 }
 
 template <typename dst_t>

ggml/src/ggml-sycl/ggml-sycl.cpp

@@ -899,6 +899,10 @@ static bool ggml_sycl_is_l0_discrete_gpu(int device) {
 static void memcpy_host_forward(sycl::queue &q_dst, sycl::queue &q_src, void *ptr_dst,
                          const void *ptr_src, size_t size) {
     char *host_buf = (char *)malloc(size);
+    if (host_buf == nullptr) {
+        GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory for host staging\n", __func__, size);
+        GGML_ABORT("fatal error");
+    }
     q_src.memcpy(host_buf, (const char *)ptr_src, size).wait();
     q_dst.memcpy((char *)ptr_dst, host_buf, size).wait();
     free(host_buf);
@@ -1177,12 +1181,13 @@ bool is_bmg_g31_arch(int device) {
 }
 
 static size_t ggml_backend_sycl_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
-    size_t max_alloc_size = dpct::get_current_device().get_max_mem_alloc_size();
+    ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context;
+    int device = ctx->device;
+    // use the buft device, the current device may differ on multi-GPU systems
+    size_t max_alloc_size = dpct::get_device(device).get_max_mem_alloc_size();
     if (g_ggml_sycl_host_pinned_mem_2g) {
         return std::min(max_alloc_size, (size_t) 2LL*1024*1024*1024);
     } else {
-        ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context;
-        int device = ctx->device;
         if(is_bmg_g31_arch(device)) {
             //Todo, it's workaround for BMG-G31, which has a known issue with large allocations.
             //The max alloc size is reduced to 60% of the reported max alloc size.
@@ -1398,7 +1403,12 @@ ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer,
     ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};
 
     ctx->tensor_extras.push_back(extra);
-    ctx->streams.push_back(&(dpct::get_current_device().default_queue()));
+    // consumers index the queues by device id, so size the vector once with one queue per device
+    if (ctx->streams.empty()) {
+        for (int i = 0; i < ggml_sycl_info().device_count; ++i) {
+            ctx->streams.push_back(&(dpct::get_device(i).default_queue()));
+        }
+    }
 
     for (int i = 0; i < ggml_sycl_info().device_count; ++i) {
         int64_t row_low, row_high;
@@ -2089,8 +2099,6 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {
     queue_ptr qptr;
     int       device;
 
-    inline static int counter{ 0 };
-
     struct ggml_sycl_buffer {
         void * ptr  = nullptr;
         size_t size = 0;
@@ -2100,6 +2108,7 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {
     static constexpr int          MAX_POOL_SIZE{ 64 };
     std::vector<ggml_sycl_buffer> buffer_pool = std::vector<ggml_sycl_buffer>(MAX_POOL_SIZE);
     size_t                        pool_size   = 0;
+    int                           counter     = 0;
 
     explicit ggml_sycl_pool_host(queue_ptr qptr_, int device_) : qptr(qptr_), device(device_) {}
 
@@ -2113,41 +2122,55 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {
                 b.size = 0;
             }
         }
-        counter = 0;
     }
 
     void * alloc(size_t size, size_t * actual_size) override {
         if (counter == MAX_POOL_SIZE) {
-            ggml_sycl_buffer b               = buffer_pool[0];
-            void *           ptr             = b.ptr;
-            *actual_size                     = b.size;
-            counter                          = 1;
-            return ptr;
-        }
-        ggml_sycl_buffer & b = buffer_pool[counter];
-
-        if (b.ptr == nullptr) {
-            void * ptr;
-
+            // the pool wrapped around, no slot is reserved for this allocation
+            void * ptr = nullptr;
             SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr)));
             if (!ptr) {
                 GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size);
                 return nullptr;
             }
-            pool_size += size;
+            pool_size    += size;
             *actual_size = size;
-            counter      = counter + 1;
+            counter      = 0;
             return ptr;
-        } else {
-            ++counter;
-            b.size = size;
-            return b.ptr;
         }
+        ggml_sycl_buffer & b = buffer_pool[counter];
+        ++counter;
+
+        if (b.ptr != nullptr) {
+            if (b.size >= size) {
+                // hand out the cached buffer, free() puts it back into the pool
+                void * ptr = b.ptr;
+                *actual_size = b.size;
+                b.ptr = nullptr;
+                b.size = 0;
+                return ptr;
+            }
+            // the cached buffer is too small, replace it
+            SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(b.ptr, *qptr)));
+            pool_size -= b.size;
+            b.ptr = nullptr;
+            b.size = 0;
+        }
+
+        void * ptr;
+        SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr)));
+        if (!ptr) {
+            GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size);
+            return nullptr;
+        }
+        pool_size    += size;
+        *actual_size = size;
+        return ptr;
     }
 
     void free(void * ptr, size_t size) override {
         // if the pool is not completed add the pointer to it in place of the first nullptr found.
-        // Otherwise do nothing, pointers will be freed once the pool is deallocated.
+        // Otherwise free the buffer, all slots are cached.
         for (int i = 0; i < MAX_POOL_SIZE; ++i) {
             ggml_sycl_buffer & b = buffer_pool[i];
             if (b.ptr == nullptr) {
@@ -2156,6 +2179,8 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {
                 return;
             }
         }
+        SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *qptr)));
+        pool_size -= size;
     }
 };
 
@@ -3154,8 +3179,8 @@ inline void ggml_sycl_op_mul_mat_sycl(
             to_fp16_sycl(src1_ddf_i, src1_as_f16.get(), ne, stream);
         }
         const sycl::half *src1_ptr = src1->type == GGML_TYPE_F16
-                ? (const sycl::half *)src1->data + src1_padded_row_size
-                                         : src1_as_f16.get();
+                ? (const sycl::half *)src1_ddf_i
+                : src1_as_f16.get();
 
         ggml_sycl_pool_alloc<sycl::half> src0_as_f16(ctx.pool());
         if (src0->type != GGML_TYPE_F16) {
@@ -3252,6 +3277,8 @@ inline void ggml_sycl_op_sum(ggml_backend_sycl_context & ctx, ggml_tensor *dst)
     float *       dst_dd  = static_cast<float *>(dst->data);
 
     const int64_t ne = ggml_nelements(dst->src[0]);
+    // the kernel indexes in 32-bit
+    GGML_ASSERT(ne <= INT32_MAX);
 
     sum_rows_f32_sycl(src0_dd, dst_dd, ne, 1, main_stream);
 }
@@ -3266,6 +3293,8 @@ inline void ggml_sycl_op_sum_rows(ggml_backend_sycl_context & ctx, ggml_tensor *
 
     const int64_t ncols = dst->src[0]->ne[0];
     const int64_t nrows = ggml_nrows(dst->src[0]);
+    // the kernel indexes in 32-bit
+    GGML_ASSERT(ncols * nrows <= INT32_MAX);
 
     sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream);
 }
@@ -3282,6 +3311,8 @@ inline void ggml_sycl_op_mean(ggml_backend_sycl_context & ctx, ggml_tensor * dst
 
     const int64_t ncols = dst->src[0]->ne[0];
     const int64_t nrows = ggml_nrows(dst->src[0]);
+    // the kernel indexes in 32-bit
+    GGML_ASSERT(ncols * nrows <= INT32_MAX);
 
     sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream);
 
@@ -3351,6 +3382,8 @@ inline void ggml_sycl_op_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * d
 
     const int64_t ncols = dst->src[0]->ne[0];
     const int64_t nrows = ggml_nrows(dst->src[0]);
+    // the kernel indexes in 32-bit
+    GGML_ASSERT(ncols * nrows <= INT32_MAX);
 
     argmax_f32_i32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream);
 }
@@ -3366,6 +3399,8 @@ inline void ggml_sycl_op_diag_mask_inf(ggml_backend_sycl_context & ctx, ggml_ten
     const int64_t ne00 = dst->src[0]->ne[0];
     const int64_t ne01 = dst->src[0]->ne[1];
     const int nrows0 = ggml_nrows(dst->src[0]);
+    // the kernel indexes in 32-bit
+    GGML_ASSERT(ne00 * nrows0 <= INT32_MAX);
 
     const int n_past = ((int32_t *) dst->op_params)[0];
 
@@ -3444,6 +3479,9 @@ inline void ggml_sycl_op_scale(ggml_backend_sycl_context & ctx, ggml_tensor * ds
     memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));
     memcpy(&bias,  (float *) dst->op_params + 1, sizeof(float));
 
+    // the kernel indexes in 32-bit
+    GGML_ASSERT(ggml_nelements(dst->src[0]) <= INT32_MAX);
+
     scale_f32_sycl(src0_dd, dst_dd, scale, bias, ggml_nelements(dst->src[0]), main_stream);
     /*
     DPCT1010:87: SYCL uses exceptions to report errors and does not use the
@@ -3524,7 +3562,6 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten
     const size_t q8_1_bs = QK8_1;
 
     ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;
-    ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra;
 
     const bool src0_is_contiguous = ggml_is_contiguous(src0);
     const bool src1_is_contiguous = ggml_is_contiguous(src1);
@@ -3606,7 +3643,8 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten
         queue_ptr stream = ctx.stream(i, 0);
 
         if (src0_is_contiguous) {
-            dev[i].src0_dd = (char *) src0->data;
+            // a split tensor keeps its rows in the per-device slices of the tensor extra
+            dev[i].src0_dd = split ? (char *) src0_extra->data_device[i] : (char *) src0->data;
         } else {
             dev[i].src0_dd = dev[i].src0_dd_alloc.alloc(ctx.pool(i), ggml_nbytes(src0));
         }
@@ -3702,7 +3740,8 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten
                                                              src1_ncols * src1_padded_col_size * q8_1_ts / q8_1_bs)
                                                     .wait()));
                         } else {
-                            const char * src1_ddf_i_source = (const char *) src1_extra->data_device[ctx.device] +
+                            // src1 is not split, on the main device it lives at dev[ctx.device].src1_ddf
+                            const char * src1_ddf_i_source = (const char *) dev[ctx.device].src1_ddf +
                                 (i0 * ne11 + src1_col_0) * ne10 * ggml_type_size(src1->type);
 
                             SYCL_CHECK(
@@ -5269,6 +5308,7 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, gg
                                              (int) ne11,
                                              /*stride_col_y=*/src1_padded_cols / QK8_1,
                                              /*stride_col_dst=*/(int) glu->ne[0], stream);
+    GGML_UNUSED(gate);
 }
 
 // Fused dense-FFN mat-vec for the {mul_mat(gate), mul_mat(up), GLU} subgraph at node_idx.
@@ -6860,7 +6900,9 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
                 case GGML_GLU_OP_GEGLU_ERF:
                 case GGML_GLU_OP_GEGLU_QUICK:
                 case GGML_GLU_OP_SWIGLU_CLAMP:
-                    return ggml_is_contiguous_1(op->src[0]);
+                    // the fused GLU kernels only have f32/f16 instantiations
+                    return ggml_is_contiguous_1(op->src[0]) &&
+                           (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16);
                 default:
                     return false;
             }
@@ -7123,7 +7165,8 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
         case GGML_OP_RMS_NORM_BACK:
             return ggml_is_contiguous(op->src[0]);
         case GGML_OP_SCALE:
-            return true;
+            // the scale kernel is f32 only
+            return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32;
         case GGML_OP_CONT:
             return true;
         case GGML_OP_TRI:
@@ -7156,12 +7199,19 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
 #endif
                    ) &&
                    op->src[0]->type == op->type;
-        case GGML_OP_CONV_3D:
+        case GGML_OP_CONV_3D: {
+            const int32_t * opts = (const int32_t *) op->op_params;
+            // zero kernel volume, channels or output makes the im2col GEMM degenerate (k == 0 or m == 0)
+            // and MKL BLAS rejects it; let those fall back to the CPU backend
+            const int64_t knl_n_total = op->src[0]->ne[0] * op->src[0]->ne[1] * op->src[0]->ne[2] * opts[9];
+            const int64_t patch_total = op->ne[0] * op->ne[1] * op->ne[2] * opts[10];
             return op->type == GGML_TYPE_F32 &&
                    (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
                    op->src[1]->type == GGML_TYPE_F32 &&
                    ggml_is_contiguous(op->src[0]) &&
-                   ggml_is_contiguous(op->src[1]);
+                   ggml_is_contiguous(op->src[1]) &&
+                   knl_n_total > 0 && patch_total > 0;
+        }
         case GGML_OP_SUM:
         case GGML_OP_SUM_ROWS:
         case GGML_OP_MEAN:

ggml/src/ggml-sycl/roll.cpp

@@ -73,6 +73,8 @@ void ggml_sycl_roll(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
     const ggml_tensor *src = dst->src[0];
     GGML_ASSERT(src && src->type == GGML_TYPE_F32);
 
+    // the kernel indexes in 32-bit
+    GGML_ASSERT(dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3] <= INT32_MAX);
     const int ne0 = (int) dst->ne[0];
     const int ne1 = (int) dst->ne[1];
     const int ne2 = (int) dst->ne[2];

gguf-py/gguf/constants.py

@@ -5950,6 +5950,11 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_GATE_EXP,
         MODEL_TENSOR.FFN_DOWN_EXP,
         MODEL_TENSOR.FFN_UP_EXP,
+        # NextN/MTP tensors
+        MODEL_TENSOR.NEXTN_EH_PROJ,
+        MODEL_TENSOR.NEXTN_ENORM,
+        MODEL_TENSOR.NEXTN_HNORM,
+        MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
     ],
     MODEL_ARCH.NANBEIGE: [
         MODEL_TENSOR.TOKEN_EMBD,
@@ -6278,6 +6283,7 @@ class DecisionType:
     VLLM_D3 = "vllm-d3"  # same as pplx-decider, non-causal attention
     LFM2_D1 = "lfm2-d1"  # same as openjev, the labels depend on the question type
     LFM2_D1_OMNI = "lfm2-d1-omni"  # same head as laya on a bidirectional LFM2 trunk, other prompt layout
+    GRANITE_SWITCH = "granite-switch"  # same as openjev, run by the decision aLoRA of granite switch
 
 
 class VisionProjectorType:

gguf-py/gguf/quants.py

@@ -654,6 +654,59 @@ class TQ2_0(__Quant, qtype=GGMLQuantizationType.TQ2_0):
         return (d * qs.astype(np.float32))
 
 
+class Q1_0(__Quant, qtype=GGMLQuantizationType.Q1_0):
+    @classmethod
+    def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+        # sequential sum like the C reference, np.sum uses pairwise summation and can round differently
+        d = np.cumsum(abs(blocks), axis=-1, dtype=np.float32)[..., -1:] / np.float32(cls.block_size)
+        qs = np.packbits(blocks >= 0, axis=-1, bitorder="little")
+
+        d = d.astype(np.float16).view(np.uint8)
+
+        return np.concatenate([d, qs], axis=-1)
+
+    @classmethod
+    def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+        d, qs = np.hsplit(blocks, [2])
+
+        d = d.view(np.float16).astype(np.float32)
+
+        qs = np.unpackbits(qs, axis=-1, bitorder="little")
+
+        return np.where(qs != 0, d, -d)
+
+
+class Q2_0(__Quant, qtype=GGMLQuantizationType.Q2_0):
+    @classmethod
+    def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+        n_blocks = blocks.shape[0]
+
+        d = abs(blocks).max(axis=-1, keepdims=True)
+        with np.errstate(divide="ignore"):
+            id = np.where(d == 0, 0, 1 / d)
+        qs = (np_roundf(blocks * id) + 1).clip(0, 3).astype(np.uint8)
+
+        qs = qs.reshape((n_blocks, -1, 4)) << np.array([0, 2, 4, 6], dtype=np.uint8)
+        qs = qs[..., 0] | qs[..., 1] | qs[..., 2] | qs[..., 3]
+
+        d = d.astype(np.float16).view(np.uint8)
+
+        return np.concatenate([d, qs], axis=-1)
+
+    @classmethod
+    def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:
+        n_blocks = blocks.shape[0]
+
+        d, qs = np.hsplit(blocks, [2])
+
+        d = d.view(np.float16).astype(np.float32)
+
+        qs = qs.reshape((n_blocks, -1, 1)) >> np.array([0, 2, 4, 6], dtype=np.uint8)
+        qs = (qs & 0x03).reshape((n_blocks, -1)).astype(np.int8) - np.int8(1)
+
+        return (d * qs.astype(np.float32))
+
+
 class MXFP4(__Quant, qtype=GGMLQuantizationType.MXFP4):
     # e2m1 values (doubled)
     # ref: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf

gguf-py/gguf/scripts/gguf_convert_endian.py

@@ -19,71 +19,27 @@ import gguf
 logger = logging.getLogger("gguf-convert-endian")
 
 
-def byteswap_noop(tensor, block_offs):
-    # this function is used when byteswapping is not needed
-    pass
-
-
-def byteswap_q4_0(tensor, block_offs):
-    # Each block_q4_0 consists of an f16 delta (scaling factor) followed by 16 int8 quantizations.
-
-    # Byte-Swap f16 sized delta field
-    delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)
-    delta.byteswap(inplace=True)
-
-
-def byteswap_q8_0(tensor, block_offs):
-    # Each block_q8_0 consists of an f16 delta (scaling factor) followed by 32 int8 quantizations.
-
-    # Byte-Swap f16 sized delta field
-    delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)
-    delta.byteswap(inplace=True)
-
-
-def byteswap_q4_k(tensor, block_offs):
-    # Each block_q4_k consists of 2 f16 values followed by 140 int8 values.
-
-    # Byte-Swap f16 sized fields
-    delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)
-    delta.byteswap(inplace=True)
-
-    delta = tensor.data[block_offs + 2:block_offs + 4].view(dtype=np.uint16)
-    delta.byteswap(inplace=True)
-
-
-def byteswap_q6_k(tensor, block_offs):
-    # Each block_q6_k consists of 208 int8 values followed by 1 f16 value.
-
-    # Byte-Swap f16 sized field
-    delta = tensor.data[block_offs + 208:block_offs + 210].view(dtype=np.uint16)
-    delta.byteswap(inplace=True)
-
-
-def byteswap_q1_0(tensor, block_offs):
-    # Each block_q1_0 consists of an f16 delta followed by 16 int8 quantizations.
-
-    # Byte-Swap f16 sized delta field
-    delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)
-    delta.byteswap(inplace=True)
-
-
-def byteswap_tq2_0(tensor, block_offs):
-    # Each block_tq2_0 consists of 64 int8 values followed by 1 f16 value.
-
-    # Byte-Swap f16 sized field
-    delta = tensor.data[block_offs + 64:block_offs + 66].view(dtype=np.uint16)
-    delta.byteswap(inplace=True)
-
-
-byteswap_tensors = {
-    gguf.GGMLQuantizationType.Q1_0:  byteswap_q1_0,
-    gguf.GGMLQuantizationType.Q4_0:  byteswap_q4_0,
-    gguf.GGMLQuantizationType.Q8_0:  byteswap_q8_0,
-    gguf.GGMLQuantizationType.Q4_K:  byteswap_q4_k,
-    gguf.GGMLQuantizationType.Q6_K:  byteswap_q6_k,
-    gguf.GGMLQuantizationType.TQ2_0: byteswap_tq2_0,
-    gguf.GGMLQuantizationType.MXFP4: byteswap_noop,
-    gguf.GGMLQuantizationType.NVFP4: byteswap_noop,
+# byte offset and type of the multi-byte fields in each block, all other fields are single bytes
+# use the width ggml reads the field with, e.g. Q5_0 qh is uint8_t[4] but read as uint32_t
+byteswap_tensors: dict[gguf.GGMLQuantizationType, tuple[tuple[int, type[np.unsignedinteger]], ...]] = {
+    gguf.GGMLQuantizationType.Q1_0:   ((0,   np.uint16),),                                 # d
+    gguf.GGMLQuantizationType.Q2_0:   ((0,   np.uint16),),                                 # d
+    gguf.GGMLQuantizationType.Q4_0:   ((0,   np.uint16),),                                 # d
+    gguf.GGMLQuantizationType.Q4_1:   ((0,   np.uint16), (2, np.uint16)),                  # d, m
+    gguf.GGMLQuantizationType.Q5_0:   ((0,   np.uint16), (2, np.uint32)),                  # d, qh
+    gguf.GGMLQuantizationType.Q5_1:   ((0,   np.uint16), (2, np.uint16), (4, np.uint32)),  # d, m, qh
+    gguf.GGMLQuantizationType.Q8_0:   ((0,   np.uint16),),                                 # d
+    gguf.GGMLQuantizationType.Q2_K:   ((80,  np.uint16), (82, np.uint16)),                 # d, dmin
+    gguf.GGMLQuantizationType.Q3_K:   ((108, np.uint16),),                                 # d
+    gguf.GGMLQuantizationType.Q4_K:   ((0,   np.uint16), (2, np.uint16)),                  # d, dmin
+    gguf.GGMLQuantizationType.Q5_K:   ((0,   np.uint16), (2, np.uint16)),                  # d, dmin
+    gguf.GGMLQuantizationType.Q6_K:   ((208, np.uint16),),                                 # d
+    gguf.GGMLQuantizationType.IQ4_NL: ((0,   np.uint16),),                                 # d
+    gguf.GGMLQuantizationType.IQ4_XS: ((0,   np.uint16), (2, np.uint16)),                  # d, scales_h
+    gguf.GGMLQuantizationType.TQ1_0:  ((52,  np.uint16),),                                 # d
+    gguf.GGMLQuantizationType.TQ2_0:  ((64,  np.uint16),),                                 # d
+    gguf.GGMLQuantizationType.MXFP4:  (),                                                  # noop
+    gguf.GGMLQuantizationType.NVFP4:  (),                                                  # noop
 }
 
 
@@ -140,28 +96,12 @@ def convert_byteorder(reader: gguf.GGUFReader, args: argparse.Namespace) -> None
 
         # Byte-swap tensor data if necessary
         if tensor.tensor_type in byteswap_tensors:
-            # first flatten structure
-            oldshape = tensor.data.shape
-            newshape = 1
-            for i in tensor.data.shape:
-                newshape *= i
-
-            tensor.data.resize(newshape)
-
-            block_size    = gguf.constants.GGML_QUANT_SIZES[tensor.tensor_type][1]
-            byteswap_func = byteswap_tensors[tensor.tensor_type]
-
-            n_blocks = len(tensor.data) // block_size
-            for block_num in (inner_pbar := tqdm(range(n_blocks), desc="Byte-swapping Blocks", leave=False)):
-                block_offs = block_num * block_size
-
-                byteswap_func(tensor, block_offs)
-
-                if block_num % 100000 == 0:
-                    inner_pbar.set_description(f"Byte-swapping Blocks [{(n_blocks - block_num) // n_blocks}]")
-
-            # restore old shape in case it's ever used
-            tensor.data.resize(oldshape)
+            # one row per block, then swap each field of all blocks at once
+            type_size = gguf.constants.GGML_QUANT_SIZES[tensor.tensor_type][1]
+            blocks = tensor.data.reshape(-1, type_size, copy=False)
+            for offs, dtype in byteswap_tensors[tensor.tensor_type]:
+                size = np.dtype(dtype).itemsize
+                blocks[:, offs:offs + size].view(dtype=dtype)[:, 0].byteswap(inplace=True)
         elif tensor.tensor_type == gguf.GGMLQuantizationType.BF16:
             # Special case for BF16
             # It is 2-bytes data, but by default view loads it as 1-byte data.
@@ -173,6 +113,7 @@ def convert_byteorder(reader: gguf.GGUFReader, args: argparse.Namespace) -> None
 
         pbar.set_description(log_message)
 
+    reader.data.flush()
     logger.info("* Completion")
 
 

gguf-py/tests/test_quants.py

@@ -69,6 +69,7 @@ class GGMLQuants:
             "tq1_0", "tq2_0",
             "mxfp4",
             "nvfp4",
+            "q1_0", "q2_0",
             "iq2_xxs", "iq2_xs", "iq2_s", "iq3_xxs", "iq3_s", "iq1_s", "iq1_m",
             "iq4_nl", "iq4_xs",
         ):

skills/code-review/SKILL.md

@@ -214,7 +214,7 @@ Adjacent data-shape rule: empty string is not a valid "unset" initial state - us
 - Only add tests that bring meaningful results. Too-trivial tests just bloat the suite and CI.
 - No time-sensitive tests (timing thresholds, sleeps, races against wall-clock); they are flaky on CI.
 - Think twice about tests that significantly increase CI run time (expensive computation, large inputs, or long sleep/wait delays) or download large amounts of data from the internet (big models, datasets). Flag them and ask whether a smaller model/input or an existing fixture would do.
-- Server: small tests of internal components (helpers, `server_tokens`, queues, format converters, ...) belong in `tests/test-server-component.cpp`. The end-to-end pytest suite in `tools/server/tests` is costly in CI time; only add to it when the test requires end-to-end behavior with a real model.
+- Server: small tests of internal components (helpers, `engine_tokens`, queues, format converters, ...) belong in `tests/test-server-component.cpp`. The end-to-end pytest suite in `tools/server/tests` is costly in CI time; only add to it when the test requires end-to-end behavior with a real model.
 
 ## General (always)
 

src/models/mellum.cpp

@@ -29,9 +29,13 @@ void llama_model_mellum::load_arch_hparams(llama_model_loader & ml) {
     }
 }
 
-void llama_model_mellum::load_arch_tensors(llama_model_loader &) {
+void llama_model_mellum::load_arch_tensors(llama_model_loader & ml) {
     LLAMA_LOAD_LOCALS;
 
+    const auto nf          = nextn_flags(ml);
+    const int  trunk_flags = nf.trunk;
+    const int  mtp_flags   = nf.mtp;
+
     tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
 
     // output
@@ -45,31 +49,56 @@ void llama_model_mellum::load_arch_tensors(llama_model_loader &) {
         throw std::runtime_error("n_expert_used must be > 0 for Mellum");
     }
 
-    for (int i = 0; i < n_layer; ++i) {
+    auto load_block = [&](int i, int flags) {
         auto & layer = layers[i];
 
-        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags);
 
-        create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, 0);
-        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
+        create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_gqa, n_embd_gqa, flags);
+        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, flags);
 
-        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
-        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
+        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, flags);
+        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, flags);
 
-        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags);
 
-        layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
+        layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, flags);
 
         const int64_t n_ff_exp = hparams.n_ff_exp() ? hparams.n_ff_exp() : n_ff / n_expert_used;
 
-        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {  n_embd, n_ff_exp, n_expert}, 0);
-        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp,   n_embd, n_expert}, 0);
-        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {  n_embd, n_ff_exp, n_expert}, 0);
+        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {  n_embd, n_ff_exp, n_expert}, flags);
+        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp,   n_embd, n_expert}, flags);
+        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {  n_embd, n_ff_exp, n_expert}, flags);
+    };
+
+    for (int i = 0; i < n_layer; ++i) {
+        load_block(i, trunk_flags);
+    }
+
+    // MTP block is a regular Mellum decoder block plus the NextN projection and norms
+    for (int i = n_layer; i < n_layer_all; ++i) {
+        auto & layer = layers[i];
+
+        load_block(i, mtp_flags);
+
+        layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ,          "weight", i), {2 * n_embd, n_embd}, mtp_flags);
+        layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM,            "weight", i), {n_embd},             mtp_flags);
+        layer.nextn.hnorm            = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM,            "weight", i), {n_embd},             mtp_flags);
+        layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), {n_embd},             mtp_flags);
     }
 }
 
 std::unique_ptr<llm_graph_context> llama_model_mellum::build_arch_graph(const llm_graph_params & params) const {
-    if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) {
+    const bool iswa = hparams.swa_type == LLAMA_SWA_TYPE_STANDARD;
+
+    if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
+        if (iswa) {
+            return std::make_unique<graph_mtp<true>>(*this, params);
+        }
+        return std::make_unique<graph_mtp<false>>(*this, params);
+    }
+
+    if (iswa) {
         return std::make_unique<graph<true>>(*this, params);
     }
     return std::make_unique<graph<false>>(*this, params);
@@ -151,7 +180,7 @@ llama_model_mellum::graph<iswa>::graph(const llama_model & model, const llm_grap
                     model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
                     Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
         }
-        if (il == n_layer - 1 && inp_out_ids) {
+        if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {
             cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);
             inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
         }
@@ -198,6 +227,14 @@ llama_model_mellum::graph<iswa>::graph(const llama_model & model, const llm_grap
             model.output_norm, nullptr,
             LLM_NORM_RMS, -1);
 
+    // post-norm hidden state feeds both the LM head and the MTP head
+    cb(cur, "h_nextn", -1);
+    res->t_h_nextn = cur;
+
+    if (crop_after_nextn(inp_out_ids)) {
+        cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+    }
+
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
@@ -208,5 +245,162 @@ llama_model_mellum::graph<iswa>::graph(const llama_model & model, const llm_grap
     res->t_logits = cur;
 }
 
+template <bool iswa>
+llama_model_mellum::graph_mtp<iswa>::graph_mtp(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
+    GGML_ASSERT(hparams.n_layer_nextn == 1 && "MELLUM MTP currently only supports a single MTP block");
+
+    const int64_t n_embd_head = hparams.n_embd_head_v();
+
+    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+    GGML_ASSERT(n_embd_head == n_rot);
+
+    const int il = hparams.n_layer();
+    const auto & layer = model.layers[il];
+
+    GGML_ASSERT(layer.nextn.eh_proj          && "MTP block missing nextn.eh_proj");
+    GGML_ASSERT(layer.nextn.enorm            && "MTP block missing nextn.enorm");
+    GGML_ASSERT(layer.nextn.hnorm            && "MTP block missing nextn.hnorm");
+    GGML_ASSERT(layer.nextn.shared_head_norm && "MTP block missing nextn.shared_head_norm");
+
+    // TODO: extract in a common llm_graph_context::build_inp_embd_h()
+    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
+
+    inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
+    ggml_set_input(inp->tokens);
+
+    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
+    ggml_set_input(inp->embd);
+
+    inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
+    ggml_set_input(inp->h);
+    ggml_set_name(inp->h, "mtp_h_input");
+
+    // TODO: make static using `ggml_build_forward_select()`
+    //       see llm_graph_context::build_inp_embd() for reference
+    ggml_tensor * tok_embd;
+    ASSERT_EMBD_OR_TOKEN(ubatch);
+    if (ubatch.token) {
+        tok_embd = ggml_get_rows(ctx0, model.tok_embd, inp->tokens);
+    } else {
+        tok_embd = inp->embd;
+    }
+    cb(tok_embd, "mtp_tok_embd", il);
+
+    ggml_tensor * h_embd = inp->h;
+
+    res->add_input(std::move(inp));
+
+    ggml_tensor * inp_pos     = build_inp_pos();
+    ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+    using inp_attn_type = std::conditional_t<iswa, llm_graph_input_attn_kv_iswa, llm_graph_input_attn_kv>;
+    inp_attn_type * inp_attn = nullptr;
+
+    if constexpr (iswa) {
+        inp_attn = build_attn_inp_kv_iswa();
+    } else {
+        inp_attn = build_attn_inp_kv();
+    }
+
+    ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il);
+    cb(e_norm, "mtp_enorm", il);
+
+    ggml_tensor * h_norm = build_norm(h_embd, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);
+    cb(h_norm, "mtp_hnorm", il);
+
+    ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, 0);
+    cb(concat, "mtp_concat", il);
+
+    ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s);
+    cb(cur, "mtp_eh_proj", il);
+
+    ggml_tensor * inpSA = cur;
+
+    cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
+    cb(cur, "mtp_attn_norm", il);
+
+    // self_attention
+    {
+        auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il);
+
+        Qcur = build_norm(Qcur, layer.attn_q_norm, nullptr, LLM_NORM_RMS, il);
+        cb(Qcur, "mtp_Qcur_normed", il);
+
+        Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il);
+        cb(Kcur, "mtp_Kcur_normed", il);
+
+        const bool is_swa = hparams.is_swa(il);
+
+        const float freq_base_l   = model.get_rope_freq_base(cparams, il);
+        const float freq_scale_l  = model.get_rope_freq_scale(cparams, il);
+        const float ext_factor_l  = is_swa ? 0.0f : ext_factor;
+        const float attn_factor_l = is_swa ? 1.0f : attn_factor;
+        const float beta_fast_l   = is_swa ? 32.0f : beta_fast;
+        const float beta_slow_l   = is_swa ? 1.0f  : beta_slow;
+
+        Qcur = ggml_rope_ext(
+            ctx0, Qcur, inp_pos, nullptr,
+            n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+            ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l
+            );
+
+        Kcur = ggml_rope_ext(
+            ctx0, Kcur, inp_pos, nullptr,
+            n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+            ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l
+            );
+
+        cb(Qcur, "mtp_Qcur", il);
+        cb(Kcur, "mtp_Kcur", il);
+        cb(Vcur, "mtp_Vcur", il);
+
+        cur = build_attn(inp_attn,
+                layer.wo, layer.wo_b, layer.wo_s,
+                Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
+        cb(cur, "mtp_attn_out", il);
+    }
+
+    ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+    cb(ffn_inp, "mtp_ffn_inp", il);
+
+    cur = build_norm(ffn_inp, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
+    cb(cur, "mtp_ffn_norm", il);
+
+    cur = build_moe_ffn(cur,
+            layer.ffn_gate_inp,
+            layer.ffn_up_exps,
+            layer.ffn_gate_exps,
+            layer.ffn_down_exps,
+            nullptr,
+            n_expert, n_expert_used,
+            LLM_FFN_SILU, hparams.expert_weights_norm,
+            hparams.expert_weights_scale,
+            (llama_expert_gating_func_type) hparams.expert_gating_func,
+            il,
+            nullptr, nullptr,
+            layer.ffn_up_exps_s,
+            layer.ffn_gate_exps_s,
+            layer.ffn_down_exps_s);
+    cb(cur, "mtp_ffn_moe_out", il);
+
+    cur = ggml_add(ctx0, cur, ffn_inp);
+    cb(cur, "mtp_ffn_out", il);
+
+    cur = build_norm(cur, layer.nextn.shared_head_norm, nullptr, LLM_NORM_RMS, -1);
+
+    cb(cur, "h_nextn", -1);
+    res->t_h_nextn = cur;
+
+    cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+    cb(cur, "mtp_shared_head_norm", -1);
+
+    cur = build_lora_mm(model.output, cur, model.output_s);
+
+    cb(cur, "result_output", -1);
+    res->t_logits = cur;
+}
+
 template struct llama_model_mellum::graph<false>;
 template struct llama_model_mellum::graph<true>;
+template struct llama_model_mellum::graph_mtp<false>;
+template struct llama_model_mellum::graph_mtp<true>;

src/models/models.h

@@ -466,6 +466,11 @@ struct llama_model_mellum : public llama_model_base {
         graph(const llama_model & model, const llm_graph_params & params);
     };
 
+    template <bool iswa>
+    struct graph_mtp : public llm_graph_context {
+        graph_mtp(const llama_model & model, const llm_graph_params & params);
+    };
+
     std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
 };
 

tests/CMakeLists.txt

@@ -188,11 +188,9 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
     llama_build_and_test(test-llama-grammar.cpp LINK llama)
     llama_build_and_test(test-batch-alloc.cpp LINK llama)
     llama_build_and_test(test-chat.cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
-    target_include_directories(test-chat PRIVATE ${PROJECT_SOURCE_DIR}/tools/server)
-    target_link_libraries(test-chat PRIVATE server-context)
+    target_link_libraries(test-chat PRIVATE llama-engine)
     llama_build_and_test(test-server-component.cpp)
-    target_include_directories(test-server-component PRIVATE ${PROJECT_SOURCE_DIR}/tools/server)
-    target_link_libraries(test-server-component PRIVATE server-context)
+    target_link_libraries(test-server-component PRIVATE llama-engine)
     llama_build_and_test(test-json-schema-to-grammar.cpp)
 
     if (NOT GGML_BACKEND_DL)

tests/test-chat.cpp

@@ -7,7 +7,7 @@
 //
 #include "../src/llama-grammar.h"
 #include "../src/unicode.h"
-#include "../tools/server/server-chat.h"
+#include "../tools/engine/engine-chat.h"
 #include "chat-auto-parser.h"
 #include "chat.h"
 #include "common.h"
@@ -1864,7 +1864,7 @@ static void test_convert_responses_to_chatcmpl() {
             }
         })");
 
-        json result = server_chat_convert_responses_to_chatcmpl(input);
+        json result = engine_chat_convert_responses_to_chatcmpl(input);
 
         // Verify messages were converted correctly
         assert_equals(true, result.contains("messages"));
@@ -1904,7 +1904,7 @@ static void test_convert_responses_to_chatcmpl() {
             "model": "test-model"
         })");
 
-        json result = server_chat_convert_responses_to_chatcmpl(input);
+        json result = engine_chat_convert_responses_to_chatcmpl(input);
 
         assert_equals((size_t)1, result.at("messages").size());
         const auto & msg = result.at("messages")[0];
@@ -1920,7 +1920,7 @@ static void test_convert_responses_to_chatcmpl() {
             "model": "test-model"
         })");
 
-        json result = server_chat_convert_responses_to_chatcmpl(input);
+        json result = engine_chat_convert_responses_to_chatcmpl(input);
 
         assert_equals((size_t)2, result.at("messages").size());
         const auto & sys_msg = result.at("messages")[0];
@@ -1936,7 +1936,7 @@ static void test_convert_responses_to_chatcmpl() {
             "max_output_tokens": 100
         })");
 
-        json result = server_chat_convert_responses_to_chatcmpl(input);
+        json result = engine_chat_convert_responses_to_chatcmpl(input);
 
         assert_equals(true, result.contains("max_tokens"));
         assert_equals(false, result.contains("max_output_tokens"));
@@ -1980,7 +1980,7 @@ static void test_convert_responses_to_chatcmpl() {
             ]
         })");
 
-        json result = server_chat_convert_responses_to_chatcmpl(input);
+        json result = engine_chat_convert_responses_to_chatcmpl(input);
 
         assert_equals(true, result.contains("tools"));
         assert_equals(true, result.at("tools").is_array());
@@ -2015,7 +2015,7 @@ static void test_convert_responses_to_chatcmpl() {
             ]
         })");
 
-        json result = server_chat_convert_responses_to_chatcmpl(input);
+        json result = engine_chat_convert_responses_to_chatcmpl(input);
 
         assert_equals(false, result.contains("tools"));
     }
@@ -7938,7 +7938,7 @@ static void test_reasoning_budget_tokens_per_request() {
     // the reasoning-budget code path in oaicompat_chat_params_parse.
     auto tmpls = read_templates("models/templates/Qwen-Qwen3-0.6B.jinja");
 
-    server_chat_params opt;
+    engine_chat_params opt;
     opt.tmpls            = std::move(tmpls);
     opt.use_jinja        = true;
     opt.enable_thinking  = true;
@@ -7970,7 +7970,7 @@ static void test_reasoning_budget_message_per_request() {
     // <think>...</think> markers enable the reasoning-budget block in oaicompat_chat_params_parse.
     auto tmpls = read_templates("models/templates/Qwen-Qwen3-0.6B.jinja");
 
-    server_chat_params opt;
+    engine_chat_params opt;
     opt.tmpls                   = std::move(tmpls);
     opt.use_jinja               = true;
     opt.enable_thinking         = true;

tests/test-server-component.cpp

@@ -1,7 +1,8 @@
 #include "testing.h"
 
-#include "../tools/server/server-common.h"
-#include "../tools/server/server-decision.h"
+#include "../tools/engine/engine-common.h"
+#include "../tools/server/server-subproc.h"
+#include "../tools/engine/engine-decision.h"
 
 #include <cstdlib>
 #include <regex>
@@ -15,11 +16,11 @@
 // see skills/code-review/SKILL.md for more
 
 //
-// server_tokens (text only, has_mtmd = false)
+// engine_tokens (text only, has_mtmd = false)
 //
 
-static void test_server_tokens_basic(testing & t) {
-    server_tokens st(llama_tokens{1, 2, 3}, false);
+static void test_engine_tokens_basic(testing & t) {
+    engine_tokens st(llama_tokens{1, 2, 3}, false);
     t.assert_equal("size", (size_t) 3, st.size());
     t.assert_true("not empty", !st.empty());
     t.assert_equal("operator[]", 2, st[1]);
@@ -47,8 +48,8 @@ static void test_server_tokens_basic(testing & t) {
     t.assert_true("clear", st.empty());
 }
 
-static void test_server_tokens_pos(testing & t) {
-    server_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
+static void test_engine_tokens_pos(testing & t) {
+    engine_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
     t.assert_equal("pos_next()", (llama_pos) 5, st.pos_next());
     t.assert_equal("pos_next(0)", (llama_pos) 0, st.pos_next(0));
     t.assert_equal("pos_next(3)", (llama_pos) 3, st.pos_next(3));
@@ -58,8 +59,8 @@ static void test_server_tokens_pos(testing & t) {
     t.assert_equal("size_up_to_pos clamps", (size_t) 5, st.size_up_to_pos(100));
 }
 
-static void test_server_tokens_keep_first(testing & t) {
-    server_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
+static void test_engine_tokens_keep_first(testing & t) {
+    engine_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
     st.keep_first(5);
     t.assert_equal("keep all", (size_t) 5, st.size());
     st.keep_first(2);
@@ -68,21 +69,21 @@ static void test_server_tokens_keep_first(testing & t) {
     t.assert_true("keep 0", st.empty());
 }
 
-static void test_server_tokens_common_prefix(testing & t) {
-    server_tokens a(llama_tokens{1, 2, 3, 4}, false);
-    server_tokens empty(llama_tokens{}, false);
+static void test_engine_tokens_common_prefix(testing & t) {
+    engine_tokens a(llama_tokens{1, 2, 3, 4}, false);
+    engine_tokens empty(llama_tokens{}, false);
 
     t.assert_equal("both empty",  (size_t) 0, empty.get_common_prefix(empty));
     t.assert_equal("one empty",   (size_t) 0, a.get_common_prefix(empty));
     t.assert_equal("identical",   (size_t) 4, a.get_common_prefix(a));
-    t.assert_equal("diverge at 2", (size_t) 2, a.get_common_prefix(server_tokens(llama_tokens{1, 2, 9, 4}, false)));
-    t.assert_equal("diverge at 0", (size_t) 0, a.get_common_prefix(server_tokens(llama_tokens{9, 2, 3, 4}, false)));
-    t.assert_equal("b is prefix",  (size_t) 2, a.get_common_prefix(server_tokens(llama_tokens{1, 2}, false)));
-    t.assert_equal("a is prefix",  (size_t) 4, a.get_common_prefix(server_tokens(llama_tokens{1, 2, 3, 4, 5, 6}, false)));
+    t.assert_equal("diverge at 2", (size_t) 2, a.get_common_prefix(engine_tokens(llama_tokens{1, 2, 9, 4}, false)));
+    t.assert_equal("diverge at 0", (size_t) 0, a.get_common_prefix(engine_tokens(llama_tokens{9, 2, 3, 4}, false)));
+    t.assert_equal("b is prefix",  (size_t) 2, a.get_common_prefix(engine_tokens(llama_tokens{1, 2}, false)));
+    t.assert_equal("a is prefix",  (size_t) 4, a.get_common_prefix(engine_tokens(llama_tokens{1, 2, 3, 4, 5, 6}, false)));
 }
 
-static void test_server_tokens_serialize(testing & t) {
-    // same packing as slot save/restore in server-context.cpp
+static void test_engine_tokens_serialize(testing & t) {
+    // same packing as slot save/restore in engine-context.cpp
     auto pack = [](const std::vector<char> & data) {
         llama_tokens packed(data.size() / sizeof(llama_token));
         memcpy(packed.data(), data.data(), data.size());
@@ -90,29 +91,29 @@ static void test_server_tokens_serialize(testing & t) {
     };
 
     t.test("roundtrip", [&](testing & t) {
-        server_tokens st(llama_tokens{10, 20, 30}, false);
+        engine_tokens st(llama_tokens{10, 20, 30}, false);
         std::vector<char> data = st.serialize();
         t.assert_equal("aligned to llama_token", (size_t) 0, data.size() % sizeof(llama_token));
 
-        server_tokens restored = server_tokens::deserialize(pack(data), false);
+        engine_tokens restored = engine_tokens::deserialize(pack(data), false);
         t.assert_true("tokens", restored.get_tokens() == st.get_tokens());
     });
 
     t.test("roundtrip empty", [&](testing & t) {
-        server_tokens st(llama_tokens{}, false);
-        server_tokens restored = server_tokens::deserialize(pack(st.serialize()), false);
+        engine_tokens st(llama_tokens{}, false);
+        engine_tokens restored = engine_tokens::deserialize(pack(st.serialize()), false);
         t.assert_true("empty", restored.empty());
     });
 
     t.test("legacy plain token list", [&](testing & t) {
-        server_tokens restored = server_tokens::deserialize(llama_tokens{7, 8, 9}, false);
+        engine_tokens restored = engine_tokens::deserialize(llama_tokens{7, 8, 9}, false);
         t.assert_true("tokens", restored.get_tokens() == llama_tokens({7, 8, 9}));
     });
 
     auto expect_throw = [](testing & t, const std::string & msg, const llama_tokens & packed) {
         bool thrown = false;
         try {
-            server_tokens::deserialize(packed, false);
+            engine_tokens::deserialize(packed, false);
         } catch (const std::exception &) {
             thrown = true;
         }
@@ -120,7 +121,7 @@ static void test_server_tokens_serialize(testing & t) {
     };
 
     t.test("corrupted", [&](testing & t) {
-        llama_tokens good = pack(server_tokens(llama_tokens{10, 20, 30}, false).serialize());
+        llama_tokens good = pack(engine_tokens(llama_tokens{10, 20, 30}, false).serialize());
 
         llama_tokens bad_version = good;
         bad_version[1] = 999;
@@ -142,22 +143,22 @@ static void test_server_tokens_serialize(testing & t) {
     });
 }
 
-static void test_server_tokens_clone(testing & t) {
-    server_tokens st(llama_tokens{1, 2, 3}, false);
-    server_tokens cl = st.clone();
+static void test_engine_tokens_clone(testing & t) {
+    engine_tokens st(llama_tokens{1, 2, 3}, false);
+    engine_tokens cl = st.clone();
     cl.push_back(4);
     t.assert_equal("original untouched", (size_t) 3, st.size());
     t.assert_equal("clone grew", (size_t) 4, cl.size());
     t.assert_true("get_text_tokens", cl.get_text_tokens() == llama_tokens({1, 2, 3, 4}));
 }
 
-static void test_server_tokens(testing & t) {
-    t.test("basic",         test_server_tokens_basic);
-    t.test("pos",           test_server_tokens_pos);
-    t.test("keep_first",    test_server_tokens_keep_first);
-    t.test("common_prefix", test_server_tokens_common_prefix);
-    t.test("serialize",     test_server_tokens_serialize);
-    t.test("clone",         test_server_tokens_clone);
+static void test_engine_tokens(testing & t) {
+    t.test("basic",         test_engine_tokens_basic);
+    t.test("pos",           test_engine_tokens_pos);
+    t.test("keep_first",    test_engine_tokens_keep_first);
+    t.test("common_prefix", test_engine_tokens_common_prefix);
+    t.test("serialize",     test_engine_tokens_serialize);
+    t.test("clone",         test_engine_tokens_clone);
 }
 
 //
@@ -374,7 +375,7 @@ static void test_server_pipe(testing & t) {
 }
 
 //
-// server_decision_escape_special_tokens
+// engine_decision_escape_special_tokens
 //
 
 // "<|name|>" with U+00A6 in place of "|"
@@ -398,7 +399,7 @@ static void test_decision_escape(testing & t) {
             { "<|abc",            "<|abc"                          },
         };
         for (const auto & [text, expected] : cases) {
-            t.assert_equal(text, expected, server_decision_escape_special_tokens(text));
+            t.assert_equal(text, expected, engine_decision_escape_special_tokens(text));
         }
     });
 
@@ -418,7 +419,7 @@ static void test_decision_escape(testing & t) {
                 for (size_t k = 0, v = idx; k < len; k++, v /= chars.size()) {
                     text += chars[v % chars.size()];
                 }
-                if (server_decision_escape_special_tokens(text) != std::regex_replace(text, re_special, "<\xC2\xA6$1\xC2\xA6>")) {
+                if (engine_decision_escape_special_tokens(text) != std::regex_replace(text, re_special, "<\xC2\xA6$1\xC2\xA6>")) {
                     n_diff++;
                 }
                 n_strings++;
@@ -430,8 +431,8 @@ static void test_decision_escape(testing & t) {
     // std::regex overflowed the stack on these
     t.test("long input", [](testing & t) {
         const std::string name(1000000, 'a');
-        t.assert_true("not closed", server_decision_escape_special_tokens("<|" + name) == "<|" + name);
-        t.assert_true("closed", server_decision_escape_special_tokens("<|" + name + "|>") == escaped(name));
+        t.assert_true("not closed", engine_decision_escape_special_tokens("<|" + name) == "<|" + name);
+        t.assert_true("closed", engine_decision_escape_special_tokens("<|" + name + "|>") == escaped(name));
     });
 }
 
@@ -447,7 +448,7 @@ int main(int argc, char ** argv) {
         t.set_filter(argv[1]);
     }
 
-    t.test("server_tokens", test_server_tokens);
+    t.test("engine_tokens", test_engine_tokens);
     t.test("utf8",          test_utf8);
     t.test("sse",           test_sse);
     t.test("json_helpers",  test_json_helpers);

tools/CMakeLists.txt

@@ -23,6 +23,7 @@ else()
     add_subdirectory(quantize)
     if (LLAMA_BUILD_SERVER)
         add_subdirectory(ui)
+        add_subdirectory(engine)
         add_subdirectory(cli)
         add_subdirectory(server)
     endif()

tools/engine/CMakeLists.txt

@@ -0,0 +1,29 @@
+# llama-engine: inference engine (context, task queue, common utils), used by llama-server and CLI
+
+set(TARGET llama-engine)
+
+add_library(${TARGET}
+    engine-chat.cpp
+    engine-chat.h
+    engine-task.cpp
+    engine-task.h
+    engine-queue.cpp
+    engine-queue.h
+    engine-common.cpp
+    engine-common.h
+    engine-decision.cpp
+    engine-decision.h
+    engine-context.cpp
+    engine-context.h
+    engine-schema.cpp
+    engine-schema.h
+)
+set_target_properties(${TARGET} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
+
+target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+target_include_directories(${TARGET} PRIVATE ../mtmd ${PROJECT_SOURCE_DIR})
+target_link_libraries(${TARGET} PUBLIC llama-common mtmd ${CMAKE_THREAD_LIBS_INIT})
+
+if(LLAMA_TOOLS_INSTALL)
+    install(TARGETS ${TARGET} LIBRARY)
+endif()

tools/engine/engine-chat.cpp

@@ -1,9 +1,9 @@
-#include "server-chat.h"
-#include "server-common.h"
+#include "engine-chat.h"
+#include "engine-common.h"
 
 #include <sstream>
 
-json server_chat_convert_responses_to_chatcmpl(const json & response_body) {
+json engine_chat_convert_responses_to_chatcmpl(const json & response_body) {
     if (!response_body.contains("input")) {
         throw std::invalid_argument("'input' is required");
     }
@@ -345,7 +345,7 @@ static void normalize_anthropic_billing_header(std::string & system_text) {
     }
 }
 
-json server_chat_convert_anthropic_to_oai(const json & body) {
+json engine_chat_convert_anthropic_to_oai(const json & body) {
     json oai_body;
 
     // Convert system prompt
@@ -618,7 +618,7 @@ json server_chat_convert_anthropic_to_oai(const json & body) {
     return oai_body;
 }
 
-json server_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff) {
+json engine_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff) {
     json delta = json::object();
     if (!diff.reasoning_content_delta.empty()) {
         delta["reasoning_content"] = diff.reasoning_content_delta;

tools/engine/engine-chat.h

@@ -3,16 +3,15 @@
 #pragma once
 
 #include "chat.h"
-#include "server-common.h"
-#include "server-http.h"
+#include "engine-common.h"
 
 #include "json.h"
 
 // Convert OpenAI Responses API format to OpenAI Chat Completions API format
-json server_chat_convert_responses_to_chatcmpl(const json & body);
+json engine_chat_convert_responses_to_chatcmpl(const json & body);
 
 // Convert Anthropic Messages API format to OpenAI Chat Completions API format
-json server_chat_convert_anthropic_to_oai(const json & body);
+json engine_chat_convert_anthropic_to_oai(const json & body);
 
 // convert OpenAI transcriptions API format to OpenAI Chat Completions API format
 json convert_transcriptions_to_chatcmpl(
@@ -21,4 +20,4 @@ json convert_transcriptions_to_chatcmpl(
     const std::map<std::string, uploaded_file> & in_files,
     std::vector<raw_buffer> & out_files);
 
-json server_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff);
+json engine_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff);

tools/engine/engine-common.h

@@ -6,7 +6,6 @@
 #include "chat.h"
 #include "mtmd.h"
 #include "mtmd-helper.h"
-#include "subproc.h"
 
 #include "json.h"
 
@@ -39,6 +38,12 @@ using json = common_json;
 
 using raw_buffer = std::vector<uint8_t>;
 
+struct uploaded_file {
+    raw_buffer data;
+    std::string filename;
+    std::string content_type;
+};
+
 template <typename T>
 static T json_value(const json & body, const std::string & key, const T & default_value) {
     // Fallback null to default value
@@ -67,12 +72,12 @@ enum error_type {
 };
 
 // thin wrapper around common_grammar_trigger with (de)serialization functions
-struct server_grammar_trigger {
+struct engine_grammar_trigger {
     common_grammar_trigger value;
 
-    server_grammar_trigger() = default;
-    server_grammar_trigger(const common_grammar_trigger & value) : value(value) {}
-    server_grammar_trigger(const json & in) {
+    engine_grammar_trigger() = default;
+    engine_grammar_trigger(const common_grammar_trigger & value) : value(value) {}
+    engine_grammar_trigger(const json & in) {
         value.type = (common_grammar_trigger_type) in.at("type").get<int>();
         value.value = in.at("value").get<std::string>();
         if (value.type == COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN) {
@@ -110,10 +115,10 @@ const char * get_media_marker();
 //
 
 // output modalities for architecture.output_modalities in GET /models
-std::vector<std::string> server_model_output_modalities(common_decision_type decision_type);
+std::vector<std::string> engine_model_output_modalities(common_decision_type decision_type);
 
 // architecture object of GET /models; shared by the direct server and the router
-json server_model_architecture_json(
+json engine_model_architecture_json(
         bool inp_image,
         bool inp_audio,
         bool inp_video,
@@ -142,14 +147,14 @@ bool are_lora_equal(
 std::vector<size_t> lora_get_enabled_ids(const std::vector<common_adapter_lora_info> & loras);
 
 //
-// server_tokens
+// engine_tokens
 //
 
 /**
- * server_tokens is a helper to manage the input tokens and image for the server.
+ * engine_tokens is a helper to manage the input tokens and image for the server.
  * it is made this way to simplify the logic of KV cache management.
  */
-struct server_tokens {
+struct engine_tokens {
     bool has_mtmd = false;
 
 private: // disallow accessing these members directly, risking out-of-sync
@@ -172,24 +177,24 @@ private: // disallow accessing these members directly, risking out-of-sync
     // map_idx_to_media will contain: {5, img0}, {8, img1}
 
 public:
-    server_tokens() = default;
-    ~server_tokens() = default;
+    engine_tokens() = default;
+    ~engine_tokens() = default;
 
     // Prevent copying
-    // TODO: server_tokens should be copyable - remove this:
-    server_tokens(const server_tokens&) = delete;
-    server_tokens& operator=(const server_tokens&) = delete;
+    // TODO: engine_tokens should be copyable - remove this:
+    engine_tokens(const engine_tokens&) = delete;
+    engine_tokens& operator=(const engine_tokens&) = delete;
 
     // Allow moving (usually implicitly generated if members are movable)
-    server_tokens(server_tokens&&) = default;
-    server_tokens& operator=(server_tokens&&) = default;
+    engine_tokens(engine_tokens&&) = default;
+    engine_tokens& operator=(engine_tokens&&) = default;
 
     // Allow accessing elements using [] operator
     llama_token operator[](size_t index) { return tokens[index]; }
     const llama_token& operator[](size_t index) const { return tokens[index]; }
 
-    server_tokens(mtmd::input_chunks & mtmd_chunks, bool has_mtmd);
-    server_tokens(const llama_tokens & tokens, bool has_mtmd);
+    engine_tokens(mtmd::input_chunks & mtmd_chunks, bool has_mtmd);
+    engine_tokens(const llama_tokens & tokens, bool has_mtmd);
 
     // for debugging
     std::string str() const;
@@ -216,7 +221,7 @@ public:
     void push_back_placeholder(const mtmd_input_chunk * chunk);
 
     // appends server tokens, updates the media map. copies media chunks.
-    void push_back(server_tokens & tokens);
+    void push_back(engine_tokens & tokens);
 
     // for compatibility with context shift and prompt truncation
     void insert(const llama_tokens & inp_tokens);
@@ -227,7 +232,7 @@ public:
     llama_tokens get_text_tokens() const;
 
     std::vector<char> serialize() const;
-    static server_tokens deserialize(const llama_tokens & packed, bool has_mtmd);
+    static engine_tokens deserialize(const llama_tokens & packed, bool has_mtmd);
 
     // for compatibility with speculative decoding
     void set_token(llama_pos pos, llama_token id);
@@ -245,7 +250,7 @@ public:
 
     std::string detokenize(const llama_context * ctx, bool special) const;
 
-    size_t get_common_prefix(const server_tokens & b) const;
+    size_t get_common_prefix(const engine_tokens & b) const;
 
     // split the tokens into message spans, skipping over media chunks
     common_chat_msg_spans find_message_spans(const common_chat_msg_delimiters & delims) const;
@@ -253,7 +258,7 @@ public:
     // check text token IDs and the mapping between media chunks and token ranges
     bool validate(const struct llama_context * ctx) const;
 
-    server_tokens clone() const;
+    engine_tokens clone() const;
 };
 
 
@@ -290,9 +295,9 @@ void handle_media(
         const std::string & url,
         const std::string & media_path);
 
-// process mtmd prompt, return the server_tokens containing both text tokens and media chunks
-// if is_placeholder is true, the media chunk will be treated as placeholder for counting tokens; the output tokens are not usable for actual inference (e.g. for submitting a task to server_queue)
-server_tokens process_mtmd_prompt(
+// process mtmd prompt, return the engine_tokens containing both text tokens and media chunks
+// if is_placeholder is true, the media chunk will be treated as placeholder for counting tokens; the output tokens are not usable for actual inference (e.g. for submitting a task to engine_queue)
+engine_tokens process_mtmd_prompt(
                                         mtmd_context * mctx,
                                         const std::string & prompt,
                                         const std::vector<raw_buffer> & files,
@@ -312,7 +317,7 @@ server_tokens process_mtmd_prompt(
  * - "prompt": [[12, 34, 56], [78, 90, 12]]
  * - "prompt": [[12, 34, "string", 56, 78], [12, 34, 56], { "prompt_string": "string", "multimodal_data": [ "base64" ]}]
  */
-std::vector<server_tokens> tokenize_input_prompts(
+std::vector<engine_tokens> tokenize_input_prompts(
                                         const llama_vocab * vocab,
                                         mtmd_context * mctx,
                                         const json & json_prompt,
@@ -321,7 +326,7 @@ std::vector<server_tokens> tokenize_input_prompts(
                                         const mtmd_helper_init_opt & init_opt);
 
 // tokenize a single prompt, see tokenize_input_prompts() for the supported shapes
-server_tokens tokenize_input_subprompt(
+engine_tokens tokenize_input_subprompt(
                                         const llama_vocab * vocab,
                                         mtmd_context * mctx,
                                         const json & json_prompt,
@@ -334,7 +339,7 @@ server_tokens tokenize_input_subprompt(
 //
 
 // global server parameters for chat formatting / parsing
-struct server_chat_params {
+struct engine_chat_params {
     bool use_jinja;
     bool prefill_assistant;
     common_reasoning_format reasoning_format;
@@ -357,28 +362,28 @@ json oaicompat_completion_params_parse(const json & body);
 json oaicompat_chat_params_parse(
     const llama_vocab * vocab,
     json & body, /* openai api json semantics */
-    const server_chat_params & opt,
+    const engine_chat_params & opt,
     std::vector<raw_buffer> & out_files,
     common_chat_session & out_session);
 
 // used by /embeddings endpoint, content has the same format as a chat message content array
-server_tokens tokenize_oai_content_array(
+engine_tokens tokenize_oai_content_array(
     const llama_vocab * vocab,
     mtmd_context * mctx,
-    const server_chat_params & opt,
+    const engine_chat_params & opt,
     json content,
     bool add_special,
     bool parse_special,
     const mtmd_helper_init_opt & init_opt);
 
-// TODO: move it to server-task.cpp
+// TODO: move it to engine-task.cpp
 json format_embeddings_response_oaicompat(
     const json & request,
     const std::string & model_name,
     const json & embeddings,
     bool use_base64 = false);
 
-// TODO: move it to server-task.cpp
+// TODO: move it to engine-task.cpp
 json format_response_rerank(
         const json & request,
         const std::string & model_name,
@@ -391,14 +396,14 @@ json format_response_rerank(
 // stats and metrics
 //
 
-// shared between server_slot and server_task_result_*
-struct server_slot_stats {
+// shared between engine_slot and engine_task_result_*
+struct engine_slot_stats {
     uint64_t n_prompt_cached    = 0;
     uint64_t n_prompt_processed = 0;
     uint64_t n_gen              = 0;
 
     // speculative decoding stats
-    // note: the per-position breakdown lives in server_slot, it is not needed in a task result
+    // note: the per-position breakdown lives in engine_slot, it is not needed in a task result
     uint64_t n_draft_tokens      = 0;
     uint64_t n_draft_accepted    = 0;
     uint64_t n_draft_verif_steps = 0;
@@ -478,9 +483,9 @@ struct server_slot_stats {
     json to_json() const;
 };
 
-// shared between server_context_impl and server_task_result_*
-// unlike server_slot_stats, server_metrics is server-global and cumulative, not tied to a slot
-struct server_metrics {
+// shared between engine_context_impl and engine_task_result_*
+// unlike engine_slot_stats, engine_metrics is server-global and cumulative, not tied to a slot
+struct engine_metrics {
     int64_t t_start = 0;
 
     struct bucket {
@@ -568,7 +573,7 @@ bool is_valid_utf8(const std::string & str);
 
 //
 // formatting output responses
-// TODO: move these to server-task.cpp
+// TODO: move these to engine-task.cpp
 //
 
 llama_tokens format_prompt_infill(
@@ -583,126 +588,10 @@ llama_tokens format_prompt_infill(
         const llama_tokens & tokens_prompt);
 
 // format rerank task: [BOS]query[EOS][SEP]doc[EOS].
-server_tokens format_prompt_rerank(
+engine_tokens format_prompt_rerank(
         const struct llama_model * model,
         const struct llama_vocab * vocab,
         mtmd_context * mctx,
         const std::string & query,
         const std::string & doc,
         const mtmd_helper_init_opt & init_opt);
-
-// simple implementation of a pipe
-// used for streaming data between threads
-template<typename T>
-struct server_pipe {
-    std::mutex mutex;
-    std::condition_variable cv;
-    std::queue<T> queue;
-    std::atomic<bool> writer_closed{false};
-    std::atomic<bool> reader_closed{false};
-
-    // 0 = unbounded (default)
-    // > 0, write() drops the oldest item once the queue is full
-    size_t max_size = 0;
-
-    void close_write() {
-        writer_closed.store(true, std::memory_order_relaxed);
-        cv.notify_all();
-    }
-
-    void close_read() {
-        reader_closed.store(true, std::memory_order_relaxed);
-        cv.notify_all();
-    }
-
-    // close_on_stop = true: should_stop means the reader is gone for good, so the writer is told the pipe is broken.
-    // close_on_stop = false: should_stop is a per-read deadline and further reads still come, so the pipe stays usable.
-    bool read(T & output, const std::function<bool()> & should_stop, bool close_on_stop = true) {
-        std::unique_lock<std::mutex> lk(mutex);
-        constexpr auto poll_interval = std::chrono::milliseconds(500);
-        while (true) {
-            if (!queue.empty()) {
-                output = std::move(queue.front());
-                queue.pop();
-                return true;
-            }
-            if (writer_closed.load()) {
-                return false; // clean EOF
-            }
-            if (should_stop && should_stop()) { // a null should_stop means "never stop"
-                if (close_on_stop) {
-                    close_read(); // signal broken pipe to writer
-                }
-                return false; // cancelled / deadline reached
-            }
-            cv.wait_for(lk, poll_interval);
-        }
-    }
-
-    bool write(T && data) {
-        std::lock_guard<std::mutex> lk(mutex);
-        if (reader_closed.load()) {
-            return false; // broken pipe
-        }
-        if (max_size > 0) {
-            while (queue.size() >= max_size) {
-                queue.pop(); // drop oldest to stay bounded
-            }
-        }
-        queue.push(std::move(data));
-        cv.notify_one();
-        return true;
-    }
-};
-
-// a child server writes its state commands to stdout and its logs to stderr
-enum server_subproc_stream {
-    SERVER_SUBPROC_STDOUT,
-    SERVER_SUBPROC_STDERR,
-    SERVER_SUBPROC_STREAMS,
-};
-
-// gives the current stdout to the caller as a stream of its own, and sends everything else written to stdout to stderr
-FILE * server_reserve_stdout();
-
-// wrapper around common_subproc to manage a child server process
-// mainly used by router mode
-struct server_subproc {
-    common_subproc sproc;
-    std::atomic<bool> stopped{false}; // set by the monitor once the process exited and was reaped
-
-    bool is_alive() { return sproc.alive(); }
-    void terminate() { sproc.terminate(); }
-    int  join() { return sproc.join(); }
-
-    // true if both output pipes of the child are available (call after create())
-    bool has_output();
-
-    // true once both output pipes are closed
-    bool output_closed() const;
-
-    // non-blocking read from one output pipe
-    // returns the number of bytes read, 0 when nothing is available, -1 when the pipe is closed or broken
-    int read_output(server_subproc_stream stream, char * buf, size_t len);
-
-    // wait until one of a set of children has output, wake() is called, or a timeout passes
-    struct waiter {
-        waiter();
-        ~waiter();
-
-        // thread-safe; on Windows this is a no-op, wait() returns within 50 ms anyway
-        void wake();
-
-        // timeout_ms < 0 waits until data or wake(); ready[i] is set for each child with data on an open pipe (or a broken pipe)
-        void wait(const std::vector<server_subproc *> & procs, std::vector<bool> & ready, int64_t timeout_ms);
-
-    private:
-#ifndef _WIN32
-        intptr_t wake_fd[2] = { -1, -1 }; // POSIX self-pipe
-#endif
-    };
-
-private:
-    intptr_t out_handles[SERVER_SUBPROC_STREAMS] = { -1, -1 }; // fd on POSIX, HANDLE on Windows; taken lazily from sproc
-    bool     out_closed [SERVER_SUBPROC_STREAMS] = { false, false };
-};

tools/engine/engine-context.h

@@ -0,0 +1,131 @@
+#pragma once
+
+#include "engine-common.h"
+#include "engine-decision.h"
+#include "engine-task.h"
+#include "engine-queue.h"
+
+#include "json.h"
+
+#include <cstddef>
+#include <memory>
+#include <mutex>
+#include <set>
+
+struct engine_context_impl; // private implementation
+
+constexpr int HTTP_POLLING_SECONDS = 1;
+
+struct engine_context_meta {
+    std::string build_info;
+    std::string model_name;
+    std::set<std::string> model_aliases;
+    std::set<std::string> model_tags;
+    std::string model_path;
+    std::vector<std::string> model_output_modalities; // output modalities for GET /models
+    bool has_mtmd;
+    bool has_inp_image;
+    bool has_inp_audio;
+    bool has_inp_video;
+    json json_ui_settings;
+    int slot_n_ctx;
+    enum llama_pooling_type pooling_type;
+
+    // chat params
+    engine_chat_params & chat_params;
+    std::map<std::string, bool> chat_template_caps;
+
+    // tokens
+    std::string bos_token_str;
+    std::string eos_token_str;
+    llama_token fim_pre_token;
+    llama_token fim_sub_token;
+    llama_token fim_mid_token;
+    llama_token fim_pad_token;
+    llama_token fim_rep_token;
+    llama_token fim_sep_token;
+
+    // sampling
+    std::vector<llama_logit_bias> logit_bias_eog;
+
+    // model meta
+    enum llama_vocab_type model_vocab_type;
+    int32_t model_vocab_n_tokens;
+    int32_t model_n_ctx_train;
+    int32_t model_n_embd_inp;
+    uint64_t model_n_params;
+    uint64_t model_size;
+    std::string model_ftype;
+};
+
+enum engine_state {
+    ENGINE_STATE_DOWNLOADING,
+    ENGINE_STATE_LOADING,
+    ENGINE_STATE_READY,
+    ENGINE_STATE_SLEEPING,
+};
+
+static std::string engine_state_to_str(engine_state state) {
+    switch (state) {
+        case ENGINE_STATE_DOWNLOADING: return "downloading";
+        case ENGINE_STATE_LOADING:     return "loading";
+        case ENGINE_STATE_READY:       return "ready";
+        case ENGINE_STATE_SLEEPING:    return "sleeping";
+        default: GGML_ASSERT(false && "invalid engine_state");
+    }
+}
+
+static engine_state engine_state_from_str(const std::string & str) {
+    if (str == "downloading") return ENGINE_STATE_DOWNLOADING;
+    if (str == "loading")     return ENGINE_STATE_LOADING;
+    if (str == "ready")       return ENGINE_STATE_READY;
+    if (str == "sleeping")    return ENGINE_STATE_SLEEPING;
+    GGML_ASSERT(false && "invalid engine_state string");
+}
+
+using engine_state_callback_t = std::function<void(engine_state, json /* payload */)>;
+
+struct engine_context {
+    std::unique_ptr<engine_context_impl> impl;
+
+    engine_context();
+    ~engine_context();
+
+    // load the model and initialize llama_context
+    // returns true on success
+    bool load_model(common_params & params);
+
+    // this function will block main thread until termination
+    void start_loop();
+
+    // terminate main loop (will unblock start_loop)
+    void terminate();
+
+    // get the underlaying llama_context, can return nullptr if sleeping
+    // not thread-safe, should only be used from the main thread
+    llama_context * get_llama_context() const;
+
+    // get a new response reader, used by CLI application
+    engine_response_reader get_response_reader();
+
+    // get server metadata (read-only), can only be called after load_model()
+    // not thread-safe, should only be used from the main thread
+    engine_context_meta get_meta() const;
+
+    // note: must be set before load_model() is called
+    void set_state_callback(engine_state_callback_t callback);
+
+    // used by the HTTP layer
+    // pointers can only be used when not sleeping, with thread-safe APIs (e.g., tokenizer calls)
+    llama_model * get_model() const;
+    const llama_vocab * get_vocab() const;
+    mtmd_context * get_mtmd_context() const;
+    const mtmd_helper_init_opt & get_mtmd_init_opt() const;
+    const engine_decision_context & get_decision() const;
+
+    engine_queue    & get_queue_tasks();
+    engine_response & get_queue_results();
+
+    engine_metrics get_metrics() const;
+    void reset_metrics_bucket();
+};

tools/engine/engine-decision.h

@@ -0,0 +1,166 @@
+#pragma once
+
+#include "engine-common.h"
+#include "engine-task.h"
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+// typed decision models (TypeSafe /v1/systemone API)
+// the model answers each question in one forward pass, no token is generated
+
+enum engine_decision_question_type {
+    ENGINE_DECISION_QUESTION_CHOICE,
+    ENGINE_DECISION_QUESTION_SCORE,
+    ENGINE_DECISION_QUESTION_NOUL,
+};
+
+struct engine_decision_option {
+    std::string key;
+    json description; // null if not provided
+};
+
+struct engine_decision_question {
+    std::string id;
+    engine_decision_question_type type;
+    json instructions;
+    std::vector<engine_decision_option> options; // in the order of the model outputs
+};
+
+struct engine_decision_context {
+    common_decision_type type = COMMON_DECISION_TYPE_NONE;
+
+    // read the "<arch>.decision.*" metadata, type stays NONE if the model has none
+    void init(const llama_model * model);
+
+    // true if the questions of a request start with the same tokens, and the model can continue from them
+    bool can_share_prompt() const {
+        switch (type) {
+            case COMMON_DECISION_TYPE_OPENJEV:
+            case COMMON_DECISION_TYPE_LEV:
+            case COMMON_DECISION_TYPE_KEV:
+            case COMMON_DECISION_TYPE_NIMBLE:
+            case COMMON_DECISION_TYPE_PPLX_DECIDER:
+            case COMMON_DECISION_TYPE_VLLM_DECISION2:
+            case COMMON_DECISION_TYPE_LFM2_D1:
+                return true;
+            default:
+                return false;
+        }
+    }
+
+    // true if all the questions of a request go in one prompt, see fill_task_joint()
+    bool is_joint() const {
+        return type == COMMON_DECISION_TYPE_CLEF;
+    }
+
+    // true if the prompt of the model has a place for images
+    bool can_use_images() const {
+        switch (type) {
+            case COMMON_DECISION_TYPE_OPENJEV:
+            case COMMON_DECISION_TYPE_CLEF:
+            case COMMON_DECISION_TYPE_PPLX_DECIDER:
+            case COMMON_DECISION_TYPE_VLLM_D3:
+            case COMMON_DECISION_TYPE_LFM2_D1:
+            case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
+                return true;
+            default:
+                return false;
+        }
+    }
+
+    // throw std::invalid_argument on bad input
+    std::vector<engine_decision_question> parse_questions(const json & body) const;
+
+    // returns the state without its images, they are appended to files in order
+    // images come from "files" (alias "images") and from the image_url and input_audio parts of a state made of chat messages
+    // an image can be an audio clip if the model supports it
+    json parse_state(const json & body, std::vector<raw_buffer> & files) const;
+
+    // number of prompts that are evaluated to answer this question, each one shows the options in a different order
+    size_t n_variants(const engine_decision_question & question) const;
+
+    // set the prompt of one variant of this question, and where to read its result
+    // mctx is only used if there are files
+    void fill_task(
+            const json & state,
+            const std::vector<engine_decision_question> & questions,
+            const engine_decision_question & question,
+            size_t variant,
+            const std::vector<raw_buffer> & files,
+            mtmd_context * mctx,
+            const mtmd_helper_init_opt & init_opt,
+            engine_task & task) const;
+
+    // set the prompt of all the questions, the result has the scores of all their options, in order
+    // mctx is only used if there are files
+    void fill_task_joint(
+            const json & state,
+            const std::vector<engine_decision_question> & questions,
+            const std::vector<raw_buffer> & files,
+            mtmd_context * mctx,
+            const mtmd_helper_init_opt & init_opt,
+            engine_task & task) const;
+
+    // scores: the raw model outputs of each variant
+    json format_answer(const engine_decision_question & question, const std::vector<std::vector<float>> & scores, bool has_media = false) const;
+
+private:
+    const llama_vocab * vocab = nullptr;
+    std::shared_ptr<const common_chat_template> tmpl; // the "systemone" template
+
+    std::map<std::string, float> temperatures; // "<type>" or "<type>.<n_options bucket>"
+    std::map<size_t, std::vector<float>> score_bias; // added to the scores of a score question, by number of levels
+    size_t n_options_max   = 0;
+    bool   noul_true_first = false; // noul options are [true, false] instead of [false, true]
+    bool   noul_criteria_order = false; // noul options are in the order of the criteria keys, if given
+    bool   choice_sorted   = false; // choice options are in the order of their keys
+    bool   strict_payload  = false; // state, instructions and descriptions must be text, an object or an array
+    bool   canonical_legend = false; // non-string score descriptions are returned as canonical JSON text
+
+    // OPENJEV, LEV, NIMBLE, PPLX_DECIDER, VLLM_D3
+    std::vector<llama_token> labels;
+    std::vector<std::string> label_texts; // only if the label of an option is given to the template
+
+    // LAYA, KEV
+    llama_token token_marker      = LLAMA_TOKEN_NULL;
+    llama_token token_sep         = LLAMA_TOKEN_NULL;
+    std::string text_marker;
+    size_t      max_head_tokens   = 0; // question + options
+    size_t      max_option_tokens = 48;
+
+    std::string render(
+            const json & state,
+            const std::vector<engine_decision_question> & questions,
+            const engine_decision_question & question,
+            size_t variant,
+            size_t n_images,
+            bool is_audio = false) const;
+    json render_options(const engine_decision_question & question, size_t variant) const;
+    size_t n_outputs(const engine_decision_question & question) const;
+    // LFM2_D1: label text and tokens of each option
+    void d1_labels(const engine_decision_question & question, std::vector<std::string> & texts, std::vector<llama_tokens> & groups) const;
+    void fill_task_laya(llama_tokens & tokens, const engine_decision_question & question, engine_task & task) const;
+    void fill_task_d1omni(
+            const json & state,
+            const std::vector<engine_decision_question> & questions,
+            const engine_decision_question & question,
+            size_t variant,
+            const std::vector<raw_buffer> & files,
+            mtmd_context * mctx,
+            const mtmd_helper_init_opt & init_opt,
+            engine_task & task) const;
+
+    float get_temperature(const engine_decision_question & question) const;
+    double get_decision_confidence(engine_decision_question_type type, const std::vector<double> & probs) const;
+};
+
+// group the tasks so that the common prefix of their prompts is evaluated only once
+// each group is one parent and its children, it takes at most n_slots slots
+// note: the order of the tasks is preserved
+std::vector<engine_task> engine_decision_group_tasks(std::vector<engine_task> && tasks, size_t n_slots);
+
+// write "<|name|>" with U+00A6 in place of "|", so text from the request is not parsed as special tokens
+std::string engine_decision_escape_special_tokens(const std::string & text);

tools/engine/engine-queue.cpp

@@ -1,5 +1,5 @@
-#include "server-task.h"
-#include "server-queue.h"
+#include "engine-task.h"
+#include "engine-queue.h"
 
 #include "log.h"
 
@@ -18,18 +18,18 @@
 #define RES_DBG(fmt, ...) LOG_DBG("res  %12.*s: " fmt, 12, __func__, __VA_ARGS__)
 
 //
-// server_queue
+// engine_queue
 //
 
-static bool task_resets_idle_timer(server_task_type type) {
-    return type != SERVER_TASK_TYPE_METRICS;
+static bool task_resets_idle_timer(engine_task_type type) {
+    return type != ENGINE_TASK_TYPE_METRICS;
 }
 
-int server_queue::post(server_task && task, bool front) {
+int engine_queue::post(engine_task && task, bool front) {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     GGML_ASSERT(task.id != -1);
     // if this is cancel task make sure to clean up pending tasks
-    if (task.type == SERVER_TASK_TYPE_CANCEL) {
+    if (task.type == ENGINE_TASK_TYPE_CANCEL) {
         cleanup_pending_task(task.id_target);
     }
     const int  task_id     = task.id;
@@ -47,7 +47,7 @@ int server_queue::post(server_task && task, bool front) {
     return task_id;
 }
 
-int server_queue::post(std::vector<server_task> && tasks, bool front) {
+int engine_queue::post(std::vector<engine_task> && tasks, bool front) {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     bool reset_timer = false;
     for (auto & task : tasks) {
@@ -55,7 +55,7 @@ int server_queue::post(std::vector<server_task> && tasks, bool front) {
             task.id = id++;
         }
         // if this is cancel task make sure to clean up pending tasks
-        if (task.type == SERVER_TASK_TYPE_CANCEL) {
+        if (task.type == ENGINE_TASK_TYPE_CANCEL) {
             cleanup_pending_task(task.id_target);
         }
         reset_timer |= task_resets_idle_timer(task.type);
@@ -73,7 +73,7 @@ int server_queue::post(std::vector<server_task> && tasks, bool front) {
     return 0;
 }
 
-void server_queue::defer(server_task && task) {
+void engine_queue::defer(engine_task && task) {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     QUE_DBG("defer task, id = %d\n", task.id);
     queue_tasks_deferred.push_back(std::move(task));
@@ -81,13 +81,13 @@ void server_queue::defer(server_task && task) {
     condition_tasks.notify_one();
 }
 
-int server_queue::get_new_id() {
+int engine_queue::get_new_id() {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     int new_id = id++;
     return new_id;
 }
 
-void server_queue::pop_deferred_task(int id_slot) {
+void engine_queue::pop_deferred_task(int id_slot) {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     if (!queue_tasks_deferred.empty()) {
         // try to find a task that uses the specified slot
@@ -112,7 +112,7 @@ void server_queue::pop_deferred_task(int id_slot) {
     condition_tasks.notify_one();
 }
 
-void server_queue::wait_until_no_sleep() {
+void engine_queue::wait_until_no_sleep() {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     if (!sleeping) {
         return;
@@ -129,13 +129,13 @@ void server_queue::wait_until_no_sleep() {
     }
 }
 
-void server_queue::terminate() {
+void engine_queue::terminate() {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     running = false;
     condition_tasks.notify_all();
 }
 
-bool server_queue::process_new_tasks(bool is_yielding) {
+bool engine_queue::process_new_tasks(bool is_yielding) {
     while (true) {
         std::unique_lock<std::mutex> lock(mutex_tasks);
         if (!running) {
@@ -145,7 +145,7 @@ bool server_queue::process_new_tasks(bool is_yielding) {
         if (queue_tasks.empty()) {
             return false;
         }
-        server_task task = std::move(queue_tasks.front());
+        engine_task task = std::move(queue_tasks.front());
         queue_tasks.pop_front();
         lock.unlock();
 
@@ -160,7 +160,7 @@ bool server_queue::process_new_tasks(bool is_yielding) {
     }
 }
 
-void server_queue::worker_loop() {
+void engine_queue::worker_loop() {
     while (true) {
         {
             std::unique_lock<std::mutex> lock(mutex_tasks);
@@ -206,7 +206,7 @@ void server_queue::worker_loop() {
     }
 }
 
-void server_queue::worker_stop() {
+void engine_queue::worker_stop() {
     if (!worker.thread.joinable()) {
         return;
     }
@@ -219,7 +219,7 @@ void server_queue::worker_stop() {
     worker.thread.join();
 }
 
-void server_queue::yield_to_queue(std::function<void()> && work) {
+void engine_queue::yield_to_queue(std::function<void()> && work) {
     GGML_ASSERT(worker.thread.joinable() && "yield_to_queue() requires start_loop() to be running");
 
     QUE_DBG("%s", "yielding to queue\n");
@@ -275,7 +275,7 @@ void server_queue::yield_to_queue(std::function<void()> && work) {
     }
 }
 
-void server_queue::start_loop(int64_t idle_sleep_ms) {
+void engine_queue::start_loop(int64_t idle_sleep_ms) {
     running = true;
     time_last_task = ggml_time_ms();
 
@@ -365,9 +365,9 @@ void server_queue::start_loop(int64_t idle_sleep_ms) {
     worker_stop();
 }
 
-void server_queue::cleanup_pending_task(int id_target) {
+void engine_queue::cleanup_pending_task(int id_target) {
     // no need lock because this is called exclusively by post()
-    auto rm_func = [id_target](const server_task & task) {
+    auto rm_func = [id_target](const engine_task & task) {
         return task.id == id_target;
     };
     queue_tasks.erase(
@@ -383,17 +383,17 @@ void server_queue::cleanup_pending_task(int id_target) {
 }
 
 //
-// server_response
+// engine_response
 //
 
-void server_response::add_waiting_task_id(int id_task) {
+void engine_response::add_waiting_task_id(int id_task) {
     RES_DBG("add task %d to waiting list. current waiting = %d (before add)\n", id_task, (int) waiting_task_ids.size());
 
     std::unique_lock<std::mutex> lock(mutex_results);
     waiting_task_ids.insert(id_task);
 }
 
-void server_response::add_waiting_task_ids(const std::unordered_set<int> & id_tasks) {
+void engine_response::add_waiting_task_ids(const std::unordered_set<int> & id_tasks) {
     std::unique_lock<std::mutex> lock(mutex_results);
 
     for (const auto & id_task : id_tasks) {
@@ -402,20 +402,20 @@ void server_response::add_waiting_task_ids(const std::unordered_set<int> & id_ta
     }
 }
 
-void server_response::remove_waiting_task_id(int id_task) {
+void engine_response::remove_waiting_task_id(int id_task) {
     RES_DBG("remove task %d from waiting list. current waiting = %d (before remove)\n", id_task, (int) waiting_task_ids.size());
 
     std::unique_lock<std::mutex> lock(mutex_results);
     waiting_task_ids.erase(id_task);
     // make sure to clean up all pending results
     queue_results.erase(
-        std::remove_if(queue_results.begin(), queue_results.end(), [id_task](const server_task_result_ptr & res) {
+        std::remove_if(queue_results.begin(), queue_results.end(), [id_task](const engine_task_result_ptr & res) {
             return res->id == id_task;
         }),
         queue_results.end());
 }
 
-void server_response::remove_waiting_task_ids(const std::unordered_set<int> & id_tasks) {
+void engine_response::remove_waiting_task_ids(const std::unordered_set<int> & id_tasks) {
     std::unique_lock<std::mutex> lock(mutex_results);
 
     for (const auto & id_task : id_tasks) {
@@ -424,7 +424,7 @@ void server_response::remove_waiting_task_ids(const std::unordered_set<int> & id
     }
 }
 
-server_task_result_ptr server_response::recv(const std::unordered_set<int> & id_tasks) {
+engine_task_result_ptr engine_response::recv(const std::unordered_set<int> & id_tasks) {
     while (true) {
         std::unique_lock<std::mutex> lock(mutex_results);
         condition_results.wait(lock, [&]{
@@ -437,7 +437,7 @@ server_task_result_ptr server_response::recv(const std::unordered_set<int> & id_
 
         for (size_t i = 0; i < queue_results.size(); i++) {
             if (id_tasks.find(queue_results[i]->id) != id_tasks.end()) {
-                server_task_result_ptr res = std::move(queue_results[i]);
+                engine_task_result_ptr res = std::move(queue_results[i]);
                 queue_results.erase(queue_results.begin() + i);
                 return res;
             }
@@ -447,13 +447,13 @@ server_task_result_ptr server_response::recv(const std::unordered_set<int> & id_
     // should never reach here
 }
 
-server_task_result_ptr server_response::recv_with_timeout(const std::unordered_set<int> & id_tasks, int timeout) {
+engine_task_result_ptr engine_response::recv_with_timeout(const std::unordered_set<int> & id_tasks, int timeout) {
     while (true) {
         std::unique_lock<std::mutex> lock(mutex_results);
 
         for (int i = 0; i < (int) queue_results.size(); i++) {
             if (id_tasks.find(queue_results[i]->id) != id_tasks.end()) {
-                server_task_result_ptr res = std::move(queue_results[i]);
+                engine_task_result_ptr res = std::move(queue_results[i]);
                 queue_results.erase(queue_results.begin() + i);
                 return res;
             }
@@ -472,12 +472,12 @@ server_task_result_ptr server_response::recv_with_timeout(const std::unordered_s
     // should never reach here
 }
 
-server_task_result_ptr server_response::recv(int id_task) {
+engine_task_result_ptr engine_response::recv(int id_task) {
     std::unordered_set<int> id_tasks = {id_task};
     return recv(id_tasks);
 }
 
-void server_response::send(server_task_result_ptr && result) {
+void engine_response::send(engine_task_result_ptr && result) {
     RES_DBG("sending result for task id = %d\n", result->id);
 
     std::unique_lock<std::mutex> lock(mutex_results);
@@ -492,27 +492,27 @@ void server_response::send(server_task_result_ptr && result) {
     }
 }
 
-void server_response::broadcast(server_task_result_ptr && result) {
+void engine_response::broadcast(engine_task_result_ptr && result) {
     std::unique_lock<std::mutex> lock(mutex_results);
     for (const auto & id_task : waiting_task_ids) {
         RES_DBG("task id = %d pushed to result queue\n", id_task);
-        server_task_result_ptr res_copy(result->clone());
+        engine_task_result_ptr res_copy(result->clone());
         res_copy->id = id_task; // override id with target task id
         queue_results.emplace_back(std::move(res_copy));
     }
     condition_results.notify_all();
 }
 
-void server_response::terminate() {
+void engine_response::terminate() {
     running = false;
     condition_results.notify_all();
 }
 
 //
-// server_response_reader
+// engine_response_reader
 //
 
-void server_response_reader::post_task(server_task && task, bool front) {
+void engine_response_reader::post_task(engine_task && task, bool front) {
     GGML_ASSERT(id_tasks.empty() && "post_task() can only be called once per reader");
     GGML_ASSERT(!task.is_parent() && "not supported, use post_tasks() instead");
     task.index = 0;
@@ -522,9 +522,9 @@ void server_response_reader::post_task(server_task && task, bool front) {
     queue_tasks.post(std::move(task), front);
 }
 
-void server_response_reader::post_tasks(std::vector<server_task> && tasks, const common_chat_session & session, bool front) {
+void engine_response_reader::post_tasks(std::vector<engine_task> && tasks, const common_chat_session & session, bool front) {
     GGML_ASSERT(id_tasks.empty() && "post_tasks() can only be called once per reader");
-    id_tasks = server_task::get_list_id(tasks);
+    id_tasks = engine_task::get_list_id(tasks);
     states.reserve(tasks.size());
     size_t index = 0;
     for (auto & task : tasks) {
@@ -541,15 +541,15 @@ void server_response_reader::post_tasks(std::vector<server_task> && tasks, const
     queue_tasks.post(std::move(tasks), front);
 }
 
-bool server_response_reader::has_next() const {
+bool engine_response_reader::has_next() const {
     return !cancelled && received_count < id_tasks.size();
 }
 
 // return nullptr if should_stop() is true before receiving a result
 // note: if one error is received, it will stop further processing and return error result
-server_task_result_ptr server_response_reader::next(const std::function<bool()> & should_stop) {
+engine_task_result_ptr engine_response_reader::next(const std::function<bool()> & should_stop) {
     while (true) {
-        server_task_result_ptr result = queue_results.recv_with_timeout(id_tasks, polling_interval_seconds);
+        engine_task_result_ptr result = queue_results.recv_with_timeout(id_tasks, polling_interval_seconds);
         if (result == nullptr) {
             // timeout, check stop condition
             if (should_stop()) {
@@ -577,7 +577,7 @@ server_task_result_ptr server_response_reader::next(const std::function<bool()>
     // should not reach here
 }
 
-server_response_reader::batch_response server_response_reader::wait_for_all(const std::function<bool()> & should_stop) {
+engine_response_reader::batch_response engine_response_reader::wait_for_all(const std::function<bool()> & should_stop) {
     batch_response batch_res;
     batch_res.results.clear();
     batch_res.results.resize(id_tasks.size());
@@ -599,16 +599,16 @@ server_response_reader::batch_response server_response_reader::wait_for_all(cons
     return batch_res;
 }
 
-void server_response_reader::stop() {
+void engine_response_reader::stop() {
     queue_results.remove_waiting_task_ids(id_tasks);
     if (has_next() && !cancelled) {
         // if tasks is not finished yet, cancel them
         cancelled = true;
-        std::vector<server_task> cancel_tasks;
+        std::vector<engine_task> cancel_tasks;
         cancel_tasks.reserve(id_tasks.size());
         for (const auto & id_task : id_tasks) {
             SRV_WRN("cancel task, id_task = %d\n", id_task);
-            server_task task(SERVER_TASK_TYPE_CANCEL);
+            engine_task task(ENGINE_TASK_TYPE_CANCEL);
             task.id_target = id_task;
             queue_results.remove_waiting_task_id(id_task);
             cancel_tasks.push_back(std::move(task));

tools/engine/engine-queue.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "server-task.h"
+#include "engine-task.h"
 
 #include <condition_variable>
 #include <deque>
@@ -11,8 +11,8 @@
 #include <unordered_set>
 
 // struct for managing server tasks
-// in most cases, use server_response_reader to post new tasks and retrieve results
-struct server_queue {
+// in most cases, use engine_response_reader to post new tasks and retrieve results
+struct engine_queue {
 private:
     int id = 0;
     bool running  = false;
@@ -21,11 +21,11 @@ private:
     int64_t time_last_task = 0;
 
     // queues
-    std::deque<server_task> queue_tasks;
-    std::deque<server_task> queue_tasks_deferred;
+    std::deque<engine_task> queue_tasks;
+    std::deque<engine_task> queue_tasks_deferred;
     // tasks declined while yielding, put back in queue_tasks once the yield is done
     // note: kept as a member so that cleanup_pending_task() can also reach them
-    std::deque<server_task> queue_tasks_unhandled;
+    std::deque<engine_task> queue_tasks_unhandled;
 
     std::mutex mutex_tasks;
     std::condition_variable condition_tasks;
@@ -42,21 +42,21 @@ private:
     worker_t worker;
 
     // callback functions
-    std::function<bool(server_task &&, bool)> callback_new_task;
+    std::function<bool(engine_task &&, bool)> callback_new_task;
     std::function<void(void)>                 callback_update_slots;
     std::vector<std::function<void(bool)>>    callback_sleeping_state;
 
 public:
-    ~server_queue() { worker_stop(); }
+    ~engine_queue() { worker_stop(); }
 
     // Add a new task to the end of the queue
-    int post(server_task && task, bool front = false);
+    int post(engine_task && task, bool front = false);
 
     // multi-task version of post()
-    int post(std::vector<server_task> && tasks, bool front = false);
+    int post(std::vector<engine_task> && tasks, bool front = false);
 
     // Add a new task, but defer until one slot is available
-    void defer(server_task && task);
+    void defer(engine_task && task);
 
     // Get the next id for creating a new task
     int get_new_id();
@@ -118,7 +118,7 @@ public:
     // only then may the callback return false to decline the task, and it must leave it
     // untouched, so that it can be put back in the queue later
     // note: while yielding, the callback runs on worker thread, not main thread
-    void on_new_task(std::function<bool(server_task &&, bool)> callback) {
+    void on_new_task(std::function<bool(engine_task &&, bool)> callback) {
         callback_new_task = std::move(callback);
     }
 
@@ -150,8 +150,8 @@ private:
 };
 
 // struct for managing server responses
-// in most cases, use server_response_reader to retrieve results
-struct server_response {
+// in most cases, use engine_response_reader to retrieve results
+struct engine_response {
 private:
     bool running = true;
 
@@ -159,7 +159,7 @@ private:
     std::unordered_set<int> waiting_task_ids;
 
     // the main result queue (using ptr for polymorphism)
-    std::vector<server_task_result_ptr> queue_results;
+    std::vector<engine_task_result_ptr> queue_results;
 
     std::mutex mutex_results;
     std::condition_variable condition_results;
@@ -177,33 +177,33 @@ public:
     void remove_waiting_task_ids(const std::unordered_set<int> & id_tasks);
 
     // This function blocks the thread until there is a response for one of the id_tasks
-    server_task_result_ptr recv(const std::unordered_set<int> & id_tasks);
+    engine_task_result_ptr recv(const std::unordered_set<int> & id_tasks);
 
     // same as recv(), but have timeout in seconds
     // if timeout is reached, nullptr is returned
-    server_task_result_ptr recv_with_timeout(const std::unordered_set<int> & id_tasks, int timeout);
+    engine_task_result_ptr recv_with_timeout(const std::unordered_set<int> & id_tasks, int timeout);
 
     // single-task version of recv()
-    server_task_result_ptr recv(int id_task);
+    engine_task_result_ptr recv(int id_task);
 
     // Send a new result to a waiting id_task
-    void send(server_task_result_ptr && result);
+    void send(engine_task_result_ptr && result);
 
     // broadcast a new result to all waiting tasks
     // (used by router mode)
-    void broadcast(server_task_result_ptr && result);
+    void broadcast(engine_task_result_ptr && result);
 
     // terminate the waiting loop
     void terminate();
 };
 
-// RAII wrapper to make working with server_queue and server_response easier
+// RAII wrapper to make working with engine_queue and engine_response easier
 // it provides a generator-like API for server responses
 // support pooling connection state and aggregating multiple results
-struct server_response_reader {
+struct engine_response_reader {
     std::unordered_set<int> id_tasks;
-    server_queue & queue_tasks;
-    server_response & queue_results;
+    engine_queue & queue_tasks;
+    engine_response & queue_results;
     size_t received_count = 0;
     bool cancelled = false;
     int polling_interval_seconds;
@@ -213,9 +213,9 @@ struct server_response_reader {
     std::vector<task_result_state> states;
 
     // should_stop function will be called each polling_interval_seconds
-    server_response_reader(server_queue & queue_tasks, server_response & queue_results, int polling_interval_seconds)
+    engine_response_reader(engine_queue & queue_tasks, engine_response & queue_results, int polling_interval_seconds)
         : queue_tasks(queue_tasks), queue_results(queue_results), polling_interval_seconds(polling_interval_seconds) {}
-    ~server_response_reader() {
+    ~engine_response_reader() {
         stop();
     }
 
@@ -224,18 +224,18 @@ struct server_response_reader {
     }
 
     // if front = true, the task will be posted to the front of the queue (high priority)
-    void post_task(server_task && task, bool front = false);
-    void post_tasks(std::vector<server_task> && tasks, const common_chat_session & session = {}, bool front = false);
+    void post_task(engine_task && task, bool front = false);
+    void post_tasks(std::vector<engine_task> && tasks, const common_chat_session & session = {}, bool front = false);
     bool has_next() const;
 
     // return nullptr if should_stop() is true before receiving a result
     // note: if one error is received, it will stop further processing and return error result
-    server_task_result_ptr next(const std::function<bool()> & should_stop);
+    engine_task_result_ptr next(const std::function<bool()> & should_stop);
 
     struct batch_response {
         bool is_terminated = false; // if true, indicates that processing was stopped before all results were received
-        std::vector<server_task_result_ptr> results;
-        server_task_result_ptr error; // nullptr if no error
+        std::vector<engine_task_result_ptr> results;
+        engine_task_result_ptr error; // nullptr if no error
     };
     // aggregate multiple results
     batch_response wait_for_all(const std::function<bool()> & should_stop);

tools/engine/engine-schema.cpp

@@ -1,8 +1,8 @@
-#include "server-schema.h"
+#include "engine-schema.h"
 
 #include "json-schema-to-grammar.h"
 
-namespace server_schema {
+namespace engine_schema {
 
 //
 // llama.cpp-specific completion schema
@@ -324,7 +324,7 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
         ->set_handler([&](field_eval_context & ctx, const json & data) {
             std::vector<common_grammar_trigger> triggers;
             for (const auto & t : data.at("grammar_triggers")) {
-                triggers.push_back(server_grammar_trigger(t).value);
+                triggers.push_back(engine_grammar_trigger(t).value);
             }
             common_sampling_add_grammar_triggers(ctx.params.sampling, ctx.vocab, std::move(triggers));
         }));
@@ -600,4 +600,4 @@ void field_nested::eval(field_eval_context & ctx, const json & data) {
     }
 }
 
-} // namespace server_schema
+} // namespace engine_schema

tools/engine/engine-schema.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "server-common.h"
-#include "server-task.h"
+#include "engine-common.h"
+#include "engine-task.h"
 
 #include "sampling.h"
 #include "speculative.h"
@@ -13,7 +13,7 @@
 #include <string>
 #include <vector>
 
-namespace server_schema {
+namespace engine_schema {
 
 struct field_eval_context {
     task_params & params;
@@ -101,4 +101,4 @@ task_params eval_llama_cmpl_schema(
                     const std::vector<llama_logit_bias> & logit_bias_eog,
                     const json & data);
 
-} // namespace server_schema
+} // namespace engine_schema

tools/engine/engine-task.cpp

@@ -1,14 +1,14 @@
-#include "server-task.h"
+#include "engine-task.h"
 
 #include "build-info.h"
-#include "server-chat.h"
+#include "engine-chat.h"
 #include "chat.h"
 #include "common.h"
 #include "json-schema-to-grammar.h"
 #include "llama.h"
 #include "sampling.h"
 #include "speculative.h"
-#include "server-common.h"
+#include "engine-common.h"
 
 #include <sstream>
 
@@ -88,7 +88,7 @@ json task_params::to_json(bool only_metrics) const {
 
     auto grammar_triggers = json::array();
     for (const auto & trigger : sampling.grammar_triggers) {
-        server_grammar_trigger ct(trigger);
+        engine_grammar_trigger ct(trigger);
         grammar_triggers.push_back(ct.to_json());
     }
 
@@ -307,9 +307,9 @@ std::vector<unsigned char> completion_token_output::str_to_bytes(const std::stri
 }
 
 //
-// server_task_result_cmpl_final
+// engine_task_result_cmpl_final
 //
-json server_task_result_cmpl_final::to_json() {
+json engine_task_result_cmpl_final::to_json() {
     GGML_ASSERT(is_updated && "update() must be called before to_json()");
     switch (res_type) {
         case TASK_RESPONSE_TYPE_NONE:
@@ -329,7 +329,7 @@ json server_task_result_cmpl_final::to_json() {
     }
 }
 
-json server_task_result_cmpl_final::to_json_non_oaicompat() {
+json engine_task_result_cmpl_final::to_json_non_oaicompat() {
     json res = json {
         {"index",               index},
         {"content",             content.text},
@@ -354,7 +354,7 @@ json server_task_result_cmpl_final::to_json_non_oaicompat() {
     return response_fields.empty() ? res : json_get_nested_values(response_fields, res);
 }
 
-json server_task_result_cmpl_final::usage_json_oaicompat() {
+json engine_task_result_cmpl_final::usage_json_oaicompat() {
     return json {
         {"completion_tokens", n_decoded},
         {"prompt_tokens",     n_prompt_tokens},
@@ -363,7 +363,7 @@ json server_task_result_cmpl_final::usage_json_oaicompat() {
     };
 }
 
-json server_task_result_cmpl_final::to_json_oaicompat() {
+json engine_task_result_cmpl_final::to_json_oaicompat() {
     std::time_t t = std::time(0);
     json logprobs = json(nullptr); // OAI default to null
     if (!stream && probs_output.size() > 0) {
@@ -403,7 +403,7 @@ json server_task_result_cmpl_final::to_json_oaicompat() {
     return res;
 }
 
-json server_task_result_cmpl_final::to_json_oaicompat_chat() {
+json engine_task_result_cmpl_final::to_json_oaicompat_chat() {
     std::string finish_reason = "length";
     common_chat_msg msg;
     if (!oaicompat_msg.empty()) {
@@ -451,7 +451,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat() {
     return res;
 }
 
-json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
+json engine_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
     std::time_t t = std::time(0);
     std::string finish_reason = "length";
     if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {
@@ -465,7 +465,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
                 json {
                     {"finish_reason", nullptr},
                     {"index", index},
-                    {"delta", server_chat_msg_diff_to_json_oaicompat(diff)},
+                    {"delta", engine_chat_msg_diff_to_json_oaicompat(diff)},
                 },
             })},
             {"created", t},
@@ -517,7 +517,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
     return deltas;
 }
 
-json server_task_result_cmpl_final::to_json_oaicompat_resp() {
+json engine_task_result_cmpl_final::to_json_oaicompat_resp() {
     common_chat_msg msg;
     if (!oaicompat_msg.empty()) {
         msg = oaicompat_msg;
@@ -588,7 +588,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp() {
     return res;
 }
 
-json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
+json engine_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
     std::vector<json> server_sent_events;
     std::vector<json> output;
 
@@ -705,7 +705,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
     return server_sent_events;
 }
 
-json server_task_result_cmpl_final::to_json_oaicompat_asr() {
+json engine_task_result_cmpl_final::to_json_oaicompat_asr() {
     json event = json {
         {"type",  "transcript.text.done"},
         {"text",  oaicompat_msg.content},
@@ -720,7 +720,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_asr() {
     return event;
 }
 
-json server_task_result_cmpl_final::to_json_anthropic() {
+json engine_task_result_cmpl_final::to_json_anthropic() {
     std::string stop_reason = "max_tokens";
     if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {
         stop_reason = oaicompat_msg.tool_calls.empty() ? "end_turn" : "tool_use";
@@ -786,7 +786,7 @@ json server_task_result_cmpl_final::to_json_anthropic() {
     return res;
 }
 
-json server_task_result_cmpl_final::to_json_anthropic_stream() {
+json engine_task_result_cmpl_final::to_json_anthropic_stream() {
     json events = json::array();
 
     std::string stop_reason = "max_tokens";
@@ -975,9 +975,9 @@ json server_task_result_cmpl_final::to_json_anthropic_stream() {
 }
 
 //
-// server_task_result_cmpl_partial
+// engine_task_result_cmpl_partial
 //
-void server_task_result_cmpl_partial::update(task_result_state & state) {
+void engine_task_result_cmpl_partial::update(task_result_state & state) {
     is_updated = true;
     if (is_begin) {
         return; // begin marker only flushes headers, skip parsing
@@ -1015,7 +1015,7 @@ void server_task_result_cmpl_partial::update(task_result_state & state) {
     }
 }
 
-json server_task_result_cmpl_partial::to_json() {
+json engine_task_result_cmpl_partial::to_json() {
     GGML_ASSERT(is_updated && "update() must be called before to_json()");
     if (is_begin) {
         return nullptr; // simply signal to HTTP handler to send the headers and status code
@@ -1038,7 +1038,7 @@ json server_task_result_cmpl_partial::to_json() {
     }
 }
 
-json server_task_result_cmpl_partial::to_json_non_oaicompat() {
+json engine_task_result_cmpl_partial::to_json_non_oaicompat() {
     // non-OAI-compat JSON
     json res = json {
         {"index",            index},
@@ -1062,7 +1062,7 @@ json server_task_result_cmpl_partial::to_json_non_oaicompat() {
     return res;
 }
 
-json server_task_result_cmpl_partial::to_json_oaicompat() {
+json engine_task_result_cmpl_partial::to_json_oaicompat() {
     std::time_t t = std::time(0);
     json logprobs = json(nullptr); // OAI default to null
     if (prob_output.probs.size() > 0) {
@@ -1100,7 +1100,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat() {
     return res;
 }
 
-json server_task_result_cmpl_partial::to_json_oaicompat_chat() {
+json engine_task_result_cmpl_partial::to_json_oaicompat_chat() {
     bool first = n_decoded == 1;
     std::time_t t = std::time(0);
     json choices;
@@ -1131,7 +1131,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat_chat() {
     }
 
     for (const auto & diff : oaicompat_msg_diffs) {
-        add_delta(server_chat_msg_diff_to_json_oaicompat(diff));
+        add_delta(engine_chat_msg_diff_to_json_oaicompat(diff));
     }
 
     if (!deltas.empty()) {
@@ -1155,7 +1155,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat_chat() {
     return deltas;
 }
 
-json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
+json engine_task_result_cmpl_partial::to_json_oaicompat_resp() {
     std::vector<json> events;
 
     if (!oai_resp_created) {
@@ -1305,7 +1305,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
     return events;
 }
 
-json server_task_result_cmpl_partial::to_json_oaicompat_asr() {
+json engine_task_result_cmpl_partial::to_json_oaicompat_asr() {
     json event = json {
         {"type", "transcript.text.delta"},
         {"delta", content.text},
@@ -1313,7 +1313,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat_asr() {
     return event;
 }
 
-json server_task_result_cmpl_partial::to_json_anthropic() {
+json engine_task_result_cmpl_partial::to_json_anthropic() {
     json events = json::array();
     bool first = (n_decoded == 1);
     // use member variables to track block state across streaming calls
@@ -1453,22 +1453,22 @@ json server_task_result_cmpl_partial::to_json_anthropic() {
 }
 
 //
-// server_task_result_embd
+// engine_task_result_embd
 //
-json server_task_result_embd::to_json() {
+json engine_task_result_embd::to_json() {
     return res_type == TASK_RESPONSE_TYPE_OAI_EMBD
         ? to_json_oaicompat()
         : to_json_non_oaicompat();
 }
 
-json server_task_result_embd::to_json_non_oaicompat() {
+json engine_task_result_embd::to_json_non_oaicompat() {
     return json {
         {"index",     index},
         {"embedding", embedding},
     };
 }
 
-json server_task_result_embd::to_json_oaicompat() {
+json engine_task_result_embd::to_json_oaicompat() {
     return json {
         {"index",            index},
         {"embedding",        embedding[0]},
@@ -1477,9 +1477,9 @@ json server_task_result_embd::to_json_oaicompat() {
 }
 
 //
-// server_task_result_rerank
+// engine_task_result_rerank
 //
-json server_task_result_rerank::to_json() {
+json engine_task_result_rerank::to_json() {
     return json {
         {"index",            index},
         {"score",            score},
@@ -1488,9 +1488,9 @@ json server_task_result_rerank::to_json() {
 }
 
 //
-// server_task_result_decision
+// engine_task_result_decision
 //
-json server_task_result_decision::to_json() {
+json engine_task_result_decision::to_json() {
     return json {
         {"index",            index},
         {"scores",           scores},
@@ -1499,9 +1499,9 @@ json server_task_result_decision::to_json() {
 }
 
 //
-// server_task_result_error
+// engine_task_result_error
 //
-json server_task_result_error::to_json() {
+json engine_task_result_error::to_json() {
     json res = format_error_response(err_msg, err_type);
     if (err_type == ERROR_TYPE_EXCEED_CONTEXT_SIZE) {
         res["n_prompt_tokens"] = n_prompt_tokens;
@@ -1511,19 +1511,19 @@ json server_task_result_error::to_json() {
 }
 
 //
-// server_task_result_metrics
+// engine_task_result_metrics
 //
-json server_task_result_slots::to_json() {
+json engine_task_result_slots::to_json() {
     return slots_data;
 }
 
-json server_task_result_metrics::to_json() {
+json engine_task_result_metrics::to_json() {
     // not used, /metrics renders prometheus text via to_metrics()
     return json{};
 }
 
 // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names
-std::string server_task_result_metrics::to_metrics() {
+std::string engine_task_result_metrics::to_metrics() {
     const std::vector<metric_item> counters = {
         {
             "prompt_tokens_total",
@@ -1620,9 +1620,9 @@ std::string server_task_result_metrics::to_metrics() {
 }
 
 //
-// server_task_result_slot_save_load
+// engine_task_result_slot_save_load
 //
-json server_task_result_slot_save_load::to_json() {
+json engine_task_result_slot_save_load::to_json() {
     if (is_save) {
         return json {
             { "id_slot",   id_slot },
@@ -1647,9 +1647,9 @@ json server_task_result_slot_save_load::to_json() {
 }
 
 //
-// server_task_result_slot_erase
+// engine_task_result_slot_erase
 //
-json server_task_result_slot_erase::to_json() {
+json engine_task_result_slot_erase::to_json() {
     return json {
         { "id_slot",  id_slot },
         { "n_erased", n_erased },
@@ -1657,10 +1657,10 @@ json server_task_result_slot_erase::to_json() {
 }
 
 //
-// server_task_result_get_lora
+// engine_task_result_get_lora
 //
 
-json server_task_result_get_lora::to_json() {
+json engine_task_result_get_lora::to_json() {
     json result = json::array();
     for (size_t i = 0; i < loras.size(); ++i) {
         auto & lora = loras[i];
@@ -1681,17 +1681,17 @@ json server_task_result_get_lora::to_json() {
 }
 
 //
-// server_task_result_apply_lora
+// engine_task_result_apply_lora
 //
 
-json server_task_result_apply_lora::to_json() {
+json engine_task_result_apply_lora::to_json() {
     return json {{ "success", true }};
 }
 
 //
-// server_prompt_cache
+// engine_prompt_cache
 //
-size_t server_prompt_cache::size() const {
+size_t engine_prompt_cache::size() const {
     size_t res = 0;
 
     for (const auto & state : states) {
@@ -1701,7 +1701,7 @@ size_t server_prompt_cache::size() const {
     return res;
 }
 
-size_t server_prompt_cache::n_tokens() const {
+size_t engine_prompt_cache::n_tokens() const {
     size_t res = 0;
 
     for (const auto & state : states) {
@@ -1711,7 +1711,7 @@ size_t server_prompt_cache::n_tokens() const {
     return res;
 }
 
-server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & prompt, size_t state_size_tgt, size_t state_size_dft) {
+engine_prompt_cache_state * engine_prompt_cache::alloc(const engine_prompt & prompt, size_t state_size_tgt, size_t state_size_dft) {
     // first check if the current state is contained fully in the cache
     for (auto it = states.begin(); it != states.end(); ++it) {
         const int cur_lcp_len = it->prompt.tokens.get_common_prefix(prompt.tokens);
@@ -1793,7 +1793,7 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro
     return &states.back();
 }
 
-bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) {
+bool engine_prompt_cache::load(engine_prompt & prompt, const engine_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) {
     const int lcp_best = prompt.tokens.get_common_prefix(tokens_new);
 
     float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins
@@ -1870,7 +1870,7 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
     return true;
 }
 
-void server_prompt_cache::update() {
+void engine_prompt_cache::update() {
     if (limit_size > 0) {
         while (!states.empty() && size() > limit_size) {
             SRV_WRN(" - cache size limit reached, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0));

tools/engine/engine-task.h

@@ -8,26 +8,26 @@
 #include <list>
 #include <map>
 
-// TODO: prevent including the whole server-common.h as we only use server_tokens
-#include "server-common.h"
+// TODO: prevent including the whole engine-common.h as we only use engine_tokens
+#include "engine-common.h"
 
 
-enum server_task_type {
-    SERVER_TASK_TYPE_COMPLETION,
-    SERVER_TASK_TYPE_EMBEDDING,
-    SERVER_TASK_TYPE_RERANK,
-    SERVER_TASK_TYPE_DECISION,
-    SERVER_TASK_TYPE_INFILL,
-    SERVER_TASK_TYPE_CANCEL,
-    SERVER_TASK_TYPE_CONTROL,
-    SERVER_TASK_TYPE_NEXT_RESPONSE,
-    SERVER_TASK_TYPE_METRICS,
-    SERVER_TASK_TYPE_SLOT_GET,
-    SERVER_TASK_TYPE_SLOT_SAVE,
-    SERVER_TASK_TYPE_SLOT_RESTORE,
-    SERVER_TASK_TYPE_SLOT_ERASE,
-    SERVER_TASK_TYPE_GET_LORA,
-    SERVER_TASK_TYPE_SET_LORA,
+enum engine_task_type {
+    ENGINE_TASK_TYPE_COMPLETION,
+    ENGINE_TASK_TYPE_EMBEDDING,
+    ENGINE_TASK_TYPE_RERANK,
+    ENGINE_TASK_TYPE_DECISION,
+    ENGINE_TASK_TYPE_INFILL,
+    ENGINE_TASK_TYPE_CANCEL,
+    ENGINE_TASK_TYPE_CONTROL,
+    ENGINE_TASK_TYPE_NEXT_RESPONSE,
+    ENGINE_TASK_TYPE_METRICS,
+    ENGINE_TASK_TYPE_SLOT_GET,
+    ENGINE_TASK_TYPE_SLOT_SAVE,
+    ENGINE_TASK_TYPE_SLOT_RESTORE,
+    ENGINE_TASK_TYPE_SLOT_ERASE,
+    ENGINE_TASK_TYPE_GET_LORA,
+    ENGINE_TASK_TYPE_SET_LORA,
 };
 
 // TODO: change this to more generic "response_format" to replace the "format_response_*" in server-common
@@ -85,7 +85,7 @@ struct task_params {
     std::string        oaicompat_model;
     std::string        oaicompat_cmpl_id;
 
-    // realtime control (SERVER_TASK_TYPE_CONTROL)
+    // realtime control (ENGINE_TASK_TYPE_CONTROL)
     std::string        control_action;
     std::string        control_cmpl_id;
 
@@ -134,37 +134,37 @@ struct task_result_state {
         bool filter_tool_calls = false);
 };
 
-struct server_task {
-    int id = -1; // to be filled by server_queue
+struct engine_task {
+    int id = -1; // to be filled by engine_queue
 
     // TODO @ngxson : remove this field and implement a mapping task_id -> idx in the response_reader
     size_t index = 0; // used when there are multiple prompts (batch request)
 
-    // used by SERVER_TASK_TYPE_CANCEL
+    // used by ENGINE_TASK_TYPE_CANCEL
     int id_target = -1;
     int id_slot   = -1;
 
     // used by parallel sampling (multiple completions from same prompt)
     int id_parent  = -1;
     // temporary store of child tasks for scheduling
-    // note: accessing to elements is invalid after the task is moved to server_slot
-    std::vector<server_task> child_tasks;
+    // note: accessing to elements is invalid after the task is moved to engine_slot
+    std::vector<engine_task> child_tasks;
     // if set on a parent, the children have their own prompt and only share its first n_tokens_shared tokens
     int32_t n_tokens_shared = 0;
 
-    // used by SERVER_TASK_TYPE_INFERENCE
+    // used by ENGINE_TASK_TYPE_INFERENCE
     task_params   params;
-    server_tokens tokens;
+    engine_tokens tokens;
 
     // only used by CLI, this allow tokenizing CLI inputs on server side
-    // we need this because mtmd_context and vocab are not accessible outside of server_context
+    // we need this because mtmd_context and vocab are not accessible outside of engine_context
     bool                    cli = false;
     std::string             cli_prompt;
     std::vector<raw_buffer> cli_files;
 
-    server_task_type type;
+    engine_task_type type;
 
-    // used by SERVER_TASK_TYPE_SLOT_SAVE, SERVER_TASK_TYPE_SLOT_RESTORE, SERVER_TASK_TYPE_SLOT_ERASE
+    // used by ENGINE_TASK_TYPE_SLOT_SAVE, ENGINE_TASK_TYPE_SLOT_RESTORE, ENGINE_TASK_TYPE_SLOT_ERASE
     struct slot_action {
         int id_slot;
         std::string filename;
@@ -172,10 +172,10 @@ struct server_task {
     };
     slot_action slot_action;
 
-    // used by SERVER_TASK_TYPE_METRICS
+    // used by ENGINE_TASK_TYPE_METRICS
     bool metrics_reset_bucket = false;
 
-    // used by SERVER_TASK_TYPE_DECISION
+    // used by ENGINE_TASK_TYPE_DECISION
     // where to read the model output of each option, exactly one of the two lists is used
     struct decision {
         std::vector<llama_token> labels;       // logits of these tokens, at the last prompt token
@@ -206,12 +206,12 @@ struct server_task {
     };
     decision decision;
 
-    // used by SERVER_TASK_TYPE_SET_LORA
+    // used by ENGINE_TASK_TYPE_SET_LORA
     std::map<int, float> set_lora; // mapping adapter ID -> scale
 
-    server_task() = default;
+    engine_task() = default;
 
-    server_task(server_task_type type) : type(type) {}
+    engine_task(engine_task_type type) : type(type) {}
 
     int32_t n_tokens() const {
         return tokens.size();
@@ -219,10 +219,10 @@ struct server_task {
 
     bool need_embd() const {
         switch (type) {
-            case SERVER_TASK_TYPE_EMBEDDING:
-            case SERVER_TASK_TYPE_RERANK:
+            case ENGINE_TASK_TYPE_EMBEDDING:
+            case ENGINE_TASK_TYPE_RERANK:
                 return true;
-            case SERVER_TASK_TYPE_DECISION:
+            case ENGINE_TASK_TYPE_DECISION:
                 return !decision.markers.empty() || !decision.order.empty();
             default:
                 return false;
@@ -231,10 +231,10 @@ struct server_task {
 
     bool need_logits() const {
         switch (type) {
-            case SERVER_TASK_TYPE_COMPLETION:
-            case SERVER_TASK_TYPE_INFILL:
+            case ENGINE_TASK_TYPE_COMPLETION:
+            case ENGINE_TASK_TYPE_INFILL:
                 return true;
-            case SERVER_TASK_TYPE_DECISION:
+            case ENGINE_TASK_TYPE_DECISION:
                 return !decision.labels.empty();
             default:
                 return false;
@@ -243,8 +243,8 @@ struct server_task {
 
     bool need_sampling() const {
         switch (type) {
-            case SERVER_TASK_TYPE_COMPLETION:
-            case SERVER_TASK_TYPE_INFILL:
+            case ENGINE_TASK_TYPE_COMPLETION:
+            case ENGINE_TASK_TYPE_INFILL:
                 return true;
             default:
                 return false;
@@ -252,7 +252,7 @@ struct server_task {
     }
 
     // utility function
-    static std::unordered_set<int> get_list_id(const std::vector<server_task> & tasks) {
+    static std::unordered_set<int> get_list_id(const std::vector<engine_task> & tasks) {
         std::unordered_set<int> ids(tasks.size());
         for (size_t i = 0; i < tasks.size(); i++) {
             ids.insert(tasks[i].id);
@@ -275,7 +275,7 @@ struct server_task {
     }
 
     void add_child(int id_parent, int id_child) {
-        server_task copy;
+        engine_task copy;
 
         copy.id        = id_child;
         copy.id_parent = id_parent;
@@ -303,7 +303,7 @@ struct server_task {
 
     // embedding and rerank tasks always process their whole prompt, their state is never reused
     bool is_stateless() const {
-        return type == SERVER_TASK_TYPE_EMBEDDING || type == SERVER_TASK_TYPE_RERANK;
+        return type == ENGINE_TASK_TYPE_EMBEDDING || type == ENGINE_TASK_TYPE_RERANK;
     }
 };
 
@@ -316,7 +316,7 @@ struct result_prompt_progress {
     json to_json() const;
 };
 
-struct server_task_result {
+struct engine_task_result {
     int id           = -1;
     int id_slot      = -1;
 
@@ -324,25 +324,25 @@ struct server_task_result {
     size_t index = 0; // to be used for batched tasks
 
     virtual bool is_error() {
-        // only used by server_task_result_error
+        // only used by engine_task_result_error
         return false;
     }
     virtual bool is_stop() {
-        // only used by server_task_result_cmpl_*
+        // only used by engine_task_result_cmpl_*
         return true;
     }
     virtual void update(task_result_state &) {
-        // only used by server_task_result_cmpl_*
+        // only used by engine_task_result_cmpl_*
     }
     virtual json to_json() = 0;
-    virtual ~server_task_result() = default;
-    virtual server_task_result * clone() const {
+    virtual ~engine_task_result() = default;
+    virtual engine_task_result * clone() const {
         GGML_ABORT("not implemented for this task type");
     }
 };
 
-// using shared_ptr for polymorphism of server_task_result
-using server_task_result_ptr = std::unique_ptr<server_task_result>;
+// using shared_ptr for polymorphism of engine_task_result
+using engine_task_result_ptr = std::unique_ptr<engine_task_result>;
 
 struct completion_token_output {
     llama_token tok;
@@ -365,13 +365,13 @@ struct completion_token_output {
 
 };
 
-struct server_task_result_cmpl_final : server_task_result {
+struct engine_task_result_cmpl_final : engine_task_result {
     common_chat_input content;
     llama_tokens tokens;
 
     bool stream;
     bool include_usage;
-    server_slot_stats stats;
+    engine_slot_stats stats;
     std::string prompt;
 
     bool truncated;
@@ -440,7 +440,7 @@ struct server_task_result_cmpl_final : server_task_result {
     json to_json_anthropic_stream();
 };
 
-struct server_task_result_cmpl_partial : server_task_result {
+struct engine_task_result_cmpl_partial : engine_task_result {
     common_chat_input content;
     llama_tokens      tokens;
 
@@ -453,7 +453,7 @@ struct server_task_result_cmpl_partial : server_task_result {
     bool is_begin = false; // whether to send 200 status to HTTP client (begin of SSE stream)
                            // ref: https://github.com/ggml-org/llama.cpp/pull/23884
     completion_token_output prob_output;
-    server_slot_stats stats;
+    engine_slot_stats stats;
     result_prompt_progress progress;
 
     // response formatting
@@ -499,7 +499,7 @@ struct server_task_result_cmpl_partial : server_task_result {
     json to_json_anthropic();
 };
 
-struct server_task_result_embd : server_task_result {
+struct engine_task_result_embd : engine_task_result {
     std::vector<std::vector<float>> embedding;
 
     int32_t n_tokens;
@@ -514,7 +514,7 @@ struct server_task_result_embd : server_task_result {
     json to_json_oaicompat();
 };
 
-struct server_task_result_rerank : server_task_result {
+struct engine_task_result_rerank : engine_task_result {
     float score = -1e6;
 
     int32_t n_tokens;
@@ -522,7 +522,7 @@ struct server_task_result_rerank : server_task_result {
     virtual json to_json() override;
 };
 
-struct server_task_result_decision : server_task_result {
+struct engine_task_result_decision : engine_task_result {
     std::vector<float> scores; // one raw model output per option
 
     int32_t n_tokens;
@@ -530,7 +530,7 @@ struct server_task_result_decision : server_task_result {
     virtual json to_json() override;
 };
 
-struct server_task_result_error : server_task_result {
+struct engine_task_result_error : engine_task_result {
     error_type err_type = ERROR_TYPE_SERVER;
     std::string err_msg;
 
@@ -546,12 +546,12 @@ struct server_task_result_error : server_task_result {
 };
 
 // used by /metrics API
-struct server_task_result_metrics : server_task_result {
-    // these are immediate stats, not accumulated (server_metrics is cumulative)
+struct engine_task_result_metrics : engine_task_result {
+    // these are immediate stats, not accumulated (engine_metrics is cumulative)
     int n_processing_slots = 0;
     int n_tasks_deferred = 0;
 
-    server_metrics metrics;
+    engine_metrics metrics;
 
     virtual json to_json() override;
 
@@ -564,17 +564,17 @@ struct server_task_result_metrics : server_task_result {
 };
 
 // used by /slots API
-struct server_task_result_slots : server_task_result {
+struct engine_task_result_slots : engine_task_result {
     int n_idle_slots = 0;
 
-    // while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
+    // while we can also use std::vector<engine_slot> this requires copying the slot object which can be quite messy
     // therefore, we use json to temporarily store the slot.to_json() result
     json slots_data = json::array();
 
     virtual json to_json() override;
 };
 
-struct server_task_result_slot_save_load : server_task_result {
+struct engine_task_result_slot_save_load : engine_task_result {
     std::string filename;
     bool is_save; // true = save, false = load
 
@@ -585,13 +585,13 @@ struct server_task_result_slot_save_load : server_task_result {
     virtual json to_json() override;
 };
 
-struct server_task_result_slot_erase : server_task_result {
+struct engine_task_result_slot_erase : engine_task_result {
     size_t n_erased;
 
     virtual json to_json() override;
 };
 
-struct server_task_result_control : server_task_result {
+struct engine_task_result_control : engine_task_result {
     bool        success = false;
     std::string message; // optional detail when success is false
 
@@ -604,7 +604,7 @@ struct server_task_result_control : server_task_result {
     }
 };
 
-struct server_task_result_get_lora : server_task_result {
+struct engine_task_result_get_lora : engine_task_result {
     struct lora {
         common_adapter_lora_info info;
         std::string  alora_invocation_string;
@@ -615,12 +615,12 @@ struct server_task_result_get_lora : server_task_result {
     virtual json to_json() override;
 };
 
-struct server_task_result_apply_lora : server_task_result {
+struct engine_task_result_apply_lora : engine_task_result {
     virtual json to_json() override;
 };
 
-struct server_prompt {
-    server_tokens tokens;
+struct engine_prompt {
+    engine_tokens tokens;
 
     std::list<common_prompt_checkpoint> checkpoints;
 
@@ -633,15 +633,15 @@ struct server_prompt {
         return tokens.size();
     }
 
-    server_prompt clone() const {
-        return server_prompt {
+    engine_prompt clone() const {
+        return engine_prompt {
             tokens.clone(),
             checkpoints,
         };
     }
 };
 
-struct server_prompt_data {
+struct engine_prompt_data {
     std::vector<uint8_t> main;
     std::vector<uint8_t> drft;
 
@@ -650,9 +650,9 @@ struct server_prompt_data {
     }
 };
 
-struct server_prompt_cache_state {
-    server_prompt prompt;
-    server_prompt_data data;
+struct engine_prompt_cache_state {
+    engine_prompt prompt;
+    engine_prompt_data data;
 
     size_t size() const {
         size_t res = data.size();
@@ -665,13 +665,13 @@ struct server_prompt_cache_state {
     }
 };
 
-struct server_prompt_cache {
-    server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens) {
+struct engine_prompt_cache {
+    engine_prompt_cache(int32_t limit_size_mib, size_t limit_tokens) {
         this->limit_size   = 1024ull*1024ull*(limit_size_mib < 0 ? 0 : limit_size_mib);
         this->limit_tokens = limit_tokens;
     }
 
-    std::list<server_prompt_cache_state> states;
+    std::list<engine_prompt_cache_state> states;
 
     // in bytes, 0 = no limit
     size_t limit_size = 0;
@@ -683,18 +683,18 @@ struct server_prompt_cache {
 
     size_t n_tokens() const;
 
-    server_prompt_cache_state * alloc(const server_prompt & prompt, size_t state_size_main, size_t state_size_drft);
+    engine_prompt_cache_state * alloc(const engine_prompt & prompt, size_t state_size_main, size_t state_size_drft);
 
-    bool load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot);
+    bool load(engine_prompt & prompt, const engine_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot);
 
     void update();
 };
 
 // used exclusively by router mode
-struct server_task_result_router : server_task_result {
+struct engine_task_result_router : engine_task_result {
     json data;
     virtual json to_json() override { return data; }
-    virtual server_task_result * clone() const override {
-        return new server_task_result_router(*this);
+    virtual engine_task_result * clone() const override {
+        return new engine_task_result_router(*this);
     }
 };

tools/server/CMakeLists.txt

@@ -1,40 +1,5 @@
 include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
 
-# server-context containing the core server logic, used by llama-server and CLI
-
-set(TARGET server-context)
-
-add_library(${TARGET} STATIC
-    server-chat.cpp
-    server-chat.h
-    server-task.cpp
-    server-task.h
-    server-queue.cpp
-    server-queue.h
-    server-common.cpp
-    server-common.h
-    server-decision.cpp
-    server-decision.h
-    server-context.cpp
-    server-context.h
-    server-stream.cpp
-    server-stream.h
-    server-tools.cpp
-    server-tools.h
-    server-mcp.cpp
-    server-mcp.h
-    server-schema.cpp
-    server-schema.h
-)
-
-if (BUILD_SHARED_LIBS)
-    set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)
-endif()
-
-target_include_directories(${TARGET} PRIVATE ../mtmd)
-target_include_directories(${TARGET} PRIVATE ${PROJECT_SOURCE_DIR})
-target_link_libraries(${TARGET} PUBLIC llama-common mtmd ${CMAKE_THREAD_LIBS_INIT})
-
 # llama-server-impl: server logic, reusable by app
 
 set(TARGET llama-server-impl)
@@ -43,6 +8,16 @@ add_library(${TARGET}
     server.cpp
     server-http.cpp
     server-http.h
+    server-routes.cpp
+    server-routes.h
+    server-stream.cpp
+    server-stream.h
+    server-subproc.cpp
+    server-subproc.h
+    server-tools.cpp
+    server-tools.h
+    server-mcp.cpp
+    server-mcp.h
     server-models.cpp
     server-models.h
 )
@@ -50,7 +25,7 @@ set_target_properties(${TARGET} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
 
 target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(${TARGET} PRIVATE ../mtmd ${PROJECT_SOURCE_DIR})
-target_link_libraries(${TARGET} PUBLIC server-context llama-ui cpp-httplib ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PUBLIC llama-engine llama-ui cpp-httplib ${CMAKE_THREAD_LIBS_INIT})
 
 add_dependencies(${TARGET} llama-ui-assets)
 

tools/server/README-dev.md

@@ -43,20 +43,25 @@ The server supports two primary operating modes:
 - **Inference mode**: The default mode for performing inference with a single loaded GGUF model.
 - **Router mode**: Enables management of multiple inference server instances behind a single API endpoint. Requests are automatically routed to the appropriate backend instance based on the requested model.
 
+The code is split into two libraries:
+
+- `libllama-engine` (`tools/engine`): the inference engine, including `engine_context`, task queue, tasks and common utils. It has no HTTP dependency.
+- `libllama-server-impl` (`tools/server`): the HTTP layer (`server_http_context`, `server_routes`), router mode, stream replay and MCP.
+
 The core architecture consists of the following components:
 
-- `server_context`: Holds the primary inference state, including the main `llama_context` and all active slots.
-- `server_slot`: An abstraction over a single “sequence” in llama.cpp, responsible for managing individual parallel inference requests.
-- `server_routes`: Middleware layer between `server_context` and the HTTP interface; handles JSON parsing/formatting and request routing logic.
+- `engine_context`: Holds the primary inference state, including the main `llama_context` and all active slots.
+- `engine_slot`: An abstraction over a single “sequence” in llama.cpp, responsible for managing individual parallel inference requests.
+- `server_routes`: Middleware layer between `engine_context` and the HTTP interface; handles JSON parsing/formatting and request routing logic.
 - `server_http_context`: Implements the HTTP server using `cpp-httplib`.
-- `server_queue`: Thread-safe queue used by HTTP workers to submit new tasks to `server_context`.
-- `server_response`: Thread-safe queue used by `server_context` to return results to HTTP workers.
-- `server_response_reader`: Higher-level wrapper around the two queues above for cleaner code.
-- `server_task`: Unit of work pushed into `server_queue`.
-- `server_task_result`: Unit of result pushed into `server_response`.
-- `server_tokens`: Unified representation of token sequences (supports both text and multimodal tokens); used by `server_task` and `server_slot`.
+- `engine_queue`: Thread-safe queue used by HTTP workers to submit new tasks to `engine_context`.
+- `engine_response`: Thread-safe queue used by `engine_context` to return results to HTTP workers.
+- `engine_response_reader`: Higher-level wrapper around the two queues above for cleaner code.
+- `engine_task`: Unit of work pushed into `engine_queue`.
+- `engine_task_result`: Unit of result pushed into `engine_response`.
+- `engine_tokens`: Unified representation of token sequences (supports both text and multimodal tokens); used by `engine_task` and `engine_slot`.
 - `server_prompt_checkpoint`: For recurrent (e.g., RWKV) and SWA models, stores snapshots of KV cache state. Enables reuse when subsequent requests share the same prompt prefix, saving redundant computation.
-- `server_models`: Standalone component for managing multiple backend instances (used in router mode). It is completely independent of `server_context`.
+- `server_models`: Standalone component for managing multiple backend instances (used in router mode). It is completely independent of `engine_context`.
 - `stream_session_manager`: process wide owner of resumable SSE stream sessions, keyed by conversation id. A file-static singleton inside `server-stream.cpp`, driven through `server_stream_session_manager_start/stop`. Backs the replay buffer that lets a client reattach to a generation after an HTTP disconnect. See the "Resumable streaming" section below.
 
 ```mermaid
@@ -64,13 +69,13 @@ graph TD
     API_User <--> server_http_context
     server_http_context <-- router mode --> server_models
     server_http_context <-- inference mode --> server_routes
-    server_routes -- server_task --> server_queue
-    subgraph server_context
-        server_queue --> server_slot
-        server_slot -- server_task_result --> server_response
-        server_slot[multiple server_slot]
+    server_routes -- engine_task --> engine_queue
+    subgraph engine_context
+        engine_queue --> engine_slot
+        engine_slot -- engine_task_result --> engine_response
+        engine_slot[multiple engine_slot]
     end
-    server_response --> server_routes
+    engine_response --> server_routes
 ```
 
 ### Batching
@@ -85,21 +90,21 @@ Following decoding, the system either retrieves embeddings or samples the next t
 
 ### Thread Management
 
-`server_context` runs on a dedicated single thread. Because it is single-threaded, heavy post-processing (especially after token generation) should be avoided, as it directly impacts multi-sequence throughput.
+`engine_context` runs on a dedicated single thread. Because it is single-threaded, heavy post-processing (especially after token generation) should be avoided, as it directly impacts multi-sequence throughput.
 
 Each incoming HTTP request is handled by its own thread managed by the HTTP library. The following operations are performed in HTTP worker threads:
 
 - JSON request parsing
 - Chat template application
 - Tokenization
-- Conversion of `server_task_result` into final JSON response
+- Conversion of `engine_task_result` into final JSON response
 - Error formatting into JSON
 - Tracking of partial/incremental responses (e.g., streaming tool calls or reasoning steps)
 
 **Best practices to follow:**
 
 - All JSON formatting and chat template logic must stay in the HTTP layer.
-- Avoid passing raw JSON between the HTTP layer and `server_slot`. Instead, parse everything into native C++ types as early as possible.
+- Avoid passing raw JSON between the HTTP layer and `engine_slot`. Instead, parse everything into native C++ types as early as possible.
 
 ### Example trace of a request
 
@@ -107,13 +112,13 @@ Here is an example trace of an API request for text completion:
 
 - A request arrives at the HTTP layer.
 - The request is routed to the corresponding handler inside `server_routes`. In this case, `handle_completions_impl` is invoked.
-- The handler parses the input request, constructs a new `server_task`, and passes it to `server_res_generator`.
+- The handler parses the input request, constructs a new `engine_task`, and passes it to `server_res_generator`.
 - `server_res_generator` creates a new `task_result_state` for each task:
     - `task_result_state` stays in the HTTP layer, responsible for keeping track of the current state of the response (e.g., parsing tool calls or thinking messages).
-    - `server_task` is moved into `server_queue` inside `server_context`.
-- `server_context` launches the task by moving it into an available slot (see `launch_slot_with_task()`).
+    - `engine_task` is moved into `engine_queue` inside `engine_context`.
+- `engine_context` launches the task by moving it into an available slot (see `launch_slot_with_task()`).
 - `update_slot()` processes the task as described in the "Batching" section above.
-- Results may be sent using `send_partial_response` or `send_final_response`, which creates a new `server_task_result` and pushes it to the response queue.
+- Results may be sent using `send_partial_response` or `send_final_response`, which creates a new `engine_task_result` and pushes it to the response queue.
 - At the same time, `server_res_generator` listens to the response queue and retrieves this response.
 - As the response is stateless, `server_res_generator` calls `response->update()` to update the response with the current state.
 - `server_res_generator` then calls `response->to_json()` and passes the response to the HTTP layer.
@@ -134,7 +139,7 @@ The implementation is hidden in `server-stream.cpp` (pimpl). The header exposes
 
 Producer side: `server_res_generator` extends `server_res_spipe`, which keeps all spipe logic out of the generic `server_http_res`. `set_req` attaches a producer when the header is present, and the wrapped `next` tees each chunk into the ring before the socket, so a chunk lost to a dead wire is already buffered. While attached, `should_stop` ignores peer disconnect: only a `DELETE` stops generation. On an early peer drop, `on_complete` drains the tail into the ring on the http worker.
 
-Lifetime safety: the session holds no back reference to the response, so `spipe` is a plain `unique_ptr` touched only by the http worker. `cancel` raises an atomic the producer polls; the producer finalizes the session from its destructor, which also runs `~server_response_reader::stop()` to cancel the generation at the queue level. A `DELETE` stops work by raising the flag and letting the worker unwind.
+Lifetime safety: the session holds no back reference to the response, so `spipe` is a plain `unique_ptr` touched only by the http worker. `cancel` raises an atomic the producer polls; the producer finalizes the session from its destructor, which also runs `~engine_response_reader::stop()` to cancel the generation at the queue level. A `DELETE` stops work by raising the flag and letting the worker unwind.
 
 Consumer side: `GET /v1/stream?conv_id=<id>&from=N` opens a `text/event-stream` that replays buffered bytes from offset `N` and blocks for live bytes, so the browser reattaches like a fresh EventSource. An offset below the dropped prefix returns 400.
 
@@ -281,7 +286,7 @@ The main goal of this API is to allow downloading models and/or removing models
 
 Instead of building everything from the ground up (like what most AI agents will do when you ask them to implement a similar feature), we built on top of existing, already well-engineered components inside the codebase:
 - Model cache infrastructure as mentioned above (`common/download.h`)
-- Server response queue (`server-queue.h`). We use this feature to broadcast events to SSE clients.
+- Server response queue (`engine-queue.h`). We use this feature to broadcast events to SSE clients.
 - Server router thread management (`server-models.h`). We re-use the same thread model that is used for managing subprocess life cycle, except that we don't create a new subprocess, but launch the download right inside the thread.
 
 The flow for downloading a new model:
@@ -294,27 +299,27 @@ The flow for downloading a new model:
 ### Sleep mode
 
 Sleep mode was initially introduced in PR [#18228](https://github.com/ggml-org/llama.cpp/pull/18228). The main idea is to have:
-- `server_queue` keeping track of the idle timeout
-- When the timeout is detected, `server_queue` signals to `server_context_impl` that it should go into sleep
-- `server_context_impl` frees all `llama_context` and `mtmd_context`
+- `engine_queue` keeping track of the idle timeout
+- When the timeout is detected, `engine_queue` signals to `engine_context_impl` that it should go into sleep
+- `engine_context_impl` frees all `llama_context` and `mtmd_context`
 
 Compared to simply exiting the whole process, this approach allows accessing some read-only endpoints during sleep, while also handling wakeup-on-request. Any inference request will wake the server up.
 
 Call stack on entering sleeping:
-- `server_queue::start_loop` (main thread) sees no task for `idle_sleep_ms` --> `sleeping = true`
+- `engine_queue::start_loop` (main thread) sees no task for `idle_sleep_ms` --> `sleeping = true`
 - `cb0(true)` --> `server_routes::update_cached_responses`
     - snapshots `/props`, `/models` and metrics; the model is still alive here
-- `cb1(true)` --> `server_context_impl::handle_sleeping_state`
-    - `callback_state(SERVER_STATE_SLEEPING)` --> reported to router in child mode
+- `cb1(true)` --> `engine_context_impl::handle_sleeping_state`
+    - `callback_state(ENGINE_STATE_SLEEPING)` --> reported to router in child mode
     - `destroy()` --> frees `llama_context` and `mtmd_context`
 - `condition_tasks.wait` until `req_stop_sleeping`
 
 Call stack on waking up:
-- `server_res_generator` constructor (HTTP thread) --> `server_queue::wait_until_no_sleep`
+- `server_res_generator` constructor (HTTP thread) --> `engine_queue::wait_until_no_sleep`
     - sets `req_stop_sleeping = true`, then waits until `sleeping == false`
-- `server_queue::start_loop` (main thread) wakes up
-- `cb1(false)` --> `server_context_impl::handle_sleeping_state`
-    - `load_model()`, which then emits `callback_state(SERVER_STATE_READY)`
+- `engine_queue::start_loop` (main thread) wakes up
+- `cb1(false)` --> `engine_context_impl::handle_sleeping_state`
+    - `load_model()`, which then emits `callback_state(ENGINE_STATE_READY)`
 - `cb0(false)` --> `server_routes::update_cached_responses`
     - nothing to do, the cache is only read during sleep
 - `sleeping = false` --> `notify_all` unblocks the HTTP thread, the request is handled as usual
@@ -325,7 +330,7 @@ Endpoints created with `create_response(true)` (`/health`, `/props`, `/models`,
 
 - Initial server implementation: https://github.com/ggml-org/llama.cpp/pull/1443
 - Parallel decoding support: https://github.com/ggml-org/llama.cpp/pull/3228
-- Refactor introducing `server_queue` and `server_response`: https://github.com/ggml-org/llama.cpp/pull/5065
+- Refactor introducing `server_queue` and `server_response` (now `engine_queue` and `engine_response`): https://github.com/ggml-org/llama.cpp/pull/5065
 - Reranking endpoint: https://github.com/ggml-org/llama.cpp/pull/9510
 - Multimodal model support (`libmtmd`): https://github.com/ggml-org/llama.cpp/pull/12898
 - Unified KV cache handling: https://github.com/ggml-org/llama.cpp/pull/16736

tools/server/README.md

@@ -405,6 +405,10 @@ Recommended `--cors-origins` setting, depending on where the server runs:
 
 Related flags: `--cors-origins`, `--cors-methods`, `--cors-headers`, `--cors-credentials` / `--no-cors-credentials`. Background and rationale: [#25655](https://github.com/ggml-org/llama.cpp/pull/25655).
 
+### Running behind a proxy
+
+If your host reaches the internet through a corporate proxy, export `HTTP_PROXY` and `HTTPS_PROXY` before running commands that download models (e.g. `export HTTPS_PROXY=http://proxy.example.com:8080`). Hosts listed in `NO_PROXY` and loopback addresses are contacted directly.
+
 ## Build
 
 `llama-server` is built alongside everything else from the root of the project
@@ -1719,9 +1723,9 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo
 
 The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly.
 
-The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
+The number of options of a `choice` question is limited by the model, for example: 52 for openjev and granite-switch, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
 
-For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available.
+For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.
 
 *Image input:*
 

tools/server/server-common.cpp

@@ -161,6 +161,8 @@ std::vector<std::string> server_model_output_modalities(common_decision_type dec
         case COMMON_DECISION_TYPE_LFM2_D1:
         case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
             return {"decisions"};
+        case COMMON_DECISION_TYPE_GRANITE_SWITCH:
+            return {"text", "decisions"};
         default:
             // fallback when there is no decision type or the metadata is bad
             return {"text"};

tools/server/server-context.h

@@ -1,196 +0,0 @@
-#pragma once
-
-#include "server-http.h"
-#include "server-task.h"
-#include "server-queue.h"
-
-#include "json.h"
-
-#include <cstddef>
-#include <memory>
-#include <mutex>
-#include <set>
-
-struct server_context_impl; // private implementation
-
-struct server_context_meta {
-    std::string build_info;
-    std::string model_name;
-    std::set<std::string> model_aliases;
-    std::set<std::string> model_tags;
-    std::string model_path;
-    std::vector<std::string> model_output_modalities; // output modalities for GET /models
-    bool has_mtmd;
-    bool has_inp_image;
-    bool has_inp_audio;
-    bool has_inp_video;
-    json json_ui_settings;
-    int slot_n_ctx;
-    enum llama_pooling_type pooling_type;
-
-    // chat params
-    server_chat_params & chat_params;
-    std::map<std::string, bool> chat_template_caps;
-
-    // tokens
-    std::string bos_token_str;
-    std::string eos_token_str;
-    llama_token fim_pre_token;
-    llama_token fim_sub_token;
-    llama_token fim_mid_token;
-    llama_token fim_pad_token;
-    llama_token fim_rep_token;
-    llama_token fim_sep_token;
-
-    // sampling
-    std::vector<llama_logit_bias> logit_bias_eog;
-
-    // model meta
-    enum llama_vocab_type model_vocab_type;
-    int32_t model_vocab_n_tokens;
-    int32_t model_n_ctx_train;
-    int32_t model_n_embd_inp;
-    uint64_t model_n_params;
-    uint64_t model_size;
-    std::string model_ftype;
-};
-
-enum server_state {
-    SERVER_STATE_DOWNLOADING,
-    SERVER_STATE_LOADING,
-    SERVER_STATE_READY,
-    SERVER_STATE_SLEEPING,
-};
-
-static std::string server_state_to_str(server_state state) {
-    switch (state) {
-        case SERVER_STATE_DOWNLOADING: return "downloading";
-        case SERVER_STATE_LOADING:     return "loading";
-        case SERVER_STATE_READY:       return "ready";
-        case SERVER_STATE_SLEEPING:    return "sleeping";
-        default: GGML_ASSERT(false && "invalid server_state");
-    }
-}
-
-static server_state server_state_from_str(const std::string & str) {
-    if (str == "downloading") return SERVER_STATE_DOWNLOADING;
-    if (str == "loading")     return SERVER_STATE_LOADING;
-    if (str == "ready")       return SERVER_STATE_READY;
-    if (str == "sleeping")    return SERVER_STATE_SLEEPING;
-    GGML_ASSERT(false && "invalid server_state string");
-}
-
-using server_state_callback_t = std::function<void(server_state, json /* payload */)>;
-
-struct server_context {
-    std::unique_ptr<server_context_impl> impl;
-
-    server_context();
-    ~server_context();
-
-    // load the model and initialize llama_context
-    // returns true on success
-    bool load_model(common_params & params);
-
-    // this function will block main thread until termination
-    void start_loop();
-
-    // terminate main loop (will unblock start_loop)
-    void terminate();
-
-    // get the underlaying llama_context, can return nullptr if sleeping
-    // not thread-safe, should only be used from the main thread
-    llama_context * get_llama_context() const;
-
-    // get a new response reader, used by CLI application
-    server_response_reader get_response_reader();
-
-    // get server metadata (read-only), can only be called after load_model()
-    // not thread-safe, should only be used from the main thread
-    server_context_meta get_meta() const;
-
-    // note: must be set before load_model() is called
-    void set_state_callback(server_state_callback_t callback);
-};
-
-
-// forward declarations
-struct server_res_generator;
-
-struct server_routes {
-    server_routes(const common_params & params, server_context & ctx_server);
-
-    void init_routes();
-
-    // note: this is not thread-safe and can only when ctx_http.is_ready is false
-    void update_meta(const server_context & ctx_server) {
-        this->meta = std::make_unique<server_context_meta>(ctx_server.get_meta());
-    }
-
-    // handlers using lambda function, so that they can capture `this` without `std::bind`
-    // they won't be called until ctx_http.is_ready is set to true
-    server_http_context::handler_t get_health;
-    server_http_context::handler_t get_metrics;
-    server_http_context::handler_t get_slots;
-    server_http_context::handler_t post_slots;
-    server_http_context::handler_t get_props;
-    server_http_context::handler_t post_props;
-    server_http_context::handler_t post_infill;
-    server_http_context::handler_t post_completions;
-    server_http_context::handler_t post_completions_oai;
-    server_http_context::handler_t post_chat_completions;
-    server_http_context::handler_t post_chat_completions_tok;
-    server_http_context::handler_t post_control;
-    server_http_context::handler_t post_responses_oai;
-    server_http_context::handler_t post_responses_tok_oai;
-    server_http_context::handler_t post_transcriptions_oai;
-    server_http_context::handler_t post_anthropic_messages;
-    server_http_context::handler_t post_anthropic_count_tokens;
-    server_http_context::handler_t post_apply_template;
-    server_http_context::handler_t get_models;
-    server_http_context::handler_t post_tokenize;
-    server_http_context::handler_t post_detokenize;
-    server_http_context::handler_t post_embeddings;
-    server_http_context::handler_t post_embeddings_oai;
-    server_http_context::handler_t post_rerank;
-    server_http_context::handler_t post_systemone;
-    server_http_context::handler_t get_lora_adapters;
-    server_http_context::handler_t post_lora_adapters;
-
-    // to be used in router mode
-    json get_model_info() const;
-
-private:
-    std::unique_ptr<server_res_generator> handle_completions_impl(
-            const server_http_req & req,
-            server_task_type type,
-            const json & data,
-            const std::vector<raw_buffer> & files,
-            task_response_type res_type,
-            const common_chat_session & chat_session = {});
-    std::unique_ptr<server_res_generator> handle_slots_save(const server_http_req & req, int id_slot);
-    std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
-    std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
-    std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
-    std::unique_ptr<server_res_generator> handle_count_tokens(const server_http_req & req, task_response_type res_type);
-
-    // using unique_ptr to allow late initialization of const
-    std::unique_ptr<const server_context_meta> meta;
-
-    const common_params & params;
-    server_context_impl & ctx_server;
-
-    server_queue & queue_tasks;
-    server_response & queue_results;
-    std::unique_ptr<server_res_generator> create_response(bool bypass_sleep = false);
-
-    // cached responses, to be used during sleep
-    std::mutex     mutex_cache;
-    json           cached_models  = nullptr;
-    json           cached_props   = nullptr;
-    server_metrics cached_metrics;
-    // set when a scrape during sleep already reported the throughput buckets
-    bool           should_reset_buckets = false;
-    // call right before sleep to update the cached responses
-    void update_cached_responses(bool is_sleeping);
-};

tools/server/server-decision.cpp

@@ -87,7 +87,7 @@ void server_decision_context::init(const llama_model * model) {
         bias[level] = std::strtof(val, nullptr);
     }
 
-    if (model_type == COMMON_DECISION_TYPE_OPENJEV) {
+    if (model_type == COMMON_DECISION_TYPE_OPENJEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) {
         // one letter per option, each must be a single token
         const std::string letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
         for (const char c : letters) {
@@ -753,7 +753,7 @@ void server_decision_context::fill_task(
 
     const std::string prompt = render(state, questions, question, variant, files.size());
 
-    if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) {
+    if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_GRANITE_SWITCH || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) {
         // lev reads the ratings of a noul question at its first labels, not at the digits
         task.decision.labels.assign(labels.begin(), labels.begin() + n_outputs(question));
     }

tools/server/server-decision.h

@@ -39,6 +39,7 @@ struct server_decision_context {
     bool can_share_prompt() const {
         switch (type) {
             case COMMON_DECISION_TYPE_OPENJEV:
+            case COMMON_DECISION_TYPE_GRANITE_SWITCH:
             case COMMON_DECISION_TYPE_LEV:
             case COMMON_DECISION_TYPE_KEV:
             case COMMON_DECISION_TYPE_NIMBLE:

tools/server/server-http.cpp

@@ -1,7 +1,7 @@
 #include "common.h"
 #include "http.h"
 #include "server-http.h"
-#include "server-common.h"
+#include "engine-common.h"
 #include "ui.h"
 
 #include <cpp-httplib/httplib.h>

tools/server/server-http.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "engine-common.h" // raw_buffer, uploaded_file
+
 #include <atomic>
 #include <functional>
 #include <map>
@@ -38,13 +40,6 @@ struct server_http_res {
 // unique pointer, used by set_chunked_content_provider
 // httplib requires the stream provider to be stored in heap
 using server_http_res_ptr = std::unique_ptr<server_http_res>;
-using raw_buffer = std::vector<uint8_t>;
-
-struct uploaded_file {
-    raw_buffer data;
-    std::string filename;
-    std::string content_type;
-};
 
 struct server_http_req {
     std::map<std::string, std::string> params; // path_params + query_params

tools/server/server-mcp.h

@@ -1,6 +1,7 @@
 #pragma once
 
-#include "server-common.h"
+#include "engine-common.h"
+#include "server-subproc.h"
 
 #include <atomic>
 #include <chrono>

tools/server/server-models.cpp

@@ -1,7 +1,7 @@
-#include "server-common.h"
+#include "engine-common.h"
 #include "http.h"
 #include "server-models.h"
-#include "server-context.h"
+#include "engine-context.h"
 #include "server-stream.h"
 
 #include "build-info.h"
@@ -539,7 +539,7 @@ void server_model_meta::update_args(common_preset_context & ctx_preset, std::str
 
 void server_model_meta::update_caps(const common_params & base) {
     // reset to the default so a failed refresh cannot keep old values
-    architecture = server_model_architecture_json(false, false, false, {"text"});
+    architecture = engine_model_architecture_json(false, false, false, {"text"});
     n_ctx_train  = 0;
 
     // resolve the model file offline; do not download
@@ -569,7 +569,7 @@ void server_model_meta::update_caps(const common_params & base) {
     std::vector<std::string> output_modalities = {"text"};
     if (!params.model.path.empty()) {
         const common_gguf_info info = common_get_gguf_info(params.model.path);
-        output_modalities = server_model_output_modalities(info.decision_type);
+        output_modalities = engine_model_output_modalities(info.decision_type);
         n_ctx_train       = info.n_ctx_train;
     }
 
@@ -587,7 +587,7 @@ void server_model_meta::update_caps(const common_params & base) {
     }
 
     // offline discovery cannot see video; a loaded model reports it
-    architecture = server_model_architecture_json(inp_image, inp_audio, false, output_modalities);
+    architecture = engine_model_architecture_json(inp_image, inp_audio, false, output_modalities);
 }
 
 //
@@ -687,7 +687,7 @@ void server_models::add_model(server_model_meta && meta) {
 }
 
 void server_models::notify_sse(const std::string & event, const std::string & model_id, const json & data) {
-    std::unique_ptr<server_task_result_router> result = std::make_unique<server_task_result_router>();
+    std::unique_ptr<engine_task_result_router> result = std::make_unique<engine_task_result_router>();
     result->data = {
         {"model", model_id},
         {"event", event},
@@ -1660,12 +1660,12 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co
 }
 
 void server_models::handle_child_state(const std::string & name, const std::string & raw_input) {
-    server_state state;
+    engine_state state;
     json payload;
 
     try {
         json data = json::parse(raw_input.substr(strlen(CMD_CHILD_TO_ROUTER_STATE)));
-        state = server_state_from_str(json_value(data, "state", std::string()));
+        state = engine_state_from_str(json_value(data, "state", std::string()));
         payload = json_value(data, "payload", json{});
     } catch (const std::exception & e) {
         SRV_ERR("failed to parse child state update for name=%s: %s\n", name.c_str(), e.what());
@@ -1673,7 +1673,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri
     }
 
     switch (state) {
-        case SERVER_STATE_DOWNLOADING:
+        case ENGINE_STATE_DOWNLOADING:
             {
                 std::string result = json_value(payload, "result", std::string());
                 std::string url    = json_value(payload, "url",    std::string());
@@ -1698,7 +1698,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri
                     update_download_progress(name, p, false);
                 }
             } break;
-        case SERVER_STATE_LOADING:
+        case ENGINE_STATE_LOADING:
             {
                 update_status(name, {
                     SERVER_MODEL_STATUS_LOADING,
@@ -1707,7 +1707,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri
                     payload,
                 });
             } break;
-        case SERVER_STATE_READY:
+        case ENGINE_STATE_READY:
             {
                 update_status(name, {
                     SERVER_MODEL_STATUS_LOADED,
@@ -1717,7 +1717,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri
                     {}, // reset progress info
                 });
             } break;
-        case SERVER_STATE_SLEEPING:
+        case ENGINE_STATE_SLEEPING:
             {
                 update_status(name, { SERVER_MODEL_STATUS_SLEEPING });
             } break;
@@ -1784,7 +1784,7 @@ struct server_download_state : public common_download_callback {
             {"downloaded", p.downloaded},
             {"total", p.total},
         };
-        self->notify_to_router(server_state_to_str(SERVER_STATE_DOWNLOADING), data);
+        self->notify_to_router(engine_state_to_str(ENGINE_STATE_DOWNLOADING), data);
     }
     void on_start(const common_download_progress & p) override {
         on_progress(p);
@@ -1820,7 +1820,7 @@ int server_child::run_download(common_params & params) {
 
     bool ok = dl.run(params);
 
-    notify_to_router(server_state_to_str(SERVER_STATE_DOWNLOADING), {
+    notify_to_router(engine_state_to_str(ENGINE_STATE_DOWNLOADING), {
         {"result", ok ? "download_finished" : "download_failed"},
     });
 
@@ -1874,12 +1874,12 @@ void server_child::notify_to_router(const std::string & state, const json & payl
 // server_models_routes
 //
 
-// RAII wrapper similar to server_response_reader, but doesn't use server_queue
+// RAII wrapper similar to engine_response_reader, but doesn't use engine_queue
 static std::atomic<int> sse_client_id_counter = 0;
 struct server_models_sse_client {
-    server_response & queue_results;
+    engine_response & queue_results;
     int client_id;
-    server_models_sse_client(server_response & q)
+    server_models_sse_client(engine_response & q)
             : queue_results(q), client_id(sse_client_id_counter.fetch_add(1, std::memory_order_relaxed)) {
         SRV_DBG("new SSE client connected, assigned client_id=%d\n", client_id);
         queue_results.add_waiting_task_id(client_id);
@@ -1891,10 +1891,10 @@ struct server_models_sse_client {
 
     // return nullptr if should_stop() is true before receiving a result
     // note: if one error is received, it will stop further processing and return error result
-    server_task_result_ptr next(const std::function<bool()> & should_stop) {
+    engine_task_result_ptr next(const std::function<bool()> & should_stop) {
         while (true) {
             static const int http_polling_seconds = 1; // check should_stop every 1 second
-            server_task_result_ptr result = queue_results.recv_with_timeout({client_id}, http_polling_seconds);
+            engine_task_result_ptr result = queue_results.recv_with_timeout({client_id}, http_polling_seconds);
             if (result == nullptr) {
                 // timeout, check stop condition
                 if (should_stop()) {

tools/server/server-models.h

@@ -3,9 +3,10 @@
 #include "common.h"
 #include "download.h"
 #include "preset.h"
-#include "server-common.h"
+#include "engine-common.h"
 #include "server-http.h"
-#include "server-queue.h"
+#include "engine-queue.h"
+#include "server-subproc.h"
 
 #include <mutex>
 #include <condition_variable>
@@ -85,7 +86,7 @@ struct server_model_meta {
     int exit_code = 0; // exit code of the model instance process (only valid if status == FAILED)
     int stop_timeout = 0; // seconds to wait before force-killing the model instance during shutdown
     bool hidden = false; // hidden from GET /models, but still accept if requested
-    json architecture = server_model_architecture_json(false, false, false, {"text"});
+    json architecture = engine_model_architecture_json(false, false, false, {"text"});
     uint32_t n_ctx_train = 0; // trained context, read from the GGUF metadata; 0 when unknown
 
     bool is_ready() const {
@@ -237,7 +238,7 @@ public:
     server_models(const common_params & params, int argc, char ** argv);
     ~server_models();
 
-    server_response sse; // for real-time updates via SSE endpoint
+    engine_response sse; // for real-time updates via SSE endpoint
 
     // (re-)load the list of models from various sources and prepare the metadata mapping
     // - if this is called the first time, simply populate the metadata

tools/server/server-routes.h

@@ -0,0 +1,90 @@
+#pragma once
+
+#include "engine-context.h"
+#include "server-http.h"
+
+#include "json.h"
+
+#include <memory>
+#include <mutex>
+
+// forward declarations
+struct server_res_generator;
+
+struct server_routes {
+    server_routes(const common_params & params, engine_context & ctx_server);
+
+    void init_routes();
+
+    // note: this is not thread-safe and can only when ctx_http.is_ready is false
+    void update_meta(const engine_context & ctx_server) {
+        this->meta = std::make_unique<engine_context_meta>(ctx_server.get_meta());
+    }
+
+    // handlers using lambda function, so that they can capture `this` without `std::bind`
+    // they won't be called until ctx_http.is_ready is set to true
+    server_http_context::handler_t get_health;
+    server_http_context::handler_t get_metrics;
+    server_http_context::handler_t get_slots;
+    server_http_context::handler_t post_slots;
+    server_http_context::handler_t get_props;
+    server_http_context::handler_t post_props;
+    server_http_context::handler_t post_infill;
+    server_http_context::handler_t post_completions;
+    server_http_context::handler_t post_completions_oai;
+    server_http_context::handler_t post_chat_completions;
+    server_http_context::handler_t post_chat_completions_tok;
+    server_http_context::handler_t post_control;
+    server_http_context::handler_t post_responses_oai;
+    server_http_context::handler_t post_responses_tok_oai;
+    server_http_context::handler_t post_transcriptions_oai;
+    server_http_context::handler_t post_anthropic_messages;
+    server_http_context::handler_t post_anthropic_count_tokens;
+    server_http_context::handler_t post_apply_template;
+    server_http_context::handler_t get_models;
+    server_http_context::handler_t post_tokenize;
+    server_http_context::handler_t post_detokenize;
+    server_http_context::handler_t post_embeddings;
+    server_http_context::handler_t post_embeddings_oai;
+    server_http_context::handler_t post_rerank;
+    server_http_context::handler_t post_systemone;
+    server_http_context::handler_t get_lora_adapters;
+    server_http_context::handler_t post_lora_adapters;
+
+    // to be used in router mode
+    json get_model_info() const;
+
+private:
+    std::unique_ptr<server_res_generator> handle_completions_impl(
+            const server_http_req & req,
+            engine_task_type type,
+            const json & data,
+            const std::vector<raw_buffer> & files,
+            task_response_type res_type,
+            const common_chat_session & chat_session = {});
+    std::unique_ptr<server_res_generator> handle_slots_save(const server_http_req & req, int id_slot);
+    std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
+    std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
+    std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
+    std::unique_ptr<server_res_generator> handle_count_tokens(const server_http_req & req, task_response_type res_type);
+
+    // using unique_ptr to allow late initialization of const
+    std::unique_ptr<const engine_context_meta> meta;
+
+    const common_params & params;
+    engine_context & ctx_server;
+
+    engine_queue & queue_tasks;
+    engine_response & queue_results;
+    std::unique_ptr<server_res_generator> create_response(bool bypass_sleep = false);
+
+    // cached responses, to be used during sleep
+    std::mutex     mutex_cache;
+    json           cached_models  = nullptr;
+    json           cached_props   = nullptr;
+    engine_metrics cached_metrics;
+    // set when a scrape during sleep already reported the throughput buckets
+    bool           should_reset_buckets = false;
+    // call right before sleep to update the cached responses
+    void update_cached_responses(bool is_sleeping);
+};

tools/server/server-stream.cpp

@@ -1,7 +1,7 @@
 #include "server-stream.h"
-#include "server-common.h"
+#include "engine-common.h"
 #include "server-http.h"
-#include "server-queue.h"
+#include "engine-queue.h"
 
 #include <chrono>
 #include <memory>

tools/server/server-subproc.cpp

@@ -0,0 +1,196 @@
+#include "server-subproc.h"
+
+#include "log.h"
+
+#include <algorithm>
+#include <chrono>
+#include <cstring>
+#include <limits>
+#include <thread>
+
+#ifdef _WIN32
+// windows.h defines min and max as macros, which breaks std::min and std::max
+#define WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#   define NOMINMAX
+#endif
+#include <windows.h>
+#include <io.h>
+#else
+#include <errno.h>
+#include <fcntl.h>
+#include <poll.h>
+#include <unistd.h>
+#endif
+
+//
+// server_subproc
+//
+
+FILE * server_reserve_stdout() {
+    fflush(stdout);
+    // the reserved stream is not inherited, grandchildren get stdout and stderr of their own
+#ifdef _WIN32
+    int fd = _dup(_fileno(stdout));
+    GGML_ASSERT(fd >= 0);
+    SetHandleInformation((HANDLE) _get_osfhandle(fd), HANDLE_FLAG_INHERIT, 0);
+    _dup2(_fileno(stderr), _fileno(stdout));
+    SetStdHandle(STD_OUTPUT_HANDLE, GetStdHandle(STD_ERROR_HANDLE));
+    FILE * f = _fdopen(fd, "w");
+#else
+    int fd = fcntl(fileno(stdout), F_DUPFD_CLOEXEC, 0);
+    GGML_ASSERT(fd >= 0);
+    dup2(fileno(stderr), fileno(stdout));
+    FILE * f = fdopen(fd, "w");
+#endif
+    GGML_ASSERT(f);
+    return f;
+}
+
+bool server_subproc::has_output() {
+    FILE * files[SERVER_SUBPROC_STREAMS] = { sproc.stdout_file(), sproc.stderr_file() };
+    for (int i = 0; i < SERVER_SUBPROC_STREAMS; i++) {
+        if (out_handles[i] >= 0) {
+            continue;
+        }
+        if (!files[i]) {
+            return false;
+        }
+#ifdef _WIN32
+        HANDLE h = (HANDLE) _get_osfhandle(_fileno(files[i]));
+        if (h == INVALID_HANDLE_VALUE) {
+            return false;
+        }
+        out_handles[i] = (intptr_t) h;
+#else
+        int fd = fileno(files[i]);
+        if (fd < 0) {
+            return false;
+        }
+        fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
+        out_handles[i] = fd;
+#endif
+    }
+    return true;
+}
+
+bool server_subproc::output_closed() const {
+    return out_closed[SERVER_SUBPROC_STDOUT] && out_closed[SERVER_SUBPROC_STDERR];
+}
+
+int server_subproc::read_output(server_subproc_stream stream, char * buf, size_t len) {
+    if (out_closed[stream]) {
+        return -1;
+    }
+    if (!has_output()) {
+        out_closed[stream] = true;
+        return -1;
+    }
+#ifdef _WIN32
+    HANDLE h     = (HANDLE) out_handles[stream];
+    DWORD  avail = 0;
+    if (!PeekNamedPipe(h, NULL, 0, NULL, &avail, NULL)) {
+        out_closed[stream] = true; // pipe broken, child gone
+        return -1;
+    }
+    if (avail == 0) {
+        return 0;
+    }
+    DWORD to_read = avail < (DWORD) len ? avail : (DWORD) len;
+    DWORD got     = 0;
+    if (!ReadFile(h, buf, to_read, &got, NULL) || got == 0) {
+        out_closed[stream] = true;
+        return -1;
+    }
+    return (int) got;
+#else
+    while (true) {
+        ssize_t r = read((int) out_handles[stream], buf, len);
+        if (r > 0) {
+            return (int) r;
+        }
+        if (r < 0 && errno == EINTR) {
+            continue;
+        }
+        if (r < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
+            return 0;
+        }
+        out_closed[stream] = true; // EOF or error
+        return -1;
+    }
+#endif
+}
+
+server_subproc::waiter::waiter() {
+#ifndef _WIN32
+    int fds[2];
+    GGML_ASSERT(pipe(fds) == 0);
+    for (int fd : fds) {
+        fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
+    }
+    wake_fd[0] = fds[0];
+    wake_fd[1] = fds[1];
+#endif
+}
+
+server_subproc::waiter::~waiter() {
+#ifndef _WIN32
+    close((int) wake_fd[0]);
+    close((int) wake_fd[1]);
+#endif
+}
+
+void server_subproc::waiter::wake() {
+#ifndef _WIN32
+    char c = 1;
+    (void) !write((int) wake_fd[1], &c, 1);
+#endif
+}
+
+void server_subproc::waiter::wait(const std::vector<server_subproc *> & procs, std::vector<bool> & ready, int64_t timeout_ms) {
+    ready.assign(procs.size(), false);
+#ifdef _WIN32
+    // no waitable wait exists for anonymous pipes, so poll them in 50 ms steps
+    bool any = false;
+    for (size_t i = 0; i < procs.size(); i++) {
+        server_subproc * p = procs[i];
+        ready[i] = !p->has_output();
+        for (int s = 0; s < SERVER_SUBPROC_STREAMS && !ready[i]; s++) {
+            DWORD avail = 0;
+            if (!p->out_closed[s] && (!PeekNamedPipe((HANDLE) p->out_handles[s], NULL, 0, NULL, &avail, NULL) || avail > 0)) {
+                ready[i] = true; // data or broken pipe, read_output() tells which
+            }
+        }
+        any = any || ready[i];
+    }
+    if (!any) {
+        int64_t step = timeout_ms < 0 ? 50 : std::min<int64_t>(timeout_ms, 50);
+        std::this_thread::sleep_for(std::chrono::milliseconds(step));
+    }
+#else
+    std::vector<pollfd> pfds;
+    pfds.reserve(procs.size() * SERVER_SUBPROC_STREAMS + 1);
+    pfds.push_back({ (int) wake_fd[0], POLLIN, 0 });
+    for (auto * p : procs) {
+        const bool open = p->has_output();
+        for (int s = 0; s < SERVER_SUBPROC_STREAMS; s++) {
+            pfds.push_back({ open && !p->out_closed[s] ? (int) p->out_handles[s] : -1, POLLIN, 0 }); // poll() skips negative fds
+        }
+    }
+    int timeout = timeout_ms < 0 ? -1 : (int) std::min<int64_t>(timeout_ms, std::numeric_limits<int>::max());
+    int r = poll(pfds.data(), pfds.size(), timeout);
+    if (r < 0 && errno != EINTR) {
+        LOG_ERR("%s: poll() failed: %s\n", __func__, strerror(errno));
+    }
+    if (pfds[0].revents) {
+        char buf[64];
+        while (read((int) wake_fd[0], buf, sizeof(buf)) > 0) {}
+    }
+    for (size_t i = 0; i < procs.size(); i++) {
+        ready[i] = !procs[i]->has_output();
+        for (int s = 0; s < SERVER_SUBPROC_STREAMS; s++) {
+            ready[i] = ready[i] || pfds[1 + i * SERVER_SUBPROC_STREAMS + s].revents != 0;
+        }
+    }
+#endif
+}

tools/server/server-subproc.h

@@ -0,0 +1,129 @@
+#pragma once
+
+#include "subproc.h"
+
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <cstdint>
+#include <cstdio>
+#include <functional>
+#include <mutex>
+#include <queue>
+#include <vector>
+
+// simple implementation of a pipe
+// used for streaming data between threads
+template<typename T>
+struct server_pipe {
+    std::mutex mutex;
+    std::condition_variable cv;
+    std::queue<T> queue;
+    std::atomic<bool> writer_closed{false};
+    std::atomic<bool> reader_closed{false};
+
+    // 0 = unbounded (default)
+    // > 0, write() drops the oldest item once the queue is full
+    size_t max_size = 0;
+
+    void close_write() {
+        writer_closed.store(true, std::memory_order_relaxed);
+        cv.notify_all();
+    }
+
+    void close_read() {
+        reader_closed.store(true, std::memory_order_relaxed);
+        cv.notify_all();
+    }
+
+    // close_on_stop = true: should_stop means the reader is gone for good, so the writer is told the pipe is broken.
+    // close_on_stop = false: should_stop is a per-read deadline and further reads still come, so the pipe stays usable.
+    bool read(T & output, const std::function<bool()> & should_stop, bool close_on_stop = true) {
+        std::unique_lock<std::mutex> lk(mutex);
+        constexpr auto poll_interval = std::chrono::milliseconds(500);
+        while (true) {
+            if (!queue.empty()) {
+                output = std::move(queue.front());
+                queue.pop();
+                return true;
+            }
+            if (writer_closed.load()) {
+                return false; // clean EOF
+            }
+            if (should_stop && should_stop()) { // a null should_stop means "never stop"
+                if (close_on_stop) {
+                    close_read(); // signal broken pipe to writer
+                }
+                return false; // cancelled / deadline reached
+            }
+            cv.wait_for(lk, poll_interval);
+        }
+    }
+
+    bool write(T && data) {
+        std::lock_guard<std::mutex> lk(mutex);
+        if (reader_closed.load()) {
+            return false; // broken pipe
+        }
+        if (max_size > 0) {
+            while (queue.size() >= max_size) {
+                queue.pop(); // drop oldest to stay bounded
+            }
+        }
+        queue.push(std::move(data));
+        cv.notify_one();
+        return true;
+    }
+};
+
+// a child server writes its state commands to stdout and its logs to stderr
+enum server_subproc_stream {
+    SERVER_SUBPROC_STDOUT,
+    SERVER_SUBPROC_STDERR,
+    SERVER_SUBPROC_STREAMS,
+};
+
+// gives the current stdout to the caller as a stream of its own, and sends everything else written to stdout to stderr
+FILE * server_reserve_stdout();
+
+// wrapper around common_subproc to manage a child server process
+// mainly used by router mode
+struct server_subproc {
+    common_subproc sproc;
+    std::atomic<bool> stopped{false}; // set by the monitor once the process exited and was reaped
+
+    bool is_alive() { return sproc.alive(); }
+    void terminate() { sproc.terminate(); }
+    int  join() { return sproc.join(); }
+
+    // true if both output pipes of the child are available (call after create())
+    bool has_output();
+
+    // true once both output pipes are closed
+    bool output_closed() const;
+
+    // non-blocking read from one output pipe
+    // returns the number of bytes read, 0 when nothing is available, -1 when the pipe is closed or broken
+    int read_output(server_subproc_stream stream, char * buf, size_t len);
+
+    // wait until one of a set of children has output, wake() is called, or a timeout passes
+    struct waiter {
+        waiter();
+        ~waiter();
+
+        // thread-safe; on Windows this is a no-op, wait() returns within 50 ms anyway
+        void wake();
+
+        // timeout_ms < 0 waits until data or wake(); ready[i] is set for each child with data on an open pipe (or a broken pipe)
+        void wait(const std::vector<server_subproc *> & procs, std::vector<bool> & ready, int64_t timeout_ms);
+
+    private:
+#ifndef _WIN32
+        intptr_t wake_fd[2] = { -1, -1 }; // POSIX self-pipe
+#endif
+    };
+
+private:
+    intptr_t out_handles[SERVER_SUBPROC_STREAMS] = { -1, -1 }; // fd on POSIX, HANDLE on Windows; taken lazily from sproc
+    bool     out_closed [SERVER_SUBPROC_STREAMS] = { false, false };
+};

tools/server/server-tools.cpp

@@ -1755,7 +1755,7 @@ struct server_tool_get_info : server_tool {
     }
 };
 
-struct server_tool_stream_result : server_task_result {
+struct server_tool_stream_result : engine_task_result {
     std::string chunk;
     bool done = false;
     std::string error_msg;
@@ -1783,7 +1783,7 @@ void server_tool::stream::push(const std::string & chunk) {
 
 struct server_tools_res : server_http_res {
     std::thread worker;
-    server_response * qr = nullptr; // set only for streaming responses
+    engine_response * qr = nullptr; // set only for streaming responses
     int id = -1;
 
     ~server_tools_res() override {

tools/server/server-tools.h

@@ -1,8 +1,8 @@
 #pragma once
 
-#include "server-common.h"
+#include "engine-common.h"
 #include "server-http.h"
-#include "server-queue.h"
+#include "engine-queue.h"
 #include "server-mcp.h"
 
 #include <atomic>
@@ -21,7 +21,7 @@ struct server_tool {
     virtual std::string type() const { return "server"; }
 
     struct stream {
-        server_response & qr;
+        engine_response & qr;
         int id;
         std::function<bool()> alive;
         void push(const std::string & chunk);
@@ -37,7 +37,7 @@ struct server_tools {
     std::vector<std::unique_ptr<server_tool>> tools;
 
     // for streaming
-    server_response queue_res;
+    engine_response queue_res;
     std::atomic<int> res_id{0};
 
     // set when --tools-runtime is configured; routes every tool call through an isolate

tools/server/server.cpp

@@ -1,4 +1,5 @@
-#include "server-context.h"
+#include "engine-context.h"
+#include "server-routes.h"
 #include "server-http.h"
 #include "server-models.h"
 #include "server-cors-proxy.h"
@@ -195,7 +196,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
     server_mcp mcp_mgr;
 
     // struct that contains llama context and inference
-    server_context ctx_server;
+    engine_context ctx_server;
 
     //
     // Router
@@ -480,8 +481,8 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
 
         // setup communication child --> router if necessary
         if (child.is_child()) {
-            ctx_server.set_state_callback([&](server_state state, json payload) {
-                child.notify_to_router(server_state_to_str(state), payload);
+            ctx_server.set_state_callback([&](engine_state state, json payload) {
+                child.notify_to_router(engine_state_to_str(state), payload);
             });
         }
 
@@ -540,7 +541,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
         std::thread monitor_thread;
         if (child.is_child()) {
             monitor_thread = child.setup(shutdown_handler);
-            child.notify_to_router(server_state_to_str(SERVER_STATE_READY), routes.get_model_info());
+            child.notify_to_router(engine_state_to_str(ENGINE_STATE_READY), routes.get_model_info());
         }
 
         // this call blocks the main thread until queue_tasks.terminate() is called

tools/server/tests/unit/test_slot_save.py

@@ -96,7 +96,7 @@ def test_slot_restore_legacy_token_list():
     assert res.status_code == 200
     assert res.body["n_saved"] == 84
 
-    # rewrite the token payload into a plain token list, as written by servers that predate the packed server_tokens format
+    # rewrite the token payload into a plain token list, as written by servers that predate the packed engine_tokens format
     path = os.path.join(server.slot_save_path, "slot_legacy.bin")
     with open(path, "rb") as f:
         data = bytearray(f.read())