sycl: fix prompt-processing regression after Q4_0/Q8_0 weight reorder (#30374)

* skills: add slice-model

* exclude from py check

* add to agents.md

.flake8

@@ -5,6 +5,8 @@ exclude =
     # Do not traverse examples and tools
     examples,
     tools,
+    # Reference scripts for skills
+    skills,
     # Do not include package initializers
     __init__.py,
     # No need to traverse our git directory

.github/labeler.yml

@@ -46,7 +46,6 @@ examples:
                 - examples/**
                 - tools/**
             - all-globs-to-all-files:
-                - '!tools/engine/**'
                 - '!tools/server/**'
                 - '!tools/mtmd/**'
                 - '!tools/ui/**'
@@ -77,7 +76,6 @@ server/ui:
 server:
     - changed-files:
         - any-glob-to-any-file:
-            - tools/engine/**
             - tools/server/**
 mtmd:
     - changed-files:

.github/workflows/build-and-test-snapdragon.yml

@@ -104,11 +104,11 @@ jobs:
     name: Test on QDC Device (${{ matrix.device }})
     needs: [android-ndk-snapdragon, linux-iot-snapdragon]
     runs-on: ubuntu-24.04-arm
-    timeout-minutes: 90
+    timeout-minutes: 180
     strategy:
       fail-fast: false
       matrix:
-        device: [SM8750, SM8850, QCS9075M]
+        device: [SM8850, QCS9075M]
 
     permissions:
       actions: read
@@ -155,7 +155,7 @@ jobs:
               --pkg-dir    pkg-snapdragon/llama.cpp \
               --model-url  "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf" \
               --device     ${{ matrix.device }} \
-              ${{ startsWith(matrix.device, 'QCS') && '--retries 2 --retry-delay 300' || '' }}
+              ${{ startsWith(matrix.device, 'QCS') && '--job-timeout 5400 --retries 2 --retry-delay 300' || '' }}
         env:
           QDC_API_KEY: ${{ secrets.QDC_API_KEY }}
 

AGENTS.md

@@ -246,3 +246,6 @@ Chat template and parser:
 - [PEG parser](docs/development/parsing.md) - alternative to regex that llama.cpp uses to parse model's output
 - [Auto parser](docs/autoparser.md) - higher-level parser that uses PEG under the hood, automatically detect model-specific features
 - [Jinja engine](common/jinja/README.md)
+
+Testing:
+- [Slice model](skills/slice-model/SKILL.md) - build a small slice (few layers/experts) of a big HF model without downloading the full weights, to test conversion and inference

CODEOWNERS

@@ -101,7 +101,6 @@
 /tools/batched-bench/                   @ggerganov
 /tools/cli/                             @ngxson
 /tools/completion/                      @ggerganov
-/tools/engine/                          @ggml-org/llama-server
 /tools/mtmd/                            @ggml-org/llama-mtmd
 /tools/perplexity/                      @ggerganov
 /tools/parser/                          @pwilkin

common/chat.cpp

@@ -1104,6 +1104,13 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
         return common_chat_params_init_llm_jp_harmony;
     }
 
+    // Harmony XML tool calls, checked before the GPT-OSS JSON calls
+    if (src.find("<|channel|>") != std::string::npos && src.find("<|constrain|>xml") != std::string::npos &&
+        src.find("rfl:parameter") != std::string::npos) {
+        LOG_DBG("Using specialized template: Harmony XML\n");
+        return common_chat_params_init_harmony_xml;
+    }
+
     // GPT-OSS - has unique channel-based structure that needs dedicated handler
     if (src.find("<|channel|>") != std::string::npos) {
         LOG_DBG("Using specialized template: GPT-OSS\n");

common/jinja/value.cpp

@@ -1299,6 +1299,15 @@ const func_builtins & value_object_t::get_builtins() const {
             }
             return result;
         }},
+        {"list", [](const func_args & args) -> value {
+            args.ensure_vals<value_object>();
+            const auto & obj = args.get_pos(0)->as_ordered_object();
+            auto result = mk_val<value_array>();
+            for (const auto & pair : obj) {
+                result->push_back(pair.first);
+            }
+            return result;
+        }},
         {"values", [](const func_args & args) -> value {
             args.ensure_vals<value_object>();
             const auto & obj = args.get_pos(0)->as_ordered_object();

common/parsers/harmony-xml.cpp

@@ -0,0 +1,207 @@
+#include "parsers.h"
+
+// Harmony with XML tool calls: <|start|>assistant to=NAME<|channel|>analysis <|constrain|>xml<|message|><rfl:parameter name="KEY">\nVALUE\n</rfl:parameter>
+common_chat_params common_chat_params_init_harmony_xml(const common_chat_template &          tmpl,
+                                                       const autoparser::generation_params & inputs) {
+    common_chat_params data;
+
+    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);
+    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
+    data.message_delimiters = {
+        { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },
+        { COMMON_CHAT_ROLE_USER,      "<|start|>user"      },
+        { COMMON_CHAT_ROLE_SYSTEM,    "<|start|>developer" },
+        { COMMON_CHAT_ROLE_SYSTEM,    "<|start|>system"    },
+    };
+
+    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
+    data.supports_thinking = true;
+
+    data.thinking_start_tag = "<|channel|>analysis<|message|>";
+    data.thinking_end_tags  = {"<|end|>"};
+
+    // EOG text must reach the parser: an XML call has no closing tag of its own
+    static const std::vector<std::string> EOG = {
+        "<|call|>", "<|return|>", "<|endoftext|>", "<|im_end|>", "<|fim_pad|>", "<|repo_name|>", "<|file_sep|>",
+    };
+
+    data.preserved_tokens = {
+        "<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",
+    };
+    data.preserved_tokens.insert(data.preserved_tokens.end(), EOG.begin(), EOG.end());
+
+    if (inputs.has_continuation()) {
+        const auto & msg = inputs.continue_msg;
+
+        data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content;
+        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+            data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content();
+        }
+
+        data.prompt.append(data.generation_prompt);
+    }
+
+    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
+    auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
+    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
+    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
+
+    static const std::string PARAM_OPEN  = "<rfl:parameter name=\"";
+    static const std::string PARAM_CLOSE = "\n</rfl:parameter>";
+
+    data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+        auto start = p.rule("start", p.literal("<|start|>assistant"));
+
+        auto eog = p.choice();
+        for (const auto & tok : EOG) {
+            eog |= p.literal(tok);
+        }
+        auto with_eog = [&](std::vector<std::string> delims) {
+            delims.insert(delims.end(), EOG.begin(), EOG.end());
+            return delims;
+        };
+
+        auto analysis_open  = p.literal("<|channel|>analysis<|message|>");
+        auto analysis_body  = p.until_one_of(with_eog({ "<|end|>", "<|constrain|><|start|>assistant", "<|start|>assistant" }));
+        auto analysis_close = p.optional(p.literal("<|end|>") | p.literal("<|constrain|>") | eog);
+        if (extract_reasoning) {
+            p.rule("analysis", analysis_open + p.reasoning(analysis_body) + analysis_close);
+        } else {
+            p.rule("analysis", p.atomic(p.content(analysis_open)) + p.content(analysis_body + analysis_close));
+        }
+        auto analysis = p.ref("analysis");
+
+        auto final_msg = p.rule("final",
+            p.literal("<|channel|>final<|message|>") +
+            p.content(p.until_one_of(with_eog({ "<|end|>" }))) +
+            p.optional(p.literal("<|end|>") | eog));
+
+        if (has_response_format) {
+            auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1);
+            auto constraint     = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
+            auto response_format = p.rule("response-format",
+                p.literal("<|channel|>final") + constraint + p.literal("<|message|>") +
+                p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
+                p.optional(p.literal("<|end|>") | eog));
+
+            return start + p.zero_or_more(analysis + start) + response_format;
+        }
+
+        if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
+            // tolerate a missing closing tag
+            std::vector<std::string> value_ends = with_eog({ PARAM_OPEN, "<|end|>" });
+            std::vector<std::string> string_ends = { PARAM_CLOSE };
+            auto next_open = p.choice();
+            for (const auto & e : value_ends) {
+                string_ends.push_back("\n" + e);
+                next_open |= p.literal(e);
+            }
+            auto arg_close = p.tool_arg_close(p.gbnf(
+                p.literal(PARAM_CLOSE) | (p.literal("\n") + p.peek(p.optional(p.literal("\n")) + next_open)),
+                gbnf_format_literal("\n") + " " + gbnf_format_literal(PARAM_CLOSE.substr(1)) + "?"));
+            auto arg_string = p.rule("rfl-arg-string",
+                p.ac(p.tool_arg_string_value(p.until_one_of(string_ends)) + arg_close, PARAM_CLOSE));
+            auto arg_sep = p.optional(p.literal("\n"));
+
+            std::string call_close_gbnf = gbnf_format_literal("<|end|>");
+            for (const auto & tok : EOG) {
+                call_close_gbnf += " | " + gbnf_format_literal(tok);
+            }
+            auto call_close = p.gbnf(p.literal("<|end|>") | eog | p.peek(p.literal("<|start|>assistant")),
+                "(" + call_close_gbnf + ")?");
+
+            auto tool_choice = p.choice();
+            foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
+                const auto & function = tool.at("function");
+                std::string  name     = function.at("name");
+
+                std::vector<common_peg_parser> arg_rules;
+                foreach_parameter(function, [&](size_t param_index, const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
+                    auto rule_name = "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index);
+                    auto arg_open  = p.tool_arg_open(p.literal(PARAM_OPEN) + p.tool_arg_name(p.literal(param.name)) + p.literal("\">\n"));
+
+                    auto types     = param.schema->value_types();
+                    auto arg_value = p.eps();
+                    if (!types.has(common_chat_schema::TYPE_STRING) &&
+                        (types.has(common_chat_schema::TYPE_OBJECT) || types.has(common_chat_schema::TYPE_ARRAY))) {
+                        // the schema grammar fixes the key order, and the model writes keys in its own order
+                        arg_value = p.tool_arg_json_value(p.json()) + arg_close;
+                    } else if (!types.has(common_chat_schema::TYPE_STRING)) {
+                        arg_value = p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close;
+                    } else if (types.is_only(common_chat_schema::TYPE_STRING)) {
+                        arg_value = arg_string;
+                    } else {
+                        // grammar keeps only the string rule; the parser tries JSON first to type the value
+                        auto json_value = p.choice();
+                        if (types.has(common_chat_schema::TYPE_OBJECT)) {
+                            json_value |= p.json_object();
+                        }
+                        if (types.has(common_chat_schema::TYPE_ARRAY)) {
+                            json_value |= p.json_array();
+                        }
+                        if (types.has(common_chat_schema::TYPE_NUMBER) || types.has(common_chat_schema::TYPE_INTEGER)) {
+                            json_value |= p.json_number();
+                        }
+                        if (types.has(common_chat_schema::TYPE_BOOLEAN)) {
+                            json_value |= p.json_bool();
+                        }
+                        if (types.has(common_chat_schema::TYPE_NULL)) {
+                            json_value |= p.json_null();
+                        }
+                        arg_value = p.gbnf(p.atomic(p.tool_arg_json_value(json_value) + arg_close) | arg_string, "rfl-arg-string");
+                    }
+
+                    arg_rules.push_back(p.rule(rule_name, p.tool_arg(arg_open + arg_value)));
+                });
+
+                if (arg_rules.empty()) {
+                    auto any_name = p.chars("[^\"\\n<]", 1, -1);
+                    arg_rules.push_back(p.rule("tool-" + std::to_string(tool_index) + "-arg-any",
+                        p.tool_arg(p.tool_arg_open(p.literal(PARAM_OPEN) + p.tool_arg_name(any_name) + p.literal("\">\n")) + arg_string)));
+                }
+
+                auto args = p.zero_or_more(p.choice(arg_rules) + arg_sep);
+
+                auto call = p.tool(
+                    p.tool_open(p.literal(" to=") + p.tool_name(p.literal(name)) +
+                                p.literal("<|channel|>analysis <|constrain|>xml<|message|>")) +
+                    p.tool_args(args) +
+                    p.tool_close(call_close));
+
+                tool_choice |= p.rule("tool-" + std::to_string(tool_index), call);
+            });
+
+            auto tool_calls = inputs.parallel_tool_calls
+                ? tool_choice + p.zero_or_more(start + tool_choice)
+                : tool_choice;
+            auto tool_call = p.trigger_rule("tool-call", tool_calls);
+
+            if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
+                return start + p.zero_or_more(analysis + start) + tool_call;
+            }
+
+            return start + p.zero_or_more(analysis + start) + (tool_call | final_msg | p.content(p.rest()));
+        }
+
+        auto unsolicited = p.rule("unsolicited", p.atomic(p.literal(" to=") + p.rest()));
+
+        // output without a channel header is returned as content instead of failing the request
+        return start + p.zero_or_more(analysis + start) + (final_msg | unsolicited | p.content(p.rest()));
+    });
+
+    if (include_grammar) {
+        data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
+        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
+            data.parser.build_grammar(builder, data.grammar_lazy);
+        });
+
+        if (data.grammar_lazy) {
+            data.grammar_triggers = {
+                { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" },
+                { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" },
+            };
+        }
+    }
+
+    return data;
+}

common/parsers/parsers.h

@@ -59,6 +59,8 @@ common_chat_params common_chat_params_init_gigachat_v3(const common_chat_templat
 
 common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
 
+common_chat_params common_chat_params_init_harmony_xml(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
+
 common_chat_params common_chat_params_init_k2_horizon(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
 
 common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);

common/parsers/sources.cmake

@@ -9,6 +9,7 @@ set(LLAMA_CHAT_PARSERS_SOURCES
     ${CMAKE_CURRENT_LIST_DIR}/gemma4.cpp
     ${CMAKE_CURRENT_LIST_DIR}/gigachat-v3.cpp
     ${CMAKE_CURRENT_LIST_DIR}/gpt-oss.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/harmony-xml.cpp
     ${CMAKE_CURRENT_LIST_DIR}/k2-horizon.cpp
     ${CMAKE_CURRENT_LIST_DIR}/kimi-k2.cpp
     ${CMAKE_CURRENT_LIST_DIR}/kimi-k3.cpp

conversion/__init__.py

@@ -24,6 +24,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "ArcticForCausalLM": "arctic",
     "AudioFlamingo3ForConditionalGeneration": "qwen",
     "BaiChuanForCausalLM": "baichuan",
+    "BeamForCausalLM": "beam",
     "BaichuanForCausalLM": "baichuan",
     "BailingMoeForCausalLM": "bailingmoe",
     "BailingMoeV2ForCausalLM": "bailingmoe",

conversion/beam.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+from typing import Callable, Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+    from torch import Tensor
+
+from .base import ModelBase, TextModel, gguf, logger
+
+
+@ModelBase.register("BeamForCausalLM")
+class BeamModel(TextModel):
+    model_arch = gguf.MODEL_ARCH.BEAM
+
+    # reference NoPE query-scale constants; not in config.json
+    NOPE_SCALE_POWER  = 0.1
+    NOPE_SCALE_LENGTH = 2048
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        hp = self.hparams
+        rope = hp.get("rope_parameters") or {}
+        if rope.get("full_attention") is not None:
+            raise ValueError("Beam: expected NoPE full-attention layers (rope_parameters.full_attention = null)")
+        if "sliding_attention" in rope:
+            swa_rope = rope["sliding_attention"] or {}
+        else:
+            # flat rope_theta / rope_scaling, already normalized by TextModel
+            swa_rope = {k: v for k, v in self.rope_parameters.items() if k not in ("sliding_attention", "full_attention", "type")}
+            swa_rope.update(self.rope_parameters.get("sliding_attention") or {})
+        if swa_rope.get("rope_type", "default") != "default":
+            raise ValueError(f"Beam: unsupported sliding-attention rope type {swa_rope.get('rope_type')!r}")
+        # full layers are NoPE, so mirror the sliding rope params for the base class
+        swa_rope = {k: v for k, v in swa_rope.items() if not (k == "rope_type" and v == "default")}
+        self.rope_parameters = {"sliding_attention": dict(swa_rope), "full_attention": dict(swa_rope)}
+
+        layer_types = hp["layer_types"]
+        mlp_types = hp["mlp_layer_types"]
+        n_layer = hp["num_hidden_layers"]
+        if len(layer_types) != n_layer or len(mlp_types) != n_layer:
+            raise ValueError("Beam: layer_types / mlp_layer_types must have num_hidden_layers entries")
+        if any(t not in ("sliding_attention", "full_attention") for t in layer_types):
+            raise ValueError(f"Beam: unknown layer type in {layer_types}")
+        if "sliding_attention" not in layer_types:
+            raise ValueError("Beam: expected at least one sliding_attention layer")
+        n_dense = 0
+        while n_dense < n_layer and mlp_types[n_dense] == "dense":
+            n_dense += 1
+        if any(t != "sparse" for t in mlp_types[n_dense:]):
+            raise ValueError("Beam: llama.cpp supports dense MLP layers only as a leading block")
+        self._n_dense_lead = n_dense
+        if hp.get("attention_bias", False):
+            raise ValueError("Beam: attention_bias=True is not supported")
+        self._experts: dict[tuple[int, str], dict[int, Tensor]] = {}
+
+    def set_vocab(self):
+        self._set_vocab_gpt2()
+        # otherwise EOT is guessed from a hash-ordered scan
+        from transformers import AutoTokenizer
+        tok = AutoTokenizer.from_pretrained(self.dir_model)
+        ret_id = tok.convert_tokens_to_ids("<|return|>")  # ty: ignore[unresolved-attribute]
+        if isinstance(ret_id, int) and ret_id >= 0:
+            self.gguf_writer.add_eot_token_id(ret_id)
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        hp = self.hparams
+
+        head_dim = hp["head_dim"]
+        self.gguf_writer.add_rope_dimension_count(head_dim)
+
+        self.gguf_writer.add_sliding_window(hp["sliding_window"])
+        self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hp["layer_types"]])
+
+        self.gguf_writer.add_attn_temperature_scale(self.NOPE_SCALE_POWER)
+        self.gguf_writer.add_attn_temperature_length(self.NOPE_SCALE_LENGTH)
+
+        self.gguf_writer.add_leading_dense_block_count(self._n_dense_lead)
+        self.gguf_writer.add_expert_feed_forward_length(hp["moe_intermediate_size"])
+        self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
+        self.gguf_writer.add_expert_weights_norm(bool(hp.get("norm_topk_prob", True)))
+        # equivalent to the reference's post-aggregation scaling
+        self.gguf_writer.add_expert_weights_scale(float(hp["routed_scaling_factor"]))
+
+        if (softcap := hp.get("final_logit_softcapping")) is not None:
+            self.gguf_writer.add_final_logit_softcapping(float(softcap))
+
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        name, gen = item
+        if name.endswith(".self_attn.gate_bias"):
+            name = name.replace(".self_attn.gate_bias", ".self_attn.gate_proj.bias")
+        return super().filter_tensors((name, gen))
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        hp = self.hparams
+        if name.endswith(".self_attn.gate_proj.bias"):
+            assert tuple(data_torch.shape) == (hp["num_attention_heads"],), f"{name}: one gate bias per head expected, got {tuple(data_torch.shape)}"
+        elif name.endswith(".self_attn.gate_proj.weight"):
+            assert tuple(data_torch.shape) == (hp["num_attention_heads"] * hp["head_dim"], hp["hidden_size"]), f"{name}: {tuple(data_torch.shape)}"
+        elif name.endswith(".mlp.gate.e_score_correction.bias"):
+            assert tuple(data_torch.shape) == (hp["num_experts"],), f"{name}: {tuple(data_torch.shape)}"
+        elif name.endswith((".self_attn.q_norm.weight", ".self_attn.k_norm.weight")):
+            assert tuple(data_torch.shape) == (hp["head_dim"],), f"{name}: per-head q/k norm expected, got {tuple(data_torch.shape)}"
+        elif bid is not None and bid < self._n_dense_lead and ".mlp.experts." in name:
+            raise ValueError(f"Beam: expert tensor {name} in a dense layer")
+        if ".mlp.experts." in name:
+            assert bid is not None
+            n_experts = self.hparams["num_experts"]
+            parts = name.split(".")
+            xid = int(parts[parts.index("experts") + 1])
+            proj = parts[parts.index("experts") + 2]
+            if proj not in ("gate_proj", "up_proj", "down_proj"):
+                raise ValueError(f"Beam: unexpected expert tensor {name}")
+            bucket = self._experts.setdefault((bid, proj), {})
+            if xid in bucket:
+                raise ValueError(f"Beam: duplicate expert tensor {name}")
+            bucket[xid] = data_torch
+            if len(bucket) < n_experts:
+                return
+            del self._experts[(bid, proj)]
+            data = torch.stack([bucket[e] for e in range(n_experts)], dim=0)
+            yield from super().modify_tensors(data, f"model.layers.{bid}.mlp.experts.{proj}.weight", bid)
+            return
+
+        yield from super().modify_tensors(data_torch, name, bid)
+
+    def prepare_tensors(self):
+        super().prepare_tensors()
+        if self._experts:
+            missing = sorted(f"layer {b} {p} ({len(v)} experts)" for (b, p), v in self._experts.items())
+            raise ValueError(f"Beam: unprocessed experts: {missing}")
+        logger.info("Beam: all routed experts stacked")

conversion/lfm2.py

@@ -188,7 +188,7 @@ def _load_d1_omni_hparams(dir_model: Path) -> dict[str, Any]:
 class D1OmniModel(LFM2Model):
     model_arch = gguf.MODEL_ARCH.LFM2
 
-    # the server cuts the text to these lengths, see engine-decision.cpp
+    # the server cuts the text to these lengths, see server-decision.cpp
     _MAX_LENGTH = 16384
     _IMAGE_TEXT_LENGTH = 896
     _AUDIO_TEXT_LENGTH = 15360

docs/development/HOWTO-add-model.md

@@ -147,11 +147,11 @@ The conversion is the same as above, but a new model needs its own `DecisionType
 
 > [!IMPORTANT]
 >
-> Most of the logic is handled in `tools/engine/engine-decision.cpp`, to avoid too many changes to `libllama`.
+> Most of the logic is handled in `tools/server/server-decision.cpp`, to avoid too many changes to `libllama`.
 
 Note:
 - If a new public API is needed in `libllama`, add it to `llama-ext.h`.
-- Metadata with a single use case must be hard-coded in `engine-decision.cpp` instead of being saved to the GGUF. This avoids bloating the conversion code.
+- Metadata with a single use case must be hard-coded in `server-decision.cpp` instead of being saved to the GGUF. This avoids bloating the conversion code.
 - Most importantly, keep your change as small and as self-contained as possible. Reuse the existing infrastructure whenever you can.
 
 For more information, see [PR #29818](https://github.com/ggml-org/llama.cpp/pull/29818).

ggml/src/ggml-hexagon/htp/unary-ops.c

@@ -690,22 +690,48 @@ static void tri_f32(const void * restrict src,
     }
 }
 
+// softplus(x) = log(1 + exp(x))
+// Match CPU reference: return x for x > 20 to avoid overflow.
+static inline void hvx_softplus_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
+    const HVX_Vector * restrict v_src = (const HVX_Vector *) src;
+    HVX_Vector * restrict v_dst       = (HVX_Vector *) dst;
+
+    const HVX_Vector twenty = hvx_vec_splat_f32(20.0f);
+    const HVX_Vector one    = hvx_vec_splat_f32(1.0f);
+
+    const uint32_t nvec = n / VLEN_FP32;
+    const uint32_t nloe = n % VLEN_FP32;
+
+    #pragma unroll(4)
+    for (uint32_t i = 0; i < nvec; i++) {
+        HVX_Vector x              = v_src[i];
+        HVX_VectorPred pred_gt20  = Q6_Q_vcmp_gt_VsfVsf(x, twenty);
+        HVX_Vector expx           = hvx_vec_exp_f32(x);
+        HVX_Vector logv           = hvx_vec_log_f32(hvx_vec_add_f32_f32(one, expx));
+        v_dst[i]                  = Q6_V_vmux_QVV(pred_gt20, x, logv);
+    }
+
+    if (nloe > 0) {
+        HVX_Vector x             = v_src[nvec];
+        HVX_VectorPred pred_gt20 = Q6_Q_vcmp_gt_VsfVsf(x, twenty);
+        HVX_Vector expx          = hvx_vec_exp_f32(x);
+        HVX_Vector logv          = hvx_vec_log_f32(hvx_vec_add_f32_f32(one, expx));
+        HVX_Vector sp            = Q6_V_vmux_QVV(pred_gt20, x, logv);
+        hvx_vec_store_a((void *) &v_dst[nvec], nloe * sizeof(float), sp);
+    }
+}
+
 static void softplus_f32(const void * restrict src,
                          void * restrict dst,
                          const uint32_t num_rows,
                          const struct htp_unary_context * uctx) {
     htp_unary_op_preamble;
-    // softplus(x) = log(1 + exp(x))
-    // Match CPU reference: ggml_compute_softplus_f32() in ggml-impl.h
-    for (uint32_t ir = 0; ir < num_rows; ir++) {
-        const float * restrict src_f = (const float *)((const uint8_t *)src + (ir * src0_row_size_aligned));
-        float * restrict dst_f       = (float *)((uint8_t *)dst + (ir * dst_row_size_aligned));
 
-        for (uint32_t i = 0; i < ne0; i++) {
-            float x = src_f[i];
-            // For x > 20: softplus(x) ~ x (avoids exp overflow)
-            dst_f[i] = (x > 20.0f) ? x : logf(1.0f + expf(x));
-        }
+    for (uint32_t ir = 0; ir < num_rows; ir++) {
+        const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
+        uint8_t * restrict dst_local       = (uint8_t *)dst + (ir * dst_row_size_aligned);
+
+        hvx_softplus_f32_aa(dst_local, src_local, ne0);
     }
 }
 
@@ -866,12 +892,7 @@ static void tile_gelu_erf_f32(void * restrict dst, const void * restrict src, ui
 
 static void tile_softplus_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) {
     (void) uctx;
-    const float * restrict sf = (const float *) src;
-    float * restrict df       = (float *) dst;
-    for (uint32_t i = 0; i < tw; i++) {
-        float x = sf[i];
-        df[i] = (x > 20.0f) ? x : logf(1.0f + expf(x));
-    }
+    hvx_softplus_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw);
 }
 
 static void tile_tanh_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) {

ggml/src/ggml-opencl/CMakeLists.txt

@@ -132,6 +132,7 @@ set(GGML_OPENCL_KERNELS
     gemm_moe_q6_k_q8_1_dp4a
     gemm_moe_q8_1_dp4a
     moe_reorder_quant_a_q8_1
+    transpose_activation_q8
     gemm_noshuffle_q4_k_q8_1_dp4a
     gemm_noshuffle_q5_k_q8_1_dp4a
     gemm_noshuffle_q6_k_q8_1_dp4a

ggml/src/ggml-opencl/ggml-opencl.cpp

@@ -720,6 +720,10 @@ struct ggml_backend_opencl_context {
     ggml_cl_buffer prealloc_moe_qa;   // int8 quants  [tok_slots * ne00]
     ggml_cl_buffer prealloc_moe_da;   // per-block d  [tok_slots * ne00/32] (half)
     ggml_cl_buffer prealloc_moe_sa;   // per-block s  [tok_slots * ne00/32] (half)
+    // per-tile transposed qa/da and on-chip scratch for the ldck dp4a MoE GEMM
+    ggml_cl_buffer prealloc_moe_qa_trans;    // int8 quants  [tok_slots * ne00]
+    ggml_cl_buffer prealloc_moe_da_trans;    // per-block d  [tok_slots * ne00/32] (float)
+    ggml_cl_buffer prealloc_moe_qda_on_chip;
     // scratch copy of the router weights to avoid dst aliasing
     ggml_cl_buffer prealloc_moe_combine_w;
     ggml_cl_buffer prealloc_splitk_partial;  // [ksplit * M] partials for split-K GEMV
@@ -1016,6 +1020,7 @@ struct ggml_backend_opencl_context {
     cl_kernel kernel_gemv_moe_q4_k_f32_ns_wimg = nullptr;  // weight-as-texture MoE decode GEMV (opt-in)
     cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a = nullptr;    // dp4a (int8) prefill GEMM variant
     cl_kernel kernel_moe_reorder_quant_a_q8_1;   // fused reorder + q8_1 quant for the dp4a GEMM
+    cl_kernel kernel_transpose_activation_q8;    // per-tile transpose of q8_1 activations for the ldck dp4a GEMM
     cl_kernel kernel_gemm_moe_q8_1_dp4a_q80 = nullptr;   // generic dp4a MoE GEMM (MOE_QT=80), opt-in
     cl_kernel kernel_moe_expand_scale_q8_0 = nullptr;    // q8_0 per-block d -> uniform scale[16]
     cl_kernel kernel_gemm_moe_q8_1_dp4a_q50 = nullptr;   // generic dp4a MoE GEMM (MOE_QT=50, q5_0), opt-in
@@ -1031,6 +1036,7 @@ struct ggml_backend_opencl_context {
     cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a = nullptr;   // dp4a (int8) mxfp4 MoE prefill GEMM
     cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a = nullptr;    // dp4a (int8) q4_0 MoE prefill GEMM
     cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr;   // binary dp4a (int8) mxfp4 MoE prefill GEMM
+    cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = nullptr;   // binary dp4a (int8) mxfp4 MoE prefill GEMM using ldck
     cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr;    // binary dp4a (int8) q4_0 MoE prefill GEMM
     cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a_bin = nullptr;   // binary dp4a (int8) q4_k MoE prefill GEMM
     cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a_bin = nullptr;    // binary dp4a (int8) q6_k MoE prefill GEMM
@@ -4903,6 +4909,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
         }
     }
 
+    // gemm_moe_mxfp4_q8_1_dp4a_ldck_bin (dp4a prefill GEMM)
+    if (backend_ctx->has_integer_dot) {
+        size_t bin_size = 0;
+        backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = nullptr;
+
+        if (use_adreno_bin_kernels(backend_ctx)) {
+            const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_q8_1_dp4a_ila_ldck", &bin_size);
+            if (kernel_bin && bin_size > 0) {
+                cl_program prog =
+                    build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size);
+
+                CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a_ila_ldck", &err), err));
+                CL_CHECK(clReleaseProgram(prog));
+                GGML_LOG_CONT(".");
+            }
+        }
+    }
+
     // gemm_moe_q4_0_q8_1_dp4a (dp4a prefill GEMM)
     if (backend_ctx->has_integer_dot) {
 #ifdef GGML_OPENCL_EMBED_KERNELS
@@ -4984,6 +5008,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
         GGML_LOG_CONT(".");
     }
 
+    // transpose_activation_q8 (per-tile transpose of q8_1 activations for the ldck dp4a GEMM)
+    {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+        const std::string kernel_src {
+            #include "transpose_activation_q8.cl.h"
+        };
+#else
+        const std::string kernel_src = read_file("transpose_activation_q8.cl");
+#endif
+        cl_program prog =
+            build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts);
+
+        CL_CHECK((backend_ctx->kernel_transpose_activation_q8 = clCreateKernel(prog, "kernel_transpose_activation_q8", &err), err));
+        CL_CHECK(clReleaseProgram(prog));
+        GGML_LOG_CONT(".");
+    }
+
     // gemv_moe_q5_k_f32_ns
     {
 #ifdef GGML_OPENCL_EMBED_KERNELS
@@ -28258,11 +28299,19 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
                     static const char * mxfp4_moe_dp4a_env = getenv("GGML_OPENCL_MXFP4_MOE_DP4A");
                     bool use_moe_dp4a = mxfp4_moe_dp4a_env
                         ? (atoi(mxfp4_moe_dp4a_env) != 0)
-                        : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+                        : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E);
                     // dot prod has to be available
                     use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;
                     // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel
-                    if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr) {
+                    // ldck dp4a bin kernel (X1) has the same priority as normal bin kernel (X2/A8x), they are not expected to coexist.
+                    const bool use_moe_dp4a_ldck = backend_ctx->has_integer_dot &&
+                        (!mxfp4_moe_dp4a_env || atoi(mxfp4_moe_dp4a_env) != 0) &&
+                        backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr &&
+                        backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr &&
+                        backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin != nullptr;
+                    if (use_moe_dp4a_ldck) {
+                        use_moe_dp4a = true;
+                    } else if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr) {
                         use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr;
                     }
 
@@ -28351,6 +28400,12 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
                         backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));
                         backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
                         backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+                        if (use_moe_dp4a_ldck) {
+                            backend_ctx->prealloc_moe_qa_trans.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));
+                            backend_ctx->prealloc_moe_da_trans.allocate(backend_ctx->context, n_blocks * sizeof(cl_float));
+                            // matches max_constant_size of src1_a_cm in the ldck kernel
+                            backend_ctx->prealloc_moe_qda_on_chip.allocate(backend_ctx->context, 2304);
+                        }
 
                         // fused reorder + q8_1 quant straight from the original
                         // activations (no intermediate f32 reorder buffer). mxfp4 has no
@@ -28375,13 +28430,33 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
                         cl_kernel dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a;
                         if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin) {
                             dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin;
+                        } else if (use_moe_dp4a_ldck) {
+                            dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin;
+
+                            // per-tile transpose of qa/da for the ldck kernel
+                            cl_kernel tk = backend_ctx->kernel_transpose_activation_q8;
+                            CL_CHECK(clSetKernelArg(tk, 0, sizeof(cl_mem),  &backend_ctx->prealloc_moe_qa.buffer));
+                            CL_CHECK(clSetKernelArg(tk, 1, sizeof(cl_mem),  &backend_ctx->prealloc_moe_qa_trans.buffer));
+                            CL_CHECK(clSetKernelArg(tk, 2, sizeof(cl_mem),  &backend_ctx->prealloc_moe_da.buffer));
+                            CL_CHECK(clSetKernelArg(tk, 3, sizeof(cl_mem),  &backend_ctx->prealloc_moe_da_trans.buffer));
+                            CL_CHECK(clSetKernelArg(tk, 4, sizeof(cl_mem),  &(backend_ctx->prealloc_total_tiles.buffer)));
+                            CL_CHECK(clSetKernelArg(tk, 5, sizeof(cl_uint), &ne00));
+                            size_t tk_local[3]  = { 32, 8, 1 };
+                            size_t tk_global[3] = { 32, (size_t)(ne00 / 4), (size_t)max_post_router_tile };
+                            backend_ctx->enqueue_ndrange_kernel(tk, 3, tk_global, tk_local, dst);
                         }
 
                         int aidx = 0;
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->q_img));
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->e));
-                        CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
-                        CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+                        if (use_moe_dp4a_ldck) {
+                            CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa_trans.buffer));
+                            CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da_trans.buffer));
+                            CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qda_on_chip.buffer));
+                        } else {
+                            CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+                            CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+                        }
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2));
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap));
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image));

ggml/src/ggml-opencl/kernels/transpose_activation_q8.cl

@@ -0,0 +1,35 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+
+#define tile_size 32
+
+kernel void kernel_transpose_activation_q8(
+        __global uint * src1_qa,
+        __global uint * src1_qa_trans,
+        __global half * src1_sa,
+        __global float * src1_sa_trans,
+        __global const int * total_tiles,
+        uint  K
+) {
+    uint row_in_tile = get_global_id(0);
+    uint blk_k = get_group_id(1);
+    uint col_in_tile = get_local_id(1);
+    uint tile_n = get_global_id(2);
+
+    if (blk_k >= (K / tile_size) || tile_n >= (uint)total_tiles[0]) {
+        return;
+    }
+
+    uint src_idx = (tile_n * tile_size + row_in_tile) * K / 4 + blk_k * 8 + col_in_tile;
+    uint dst_idx = tile_n * tile_size * K / 4 + (blk_k * 8 + col_in_tile) * tile_size + row_in_tile;
+
+    src1_qa_trans[dst_idx] = src1_qa[src_idx];
+
+    if (col_in_tile == 0) {
+        src_idx = (tile_n * tile_size + row_in_tile) * K / 32 + blk_k;
+        dst_idx = tile_n * tile_size * K / 32 + blk_k * tile_size + row_in_tile;
+
+        src1_sa_trans[dst_idx] = (float)(src1_sa[src_idx]);
+    }
+
+}

ggml/src/ggml-sycl/convert.cpp

@@ -158,16 +158,16 @@ static void dequantize_row_q4_0_sycl_reorder(const void *vx, dst_t *y, const int
     dpct::has_capability_or_fail(stream->get_device(),
                                     {sycl::aspect::fp16});
 
-    int constexpr WARP_K = WARP_SIZE * QK4_0;
-    const int n_warp = (k + WARP_K - 1) / WARP_K;
-    GGML_ASSERT(k % 2 == 0);
-    stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) *
-        sycl::range<3>(1, 1, WARP_SIZE),
-        sycl::range<3>(1, 1, WARP_SIZE)),
-        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{
+    GGML_ASSERT(k % QK4_0 == 0);
+    const int64_t n_groups = ceil_div(k / 2, SYCL_DEQUANTIZE_BLOCK_SIZE);
+    // decrease global range when it exceeds the max int; the kernel loops to cover the rest
+    const int64_t wg_size  = downsample_sycl_global_range(n_groups, SYCL_DEQUANTIZE_BLOCK_SIZE);
+    stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_groups) *
+        sycl::range<3>(1, 1, wg_size),
+        sycl::range<3>(1, 1, wg_size)),
+        [=](sycl::nd_item<3> item_ct1) {
             dequantize_block_q4_0_reorder(vx, y, k, item_ct1);
         });
-
 }
 
 template <typename dst_t>
@@ -177,16 +177,16 @@ static void dequantize_row_q8_0_sycl_reorder(const void *vx, dst_t *y, const int
     dpct::has_capability_or_fail(stream->get_device(),
                                     {sycl::aspect::fp16});
 
-    int constexpr WARP_K = WARP_SIZE * QK8_0;
-    const int n_warp = (k + WARP_K - 1) / WARP_K;
     GGML_ASSERT(k % QK8_0 == 0);
-    stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) *
-        sycl::range<3>(1, 1, WARP_SIZE),
-        sycl::range<3>(1, 1, WARP_SIZE)),
-        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{
+    const int64_t n_groups = ceil_div(k / 2, SYCL_DEQUANTIZE_BLOCK_SIZE);
+    // decrease global range when it exceeds the max int; the kernel loops to cover the rest
+    const int64_t wg_size  = downsample_sycl_global_range(n_groups, SYCL_DEQUANTIZE_BLOCK_SIZE);
+    stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_groups) *
+        sycl::range<3>(1, 1, wg_size),
+        sycl::range<3>(1, 1, wg_size)),
+        [=](sycl::nd_item<3> item_ct1) {
             dequantize_block_q8_0_reorder(vx, y, k, item_ct1);
         });
-
 }
 
 template <typename dst_t>

ggml/src/ggml-sycl/dequantize.hpp

@@ -818,61 +818,43 @@ static void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restri
 }
 
 template<typename dst_t>
-static void dequantize_block_q4_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb32,
+static void dequantize_block_q4_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k,
                                   const sycl::nd_item<3> &item_ct1) {
+    // one thread per qs byte, like dequantize_block<dequantize_q4_0>; the reorder layout makes the byte reads contiguous
+    // grid-stride loop because the launcher may shrink the work-group to keep the global range below INT_MAX
+    // float math, not dequantize_q4_0_reorder: that helper computes in half with GGML_SYCL_F16, which changes f32 output
+    const sycl::half * d_ptr  = (const sycl::half *) ((const uint8_t *) vx + k/2);
+    const int64_t      stride = item_ct1.get_local_range(2) * item_ct1.get_group_range(2);
 
-    const int64_t i = item_ct1.get_group(2);
-    auto k=nb32;
-    // assume 32 threads
-    const int64_t tid = item_ct1.get_local_id(2);
-    const int lane_ib = i * WARP_SIZE + tid;
+    for (int64_t iq = item_ct1.get_global_id(2); iq < k/2; iq += stride) {
+        const int64_t ib  = iq / (QK4_0/2);
+        const int64_t iqs = iq % (QK4_0/2);
+        const int     vq  = ((const uint8_t *) vx)[iq];
+        const float   d   = float(d_ptr[ib]);
 
-    if (lane_ib >= k / QK4_0) {
-        return;
+        dst_t * y = yy + ib*QK4_0 + iqs;
+        y[0]       = d * ((vq & 0xF) - 8);
+        y[QK4_0/2] = d * ((vq >>  4) - 8);
     }
-
-    dst_t * y_ptr = yy + lane_ib * QK4_0;
-
-    auto qs = (const uint8_t*)vx + lane_ib * QK4_0 / 2;
-    auto s_ptr = (const sycl::half*)((const uint8_t*)vx + k / 2) + lane_ib;
-
-    const float d = float(*s_ptr);
-
-#pragma unroll
-    for (int l = 0; l < QK4_0 / 2; ++l) {
-        int vq = qs[l];
-        y_ptr[l + 0] = d * ((vq & 0xF) - 8);
-        y_ptr[l + 16] = d * ((vq >> 4) - 8);
-    }
-
 }
 
 // Dequantize Q8_0 from reorder layout: [all qs (k bytes)][all d values]
-// Each thread handles one block of QK8_0 elements.
 template<typename dst_t>
 static void dequantize_block_q8_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k,
                                   const sycl::nd_item<3> &item_ct1) {
+    // two values per thread, like dequantize_block<dequantize_q8_0>; the reorder layout keeps all qs contiguous
+    // grid-stride loop because the launcher may shrink the work-group to keep the global range below INT_MAX
+    // float math, not dequantize_q8_0_reorder: that helper computes in half with GGML_SYCL_F16, which changes f32 output
+    const int8_t *     qs     = (const int8_t *) vx;
+    const sycl::half * d_ptr  = (const sycl::half *) ((const uint8_t *) vx + k);
+    const int64_t      stride = 2 * item_ct1.get_local_range(2) * item_ct1.get_group_range(2);
 
-    const int64_t i = item_ct1.get_group(2);
-    const int64_t tid = item_ct1.get_local_id(2);
-    const int lane_ib = i * WARP_SIZE + tid;
+    for (int64_t i = 2 * item_ct1.get_global_id(2); i < k; i += stride) {
+        const float d = float(d_ptr[i / QK8_0]);
 
-    if (lane_ib >= k / QK8_0) {
-        return;
+        yy[i + 0] = d * qs[i + 0];
+        yy[i + 1] = d * qs[i + 1];
     }
-
-    dst_t * y_ptr = yy + lane_ib * QK8_0;
-
-    auto qs = (const int8_t*)vx + lane_ib * QK8_0;
-    auto s_ptr = (const sycl::half*)((const uint8_t*)vx + k) + lane_ib;
-
-    const float d = float(*s_ptr);
-
-#pragma unroll
-    for (int l = 0; l < QK8_0; ++l) {
-        y_ptr[l] = d * qs[l];
-    }
-
 }
 
 template<typename dst_t>

gguf-py/gguf/constants.py

@@ -573,6 +573,7 @@ class MODEL_ARCH(IntEnum):
     QWEN3MOE         = auto()
     QWEN3NEXT        = auto()
     HRM_TEXT         = auto()
+    BEAM             = auto()
     QWEN3VL          = auto()
     QWEN3VLMOE       = auto()
     QWEN35           = auto()
@@ -1388,6 +1389,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.QWEN3MOE:         "qwen3moe",
     MODEL_ARCH.QWEN3NEXT:        "qwen3next",
     MODEL_ARCH.HRM_TEXT:         "hrm_text",
+    MODEL_ARCH.BEAM:             "beam",
     MODEL_ARCH.QWEN3VL:          "qwen3vl",
     MODEL_ARCH.QWEN3VLMOE:       "qwen3vlmoe",
     MODEL_ARCH.QWEN35:           "qwen35",
@@ -2981,6 +2983,30 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
         MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
     ],
+    MODEL_ARCH.BEAM: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_POST_NORM,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.ATTN_Q_NORM,
+        MODEL_TENSOR.ATTN_K_NORM,
+        MODEL_TENSOR.ATTN_GATE,
+        MODEL_TENSOR.FFN_PRE_NORM,
+        MODEL_TENSOR.FFN_POST_NORM,
+        MODEL_TENSOR.FFN_GATE,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.FFN_GATE_INP,
+        MODEL_TENSOR.FFN_EXP_PROBS_B,
+        MODEL_TENSOR.FFN_GATE_EXP,
+        MODEL_TENSOR.FFN_DOWN_EXP,
+        MODEL_TENSOR.FFN_UP_EXP,
+    ],
     MODEL_ARCH.HRM_TEXT: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT,

skills/code-review/SKILL.md

@@ -214,7 +214,7 @@ Adjacent data-shape rule: empty string is not a valid "unset" initial state - us
 - Only add tests that bring meaningful results. Too-trivial tests just bloat the suite and CI.
 - No time-sensitive tests (timing thresholds, sleeps, races against wall-clock); they are flaky on CI.
 - Think twice about tests that significantly increase CI run time (expensive computation, large inputs, or long sleep/wait delays) or download large amounts of data from the internet (big models, datasets). Flag them and ask whether a smaller model/input or an existing fixture would do.
-- Server: small tests of internal components (helpers, `engine_tokens`, queues, format converters, ...) belong in `tests/test-server-component.cpp`. The end-to-end pytest suite in `tools/server/tests` is costly in CI time; only add to it when the test requires end-to-end behavior with a real model.
+- Server: small tests of internal components (helpers, `server_tokens`, queues, format converters, ...) belong in `tests/test-server-component.cpp`. The end-to-end pytest suite in `tools/server/tests` is costly in CI time; only add to it when the test requires end-to-end behavior with a real model.
 
 ## General (always)
 

skills/slice-model/SKILL.md

@@ -0,0 +1,26 @@
+---
+name: slice-model
+description: Build a small test "slice" of a HF model (fewer layers/experts) by downloading only the needed tensors via HTTP range requests. Use when the user needs a small version of a big model to test conversion or inference.
+---
+
+# Slice a HF model for testing
+
+[slice_model.py](slice_model.py) is a **reference** implementation. Read it first: its docstring and `--help` explain how it works and what it does. Its heuristics only cover common layouts, and new models often have tensor names, shapes or config keys it does not handle. Do not run it blindly. If the model needs special handling, copy it to `tmp/slice_<model>.py` and adapt the copy, or write a new script with the same approach.
+
+## Steps
+
+1. Ask the user for the repo, what the slice must exercise (e.g. a MoE layer, each layer type of a hybrid model, MTP, the vision encoder), the size budget and the output dir.
+2. Inspect the model: run the script with `--inspect`, read `config.json`, and read the HF modeling code if anything is unclear. Check the plan against the pitfalls below.
+3. Build the slice, then read the config patches in the log and check each one.
+4. Verify: load the slice in transformers (no missing/unexpected weight warnings, no NaN logits), then convert it with `convert_hf_to_gguf.py` and run it. For numerical checks, see `examples/model-conversion/`.
+
+## Pitfalls
+
+- Layer counts in nested or unknown config keys, or several towers with the same layer count.
+- Hybrid layer patterns (`layer_types`, `full_attention_interval`, ...): keep at least one layer of each type, and check the patched config still matches the kept layers.
+- Dense leading layers (`first_k_dense_replace`) and MTP/nextn layers numbered after the regular layers.
+- Every per-expert tensor must be sliced to the same expert count: experts, router weight and biases. Tensors with the expert dim not at dim 0 are not contiguous.
+- Routing constraints: top-k, expert groups, shared experts, zero-compute experts.
+- Quantized weights (FP8, MXFP4): slice the scale tensors in the same way as their weight.
+- Sizes derived in modeling code instead of stored in the config: follow the exact formula if you shrink them.
+- Slicing the vocab without the tokenizer breaks conversion.

skills/slice-model/slice_model.py

@@ -0,0 +1,558 @@
+#!/usr/bin/env python3
+"""
+REFERENCE ONLY - build a small test "slice" of a HF safetensors model without downloading the full weights.
+
+The heuristics below (layer/expert detection, config patching) cover common layouts only. New models often use tensor names, shapes or config keys that this script does not know about. Always run --inspect first, check the plan, and adapt the script to the model.
+
+How it works: each safetensors shard starts with an 8-byte header length + JSON header, which gives the exact byte span of every tensor. We fetch only the headers, then use HTTP range requests to fetch the tensors we keep:
+  1. keep only some layers (per numeric "family" in tensor names whose count matches a layer count in config.json), renumbered to 0..n-1
+  2. keep only the first E experts (per-expert tensors are dropped, stacked expert tensors and router weights are row-sliced along dim 0)
+  3. optionally keep only the first V rows of vocab-sized tensors (off by default, see --vocab-size)
+config.json is patched to match. Other small repo files are copied as-is.
+
+Usage:
+    python slice_model.py Qwen/Qwen3-0.6B --inspect
+    python slice_model.py Qwen/Qwen3-30B-A3B --num-layers 2 --num-experts 8 --dry-run
+    python slice_model.py Qwen/Qwen3-30B-A3B --keep-layers 0,3 --output ../models/qwen3moe-slice
+"""
+from __future__ import annotations
+
+import argparse
+import concurrent.futures as cf
+import fnmatch
+import json
+import logging
+import os
+import re
+import struct
+import sys
+import threading
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import requests
+from huggingface_hub import HfApi, get_token, hf_hub_url
+
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+logger = logging.getLogger("slice_model")
+
+INDEX_FILE = "model.safetensors.index.json"
+CONFIG_FILE = "config.json"
+
+# files not worth copying into a slice
+AUX_BLACKLIST = [
+    # weights in other formats (the slice replaces them)
+    "*.bin", "*.pth", "*.pt", "*.ckpt", "*.h5", "*.msgpack",
+    "*.onnx", "*.tflite", "*.mlmodel", "*.mlpackage/*", "*.gguf", "*.nemo",
+    "onnx/*", "openvino/*", "coreml/*", "original/*", "metal/*",
+    # media / demo assets
+    "*.png", "*.jpg", "*.jpeg", "*.gif", "*.webp", "*.avif", "*.svg",
+    "*.mp4", "*.webm", "*.mov", "*.wav", "*.mp3", "*.flac", "*.ogg", "*.pdf",
+    ".gitattributes",
+]
+AUX_MAX_SIZE = 50 * 1024 * 1024
+
+LAYER_CONTAINERS = {"layers", "layer", "blocks", "block", "h"}
+EXPERT_ROW_KEYWORDS = ("expert", "router", "gate", "score")
+
+LAYER_KEYS = {
+    "num_hidden_layers", "num_layers", "n_layer", "n_layers", "num_layer",
+    "num_decoder_layers", "encoder_layers", "decoder_layers", "depth",
+    "num_encoder_layers", "num_attention_layers", "layer_count",
+}
+VOCAB_KEYS = {"vocab_size", "padded_vocab_size"}
+NEXTN_KEYS = {"num_nextn_predict_layers", "num_mtp_layers"}
+TOPK_SUBSTR = ("per_tok", "topk", "top_k", "moe_k")
+EXPERT_KEY_EXCLUDE = ("per_tok", "topk", "top_k", "shared", "group", "intermediate", "size", "dim", "dtype")
+
+
+def parse_size(text: str) -> int:
+    m = re.fullmatch(r"(\d+(?:\.\d+)?)\s*([kKmMgGtT]?)[bB]?", text.strip())
+    if not m:
+        raise argparse.ArgumentTypeError(f"invalid size: {text!r}")
+    mult = {"": 1, "k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4}[m.group(2).lower()]
+    return int(float(m.group(1)) * mult)
+
+
+def human(n: float) -> str:
+    for unit in ("B", "KB", "MB", "GB"):
+        if n < 1024 or unit == "GB":
+            return f"{n:.1f}{unit}" if unit != "B" else f"{int(n)}B"
+        n /= 1024
+    return f"{n}GB"
+
+
+# --------------------------------------------------------------------------------------
+# HTTP
+# --------------------------------------------------------------------------------------
+
+_tls = threading.local()
+
+
+def _session() -> requests.Session:
+    s = getattr(_tls, "s", None)
+    if s is None:
+        s = requests.Session()
+        _tls.s = s
+    return s
+
+
+class HubClient:
+    def __init__(self, repo_id: str, revision: str, token: str | None):
+        self.repo_id = repo_id
+        self.revision = revision
+        self.headers = {"Authorization": f"Bearer {token}"} if token else {}
+
+    def url(self, filename: str) -> str:
+        return hf_hub_url(self.repo_id, filename, revision=self.revision)
+
+    def _request(self, filename: str, headers: dict, stream: bool = False, retries: int = 5):
+        last: Exception | None = None
+        for attempt in range(1, retries + 1):
+            try:
+                r = _session().get(self.url(filename), headers=headers, stream=stream, timeout=90, allow_redirects=True)
+                if r.status_code in (429, 500, 502, 503, 504):
+                    raise requests.HTTPError(f"HTTP {r.status_code}", response=r)
+                r.raise_for_status()
+                return r
+            except Exception as e:  # noqa: BLE001
+                last = e
+                if attempt == retries or "404" in str(e) or "401" in str(e) or "403" in str(e):
+                    break
+                time.sleep(2.0 * attempt)
+        raise RuntimeError(f"{self.repo_id}/{filename}: {last}") from last
+
+    def get_range(self, filename: str, start: int, end_inclusive: int) -> bytes:
+        h = dict(self.headers)
+        h["Range"] = f"bytes={start}-{end_inclusive}"
+        return self._request(filename, h).content
+
+    def get_json(self, filename: str) -> Any:
+        return json.loads(self._request(filename, dict(self.headers)).content)
+
+    def download_file(self, filename: str, dest: Path):
+        r = self._request(filename, dict(self.headers), stream=True)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        with open(dest, "wb") as f:
+            for chunk in r.iter_content(chunk_size=4 * 1024 * 1024):
+                if chunk:
+                    f.write(chunk)
+
+    def stream_range_into_fd(self, filename: str, start: int, end_inclusive: int, fd: int, write_offset: int):
+        h = dict(self.headers)
+        h["Range"] = f"bytes={start}-{end_inclusive}"
+        r = self._request(filename, h, stream=True)
+        pos = write_offset
+        for chunk in r.iter_content(chunk_size=8 * 1024 * 1024):
+            if chunk:
+                os.pwrite(fd, chunk, pos)
+                pos += len(chunk)
+        if pos - write_offset != end_inclusive - start + 1:
+            raise RuntimeError(f"short read: {filename} bytes={start}-{end_inclusive}")
+
+
+def fetch_shard_header(client: HubClient, filename: str) -> tuple[dict, int]:
+    (header_len,) = struct.unpack("<Q", client.get_range(filename, 0, 7))
+    header = json.loads(client.get_range(filename, 8, 8 + header_len - 1))
+    header.pop("__metadata__", None)
+    return header, 8 + header_len
+
+
+# --------------------------------------------------------------------------------------
+# config.json analysis / patching
+# --------------------------------------------------------------------------------------
+
+def walk_config(cfg: Any):
+    """Yield (container_dict, key, value) for every leaf in nested dicts."""
+    if isinstance(cfg, dict):
+        for k, v in cfg.items():
+            if isinstance(v, dict):
+                yield from walk_config(v)
+            else:
+                yield cfg, k, v
+
+
+@dataclass
+class ConfigInfo:
+    cfg: dict | None
+    layer_counts: set[int] = field(default_factory=set)
+    nextn_counts: set[int] = field(default_factory=set)
+    expert_counts: set[int] = field(default_factory=set)
+    vocab_sizes: set[int] = field(default_factory=set)
+    first_k_dense: int = 0
+
+
+def analyze_config(cfg: dict | None) -> ConfigInfo:
+    info = ConfigInfo(cfg=cfg)
+    if cfg is None:
+        return info
+    for _d, key, val in walk_config(cfg):
+        if not isinstance(val, int) or isinstance(val, bool) or val <= 0:
+            continue
+        if key in LAYER_KEYS:
+            info.layer_counts.add(val)
+        elif key in NEXTN_KEYS:
+            info.nextn_counts.add(val)
+        elif key in VOCAB_KEYS:
+            info.vocab_sizes.add(val)
+        elif key == "first_k_dense_replace":
+            info.first_k_dense = val
+        elif "expert" in key and val > 1 and not any(x in key for x in EXPERT_KEY_EXCLUDE):
+            info.expert_counts.add(val)
+    return info
+
+
+def patch_config(cfg: dict, families: dict[str, Family], new_vocab: int | None, orig_vocabs: set[int]) -> list[str]:
+    """Patch counts in-place to match the slice. Returns log lines."""
+    log = []
+    by_card: dict[int, Family] = {}
+    for fam in families.values():
+        if fam.kept is not None:
+            by_card.setdefault(fam.card, fam)
+            by_card.setdefault(fam.regular_total, fam)
+    max_new_experts = max((f.new_count for f in families.values() if f.kind == "expert" and f.kept is not None), default=None)
+
+    for d, key, val in list(walk_config(cfg)):
+        if isinstance(val, int) and not isinstance(val, bool):
+            fam = by_card.get(val)
+            if key in LAYER_KEYS and fam is not None and fam.kind == "layer":
+                d[key] = fam.new_regular
+            elif key == "first_k_dense_replace":
+                lfam = next((f for f in families.values() if f.kind == "layer"), None)
+                if lfam is None:
+                    continue
+                d[key] = sum(1 for i in lfam.kept if i < val)
+            elif "expert" in key and fam is not None and fam.kind == "expert" and not any(x in key for x in EXPERT_KEY_EXCLUDE):
+                d[key] = fam.new_count
+            elif "expert" in key and any(x in key for x in TOPK_SUBSTR) and max_new_experts is not None and val > max_new_experts:
+                d[key] = max_new_experts
+            elif key in VOCAB_KEYS and new_vocab is not None and val in orig_vocabs:
+                d[key] = new_vocab
+            if d[key] != val:
+                log.append(f"{key}: {val} -> {d[key]}")
+        elif isinstance(val, list) and val and all(not isinstance(x, (dict, list)) for x in val):
+            # per-layer lists, e.g. layer_types
+            fam = by_card.get(len(val))
+            if fam is not None and fam.kind == "layer" and len(val) in (fam.card, fam.regular_total):
+                d[key] = [val[i] for i in fam.kept if i < len(val)]
+                log.append(f"{key}: list[{len(val)}] -> list[{len(d[key])}]")
+    return log
+
+
+# --------------------------------------------------------------------------------------
+# tensor name analysis
+# --------------------------------------------------------------------------------------
+
+@dataclass
+class Family:
+    """One numeric slot in tensor names, e.g. 'model.layers.#'."""
+    prefix: str
+    kind: str            # "layer" | "expert" | "other"
+    indices: set[int] = field(default_factory=set)
+    kept: list[int] | None = None    # None = keep all, no renumbering
+    renumber: dict[int, int] = field(default_factory=dict)
+    regular_total: int = 0           # layer count excluding MTP tail
+    new_regular: int = 0
+
+    @property
+    def card(self) -> int:
+        return len(self.indices)
+
+    @property
+    def new_count(self) -> int:
+        return len(self.kept) if self.kept is not None else self.card
+
+
+def name_slots(name: str):
+    """Yield (slot_pos, family_prefix, index) for each numeric path component."""
+    tokens = name.split(".")
+    norm: list[str] = []
+    for i, tok in enumerate(tokens):
+        if tok.isdigit():
+            yield i, ".".join(norm), int(tok)
+            norm.append("#")
+        else:
+            norm.append(tok)
+
+
+def build_families(names: list[str], cfg_info: ConfigInfo) -> dict[str, Family]:
+    families: dict[str, Family] = {}
+    for name in names:
+        tokens = name.split(".")
+        for i, prefix, idx in name_slots(name):
+            container = tokens[i - 1] if i > 0 else ""
+            fam = families.get(prefix)
+            if fam is None:
+                fam = Family(prefix=prefix, kind="expert" if "expert" in container else "other")
+                families[prefix] = fam
+            fam.indices.add(idx)
+
+    acceptable_layer_cards: dict[int, int] = {}  # card -> regular layer count
+    for c in cfg_info.layer_counts:
+        acceptable_layer_cards[c] = c
+        for m in cfg_info.nextn_counts:
+            acceptable_layer_cards[c + m] = c
+
+    for fam in families.values():
+        if fam.kind == "expert":
+            continue
+        container = fam.prefix.rsplit(".", 1)[-1]
+        if cfg_info.cfg is not None:
+            if fam.card in acceptable_layer_cards:
+                fam.kind = "layer"
+                fam.regular_total = acceptable_layer_cards[fam.card]
+        elif fam.card >= 4 and container in LAYER_CONTAINERS:
+            fam.kind = "layer"
+            fam.regular_total = fam.card
+    return families
+
+
+def plan_families(families: dict[str, Family], num_layers: int, keep_layers: list[int] | None, num_experts: int, first_k_dense: int):
+    for fam in families.values():
+        if fam.kind == "layer":
+            if keep_layers is not None:
+                kept = [i for i in keep_layers if i < fam.regular_total]
+            else:
+                base = min(num_layers, fam.regular_total)
+                kept = list(range(base))
+                # keep one MoE layer for models whose first k layers are dense
+                if 0 < first_k_dense < fam.regular_total and first_k_dense >= base >= 1:
+                    kept = list(range(base - 1)) + [first_k_dense]
+            fam.new_regular = len(kept)
+            # keep the MTP tail (layer indices beyond the regular count)
+            kept += [i for i in sorted(fam.indices) if i >= fam.regular_total]
+            fam.kept = sorted(set(kept))
+        elif fam.kind == "expert":
+            fam.kept = sorted(fam.indices)[:num_experts]
+            fam.new_regular = len(fam.kept)
+        else:
+            fam.kept = None
+            continue
+        fam.renumber = {old: new for new, old in enumerate(fam.kept)}
+
+
+def slice_tensor_name(name: str, families: dict[str, Family]) -> str | None:
+    """Return the renumbered output name, or None if the tensor is dropped."""
+    tokens = name.split(".")
+    for i, prefix, idx in name_slots(name):
+        fam = families[prefix]
+        if fam.kept is None:
+            continue
+        if idx not in fam.renumber:
+            return None
+        tokens[i] = str(fam.renumber[idx])
+    return ".".join(tokens)
+
+
+# --------------------------------------------------------------------------------------
+# slicing
+# --------------------------------------------------------------------------------------
+
+@dataclass
+class TensorPlan:
+    src_name: str
+    out_name: str
+    dtype: str
+    shape: list[int]
+    orig_shape: list[int]
+    size: int
+    src_file: str
+    seg_start: int       # absolute byte range in src_file
+    seg_end: int         # inclusive
+
+
+def pick_weight_files(filenames: list[str]) -> tuple[list[str], bool]:
+    """Return (weight files, has_index). Preference: index -> model.safetensors -> any root safetensors."""
+    if INDEX_FILE in filenames:
+        return [], True
+    if "model.safetensors" in filenames:
+        return ["model.safetensors"], False
+    root_st = [f for f in filenames if f.endswith(".safetensors") and "/" not in f]
+    if "consolidated.safetensors" in root_st:
+        return ["consolidated.safetensors"], False
+    return root_st[:1], False
+
+
+def row_slice(shape: list[int], size: int, keep_rows: int) -> tuple[list[int], int] | None:
+    # only a prefix of dim 0 is a single contiguous byte range
+    if len(shape) < 1 or shape[0] <= keep_rows or size % shape[0] != 0:
+        return None
+    row_bytes = size // shape[0]
+    return [keep_rows] + shape[1:], keep_rows * row_bytes
+
+
+def build_plan(args, families, headers, weight_files, src_file_of, cfg_info) -> list[TensorPlan]:
+    plan_families(families, args.num_layers, args.keep_layers, args.num_experts, cfg_info.first_k_dense)
+    plans = []
+    for fname in weight_files:
+        header, data_start = headers[fname]
+        for name, meta in header.items():
+            out_name = slice_tensor_name(name, families)
+            if out_name is None:
+                continue
+            start, end = meta["data_offsets"]
+            shape, size = list(meta["shape"]), end - start
+            new_rows = None
+            if args.vocab_size and shape and shape[0] > args.vocab_size \
+                    and any(shape[0] >= vs and shape[0] - vs <= 1024 for vs in cfg_info.vocab_sizes):
+                new_rows = args.vocab_size
+            elif shape and shape[0] in cfg_info.expert_counts and shape[0] > args.num_experts \
+                    and any(k in name for k in EXPERT_ROW_KEYWORDS):
+                new_rows = args.num_experts
+            if new_rows is not None:
+                sliced = row_slice(shape, size, new_rows)
+                if sliced is not None:
+                    shape, size = sliced
+            plans.append(TensorPlan(name, out_name, meta["dtype"], shape, list(meta["shape"]), size, src_file_of[name],
+                                    data_start + start, data_start + start + size - 1))
+    return plans
+
+
+def print_inspect(cfg_info: ConfigInfo, families: dict[str, Family], plans: list[TensorPlan], all_names: list[str]):
+    logger.info("=== config analysis ===")
+    logger.info(f"layer counts: {sorted(cfg_info.layer_counts)}  nextn: {sorted(cfg_info.nextn_counts)}  "
+                f"experts: {sorted(cfg_info.expert_counts)}  vocab: {sorted(cfg_info.vocab_sizes)}  first_k_dense: {cfg_info.first_k_dense}")
+    logger.info("\n=== numeric families in tensor names ===")
+    for fam in sorted(families.values(), key=lambda f: f.prefix):
+        kept = "all" if fam.kept is None else f"{fam.kept[:16]}{'...' if len(fam.kept) > 16 else ''}"
+        logger.info(f"{fam.prefix:60s} kind={fam.kind:6s} count={fam.card:4d} kept={kept}")
+    logger.info("\n=== kept tensors (first index of each family only) ===")
+    seen = set()
+    for t in plans:
+        key = re.sub(r"\.\d+\.", ".#.", t.out_name)
+        if key in seen and t.shape == t.orig_shape:
+            continue
+        seen.add(key)
+        mark = f"  (sliced from {t.orig_shape})" if t.shape != t.orig_shape else ""
+        logger.info(f"{t.out_name:70s} {t.dtype:5s} {t.shape}{mark}")
+    kept_src = {t.src_name for t in plans}
+    dropped = [n for n in all_names if n not in kept_src]
+    logger.info(f"\n{len(plans)}/{len(all_names)} tensors kept, {len(dropped)} dropped")
+
+
+def write_slice(out_dir: Path, client: HubClient, plans: list[TensorPlan], jobs: int):
+    header_out: dict[str, Any] = {}
+    offset = 0
+    for t in plans:
+        header_out[t.out_name] = {"dtype": t.dtype, "shape": t.shape, "data_offsets": [offset, offset + t.size]}
+        offset += t.size
+    header_out["__metadata__"] = {"format": "pt"}
+    hbytes = json.dumps(header_out, separators=(",", ":")).encode()
+    hbytes += b" " * ((-(len(hbytes) + 8)) % 8)
+    data_start = 8 + len(hbytes)
+
+    shard_path = out_dir / "model.safetensors"
+    with open(shard_path, "wb") as f:
+        f.write(struct.pack("<Q", len(hbytes)))
+        f.write(hbytes)
+        f.truncate(data_start + offset)
+    fd = os.open(shard_path, os.O_WRONLY)
+    try:
+        def fetch(t: TensorPlan):
+            dst = data_start + header_out[t.out_name]["data_offsets"][0]
+            client.stream_range_into_fd(t.src_file, t.seg_start, t.seg_end, fd, dst)
+
+        with cf.ThreadPoolExecutor(max_workers=jobs) as ex:
+            list(ex.map(fetch, plans))
+    finally:
+        os.close(fd)
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("repo", help="HF repo id, e.g. Qwen/Qwen3-0.6B")
+    ap.add_argument("--revision", default=None)
+    ap.add_argument("--output", type=Path, default=None, help="output dir (default: ./tmp/hf_slices/{user}--{model})")
+    ap.add_argument("--inspect", action="store_true", help="print config analysis, tensor families and the plan, write nothing")
+    ap.add_argument("--dry-run", action="store_true", help="plan and estimate size only, write nothing")
+    ap.add_argument("--num-layers", type=int, default=2, help="keep the first N layers (default: 2)")
+    ap.add_argument("--keep-layers", default=None, help="explicit comma-separated layer indices to keep, overrides --num-layers")
+    ap.add_argument("--num-experts", type=int, default=8)
+    ap.add_argument("--vocab-size", type=int, default=0,
+                    help="row-slice vocab-sized tensors to this size (default: 0 = off; tokenizer is NOT sliced, so this breaks conversion for most tokenizers)")
+    ap.add_argument("--max-size", type=parse_size, default="1G", help="abort if the sliced weights exceed this (default: 1G)")
+    ap.add_argument("--hf-token", default=None, help="HF token (default: HF_TOKEN env var or 'hf auth login' token)")
+    ap.add_argument("--jobs", type=int, default=8, help="concurrent HTTP requests")
+    args = ap.parse_args()
+    args.keep_layers = sorted({int(x) for x in args.keep_layers.split(",")}) if args.keep_layers else None
+
+    token = args.hf_token or get_token()
+    api = HfApi(token=token)
+    info = api.model_info(args.repo, revision=args.revision, files_metadata=True)
+    client = HubClient(args.repo, info.sha or "main", token)
+    siblings = {s.rfilename: (s.size or 0) for s in info.siblings or []}
+    filenames = sorted(siblings)
+
+    weight_files, has_index = pick_weight_files(filenames)
+    weight_map: dict[str, str] = {}
+    if has_index:
+        weight_map = client.get_json(INDEX_FILE)["weight_map"]
+        weight_files = sorted(set(weight_map.values()))
+    elif not weight_files:
+        logger.error("no .safetensors found")
+        return 1
+
+    cfg = client.get_json(CONFIG_FILE) if CONFIG_FILE in filenames else None
+    cfg_info = analyze_config(cfg)
+
+    with cf.ThreadPoolExecutor(max_workers=args.jobs) as ex:
+        headers = dict(zip(weight_files, ex.map(lambda f: fetch_shard_header(client, f), weight_files)))
+    all_names = [n for fname in weight_files for n in headers[fname][0]]
+    src_file_of = weight_map if has_index else {n: weight_files[0] for n in all_names}
+
+    families = build_families(all_names, cfg_info)
+    plans = build_plan(args, families, headers, weight_files, src_file_of, cfg_info)
+    est = sum(t.size for t in plans)
+
+    if args.inspect:
+        print_inspect(cfg_info, families, plans, all_names)
+    logger.info(f"{args.repo}@{info.sha}: {len(plans)}/{len(all_names)} tensors, {human(est)}")
+    if not any(f.kind == "layer" for f in families.values()):
+        logger.warning("no layer family detected - check --inspect output and adapt the script")
+    if est > args.max_size:
+        logger.error(f"slice is {human(est)} > --max-size {human(args.max_size)}")
+        return 1
+    if args.inspect or args.dry_run:
+        return 0
+
+    out_dir = args.output or Path("tmp") / "hf_slices" / args.repo.replace("/", "--")
+    out_dir.mkdir(parents=True, exist_ok=True)
+    write_slice(out_dir, client, plans, args.jobs)
+
+    skipped_aux = []
+    for fname in filenames:
+        if fname in weight_files or fname in (INDEX_FILE, CONFIG_FILE) or fname.endswith(".safetensors"):
+            continue
+        if any(fnmatch.fnmatch(fname, p) or fnmatch.fnmatch(Path(fname).name, p) for p in AUX_BLACKLIST):
+            continue
+        if siblings[fname] > AUX_MAX_SIZE:
+            skipped_aux.append(f"{fname} ({human(siblings[fname])})")
+            continue
+        client.download_file(fname, out_dir / fname)
+    for f in skipped_aux:
+        logger.warning(f"skipped large aux file {f}")
+
+    patch_log = []
+    if cfg is not None:
+        new_vocab = args.vocab_size if args.vocab_size and any(vs > args.vocab_size for vs in cfg_info.vocab_sizes) else None
+        patch_log = patch_config(cfg, families, new_vocab, cfg_info.vocab_sizes)
+        with open(out_dir / CONFIG_FILE, "w") as f:
+            json.dump(cfg, f, indent=2)
+            f.write("\n")
+    for line in patch_log:
+        logger.info(f"config: {line}")
+
+    with open(out_dir / ".slice_meta.json", "w") as f:
+        json.dump({
+            "repo": args.repo, "revision": info.sha,
+            "tensors": len(plans), "total_tensors": len(all_names), "size": est,
+            "config_patches": patch_log, "skipped_aux": skipped_aux,
+        }, f, indent=2)
+    logger.info(f"wrote {out_dir}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/llama-arch.cpp

@@ -165,6 +165,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_QWEN3TTS,         "qwen3tts"         },
     { LLM_ARCH_POCKETTTS,        "pockettts"        },
     { LLM_ARCH_K2_HORIZON,       "k2-horizon"       },
+    { LLM_ARCH_BEAM,             "beam"             },
     { LLM_ARCH_UNKNOWN,          "(unknown)"        },
 };
 

src/llama-arch.h

@@ -170,6 +170,7 @@ enum llm_arch {
     LLM_ARCH_MINIMAX_01,
     LLM_ARCH_HRM_TEXT,
     LLM_ARCH_K2_HORIZON,
+    LLM_ARCH_BEAM,
     LLM_ARCH_UNKNOWN,
 };
 

src/llama-graph.cpp

@@ -218,17 +218,31 @@ void llm_graph_input_attn_temp::set_input(const llama_ubatch * ubatch) {
         GGML_ASSERT(n_attn_temp_floor_scale != 0);
 
         std::vector<float> attn_scale_data(n_tokens, 0.0f);
-        for (int i = 0; i < n_tokens; ++i) {
-            const float pos = ubatch->pos[i];
-            attn_scale_data[i] = std::log(
-                std::floor((pos + f_attn_temp_offset) / n_attn_temp_floor_scale) + 1.0
-            ) * f_attn_temp_scale + 1.0;
+        if (attn_temp_pow) {
+            // (1 + (pos + offset) / length) ^ power
+            for (int i = 0; i < n_tokens; ++i) {
+                const double pos = ubatch->pos[i];
+                attn_scale_data[i] = (float) std::exp((double) f_attn_temp_scale *
+                    std::log1p((pos + f_attn_temp_offset) / (double) n_attn_temp_floor_scale));
+            }
+        } else {
+            for (int i = 0; i < n_tokens; ++i) {
+                const float pos = ubatch->pos[i];
+                attn_scale_data[i] = std::log(
+                    std::floor((pos + f_attn_temp_offset) / n_attn_temp_floor_scale) + 1.0
+                ) * f_attn_temp_scale + 1.0;
+            }
         }
 
         ggml_backend_tensor_set(attn_scale, attn_scale_data.data(), 0, n_tokens*ggml_element_size(attn_scale));
     }
 }
 
+bool llm_graph_input_attn_temp::can_reuse(const llm_graph_params & params) {
+    // only the power form opts in; the log form keeps no reuse
+    return attn_temp_pow && attn_scale != nullptr && attn_scale->ne[2] == params.ubatch.n_tokens;
+}
+
 void llm_graph_input_pos_bucket::set_input(const llama_ubatch * ubatch) {
     if (pos_bucket) {
         const int64_t n_tokens = ubatch->n_tokens;
@@ -2743,7 +2757,7 @@ ggml_tensor * llm_graph_context::build_inp_pos() const {
 }
 
 ggml_tensor * llm_graph_context::build_inp_attn_scale() const {
-    auto inp = std::make_unique<llm_graph_input_attn_temp>(hparams.n_attn_temp_floor_scale, hparams.f_attn_temp_scale, hparams.f_attn_temp_offset);
+    auto inp = std::make_unique<llm_graph_input_attn_temp>(hparams.n_attn_temp_floor_scale, hparams.f_attn_temp_scale, hparams.f_attn_temp_offset, hparams.attn_temp_pow);
 
     auto & cur = inp->attn_scale;
 

src/llama-graph.h

@@ -188,17 +188,20 @@ public:
 // temperature tuning, used by llama4
 class llm_graph_input_attn_temp : public llm_graph_input_i {
 public:
-    llm_graph_input_attn_temp(uint32_t n_attn_temp_floor_scale, float f_attn_temp_scale, float f_attn_temp_offset)
-        : n_attn_temp_floor_scale(n_attn_temp_floor_scale), f_attn_temp_scale(f_attn_temp_scale), f_attn_temp_offset(f_attn_temp_offset) {}
+    llm_graph_input_attn_temp(uint32_t n_attn_temp_floor_scale, float f_attn_temp_scale, float f_attn_temp_offset, bool attn_temp_pow = false)
+        : n_attn_temp_floor_scale(n_attn_temp_floor_scale), f_attn_temp_scale(f_attn_temp_scale), f_attn_temp_offset(f_attn_temp_offset), attn_temp_pow(attn_temp_pow) {}
     virtual ~llm_graph_input_attn_temp() = default;
 
     void set_input(const llama_ubatch * ubatch) override;
 
+    bool can_reuse(const llm_graph_params & params) override;
+
     ggml_tensor * attn_scale = nullptr; // F32 [n_batch]
 
     const uint32_t n_attn_temp_floor_scale;
     const float    f_attn_temp_scale;
     const float    f_attn_temp_offset;
+    const bool     attn_temp_pow;
 };
 
 class llm_graph_input_pos_bucket : public llm_graph_input_i {

src/llama-hparams.h

@@ -280,6 +280,8 @@ struct llama_hparams {
     uint32_t n_attn_temp_floor_scale = 0;
     float    f_attn_temp_scale       = 0.0f;
     float    f_attn_temp_offset      = 0.0f; // offset position index
+    // use (1 + (pos + offset) / floor_scale) ^ scale instead of the llama4 log form
+    bool     attn_temp_pow           = false;
 
     // realtime models: number of tokens the output lags behind the audio input
     uint32_t n_tokens_delay          = 0;

src/llama-model.cpp

@@ -322,6 +322,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_minimax_m3(params);
         case LLM_ARCH_HRM_TEXT:
             return new llama_model_hrm_text(params);
+        case LLM_ARCH_BEAM:
+            return new llama_model_beam(params);
         case LLM_ARCH_COGVLM:
             return new llama_model_cogvlm(params);
         case LLM_ARCH_PANGU_EMBED:
@@ -405,6 +407,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
     static const std::regex pattern_idx_cache       ("cache_idx_(k|v)_l\\d*");
     static const std::regex pattern_dsv4_state      ("dsv4_(csa|hca|lid)_state_(kv|score)_l\\d*");
     static const std::regex pattern_attn_sinks      ("blk\\.\\d*\\.attn_sinks.weight");
+    static const std::regex pattern_attn_gate_bias  ("blk\\.\\d*\\.attn_gate\\.bias");
     static const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight");
     static const std::regex pattern_attn_out_bias   ("blk\\.\\d*\\.attn_output.bias");
     static const std::regex pattern_attn_out_a_weight("blk\\.\\d*\\.attn_output_a\\.weight");
@@ -550,7 +553,8 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
         if (std::regex_match(tensor_name, pattern_qk_norm)) {
             return get_tensor_config_impl(tensor->ne[1] == 1 ? GGML_BACKEND_SPLIT_AXIS_MIRRORED : GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight");
         }
-        if (std::regex_match(tensor_name, pattern_kv_cache) || std::regex_match(tensor_name, pattern_attn_sinks)) {
+        if (std::regex_match(tensor_name, pattern_kv_cache) || std::regex_match(tensor_name, pattern_attn_sinks) ||
+                std::regex_match(tensor_name, pattern_attn_gate_bias)) {
             return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight");
         }
         if (std::regex_match(tensor_name, pattern_attn_out_weight)) {
@@ -749,6 +753,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
 
             const int64_t granularity_q    = std::lcm(n_embd_q, blck_size_perf);
             const int64_t granularity_head = granularity_q / hparams.n_embd_head_k(il); // for tensors with one value per head
+            if (std::regex_match(tensor_name, pattern_attn_gate_bias)) {
+                GGML_ASSERT(segments.size() == 1);
+                return {granularity_head};
+            }
             if (std::regex_match(tensor_name, pattern_attn_sinks)) {
                 GGML_ASSERT(segments.size() == 1);
                 if (is_dsv4) {
@@ -3606,6 +3614,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_MELLUM:
         case LLM_ARCH_MAPLE:
         case LLM_ARCH_HRM_TEXT:
+        case LLM_ARCH_BEAM:
             return LLAMA_ROPE_TYPE_NEOX;
 
         case LLM_ARCH_DFLASH:

src/llama-model.h

@@ -310,6 +310,7 @@ struct llama_layer {
     struct ggml_tensor * wv_enc    = nullptr;
     struct ggml_tensor * wo_enc    = nullptr;
     struct ggml_tensor * wqkv_gate = nullptr;
+    struct ggml_tensor * wqkv_gate_b = nullptr;
     // K2 Horizon MoVA
     struct ggml_tensor * attn_v_gate   = nullptr;
     struct ggml_tensor * attn_v_gate_b = nullptr;

src/llama-vocab.cpp

@@ -3142,9 +3142,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
             }
         }
 
-        // gemma4 and plamo have a normal </s> token, unlike paddleocr
+        // gemma4, plamo and beam have a normal </s> token, unlike paddleocr
         {
-            bool has_normal_s_marker = false;
+            // beam's tool call marker is not an EOG token, so it is looked up in the whole vocab
+            bool has_normal_s_marker = token_to_id.count("<rfl:parameter name=\"") > 0;
             bool has_s = false;
 
             llama_token s_id = LLAMA_TOKEN_NULL;

src/models/beam.cpp

@@ -0,0 +1,261 @@
+#include "models.h"
+
+void llama_model_beam::load_arch_hparams(llama_model_loader & ml) {
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,       hparams.f_norm_rms_eps);
+    ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,         hparams.n_layer_dense_lead);
+    ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
+    ml.get_key(LLM_KV_EXPERT_GATING_FUNC,                hparams.expert_gating_func, false);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,              hparams.expert_weights_scale);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,               hparams.expert_weights_norm);
+    hparams.f_final_logit_softcapping = 0.0f; // absent key = no softcap (default is 30)
+    ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING,           hparams.f_final_logit_softcapping, false);
+
+    if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {
+        hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;
+    }
+    if (hparams.n_layer_dense_lead > hparams.n_layer()) {
+        throw std::runtime_error("beam: leading_dense_block_count exceeds the layer count");
+    }
+
+    ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; // masked iff p1 - p0 >= n_swa
+    uint32_t n_pattern = 0;
+    ml.get_arr_n(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern);
+    if (hparams.n_swa == 0 || n_pattern != hparams.n_layer_all) {
+        throw std::runtime_error("beam: requires a positive sliding window and a sliding window pattern for each layer");
+    }
+    ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+    if (!hparams.is_swa_any()) {
+        throw std::runtime_error("beam: sliding window pattern has no sliding window layer");
+    }
+
+    // full layers are NoPE
+    hparams.rope_pattern = hparams.is_swa_impl;
+    hparams.rope_freq_base_train_swa  = hparams.rope_freq_base_train;
+    hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
+    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
+
+    // NoPE query scale uses pos + 1
+    hparams.f_attn_temp_scale = 0.1f;
+    hparams.attn_temp_length  = 2048;
+    ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_SCALE,  hparams.f_attn_temp_scale, false);
+    ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.attn_temp_length,  false);
+    if (hparams.f_attn_temp_scale == 0.0f) {
+        throw std::runtime_error("beam: attention.temperature_scale must be nonzero");
+    }
+    if (hparams.attn_temp_length == 0) {
+        throw std::runtime_error("beam: attention.temperature_length must be positive");
+    }
+    hparams.n_attn_temp_floor_scale = hparams.attn_temp_length;
+    hparams.f_attn_temp_offset      = 1.0f;
+    hparams.attn_temp_pow           = true;
+
+    switch (hparams.n_layer()) {
+        case 52: type = LLM_TYPE_UNKNOWN; break;
+        default: type = LLM_TYPE_UNKNOWN;
+    }
+}
+
+void llama_model_beam::load_arch_tensors(llama_model_loader &) {
+    LLAMA_LOAD_LOCALS;
+
+    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+    if (output == NULL) {
+        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+    }
+
+    for (int i = 0; i < n_layer; ++i) {
+        auto & layer = layers[i];
+
+        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM,      "weight", i), {n_embd}, 0);
+        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0);
+
+        create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
+        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
+
+        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
+        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
+
+        layer.wqkv_gate   = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
+        layer.wqkv_gate_b = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "bias",   i), {n_head}, 0);
+
+        layer.ffn_norm      = create_tensor(tn(LLM_TENSOR_FFN_NORM,      "weight", i), {n_embd}, 0);
+        layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0);
+
+        if ((uint32_t) i < hparams.n_layer_dense_lead) {
+            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
+            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
+            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd, n_ff}, 0);
+        } else {
+            const int64_t n_ff_exp = hparams.n_ff_exp(i);
+
+            layer.ffn_gate_inp    = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,    "weight", i), {n_embd, n_expert}, 0);
+            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias",   i), {n_expert}, 0);
+
+            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
+            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
+            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
+        }
+    }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_beam::build_arch_graph(const llm_graph_params & params) const {
+    return std::make_unique<graph>(*this, params);
+}
+
+static ggml_tensor * beam_residual_add(ggml_context * ctx, ggml_tensor * h, ggml_tensor * r, int i) {
+    const float h_scale = (float) (1.0 / std::sqrt((double) i));
+    const float r_scale = (float) std::sqrt((double) (i - 1) / (double) i);
+    return ggml_add(ctx, ggml_scale(ctx, h, h_scale), ggml_scale(ctx, r, r_scale));
+}
+
+llama_model_beam::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
+    const int64_t n_embd_head = hparams.n_embd_head_v();
+    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+
+    ggml_tensor * cur;
+    ggml_tensor * inpL;
+
+    inpL = build_inp_embd(model.tok_embd);
+
+    ggml_tensor * inp_pos = build_inp_pos();
+
+    bool has_nope = false;
+    for (int il = 0; il < n_layer; ++il) {
+        has_nope = has_nope || !hparams.is_swa(il);
+    }
+    ggml_tensor * inp_attn_scale = has_nope ? build_inp_attn_scale() : nullptr;
+
+    auto * inp_attn = build_attn_inp_kv_iswa();
+
+    ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+    const float kq_scale = 1.0f / sqrtf(float(n_embd_head));
+
+    for (int il = 0; il < n_layer; ++il) {
+        const auto & layer = model.layers[il];
+
+        const bool is_swa = hparams.is_swa(il);
+
+        ggml_tensor * inpSA = inpL;
+
+        cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "attn_norm", il);
+
+        {
+            ggml_tensor * attn_inp = cur;
+
+            auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il);
+
+            Qcur = build_norm(Qcur, layer.attn_q_norm, NULL, LLM_NORM_RMS, il);
+            Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il);
+            cb(Qcur, "Qcur_normed", il);
+            cb(Kcur, "Kcur_normed", il);
+
+            if (is_swa) {
+                const float freq_base_l  = model.get_rope_freq_base (cparams, il);
+                const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
+
+                Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,
+                        n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+                        ext_factor, attn_factor, beta_fast, beta_slow);
+                Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
+                        n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+                        ext_factor, attn_factor, beta_fast, beta_slow);
+                cb(Qcur, "Qcur_rope", il);
+                cb(Kcur, "Kcur_rope", il);
+            } else {
+                Qcur = ggml_mul(ctx0, Qcur, inp_attn_scale);
+                cb(Qcur, "Qcur_scaled", il);
+            }
+
+            cur = build_attn(inp_attn,
+                    NULL, NULL, NULL, // o_proj is applied after the gate
+                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
+            cb(cur, "attn_out", il);
+
+            ggml_tensor * gate = build_lora_mm(layer.wqkv_gate, attn_inp);
+            cb(gate, "attn_gate_proj", il);
+            gate = ggml_reshape_3d(ctx0, gate, n_embd_head, n_head, n_tokens);
+            gate = ggml_add(ctx0, gate, ggml_reshape_3d(ctx0, layer.wqkv_gate_b, 1, n_head, 1));
+            gate = ggml_sigmoid(ctx0, gate);
+            gate = ggml_reshape_2d(ctx0, gate, n_embd_head * n_head, n_tokens);
+            cb(gate, "attn_gate_sig", il);
+
+            cur = ggml_mul(ctx0, cur, gate);
+            cb(cur, "attn_gated", il);
+
+            cur = build_lora_mm(layer.wo, cur, layer.wo_s);
+            cb(cur, "attn_o_proj", il);
+        }
+
+        cur = build_norm(cur, layer.attn_post_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "attn_post_norm", il);
+
+        if (il == n_layer - 1 && inp_out_ids) {
+            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);
+            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+        }
+
+        const int i_attn = 2 * (il + 1);
+        ggml_tensor * ffn_inp = beam_residual_add(ctx0, cur, inpSA, i_attn);
+        cb(ffn_inp, "ffn_inp", il);
+
+        cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "ffn_norm", il);
+
+        if ((uint32_t) il < hparams.n_layer_dense_lead) {
+            cur = build_ffn(cur,
+                    layer.ffn_up,   NULL, NULL,
+                    layer.ffn_gate, NULL, NULL,
+                    layer.ffn_down, NULL, NULL,
+                    NULL,
+                    LLM_FFN_SILU, LLM_FFN_PAR, il);
+            cb(cur, "ffn_out", il);
+        } else {
+            cur = build_moe_ffn(cur,
+                    layer.ffn_gate_inp,
+                    layer.ffn_up_exps,
+                    layer.ffn_gate_exps,
+                    layer.ffn_down_exps,
+                    layer.ffn_exp_probs_b,
+                    n_expert, n_expert_used,
+                    LLM_FFN_SILU,
+                    hparams.expert_weights_norm,
+                    hparams.expert_weights_scale,
+                    (llama_expert_gating_func_type) hparams.expert_gating_func,
+                    il);
+            cb(cur, "ffn_moe_out", il);
+        }
+
+        cur = build_norm(cur, layer.ffn_post_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "ffn_post_norm", il);
+
+        cur = beam_residual_add(ctx0, cur, ffn_inp, i_attn + 1);
+
+        cur = build_cvec(cur, il);
+        cb(cur, "l_out", il);
+
+        inpL = cur;
+    }
+
+    cur = inpL;
+
+    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
+    cb(cur, "result_norm", -1);
+    res->t_embd = cur;
+
+    cur = build_lora_mm(model.output, cur, model.output_s);
+
+    if (hparams.f_final_logit_softcapping != 0.0f) {
+        cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping);
+        cur = ggml_tanh(ctx0, cur);
+        cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping);
+    }
+
+    cb(cur, "result_output", -1);
+    res->t_logits = cur;
+}

src/models/models.h

@@ -2041,6 +2041,19 @@ struct llama_model_afmoe : public llama_model_base {
 };
 
 
+struct llama_model_beam : public llama_model_base {
+    llama_model_beam(const struct llama_model_params & params) : llama_model_base(params) {}
+    void load_arch_hparams(llama_model_loader & ml) override;
+    void load_arch_tensors(llama_model_loader & ml) override;
+
+    struct graph : public llm_graph_context {
+        graph(const llama_model & model, const llm_graph_params & params);
+    };
+
+    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
 struct llama_model_laguna : public llama_model_base {
     llama_model_laguna(const struct llama_model_params & params) : llama_model_base(params) {}
     void load_arch_hparams(llama_model_loader & ml) override;

tests/CMakeLists.txt

@@ -188,9 +188,11 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
     llama_build_and_test(test-llama-grammar.cpp LINK llama)
     llama_build_and_test(test-batch-alloc.cpp LINK llama)
     llama_build_and_test(test-chat.cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
-    target_link_libraries(test-chat PRIVATE llama-engine)
+    target_include_directories(test-chat PRIVATE ${PROJECT_SOURCE_DIR}/tools/server)
+    target_link_libraries(test-chat PRIVATE server-context)
     llama_build_and_test(test-server-component.cpp)
-    target_link_libraries(test-server-component PRIVATE llama-engine)
+    target_include_directories(test-server-component PRIVATE ${PROJECT_SOURCE_DIR}/tools/server)
+    target_link_libraries(test-server-component PRIVATE server-context)
     llama_build_and_test(test-json-schema-to-grammar.cpp)
 
     if (NOT GGML_BACKEND_DL)

tests/test-chat.cpp

@@ -7,7 +7,7 @@
 //
 #include "../src/llama-grammar.h"
 #include "../src/unicode.h"
-#include "../tools/engine/engine-chat.h"
+#include "../tools/server/server-chat.h"
 #include "chat-auto-parser.h"
 #include "chat.h"
 #include "common.h"
@@ -1864,7 +1864,7 @@ static void test_convert_responses_to_chatcmpl() {
             }
         })");
 
-        json result = engine_chat_convert_responses_to_chatcmpl(input);
+        json result = server_chat_convert_responses_to_chatcmpl(input);
 
         // Verify messages were converted correctly
         assert_equals(true, result.contains("messages"));
@@ -1904,7 +1904,7 @@ static void test_convert_responses_to_chatcmpl() {
             "model": "test-model"
         })");
 
-        json result = engine_chat_convert_responses_to_chatcmpl(input);
+        json result = server_chat_convert_responses_to_chatcmpl(input);
 
         assert_equals((size_t)1, result.at("messages").size());
         const auto & msg = result.at("messages")[0];
@@ -1920,7 +1920,7 @@ static void test_convert_responses_to_chatcmpl() {
             "model": "test-model"
         })");
 
-        json result = engine_chat_convert_responses_to_chatcmpl(input);
+        json result = server_chat_convert_responses_to_chatcmpl(input);
 
         assert_equals((size_t)2, result.at("messages").size());
         const auto & sys_msg = result.at("messages")[0];
@@ -1936,7 +1936,7 @@ static void test_convert_responses_to_chatcmpl() {
             "max_output_tokens": 100
         })");
 
-        json result = engine_chat_convert_responses_to_chatcmpl(input);
+        json result = server_chat_convert_responses_to_chatcmpl(input);
 
         assert_equals(true, result.contains("max_tokens"));
         assert_equals(false, result.contains("max_output_tokens"));
@@ -1980,7 +1980,7 @@ static void test_convert_responses_to_chatcmpl() {
             ]
         })");
 
-        json result = engine_chat_convert_responses_to_chatcmpl(input);
+        json result = server_chat_convert_responses_to_chatcmpl(input);
 
         assert_equals(true, result.contains("tools"));
         assert_equals(true, result.at("tools").is_array());
@@ -2015,7 +2015,7 @@ static void test_convert_responses_to_chatcmpl() {
             ]
         })");
 
-        json result = engine_chat_convert_responses_to_chatcmpl(input);
+        json result = server_chat_convert_responses_to_chatcmpl(input);
 
         assert_equals(false, result.contains("tools"));
     }
@@ -7938,7 +7938,7 @@ static void test_reasoning_budget_tokens_per_request() {
     // the reasoning-budget code path in oaicompat_chat_params_parse.
     auto tmpls = read_templates("models/templates/Qwen-Qwen3-0.6B.jinja");
 
-    engine_chat_params opt;
+    server_chat_params opt;
     opt.tmpls            = std::move(tmpls);
     opt.use_jinja        = true;
     opt.enable_thinking  = true;
@@ -7970,7 +7970,7 @@ static void test_reasoning_budget_message_per_request() {
     // <think>...</think> markers enable the reasoning-budget block in oaicompat_chat_params_parse.
     auto tmpls = read_templates("models/templates/Qwen-Qwen3-0.6B.jinja");
 
-    engine_chat_params opt;
+    server_chat_params opt;
     opt.tmpls                   = std::move(tmpls);
     opt.use_jinja               = true;
     opt.enable_thinking         = true;

tests/test-jinja.cpp

@@ -2011,6 +2011,12 @@ static void test_object_methods(testing & t) {
         "a b "
     );
 
+    test_template(t, "object | list",
+        "{{ (obj | list) == ['b', 'a'] }} {{ obj | list | join(',') }}",
+        {{"obj", {{"b", 1}, {"a", 2}}}},
+        "True b,a"
+    );
+
     test_template(t, "object.values()",
         "{% for v in obj.values() %}{{ v }} {% endfor %}",
         {{"obj", {{"a", 1}, {"b", 2}}}},

tests/test-llama-archs.cpp

@@ -188,6 +188,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
     } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER) {
         n_head_kv = 2; // GQA coverage
     }
+    if (arch == LLM_ARCH_BEAM) {
+        n_head    = 4;
+        n_head_kv = 2;
+        n_layer   = 4;
+    }
     const uint32_t n_embd_head = n_embd / n_head;
 
     ms.add_kv(LLM_KV_GENERAL_ARCHITECTURE,      llm_arch_name(arch));
@@ -294,6 +299,13 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
 
     if (arch == LLM_ARCH_MAION_CODER) {
         ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector<uint32_t>{1, 1, 0, 0});
+    } else if (arch == LLM_ARCH_BEAM) {
+        std::vector<uint32_t> pattern;
+        pattern.reserve(n_layer);
+        for (uint32_t il = 0; il < n_layer; il++) {
+            pattern.push_back(il % 2);
+        }
+        ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);
     } else if (arch == LLM_ARCH_GEMMA4) {
         ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER,      n_embd/2);
         ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS,      uint32_t(0));
@@ -470,6 +482,14 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         }
     }
 
+    if (arch == LLM_ARCH_BEAM) {
+        ms.add_kv(LLM_KV_EXPERT_COUNT,            uint32_t(8));
+        ms.add_kv(LLM_KV_EXPERT_GROUP_COUNT,      uint32_t(4));
+        ms.add_kv(LLM_KV_EXPERT_GROUP_USED_COUNT, uint32_t(2));
+        ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE,    2.5f);
+        ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM,     true);
+    }
+
     if (arch == LLM_ARCH_MAION_CODER) {
         ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd / 2);
         ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, std::vector<uint32_t>{n_ff, n_ff / 2, n_ff, n_ff});
@@ -712,6 +732,7 @@ static bool moe_mandatory(const llm_arch arch) {
         case LLM_ARCH_BAILINGMOE3:
         case LLM_ARCH_DOTS1:
         case LLM_ARCH_AFMOE:
+        case LLM_ARCH_BEAM:
         case LLM_ARCH_ERNIE4_5:
         case LLM_ARCH_ERNIE4_5_MOE:
         case LLM_ARCH_HUNYUAN_MOE:

tests/test-server-component.cpp

@@ -1,8 +1,7 @@
 #include "testing.h"
 
-#include "../tools/engine/engine-common.h"
-#include "../tools/server/server-subproc.h"
-#include "../tools/engine/engine-decision.h"
+#include "../tools/server/server-common.h"
+#include "../tools/server/server-decision.h"
 
 #include <cstdlib>
 #include <regex>
@@ -16,11 +15,11 @@
 // see skills/code-review/SKILL.md for more
 
 //
-// engine_tokens (text only, has_mtmd = false)
+// server_tokens (text only, has_mtmd = false)
 //
 
-static void test_engine_tokens_basic(testing & t) {
-    engine_tokens st(llama_tokens{1, 2, 3}, false);
+static void test_server_tokens_basic(testing & t) {
+    server_tokens st(llama_tokens{1, 2, 3}, false);
     t.assert_equal("size", (size_t) 3, st.size());
     t.assert_true("not empty", !st.empty());
     t.assert_equal("operator[]", 2, st[1]);
@@ -48,8 +47,8 @@ static void test_engine_tokens_basic(testing & t) {
     t.assert_true("clear", st.empty());
 }
 
-static void test_engine_tokens_pos(testing & t) {
-    engine_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
+static void test_server_tokens_pos(testing & t) {
+    server_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
     t.assert_equal("pos_next()", (llama_pos) 5, st.pos_next());
     t.assert_equal("pos_next(0)", (llama_pos) 0, st.pos_next(0));
     t.assert_equal("pos_next(3)", (llama_pos) 3, st.pos_next(3));
@@ -59,8 +58,8 @@ static void test_engine_tokens_pos(testing & t) {
     t.assert_equal("size_up_to_pos clamps", (size_t) 5, st.size_up_to_pos(100));
 }
 
-static void test_engine_tokens_keep_first(testing & t) {
-    engine_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
+static void test_server_tokens_keep_first(testing & t) {
+    server_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
     st.keep_first(5);
     t.assert_equal("keep all", (size_t) 5, st.size());
     st.keep_first(2);
@@ -69,21 +68,21 @@ static void test_engine_tokens_keep_first(testing & t) {
     t.assert_true("keep 0", st.empty());
 }
 
-static void test_engine_tokens_common_prefix(testing & t) {
-    engine_tokens a(llama_tokens{1, 2, 3, 4}, false);
-    engine_tokens empty(llama_tokens{}, false);
+static void test_server_tokens_common_prefix(testing & t) {
+    server_tokens a(llama_tokens{1, 2, 3, 4}, false);
+    server_tokens empty(llama_tokens{}, false);
 
     t.assert_equal("both empty",  (size_t) 0, empty.get_common_prefix(empty));
     t.assert_equal("one empty",   (size_t) 0, a.get_common_prefix(empty));
     t.assert_equal("identical",   (size_t) 4, a.get_common_prefix(a));
-    t.assert_equal("diverge at 2", (size_t) 2, a.get_common_prefix(engine_tokens(llama_tokens{1, 2, 9, 4}, false)));
-    t.assert_equal("diverge at 0", (size_t) 0, a.get_common_prefix(engine_tokens(llama_tokens{9, 2, 3, 4}, false)));
-    t.assert_equal("b is prefix",  (size_t) 2, a.get_common_prefix(engine_tokens(llama_tokens{1, 2}, false)));
-    t.assert_equal("a is prefix",  (size_t) 4, a.get_common_prefix(engine_tokens(llama_tokens{1, 2, 3, 4, 5, 6}, false)));
+    t.assert_equal("diverge at 2", (size_t) 2, a.get_common_prefix(server_tokens(llama_tokens{1, 2, 9, 4}, false)));
+    t.assert_equal("diverge at 0", (size_t) 0, a.get_common_prefix(server_tokens(llama_tokens{9, 2, 3, 4}, false)));
+    t.assert_equal("b is prefix",  (size_t) 2, a.get_common_prefix(server_tokens(llama_tokens{1, 2}, false)));
+    t.assert_equal("a is prefix",  (size_t) 4, a.get_common_prefix(server_tokens(llama_tokens{1, 2, 3, 4, 5, 6}, false)));
 }
 
-static void test_engine_tokens_serialize(testing & t) {
-    // same packing as slot save/restore in engine-context.cpp
+static void test_server_tokens_serialize(testing & t) {
+    // same packing as slot save/restore in server-context.cpp
     auto pack = [](const std::vector<char> & data) {
         llama_tokens packed(data.size() / sizeof(llama_token));
         memcpy(packed.data(), data.data(), data.size());
@@ -91,29 +90,29 @@ static void test_engine_tokens_serialize(testing & t) {
     };
 
     t.test("roundtrip", [&](testing & t) {
-        engine_tokens st(llama_tokens{10, 20, 30}, false);
+        server_tokens st(llama_tokens{10, 20, 30}, false);
         std::vector<char> data = st.serialize();
         t.assert_equal("aligned to llama_token", (size_t) 0, data.size() % sizeof(llama_token));
 
-        engine_tokens restored = engine_tokens::deserialize(pack(data), false);
+        server_tokens restored = server_tokens::deserialize(pack(data), false);
         t.assert_true("tokens", restored.get_tokens() == st.get_tokens());
     });
 
     t.test("roundtrip empty", [&](testing & t) {
-        engine_tokens st(llama_tokens{}, false);
-        engine_tokens restored = engine_tokens::deserialize(pack(st.serialize()), false);
+        server_tokens st(llama_tokens{}, false);
+        server_tokens restored = server_tokens::deserialize(pack(st.serialize()), false);
         t.assert_true("empty", restored.empty());
     });
 
     t.test("legacy plain token list", [&](testing & t) {
-        engine_tokens restored = engine_tokens::deserialize(llama_tokens{7, 8, 9}, false);
+        server_tokens restored = server_tokens::deserialize(llama_tokens{7, 8, 9}, false);
         t.assert_true("tokens", restored.get_tokens() == llama_tokens({7, 8, 9}));
     });
 
     auto expect_throw = [](testing & t, const std::string & msg, const llama_tokens & packed) {
         bool thrown = false;
         try {
-            engine_tokens::deserialize(packed, false);
+            server_tokens::deserialize(packed, false);
         } catch (const std::exception &) {
             thrown = true;
         }
@@ -121,7 +120,7 @@ static void test_engine_tokens_serialize(testing & t) {
     };
 
     t.test("corrupted", [&](testing & t) {
-        llama_tokens good = pack(engine_tokens(llama_tokens{10, 20, 30}, false).serialize());
+        llama_tokens good = pack(server_tokens(llama_tokens{10, 20, 30}, false).serialize());
 
         llama_tokens bad_version = good;
         bad_version[1] = 999;
@@ -143,22 +142,22 @@ static void test_engine_tokens_serialize(testing & t) {
     });
 }
 
-static void test_engine_tokens_clone(testing & t) {
-    engine_tokens st(llama_tokens{1, 2, 3}, false);
-    engine_tokens cl = st.clone();
+static void test_server_tokens_clone(testing & t) {
+    server_tokens st(llama_tokens{1, 2, 3}, false);
+    server_tokens cl = st.clone();
     cl.push_back(4);
     t.assert_equal("original untouched", (size_t) 3, st.size());
     t.assert_equal("clone grew", (size_t) 4, cl.size());
     t.assert_true("get_text_tokens", cl.get_text_tokens() == llama_tokens({1, 2, 3, 4}));
 }
 
-static void test_engine_tokens(testing & t) {
-    t.test("basic",         test_engine_tokens_basic);
-    t.test("pos",           test_engine_tokens_pos);
-    t.test("keep_first",    test_engine_tokens_keep_first);
-    t.test("common_prefix", test_engine_tokens_common_prefix);
-    t.test("serialize",     test_engine_tokens_serialize);
-    t.test("clone",         test_engine_tokens_clone);
+static void test_server_tokens(testing & t) {
+    t.test("basic",         test_server_tokens_basic);
+    t.test("pos",           test_server_tokens_pos);
+    t.test("keep_first",    test_server_tokens_keep_first);
+    t.test("common_prefix", test_server_tokens_common_prefix);
+    t.test("serialize",     test_server_tokens_serialize);
+    t.test("clone",         test_server_tokens_clone);
 }
 
 //
@@ -375,7 +374,7 @@ static void test_server_pipe(testing & t) {
 }
 
 //
-// engine_decision_escape_special_tokens
+// server_decision_escape_special_tokens
 //
 
 // "<|name|>" with U+00A6 in place of "|"
@@ -399,7 +398,7 @@ static void test_decision_escape(testing & t) {
             { "<|abc",            "<|abc"                          },
         };
         for (const auto & [text, expected] : cases) {
-            t.assert_equal(text, expected, engine_decision_escape_special_tokens(text));
+            t.assert_equal(text, expected, server_decision_escape_special_tokens(text));
         }
     });
 
@@ -419,7 +418,7 @@ static void test_decision_escape(testing & t) {
                 for (size_t k = 0, v = idx; k < len; k++, v /= chars.size()) {
                     text += chars[v % chars.size()];
                 }
-                if (engine_decision_escape_special_tokens(text) != std::regex_replace(text, re_special, "<\xC2\xA6$1\xC2\xA6>")) {
+                if (server_decision_escape_special_tokens(text) != std::regex_replace(text, re_special, "<\xC2\xA6$1\xC2\xA6>")) {
                     n_diff++;
                 }
                 n_strings++;
@@ -431,8 +430,8 @@ static void test_decision_escape(testing & t) {
     // std::regex overflowed the stack on these
     t.test("long input", [](testing & t) {
         const std::string name(1000000, 'a');
-        t.assert_true("not closed", engine_decision_escape_special_tokens("<|" + name) == "<|" + name);
-        t.assert_true("closed", engine_decision_escape_special_tokens("<|" + name + "|>") == escaped(name));
+        t.assert_true("not closed", server_decision_escape_special_tokens("<|" + name) == "<|" + name);
+        t.assert_true("closed", server_decision_escape_special_tokens("<|" + name + "|>") == escaped(name));
     });
 }
 
@@ -448,7 +447,7 @@ int main(int argc, char ** argv) {
         t.set_filter(argv[1]);
     }
 
-    t.test("engine_tokens", test_engine_tokens);
+    t.test("server_tokens", test_server_tokens);
     t.test("utf8",          test_utf8);
     t.test("sse",           test_sse);
     t.test("json_helpers",  test_json_helpers);

tools/CMakeLists.txt

@@ -23,7 +23,6 @@ else()
     add_subdirectory(quantize)
     if (LLAMA_BUILD_SERVER)
         add_subdirectory(ui)
-        add_subdirectory(engine)
         add_subdirectory(cli)
         add_subdirectory(server)
     endif()

tools/engine/CMakeLists.txt

@@ -1,29 +0,0 @@
-# llama-engine: inference engine (context, task queue, common utils), used by llama-server and CLI
-
-set(TARGET llama-engine)
-
-add_library(${TARGET}
-    engine-chat.cpp
-    engine-chat.h
-    engine-task.cpp
-    engine-task.h
-    engine-queue.cpp
-    engine-queue.h
-    engine-common.cpp
-    engine-common.h
-    engine-decision.cpp
-    engine-decision.h
-    engine-context.cpp
-    engine-context.h
-    engine-schema.cpp
-    engine-schema.h
-)
-set_target_properties(${TARGET} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
-
-target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
-target_include_directories(${TARGET} PRIVATE ../mtmd ${PROJECT_SOURCE_DIR})
-target_link_libraries(${TARGET} PUBLIC llama-common mtmd ${CMAKE_THREAD_LIBS_INIT})
-
-if(LLAMA_TOOLS_INSTALL)
-    install(TARGETS ${TARGET} LIBRARY)
-endif()

tools/engine/engine-context.h

@@ -1,131 +0,0 @@
-#pragma once
-
-#include "engine-common.h"
-#include "engine-decision.h"
-#include "engine-task.h"
-#include "engine-queue.h"
-
-#include "json.h"
-
-#include <cstddef>
-#include <memory>
-#include <mutex>
-#include <set>
-
-struct engine_context_impl; // private implementation
-
-constexpr int HTTP_POLLING_SECONDS = 1;
-
-struct engine_context_meta {
-    std::string build_info;
-    std::string model_name;
-    std::set<std::string> model_aliases;
-    std::set<std::string> model_tags;
-    std::string model_path;
-    std::vector<std::string> model_output_modalities; // output modalities for GET /models
-    bool has_mtmd;
-    bool has_inp_image;
-    bool has_inp_audio;
-    bool has_inp_video;
-    json json_ui_settings;
-    int slot_n_ctx;
-    enum llama_pooling_type pooling_type;
-
-    // chat params
-    engine_chat_params & chat_params;
-    std::map<std::string, bool> chat_template_caps;
-
-    // tokens
-    std::string bos_token_str;
-    std::string eos_token_str;
-    llama_token fim_pre_token;
-    llama_token fim_sub_token;
-    llama_token fim_mid_token;
-    llama_token fim_pad_token;
-    llama_token fim_rep_token;
-    llama_token fim_sep_token;
-
-    // sampling
-    std::vector<llama_logit_bias> logit_bias_eog;
-
-    // model meta
-    enum llama_vocab_type model_vocab_type;
-    int32_t model_vocab_n_tokens;
-    int32_t model_n_ctx_train;
-    int32_t model_n_embd_inp;
-    uint64_t model_n_params;
-    uint64_t model_size;
-    std::string model_ftype;
-};
-
-enum engine_state {
-    ENGINE_STATE_DOWNLOADING,
-    ENGINE_STATE_LOADING,
-    ENGINE_STATE_READY,
-    ENGINE_STATE_SLEEPING,
-};
-
-static std::string engine_state_to_str(engine_state state) {
-    switch (state) {
-        case ENGINE_STATE_DOWNLOADING: return "downloading";
-        case ENGINE_STATE_LOADING:     return "loading";
-        case ENGINE_STATE_READY:       return "ready";
-        case ENGINE_STATE_SLEEPING:    return "sleeping";
-        default: GGML_ASSERT(false && "invalid engine_state");
-    }
-}
-
-static engine_state engine_state_from_str(const std::string & str) {
-    if (str == "downloading") return ENGINE_STATE_DOWNLOADING;
-    if (str == "loading")     return ENGINE_STATE_LOADING;
-    if (str == "ready")       return ENGINE_STATE_READY;
-    if (str == "sleeping")    return ENGINE_STATE_SLEEPING;
-    GGML_ASSERT(false && "invalid engine_state string");
-}
-
-using engine_state_callback_t = std::function<void(engine_state, json /* payload */)>;
-
-struct engine_context {
-    std::unique_ptr<engine_context_impl> impl;
-
-    engine_context();
-    ~engine_context();
-
-    // load the model and initialize llama_context
-    // returns true on success
-    bool load_model(common_params & params);
-
-    // this function will block main thread until termination
-    void start_loop();
-
-    // terminate main loop (will unblock start_loop)
-    void terminate();
-
-    // get the underlaying llama_context, can return nullptr if sleeping
-    // not thread-safe, should only be used from the main thread
-    llama_context * get_llama_context() const;
-
-    // get a new response reader, used by CLI application
-    engine_response_reader get_response_reader();
-
-    // get server metadata (read-only), can only be called after load_model()
-    // not thread-safe, should only be used from the main thread
-    engine_context_meta get_meta() const;
-
-    // note: must be set before load_model() is called
-    void set_state_callback(engine_state_callback_t callback);
-
-    // used by the HTTP layer
-    // pointers can only be used when not sleeping, with thread-safe APIs (e.g., tokenizer calls)
-    llama_model * get_model() const;
-    const llama_vocab * get_vocab() const;
-    mtmd_context * get_mtmd_context() const;
-    const mtmd_helper_init_opt & get_mtmd_init_opt() const;
-    const engine_decision_context & get_decision() const;
-
-    engine_queue    & get_queue_tasks();
-    engine_response & get_queue_results();
-
-    engine_metrics get_metrics() const;
-    void reset_metrics_bucket();
-};

tools/engine/engine-decision.h

@@ -1,166 +0,0 @@
-#pragma once
-
-#include "engine-common.h"
-#include "engine-task.h"
-
-#include <map>
-#include <memory>
-#include <string>
-#include <vector>
-
-// typed decision models (TypeSafe /v1/systemone API)
-// the model answers each question in one forward pass, no token is generated
-
-enum engine_decision_question_type {
-    ENGINE_DECISION_QUESTION_CHOICE,
-    ENGINE_DECISION_QUESTION_SCORE,
-    ENGINE_DECISION_QUESTION_NOUL,
-};
-
-struct engine_decision_option {
-    std::string key;
-    json description; // null if not provided
-};
-
-struct engine_decision_question {
-    std::string id;
-    engine_decision_question_type type;
-    json instructions;
-    std::vector<engine_decision_option> options; // in the order of the model outputs
-};
-
-struct engine_decision_context {
-    common_decision_type type = COMMON_DECISION_TYPE_NONE;
-
-    // read the "<arch>.decision.*" metadata, type stays NONE if the model has none
-    void init(const llama_model * model);
-
-    // true if the questions of a request start with the same tokens, and the model can continue from them
-    bool can_share_prompt() const {
-        switch (type) {
-            case COMMON_DECISION_TYPE_OPENJEV:
-            case COMMON_DECISION_TYPE_LEV:
-            case COMMON_DECISION_TYPE_KEV:
-            case COMMON_DECISION_TYPE_NIMBLE:
-            case COMMON_DECISION_TYPE_PPLX_DECIDER:
-            case COMMON_DECISION_TYPE_VLLM_DECISION2:
-            case COMMON_DECISION_TYPE_LFM2_D1:
-                return true;
-            default:
-                return false;
-        }
-    }
-
-    // true if all the questions of a request go in one prompt, see fill_task_joint()
-    bool is_joint() const {
-        return type == COMMON_DECISION_TYPE_CLEF;
-    }
-
-    // true if the prompt of the model has a place for images
-    bool can_use_images() const {
-        switch (type) {
-            case COMMON_DECISION_TYPE_OPENJEV:
-            case COMMON_DECISION_TYPE_CLEF:
-            case COMMON_DECISION_TYPE_PPLX_DECIDER:
-            case COMMON_DECISION_TYPE_VLLM_D3:
-            case COMMON_DECISION_TYPE_LFM2_D1:
-            case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
-                return true;
-            default:
-                return false;
-        }
-    }
-
-    // throw std::invalid_argument on bad input
-    std::vector<engine_decision_question> parse_questions(const json & body) const;
-
-    // returns the state without its images, they are appended to files in order
-    // images come from "files" (alias "images") and from the image_url and input_audio parts of a state made of chat messages
-    // an image can be an audio clip if the model supports it
-    json parse_state(const json & body, std::vector<raw_buffer> & files) const;
-
-    // number of prompts that are evaluated to answer this question, each one shows the options in a different order
-    size_t n_variants(const engine_decision_question & question) const;
-
-    // set the prompt of one variant of this question, and where to read its result
-    // mctx is only used if there are files
-    void fill_task(
-            const json & state,
-            const std::vector<engine_decision_question> & questions,
-            const engine_decision_question & question,
-            size_t variant,
-            const std::vector<raw_buffer> & files,
-            mtmd_context * mctx,
-            const mtmd_helper_init_opt & init_opt,
-            engine_task & task) const;
-
-    // set the prompt of all the questions, the result has the scores of all their options, in order
-    // mctx is only used if there are files
-    void fill_task_joint(
-            const json & state,
-            const std::vector<engine_decision_question> & questions,
-            const std::vector<raw_buffer> & files,
-            mtmd_context * mctx,
-            const mtmd_helper_init_opt & init_opt,
-            engine_task & task) const;
-
-    // scores: the raw model outputs of each variant
-    json format_answer(const engine_decision_question & question, const std::vector<std::vector<float>> & scores, bool has_media = false) const;
-
-private:
-    const llama_vocab * vocab = nullptr;
-    std::shared_ptr<const common_chat_template> tmpl; // the "systemone" template
-
-    std::map<std::string, float> temperatures; // "<type>" or "<type>.<n_options bucket>"
-    std::map<size_t, std::vector<float>> score_bias; // added to the scores of a score question, by number of levels
-    size_t n_options_max   = 0;
-    bool   noul_true_first = false; // noul options are [true, false] instead of [false, true]
-    bool   noul_criteria_order = false; // noul options are in the order of the criteria keys, if given
-    bool   choice_sorted   = false; // choice options are in the order of their keys
-    bool   strict_payload  = false; // state, instructions and descriptions must be text, an object or an array
-    bool   canonical_legend = false; // non-string score descriptions are returned as canonical JSON text
-
-    // OPENJEV, LEV, NIMBLE, PPLX_DECIDER, VLLM_D3
-    std::vector<llama_token> labels;
-    std::vector<std::string> label_texts; // only if the label of an option is given to the template
-
-    // LAYA, KEV
-    llama_token token_marker      = LLAMA_TOKEN_NULL;
-    llama_token token_sep         = LLAMA_TOKEN_NULL;
-    std::string text_marker;
-    size_t      max_head_tokens   = 0; // question + options
-    size_t      max_option_tokens = 48;
-
-    std::string render(
-            const json & state,
-            const std::vector<engine_decision_question> & questions,
-            const engine_decision_question & question,
-            size_t variant,
-            size_t n_images,
-            bool is_audio = false) const;
-    json render_options(const engine_decision_question & question, size_t variant) const;
-    size_t n_outputs(const engine_decision_question & question) const;
-    // LFM2_D1: label text and tokens of each option
-    void d1_labels(const engine_decision_question & question, std::vector<std::string> & texts, std::vector<llama_tokens> & groups) const;
-    void fill_task_laya(llama_tokens & tokens, const engine_decision_question & question, engine_task & task) const;
-    void fill_task_d1omni(
-            const json & state,
-            const std::vector<engine_decision_question> & questions,
-            const engine_decision_question & question,
-            size_t variant,
-            const std::vector<raw_buffer> & files,
-            mtmd_context * mctx,
-            const mtmd_helper_init_opt & init_opt,
-            engine_task & task) const;
-
-    float get_temperature(const engine_decision_question & question) const;
-    double get_decision_confidence(engine_decision_question_type type, const std::vector<double> & probs) const;
-};
-
-// group the tasks so that the common prefix of their prompts is evaluated only once
-// each group is one parent and its children, it takes at most n_slots slots
-// note: the order of the tasks is preserved
-std::vector<engine_task> engine_decision_group_tasks(std::vector<engine_task> && tasks, size_t n_slots);
-
-// write "<|name|>" with U+00A6 in place of "|", so text from the request is not parsed as special tokens
-std::string engine_decision_escape_special_tokens(const std::string & text);

tools/server/CMakeLists.txt

@@ -1,5 +1,40 @@
 include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
 
+# server-context containing the core server logic, used by llama-server and CLI
+
+set(TARGET server-context)
+
+add_library(${TARGET} STATIC
+    server-chat.cpp
+    server-chat.h
+    server-task.cpp
+    server-task.h
+    server-queue.cpp
+    server-queue.h
+    server-common.cpp
+    server-common.h
+    server-decision.cpp
+    server-decision.h
+    server-context.cpp
+    server-context.h
+    server-stream.cpp
+    server-stream.h
+    server-tools.cpp
+    server-tools.h
+    server-mcp.cpp
+    server-mcp.h
+    server-schema.cpp
+    server-schema.h
+)
+
+if (BUILD_SHARED_LIBS)
+    set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+endif()
+
+target_include_directories(${TARGET} PRIVATE ../mtmd)
+target_include_directories(${TARGET} PRIVATE ${PROJECT_SOURCE_DIR})
+target_link_libraries(${TARGET} PUBLIC llama-common mtmd ${CMAKE_THREAD_LIBS_INIT})
+
 # llama-server-impl: server logic, reusable by app
 
 set(TARGET llama-server-impl)
@@ -8,16 +43,6 @@ add_library(${TARGET}
     server.cpp
     server-http.cpp
     server-http.h
-    server-routes.cpp
-    server-routes.h
-    server-stream.cpp
-    server-stream.h
-    server-subproc.cpp
-    server-subproc.h
-    server-tools.cpp
-    server-tools.h
-    server-mcp.cpp
-    server-mcp.h
     server-models.cpp
     server-models.h
 )
@@ -25,7 +50,7 @@ set_target_properties(${TARGET} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
 
 target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(${TARGET} PRIVATE ../mtmd ${PROJECT_SOURCE_DIR})
-target_link_libraries(${TARGET} PUBLIC llama-engine llama-ui cpp-httplib ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PUBLIC server-context llama-ui cpp-httplib ${CMAKE_THREAD_LIBS_INIT})
 
 add_dependencies(${TARGET} llama-ui-assets)
 

tools/server/README-dev.md

@@ -43,25 +43,20 @@ The server supports two primary operating modes:
 - **Inference mode**: The default mode for performing inference with a single loaded GGUF model.
 - **Router mode**: Enables management of multiple inference server instances behind a single API endpoint. Requests are automatically routed to the appropriate backend instance based on the requested model.
 
-The code is split into two libraries:
-
-- `libllama-engine` (`tools/engine`): the inference engine, including `engine_context`, task queue, tasks and common utils. It has no HTTP dependency.
-- `libllama-server-impl` (`tools/server`): the HTTP layer (`server_http_context`, `server_routes`), router mode, stream replay and MCP.
-
 The core architecture consists of the following components:
 
-- `engine_context`: Holds the primary inference state, including the main `llama_context` and all active slots.
-- `engine_slot`: An abstraction over a single “sequence” in llama.cpp, responsible for managing individual parallel inference requests.
-- `server_routes`: Middleware layer between `engine_context` and the HTTP interface; handles JSON parsing/formatting and request routing logic.
+- `server_context`: Holds the primary inference state, including the main `llama_context` and all active slots.
+- `server_slot`: An abstraction over a single “sequence” in llama.cpp, responsible for managing individual parallel inference requests.
+- `server_routes`: Middleware layer between `server_context` and the HTTP interface; handles JSON parsing/formatting and request routing logic.
 - `server_http_context`: Implements the HTTP server using `cpp-httplib`.
-- `engine_queue`: Thread-safe queue used by HTTP workers to submit new tasks to `engine_context`.
-- `engine_response`: Thread-safe queue used by `engine_context` to return results to HTTP workers.
-- `engine_response_reader`: Higher-level wrapper around the two queues above for cleaner code.
-- `engine_task`: Unit of work pushed into `engine_queue`.
-- `engine_task_result`: Unit of result pushed into `engine_response`.
-- `engine_tokens`: Unified representation of token sequences (supports both text and multimodal tokens); used by `engine_task` and `engine_slot`.
+- `server_queue`: Thread-safe queue used by HTTP workers to submit new tasks to `server_context`.
+- `server_response`: Thread-safe queue used by `server_context` to return results to HTTP workers.
+- `server_response_reader`: Higher-level wrapper around the two queues above for cleaner code.
+- `server_task`: Unit of work pushed into `server_queue`.
+- `server_task_result`: Unit of result pushed into `server_response`.
+- `server_tokens`: Unified representation of token sequences (supports both text and multimodal tokens); used by `server_task` and `server_slot`.
 - `server_prompt_checkpoint`: For recurrent (e.g., RWKV) and SWA models, stores snapshots of KV cache state. Enables reuse when subsequent requests share the same prompt prefix, saving redundant computation.
-- `server_models`: Standalone component for managing multiple backend instances (used in router mode). It is completely independent of `engine_context`.
+- `server_models`: Standalone component for managing multiple backend instances (used in router mode). It is completely independent of `server_context`.
 - `stream_session_manager`: process wide owner of resumable SSE stream sessions, keyed by conversation id. A file-static singleton inside `server-stream.cpp`, driven through `server_stream_session_manager_start/stop`. Backs the replay buffer that lets a client reattach to a generation after an HTTP disconnect. See the "Resumable streaming" section below.
 
 ```mermaid
@@ -69,13 +64,13 @@ graph TD
     API_User <--> server_http_context
     server_http_context <-- router mode --> server_models
     server_http_context <-- inference mode --> server_routes
-    server_routes -- engine_task --> engine_queue
-    subgraph engine_context
-        engine_queue --> engine_slot
-        engine_slot -- engine_task_result --> engine_response
-        engine_slot[multiple engine_slot]
+    server_routes -- server_task --> server_queue
+    subgraph server_context
+        server_queue --> server_slot
+        server_slot -- server_task_result --> server_response
+        server_slot[multiple server_slot]
     end
-    engine_response --> server_routes
+    server_response --> server_routes
 ```
 
 ### Batching
@@ -90,21 +85,21 @@ Following decoding, the system either retrieves embeddings or samples the next t
 
 ### Thread Management
 
-`engine_context` runs on a dedicated single thread. Because it is single-threaded, heavy post-processing (especially after token generation) should be avoided, as it directly impacts multi-sequence throughput.
+`server_context` runs on a dedicated single thread. Because it is single-threaded, heavy post-processing (especially after token generation) should be avoided, as it directly impacts multi-sequence throughput.
 
 Each incoming HTTP request is handled by its own thread managed by the HTTP library. The following operations are performed in HTTP worker threads:
 
 - JSON request parsing
 - Chat template application
 - Tokenization
-- Conversion of `engine_task_result` into final JSON response
+- Conversion of `server_task_result` into final JSON response
 - Error formatting into JSON
 - Tracking of partial/incremental responses (e.g., streaming tool calls or reasoning steps)
 
 **Best practices to follow:**
 
 - All JSON formatting and chat template logic must stay in the HTTP layer.
-- Avoid passing raw JSON between the HTTP layer and `engine_slot`. Instead, parse everything into native C++ types as early as possible.
+- Avoid passing raw JSON between the HTTP layer and `server_slot`. Instead, parse everything into native C++ types as early as possible.
 
 ### Example trace of a request
 
@@ -112,13 +107,13 @@ Here is an example trace of an API request for text completion:
 
 - A request arrives at the HTTP layer.
 - The request is routed to the corresponding handler inside `server_routes`. In this case, `handle_completions_impl` is invoked.
-- The handler parses the input request, constructs a new `engine_task`, and passes it to `server_res_generator`.
+- The handler parses the input request, constructs a new `server_task`, and passes it to `server_res_generator`.
 - `server_res_generator` creates a new `task_result_state` for each task:
     - `task_result_state` stays in the HTTP layer, responsible for keeping track of the current state of the response (e.g., parsing tool calls or thinking messages).
-    - `engine_task` is moved into `engine_queue` inside `engine_context`.
-- `engine_context` launches the task by moving it into an available slot (see `launch_slot_with_task()`).
+    - `server_task` is moved into `server_queue` inside `server_context`.
+- `server_context` launches the task by moving it into an available slot (see `launch_slot_with_task()`).
 - `update_slot()` processes the task as described in the "Batching" section above.
-- Results may be sent using `send_partial_response` or `send_final_response`, which creates a new `engine_task_result` and pushes it to the response queue.
+- Results may be sent using `send_partial_response` or `send_final_response`, which creates a new `server_task_result` and pushes it to the response queue.
 - At the same time, `server_res_generator` listens to the response queue and retrieves this response.
 - As the response is stateless, `server_res_generator` calls `response->update()` to update the response with the current state.
 - `server_res_generator` then calls `response->to_json()` and passes the response to the HTTP layer.
@@ -139,7 +134,7 @@ The implementation is hidden in `server-stream.cpp` (pimpl). The header exposes
 
 Producer side: `server_res_generator` extends `server_res_spipe`, which keeps all spipe logic out of the generic `server_http_res`. `set_req` attaches a producer when the header is present, and the wrapped `next` tees each chunk into the ring before the socket, so a chunk lost to a dead wire is already buffered. While attached, `should_stop` ignores peer disconnect: only a `DELETE` stops generation. On an early peer drop, `on_complete` drains the tail into the ring on the http worker.
 
-Lifetime safety: the session holds no back reference to the response, so `spipe` is a plain `unique_ptr` touched only by the http worker. `cancel` raises an atomic the producer polls; the producer finalizes the session from its destructor, which also runs `~engine_response_reader::stop()` to cancel the generation at the queue level. A `DELETE` stops work by raising the flag and letting the worker unwind.
+Lifetime safety: the session holds no back reference to the response, so `spipe` is a plain `unique_ptr` touched only by the http worker. `cancel` raises an atomic the producer polls; the producer finalizes the session from its destructor, which also runs `~server_response_reader::stop()` to cancel the generation at the queue level. A `DELETE` stops work by raising the flag and letting the worker unwind.
 
 Consumer side: `GET /v1/stream?conv_id=<id>&from=N` opens a `text/event-stream` that replays buffered bytes from offset `N` and blocks for live bytes, so the browser reattaches like a fresh EventSource. An offset below the dropped prefix returns 400.
 
@@ -286,7 +281,7 @@ The main goal of this API is to allow downloading models and/or removing models
 
 Instead of building everything from the ground up (like what most AI agents will do when you ask them to implement a similar feature), we built on top of existing, already well-engineered components inside the codebase:
 - Model cache infrastructure as mentioned above (`common/download.h`)
-- Server response queue (`engine-queue.h`). We use this feature to broadcast events to SSE clients.
+- Server response queue (`server-queue.h`). We use this feature to broadcast events to SSE clients.
 - Server router thread management (`server-models.h`). We re-use the same thread model that is used for managing subprocess life cycle, except that we don't create a new subprocess, but launch the download right inside the thread.
 
 The flow for downloading a new model:
@@ -299,27 +294,27 @@ The flow for downloading a new model:
 ### Sleep mode
 
 Sleep mode was initially introduced in PR [#18228](https://github.com/ggml-org/llama.cpp/pull/18228). The main idea is to have:
-- `engine_queue` keeping track of the idle timeout
-- When the timeout is detected, `engine_queue` signals to `engine_context_impl` that it should go into sleep
-- `engine_context_impl` frees all `llama_context` and `mtmd_context`
+- `server_queue` keeping track of the idle timeout
+- When the timeout is detected, `server_queue` signals to `server_context_impl` that it should go into sleep
+- `server_context_impl` frees all `llama_context` and `mtmd_context`
 
 Compared to simply exiting the whole process, this approach allows accessing some read-only endpoints during sleep, while also handling wakeup-on-request. Any inference request will wake the server up.
 
 Call stack on entering sleeping:
-- `engine_queue::start_loop` (main thread) sees no task for `idle_sleep_ms` --> `sleeping = true`
+- `server_queue::start_loop` (main thread) sees no task for `idle_sleep_ms` --> `sleeping = true`
 - `cb0(true)` --> `server_routes::update_cached_responses`
     - snapshots `/props`, `/models` and metrics; the model is still alive here
-- `cb1(true)` --> `engine_context_impl::handle_sleeping_state`
-    - `callback_state(ENGINE_STATE_SLEEPING)` --> reported to router in child mode
+- `cb1(true)` --> `server_context_impl::handle_sleeping_state`
+    - `callback_state(SERVER_STATE_SLEEPING)` --> reported to router in child mode
     - `destroy()` --> frees `llama_context` and `mtmd_context`
 - `condition_tasks.wait` until `req_stop_sleeping`
 
 Call stack on waking up:
-- `server_res_generator` constructor (HTTP thread) --> `engine_queue::wait_until_no_sleep`
+- `server_res_generator` constructor (HTTP thread) --> `server_queue::wait_until_no_sleep`
     - sets `req_stop_sleeping = true`, then waits until `sleeping == false`
-- `engine_queue::start_loop` (main thread) wakes up
-- `cb1(false)` --> `engine_context_impl::handle_sleeping_state`
-    - `load_model()`, which then emits `callback_state(ENGINE_STATE_READY)`
+- `server_queue::start_loop` (main thread) wakes up
+- `cb1(false)` --> `server_context_impl::handle_sleeping_state`
+    - `load_model()`, which then emits `callback_state(SERVER_STATE_READY)`
 - `cb0(false)` --> `server_routes::update_cached_responses`
     - nothing to do, the cache is only read during sleep
 - `sleeping = false` --> `notify_all` unblocks the HTTP thread, the request is handled as usual
@@ -330,7 +325,7 @@ Endpoints created with `create_response(true)` (`/health`, `/props`, `/models`,
 
 - Initial server implementation: https://github.com/ggml-org/llama.cpp/pull/1443
 - Parallel decoding support: https://github.com/ggml-org/llama.cpp/pull/3228
-- Refactor introducing `server_queue` and `server_response` (now `engine_queue` and `engine_response`): https://github.com/ggml-org/llama.cpp/pull/5065
+- Refactor introducing `server_queue` and `server_response`: https://github.com/ggml-org/llama.cpp/pull/5065
 - Reranking endpoint: https://github.com/ggml-org/llama.cpp/pull/9510
 - Multimodal model support (`libmtmd`): https://github.com/ggml-org/llama.cpp/pull/12898
 - Unified KV cache handling: https://github.com/ggml-org/llama.cpp/pull/16736

tools/server/server-chat.cpp

@@ -1,9 +1,9 @@
-#include "engine-chat.h"
-#include "engine-common.h"
+#include "server-chat.h"
+#include "server-common.h"
 
 #include <sstream>
 
-json engine_chat_convert_responses_to_chatcmpl(const json & response_body) {
+json server_chat_convert_responses_to_chatcmpl(const json & response_body) {
     if (!response_body.contains("input")) {
         throw std::invalid_argument("'input' is required");
     }
@@ -345,7 +345,7 @@ static void normalize_anthropic_billing_header(std::string & system_text) {
     }
 }
 
-json engine_chat_convert_anthropic_to_oai(const json & body) {
+json server_chat_convert_anthropic_to_oai(const json & body) {
     json oai_body;
 
     // Convert system prompt
@@ -618,7 +618,7 @@ json engine_chat_convert_anthropic_to_oai(const json & body) {
     return oai_body;
 }
 
-json engine_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff) {
+json server_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff) {
     json delta = json::object();
     if (!diff.reasoning_content_delta.empty()) {
         delta["reasoning_content"] = diff.reasoning_content_delta;

tools/server/server-chat.h

@@ -3,15 +3,16 @@
 #pragma once
 
 #include "chat.h"
-#include "engine-common.h"
+#include "server-common.h"
+#include "server-http.h"
 
 #include "json.h"
 
 // Convert OpenAI Responses API format to OpenAI Chat Completions API format
-json engine_chat_convert_responses_to_chatcmpl(const json & body);
+json server_chat_convert_responses_to_chatcmpl(const json & body);
 
 // Convert Anthropic Messages API format to OpenAI Chat Completions API format
-json engine_chat_convert_anthropic_to_oai(const json & body);
+json server_chat_convert_anthropic_to_oai(const json & body);
 
 // convert OpenAI transcriptions API format to OpenAI Chat Completions API format
 json convert_transcriptions_to_chatcmpl(
@@ -20,4 +21,4 @@ json convert_transcriptions_to_chatcmpl(
     const std::map<std::string, uploaded_file> & in_files,
     std::vector<raw_buffer> & out_files);
 
-json engine_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff);
+json server_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff);

tools/server/server-common.h

@@ -6,6 +6,7 @@
 #include "chat.h"
 #include "mtmd.h"
 #include "mtmd-helper.h"
+#include "subproc.h"
 
 #include "json.h"
 
@@ -38,12 +39,6 @@ using json = common_json;
 
 using raw_buffer = std::vector<uint8_t>;
 
-struct uploaded_file {
-    raw_buffer data;
-    std::string filename;
-    std::string content_type;
-};
-
 template <typename T>
 static T json_value(const json & body, const std::string & key, const T & default_value) {
     // Fallback null to default value
@@ -72,12 +67,12 @@ enum error_type {
 };
 
 // thin wrapper around common_grammar_trigger with (de)serialization functions
-struct engine_grammar_trigger {
+struct server_grammar_trigger {
     common_grammar_trigger value;
 
-    engine_grammar_trigger() = default;
-    engine_grammar_trigger(const common_grammar_trigger & value) : value(value) {}
-    engine_grammar_trigger(const json & in) {
+    server_grammar_trigger() = default;
+    server_grammar_trigger(const common_grammar_trigger & value) : value(value) {}
+    server_grammar_trigger(const json & in) {
         value.type = (common_grammar_trigger_type) in.at("type").get<int>();
         value.value = in.at("value").get<std::string>();
         if (value.type == COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN) {
@@ -115,10 +110,10 @@ const char * get_media_marker();
 //
 
 // output modalities for architecture.output_modalities in GET /models
-std::vector<std::string> engine_model_output_modalities(common_decision_type decision_type);
+std::vector<std::string> server_model_output_modalities(common_decision_type decision_type);
 
 // architecture object of GET /models; shared by the direct server and the router
-json engine_model_architecture_json(
+json server_model_architecture_json(
         bool inp_image,
         bool inp_audio,
         bool inp_video,
@@ -147,14 +142,14 @@ bool are_lora_equal(
 std::vector<size_t> lora_get_enabled_ids(const std::vector<common_adapter_lora_info> & loras);
 
 //
-// engine_tokens
+// server_tokens
 //
 
 /**
- * engine_tokens is a helper to manage the input tokens and image for the server.
+ * server_tokens is a helper to manage the input tokens and image for the server.
  * it is made this way to simplify the logic of KV cache management.
  */
-struct engine_tokens {
+struct server_tokens {
     bool has_mtmd = false;
 
 private: // disallow accessing these members directly, risking out-of-sync
@@ -177,24 +172,24 @@ private: // disallow accessing these members directly, risking out-of-sync
     // map_idx_to_media will contain: {5, img0}, {8, img1}
 
 public:
-    engine_tokens() = default;
-    ~engine_tokens() = default;
+    server_tokens() = default;
+    ~server_tokens() = default;
 
     // Prevent copying
-    // TODO: engine_tokens should be copyable - remove this:
-    engine_tokens(const engine_tokens&) = delete;
-    engine_tokens& operator=(const engine_tokens&) = delete;
+    // TODO: server_tokens should be copyable - remove this:
+    server_tokens(const server_tokens&) = delete;
+    server_tokens& operator=(const server_tokens&) = delete;
 
     // Allow moving (usually implicitly generated if members are movable)
-    engine_tokens(engine_tokens&&) = default;
-    engine_tokens& operator=(engine_tokens&&) = default;
+    server_tokens(server_tokens&&) = default;
+    server_tokens& operator=(server_tokens&&) = default;
 
     // Allow accessing elements using [] operator
     llama_token operator[](size_t index) { return tokens[index]; }
     const llama_token& operator[](size_t index) const { return tokens[index]; }
 
-    engine_tokens(mtmd::input_chunks & mtmd_chunks, bool has_mtmd);
-    engine_tokens(const llama_tokens & tokens, bool has_mtmd);
+    server_tokens(mtmd::input_chunks & mtmd_chunks, bool has_mtmd);
+    server_tokens(const llama_tokens & tokens, bool has_mtmd);
 
     // for debugging
     std::string str() const;
@@ -221,7 +216,7 @@ public:
     void push_back_placeholder(const mtmd_input_chunk * chunk);
 
     // appends server tokens, updates the media map. copies media chunks.
-    void push_back(engine_tokens & tokens);
+    void push_back(server_tokens & tokens);
 
     // for compatibility with context shift and prompt truncation
     void insert(const llama_tokens & inp_tokens);
@@ -232,7 +227,7 @@ public:
     llama_tokens get_text_tokens() const;
 
     std::vector<char> serialize() const;
-    static engine_tokens deserialize(const llama_tokens & packed, bool has_mtmd);
+    static server_tokens deserialize(const llama_tokens & packed, bool has_mtmd);
 
     // for compatibility with speculative decoding
     void set_token(llama_pos pos, llama_token id);
@@ -250,7 +245,7 @@ public:
 
     std::string detokenize(const llama_context * ctx, bool special) const;
 
-    size_t get_common_prefix(const engine_tokens & b) const;
+    size_t get_common_prefix(const server_tokens & b) const;
 
     // split the tokens into message spans, skipping over media chunks
     common_chat_msg_spans find_message_spans(const common_chat_msg_delimiters & delims) const;
@@ -258,7 +253,7 @@ public:
     // check text token IDs and the mapping between media chunks and token ranges
     bool validate(const struct llama_context * ctx) const;
 
-    engine_tokens clone() const;
+    server_tokens clone() const;
 };
 
 
@@ -295,9 +290,9 @@ void handle_media(
         const std::string & url,
         const std::string & media_path);
 
-// process mtmd prompt, return the engine_tokens containing both text tokens and media chunks
-// if is_placeholder is true, the media chunk will be treated as placeholder for counting tokens; the output tokens are not usable for actual inference (e.g. for submitting a task to engine_queue)
-engine_tokens process_mtmd_prompt(
+// process mtmd prompt, return the server_tokens containing both text tokens and media chunks
+// if is_placeholder is true, the media chunk will be treated as placeholder for counting tokens; the output tokens are not usable for actual inference (e.g. for submitting a task to server_queue)
+server_tokens process_mtmd_prompt(
                                         mtmd_context * mctx,
                                         const std::string & prompt,
                                         const std::vector<raw_buffer> & files,
@@ -317,7 +312,7 @@ engine_tokens process_mtmd_prompt(
  * - "prompt": [[12, 34, 56], [78, 90, 12]]
  * - "prompt": [[12, 34, "string", 56, 78], [12, 34, 56], { "prompt_string": "string", "multimodal_data": [ "base64" ]}]
  */
-std::vector<engine_tokens> tokenize_input_prompts(
+std::vector<server_tokens> tokenize_input_prompts(
                                         const llama_vocab * vocab,
                                         mtmd_context * mctx,
                                         const json & json_prompt,
@@ -326,7 +321,7 @@ std::vector<engine_tokens> tokenize_input_prompts(
                                         const mtmd_helper_init_opt & init_opt);
 
 // tokenize a single prompt, see tokenize_input_prompts() for the supported shapes
-engine_tokens tokenize_input_subprompt(
+server_tokens tokenize_input_subprompt(
                                         const llama_vocab * vocab,
                                         mtmd_context * mctx,
                                         const json & json_prompt,
@@ -339,7 +334,7 @@ engine_tokens tokenize_input_subprompt(
 //
 
 // global server parameters for chat formatting / parsing
-struct engine_chat_params {
+struct server_chat_params {
     bool use_jinja;
     bool prefill_assistant;
     common_reasoning_format reasoning_format;
@@ -362,28 +357,28 @@ json oaicompat_completion_params_parse(const json & body);
 json oaicompat_chat_params_parse(
     const llama_vocab * vocab,
     json & body, /* openai api json semantics */
-    const engine_chat_params & opt,
+    const server_chat_params & opt,
     std::vector<raw_buffer> & out_files,
     common_chat_session & out_session);
 
 // used by /embeddings endpoint, content has the same format as a chat message content array
-engine_tokens tokenize_oai_content_array(
+server_tokens tokenize_oai_content_array(
     const llama_vocab * vocab,
     mtmd_context * mctx,
-    const engine_chat_params & opt,
+    const server_chat_params & opt,
     json content,
     bool add_special,
     bool parse_special,
     const mtmd_helper_init_opt & init_opt);
 
-// TODO: move it to engine-task.cpp
+// TODO: move it to server-task.cpp
 json format_embeddings_response_oaicompat(
     const json & request,
     const std::string & model_name,
     const json & embeddings,
     bool use_base64 = false);
 
-// TODO: move it to engine-task.cpp
+// TODO: move it to server-task.cpp
 json format_response_rerank(
         const json & request,
         const std::string & model_name,
@@ -396,14 +391,14 @@ json format_response_rerank(
 // stats and metrics
 //
 
-// shared between engine_slot and engine_task_result_*
-struct engine_slot_stats {
+// shared between server_slot and server_task_result_*
+struct server_slot_stats {
     uint64_t n_prompt_cached    = 0;
     uint64_t n_prompt_processed = 0;
     uint64_t n_gen              = 0;
 
     // speculative decoding stats
-    // note: the per-position breakdown lives in engine_slot, it is not needed in a task result
+    // note: the per-position breakdown lives in server_slot, it is not needed in a task result
     uint64_t n_draft_tokens      = 0;
     uint64_t n_draft_accepted    = 0;
     uint64_t n_draft_verif_steps = 0;
@@ -483,9 +478,9 @@ struct engine_slot_stats {
     json to_json() const;
 };
 
-// shared between engine_context_impl and engine_task_result_*
-// unlike engine_slot_stats, engine_metrics is server-global and cumulative, not tied to a slot
-struct engine_metrics {
+// shared between server_context_impl and server_task_result_*
+// unlike server_slot_stats, server_metrics is server-global and cumulative, not tied to a slot
+struct server_metrics {
     int64_t t_start = 0;
 
     struct bucket {
@@ -573,7 +568,7 @@ bool is_valid_utf8(const std::string & str);
 
 //
 // formatting output responses
-// TODO: move these to engine-task.cpp
+// TODO: move these to server-task.cpp
 //
 
 llama_tokens format_prompt_infill(
@@ -588,10 +583,126 @@ llama_tokens format_prompt_infill(
         const llama_tokens & tokens_prompt);
 
 // format rerank task: [BOS]query[EOS][SEP]doc[EOS].
-engine_tokens format_prompt_rerank(
+server_tokens format_prompt_rerank(
         const struct llama_model * model,
         const struct llama_vocab * vocab,
         mtmd_context * mctx,
         const std::string & query,
         const std::string & doc,
         const mtmd_helper_init_opt & init_opt);
+
+// simple implementation of a pipe
+// used for streaming data between threads
+template<typename T>
+struct server_pipe {
+    std::mutex mutex;
+    std::condition_variable cv;
+    std::queue<T> queue;
+    std::atomic<bool> writer_closed{false};
+    std::atomic<bool> reader_closed{false};
+
+    // 0 = unbounded (default)
+    // > 0, write() drops the oldest item once the queue is full
+    size_t max_size = 0;
+
+    void close_write() {
+        writer_closed.store(true, std::memory_order_relaxed);
+        cv.notify_all();
+    }
+
+    void close_read() {
+        reader_closed.store(true, std::memory_order_relaxed);
+        cv.notify_all();
+    }
+
+    // close_on_stop = true: should_stop means the reader is gone for good, so the writer is told the pipe is broken.
+    // close_on_stop = false: should_stop is a per-read deadline and further reads still come, so the pipe stays usable.
+    bool read(T & output, const std::function<bool()> & should_stop, bool close_on_stop = true) {
+        std::unique_lock<std::mutex> lk(mutex);
+        constexpr auto poll_interval = std::chrono::milliseconds(500);
+        while (true) {
+            if (!queue.empty()) {
+                output = std::move(queue.front());
+                queue.pop();
+                return true;
+            }
+            if (writer_closed.load()) {
+                return false; // clean EOF
+            }
+            if (should_stop && should_stop()) { // a null should_stop means "never stop"
+                if (close_on_stop) {
+                    close_read(); // signal broken pipe to writer
+                }
+                return false; // cancelled / deadline reached
+            }
+            cv.wait_for(lk, poll_interval);
+        }
+    }
+
+    bool write(T && data) {
+        std::lock_guard<std::mutex> lk(mutex);
+        if (reader_closed.load()) {
+            return false; // broken pipe
+        }
+        if (max_size > 0) {
+            while (queue.size() >= max_size) {
+                queue.pop(); // drop oldest to stay bounded
+            }
+        }
+        queue.push(std::move(data));
+        cv.notify_one();
+        return true;
+    }
+};
+
+// a child server writes its state commands to stdout and its logs to stderr
+enum server_subproc_stream {
+    SERVER_SUBPROC_STDOUT,
+    SERVER_SUBPROC_STDERR,
+    SERVER_SUBPROC_STREAMS,
+};
+
+// gives the current stdout to the caller as a stream of its own, and sends everything else written to stdout to stderr
+FILE * server_reserve_stdout();
+
+// wrapper around common_subproc to manage a child server process
+// mainly used by router mode
+struct server_subproc {
+    common_subproc sproc;
+    std::atomic<bool> stopped{false}; // set by the monitor once the process exited and was reaped
+
+    bool is_alive() { return sproc.alive(); }
+    void terminate() { sproc.terminate(); }
+    int  join() { return sproc.join(); }
+
+    // true if both output pipes of the child are available (call after create())
+    bool has_output();
+
+    // true once both output pipes are closed
+    bool output_closed() const;
+
+    // non-blocking read from one output pipe
+    // returns the number of bytes read, 0 when nothing is available, -1 when the pipe is closed or broken
+    int read_output(server_subproc_stream stream, char * buf, size_t len);
+
+    // wait until one of a set of children has output, wake() is called, or a timeout passes
+    struct waiter {
+        waiter();
+        ~waiter();
+
+        // thread-safe; on Windows this is a no-op, wait() returns within 50 ms anyway
+        void wake();
+
+        // timeout_ms < 0 waits until data or wake(); ready[i] is set for each child with data on an open pipe (or a broken pipe)
+        void wait(const std::vector<server_subproc *> & procs, std::vector<bool> & ready, int64_t timeout_ms);
+
+    private:
+#ifndef _WIN32
+        intptr_t wake_fd[2] = { -1, -1 }; // POSIX self-pipe
+#endif
+    };
+
+private:
+    intptr_t out_handles[SERVER_SUBPROC_STREAMS] = { -1, -1 }; // fd on POSIX, HANDLE on Windows; taken lazily from sproc
+    bool     out_closed [SERVER_SUBPROC_STREAMS] = { false, false };
+};

tools/server/server-context.h

@@ -0,0 +1,196 @@
+#pragma once
+
+#include "server-http.h"
+#include "server-task.h"
+#include "server-queue.h"
+
+#include "json.h"
+
+#include <cstddef>
+#include <memory>
+#include <mutex>
+#include <set>
+
+struct server_context_impl; // private implementation
+
+struct server_context_meta {
+    std::string build_info;
+    std::string model_name;
+    std::set<std::string> model_aliases;
+    std::set<std::string> model_tags;
+    std::string model_path;
+    std::vector<std::string> model_output_modalities; // output modalities for GET /models
+    bool has_mtmd;
+    bool has_inp_image;
+    bool has_inp_audio;
+    bool has_inp_video;
+    json json_ui_settings;
+    int slot_n_ctx;
+    enum llama_pooling_type pooling_type;
+
+    // chat params
+    server_chat_params & chat_params;
+    std::map<std::string, bool> chat_template_caps;
+
+    // tokens
+    std::string bos_token_str;
+    std::string eos_token_str;
+    llama_token fim_pre_token;
+    llama_token fim_sub_token;
+    llama_token fim_mid_token;
+    llama_token fim_pad_token;
+    llama_token fim_rep_token;
+    llama_token fim_sep_token;
+
+    // sampling
+    std::vector<llama_logit_bias> logit_bias_eog;
+
+    // model meta
+    enum llama_vocab_type model_vocab_type;
+    int32_t model_vocab_n_tokens;
+    int32_t model_n_ctx_train;
+    int32_t model_n_embd_inp;
+    uint64_t model_n_params;
+    uint64_t model_size;
+    std::string model_ftype;
+};
+
+enum server_state {
+    SERVER_STATE_DOWNLOADING,
+    SERVER_STATE_LOADING,
+    SERVER_STATE_READY,
+    SERVER_STATE_SLEEPING,
+};
+
+static std::string server_state_to_str(server_state state) {
+    switch (state) {
+        case SERVER_STATE_DOWNLOADING: return "downloading";
+        case SERVER_STATE_LOADING:     return "loading";
+        case SERVER_STATE_READY:       return "ready";
+        case SERVER_STATE_SLEEPING:    return "sleeping";
+        default: GGML_ASSERT(false && "invalid server_state");
+    }
+}
+
+static server_state server_state_from_str(const std::string & str) {
+    if (str == "downloading") return SERVER_STATE_DOWNLOADING;
+    if (str == "loading")     return SERVER_STATE_LOADING;
+    if (str == "ready")       return SERVER_STATE_READY;
+    if (str == "sleeping")    return SERVER_STATE_SLEEPING;
+    GGML_ASSERT(false && "invalid server_state string");
+}
+
+using server_state_callback_t = std::function<void(server_state, json /* payload */)>;
+
+struct server_context {
+    std::unique_ptr<server_context_impl> impl;
+
+    server_context();
+    ~server_context();
+
+    // load the model and initialize llama_context
+    // returns true on success
+    bool load_model(common_params & params);
+
+    // this function will block main thread until termination
+    void start_loop();
+
+    // terminate main loop (will unblock start_loop)
+    void terminate();
+
+    // get the underlaying llama_context, can return nullptr if sleeping
+    // not thread-safe, should only be used from the main thread
+    llama_context * get_llama_context() const;
+
+    // get a new response reader, used by CLI application
+    server_response_reader get_response_reader();
+
+    // get server metadata (read-only), can only be called after load_model()
+    // not thread-safe, should only be used from the main thread
+    server_context_meta get_meta() const;
+
+    // note: must be set before load_model() is called
+    void set_state_callback(server_state_callback_t callback);
+};
+
+
+// forward declarations
+struct server_res_generator;
+
+struct server_routes {
+    server_routes(const common_params & params, server_context & ctx_server);
+
+    void init_routes();
+
+    // note: this is not thread-safe and can only when ctx_http.is_ready is false
+    void update_meta(const server_context & ctx_server) {
+        this->meta = std::make_unique<server_context_meta>(ctx_server.get_meta());
+    }
+
+    // handlers using lambda function, so that they can capture `this` without `std::bind`
+    // they won't be called until ctx_http.is_ready is set to true
+    server_http_context::handler_t get_health;
+    server_http_context::handler_t get_metrics;
+    server_http_context::handler_t get_slots;
+    server_http_context::handler_t post_slots;
+    server_http_context::handler_t get_props;
+    server_http_context::handler_t post_props;
+    server_http_context::handler_t post_infill;
+    server_http_context::handler_t post_completions;
+    server_http_context::handler_t post_completions_oai;
+    server_http_context::handler_t post_chat_completions;
+    server_http_context::handler_t post_chat_completions_tok;
+    server_http_context::handler_t post_control;
+    server_http_context::handler_t post_responses_oai;
+    server_http_context::handler_t post_responses_tok_oai;
+    server_http_context::handler_t post_transcriptions_oai;
+    server_http_context::handler_t post_anthropic_messages;
+    server_http_context::handler_t post_anthropic_count_tokens;
+    server_http_context::handler_t post_apply_template;
+    server_http_context::handler_t get_models;
+    server_http_context::handler_t post_tokenize;
+    server_http_context::handler_t post_detokenize;
+    server_http_context::handler_t post_embeddings;
+    server_http_context::handler_t post_embeddings_oai;
+    server_http_context::handler_t post_rerank;
+    server_http_context::handler_t post_systemone;
+    server_http_context::handler_t get_lora_adapters;
+    server_http_context::handler_t post_lora_adapters;
+
+    // to be used in router mode
+    json get_model_info() const;
+
+private:
+    std::unique_ptr<server_res_generator> handle_completions_impl(
+            const server_http_req & req,
+            server_task_type type,
+            const json & data,
+            const std::vector<raw_buffer> & files,
+            task_response_type res_type,
+            const common_chat_session & chat_session = {});
+    std::unique_ptr<server_res_generator> handle_slots_save(const server_http_req & req, int id_slot);
+    std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
+    std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
+    std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
+    std::unique_ptr<server_res_generator> handle_count_tokens(const server_http_req & req, task_response_type res_type);
+
+    // using unique_ptr to allow late initialization of const
+    std::unique_ptr<const server_context_meta> meta;
+
+    const common_params & params;
+    server_context_impl & ctx_server;
+
+    server_queue & queue_tasks;
+    server_response & queue_results;
+    std::unique_ptr<server_res_generator> create_response(bool bypass_sleep = false);
+
+    // cached responses, to be used during sleep
+    std::mutex     mutex_cache;
+    json           cached_models  = nullptr;
+    json           cached_props   = nullptr;
+    server_metrics cached_metrics;
+    // set when a scrape during sleep already reported the throughput buckets
+    bool           should_reset_buckets = false;
+    // call right before sleep to update the cached responses
+    void update_cached_responses(bool is_sleeping);
+};

tools/server/server-http.cpp

@@ -1,7 +1,7 @@
 #include "common.h"
 #include "http.h"
 #include "server-http.h"
-#include "engine-common.h"
+#include "server-common.h"
 #include "ui.h"
 
 #include <cpp-httplib/httplib.h>

tools/server/server-http.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#include "engine-common.h" // raw_buffer, uploaded_file
-
 #include <atomic>
 #include <functional>
 #include <map>
@@ -40,6 +38,13 @@ struct server_http_res {
 // unique pointer, used by set_chunked_content_provider
 // httplib requires the stream provider to be stored in heap
 using server_http_res_ptr = std::unique_ptr<server_http_res>;
+using raw_buffer = std::vector<uint8_t>;
+
+struct uploaded_file {
+    raw_buffer data;
+    std::string filename;
+    std::string content_type;
+};
 
 struct server_http_req {
     std::map<std::string, std::string> params; // path_params + query_params

tools/server/server-mcp.h

@@ -1,7 +1,6 @@
 #pragma once
 
-#include "engine-common.h"
-#include "server-subproc.h"
+#include "server-common.h"
 
 #include <atomic>
 #include <chrono>

tools/server/server-models.cpp

@@ -1,7 +1,7 @@
-#include "engine-common.h"
+#include "server-common.h"
 #include "http.h"
 #include "server-models.h"
-#include "engine-context.h"
+#include "server-context.h"
 #include "server-stream.h"
 
 #include "build-info.h"
@@ -539,7 +539,7 @@ void server_model_meta::update_args(common_preset_context & ctx_preset, std::str
 
 void server_model_meta::update_caps(const common_params & base) {
     // reset to the default so a failed refresh cannot keep old values
-    architecture = engine_model_architecture_json(false, false, false, {"text"});
+    architecture = server_model_architecture_json(false, false, false, {"text"});
     n_ctx_train  = 0;
 
     // resolve the model file offline; do not download
@@ -569,7 +569,7 @@ void server_model_meta::update_caps(const common_params & base) {
     std::vector<std::string> output_modalities = {"text"};
     if (!params.model.path.empty()) {
         const common_gguf_info info = common_get_gguf_info(params.model.path);
-        output_modalities = engine_model_output_modalities(info.decision_type);
+        output_modalities = server_model_output_modalities(info.decision_type);
         n_ctx_train       = info.n_ctx_train;
     }
 
@@ -587,7 +587,7 @@ void server_model_meta::update_caps(const common_params & base) {
     }
 
     // offline discovery cannot see video; a loaded model reports it
-    architecture = engine_model_architecture_json(inp_image, inp_audio, false, output_modalities);
+    architecture = server_model_architecture_json(inp_image, inp_audio, false, output_modalities);
 }
 
 //
@@ -687,7 +687,7 @@ void server_models::add_model(server_model_meta && meta) {
 }
 
 void server_models::notify_sse(const std::string & event, const std::string & model_id, const json & data) {
-    std::unique_ptr<engine_task_result_router> result = std::make_unique<engine_task_result_router>();
+    std::unique_ptr<server_task_result_router> result = std::make_unique<server_task_result_router>();
     result->data = {
         {"model", model_id},
         {"event", event},
@@ -1660,12 +1660,12 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co
 }
 
 void server_models::handle_child_state(const std::string & name, const std::string & raw_input) {
-    engine_state state;
+    server_state state;
     json payload;
 
     try {
         json data = json::parse(raw_input.substr(strlen(CMD_CHILD_TO_ROUTER_STATE)));
-        state = engine_state_from_str(json_value(data, "state", std::string()));
+        state = server_state_from_str(json_value(data, "state", std::string()));
         payload = json_value(data, "payload", json{});
     } catch (const std::exception & e) {
         SRV_ERR("failed to parse child state update for name=%s: %s\n", name.c_str(), e.what());
@@ -1673,7 +1673,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri
     }
 
     switch (state) {
-        case ENGINE_STATE_DOWNLOADING:
+        case SERVER_STATE_DOWNLOADING:
             {
                 std::string result = json_value(payload, "result", std::string());
                 std::string url    = json_value(payload, "url",    std::string());
@@ -1698,7 +1698,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri
                     update_download_progress(name, p, false);
                 }
             } break;
-        case ENGINE_STATE_LOADING:
+        case SERVER_STATE_LOADING:
             {
                 update_status(name, {
                     SERVER_MODEL_STATUS_LOADING,
@@ -1707,7 +1707,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri
                     payload,
                 });
             } break;
-        case ENGINE_STATE_READY:
+        case SERVER_STATE_READY:
             {
                 update_status(name, {
                     SERVER_MODEL_STATUS_LOADED,
@@ -1717,7 +1717,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri
                     {}, // reset progress info
                 });
             } break;
-        case ENGINE_STATE_SLEEPING:
+        case SERVER_STATE_SLEEPING:
             {
                 update_status(name, { SERVER_MODEL_STATUS_SLEEPING });
             } break;
@@ -1784,7 +1784,7 @@ struct server_download_state : public common_download_callback {
             {"downloaded", p.downloaded},
             {"total", p.total},
         };
-        self->notify_to_router(engine_state_to_str(ENGINE_STATE_DOWNLOADING), data);
+        self->notify_to_router(server_state_to_str(SERVER_STATE_DOWNLOADING), data);
     }
     void on_start(const common_download_progress & p) override {
         on_progress(p);
@@ -1820,7 +1820,7 @@ int server_child::run_download(common_params & params) {
 
     bool ok = dl.run(params);
 
-    notify_to_router(engine_state_to_str(ENGINE_STATE_DOWNLOADING), {
+    notify_to_router(server_state_to_str(SERVER_STATE_DOWNLOADING), {
         {"result", ok ? "download_finished" : "download_failed"},
     });
 
@@ -1874,12 +1874,12 @@ void server_child::notify_to_router(const std::string & state, const json & payl
 // server_models_routes
 //
 
-// RAII wrapper similar to engine_response_reader, but doesn't use engine_queue
+// RAII wrapper similar to server_response_reader, but doesn't use server_queue
 static std::atomic<int> sse_client_id_counter = 0;
 struct server_models_sse_client {
-    engine_response & queue_results;
+    server_response & queue_results;
     int client_id;
-    server_models_sse_client(engine_response & q)
+    server_models_sse_client(server_response & q)
             : queue_results(q), client_id(sse_client_id_counter.fetch_add(1, std::memory_order_relaxed)) {
         SRV_DBG("new SSE client connected, assigned client_id=%d\n", client_id);
         queue_results.add_waiting_task_id(client_id);
@@ -1891,10 +1891,10 @@ struct server_models_sse_client {
 
     // return nullptr if should_stop() is true before receiving a result
     // note: if one error is received, it will stop further processing and return error result
-    engine_task_result_ptr next(const std::function<bool()> & should_stop) {
+    server_task_result_ptr next(const std::function<bool()> & should_stop) {
         while (true) {
             static const int http_polling_seconds = 1; // check should_stop every 1 second
-            engine_task_result_ptr result = queue_results.recv_with_timeout({client_id}, http_polling_seconds);
+            server_task_result_ptr result = queue_results.recv_with_timeout({client_id}, http_polling_seconds);
             if (result == nullptr) {
                 // timeout, check stop condition
                 if (should_stop()) {

tools/server/server-models.h

@@ -3,10 +3,9 @@
 #include "common.h"
 #include "download.h"
 #include "preset.h"
-#include "engine-common.h"
+#include "server-common.h"
 #include "server-http.h"
-#include "engine-queue.h"
-#include "server-subproc.h"
+#include "server-queue.h"
 
 #include <mutex>
 #include <condition_variable>
@@ -86,7 +85,7 @@ struct server_model_meta {
     int exit_code = 0; // exit code of the model instance process (only valid if status == FAILED)
     int stop_timeout = 0; // seconds to wait before force-killing the model instance during shutdown
     bool hidden = false; // hidden from GET /models, but still accept if requested
-    json architecture = engine_model_architecture_json(false, false, false, {"text"});
+    json architecture = server_model_architecture_json(false, false, false, {"text"});
     uint32_t n_ctx_train = 0; // trained context, read from the GGUF metadata; 0 when unknown
 
     bool is_ready() const {
@@ -238,7 +237,7 @@ public:
     server_models(const common_params & params, int argc, char ** argv);
     ~server_models();
 
-    engine_response sse; // for real-time updates via SSE endpoint
+    server_response sse; // for real-time updates via SSE endpoint
 
     // (re-)load the list of models from various sources and prepare the metadata mapping
     // - if this is called the first time, simply populate the metadata

tools/server/server-queue.cpp

@@ -1,5 +1,5 @@
-#include "engine-task.h"
-#include "engine-queue.h"
+#include "server-task.h"
+#include "server-queue.h"
 
 #include "log.h"
 
@@ -18,18 +18,18 @@
 #define RES_DBG(fmt, ...) LOG_DBG("res  %12.*s: " fmt, 12, __func__, __VA_ARGS__)
 
 //
-// engine_queue
+// server_queue
 //
 
-static bool task_resets_idle_timer(engine_task_type type) {
-    return type != ENGINE_TASK_TYPE_METRICS;
+static bool task_resets_idle_timer(server_task_type type) {
+    return type != SERVER_TASK_TYPE_METRICS;
 }
 
-int engine_queue::post(engine_task && task, bool front) {
+int server_queue::post(server_task && task, bool front) {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     GGML_ASSERT(task.id != -1);
     // if this is cancel task make sure to clean up pending tasks
-    if (task.type == ENGINE_TASK_TYPE_CANCEL) {
+    if (task.type == SERVER_TASK_TYPE_CANCEL) {
         cleanup_pending_task(task.id_target);
     }
     const int  task_id     = task.id;
@@ -47,7 +47,7 @@ int engine_queue::post(engine_task && task, bool front) {
     return task_id;
 }
 
-int engine_queue::post(std::vector<engine_task> && tasks, bool front) {
+int server_queue::post(std::vector<server_task> && tasks, bool front) {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     bool reset_timer = false;
     for (auto & task : tasks) {
@@ -55,7 +55,7 @@ int engine_queue::post(std::vector<engine_task> && tasks, bool front) {
             task.id = id++;
         }
         // if this is cancel task make sure to clean up pending tasks
-        if (task.type == ENGINE_TASK_TYPE_CANCEL) {
+        if (task.type == SERVER_TASK_TYPE_CANCEL) {
             cleanup_pending_task(task.id_target);
         }
         reset_timer |= task_resets_idle_timer(task.type);
@@ -73,7 +73,7 @@ int engine_queue::post(std::vector<engine_task> && tasks, bool front) {
     return 0;
 }
 
-void engine_queue::defer(engine_task && task) {
+void server_queue::defer(server_task && task) {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     QUE_DBG("defer task, id = %d\n", task.id);
     queue_tasks_deferred.push_back(std::move(task));
@@ -81,13 +81,13 @@ void engine_queue::defer(engine_task && task) {
     condition_tasks.notify_one();
 }
 
-int engine_queue::get_new_id() {
+int server_queue::get_new_id() {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     int new_id = id++;
     return new_id;
 }
 
-void engine_queue::pop_deferred_task(int id_slot) {
+void server_queue::pop_deferred_task(int id_slot) {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     if (!queue_tasks_deferred.empty()) {
         // try to find a task that uses the specified slot
@@ -112,7 +112,7 @@ void engine_queue::pop_deferred_task(int id_slot) {
     condition_tasks.notify_one();
 }
 
-void engine_queue::wait_until_no_sleep() {
+void server_queue::wait_until_no_sleep() {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     if (!sleeping) {
         return;
@@ -129,13 +129,13 @@ void engine_queue::wait_until_no_sleep() {
     }
 }
 
-void engine_queue::terminate() {
+void server_queue::terminate() {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     running = false;
     condition_tasks.notify_all();
 }
 
-bool engine_queue::process_new_tasks(bool is_yielding) {
+bool server_queue::process_new_tasks(bool is_yielding) {
     while (true) {
         std::unique_lock<std::mutex> lock(mutex_tasks);
         if (!running) {
@@ -145,7 +145,7 @@ bool engine_queue::process_new_tasks(bool is_yielding) {
         if (queue_tasks.empty()) {
             return false;
         }
-        engine_task task = std::move(queue_tasks.front());
+        server_task task = std::move(queue_tasks.front());
         queue_tasks.pop_front();
         lock.unlock();
 
@@ -160,7 +160,7 @@ bool engine_queue::process_new_tasks(bool is_yielding) {
     }
 }
 
-void engine_queue::worker_loop() {
+void server_queue::worker_loop() {
     while (true) {
         {
             std::unique_lock<std::mutex> lock(mutex_tasks);
@@ -206,7 +206,7 @@ void engine_queue::worker_loop() {
     }
 }
 
-void engine_queue::worker_stop() {
+void server_queue::worker_stop() {
     if (!worker.thread.joinable()) {
         return;
     }
@@ -219,7 +219,7 @@ void engine_queue::worker_stop() {
     worker.thread.join();
 }
 
-void engine_queue::yield_to_queue(std::function<void()> && work) {
+void server_queue::yield_to_queue(std::function<void()> && work) {
     GGML_ASSERT(worker.thread.joinable() && "yield_to_queue() requires start_loop() to be running");
 
     QUE_DBG("%s", "yielding to queue\n");
@@ -275,7 +275,7 @@ void engine_queue::yield_to_queue(std::function<void()> && work) {
     }
 }
 
-void engine_queue::start_loop(int64_t idle_sleep_ms) {
+void server_queue::start_loop(int64_t idle_sleep_ms) {
     running = true;
     time_last_task = ggml_time_ms();
 
@@ -365,9 +365,9 @@ void engine_queue::start_loop(int64_t idle_sleep_ms) {
     worker_stop();
 }
 
-void engine_queue::cleanup_pending_task(int id_target) {
+void server_queue::cleanup_pending_task(int id_target) {
     // no need lock because this is called exclusively by post()
-    auto rm_func = [id_target](const engine_task & task) {
+    auto rm_func = [id_target](const server_task & task) {
         return task.id == id_target;
     };
     queue_tasks.erase(
@@ -383,17 +383,17 @@ void engine_queue::cleanup_pending_task(int id_target) {
 }
 
 //
-// engine_response
+// server_response
 //
 
-void engine_response::add_waiting_task_id(int id_task) {
+void server_response::add_waiting_task_id(int id_task) {
     RES_DBG("add task %d to waiting list. current waiting = %d (before add)\n", id_task, (int) waiting_task_ids.size());
 
     std::unique_lock<std::mutex> lock(mutex_results);
     waiting_task_ids.insert(id_task);
 }
 
-void engine_response::add_waiting_task_ids(const std::unordered_set<int> & id_tasks) {
+void server_response::add_waiting_task_ids(const std::unordered_set<int> & id_tasks) {
     std::unique_lock<std::mutex> lock(mutex_results);
 
     for (const auto & id_task : id_tasks) {
@@ -402,20 +402,20 @@ void engine_response::add_waiting_task_ids(const std::unordered_set<int> & id_ta
     }
 }
 
-void engine_response::remove_waiting_task_id(int id_task) {
+void server_response::remove_waiting_task_id(int id_task) {
     RES_DBG("remove task %d from waiting list. current waiting = %d (before remove)\n", id_task, (int) waiting_task_ids.size());
 
     std::unique_lock<std::mutex> lock(mutex_results);
     waiting_task_ids.erase(id_task);
     // make sure to clean up all pending results
     queue_results.erase(
-        std::remove_if(queue_results.begin(), queue_results.end(), [id_task](const engine_task_result_ptr & res) {
+        std::remove_if(queue_results.begin(), queue_results.end(), [id_task](const server_task_result_ptr & res) {
             return res->id == id_task;
         }),
         queue_results.end());
 }
 
-void engine_response::remove_waiting_task_ids(const std::unordered_set<int> & id_tasks) {
+void server_response::remove_waiting_task_ids(const std::unordered_set<int> & id_tasks) {
     std::unique_lock<std::mutex> lock(mutex_results);
 
     for (const auto & id_task : id_tasks) {
@@ -424,7 +424,7 @@ void engine_response::remove_waiting_task_ids(const std::unordered_set<int> & id
     }
 }
 
-engine_task_result_ptr engine_response::recv(const std::unordered_set<int> & id_tasks) {
+server_task_result_ptr server_response::recv(const std::unordered_set<int> & id_tasks) {
     while (true) {
         std::unique_lock<std::mutex> lock(mutex_results);
         condition_results.wait(lock, [&]{
@@ -437,7 +437,7 @@ engine_task_result_ptr engine_response::recv(const std::unordered_set<int> & id_
 
         for (size_t i = 0; i < queue_results.size(); i++) {
             if (id_tasks.find(queue_results[i]->id) != id_tasks.end()) {
-                engine_task_result_ptr res = std::move(queue_results[i]);
+                server_task_result_ptr res = std::move(queue_results[i]);
                 queue_results.erase(queue_results.begin() + i);
                 return res;
             }
@@ -447,13 +447,13 @@ engine_task_result_ptr engine_response::recv(const std::unordered_set<int> & id_
     // should never reach here
 }
 
-engine_task_result_ptr engine_response::recv_with_timeout(const std::unordered_set<int> & id_tasks, int timeout) {
+server_task_result_ptr server_response::recv_with_timeout(const std::unordered_set<int> & id_tasks, int timeout) {
     while (true) {
         std::unique_lock<std::mutex> lock(mutex_results);
 
         for (int i = 0; i < (int) queue_results.size(); i++) {
             if (id_tasks.find(queue_results[i]->id) != id_tasks.end()) {
-                engine_task_result_ptr res = std::move(queue_results[i]);
+                server_task_result_ptr res = std::move(queue_results[i]);
                 queue_results.erase(queue_results.begin() + i);
                 return res;
             }
@@ -472,12 +472,12 @@ engine_task_result_ptr engine_response::recv_with_timeout(const std::unordered_s
     // should never reach here
 }
 
-engine_task_result_ptr engine_response::recv(int id_task) {
+server_task_result_ptr server_response::recv(int id_task) {
     std::unordered_set<int> id_tasks = {id_task};
     return recv(id_tasks);
 }
 
-void engine_response::send(engine_task_result_ptr && result) {
+void server_response::send(server_task_result_ptr && result) {
     RES_DBG("sending result for task id = %d\n", result->id);
 
     std::unique_lock<std::mutex> lock(mutex_results);
@@ -492,27 +492,27 @@ void engine_response::send(engine_task_result_ptr && result) {
     }
 }
 
-void engine_response::broadcast(engine_task_result_ptr && result) {
+void server_response::broadcast(server_task_result_ptr && result) {
     std::unique_lock<std::mutex> lock(mutex_results);
     for (const auto & id_task : waiting_task_ids) {
         RES_DBG("task id = %d pushed to result queue\n", id_task);
-        engine_task_result_ptr res_copy(result->clone());
+        server_task_result_ptr res_copy(result->clone());
         res_copy->id = id_task; // override id with target task id
         queue_results.emplace_back(std::move(res_copy));
     }
     condition_results.notify_all();
 }
 
-void engine_response::terminate() {
+void server_response::terminate() {
     running = false;
     condition_results.notify_all();
 }
 
 //
-// engine_response_reader
+// server_response_reader
 //
 
-void engine_response_reader::post_task(engine_task && task, bool front) {
+void server_response_reader::post_task(server_task && task, bool front) {
     GGML_ASSERT(id_tasks.empty() && "post_task() can only be called once per reader");
     GGML_ASSERT(!task.is_parent() && "not supported, use post_tasks() instead");
     task.index = 0;
@@ -522,9 +522,9 @@ void engine_response_reader::post_task(engine_task && task, bool front) {
     queue_tasks.post(std::move(task), front);
 }
 
-void engine_response_reader::post_tasks(std::vector<engine_task> && tasks, const common_chat_session & session, bool front) {
+void server_response_reader::post_tasks(std::vector<server_task> && tasks, const common_chat_session & session, bool front) {
     GGML_ASSERT(id_tasks.empty() && "post_tasks() can only be called once per reader");
-    id_tasks = engine_task::get_list_id(tasks);
+    id_tasks = server_task::get_list_id(tasks);
     states.reserve(tasks.size());
     size_t index = 0;
     for (auto & task : tasks) {
@@ -541,15 +541,15 @@ void engine_response_reader::post_tasks(std::vector<engine_task> && tasks, const
     queue_tasks.post(std::move(tasks), front);
 }
 
-bool engine_response_reader::has_next() const {
+bool server_response_reader::has_next() const {
     return !cancelled && received_count < id_tasks.size();
 }
 
 // return nullptr if should_stop() is true before receiving a result
 // note: if one error is received, it will stop further processing and return error result
-engine_task_result_ptr engine_response_reader::next(const std::function<bool()> & should_stop) {
+server_task_result_ptr server_response_reader::next(const std::function<bool()> & should_stop) {
     while (true) {
-        engine_task_result_ptr result = queue_results.recv_with_timeout(id_tasks, polling_interval_seconds);
+        server_task_result_ptr result = queue_results.recv_with_timeout(id_tasks, polling_interval_seconds);
         if (result == nullptr) {
             // timeout, check stop condition
             if (should_stop()) {
@@ -577,7 +577,7 @@ engine_task_result_ptr engine_response_reader::next(const std::function<bool()>
     // should not reach here
 }
 
-engine_response_reader::batch_response engine_response_reader::wait_for_all(const std::function<bool()> & should_stop) {
+server_response_reader::batch_response server_response_reader::wait_for_all(const std::function<bool()> & should_stop) {
     batch_response batch_res;
     batch_res.results.clear();
     batch_res.results.resize(id_tasks.size());
@@ -599,16 +599,16 @@ engine_response_reader::batch_response engine_response_reader::wait_for_all(cons
     return batch_res;
 }
 
-void engine_response_reader::stop() {
+void server_response_reader::stop() {
     queue_results.remove_waiting_task_ids(id_tasks);
     if (has_next() && !cancelled) {
         // if tasks is not finished yet, cancel them
         cancelled = true;
-        std::vector<engine_task> cancel_tasks;
+        std::vector<server_task> cancel_tasks;
         cancel_tasks.reserve(id_tasks.size());
         for (const auto & id_task : id_tasks) {
             SRV_WRN("cancel task, id_task = %d\n", id_task);
-            engine_task task(ENGINE_TASK_TYPE_CANCEL);
+            server_task task(SERVER_TASK_TYPE_CANCEL);
             task.id_target = id_task;
             queue_results.remove_waiting_task_id(id_task);
             cancel_tasks.push_back(std::move(task));

tools/server/server-queue.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "engine-task.h"
+#include "server-task.h"
 
 #include <condition_variable>
 #include <deque>
@@ -11,8 +11,8 @@
 #include <unordered_set>
 
 // struct for managing server tasks
-// in most cases, use engine_response_reader to post new tasks and retrieve results
-struct engine_queue {
+// in most cases, use server_response_reader to post new tasks and retrieve results
+struct server_queue {
 private:
     int id = 0;
     bool running  = false;
@@ -21,11 +21,11 @@ private:
     int64_t time_last_task = 0;
 
     // queues
-    std::deque<engine_task> queue_tasks;
-    std::deque<engine_task> queue_tasks_deferred;
+    std::deque<server_task> queue_tasks;
+    std::deque<server_task> queue_tasks_deferred;
     // tasks declined while yielding, put back in queue_tasks once the yield is done
     // note: kept as a member so that cleanup_pending_task() can also reach them
-    std::deque<engine_task> queue_tasks_unhandled;
+    std::deque<server_task> queue_tasks_unhandled;
 
     std::mutex mutex_tasks;
     std::condition_variable condition_tasks;
@@ -42,21 +42,21 @@ private:
     worker_t worker;
 
     // callback functions
-    std::function<bool(engine_task &&, bool)> callback_new_task;
+    std::function<bool(server_task &&, bool)> callback_new_task;
     std::function<void(void)>                 callback_update_slots;
     std::vector<std::function<void(bool)>>    callback_sleeping_state;
 
 public:
-    ~engine_queue() { worker_stop(); }
+    ~server_queue() { worker_stop(); }
 
     // Add a new task to the end of the queue
-    int post(engine_task && task, bool front = false);
+    int post(server_task && task, bool front = false);
 
     // multi-task version of post()
-    int post(std::vector<engine_task> && tasks, bool front = false);
+    int post(std::vector<server_task> && tasks, bool front = false);
 
     // Add a new task, but defer until one slot is available
-    void defer(engine_task && task);
+    void defer(server_task && task);
 
     // Get the next id for creating a new task
     int get_new_id();
@@ -118,7 +118,7 @@ public:
     // only then may the callback return false to decline the task, and it must leave it
     // untouched, so that it can be put back in the queue later
     // note: while yielding, the callback runs on worker thread, not main thread
-    void on_new_task(std::function<bool(engine_task &&, bool)> callback) {
+    void on_new_task(std::function<bool(server_task &&, bool)> callback) {
         callback_new_task = std::move(callback);
     }
 
@@ -150,8 +150,8 @@ private:
 };
 
 // struct for managing server responses
-// in most cases, use engine_response_reader to retrieve results
-struct engine_response {
+// in most cases, use server_response_reader to retrieve results
+struct server_response {
 private:
     bool running = true;
 
@@ -159,7 +159,7 @@ private:
     std::unordered_set<int> waiting_task_ids;
 
     // the main result queue (using ptr for polymorphism)
-    std::vector<engine_task_result_ptr> queue_results;
+    std::vector<server_task_result_ptr> queue_results;
 
     std::mutex mutex_results;
     std::condition_variable condition_results;
@@ -177,33 +177,33 @@ public:
     void remove_waiting_task_ids(const std::unordered_set<int> & id_tasks);
 
     // This function blocks the thread until there is a response for one of the id_tasks
-    engine_task_result_ptr recv(const std::unordered_set<int> & id_tasks);
+    server_task_result_ptr recv(const std::unordered_set<int> & id_tasks);
 
     // same as recv(), but have timeout in seconds
     // if timeout is reached, nullptr is returned
-    engine_task_result_ptr recv_with_timeout(const std::unordered_set<int> & id_tasks, int timeout);
+    server_task_result_ptr recv_with_timeout(const std::unordered_set<int> & id_tasks, int timeout);
 
     // single-task version of recv()
-    engine_task_result_ptr recv(int id_task);
+    server_task_result_ptr recv(int id_task);
 
     // Send a new result to a waiting id_task
-    void send(engine_task_result_ptr && result);
+    void send(server_task_result_ptr && result);
 
     // broadcast a new result to all waiting tasks
     // (used by router mode)
-    void broadcast(engine_task_result_ptr && result);
+    void broadcast(server_task_result_ptr && result);
 
     // terminate the waiting loop
     void terminate();
 };
 
-// RAII wrapper to make working with engine_queue and engine_response easier
+// RAII wrapper to make working with server_queue and server_response easier
 // it provides a generator-like API for server responses
 // support pooling connection state and aggregating multiple results
-struct engine_response_reader {
+struct server_response_reader {
     std::unordered_set<int> id_tasks;
-    engine_queue & queue_tasks;
-    engine_response & queue_results;
+    server_queue & queue_tasks;
+    server_response & queue_results;
     size_t received_count = 0;
     bool cancelled = false;
     int polling_interval_seconds;
@@ -213,9 +213,9 @@ struct engine_response_reader {
     std::vector<task_result_state> states;
 
     // should_stop function will be called each polling_interval_seconds
-    engine_response_reader(engine_queue & queue_tasks, engine_response & queue_results, int polling_interval_seconds)
+    server_response_reader(server_queue & queue_tasks, server_response & queue_results, int polling_interval_seconds)
         : queue_tasks(queue_tasks), queue_results(queue_results), polling_interval_seconds(polling_interval_seconds) {}
-    ~engine_response_reader() {
+    ~server_response_reader() {
         stop();
     }
 
@@ -224,18 +224,18 @@ struct engine_response_reader {
     }
 
     // if front = true, the task will be posted to the front of the queue (high priority)
-    void post_task(engine_task && task, bool front = false);
-    void post_tasks(std::vector<engine_task> && tasks, const common_chat_session & session = {}, bool front = false);
+    void post_task(server_task && task, bool front = false);
+    void post_tasks(std::vector<server_task> && tasks, const common_chat_session & session = {}, bool front = false);
     bool has_next() const;
 
     // return nullptr if should_stop() is true before receiving a result
     // note: if one error is received, it will stop further processing and return error result
-    engine_task_result_ptr next(const std::function<bool()> & should_stop);
+    server_task_result_ptr next(const std::function<bool()> & should_stop);
 
     struct batch_response {
         bool is_terminated = false; // if true, indicates that processing was stopped before all results were received
-        std::vector<engine_task_result_ptr> results;
-        engine_task_result_ptr error; // nullptr if no error
+        std::vector<server_task_result_ptr> results;
+        server_task_result_ptr error; // nullptr if no error
     };
     // aggregate multiple results
     batch_response wait_for_all(const std::function<bool()> & should_stop);

tools/server/server-routes.h

@@ -1,90 +0,0 @@
-#pragma once
-
-#include "engine-context.h"
-#include "server-http.h"
-
-#include "json.h"
-
-#include <memory>
-#include <mutex>
-
-// forward declarations
-struct server_res_generator;
-
-struct server_routes {
-    server_routes(const common_params & params, engine_context & ctx_server);
-
-    void init_routes();
-
-    // note: this is not thread-safe and can only when ctx_http.is_ready is false
-    void update_meta(const engine_context & ctx_server) {
-        this->meta = std::make_unique<engine_context_meta>(ctx_server.get_meta());
-    }
-
-    // handlers using lambda function, so that they can capture `this` without `std::bind`
-    // they won't be called until ctx_http.is_ready is set to true
-    server_http_context::handler_t get_health;
-    server_http_context::handler_t get_metrics;
-    server_http_context::handler_t get_slots;
-    server_http_context::handler_t post_slots;
-    server_http_context::handler_t get_props;
-    server_http_context::handler_t post_props;
-    server_http_context::handler_t post_infill;
-    server_http_context::handler_t post_completions;
-    server_http_context::handler_t post_completions_oai;
-    server_http_context::handler_t post_chat_completions;
-    server_http_context::handler_t post_chat_completions_tok;
-    server_http_context::handler_t post_control;
-    server_http_context::handler_t post_responses_oai;
-    server_http_context::handler_t post_responses_tok_oai;
-    server_http_context::handler_t post_transcriptions_oai;
-    server_http_context::handler_t post_anthropic_messages;
-    server_http_context::handler_t post_anthropic_count_tokens;
-    server_http_context::handler_t post_apply_template;
-    server_http_context::handler_t get_models;
-    server_http_context::handler_t post_tokenize;
-    server_http_context::handler_t post_detokenize;
-    server_http_context::handler_t post_embeddings;
-    server_http_context::handler_t post_embeddings_oai;
-    server_http_context::handler_t post_rerank;
-    server_http_context::handler_t post_systemone;
-    server_http_context::handler_t get_lora_adapters;
-    server_http_context::handler_t post_lora_adapters;
-
-    // to be used in router mode
-    json get_model_info() const;
-
-private:
-    std::unique_ptr<server_res_generator> handle_completions_impl(
-            const server_http_req & req,
-            engine_task_type type,
-            const json & data,
-            const std::vector<raw_buffer> & files,
-            task_response_type res_type,
-            const common_chat_session & chat_session = {});
-    std::unique_ptr<server_res_generator> handle_slots_save(const server_http_req & req, int id_slot);
-    std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
-    std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
-    std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
-    std::unique_ptr<server_res_generator> handle_count_tokens(const server_http_req & req, task_response_type res_type);
-
-    // using unique_ptr to allow late initialization of const
-    std::unique_ptr<const engine_context_meta> meta;
-
-    const common_params & params;
-    engine_context & ctx_server;
-
-    engine_queue & queue_tasks;
-    engine_response & queue_results;
-    std::unique_ptr<server_res_generator> create_response(bool bypass_sleep = false);
-
-    // cached responses, to be used during sleep
-    std::mutex     mutex_cache;
-    json           cached_models  = nullptr;
-    json           cached_props   = nullptr;
-    engine_metrics cached_metrics;
-    // set when a scrape during sleep already reported the throughput buckets
-    bool           should_reset_buckets = false;
-    // call right before sleep to update the cached responses
-    void update_cached_responses(bool is_sleeping);
-};

tools/server/server-schema.cpp

@@ -1,8 +1,8 @@
-#include "engine-schema.h"
+#include "server-schema.h"
 
 #include "json-schema-to-grammar.h"
 
-namespace engine_schema {
+namespace server_schema {
 
 //
 // llama.cpp-specific completion schema
@@ -324,7 +324,7 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
         ->set_handler([&](field_eval_context & ctx, const json & data) {
             std::vector<common_grammar_trigger> triggers;
             for (const auto & t : data.at("grammar_triggers")) {
-                triggers.push_back(engine_grammar_trigger(t).value);
+                triggers.push_back(server_grammar_trigger(t).value);
             }
             common_sampling_add_grammar_triggers(ctx.params.sampling, ctx.vocab, std::move(triggers));
         }));
@@ -600,4 +600,4 @@ void field_nested::eval(field_eval_context & ctx, const json & data) {
     }
 }
 
-} // namespace engine_schema
+} // namespace server_schema

tools/server/server-schema.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "engine-common.h"
-#include "engine-task.h"
+#include "server-common.h"
+#include "server-task.h"
 
 #include "sampling.h"
 #include "speculative.h"
@@ -13,7 +13,7 @@
 #include <string>
 #include <vector>
 
-namespace engine_schema {
+namespace server_schema {
 
 struct field_eval_context {
     task_params & params;
@@ -101,4 +101,4 @@ task_params eval_llama_cmpl_schema(
                     const std::vector<llama_logit_bias> & logit_bias_eog,
                     const json & data);
 
-} // namespace engine_schema
+} // namespace server_schema

tools/server/server-stream.cpp

@@ -1,7 +1,7 @@
 #include "server-stream.h"
-#include "engine-common.h"
+#include "server-common.h"
 #include "server-http.h"
-#include "engine-queue.h"
+#include "server-queue.h"
 
 #include <chrono>
 #include <memory>

tools/server/server-subproc.cpp

@@ -1,196 +0,0 @@
-#include "server-subproc.h"
-
-#include "log.h"
-
-#include <algorithm>
-#include <chrono>
-#include <cstring>
-#include <limits>
-#include <thread>
-
-#ifdef _WIN32
-// windows.h defines min and max as macros, which breaks std::min and std::max
-#define WIN32_LEAN_AND_MEAN
-#ifndef NOMINMAX
-#   define NOMINMAX
-#endif
-#include <windows.h>
-#include <io.h>
-#else
-#include <errno.h>
-#include <fcntl.h>
-#include <poll.h>
-#include <unistd.h>
-#endif
-
-//
-// server_subproc
-//
-
-FILE * server_reserve_stdout() {
-    fflush(stdout);
-    // the reserved stream is not inherited, grandchildren get stdout and stderr of their own
-#ifdef _WIN32
-    int fd = _dup(_fileno(stdout));
-    GGML_ASSERT(fd >= 0);
-    SetHandleInformation((HANDLE) _get_osfhandle(fd), HANDLE_FLAG_INHERIT, 0);
-    _dup2(_fileno(stderr), _fileno(stdout));
-    SetStdHandle(STD_OUTPUT_HANDLE, GetStdHandle(STD_ERROR_HANDLE));
-    FILE * f = _fdopen(fd, "w");
-#else
-    int fd = fcntl(fileno(stdout), F_DUPFD_CLOEXEC, 0);
-    GGML_ASSERT(fd >= 0);
-    dup2(fileno(stderr), fileno(stdout));
-    FILE * f = fdopen(fd, "w");
-#endif
-    GGML_ASSERT(f);
-    return f;
-}
-
-bool server_subproc::has_output() {
-    FILE * files[SERVER_SUBPROC_STREAMS] = { sproc.stdout_file(), sproc.stderr_file() };
-    for (int i = 0; i < SERVER_SUBPROC_STREAMS; i++) {
-        if (out_handles[i] >= 0) {
-            continue;
-        }
-        if (!files[i]) {
-            return false;
-        }
-#ifdef _WIN32
-        HANDLE h = (HANDLE) _get_osfhandle(_fileno(files[i]));
-        if (h == INVALID_HANDLE_VALUE) {
-            return false;
-        }
-        out_handles[i] = (intptr_t) h;
-#else
-        int fd = fileno(files[i]);
-        if (fd < 0) {
-            return false;
-        }
-        fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
-        out_handles[i] = fd;
-#endif
-    }
-    return true;
-}
-
-bool server_subproc::output_closed() const {
-    return out_closed[SERVER_SUBPROC_STDOUT] && out_closed[SERVER_SUBPROC_STDERR];
-}
-
-int server_subproc::read_output(server_subproc_stream stream, char * buf, size_t len) {
-    if (out_closed[stream]) {
-        return -1;
-    }
-    if (!has_output()) {
-        out_closed[stream] = true;
-        return -1;
-    }
-#ifdef _WIN32
-    HANDLE h     = (HANDLE) out_handles[stream];
-    DWORD  avail = 0;
-    if (!PeekNamedPipe(h, NULL, 0, NULL, &avail, NULL)) {
-        out_closed[stream] = true; // pipe broken, child gone
-        return -1;
-    }
-    if (avail == 0) {
-        return 0;
-    }
-    DWORD to_read = avail < (DWORD) len ? avail : (DWORD) len;
-    DWORD got     = 0;
-    if (!ReadFile(h, buf, to_read, &got, NULL) || got == 0) {
-        out_closed[stream] = true;
-        return -1;
-    }
-    return (int) got;
-#else
-    while (true) {
-        ssize_t r = read((int) out_handles[stream], buf, len);
-        if (r > 0) {
-            return (int) r;
-        }
-        if (r < 0 && errno == EINTR) {
-            continue;
-        }
-        if (r < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
-            return 0;
-        }
-        out_closed[stream] = true; // EOF or error
-        return -1;
-    }
-#endif
-}
-
-server_subproc::waiter::waiter() {
-#ifndef _WIN32
-    int fds[2];
-    GGML_ASSERT(pipe(fds) == 0);
-    for (int fd : fds) {
-        fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
-    }
-    wake_fd[0] = fds[0];
-    wake_fd[1] = fds[1];
-#endif
-}
-
-server_subproc::waiter::~waiter() {
-#ifndef _WIN32
-    close((int) wake_fd[0]);
-    close((int) wake_fd[1]);
-#endif
-}
-
-void server_subproc::waiter::wake() {
-#ifndef _WIN32
-    char c = 1;
-    (void) !write((int) wake_fd[1], &c, 1);
-#endif
-}
-
-void server_subproc::waiter::wait(const std::vector<server_subproc *> & procs, std::vector<bool> & ready, int64_t timeout_ms) {
-    ready.assign(procs.size(), false);
-#ifdef _WIN32
-    // no waitable wait exists for anonymous pipes, so poll them in 50 ms steps
-    bool any = false;
-    for (size_t i = 0; i < procs.size(); i++) {
-        server_subproc * p = procs[i];
-        ready[i] = !p->has_output();
-        for (int s = 0; s < SERVER_SUBPROC_STREAMS && !ready[i]; s++) {
-            DWORD avail = 0;
-            if (!p->out_closed[s] && (!PeekNamedPipe((HANDLE) p->out_handles[s], NULL, 0, NULL, &avail, NULL) || avail > 0)) {
-                ready[i] = true; // data or broken pipe, read_output() tells which
-            }
-        }
-        any = any || ready[i];
-    }
-    if (!any) {
-        int64_t step = timeout_ms < 0 ? 50 : std::min<int64_t>(timeout_ms, 50);
-        std::this_thread::sleep_for(std::chrono::milliseconds(step));
-    }
-#else
-    std::vector<pollfd> pfds;
-    pfds.reserve(procs.size() * SERVER_SUBPROC_STREAMS + 1);
-    pfds.push_back({ (int) wake_fd[0], POLLIN, 0 });
-    for (auto * p : procs) {
-        const bool open = p->has_output();
-        for (int s = 0; s < SERVER_SUBPROC_STREAMS; s++) {
-            pfds.push_back({ open && !p->out_closed[s] ? (int) p->out_handles[s] : -1, POLLIN, 0 }); // poll() skips negative fds
-        }
-    }
-    int timeout = timeout_ms < 0 ? -1 : (int) std::min<int64_t>(timeout_ms, std::numeric_limits<int>::max());
-    int r = poll(pfds.data(), pfds.size(), timeout);
-    if (r < 0 && errno != EINTR) {
-        LOG_ERR("%s: poll() failed: %s\n", __func__, strerror(errno));
-    }
-    if (pfds[0].revents) {
-        char buf[64];
-        while (read((int) wake_fd[0], buf, sizeof(buf)) > 0) {}
-    }
-    for (size_t i = 0; i < procs.size(); i++) {
-        ready[i] = !procs[i]->has_output();
-        for (int s = 0; s < SERVER_SUBPROC_STREAMS; s++) {
-            ready[i] = ready[i] || pfds[1 + i * SERVER_SUBPROC_STREAMS + s].revents != 0;
-        }
-    }
-#endif
-}

tools/server/server-subproc.h

@@ -1,129 +0,0 @@
-#pragma once
-
-#include "subproc.h"
-
-#include <atomic>
-#include <chrono>
-#include <condition_variable>
-#include <cstdint>
-#include <cstdio>
-#include <functional>
-#include <mutex>
-#include <queue>
-#include <vector>
-
-// simple implementation of a pipe
-// used for streaming data between threads
-template<typename T>
-struct server_pipe {
-    std::mutex mutex;
-    std::condition_variable cv;
-    std::queue<T> queue;
-    std::atomic<bool> writer_closed{false};
-    std::atomic<bool> reader_closed{false};
-
-    // 0 = unbounded (default)
-    // > 0, write() drops the oldest item once the queue is full
-    size_t max_size = 0;
-
-    void close_write() {
-        writer_closed.store(true, std::memory_order_relaxed);
-        cv.notify_all();
-    }
-
-    void close_read() {
-        reader_closed.store(true, std::memory_order_relaxed);
-        cv.notify_all();
-    }
-
-    // close_on_stop = true: should_stop means the reader is gone for good, so the writer is told the pipe is broken.
-    // close_on_stop = false: should_stop is a per-read deadline and further reads still come, so the pipe stays usable.
-    bool read(T & output, const std::function<bool()> & should_stop, bool close_on_stop = true) {
-        std::unique_lock<std::mutex> lk(mutex);
-        constexpr auto poll_interval = std::chrono::milliseconds(500);
-        while (true) {
-            if (!queue.empty()) {
-                output = std::move(queue.front());
-                queue.pop();
-                return true;
-            }
-            if (writer_closed.load()) {
-                return false; // clean EOF
-            }
-            if (should_stop && should_stop()) { // a null should_stop means "never stop"
-                if (close_on_stop) {
-                    close_read(); // signal broken pipe to writer
-                }
-                return false; // cancelled / deadline reached
-            }
-            cv.wait_for(lk, poll_interval);
-        }
-    }
-
-    bool write(T && data) {
-        std::lock_guard<std::mutex> lk(mutex);
-        if (reader_closed.load()) {
-            return false; // broken pipe
-        }
-        if (max_size > 0) {
-            while (queue.size() >= max_size) {
-                queue.pop(); // drop oldest to stay bounded
-            }
-        }
-        queue.push(std::move(data));
-        cv.notify_one();
-        return true;
-    }
-};
-
-// a child server writes its state commands to stdout and its logs to stderr
-enum server_subproc_stream {
-    SERVER_SUBPROC_STDOUT,
-    SERVER_SUBPROC_STDERR,
-    SERVER_SUBPROC_STREAMS,
-};
-
-// gives the current stdout to the caller as a stream of its own, and sends everything else written to stdout to stderr
-FILE * server_reserve_stdout();
-
-// wrapper around common_subproc to manage a child server process
-// mainly used by router mode
-struct server_subproc {
-    common_subproc sproc;
-    std::atomic<bool> stopped{false}; // set by the monitor once the process exited and was reaped
-
-    bool is_alive() { return sproc.alive(); }
-    void terminate() { sproc.terminate(); }
-    int  join() { return sproc.join(); }
-
-    // true if both output pipes of the child are available (call after create())
-    bool has_output();
-
-    // true once both output pipes are closed
-    bool output_closed() const;
-
-    // non-blocking read from one output pipe
-    // returns the number of bytes read, 0 when nothing is available, -1 when the pipe is closed or broken
-    int read_output(server_subproc_stream stream, char * buf, size_t len);
-
-    // wait until one of a set of children has output, wake() is called, or a timeout passes
-    struct waiter {
-        waiter();
-        ~waiter();
-
-        // thread-safe; on Windows this is a no-op, wait() returns within 50 ms anyway
-        void wake();
-
-        // timeout_ms < 0 waits until data or wake(); ready[i] is set for each child with data on an open pipe (or a broken pipe)
-        void wait(const std::vector<server_subproc *> & procs, std::vector<bool> & ready, int64_t timeout_ms);
-
-    private:
-#ifndef _WIN32
-        intptr_t wake_fd[2] = { -1, -1 }; // POSIX self-pipe
-#endif
-    };
-
-private:
-    intptr_t out_handles[SERVER_SUBPROC_STREAMS] = { -1, -1 }; // fd on POSIX, HANDLE on Windows; taken lazily from sproc
-    bool     out_closed [SERVER_SUBPROC_STREAMS] = { false, false };
-};

tools/server/server-task.cpp

@@ -1,14 +1,14 @@
-#include "engine-task.h"
+#include "server-task.h"
 
 #include "build-info.h"
-#include "engine-chat.h"
+#include "server-chat.h"
 #include "chat.h"
 #include "common.h"
 #include "json-schema-to-grammar.h"
 #include "llama.h"
 #include "sampling.h"
 #include "speculative.h"
-#include "engine-common.h"
+#include "server-common.h"
 
 #include <sstream>
 
@@ -88,7 +88,7 @@ json task_params::to_json(bool only_metrics) const {
 
     auto grammar_triggers = json::array();
     for (const auto & trigger : sampling.grammar_triggers) {
-        engine_grammar_trigger ct(trigger);
+        server_grammar_trigger ct(trigger);
         grammar_triggers.push_back(ct.to_json());
     }
 
@@ -307,9 +307,9 @@ std::vector<unsigned char> completion_token_output::str_to_bytes(const std::stri
 }
 
 //
-// engine_task_result_cmpl_final
+// server_task_result_cmpl_final
 //
-json engine_task_result_cmpl_final::to_json() {
+json server_task_result_cmpl_final::to_json() {
     GGML_ASSERT(is_updated && "update() must be called before to_json()");
     switch (res_type) {
         case TASK_RESPONSE_TYPE_NONE:
@@ -329,7 +329,7 @@ json engine_task_result_cmpl_final::to_json() {
     }
 }
 
-json engine_task_result_cmpl_final::to_json_non_oaicompat() {
+json server_task_result_cmpl_final::to_json_non_oaicompat() {
     json res = json {
         {"index",               index},
         {"content",             content.text},
@@ -354,7 +354,7 @@ json engine_task_result_cmpl_final::to_json_non_oaicompat() {
     return response_fields.empty() ? res : json_get_nested_values(response_fields, res);
 }
 
-json engine_task_result_cmpl_final::usage_json_oaicompat() {
+json server_task_result_cmpl_final::usage_json_oaicompat() {
     return json {
         {"completion_tokens", n_decoded},
         {"prompt_tokens",     n_prompt_tokens},
@@ -363,7 +363,7 @@ json engine_task_result_cmpl_final::usage_json_oaicompat() {
     };
 }
 
-json engine_task_result_cmpl_final::to_json_oaicompat() {
+json server_task_result_cmpl_final::to_json_oaicompat() {
     std::time_t t = std::time(0);
     json logprobs = json(nullptr); // OAI default to null
     if (!stream && probs_output.size() > 0) {
@@ -403,7 +403,7 @@ json engine_task_result_cmpl_final::to_json_oaicompat() {
     return res;
 }
 
-json engine_task_result_cmpl_final::to_json_oaicompat_chat() {
+json server_task_result_cmpl_final::to_json_oaicompat_chat() {
     std::string finish_reason = "length";
     common_chat_msg msg;
     if (!oaicompat_msg.empty()) {
@@ -451,7 +451,7 @@ json engine_task_result_cmpl_final::to_json_oaicompat_chat() {
     return res;
 }
 
-json engine_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
+json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
     std::time_t t = std::time(0);
     std::string finish_reason = "length";
     if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {
@@ -465,7 +465,7 @@ json engine_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
                 json {
                     {"finish_reason", nullptr},
                     {"index", index},
-                    {"delta", engine_chat_msg_diff_to_json_oaicompat(diff)},
+                    {"delta", server_chat_msg_diff_to_json_oaicompat(diff)},
                 },
             })},
             {"created", t},
@@ -517,7 +517,7 @@ json engine_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
     return deltas;
 }
 
-json engine_task_result_cmpl_final::to_json_oaicompat_resp() {
+json server_task_result_cmpl_final::to_json_oaicompat_resp() {
     common_chat_msg msg;
     if (!oaicompat_msg.empty()) {
         msg = oaicompat_msg;
@@ -588,7 +588,7 @@ json engine_task_result_cmpl_final::to_json_oaicompat_resp() {
     return res;
 }
 
-json engine_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
+json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
     std::vector<json> server_sent_events;
     std::vector<json> output;
 
@@ -705,7 +705,7 @@ json engine_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
     return server_sent_events;
 }
 
-json engine_task_result_cmpl_final::to_json_oaicompat_asr() {
+json server_task_result_cmpl_final::to_json_oaicompat_asr() {
     json event = json {
         {"type",  "transcript.text.done"},
         {"text",  oaicompat_msg.content},
@@ -720,7 +720,7 @@ json engine_task_result_cmpl_final::to_json_oaicompat_asr() {
     return event;
 }
 
-json engine_task_result_cmpl_final::to_json_anthropic() {
+json server_task_result_cmpl_final::to_json_anthropic() {
     std::string stop_reason = "max_tokens";
     if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {
         stop_reason = oaicompat_msg.tool_calls.empty() ? "end_turn" : "tool_use";
@@ -786,7 +786,7 @@ json engine_task_result_cmpl_final::to_json_anthropic() {
     return res;
 }
 
-json engine_task_result_cmpl_final::to_json_anthropic_stream() {
+json server_task_result_cmpl_final::to_json_anthropic_stream() {
     json events = json::array();
 
     std::string stop_reason = "max_tokens";
@@ -975,9 +975,9 @@ json engine_task_result_cmpl_final::to_json_anthropic_stream() {
 }
 
 //
-// engine_task_result_cmpl_partial
+// server_task_result_cmpl_partial
 //
-void engine_task_result_cmpl_partial::update(task_result_state & state) {
+void server_task_result_cmpl_partial::update(task_result_state & state) {
     is_updated = true;
     if (is_begin) {
         return; // begin marker only flushes headers, skip parsing
@@ -1015,7 +1015,7 @@ void engine_task_result_cmpl_partial::update(task_result_state & state) {
     }
 }
 
-json engine_task_result_cmpl_partial::to_json() {
+json server_task_result_cmpl_partial::to_json() {
     GGML_ASSERT(is_updated && "update() must be called before to_json()");
     if (is_begin) {
         return nullptr; // simply signal to HTTP handler to send the headers and status code
@@ -1038,7 +1038,7 @@ json engine_task_result_cmpl_partial::to_json() {
     }
 }
 
-json engine_task_result_cmpl_partial::to_json_non_oaicompat() {
+json server_task_result_cmpl_partial::to_json_non_oaicompat() {
     // non-OAI-compat JSON
     json res = json {
         {"index",            index},
@@ -1062,7 +1062,7 @@ json engine_task_result_cmpl_partial::to_json_non_oaicompat() {
     return res;
 }
 
-json engine_task_result_cmpl_partial::to_json_oaicompat() {
+json server_task_result_cmpl_partial::to_json_oaicompat() {
     std::time_t t = std::time(0);
     json logprobs = json(nullptr); // OAI default to null
     if (prob_output.probs.size() > 0) {
@@ -1100,7 +1100,7 @@ json engine_task_result_cmpl_partial::to_json_oaicompat() {
     return res;
 }
 
-json engine_task_result_cmpl_partial::to_json_oaicompat_chat() {
+json server_task_result_cmpl_partial::to_json_oaicompat_chat() {
     bool first = n_decoded == 1;
     std::time_t t = std::time(0);
     json choices;
@@ -1131,7 +1131,7 @@ json engine_task_result_cmpl_partial::to_json_oaicompat_chat() {
     }
 
     for (const auto & diff : oaicompat_msg_diffs) {
-        add_delta(engine_chat_msg_diff_to_json_oaicompat(diff));
+        add_delta(server_chat_msg_diff_to_json_oaicompat(diff));
     }
 
     if (!deltas.empty()) {
@@ -1155,7 +1155,7 @@ json engine_task_result_cmpl_partial::to_json_oaicompat_chat() {
     return deltas;
 }
 
-json engine_task_result_cmpl_partial::to_json_oaicompat_resp() {
+json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
     std::vector<json> events;
 
     if (!oai_resp_created) {
@@ -1305,7 +1305,7 @@ json engine_task_result_cmpl_partial::to_json_oaicompat_resp() {
     return events;
 }
 
-json engine_task_result_cmpl_partial::to_json_oaicompat_asr() {
+json server_task_result_cmpl_partial::to_json_oaicompat_asr() {
     json event = json {
         {"type", "transcript.text.delta"},
         {"delta", content.text},
@@ -1313,7 +1313,7 @@ json engine_task_result_cmpl_partial::to_json_oaicompat_asr() {
     return event;
 }
 
-json engine_task_result_cmpl_partial::to_json_anthropic() {
+json server_task_result_cmpl_partial::to_json_anthropic() {
     json events = json::array();
     bool first = (n_decoded == 1);
     // use member variables to track block state across streaming calls
@@ -1453,22 +1453,22 @@ json engine_task_result_cmpl_partial::to_json_anthropic() {
 }
 
 //
-// engine_task_result_embd
+// server_task_result_embd
 //
-json engine_task_result_embd::to_json() {
+json server_task_result_embd::to_json() {
     return res_type == TASK_RESPONSE_TYPE_OAI_EMBD
         ? to_json_oaicompat()
         : to_json_non_oaicompat();
 }
 
-json engine_task_result_embd::to_json_non_oaicompat() {
+json server_task_result_embd::to_json_non_oaicompat() {
     return json {
         {"index",     index},
         {"embedding", embedding},
     };
 }
 
-json engine_task_result_embd::to_json_oaicompat() {
+json server_task_result_embd::to_json_oaicompat() {
     return json {
         {"index",            index},
         {"embedding",        embedding[0]},
@@ -1477,9 +1477,9 @@ json engine_task_result_embd::to_json_oaicompat() {
 }
 
 //
-// engine_task_result_rerank
+// server_task_result_rerank
 //
-json engine_task_result_rerank::to_json() {
+json server_task_result_rerank::to_json() {
     return json {
         {"index",            index},
         {"score",            score},
@@ -1488,9 +1488,9 @@ json engine_task_result_rerank::to_json() {
 }
 
 //
-// engine_task_result_decision
+// server_task_result_decision
 //
-json engine_task_result_decision::to_json() {
+json server_task_result_decision::to_json() {
     return json {
         {"index",            index},
         {"scores",           scores},
@@ -1499,9 +1499,9 @@ json engine_task_result_decision::to_json() {
 }
 
 //
-// engine_task_result_error
+// server_task_result_error
 //
-json engine_task_result_error::to_json() {
+json server_task_result_error::to_json() {
     json res = format_error_response(err_msg, err_type);
     if (err_type == ERROR_TYPE_EXCEED_CONTEXT_SIZE) {
         res["n_prompt_tokens"] = n_prompt_tokens;
@@ -1511,19 +1511,19 @@ json engine_task_result_error::to_json() {
 }
 
 //
-// engine_task_result_metrics
+// server_task_result_metrics
 //
-json engine_task_result_slots::to_json() {
+json server_task_result_slots::to_json() {
     return slots_data;
 }
 
-json engine_task_result_metrics::to_json() {
+json server_task_result_metrics::to_json() {
     // not used, /metrics renders prometheus text via to_metrics()
     return json{};
 }
 
 // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names
-std::string engine_task_result_metrics::to_metrics() {
+std::string server_task_result_metrics::to_metrics() {
     const std::vector<metric_item> counters = {
         {
             "prompt_tokens_total",
@@ -1620,9 +1620,9 @@ std::string engine_task_result_metrics::to_metrics() {
 }
 
 //
-// engine_task_result_slot_save_load
+// server_task_result_slot_save_load
 //
-json engine_task_result_slot_save_load::to_json() {
+json server_task_result_slot_save_load::to_json() {
     if (is_save) {
         return json {
             { "id_slot",   id_slot },
@@ -1647,9 +1647,9 @@ json engine_task_result_slot_save_load::to_json() {
 }
 
 //
-// engine_task_result_slot_erase
+// server_task_result_slot_erase
 //
-json engine_task_result_slot_erase::to_json() {
+json server_task_result_slot_erase::to_json() {
     return json {
         { "id_slot",  id_slot },
         { "n_erased", n_erased },
@@ -1657,10 +1657,10 @@ json engine_task_result_slot_erase::to_json() {
 }
 
 //
-// engine_task_result_get_lora
+// server_task_result_get_lora
 //
 
-json engine_task_result_get_lora::to_json() {
+json server_task_result_get_lora::to_json() {
     json result = json::array();
     for (size_t i = 0; i < loras.size(); ++i) {
         auto & lora = loras[i];
@@ -1681,17 +1681,17 @@ json engine_task_result_get_lora::to_json() {
 }
 
 //
-// engine_task_result_apply_lora
+// server_task_result_apply_lora
 //
 
-json engine_task_result_apply_lora::to_json() {
+json server_task_result_apply_lora::to_json() {
     return json {{ "success", true }};
 }
 
 //
-// engine_prompt_cache
+// server_prompt_cache
 //
-size_t engine_prompt_cache::size() const {
+size_t server_prompt_cache::size() const {
     size_t res = 0;
 
     for (const auto & state : states) {
@@ -1701,7 +1701,7 @@ size_t engine_prompt_cache::size() const {
     return res;
 }
 
-size_t engine_prompt_cache::n_tokens() const {
+size_t server_prompt_cache::n_tokens() const {
     size_t res = 0;
 
     for (const auto & state : states) {
@@ -1711,7 +1711,7 @@ size_t engine_prompt_cache::n_tokens() const {
     return res;
 }
 
-engine_prompt_cache_state * engine_prompt_cache::alloc(const engine_prompt & prompt, size_t state_size_tgt, size_t state_size_dft) {
+server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & prompt, size_t state_size_tgt, size_t state_size_dft) {
     // first check if the current state is contained fully in the cache
     for (auto it = states.begin(); it != states.end(); ++it) {
         const int cur_lcp_len = it->prompt.tokens.get_common_prefix(prompt.tokens);
@@ -1793,7 +1793,7 @@ engine_prompt_cache_state * engine_prompt_cache::alloc(const engine_prompt & pro
     return &states.back();
 }
 
-bool engine_prompt_cache::load(engine_prompt & prompt, const engine_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) {
+bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) {
     const int lcp_best = prompt.tokens.get_common_prefix(tokens_new);
 
     float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins
@@ -1870,7 +1870,7 @@ bool engine_prompt_cache::load(engine_prompt & prompt, const engine_tokens & tok
     return true;
 }
 
-void engine_prompt_cache::update() {
+void server_prompt_cache::update() {
     if (limit_size > 0) {
         while (!states.empty() && size() > limit_size) {
             SRV_WRN(" - cache size limit reached, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0));

tools/server/server-task.h

@@ -8,26 +8,26 @@
 #include <list>
 #include <map>
 
-// TODO: prevent including the whole engine-common.h as we only use engine_tokens
-#include "engine-common.h"
+// TODO: prevent including the whole server-common.h as we only use server_tokens
+#include "server-common.h"
 
 
-enum engine_task_type {
-    ENGINE_TASK_TYPE_COMPLETION,
-    ENGINE_TASK_TYPE_EMBEDDING,
-    ENGINE_TASK_TYPE_RERANK,
-    ENGINE_TASK_TYPE_DECISION,
-    ENGINE_TASK_TYPE_INFILL,
-    ENGINE_TASK_TYPE_CANCEL,
-    ENGINE_TASK_TYPE_CONTROL,
-    ENGINE_TASK_TYPE_NEXT_RESPONSE,
-    ENGINE_TASK_TYPE_METRICS,
-    ENGINE_TASK_TYPE_SLOT_GET,
-    ENGINE_TASK_TYPE_SLOT_SAVE,
-    ENGINE_TASK_TYPE_SLOT_RESTORE,
-    ENGINE_TASK_TYPE_SLOT_ERASE,
-    ENGINE_TASK_TYPE_GET_LORA,
-    ENGINE_TASK_TYPE_SET_LORA,
+enum server_task_type {
+    SERVER_TASK_TYPE_COMPLETION,
+    SERVER_TASK_TYPE_EMBEDDING,
+    SERVER_TASK_TYPE_RERANK,
+    SERVER_TASK_TYPE_DECISION,
+    SERVER_TASK_TYPE_INFILL,
+    SERVER_TASK_TYPE_CANCEL,
+    SERVER_TASK_TYPE_CONTROL,
+    SERVER_TASK_TYPE_NEXT_RESPONSE,
+    SERVER_TASK_TYPE_METRICS,
+    SERVER_TASK_TYPE_SLOT_GET,
+    SERVER_TASK_TYPE_SLOT_SAVE,
+    SERVER_TASK_TYPE_SLOT_RESTORE,
+    SERVER_TASK_TYPE_SLOT_ERASE,
+    SERVER_TASK_TYPE_GET_LORA,
+    SERVER_TASK_TYPE_SET_LORA,
 };
 
 // TODO: change this to more generic "response_format" to replace the "format_response_*" in server-common
@@ -85,7 +85,7 @@ struct task_params {
     std::string        oaicompat_model;
     std::string        oaicompat_cmpl_id;
 
-    // realtime control (ENGINE_TASK_TYPE_CONTROL)
+    // realtime control (SERVER_TASK_TYPE_CONTROL)
     std::string        control_action;
     std::string        control_cmpl_id;
 
@@ -134,37 +134,37 @@ struct task_result_state {
         bool filter_tool_calls = false);
 };
 
-struct engine_task {
-    int id = -1; // to be filled by engine_queue
+struct server_task {
+    int id = -1; // to be filled by server_queue
 
     // TODO @ngxson : remove this field and implement a mapping task_id -> idx in the response_reader
     size_t index = 0; // used when there are multiple prompts (batch request)
 
-    // used by ENGINE_TASK_TYPE_CANCEL
+    // used by SERVER_TASK_TYPE_CANCEL
     int id_target = -1;
     int id_slot   = -1;
 
     // used by parallel sampling (multiple completions from same prompt)
     int id_parent  = -1;
     // temporary store of child tasks for scheduling
-    // note: accessing to elements is invalid after the task is moved to engine_slot
-    std::vector<engine_task> child_tasks;
+    // note: accessing to elements is invalid after the task is moved to server_slot
+    std::vector<server_task> child_tasks;
     // if set on a parent, the children have their own prompt and only share its first n_tokens_shared tokens
     int32_t n_tokens_shared = 0;
 
-    // used by ENGINE_TASK_TYPE_INFERENCE
+    // used by SERVER_TASK_TYPE_INFERENCE
     task_params   params;
-    engine_tokens tokens;
+    server_tokens tokens;
 
     // only used by CLI, this allow tokenizing CLI inputs on server side
-    // we need this because mtmd_context and vocab are not accessible outside of engine_context
+    // we need this because mtmd_context and vocab are not accessible outside of server_context
     bool                    cli = false;
     std::string             cli_prompt;
     std::vector<raw_buffer> cli_files;
 
-    engine_task_type type;
+    server_task_type type;
 
-    // used by ENGINE_TASK_TYPE_SLOT_SAVE, ENGINE_TASK_TYPE_SLOT_RESTORE, ENGINE_TASK_TYPE_SLOT_ERASE
+    // used by SERVER_TASK_TYPE_SLOT_SAVE, SERVER_TASK_TYPE_SLOT_RESTORE, SERVER_TASK_TYPE_SLOT_ERASE
     struct slot_action {
         int id_slot;
         std::string filename;
@@ -172,10 +172,10 @@ struct engine_task {
     };
     slot_action slot_action;
 
-    // used by ENGINE_TASK_TYPE_METRICS
+    // used by SERVER_TASK_TYPE_METRICS
     bool metrics_reset_bucket = false;
 
-    // used by ENGINE_TASK_TYPE_DECISION
+    // used by SERVER_TASK_TYPE_DECISION
     // where to read the model output of each option, exactly one of the two lists is used
     struct decision {
         std::vector<llama_token> labels;       // logits of these tokens, at the last prompt token
@@ -206,12 +206,12 @@ struct engine_task {
     };
     decision decision;
 
-    // used by ENGINE_TASK_TYPE_SET_LORA
+    // used by SERVER_TASK_TYPE_SET_LORA
     std::map<int, float> set_lora; // mapping adapter ID -> scale
 
-    engine_task() = default;
+    server_task() = default;
 
-    engine_task(engine_task_type type) : type(type) {}
+    server_task(server_task_type type) : type(type) {}
 
     int32_t n_tokens() const {
         return tokens.size();
@@ -219,10 +219,10 @@ struct engine_task {
 
     bool need_embd() const {
         switch (type) {
-            case ENGINE_TASK_TYPE_EMBEDDING:
-            case ENGINE_TASK_TYPE_RERANK:
+            case SERVER_TASK_TYPE_EMBEDDING:
+            case SERVER_TASK_TYPE_RERANK:
                 return true;
-            case ENGINE_TASK_TYPE_DECISION:
+            case SERVER_TASK_TYPE_DECISION:
                 return !decision.markers.empty() || !decision.order.empty();
             default:
                 return false;
@@ -231,10 +231,10 @@ struct engine_task {
 
     bool need_logits() const {
         switch (type) {
-            case ENGINE_TASK_TYPE_COMPLETION:
-            case ENGINE_TASK_TYPE_INFILL:
+            case SERVER_TASK_TYPE_COMPLETION:
+            case SERVER_TASK_TYPE_INFILL:
                 return true;
-            case ENGINE_TASK_TYPE_DECISION:
+            case SERVER_TASK_TYPE_DECISION:
                 return !decision.labels.empty();
             default:
                 return false;
@@ -243,8 +243,8 @@ struct engine_task {
 
     bool need_sampling() const {
         switch (type) {
-            case ENGINE_TASK_TYPE_COMPLETION:
-            case ENGINE_TASK_TYPE_INFILL:
+            case SERVER_TASK_TYPE_COMPLETION:
+            case SERVER_TASK_TYPE_INFILL:
                 return true;
             default:
                 return false;
@@ -252,7 +252,7 @@ struct engine_task {
     }
 
     // utility function
-    static std::unordered_set<int> get_list_id(const std::vector<engine_task> & tasks) {
+    static std::unordered_set<int> get_list_id(const std::vector<server_task> & tasks) {
         std::unordered_set<int> ids(tasks.size());
         for (size_t i = 0; i < tasks.size(); i++) {
             ids.insert(tasks[i].id);
@@ -275,7 +275,7 @@ struct engine_task {
     }
 
     void add_child(int id_parent, int id_child) {
-        engine_task copy;
+        server_task copy;
 
         copy.id        = id_child;
         copy.id_parent = id_parent;
@@ -303,7 +303,7 @@ struct engine_task {
 
     // embedding and rerank tasks always process their whole prompt, their state is never reused
     bool is_stateless() const {
-        return type == ENGINE_TASK_TYPE_EMBEDDING || type == ENGINE_TASK_TYPE_RERANK;
+        return type == SERVER_TASK_TYPE_EMBEDDING || type == SERVER_TASK_TYPE_RERANK;
     }
 };
 
@@ -316,7 +316,7 @@ struct result_prompt_progress {
     json to_json() const;
 };
 
-struct engine_task_result {
+struct server_task_result {
     int id           = -1;
     int id_slot      = -1;
 
@@ -324,25 +324,25 @@ struct engine_task_result {
     size_t index = 0; // to be used for batched tasks
 
     virtual bool is_error() {
-        // only used by engine_task_result_error
+        // only used by server_task_result_error
         return false;
     }
     virtual bool is_stop() {
-        // only used by engine_task_result_cmpl_*
+        // only used by server_task_result_cmpl_*
         return true;
     }
     virtual void update(task_result_state &) {
-        // only used by engine_task_result_cmpl_*
+        // only used by server_task_result_cmpl_*
     }
     virtual json to_json() = 0;
-    virtual ~engine_task_result() = default;
-    virtual engine_task_result * clone() const {
+    virtual ~server_task_result() = default;
+    virtual server_task_result * clone() const {
         GGML_ABORT("not implemented for this task type");
     }
 };
 
-// using shared_ptr for polymorphism of engine_task_result
-using engine_task_result_ptr = std::unique_ptr<engine_task_result>;
+// using shared_ptr for polymorphism of server_task_result
+using server_task_result_ptr = std::unique_ptr<server_task_result>;
 
 struct completion_token_output {
     llama_token tok;
@@ -365,13 +365,13 @@ struct completion_token_output {
 
 };
 
-struct engine_task_result_cmpl_final : engine_task_result {
+struct server_task_result_cmpl_final : server_task_result {
     common_chat_input content;
     llama_tokens tokens;
 
     bool stream;
     bool include_usage;
-    engine_slot_stats stats;
+    server_slot_stats stats;
     std::string prompt;
 
     bool truncated;
@@ -440,7 +440,7 @@ struct engine_task_result_cmpl_final : engine_task_result {
     json to_json_anthropic_stream();
 };
 
-struct engine_task_result_cmpl_partial : engine_task_result {
+struct server_task_result_cmpl_partial : server_task_result {
     common_chat_input content;
     llama_tokens      tokens;
 
@@ -453,7 +453,7 @@ struct engine_task_result_cmpl_partial : engine_task_result {
     bool is_begin = false; // whether to send 200 status to HTTP client (begin of SSE stream)
                            // ref: https://github.com/ggml-org/llama.cpp/pull/23884
     completion_token_output prob_output;
-    engine_slot_stats stats;
+    server_slot_stats stats;
     result_prompt_progress progress;
 
     // response formatting
@@ -499,7 +499,7 @@ struct engine_task_result_cmpl_partial : engine_task_result {
     json to_json_anthropic();
 };
 
-struct engine_task_result_embd : engine_task_result {
+struct server_task_result_embd : server_task_result {
     std::vector<std::vector<float>> embedding;
 
     int32_t n_tokens;
@@ -514,7 +514,7 @@ struct engine_task_result_embd : engine_task_result {
     json to_json_oaicompat();
 };
 
-struct engine_task_result_rerank : engine_task_result {
+struct server_task_result_rerank : server_task_result {
     float score = -1e6;
 
     int32_t n_tokens;
@@ -522,7 +522,7 @@ struct engine_task_result_rerank : engine_task_result {
     virtual json to_json() override;
 };
 
-struct engine_task_result_decision : engine_task_result {
+struct server_task_result_decision : server_task_result {
     std::vector<float> scores; // one raw model output per option
 
     int32_t n_tokens;
@@ -530,7 +530,7 @@ struct engine_task_result_decision : engine_task_result {
     virtual json to_json() override;
 };
 
-struct engine_task_result_error : engine_task_result {
+struct server_task_result_error : server_task_result {
     error_type err_type = ERROR_TYPE_SERVER;
     std::string err_msg;
 
@@ -546,12 +546,12 @@ struct engine_task_result_error : engine_task_result {
 };
 
 // used by /metrics API
-struct engine_task_result_metrics : engine_task_result {
-    // these are immediate stats, not accumulated (engine_metrics is cumulative)
+struct server_task_result_metrics : server_task_result {
+    // these are immediate stats, not accumulated (server_metrics is cumulative)
     int n_processing_slots = 0;
     int n_tasks_deferred = 0;
 
-    engine_metrics metrics;
+    server_metrics metrics;
 
     virtual json to_json() override;
 
@@ -564,17 +564,17 @@ struct engine_task_result_metrics : engine_task_result {
 };
 
 // used by /slots API
-struct engine_task_result_slots : engine_task_result {
+struct server_task_result_slots : server_task_result {
     int n_idle_slots = 0;
 
-    // while we can also use std::vector<engine_slot> this requires copying the slot object which can be quite messy
+    // while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
     // therefore, we use json to temporarily store the slot.to_json() result
     json slots_data = json::array();
 
     virtual json to_json() override;
 };
 
-struct engine_task_result_slot_save_load : engine_task_result {
+struct server_task_result_slot_save_load : server_task_result {
     std::string filename;
     bool is_save; // true = save, false = load
 
@@ -585,13 +585,13 @@ struct engine_task_result_slot_save_load : engine_task_result {
     virtual json to_json() override;
 };
 
-struct engine_task_result_slot_erase : engine_task_result {
+struct server_task_result_slot_erase : server_task_result {
     size_t n_erased;
 
     virtual json to_json() override;
 };
 
-struct engine_task_result_control : engine_task_result {
+struct server_task_result_control : server_task_result {
     bool        success = false;
     std::string message; // optional detail when success is false
 
@@ -604,7 +604,7 @@ struct engine_task_result_control : engine_task_result {
     }
 };
 
-struct engine_task_result_get_lora : engine_task_result {
+struct server_task_result_get_lora : server_task_result {
     struct lora {
         common_adapter_lora_info info;
         std::string  alora_invocation_string;
@@ -615,12 +615,12 @@ struct engine_task_result_get_lora : engine_task_result {
     virtual json to_json() override;
 };
 
-struct engine_task_result_apply_lora : engine_task_result {
+struct server_task_result_apply_lora : server_task_result {
     virtual json to_json() override;
 };
 
-struct engine_prompt {
-    engine_tokens tokens;
+struct server_prompt {
+    server_tokens tokens;
 
     std::list<common_prompt_checkpoint> checkpoints;
 
@@ -633,15 +633,15 @@ struct engine_prompt {
         return tokens.size();
     }
 
-    engine_prompt clone() const {
-        return engine_prompt {
+    server_prompt clone() const {
+        return server_prompt {
             tokens.clone(),
             checkpoints,
         };
     }
 };
 
-struct engine_prompt_data {
+struct server_prompt_data {
     std::vector<uint8_t> main;
     std::vector<uint8_t> drft;
 
@@ -650,9 +650,9 @@ struct engine_prompt_data {
     }
 };
 
-struct engine_prompt_cache_state {
-    engine_prompt prompt;
-    engine_prompt_data data;
+struct server_prompt_cache_state {
+    server_prompt prompt;
+    server_prompt_data data;
 
     size_t size() const {
         size_t res = data.size();
@@ -665,13 +665,13 @@ struct engine_prompt_cache_state {
     }
 };
 
-struct engine_prompt_cache {
-    engine_prompt_cache(int32_t limit_size_mib, size_t limit_tokens) {
+struct server_prompt_cache {
+    server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens) {
         this->limit_size   = 1024ull*1024ull*(limit_size_mib < 0 ? 0 : limit_size_mib);
         this->limit_tokens = limit_tokens;
     }
 
-    std::list<engine_prompt_cache_state> states;
+    std::list<server_prompt_cache_state> states;
 
     // in bytes, 0 = no limit
     size_t limit_size = 0;
@@ -683,18 +683,18 @@ struct engine_prompt_cache {
 
     size_t n_tokens() const;
 
-    engine_prompt_cache_state * alloc(const engine_prompt & prompt, size_t state_size_main, size_t state_size_drft);
+    server_prompt_cache_state * alloc(const server_prompt & prompt, size_t state_size_main, size_t state_size_drft);
 
-    bool load(engine_prompt & prompt, const engine_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot);
+    bool load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot);
 
     void update();
 };
 
 // used exclusively by router mode
-struct engine_task_result_router : engine_task_result {
+struct server_task_result_router : server_task_result {
     json data;
     virtual json to_json() override { return data; }
-    virtual engine_task_result * clone() const override {
-        return new engine_task_result_router(*this);
+    virtual server_task_result * clone() const override {
+        return new server_task_result_router(*this);
     }
 };

tools/server/server-tools.cpp

@@ -1755,7 +1755,7 @@ struct server_tool_get_info : server_tool {
     }
 };
 
-struct server_tool_stream_result : engine_task_result {
+struct server_tool_stream_result : server_task_result {
     std::string chunk;
     bool done = false;
     std::string error_msg;
@@ -1783,7 +1783,7 @@ void server_tool::stream::push(const std::string & chunk) {
 
 struct server_tools_res : server_http_res {
     std::thread worker;
-    engine_response * qr = nullptr; // set only for streaming responses
+    server_response * qr = nullptr; // set only for streaming responses
     int id = -1;
 
     ~server_tools_res() override {

tools/server/server-tools.h

@@ -1,8 +1,8 @@
 #pragma once
 
-#include "engine-common.h"
+#include "server-common.h"
 #include "server-http.h"
-#include "engine-queue.h"
+#include "server-queue.h"
 #include "server-mcp.h"
 
 #include <atomic>
@@ -21,7 +21,7 @@ struct server_tool {
     virtual std::string type() const { return "server"; }
 
     struct stream {
-        engine_response & qr;
+        server_response & qr;
         int id;
         std::function<bool()> alive;
         void push(const std::string & chunk);
@@ -37,7 +37,7 @@ struct server_tools {
     std::vector<std::unique_ptr<server_tool>> tools;
 
     // for streaming
-    engine_response queue_res;
+    server_response queue_res;
     std::atomic<int> res_id{0};
 
     // set when --tools-runtime is configured; routes every tool call through an isolate

tools/server/server.cpp

@@ -1,5 +1,4 @@
-#include "engine-context.h"
-#include "server-routes.h"
+#include "server-context.h"
 #include "server-http.h"
 #include "server-models.h"
 #include "server-cors-proxy.h"
@@ -196,7 +195,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
     server_mcp mcp_mgr;
 
     // struct that contains llama context and inference
-    engine_context ctx_server;
+    server_context ctx_server;
 
     //
     // Router
@@ -481,8 +480,8 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
 
         // setup communication child --> router if necessary
         if (child.is_child()) {
-            ctx_server.set_state_callback([&](engine_state state, json payload) {
-                child.notify_to_router(engine_state_to_str(state), payload);
+            ctx_server.set_state_callback([&](server_state state, json payload) {
+                child.notify_to_router(server_state_to_str(state), payload);
             });
         }
 
@@ -541,7 +540,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
         std::thread monitor_thread;
         if (child.is_child()) {
             monitor_thread = child.setup(shutdown_handler);
-            child.notify_to_router(engine_state_to_str(ENGINE_STATE_READY), routes.get_model_info());
+            child.notify_to_router(server_state_to_str(SERVER_STATE_READY), routes.get_model_info());
         }
 
         // this call blocks the main thread until queue_tasks.terminate() is called

tools/server/tests/unit/test_slot_save.py

@@ -96,7 +96,7 @@ def test_slot_restore_legacy_token_list():
     assert res.status_code == 200
     assert res.body["n_saved"] == 84
 
-    # rewrite the token payload into a plain token list, as written by servers that predate the packed engine_tokens format
+    # rewrite the token payload into a plain token list, as written by servers that predate the packed server_tokens format
     path = os.path.join(server.slot_save_path, "slot_legacy.bin")
     with open(path, "rb") as f:
         data = bytearray(f.read())

ty.toml

@@ -8,6 +8,7 @@ deprecated = "warn"
 [src]
 exclude = [
     "./tools/mtmd/legacy-models/**",
+    "./skills/**",
 ]
 
 [[overrides]]