opencl: Add MxFP4 MoE bin kernel for Adreno X1 devices (#30691)

* opencl: add bin kernel `gemm_moe_mxfp4_q8_1_dp4a_ila_ldck` for X1

* opencl: fix whitespace

* opencl: undo unnecessary change

* opencl: enable dp4a clc for X1E

---------

Co-authored-by: Li He <lih@qti.qualcomm.com>

.github/labeler.yml

@@ -46,7 +46,6 @@ examples:
                 - examples/**
                 - tools/**
             - all-globs-to-all-files:
-                - '!tools/engine/**'
                 - '!tools/server/**'
                 - '!tools/mtmd/**'
                 - '!tools/ui/**'
@@ -77,7 +76,6 @@ server/ui:
 server:
     - changed-files:
         - any-glob-to-any-file:
-            - tools/engine/**
             - tools/server/**
 mtmd:
     - changed-files:

CODEOWNERS

@@ -101,7 +101,6 @@
 /tools/batched-bench/                   @ggerganov
 /tools/cli/                             @ngxson
 /tools/completion/                      @ggerganov
-/tools/engine/                          @ggml-org/llama-server
 /tools/mtmd/                            @ggml-org/llama-mtmd
 /tools/perplexity/                      @ggerganov
 /tools/parser/                          @pwilkin

common/chat.cpp

@@ -1104,6 +1104,13 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
         return common_chat_params_init_llm_jp_harmony;
     }
 
+    // Harmony XML tool calls, checked before the GPT-OSS JSON calls
+    if (src.find("<|channel|>") != std::string::npos && src.find("<|constrain|>xml") != std::string::npos &&
+        src.find("rfl:parameter") != std::string::npos) {
+        LOG_DBG("Using specialized template: Harmony XML\n");
+        return common_chat_params_init_harmony_xml;
+    }
+
     // GPT-OSS - has unique channel-based structure that needs dedicated handler
     if (src.find("<|channel|>") != std::string::npos) {
         LOG_DBG("Using specialized template: GPT-OSS\n");

common/jinja/value.cpp

@@ -1299,6 +1299,15 @@ const func_builtins & value_object_t::get_builtins() const {
             }
             return result;
         }},
+        {"list", [](const func_args & args) -> value {
+            args.ensure_vals<value_object>();
+            const auto & obj = args.get_pos(0)->as_ordered_object();
+            auto result = mk_val<value_array>();
+            for (const auto & pair : obj) {
+                result->push_back(pair.first);
+            }
+            return result;
+        }},
         {"values", [](const func_args & args) -> value {
             args.ensure_vals<value_object>();
             const auto & obj = args.get_pos(0)->as_ordered_object();

common/parsers/harmony-xml.cpp

@@ -0,0 +1,207 @@
+#include "parsers.h"
+
+// Harmony with XML tool calls: <|start|>assistant to=NAME<|channel|>analysis <|constrain|>xml<|message|><rfl:parameter name="KEY">\nVALUE\n</rfl:parameter>
+common_chat_params common_chat_params_init_harmony_xml(const common_chat_template &          tmpl,
+                                                       const autoparser::generation_params & inputs) {
+    common_chat_params data;
+
+    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);
+    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
+    data.message_delimiters = {
+        { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },
+        { COMMON_CHAT_ROLE_USER,      "<|start|>user"      },
+        { COMMON_CHAT_ROLE_SYSTEM,    "<|start|>developer" },
+        { COMMON_CHAT_ROLE_SYSTEM,    "<|start|>system"    },
+    };
+
+    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
+    data.supports_thinking = true;
+
+    data.thinking_start_tag = "<|channel|>analysis<|message|>";
+    data.thinking_end_tags  = {"<|end|>"};
+
+    // EOG text must reach the parser: an XML call has no closing tag of its own
+    static const std::vector<std::string> EOG = {
+        "<|call|>", "<|return|>", "<|endoftext|>", "<|im_end|>", "<|fim_pad|>", "<|repo_name|>", "<|file_sep|>",
+    };
+
+    data.preserved_tokens = {
+        "<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",
+    };
+    data.preserved_tokens.insert(data.preserved_tokens.end(), EOG.begin(), EOG.end());
+
+    if (inputs.has_continuation()) {
+        const auto & msg = inputs.continue_msg;
+
+        data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content;
+        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+            data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content();
+        }
+
+        data.prompt.append(data.generation_prompt);
+    }
+
+    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
+    auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
+    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
+    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
+
+    static const std::string PARAM_OPEN  = "<rfl:parameter name=\"";
+    static const std::string PARAM_CLOSE = "\n</rfl:parameter>";
+
+    data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+        auto start = p.rule("start", p.literal("<|start|>assistant"));
+
+        auto eog = p.choice();
+        for (const auto & tok : EOG) {
+            eog |= p.literal(tok);
+        }
+        auto with_eog = [&](std::vector<std::string> delims) {
+            delims.insert(delims.end(), EOG.begin(), EOG.end());
+            return delims;
+        };
+
+        auto analysis_open  = p.literal("<|channel|>analysis<|message|>");
+        auto analysis_body  = p.until_one_of(with_eog({ "<|end|>", "<|constrain|><|start|>assistant", "<|start|>assistant" }));
+        auto analysis_close = p.optional(p.literal("<|end|>") | p.literal("<|constrain|>") | eog);
+        if (extract_reasoning) {
+            p.rule("analysis", analysis_open + p.reasoning(analysis_body) + analysis_close);
+        } else {
+            p.rule("analysis", p.atomic(p.content(analysis_open)) + p.content(analysis_body + analysis_close));
+        }
+        auto analysis = p.ref("analysis");
+
+        auto final_msg = p.rule("final",
+            p.literal("<|channel|>final<|message|>") +
+            p.content(p.until_one_of(with_eog({ "<|end|>" }))) +
+            p.optional(p.literal("<|end|>") | eog));
+
+        if (has_response_format) {
+            auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1);
+            auto constraint     = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
+            auto response_format = p.rule("response-format",
+                p.literal("<|channel|>final") + constraint + p.literal("<|message|>") +
+                p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
+                p.optional(p.literal("<|end|>") | eog));
+
+            return start + p.zero_or_more(analysis + start) + response_format;
+        }
+
+        if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
+            // tolerate a missing closing tag
+            std::vector<std::string> value_ends = with_eog({ PARAM_OPEN, "<|end|>" });
+            std::vector<std::string> string_ends = { PARAM_CLOSE };
+            auto next_open = p.choice();
+            for (const auto & e : value_ends) {
+                string_ends.push_back("\n" + e);
+                next_open |= p.literal(e);
+            }
+            auto arg_close = p.tool_arg_close(p.gbnf(
+                p.literal(PARAM_CLOSE) | (p.literal("\n") + p.peek(p.optional(p.literal("\n")) + next_open)),
+                gbnf_format_literal("\n") + " " + gbnf_format_literal(PARAM_CLOSE.substr(1)) + "?"));
+            auto arg_string = p.rule("rfl-arg-string",
+                p.ac(p.tool_arg_string_value(p.until_one_of(string_ends)) + arg_close, PARAM_CLOSE));
+            auto arg_sep = p.optional(p.literal("\n"));
+
+            std::string call_close_gbnf = gbnf_format_literal("<|end|>");
+            for (const auto & tok : EOG) {
+                call_close_gbnf += " | " + gbnf_format_literal(tok);
+            }
+            auto call_close = p.gbnf(p.literal("<|end|>") | eog | p.peek(p.literal("<|start|>assistant")),
+                "(" + call_close_gbnf + ")?");
+
+            auto tool_choice = p.choice();
+            foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
+                const auto & function = tool.at("function");
+                std::string  name     = function.at("name");
+
+                std::vector<common_peg_parser> arg_rules;
+                foreach_parameter(function, [&](size_t param_index, const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
+                    auto rule_name = "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index);
+                    auto arg_open  = p.tool_arg_open(p.literal(PARAM_OPEN) + p.tool_arg_name(p.literal(param.name)) + p.literal("\">\n"));
+
+                    auto types     = param.schema->value_types();
+                    auto arg_value = p.eps();
+                    if (!types.has(common_chat_schema::TYPE_STRING) &&
+                        (types.has(common_chat_schema::TYPE_OBJECT) || types.has(common_chat_schema::TYPE_ARRAY))) {
+                        // the schema grammar fixes the key order, and the model writes keys in its own order
+                        arg_value = p.tool_arg_json_value(p.json()) + arg_close;
+                    } else if (!types.has(common_chat_schema::TYPE_STRING)) {
+                        arg_value = p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close;
+                    } else if (types.is_only(common_chat_schema::TYPE_STRING)) {
+                        arg_value = arg_string;
+                    } else {
+                        // grammar keeps only the string rule; the parser tries JSON first to type the value
+                        auto json_value = p.choice();
+                        if (types.has(common_chat_schema::TYPE_OBJECT)) {
+                            json_value |= p.json_object();
+                        }
+                        if (types.has(common_chat_schema::TYPE_ARRAY)) {
+                            json_value |= p.json_array();
+                        }
+                        if (types.has(common_chat_schema::TYPE_NUMBER) || types.has(common_chat_schema::TYPE_INTEGER)) {
+                            json_value |= p.json_number();
+                        }
+                        if (types.has(common_chat_schema::TYPE_BOOLEAN)) {
+                            json_value |= p.json_bool();
+                        }
+                        if (types.has(common_chat_schema::TYPE_NULL)) {
+                            json_value |= p.json_null();
+                        }
+                        arg_value = p.gbnf(p.atomic(p.tool_arg_json_value(json_value) + arg_close) | arg_string, "rfl-arg-string");
+                    }
+
+                    arg_rules.push_back(p.rule(rule_name, p.tool_arg(arg_open + arg_value)));
+                });
+
+                if (arg_rules.empty()) {
+                    auto any_name = p.chars("[^\"\\n<]", 1, -1);
+                    arg_rules.push_back(p.rule("tool-" + std::to_string(tool_index) + "-arg-any",
+                        p.tool_arg(p.tool_arg_open(p.literal(PARAM_OPEN) + p.tool_arg_name(any_name) + p.literal("\">\n")) + arg_string)));
+                }
+
+                auto args = p.zero_or_more(p.choice(arg_rules) + arg_sep);
+
+                auto call = p.tool(
+                    p.tool_open(p.literal(" to=") + p.tool_name(p.literal(name)) +
+                                p.literal("<|channel|>analysis <|constrain|>xml<|message|>")) +
+                    p.tool_args(args) +
+                    p.tool_close(call_close));
+
+                tool_choice |= p.rule("tool-" + std::to_string(tool_index), call);
+            });
+
+            auto tool_calls = inputs.parallel_tool_calls
+                ? tool_choice + p.zero_or_more(start + tool_choice)
+                : tool_choice;
+            auto tool_call = p.trigger_rule("tool-call", tool_calls);
+
+            if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
+                return start + p.zero_or_more(analysis + start) + tool_call;
+            }
+
+            return start + p.zero_or_more(analysis + start) + (tool_call | final_msg | p.content(p.rest()));
+        }
+
+        auto unsolicited = p.rule("unsolicited", p.atomic(p.literal(" to=") + p.rest()));
+
+        // output without a channel header is returned as content instead of failing the request
+        return start + p.zero_or_more(analysis + start) + (final_msg | unsolicited | p.content(p.rest()));
+    });
+
+    if (include_grammar) {
+        data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
+        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
+            data.parser.build_grammar(builder, data.grammar_lazy);
+        });
+
+        if (data.grammar_lazy) {
+            data.grammar_triggers = {
+                { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" },
+                { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" },
+            };
+        }
+    }
+
+    return data;
+}

common/parsers/parsers.h

@@ -59,6 +59,8 @@ common_chat_params common_chat_params_init_gigachat_v3(const common_chat_templat
 
 common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
 
+common_chat_params common_chat_params_init_harmony_xml(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
+
 common_chat_params common_chat_params_init_k2_horizon(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
 
 common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);

common/parsers/sources.cmake

@@ -9,6 +9,7 @@ set(LLAMA_CHAT_PARSERS_SOURCES
     ${CMAKE_CURRENT_LIST_DIR}/gemma4.cpp
     ${CMAKE_CURRENT_LIST_DIR}/gigachat-v3.cpp
     ${CMAKE_CURRENT_LIST_DIR}/gpt-oss.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/harmony-xml.cpp
     ${CMAKE_CURRENT_LIST_DIR}/k2-horizon.cpp
     ${CMAKE_CURRENT_LIST_DIR}/kimi-k2.cpp
     ${CMAKE_CURRENT_LIST_DIR}/kimi-k3.cpp

conversion/__init__.py

@@ -24,6 +24,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "ArcticForCausalLM": "arctic",
     "AudioFlamingo3ForConditionalGeneration": "qwen",
     "BaiChuanForCausalLM": "baichuan",
+    "BeamForCausalLM": "beam",
     "BaichuanForCausalLM": "baichuan",
     "BailingMoeForCausalLM": "bailingmoe",
     "BailingMoeV2ForCausalLM": "bailingmoe",

conversion/beam.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+from typing import Callable, Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+    from torch import Tensor
+
+from .base import ModelBase, TextModel, gguf, logger
+
+
+@ModelBase.register("BeamForCausalLM")
+class BeamModel(TextModel):
+    model_arch = gguf.MODEL_ARCH.BEAM
+
+    # reference NoPE query-scale constants; not in config.json
+    NOPE_SCALE_POWER  = 0.1
+    NOPE_SCALE_LENGTH = 2048
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        hp = self.hparams
+        rope = hp.get("rope_parameters") or {}
+        if rope.get("full_attention") is not None:
+            raise ValueError("Beam: expected NoPE full-attention layers (rope_parameters.full_attention = null)")
+        if "sliding_attention" in rope:
+            swa_rope = rope["sliding_attention"] or {}
+        else:
+            # flat rope_theta / rope_scaling, already normalized by TextModel
+            swa_rope = {k: v for k, v in self.rope_parameters.items() if k not in ("sliding_attention", "full_attention", "type")}
+            swa_rope.update(self.rope_parameters.get("sliding_attention") or {})
+        if swa_rope.get("rope_type", "default") != "default":
+            raise ValueError(f"Beam: unsupported sliding-attention rope type {swa_rope.get('rope_type')!r}")
+        # full layers are NoPE, so mirror the sliding rope params for the base class
+        swa_rope = {k: v for k, v in swa_rope.items() if not (k == "rope_type" and v == "default")}
+        self.rope_parameters = {"sliding_attention": dict(swa_rope), "full_attention": dict(swa_rope)}
+
+        layer_types = hp["layer_types"]
+        mlp_types = hp["mlp_layer_types"]
+        n_layer = hp["num_hidden_layers"]
+        if len(layer_types) != n_layer or len(mlp_types) != n_layer:
+            raise ValueError("Beam: layer_types / mlp_layer_types must have num_hidden_layers entries")
+        if any(t not in ("sliding_attention", "full_attention") for t in layer_types):
+            raise ValueError(f"Beam: unknown layer type in {layer_types}")
+        if "sliding_attention" not in layer_types:
+            raise ValueError("Beam: expected at least one sliding_attention layer")
+        n_dense = 0
+        while n_dense < n_layer and mlp_types[n_dense] == "dense":
+            n_dense += 1
+        if any(t != "sparse" for t in mlp_types[n_dense:]):
+            raise ValueError("Beam: llama.cpp supports dense MLP layers only as a leading block")
+        self._n_dense_lead = n_dense
+        if hp.get("attention_bias", False):
+            raise ValueError("Beam: attention_bias=True is not supported")
+        self._experts: dict[tuple[int, str], dict[int, Tensor]] = {}
+
+    def set_vocab(self):
+        self._set_vocab_gpt2()
+        # otherwise EOT is guessed from a hash-ordered scan
+        from transformers import AutoTokenizer
+        tok = AutoTokenizer.from_pretrained(self.dir_model)
+        ret_id = tok.convert_tokens_to_ids("<|return|>")  # ty: ignore[unresolved-attribute]
+        if isinstance(ret_id, int) and ret_id >= 0:
+            self.gguf_writer.add_eot_token_id(ret_id)
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        hp = self.hparams
+
+        head_dim = hp["head_dim"]
+        self.gguf_writer.add_rope_dimension_count(head_dim)
+
+        self.gguf_writer.add_sliding_window(hp["sliding_window"])
+        self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hp["layer_types"]])
+
+        self.gguf_writer.add_attn_temperature_scale(self.NOPE_SCALE_POWER)
+        self.gguf_writer.add_attn_temperature_length(self.NOPE_SCALE_LENGTH)
+
+        self.gguf_writer.add_leading_dense_block_count(self._n_dense_lead)
+        self.gguf_writer.add_expert_feed_forward_length(hp["moe_intermediate_size"])
+        self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
+        self.gguf_writer.add_expert_weights_norm(bool(hp.get("norm_topk_prob", True)))
+        # equivalent to the reference's post-aggregation scaling
+        self.gguf_writer.add_expert_weights_scale(float(hp["routed_scaling_factor"]))
+
+        if (softcap := hp.get("final_logit_softcapping")) is not None:
+            self.gguf_writer.add_final_logit_softcapping(float(softcap))
+
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        name, gen = item
+        if name.endswith(".self_attn.gate_bias"):
+            name = name.replace(".self_attn.gate_bias", ".self_attn.gate_proj.bias")
+        return super().filter_tensors((name, gen))
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        hp = self.hparams
+        if name.endswith(".self_attn.gate_proj.bias"):
+            assert tuple(data_torch.shape) == (hp["num_attention_heads"],), f"{name}: one gate bias per head expected, got {tuple(data_torch.shape)}"
+        elif name.endswith(".self_attn.gate_proj.weight"):
+            assert tuple(data_torch.shape) == (hp["num_attention_heads"] * hp["head_dim"], hp["hidden_size"]), f"{name}: {tuple(data_torch.shape)}"
+        elif name.endswith(".mlp.gate.e_score_correction.bias"):
+            assert tuple(data_torch.shape) == (hp["num_experts"],), f"{name}: {tuple(data_torch.shape)}"
+        elif name.endswith((".self_attn.q_norm.weight", ".self_attn.k_norm.weight")):
+            assert tuple(data_torch.shape) == (hp["head_dim"],), f"{name}: per-head q/k norm expected, got {tuple(data_torch.shape)}"
+        elif bid is not None and bid < self._n_dense_lead and ".mlp.experts." in name:
+            raise ValueError(f"Beam: expert tensor {name} in a dense layer")
+        if ".mlp.experts." in name:
+            assert bid is not None
+            n_experts = self.hparams["num_experts"]
+            parts = name.split(".")
+            xid = int(parts[parts.index("experts") + 1])
+            proj = parts[parts.index("experts") + 2]
+            if proj not in ("gate_proj", "up_proj", "down_proj"):
+                raise ValueError(f"Beam: unexpected expert tensor {name}")
+            bucket = self._experts.setdefault((bid, proj), {})
+            if xid in bucket:
+                raise ValueError(f"Beam: duplicate expert tensor {name}")
+            bucket[xid] = data_torch
+            if len(bucket) < n_experts:
+                return
+            del self._experts[(bid, proj)]
+            data = torch.stack([bucket[e] for e in range(n_experts)], dim=0)
+            yield from super().modify_tensors(data, f"model.layers.{bid}.mlp.experts.{proj}.weight", bid)
+            return
+
+        yield from super().modify_tensors(data_torch, name, bid)
+
+    def prepare_tensors(self):
+        super().prepare_tensors()
+        if self._experts:
+            missing = sorted(f"layer {b} {p} ({len(v)} experts)" for (b, p), v in self._experts.items())
+            raise ValueError(f"Beam: unprocessed experts: {missing}")
+        logger.info("Beam: all routed experts stacked")

conversion/lfm2.py

@@ -188,7 +188,7 @@ def _load_d1_omni_hparams(dir_model: Path) -> dict[str, Any]:
 class D1OmniModel(LFM2Model):
     model_arch = gguf.MODEL_ARCH.LFM2
 
-    # the server cuts the text to these lengths, see engine-decision.cpp
+    # the server cuts the text to these lengths, see server-decision.cpp
     _MAX_LENGTH = 16384
     _IMAGE_TEXT_LENGTH = 896
     _AUDIO_TEXT_LENGTH = 15360

docs/development/HOWTO-add-model.md

@@ -147,11 +147,11 @@ The conversion is the same as above, but a new model needs its own `DecisionType
 
 > [!IMPORTANT]
 >
-> Most of the logic is handled in `tools/engine/engine-decision.cpp`, to avoid too many changes to `libllama`.
+> Most of the logic is handled in `tools/server/server-decision.cpp`, to avoid too many changes to `libllama`.
 
 Note:
 - If a new public API is needed in `libllama`, add it to `llama-ext.h`.
-- Metadata with a single use case must be hard-coded in `engine-decision.cpp` instead of being saved to the GGUF. This avoids bloating the conversion code.
+- Metadata with a single use case must be hard-coded in `server-decision.cpp` instead of being saved to the GGUF. This avoids bloating the conversion code.
 - Most importantly, keep your change as small and as self-contained as possible. Reuse the existing infrastructure whenever you can.
 
 For more information, see [PR #29818](https://github.com/ggml-org/llama.cpp/pull/29818).

ggml/src/ggml-opencl/CMakeLists.txt

@@ -132,6 +132,7 @@ set(GGML_OPENCL_KERNELS
     gemm_moe_q6_k_q8_1_dp4a
     gemm_moe_q8_1_dp4a
     moe_reorder_quant_a_q8_1
+    transpose_activation_q8
     gemm_noshuffle_q4_k_q8_1_dp4a
     gemm_noshuffle_q5_k_q8_1_dp4a
     gemm_noshuffle_q6_k_q8_1_dp4a

ggml/src/ggml-opencl/ggml-opencl.cpp

@@ -720,6 +720,10 @@ struct ggml_backend_opencl_context {
     ggml_cl_buffer prealloc_moe_qa;   // int8 quants  [tok_slots * ne00]
     ggml_cl_buffer prealloc_moe_da;   // per-block d  [tok_slots * ne00/32] (half)
     ggml_cl_buffer prealloc_moe_sa;   // per-block s  [tok_slots * ne00/32] (half)
+    // per-tile transposed qa/da and on-chip scratch for the ldck dp4a MoE GEMM
+    ggml_cl_buffer prealloc_moe_qa_trans;    // int8 quants  [tok_slots * ne00]
+    ggml_cl_buffer prealloc_moe_da_trans;    // per-block d  [tok_slots * ne00/32] (float)
+    ggml_cl_buffer prealloc_moe_qda_on_chip;
     // scratch copy of the router weights to avoid dst aliasing
     ggml_cl_buffer prealloc_moe_combine_w;
     ggml_cl_buffer prealloc_splitk_partial;  // [ksplit * M] partials for split-K GEMV
@@ -1016,6 +1020,7 @@ struct ggml_backend_opencl_context {
     cl_kernel kernel_gemv_moe_q4_k_f32_ns_wimg = nullptr;  // weight-as-texture MoE decode GEMV (opt-in)
     cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a = nullptr;    // dp4a (int8) prefill GEMM variant
     cl_kernel kernel_moe_reorder_quant_a_q8_1;   // fused reorder + q8_1 quant for the dp4a GEMM
+    cl_kernel kernel_transpose_activation_q8;    // per-tile transpose of q8_1 activations for the ldck dp4a GEMM
     cl_kernel kernel_gemm_moe_q8_1_dp4a_q80 = nullptr;   // generic dp4a MoE GEMM (MOE_QT=80), opt-in
     cl_kernel kernel_moe_expand_scale_q8_0 = nullptr;    // q8_0 per-block d -> uniform scale[16]
     cl_kernel kernel_gemm_moe_q8_1_dp4a_q50 = nullptr;   // generic dp4a MoE GEMM (MOE_QT=50, q5_0), opt-in
@@ -1031,6 +1036,7 @@ struct ggml_backend_opencl_context {
     cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a = nullptr;   // dp4a (int8) mxfp4 MoE prefill GEMM
     cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a = nullptr;    // dp4a (int8) q4_0 MoE prefill GEMM
     cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr;   // binary dp4a (int8) mxfp4 MoE prefill GEMM
+    cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = nullptr;   // binary dp4a (int8) mxfp4 MoE prefill GEMM using ldck
     cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr;    // binary dp4a (int8) q4_0 MoE prefill GEMM
     cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a_bin = nullptr;   // binary dp4a (int8) q4_k MoE prefill GEMM
     cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a_bin = nullptr;    // binary dp4a (int8) q6_k MoE prefill GEMM
@@ -4903,6 +4909,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
         }
     }
 
+    // gemm_moe_mxfp4_q8_1_dp4a_ldck_bin (dp4a prefill GEMM)
+    if (backend_ctx->has_integer_dot) {
+        size_t bin_size = 0;
+        backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = nullptr;
+
+        if (use_adreno_bin_kernels(backend_ctx)) {
+            const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_q8_1_dp4a_ila_ldck", &bin_size);
+            if (kernel_bin && bin_size > 0) {
+                cl_program prog =
+                    build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size);
+
+                CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a_ila_ldck", &err), err));
+                CL_CHECK(clReleaseProgram(prog));
+                GGML_LOG_CONT(".");
+            }
+        }
+    }
+
     // gemm_moe_q4_0_q8_1_dp4a (dp4a prefill GEMM)
     if (backend_ctx->has_integer_dot) {
 #ifdef GGML_OPENCL_EMBED_KERNELS
@@ -4984,6 +5008,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
         GGML_LOG_CONT(".");
     }
 
+    // transpose_activation_q8 (per-tile transpose of q8_1 activations for the ldck dp4a GEMM)
+    {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+        const std::string kernel_src {
+            #include "transpose_activation_q8.cl.h"
+        };
+#else
+        const std::string kernel_src = read_file("transpose_activation_q8.cl");
+#endif
+        cl_program prog =
+            build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts);
+
+        CL_CHECK((backend_ctx->kernel_transpose_activation_q8 = clCreateKernel(prog, "kernel_transpose_activation_q8", &err), err));
+        CL_CHECK(clReleaseProgram(prog));
+        GGML_LOG_CONT(".");
+    }
+
     // gemv_moe_q5_k_f32_ns
     {
 #ifdef GGML_OPENCL_EMBED_KERNELS
@@ -28258,11 +28299,19 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
                     static const char * mxfp4_moe_dp4a_env = getenv("GGML_OPENCL_MXFP4_MOE_DP4A");
                     bool use_moe_dp4a = mxfp4_moe_dp4a_env
                         ? (atoi(mxfp4_moe_dp4a_env) != 0)
-                        : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+                        : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E);
                     // dot prod has to be available
                     use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;
                     // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel
-                    if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr) {
+                    // ldck dp4a bin kernel (X1) has the same priority as normal bin kernel (X2/A8x), they are not expected to coexist.
+                    const bool use_moe_dp4a_ldck = backend_ctx->has_integer_dot &&
+                        (!mxfp4_moe_dp4a_env || atoi(mxfp4_moe_dp4a_env) != 0) &&
+                        backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr &&
+                        backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr &&
+                        backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin != nullptr;
+                    if (use_moe_dp4a_ldck) {
+                        use_moe_dp4a = true;
+                    } else if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr) {
                         use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr;
                     }
 
@@ -28351,6 +28400,12 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
                         backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));
                         backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
                         backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+                        if (use_moe_dp4a_ldck) {
+                            backend_ctx->prealloc_moe_qa_trans.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));
+                            backend_ctx->prealloc_moe_da_trans.allocate(backend_ctx->context, n_blocks * sizeof(cl_float));
+                            // matches max_constant_size of src1_a_cm in the ldck kernel
+                            backend_ctx->prealloc_moe_qda_on_chip.allocate(backend_ctx->context, 2304);
+                        }
 
                         // fused reorder + q8_1 quant straight from the original
                         // activations (no intermediate f32 reorder buffer). mxfp4 has no
@@ -28375,13 +28430,33 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
                         cl_kernel dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a;
                         if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin) {
                             dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin;
+                        } else if (use_moe_dp4a_ldck) {
+                            dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin;
+
+                            // per-tile transpose of qa/da for the ldck kernel
+                            cl_kernel tk = backend_ctx->kernel_transpose_activation_q8;
+                            CL_CHECK(clSetKernelArg(tk, 0, sizeof(cl_mem),  &backend_ctx->prealloc_moe_qa.buffer));
+                            CL_CHECK(clSetKernelArg(tk, 1, sizeof(cl_mem),  &backend_ctx->prealloc_moe_qa_trans.buffer));
+                            CL_CHECK(clSetKernelArg(tk, 2, sizeof(cl_mem),  &backend_ctx->prealloc_moe_da.buffer));
+                            CL_CHECK(clSetKernelArg(tk, 3, sizeof(cl_mem),  &backend_ctx->prealloc_moe_da_trans.buffer));
+                            CL_CHECK(clSetKernelArg(tk, 4, sizeof(cl_mem),  &(backend_ctx->prealloc_total_tiles.buffer)));
+                            CL_CHECK(clSetKernelArg(tk, 5, sizeof(cl_uint), &ne00));
+                            size_t tk_local[3]  = { 32, 8, 1 };
+                            size_t tk_global[3] = { 32, (size_t)(ne00 / 4), (size_t)max_post_router_tile };
+                            backend_ctx->enqueue_ndrange_kernel(tk, 3, tk_global, tk_local, dst);
                         }
 
                         int aidx = 0;
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->q_img));
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->e));
-                        CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
-                        CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+                        if (use_moe_dp4a_ldck) {
+                            CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa_trans.buffer));
+                            CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da_trans.buffer));
+                            CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qda_on_chip.buffer));
+                        } else {
+                            CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+                            CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+                        }
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2));
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap));
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image));

ggml/src/ggml-opencl/kernels/transpose_activation_q8.cl

@@ -0,0 +1,35 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+
+#define tile_size 32
+
+kernel void kernel_transpose_activation_q8(
+        __global uint * src1_qa,
+        __global uint * src1_qa_trans,
+        __global half * src1_sa,
+        __global float * src1_sa_trans,
+        __global const int * total_tiles,
+        uint  K
+) {
+    uint row_in_tile = get_global_id(0);
+    uint blk_k = get_group_id(1);
+    uint col_in_tile = get_local_id(1);
+    uint tile_n = get_global_id(2);
+
+    if (blk_k >= (K / tile_size) || tile_n >= (uint)total_tiles[0]) {
+        return;
+    }
+
+    uint src_idx = (tile_n * tile_size + row_in_tile) * K / 4 + blk_k * 8 + col_in_tile;
+    uint dst_idx = tile_n * tile_size * K / 4 + (blk_k * 8 + col_in_tile) * tile_size + row_in_tile;
+
+    src1_qa_trans[dst_idx] = src1_qa[src_idx];
+
+    if (col_in_tile == 0) {
+        src_idx = (tile_n * tile_size + row_in_tile) * K / 32 + blk_k;
+        dst_idx = tile_n * tile_size * K / 32 + blk_k * tile_size + row_in_tile;
+
+        src1_sa_trans[dst_idx] = (float)(src1_sa[src_idx]);
+    }
+
+}

gguf-py/gguf/constants.py

@@ -573,6 +573,7 @@ class MODEL_ARCH(IntEnum):
     QWEN3MOE         = auto()
     QWEN3NEXT        = auto()
     HRM_TEXT         = auto()
+    BEAM             = auto()
     QWEN3VL          = auto()
     QWEN3VLMOE       = auto()
     QWEN35           = auto()
@@ -1388,6 +1389,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.QWEN3MOE:         "qwen3moe",
     MODEL_ARCH.QWEN3NEXT:        "qwen3next",
     MODEL_ARCH.HRM_TEXT:         "hrm_text",
+    MODEL_ARCH.BEAM:             "beam",
     MODEL_ARCH.QWEN3VL:          "qwen3vl",
     MODEL_ARCH.QWEN3VLMOE:       "qwen3vlmoe",
     MODEL_ARCH.QWEN35:           "qwen35",
@@ -2981,6 +2983,30 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
         MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
     ],
+    MODEL_ARCH.BEAM: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_POST_NORM,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.ATTN_Q_NORM,
+        MODEL_TENSOR.ATTN_K_NORM,
+        MODEL_TENSOR.ATTN_GATE,
+        MODEL_TENSOR.FFN_PRE_NORM,
+        MODEL_TENSOR.FFN_POST_NORM,
+        MODEL_TENSOR.FFN_GATE,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.FFN_GATE_INP,
+        MODEL_TENSOR.FFN_EXP_PROBS_B,
+        MODEL_TENSOR.FFN_GATE_EXP,
+        MODEL_TENSOR.FFN_DOWN_EXP,
+        MODEL_TENSOR.FFN_UP_EXP,
+    ],
     MODEL_ARCH.HRM_TEXT: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT,

skills/code-review/SKILL.md

@@ -214,7 +214,7 @@ Adjacent data-shape rule: empty string is not a valid "unset" initial state - us
 - Only add tests that bring meaningful results. Too-trivial tests just bloat the suite and CI.
 - No time-sensitive tests (timing thresholds, sleeps, races against wall-clock); they are flaky on CI.
 - Think twice about tests that significantly increase CI run time (expensive computation, large inputs, or long sleep/wait delays) or download large amounts of data from the internet (big models, datasets). Flag them and ask whether a smaller model/input or an existing fixture would do.
-- Server: small tests of internal components (helpers, `engine_tokens`, queues, format converters, ...) belong in `tests/test-server-component.cpp`. The end-to-end pytest suite in `tools/server/tests` is costly in CI time; only add to it when the test requires end-to-end behavior with a real model.
+- Server: small tests of internal components (helpers, `server_tokens`, queues, format converters, ...) belong in `tests/test-server-component.cpp`. The end-to-end pytest suite in `tools/server/tests` is costly in CI time; only add to it when the test requires end-to-end behavior with a real model.
 
 ## General (always)
 

src/llama-arch.cpp

@@ -165,6 +165,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_QWEN3TTS,         "qwen3tts"         },
     { LLM_ARCH_POCKETTTS,        "pockettts"        },
     { LLM_ARCH_K2_HORIZON,       "k2-horizon"       },
+    { LLM_ARCH_BEAM,             "beam"             },
     { LLM_ARCH_UNKNOWN,          "(unknown)"        },
 };
 

src/llama-arch.h

@@ -170,6 +170,7 @@ enum llm_arch {
     LLM_ARCH_MINIMAX_01,
     LLM_ARCH_HRM_TEXT,
     LLM_ARCH_K2_HORIZON,
+    LLM_ARCH_BEAM,
     LLM_ARCH_UNKNOWN,
 };
 

src/llama-graph.cpp

@@ -218,17 +218,31 @@ void llm_graph_input_attn_temp::set_input(const llama_ubatch * ubatch) {
         GGML_ASSERT(n_attn_temp_floor_scale != 0);
 
         std::vector<float> attn_scale_data(n_tokens, 0.0f);
-        for (int i = 0; i < n_tokens; ++i) {
-            const float pos = ubatch->pos[i];
-            attn_scale_data[i] = std::log(
-                std::floor((pos + f_attn_temp_offset) / n_attn_temp_floor_scale) + 1.0
-            ) * f_attn_temp_scale + 1.0;
+        if (attn_temp_pow) {
+            // (1 + (pos + offset) / length) ^ power
+            for (int i = 0; i < n_tokens; ++i) {
+                const double pos = ubatch->pos[i];
+                attn_scale_data[i] = (float) std::exp((double) f_attn_temp_scale *
+                    std::log1p((pos + f_attn_temp_offset) / (double) n_attn_temp_floor_scale));
+            }
+        } else {
+            for (int i = 0; i < n_tokens; ++i) {
+                const float pos = ubatch->pos[i];
+                attn_scale_data[i] = std::log(
+                    std::floor((pos + f_attn_temp_offset) / n_attn_temp_floor_scale) + 1.0
+                ) * f_attn_temp_scale + 1.0;
+            }
         }
 
         ggml_backend_tensor_set(attn_scale, attn_scale_data.data(), 0, n_tokens*ggml_element_size(attn_scale));
     }
 }
 
+bool llm_graph_input_attn_temp::can_reuse(const llm_graph_params & params) {
+    // only the power form opts in; the log form keeps no reuse
+    return attn_temp_pow && attn_scale != nullptr && attn_scale->ne[2] == params.ubatch.n_tokens;
+}
+
 void llm_graph_input_pos_bucket::set_input(const llama_ubatch * ubatch) {
     if (pos_bucket) {
         const int64_t n_tokens = ubatch->n_tokens;
@@ -2743,7 +2757,7 @@ ggml_tensor * llm_graph_context::build_inp_pos() const {
 }
 
 ggml_tensor * llm_graph_context::build_inp_attn_scale() const {
-    auto inp = std::make_unique<llm_graph_input_attn_temp>(hparams.n_attn_temp_floor_scale, hparams.f_attn_temp_scale, hparams.f_attn_temp_offset);
+    auto inp = std::make_unique<llm_graph_input_attn_temp>(hparams.n_attn_temp_floor_scale, hparams.f_attn_temp_scale, hparams.f_attn_temp_offset, hparams.attn_temp_pow);
 
     auto & cur = inp->attn_scale;
 

src/llama-graph.h

@@ -188,17 +188,20 @@ public:
 // temperature tuning, used by llama4
 class llm_graph_input_attn_temp : public llm_graph_input_i {
 public:
-    llm_graph_input_attn_temp(uint32_t n_attn_temp_floor_scale, float f_attn_temp_scale, float f_attn_temp_offset)
-        : n_attn_temp_floor_scale(n_attn_temp_floor_scale), f_attn_temp_scale(f_attn_temp_scale), f_attn_temp_offset(f_attn_temp_offset) {}
+    llm_graph_input_attn_temp(uint32_t n_attn_temp_floor_scale, float f_attn_temp_scale, float f_attn_temp_offset, bool attn_temp_pow = false)
+        : n_attn_temp_floor_scale(n_attn_temp_floor_scale), f_attn_temp_scale(f_attn_temp_scale), f_attn_temp_offset(f_attn_temp_offset), attn_temp_pow(attn_temp_pow) {}
     virtual ~llm_graph_input_attn_temp() = default;
 
     void set_input(const llama_ubatch * ubatch) override;
 
+    bool can_reuse(const llm_graph_params & params) override;
+
     ggml_tensor * attn_scale = nullptr; // F32 [n_batch]
 
     const uint32_t n_attn_temp_floor_scale;
     const float    f_attn_temp_scale;
     const float    f_attn_temp_offset;
+    const bool     attn_temp_pow;
 };
 
 class llm_graph_input_pos_bucket : public llm_graph_input_i {

src/llama-hparams.h

@@ -280,6 +280,8 @@ struct llama_hparams {
     uint32_t n_attn_temp_floor_scale = 0;
     float    f_attn_temp_scale       = 0.0f;
     float    f_attn_temp_offset      = 0.0f; // offset position index
+    // use (1 + (pos + offset) / floor_scale) ^ scale instead of the llama4 log form
+    bool     attn_temp_pow           = false;
 
     // realtime models: number of tokens the output lags behind the audio input
     uint32_t n_tokens_delay          = 0;

src/llama-model.cpp

@@ -322,6 +322,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_minimax_m3(params);
         case LLM_ARCH_HRM_TEXT:
             return new llama_model_hrm_text(params);
+        case LLM_ARCH_BEAM:
+            return new llama_model_beam(params);
         case LLM_ARCH_COGVLM:
             return new llama_model_cogvlm(params);
         case LLM_ARCH_PANGU_EMBED:
@@ -405,6 +407,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
     static const std::regex pattern_idx_cache       ("cache_idx_(k|v)_l\\d*");
     static const std::regex pattern_dsv4_state      ("dsv4_(csa|hca|lid)_state_(kv|score)_l\\d*");
     static const std::regex pattern_attn_sinks      ("blk\\.\\d*\\.attn_sinks.weight");
+    static const std::regex pattern_attn_gate_bias  ("blk\\.\\d*\\.attn_gate\\.bias");
     static const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight");
     static const std::regex pattern_attn_out_bias   ("blk\\.\\d*\\.attn_output.bias");
     static const std::regex pattern_attn_out_a_weight("blk\\.\\d*\\.attn_output_a\\.weight");
@@ -550,7 +553,8 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
         if (std::regex_match(tensor_name, pattern_qk_norm)) {
             return get_tensor_config_impl(tensor->ne[1] == 1 ? GGML_BACKEND_SPLIT_AXIS_MIRRORED : GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight");
         }
-        if (std::regex_match(tensor_name, pattern_kv_cache) || std::regex_match(tensor_name, pattern_attn_sinks)) {
+        if (std::regex_match(tensor_name, pattern_kv_cache) || std::regex_match(tensor_name, pattern_attn_sinks) ||
+                std::regex_match(tensor_name, pattern_attn_gate_bias)) {
             return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight");
         }
         if (std::regex_match(tensor_name, pattern_attn_out_weight)) {
@@ -749,6 +753,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
 
             const int64_t granularity_q    = std::lcm(n_embd_q, blck_size_perf);
             const int64_t granularity_head = granularity_q / hparams.n_embd_head_k(il); // for tensors with one value per head
+            if (std::regex_match(tensor_name, pattern_attn_gate_bias)) {
+                GGML_ASSERT(segments.size() == 1);
+                return {granularity_head};
+            }
             if (std::regex_match(tensor_name, pattern_attn_sinks)) {
                 GGML_ASSERT(segments.size() == 1);
                 if (is_dsv4) {
@@ -3606,6 +3614,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_MELLUM:
         case LLM_ARCH_MAPLE:
         case LLM_ARCH_HRM_TEXT:
+        case LLM_ARCH_BEAM:
             return LLAMA_ROPE_TYPE_NEOX;
 
         case LLM_ARCH_DFLASH:

src/llama-model.h

@@ -310,6 +310,7 @@ struct llama_layer {
     struct ggml_tensor * wv_enc    = nullptr;
     struct ggml_tensor * wo_enc    = nullptr;
     struct ggml_tensor * wqkv_gate = nullptr;
+    struct ggml_tensor * wqkv_gate_b = nullptr;
     // K2 Horizon MoVA
     struct ggml_tensor * attn_v_gate   = nullptr;
     struct ggml_tensor * attn_v_gate_b = nullptr;

src/llama-vocab.cpp

@@ -3142,9 +3142,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
             }
         }
 
-        // gemma4 and plamo have a normal </s> token, unlike paddleocr
+        // gemma4, plamo and beam have a normal </s> token, unlike paddleocr
         {
-            bool has_normal_s_marker = false;
+            // beam's tool call marker is not an EOG token, so it is looked up in the whole vocab
+            bool has_normal_s_marker = token_to_id.count("<rfl:parameter name=\"") > 0;
             bool has_s = false;
 
             llama_token s_id = LLAMA_TOKEN_NULL;

src/models/beam.cpp

@@ -0,0 +1,261 @@
+#include "models.h"
+
+void llama_model_beam::load_arch_hparams(llama_model_loader & ml) {
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,       hparams.f_norm_rms_eps);
+    ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,         hparams.n_layer_dense_lead);
+    ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
+    ml.get_key(LLM_KV_EXPERT_GATING_FUNC,                hparams.expert_gating_func, false);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,              hparams.expert_weights_scale);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,               hparams.expert_weights_norm);
+    hparams.f_final_logit_softcapping = 0.0f; // absent key = no softcap (default is 30)
+    ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING,           hparams.f_final_logit_softcapping, false);
+
+    if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {
+        hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;
+    }
+    if (hparams.n_layer_dense_lead > hparams.n_layer()) {
+        throw std::runtime_error("beam: leading_dense_block_count exceeds the layer count");
+    }
+
+    ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; // masked iff p1 - p0 >= n_swa
+    uint32_t n_pattern = 0;
+    ml.get_arr_n(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern);
+    if (hparams.n_swa == 0 || n_pattern != hparams.n_layer_all) {
+        throw std::runtime_error("beam: requires a positive sliding window and a sliding window pattern for each layer");
+    }
+    ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+    if (!hparams.is_swa_any()) {
+        throw std::runtime_error("beam: sliding window pattern has no sliding window layer");
+    }
+
+    // full layers are NoPE
+    hparams.rope_pattern = hparams.is_swa_impl;
+    hparams.rope_freq_base_train_swa  = hparams.rope_freq_base_train;
+    hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
+    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
+
+    // NoPE query scale uses pos + 1
+    hparams.f_attn_temp_scale = 0.1f;
+    hparams.attn_temp_length  = 2048;
+    ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_SCALE,  hparams.f_attn_temp_scale, false);
+    ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.attn_temp_length,  false);
+    if (hparams.f_attn_temp_scale == 0.0f) {
+        throw std::runtime_error("beam: attention.temperature_scale must be nonzero");
+    }
+    if (hparams.attn_temp_length == 0) {
+        throw std::runtime_error("beam: attention.temperature_length must be positive");
+    }
+    hparams.n_attn_temp_floor_scale = hparams.attn_temp_length;
+    hparams.f_attn_temp_offset      = 1.0f;
+    hparams.attn_temp_pow           = true;
+
+    switch (hparams.n_layer()) {
+        case 52: type = LLM_TYPE_UNKNOWN; break;
+        default: type = LLM_TYPE_UNKNOWN;
+    }
+}
+
+void llama_model_beam::load_arch_tensors(llama_model_loader &) {
+    LLAMA_LOAD_LOCALS;
+
+    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+    if (output == NULL) {
+        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+    }
+
+    for (int i = 0; i < n_layer; ++i) {
+        auto & layer = layers[i];
+
+        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM,      "weight", i), {n_embd}, 0);
+        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0);
+
+        create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
+        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
+
+        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
+        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
+
+        layer.wqkv_gate   = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
+        layer.wqkv_gate_b = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "bias",   i), {n_head}, 0);
+
+        layer.ffn_norm      = create_tensor(tn(LLM_TENSOR_FFN_NORM,      "weight", i), {n_embd}, 0);
+        layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0);
+
+        if ((uint32_t) i < hparams.n_layer_dense_lead) {
+            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
+            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
+            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd, n_ff}, 0);
+        } else {
+            const int64_t n_ff_exp = hparams.n_ff_exp(i);
+
+            layer.ffn_gate_inp    = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,    "weight", i), {n_embd, n_expert}, 0);
+            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias",   i), {n_expert}, 0);
+
+            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
+            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
+            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
+        }
+    }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_beam::build_arch_graph(const llm_graph_params & params) const {
+    return std::make_unique<graph>(*this, params);
+}
+
+static ggml_tensor * beam_residual_add(ggml_context * ctx, ggml_tensor * h, ggml_tensor * r, int i) {
+    const float h_scale = (float) (1.0 / std::sqrt((double) i));
+    const float r_scale = (float) std::sqrt((double) (i - 1) / (double) i);
+    return ggml_add(ctx, ggml_scale(ctx, h, h_scale), ggml_scale(ctx, r, r_scale));
+}
+
+llama_model_beam::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
+    const int64_t n_embd_head = hparams.n_embd_head_v();
+    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+
+    ggml_tensor * cur;
+    ggml_tensor * inpL;
+
+    inpL = build_inp_embd(model.tok_embd);
+
+    ggml_tensor * inp_pos = build_inp_pos();
+
+    bool has_nope = false;
+    for (int il = 0; il < n_layer; ++il) {
+        has_nope = has_nope || !hparams.is_swa(il);
+    }
+    ggml_tensor * inp_attn_scale = has_nope ? build_inp_attn_scale() : nullptr;
+
+    auto * inp_attn = build_attn_inp_kv_iswa();
+
+    ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+    const float kq_scale = 1.0f / sqrtf(float(n_embd_head));
+
+    for (int il = 0; il < n_layer; ++il) {
+        const auto & layer = model.layers[il];
+
+        const bool is_swa = hparams.is_swa(il);
+
+        ggml_tensor * inpSA = inpL;
+
+        cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "attn_norm", il);
+
+        {
+            ggml_tensor * attn_inp = cur;
+
+            auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il);
+
+            Qcur = build_norm(Qcur, layer.attn_q_norm, NULL, LLM_NORM_RMS, il);
+            Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il);
+            cb(Qcur, "Qcur_normed", il);
+            cb(Kcur, "Kcur_normed", il);
+
+            if (is_swa) {
+                const float freq_base_l  = model.get_rope_freq_base (cparams, il);
+                const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
+
+                Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,
+                        n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+                        ext_factor, attn_factor, beta_fast, beta_slow);
+                Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
+                        n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
+                        ext_factor, attn_factor, beta_fast, beta_slow);
+                cb(Qcur, "Qcur_rope", il);
+                cb(Kcur, "Kcur_rope", il);
+            } else {
+                Qcur = ggml_mul(ctx0, Qcur, inp_attn_scale);
+                cb(Qcur, "Qcur_scaled", il);
+            }
+
+            cur = build_attn(inp_attn,
+                    NULL, NULL, NULL, // o_proj is applied after the gate
+                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
+            cb(cur, "attn_out", il);
+
+            ggml_tensor * gate = build_lora_mm(layer.wqkv_gate, attn_inp);
+            cb(gate, "attn_gate_proj", il);
+            gate = ggml_reshape_3d(ctx0, gate, n_embd_head, n_head, n_tokens);
+            gate = ggml_add(ctx0, gate, ggml_reshape_3d(ctx0, layer.wqkv_gate_b, 1, n_head, 1));
+            gate = ggml_sigmoid(ctx0, gate);
+            gate = ggml_reshape_2d(ctx0, gate, n_embd_head * n_head, n_tokens);
+            cb(gate, "attn_gate_sig", il);
+
+            cur = ggml_mul(ctx0, cur, gate);
+            cb(cur, "attn_gated", il);
+
+            cur = build_lora_mm(layer.wo, cur, layer.wo_s);
+            cb(cur, "attn_o_proj", il);
+        }
+
+        cur = build_norm(cur, layer.attn_post_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "attn_post_norm", il);
+
+        if (il == n_layer - 1 && inp_out_ids) {
+            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);
+            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+        }
+
+        const int i_attn = 2 * (il + 1);
+        ggml_tensor * ffn_inp = beam_residual_add(ctx0, cur, inpSA, i_attn);
+        cb(ffn_inp, "ffn_inp", il);
+
+        cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "ffn_norm", il);
+
+        if ((uint32_t) il < hparams.n_layer_dense_lead) {
+            cur = build_ffn(cur,
+                    layer.ffn_up,   NULL, NULL,
+                    layer.ffn_gate, NULL, NULL,
+                    layer.ffn_down, NULL, NULL,
+                    NULL,
+                    LLM_FFN_SILU, LLM_FFN_PAR, il);
+            cb(cur, "ffn_out", il);
+        } else {
+            cur = build_moe_ffn(cur,
+                    layer.ffn_gate_inp,
+                    layer.ffn_up_exps,
+                    layer.ffn_gate_exps,
+                    layer.ffn_down_exps,
+                    layer.ffn_exp_probs_b,
+                    n_expert, n_expert_used,
+                    LLM_FFN_SILU,
+                    hparams.expert_weights_norm,
+                    hparams.expert_weights_scale,
+                    (llama_expert_gating_func_type) hparams.expert_gating_func,
+                    il);
+            cb(cur, "ffn_moe_out", il);
+        }
+
+        cur = build_norm(cur, layer.ffn_post_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "ffn_post_norm", il);
+
+        cur = beam_residual_add(ctx0, cur, ffn_inp, i_attn + 1);
+
+        cur = build_cvec(cur, il);
+        cb(cur, "l_out", il);
+
+        inpL = cur;
+    }
+
+    cur = inpL;
+
+    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
+    cb(cur, "result_norm", -1);
+    res->t_embd = cur;
+
+    cur = build_lora_mm(model.output, cur, model.output_s);
+
+    if (hparams.f_final_logit_softcapping != 0.0f) {
+        cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping);
+        cur = ggml_tanh(ctx0, cur);
+        cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping);
+    }
+
+    cb(cur, "result_output", -1);
+    res->t_logits = cur;
+}

src/models/models.h

@@ -2041,6 +2041,19 @@ struct llama_model_afmoe : public llama_model_base {
 };
 
 
+struct llama_model_beam : public llama_model_base {
+    llama_model_beam(const struct llama_model_params & params) : llama_model_base(params) {}
+    void load_arch_hparams(llama_model_loader & ml) override;
+    void load_arch_tensors(llama_model_loader & ml) override;
+
+    struct graph : public llm_graph_context {
+        graph(const llama_model & model, const llm_graph_params & params);
+    };
+
+    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
 struct llama_model_laguna : public llama_model_base {
     llama_model_laguna(const struct llama_model_params & params) : llama_model_base(params) {}
     void load_arch_hparams(llama_model_loader & ml) override;

tests/CMakeLists.txt

@@ -188,9 +188,11 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
     llama_build_and_test(test-llama-grammar.cpp LINK llama)
     llama_build_and_test(test-batch-alloc.cpp LINK llama)
     llama_build_and_test(test-chat.cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
-    target_link_libraries(test-chat PRIVATE llama-engine)
+    target_include_directories(test-chat PRIVATE ${PROJECT_SOURCE_DIR}/tools/server)
+    target_link_libraries(test-chat PRIVATE server-context)
     llama_build_and_test(test-server-component.cpp)
-    target_link_libraries(test-server-component PRIVATE llama-engine)
+    target_include_directories(test-server-component PRIVATE ${PROJECT_SOURCE_DIR}/tools/server)
+    target_link_libraries(test-server-component PRIVATE server-context)
     llama_build_and_test(test-json-schema-to-grammar.cpp)
 
     if (NOT GGML_BACKEND_DL)

tests/test-chat.cpp

@@ -7,7 +7,7 @@
 //
 #include "../src/llama-grammar.h"
 #include "../src/unicode.h"
-#include "../tools/engine/engine-chat.h"
+#include "../tools/server/server-chat.h"
 #include "chat-auto-parser.h"
 #include "chat.h"
 #include "common.h"
@@ -1864,7 +1864,7 @@ static void test_convert_responses_to_chatcmpl() {
             }
         })");
 
-        json result = engine_chat_convert_responses_to_chatcmpl(input);
+        json result = server_chat_convert_responses_to_chatcmpl(input);
 
         // Verify messages were converted correctly
         assert_equals(true, result.contains("messages"));
@@ -1904,7 +1904,7 @@ static void test_convert_responses_to_chatcmpl() {
             "model": "test-model"
         })");
 
-        json result = engine_chat_convert_responses_to_chatcmpl(input);
+        json result = server_chat_convert_responses_to_chatcmpl(input);
 
         assert_equals((size_t)1, result.at("messages").size());
         const auto & msg = result.at("messages")[0];
@@ -1920,7 +1920,7 @@ static void test_convert_responses_to_chatcmpl() {
             "model": "test-model"
         })");
 
-        json result = engine_chat_convert_responses_to_chatcmpl(input);
+        json result = server_chat_convert_responses_to_chatcmpl(input);
 
         assert_equals((size_t)2, result.at("messages").size());
         const auto & sys_msg = result.at("messages")[0];
@@ -1936,7 +1936,7 @@ static void test_convert_responses_to_chatcmpl() {
             "max_output_tokens": 100
         })");
 
-        json result = engine_chat_convert_responses_to_chatcmpl(input);
+        json result = server_chat_convert_responses_to_chatcmpl(input);
 
         assert_equals(true, result.contains("max_tokens"));
         assert_equals(false, result.contains("max_output_tokens"));
@@ -1980,7 +1980,7 @@ static void test_convert_responses_to_chatcmpl() {
             ]
         })");
 
-        json result = engine_chat_convert_responses_to_chatcmpl(input);
+        json result = server_chat_convert_responses_to_chatcmpl(input);
 
         assert_equals(true, result.contains("tools"));
         assert_equals(true, result.at("tools").is_array());
@@ -2015,7 +2015,7 @@ static void test_convert_responses_to_chatcmpl() {
             ]
         })");
 
-        json result = engine_chat_convert_responses_to_chatcmpl(input);
+        json result = server_chat_convert_responses_to_chatcmpl(input);
 
         assert_equals(false, result.contains("tools"));
     }
@@ -7938,7 +7938,7 @@ static void test_reasoning_budget_tokens_per_request() {
     // the reasoning-budget code path in oaicompat_chat_params_parse.
     auto tmpls = read_templates("models/templates/Qwen-Qwen3-0.6B.jinja");
 
-    engine_chat_params opt;
+    server_chat_params opt;
     opt.tmpls            = std::move(tmpls);
     opt.use_jinja        = true;
     opt.enable_thinking  = true;
@@ -7970,7 +7970,7 @@ static void test_reasoning_budget_message_per_request() {
     // <think>...</think> markers enable the reasoning-budget block in oaicompat_chat_params_parse.
     auto tmpls = read_templates("models/templates/Qwen-Qwen3-0.6B.jinja");
 
-    engine_chat_params opt;
+    server_chat_params opt;
     opt.tmpls                   = std::move(tmpls);
     opt.use_jinja               = true;
     opt.enable_thinking         = true;

tests/test-jinja.cpp

@@ -2011,6 +2011,12 @@ static void test_object_methods(testing & t) {
         "a b "
     );
 
+    test_template(t, "object | list",
+        "{{ (obj | list) == ['b', 'a'] }} {{ obj | list | join(',') }}",
+        {{"obj", {{"b", 1}, {"a", 2}}}},
+        "True b,a"
+    );
+
     test_template(t, "object.values()",
         "{% for v in obj.values() %}{{ v }} {% endfor %}",
         {{"obj", {{"a", 1}, {"b", 2}}}},

tests/test-llama-archs.cpp

@@ -188,6 +188,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
     } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER) {
         n_head_kv = 2; // GQA coverage
     }
+    if (arch == LLM_ARCH_BEAM) {
+        n_head    = 4;
+        n_head_kv = 2;
+        n_layer   = 4;
+    }
     const uint32_t n_embd_head = n_embd / n_head;
 
     ms.add_kv(LLM_KV_GENERAL_ARCHITECTURE,      llm_arch_name(arch));
@@ -294,6 +299,13 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
 
     if (arch == LLM_ARCH_MAION_CODER) {
         ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector<uint32_t>{1, 1, 0, 0});
+    } else if (arch == LLM_ARCH_BEAM) {
+        std::vector<uint32_t> pattern;
+        pattern.reserve(n_layer);
+        for (uint32_t il = 0; il < n_layer; il++) {
+            pattern.push_back(il % 2);
+        }
+        ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);
     } else if (arch == LLM_ARCH_GEMMA4) {
         ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER,      n_embd/2);
         ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS,      uint32_t(0));
@@ -470,6 +482,14 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         }
     }
 
+    if (arch == LLM_ARCH_BEAM) {
+        ms.add_kv(LLM_KV_EXPERT_COUNT,            uint32_t(8));
+        ms.add_kv(LLM_KV_EXPERT_GROUP_COUNT,      uint32_t(4));
+        ms.add_kv(LLM_KV_EXPERT_GROUP_USED_COUNT, uint32_t(2));
+        ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE,    2.5f);
+        ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM,     true);
+    }
+
     if (arch == LLM_ARCH_MAION_CODER) {
         ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd / 2);
         ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, std::vector<uint32_t>{n_ff, n_ff / 2, n_ff, n_ff});
@@ -712,6 +732,7 @@ static bool moe_mandatory(const llm_arch arch) {
         case LLM_ARCH_BAILINGMOE3:
         case LLM_ARCH_DOTS1:
         case LLM_ARCH_AFMOE:
+        case LLM_ARCH_BEAM:
         case LLM_ARCH_ERNIE4_5:
         case LLM_ARCH_ERNIE4_5_MOE:
         case LLM_ARCH_HUNYUAN_MOE:

tests/test-server-component.cpp

@@ -1,8 +1,7 @@
 #include "testing.h"
 
-#include "../tools/engine/engine-common.h"
-#include "../tools/server/server-subproc.h"
-#include "../tools/engine/engine-decision.h"
+#include "../tools/server/server-common.h"
+#include "../tools/server/server-decision.h"
 
 #include <cstdlib>
 #include <regex>
@@ -16,11 +15,11 @@
 // see skills/code-review/SKILL.md for more
 
 //
-// engine_tokens (text only, has_mtmd = false)
+// server_tokens (text only, has_mtmd = false)
 //
 
-static void test_engine_tokens_basic(testing & t) {
-    engine_tokens st(llama_tokens{1, 2, 3}, false);
+static void test_server_tokens_basic(testing & t) {
+    server_tokens st(llama_tokens{1, 2, 3}, false);
     t.assert_equal("size", (size_t) 3, st.size());
     t.assert_true("not empty", !st.empty());
     t.assert_equal("operator[]", 2, st[1]);
@@ -48,8 +47,8 @@ static void test_engine_tokens_basic(testing & t) {
     t.assert_true("clear", st.empty());
 }
 
-static void test_engine_tokens_pos(testing & t) {
-    engine_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
+static void test_server_tokens_pos(testing & t) {
+    server_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
     t.assert_equal("pos_next()", (llama_pos) 5, st.pos_next());
     t.assert_equal("pos_next(0)", (llama_pos) 0, st.pos_next(0));
     t.assert_equal("pos_next(3)", (llama_pos) 3, st.pos_next(3));
@@ -59,8 +58,8 @@ static void test_engine_tokens_pos(testing & t) {
     t.assert_equal("size_up_to_pos clamps", (size_t) 5, st.size_up_to_pos(100));
 }
 
-static void test_engine_tokens_keep_first(testing & t) {
-    engine_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
+static void test_server_tokens_keep_first(testing & t) {
+    server_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
     st.keep_first(5);
     t.assert_equal("keep all", (size_t) 5, st.size());
     st.keep_first(2);
@@ -69,21 +68,21 @@ static void test_engine_tokens_keep_first(testing & t) {
     t.assert_true("keep 0", st.empty());
 }
 
-static void test_engine_tokens_common_prefix(testing & t) {
-    engine_tokens a(llama_tokens{1, 2, 3, 4}, false);
-    engine_tokens empty(llama_tokens{}, false);
+static void test_server_tokens_common_prefix(testing & t) {
+    server_tokens a(llama_tokens{1, 2, 3, 4}, false);
+    server_tokens empty(llama_tokens{}, false);
 
     t.assert_equal("both empty",  (size_t) 0, empty.get_common_prefix(empty));
     t.assert_equal("one empty",   (size_t) 0, a.get_common_prefix(empty));
     t.assert_equal("identical",   (size_t) 4, a.get_common_prefix(a));
-    t.assert_equal("diverge at 2", (size_t) 2, a.get_common_prefix(engine_tokens(llama_tokens{1, 2, 9, 4}, false)));
-    t.assert_equal("diverge at 0", (size_t) 0, a.get_common_prefix(engine_tokens(llama_tokens{9, 2, 3, 4}, false)));
-    t.assert_equal("b is prefix",  (size_t) 2, a.get_common_prefix(engine_tokens(llama_tokens{1, 2}, false)));
-    t.assert_equal("a is prefix",  (size_t) 4, a.get_common_prefix(engine_tokens(llama_tokens{1, 2, 3, 4, 5, 6}, false)));
+    t.assert_equal("diverge at 2", (size_t) 2, a.get_common_prefix(server_tokens(llama_tokens{1, 2, 9, 4}, false)));
+    t.assert_equal("diverge at 0", (size_t) 0, a.get_common_prefix(server_tokens(llama_tokens{9, 2, 3, 4}, false)));
+    t.assert_equal("b is prefix",  (size_t) 2, a.get_common_prefix(server_tokens(llama_tokens{1, 2}, false)));
+    t.assert_equal("a is prefix",  (size_t) 4, a.get_common_prefix(server_tokens(llama_tokens{1, 2, 3, 4, 5, 6}, false)));
 }
 
-static void test_engine_tokens_serialize(testing & t) {
-    // same packing as slot save/restore in engine-context.cpp
+static void test_server_tokens_serialize(testing & t) {
+    // same packing as slot save/restore in server-context.cpp
     auto pack = [](const std::vector<char> & data) {
         llama_tokens packed(data.size() / sizeof(llama_token));
         memcpy(packed.data(), data.data(), data.size());
@@ -91,29 +90,29 @@ static void test_engine_tokens_serialize(testing & t) {
     };
 
     t.test("roundtrip", [&](testing & t) {
-        engine_tokens st(llama_tokens{10, 20, 30}, false);
+        server_tokens st(llama_tokens{10, 20, 30}, false);
         std::vector<char> data = st.serialize();
         t.assert_equal("aligned to llama_token", (size_t) 0, data.size() % sizeof(llama_token));
 
-        engine_tokens restored = engine_tokens::deserialize(pack(data), false);
+        server_tokens restored = server_tokens::deserialize(pack(data), false);
         t.assert_true("tokens", restored.get_tokens() == st.get_tokens());
     });
 
     t.test("roundtrip empty", [&](testing & t) {
-        engine_tokens st(llama_tokens{}, false);
-        engine_tokens restored = engine_tokens::deserialize(pack(st.serialize()), false);
+        server_tokens st(llama_tokens{}, false);
+        server_tokens restored = server_tokens::deserialize(pack(st.serialize()), false);
         t.assert_true("empty", restored.empty());
     });
 
     t.test("legacy plain token list", [&](testing & t) {
-        engine_tokens restored = engine_tokens::deserialize(llama_tokens{7, 8, 9}, false);
+        server_tokens restored = server_tokens::deserialize(llama_tokens{7, 8, 9}, false);
         t.assert_true("tokens", restored.get_tokens() == llama_tokens({7, 8, 9}));
     });
 
     auto expect_throw = [](testing & t, const std::string & msg, const llama_tokens & packed) {
         bool thrown = false;
         try {
-            engine_tokens::deserialize(packed, false);
+            server_tokens::deserialize(packed, false);
         } catch (const std::exception &) {
             thrown = true;
         }
@@ -121,7 +120,7 @@ static void test_engine_tokens_serialize(testing & t) {
     };
 
     t.test("corrupted", [&](testing & t) {
-        llama_tokens good = pack(engine_tokens(llama_tokens{10, 20, 30}, false).serialize());
+        llama_tokens good = pack(server_tokens(llama_tokens{10, 20, 30}, false).serialize());
 
         llama_tokens bad_version = good;
         bad_version[1] = 999;
@@ -143,22 +142,22 @@ static void test_engine_tokens_serialize(testing & t) {
     });
 }
 
-static void test_engine_tokens_clone(testing & t) {
-    engine_tokens st(llama_tokens{1, 2, 3}, false);
-    engine_tokens cl = st.clone();
+static void test_server_tokens_clone(testing & t) {
+    server_tokens st(llama_tokens{1, 2, 3}, false);
+    server_tokens cl = st.clone();
     cl.push_back(4);
     t.assert_equal("original untouched", (size_t) 3, st.size());
     t.assert_equal("clone grew", (size_t) 4, cl.size());
     t.assert_true("get_text_tokens", cl.get_text_tokens() == llama_tokens({1, 2, 3, 4}));
 }
 
-static void test_engine_tokens(testing & t) {
-    t.test("basic",         test_engine_tokens_basic);
-    t.test("pos",           test_engine_tokens_pos);
-    t.test("keep_first",    test_engine_tokens_keep_first);
-    t.test("common_prefix", test_engine_tokens_common_prefix);
-    t.test("serialize",     test_engine_tokens_serialize);
-    t.test("clone",         test_engine_tokens_clone);
+static void test_server_tokens(testing & t) {
+    t.test("basic",         test_server_tokens_basic);
+    t.test("pos",           test_server_tokens_pos);
+    t.test("keep_first",    test_server_tokens_keep_first);
+    t.test("common_prefix", test_server_tokens_common_prefix);
+    t.test("serialize",     test_server_tokens_serialize);
+    t.test("clone",         test_server_tokens_clone);
 }
 
 //
@@ -375,7 +374,7 @@ static void test_server_pipe(testing & t) {
 }
 
 //
-// engine_decision_escape_special_tokens
+// server_decision_escape_special_tokens
 //
 
 // "<|name|>" with U+00A6 in place of "|"
@@ -399,7 +398,7 @@ static void test_decision_escape(testing & t) {
             { "<|abc",            "<|abc"                          },
         };
         for (const auto & [text, expected] : cases) {
-            t.assert_equal(text, expected, engine_decision_escape_special_tokens(text));
+            t.assert_equal(text, expected, server_decision_escape_special_tokens(text));
         }
     });
 
@@ -419,7 +418,7 @@ static void test_decision_escape(testing & t) {
                 for (size_t k = 0, v = idx; k < len; k++, v /= chars.size()) {
                     text += chars[v % chars.size()];
                 }
-                if (engine_decision_escape_special_tokens(text) != std::regex_replace(text, re_special, "<\xC2\xA6$1\xC2\xA6>")) {
+                if (server_decision_escape_special_tokens(text) != std::regex_replace(text, re_special, "<\xC2\xA6$1\xC2\xA6>")) {
                     n_diff++;
                 }
                 n_strings++;
@@ -431,8 +430,8 @@ static void test_decision_escape(testing & t) {
     // std::regex overflowed the stack on these
     t.test("long input", [](testing & t) {
         const std::string name(1000000, 'a');
-        t.assert_true("not closed", engine_decision_escape_special_tokens("<|" + name) == "<|" + name);
-        t.assert_true("closed", engine_decision_escape_special_tokens("<|" + name + "|>") == escaped(name));
+        t.assert_true("not closed", server_decision_escape_special_tokens("<|" + name) == "<|" + name);
+        t.assert_true("closed", server_decision_escape_special_tokens("<|" + name + "|>") == escaped(name));
     });
 }
 
@@ -448,7 +447,7 @@ int main(int argc, char ** argv) {
         t.set_filter(argv[1]);
     }
 
-    t.test("engine_tokens", test_engine_tokens);
+    t.test("server_tokens", test_server_tokens);
     t.test("utf8",          test_utf8);
     t.test("sse",           test_sse);
     t.test("json_helpers",  test_json_helpers);

tools/CMakeLists.txt

@@ -23,7 +23,6 @@ else()
     add_subdirectory(quantize)
     if (LLAMA_BUILD_SERVER)
         add_subdirectory(ui)
-        add_subdirectory(engine)
         add_subdirectory(cli)
         add_subdirectory(server)
     endif()

tools/engine/CMakeLists.txt

@@ -1,29 +0,0 @@
-# llama-engine: inference engine (context, task queue, common utils), used by llama-server and CLI
-
-set(TARGET llama-engine)
-
-add_library(${TARGET}
-    engine-chat.cpp
-    engine-chat.h
-    engine-task.cpp
-    engine-task.h
-    engine-queue.cpp
-    engine-queue.h
-    engine-common.cpp
-    engine-common.h
-    engine-decision.cpp
-    engine-decision.h
-    engine-context.cpp
-    engine-context.h
-    engine-schema.cpp
-    engine-schema.h
-)
-set_target_properties(${TARGET} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
-
-target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
-target_include_directories(${TARGET} PRIVATE ../mtmd ${PROJECT_SOURCE_DIR})
-target_link_libraries(${TARGET} PUBLIC llama-common mtmd ${CMAKE_THREAD_LIBS_INIT})
-
-if(LLAMA_TOOLS_INSTALL)
-    install(TARGETS ${TARGET} LIBRARY)
-endif()

tools/engine/engine-context.h

@@ -1,131 +0,0 @@
-#pragma once
-
-#include "engine-common.h"
-#include "engine-decision.h"
-#include "engine-task.h"
-#include "engine-queue.h"
-
-#include "json.h"
-
-#include <cstddef>
-#include <memory>
-#include <mutex>
-#include <set>
-
-struct engine_context_impl; // private implementation
-
-constexpr int HTTP_POLLING_SECONDS = 1;
-
-struct engine_context_meta {
-    std::string build_info;
-    std::string model_name;
-    std::set<std::string> model_aliases;
-    std::set<std::string> model_tags;
-    std::string model_path;
-    std::vector<std::string> model_output_modalities; // output modalities for GET /models
-    bool has_mtmd;
-    bool has_inp_image;
-    bool has_inp_audio;
-    bool has_inp_video;
-    json json_ui_settings;
-    int slot_n_ctx;
-    enum llama_pooling_type pooling_type;
-
-    // chat params
-    engine_chat_params & chat_params;
-    std::map<std::string, bool> chat_template_caps;
-
-    // tokens
-    std::string bos_token_str;
-    std::string eos_token_str;
-    llama_token fim_pre_token;
-    llama_token fim_sub_token;
-    llama_token fim_mid_token;
-    llama_token fim_pad_token;
-    llama_token fim_rep_token;
-    llama_token fim_sep_token;
-
-    // sampling
-    std::vector<llama_logit_bias> logit_bias_eog;
-
-    // model meta
-    enum llama_vocab_type model_vocab_type;
-    int32_t model_vocab_n_tokens;
-    int32_t model_n_ctx_train;
-    int32_t model_n_embd_inp;
-    uint64_t model_n_params;
-    uint64_t model_size;
-    std::string model_ftype;
-};
-
-enum engine_state {
-    ENGINE_STATE_DOWNLOADING,
-    ENGINE_STATE_LOADING,
-    ENGINE_STATE_READY,
-    ENGINE_STATE_SLEEPING,
-};
-
-static std::string engine_state_to_str(engine_state state) {
-    switch (state) {
-        case ENGINE_STATE_DOWNLOADING: return "downloading";
-        case ENGINE_STATE_LOADING:     return "loading";
-        case ENGINE_STATE_READY:       return "ready";
-        case ENGINE_STATE_SLEEPING:    return "sleeping";
-        default: GGML_ASSERT(false && "invalid engine_state");
-    }
-}
-
-static engine_state engine_state_from_str(const std::string & str) {
-    if (str == "downloading") return ENGINE_STATE_DOWNLOADING;
-    if (str == "loading")     return ENGINE_STATE_LOADING;
-    if (str == "ready")       return ENGINE_STATE_READY;
-    if (str == "sleeping")    return ENGINE_STATE_SLEEPING;
-    GGML_ASSERT(false && "invalid engine_state string");
-}
-
-using engine_state_callback_t = std::function<void(engine_state, json /* payload */)>;
-
-struct engine_context {
-    std::unique_ptr<engine_context_impl> impl;
-
-    engine_context();
-    ~engine_context();
-
-    // load the model and initialize llama_context
-    // returns true on success
-    bool load_model(common_params & params);
-
-    // this function will block main thread until termination
-    void start_loop();
-
-    // terminate main loop (will unblock start_loop)
-    void terminate();
-
-    // get the underlaying llama_context, can return nullptr if sleeping
-    // not thread-safe, should only be used from the main thread
-    llama_context * get_llama_context() const;
-
-    // get a new response reader, used by CLI application
-    engine_response_reader get_response_reader();
-
-    // get server metadata (read-only), can only be called after load_model()
-    // not thread-safe, should only be used from the main thread
-    engine_context_meta get_meta() const;
-
-    // note: must be set before load_model() is called
-    void set_state_callback(engine_state_callback_t callback);
-
-    // used by the HTTP layer
-    // pointers can only be used when not sleeping, with thread-safe APIs (e.g., tokenizer calls)
-    llama_model * get_model() const;
-    const llama_vocab * get_vocab() const;
-    mtmd_context * get_mtmd_context() const;
-    const mtmd_helper_init_opt & get_mtmd_init_opt() const;
-    const engine_decision_context & get_decision() const;
-
-    engine_queue    & get_queue_tasks();
-    engine_response & get_queue_results();
-
-    engine_metrics get_metrics() const;
-    void reset_metrics_bucket();
-};

tools/engine/engine-decision.h

@@ -1,166 +0,0 @@
-#pragma once
-
-#include "engine-common.h"
-#include "engine-task.h"
-
-#include <map>
-#include <memory>
-#include <string>
-#include <vector>
-
-// typed decision models (TypeSafe /v1/systemone API)
-// the model answers each question in one forward pass, no token is generated
-
-enum engine_decision_question_type {
-    ENGINE_DECISION_QUESTION_CHOICE,
-    ENGINE_DECISION_QUESTION_SCORE,
-    ENGINE_DECISION_QUESTION_NOUL,
-};
-
-struct engine_decision_option {
-    std::string key;
-    json description; // null if not provided
-};
-
-struct engine_decision_question {
-    std::string id;
-    engine_decision_question_type type;
-    json instructions;
-    std::vector<engine_decision_option> options; // in the order of the model outputs
-};
-
-struct engine_decision_context {
-    common_decision_type type = COMMON_DECISION_TYPE_NONE;
-
-    // read the "<arch>.decision.*" metadata, type stays NONE if the model has none
-    void init(const llama_model * model);
-
-    // true if the questions of a request start with the same tokens, and the model can continue from them
-    bool can_share_prompt() const {
-        switch (type) {
-            case COMMON_DECISION_TYPE_OPENJEV:
-            case COMMON_DECISION_TYPE_LEV:
-            case COMMON_DECISION_TYPE_KEV:
-            case COMMON_DECISION_TYPE_NIMBLE:
-            case COMMON_DECISION_TYPE_PPLX_DECIDER:
-            case COMMON_DECISION_TYPE_VLLM_DECISION2:
-            case COMMON_DECISION_TYPE_LFM2_D1:
-                return true;
-            default:
-                return false;
-        }
-    }
-
-    // true if all the questions of a request go in one prompt, see fill_task_joint()
-    bool is_joint() const {
-        return type == COMMON_DECISION_TYPE_CLEF;
-    }
-
-    // true if the prompt of the model has a place for images
-    bool can_use_images() const {
-        switch (type) {
-            case COMMON_DECISION_TYPE_OPENJEV:
-            case COMMON_DECISION_TYPE_CLEF:
-            case COMMON_DECISION_TYPE_PPLX_DECIDER:
-            case COMMON_DECISION_TYPE_VLLM_D3:
-            case COMMON_DECISION_TYPE_LFM2_D1:
-            case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
-                return true;
-            default:
-                return false;
-        }
-    }
-
-    // throw std::invalid_argument on bad input
-    std::vector<engine_decision_question> parse_questions(const json & body) const;
-
-    // returns the state without its images, they are appended to files in order
-    // images come from "files" (alias "images") and from the image_url and input_audio parts of a state made of chat messages
-    // an image can be an audio clip if the model supports it
-    json parse_state(const json & body, std::vector<raw_buffer> & files) const;
-
-    // number of prompts that are evaluated to answer this question, each one shows the options in a different order
-    size_t n_variants(const engine_decision_question & question) const;
-
-    // set the prompt of one variant of this question, and where to read its result
-    // mctx is only used if there are files
-    void fill_task(
-            const json & state,
-            const std::vector<engine_decision_question> & questions,
-            const engine_decision_question & question,
-            size_t variant,
-            const std::vector<raw_buffer> & files,
-            mtmd_context * mctx,
-            const mtmd_helper_init_opt & init_opt,
-            engine_task & task) const;
-
-    // set the prompt of all the questions, the result has the scores of all their options, in order
-    // mctx is only used if there are files
-    void fill_task_joint(
-            const json & state,
-            const std::vector<engine_decision_question> & questions,
-            const std::vector<raw_buffer> & files,
-            mtmd_context * mctx,
-            const mtmd_helper_init_opt & init_opt,
-            engine_task & task) const;
-
-    // scores: the raw model outputs of each variant
-    json format_answer(const engine_decision_question & question, const std::vector<std::vector<float>> & scores, bool has_media = false) const;
-
-private:
-    const llama_vocab * vocab = nullptr;
-    std::shared_ptr<const common_chat_template> tmpl; // the "systemone" template
-
-    std::map<std::string, float> temperatures; // "<type>" or "<type>.<n_options bucket>"
-    std::map<size_t, std::vector<float>> score_bias; // added to the scores of a score question, by number of levels
-    size_t n_options_max   = 0;
-    bool   noul_true_first = false; // noul options are [true, false] instead of [false, true]
-    bool   noul_criteria_order = false; // noul options are in the order of the criteria keys, if given
-    bool   choice_sorted   = false; // choice options are in the order of their keys
-    bool   strict_payload  = false; // state, instructions and descriptions must be text, an object or an array
-    bool   canonical_legend = false; // non-string score descriptions are returned as canonical JSON text
-
-    // OPENJEV, LEV, NIMBLE, PPLX_DECIDER, VLLM_D3
-    std::vector<llama_token> labels;
-    std::vector<std::string> label_texts; // only if the label of an option is given to the template
-
-    // LAYA, KEV
-    llama_token token_marker      = LLAMA_TOKEN_NULL;
-    llama_token token_sep         = LLAMA_TOKEN_NULL;
-    std::string text_marker;
-    size_t      max_head_tokens   = 0; // question + options
-    size_t      max_option_tokens = 48;
-
-    std::string render(
-            const json & state,
-            const std::vector<engine_decision_question> & questions,
-            const engine_decision_question & question,
-            size_t variant,
-            size_t n_images,
-            bool is_audio = false) const;
-    json render_options(const engine_decision_question & question, size_t variant) const;
-    size_t n_outputs(const engine_decision_question & question) const;
-    // LFM2_D1: label text and tokens of each option
-    void d1_labels(const engine_decision_question & question, std::vector<std::string> & texts, std::vector<llama_tokens> & groups) const;
-    void fill_task_laya(llama_tokens & tokens, const engine_decision_question & question, engine_task & task) const;
-    void fill_task_d1omni(
-            const json & state,
-            const std::vector<engine_decision_question> & questions,
-            const engine_decision_question & question,
-            size_t variant,
-            const std::vector<raw_buffer> & files,
-            mtmd_context * mctx,
-            const mtmd_helper_init_opt & init_opt,
-            engine_task & task) const;
-
-    float get_temperature(const engine_decision_question & question) const;
-    double get_decision_confidence(engine_decision_question_type type, const std::vector<double> & probs) const;
-};
-
-// group the tasks so that the common prefix of their prompts is evaluated only once
-// each group is one parent and its children, it takes at most n_slots slots
-// note: the order of the tasks is preserved
-std::vector<engine_task> engine_decision_group_tasks(std::vector<engine_task> && tasks, size_t n_slots);
-
-// write "<|name|>" with U+00A6 in place of "|", so text from the request is not parsed as special tokens
-std::string engine_decision_escape_special_tokens(const std::string & text);

tools/server/CMakeLists.txt

@@ -1,5 +1,40 @@
 include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
 
+# server-context containing the core server logic, used by llama-server and CLI
+
+set(TARGET server-context)
+
+add_library(${TARGET} STATIC
+    server-chat.cpp
+    server-chat.h
+    server-task.cpp
+    server-task.h
+    server-queue.cpp
+    server-queue.h
+    server-common.cpp
+    server-common.h
+    server-decision.cpp
+    server-decision.h
+    server-context.cpp
+    server-context.h
+    server-stream.cpp
+    server-stream.h
+    server-tools.cpp
+    server-tools.h
+    server-mcp.cpp
+    server-mcp.h
+    server-schema.cpp
+    server-schema.h
+)
+
+if (BUILD_SHARED_LIBS)
+    set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+endif()
+
+target_include_directories(${TARGET} PRIVATE ../mtmd)
+target_include_directories(${TARGET} PRIVATE ${PROJECT_SOURCE_DIR})
+target_link_libraries(${TARGET} PUBLIC llama-common mtmd ${CMAKE_THREAD_LIBS_INIT})
+
 # llama-server-impl: server logic, reusable by app
 
 set(TARGET llama-server-impl)
@@ -8,16 +43,6 @@ add_library(${TARGET}
     server.cpp
     server-http.cpp
     server-http.h
-    server-routes.cpp
-    server-routes.h
-    server-stream.cpp
-    server-stream.h
-    server-subproc.cpp
-    server-subproc.h
-    server-tools.cpp
-    server-tools.h
-    server-mcp.cpp
-    server-mcp.h
     server-models.cpp
     server-models.h
 )
@@ -25,7 +50,7 @@ set_target_properties(${TARGET} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
 
 target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(${TARGET} PRIVATE ../mtmd ${PROJECT_SOURCE_DIR})
-target_link_libraries(${TARGET} PUBLIC llama-engine llama-ui cpp-httplib ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PUBLIC server-context llama-ui cpp-httplib ${CMAKE_THREAD_LIBS_INIT})
 
 add_dependencies(${TARGET} llama-ui-assets)
 

tools/server/README-dev.md

@@ -43,25 +43,20 @@ The server supports two primary operating modes:
 - **Inference mode**: The default mode for performing inference with a single loaded GGUF model.
 - **Router mode**: Enables management of multiple inference server instances behind a single API endpoint. Requests are automatically routed to the appropriate backend instance based on the requested model.
 
-The code is split into two libraries:
-
-- `libllama-engine` (`tools/engine`): the inference engine, including `engine_context`, task queue, tasks and common utils. It has no HTTP dependency.
-- `libllama-server-impl` (`tools/server`): the HTTP layer (`server_http_context`, `server_routes`), router mode, stream replay and MCP.
-
 The core architecture consists of the following components:
 
-- `engine_context`: Holds the primary inference state, including the main `llama_context` and all active slots.
-- `engine_slot`: An abstraction over a single “sequence” in llama.cpp, responsible for managing individual parallel inference requests.
-- `server_routes`: Middleware layer between `engine_context` and the HTTP interface; handles JSON parsing/formatting and request routing logic.
+- `server_context`: Holds the primary inference state, including the main `llama_context` and all active slots.
+- `server_slot`: An abstraction over a single “sequence” in llama.cpp, responsible for managing individual parallel inference requests.
+- `server_routes`: Middleware layer between `server_context` and the HTTP interface; handles JSON parsing/formatting and request routing logic.
 - `server_http_context`: Implements the HTTP server using `cpp-httplib`.
-- `engine_queue`: Thread-safe queue used by HTTP workers to submit new tasks to `engine_context`.
-- `engine_response`: Thread-safe queue used by `engine_context` to return results to HTTP workers.
-- `engine_response_reader`: Higher-level wrapper around the two queues above for cleaner code.
-- `engine_task`: Unit of work pushed into `engine_queue`.
-- `engine_task_result`: Unit of result pushed into `engine_response`.
-- `engine_tokens`: Unified representation of token sequences (supports both text and multimodal tokens); used by `engine_task` and `engine_slot`.
+- `server_queue`: Thread-safe queue used by HTTP workers to submit new tasks to `server_context`.
+- `server_response`: Thread-safe queue used by `server_context` to return results to HTTP workers.
+- `server_response_reader`: Higher-level wrapper around the two queues above for cleaner code.
+- `server_task`: Unit of work pushed into `server_queue`.
+- `server_task_result`: Unit of result pushed into `server_response`.
+- `server_tokens`: Unified representation of token sequences (supports both text and multimodal tokens); used by `server_task` and `server_slot`.
 - `server_prompt_checkpoint`: For recurrent (e.g., RWKV) and SWA models, stores snapshots of KV cache state. Enables reuse when subsequent requests share the same prompt prefix, saving redundant computation.
-- `server_models`: Standalone component for managing multiple backend instances (used in router mode). It is completely independent of `engine_context`.
+- `server_models`: Standalone component for managing multiple backend instances (used in router mode). It is completely independent of `server_context`.
 - `stream_session_manager`: process wide owner of resumable SSE stream sessions, keyed by conversation id. A file-static singleton inside `server-stream.cpp`, driven through `server_stream_session_manager_start/stop`. Backs the replay buffer that lets a client reattach to a generation after an HTTP disconnect. See the "Resumable streaming" section below.
 
 ```mermaid
@@ -69,13 +64,13 @@ graph TD
     API_User <--> server_http_context
     server_http_context <-- router mode --> server_models
     server_http_context <-- inference mode --> server_routes
-    server_routes -- engine_task --> engine_queue
-    subgraph engine_context
-        engine_queue --> engine_slot
-        engine_slot -- engine_task_result --> engine_response
-        engine_slot[multiple engine_slot]
+    server_routes -- server_task --> server_queue
+    subgraph server_context
+        server_queue --> server_slot
+        server_slot -- server_task_result --> server_response
+        server_slot[multiple server_slot]
     end
-    engine_response --> server_routes
+    server_response --> server_routes
 ```
 
 ### Batching
@@ -90,21 +85,21 @@ Following decoding, the system either retrieves embeddings or samples the next t
 
 ### Thread Management
 
-`engine_context` runs on a dedicated single thread. Because it is single-threaded, heavy post-processing (especially after token generation) should be avoided, as it directly impacts multi-sequence throughput.
+`server_context` runs on a dedicated single thread. Because it is single-threaded, heavy post-processing (especially after token generation) should be avoided, as it directly impacts multi-sequence throughput.
 
 Each incoming HTTP request is handled by its own thread managed by the HTTP library. The following operations are performed in HTTP worker threads:
 
 - JSON request parsing
 - Chat template application
 - Tokenization
-- Conversion of `engine_task_result` into final JSON response
+- Conversion of `server_task_result` into final JSON response
 - Error formatting into JSON
 - Tracking of partial/incremental responses (e.g., streaming tool calls or reasoning steps)
 
 **Best practices to follow:**
 
 - All JSON formatting and chat template logic must stay in the HTTP layer.
-- Avoid passing raw JSON between the HTTP layer and `engine_slot`. Instead, parse everything into native C++ types as early as possible.
+- Avoid passing raw JSON between the HTTP layer and `server_slot`. Instead, parse everything into native C++ types as early as possible.
 
 ### Example trace of a request
 
@@ -112,13 +107,13 @@ Here is an example trace of an API request for text completion:
 
 - A request arrives at the HTTP layer.
 - The request is routed to the corresponding handler inside `server_routes`. In this case, `handle_completions_impl` is invoked.
-- The handler parses the input request, constructs a new `engine_task`, and passes it to `server_res_generator`.
+- The handler parses the input request, constructs a new `server_task`, and passes it to `server_res_generator`.
 - `server_res_generator` creates a new `task_result_state` for each task:
     - `task_result_state` stays in the HTTP layer, responsible for keeping track of the current state of the response (e.g., parsing tool calls or thinking messages).
-    - `engine_task` is moved into `engine_queue` inside `engine_context`.
-- `engine_context` launches the task by moving it into an available slot (see `launch_slot_with_task()`).
+    - `server_task` is moved into `server_queue` inside `server_context`.
+- `server_context` launches the task by moving it into an available slot (see `launch_slot_with_task()`).
 - `update_slot()` processes the task as described in the "Batching" section above.
-- Results may be sent using `send_partial_response` or `send_final_response`, which creates a new `engine_task_result` and pushes it to the response queue.
+- Results may be sent using `send_partial_response` or `send_final_response`, which creates a new `server_task_result` and pushes it to the response queue.
 - At the same time, `server_res_generator` listens to the response queue and retrieves this response.
 - As the response is stateless, `server_res_generator` calls `response->update()` to update the response with the current state.
 - `server_res_generator` then calls `response->to_json()` and passes the response to the HTTP layer.
@@ -139,7 +134,7 @@ The implementation is hidden in `server-stream.cpp` (pimpl). The header exposes
 
 Producer side: `server_res_generator` extends `server_res_spipe`, which keeps all spipe logic out of the generic `server_http_res`. `set_req` attaches a producer when the header is present, and the wrapped `next` tees each chunk into the ring before the socket, so a chunk lost to a dead wire is already buffered. While attached, `should_stop` ignores peer disconnect: only a `DELETE` stops generation. On an early peer drop, `on_complete` drains the tail into the ring on the http worker.
 
-Lifetime safety: the session holds no back reference to the response, so `spipe` is a plain `unique_ptr` touched only by the http worker. `cancel` raises an atomic the producer polls; the producer finalizes the session from its destructor, which also runs `~engine_response_reader::stop()` to cancel the generation at the queue level. A `DELETE` stops work by raising the flag and letting the worker unwind.
+Lifetime safety: the session holds no back reference to the response, so `spipe` is a plain `unique_ptr` touched only by the http worker. `cancel` raises an atomic the producer polls; the producer finalizes the session from its destructor, which also runs `~server_response_reader::stop()` to cancel the generation at the queue level. A `DELETE` stops work by raising the flag and letting the worker unwind.
 
 Consumer side: `GET /v1/stream?conv_id=<id>&from=N` opens a `text/event-stream` that replays buffered bytes from offset `N` and blocks for live bytes, so the browser reattaches like a fresh EventSource. An offset below the dropped prefix returns 400.
 
@@ -286,7 +281,7 @@ The main goal of this API is to allow downloading models and/or removing models
 
 Instead of building everything from the ground up (like what most AI agents will do when you ask them to implement a similar feature), we built on top of existing, already well-engineered components inside the codebase:
 - Model cache infrastructure as mentioned above (`common/download.h`)
-- Server response queue (`engine-queue.h`). We use this feature to broadcast events to SSE clients.
+- Server response queue (`server-queue.h`). We use this feature to broadcast events to SSE clients.
 - Server router thread management (`server-models.h`). We re-use the same thread model that is used for managing subprocess life cycle, except that we don't create a new subprocess, but launch the download right inside the thread.
 
 The flow for downloading a new model:
@@ -299,27 +294,27 @@ The flow for downloading a new model:
 ### Sleep mode
 
 Sleep mode was initially introduced in PR [#18228](https://github.com/ggml-org/llama.cpp/pull/18228). The main idea is to have:
-- `engine_queue` keeping track of the idle timeout
-- When the timeout is detected, `engine_queue` signals to `engine_context_impl` that it should go into sleep
-- `engine_context_impl` frees all `llama_context` and `mtmd_context`
+- `server_queue` keeping track of the idle timeout
+- When the timeout is detected, `server_queue` signals to `server_context_impl` that it should go into sleep
+- `server_context_impl` frees all `llama_context` and `mtmd_context`
 
 Compared to simply exiting the whole process, this approach allows accessing some read-only endpoints during sleep, while also handling wakeup-on-request. Any inference request will wake the server up.
 
 Call stack on entering sleeping:
-- `engine_queue::start_loop` (main thread) sees no task for `idle_sleep_ms` --> `sleeping = true`
+- `server_queue::start_loop` (main thread) sees no task for `idle_sleep_ms` --> `sleeping = true`
 - `cb0(true)` --> `server_routes::update_cached_responses`
     - snapshots `/props`, `/models` and metrics; the model is still alive here
-- `cb1(true)` --> `engine_context_impl::handle_sleeping_state`
-    - `callback_state(ENGINE_STATE_SLEEPING)` --> reported to router in child mode
+- `cb1(true)` --> `server_context_impl::handle_sleeping_state`
+    - `callback_state(SERVER_STATE_SLEEPING)` --> reported to router in child mode
     - `destroy()` --> frees `llama_context` and `mtmd_context`
 - `condition_tasks.wait` until `req_stop_sleeping`
 
 Call stack on waking up:
-- `server_res_generator` constructor (HTTP thread) --> `engine_queue::wait_until_no_sleep`
+- `server_res_generator` constructor (HTTP thread) --> `server_queue::wait_until_no_sleep`
     - sets `req_stop_sleeping = true`, then waits until `sleeping == false`
-- `engine_queue::start_loop` (main thread) wakes up
-- `cb1(false)` --> `engine_context_impl::handle_sleeping_state`
-    - `load_model()`, which then emits `callback_state(ENGINE_STATE_READY)`
+- `server_queue::start_loop` (main thread) wakes up
+- `cb1(false)` --> `server_context_impl::handle_sleeping_state`
+    - `load_model()`, which then emits `callback_state(SERVER_STATE_READY)`
 - `cb0(false)` --> `server_routes::update_cached_responses`
     - nothing to do, the cache is only read during sleep
 - `sleeping = false` --> `notify_all` unblocks the HTTP thread, the request is handled as usual
@@ -330,7 +325,7 @@ Endpoints created with `create_response(true)` (`/health`, `/props`, `/models`,
 
 - Initial server implementation: https://github.com/ggml-org/llama.cpp/pull/1443
 - Parallel decoding support: https://github.com/ggml-org/llama.cpp/pull/3228
-- Refactor introducing `server_queue` and `server_response` (now `engine_queue` and `engine_response`): https://github.com/ggml-org/llama.cpp/pull/5065
+- Refactor introducing `server_queue` and `server_response`: https://github.com/ggml-org/llama.cpp/pull/5065
 - Reranking endpoint: https://github.com/ggml-org/llama.cpp/pull/9510
 - Multimodal model support (`libmtmd`): https://github.com/ggml-org/llama.cpp/pull/12898
 - Unified KV cache handling: https://github.com/ggml-org/llama.cpp/pull/16736

tools/server/server-chat.cpp

@@ -1,9 +1,9 @@
-#include "engine-chat.h"
-#include "engine-common.h"
+#include "server-chat.h"
+#include "server-common.h"
 
 #include <sstream>
 
-json engine_chat_convert_responses_to_chatcmpl(const json & response_body) {
+json server_chat_convert_responses_to_chatcmpl(const json & response_body) {
     if (!response_body.contains("input")) {
         throw std::invalid_argument("'input' is required");
     }
@@ -345,7 +345,7 @@ static void normalize_anthropic_billing_header(std::string & system_text) {
     }
 }
 
-json engine_chat_convert_anthropic_to_oai(const json & body) {
+json server_chat_convert_anthropic_to_oai(const json & body) {
     json oai_body;
 
     // Convert system prompt
@@ -618,7 +618,7 @@ json engine_chat_convert_anthropic_to_oai(const json & body) {
     return oai_body;
 }
 
-json engine_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff) {
+json server_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff) {
     json delta = json::object();
     if (!diff.reasoning_content_delta.empty()) {
         delta["reasoning_content"] = diff.reasoning_content_delta;

tools/server/server-chat.h

@@ -3,15 +3,16 @@
 #pragma once
 
 #include "chat.h"
-#include "engine-common.h"
+#include "server-common.h"
+#include "server-http.h"
 
 #include "json.h"
 
 // Convert OpenAI Responses API format to OpenAI Chat Completions API format
-json engine_chat_convert_responses_to_chatcmpl(const json & body);
+json server_chat_convert_responses_to_chatcmpl(const json & body);
 
 // Convert Anthropic Messages API format to OpenAI Chat Completions API format
-json engine_chat_convert_anthropic_to_oai(const json & body);
+json server_chat_convert_anthropic_to_oai(const json & body);
 
 // convert OpenAI transcriptions API format to OpenAI Chat Completions API format
 json convert_transcriptions_to_chatcmpl(
@@ -20,4 +21,4 @@ json convert_transcriptions_to_chatcmpl(
     const std::map<std::string, uploaded_file> & in_files,
     std::vector<raw_buffer> & out_files);
 
-json engine_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff);
+json server_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff);

tools/server/server-common.h

@@ -6,6 +6,7 @@
 #include "chat.h"
 #include "mtmd.h"
 #include "mtmd-helper.h"
+#include "subproc.h"
 
 #include "json.h"
 
@@ -38,12 +39,6 @@ using json = common_json;
 
 using raw_buffer = std::vector<uint8_t>;
 
-struct uploaded_file {
-    raw_buffer data;
-    std::string filename;
-    std::string content_type;
-};
-
 template <typename T>
 static T json_value(const json & body, const std::string & key, const T & default_value) {
     // Fallback null to default value
@@ -72,12 +67,12 @@ enum error_type {
 };
 
 // thin wrapper around common_grammar_trigger with (de)serialization functions
-struct engine_grammar_trigger {
+struct server_grammar_trigger {
     common_grammar_trigger value;
 
-    engine_grammar_trigger() = default;
-    engine_grammar_trigger(const common_grammar_trigger & value) : value(value) {}
-    engine_grammar_trigger(const json & in) {
+    server_grammar_trigger() = default;
+    server_grammar_trigger(const common_grammar_trigger & value) : value(value) {}
+    server_grammar_trigger(const json & in) {
         value.type = (common_grammar_trigger_type) in.at("type").get<int>();
         value.value = in.at("value").get<std::string>();
         if (value.type == COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN) {
@@ -115,10 +110,10 @@ const char * get_media_marker();
 //
 
 // output modalities for architecture.output_modalities in GET /models
-std::vector<std::string> engine_model_output_modalities(common_decision_type decision_type);
+std::vector<std::string> server_model_output_modalities(common_decision_type decision_type);
 
 // architecture object of GET /models; shared by the direct server and the router
-json engine_model_architecture_json(
+json server_model_architecture_json(
         bool inp_image,
         bool inp_audio,
         bool inp_video,
@@ -147,14 +142,14 @@ bool are_lora_equal(
 std::vector<size_t> lora_get_enabled_ids(const std::vector<common_adapter_lora_info> & loras);
 
 //
-// engine_tokens
+// server_tokens
 //
 
 /**
- * engine_tokens is a helper to manage the input tokens and image for the server.
+ * server_tokens is a helper to manage the input tokens and image for the server.
  * it is made this way to simplify the logic of KV cache management.
  */
-struct engine_tokens {
+struct server_tokens {
     bool has_mtmd = false;
 
 private: // disallow accessing these members directly, risking out-of-sync
@@ -177,24 +172,24 @@ private: // disallow accessing these members directly, risking out-of-sync
     // map_idx_to_media will contain: {5, img0}, {8, img1}
 
 public:
-    engine_tokens() = default;
-    ~engine_tokens() = default;
+    server_tokens() = default;
+    ~server_tokens() = default;
 
     // Prevent copying
-    // TODO: engine_tokens should be copyable - remove this:
-    engine_tokens(const engine_tokens&) = delete;
-    engine_tokens& operator=(const engine_tokens&) = delete;
+    // TODO: server_tokens should be copyable - remove this:
+    server_tokens(const server_tokens&) = delete;
+    server_tokens& operator=(const server_tokens&) = delete;
 
     // Allow moving (usually implicitly generated if members are movable)
-    engine_tokens(engine_tokens&&) = default;
-    engine_tokens& operator=(engine_tokens&&) = default;
+    server_tokens(server_tokens&&) = default;
+    server_tokens& operator=(server_tokens&&) = default;
 
     // Allow accessing elements using [] operator
     llama_token operator[](size_t index) { return tokens[index]; }
     const llama_token& operator[](size_t index) const { return tokens[index]; }
 
-    engine_tokens(mtmd::input_chunks & mtmd_chunks, bool has_mtmd);
-    engine_tokens(const llama_tokens & tokens, bool has_mtmd);
+    server_tokens(mtmd::input_chunks & mtmd_chunks, bool has_mtmd);
+    server_tokens(const llama_tokens & tokens, bool has_mtmd);
 
     // for debugging
     std::string str() const;
@@ -221,7 +216,7 @@ public:
     void push_back_placeholder(const mtmd_input_chunk * chunk);
 
     // appends server tokens, updates the media map. copies media chunks.
-    void push_back(engine_tokens & tokens);
+    void push_back(server_tokens & tokens);
 
     // for compatibility with context shift and prompt truncation
     void insert(const llama_tokens & inp_tokens);
@@ -232,7 +227,7 @@ public:
     llama_tokens get_text_tokens() const;
 
     std::vector<char> serialize() const;
-    static engine_tokens deserialize(const llama_tokens & packed, bool has_mtmd);
+    static server_tokens deserialize(const llama_tokens & packed, bool has_mtmd);
 
     // for compatibility with speculative decoding
     void set_token(llama_pos pos, llama_token id);
@@ -250,7 +245,7 @@ public:
 
     std::string detokenize(const llama_context * ctx, bool special) const;
 
-    size_t get_common_prefix(const engine_tokens & b) const;
+    size_t get_common_prefix(const server_tokens & b) const;
 
     // split the tokens into message spans, skipping over media chunks
     common_chat_msg_spans find_message_spans(const common_chat_msg_delimiters & delims) const;
@@ -258,7 +253,7 @@ public:
     // check text token IDs and the mapping between media chunks and token ranges
     bool validate(const struct llama_context * ctx) const;
 
-    engine_tokens clone() const;
+    server_tokens clone() const;
 };
 
 
@@ -295,9 +290,9 @@ void handle_media(
         const std::string & url,
         const std::string & media_path);
 
-// process mtmd prompt, return the engine_tokens containing both text tokens and media chunks
-// if is_placeholder is true, the media chunk will be treated as placeholder for counting tokens; the output tokens are not usable for actual inference (e.g. for submitting a task to engine_queue)
-engine_tokens process_mtmd_prompt(
+// process mtmd prompt, return the server_tokens containing both text tokens and media chunks
+// if is_placeholder is true, the media chunk will be treated as placeholder for counting tokens; the output tokens are not usable for actual inference (e.g. for submitting a task to server_queue)
+server_tokens process_mtmd_prompt(
                                         mtmd_context * mctx,
                                         const std::string & prompt,
                                         const std::vector<raw_buffer> & files,
@@ -317,7 +312,7 @@ engine_tokens process_mtmd_prompt(
  * - "prompt": [[12, 34, 56], [78, 90, 12]]
  * - "prompt": [[12, 34, "string", 56, 78], [12, 34, 56], { "prompt_string": "string", "multimodal_data": [ "base64" ]}]
  */
-std::vector<engine_tokens> tokenize_input_prompts(
+std::vector<server_tokens> tokenize_input_prompts(
                                         const llama_vocab * vocab,
                                         mtmd_context * mctx,
                                         const json & json_prompt,
@@ -326,7 +321,7 @@ std::vector<engine_tokens> tokenize_input_prompts(
                                         const mtmd_helper_init_opt & init_opt);
 
 // tokenize a single prompt, see tokenize_input_prompts() for the supported shapes
-engine_tokens tokenize_input_subprompt(
+server_tokens tokenize_input_subprompt(
                                         const llama_vocab * vocab,
                                         mtmd_context * mctx,
                                         const json & json_prompt,
@@ -339,7 +334,7 @@ engine_tokens tokenize_input_subprompt(
 //
 
 // global server parameters for chat formatting / parsing
-struct engine_chat_params {
+struct server_chat_params {
     bool use_jinja;
     bool prefill_assistant;
     common_reasoning_format reasoning_format;
@@ -362,28 +357,28 @@ json oaicompat_completion_params_parse(const json & body);
 json oaicompat_chat_params_parse(
     const llama_vocab * vocab,
     json & body, /* openai api json semantics */
-    const engine_chat_params & opt,
+    const server_chat_params & opt,
     std::vector<raw_buffer> & out_files,
     common_chat_session & out_session);
 
 // used by /embeddings endpoint, content has the same format as a chat message content array
-engine_tokens tokenize_oai_content_array(
+server_tokens tokenize_oai_content_array(
     const llama_vocab * vocab,
     mtmd_context * mctx,
-    const engine_chat_params & opt,
+    const server_chat_params & opt,
     json content,
     bool add_special,
     bool parse_special,
     const mtmd_helper_init_opt & init_opt);
 
-// TODO: move it to engine-task.cpp
+// TODO: move it to server-task.cpp
 json format_embeddings_response_oaicompat(
     const json & request,
     const std::string & model_name,
     const json & embeddings,
     bool use_base64 = false);
 
-// TODO: move it to engine-task.cpp
+// TODO: move it to server-task.cpp
 json format_response_rerank(
         const json & request,
         const std::string & model_name,
@@ -396,14 +391,14 @@ json format_response_rerank(
 // stats and metrics
 //
 
-// shared between engine_slot and engine_task_result_*
-struct engine_slot_stats {
+// shared between server_slot and server_task_result_*
+struct server_slot_stats {
     uint64_t n_prompt_cached    = 0;
     uint64_t n_prompt_processed = 0;
     uint64_t n_gen              = 0;
 
     // speculative decoding stats
-    // note: the per-position breakdown lives in engine_slot, it is not needed in a task result
+    // note: the per-position breakdown lives in server_slot, it is not needed in a task result
     uint64_t n_draft_tokens      = 0;
     uint64_t n_draft_accepted    = 0;
     uint64_t n_draft_verif_steps = 0;
@@ -483,9 +478,9 @@ struct engine_slot_stats {
     json to_json() const;
 };
 
-// shared between engine_context_impl and engine_task_result_*
-// unlike engine_slot_stats, engine_metrics is server-global and cumulative, not tied to a slot
-struct engine_metrics {
+// shared between server_context_impl and server_task_result_*
+// unlike server_slot_stats, server_metrics is server-global and cumulative, not tied to a slot
+struct server_metrics {
     int64_t t_start = 0;
 
     struct bucket {
@@ -573,7 +568,7 @@ bool is_valid_utf8(const std::string & str);
 
 //
 // formatting output responses
-// TODO: move these to engine-task.cpp
+// TODO: move these to server-task.cpp
 //
 
 llama_tokens format_prompt_infill(
@@ -588,10 +583,126 @@ llama_tokens format_prompt_infill(
         const llama_tokens & tokens_prompt);
 
 // format rerank task: [BOS]query[EOS][SEP]doc[EOS].
-engine_tokens format_prompt_rerank(
+server_tokens format_prompt_rerank(
         const struct llama_model * model,
         const struct llama_vocab * vocab,
         mtmd_context * mctx,
         const std::string & query,
         const std::string & doc,
         const mtmd_helper_init_opt & init_opt);
+
+// simple implementation of a pipe
+// used for streaming data between threads
+template<typename T>
+struct server_pipe {
+    std::mutex mutex;
+    std::condition_variable cv;
+    std::queue<T> queue;
+    std::atomic<bool> writer_closed{false};
+    std::atomic<bool> reader_closed{false};
+
+    // 0 = unbounded (default)
+    // > 0, write() drops the oldest item once the queue is full
+    size_t max_size = 0;
+
+    void close_write() {
+        writer_closed.store(true, std::memory_order_relaxed);
+        cv.notify_all();
+    }
+
+    void close_read() {
+        reader_closed.store(true, std::memory_order_relaxed);
+        cv.notify_all();
+    }
+
+    // close_on_stop = true: should_stop means the reader is gone for good, so the writer is told the pipe is broken.
+    // close_on_stop = false: should_stop is a per-read deadline and further reads still come, so the pipe stays usable.
+    bool read(T & output, const std::function<bool()> & should_stop, bool close_on_stop = true) {
+        std::unique_lock<std::mutex> lk(mutex);
+        constexpr auto poll_interval = std::chrono::milliseconds(500);
+        while (true) {
+            if (!queue.empty()) {
+                output = std::move(queue.front());
+                queue.pop();
+                return true;
+            }
+            if (writer_closed.load()) {
+                return false; // clean EOF
+            }
+            if (should_stop && should_stop()) { // a null should_stop means "never stop"
+                if (close_on_stop) {
+                    close_read(); // signal broken pipe to writer
+                }
+                return false; // cancelled / deadline reached
+            }
+            cv.wait_for(lk, poll_interval);
+        }
+    }
+
+    bool write(T && data) {
+        std::lock_guard<std::mutex> lk(mutex);
+        if (reader_closed.load()) {
+            return false; // broken pipe
+        }
+        if (max_size > 0) {
+            while (queue.size() >= max_size) {
+                queue.pop(); // drop oldest to stay bounded
+            }
+        }
+        queue.push(std::move(data));
+        cv.notify_one();
+        return true;
+    }
+};
+
+// a child server writes its state commands to stdout and its logs to stderr
+enum server_subproc_stream {
+    SERVER_SUBPROC_STDOUT,
+    SERVER_SUBPROC_STDERR,
+    SERVER_SUBPROC_STREAMS,
+};
+
+// gives the current stdout to the caller as a stream of its own, and sends everything else written to stdout to stderr
+FILE * server_reserve_stdout();
+
+// wrapper around common_subproc to manage a child server process
+// mainly used by router mode
+struct server_subproc {
+    common_subproc sproc;
+    std::atomic<bool> stopped{false}; // set by the monitor once the process exited and was reaped
+
+    bool is_alive() { return sproc.alive(); }
+    void terminate() { sproc.terminate(); }
+    int  join() { return sproc.join(); }
+
+    // true if both output pipes of the child are available (call after create())
+    bool has_output();
+
+    // true once both output pipes are closed
+    bool output_closed() const;
+
+    // non-blocking read from one output pipe
+    // returns the number of bytes read, 0 when nothing is available, -1 when the pipe is closed or broken
+    int read_output(server_subproc_stream stream, char * buf, size_t len);
+
+    // wait until one of a set of children has output, wake() is called, or a timeout passes
+    struct waiter {
+        waiter();
+        ~waiter();
+
+        // thread-safe; on Windows this is a no-op, wait() returns within 50 ms anyway
+        void wake();
+
+        // timeout_ms < 0 waits until data or wake(); ready[i] is set for each child with data on an open pipe (or a broken pipe)
+        void wait(const std::vector<server_subproc *> & procs, std::vector<bool> & ready, int64_t timeout_ms);
+
+    private:
+#ifndef _WIN32
+        intptr_t wake_fd[2] = { -1, -1 }; // POSIX self-pipe
+#endif
+    };
+
+private:
+    intptr_t out_handles[SERVER_SUBPROC_STREAMS] = { -1, -1 }; // fd on POSIX, HANDLE on Windows; taken lazily from sproc
+    bool     out_closed [SERVER_SUBPROC_STREAMS] = { false, false };
+};

tools/server/server-context.h

@@ -0,0 +1,196 @@
+#pragma once
+
+#include "server-http.h"
+#include "server-task.h"
+#include "server-queue.h"
+
+#include "json.h"
+
+#include <cstddef>
+#include <memory>
+#include <mutex>
+#include <set>
+
+struct server_context_impl; // private implementation
+
+struct server_context_meta {
+    std::string build_info;
+    std::string model_name;
+    std::set<std::string> model_aliases;
+    std::set<std::string> model_tags;
+    std::string model_path;
+    std::vector<std::string> model_output_modalities; // output modalities for GET /models
+    bool has_mtmd;
+    bool has_inp_image;
+    bool has_inp_audio;
+    bool has_inp_video;
+    json json_ui_settings;
+    int slot_n_ctx;
+    enum llama_pooling_type pooling_type;
+
+    // chat params
+    server_chat_params & chat_params;
+    std::map<std::string, bool> chat_template_caps;
+
+    // tokens
+    std::string bos_token_str;
+    std::string eos_token_str;
+    llama_token fim_pre_token;
+    llama_token fim_sub_token;
+    llama_token fim_mid_token;
+    llama_token fim_pad_token;
+    llama_token fim_rep_token;
+    llama_token fim_sep_token;
+
+    // sampling
+    std::vector<llama_logit_bias> logit_bias_eog;
+
+    // model meta
+    enum llama_vocab_type model_vocab_type;
+    int32_t model_vocab_n_tokens;
+    int32_t model_n_ctx_train;
+    int32_t model_n_embd_inp;
+    uint64_t model_n_params;
+    uint64_t model_size;
+    std::string model_ftype;
+};
+
+enum server_state {
+    SERVER_STATE_DOWNLOADING,
+    SERVER_STATE_LOADING,
+    SERVER_STATE_READY,
+    SERVER_STATE_SLEEPING,
+};
+
+static std::string server_state_to_str(server_state state) {
+    switch (state) {
+        case SERVER_STATE_DOWNLOADING: return "downloading";
+        case SERVER_STATE_LOADING:     return "loading";
+        case SERVER_STATE_READY:       return "ready";
+        case SERVER_STATE_SLEEPING:    return "sleeping";
+        default: GGML_ASSERT(false && "invalid server_state");
+    }
+}
+
+static server_state server_state_from_str(const std::string & str) {
+    if (str == "downloading") return SERVER_STATE_DOWNLOADING;
+    if (str == "loading")     return SERVER_STATE_LOADING;
+    if (str == "ready")       return SERVER_STATE_READY;
+    if (str == "sleeping")    return SERVER_STATE_SLEEPING;
+    GGML_ASSERT(false && "invalid server_state string");
+}
+
+using server_state_callback_t = std::function<void(server_state, json /* payload */)>;
+
+struct server_context {
+    std::unique_ptr<server_context_impl> impl;
+
+    server_context();
+    ~server_context();
+
+    // load the model and initialize llama_context
+    // returns true on success
+    bool load_model(common_params & params);
+
+    // this function will block main thread until termination
+    void start_loop();
+
+    // terminate main loop (will unblock start_loop)
+    void terminate();
+
+    // get the underlaying llama_context, can return nullptr if sleeping
+    // not thread-safe, should only be used from the main thread
+    llama_context * get_llama_context() const;
+
+    // get a new response reader, used by CLI application
+    server_response_reader get_response_reader();
+
+    // get server metadata (read-only), can only be called after load_model()
+    // not thread-safe, should only be used from the main thread
+    server_context_meta get_meta() const;
+
+    // note: must be set before load_model() is called
+    void set_state_callback(server_state_callback_t callback);
+};
+
+
+// forward declarations
+struct server_res_generator;
+
+struct server_routes {
+    server_routes(const common_params & params, server_context & ctx_server);
+
+    void init_routes();
+
+    // note: this is not thread-safe and can only when ctx_http.is_ready is false
+    void update_meta(const server_context & ctx_server) {
+        this->meta = std::make_unique<server_context_meta>(ctx_server.get_meta());
+    }
+
+    // handlers using lambda function, so that they can capture `this` without `std::bind`
+    // they won't be called until ctx_http.is_ready is set to true
+    server_http_context::handler_t get_health;
+    server_http_context::handler_t get_metrics;
+    server_http_context::handler_t get_slots;
+    server_http_context::handler_t post_slots;
+    server_http_context::handler_t get_props;
+    server_http_context::handler_t post_props;
+    server_http_context::handler_t post_infill;
+    server_http_context::handler_t post_completions;
+    server_http_context::handler_t post_completions_oai;
+    server_http_context::handler_t post_chat_completions;
+    server_http_context::handler_t post_chat_completions_tok;
+    server_http_context::handler_t post_control;
+    server_http_context::handler_t post_responses_oai;
+    server_http_context::handler_t post_responses_tok_oai;
+    server_http_context::handler_t post_transcriptions_oai;
+    server_http_context::handler_t post_anthropic_messages;
+    server_http_context::handler_t post_anthropic_count_tokens;
+    server_http_context::handler_t post_apply_template;
+    server_http_context::handler_t get_models;
+    server_http_context::handler_t post_tokenize;
+    server_http_context::handler_t post_detokenize;
+    server_http_context::handler_t post_embeddings;
+    server_http_context::handler_t post_embeddings_oai;
+    server_http_context::handler_t post_rerank;
+    server_http_context::handler_t post_systemone;
+    server_http_context::handler_t get_lora_adapters;
+    server_http_context::handler_t post_lora_adapters;
+
+    // to be used in router mode
+    json get_model_info() const;
+
+private:
+    std::unique_ptr<server_res_generator> handle_completions_impl(
+            const server_http_req & req,
+            server_task_type type,
+            const json & data,
+            const std::vector<raw_buffer> & files,
+            task_response_type res_type,
+            const common_chat_session & chat_session = {});
+    std::unique_ptr<server_res_generator> handle_slots_save(const server_http_req & req, int id_slot);
+    std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
+    std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
+    std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
+    std::unique_ptr<server_res_generator> handle_count_tokens(const server_http_req & req, task_response_type res_type);
+
+    // using unique_ptr to allow late initialization of const
+    std::unique_ptr<const server_context_meta> meta;
+
+    const common_params & params;
+    server_context_impl & ctx_server;
+
+    server_queue & queue_tasks;
+    server_response & queue_results;
+    std::unique_ptr<server_res_generator> create_response(bool bypass_sleep = false);
+
+    // cached responses, to be used during sleep
+    std::mutex     mutex_cache;
+    json           cached_models  = nullptr;
+    json           cached_props   = nullptr;
+    server_metrics cached_metrics;
+    // set when a scrape during sleep already reported the throughput buckets
+    bool           should_reset_buckets = false;
+    // call right before sleep to update the cached responses
+    void update_cached_responses(bool is_sleeping);
+};

tools/server/server-http.cpp

@@ -1,7 +1,7 @@
 #include "common.h"
 #include "http.h"
 #include "server-http.h"
-#include "engine-common.h"
+#include "server-common.h"
 #include "ui.h"
 
 #include <cpp-httplib/httplib.h>

tools/server/server-http.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#include "engine-common.h" // raw_buffer, uploaded_file
-
 #include <atomic>
 #include <functional>
 #include <map>
@@ -40,6 +38,13 @@ struct server_http_res {
 // unique pointer, used by set_chunked_content_provider
 // httplib requires the stream provider to be stored in heap
 using server_http_res_ptr = std::unique_ptr<server_http_res>;
+using raw_buffer = std::vector<uint8_t>;
+
+struct uploaded_file {
+    raw_buffer data;
+    std::string filename;
+    std::string content_type;
+};
 
 struct server_http_req {
     std::map<std::string, std::string> params; // path_params + query_params

tools/server/server-mcp.h

@@ -1,7 +1,6 @@
 #pragma once
 
-#include "engine-common.h"
-#include "server-subproc.h"
+#include "server-common.h"
 
 #include <atomic>
 #include <chrono>

tools/server/server-models.cpp

@@ -1,7 +1,7 @@
-#include "engine-common.h"
+#include "server-common.h"
 #include "http.h"
 #include "server-models.h"
-#include "engine-context.h"
+#include "server-context.h"
 #include "server-stream.h"
 
 #include "build-info.h"
@@ -539,7 +539,7 @@ void server_model_meta::update_args(common_preset_context & ctx_preset, std::str
 
 void server_model_meta::update_caps(const common_params & base) {
     // reset to the default so a failed refresh cannot keep old values
-    architecture = engine_model_architecture_json(false, false, false, {"text"});
+    architecture = server_model_architecture_json(false, false, false, {"text"});
     n_ctx_train  = 0;
 
     // resolve the model file offline; do not download
@@ -569,7 +569,7 @@ void server_model_meta::update_caps(const common_params & base) {
     std::vector<std::string> output_modalities = {"text"};
     if (!params.model.path.empty()) {
         const common_gguf_info info = common_get_gguf_info(params.model.path);
-        output_modalities = engine_model_output_modalities(info.decision_type);
+        output_modalities = server_model_output_modalities(info.decision_type);
         n_ctx_train       = info.n_ctx_train;
     }
 
@@ -587,7 +587,7 @@ void server_model_meta::update_caps(const common_params & base) {
     }
 
     // offline discovery cannot see video; a loaded model reports it
-    architecture = engine_model_architecture_json(inp_image, inp_audio, false, output_modalities);
+    architecture = server_model_architecture_json(inp_image, inp_audio, false, output_modalities);
 }
 
 //
@@ -687,7 +687,7 @@ void server_models::add_model(server_model_meta && meta) {
 }
 
 void server_models::notify_sse(const std::string & event, const std::string & model_id, const json & data) {
-    std::unique_ptr<engine_task_result_router> result = std::make_unique<engine_task_result_router>();
+    std::unique_ptr<server_task_result_router> result = std::make_unique<server_task_result_router>();
     result->data = {
         {"model", model_id},
         {"event", event},
@@ -1660,12 +1660,12 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co
 }
 
 void server_models::handle_child_state(const std::string & name, const std::string & raw_input) {
-    engine_state state;
+    server_state state;
     json payload;
 
     try {
         json data = json::parse(raw_input.substr(strlen(CMD_CHILD_TO_ROUTER_STATE)));
-        state = engine_state_from_str(json_value(data, "state", std::string()));
+        state = server_state_from_str(json_value(data, "state", std::string()));
         payload = json_value(data, "payload", json{});
     } catch (const std::exception & e) {
         SRV_ERR("failed to parse child state update for name=%s: %s\n", name.c_str(), e.what());
@@ -1673,7 +1673,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri
     }
 
     switch (state) {
-        case ENGINE_STATE_DOWNLOADING:
+        case SERVER_STATE_DOWNLOADING:
             {
                 std::string result = json_value(payload, "result", std::string());
                 std::string url    = json_value(payload, "url",    std::string());
@@ -1698,7 +1698,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri
                     update_download_progress(name, p, false);
                 }
             } break;
-        case ENGINE_STATE_LOADING:
+        case SERVER_STATE_LOADING:
             {
                 update_status(name, {
                     SERVER_MODEL_STATUS_LOADING,
@@ -1707,7 +1707,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri
                     payload,
                 });
             } break;
-        case ENGINE_STATE_READY:
+        case SERVER_STATE_READY:
             {
                 update_status(name, {
                     SERVER_MODEL_STATUS_LOADED,
@@ -1717,7 +1717,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri
                     {}, // reset progress info
                 });
             } break;
-        case ENGINE_STATE_SLEEPING:
+        case SERVER_STATE_SLEEPING:
             {
                 update_status(name, { SERVER_MODEL_STATUS_SLEEPING });
             } break;
@@ -1784,7 +1784,7 @@ struct server_download_state : public common_download_callback {
             {"downloaded", p.downloaded},
             {"total", p.total},
         };
-        self->notify_to_router(engine_state_to_str(ENGINE_STATE_DOWNLOADING), data);
+        self->notify_to_router(server_state_to_str(SERVER_STATE_DOWNLOADING), data);
     }
     void on_start(const common_download_progress & p) override {
         on_progress(p);
@@ -1820,7 +1820,7 @@ int server_child::run_download(common_params & params) {
 
     bool ok = dl.run(params);
 
-    notify_to_router(engine_state_to_str(ENGINE_STATE_DOWNLOADING), {
+    notify_to_router(server_state_to_str(SERVER_STATE_DOWNLOADING), {
         {"result", ok ? "download_finished" : "download_failed"},
     });
 
@@ -1874,12 +1874,12 @@ void server_child::notify_to_router(const std::string & state, const json & payl
 // server_models_routes
 //
 
-// RAII wrapper similar to engine_response_reader, but doesn't use engine_queue
+// RAII wrapper similar to server_response_reader, but doesn't use server_queue
 static std::atomic<int> sse_client_id_counter = 0;
 struct server_models_sse_client {
-    engine_response & queue_results;
+    server_response & queue_results;
     int client_id;
-    server_models_sse_client(engine_response & q)
+    server_models_sse_client(server_response & q)
             : queue_results(q), client_id(sse_client_id_counter.fetch_add(1, std::memory_order_relaxed)) {
         SRV_DBG("new SSE client connected, assigned client_id=%d\n", client_id);
         queue_results.add_waiting_task_id(client_id);
@@ -1891,10 +1891,10 @@ struct server_models_sse_client {
 
     // return nullptr if should_stop() is true before receiving a result
     // note: if one error is received, it will stop further processing and return error result
-    engine_task_result_ptr next(const std::function<bool()> & should_stop) {
+    server_task_result_ptr next(const std::function<bool()> & should_stop) {
         while (true) {
             static const int http_polling_seconds = 1; // check should_stop every 1 second
-            engine_task_result_ptr result = queue_results.recv_with_timeout({client_id}, http_polling_seconds);
+            server_task_result_ptr result = queue_results.recv_with_timeout({client_id}, http_polling_seconds);
             if (result == nullptr) {
                 // timeout, check stop condition
                 if (should_stop()) {

tools/server/server-models.h

@@ -3,10 +3,9 @@
 #include "common.h"
 #include "download.h"
 #include "preset.h"
-#include "engine-common.h"
+#include "server-common.h"
 #include "server-http.h"
-#include "engine-queue.h"
-#include "server-subproc.h"
+#include "server-queue.h"
 
 #include <mutex>
 #include <condition_variable>
@@ -86,7 +85,7 @@ struct server_model_meta {
     int exit_code = 0; // exit code of the model instance process (only valid if status == FAILED)
     int stop_timeout = 0; // seconds to wait before force-killing the model instance during shutdown
     bool hidden = false; // hidden from GET /models, but still accept if requested
-    json architecture = engine_model_architecture_json(false, false, false, {"text"});
+    json architecture = server_model_architecture_json(false, false, false, {"text"});
     uint32_t n_ctx_train = 0; // trained context, read from the GGUF metadata; 0 when unknown
 
     bool is_ready() const {
@@ -238,7 +237,7 @@ public:
     server_models(const common_params & params, int argc, char ** argv);
     ~server_models();
 
-    engine_response sse; // for real-time updates via SSE endpoint
+    server_response sse; // for real-time updates via SSE endpoint
 
     // (re-)load the list of models from various sources and prepare the metadata mapping
     // - if this is called the first time, simply populate the metadata

tools/server/server-queue.cpp

@@ -1,5 +1,5 @@
-#include "engine-task.h"
-#include "engine-queue.h"
+#include "server-task.h"
+#include "server-queue.h"
 
 #include "log.h"
 
@@ -18,18 +18,18 @@
 #define RES_DBG(fmt, ...) LOG_DBG("res  %12.*s: " fmt, 12, __func__, __VA_ARGS__)
 
 //
-// engine_queue
+// server_queue
 //
 
-static bool task_resets_idle_timer(engine_task_type type) {
-    return type != ENGINE_TASK_TYPE_METRICS;
+static bool task_resets_idle_timer(server_task_type type) {
+    return type != SERVER_TASK_TYPE_METRICS;
 }
 
-int engine_queue::post(engine_task && task, bool front) {
+int server_queue::post(server_task && task, bool front) {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     GGML_ASSERT(task.id != -1);
     // if this is cancel task make sure to clean up pending tasks
-    if (task.type == ENGINE_TASK_TYPE_CANCEL) {
+    if (task.type == SERVER_TASK_TYPE_CANCEL) {
         cleanup_pending_task(task.id_target);
     }
     const int  task_id     = task.id;
@@ -47,7 +47,7 @@ int engine_queue::post(engine_task && task, bool front) {
     return task_id;
 }
 
-int engine_queue::post(std::vector<engine_task> && tasks, bool front) {
+int server_queue::post(std::vector<server_task> && tasks, bool front) {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     bool reset_timer = false;
     for (auto & task : tasks) {
@@ -55,7 +55,7 @@ int engine_queue::post(std::vector<engine_task> && tasks, bool front) {
             task.id = id++;
         }
         // if this is cancel task make sure to clean up pending tasks
-        if (task.type == ENGINE_TASK_TYPE_CANCEL) {
+        if (task.type == SERVER_TASK_TYPE_CANCEL) {
             cleanup_pending_task(task.id_target);
         }
         reset_timer |= task_resets_idle_timer(task.type);
@@ -73,7 +73,7 @@ int engine_queue::post(std::vector<engine_task> && tasks, bool front) {
     return 0;
 }
 
-void engine_queue::defer(engine_task && task) {
+void server_queue::defer(server_task && task) {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     QUE_DBG("defer task, id = %d\n", task.id);
     queue_tasks_deferred.push_back(std::move(task));
@@ -81,13 +81,13 @@ void engine_queue::defer(engine_task && task) {
     condition_tasks.notify_one();
 }
 
-int engine_queue::get_new_id() {
+int server_queue::get_new_id() {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     int new_id = id++;
     return new_id;
 }
 
-void engine_queue::pop_deferred_task(int id_slot) {
+void server_queue::pop_deferred_task(int id_slot) {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     if (!queue_tasks_deferred.empty()) {
         // try to find a task that uses the specified slot
@@ -112,7 +112,7 @@ void engine_queue::pop_deferred_task(int id_slot) {
     condition_tasks.notify_one();
 }
 
-void engine_queue::wait_until_no_sleep() {
+void server_queue::wait_until_no_sleep() {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     if (!sleeping) {
         return;
@@ -129,13 +129,13 @@ void engine_queue::wait_until_no_sleep() {
     }
 }
 
-void engine_queue::terminate() {
+void server_queue::terminate() {
     std::unique_lock<std::mutex> lock(mutex_tasks);
     running = false;
     condition_tasks.notify_all();
 }
 
-bool engine_queue::process_new_tasks(bool is_yielding) {
+bool server_queue::process_new_tasks(bool is_yielding) {
     while (true) {
         std::unique_lock<std::mutex> lock(mutex_tasks);
         if (!running) {
@@ -145,7 +145,7 @@ bool engine_queue::process_new_tasks(bool is_yielding) {
         if (queue_tasks.empty()) {
             return false;
         }
-        engine_task task = std::move(queue_tasks.front());
+        server_task task = std::move(queue_tasks.front());
         queue_tasks.pop_front();
         lock.unlock();
 
@@ -160,7 +160,7 @@ bool engine_queue::process_new_tasks(bool is_yielding) {
     }
 }
 
-void engine_queue::worker_loop() {
+void server_queue::worker_loop() {
     while (true) {
         {
             std::unique_lock<std::mutex> lock(mutex_tasks);
@@ -206,7 +206,7 @@ void engine_queue::worker_loop() {
     }
 }
 
-void engine_queue::worker_stop() {
+void server_queue::worker_stop() {
     if (!worker.thread.joinable()) {
         return;
     }
@@ -219,7 +219,7 @@ void engine_queue::worker_stop() {
     worker.thread.join();
 }
 
-void engine_queue::yield_to_queue(std::function<void()> && work) {
+void server_queue::yield_to_queue(std::function<void()> && work) {
     GGML_ASSERT(worker.thread.joinable() && "yield_to_queue() requires start_loop() to be running");
 
     QUE_DBG("%s", "yielding to queue\n");
@@ -275,7 +275,7 @@ void engine_queue::yield_to_queue(std::function<void()> && work) {
     }
 }
 
-void engine_queue::start_loop(int64_t idle_sleep_ms) {
+void server_queue::start_loop(int64_t idle_sleep_ms) {
     running = true;
     time_last_task = ggml_time_ms();
 
@@ -365,9 +365,9 @@ void engine_queue::start_loop(int64_t idle_sleep_ms) {
     worker_stop();
 }
 
-void engine_queue::cleanup_pending_task(int id_target) {
+void server_queue::cleanup_pending_task(int id_target) {
     // no need lock because this is called exclusively by post()
-    auto rm_func = [id_target](const engine_task & task) {
+    auto rm_func = [id_target](const server_task & task) {
         return task.id == id_target;
     };
     queue_tasks.erase(
@@ -383,17 +383,17 @@ void engine_queue::cleanup_pending_task(int id_target) {
 }
 
 //
-// engine_response
+// server_response
 //
 
-void engine_response::add_waiting_task_id(int id_task) {
+void server_response::add_waiting_task_id(int id_task) {
     RES_DBG("add task %d to waiting list. current waiting = %d (before add)\n", id_task, (int) waiting_task_ids.size());
 
     std::unique_lock<std::mutex> lock(mutex_results);
     waiting_task_ids.insert(id_task);
 }
 
-void engine_response::add_waiting_task_ids(const std::unordered_set<int> & id_tasks) {
+void server_response::add_waiting_task_ids(const std::unordered_set<int> & id_tasks) {
     std::unique_lock<std::mutex> lock(mutex_results);
 
     for (const auto & id_task : id_tasks) {
@@ -402,20 +402,20 @@ void engine_response::add_waiting_task_ids(const std::unordered_set<int> & id_ta
     }
 }
 
-void engine_response::remove_waiting_task_id(int id_task) {
+void server_response::remove_waiting_task_id(int id_task) {
     RES_DBG("remove task %d from waiting list. current waiting = %d (before remove)\n", id_task, (int) waiting_task_ids.size());
 
     std::unique_lock<std::mutex> lock(mutex_results);
     waiting_task_ids.erase(id_task);
     // make sure to clean up all pending results
     queue_results.erase(
-        std::remove_if(queue_results.begin(), queue_results.end(), [id_task](const engine_task_result_ptr & res) {
+        std::remove_if(queue_results.begin(), queue_results.end(), [id_task](const server_task_result_ptr & res) {
             return res->id == id_task;
         }),
         queue_results.end());
 }
 
-void engine_response::remove_waiting_task_ids(const std::unordered_set<int> & id_tasks) {
+void server_response::remove_waiting_task_ids(const std::unordered_set<int> & id_tasks) {
     std::unique_lock<std::mutex> lock(mutex_results);
 
     for (const auto & id_task : id_tasks) {
@@ -424,7 +424,7 @@ void engine_response::remove_waiting_task_ids(const std::unordered_set<int> & id
     }
 }
 
-engine_task_result_ptr engine_response::recv(const std::unordered_set<int> & id_tasks) {
+server_task_result_ptr server_response::recv(const std::unordered_set<int> & id_tasks) {
     while (true) {
         std::unique_lock<std::mutex> lock(mutex_results);
         condition_results.wait(lock, [&]{
@@ -437,7 +437,7 @@ engine_task_result_ptr engine_response::recv(const std::unordered_set<int> & id_
 
         for (size_t i = 0; i < queue_results.size(); i++) {
             if (id_tasks.find(queue_results[i]->id) != id_tasks.end()) {
-                engine_task_result_ptr res = std::move(queue_results[i]);
+                server_task_result_ptr res = std::move(queue_results[i]);
                 queue_results.erase(queue_results.begin() + i);
                 return res;
             }
@@ -447,13 +447,13 @@ engine_task_result_ptr engine_response::recv(const std::unordered_set<int> & id_
     // should never reach here
 }
 
-engine_task_result_ptr engine_response::recv_with_timeout(const std::unordered_set<int> & id_tasks, int timeout) {
+server_task_result_ptr server_response::recv_with_timeout(const std::unordered_set<int> & id_tasks, int timeout) {
     while (true) {
         std::unique_lock<std::mutex> lock(mutex_results);
 
         for (int i = 0; i < (int) queue_results.size(); i++) {
             if (id_tasks.find(queue_results[i]->id) != id_tasks.end()) {
-                engine_task_result_ptr res = std::move(queue_results[i]);
+                server_task_result_ptr res = std::move(queue_results[i]);
                 queue_results.erase(queue_results.begin() + i);
                 return res;
             }
@@ -472,12 +472,12 @@ engine_task_result_ptr engine_response::recv_with_timeout(const std::unordered_s
     // should never reach here
 }
 
-engine_task_result_ptr engine_response::recv(int id_task) {
+server_task_result_ptr server_response::recv(int id_task) {
     std::unordered_set<int> id_tasks = {id_task};
     return recv(id_tasks);
 }
 
-void engine_response::send(engine_task_result_ptr && result) {
+void server_response::send(server_task_result_ptr && result) {
     RES_DBG("sending result for task id = %d\n", result->id);
 
     std::unique_lock<std::mutex> lock(mutex_results);
@@ -492,27 +492,27 @@ void engine_response::send(engine_task_result_ptr && result) {
     }
 }
 
-void engine_response::broadcast(engine_task_result_ptr && result) {
+void server_response::broadcast(server_task_result_ptr && result) {
     std::unique_lock<std::mutex> lock(mutex_results);
     for (const auto & id_task : waiting_task_ids) {
         RES_DBG("task id = %d pushed to result queue\n", id_task);
-        engine_task_result_ptr res_copy(result->clone());
+        server_task_result_ptr res_copy(result->clone());
         res_copy->id = id_task; // override id with target task id
         queue_results.emplace_back(std::move(res_copy));
     }
     condition_results.notify_all();
 }
 
-void engine_response::terminate() {
+void server_response::terminate() {
     running = false;
     condition_results.notify_all();
 }
 
 //
-// engine_response_reader
+// server_response_reader
 //
 
-void engine_response_reader::post_task(engine_task && task, bool front) {
+void server_response_reader::post_task(server_task && task, bool front) {
     GGML_ASSERT(id_tasks.empty() && "post_task() can only be called once per reader");
     GGML_ASSERT(!task.is_parent() && "not supported, use post_tasks() instead");
     task.index = 0;
@@ -522,9 +522,9 @@ void engine_response_reader::post_task(engine_task && task, bool front) {
     queue_tasks.post(std::move(task), front);
 }
 
-void engine_response_reader::post_tasks(std::vector<engine_task> && tasks, const common_chat_session & session, bool front) {
+void server_response_reader::post_tasks(std::vector<server_task> && tasks, const common_chat_session & session, bool front) {
     GGML_ASSERT(id_tasks.empty() && "post_tasks() can only be called once per reader");
-    id_tasks = engine_task::get_list_id(tasks);
+    id_tasks = server_task::get_list_id(tasks);
     states.reserve(tasks.size());
     size_t index = 0;
     for (auto & task : tasks) {
@@ -541,15 +541,15 @@ void engine_response_reader::post_tasks(std::vector<engine_task> && tasks, const
     queue_tasks.post(std::move(tasks), front);
 }
 
-bool engine_response_reader::has_next() const {
+bool server_response_reader::has_next() const {
     return !cancelled && received_count < id_tasks.size();
 }
 
 // return nullptr if should_stop() is true before receiving a result
 // note: if one error is received, it will stop further processing and return error result
-engine_task_result_ptr engine_response_reader::next(const std::function<bool()> & should_stop) {
+server_task_result_ptr server_response_reader::next(const std::function<bool()> & should_stop) {
     while (true) {
-        engine_task_result_ptr result = queue_results.recv_with_timeout(id_tasks, polling_interval_seconds);
+        server_task_result_ptr result = queue_results.recv_with_timeout(id_tasks, polling_interval_seconds);
         if (result == nullptr) {
             // timeout, check stop condition
             if (should_stop()) {
@@ -577,7 +577,7 @@ engine_task_result_ptr engine_response_reader::next(const std::function<bool()>
     // should not reach here
 }
 
-engine_response_reader::batch_response engine_response_reader::wait_for_all(const std::function<bool()> & should_stop) {
+server_response_reader::batch_response server_response_reader::wait_for_all(const std::function<bool()> & should_stop) {
     batch_response batch_res;
     batch_res.results.clear();
     batch_res.results.resize(id_tasks.size());
@@ -599,16 +599,16 @@ engine_response_reader::batch_response engine_response_reader::wait_for_all(cons
     return batch_res;
 }
 
-void engine_response_reader::stop() {
+void server_response_reader::stop() {
     queue_results.remove_waiting_task_ids(id_tasks);
     if (has_next() && !cancelled) {
         // if tasks is not finished yet, cancel them
         cancelled = true;
-        std::vector<engine_task> cancel_tasks;
+        std::vector<server_task> cancel_tasks;
         cancel_tasks.reserve(id_tasks.size());
         for (const auto & id_task : id_tasks) {
             SRV_WRN("cancel task, id_task = %d\n", id_task);
-            engine_task task(ENGINE_TASK_TYPE_CANCEL);
+            server_task task(SERVER_TASK_TYPE_CANCEL);
             task.id_target = id_task;
             queue_results.remove_waiting_task_id(id_task);
             cancel_tasks.push_back(std::move(task));

tools/server/server-queue.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "engine-task.h"
+#include "server-task.h"
 
 #include <condition_variable>
 #include <deque>
@@ -11,8 +11,8 @@
 #include <unordered_set>
 
 // struct for managing server tasks
-// in most cases, use engine_response_reader to post new tasks and retrieve results
-struct engine_queue {
+// in most cases, use server_response_reader to post new tasks and retrieve results
+struct server_queue {
 private:
     int id = 0;
     bool running  = false;
@@ -21,11 +21,11 @@ private:
     int64_t time_last_task = 0;
 
     // queues
-    std::deque<engine_task> queue_tasks;
-    std::deque<engine_task> queue_tasks_deferred;
+    std::deque<server_task> queue_tasks;
+    std::deque<server_task> queue_tasks_deferred;
     // tasks declined while yielding, put back in queue_tasks once the yield is done
     // note: kept as a member so that cleanup_pending_task() can also reach them
-    std::deque<engine_task> queue_tasks_unhandled;
+    std::deque<server_task> queue_tasks_unhandled;
 
     std::mutex mutex_tasks;
     std::condition_variable condition_tasks;
@@ -42,21 +42,21 @@ private:
     worker_t worker;
 
     // callback functions
-    std::function<bool(engine_task &&, bool)> callback_new_task;
+    std::function<bool(server_task &&, bool)> callback_new_task;
     std::function<void(void)>                 callback_update_slots;
     std::vector<std::function<void(bool)>>    callback_sleeping_state;
 
 public:
-    ~engine_queue() { worker_stop(); }
+    ~server_queue() { worker_stop(); }
 
     // Add a new task to the end of the queue
-    int post(engine_task && task, bool front = false);
+    int post(server_task && task, bool front = false);
 
     // multi-task version of post()
-    int post(std::vector<engine_task> && tasks, bool front = false);
+    int post(std::vector<server_task> && tasks, bool front = false);
 
     // Add a new task, but defer until one slot is available
-    void defer(engine_task && task);
+    void defer(server_task && task);
 
     // Get the next id for creating a new task
     int get_new_id();
@@ -118,7 +118,7 @@ public:
     // only then may the callback return false to decline the task, and it must leave it
     // untouched, so that it can be put back in the queue later
     // note: while yielding, the callback runs on worker thread, not main thread
-    void on_new_task(std::function<bool(engine_task &&, bool)> callback) {
+    void on_new_task(std::function<bool(server_task &&, bool)> callback) {
         callback_new_task = std::move(callback);
     }
 
@@ -150,8 +150,8 @@ private:
 };
 
 // struct for managing server responses
-// in most cases, use engine_response_reader to retrieve results
-struct engine_response {
+// in most cases, use server_response_reader to retrieve results
+struct server_response {
 private:
     bool running = true;
 
@@ -159,7 +159,7 @@ private:
     std::unordered_set<int> waiting_task_ids;
 
     // the main result queue (using ptr for polymorphism)
-    std::vector<engine_task_result_ptr> queue_results;
+    std::vector<server_task_result_ptr> queue_results;
 
     std::mutex mutex_results;
     std::condition_variable condition_results;
@@ -177,33 +177,33 @@ public:
     void remove_waiting_task_ids(const std::unordered_set<int> & id_tasks);
 
     // This function blocks the thread until there is a response for one of the id_tasks
-    engine_task_result_ptr recv(const std::unordered_set<int> & id_tasks);
+    server_task_result_ptr recv(const std::unordered_set<int> & id_tasks);
 
     // same as recv(), but have timeout in seconds
     // if timeout is reached, nullptr is returned
-    engine_task_result_ptr recv_with_timeout(const std::unordered_set<int> & id_tasks, int timeout);
+    server_task_result_ptr recv_with_timeout(const std::unordered_set<int> & id_tasks, int timeout);
 
     // single-task version of recv()
-    engine_task_result_ptr recv(int id_task);
+    server_task_result_ptr recv(int id_task);
 
     // Send a new result to a waiting id_task
-    void send(engine_task_result_ptr && result);
+    void send(server_task_result_ptr && result);
 
     // broadcast a new result to all waiting tasks
     // (used by router mode)
-    void broadcast(engine_task_result_ptr && result);
+    void broadcast(server_task_result_ptr && result);
 
     // terminate the waiting loop
     void terminate();
 };
 
-// RAII wrapper to make working with engine_queue and engine_response easier
+// RAII wrapper to make working with server_queue and server_response easier
 // it provides a generator-like API for server responses
 // support pooling connection state and aggregating multiple results
-struct engine_response_reader {
+struct server_response_reader {
     std::unordered_set<int> id_tasks;
-    engine_queue & queue_tasks;
-    engine_response & queue_results;
+    server_queue & queue_tasks;
+    server_response & queue_results;
     size_t received_count = 0;
     bool cancelled = false;
     int polling_interval_seconds;
@@ -213,9 +213,9 @@ struct engine_response_reader {
     std::vector<task_result_state> states;
 
     // should_stop function will be called each polling_interval_seconds
-    engine_response_reader(engine_queue & queue_tasks, engine_response & queue_results, int polling_interval_seconds)
+    server_response_reader(server_queue & queue_tasks, server_response & queue_results, int polling_interval_seconds)
         : queue_tasks(queue_tasks), queue_results(queue_results), polling_interval_seconds(polling_interval_seconds) {}
-    ~engine_response_reader() {
+    ~server_response_reader() {
         stop();
     }
 
@@ -224,18 +224,18 @@ struct engine_response_reader {
     }
 
     // if front = true, the task will be posted to the front of the queue (high priority)
-    void post_task(engine_task && task, bool front = false);
-    void post_tasks(std::vector<engine_task> && tasks, const common_chat_session & session = {}, bool front = false);
+    void post_task(server_task && task, bool front = false);
+    void post_tasks(std::vector<server_task> && tasks, const common_chat_session & session = {}, bool front = false);
     bool has_next() const;
 
     // return nullptr if should_stop() is true before receiving a result
     // note: if one error is received, it will stop further processing and return error result
-    engine_task_result_ptr next(const std::function<bool()> & should_stop);
+    server_task_result_ptr next(const std::function<bool()> & should_stop);
 
     struct batch_response {
         bool is_terminated = false; // if true, indicates that processing was stopped before all results were received
-        std::vector<engine_task_result_ptr> results;
-        engine_task_result_ptr error; // nullptr if no error
+        std::vector<server_task_result_ptr> results;
+        server_task_result_ptr error; // nullptr if no error
     };
     // aggregate multiple results
     batch_response wait_for_all(const std::function<bool()> & should_stop);

tools/server/server-routes.h

@@ -1,90 +0,0 @@
-#pragma once
-
-#include "engine-context.h"
-#include "server-http.h"
-
-#include "json.h"
-
-#include <memory>
-#include <mutex>
-
-// forward declarations
-struct server_res_generator;
-
-struct server_routes {
-    server_routes(const common_params & params, engine_context & ctx_server);
-
-    void init_routes();
-
-    // note: this is not thread-safe and can only when ctx_http.is_ready is false
-    void update_meta(const engine_context & ctx_server) {
-        this->meta = std::make_unique<engine_context_meta>(ctx_server.get_meta());
-    }
-
-    // handlers using lambda function, so that they can capture `this` without `std::bind`
-    // they won't be called until ctx_http.is_ready is set to true
-    server_http_context::handler_t get_health;
-    server_http_context::handler_t get_metrics;
-    server_http_context::handler_t get_slots;
-    server_http_context::handler_t post_slots;
-    server_http_context::handler_t get_props;
-    server_http_context::handler_t post_props;
-    server_http_context::handler_t post_infill;
-    server_http_context::handler_t post_completions;
-    server_http_context::handler_t post_completions_oai;
-    server_http_context::handler_t post_chat_completions;
-    server_http_context::handler_t post_chat_completions_tok;
-    server_http_context::handler_t post_control;
-    server_http_context::handler_t post_responses_oai;
-    server_http_context::handler_t post_responses_tok_oai;
-    server_http_context::handler_t post_transcriptions_oai;
-    server_http_context::handler_t post_anthropic_messages;
-    server_http_context::handler_t post_anthropic_count_tokens;
-    server_http_context::handler_t post_apply_template;
-    server_http_context::handler_t get_models;
-    server_http_context::handler_t post_tokenize;
-    server_http_context::handler_t post_detokenize;
-    server_http_context::handler_t post_embeddings;
-    server_http_context::handler_t post_embeddings_oai;
-    server_http_context::handler_t post_rerank;
-    server_http_context::handler_t post_systemone;
-    server_http_context::handler_t get_lora_adapters;
-    server_http_context::handler_t post_lora_adapters;
-
-    // to be used in router mode
-    json get_model_info() const;
-
-private:
-    std::unique_ptr<server_res_generator> handle_completions_impl(
-            const server_http_req & req,
-            engine_task_type type,
-            const json & data,
-            const std::vector<raw_buffer> & files,
-            task_response_type res_type,
-            const common_chat_session & chat_session = {});
-    std::unique_ptr<server_res_generator> handle_slots_save(const server_http_req & req, int id_slot);
-    std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
-    std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
-    std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
-    std::unique_ptr<server_res_generator> handle_count_tokens(const server_http_req & req, task_response_type res_type);
-
-    // using unique_ptr to allow late initialization of const
-    std::unique_ptr<const engine_context_meta> meta;
-
-    const common_params & params;
-    engine_context & ctx_server;
-
-    engine_queue & queue_tasks;
-    engine_response & queue_results;
-    std::unique_ptr<server_res_generator> create_response(bool bypass_sleep = false);
-
-    // cached responses, to be used during sleep
-    std::mutex     mutex_cache;
-    json           cached_models  = nullptr;
-    json           cached_props   = nullptr;
-    engine_metrics cached_metrics;
-    // set when a scrape during sleep already reported the throughput buckets
-    bool           should_reset_buckets = false;
-    // call right before sleep to update the cached responses
-    void update_cached_responses(bool is_sleeping);
-};

tools/server/server-schema.cpp

@@ -1,8 +1,8 @@
-#include "engine-schema.h"
+#include "server-schema.h"
 
 #include "json-schema-to-grammar.h"
 
-namespace engine_schema {
+namespace server_schema {
 
 //
 // llama.cpp-specific completion schema
@@ -324,7 +324,7 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
         ->set_handler([&](field_eval_context & ctx, const json & data) {
             std::vector<common_grammar_trigger> triggers;
             for (const auto & t : data.at("grammar_triggers")) {
-                triggers.push_back(engine_grammar_trigger(t).value);
+                triggers.push_back(server_grammar_trigger(t).value);
             }
             common_sampling_add_grammar_triggers(ctx.params.sampling, ctx.vocab, std::move(triggers));
         }));
@@ -600,4 +600,4 @@ void field_nested::eval(field_eval_context & ctx, const json & data) {
     }
 }
 
-} // namespace engine_schema
+} // namespace server_schema

tools/server/server-schema.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "engine-common.h"
-#include "engine-task.h"
+#include "server-common.h"
+#include "server-task.h"
 
 #include "sampling.h"
 #include "speculative.h"
@@ -13,7 +13,7 @@
 #include <string>
 #include <vector>
 
-namespace engine_schema {
+namespace server_schema {
 
 struct field_eval_context {
     task_params & params;
@@ -101,4 +101,4 @@ task_params eval_llama_cmpl_schema(
                     const std::vector<llama_logit_bias> & logit_bias_eog,
                     const json & data);
 
-} // namespace engine_schema
+} // namespace server_schema

tools/server/server-stream.cpp

@@ -1,7 +1,7 @@
 #include "server-stream.h"
-#include "engine-common.h"
+#include "server-common.h"
 #include "server-http.h"
-#include "engine-queue.h"
+#include "server-queue.h"
 
 #include <chrono>
 #include <memory>

tools/server/server-subproc.cpp

@@ -1,196 +0,0 @@
-#include "server-subproc.h"
-
-#include "log.h"
-
-#include <algorithm>
-#include <chrono>
-#include <cstring>
-#include <limits>
-#include <thread>
-
-#ifdef _WIN32
-// windows.h defines min and max as macros, which breaks std::min and std::max
-#define WIN32_LEAN_AND_MEAN
-#ifndef NOMINMAX
-#   define NOMINMAX
-#endif
-#include <windows.h>
-#include <io.h>
-#else
-#include <errno.h>
-#include <fcntl.h>
-#include <poll.h>
-#include <unistd.h>
-#endif
-
-//
-// server_subproc
-//
-
-FILE * server_reserve_stdout() {
-    fflush(stdout);
-    // the reserved stream is not inherited, grandchildren get stdout and stderr of their own
-#ifdef _WIN32
-    int fd = _dup(_fileno(stdout));
-    GGML_ASSERT(fd >= 0);
-    SetHandleInformation((HANDLE) _get_osfhandle(fd), HANDLE_FLAG_INHERIT, 0);
-    _dup2(_fileno(stderr), _fileno(stdout));
-    SetStdHandle(STD_OUTPUT_HANDLE, GetStdHandle(STD_ERROR_HANDLE));
-    FILE * f = _fdopen(fd, "w");
-#else
-    int fd = fcntl(fileno(stdout), F_DUPFD_CLOEXEC, 0);
-    GGML_ASSERT(fd >= 0);
-    dup2(fileno(stderr), fileno(stdout));
-    FILE * f = fdopen(fd, "w");
-#endif
-    GGML_ASSERT(f);
-    return f;
-}
-
-bool server_subproc::has_output() {
-    FILE * files[SERVER_SUBPROC_STREAMS] = { sproc.stdout_file(), sproc.stderr_file() };
-    for (int i = 0; i < SERVER_SUBPROC_STREAMS; i++) {
-        if (out_handles[i] >= 0) {
-            continue;
-        }
-        if (!files[i]) {
-            return false;
-        }
-#ifdef _WIN32
-        HANDLE h = (HANDLE) _get_osfhandle(_fileno(files[i]));
-        if (h == INVALID_HANDLE_VALUE) {
-            return false;
-        }
-        out_handles[i] = (intptr_t) h;
-#else
-        int fd = fileno(files[i]);
-        if (fd < 0) {
-            return false;
-        }
-        fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
-        out_handles[i] = fd;
-#endif
-    }
-    return true;
-}
-
-bool server_subproc::output_closed() const {
-    return out_closed[SERVER_SUBPROC_STDOUT] && out_closed[SERVER_SUBPROC_STDERR];
-}
-
-int server_subproc::read_output(server_subproc_stream stream, char * buf, size_t len) {
-    if (out_closed[stream]) {
-        return -1;
-    }
-    if (!has_output()) {
-        out_closed[stream] = true;
-        return -1;
-    }
-#ifdef _WIN32
-    HANDLE h     = (HANDLE) out_handles[stream];
-    DWORD  avail = 0;
-    if (!PeekNamedPipe(h, NULL, 0, NULL, &avail, NULL)) {
-        out_closed[stream] = true; // pipe broken, child gone
-        return -1;
-    }
-    if (avail == 0) {
-        return 0;
-    }
-    DWORD to_read = avail < (DWORD) len ? avail : (DWORD) len;
-    DWORD got     = 0;
-    if (!ReadFile(h, buf, to_read, &got, NULL) || got == 0) {
-        out_closed[stream] = true;
-        return -1;
-    }
-    return (int) got;
-#else
-    while (true) {
-        ssize_t r = read((int) out_handles[stream], buf, len);
-        if (r > 0) {
-            return (int) r;
-        }
-        if (r < 0 && errno == EINTR) {
-            continue;
-        }
-        if (r < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
-            return 0;
-        }
-        out_closed[stream] = true; // EOF or error
-        return -1;
-    }
-#endif
-}
-
-server_subproc::waiter::waiter() {
-#ifndef _WIN32
-    int fds[2];
-    GGML_ASSERT(pipe(fds) == 0);
-    for (int fd : fds) {
-        fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
-    }
-    wake_fd[0] = fds[0];
-    wake_fd[1] = fds[1];
-#endif
-}
-
-server_subproc::waiter::~waiter() {
-#ifndef _WIN32
-    close((int) wake_fd[0]);
-    close((int) wake_fd[1]);
-#endif
-}
-
-void server_subproc::waiter::wake() {
-#ifndef _WIN32
-    char c = 1;
-    (void) !write((int) wake_fd[1], &c, 1);
-#endif
-}
-
-void server_subproc::waiter::wait(const std::vector<server_subproc *> & procs, std::vector<bool> & ready, int64_t timeout_ms) {
-    ready.assign(procs.size(), false);
-#ifdef _WIN32
-    // no waitable wait exists for anonymous pipes, so poll them in 50 ms steps
-    bool any = false;
-    for (size_t i = 0; i < procs.size(); i++) {
-        server_subproc * p = procs[i];
-        ready[i] = !p->has_output();
-        for (int s = 0; s < SERVER_SUBPROC_STREAMS && !ready[i]; s++) {
-            DWORD avail = 0;
-            if (!p->out_closed[s] && (!PeekNamedPipe((HANDLE) p->out_handles[s], NULL, 0, NULL, &avail, NULL) || avail > 0)) {
-                ready[i] = true; // data or broken pipe, read_output() tells which
-            }
-        }
-        any = any || ready[i];
-    }
-    if (!any) {
-        int64_t step = timeout_ms < 0 ? 50 : std::min<int64_t>(timeout_ms, 50);
-        std::this_thread::sleep_for(std::chrono::milliseconds(step));
-    }
-#else
-    std::vector<pollfd> pfds;
-    pfds.reserve(procs.size() * SERVER_SUBPROC_STREAMS + 1);
-    pfds.push_back({ (int) wake_fd[0], POLLIN, 0 });
-    for (auto * p : procs) {
-        const bool open = p->has_output();
-        for (int s = 0; s < SERVER_SUBPROC_STREAMS; s++) {
-            pfds.push_back({ open && !p->out_closed[s] ? (int) p->out_handles[s] : -1, POLLIN, 0 }); // poll() skips negative fds
-        }
-    }
-    int timeout = timeout_ms < 0 ? -1 : (int) std::min<int64_t>(timeout_ms, std::numeric_limits<int>::max());
-    int r = poll(pfds.data(), pfds.size(), timeout);
-    if (r < 0 && errno != EINTR) {
-        LOG_ERR("%s: poll() failed: %s\n", __func__, strerror(errno));
-    }
-    if (pfds[0].revents) {
-        char buf[64];
-        while (read((int) wake_fd[0], buf, sizeof(buf)) > 0) {}
-    }
-    for (size_t i = 0; i < procs.size(); i++) {
-        ready[i] = !procs[i]->has_output();
-        for (int s = 0; s < SERVER_SUBPROC_STREAMS; s++) {
-            ready[i] = ready[i] || pfds[1 + i * SERVER_SUBPROC_STREAMS + s].revents != 0;
-        }
-    }
-#endif
-}

tools/server/server-subproc.h

@@ -1,129 +0,0 @@
-#pragma once
-
-#include "subproc.h"
-
-#include <atomic>
-#include <chrono>
-#include <condition_variable>
-#include <cstdint>
-#include <cstdio>
-#include <functional>
-#include <mutex>
-#include <queue>
-#include <vector>
-
-// simple implementation of a pipe
-// used for streaming data between threads
-template<typename T>
-struct server_pipe {
-    std::mutex mutex;
-    std::condition_variable cv;
-    std::queue<T> queue;
-    std::atomic<bool> writer_closed{false};
-    std::atomic<bool> reader_closed{false};
-
-    // 0 = unbounded (default)
-    // > 0, write() drops the oldest item once the queue is full
-    size_t max_size = 0;
-
-    void close_write() {
-        writer_closed.store(true, std::memory_order_relaxed);
-        cv.notify_all();
-    }
-
-    void close_read() {
-        reader_closed.store(true, std::memory_order_relaxed);
-        cv.notify_all();
-    }
-
-    // close_on_stop = true: should_stop means the reader is gone for good, so the writer is told the pipe is broken.
-    // close_on_stop = false: should_stop is a per-read deadline and further reads still come, so the pipe stays usable.
-    bool read(T & output, const std::function<bool()> & should_stop, bool close_on_stop = true) {
-        std::unique_lock<std::mutex> lk(mutex);
-        constexpr auto poll_interval = std::chrono::milliseconds(500);
-        while (true) {
-            if (!queue.empty()) {
-                output = std::move(queue.front());
-                queue.pop();
-                return true;
-            }
-            if (writer_closed.load()) {
-                return false; // clean EOF
-            }
-            if (should_stop && should_stop()) { // a null should_stop means "never stop"
-                if (close_on_stop) {
-                    close_read(); // signal broken pipe to writer
-                }
-                return false; // cancelled / deadline reached
-            }
-            cv.wait_for(lk, poll_interval);
-        }
-    }
-
-    bool write(T && data) {
-        std::lock_guard<std::mutex> lk(mutex);
-        if (reader_closed.load()) {
-            return false; // broken pipe
-        }
-        if (max_size > 0) {
-            while (queue.size() >= max_size) {
-                queue.pop(); // drop oldest to stay bounded
-            }
-        }
-        queue.push(std::move(data));
-        cv.notify_one();
-        return true;
-    }
-};
-
-// a child server writes its state commands to stdout and its logs to stderr
-enum server_subproc_stream {
-    SERVER_SUBPROC_STDOUT,
-    SERVER_SUBPROC_STDERR,
-    SERVER_SUBPROC_STREAMS,
-};
-
-// gives the current stdout to the caller as a stream of its own, and sends everything else written to stdout to stderr
-FILE * server_reserve_stdout();
-
-// wrapper around common_subproc to manage a child server process
-// mainly used by router mode
-struct server_subproc {
-    common_subproc sproc;
-    std::atomic<bool> stopped{false}; // set by the monitor once the process exited and was reaped
-
-    bool is_alive() { return sproc.alive(); }
-    void terminate() { sproc.terminate(); }
-    int  join() { return sproc.join(); }
-
-    // true if both output pipes of the child are available (call after create())
-    bool has_output();
-
-    // true once both output pipes are closed
-    bool output_closed() const;
-
-    // non-blocking read from one output pipe
-    // returns the number of bytes read, 0 when nothing is available, -1 when the pipe is closed or broken
-    int read_output(server_subproc_stream stream, char * buf, size_t len);
-
-    // wait until one of a set of children has output, wake() is called, or a timeout passes
-    struct waiter {
-        waiter();
-        ~waiter();
-
-        // thread-safe; on Windows this is a no-op, wait() returns within 50 ms anyway
-        void wake();
-
-        // timeout_ms < 0 waits until data or wake(); ready[i] is set for each child with data on an open pipe (or a broken pipe)
-        void wait(const std::vector<server_subproc *> & procs, std::vector<bool> & ready, int64_t timeout_ms);
-
-    private:
-#ifndef _WIN32
-        intptr_t wake_fd[2] = { -1, -1 }; // POSIX self-pipe
-#endif
-    };
-
-private:
-    intptr_t out_handles[SERVER_SUBPROC_STREAMS] = { -1, -1 }; // fd on POSIX, HANDLE on Windows; taken lazily from sproc
-    bool     out_closed [SERVER_SUBPROC_STREAMS] = { false, false };
-};

tools/server/server-task.cpp

@@ -1,14 +1,14 @@
-#include "engine-task.h"
+#include "server-task.h"
 
 #include "build-info.h"
-#include "engine-chat.h"
+#include "server-chat.h"
 #include "chat.h"
 #include "common.h"
 #include "json-schema-to-grammar.h"
 #include "llama.h"
 #include "sampling.h"
 #include "speculative.h"
-#include "engine-common.h"
+#include "server-common.h"
 
 #include <sstream>
 
@@ -88,7 +88,7 @@ json task_params::to_json(bool only_metrics) const {
 
     auto grammar_triggers = json::array();
     for (const auto & trigger : sampling.grammar_triggers) {
-        engine_grammar_trigger ct(trigger);
+        server_grammar_trigger ct(trigger);
         grammar_triggers.push_back(ct.to_json());
     }
 
@@ -307,9 +307,9 @@ std::vector<unsigned char> completion_token_output::str_to_bytes(const std::stri
 }
 
 //
-// engine_task_result_cmpl_final
+// server_task_result_cmpl_final
 //
-json engine_task_result_cmpl_final::to_json() {
+json server_task_result_cmpl_final::to_json() {
     GGML_ASSERT(is_updated && "update() must be called before to_json()");
     switch (res_type) {
         case TASK_RESPONSE_TYPE_NONE:
@@ -329,7 +329,7 @@ json engine_task_result_cmpl_final::to_json() {
     }
 }
 
-json engine_task_result_cmpl_final::to_json_non_oaicompat() {
+json server_task_result_cmpl_final::to_json_non_oaicompat() {
     json res = json {
         {"index",               index},
         {"content",             content.text},
@@ -354,7 +354,7 @@ json engine_task_result_cmpl_final::to_json_non_oaicompat() {
     return response_fields.empty() ? res : json_get_nested_values(response_fields, res);
 }
 
-json engine_task_result_cmpl_final::usage_json_oaicompat() {
+json server_task_result_cmpl_final::usage_json_oaicompat() {
     return json {
         {"completion_tokens", n_decoded},
         {"prompt_tokens",     n_prompt_tokens},
@@ -363,7 +363,7 @@ json engine_task_result_cmpl_final::usage_json_oaicompat() {
     };
 }
 
-json engine_task_result_cmpl_final::to_json_oaicompat() {
+json server_task_result_cmpl_final::to_json_oaicompat() {
     std::time_t t = std::time(0);
     json logprobs = json(nullptr); // OAI default to null
     if (!stream && probs_output.size() > 0) {
@@ -403,7 +403,7 @@ json engine_task_result_cmpl_final::to_json_oaicompat() {
     return res;
 }
 
-json engine_task_result_cmpl_final::to_json_oaicompat_chat() {
+json server_task_result_cmpl_final::to_json_oaicompat_chat() {
     std::string finish_reason = "length";
     common_chat_msg msg;
     if (!oaicompat_msg.empty()) {
@@ -451,7 +451,7 @@ json engine_task_result_cmpl_final::to_json_oaicompat_chat() {
     return res;
 }
 
-json engine_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
+json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
     std::time_t t = std::time(0);
     std::string finish_reason = "length";
     if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {
@@ -465,7 +465,7 @@ json engine_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
                 json {
                     {"finish_reason", nullptr},
                     {"index", index},
-                    {"delta", engine_chat_msg_diff_to_json_oaicompat(diff)},
+                    {"delta", server_chat_msg_diff_to_json_oaicompat(diff)},
                 },
             })},
             {"created", t},
@@ -517,7 +517,7 @@ json engine_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
     return deltas;
 }
 
-json engine_task_result_cmpl_final::to_json_oaicompat_resp() {
+json server_task_result_cmpl_final::to_json_oaicompat_resp() {
     common_chat_msg msg;
     if (!oaicompat_msg.empty()) {
         msg = oaicompat_msg;
@@ -588,7 +588,7 @@ json engine_task_result_cmpl_final::to_json_oaicompat_resp() {
     return res;
 }
 
-json engine_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
+json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
     std::vector<json> server_sent_events;
     std::vector<json> output;
 
@@ -705,7 +705,7 @@ json engine_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
     return server_sent_events;
 }
 
-json engine_task_result_cmpl_final::to_json_oaicompat_asr() {
+json server_task_result_cmpl_final::to_json_oaicompat_asr() {
     json event = json {
         {"type",  "transcript.text.done"},
         {"text",  oaicompat_msg.content},
@@ -720,7 +720,7 @@ json engine_task_result_cmpl_final::to_json_oaicompat_asr() {
     return event;
 }
 
-json engine_task_result_cmpl_final::to_json_anthropic() {
+json server_task_result_cmpl_final::to_json_anthropic() {
     std::string stop_reason = "max_tokens";
     if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {
         stop_reason = oaicompat_msg.tool_calls.empty() ? "end_turn" : "tool_use";
@@ -786,7 +786,7 @@ json engine_task_result_cmpl_final::to_json_anthropic() {
     return res;
 }
 
-json engine_task_result_cmpl_final::to_json_anthropic_stream() {
+json server_task_result_cmpl_final::to_json_anthropic_stream() {
     json events = json::array();
 
     std::string stop_reason = "max_tokens";
@@ -975,9 +975,9 @@ json engine_task_result_cmpl_final::to_json_anthropic_stream() {
 }
 
 //
-// engine_task_result_cmpl_partial
+// server_task_result_cmpl_partial
 //
-void engine_task_result_cmpl_partial::update(task_result_state & state) {
+void server_task_result_cmpl_partial::update(task_result_state & state) {
     is_updated = true;
     if (is_begin) {
         return; // begin marker only flushes headers, skip parsing
@@ -1015,7 +1015,7 @@ void engine_task_result_cmpl_partial::update(task_result_state & state) {
     }
 }
 
-json engine_task_result_cmpl_partial::to_json() {
+json server_task_result_cmpl_partial::to_json() {
     GGML_ASSERT(is_updated && "update() must be called before to_json()");
     if (is_begin) {
         return nullptr; // simply signal to HTTP handler to send the headers and status code
@@ -1038,7 +1038,7 @@ json engine_task_result_cmpl_partial::to_json() {
     }
 }
 
-json engine_task_result_cmpl_partial::to_json_non_oaicompat() {
+json server_task_result_cmpl_partial::to_json_non_oaicompat() {
     // non-OAI-compat JSON
     json res = json {
         {"index",            index},
@@ -1062,7 +1062,7 @@ json engine_task_result_cmpl_partial::to_json_non_oaicompat() {
     return res;
 }
 
-json engine_task_result_cmpl_partial::to_json_oaicompat() {
+json server_task_result_cmpl_partial::to_json_oaicompat() {
     std::time_t t = std::time(0);
     json logprobs = json(nullptr); // OAI default to null
     if (prob_output.probs.size() > 0) {
@@ -1100,7 +1100,7 @@ json engine_task_result_cmpl_partial::to_json_oaicompat() {
     return res;
 }
 
-json engine_task_result_cmpl_partial::to_json_oaicompat_chat() {
+json server_task_result_cmpl_partial::to_json_oaicompat_chat() {
     bool first = n_decoded == 1;
     std::time_t t = std::time(0);
     json choices;
@@ -1131,7 +1131,7 @@ json engine_task_result_cmpl_partial::to_json_oaicompat_chat() {
     }
 
     for (const auto & diff : oaicompat_msg_diffs) {
-        add_delta(engine_chat_msg_diff_to_json_oaicompat(diff));
+        add_delta(server_chat_msg_diff_to_json_oaicompat(diff));
     }
 
     if (!deltas.empty()) {
@@ -1155,7 +1155,7 @@ json engine_task_result_cmpl_partial::to_json_oaicompat_chat() {
     return deltas;
 }
 
-json engine_task_result_cmpl_partial::to_json_oaicompat_resp() {
+json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
     std::vector<json> events;
 
     if (!oai_resp_created) {
@@ -1305,7 +1305,7 @@ json engine_task_result_cmpl_partial::to_json_oaicompat_resp() {
     return events;
 }
 
-json engine_task_result_cmpl_partial::to_json_oaicompat_asr() {
+json server_task_result_cmpl_partial::to_json_oaicompat_asr() {
     json event = json {
         {"type", "transcript.text.delta"},
         {"delta", content.text},
@@ -1313,7 +1313,7 @@ json engine_task_result_cmpl_partial::to_json_oaicompat_asr() {
     return event;
 }
 
-json engine_task_result_cmpl_partial::to_json_anthropic() {
+json server_task_result_cmpl_partial::to_json_anthropic() {
     json events = json::array();
     bool first = (n_decoded == 1);
     // use member variables to track block state across streaming calls
@@ -1453,22 +1453,22 @@ json engine_task_result_cmpl_partial::to_json_anthropic() {
 }
 
 //
-// engine_task_result_embd
+// server_task_result_embd
 //
-json engine_task_result_embd::to_json() {
+json server_task_result_embd::to_json() {
     return res_type == TASK_RESPONSE_TYPE_OAI_EMBD
         ? to_json_oaicompat()
         : to_json_non_oaicompat();
 }
 
-json engine_task_result_embd::to_json_non_oaicompat() {
+json server_task_result_embd::to_json_non_oaicompat() {
     return json {
         {"index",     index},
         {"embedding", embedding},
     };
 }
 
-json engine_task_result_embd::to_json_oaicompat() {
+json server_task_result_embd::to_json_oaicompat() {
     return json {
         {"index",            index},
         {"embedding",        embedding[0]},
@@ -1477,9 +1477,9 @@ json engine_task_result_embd::to_json_oaicompat() {
 }
 
 //
-// engine_task_result_rerank
+// server_task_result_rerank
 //
-json engine_task_result_rerank::to_json() {
+json server_task_result_rerank::to_json() {
     return json {
         {"index",            index},
         {"score",            score},
@@ -1488,9 +1488,9 @@ json engine_task_result_rerank::to_json() {
 }
 
 //
-// engine_task_result_decision
+// server_task_result_decision
 //
-json engine_task_result_decision::to_json() {
+json server_task_result_decision::to_json() {
     return json {
         {"index",            index},
         {"scores",           scores},
@@ -1499,9 +1499,9 @@ json engine_task_result_decision::to_json() {
 }
 
 //
-// engine_task_result_error
+// server_task_result_error
 //
-json engine_task_result_error::to_json() {
+json server_task_result_error::to_json() {
     json res = format_error_response(err_msg, err_type);
     if (err_type == ERROR_TYPE_EXCEED_CONTEXT_SIZE) {
         res["n_prompt_tokens"] = n_prompt_tokens;
@@ -1511,19 +1511,19 @@ json engine_task_result_error::to_json() {
 }
 
 //
-// engine_task_result_metrics
+// server_task_result_metrics
 //
-json engine_task_result_slots::to_json() {
+json server_task_result_slots::to_json() {
     return slots_data;
 }
 
-json engine_task_result_metrics::to_json() {
+json server_task_result_metrics::to_json() {
     // not used, /metrics renders prometheus text via to_metrics()
     return json{};
 }
 
 // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names
-std::string engine_task_result_metrics::to_metrics() {
+std::string server_task_result_metrics::to_metrics() {
     const std::vector<metric_item> counters = {
         {
             "prompt_tokens_total",
@@ -1620,9 +1620,9 @@ std::string engine_task_result_metrics::to_metrics() {
 }
 
 //
-// engine_task_result_slot_save_load
+// server_task_result_slot_save_load
 //
-json engine_task_result_slot_save_load::to_json() {
+json server_task_result_slot_save_load::to_json() {
     if (is_save) {
         return json {
             { "id_slot",   id_slot },
@@ -1647,9 +1647,9 @@ json engine_task_result_slot_save_load::to_json() {
 }
 
 //
-// engine_task_result_slot_erase
+// server_task_result_slot_erase
 //
-json engine_task_result_slot_erase::to_json() {
+json server_task_result_slot_erase::to_json() {
     return json {
         { "id_slot",  id_slot },
         { "n_erased", n_erased },
@@ -1657,10 +1657,10 @@ json engine_task_result_slot_erase::to_json() {
 }
 
 //
-// engine_task_result_get_lora
+// server_task_result_get_lora
 //
 
-json engine_task_result_get_lora::to_json() {
+json server_task_result_get_lora::to_json() {
     json result = json::array();
     for (size_t i = 0; i < loras.size(); ++i) {
         auto & lora = loras[i];
@@ -1681,17 +1681,17 @@ json engine_task_result_get_lora::to_json() {
 }
 
 //
-// engine_task_result_apply_lora
+// server_task_result_apply_lora
 //
 
-json engine_task_result_apply_lora::to_json() {
+json server_task_result_apply_lora::to_json() {
     return json {{ "success", true }};
 }
 
 //
-// engine_prompt_cache
+// server_prompt_cache
 //
-size_t engine_prompt_cache::size() const {
+size_t server_prompt_cache::size() const {
     size_t res = 0;
 
     for (const auto & state : states) {
@@ -1701,7 +1701,7 @@ size_t engine_prompt_cache::size() const {
     return res;
 }
 
-size_t engine_prompt_cache::n_tokens() const {
+size_t server_prompt_cache::n_tokens() const {
     size_t res = 0;
 
     for (const auto & state : states) {
@@ -1711,7 +1711,7 @@ size_t engine_prompt_cache::n_tokens() const {
     return res;
 }
 
-engine_prompt_cache_state * engine_prompt_cache::alloc(const engine_prompt & prompt, size_t state_size_tgt, size_t state_size_dft) {
+server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & prompt, size_t state_size_tgt, size_t state_size_dft) {
     // first check if the current state is contained fully in the cache
     for (auto it = states.begin(); it != states.end(); ++it) {
         const int cur_lcp_len = it->prompt.tokens.get_common_prefix(prompt.tokens);
@@ -1793,7 +1793,7 @@ engine_prompt_cache_state * engine_prompt_cache::alloc(const engine_prompt & pro
     return &states.back();
 }
 
-bool engine_prompt_cache::load(engine_prompt & prompt, const engine_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) {
+bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) {
     const int lcp_best = prompt.tokens.get_common_prefix(tokens_new);
 
     float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins
@@ -1870,7 +1870,7 @@ bool engine_prompt_cache::load(engine_prompt & prompt, const engine_tokens & tok
     return true;
 }
 
-void engine_prompt_cache::update() {
+void server_prompt_cache::update() {
     if (limit_size > 0) {
         while (!states.empty() && size() > limit_size) {
             SRV_WRN(" - cache size limit reached, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0));

tools/server/server-task.h

@@ -8,26 +8,26 @@
 #include <list>
 #include <map>
 
-// TODO: prevent including the whole engine-common.h as we only use engine_tokens
-#include "engine-common.h"
+// TODO: prevent including the whole server-common.h as we only use server_tokens
+#include "server-common.h"
 
 
-enum engine_task_type {
-    ENGINE_TASK_TYPE_COMPLETION,
-    ENGINE_TASK_TYPE_EMBEDDING,
-    ENGINE_TASK_TYPE_RERANK,
-    ENGINE_TASK_TYPE_DECISION,
-    ENGINE_TASK_TYPE_INFILL,
-    ENGINE_TASK_TYPE_CANCEL,
-    ENGINE_TASK_TYPE_CONTROL,
-    ENGINE_TASK_TYPE_NEXT_RESPONSE,
-    ENGINE_TASK_TYPE_METRICS,
-    ENGINE_TASK_TYPE_SLOT_GET,
-    ENGINE_TASK_TYPE_SLOT_SAVE,
-    ENGINE_TASK_TYPE_SLOT_RESTORE,
-    ENGINE_TASK_TYPE_SLOT_ERASE,
-    ENGINE_TASK_TYPE_GET_LORA,
-    ENGINE_TASK_TYPE_SET_LORA,
+enum server_task_type {
+    SERVER_TASK_TYPE_COMPLETION,
+    SERVER_TASK_TYPE_EMBEDDING,
+    SERVER_TASK_TYPE_RERANK,
+    SERVER_TASK_TYPE_DECISION,
+    SERVER_TASK_TYPE_INFILL,
+    SERVER_TASK_TYPE_CANCEL,
+    SERVER_TASK_TYPE_CONTROL,
+    SERVER_TASK_TYPE_NEXT_RESPONSE,
+    SERVER_TASK_TYPE_METRICS,
+    SERVER_TASK_TYPE_SLOT_GET,
+    SERVER_TASK_TYPE_SLOT_SAVE,
+    SERVER_TASK_TYPE_SLOT_RESTORE,
+    SERVER_TASK_TYPE_SLOT_ERASE,
+    SERVER_TASK_TYPE_GET_LORA,
+    SERVER_TASK_TYPE_SET_LORA,
 };
 
 // TODO: change this to more generic "response_format" to replace the "format_response_*" in server-common
@@ -85,7 +85,7 @@ struct task_params {
     std::string        oaicompat_model;
     std::string        oaicompat_cmpl_id;
 
-    // realtime control (ENGINE_TASK_TYPE_CONTROL)
+    // realtime control (SERVER_TASK_TYPE_CONTROL)
     std::string        control_action;
     std::string        control_cmpl_id;
 
@@ -134,37 +134,37 @@ struct task_result_state {
         bool filter_tool_calls = false);
 };
 
-struct engine_task {
-    int id = -1; // to be filled by engine_queue
+struct server_task {
+    int id = -1; // to be filled by server_queue
 
     // TODO @ngxson : remove this field and implement a mapping task_id -> idx in the response_reader
     size_t index = 0; // used when there are multiple prompts (batch request)
 
-    // used by ENGINE_TASK_TYPE_CANCEL
+    // used by SERVER_TASK_TYPE_CANCEL
     int id_target = -1;
     int id_slot   = -1;
 
     // used by parallel sampling (multiple completions from same prompt)
     int id_parent  = -1;
     // temporary store of child tasks for scheduling
-    // note: accessing to elements is invalid after the task is moved to engine_slot
-    std::vector<engine_task> child_tasks;
+    // note: accessing to elements is invalid after the task is moved to server_slot
+    std::vector<server_task> child_tasks;
     // if set on a parent, the children have their own prompt and only share its first n_tokens_shared tokens
     int32_t n_tokens_shared = 0;
 
-    // used by ENGINE_TASK_TYPE_INFERENCE
+    // used by SERVER_TASK_TYPE_INFERENCE
     task_params   params;
-    engine_tokens tokens;
+    server_tokens tokens;
 
     // only used by CLI, this allow tokenizing CLI inputs on server side
-    // we need this because mtmd_context and vocab are not accessible outside of engine_context
+    // we need this because mtmd_context and vocab are not accessible outside of server_context
     bool                    cli = false;
     std::string             cli_prompt;
     std::vector<raw_buffer> cli_files;
 
-    engine_task_type type;
+    server_task_type type;
 
-    // used by ENGINE_TASK_TYPE_SLOT_SAVE, ENGINE_TASK_TYPE_SLOT_RESTORE, ENGINE_TASK_TYPE_SLOT_ERASE
+    // used by SERVER_TASK_TYPE_SLOT_SAVE, SERVER_TASK_TYPE_SLOT_RESTORE, SERVER_TASK_TYPE_SLOT_ERASE
     struct slot_action {
         int id_slot;
         std::string filename;
@@ -172,10 +172,10 @@ struct engine_task {
     };
     slot_action slot_action;
 
-    // used by ENGINE_TASK_TYPE_METRICS
+    // used by SERVER_TASK_TYPE_METRICS
     bool metrics_reset_bucket = false;
 
-    // used by ENGINE_TASK_TYPE_DECISION
+    // used by SERVER_TASK_TYPE_DECISION
     // where to read the model output of each option, exactly one of the two lists is used
     struct decision {
         std::vector<llama_token> labels;       // logits of these tokens, at the last prompt token
@@ -206,12 +206,12 @@ struct engine_task {
     };
     decision decision;
 
-    // used by ENGINE_TASK_TYPE_SET_LORA
+    // used by SERVER_TASK_TYPE_SET_LORA
     std::map<int, float> set_lora; // mapping adapter ID -> scale
 
-    engine_task() = default;
+    server_task() = default;
 
-    engine_task(engine_task_type type) : type(type) {}
+    server_task(server_task_type type) : type(type) {}
 
     int32_t n_tokens() const {
         return tokens.size();
@@ -219,10 +219,10 @@ struct engine_task {
 
     bool need_embd() const {
         switch (type) {
-            case ENGINE_TASK_TYPE_EMBEDDING:
-            case ENGINE_TASK_TYPE_RERANK:
+            case SERVER_TASK_TYPE_EMBEDDING:
+            case SERVER_TASK_TYPE_RERANK:
                 return true;
-            case ENGINE_TASK_TYPE_DECISION:
+            case SERVER_TASK_TYPE_DECISION:
                 return !decision.markers.empty() || !decision.order.empty();
             default:
                 return false;
@@ -231,10 +231,10 @@ struct engine_task {
 
     bool need_logits() const {
         switch (type) {
-            case ENGINE_TASK_TYPE_COMPLETION:
-            case ENGINE_TASK_TYPE_INFILL:
+            case SERVER_TASK_TYPE_COMPLETION:
+            case SERVER_TASK_TYPE_INFILL:
                 return true;
-            case ENGINE_TASK_TYPE_DECISION:
+            case SERVER_TASK_TYPE_DECISION:
                 return !decision.labels.empty();
             default:
                 return false;
@@ -243,8 +243,8 @@ struct engine_task {
 
     bool need_sampling() const {
         switch (type) {
-            case ENGINE_TASK_TYPE_COMPLETION:
-            case ENGINE_TASK_TYPE_INFILL:
+            case SERVER_TASK_TYPE_COMPLETION:
+            case SERVER_TASK_TYPE_INFILL:
                 return true;
             default:
                 return false;
@@ -252,7 +252,7 @@ struct engine_task {
     }
 
     // utility function
-    static std::unordered_set<int> get_list_id(const std::vector<engine_task> & tasks) {
+    static std::unordered_set<int> get_list_id(const std::vector<server_task> & tasks) {
         std::unordered_set<int> ids(tasks.size());
         for (size_t i = 0; i < tasks.size(); i++) {
             ids.insert(tasks[i].id);
@@ -275,7 +275,7 @@ struct engine_task {
     }
 
     void add_child(int id_parent, int id_child) {
-        engine_task copy;
+        server_task copy;
 
         copy.id        = id_child;
         copy.id_parent = id_parent;
@@ -303,7 +303,7 @@ struct engine_task {
 
     // embedding and rerank tasks always process their whole prompt, their state is never reused
     bool is_stateless() const {
-        return type == ENGINE_TASK_TYPE_EMBEDDING || type == ENGINE_TASK_TYPE_RERANK;
+        return type == SERVER_TASK_TYPE_EMBEDDING || type == SERVER_TASK_TYPE_RERANK;
     }
 };
 
@@ -316,7 +316,7 @@ struct result_prompt_progress {
     json to_json() const;
 };
 
-struct engine_task_result {
+struct server_task_result {
     int id           = -1;
     int id_slot      = -1;
 
@@ -324,25 +324,25 @@ struct engine_task_result {
     size_t index = 0; // to be used for batched tasks
 
     virtual bool is_error() {
-        // only used by engine_task_result_error
+        // only used by server_task_result_error
         return false;
     }
     virtual bool is_stop() {
-        // only used by engine_task_result_cmpl_*
+        // only used by server_task_result_cmpl_*
         return true;
     }
     virtual void update(task_result_state &) {
-        // only used by engine_task_result_cmpl_*
+        // only used by server_task_result_cmpl_*
     }
     virtual json to_json() = 0;
-    virtual ~engine_task_result() = default;
-    virtual engine_task_result * clone() const {
+    virtual ~server_task_result() = default;
+    virtual server_task_result * clone() const {
         GGML_ABORT("not implemented for this task type");
     }
 };
 
-// using shared_ptr for polymorphism of engine_task_result
-using engine_task_result_ptr = std::unique_ptr<engine_task_result>;
+// using shared_ptr for polymorphism of server_task_result
+using server_task_result_ptr = std::unique_ptr<server_task_result>;
 
 struct completion_token_output {
     llama_token tok;
@@ -365,13 +365,13 @@ struct completion_token_output {
 
 };
 
-struct engine_task_result_cmpl_final : engine_task_result {
+struct server_task_result_cmpl_final : server_task_result {
     common_chat_input content;
     llama_tokens tokens;
 
     bool stream;
     bool include_usage;
-    engine_slot_stats stats;
+    server_slot_stats stats;
     std::string prompt;
 
     bool truncated;
@@ -440,7 +440,7 @@ struct engine_task_result_cmpl_final : engine_task_result {
     json to_json_anthropic_stream();
 };
 
-struct engine_task_result_cmpl_partial : engine_task_result {
+struct server_task_result_cmpl_partial : server_task_result {
     common_chat_input content;
     llama_tokens      tokens;
 
@@ -453,7 +453,7 @@ struct engine_task_result_cmpl_partial : engine_task_result {
     bool is_begin = false; // whether to send 200 status to HTTP client (begin of SSE stream)
                            // ref: https://github.com/ggml-org/llama.cpp/pull/23884
     completion_token_output prob_output;
-    engine_slot_stats stats;
+    server_slot_stats stats;
     result_prompt_progress progress;
 
     // response formatting
@@ -499,7 +499,7 @@ struct engine_task_result_cmpl_partial : engine_task_result {
     json to_json_anthropic();
 };
 
-struct engine_task_result_embd : engine_task_result {
+struct server_task_result_embd : server_task_result {
     std::vector<std::vector<float>> embedding;
 
     int32_t n_tokens;
@@ -514,7 +514,7 @@ struct engine_task_result_embd : engine_task_result {
     json to_json_oaicompat();
 };
 
-struct engine_task_result_rerank : engine_task_result {
+struct server_task_result_rerank : server_task_result {
     float score = -1e6;
 
     int32_t n_tokens;
@@ -522,7 +522,7 @@ struct engine_task_result_rerank : engine_task_result {
     virtual json to_json() override;
 };
 
-struct engine_task_result_decision : engine_task_result {
+struct server_task_result_decision : server_task_result {
     std::vector<float> scores; // one raw model output per option
 
     int32_t n_tokens;
@@ -530,7 +530,7 @@ struct engine_task_result_decision : engine_task_result {
     virtual json to_json() override;
 };
 
-struct engine_task_result_error : engine_task_result {
+struct server_task_result_error : server_task_result {
     error_type err_type = ERROR_TYPE_SERVER;
     std::string err_msg;
 
@@ -546,12 +546,12 @@ struct engine_task_result_error : engine_task_result {
 };
 
 // used by /metrics API
-struct engine_task_result_metrics : engine_task_result {
-    // these are immediate stats, not accumulated (engine_metrics is cumulative)
+struct server_task_result_metrics : server_task_result {
+    // these are immediate stats, not accumulated (server_metrics is cumulative)
     int n_processing_slots = 0;
     int n_tasks_deferred = 0;
 
-    engine_metrics metrics;
+    server_metrics metrics;
 
     virtual json to_json() override;
 
@@ -564,17 +564,17 @@ struct engine_task_result_metrics : engine_task_result {
 };
 
 // used by /slots API
-struct engine_task_result_slots : engine_task_result {
+struct server_task_result_slots : server_task_result {
     int n_idle_slots = 0;
 
-    // while we can also use std::vector<engine_slot> this requires copying the slot object which can be quite messy
+    // while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy
     // therefore, we use json to temporarily store the slot.to_json() result
     json slots_data = json::array();
 
     virtual json to_json() override;
 };
 
-struct engine_task_result_slot_save_load : engine_task_result {
+struct server_task_result_slot_save_load : server_task_result {
     std::string filename;
     bool is_save; // true = save, false = load
 
@@ -585,13 +585,13 @@ struct engine_task_result_slot_save_load : engine_task_result {
     virtual json to_json() override;
 };
 
-struct engine_task_result_slot_erase : engine_task_result {
+struct server_task_result_slot_erase : server_task_result {
     size_t n_erased;
 
     virtual json to_json() override;
 };
 
-struct engine_task_result_control : engine_task_result {
+struct server_task_result_control : server_task_result {
     bool        success = false;
     std::string message; // optional detail when success is false
 
@@ -604,7 +604,7 @@ struct engine_task_result_control : engine_task_result {
     }
 };
 
-struct engine_task_result_get_lora : engine_task_result {
+struct server_task_result_get_lora : server_task_result {
     struct lora {
         common_adapter_lora_info info;
         std::string  alora_invocation_string;
@@ -615,12 +615,12 @@ struct engine_task_result_get_lora : engine_task_result {
     virtual json to_json() override;
 };
 
-struct engine_task_result_apply_lora : engine_task_result {
+struct server_task_result_apply_lora : server_task_result {
     virtual json to_json() override;
 };
 
-struct engine_prompt {
-    engine_tokens tokens;
+struct server_prompt {
+    server_tokens tokens;
 
     std::list<common_prompt_checkpoint> checkpoints;
 
@@ -633,15 +633,15 @@ struct engine_prompt {
         return tokens.size();
     }
 
-    engine_prompt clone() const {
-        return engine_prompt {
+    server_prompt clone() const {
+        return server_prompt {
             tokens.clone(),
             checkpoints,
         };
     }
 };
 
-struct engine_prompt_data {
+struct server_prompt_data {
     std::vector<uint8_t> main;
     std::vector<uint8_t> drft;
 
@@ -650,9 +650,9 @@ struct engine_prompt_data {
     }
 };
 
-struct engine_prompt_cache_state {
-    engine_prompt prompt;
-    engine_prompt_data data;
+struct server_prompt_cache_state {
+    server_prompt prompt;
+    server_prompt_data data;
 
     size_t size() const {
         size_t res = data.size();
@@ -665,13 +665,13 @@ struct engine_prompt_cache_state {
     }
 };
 
-struct engine_prompt_cache {
-    engine_prompt_cache(int32_t limit_size_mib, size_t limit_tokens) {
+struct server_prompt_cache {
+    server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens) {
         this->limit_size   = 1024ull*1024ull*(limit_size_mib < 0 ? 0 : limit_size_mib);
         this->limit_tokens = limit_tokens;
     }
 
-    std::list<engine_prompt_cache_state> states;
+    std::list<server_prompt_cache_state> states;
 
     // in bytes, 0 = no limit
     size_t limit_size = 0;
@@ -683,18 +683,18 @@ struct engine_prompt_cache {
 
     size_t n_tokens() const;
 
-    engine_prompt_cache_state * alloc(const engine_prompt & prompt, size_t state_size_main, size_t state_size_drft);
+    server_prompt_cache_state * alloc(const server_prompt & prompt, size_t state_size_main, size_t state_size_drft);
 
-    bool load(engine_prompt & prompt, const engine_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot);
+    bool load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot);
 
     void update();
 };
 
 // used exclusively by router mode
-struct engine_task_result_router : engine_task_result {
+struct server_task_result_router : server_task_result {
     json data;
     virtual json to_json() override { return data; }
-    virtual engine_task_result * clone() const override {
-        return new engine_task_result_router(*this);
+    virtual server_task_result * clone() const override {
+        return new server_task_result_router(*this);
     }
 };

tools/server/server-tools.cpp

@@ -1755,7 +1755,7 @@ struct server_tool_get_info : server_tool {
     }
 };
 
-struct server_tool_stream_result : engine_task_result {
+struct server_tool_stream_result : server_task_result {
     std::string chunk;
     bool done = false;
     std::string error_msg;
@@ -1783,7 +1783,7 @@ void server_tool::stream::push(const std::string & chunk) {
 
 struct server_tools_res : server_http_res {
     std::thread worker;
-    engine_response * qr = nullptr; // set only for streaming responses
+    server_response * qr = nullptr; // set only for streaming responses
     int id = -1;
 
     ~server_tools_res() override {

tools/server/server-tools.h

@@ -1,8 +1,8 @@
 #pragma once
 
-#include "engine-common.h"
+#include "server-common.h"
 #include "server-http.h"
-#include "engine-queue.h"
+#include "server-queue.h"
 #include "server-mcp.h"
 
 #include <atomic>
@@ -21,7 +21,7 @@ struct server_tool {
     virtual std::string type() const { return "server"; }
 
     struct stream {
-        engine_response & qr;
+        server_response & qr;
         int id;
         std::function<bool()> alive;
         void push(const std::string & chunk);
@@ -37,7 +37,7 @@ struct server_tools {
     std::vector<std::unique_ptr<server_tool>> tools;
 
     // for streaming
-    engine_response queue_res;
+    server_response queue_res;
     std::atomic<int> res_id{0};
 
     // set when --tools-runtime is configured; routes every tool call through an isolate

tools/server/server.cpp

@@ -1,5 +1,4 @@
-#include "engine-context.h"
-#include "server-routes.h"
+#include "server-context.h"
 #include "server-http.h"
 #include "server-models.h"
 #include "server-cors-proxy.h"
@@ -196,7 +195,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
     server_mcp mcp_mgr;
 
     // struct that contains llama context and inference
-    engine_context ctx_server;
+    server_context ctx_server;
 
     //
     // Router
@@ -481,8 +480,8 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
 
         // setup communication child --> router if necessary
         if (child.is_child()) {
-            ctx_server.set_state_callback([&](engine_state state, json payload) {
-                child.notify_to_router(engine_state_to_str(state), payload);
+            ctx_server.set_state_callback([&](server_state state, json payload) {
+                child.notify_to_router(server_state_to_str(state), payload);
             });
         }
 
@@ -541,7 +540,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
         std::thread monitor_thread;
         if (child.is_child()) {
             monitor_thread = child.setup(shutdown_handler);
-            child.notify_to_router(engine_state_to_str(ENGINE_STATE_READY), routes.get_model_info());
+            child.notify_to_router(server_state_to_str(SERVER_STATE_READY), routes.get_model_info());
         }
 
         // this call blocks the main thread until queue_tasks.terminate() is called

tools/server/tests/unit/test_slot_save.py

@@ -96,7 +96,7 @@ def test_slot_restore_legacy_token_list():
     assert res.status_code == 200
     assert res.body["n_saved"] == 84
 
-    # rewrite the token payload into a plain token list, as written by servers that predate the packed engine_tokens format
+    # rewrite the token payload into a plain token list, as written by servers that predate the packed server_tokens format
     path = os.path.join(server.slot_save_path, "slot_legacy.bin")
     with open(path, "rb") as f:
         data = bytearray(f.read())