ci : run the server-speculative steps with bash

.github/workflows/server-speculative.yml

@@ -0,0 +1,65 @@
+name: Server (speculative decoding)
+
+on:
+  workflow_dispatch:
+    inputs:
+      sha:
+        description: 'Commit SHA1 to build'
+        required: false
+        type: string
+
+cache-mode: none
+permissions:
+  contents: read
+
+env:
+  LLAMA_ARG_LOG_COLORS: 1
+  LLAMA_ARG_LOG_PREFIX: 1
+  LLAMA_ARG_LOG_TIMESTAMPS: 1
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  server-speculative:
+    runs-on: [self-hosted, Linux, gb10, perf]
+    container: "nvidia/cuda:13.4.3-devel-ubuntu26.04"
+    timeout-minutes: 60
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
+
+      - name: Install dependencies
+        run: |
+          apt update
+          apt install -y cmake libssl-dev python3 python3-venv python3-pip
+
+      - name: Build
+        run: |
+          cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
+          cmake --build build --config Release -j $(nproc) --target llama-server
+
+      - name: Python setup
+        run: |
+          cd tools/server/tests
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install -r requirements.txt
+
+      - name: Test speculative decoding
+        env:
+          SPEC_METHODS: compact
+          SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || format('{0}/spec-models', runner.temp) }}
+          N_GPU_LAYERS: 99
+        run: |
+          cd tools/server/tests
+          source venv/bin/activate
+          PYTEST_WORKERS=1 ./tests.sh -s -v unit/test_spec_methods.py

tools/server/tests/README.md

@@ -39,6 +39,46 @@ To run slow tests (will download many models, make sure to set `LLAMA_CACHE` if
 SLOW_TESTS=1 ./tests.sh
 ```
 
+The speculative-decoding method tests are opt-in and marked `heavy`. The compact suite covers MTP,
+DFlash, DSpark, and EAGLE-3 with five pinned files totaling 6.271 GiB:
+
+```shell
+SPEC_METHODS=compact SPEC_MODEL_DIR=/path/to/model-cache PYTEST_WORKERS=1 ./tests.sh -s -v unit/test_spec_methods.py
+```
+
+- MTP target: `Qwen3.5-4B-MTP-Q4_K_M.gguf`, 2.904 GiB.
+- Shared DFlash/DSpark/EAGLE-3 target: `Qwen3-4B-Q4_K_M.gguf`, 2.326 GiB.
+- DFlash draft: `dflash-Qwen3-4B-Q4_K_M.gguf`, 294.065 MiB.
+- DSpark draft: `dspark-Qwen3-4B-Q4_K_M.gguf`, 544.453 MiB.
+- EAGLE-3 draft: `eagle3-Qwen3-4B-Q8_0.gguf`, 227.075 MiB.
+
+All files come from `ggml-org/spec-for-testing-GGUF`, pinned to revision
+`a0736c8e98d3fe580352dacffa3fbd5e9c22a437` in `unit/test_spec_methods.py`.
+The consolidated repository retains the licenses of its source models; review its model card and
+the source model cards before use.
+
+Use `SPEC_METHODS=nightly` for DFlash2 and LiLiCorr, or a comma-separated subset such as
+`SPEC_METHODS=mtp,dflash`. The nightly pair requires 41.422 GiB and remains separate because no
+verified compact DFlash2 GGUF or compact LiLiCorr checkpoint is available. `SPEC_METHODS=all` runs both
+tiers and downloads about 47.693 GiB.
+
+The `Server (speculative decoding)` workflow runs only by explicit manual dispatch on the dedicated
+DGX Spark runner. It does not use the GitHub Actions cache. Set the `SPEC_METHOD_MODEL_DIR` repository
+variable to the runner's persistent model directory when its mount path is confirmed. Heavyweight
+DFlash2 and LiLiCorr measurements belong in the internal nightly performance/quality runs.
+
+The acceptance floors use one-sided regression checks: improvements do not fail. Three identical
+DGX Spark calibration runs produced MTP `0.7155 / 3.1230`, DFlash `0.1867 / 2.2754`, DSpark
+`0.2878 / 2.9766`, EAGLE-3 `0.3047 / 1.9095`, DFlash2 `0.5480 / 4.7414`, and LiLiCorr
+`0.6699 / 5.6444` for acceptance rate and length. The enforced floors subtract three percentage
+points and 0.2 accepted tokens per verification step. The committed six-method suite passed on
+DGX Spark in 41.12 seconds.
+
+DFlash's token-acceptance rate is lower because the denominator counts every token proposed in each
+seven-token draft block. It accepted 213 of 1,141 proposed tokens over 167 verification steps,
+equivalent to 1.275 accepted draft tokens per step and a mean acceptance length of 2.2754. The floor
+is intended to detect regressions from that deterministic baseline, not to define a quality target.
+
 To run with stdout/stderr display in real time (verbose output, but useful for debugging):
 
 ```shell

tools/server/tests/pytest.ini

@@ -1,4 +1,5 @@
 [pytest]
 markers =
+    heavy: requires a dedicated runner and large model downloads
     slow: marks tests as slow (deselect with '-m "not slow"')
     serial

tools/server/tests/unit/test_spec_methods.py

@@ -0,0 +1,211 @@
+import os
+import re
+from dataclasses import dataclass
+
+import pytest
+from utils import ServerProcess, download_file
+
+# Runs each speculative method greedily and checks that it still drafts and accepts as expected.
+# accept rate = accepted / drafted, accept len = 1 + accepted / verification steps (from /metrics).
+# DFlash2 and LiLiCorr use --spec-type draft-dflash; the draft GGUF metadata selects the variant.
+# Needs real models and a GPU. Use SPEC_METHODS=compact, nightly, all, or a comma-separated method list.
+
+HF = "https://huggingface.co"
+MODEL_REPO = "ggml-org/spec-for-testing-GGUF"
+MODEL_REVISION = "a0736c8e98d3fe580352dacffa3fbd5e9c22a437"
+
+
+@dataclass(frozen=True)
+class SpecCase:
+    name: str
+    spec_type: str
+    target: str
+    draft: str | None  # None when the heads are part of the target (MTP)
+    n_max: int
+    min_accept_rate: float
+    min_accept_len: float
+
+
+CASES = [
+    # Floors are the minimum of three identical DGX Spark calibration runs minus
+    # three acceptance-rate percentage points and 0.2 accepted tokens per step.
+    SpecCase(
+        name="mtp",
+        spec_type="draft-mtp",
+        target="Qwen3.5-4B-MTP-Q4_K_M.gguf",
+        draft=None,
+        n_max=3,
+        min_accept_rate=0.6855,
+        min_accept_len=2.9230,
+    ),
+    SpecCase(
+        name="dflash",
+        spec_type="draft-dflash",
+        target="Qwen3-4B-Q4_K_M.gguf",
+        draft="dflash-Qwen3-4B-Q4_K_M.gguf",
+        n_max=7,
+        min_accept_rate=0.1567,
+        min_accept_len=2.0754,
+    ),
+    SpecCase(
+        name="dspark",
+        spec_type="draft-dspark",
+        target="Qwen3-4B-Q4_K_M.gguf",
+        draft="dspark-Qwen3-4B-Q4_K_M.gguf",
+        n_max=7,
+        min_accept_rate=0.2578,
+        min_accept_len=2.7766,
+    ),
+    SpecCase(
+        name="eagle3",
+        spec_type="draft-eagle3",
+        target="Qwen3-4B-Q4_K_M.gguf",
+        draft="eagle3-Qwen3-4B-Q8_0.gguf",
+        n_max=3,
+        min_accept_rate=0.2747,
+        min_accept_len=1.7095,
+    ),
+    SpecCase(
+        name="dflash2",
+        spec_type="draft-dflash",
+        target="Qwen3.8-27B-Q4_K_M.gguf",
+        draft="dflash2-Qwen3.8-27B-Q8_0.gguf",
+        n_max=7,
+        min_accept_rate=0.5180,
+        min_accept_len=4.5414,
+    ),
+    SpecCase(
+        name="lilicorr",
+        spec_type="draft-dflash",
+        target="NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf",
+        draft="lilicorr-NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-NVFP4.gguf",
+        n_max=7,
+        min_accept_rate=0.6399,
+        min_accept_len=5.4444,
+    ),
+]
+
+COMPACT_METHODS = {"mtp", "dflash", "dspark", "eagle3"}
+NIGHTLY_METHODS = {"dflash2", "lilicorr"}
+
+PROMPTS = [
+    "Write a Python function that returns the n-th Fibonacci number using memoization.\n\n```python\n",
+    "The three main differences between TCP and UDP are:\n1.",
+    "Question: What is the capital of France, and what river flows through it?\nAnswer:",
+]
+
+N_PREDICT = 128
+
+
+def model_path(file: str) -> str:
+    """Download a pinned GGUF from the consolidated test repository."""
+    model_dir = os.environ.get("SPEC_MODEL_DIR", "./tmp")
+    os.makedirs(model_dir, exist_ok=True)
+    return download_file(
+        f"{HF}/{MODEL_REPO}/resolve/{MODEL_REVISION}/{file}",
+        os.path.join(model_dir, file),
+    )
+
+
+def make_server(case: SpecCase) -> ServerProcess:
+    server = ServerProcess()
+    server.model_hf_repo = None
+    server.model_hf_file = None
+    server.model_file = model_path(case.target)
+    server.n_ctx = 1024
+    server.n_slots = 1
+    server.n_gpu_layer = int(os.environ.get("N_GPU_LAYERS", 99))
+    server.fa = "on"
+    server.temperature = 0.0
+    server.server_metrics = True
+    server.spec_type = case.spec_type
+    server.spec_draft_n_gpu_layer = server.n_gpu_layer
+    if case.draft:
+        server.model_draft = model_path(case.draft)
+    if case.n_max:
+        server.spec_draft_n_max = case.n_max
+    return server
+
+
+def generate(server: ServerProcess) -> list[list[int]]:
+    outputs = []
+    for prompt in PROMPTS:
+        res = server.make_request(
+            "POST",
+            "/completion",
+            data={
+                "prompt": prompt,
+                "temperature": 0.0,
+                "top_k": 1,
+                "seed": 42,
+                "n_predict": N_PREDICT,
+                "cache_prompt": False,
+                "return_tokens": True,
+            },
+        )
+        assert res.status_code == 200
+        outputs.append(res.body["tokens"])
+    return outputs
+
+
+def spec_metrics(server: ServerProcess) -> dict[str, float]:
+    # Use exact cumulative counters: the equivalent server log values are rounded
+    # and emitted separately for each request.
+    res = server.make_request("GET", "/metrics")
+    assert res.status_code == 200
+    values = {}
+    for name in ("draft_tokens", "accepted_tokens", "drafts"):
+        m = re.search(
+            rf"^llamacpp:spec_decode_num_{name}_total (\S+)$", res.body, re.MULTILINE
+        )
+        assert m is not None, f"missing spec_decode_num_{name}_total in /metrics"
+        values[name] = float(m.group(1))
+    return values
+
+
+def selected_cases():
+    wanted = os.environ.get("SPEC_METHODS")
+    if not wanted:
+        return []
+    aliases = {
+        "all": {case.name for case in CASES},
+        "compact": COMPACT_METHODS,
+        "nightly": NIGHTLY_METHODS,
+    }
+    selected = aliases.get(
+        wanted, {name.strip() for name in wanted.split(",") if name.strip()}
+    )
+    unknown = selected - {c.name for c in CASES}
+    if unknown:
+        raise ValueError(f"unknown SPEC_METHODS: {', '.join(sorted(unknown))}")
+    return [case for case in CASES if case.name in selected]
+
+
+@pytest.mark.heavy
+@pytest.mark.slow
+@pytest.mark.parametrize("case", selected_cases(), ids=lambda c: c.name)
+def test_spec_method(case: SpecCase):
+    server = make_server(case)
+    try:
+        server.start(timeout_seconds=1800)
+        outputs = generate(server)
+        m = spec_metrics(server)
+    finally:
+        server.stop()
+
+    accept_rate = m["accepted_tokens"] / m["draft_tokens"] if m["draft_tokens"] else 0.0
+    accept_len = 1.0 + m["accepted_tokens"] / m["drafts"] if m["drafts"] else 1.0
+    print(
+        f"SPEC_RESULT {case.name}: drafted={m['draft_tokens']:.0f} accepted={m['accepted_tokens']:.0f} "
+        f"steps={m['drafts']:.0f} accept_rate={accept_rate:.4f} accept_len={accept_len:.4f}"
+    )
+
+    assert all(outputs), f"{case.name}: at least one request returned no tokens"
+    assert m["draft_tokens"] > 0, f"{case.name}: no tokens were drafted"
+    assert m["drafts"] > 0, f"{case.name}: no speculative verification steps ran"
+    assert (
+        accept_rate >= case.min_accept_rate
+    ), f"{case.name}: accept rate {accept_rate:.3f} < {case.min_accept_rate}"
+    assert (
+        accept_len >= case.min_accept_len
+    ), f"{case.name}: accept length {accept_len:.3f} < {case.min_accept_len}"

tools/server/tests/utils.py

@@ -102,6 +102,7 @@ class ServerProcess:
     spec_draft_n_min: int | None = None
     spec_draft_n_max: int | None = None
     spec_draft_n_ubatch: int | None = None
+    spec_draft_n_gpu_layer: int | None = None
     spec_synth_len: float | None = None
     spec_synth_rates: List[float] | None = None
     no_ui: bool | None = None
@@ -259,6 +260,8 @@ class ServerProcess:
             server_args.extend(["--spec-draft-n-min", self.spec_draft_n_min])
         if self.spec_draft_n_ubatch:
             server_args.extend(["--spec-draft-n-ubatch", self.spec_draft_n_ubatch])
+        if self.spec_draft_n_gpu_layer:
+            server_args.extend(["--spec-draft-ngl", self.spec_draft_n_gpu_layer])
         if self.spec_synth_len is not None:
             server_args.extend(["--spec-synth-len", self.spec_synth_len])
         if self.spec_synth_rates is not None: