From a035a88878ad4d48c1e1b41cf83b0c11aea64bdb Mon Sep 17 00:00:00 2001 From: Bhavik Sharda <10757940+BLSharda@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:06:01 +0530 Subject: [PATCH] server: Adding spec-decode counters to /metrics endpoint (#26389) * * server: add spec-decode counters to /metrics endpoint * server: fixed review comments and now aligned param names exactly with vLLM. --- tools/server/README.md | 4 +++ tools/server/server-context.cpp | 44 +++++++++++++++++++++++++++++++++ tools/server/server-task.cpp | 5 ++++ tools/server/server-task.h | 5 ++++ 4 files changed, 58 insertions(+) diff --git a/tools/server/README.md b/tools/server/README.md index c8e7fcdcf2..4d80f059d5 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1076,6 +1076,10 @@ In *router mode* the query param `?model={model_id}` has to be set. This endpoin | `llamacpp:n_tokens_max` | Counter | High watermark of the context size observed. | | `llamacpp:n_decode_total` | Counter | Total Number of llama_decode() calls. | | `llamacpp:n_busy_slots_per_decode` | Gauge | Average number of busy slots per llama_decode() call. | +| `llamacpp:spec_decode_num_draft_tokens_total` | Counter | Total draft tokens generated (0 when spec-decode is off). | +| `llamacpp:spec_decode_num_accepted_tokens_total` | Counter | Total draft tokens accepted by the target model (0 when spec-decode is off). | +| `llamacpp:spec_decode_num_drafts_total` | Counter | Total speculative decoding verification steps (0 when spec-decode is off). | +| `llamacpp:spec_decode_num_accepted_tokens_per_pos_total` | Counter | Accepted tokens per draft position (labeled `position="N"`; absent when spec-decode is off or before the first completed speculative request). | ### POST `/slots/{id_slot}?action=save`: Save the prompt cache of the specified slot to a file. diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 380e62af67..38d2e5c7a0 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -845,6 +845,11 @@ struct server_metrics { uint64_t n_decode_total = 0; uint64_t n_busy_slots_total = 0; + uint64_t n_draft_tokens_total = 0; + uint64_t n_draft_accepted_total = 0; + uint64_t n_draft_verif_steps_total = 0; + std::vector n_accepted_per_pos_total; + void init() { t_start = ggml_time_us(); } @@ -863,6 +868,17 @@ struct server_metrics { n_tokens_predicted += slot.n_decoded; t_tokens_generation += slot.t_token_generation; t_tokens_generation_total += slot.t_token_generation; + + n_draft_tokens_total += slot.n_draft_total; + n_draft_accepted_total += slot.n_draft_accepted; + n_draft_verif_steps_total += slot.n_draft_verif_steps; + + if (n_accepted_per_pos_total.size() < slot.n_accepted_per_pos.size()) { + n_accepted_per_pos_total.resize(slot.n_accepted_per_pos.size(), 0); + } + for (size_t i = 0; i < slot.n_accepted_per_pos.size(); i++) { + n_accepted_per_pos_total[i] += slot.n_accepted_per_pos[i]; + } } void on_decoded(const std::vector & slots) { @@ -2552,6 +2568,11 @@ private: res->n_decode_total = metrics.n_decode_total; res->n_busy_slots_total = metrics.n_busy_slots_total; + res->n_draft_tokens_total = metrics.n_draft_tokens_total; + res->n_draft_accepted_total = metrics.n_draft_accepted_total; + res->n_draft_verif_steps_total = metrics.n_draft_verif_steps_total; + res->n_accepted_per_pos_total = metrics.n_accepted_per_pos_total; + if (task.metrics_reset_bucket) { metrics.reset_bucket(); } @@ -4440,6 +4461,18 @@ void server_routes::init_routes() { {"name", "n_tokens_max"}, {"help", "Largest observed n_tokens."}, {"value", res_task->n_tokens_max} + }, { + {"name", "spec_decode_num_draft_tokens_total"}, + {"help", "Total draft tokens generated"}, + {"value", res_task->n_draft_tokens_total} + }, { + {"name", "spec_decode_num_accepted_tokens_total"}, + {"help", "Total draft tokens accepted by the target model"}, + {"value", res_task->n_draft_accepted_total} + }, { + {"name", "spec_decode_num_drafts_total"}, + {"help", "Total speculative decoding verification steps"}, + {"value", res_task->n_draft_verif_steps_total} }}}, {"gauge", {{ {"name", "prompt_tokens_seconds"}, @@ -4481,6 +4514,17 @@ void server_routes::init_routes() { } } + // labeled counter: one time series per draft position + if (!res_task->n_accepted_per_pos_total.empty()) { + prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos_total" + " Accepted tokens per draft position\n" + << "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos_total counter\n"; + for (size_t i = 0; i < res_task->n_accepted_per_pos_total.size(); i++) { + prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\"" + << i << "\"} " << res_task->n_accepted_per_pos_total[i] << "\n"; + } + } + res->headers["Process-Start-Time-Unix"] = std::to_string(res_task->t_start); res->content_type = "text/plain; version=0.0.4"; res->status = 200; diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 070f1ade24..1ee6775530 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -1560,6 +1560,11 @@ json server_task_result_metrics::to_json() { { "n_decode_total", n_decode_total }, { "n_busy_slots_total", n_busy_slots_total }, + { "n_draft_tokens_total", n_draft_tokens_total }, + { "n_draft_accepted_total", n_draft_accepted_total }, + { "n_draft_verif_steps_total", n_draft_verif_steps_total }, + { "n_accepted_per_pos_total", n_accepted_per_pos_total }, + { "slots", slots_data }, }; } diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 411d918079..6275ec7604 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -532,6 +532,11 @@ struct server_task_result_metrics : server_task_result { uint64_t n_decode_total = 0; uint64_t n_busy_slots_total = 0; + uint64_t n_draft_tokens_total = 0; + uint64_t n_draft_accepted_total = 0; + uint64_t n_draft_verif_steps_total = 0; + std::vector n_accepted_per_pos_total; + // while we can also use std::vector this requires copying the slot object which can be quite messy // therefore, we use json to temporarily store the slot.to_json() result json slots_data = json::array();