upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-02 19:07:25 -05:00
Code Issues Packages Projects Releases Wiki Activity

metrics: seperate cache/processed prompt tokens

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-12 13:54:36 +02:00
parent a55abe44aa
commit e0f113decb
4 changed files with 32 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-common.h
+16 -6
View File
@@ -443,13 +443,17 @@ struct server_metrics {
}
};
// prompt tokens reused from the cache need no decode, so they only have a count
// these are reset by reset_bucket()
bucket prompt_bucket;
bucket predict_bucket;
bucket prompt_bucket;
bucket predict_bucket;
uint64_t n_prompt_cached_bucket = 0;
// metrics below are cumulative since the server started
bucket prompt;
bucket predict;
bucket prompt; // only processed tokens, not cached tokens
bucket predict;
uint64_t n_prompt_cached = 0;
uint64_t n_tokens_max = 0;
@@ -472,8 +476,9 @@ struct server_metrics {
}
void reset_bucket() {
prompt_bucket = {};
predict_bucket = {};
prompt_bucket = {};
predict_bucket = {};
n_prompt_cached_bucket = 0;
}
void add_prompt(uint64_t n_tokens, uint64_t t_us) {
@@ -481,6 +486,11 @@ struct server_metrics {
prompt_bucket.add(n_tokens, n_tokens, t_us);
}
void add_prompt_cached(uint64_t n_tokens) {
n_prompt_cached += n_tokens;
n_prompt_cached_bucket += n_tokens;
}
void on_decode_start() {
t_decode_start = ggml_time_us();
}
tools/server/server-context.cpp
+12 -4
View File
@@ -2434,11 +2434,13 @@ private:
res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size();
res->t_start = metrics.t_start;
res->prompt_bucket = metrics.prompt_bucket;
res->predict_bucket = metrics.predict_bucket;
res->prompt_bucket = metrics.prompt_bucket;
res->predict_bucket = metrics.predict_bucket;
res->n_prompt_cached_bucket = metrics.n_prompt_cached_bucket;
res->prompt = metrics.prompt;
res->predict = metrics.predict;
res->prompt = metrics.prompt;
res->predict = metrics.predict;
res->n_prompt_cached = metrics.n_prompt_cached;
res->n_tokens_max = metrics.n_tokens_max;
@@ -3290,6 +3292,8 @@ private:
slot.stats.n_prompt_cached = n_past;
slot.stats.n_prompt_processed = 0;
metrics.add_prompt_cached(n_past);
slot.prompt.tokens.keep_first(n_past);
// this is to signal the client that the request has started processing
@@ -4402,6 +4406,10 @@ void server_routes::init_routes() {
{"name", "prompt_tokens_total"},
{"help", "Number of prompt tokens processed."},
{"value", res_task->prompt.count}
}, {
{"name", "prompt_tokens_cached_total"},
{"help", "Number of prompt tokens reused from the cache."},
{"value", res_task->n_prompt_cached}
}, {
{"name", "prompt_seconds_total"},
{"help", "Prompt process time"},
tools/server/server-task.cpp
+2
View File
@@ -1518,6 +1518,7 @@ json server_task_result_metrics::to_json() {
{ "t_start", t_start },
{ "n_prompt_tokens_processed_total", prompt.count },
{ "n_prompt_tokens_cached_total", n_prompt_cached },
{ "t_tokens_generation_total", predict.time / 1e3 },
{ "n_tokens_predicted_total", predict.count },
{ "t_prompt_processing_total", prompt.time / 1e3 },
@@ -1525,6 +1526,7 @@ json server_task_result_metrics::to_json() {
{ "n_tokens_max", n_tokens_max },
{ "n_prompt_tokens_processed", prompt_bucket.count },
{ "n_prompt_tokens_cached", n_prompt_cached_bucket },
{ "t_prompt_processing", prompt_bucket.time / 1e3 },
{ "n_tokens_predicted", predict_bucket.count },
{ "t_tokens_generation", predict_bucket.time / 1e3 },
tools/server/server-task.h
+2
View File
@@ -499,9 +499,11 @@ struct server_task_result_metrics : server_task_result {
// note: the fields below mirror server_metrics, keep the names in-sync
server_metrics::bucket prompt_bucket;
server_metrics::bucket predict_bucket;
uint64_t n_prompt_cached_bucket = 0;
server_metrics::bucket prompt;
server_metrics::bucket predict;
uint64_t n_prompt_cached = 0;
uint64_t n_tokens_max = 0;