Compare commits

...
Author SHA1 Message Date
Xuan Son Nguyen 801aec5270 add experimental warn 2026-11-15 18:31:24 +01:00
Xuan Son Nguyen cb227fef47 serialize ops on shared cache dir (for router mode) 2026-11-15 18:26:05 +01:00
Xuan Son Nguyen 65ba486468 improve file header 2026-11-15 17:45:37 +01:00
Xuan Son Nguyen a1c4ad57b0 fix prometheus bug on large number 2026-11-15 17:35:47 +01:00
Xuan Son Nguyen 4348c46dc1 --cache-disk-max-size 2026-11-15 09:51:23 +01:00
Xuan Son Nguyen 65810420d2 add docs 2026-11-15 09:50:10 +01:00
Xuan Son Nguyen 6833482947 add metrics 2026-11-15 09:50:06 +01:00
Xuan Son Nguyen 0a68f9b8d4 server: implement disk cache 2026-11-15 09:38:47 +01:00
13 changed files with 1285 additions and 29 deletions
+20
View File
@@ -1700,6 +1700,26 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.cache_ram_explicit = true;
}
).set_env("LLAMA_ARG_CACHE_RAM").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
add_opt(common_arg(
{"-cdisk", "--cache-disk"}, "PATH",
"experimental: directory of the disk prompt cache: the prompts that are dropped from the RAM cache are saved there and loaded back when needed, also after a restart (default: disabled, requires cache-ram)",
[](common_params & params, const std::string & value) {
if (!fs_is_directory(value)) {
throw std::invalid_argument("not a directory: " + value);
}
params.cache_disk_path = value;
}
).set_env("LLAMA_ARG_CACHE_DISK").set_examples({LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"--cache-disk-max-size"}, "N",
string_format("set the maximum size of the disk prompt cache directory in MiB, the oldest files are removed first (default: %d, -1 - no limit)", params.cache_disk_max_size_mib),
[](common_params & params, int value) {
if (value == 0 || value < -1) {
throw std::invalid_argument("cache-disk-max-size must be positive or -1 (no limit)");
}
params.cache_disk_max_size_mib = value;
}
).set_env("LLAMA_ARG_CACHE_DISK_MAX_SIZE").set_examples({LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"-kvu", "--kv-unified"},
{"-no-kvu", "--no-kv-unified"},
+2
View File
@@ -663,6 +663,8 @@ struct common_params {
int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints
int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
bool cache_ram_explicit = false; // set by --cache-ram, decision models disable the cache otherwise
std::string cache_disk_path; // directory of the disk prompt cache, empty = disabled
int32_t cache_disk_max_size_mib = -1; // -1 = no limit
// llama-connect params
bool server_connect = false;
+188
View File
@@ -4,11 +4,15 @@
#include "../tools/server/server-common.h"
#include "../tools/server/server-decision.h"
#include "../tools/server/server-schema.h"
#include "../tools/server/server-slot-sched.h"
#include <algorithm>
#include <cmath>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <random>
#include <regex>
#include <string>
#include <thread>
@@ -232,6 +236,189 @@ static void test_server_tokens_prefix_hashes(testing & t) {
}
}
//
// server_prompt_cache_disk
//
// the prompts share the first n_tokens - 1 tokens when only "last" differs
static server_prompt_cache_state make_cache_state(size_t n_tokens, llama_token last, size_t size_main) {
llama_tokens toks(n_tokens);
for (size_t i = 0; i < n_tokens; i++) {
toks[i] = 100 + (llama_token) i;
}
toks.back() = last;
server_prompt_cache_state res;
res.prompt.tokens = server_tokens(toks, false);
res.data.main.assign(size_main, (uint8_t) last);
res.data.drft.assign(3, 7);
auto ckpt = std::make_shared<common_prompt_checkpoint>();
ckpt->n_tokens = n_tokens / 2;
ckpt->pos_min = 1;
ckpt->pos_max = (llama_pos) n_tokens / 2;
ckpt->data_tgt = { 1, 2, 3, 4 };
ckpt->data_spec = { 5 };
res.prompt.checkpoints.push_back(std::move(ckpt));
return res;
}
static void test_server_prompt_cache_disk(testing & t) {
namespace fs = std::filesystem;
const size_t n_blk = server_tokens::hash_block_size;
const size_t n_tok = 2 * n_blk + 10;
const fs::path dir = fs::temp_directory_path() / ("test-server-cache-disk-" + std::to_string(std::random_device{}()));
fs::create_directories(dir);
const std::string compat = "0123456789abcdef";
const auto state = make_cache_state(n_tok, 1, 1000);
const auto & tokens = state.prompt.tokens;
{
const auto other = make_cache_state(n_tok, 2, 1000);
t.assert_true("get_hash, same tokens", tokens.get_hash(n_tok) == tokens.clone().get_hash(n_tok));
t.assert_true("get_hash, same prefix", tokens.get_hash(n_tok - 1) == other.prompt.tokens.get_hash(n_tok - 1));
t.assert_true("get_hash, other last token", tokens.get_hash(n_tok) != other.prompt.tokens.get_hash(n_tok));
t.assert_true("get_hash, other length", tokens.get_hash(n_tok) != tokens.get_hash(n_tok - 1));
t.assert_true("get_hash, not a chain hash", tokens.get_hash(n_blk) != tokens.get_prefix_hashes()[0]);
}
{
server_prompt_cache_disk disk(dir.string(), compat, 0, false);
t.assert_equal("empty dir", (size_t) 0, disk.get_entries().size());
t.assert_true("save", disk.save(state));
t.assert_true("save again", disk.save(state));
t.assert_equal("one file", (size_t) 1, disk.get_entries().size());
t.assert_equal("size", (size_t) fs::file_size(dir / disk.get_entries().front().name), disk.size());
}
{
// the files are found again after a restart
server_prompt_cache_disk disk(dir.string(), compat, 0, false);
t.assert_equal("scan", (size_t) 1, disk.get_entries().size());
const auto & entry = disk.get_entries().front();
t.assert_equal("n_tokens", n_tok, entry.n_tokens);
t.assert_true("hashes", entry.hashes == tokens.get_prefix_hashes());
const auto other = make_cache_state(n_tok + 50, 2, 10);
t.assert_equal("lookup", 2 * n_blk, other.prompt.tokens.get_common_prefix_lower(entry.hashes));
const size_t n_max = 100000;
server_prompt_cache_state loaded;
t.assert_true("load, not enough shared", !disk.load(other.prompt.tokens, 2 * n_blk, n_max, loaded));
t.assert_true("load, prompt is too long", !disk.load(other.prompt.tokens, 0, n_tok - 1, loaded));
t.assert_true("load", disk.load(other.prompt.tokens, 2 * n_blk - 1, n_max, loaded));
t.assert_equal("tokens", n_tok, loaded.prompt.tokens.get_common_prefix(tokens));
t.assert_equal("tokens size", n_tok, loaded.prompt.tokens.size());
t.assert_true("state", loaded.data.main == state.data.main);
t.assert_true("draft state", loaded.data.drft == state.data.drft);
t.assert_equal("checkpoints", (size_t) 1, loaded.prompt.checkpoints.size());
const auto & ckpt = *loaded.prompt.checkpoints.front();
const auto & ckpt_ref = *state.prompt.checkpoints.front();
t.assert_equal("checkpoint n_tokens", ckpt_ref.n_tokens, ckpt.n_tokens);
t.assert_equal("checkpoint pos_min", ckpt_ref.pos_min, ckpt.pos_min);
t.assert_equal("checkpoint pos_max", ckpt_ref.pos_max, ckpt.pos_max);
t.assert_true("checkpoint data", ckpt.data_tgt == ckpt_ref.data_tgt && ckpt.data_dft.empty() && ckpt.data_spec == ckpt_ref.data_spec);
// same length, other last token: both are kept
t.assert_true("save other", disk.save(make_cache_state(n_tok, 2, 1000)));
t.assert_equal("two files", (size_t) 2, disk.get_entries().size());
// a longer prompt replaces the prompts that are its prefix
auto longer = make_cache_state(n_tok, 1, 1000);
longer.prompt.tokens.push_back(3);
t.assert_true("save longer", disk.save(longer));
t.assert_equal("prefix removed", (size_t) 2, disk.get_entries().size());
t.assert_equal("longer is the newest", n_tok + 1, disk.get_entries().back().n_tokens);
}
{
// a file that is not a cache file is left alone
const fs::path path_other = dir / "notes.cache";
std::ofstream(path_other) << "not a cache file";
// another model sees the files but cannot use them
server_prompt_cache_disk disk(dir.string(), "fedcba9876543210", 0, false);
t.assert_equal("other compat, files", (size_t) 2, disk.get_entries().size());
t.assert_equal("other compat, n_tokens", (size_t) 0, disk.get_entries().front().n_tokens);
t.assert_true("other compat, size", disk.size() > 2000);
t.assert_true("file name", string_starts_with(disk.get_entries().front().name, "0123456789abcdef-") && string_ends_with(disk.get_entries().front().name, ".cache"));
}
{
// room for two files of this size
server_prompt_cache_disk disk(dir.string(), compat, 4000, false);
const std::string oldest = disk.get_entries().front().name;
t.assert_true("save over the limit", disk.save(make_cache_state(n_tok, 4, 1000)));
t.assert_equal("oldest removed", (size_t) 2, disk.get_entries().size());
t.assert_true("oldest file removed", !fs::exists(dir / oldest));
t.assert_true("too large", !disk.save(make_cache_state(n_tok, 5, 5000)));
}
{
// a broken file is removed
server_prompt_cache_disk disk(dir.string(), compat, 0, false);
const auto & entry = disk.get_entries().front();
const fs::path path = dir / entry.name;
fs::resize_file(path, fs::file_size(path) - 1);
// the other file matches the same prompt and is loaded in its place
server_prompt_cache_state loaded;
t.assert_true("load with a broken file", disk.load(tokens, 0, 100000, loaded));
t.assert_equal("broken file removed", (size_t) 1, disk.get_entries().size());
t.assert_true("broken file deleted", !fs::exists(path));
const fs::path path_last = dir / disk.get_entries().front().name;
fs::resize_file(path_last, fs::file_size(path_last) - 1);
server_prompt_cache_disk disk_new(dir.string(), compat, 0, false);
t.assert_equal("broken file removed by the scan", (size_t) 0, disk_new.get_entries().size());
}
t.assert_true("other file is kept", fs::exists(dir / "notes.cache"));
{
// two servers with different models share the directory, both started on an empty directory
fs::remove_all(dir);
fs::create_directories(dir);
server_prompt_cache_disk disk_a(dir.string(), compat, 0, false);
server_prompt_cache_disk disk_b(dir.string(), "fedcba9876543210", 4000, false);
t.assert_true("shared, save a", disk_a.save(make_cache_state(n_tok, 1, 1000)));
t.assert_true("shared, save a again", disk_a.save(make_cache_state(n_tok, 2, 1000)));
const std::string oldest = disk_a.get_entries().front().name;
// b sees the files of a and removes the oldest one to stay under its limit
t.assert_true("shared, save b", disk_b.save(make_cache_state(n_tok, 3, 1000)));
t.assert_equal("shared, b sees all files", (size_t) 2, disk_b.get_entries().size());
t.assert_true("shared, oldest file removed", !fs::exists(dir / oldest));
// a finds out when it looks for the removed prompt, the other one is still there
const auto state_1 = make_cache_state(n_tok, 1, 1000);
const auto state_2 = make_cache_state(n_tok, 2, 1000);
server_prompt_cache_state loaded;
t.assert_true("shared, removed prompt", !disk_a.load(state_1.prompt.tokens, n_tok - 1, 100000, loaded));
t.assert_equal("shared, a sees all files", (size_t) 2, disk_a.get_entries().size());
t.assert_true("shared, kept prompt", disk_a.load(state_2.prompt.tokens, 0, 100000, loaded));
t.assert_equal("shared, kept prompt tokens", n_tok, loaded.prompt.tokens.get_common_prefix(state_2.prompt.tokens));
// a file left by a killed server is removed
const fs::path path_tmp = dir / (oldest + ".tmp");
std::ofstream(path_tmp) << "partial";
t.assert_true("shared, save a after tmp", disk_a.save(make_cache_state(n_tok, 4, 1000)));
t.assert_true("shared, tmp file removed", !fs::exists(path_tmp));
}
fs::remove_all(dir);
}
static void test_server_tokens(testing & t) {
t.test("basic", test_server_tokens_basic);
t.test("pos", test_server_tokens_pos);
@@ -830,6 +1017,7 @@ int main(int argc, char ** argv) {
}
t.test("server_tokens", test_server_tokens);
t.test("server_prompt_cache_disk", test_server_prompt_cache_disk);
t.test("utf8", test_utf8);
t.test("sse", test_sse);
t.test("json_helpers", test_json_helpers);
+35
View File
@@ -167,6 +167,8 @@ For the full list of features, please refer to [server's changelog](https://gith
| `-ctxcp, --ctx-checkpoints, --swa-checkpoints N` | max number of context checkpoints to create per slot (default: 32)[(more info)](https://github.com/ggml-org/llama.cpp/pull/15293)<br/>(env: LLAMA_ARG_CTX_CHECKPOINTS) |
| `-cms, --checkpoint-min-step N` | minimum spacing between context checkpoints in tokens (default: 8192, 0 = no minimum)<br/>(env: LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT) |
| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, 0 for decision models, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)<br/>(env: LLAMA_ARG_CACHE_RAM) |
| `-cdisk, --cache-disk PATH` | experimental: directory of the disk prompt cache: the prompts that are dropped from the RAM cache are saved there and loaded back when needed, also after a restart (default: disabled, requires cache-ram)<br/>(env: LLAMA_ARG_CACHE_DISK) |
| `--cache-disk-max-size N` | set the maximum size of the disk prompt cache directory in MiB, the oldest files are removed first (default: -1, -1 - no limit)<br/>(env: LLAMA_ARG_CACHE_DISK_MAX_SIZE) |
| `-kvu, --kv-unified, -no-kvu, --no-kv-unified` | use single unified KV buffer shared across all sequences (default: enabled if number of slots is auto)<br/>(env: LLAMA_ARG_KV_UNIFIED) |
| `--cache-idle-slots, --no-cache-idle-slots` | save idle slots to the prompt cache on new task, and clear them when using unified KV (default: enabled, requires cache-ram)<br/>(env: LLAMA_ARG_CACHE_IDLE_SLOTS) |
| `--context-shift, --no-context-shift` | whether to use context shift on infinite text generation (default: disabled)<br/>(env: LLAMA_ARG_CONTEXT_SHIFT) |
@@ -345,6 +347,30 @@ services:
LLAMA_ARG_PORT: 8080
```
### Prompt cache on disk
This feature is experimental: its behavior, options and file format can change.
With `--cache-disk PATH`, the prompts that no longer fit in the RAM cache (`--cache-ram`) are kept in a directory, and they are still there after a restart. The directory must exist, and the RAM cache must be enabled.
```sh
llama-server -m model.gguf --cache-disk /path/to/cache --cache-disk-max-size 32768
```
- **Saving:** a prompt is saved when the RAM cache drops it, and when the server shuts down or goes to sleep (idle slots included). A server that is killed loses the prompts that were only in RAM.
- **Loading:** for each request, the saved prompt that shares the longest prefix with it is loaded, if it shares more than the slot and the RAM cache do. It does not need to be a full prefix: the state is cut where the prompts differ. Context checkpoints are saved too.
- **Size:** a file is about as large as the same prompt in the RAM cache. With `--cache-disk-max-size N`, the least recently used files are removed to stay under `N` MiB. There is no limit by default.
- **Compatibility:** a file is only used with the same model, mmproj, draft model, LoRA adapters, control vectors, KV cache types, `--swa-full` and RoPE/YaRN options. Other files are ignored, but they count in the size limit.
- **Not saved:** prompts shorter than 64 tokens, prompts processed with a LoRA configuration other than the one given at startup, and prompts restored from a slot file.
Notes:
- The files hold the prompt tokens as they are, so anyone who can read the directory can read the prompts.
- While a large state is written or read, the other requests wait.
- Several servers can share the directory, for example the models of a router, and `--cache-disk-max-size` then applies to all of them together. They take turns: while one reads or writes a file, the others wait.
- In router mode, a model that is unloaded has `stop-timeout` seconds to save its prompts.
- With `--metrics`, see the `llamacpp:prompt_cache_disk_*` metrics in [`/metrics`](#get-metrics-prometheus-compatible-metrics-exporter).
### Multimodal support
Multimodal support was added in [#12898](https://github.com/ggml-org/llama.cpp/pull/12898) and is currently an experimental feature.
@@ -1251,6 +1277,15 @@ In *router mode* the query param `?model={model_id}` has to be set. This endpoin
| `llamacpp:spec_decode_num_draft_tokens_total` | Counter | Total draft tokens generated (0 when spec-decode is off). |
| `llamacpp:spec_decode_num_accepted_tokens_total` | Counter | Total draft tokens accepted by the target model (0 when spec-decode is off). |
| `llamacpp:spec_decode_num_drafts_total` | Counter | Total speculative decoding verification steps (0 when spec-decode is off). |
| `llamacpp:prompt_cache_disk_saves_total` | Counter | Number of prompts saved to the disk prompt cache (the disk cache metrics are 0 without `--cache-disk`). |
| `llamacpp:prompt_cache_disk_save_bytes_total` | Counter | Number of bytes written to the disk prompt cache. |
| `llamacpp:prompt_cache_disk_save_seconds_total` | Counter | Time spent saving prompts to the disk prompt cache in seconds. |
| `llamacpp:prompt_cache_disk_loads_total` | Counter | Number of prompts loaded from the disk prompt cache. |
| `llamacpp:prompt_cache_disk_load_bytes_total` | Counter | Number of bytes read from the disk prompt cache. |
| `llamacpp:prompt_cache_disk_load_seconds_total` | Counter | Time spent loading prompts from the disk prompt cache in seconds. |
| `llamacpp:prompt_cache_disk_evictions_total` | Counter | Number of files removed from the disk prompt cache to stay under `--cache-disk-max-size`. |
| `llamacpp:prompt_cache_disk_files` | Gauge | Number of files in the disk prompt cache directory, the files of other models included. |
| `llamacpp:prompt_cache_disk_bytes` | Gauge | Size of the disk prompt cache directory in bytes, the files of other models included. |
| `llamacpp:spec_decode_num_accepted_tokens_per_pos_total` | Counter | Accepted tokens per draft position (labeled `position="N"`; absent when spec-decode is off or before the first completed speculative request). |
### POST `/slots/{id_slot}?action=save`: Save the prompt cache of the specified slot to a file.
+37 -20
View File
@@ -834,6 +834,7 @@ void server_tokens::keep_first(size_t n) {
}
tokens.resize(n);
hashes.resize(std::min(hashes.size(), n / hash_block_size));
update_hashes();
}
std::string server_tokens::detokenize(const llama_context * ctx, bool special) const {
@@ -900,37 +901,53 @@ size_t server_tokens::get_common_prefix(const server_tokens & b) const {
return max_idx; // all tokens are equal
}
void server_tokens::update_hashes() {
const size_t n_blocks = tokens.size() / hash_block_size;
server_tokens::prefix_hash server_tokens::hash_range(const prefix_hash & prev, size_t i_begin, size_t i_end) const {
std::string buf;
const auto append = [&buf](const void * data, size_t len) {
buf.append((const char *) data, len);
};
for (size_t k = hashes.size(); k < n_blocks; k++) {
const size_t i_begin = k * hash_block_size;
const size_t i_end = i_begin + hash_block_size;
append(prev.data(), prev.size());
append(tokens.data() + i_begin, (i_end - i_begin) * sizeof(llama_token));
// the previous hash comes first, zeros for the first block
// a media chunk is hashed at its first token, it can span over many blocks
for (auto it = map_idx_to_media.lower_bound(i_begin); it != map_idx_to_media.end() && it->first < i_end; ++it) {
const std::string_view id = mtmd_input_chunk_get_id(it->second.get());
const uint64_t fields[] = { it->first, mtmd_input_chunk_get_n_tokens(it->second.get()), id.size() };
append(fields, sizeof(fields));
append(id.data(), id.size());
}
return hash_sha256(buf.data(), buf.size());
}
void server_tokens::update_hashes() {
const size_t n_blocks = tokens.size() / hash_block_size;
for (size_t k = hashes.size(); k < n_blocks; k++) {
// zeros for the first block
const prefix_hash prev = k > 0 ? hashes[k - 1] : prefix_hash{};
buf.clear();
append(prev.data(), prev.size());
append(tokens.data() + i_begin, hash_block_size * sizeof(llama_token));
// a media chunk is hashed in the block of its first token, it can span over many blocks
for (auto it = map_idx_to_media.lower_bound(i_begin); it != map_idx_to_media.end() && it->first < i_end; ++it) {
const std::string_view id = mtmd_input_chunk_get_id(it->second.get());
const uint64_t fields[] = { it->first, mtmd_input_chunk_get_n_tokens(it->second.get()), id.size() };
append(fields, sizeof(fields));
append(id.data(), id.size());
}
hashes.push_back(hash_sha256(buf.data(), buf.size()));
hashes.push_back(hash_range(prev, k * hash_block_size, (k + 1) * hash_block_size));
}
}
server_tokens::prefix_hash server_tokens::get_hash(size_t n) const {
const size_t n_blocks = n / hash_block_size;
GGML_ASSERT(n <= tokens.size() && n_blocks <= hashes.size());
const prefix_hash prev = n_blocks > 0 ? hashes[n_blocks - 1] : prefix_hash{};
// the tokens after the last full block and the length, so it never equals a chain entry
const prefix_hash tail = hash_range(prev, n_blocks * hash_block_size, n);
std::string buf((const char *) tail.data(), tail.size());
const uint64_t n_tokens = n;
buf.append((const char *) &n_tokens, sizeof(n_tokens));
return hash_sha256(buf.data(), buf.size());
}
size_t server_tokens::get_common_prefix_lower(const std::vector<prefix_hash> & b_hashes) const {
const auto & a_hashes = hashes;
+21
View File
@@ -197,6 +197,9 @@ private: // disallow accessing these members directly, risking out-of-sync
void update_hashes();
// hash of the tokens in [i_begin, i_end), chained after prev
prefix_hash hash_range(const prefix_hash & prev, size_t i_begin, size_t i_end) const;
public:
server_tokens() = default;
~server_tokens() = default;
@@ -276,6 +279,9 @@ public:
const std::vector<prefix_hash> & get_prefix_hashes() const { return hashes; }
// hash of the first n tokens, n does not need to be at a block boundary
prefix_hash get_hash(size_t n) const;
// lower bound of get_common_prefix(), b is known only by its prefix hashes
// the result is less than one block plus one media chunk below the exact value
size_t get_common_prefix_lower(const std::vector<prefix_hash> & b_hashes) const;
@@ -523,6 +529,21 @@ struct server_slot_stats {
json to_json() const;
};
// for /metrics, the counters restart when the model is loaded again after sleeping
struct server_prompt_cache_disk_stats {
uint64_t n_saves = 0; // files written
uint64_t n_loads = 0; // files read
uint64_t n_evictions = 0; // files removed to stay under the size limit
uint64_t size_saved = 0; // in bytes
uint64_t size_loaded = 0; // in bytes
uint64_t t_save = 0; // in microseconds
uint64_t t_load = 0; // in microseconds
// current state of the directory, the files of other models included
uint64_t n_files = 0;
uint64_t size = 0; // in bytes
};
// shared between server_context_impl and server_task_result_*
// unlike server_slot_stats, server_metrics is server-global and cumulative, not tied to a slot
struct server_metrics {
+47 -1
View File
@@ -371,7 +371,7 @@ struct server_slot {
}
bool prompt_load(server_prompt_cache & prompt_cache, const server_tokens & tokens) {
bool res = prompt_cache.load(prompt, tokens, ctx_tgt, ctx_dft, id);
bool res = prompt_cache.load(prompt, tokens, ctx_tgt, ctx_dft, id, n_ctx);
if (!res) {
SLT_WRN(*this, "%s", "failed to load prompt from cache\n");
}
@@ -1115,6 +1115,9 @@ private:
std::unique_ptr<server_prompt_cache> prompt_cache;
// the lora config that the disk prompt cache is valid for
std::vector<common_adapter_lora_info> lora_init;
server_metrics metrics;
// queued prompt stats - llama_decode() is async, so the timing is only valid after a sync
@@ -1135,7 +1138,24 @@ private:
int64_t t_last_load_progress_ms = 0;
// keeps the prompts for the next start, the idle slots included
void flush_prompt_cache() {
if (!prompt_cache || !prompt_cache->disk) {
return;
}
for (auto & slot : slots) {
if (!slot.is_processing()) {
slot.prompt_save(*prompt_cache);
}
}
prompt_cache->flush_to_disk();
}
void destroy() {
flush_prompt_cache();
spec.reset();
spec_init.reset();
@@ -1605,8 +1625,23 @@ private:
SRV_TRC("%s", "use `--cache-ram 0` to disable the prompt cache\n");
prompt_cache = std::make_unique<server_prompt_cache>(params_base.cache_ram_mib, n_ctx);
if (!params_base.cache_disk_path.empty()) {
const size_t limit_size = 1024ull*1024ull*(params_base.cache_disk_max_size_mib < 0 ? 0 : params_base.cache_disk_max_size_mib);
lora_init = params_base.lora_adapters;
SRV_WRN("%s", "--cache-disk is experimental, its behavior and file format can change in the future\n");
prompt_cache->disk = std::make_unique<server_prompt_cache_disk>(
params_base.cache_disk_path, server_prompt_cache_disk_compat(params_base), limit_size, mctx != nullptr);
}
} else {
SRV_TRC("%s", "prompt cache is disabled - use `--cache-ram N` to enable it\n");
if (!params_base.cache_disk_path.empty()) {
SRV_WRN("%s", "--cache-disk needs the prompt cache, it is ignored\n");
}
}
SRV_TRC("%s", "for more info see https://github.com/ggml-org/llama.cpp/pull/16391\n");
@@ -1884,6 +1919,10 @@ private:
slot.lora = task_loras;
}
if (prompt_cache && prompt_cache->disk && !are_lora_equal(slot.lora, lora_init)) {
slot.prompt.lora_changed = true;
}
// if using alora, make sure it's only a single one requested and active
size_t alora_invocation_start = task.tokens.size();
if (lora_all_alora(slot.lora)) {
@@ -3108,6 +3147,10 @@ private:
res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size();
res->metrics = metrics;
if (prompt_cache && prompt_cache->disk) {
res->cache_disk = prompt_cache->disk->get_stats();
}
if (task.metrics_reset_bucket) {
metrics.reset_bucket();
}
@@ -3243,6 +3286,9 @@ private:
slot->prompt.clear();
slot->prompt.tokens = std::move(restored);
// the lora config of a slot file is not known
slot->prompt.lora_changed = true;
} catch (const std::exception & err) {
slot->prompt_clear();
send_error(task, std::string("Unable to restore slot: ") + err.what(), ERROR_TYPE_INVALID_REQUEST);
+572 -6
View File
@@ -1,10 +1,13 @@
#include "server-slot-sched.h"
#include "common.h"
#include "hash/hash.h"
#include "llama.h"
#include "log.h"
#include <algorithm>
#include <fstream>
#include <regex>
//
// server_prompt
@@ -57,6 +60,531 @@ server_prompt_reuse server_prompt::get_reuse(int32_t n_prefix, bool has_new_toke
return res;
}
//
// server_prompt_cache_disk
//
namespace fs = std::filesystem;
static const char * FILE_EXT = ".cache";
static const char * FILE_EXT_TMP = ".tmp";
// of any compat id, other files in the directory are left alone
static const std::regex re_file_name("[0-9a-f]{16}-[0-9a-f]{32}\\.cache");
static const std::regex re_file_name_tmp("[0-9a-f]{16}-[0-9a-f]{32}\\.cache\\.tmp");
// TODO: the lock is held for a whole operation, the reads and writes of the large files included
// this is simple and correct, but the servers that share the directory wait for each other
struct server_cache_disk_lock {
common_file_lock & lock;
server_cache_disk_lock(common_file_lock & lock) : lock(lock) {
static bool warned = false;
if (!lock.acquire() && !warned) {
warned = true;
SRV_WRN("%s", "failed to lock the disk prompt cache directory, do not share it with another server\n");
}
}
~server_cache_disk_lock() {
lock.close();
}
};
struct server_cache_disk_header {
uint32_t magic = 0x48434143; // "CACH"
uint32_t version = 1; // to be increased when the file format changes
uint64_t n_tokens = 0;
uint64_t n_hashes = 0;
uint64_t n_checkpoints = 0;
// in bytes
uint64_t size_tokens = 0; // server_tokens::serialize()
uint64_t size_main = 0;
uint64_t size_drft = 0;
uint64_t size_checkpoints = 0; // headers included
server_tokens::prefix_hash hash = {};
};
static_assert(sizeof(server_cache_disk_header) == 96, "unexpected header size");
struct server_cache_disk_checkpoint {
int64_t n_tokens = 0;
int32_t pos_min = 0;
int32_t pos_max = 0;
// in bytes
uint64_t size_tgt = 0;
uint64_t size_dft = 0;
uint64_t size_spec = 0;
};
static_assert(sizeof(server_cache_disk_checkpoint) == 40, "unexpected checkpoint header size");
static std::string make_name(const std::string & compat, const server_tokens::prefix_hash & hash) {
static const char hex[] = "0123456789abcdef";
// half of the hash is enough for a file name
std::string res = compat + "-";
for (size_t i = 0; i < hash.size() / 2; i++) {
res += hex[hash[i] >> 4];
res += hex[hash[i] & 0xf];
}
return res + FILE_EXT;
}
static void read_bytes(std::ifstream & in, void * data, size_t size) {
in.read((char *) data, size);
if (!in.good()) {
throw std::runtime_error("unexpected end of file");
}
}
static void write_bytes(std::ofstream & out, const void * data, size_t size) {
out.write((const char *) data, size);
}
// reads the header and the hash chain, the stream is left at the start of the tokens
static server_cache_disk_header read_header(std::ifstream & in, const std::string & compat, server_prompt_cache_disk_entry & entry) {
server_cache_disk_header header;
read_bytes(in, &header, sizeof(header));
if (header.magic != server_cache_disk_header().magic || header.version != server_cache_disk_header().version) {
throw std::runtime_error("unsupported file version");
}
const uint64_t size_hashes = header.n_hashes * sizeof(server_tokens::prefix_hash);
const uint64_t size_file = sizeof(header) + size_hashes + header.size_tokens + header.size_main + header.size_drft + header.size_checkpoints;
if (header.n_hashes != header.n_tokens / server_tokens::hash_block_size ||
header.size_tokens % sizeof(llama_token) != 0 ||
size_file != entry.size ||
make_name(compat, header.hash) != entry.name) {
throw std::runtime_error("invalid header");
}
entry.n_tokens = header.n_tokens;
entry.hash = header.hash;
entry.hashes.resize(header.n_hashes);
if (size_hashes > 0) {
read_bytes(in, entry.hashes.data(), size_hashes);
}
return header;
}
server_prompt_cache_disk::server_prompt_cache_disk(const std::string & dir, const std::string & compat, size_t limit_size, bool has_mtmd)
: dir(dir), compat(compat), limit_size(limit_size), has_mtmd(has_mtmd), lock((this->dir / ".lock").string()) {
server_cache_disk_lock guard(lock);
refresh();
const size_t n_usable = std::count_if(entries.begin(), entries.end(), [](const auto & entry) {
return entry.n_tokens > 0;
});
SRV_INF("disk prompt cache '%s': %zu prompts, %zu files of other models, %.3f MiB (limit: %.3f MiB)\n",
this->dir.string().c_str(), n_usable, entries.size() - n_usable, size() / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0));
}
void server_prompt_cache_disk::refresh() {
entry_list known = std::move(entries);
entries.clear();
std::error_code ec;
for (const auto & file : fs::directory_iterator(dir, ec)) {
if (!file.is_regular_file(ec)) {
continue;
}
const std::string name = file.path().filename().string();
// a file is written under the lock, so this one was left by a server that was killed
if (std::regex_match(name, re_file_name_tmp)) {
fs::remove(file.path(), ec);
continue;
}
if (!std::regex_match(name, re_file_name)) {
continue;
}
server_prompt_cache_disk_entry entry;
entry.name = name;
entry.size = file.file_size(ec);
entry.t_used = file.last_write_time(ec);
if (string_starts_with(name, compat + "-")) {
const auto it = std::find_if(known.begin(), known.end(), [&entry](const auto & cur) {
return cur.name == entry.name && cur.size == entry.size;
});
if (it != known.end()) {
// the header of a known file is not read again
entry.n_tokens = it->n_tokens;
entry.hash = it->hash;
entry.hashes = std::move(it->hashes);
known.erase(it);
} else {
try {
std::ifstream in(file.path(), std::ios::binary);
read_header(in, compat, entry);
} catch (const std::exception & e) {
SRV_WRN("removing '%s': %s\n", name.c_str(), e.what());
fs::remove(file.path(), ec);
continue;
}
}
}
entries.push_back(std::move(entry));
}
entries.sort([](const auto & a, const auto & b) {
return a.t_used < b.t_used;
});
}
size_t server_prompt_cache_disk::size() const {
size_t res = 0;
for (const auto & entry : entries) {
res += entry.size;
}
return res;
}
server_prompt_cache_disk_stats server_prompt_cache_disk::get_stats() const {
server_prompt_cache_disk_stats res = stats;
res.n_files = entries.size();
res.size = size();
return res;
}
server_prompt_cache_disk::entry_list::iterator server_prompt_cache_disk::remove(entry_list::iterator it) {
std::error_code ec;
fs::remove(dir / it->name, ec);
return entries.erase(it);
}
void server_prompt_cache_disk::touch(entry_list::iterator it) {
std::error_code ec;
it->t_used = fs::file_time_type::clock::now();
fs::last_write_time(dir / it->name, it->t_used, ec);
entries.splice(entries.end(), entries, it);
}
bool server_prompt_cache_disk::save(const server_prompt_cache_state & state) {
const auto & tokens = state.prompt.tokens;
// a prompt without a full hash block can never be found
if (tokens.size() < server_tokens::hash_block_size || state.data.main.empty()) {
return false;
}
server_cache_disk_header header;
header.hash = tokens.get_hash(tokens.size());
const std::string name = make_name(compat, header.hash);
std::vector<char> tokens_data;
try {
tokens_data = tokens.serialize();
} catch (const std::exception & e) {
SRV_WRN("failed to serialize the prompt: %s\n", e.what());
return false;
}
const auto & hashes = tokens.get_prefix_hashes();
header.n_tokens = tokens.size();
header.n_hashes = hashes.size();
header.n_checkpoints = state.prompt.checkpoints.size();
header.size_tokens = tokens_data.size();
header.size_main = state.data.main.size();
header.size_drft = state.data.drft.size();
for (const auto & ckpt : state.prompt.checkpoints) {
header.size_checkpoints += sizeof(server_cache_disk_checkpoint) + ckpt->data_tgt.size() + ckpt->data_dft.size() + ckpt->data_spec.size();
}
const size_t size_hashes = hashes.size() * sizeof(server_tokens::prefix_hash);
const size_t size_file = sizeof(header) + size_hashes + header.size_tokens + header.size_main + header.size_drft + header.size_checkpoints;
if (limit_size > 0 && size_file > limit_size) {
SRV_WRN(" - prompt size %.3f MiB exceeds disk cache size limit %.3f MiB, skipping\n",
size_file / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0));
return false;
}
server_cache_disk_lock guard(lock);
refresh();
for (auto it = entries.begin(); it != entries.end();) {
if (it->name == name) {
SRV_TRC(" - prompt is already on disk, '%s'\n", name.c_str());
touch(it);
return true;
}
if (0 < it->n_tokens && it->n_tokens < tokens.size() && tokens.get_hash(it->n_tokens) == it->hash) {
SRV_TRC(" - removing obsolete prompt with length %zu from disk, '%s'\n", it->n_tokens, it->name.c_str());
it = remove(it);
} else {
++it;
}
}
if (limit_size > 0) {
while (!entries.empty() && size() + size_file > limit_size) {
SRV_TRC(" - making room on disk, removing oldest file '%s' (size = %.3f MiB)\n",
entries.front().name.c_str(), entries.front().size / (1024.0 * 1024.0));
remove(entries.begin());
stats.n_evictions++;
}
}
const int64_t t_start = ggml_time_us();
const fs::path path = dir / name;
const fs::path path_tmp = dir / (name + FILE_EXT_TMP);
std::error_code ec;
{
std::ofstream out(path_tmp, std::ios::binary | std::ios::trunc);
write_bytes(out, &header, sizeof(header));
write_bytes(out, hashes.data(), size_hashes);
write_bytes(out, tokens_data.data(), tokens_data.size());
write_bytes(out, state.data.main.data(), state.data.main.size());
write_bytes(out, state.data.drft.data(), state.data.drft.size());
for (const auto & ckpt : state.prompt.checkpoints) {
server_cache_disk_checkpoint ckpt_header;
ckpt_header.n_tokens = ckpt->n_tokens;
ckpt_header.pos_min = ckpt->pos_min;
ckpt_header.pos_max = ckpt->pos_max;
ckpt_header.size_tgt = ckpt->data_tgt.size();
ckpt_header.size_dft = ckpt->data_dft.size();
ckpt_header.size_spec = ckpt->data_spec.size();
write_bytes(out, &ckpt_header, sizeof(ckpt_header));
write_bytes(out, ckpt->data_tgt.data(), ckpt->data_tgt.size());
write_bytes(out, ckpt->data_dft.data(), ckpt->data_dft.size());
write_bytes(out, ckpt->data_spec.data(), ckpt->data_spec.size());
}
out.flush();
if (!out.good()) {
SRV_ERR("failed to write '%s'\n", path_tmp.string().c_str());
out.close();
fs::remove(path_tmp, ec);
return false;
}
}
// a reader never sees a half written file
fs::rename(path_tmp, path, ec);
if (ec) {
SRV_ERR("failed to rename '%s': %s\n", path_tmp.string().c_str(), ec.message().c_str());
fs::remove(path_tmp, ec);
return false;
}
server_prompt_cache_disk_entry entry;
entry.name = name;
entry.size = size_file;
entry.t_used = fs::last_write_time(path, ec);
entry.n_tokens = tokens.size();
entry.hash = header.hash;
entry.hashes = hashes;
entries.push_back(std::move(entry));
stats.n_saves++;
stats.size_saved += size_file;
stats.t_save += ggml_time_us() - t_start;
SRV_INF("saved prompt with %zu tokens to disk, %.3f MiB, '%s'\n", tokens.size(), size_file / (1024.0 * 1024.0), name.c_str());
return true;
}
server_prompt_cache_disk::entry_list::iterator server_prompt_cache_disk::find(const server_tokens & tokens, size_t n_prefix_min, size_t n_tokens_max) {
auto it_best = entries.end();
size_t n_prefix_best = n_prefix_min;
for (auto it = entries.begin(); it != entries.end(); ++it) {
if (it->n_tokens == 0 || it->n_tokens > n_tokens_max) {
continue;
}
// only the hashes are known here, so this is a lower bound
const size_t n_prefix_cur = tokens.get_common_prefix_lower(it->hashes);
// don't trash large prompts
if (n_prefix_cur < 0.25f * it->n_tokens) {
continue;
}
if (n_prefix_cur > n_prefix_best) {
n_prefix_best = n_prefix_cur;
it_best = it;
}
}
return it_best;
}
bool server_prompt_cache_disk::load(const server_tokens & tokens, size_t n_prefix_min, size_t n_tokens_max, server_prompt_cache_state & state) {
// the lock is taken only if the files that are known so far have a match
if (find(tokens, n_prefix_min, n_tokens_max) == entries.end()) {
return false;
}
server_cache_disk_lock guard(lock);
refresh();
const auto it = find(tokens, n_prefix_min, n_tokens_max);
if (it == entries.end()) {
return false;
}
SRV_TRC(" - found better prompt on disk with length %zu\n", it->n_tokens);
return read(it, state);
}
bool server_prompt_cache_disk::read(entry_list::iterator it, server_prompt_cache_state & state) {
server_prompt_cache_state res;
const int64_t t_start = ggml_time_us();
try {
std::ifstream in(dir / it->name, std::ios::binary);
server_prompt_cache_disk_entry cur;
cur.name = it->name;
cur.size = it->size;
const auto header = read_header(in, compat, cur);
llama_tokens tokens_data(header.size_tokens / sizeof(llama_token));
read_bytes(in, tokens_data.data(), header.size_tokens);
res.prompt.tokens = server_tokens::deserialize(tokens_data, has_mtmd);
if (res.prompt.tokens.size() != it->n_tokens || res.prompt.tokens.get_hash(it->n_tokens) != it->hash) {
throw std::runtime_error("the tokens do not match the hash");
}
res.data.main.resize(header.size_main);
res.data.drft.resize(header.size_drft);
read_bytes(in, res.data.main.data(), header.size_main);
read_bytes(in, res.data.drft.data(), header.size_drft);
uint64_t size_checkpoints = 0;
for (uint64_t i = 0; i < header.n_checkpoints; i++) {
server_cache_disk_checkpoint ckpt_header;
read_bytes(in, &ckpt_header, sizeof(ckpt_header));
size_checkpoints += sizeof(ckpt_header) + ckpt_header.size_tgt + ckpt_header.size_dft + ckpt_header.size_spec;
if (size_checkpoints > header.size_checkpoints) {
throw std::runtime_error("invalid checkpoint size");
}
auto ckpt = std::make_shared<common_prompt_checkpoint>();
ckpt->n_tokens = ckpt_header.n_tokens;
ckpt->pos_min = ckpt_header.pos_min;
ckpt->pos_max = ckpt_header.pos_max;
ckpt->data_tgt.resize(ckpt_header.size_tgt);
ckpt->data_dft.resize(ckpt_header.size_dft);
ckpt->data_spec.resize(ckpt_header.size_spec);
read_bytes(in, ckpt->data_tgt.data(), ckpt_header.size_tgt);
read_bytes(in, ckpt->data_dft.data(), ckpt_header.size_dft);
read_bytes(in, ckpt->data_spec.data(), ckpt_header.size_spec);
res.prompt.checkpoints.push_back(std::move(ckpt));
}
} catch (const std::bad_alloc & e) {
SRV_ERR("failed to allocate memory for '%s': %s\n", it->name.c_str(), e.what());
return false;
} catch (const std::exception & e) {
SRV_WRN("removing '%s': %s\n", it->name.c_str(), e.what());
remove(it);
return false;
}
SRV_INF("loaded prompt with %zu tokens from disk, %.3f MiB, '%s'\n", it->n_tokens, it->size / (1024.0 * 1024.0), it->name.c_str());
stats.n_loads++;
stats.size_loaded += it->size;
stats.t_load += ggml_time_us() - t_start;
touch(it);
state = std::move(res);
return true;
}
template <typename T>
static void compat_add(std::string & buf, const T & value) {
static_assert(std::is_trivially_copyable<T>::value, "T must be trivially copyable");
buf.append((const char *) &value, sizeof(value));
}
static void compat_add(std::string & buf, const std::string & value) {
compat_add(buf, (uint64_t) value.size());
buf += value;
}
// a file is known by its path, size and time, reading a model to hash it takes too long
static void compat_add_file(std::string & buf, const std::string & path) {
std::error_code ec;
compat_add(buf, path);
compat_add(buf, (uint64_t) (path.empty() ? 0 : fs::file_size(path, ec)));
compat_add(buf, (int64_t) (path.empty() ? 0 : fs::last_write_time(path, ec).time_since_epoch().count()));
}
std::string server_prompt_cache_disk_compat(const common_params & params) {
std::string buf;
compat_add(buf, (uint32_t) LLAMA_STATE_SEQ_VERSION);
compat_add(buf, (uint64_t) server_tokens::hash_block_size);
compat_add_file(buf, params.model.path);
compat_add_file(buf, params.mmproj.path);
compat_add_file(buf, params.speculative.draft.mparams.path);
for (const auto & lora : params.lora_adapters) {
compat_add_file(buf, lora.path);
compat_add(buf, lora.scale);
}
for (const auto & cvec : params.control_vectors) {
compat_add_file(buf, cvec.fname);
compat_add(buf, cvec.strength);
}
compat_add(buf, params.control_vector_layer_start);
compat_add(buf, params.control_vector_layer_end);
compat_add(buf, (int32_t) params.cache_type_k);
compat_add(buf, (int32_t) params.cache_type_v);
compat_add(buf, (uint8_t) params.swa_full);
compat_add(buf, params.rope_freq_base);
compat_add(buf, params.rope_freq_scale);
compat_add(buf, (int32_t) params.rope_scaling_type);
compat_add(buf, params.yarn_ext_factor);
compat_add(buf, params.yarn_attn_factor);
compat_add(buf, params.yarn_beta_fast);
compat_add(buf, params.yarn_beta_slow);
compat_add(buf, params.yarn_orig_ctx);
return hash_sha256_hex(buf.data(), buf.size()).substr(0, 16);
}
//
// server_prompt_cache
//
@@ -126,7 +654,7 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro
SRV_WRN(" - making room for prompt cache entry, removing oldest entry (size = %.3f MiB)\n",
states.front().size() / (1024.0 * 1024.0));
states.pop_front();
pop_oldest();
}
}
@@ -151,8 +679,9 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro
states.push_back({
/*.prompt =*/ {
/*.tokens =*/ prompt.tokens.clone(),
/*.checkpoints =*/ prompt.checkpoints,
/*.tokens =*/ prompt.tokens.clone(),
/*.checkpoints =*/ prompt.checkpoints,
/*.lora_changed =*/ prompt.lora_changed,
},
/*.data =*/ {
/*.main =*/ std::move(state_data_tgt),
@@ -163,9 +692,46 @@ server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & pro
return &states.back();
}
bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) {
void server_prompt_cache::save_to_disk(const server_prompt_cache_state & state) {
if (!disk || state.prompt.lora_changed) {
return;
}
disk->save(state);
}
void server_prompt_cache::flush_to_disk() {
for (const auto & state : states) {
save_to_disk(state);
}
}
void server_prompt_cache::pop_oldest() {
save_to_disk(states.front());
states.pop_front();
}
void server_prompt_cache::load_from_disk(const server_tokens & tokens_new, size_t lcp_slot, int32_t n_ctx_slot) {
size_t lcp_best = lcp_slot;
for (const auto & state : states) {
lcp_best = std::max(lcp_best, state.prompt.tokens.get_common_prefix(tokens_new));
}
server_prompt_cache_state state;
if (disk->load(tokens_new, lcp_best, n_ctx_slot, state)) {
SRV_DBG("%s", "__TEST_TAG_PROMPT_CACHE_DISK_LOAD__\n");
states.push_back(std::move(state));
}
}
bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot, int32_t n_ctx_slot) {
int lcp_best = prompt.tokens.get_common_prefix(tokens_new);
if (disk) {
load_from_disk(tokens_new, lcp_best, n_ctx_slot);
}
float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins
float f_sim_best = float(lcp_best) / tokens_new.size();
@@ -253,7 +819,7 @@ void server_prompt_cache::update() {
while (!states.empty() && size() > limit_size) {
SRV_WRN(" - cache size limit reached, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0));
states.pop_front();
pop_oldest();
}
}
@@ -268,7 +834,7 @@ void server_prompt_cache::update() {
SRV_WRN(" - cache token limit (%zu, est: %zu) reached, removing oldest entry (size = %.3f MiB)\n",
limit_tokens, limit_tokens_cur, states.front().size() / (1024.0 * 1024.0));
states.pop_front();
pop_oldest();
}
}
+96 -1
View File
@@ -3,8 +3,12 @@
#include "server-common.h"
#include "server-task.h"
#include "file-lock.h"
#include <filesystem>
#include <list>
#include <memory>
#include <string>
#include <vector>
// which prompt goes to which slot: slot prompt state, prompt cache and slot selection
@@ -24,9 +28,14 @@ struct server_prompt {
// a checkpoint does not change after it is created, so the prompt cache shares it with the slot instead of a copy
std::list<std::shared_ptr<const common_prompt_checkpoint>> checkpoints;
// some tokens were processed with a lora config other than the one at startup
// TODO: such a prompt is not saved to disk, because the lora config is not a part of the prompt hash
bool lora_changed = false;
void clear() {
tokens.clear();
checkpoints.clear();
lora_changed = false;
}
int n_tokens() const {
@@ -37,6 +46,7 @@ struct server_prompt {
return server_prompt {
tokens.clone(),
checkpoints,
lora_changed,
};
}
@@ -71,6 +81,76 @@ struct server_prompt_cache_state {
}
};
// prompt cache on disk, one file per prompt in a flat directory: {compat}-{prompt hash}.cache
// file: header | hash chain | tokens | state | draft state | checkpoints
// the directory can be shared by many servers, every operation holds a lock on it
struct server_prompt_cache_disk_entry {
std::string name;
size_t size = 0; // in bytes
std::filesystem::file_time_type t_used;
// the fields below are not set for the files of another compat id, such files only count in the size limit
size_t n_tokens = 0;
server_tokens::prefix_hash hash = {}; // of the whole prompt
std::vector<server_tokens::prefix_hash> hashes;
};
struct server_prompt_cache_disk {
// compat: 16 hex chars, changes when the saved states can no longer be used (other model, mmproj, etc.)
// limit_size: in bytes, 0 = no limit
server_prompt_cache_disk(const std::string & dir, const std::string & compat, size_t limit_size, bool has_mtmd);
// the files as of the last operation, another server can change them at any time
const std::list<server_prompt_cache_disk_entry> & get_entries() const { return entries; }
size_t size() const;
server_prompt_cache_disk_stats get_stats() const;
// removes the saved prompts that are a prefix of this one, then the oldest files if over the limit
bool save(const server_prompt_cache_state & state);
// loads the saved prompt that shares the longest prefix with the tokens
// n_prefix_min: the saved prompt must share more tokens than this
// n_tokens_max: longer saved prompts are skipped
bool load(const server_tokens & tokens, size_t n_prefix_min, size_t n_tokens_max, server_prompt_cache_state & state);
private:
using entry_list = std::list<server_prompt_cache_disk_entry>;
// reads the directory again, the lock must be held
void refresh();
// the prompts are compared by their hashes, no file is read
entry_list::iterator find(const server_tokens & tokens, size_t n_prefix_min, size_t n_tokens_max);
// a file that cannot be read is removed
bool read(entry_list::iterator it, server_prompt_cache_state & state);
entry_list::iterator remove(entry_list::iterator it);
// marks the entry as the most recently used one
void touch(entry_list::iterator it);
std::filesystem::path dir;
std::string compat;
size_t limit_size;
bool has_mtmd;
// oldest first
entry_list entries;
common_file_lock lock;
server_prompt_cache_disk_stats stats;
};
// id of everything that makes a saved state unusable when it changes
std::string server_prompt_cache_disk_compat(const common_params & params);
struct server_prompt_cache {
server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens) {
this->limit_size = 1024ull*1024ull*(limit_size_mib < 0 ? 0 : limit_size_mib);
@@ -79,6 +159,9 @@ struct server_prompt_cache {
std::list<server_prompt_cache_state> states;
// optional, the states that are dropped from the RAM are saved there
std::unique_ptr<server_prompt_cache_disk> disk;
// in bytes, 0 = no limit
size_t limit_size = 0;
@@ -91,9 +174,21 @@ struct server_prompt_cache {
server_prompt_cache_state * alloc(const server_prompt & prompt, size_t state_size_main, size_t state_size_drft);
bool load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot);
// n_ctx_slot: longer prompts are not loaded from the disk
bool load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot, int32_t n_ctx_slot);
void update();
// saves all the states to disk, they stay in the RAM
void flush_to_disk();
private:
void save_to_disk(const server_prompt_cache_state & state);
void pop_oldest();
// moves the best match from the disk to the RAM if it beats the slot and the states in RAM
void load_from_disk(const server_tokens & tokens_new, size_t lcp_slot, int32_t n_ctx_slot);
};
// what the scheduler needs to know about a slot
+49 -1
View File
@@ -10,6 +10,7 @@
#include "speculative.h"
#include "server-common.h"
#include <cmath>
#include <sstream>
//
@@ -1564,6 +1565,17 @@ json server_task_result_metrics::to_json() {
return json{};
}
// whole numbers are printed in full, the default stream precision rounds them to 6 digits
static std::string metric_value_str(double value) {
if (value == std::floor(value) && std::fabs(value) < 9007199254740992.0) { // 2^53
return std::to_string((int64_t) value);
}
std::ostringstream ss;
ss << value;
return ss.str();
}
// metrics definition: https://prometheus.io/docs/practices/naming/#metric-names
std::string server_task_result_metrics::to_metrics() {
const std::vector<metric_item> counters = {
@@ -1607,6 +1619,34 @@ std::string server_task_result_metrics::to_metrics() {
"spec_decode_num_drafts_total",
"Speculative: Total speculative decoding verification steps",
(double) metrics.n_draft_verif_steps
}, {
"prompt_cache_disk_saves_total",
"Disk prompt cache: Number of prompts saved",
(double) cache_disk.n_saves
}, {
"prompt_cache_disk_save_bytes_total",
"Disk prompt cache: Number of bytes written",
(double) cache_disk.size_saved
}, {
"prompt_cache_disk_save_seconds_total",
"Disk prompt cache: Total time spent saving prompts",
cache_disk.t_save / 1.e6
}, {
"prompt_cache_disk_loads_total",
"Disk prompt cache: Number of prompts loaded",
(double) cache_disk.n_loads
}, {
"prompt_cache_disk_load_bytes_total",
"Disk prompt cache: Number of bytes read",
(double) cache_disk.size_loaded
}, {
"prompt_cache_disk_load_seconds_total",
"Disk prompt cache: Total time spent loading prompts",
cache_disk.t_load / 1.e6
}, {
"prompt_cache_disk_evictions_total",
"Disk prompt cache: Number of files removed to stay under the size limit",
(double) cache_disk.n_evictions
},
};
@@ -1631,6 +1671,14 @@ std::string server_task_result_metrics::to_metrics() {
"n_busy_slots_per_decode",
"Average number of busy slots per llama_decode() call",
(double) metrics.n_busy_slots / std::max((double) metrics.n_decode, 1.0)
}, {
"prompt_cache_disk_files",
"Disk prompt cache: Number of files in the directory",
(double) cache_disk.n_files
}, {
"prompt_cache_disk_bytes",
"Disk prompt cache: Size of the directory in bytes",
(double) cache_disk.size
},
};
@@ -1640,7 +1688,7 @@ std::string server_task_result_metrics::to_metrics() {
for (const auto & item : items) {
prometheus << "# HELP llamacpp:" << item.name << " " << item.description << "\n"
<< "# TYPE llamacpp:" << item.name << " " << type << "\n"
<< "llamacpp:" << item.name << " " << item.value << "\n";
<< "llamacpp:" << item.name << " " << metric_value_str(item.value) << "\n";
}
};
+3
View File
@@ -619,6 +619,9 @@ struct server_task_result_metrics : server_task_result {
server_metrics metrics;
// all zeros when the disk prompt cache is disabled
server_prompt_cache_disk_stats cache_disk;
virtual json to_json() override;
struct metric_item {
+209
View File
@@ -0,0 +1,209 @@
import glob
import os
import shutil
import tempfile
import pytest
from utils import *
server = ServerPreset.tinyllama2()
cache_dir: str = ""
TAG_DISK_LOAD = "__TEST_TAG_PROMPT_CACHE_DISK_LOAD__"
# longer than a few hash blocks (64 tokens each), the token ids fit the small vocab of the test model
PROMPT_A = [10 + i for i in range(200)]
PROMPT_B = [300 + i for i in range(200)]
def read_log() -> str:
with open(server.log_path) as f:
return f.read()
def cache_files() -> list[str]:
return sorted(glob.glob(os.path.join(cache_dir, "*.cache")))
def process_prompt(prompt, **kwargs):
res = server.make_request("POST", "/completion", data={
"prompt": prompt,
"n_predict": 0,
"cache_prompt": True,
**kwargs,
})
assert res.status_code == 200
return res.body["timings"]
def get_metric(name: str) -> float:
res = server.make_request("GET", "/metrics")
assert res.status_code == 200
for line in res.body.splitlines():
if line.startswith(f"llamacpp:prompt_cache_disk_{name} "):
return float(line.split(" ")[1])
raise AssertionError(f"metric not found: {name}")
def create_server_preset(preset):
global server, cache_dir
cache_dir = tempfile.mkdtemp(prefix="llama_cache_disk_")
server = preset
server.n_slots = 1
server.n_ctx = 512
server.temperature = 0.0
server.debug = True
server.server_metrics = True
server.cache_disk = cache_dir
fd, server.log_path = tempfile.mkstemp(suffix=".log")
os.close(fd)
@pytest.fixture(autouse=True)
def create_server():
create_server_preset(ServerPreset.tinyllama2())
yield
shutil.rmtree(cache_dir, ignore_errors=True)
def test_cache_disk_restart():
server.start()
process_prompt(PROMPT_A)
assert len(cache_files()) == 0
# the state of the idle slot is saved on shutdown
server.stop()
assert len(cache_files()) == 1
server.start()
timings = process_prompt(PROMPT_A + [7, 8, 9])
assert TAG_DISK_LOAD in read_log()
assert timings["cache_n"] == len(PROMPT_A)
assert timings["prompt_n"] == 3
file_size = os.path.getsize(cache_files()[0])
assert get_metric("loads_total") == 1
assert get_metric("load_bytes_total") == file_size
assert get_metric("saves_total") == 0
assert get_metric("files") == 1
assert get_metric("bytes") == file_size
# the longer prompt replaces its prefix on disk
server.stop()
assert len(cache_files()) == 1
def test_cache_disk_partial_match():
server.start()
process_prompt(PROMPT_A)
server.stop()
# the prompts differ in the middle of a hash block
server.start()
timings = process_prompt(PROMPT_A[:150] + PROMPT_B[:50])
assert TAG_DISK_LOAD in read_log()
assert timings["cache_n"] == 150
assert timings["prompt_n"] == 50
server.stop()
assert len(cache_files()) == 2
def test_cache_disk_no_match():
server.start()
process_prompt(PROMPT_A)
server.stop()
server.start()
timings = process_prompt(PROMPT_B)
assert TAG_DISK_LOAD not in read_log()
assert timings["cache_n"] == 0
def test_cache_disk_ram_first():
server.start()
process_prompt(PROMPT_A)
# PROMPT_A goes to the RAM cache, nothing is written while the server runs
process_prompt(PROMPT_B)
assert len(cache_files()) == 0
timings = process_prompt(PROMPT_A)
assert TAG_DISK_LOAD not in read_log()
assert timings["cache_n"] > 0
def test_cache_disk_ram_eviction():
# the RAM cache has room for one of the two states
server.cache_ram = 2
server.n_ctx = 2048
server.start()
process_prompt(PROMPT_A * 9)
process_prompt(PROMPT_B * 9)
process_prompt(PROMPT_A)
assert len(cache_files()) >= 1
assert get_metric("saves_total") == len(cache_files())
# large values are not rounded
size_saved = sum(os.path.getsize(f) for f in cache_files())
assert size_saved > 1000000
assert get_metric("save_bytes_total") == size_saved
assert get_metric("loads_total") == 0
def test_cache_disk_broken_file():
server.start()
process_prompt(PROMPT_A)
server.stop()
files = cache_files()
assert len(files) == 1
with open(files[0], "r+b") as f:
f.truncate(os.path.getsize(files[0]) - 16)
server.start()
timings = process_prompt(PROMPT_A)
assert timings["cache_n"] == 0
assert files[0] not in cache_files()
def test_cache_disk_shared_dir():
# two servers with different models use the same directory
other = ServerPreset.stories15m_moe()
other.server_port = server.server_port + 1
other.n_slots = 1
other.n_ctx = 512
other.cache_disk = cache_dir
server.start()
other.start()
process_prompt(PROMPT_A)
res = other.make_request("POST", "/completion", data={"prompt": PROMPT_B, "n_predict": 0, "cache_prompt": True})
assert res.status_code == 200
other.stop()
server.stop()
assert len(cache_files()) == 2
# each server finds its own prompt
server.start()
timings = process_prompt(PROMPT_A)
assert TAG_DISK_LOAD in read_log()
assert timings["cache_n"] == len(PROMPT_A) - 1
assert get_metric("files") == 2
def test_cache_disk_lora():
create_server_preset(ServerPreset.stories15m_moe())
server.lora_files = [download_file("https://huggingface.co/ggml-org-dev/stories15M_MOE/resolve/main/moe_shakespeare15M.gguf")]
server.start()
# a state made with another lora config is not saved
process_prompt(PROMPT_A, lora=[{"id": 0, "scale": 0.5}])
server.stop()
assert len(cache_files()) == 0
server.start()
process_prompt(PROMPT_A)
server.stop()
assert len(cache_files()) == 1
+6
View File
@@ -121,6 +121,8 @@ class ServerProcess:
media_path: str | None = None
sleep_idle_seconds: int | None = None
cache_ram: int | None = None
cache_disk: str | None = None
cache_disk_max_size: int | None = None
override_kv: str | None = None
no_cache_idle_slots: bool = False
log_path: str | None = None
@@ -304,6 +306,10 @@ class ServerProcess:
server_args.extend(["--sleep-idle-seconds", self.sleep_idle_seconds])
if self.cache_ram is not None:
server_args.extend(["--cache-ram", self.cache_ram])
if self.cache_disk is not None:
server_args.extend(["--cache-disk", self.cache_disk])
if self.cache_disk_max_size is not None:
server_args.extend(["--cache-disk-max-size", self.cache_disk_max_size])
if self.override_kv:
server_args.extend(["--override-kv", self.override_kv])
if self.no_cache_idle_slots: