upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-23 11:10:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

args: refactor mlock/mmap/directio into load-mode (#20834)

Browse Source 
* args: overhaul mmap/mlock/dio into single arg

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* docs: update docs with llama-gen-docs

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* chore: satisfy code quality

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* args: make the `+` sign an actual modifier now

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* chore: general code clean up + comments

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* arg: fix deprecated flags support

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* arg: quick sanity check

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* bench: sync llama-bench argument parsing

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* fix: bugfix variable behaviour + llama-bench lm column size

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* arg: inverse commands should do the opposite instead of doing nothing

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* bench: fix incorrect dash

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* bench: fix missing modifiers for deprecated flags

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* llama: switch back to thread_local

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* arg: switch back to single enum

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* docs: update arg docs

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* chore: fix missing `mlock` from llama_load_mode_from_str + cleanup llama-bench

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* llama: fix mlock not activating

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* arg: add deprecation warning when old and new flags are combined

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* arg: cont add comment for todo in the future

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* docs: sync with upstream

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* docs: re-sync with upstream again

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

---------

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>
This commit is contained in:
Aaron Teo
2026-07-23 20:32:56 +08:00
committed by GitHub
parent da296d6e72
commit e6dd0e29a6
20 changed files with 315 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
scripts/compare-llama-bench.py
View File

@@ -28,7 +28,7 @@ LLAMA_BENCH_DB_FIELDS = [
"model_type", "model_size", "model_n_params", "n_batch", "n_ubatch", "n_threads",
"cpu_mask", "cpu_strict", "poll", "type_k", "type_v", "n_gpu_layers",
"split_mode", "main_gpu", "no_kv_offload", "flash_attn", "tensor_split", "tensor_buft_overrides",
"use_mmap", "embeddings", "no_op_offload", "n_prompt", "n_gen", "n_depth",
"load_mode", "embeddings", "no_op_offload", "n_prompt", "n_gen", "n_depth",
"test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts", "n_cpu_moe",
"fit_target", "fit_min_ctx"
]
@@ -38,7 +38,7 @@ LLAMA_BENCH_DB_TYPES = [
"TEXT", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER",
"TEXT", "INTEGER", "INTEGER", "TEXT", "TEXT", "INTEGER",
"TEXT", "INTEGER", "INTEGER", "INTEGER", "TEXT", "TEXT",
"INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER",
"TEXT", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER",
"TEXT", "INTEGER", "INTEGER", "REAL", "REAL", "INTEGER",
"INTEGER", "INTEGER"
]
@@ -63,7 +63,7 @@ assert len(TEST_BACKEND_OPS_DB_FIELDS) == len(TEST_BACKEND_OPS_DB_TYPES)
LLAMA_BENCH_KEY_PROPERTIES = [
"cpu_info", "gpu_info", "backends", "n_gpu_layers", "n_cpu_moe", "tensor_buft_overrides", "model_filename", "model_type",
"n_batch", "n_ubatch", "embeddings", "cpu_mask", "cpu_strict", "poll", "n_threads", "type_k", "type_v",
"use_mmap", "no_kv_offload", "split_mode", "main_gpu", "tensor_split", "flash_attn", "n_prompt", "n_gen", "n_depth",
"load_mode", "no_kv_offload", "split_mode", "main_gpu", "tensor_split", "flash_attn", "n_prompt", "n_gen", "n_depth",
"fit_target", "fit_min_ctx"
]
@@ -73,7 +73,7 @@ TEST_BACKEND_OPS_KEY_PROPERTIES = [
]
# Properties that are boolean and are converted to Yes/No for the table:
LLAMA_BENCH_BOOL_PROPERTIES = ["embeddings", "cpu_strict", "use_mmap", "no_kv_offload", "flash_attn"]
LLAMA_BENCH_BOOL_PROPERTIES = ["embeddings", "cpu_strict", "no_kv_offload", "flash_attn"]
TEST_BACKEND_OPS_BOOL_PROPERTIES = ["supported", "passed"]
# Header names for the table (llama-bench):
@@ -82,7 +82,7 @@ LLAMA_BENCH_PRETTY_NAMES = {
"tensor_buft_overrides": "Tensor overrides", "model_filename": "File", "model_type": "Model", "model_size": "Model size [GiB]",
"model_n_params": "Num. of par.", "n_batch": "Batch size", "n_ubatch": "Microbatch size", "embeddings": "Embeddings",
"cpu_mask": "CPU mask", "cpu_strict": "CPU strict", "poll": "Poll", "n_threads": "Threads", "type_k": "K type", "type_v": "V type",
"use_mmap": "Use mmap", "no_kv_offload": "NKVO", "split_mode": "Split mode", "main_gpu": "Main GPU", "tensor_split": "Tensor split",
"load_mode": "Load mode", "no_kv_offload": "NKVO", "split_mode": "Split mode", "main_gpu": "Main GPU", "tensor_split": "Tensor split",
"flash_attn": "FlashAttention",
}