upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-07 02:20:48 -05:00
Code Issues Packages Projects Releases Wiki Activity

restore llama-bench.cpp changes

Browse Source
This commit is contained in:
Scott Cutler
2026-04-23 13:17:38 -07:00
parent 239f20e1e9
commit cfcaee3509
1 changed files with 4 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
tools/llama-bench/llama-bench.cpp
View File

@@ -335,7 +335,6 @@ struct cmd_params {
std::vector<int> n_gpu_layers;
std::vector<int> n_cpu_moe;
std::vector<llama_split_mode> split_mode;
std::vector<std::string> reduction_provider;
std::vector<int> main_gpu;
std::vector<bool> no_kv_offload;
std::vector<bool> flash_attn;
@@ -380,7 +379,6 @@ static const cmd_params cmd_params_defaults = {
/* n_gpu_layers */ { 99 },
/* n_cpu_moe */ { 0 },
/* split_mode */ { LLAMA_SPLIT_MODE_LAYER },
/* reduction_provider */ { "auto" },
/* main_gpu */ { 0 },
/* no_kv_offload */ { false },
/* flash_attn */ { false },
@@ -451,7 +449,6 @@ static void print_usage(int /* argc */, char ** argv) {
printf(" -ngl, --n-gpu-layers <n> (default: %s)\n", join(cmd_params_defaults.n_gpu_layers, ",").c_str());
printf(" -ncmoe, --n-cpu-moe <n> (default: %s)\n", join(cmd_params_defaults.n_cpu_moe, ",").c_str());
printf(" -sm, --split-mode <none|layer|row|tensor> (default: %s)\n", join(transform_to_str(cmd_params_defaults.split_mode, split_mode_str), ",").c_str());
printf(" -rp, --reduction-provider <auto|nccl|internal> allreduce provider for tensor split mode (default: %s)\n", join(cmd_params_defaults.reduction_provider, ",").c_str());
printf(" -mg, --main-gpu <i> (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str());
printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str());
printf(" -fa, --flash-attn <0|1> (default: %s)\n", join(cmd_params_defaults.flash_attn, ",").c_str());
@@ -762,22 +759,6 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
break;
}
params.split_mode.insert(params.split_mode.end(), modes.begin(), modes.end());
} else if (arg == "-rp" || arg == "--reduction-provider") {
if (++i >= argc) {
invalid_param = true;
break;
}
auto p = string_split<std::string>(argv[i], split_delim);
for (const auto & v : p) {
if (v != "auto" && v != "nccl" && v != "internal") {
invalid_param = true;
break;
}
}
if (invalid_param) {
break;
}
params.reduction_provider.insert(params.reduction_provider.end(), p.begin(), p.end());
} else if (arg == "-mg" || arg == "--main-gpu") {
if (++i >= argc) {
invalid_param = true;
@@ -1084,9 +1065,6 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
if (params.split_mode.empty()) {
params.split_mode = cmd_params_defaults.split_mode;
}
if (params.reduction_provider.empty()) {
params.reduction_provider = cmd_params_defaults.reduction_provider;
}
if (params.main_gpu.empty()) {
params.main_gpu = cmd_params_defaults.main_gpu;
}
@@ -1158,7 +1136,6 @@ struct cmd_params_instance {
int n_gpu_layers;
int n_cpu_moe;
llama_split_mode split_mode;
std::string reduction_provider;
int main_gpu;
bool no_kv_offload;
bool flash_attn;
@@ -1228,7 +1205,7 @@ struct cmd_params_instance {
bool equal_mparams(const cmd_params_instance & other) const {
return model == other.model && n_gpu_layers == other.n_gpu_layers && n_cpu_moe == other.n_cpu_moe &&
split_mode == other.split_mode && reduction_provider == other.reduction_provider &&
split_mode == other.split_mode &&
main_gpu == other.main_gpu && tensor_split == other.tensor_split &&
use_mmap == other.use_mmap && use_direct_io == other.use_direct_io &&
devices == other.devices &&
@@ -1265,7 +1242,6 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
for (const auto & nl : params.n_gpu_layers)
for (const auto & ncmoe : params.n_cpu_moe)
for (const auto & sm : params.split_mode)
for (const auto & rp : params.reduction_provider)
for (const auto & mg : params.main_gpu)
for (const auto & devs : params.devices)
for (const auto & ts : params.tensor_split)
@@ -1306,7 +1282,6 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
/* .n_gpu_layers = */ nl,
/* .n_cpu_moe = */ ncmoe,
/* .split_mode = */ sm,
/* .reduction_provider = */ rp,
/* .main_gpu = */ mg,
/* .no_kv_offload= */ nkvo,
/* .flash_attn = */ fa,
@@ -1344,7 +1319,6 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
/* .n_gpu_layers = */ nl,
/* .n_cpu_moe = */ ncmoe,
/* .split_mode = */ sm,
/* .reduction_provider = */ rp,
/* .main_gpu = */ mg,
/* .no_kv_offload= */ nkvo,
/* .flash_attn = */ fa,
@@ -1382,7 +1356,6 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
/* .n_gpu_layers = */ nl,
/* .n_cpu_moe = */ ncmoe,
/* .split_mode = */ sm,
/* .reduction_provider = */ rp,
/* .main_gpu = */ mg,
/* .no_kv_offload= */ nkvo,
/* .flash_attn = */ fa,
@@ -1425,7 +1398,6 @@ struct test {
int n_gpu_layers;
int n_cpu_moe;
llama_split_mode split_mode;
std::string reduction_provider;
int main_gpu;
bool no_kv_offload;
bool flash_attn;
@@ -1466,7 +1438,6 @@ struct test {
n_gpu_layers = inst.n_gpu_layers;
n_cpu_moe = inst.n_cpu_moe;
split_mode = inst.split_mode;
reduction_provider = inst.reduction_provider;
main_gpu = inst.main_gpu;
no_kv_offload = inst.no_kv_offload;
flash_attn = inst.flash_attn;
@@ -1535,7 +1506,7 @@ struct test {
"model_filename", "model_type", "model_size", "model_n_params", "n_batch",
"n_ubatch", "n_threads", "cpu_mask", "cpu_strict", "poll",
"type_k", "type_v", "n_gpu_layers", "n_cpu_moe", "split_mode",
"reduction_provider", "main_gpu", "no_kv_offload", "flash_attn", "devices", "tensor_split",
"main_gpu", "no_kv_offload", "flash_attn", "devices", "tensor_split",
"tensor_buft_overrides", "use_mmap", "use_direct_io", "embeddings",
"no_op_offload", "no_host", "fit_target", "fit_min_ctx",
"n_prompt", "n_gen", "n_depth",
@@ -1621,7 +1592,6 @@ struct test {
std::to_string(n_gpu_layers),
std::to_string(n_cpu_moe),
split_mode_str(split_mode),
reduction_provider,
std::to_string(main_gpu),
std::to_string(no_kv_offload),
std::to_string(flash_attn),
@@ -2143,10 +2113,9 @@ static bool test_gen(llama_context * ctx, int n_gen, int n_threads) {
}
static void llama_null_log_callback(enum ggml_log_level level, const char * text, void * user_data) {
(void) level;
(void) text;
(void) user_data;
if (level >= GGML_LOG_LEVEL_WARN) {
fputs(text, stderr);
}
}
static std::unique_ptr<printer> create_printer(output_formats format) {
@@ -2276,16 +2245,6 @@ int main(int argc, char ** argv) {
params.verbose ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
}
// set reduction provider env var before model load (comm_init reads it)
{
const char * ar_val = (inst.reduction_provider == "auto") ? "" : inst.reduction_provider.c_str();
#ifdef _WIN32
_putenv_s("GGML_CUDA_ALLREDUCE", ar_val);
#else
setenv("GGML_CUDA_ALLREDUCE", ar_val, 1);
#endif
}
// keep the same model between tests when possible
if (!lmodel || !prev_inst || !inst.equal_mparams(*prev_inst)) {
if (lmodel) {