diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp index 366bf1cb57..e21a80e697 100644 --- a/tools/llama-bench/llama-bench.cpp +++ b/tools/llama-bench/llama-bench.cpp @@ -335,7 +335,6 @@ struct cmd_params { std::vector n_gpu_layers; std::vector n_cpu_moe; std::vector split_mode; - std::vector reduction_provider; std::vector main_gpu; std::vector no_kv_offload; std::vector flash_attn; @@ -380,7 +379,6 @@ static const cmd_params cmd_params_defaults = { /* n_gpu_layers */ { 99 }, /* n_cpu_moe */ { 0 }, /* split_mode */ { LLAMA_SPLIT_MODE_LAYER }, - /* reduction_provider */ { "auto" }, /* main_gpu */ { 0 }, /* no_kv_offload */ { false }, /* flash_attn */ { false }, @@ -451,7 +449,6 @@ static void print_usage(int /* argc */, char ** argv) { printf(" -ngl, --n-gpu-layers (default: %s)\n", join(cmd_params_defaults.n_gpu_layers, ",").c_str()); printf(" -ncmoe, --n-cpu-moe (default: %s)\n", join(cmd_params_defaults.n_cpu_moe, ",").c_str()); printf(" -sm, --split-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.split_mode, split_mode_str), ",").c_str()); - printf(" -rp, --reduction-provider allreduce provider for tensor split mode (default: %s)\n", join(cmd_params_defaults.reduction_provider, ",").c_str()); printf(" -mg, --main-gpu (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str()); printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str()); printf(" -fa, --flash-attn <0|1> (default: %s)\n", join(cmd_params_defaults.flash_attn, ",").c_str()); @@ -762,22 +759,6 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { break; } params.split_mode.insert(params.split_mode.end(), modes.begin(), modes.end()); - } else if (arg == "-rp" || arg == "--reduction-provider") { - if (++i >= argc) { - invalid_param = true; - break; - } - auto p = string_split(argv[i], split_delim); - for (const auto & v : p) { - if (v != "auto" && v != "nccl" && v != "internal") { - invalid_param = true; - break; - } - } - if (invalid_param) { - break; - } - params.reduction_provider.insert(params.reduction_provider.end(), p.begin(), p.end()); } else if (arg == "-mg" || arg == "--main-gpu") { if (++i >= argc) { invalid_param = true; @@ -1084,9 +1065,6 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { if (params.split_mode.empty()) { params.split_mode = cmd_params_defaults.split_mode; } - if (params.reduction_provider.empty()) { - params.reduction_provider = cmd_params_defaults.reduction_provider; - } if (params.main_gpu.empty()) { params.main_gpu = cmd_params_defaults.main_gpu; } @@ -1158,7 +1136,6 @@ struct cmd_params_instance { int n_gpu_layers; int n_cpu_moe; llama_split_mode split_mode; - std::string reduction_provider; int main_gpu; bool no_kv_offload; bool flash_attn; @@ -1228,7 +1205,7 @@ struct cmd_params_instance { bool equal_mparams(const cmd_params_instance & other) const { return model == other.model && n_gpu_layers == other.n_gpu_layers && n_cpu_moe == other.n_cpu_moe && - split_mode == other.split_mode && reduction_provider == other.reduction_provider && + split_mode == other.split_mode && main_gpu == other.main_gpu && tensor_split == other.tensor_split && use_mmap == other.use_mmap && use_direct_io == other.use_direct_io && devices == other.devices && @@ -1265,7 +1242,6 @@ static std::vector get_cmd_params_instances(const cmd_param for (const auto & nl : params.n_gpu_layers) for (const auto & ncmoe : params.n_cpu_moe) for (const auto & sm : params.split_mode) - for (const auto & rp : params.reduction_provider) for (const auto & mg : params.main_gpu) for (const auto & devs : params.devices) for (const auto & ts : params.tensor_split) @@ -1306,7 +1282,6 @@ static std::vector get_cmd_params_instances(const cmd_param /* .n_gpu_layers = */ nl, /* .n_cpu_moe = */ ncmoe, /* .split_mode = */ sm, - /* .reduction_provider = */ rp, /* .main_gpu = */ mg, /* .no_kv_offload= */ nkvo, /* .flash_attn = */ fa, @@ -1344,7 +1319,6 @@ static std::vector get_cmd_params_instances(const cmd_param /* .n_gpu_layers = */ nl, /* .n_cpu_moe = */ ncmoe, /* .split_mode = */ sm, - /* .reduction_provider = */ rp, /* .main_gpu = */ mg, /* .no_kv_offload= */ nkvo, /* .flash_attn = */ fa, @@ -1382,7 +1356,6 @@ static std::vector get_cmd_params_instances(const cmd_param /* .n_gpu_layers = */ nl, /* .n_cpu_moe = */ ncmoe, /* .split_mode = */ sm, - /* .reduction_provider = */ rp, /* .main_gpu = */ mg, /* .no_kv_offload= */ nkvo, /* .flash_attn = */ fa, @@ -1425,7 +1398,6 @@ struct test { int n_gpu_layers; int n_cpu_moe; llama_split_mode split_mode; - std::string reduction_provider; int main_gpu; bool no_kv_offload; bool flash_attn; @@ -1466,7 +1438,6 @@ struct test { n_gpu_layers = inst.n_gpu_layers; n_cpu_moe = inst.n_cpu_moe; split_mode = inst.split_mode; - reduction_provider = inst.reduction_provider; main_gpu = inst.main_gpu; no_kv_offload = inst.no_kv_offload; flash_attn = inst.flash_attn; @@ -1535,7 +1506,7 @@ struct test { "model_filename", "model_type", "model_size", "model_n_params", "n_batch", "n_ubatch", "n_threads", "cpu_mask", "cpu_strict", "poll", "type_k", "type_v", "n_gpu_layers", "n_cpu_moe", "split_mode", - "reduction_provider", "main_gpu", "no_kv_offload", "flash_attn", "devices", "tensor_split", + "main_gpu", "no_kv_offload", "flash_attn", "devices", "tensor_split", "tensor_buft_overrides", "use_mmap", "use_direct_io", "embeddings", "no_op_offload", "no_host", "fit_target", "fit_min_ctx", "n_prompt", "n_gen", "n_depth", @@ -1621,7 +1592,6 @@ struct test { std::to_string(n_gpu_layers), std::to_string(n_cpu_moe), split_mode_str(split_mode), - reduction_provider, std::to_string(main_gpu), std::to_string(no_kv_offload), std::to_string(flash_attn), @@ -2143,10 +2113,9 @@ static bool test_gen(llama_context * ctx, int n_gen, int n_threads) { } static void llama_null_log_callback(enum ggml_log_level level, const char * text, void * user_data) { + (void) level; + (void) text; (void) user_data; - if (level >= GGML_LOG_LEVEL_WARN) { - fputs(text, stderr); - } } static std::unique_ptr create_printer(output_formats format) { @@ -2276,16 +2245,6 @@ int main(int argc, char ** argv) { params.verbose ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR); } - // set reduction provider env var before model load (comm_init reads it) - { - const char * ar_val = (inst.reduction_provider == "auto") ? "" : inst.reduction_provider.c_str(); -#ifdef _WIN32 - _putenv_s("GGML_CUDA_ALLREDUCE", ar_val); -#else - setenv("GGML_CUDA_ALLREDUCE", ar_val, 1); -#endif - } - // keep the same model between tests when possible if (!lmodel || !prev_inst || !inst.equal_mparams(*prev_inst)) { if (lmodel) {