upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-06 01:50:50 -05:00
Code Issues Packages Projects Releases Wiki Activity

common: add option --spec-draftless

Browse Source
This commit is contained in:
Sascha Rogmann
2026-01-22 23:17:56 +01:00
parent eb43748b05
commit 1e29af4ea5
3 changed files with 62 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
common/arg.cpp
View File

@@ -630,9 +630,11 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
bool has_draft =!params.speculative.model.path.empty();
bool has_draft_eagle3 = false; // TODO PR-18039: if params.speculative.eagle3
bool has_lookup_caches = !params.lookup_cache_static.empty() && !params.lookup_cache_dynamic.empty();
bool has_simple = (params.speculative.draftless_type == COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE);
bool found_config_draft = false;
bool found_config_eagle3 = false;
bool found_config_ngram_cache = false;
bool found_config_ngram_simple = false;
for (const auto & config : params.speculative.configs) {
if (config.type == COMMON_SPECULATIVE_TYPE_DRAFT) {
found_config_draft = true;
@@ -643,6 +645,15 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
if (config.type == COMMON_SPECULATIVE_TYPE_NGRAM_CACHE) {
found_config_ngram_cache = true;
}
if (config.type == COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE) {
found_config_ngram_simple = true;
}
}
if (has_simple && !found_config_ngram_simple) {
params.speculative.configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE));
}
if (has_lookup_caches && !found_config_ngram_cache) {
params.speculative.configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE));
}
if (has_draft && !found_config_draft) {
params.speculative.configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT));
@@ -650,9 +661,6 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
if (has_draft_eagle3 && !found_config_eagle3) {
params.speculative.configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_EAGLE3));
}
if (has_lookup_caches && !found_config_ngram_cache) {
params.speculative.configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE));
}
}
if (!params.chat_template.empty() && !common_chat_verify_template(params.chat_template, params.use_jinja)) {
@@ -3421,6 +3429,42 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.speculative.replacements.push_back({ tgt, dft });
}
).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
add_opt(common_arg(
{"--spec-draftless"}, "[none|ngram-cache|ngram-simple]",
string_format("type of speculative decoding to use when no draft model is provided (default: %s)\n",
common_speculative_type_to_str(params.speculative.draftless_type).c_str()),
[](common_params & params, const std::string & value) {
if (value == "none") {
params.speculative.draftless_type = COMMON_SPECULATIVE_TYPE_NONE;
} else if (value == "ngram-cache") {
params.speculative.draftless_type = COMMON_SPECULATIVE_TYPE_NGRAM_CACHE;
} else if (value == "ngram-simple") {
params.speculative.draftless_type = COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE;
} else {
throw std::invalid_argument("unknown speculative decoding type without draft model");
}
}
).set_examples({LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"--spec-ngram-size-n"}, "N",
string_format("ngram size N for ngram-map speculative decoding, length of lookup n-gram (default: %d)", params.speculative.spec_ngram_size_n),
[](common_params & params, int value) {
if (value < 1 || value > 1024) {
throw std::invalid_argument("ngram size N must be between 1 and 1024 inclusive");
}
params.speculative.spec_ngram_size_n = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"--spec-ngram-size-m"}, "N",
string_format("ngram size M for ngram-map speculative decoding, length of draft m-gram (default: %d)", params.speculative.spec_ngram_size_m),
[](common_params & params, int value) {
if (value < 1 || value > 1024) {
throw std::invalid_argument("ngram size M must be between 1 and 1024 inclusive");
}
params.speculative.spec_ngram_size_m = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"--spec-config"}, "SPECULATIVE_CONFIG",
string_format("list of speculative decoding types, separated by ';', optionally followed by a colon and a comma-separated list of key=value pairs\n(types: %s)\n", common_speculative_type_name_str().c_str()),

4
common/common.h
View File

@@ -281,6 +281,10 @@ struct common_params_speculative {
struct common_params_model model;
common_speculative_type draftless_type = COMMON_SPECULATIVE_TYPE_NONE; // type of speculative decoding without a draft model
uint16_t spec_ngram_size_n = 12;
uint16_t spec_ngram_size_m = 48;
std::vector<common_speculative_config> configs = {}; // list of speculative configs to try
};

19
common/speculative.cpp
View File

@@ -149,16 +149,16 @@ struct common_speculative {
common_speculative_state * curr_impl = nullptr; // current implementation in use (for stats)
};
common_ngram_map get_common_ngram_map(const common_speculative_config config);
common_ngram_map get_common_ngram_map(const common_speculative_config config, uint16_t size_ngram, uint16_t size_mgram);
struct common_speculative_state_ngram_cache create_state_ngram_cache(
std::string path_static, std::string path_dynamic,
common_speculative_config config);
common_ngram_map get_common_ngram_map(const common_speculative_config config) {
uint16_t size_key = 12;
uint16_t size_value = 48;
common_ngram_map get_common_ngram_map(const common_speculative_config config, uint16_t size_ngram, uint16_t size_mgram) {
uint16_t size_key = size_ngram;
uint16_t size_value = size_mgram;
bool key_only = false;
uint16_t check_rate = 3;
uint16_t check_rate = 2;
uint16_t min_hits = 1;
const std::map<std::string, std::string> & cfg = config.config;
if (cfg.find("size_ngram") != cfg.end()) {
@@ -270,7 +270,8 @@ struct common_speculative * common_speculative_init(
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: {
common_ngram_map ngram_map = get_common_ngram_map(config);
common_ngram_map ngram_map = get_common_ngram_map(config,
params.speculative.spec_ngram_size_n, params.speculative.spec_ngram_size_m);
uint16_t ngram_size_key = ngram_map.size_key;
uint16_t mgram_size_value = ngram_map.size_value;
uint16_t check_rate = ngram_map.check_rate;
@@ -285,13 +286,15 @@ struct common_speculative * common_speculative_init(
}
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: {
implementations.push_back(std::make_unique<common_speculative_state_ngram_map_k>(
(config.type), get_common_ngram_map(config)
(config.type), get_common_ngram_map(config,
params.speculative.spec_ngram_size_n, params.speculative.spec_ngram_size_m)
));
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: {
implementations.push_back(std::make_unique<common_speculative_state_ngram_map_k4v>(
(config.type), get_common_ngram_map(config)));
(config.type), get_common_ngram_map(config,
params.speculative.spec_ngram_size_n, params.speculative.spec_ngram_size_m)));
break;
}
case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: {