upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-23 11:10:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

common: infer the speculative type from the draft repo sidecars (#25989)

Browse Source 
With -hfd pointing to a repo that ships mtp-/dflash-/eagle3- sidecars
and no --spec-type given, the draft resolved to a full model while the
sidecar was the intended draft.

When the speculative types are still at their default, discover the
sidecars of the draft repo, pick the first available following the
existing mtp > dflash > eagle3 priority, and set the corresponding
type, so this now works without any extra flag:

llama-server -hf repo:Q3_K_M -hfd repo:Q8_0

An explicit --spec-type disables the inference, and a draft repo
without sidecars keeps resolving to a full model as before.
This commit is contained in:
Pascal
2026-07-22 13:06:35 +02:00
committed by GitHub
parent f534da26e4
commit 6d5a910c50
1 changed files with 26 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
common/arg.cpp
View File

@@ -351,6 +351,10 @@ static std::string get_default_local_path(const std::string & url) {
return fs_get_cache_file(string_split<std::string>(f, '/').back());
}
static bool spec_types_is_default(const common_params & params) {
return params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_NONE};
}
common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex) {
common_download_hf_plan plan;
common_download_hf_plan plan_spec;
@@ -391,7 +395,14 @@ common_models_handler common_models_handler_init(const common_params & params, l
}
if (!params.speculative.draft.mparams.hf_repo.empty()) {
plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts);
// without a requested type, discover every sidecar the draft repo ships to infer the type later
auto opts_spec = opts;
if (spec_types_is_default(params)) {
opts_spec.download_mtp = true;
opts_spec.download_dflash = true;
opts_spec.download_eagle3 = true;
}
plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts_spec);
}
if (!params.vocoder.model.hf_repo.empty()) {
@@ -527,6 +538,20 @@ void common_models_handler_apply(common_models_handler & handler, common_params
}
};
// infer the speculative type from the sidecar shipped by the draft repo when none is requested
if (spec_types_is_default(params)) {
if (!plan_spec.mtp.local_path.empty()) {
params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_MTP };
plan_spec.dflash = {};
plan_spec.eagle3 = {};
} else if (!plan_spec.dflash.local_path.empty()) {
params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH };
plan_spec.eagle3 = {};
} else if (!plan_spec.eagle3.local_path.empty()) {
params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 };
}
}
// when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full model
const bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() ||
!plan_spec.dflash.local_path.empty() ||