diff --git a/common/arg.cpp b/common/arg.cpp index 48e2747742..1f5b5a8abf 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -351,6 +351,10 @@ static std::string get_default_local_path(const std::string & url) { return fs_get_cache_file(string_split(f, '/').back()); } +static bool spec_types_is_default(const common_params & params) { + return params.speculative.types == std::vector{COMMON_SPECULATIVE_TYPE_NONE}; +} + common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex) { common_download_hf_plan plan; common_download_hf_plan plan_spec; @@ -391,7 +395,14 @@ common_models_handler common_models_handler_init(const common_params & params, l } if (!params.speculative.draft.mparams.hf_repo.empty()) { - plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts); + // without a requested type, discover every sidecar the draft repo ships to infer the type later + auto opts_spec = opts; + if (spec_types_is_default(params)) { + opts_spec.download_mtp = true; + opts_spec.download_dflash = true; + opts_spec.download_eagle3 = true; + } + plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts_spec); } if (!params.vocoder.model.hf_repo.empty()) { @@ -527,6 +538,20 @@ void common_models_handler_apply(common_models_handler & handler, common_params } }; + // infer the speculative type from the sidecar shipped by the draft repo when none is requested + if (spec_types_is_default(params)) { + if (!plan_spec.mtp.local_path.empty()) { + params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_MTP }; + plan_spec.dflash = {}; + plan_spec.eagle3 = {}; + } else if (!plan_spec.dflash.local_path.empty()) { + params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH }; + plan_spec.eagle3 = {}; + } else if (!plan_spec.eagle3.local_path.empty()) { + params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 }; + } + } + // when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full model const bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() || !plan_spec.dflash.local_path.empty() ||