upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-05 17:40:46 -05:00
Code Issues Packages Projects Releases Wiki Activity

add Q rot when cache is quantized

Browse Source
This commit is contained in:
Aman Gupta
2026-05-22 00:17:02 +08:00
parent 7b87cd3598
commit 27461cd888
3 changed files with 46 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
tools/server/server-context.cpp
View File

@@ -913,13 +913,23 @@ private:
SRV_INF("loading draft model '%s'\n", params_spec.mparams.path.c_str());
const bool spec_mtp = std::find(params_base.speculative.types.begin(),
params_base.speculative.types.end(),
COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params_base.speculative.types.end();
auto params_dft = params_base;
params_dft.devices = params_spec.devices;
params_dft.model = params_spec.mparams;
params_dft.n_gpu_layers = params_spec.n_gpu_layers;
params_dft.cache_type_k = params_spec.cache_type_k;
params_dft.cache_type_v = params_spec.cache_type_v;
// TODO: find a better way to expose that the cache is shared
if (spec_mtp) {
params_dft.cache_type_k = params_base.cache_type_k;
params_dft.cache_type_v = params_base.cache_type_v;
} else {
params_dft.cache_type_k = params_spec.cache_type_k;
params_dft.cache_type_v = params_spec.cache_type_v;
}
if (params_spec.cpuparams.n_threads > 0) {
params_dft.cpuparams.n_threads = params_spec.cpuparams.n_threads;
@@ -938,9 +948,6 @@ private:
auto cparams = common_context_params_to_llama(params_dft);
const bool spec_mtp = std::find(params_base.speculative.types.begin(),
params_base.speculative.types.end(),
COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params_base.speculative.types.end();
if (spec_mtp) {
cparams.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
}