upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-03 19:37:29 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9 from huggingface/onyx-swa-rope-theta

Browse Source 
onyx: use the model rope theta on sliding-window layers
This commit is contained in:
Pedro Cuenca
2026-08-07 19:19:29 +02:00
committed by GitHub
parent eb6046209f 46f2dc52df
commit fe54c4f6f6
1 changed files with 5 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/models/onyx.cpp
+5
View File
@@ -7,6 +7,11 @@ void llama_model_onyx::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);
ml.get_key(LLM_KV_ATTENTION_POST_NORM_RMS_EPS, hparams.f_post_norm_rms_eps);
// SWA layers share the model rope theta; they are also the only layers that use rope
// here (global layers are NoPE), so the 10000.0 default would apply to all of them.
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
// SWA + NoPE: [SW, SW, SW, Full], NoPE used on Full layers.
if (hparams.n_swa > 0) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;