From d5e61c03316174c92a235b97bbcec77b57104ea5 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Fri, 21 Aug 2026 12:07:53 +0200 Subject: [PATCH] partially apply to deepseek2 --- docs/development/HOWTO-add-model.md | 6 +++-- src/models/deepseek2.cpp | 40 +++++++++++++++++------------ 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/docs/development/HOWTO-add-model.md b/docs/development/HOWTO-add-model.md index e24f5570df..31b3f26861 100644 --- a/docs/development/HOWTO-add-model.md +++ b/docs/development/HOWTO-add-model.md @@ -168,15 +168,17 @@ Examples: ### Rotating only a part of the head -Many models rotate only a part of each head and leave the rest untouched (often called the "nope" part). Do not build this with views plus `ggml_concat`: `ggml_concat` allocates a new tensor and copies every element, which costs more than the RoPE itself. Both layouts can be done with a single RoPE op that copies the untouched dims through for you: +Many models rotate only a part of each head and leave the rest untouched (often called the "nope" part). Do not build this with views plus `ggml_concat`, it's not efficient. Both layouts can be done with a single RoPE op: - `[rope|nope]`, rotated dims first: pass `n_dims` smaller than the head size to `ggml_rope_ext`. Dims from `n_dims` to the end are copied as-is. - `[nope|rope]`, rotated dims last: call `ggml_rope_set_offset(cur, n_offs)` on the result of the RoPE, where `n_offs` is the size of the leading untouched part. Dims outside `[n_offs, n_offs + n_dims)` are copied as-is. -`n_offs` must be even, `n_offs + n_dims` must fit in the row, and vision RoPE is not supported. Note that the frequencies are computed relative to the rotated window, so the result matches applying `ggml_rope_ext` to that slice on its own. +`n_offs` must be even, `n_offs + n_dims` must fit in the row, and vision RoPE is not supported. Note that the frequencies are computed relative to the rotated window. Example: DeepSeek-V4 uses `[nope|rope]` for its query, key and compressed KV tensors, so `src/models/deepseek4.cpp` ropes the whole tensor and then calls `ggml_rope_set_offset(cur, n_embd_head_nope)`. +Exception: some models apply an extra op to the `nope` part, for example `deepseek32.cpp`, and may not use this optimization. While RoPE can be applied selectively to a part of the head, the extra op may not, so these models still need views plus `ggml_concat`. + ## GGUF specification https://github.com/ggml-org/ggml/blob/master/docs/gguf.md diff --git a/src/models/deepseek2.cpp b/src/models/deepseek2.cpp index ba90c0d077..e0e537e005 100644 --- a/src/models/deepseek2.cpp +++ b/src/models/deepseek2.cpp @@ -524,17 +524,9 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p q = ggml_mul_mat(ctx0, model.layers[il].wq, cur); cb(q, "q", il); } - // split into {n_embd_head_qk_nope, n_head, n_tokens} - ggml_tensor * q_nope = - ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k), - ggml_row_size(q->type, n_embd_head_k) * n_head, 0); - cb(q_nope, "q_nope", il); - - // and {n_embd_head_qk_rope, n_head, n_tokens} - ggml_tensor * q_pe = ggml_view_3d( - ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k), - ggml_row_size(q->type, n_embd_head_k) * n_head, ggml_row_size(q->type, n_embd_head_qk_nope)); - cb(q_pe, "q_pe", il); + // {n_embd_head_k, n_head, n_tokens} + q = ggml_reshape_3d(ctx0, q, n_embd_head_k, n_head, n_tokens); + cb(q, "q", il); ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur); cb(kv_cmpr_pe, "kv_cmpr_pe", il); @@ -552,10 +544,6 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); cb(k_pe, "k_pe", il); - q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, - ext_factor, attn_factor, beta_fast, beta_slow); - cb(q_pe, "q_pe", il); - k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); cb(k_pe, "k_pe", il); @@ -564,6 +552,20 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p cb(kv_cmpr, "kv_cmpr", il); if (is_mla) { + // split into {n_embd_head_qk_nope, n_head, n_tokens} + ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, + q->nb[1], q->nb[2], 0); + cb(q_nope, "q_nope", il); + + // and {n_embd_head_qk_rope, n_head, n_tokens} + ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, + q->nb[1], q->nb[2], ggml_row_size(q->type, n_embd_head_qk_nope)); + cb(q_pe, "q_pe", il); + + q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(q_pe, "q_pe", il); + // {n_embd_head_qk_nope, n_tokens, n_head} q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); cb(q_nope, "q_nope_perm", il); @@ -623,10 +625,14 @@ llama_model_deepseek2::graph::graph(const llama_model & model, const llm_graph_p Vcur = ggml_cont(ctx0, Vcur); cb(Vcur, "Vcur_cont", il); - ggml_tensor * Qcur = ggml_concat(ctx0, q_nope, q_pe, 0); + // RoPE is applied to the trailing dims only + ggml_tensor * Qcur = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + Qcur = ggml_rope_set_offset(Qcur, n_embd_head_qk_nope); cb(Qcur, "Qcur", il); - ggml_tensor * Kcur = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0); + ggml_tensor * Kcur = ggml_concat(ctx0, k_nope, + ggml_repeat_4d(ctx0, k_pe, n_embd_head_qk_rope, n_head, n_tokens, 1), 0); cb(Kcur, "Kcur", il); if (inp_attn_scale) {