diff --git a/conversion/dots3.py b/conversion/dots3.py index 93362537ee..7c36b84826 100644 --- a/conversion/dots3.py +++ b/conversion/dots3.py @@ -1,6 +1,7 @@ from __future__ import annotations import math +import re from typing import TYPE_CHECKING, Callable, Iterable @@ -16,6 +17,14 @@ from .deepseek import DeepseekV2Model class Dots3NoteModel(DeepseekV2Model): model_arch = gguf.MODEL_ARCH.DOTS3NOTE skip_mtp = False + supports_mtp_export = True + + # trunk layer count, stashed before indexing for filter_tensors (mirrors DeepseekV32Model) + _n_main_layers: int | None = None + + def index_tensors(self, remote_hf_model_id: str | None = None): + type(self)._n_main_layers = self.hparams["num_hidden_layers"] + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -75,10 +84,25 @@ class Dots3NoteModel(DeepseekV2Model): @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - name, _ = item + if (titem := super().filter_tensors(item)) is None: + return None + name, gen = titem if name.startswith(("vision_encoder.", "audio_encoder.")): return None - return super().filter_tensors(item) + + assert cls._n_main_layers is not None + is_mtp = name.startswith("model.mtp.") or \ + ((m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers) + + # --no-mtp: drop the NextN/MTP block; --mtp: keep only that block plus the shared embeddings/norm/lm_head + if is_mtp and cls.no_mtp: + return None + if cls.mtp_only and not is_mtp and name not in ( + "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", + ): + return None + + return name, gen def set_gguf_parameters(self): hparams = self.hparams @@ -116,6 +140,19 @@ class Dots3NoteModel(DeepseekV2Model): self.gguf_writer.add_indexer_top_k(hparams["index_topk"]) self.gguf_writer.add_indexer_types([not self._is_swa_layer(il) for il in range(n_layer)]) + def prepare_metadata(self, vocab_only: bool): + from_dir = self.fname_out.is_dir() + super().prepare_metadata(vocab_only=vocab_only) + + if not self.mtp_only or not from_dir: + return + + output_type: str = self.ftype.name.partition("_")[2] + fname_default: str = gguf.naming_convention( + self.metadata.name, self.metadata.basename, self.metadata.finetune, + self.metadata.version, size_label=None, output_type=output_type, model_type=None) + self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: # move the MTP token embedding into the NextN block so the standard nextn mapping picks it up if name == "model.mtp.embed_tokens.weight": diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 8bedb8914f..faaa69ceed 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -847,6 +847,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_230B_A10B: return "230B.A10B"; case LLM_TYPE_428B_A23B: return "428B.A23B"; case LLM_TYPE_235B_A22B: return "235B.A22B"; + case LLM_TYPE_288B_A19B: return "288B.A19B"; case LLM_TYPE_300B_A47B: return "300B.A47B"; case LLM_TYPE_310B_A15B: return "310B.A15B"; case LLM_TYPE_355B_A32B: return "355B.A32B"; diff --git a/src/llama-model.h b/src/llama-model.h index 4412ef08e7..44bd967575 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -140,6 +140,7 @@ enum llm_type { LLM_TYPE_230B_A10B, // Minimax M2 LLM_TYPE_428B_A23B, // Minimax M3 LLM_TYPE_235B_A22B, + LLM_TYPE_288B_A19B, // dots3-note LLM_TYPE_300B_A47B, // Ernie MoE big LLM_TYPE_310B_A15B, // /MiMo-V2-Flash LLM_TYPE_355B_A32B, // GLM-4.5 diff --git a/src/models/dots3note.cpp b/src/models/dots3note.cpp index ba376dcc84..d9ac0b3d07 100644 --- a/src/models/dots3note.cpp +++ b/src/models/dots3note.cpp @@ -37,19 +37,16 @@ void llama_model_dots3note::load_arch_hparams(llama_model_loader & ml) { hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa); - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); - // DSA parameters - by default the indexer exists exactly on the full-attention layers + // DSA parameters ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head); ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size); ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k); - for (uint32_t il = 0; il < hparams.n_layer(); ++il) { - hparams.is_indexer_full_impl[il] = hparams.is_swa(il) ? 0 : 1; - } - ml.get_key_or_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, hparams.n_layer(), false); + ml.get_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl); switch (hparams.n_layer()) { - case 46: type = LLM_TYPE_UNKNOWN; break; // 288B-A19B + case 46: type = LLM_TYPE_288B_A19B; break; default: type = LLM_TYPE_UNKNOWN; } } diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 5a3322880d..acbc0984f6 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -185,6 +185,13 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, uint32_t(192)); ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, uint32_t(128)); ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f); + // indexer on the full-attention layers (inverse of the swa pattern) + std::vector indexer_types; + indexer_types.reserve(n_layer); + for (uint32_t il = 0; il < n_layer; il++) { + indexer_types.push_back(il % 2 ? 0 : 1); + } + ms.add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, indexer_types); } } else if (arch == LLM_ARCH_MINIMAX_M3) { // partial rotary: n_rot must not exceed the indexer key length (64)