From 3581ba0cf591b3f772fbb002de0f70e294bc0396 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Sun, 2 Aug 2026 23:16:31 +0800 Subject: [PATCH] convert: add option to create separate dspark GGUF (#26452) * convert: add option to create separate dspark GGUF * add --no-nextn * fix convert bug --- conversion/__init__.py | 1 + conversion/deepseek.py | 105 +++++++++++++++++++++++++++++++++++++- convert_hf_to_gguf.py | 21 ++++++-- gguf-py/gguf/constants.py | 25 +++++++++ 4 files changed, 146 insertions(+), 6 deletions(-) diff --git a/conversion/__init__.py b/conversion/__init__.py index 1a47b851a0..534f9e309a 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -55,6 +55,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "DFlashDraftModel": "qwen", "Qwen3DSparkModel": "qwen", "DeepseekV4ForCausalLM": "deepseek", + "DeepseekV4DSparkModel": "deepseek", "DistilBertForMaskedLM": "bert", "DistilBertForSequenceClassification": "bert", "DistilBertModel": "bert", diff --git a/conversion/deepseek.py b/conversion/deepseek.py index 0bf69be3be..5b69e23437 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -620,7 +620,8 @@ class DeepseekV4Model(TextModel): self.gguf_writer.add_hyper_connection_sinkhorn_iterations(hparams["hc_sinkhorn_iters"]) self.gguf_writer.add_hyper_connection_epsilon(hparams["hc_eps"]) self.gguf_writer.add_hash_layer_count(hparams["num_hash_layers"]) - self.gguf_writer.add_embedding_length_out(hparams["hidden_size"] * hparams["hc_mult"]) + if self.model_arch == gguf.MODEL_ARCH.DEEPSEEK4: + self.gguf_writer.add_embedding_length_out(hparams["hidden_size"] * hparams["hc_mult"]) if self.mtp_only and (num_nextn_predict_layers := hparams.get("num_nextn_predict_layers", 0)) > 0: self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers) @@ -878,3 +879,105 @@ class DeepseekV4Model(TextModel): super().prepare_tensors() self._is_mxfp4 = True self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE + + +@ModelBase.register("DeepseekV4DSparkModel") +class DeepseekV4DSparkModel(DeepseekV4Model): + model_arch = gguf.MODEL_ARCH.DFLASH + + _DSPARK_ROOT_MAP: dict[str, tuple[gguf.MODEL_TENSOR, str]] = { + "main_proj.weight": (gguf.MODEL_TENSOR.FC, ".weight"), + "main_norm.weight": (gguf.MODEL_TENSOR.ENC_OUTPUT_NORM, ".weight"), + "markov_head.markov_w1.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W1, ".weight"), + "markov_head.markov_w2.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W2, ".weight"), + "confidence_head.proj.weight": (gguf.MODEL_TENSOR.DSPARK_CONF_PROJ, ".weight"), + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.block_count = 1 + max( + int(match.group(1)) for name in self.model_tensors + if (match := re.match(r"layers\.(\d+)\.", name)) + ) + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + + self.hparams["compress_ratios"] = [0] * self.block_count + self.hparams["num_hash_layers"] = 0 + + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + if remote_hf_model_id is None: + return super().index_tensors() + + with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f: + weight_map = json.load(f)["weight_map"] + + part_names = sorted({ + part_name for name, part_name in weight_map.items() + if name.startswith("mtp.") + }) + tensors: dict[str, Callable[[], Tensor]] = {} + + for part_name in part_names: + from huggingface_hub import hf_hub_download + + logger.info("gguf: caching remote DSpark part '%s'", part_name) + part_path = Path(hf_hub_download(repo_id=remote_hf_model_id, filename=part_name)) + with gguf.utility.SafetensorsLocal(part_path) as model_part: + for name in model_part: + data = model_part[name] + data_gen = lambda data=data: LazyTorchTensor.from_local_tensor(data) # noqa: E731 + if titem := self.filter_tensors((name, data_gen)): + tensor_name, tensor_gen = titem + tensors[tensor_name] = tensor_gen + + return tensors + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + if not name.startswith("mtp."): + return None + return super().filter_tensors((cls._rekey_mtp_tensor_name(name), gen)) + + @staticmethod + def _rekey_mtp_tensor_name(name: str) -> str: + match = re.match(r"mtp\.(\d+)\.(.+)$", name) + if match is None: + raise ValueError(f"Unexpected DSpark tensor {name!r}") + + stage, rest = match.group(1), match.group(2) + root_names = ( + "main_proj.scale", + "norm.weight", + "hc_head_fn", + "hc_head_base", + "hc_head_scale", + ) + if rest in DeepseekV4DSparkModel._DSPARK_ROOT_MAP or rest in root_names: + return rest + return f"layers.{stage}.{rest}" + + def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]: + if name in self._DSPARK_ROOT_MAP: + return self._DSPARK_ROOT_MAP[name] + return super()._map_dsv4_tensor_name(name, bid) + + def set_vocab(self): + if self.target_model_dir is None: + raise ValueError("DeepSeek-V4 DSpark requires --target-model-dir with the target tokenizer") + + original_dir = self.dir_model + try: + self.dir_model = self.target_model_dir + super().set_vocab() + finally: + self.dir_model = original_dir + + self.gguf_writer.add_mask_token_id(self.hparams["dspark_noise_token_id"]) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + self.gguf_writer.add_block_size(self.hparams["dspark_block_size"]) + self.gguf_writer.add_target_layers([layer + 1 for layer in self.hparams["dspark_target_layer_ids"]]) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 2c5e62a16f..78ad26c656 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -122,8 +122,12 @@ def parse_args() -> argparse.Namespace: help="Export only the multi-token prediction (MTP) head as a separate GGUF, suitable for use as a speculative draft. An 'mtp-' prefix will be added to the output file name.", ) parser.add_argument( - "--no-mtp", action="store_true", - help="Exclude the multi-token prediction (MTP) head from the converted GGUF. Pair with --mtp on a second run to publish trunk and MTP as two files. Note: the split form duplicates embeddings, but even though the bundled default is more space-efficient overall, this allows differing quantization which may be more performant.", + "--no-nextn", "--no-mtp", dest="no_mtp", action="store_true", + help="Exclude NextN speculative draft tensors from the converted GGUF. Pair with --mtp or --dspark on a second run to publish target and draft as two files.", + ) + parser.add_argument( + "--dspark", action="store_true", + help="Export only the DeepSeek-V4 DSpark draft tensors as a separate GGUF.", ) parser.add_argument( "--mistral-format", action="store_true", @@ -254,13 +258,20 @@ def main() -> None: from conversion.mistral import MistralModel model_class = MistralModel - if args.mtp and args.no_mtp: - logger.error("--mtp and --no-mtp are mutually exclusive") + if sum((args.mtp, args.no_mtp, args.dspark)) > 1: + logger.error("--mtp, --no-nextn, and --dspark are mutually exclusive") sys.exit(1) + if args.dspark: + if is_mistral_format or model_architecture != "DeepseekV4ForCausalLM": + logger.error("--dspark is only supported for DeepseekV4ForCausalLM") + sys.exit(1) + from conversion.deepseek import DeepseekV4DSparkModel + model_class = DeepseekV4DSparkModel + if args.mtp or args.no_mtp: if not model_class.supports_mtp_export: - logger.error("--mtp / --no-mtp are not supported for %s", model_architecture) + logger.error("--mtp / --no-nextn are not supported for %s", model_architecture) sys.exit(1) if args.no_mtp: model_class.no_mtp = True diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 978231091c..a0e55781ba 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -4383,10 +4383,35 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_SINKS, + MODEL_TENSOR.ATTN_Q_A, + MODEL_TENSOR.ATTN_Q_B, + MODEL_TENSOR.ATTN_Q_A_NORM, + MODEL_TENSOR.ATTN_KV, + MODEL_TENSOR.ATTN_KV_NORM, + MODEL_TENSOR.ATTN_OUT_A, + MODEL_TENSOR.ATTN_OUT_B, + MODEL_TENSOR.HC_ATTN_FN, + MODEL_TENSOR.HC_ATTN_BASE, + MODEL_TENSOR.HC_ATTN_SCALE, + MODEL_TENSOR.HC_FFN_FN, + MODEL_TENSOR.HC_FFN_BASE, + MODEL_TENSOR.HC_FFN_SCALE, + MODEL_TENSOR.HC_HEAD_FN, + MODEL_TENSOR.HC_HEAD_BASE, + MODEL_TENSOR.HC_HEAD_SCALE, MODEL_TENSOR.FFN_NORM, MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, MODEL_TENSOR.FC, MODEL_TENSOR.ENC_OUTPUT_NORM, # optional DSpark heads