upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-03 00:20:50 -05:00
Code Issues Packages Projects Releases Wiki Activity

convert: add option to create separate dspark GGUF (#26452)

Browse Source 
* convert: add option to create separate dspark GGUF

* add --no-nextn

* fix convert bug
This commit is contained in:
Aman Gupta
2026-08-02 23:16:31 +08:00
committed by GitHub
parent c745be2a2c
commit 3581ba0cf5
4 changed files with 146 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
conversion/__init__.py
View File

@@ -55,6 +55,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"DFlashDraftModel": "qwen",
"Qwen3DSparkModel": "qwen",
"DeepseekV4ForCausalLM": "deepseek",
"DeepseekV4DSparkModel": "deepseek",
"DistilBertForMaskedLM": "bert",
"DistilBertForSequenceClassification": "bert",
"DistilBertModel": "bert",

105
conversion/deepseek.py
View File

@@ -620,7 +620,8 @@ class DeepseekV4Model(TextModel):
self.gguf_writer.add_hyper_connection_sinkhorn_iterations(hparams["hc_sinkhorn_iters"])
self.gguf_writer.add_hyper_connection_epsilon(hparams["hc_eps"])
self.gguf_writer.add_hash_layer_count(hparams["num_hash_layers"])
self.gguf_writer.add_embedding_length_out(hparams["hidden_size"] * hparams["hc_mult"])
if self.model_arch == gguf.MODEL_ARCH.DEEPSEEK4:
self.gguf_writer.add_embedding_length_out(hparams["hidden_size"] * hparams["hc_mult"])
if self.mtp_only and (num_nextn_predict_layers := hparams.get("num_nextn_predict_layers", 0)) > 0:
self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers)
@@ -878,3 +879,105 @@ class DeepseekV4Model(TextModel):
super().prepare_tensors()
self._is_mxfp4 = True
self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE
@ModelBase.register("DeepseekV4DSparkModel")
class DeepseekV4DSparkModel(DeepseekV4Model):
model_arch = gguf.MODEL_ARCH.DFLASH
_DSPARK_ROOT_MAP: dict[str, tuple[gguf.MODEL_TENSOR, str]] = {
"main_proj.weight": (gguf.MODEL_TENSOR.FC, ".weight"),
"main_norm.weight": (gguf.MODEL_TENSOR.ENC_OUTPUT_NORM, ".weight"),
"markov_head.markov_w1.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W1, ".weight"),
"markov_head.markov_w2.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W2, ".weight"),
"confidence_head.proj.weight": (gguf.MODEL_TENSOR.DSPARK_CONF_PROJ, ".weight"),
}
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.block_count = 1 + max(
int(match.group(1)) for name in self.model_tensors
if (match := re.match(r"layers\.(\d+)\.", name))
)
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
self.hparams["compress_ratios"] = [0] * self.block_count
self.hparams["num_hash_layers"] = 0
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
if remote_hf_model_id is None:
return super().index_tensors()
with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f:
weight_map = json.load(f)["weight_map"]
part_names = sorted({
part_name for name, part_name in weight_map.items()
if name.startswith("mtp.")
})
tensors: dict[str, Callable[[], Tensor]] = {}
for part_name in part_names:
from huggingface_hub import hf_hub_download
logger.info("gguf: caching remote DSpark part '%s'", part_name)
part_path = Path(hf_hub_download(repo_id=remote_hf_model_id, filename=part_name))
with gguf.utility.SafetensorsLocal(part_path) as model_part:
for name in model_part:
data = model_part[name]
data_gen = lambda data=data: LazyTorchTensor.from_local_tensor(data) # noqa: E731
if titem := self.filter_tensors((name, data_gen)):
tensor_name, tensor_gen = titem
tensors[tensor_name] = tensor_gen
return tensors
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, gen = item
if not name.startswith("mtp."):
return None
return super().filter_tensors((cls._rekey_mtp_tensor_name(name), gen))
@staticmethod
def _rekey_mtp_tensor_name(name: str) -> str:
match = re.match(r"mtp\.(\d+)\.(.+)$", name)
if match is None:
raise ValueError(f"Unexpected DSpark tensor {name!r}")
stage, rest = match.group(1), match.group(2)
root_names = (
"main_proj.scale",
"norm.weight",
"hc_head_fn",
"hc_head_base",
"hc_head_scale",
)
if rest in DeepseekV4DSparkModel._DSPARK_ROOT_MAP or rest in root_names:
return rest
return f"layers.{stage}.{rest}"
def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]:
if name in self._DSPARK_ROOT_MAP:
return self._DSPARK_ROOT_MAP[name]
return super()._map_dsv4_tensor_name(name, bid)
def set_vocab(self):
if self.target_model_dir is None:
raise ValueError("DeepSeek-V4 DSpark requires --target-model-dir with the target tokenizer")
original_dir = self.dir_model
try:
self.dir_model = self.target_model_dir
super().set_vocab()
finally:
self.dir_model = original_dir
self.gguf_writer.add_mask_token_id(self.hparams["dspark_noise_token_id"])
def set_gguf_parameters(self):
super().set_gguf_parameters()
self.gguf_writer.add_block_size(self.hparams["dspark_block_size"])
self.gguf_writer.add_target_layers([layer + 1 for layer in self.hparams["dspark_target_layer_ids"]])

21
convert_hf_to_gguf.py
View File

@@ -122,8 +122,12 @@ def parse_args() -> argparse.Namespace:
help="Export only the multi-token prediction (MTP) head as a separate GGUF, suitable for use as a speculative draft. An 'mtp-' prefix will be added to the output file name.",
)
parser.add_argument(
"--no-mtp", action="store_true",
help="Exclude the multi-token prediction (MTP) head from the converted GGUF. Pair with --mtp on a second run to publish trunk and MTP as two files. Note: the split form duplicates embeddings, but even though the bundled default is more space-efficient overall, this allows differing quantization which may be more performant.",
"--no-nextn", "--no-mtp", dest="no_mtp", action="store_true",
help="Exclude NextN speculative draft tensors from the converted GGUF. Pair with --mtp or --dspark on a second run to publish target and draft as two files.",
)
parser.add_argument(
"--dspark", action="store_true",
help="Export only the DeepSeek-V4 DSpark draft tensors as a separate GGUF.",
)
parser.add_argument(
"--mistral-format", action="store_true",
@@ -254,13 +258,20 @@ def main() -> None:
from conversion.mistral import MistralModel
model_class = MistralModel
if args.mtp and args.no_mtp:
logger.error("--mtp and --no-mtp are mutually exclusive")
if sum((args.mtp, args.no_mtp, args.dspark)) > 1:
logger.error("--mtp, --no-nextn, and --dspark are mutually exclusive")
sys.exit(1)
if args.dspark:
if is_mistral_format or model_architecture != "DeepseekV4ForCausalLM":
logger.error("--dspark is only supported for DeepseekV4ForCausalLM")
sys.exit(1)
from conversion.deepseek import DeepseekV4DSparkModel
model_class = DeepseekV4DSparkModel
if args.mtp or args.no_mtp:
if not model_class.supports_mtp_export:
logger.error("--mtp / --no-mtp are not supported for %s", model_architecture)
logger.error("--mtp / --no-nextn are not supported for %s", model_architecture)
sys.exit(1)
if args.no_mtp:
model_class.no_mtp = True

25
gguf-py/gguf/constants.py
View File

@@ -4383,10 +4383,35 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.ATTN_OUT,
MODEL_TENSOR.ATTN_Q_NORM,
MODEL_TENSOR.ATTN_K_NORM,
MODEL_TENSOR.ATTN_SINKS,
MODEL_TENSOR.ATTN_Q_A,
MODEL_TENSOR.ATTN_Q_B,
MODEL_TENSOR.ATTN_Q_A_NORM,
MODEL_TENSOR.ATTN_KV,
MODEL_TENSOR.ATTN_KV_NORM,
MODEL_TENSOR.ATTN_OUT_A,
MODEL_TENSOR.ATTN_OUT_B,
MODEL_TENSOR.HC_ATTN_FN,
MODEL_TENSOR.HC_ATTN_BASE,
MODEL_TENSOR.HC_ATTN_SCALE,
MODEL_TENSOR.HC_FFN_FN,
MODEL_TENSOR.HC_FFN_BASE,
MODEL_TENSOR.HC_FFN_SCALE,
MODEL_TENSOR.HC_HEAD_FN,
MODEL_TENSOR.HC_HEAD_BASE,
MODEL_TENSOR.HC_HEAD_SCALE,
MODEL_TENSOR.FFN_NORM,
MODEL_TENSOR.FFN_GATE,
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_UP,
MODEL_TENSOR.FFN_GATE_INP,
MODEL_TENSOR.FFN_EXP_PROBS_B,
MODEL_TENSOR.FFN_GATE_EXP,
MODEL_TENSOR.FFN_DOWN_EXP,
MODEL_TENSOR.FFN_UP_EXP,
MODEL_TENSOR.FFN_GATE_SHEXP,
MODEL_TENSOR.FFN_DOWN_SHEXP,
MODEL_TENSOR.FFN_UP_SHEXP,
MODEL_TENSOR.FC,
MODEL_TENSOR.ENC_OUTPUT_NORM,
# optional DSpark heads