upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-02 02:47:26 -05:00
Code Issues Packages Projects Releases Wiki Activity

improve conversion script

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-10-01 19:51:06 +02:00
parent 1b89754bb4
commit 85eae8aeee
4 changed files with 30 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/base.py
+7 -2
View File
@@ -1268,14 +1268,14 @@ class ModelBase:
return inner
@staticmethod
def load_hparams(dir_model: Path, is_mistral_format: bool):
def load_hparams(dir_model: Path, is_mistral_format: bool, guess: bool = True):
if is_mistral_format:
with open(dir_model / "params.json", "r", encoding="utf-8") as f:
config = json.load(f)
return config
# checkpoints with a non-HF layout are matched by their own loader
config = ModelBase.load_hparams_guess(dir_model) if dir_model.is_dir() else None
config = ModelBase.load_hparams_guess(dir_model) if guess and dir_model.is_dir() else None
if config is not None:
return config
@@ -2884,6 +2884,11 @@ else:
LazyTorchTensor._dtype_str_map["F8_E8M0"] = torch.uint8
def jinja_str_or_json(name: str) -> str:
# jinja expression that renders a variable as-is if it is a string, as JSON otherwise
return "{{ " + name + " if " + name + " is string else " + name + " | tojson }}"
def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> str:
# TODO @ngxson : this won't work correctly if the model has both audio & vision encoders
# maybe we should fallback to text model's arch in that case, since not many models have both
conversion/bert.py
+8 -39
View File
@@ -11,7 +11,7 @@ import torch
if TYPE_CHECKING:
from torch import Tensor
from .base import ModelBase, SentencePieceTokenTypes, TextModel, gguf, logger
from .base import ModelBase, SentencePieceTokenTypes, TextModel, gguf, jinja_str_or_json, logger
@ModelBase.register("BertModel", "BertForMaskedLM", "CamembertModel", "BertForSequenceClassification")
@@ -641,11 +641,6 @@ class ModernBertModel(BertModel):
yield from super().modify_tensors(data_torch, name, bid)
def _jinja_str(name: str) -> str:
# non-string values are rendered as JSON
return "{{ " + name + " if " + name + " is string else " + name + " | tojson }}"
def _is_decision_checkpoint(dir_model: Path) -> bool:
if not (dir_model / "encoder" / "config.json").is_file():
return False
@@ -655,8 +650,7 @@ def _is_decision_checkpoint(dir_model: Path) -> bool:
@ModelBase.register_hparams_loader(_is_decision_checkpoint)
def _load_decision_hparams(dir_model: Path) -> dict[str, Any]:
logger.info("gguf: detected ModernBert decision checkpoint")
with open(dir_model / "encoder" / "config.json", encoding="utf-8") as f:
hparams = json.load(f)
hparams = ModelBase.load_hparams(dir_model / "encoder", False, guess=False)
is_julia = (dir_model / "julia_config.json").is_file()
with open(dir_model / ("julia_config.json" if is_julia else "rl_agent_config.json"), encoding="utf-8") as f:
decision = json.load(f)
@@ -675,23 +669,6 @@ def _load_decision_hparams(dir_model: Path) -> dict[str, Any]:
class ModernBertDecisionModel(ModernBertModel):
model_arch = gguf.MODEL_ARCH.MODERN_BERT
# nn.TransformerEncoderLayer -> ModernBert block
_HEAD_BLOCK_NAMES = {
"self_attn.in_proj_weight": "attn.Wqkv.weight",
"self_attn.in_proj_bias": "attn.Wqkv.bias",
"self_attn.out_proj": "attn.Wo",
"norm1": "attn_norm",
"norm2": "mlp_norm",
"linear1": "mlp.Wi",
"linear2": "mlp.Wo",
}
_HEAD_NAMES = {
"type_emb": "embeddings.token_type_embeddings",
"scorer.0": "head.norm",
"scorer.1": "head.dense",
"scorer.3": "classifier.out_proj",
}
def set_vocab(self):
# vocab loaders read self.dir_model, point it to the tokenizer sub-directory
dir_model = self.dir_model
@@ -707,7 +684,7 @@ class ModernBertDecisionModel(ModernBertModel):
with open(self.dir_model / "tokenizer" / "tokenizer_config.json", encoding="utf-8") as f:
tokenizer_config = json.load(f)
tok_cls, tok_sep, tok_mask = (tokenizer_config[k] for k in ("cls_token", "sep_token", "mask_token"))
description = _jinja_str("o.description")
description = jinja_str_or_json("o.description")
if self.hparams["decision"].get("architecture") == "JuliaDecisionModel":
option = "{% if o.description %}" + description + "{% else %}{{ o.key }}{% endif %}"
else:
@@ -718,11 +695,10 @@ class ModernBertDecisionModel(ModernBertModel):
+ "{% elif o.key == 'true' %}yes, the statement holds"
"{% else %}no, the statement does not hold{% endif %}{% endif %}"
)
# one marker token per option
return (
tok_cls + "{{ type }} question: " + _jinja_str("instructions") + tok_sep
tok_cls + "{{ type }} question: " + jinja_str_or_json("instructions") + tok_sep
+ "{% for o in options %}" + tok_mask + " " + option + "{% endfor %}"
+ tok_sep + _jinja_str("state") + tok_sep
+ tok_sep + jinja_str_or_json("state") + tok_sep
)
def set_gguf_parameters(self):
@@ -752,16 +728,9 @@ class ModernBertDecisionModel(ModernBertModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if name.startswith("head.layers.") and bid is not None:
# the head blocks come after the encoder blocks
suffix = name.split(".", 3)[3].replace("in_proj_", "in_proj.")
bid += self.block_count - self.hparams["decision"]["head_layers"]
suffix = name.split(".", 3)[3]
for old, new in self._HEAD_BLOCK_NAMES.items():
if suffix.startswith(old):
name = f"layers.{bid}.{new}{suffix[len(old):]}"
break
else:
for old, new in self._HEAD_NAMES.items():
if name.startswith(old + "."):
name = new + name[len(old):]
break
name = f"head.layers.{bid}.{suffix}"
yield from super().modify_tensors(data_torch, name, bid)
conversion/qwen.py
+4 -11
View File
@@ -11,7 +11,7 @@ import torch
if TYPE_CHECKING:
from torch import Tensor
from .base import LazyTorchTensor, ModelBase, ModelType, TextModel, get_model_architecture, gguf, logger
from .base import LazyTorchTensor, ModelBase, ModelType, TextModel, get_model_architecture, gguf, jinja_str_or_json, logger
@ModelBase.register("QWenLMHeadModel")
@@ -656,11 +656,6 @@ class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
model_arch = gguf.MODEL_ARCH.QWEN35
def _jinja_str(name: str) -> str:
# non-string values are rendered as JSON
return "{{ " + name + " if " + name + " is string else " + name + " | tojson }}"
def _is_openjev_checkpoint(dir_model: Path) -> bool:
return (dir_model / "helper" / "shim.py").is_file() and (dir_model / "config.json").is_file()
@@ -668,8 +663,7 @@ def _is_openjev_checkpoint(dir_model: Path) -> bool:
@ModelBase.register_hparams_loader(_is_openjev_checkpoint)
def _load_openjev_hparams(dir_model: Path) -> dict[str, Any]:
logger.info("gguf: detected OpenJev checkpoint")
with open(dir_model / "config.json", encoding="utf-8") as f:
hparams = json.load(f)
hparams = ModelBase.load_hparams(dir_model, False, guess=False)
hparams["architectures"] = ["OpenJevModel"]
return hparams
@@ -690,16 +684,15 @@ class OpenJevModel(Qwen3_5TextModel):
self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])
def _systemone_template(self) -> str:
description = _jinja_str("o.description")
description = jinja_str_or_json("o.description")
option = (
"{% if type != 'noul' %}{{ o.key }}: {% if o.description %}" + description + "{% endif %}"
"{% elif o.key == 'true' %}yes: {% if o.description %}" + description + "{% else %}The statement is true.{% endif %}"
"{% else %}no: {% if o.description %}" + description + "{% else %}The statement is false.{% endif %}{% endif %}"
)
# newlines next to a block tag are emitted as expressions, so that trim_blocks cannot drop them
return (
"{% set letters = '" + self._LETTERS + "' %}"
"<|im_start|>user\nState:\n" + _jinja_str("state") + "\n\nQuestion: " + _jinja_str("instructions")
"<|im_start|>user\nState:\n" + jinja_str_or_json("state") + "\n\nQuestion: " + jinja_str_or_json("instructions")
+ "{% if type == 'score' %} Rate along the ordered levels below (lowest first).{% endif %}"
"{{ '\\nOptions:\\n' }}"
"{% for o in options %}[{{ letters[loop.index0] }}] " + option + "{{ '\\n' }}{% endfor %}"
gguf-py/gguf/tensor_mapping.py
+11 -1
View File
@@ -48,6 +48,7 @@ class TensorNameMap:
# Token type embeddings
MODEL_TENSOR.TOKEN_TYPES: (
"embeddings.token_type_embeddings", # bert nomic-bert
"type_emb", # laya
),
# Normalization of token embeddings
@@ -216,6 +217,7 @@ class TensorNameMap:
"layers.{bid}.input_layernorm", # qwen3-embedding
"model.layers.{bid}.attention_layernorm", # apertus
"model.layers.{bid}.pre_attention_layernorm", # kormo
"head.layers.{bid}.norm1", # laya
),
# Attention norm 2
@@ -250,6 +252,7 @@ class TensorNameMap:
"layers.{bid}.attn.Wqkv", # modern-bert
"model.layers.{bid}.self_attn.language_expert_query_key_value", # cogvlm
"model.layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5
"head.layers.{bid}.self_attn.in_proj", # laya
),
# Attention query
@@ -355,6 +358,7 @@ class TensorNameMap:
"backbone.layers.{bid}.mixer.o_proj", # nemotron-h
"model.layers.{bid}.self_attn.language_expert_dense", # cogvlm
"model.blocks.{bid}.attn.attn_resid", # talkie
"head.layers.{bid}.self_attn.out_proj", # laya
),
# Attention output norm
@@ -420,7 +424,8 @@ class TensorNameMap:
"layers.{bid}.post_attention_layernorm", # qwen3-embedding
"model.layers.{bid}.feedforward_layernorm", # apertus
"model.layers.{bid}.pre_mlp_layernorm", # kormo
"layers.{bid}.mlp_norm" # modern-bert
"layers.{bid}.mlp_norm", # modern-bert
"head.layers.{bid}.norm2", # laya
),
# Pre feed-forward norm
@@ -533,6 +538,7 @@ class TensorNameMap:
"backbone.layers.{bid}.mixer.up_proj", # nemotron-h
"model.layers.{bid}.mlp.language_mlp.up_proj", # cogvlm
"model.blocks.{bid}.mlp.mlp_linear", # talkie
"head.layers.{bid}.linear1", # laya
),
MODEL_TENSOR.FFN_UP_EXP: (
@@ -663,6 +669,7 @@ class TensorNameMap:
"backbone.layers.{bid}.mixer.down_proj", # nemotron-h
"model.layers.{bid}.mlp.language_mlp.down_proj", # cogvlm
"model.blocks.{bid}.mlp.mlp_resid", # talkie
"head.layers.{bid}.linear2", # laya
),
MODEL_TENSOR.FFN_DOWN_EXP: (
@@ -1441,14 +1448,17 @@ class TensorNameMap:
"pre_classifier", # distillbert
"dense", # neobert
"head.dense", # modern-bert
"scorer.1", # laya
),
MODEL_TENSOR.CLS_OUT: (
"classifier.out_proj", # roberta
"scorer.3", # laya
),
MODEL_TENSOR.CLS_NORM: (
"head.norm", # modern-bert
"scorer.0", # laya
),
#############################################################################