diff --git a/conversion/base.py b/conversion/base.py index e61d142f68..f13ef56dc2 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -1268,14 +1268,14 @@ class ModelBase: return inner @staticmethod - def load_hparams(dir_model: Path, is_mistral_format: bool): + def load_hparams(dir_model: Path, is_mistral_format: bool, guess: bool = True): if is_mistral_format: with open(dir_model / "params.json", "r", encoding="utf-8") as f: config = json.load(f) return config # checkpoints with a non-HF layout are matched by their own loader - config = ModelBase.load_hparams_guess(dir_model) if dir_model.is_dir() else None + config = ModelBase.load_hparams_guess(dir_model) if guess and dir_model.is_dir() else None if config is not None: return config @@ -2884,6 +2884,11 @@ else: LazyTorchTensor._dtype_str_map["F8_E8M0"] = torch.uint8 +def jinja_str_or_json(name: str) -> str: + # jinja expression that renders a variable as-is if it is a string, as JSON otherwise + return "{{ " + name + " if " + name + " is string else " + name + " | tojson }}" + + def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> str: # TODO @ngxson : this won't work correctly if the model has both audio & vision encoders # maybe we should fallback to text model's arch in that case, since not many models have both diff --git a/conversion/bert.py b/conversion/bert.py index 32b71278fb..0d9001bc43 100644 --- a/conversion/bert.py +++ b/conversion/bert.py @@ -11,7 +11,7 @@ import torch if TYPE_CHECKING: from torch import Tensor -from .base import ModelBase, SentencePieceTokenTypes, TextModel, gguf, logger +from .base import ModelBase, SentencePieceTokenTypes, TextModel, gguf, jinja_str_or_json, logger @ModelBase.register("BertModel", "BertForMaskedLM", "CamembertModel", "BertForSequenceClassification") @@ -641,11 +641,6 @@ class ModernBertModel(BertModel): yield from super().modify_tensors(data_torch, name, bid) -def _jinja_str(name: str) -> str: - # non-string values are rendered as JSON - return "{{ " + name + " if " + name + " is string else " + name + " | tojson }}" - - def _is_decision_checkpoint(dir_model: Path) -> bool: if not (dir_model / "encoder" / "config.json").is_file(): return False @@ -655,8 +650,7 @@ def _is_decision_checkpoint(dir_model: Path) -> bool: @ModelBase.register_hparams_loader(_is_decision_checkpoint) def _load_decision_hparams(dir_model: Path) -> dict[str, Any]: logger.info("gguf: detected ModernBert decision checkpoint") - with open(dir_model / "encoder" / "config.json", encoding="utf-8") as f: - hparams = json.load(f) + hparams = ModelBase.load_hparams(dir_model / "encoder", False, guess=False) is_julia = (dir_model / "julia_config.json").is_file() with open(dir_model / ("julia_config.json" if is_julia else "rl_agent_config.json"), encoding="utf-8") as f: decision = json.load(f) @@ -675,23 +669,6 @@ def _load_decision_hparams(dir_model: Path) -> dict[str, Any]: class ModernBertDecisionModel(ModernBertModel): model_arch = gguf.MODEL_ARCH.MODERN_BERT - # nn.TransformerEncoderLayer -> ModernBert block - _HEAD_BLOCK_NAMES = { - "self_attn.in_proj_weight": "attn.Wqkv.weight", - "self_attn.in_proj_bias": "attn.Wqkv.bias", - "self_attn.out_proj": "attn.Wo", - "norm1": "attn_norm", - "norm2": "mlp_norm", - "linear1": "mlp.Wi", - "linear2": "mlp.Wo", - } - _HEAD_NAMES = { - "type_emb": "embeddings.token_type_embeddings", - "scorer.0": "head.norm", - "scorer.1": "head.dense", - "scorer.3": "classifier.out_proj", - } - def set_vocab(self): # vocab loaders read self.dir_model, point it to the tokenizer sub-directory dir_model = self.dir_model @@ -707,7 +684,7 @@ class ModernBertDecisionModel(ModernBertModel): with open(self.dir_model / "tokenizer" / "tokenizer_config.json", encoding="utf-8") as f: tokenizer_config = json.load(f) tok_cls, tok_sep, tok_mask = (tokenizer_config[k] for k in ("cls_token", "sep_token", "mask_token")) - description = _jinja_str("o.description") + description = jinja_str_or_json("o.description") if self.hparams["decision"].get("architecture") == "JuliaDecisionModel": option = "{% if o.description %}" + description + "{% else %}{{ o.key }}{% endif %}" else: @@ -718,11 +695,10 @@ class ModernBertDecisionModel(ModernBertModel): + "{% elif o.key == 'true' %}yes, the statement holds" "{% else %}no, the statement does not hold{% endif %}{% endif %}" ) - # one marker token per option return ( - tok_cls + "{{ type }} question: " + _jinja_str("instructions") + tok_sep + tok_cls + "{{ type }} question: " + jinja_str_or_json("instructions") + tok_sep + "{% for o in options %}" + tok_mask + " " + option + "{% endfor %}" - + tok_sep + _jinja_str("state") + tok_sep + + tok_sep + jinja_str_or_json("state") + tok_sep ) def set_gguf_parameters(self): @@ -752,16 +728,9 @@ class ModernBertDecisionModel(ModernBertModel): def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: if name.startswith("head.layers.") and bid is not None: + # the head blocks come after the encoder blocks + suffix = name.split(".", 3)[3].replace("in_proj_", "in_proj.") bid += self.block_count - self.hparams["decision"]["head_layers"] - suffix = name.split(".", 3)[3] - for old, new in self._HEAD_BLOCK_NAMES.items(): - if suffix.startswith(old): - name = f"layers.{bid}.{new}{suffix[len(old):]}" - break - else: - for old, new in self._HEAD_NAMES.items(): - if name.startswith(old + "."): - name = new + name[len(old):] - break + name = f"head.layers.{bid}.{suffix}" yield from super().modify_tensors(data_torch, name, bid) diff --git a/conversion/qwen.py b/conversion/qwen.py index 17b31aef22..edc77e817f 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -11,7 +11,7 @@ import torch if TYPE_CHECKING: from torch import Tensor -from .base import LazyTorchTensor, ModelBase, ModelType, TextModel, get_model_architecture, gguf, logger +from .base import LazyTorchTensor, ModelBase, ModelType, TextModel, get_model_architecture, gguf, jinja_str_or_json, logger @ModelBase.register("QWenLMHeadModel") @@ -656,11 +656,6 @@ class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): model_arch = gguf.MODEL_ARCH.QWEN35 -def _jinja_str(name: str) -> str: - # non-string values are rendered as JSON - return "{{ " + name + " if " + name + " is string else " + name + " | tojson }}" - - def _is_openjev_checkpoint(dir_model: Path) -> bool: return (dir_model / "helper" / "shim.py").is_file() and (dir_model / "config.json").is_file() @@ -668,8 +663,7 @@ def _is_openjev_checkpoint(dir_model: Path) -> bool: @ModelBase.register_hparams_loader(_is_openjev_checkpoint) def _load_openjev_hparams(dir_model: Path) -> dict[str, Any]: logger.info("gguf: detected OpenJev checkpoint") - with open(dir_model / "config.json", encoding="utf-8") as f: - hparams = json.load(f) + hparams = ModelBase.load_hparams(dir_model, False, guess=False) hparams["architectures"] = ["OpenJevModel"] return hparams @@ -690,16 +684,15 @@ class OpenJevModel(Qwen3_5TextModel): self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}]) def _systemone_template(self) -> str: - description = _jinja_str("o.description") + description = jinja_str_or_json("o.description") option = ( "{% if type != 'noul' %}{{ o.key }}: {% if o.description %}" + description + "{% endif %}" "{% elif o.key == 'true' %}yes: {% if o.description %}" + description + "{% else %}The statement is true.{% endif %}" "{% else %}no: {% if o.description %}" + description + "{% else %}The statement is false.{% endif %}{% endif %}" ) - # newlines next to a block tag are emitted as expressions, so that trim_blocks cannot drop them return ( "{% set letters = '" + self._LETTERS + "' %}" - "<|im_start|>user\nState:\n" + _jinja_str("state") + "\n\nQuestion: " + _jinja_str("instructions") + "<|im_start|>user\nState:\n" + jinja_str_or_json("state") + "\n\nQuestion: " + jinja_str_or_json("instructions") + "{% if type == 'score' %} Rate along the ordered levels below (lowest first).{% endif %}" "{{ '\\nOptions:\\n' }}" "{% for o in options %}[{{ letters[loop.index0] }}] " + option + "{{ '\\n' }}{% endfor %}" diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index ed7f1c2beb..c8d52f2fe2 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -48,6 +48,7 @@ class TensorNameMap: # Token type embeddings MODEL_TENSOR.TOKEN_TYPES: ( "embeddings.token_type_embeddings", # bert nomic-bert + "type_emb", # laya ), # Normalization of token embeddings @@ -216,6 +217,7 @@ class TensorNameMap: "layers.{bid}.input_layernorm", # qwen3-embedding "model.layers.{bid}.attention_layernorm", # apertus "model.layers.{bid}.pre_attention_layernorm", # kormo + "head.layers.{bid}.norm1", # laya ), # Attention norm 2 @@ -250,6 +252,7 @@ class TensorNameMap: "layers.{bid}.attn.Wqkv", # modern-bert "model.layers.{bid}.self_attn.language_expert_query_key_value", # cogvlm "model.layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5 + "head.layers.{bid}.self_attn.in_proj", # laya ), # Attention query @@ -355,6 +358,7 @@ class TensorNameMap: "backbone.layers.{bid}.mixer.o_proj", # nemotron-h "model.layers.{bid}.self_attn.language_expert_dense", # cogvlm "model.blocks.{bid}.attn.attn_resid", # talkie + "head.layers.{bid}.self_attn.out_proj", # laya ), # Attention output norm @@ -420,7 +424,8 @@ class TensorNameMap: "layers.{bid}.post_attention_layernorm", # qwen3-embedding "model.layers.{bid}.feedforward_layernorm", # apertus "model.layers.{bid}.pre_mlp_layernorm", # kormo - "layers.{bid}.mlp_norm" # modern-bert + "layers.{bid}.mlp_norm", # modern-bert + "head.layers.{bid}.norm2", # laya ), # Pre feed-forward norm @@ -533,6 +538,7 @@ class TensorNameMap: "backbone.layers.{bid}.mixer.up_proj", # nemotron-h "model.layers.{bid}.mlp.language_mlp.up_proj", # cogvlm "model.blocks.{bid}.mlp.mlp_linear", # talkie + "head.layers.{bid}.linear1", # laya ), MODEL_TENSOR.FFN_UP_EXP: ( @@ -663,6 +669,7 @@ class TensorNameMap: "backbone.layers.{bid}.mixer.down_proj", # nemotron-h "model.layers.{bid}.mlp.language_mlp.down_proj", # cogvlm "model.blocks.{bid}.mlp.mlp_resid", # talkie + "head.layers.{bid}.linear2", # laya ), MODEL_TENSOR.FFN_DOWN_EXP: ( @@ -1441,14 +1448,17 @@ class TensorNameMap: "pre_classifier", # distillbert "dense", # neobert "head.dense", # modern-bert + "scorer.1", # laya ), MODEL_TENSOR.CLS_OUT: ( "classifier.out_proj", # roberta + "scorer.3", # laya ), MODEL_TENSOR.CLS_NORM: ( "head.norm", # modern-bert + "scorer.0", # laya ), #############################################################################