From a55a8c5266c86ca60b9e5251231f24140e2b34b0 Mon Sep 17 00:00:00 2001 From: Ruixiang Wang Date: Thu, 16 Jul 2026 14:19:47 +0200 Subject: [PATCH] convert : fix dflash target tokenizer mismatch during conversion (#25733) * spec: fix dflash target tokenizer mismatch during conversion * fix ci ty check --- conversion/qwen.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/conversion/qwen.py b/conversion/qwen.py index 82d42fcc16..9bc2b99fde 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json + from typing import Any, Callable, Iterable, TYPE_CHECKING import torch @@ -641,7 +643,19 @@ class DFlashModel(Qwen3Model): logger.info(f"DFlash: Using tokenizer from target model: {self.target_model_dir}") original_dir = self.dir_model self.dir_model = self.target_model_dir - super().set_vocab() + + # Reuse the target model's own vocab handler (e.g. Gemma-4 needs its + # own tokenizer logic, not the Qwen default). + from . import get_model_class + with open(self.target_model_dir / "config.json", "r", encoding="utf-8") as f: + target_arch = json.load(f)["architectures"][0] + target_cls = get_model_class(target_arch) + + if target_cls is not type(self): + target_cls.set_vocab(self) # ty: ignore[unresolved-attribute] + else: + super().set_vocab() + self.dir_model = original_dir mask_token_id = self.hparams.get("dflash_config", {}).get("mask_token_id")