diff --git a/conversion/__init__.py b/conversion/__init__.py index f9bfce4833..c89d734902 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -303,6 +303,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "Qwen2_5_VLForConditionalGeneration": "qwenvl", "Qwen3ASRForConditionalGeneration": "qwen3vl", "Qwen3OmniMoeForConditionalGeneration": "qwen3vl", + "Qwen3TTSForConditionalGeneration": "qwen3tts", "Qwen3VLForConditionalGeneration": "qwen3vl", "Qwen3VLMoeForConditionalGeneration": "qwen3vl", "Qwen3_5ForConditionalGeneration": "qwen3vl", diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py index 0aa2388f3a..de52f01e1a 100644 --- a/conversion/qwen3tts.py +++ b/conversion/qwen3tts.py @@ -1,14 +1,14 @@ from __future__ import annotations from pathlib import Path -from typing import Callable, Iterable, TYPE_CHECKING +from typing import Any, Callable, Iterable, TYPE_CHECKING import torch.nn.functional as F if TYPE_CHECKING: from torch import Tensor -from .base import ModelBase, TextModel, gguf, logger +from .base import ModelBase, MmprojModel, TextModel, gguf, logger # torch activation functions used by Qwen3TTSTalkerResizeMLP (config's hidden_act) @@ -21,10 +21,6 @@ _ACT2FN = { @ModelBase.register("Qwen3TTSForConditionalGeneration") class Qwen3TTSTalkerModel(TextModel): - """Converts only the talker's backbone transformer (text-conditioned codec - token predictor). The speaker encoder and the small code_predictor - sub-model are not handled yet.""" - model_arch = gguf.MODEL_ARCH.QWEN3TTS _TEXT_PROJ_KEYS = ( @@ -41,11 +37,7 @@ class Qwen3TTSTalkerModel(TextModel): hparams = kwargs.pop("hparams", None) if hparams is None: hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False) - # reuse TextModel's generic "text_config" flattening for the talker's own config talker_config = dict(hparams["talker_config"]) - # talker_config's own "vocab_size" is the codec vocab (talker.codec_head / - # talker.model.codec_embedding), not the BPE text vocab that "vocab_size" is - # normally expected to describe; use text_vocab_size (matches embed_tokens) instead talker_config["vocab_size"] = talker_config["text_vocab_size"] hparams["text_config"] = talker_config super().__init__(dir_model, *args, hparams=hparams, **kwargs) @@ -84,9 +76,7 @@ class Qwen3TTSTalkerModel(TextModel): if len(self._text_proj_buffer) < len(self._TEXT_PROJ_KEYS): return - # the talker only ever consumes text_embedding through text_projection - # (a 2-layer MLP: fc2(act(fc1(x)))), so fold it into the embedding table - # at conversion time instead of carrying the MLP weights around + # fold MLP into the embedding table at conversion time, MLP won't be used at inference time anyway act_fn = _ACT2FN[self.hparams["hidden_act"]] embed = self._text_proj_buffer["model.text_embedding.weight"] hidden = act_fn(F.linear(embed, @@ -99,3 +89,48 @@ class Qwen3TTSTalkerModel(TextModel): return yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("Qwen3TTSForConditionalGeneration") +class Qwen3TTSSpeakerEncoderModel(MmprojModel): + has_vision_encoder = False + has_audio_encoder = True + + def __init__(self, dir_model: Path, *args, **kwargs): + hparams = kwargs.pop("hparams", None) + if hparams is None: + hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False) + hparams["text_config"] = {"hidden_size": hparams["talker_config"]["hidden_size"]} + # ECAPA-TDNN has a fixed 4-stage backbone, not a configurable transformer depth; + # MmprojModel.__init__ still needs one of the n_block_keys to build its tensor map + hparams["speaker_encoder_config"]["n_layers"] = 4 + super().__init__(dir_model, *args, hparams=hparams, **kwargs) + + def get_audio_config(self) -> dict[str, Any] | None: + return self.global_config.get("speaker_encoder_config") + + def set_gguf_parameters(self): + self.gguf_writer.add_file_type(self.ftype) + self.gguf_writer.add_clip_has_audio_encoder(True) + self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.QWEN3TTS_SPKENC) + self.gguf_writer.add_audio_projection_dim(self.n_embd_text) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + + if not name.startswith("speaker_encoder."): + return None + + return super().filter_tensors((name, gen)) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if "res2net_block.blocks." in name: + assert bid is not None # the outer stage index, picked up from the tensor name automatically + xid = int(name.split("res2net_block.blocks.")[1].split(".")[0]) + suffix = "." + name.rsplit(".", 1)[1] + new_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_ENC_CONV_RES2].format(bid=bid, xid=xid) + suffix + yield (new_name, data_torch) + return + + yield from super().modify_tensors(data_torch, name, bid) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 18c9d3a89b..0c7d7e7991 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -956,6 +956,11 @@ class MODEL_TENSOR(IntEnum): A_ENC_DOWNSAMPLE_CONV = auto() # mimo-audio-tokenizer: post-transformer downsample conv A_ENC_DOWNSAMPLE_NORM = auto() # mimo-audio-tokenizer: post-transformer downsample norm A_ENC_RVQ_CODEBOOK = auto() # mimo-audio-tokenizer: residual vector quantizer codebook, per quantizer index + A_ENC_CONV_RES2 = auto() # qwen3tts + A_ENC_SE_CONV1 = auto() # qwen3tts + A_ENC_SE_CONV2 = auto() # qwen3tts + A_ENC_ASP_ATTN = auto() # qwen3tts + A_ENC_ASP_TDNN = auto() # qwen3tts A_MMPROJ = auto() A_MMPROJ_FC = auto() A_MM_NORM_PRE = auto() @@ -1558,6 +1563,11 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV: "a.downsample.conv", MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM: "a.downsample.norm", MODEL_TENSOR.A_ENC_RVQ_CODEBOOK: "a.rvq.codebook", + MODEL_TENSOR.A_ENC_CONV_RES2: "a.blk.{bid}.res2.{xid}", + MODEL_TENSOR.A_ENC_SE_CONV1: "a.blk.{bid}.se_conv1", + MODEL_TENSOR.A_ENC_SE_CONV2: "a.blk.{bid}.se_conv2", + MODEL_TENSOR.A_ENC_ASP_ATTN: "a.asp_attn", + MODEL_TENSOR.A_ENC_ASP_TDNN: "a.asp_tdnn", MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}", MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc", MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre", @@ -1805,6 +1815,11 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.A_ENC_CONV_NORM, MODEL_TENSOR.A_ENC_CONV_PW1, MODEL_TENSOR.A_ENC_CONV_PW2, + MODEL_TENSOR.A_ENC_CONV_RES2, + MODEL_TENSOR.A_ENC_SE_CONV1, + MODEL_TENSOR.A_ENC_SE_CONV2, + MODEL_TENSOR.A_ENC_ASP_ATTN, + MODEL_TENSOR.A_ENC_ASP_TDNN, MODEL_TENSOR.A_MM_INP_PROJ, MODEL_TENSOR.A_MM_SOFT_EMB_NORM, MODEL_TENSOR.A_MM_EMBEDDING, @@ -4867,6 +4882,7 @@ class VisionProjectorType: GLM4V = "glm4v" YOUTUVL = "youtuvl" NEMOTRON_V2_VL = "nemotron_v2_vl" + QWEN3TTS_SPKENC = "qwen3tts_spkenc" # audio: ECAPA-TDNN speaker encoder HUNYUANVL = "hunyuanvl" MINIMAXM3 = "minimax_m3" MINICPMV4_6 = "minicpmv4_6" diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 8299ac25b4..5bc624ac4c 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -2096,6 +2096,7 @@ class TensorNameMap: "model.audio_tower.subsample_conv_projection.conv_{bid}.conv", # gemma3n "conformer.subsample_conv_projection.layer{bid}.conv", # gemma4 "encoder.conv{bid}", # mimo-audio-tokenizer + "speaker_encoder.blocks.{bid}.conv", # qwen3tts speaker encoder (only bid=0, the stem TDNN) ), MODEL_TENSOR.A_ENC_CONV1D_NORM: ( @@ -2113,6 +2114,7 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_CONV_OUT: ( "audio_tower.conv_out", # qwen3omni + "speaker_encoder.mfa.conv", # qwen3tts speaker encoder: multi-layer feature aggregation ), MODEL_TENSOR.A_PRE_NORM: (), @@ -2306,7 +2308,8 @@ class TensorNameMap: MODEL_TENSOR.A_MMPROJ_FC: ( "audio.multi_modal_projector.linear", # qwen2audio "audio_tower.proj", # qwen2omni - "model.audio_tower.output_proj" # gemma4 + "model.audio_tower.output_proj", # gemma4 + "speaker_encoder.fc", # qwen3tts speaker encoder: final speaker embedding projection ), MODEL_TENSOR.A_MM_NORM_PRE: ( @@ -2369,12 +2372,30 @@ class TensorNameMap: "conformer.layers.{bid}.conv.pointwise_conv1", # lfm2 "conformer.layers.{bid}.lconv1d.linear_start", # gemma3n "encoder.layers.{bid}.conv.up_conv", # granite_speech + "speaker_encoder.blocks.{bid}.tdnn1.conv", # qwen3tts speaker encoder ), MODEL_TENSOR.A_ENC_CONV_PW2: ( "conformer.layers.{bid}.conv.pointwise_conv2", # lfm2 "conformer.layers.{bid}.lconv1d.linear_end", # gemma3n "encoder.layers.{bid}.conv.down_conv", # granite_speech + "speaker_encoder.blocks.{bid}.tdnn2.conv", # qwen3tts speaker encoder + ), + + MODEL_TENSOR.A_ENC_SE_CONV1: ( + "speaker_encoder.blocks.{bid}.se_block.conv1", # qwen3tts + ), + + MODEL_TENSOR.A_ENC_SE_CONV2: ( + "speaker_encoder.blocks.{bid}.se_block.conv2", # qwen3tts + ), + + MODEL_TENSOR.A_ENC_ASP_ATTN: ( + "speaker_encoder.asp.conv", # qwen3tts + ), + + MODEL_TENSOR.A_ENC_ASP_TDNN: ( + "speaker_encoder.asp.tdnn.conv", # qwen3tts ), MODEL_TENSOR.A_ENC_NORM_CONV: (