diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py index f52e6ac508..206978f06b 100644 --- a/conversion/qwen3tts.py +++ b/conversion/qwen3tts.py @@ -235,7 +235,10 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel): self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) # handle code predictor config + self.gguf_writer.add_clip_has_gen_audio_encoder(True) + self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.QWEN3TTS_GEN) code_predictor_config = self.global_config["talker_config"]["code_predictor_config"] + self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text) self.gguf_writer.add_gen_audio_embedding_length(code_predictor_config["hidden_size"]) self.gguf_writer.add_gen_audio_feed_forward_length(code_predictor_config["intermediate_size"]) self.gguf_writer.add_gen_audio_block_count(code_predictor_config["num_hidden_layers"]) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index ea46ac3af2..469940b950 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -322,6 +322,7 @@ class Keys: PROJECTOR_TYPE = "clip.projector_type" HAS_VISION_ENCODER = "clip.has_vision_encoder" HAS_AUDIO_ENCODER = "clip.has_audio_encoder" + HAS_GEN_AUDIO_ENCODER = "clip.has_gen_audio_encoder" HAS_LLAVA_PROJECTOR = "clip.has_llava_projector" class ClipVision: @@ -396,9 +397,11 @@ class Keys: HEAD_COUNT = "clip.audio.projector.head_count" class ClipGenAudio: + PROJECTOR_TYPE = "clip.gen.audio.projector_type" # for mixed modality models EMBEDDING_LENGTH = "clip.gen.audio.embedding_length" FEED_FORWARD_LENGTH = "clip.gen.audio.feed_forward_length" BLOCK_COUNT = "clip.gen.audio.block_count" + PROJECTION_DIM = "clip.gen.audio.projection_dim" class Attention: HEAD_COUNT = "clip.gen.audio.attention.head_count" @@ -1607,14 +1610,14 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.A_GEN_CODE_EMBD: "a.gen.code.embd", MODEL_TENSOR.A_GEN_CODE_HEAD: "a.gen.code.head", MODEL_TENSOR.A_GEN_CODE_OUT_EMBD: "a.gen.code.out_embd", - MODEL_TENSOR.A_GEN_CODE_ATTN_NORM: "a.gen.code.blk.{bid}.attn_norm", + MODEL_TENSOR.A_GEN_CODE_ATTN_NORM: "a.gen.code.blk.{bid}.ln1", # reuses the generic clip.cpp block loader (TN_LN_1) MODEL_TENSOR.A_GEN_CODE_ATTN_Q: "a.gen.code.blk.{bid}.attn_q", MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM: "a.gen.code.blk.{bid}.attn_q_norm", MODEL_TENSOR.A_GEN_CODE_ATTN_K: "a.gen.code.blk.{bid}.attn_k", MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM: "a.gen.code.blk.{bid}.attn_k_norm", MODEL_TENSOR.A_GEN_CODE_ATTN_V: "a.gen.code.blk.{bid}.attn_v", MODEL_TENSOR.A_GEN_CODE_ATTN_OUT: "a.gen.code.blk.{bid}.attn_out", - MODEL_TENSOR.A_GEN_CODE_FFN_NORM: "a.gen.code.blk.{bid}.ffn_norm", + MODEL_TENSOR.A_GEN_CODE_FFN_NORM: "a.gen.code.blk.{bid}.ln2", # reuses the generic clip.cpp block loader (TN_LN_2) MODEL_TENSOR.A_GEN_CODE_FFN_GATE: "a.gen.code.blk.{bid}.ffn_gate", MODEL_TENSOR.A_GEN_CODE_FFN_UP: "a.gen.code.blk.{bid}.ffn_up", MODEL_TENSOR.A_GEN_CODE_FFN_DOWN: "a.gen.code.blk.{bid}.ffn_down", @@ -4965,6 +4968,7 @@ class VisionProjectorType: YOUTUVL = "youtuvl" NEMOTRON_V2_VL = "nemotron_v2_vl" QWEN3TTS_SPKENC = "qwen3tts_spkenc" # audio: ECAPA-TDNN speaker encoder + QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor HUNYUANVL = "hunyuanvl" PARAKEET = "parakeet" # audio MINIMAXM3 = "minimax_m3" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 2ef09759c5..b30a0954e5 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1199,6 +1199,9 @@ class GGUFWriter: def add_clip_has_audio_encoder(self, value: bool) -> None: self.add_bool(Keys.Clip.HAS_AUDIO_ENCODER, value) + def add_clip_has_gen_audio_encoder(self, value: bool) -> None: + self.add_bool(Keys.Clip.HAS_GEN_AUDIO_ENCODER, value) + def add_clip_projector_type(self, value: str) -> None: self.add_string(Keys.Clip.PROJECTOR_TYPE, value) @@ -1398,11 +1401,17 @@ class GGUFWriter: def add_audio_projector_head_count(self, value: int) -> None: self.add_uint32(Keys.ClipAudio.Projector.HEAD_COUNT, value) - def add_gen_audio_embedding_length(self, value: int) -> None: - self.add_uint32(Keys.ClipGenAudio.EMBEDDING_LENGTH, value) - # audio generation (mmproj) + def add_clip_gen_audio_projector_type(self, value: str) -> None: + self.add_string(Keys.ClipGenAudio.PROJECTOR_TYPE, value) + + def add_gen_audio_projection_dim(self, value: int) -> None: + self.add_uint32(Keys.ClipGenAudio.PROJECTION_DIM, value) + + def add_gen_audio_embedding_length(self, value: int) -> None: + self.add_uint32(Keys.ClipGenAudio.EMBEDDING_LENGTH, value) + def add_gen_audio_feed_forward_length(self, value: int) -> None: self.add_uint32(Keys.ClipGenAudio.FEED_FORWARD_LENGTH, value) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index f73e80a8aa..b86adc5688 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -32,6 +32,7 @@ #define KEY_PROJ_TYPE "clip.projector_type" #define KEY_HAS_AUDIO_ENC "clip.has_audio_encoder" #define KEY_HAS_VISION_ENC "clip.has_vision_encoder" +#define KEY_HAS_GEN_AUDIO_ENC "clip.has_gen_audio_encoder" #define KEY_USE_GELU "clip.use_gelu" #define KEY_USE_SILU "clip.use_silu" @@ -88,6 +89,8 @@ #define KEY_A_ATTN_WINDOW_SIZE "clip.audio.window_size" // mimo-audio-tokenizer: sliding-window radius #define KEY_A_LOCAL_BLOCK_COUNT "clip.audio.local_block_count" // mimo-v2.5: input_local_transformer layer count #define KEY_A_LOCAL_GROUP_SIZE "clip.audio.local_group_size" // mimo-v2.5: input_local_transformer grouping size +// audio generation (gen-audio)-specific +#define KEY_GEN_AUDIO_PROJ_TYPE "clip.gen.audio.projector_type" // for models with mixed modalities #define KEY_AUDIO_SUBSAMPLING_FACTOR "clip.audio.subsampling_factor" // @@ -207,6 +210,13 @@ #define TN_A_ASP_ATTN "a.asp_attn.%s" #define TN_A_ASP_TDNN "a.asp_tdnn.%s" +// qwen3tts code_predictor +#define TN_A_GEN_CODE_PROJ_IN "a.gen.code.proj_in.%s" +#define TN_A_GEN_CODE_EMBD "a.gen.code.embd.%s" +#define TN_A_GEN_CODE_HEAD "a.gen.code.head.%s" +#define TN_A_GEN_CODE_OUT_EMBD "a.gen.code.out_embd.%s" +#define TN_A_GEN_CODE_NORM "a.gen.code.output_norm.%s" + // cogvlm #define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s" #define TN_MM_H_TO_4H "mm.up.%s" @@ -415,6 +425,7 @@ enum projector_type { PROJECTOR_TYPE_GRANITE4_VISION, PROJECTOR_TYPE_MIMO_AUDIO, PROJECTOR_TYPE_QWEN3TTS_SPKENC, + PROJECTOR_TYPE_QWEN3TTS_GEN, PROJECTOR_TYPE_UNKNOWN, }; @@ -471,8 +482,9 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"}, { PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"}, { PROJECTOR_TYPE_MIMO_AUDIO, "mimo_audio"}, - { PROJECTOR_TYPE_QWEN3TTS_SPKENC, "qwen3tts_spkenc"}, { PROJECTOR_TYPE_PARAKEET, "parakeet"}, + { PROJECTOR_TYPE_QWEN3TTS_SPKENC, "qwen3tts_spkenc"}, + { PROJECTOR_TYPE_QWEN3TTS_GEN, "qwen3tts_gen"}, }; static projector_type clip_projector_type_from_string(const std::string & str) { diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 07d36b6b9c..3cf513c943 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -592,6 +592,14 @@ struct clip_model { ggml_tensor * spk_asp_tdnn_w = nullptr; ggml_tensor * spk_asp_tdnn_b = nullptr; + // qwen3tts code_predictor + ggml_tensor * gen_code_proj_in_w = nullptr; // small_to_mtp_projection + ggml_tensor * gen_code_proj_in_b = nullptr; + ggml_tensor * gen_code_embd_w = nullptr; // per-codebook embedding, merged 3D + ggml_tensor * gen_code_head_w = nullptr; // per-codebook output head, merged 3D + ggml_tensor * gen_code_out_embd_w = nullptr; // codebook-0 embedding, fed back into the talker + ggml_tensor * gen_code_norm_w = nullptr; // final norm + // cogvlm ggml_tensor * mm_post_fc_norm_w = nullptr; ggml_tensor * mm_post_fc_norm_b = nullptr; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 73e9947824..c43c0d2572 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1069,8 +1069,9 @@ struct clip_model_loader { size_t model_size = 0; // in bytes - bool has_vision = false; - bool has_audio = false; + bool has_vision = false; + bool has_audio = false; + bool has_gen_audio = false; mtmd_progress_callback progress_callback = nullptr; void * progress_callback_user_data = nullptr; @@ -1116,8 +1117,9 @@ struct clip_model_loader { // modalities { - get_bool(KEY_HAS_VISION_ENC, has_vision, false); - get_bool(KEY_HAS_AUDIO_ENC, has_audio, false); + get_bool(KEY_HAS_VISION_ENC, has_vision, false); + get_bool(KEY_HAS_AUDIO_ENC, has_audio, false); + get_bool(KEY_HAS_GEN_AUDIO_ENC, has_gen_audio, false); if (has_vision) { LOG_INF("%s: has vision encoder\n", __func__); @@ -1125,6 +1127,9 @@ struct clip_model_loader { if (has_audio) { LOG_INF("%s: has audio encoder\n", __func__); } + if (has_gen_audio) { + LOG_INF("%s: has audio generation (gen) encoder\n", __func__); + } } // tensors @@ -1151,6 +1156,8 @@ struct clip_model_loader { GGML_ASSERT(has_vision); } else if (modality == CLIP_MODALITY_AUDIO) { GGML_ASSERT(has_audio); + } else if (modality == CLIP_MODALITY_GEN_AUDIO) { + GGML_ASSERT(has_gen_audio); } model.modality = modality; @@ -1167,6 +1174,8 @@ struct clip_model_loader { get_string(KEY_VISION_PROJ_TYPE, proj_type, false); } else if (modality == CLIP_MODALITY_AUDIO) { get_string(KEY_AUDIO_PROJ_TYPE, proj_type, false); + } else if (modality == CLIP_MODALITY_GEN_AUDIO) { + get_string(KEY_GEN_AUDIO_PROJ_TYPE, proj_type, false); } else { GGML_ABORT("unknown modality"); } @@ -1186,12 +1195,13 @@ struct clip_model_loader { } } - const bool is_vision = model.modality == CLIP_MODALITY_VISION; - const bool is_audio = model.modality == CLIP_MODALITY_AUDIO; + const bool is_vision = model.modality == CLIP_MODALITY_VISION; + const bool is_audio = model.modality == CLIP_MODALITY_AUDIO; + const bool is_gen_audio = model.modality == CLIP_MODALITY_GEN_AUDIO; // other hparams { - const char * prefix = is_vision ? "vision" : "audio"; + const char * prefix = is_vision ? "vision" : (is_audio ? "audio" : "gen.audio"); get_u32(string_format(KEY_N_EMBD, prefix), hparams.n_embd); get_u32(string_format(KEY_N_HEAD, prefix), hparams.n_head); get_u32(string_format(KEY_N_FF, prefix), hparams.n_ff); @@ -1229,6 +1239,11 @@ struct clip_model_loader { hparams.image_size = 0; hparams.patch_size = 1; + } else if (is_gen_audio) { + // these are unused, but still need to be set to avoid issues + hparams.image_size = 0; + hparams.patch_size = 1; + } else { GGML_ASSERT(false && "unknown modality"); } @@ -1658,6 +1673,10 @@ struct clip_model_loader { hparams.audio_window_len = 1024; hparams.audio_hop_len = 256; } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + // discrete-token autoregressive predictor, no mel-frontend needed + } break; case PROJECTOR_TYPE_PADDLEOCR: { hparams.n_merge = 2; @@ -1882,7 +1901,9 @@ struct clip_model_loader { } // TODO @ngxson : support both audio and video in the future - const char * prefix = model.modality == CLIP_MODALITY_AUDIO ? "a" : "v"; + const char * prefix = model.modality == CLIP_MODALITY_AUDIO ? "a" + : model.modality == CLIP_MODALITY_GEN_AUDIO ? "a.gen.code" + : "v"; // get offsets for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { @@ -2647,6 +2668,16 @@ struct clip_model_loader { model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight")); model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias")); } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + // code_predictor + model.gen_code_proj_in_w = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "weight")); + model.gen_code_proj_in_b = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "bias")); + model.gen_code_embd_w = get_tensor(string_format(TN_A_GEN_CODE_EMBD, "weight")); + model.gen_code_head_w = get_tensor(string_format(TN_A_GEN_CODE_HEAD, "weight")); + model.gen_code_out_embd_w = get_tensor(string_format(TN_A_GEN_CODE_OUT_EMBD, "weight")); + model.gen_code_norm_w = get_tensor(string_format(TN_A_GEN_CODE_NORM, "weight")); + } break; case PROJECTOR_TYPE_VOXTRAL: { model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight")); @@ -3475,6 +3506,7 @@ struct clip_model_loader { struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params) { clip_ctx * ctx_vision = nullptr; clip_ctx * ctx_audio = nullptr; + clip_ctx * ctx_gen_audio = nullptr; try { clip_model_loader loader(fname, @@ -3507,16 +3539,23 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params } } + if (loader.has_gen_audio) { + ctx_gen_audio = new clip_ctx(ctx_params); + loader.load_hparams(ctx_gen_audio->model, CLIP_MODALITY_GEN_AUDIO); + loader.load_tensors(*ctx_gen_audio); + } + } catch (const std::exception & e) { LOG_ERR("%s: failed to load model '%s': %s\n", __func__, fname, e.what()); delete ctx_vision; delete ctx_audio; + delete ctx_gen_audio; - return {nullptr, nullptr}; + return {nullptr, nullptr, nullptr}; } - return {ctx_vision, ctx_audio}; + return {ctx_vision, ctx_audio, ctx_gen_audio}; } struct clip_cap clip_get_cap(const char * fname) { diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index 967093a812..18836ecd0c 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -37,6 +37,7 @@ struct clip_image_f32_batch; enum clip_modality { CLIP_MODALITY_VISION, CLIP_MODALITY_AUDIO, + CLIP_MODALITY_GEN_AUDIO, }; enum clip_flash_attn_type { @@ -61,6 +62,7 @@ struct clip_context_params { struct clip_init_result { struct clip_ctx * ctx_v; // vision context struct clip_ctx * ctx_a; // audio context + struct clip_ctx * ctx_gen_a; // audio generation context }; struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params); diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 4841cde564..5e83bf039a 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -262,6 +262,9 @@ struct mtmd_context { struct clip_ctx * ctx_a; // audio std::vector out_embd; // image embedding vector + // generation context + struct clip_ctx * ctx_gen_a; // audio + bool print_timings; int n_threads; std::string media_marker; @@ -354,6 +357,7 @@ struct mtmd_context { auto res = clip_init(mmproj_fname, ctx_clip_params); ctx_v = res.ctx_v; ctx_a = res.ctx_a; + ctx_gen_a = res.ctx_gen_a; if (!ctx_v && !ctx_a) { throw std::runtime_error(string_format("Failed to load CLIP model from %s\n", mmproj_fname)); } @@ -784,6 +788,7 @@ struct mtmd_context { ~mtmd_context() { clip_free(ctx_a); clip_free(ctx_v); + clip_free(ctx_gen_a); } private: