diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py index d73383eb75..096b1135f3 100644 --- a/conversion/qwen3tts.py +++ b/conversion/qwen3tts.py @@ -19,6 +19,17 @@ _ACT2FN = { } +DEFAULT_TEMPLATE = """{% for message in messages %} +{% if message['role'] == 'system' %}<|im_start|>system +{{ message['content'] }}<|im_end|> +{% elif message['role'] == 'user' %}<|im_start|>user +{{ message['content'] }}<|im_end|> +{% endif %} +{% endfor %}<|im_start|>assistant +{{ text_to_speak }}<|im_end|> +<|im_start|>assistant""" + + @ModelBase.register("Qwen3TTSForConditionalGeneration") class Qwen3TTSTalkerModel(TextModel): model_arch = gguf.MODEL_ARCH.QWEN3TTS @@ -51,8 +62,7 @@ class Qwen3TTSTalkerModel(TextModel): def set_gguf_parameters(self): super().set_gguf_parameters() - # TODO: figure out the template - self.gguf_writer.add_chat_template("{% for m in messages %}{{m['content']}}{% endfor %}") + self.gguf_writer.add_chat_template(DEFAULT_TEMPLATE) @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 18a8288ba0..d7b7a812fc 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -52,6 +52,7 @@ add_library(mtmd models/mimovl.cpp models/qwen3a.cpp models/mimo-audio.cpp + models/qwen3tts-spkenc.cpp models/step3vl.cpp models/siglip.cpp models/whisper-enc.cpp diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 0f7bd1596c..55f69b67c4 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1025,6 +1025,10 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_YOUTUVL: { builder = std::make_unique(ctx, img); @@ -4408,6 +4412,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 case PROJECTOR_TYPE_COGVLM: case PROJECTOR_TYPE_YASA2: case PROJECTOR_TYPE_GEMMA4UA: + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: { // do nothing } break; @@ -4899,7 +4904,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_MIMO_AUDIO: return ctx->model.mm_2_w->ne[1]; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: - return ctx->model.mm_fc_w->ne[1]; + return ctx->model.mm_fc_w->ne[2]; default: GGML_ABORT("Unknown projector type"); } diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index caed438ec5..83db37ef3f 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -215,6 +215,17 @@ struct clip_graph_mimo_audio : clip_graph { ggml_cgraph * build() override; }; +struct clip_graph_qwen3tts_spkenc : clip_graph { + clip_graph_qwen3tts_spkenc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; + + ggml_tensor * conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const; + ggml_tensor * res2net(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const; + ggml_tensor * se_block(ggml_tensor * x, const clip_layer & layer) const; + ggml_tensor * se_res2net_block(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const; + ggml_tensor * attentive_stats_pool(ggml_tensor * x) const; +}; + struct clip_graph_kimik25 : clip_graph { clip_graph_kimik25(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/models/qwen3tts-spkenc.cpp b/tools/mtmd/models/qwen3tts-spkenc.cpp new file mode 100644 index 0000000000..5c3f633124 --- /dev/null +++ b/tools/mtmd/models/qwen3tts-spkenc.cpp @@ -0,0 +1,201 @@ +#include "models.h" + +static constexpr int SPK_RES2NET_SCALE = 8; // enc_res2net_scale +static constexpr int SPK_DILATIONS[3] = { 2, 3, 4 }; // enc_dilations[1..3] + +// Conv1d, kernel K, padding "same" (reflect), dilation d. +// x: [C, T] (ne[0]=C, ne[1]=T) -> [out_c, T] +ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const { + const int K = (int) w->ne[0]; + const int IC = (int) w->ne[1]; + const int OC = (int) w->ne[2]; + const int pad = ((K - 1) * dilation) / 2; + + // ggml_pad_reflect_1d pads ne[0], so bring T onto ne[0] first; im2col + // below expects the same [T, IC] layout. + ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, IC] + if (pad > 0) { + x_t = ggml_pad_reflect_1d(ctx0, x_t, pad, pad); // [T + 2*pad, IC] + } + ggml_tensor * x4d = ggml_reshape_4d(ctx0, x_t, x_t->ne[0], IC, 1, 1); + + // Dummy F32 kernel: im2col only reads its shape (K, IC), never its data, + // so this avoids a type assert when w is quantized. + ggml_tensor * dummy = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, K, IC, 1, 1); + + ggml_tensor * col = ggml_im2col(ctx0, dummy, x4d, 1, 1, 0, 0, dilation, 1, false, GGML_TYPE_F32); + const int64_t T_out = col->ne[1]; + col = ggml_reshape_2d(ctx0, col, (int64_t) K * IC, T_out); + + ggml_tensor * w2d = ggml_reshape_2d(ctx0, w, (int64_t) K * IC, OC); + ggml_tensor * y = ggml_mul_mat(ctx0, w2d, col); // [OC, T_out] + ggml_mul_mat_set_prec(y, GGML_PREC_F32); + + ggml_tensor * b2d = ggml_reshape_2d(ctx0, b, OC, 1); + y = ggml_add(ctx0, y, b2d); + return y; +} + +// Res2Net: split channel axis into `scale` chunks, chain dilated conv1d +// branches. x: [C, T] -> [C, T] +ggml_tensor * clip_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const { + const int64_t C = x->ne[0]; + const int64_t T = x->ne[1]; + const int64_t Cs = C / scale; + + std::vector outs; + outs.reserve(scale); + + auto chunk = [&](int i) -> ggml_tensor * { + return ggml_view_2d(ctx0, x, Cs, T, x->nb[1], (size_t) i * Cs * x->nb[0]); + }; + + ggml_tensor * prev = nullptr; + for (int i = 0; i < scale; i++) { + ggml_tensor * c = ggml_cont(ctx0, chunk(i)); + if (i == 0) { + outs.push_back(c); + continue; + } + ggml_tensor * inp = (i >= 2) ? ggml_add(ctx0, c, prev) : c; + ggml_tensor * y = conv1d_same(inp, layer.res2_conv_w[i - 1], layer.res2_conv_b[i - 1], dilation); + y = ggml_relu(ctx0, y); + outs.push_back(y); + prev = y; + } + + ggml_tensor * acc = outs[0]; + for (int i = 1; i < scale; i++) { + acc = ggml_concat(ctx0, acc, outs[i], 0); + } + return acc; +} + +// Squeeze-and-excitation gate. x: [C, T] -> [C, T] +ggml_tensor * clip_graph_qwen3tts_spkenc::se_block(ggml_tensor * x, const clip_layer & layer) const { + // temporal mean, keepdim: transpose so T is on ne[0], reduce, transpose back + ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, C] + ggml_tensor * mean = ggml_mean(ctx0, x_t); // [1, C] + mean = ggml_cont(ctx0, ggml_transpose(ctx0, mean)); // [C, 1] + + ggml_tensor * h = conv1d_same(mean, layer.se_conv1_w, layer.se_conv1_b, 1); + h = ggml_relu(ctx0, h); + h = conv1d_same(h, layer.se_conv2_w, layer.se_conv2_b, 1); + h = ggml_sigmoid(ctx0, h); // [C, 1] + + return ggml_mul(ctx0, x, h); // broadcast gate over T +} + +// tdnn1 -> res2net -> tdnn2 -> se, plus residual. x: [C, T] -> [C, T] +ggml_tensor * clip_graph_qwen3tts_spkenc::se_res2net_block(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const { + ggml_tensor * residual = x; + ggml_tensor * h = conv1d_same(x, layer.conv_pw1_w, layer.conv_pw1_b, 1); // tdnn1 + h = ggml_relu(ctx0, h); + h = res2net(h, layer, dilation, scale); + h = conv1d_same(h, layer.conv_pw2_w, layer.conv_pw2_b, 1); // tdnn2 + h = ggml_relu(ctx0, h); + h = se_block(h, layer); + return ggml_add(ctx0, h, residual); +} + +// Attentive statistics pooling. x: [C, T] -> [2*C, 1] +ggml_tensor * clip_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x) const { + const int64_t T = x->ne[1]; + + // mean over T: [C, 1] + ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x)); + ggml_tensor * mean = ggml_mean(ctx0, x_t); + mean = ggml_cont(ctx0, ggml_transpose(ctx0, mean)); + + // std over T: sqrt(clamp(mean((x - mean)^2), eps)) + ggml_tensor * mean_rep = ggml_repeat(ctx0, mean, x); + ggml_tensor * centered = ggml_sub(ctx0, x, mean_rep); + ggml_tensor * var_t = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_sqr(ctx0, centered))); + ggml_tensor * var = ggml_mean(ctx0, var_t); + var = ggml_cont(ctx0, ggml_transpose(ctx0, var)); + var = ggml_scale_bias(ctx0, var, 1.0f, 1e-12f); + ggml_tensor * std = ggml_sqrt(ctx0, var); + + // attention input: cat([x, mean, std]) along channel axis -> [3C, T] + ggml_tensor * std_rep = ggml_repeat(ctx0, std, x); + ggml_tensor * cat = ggml_concat(ctx0, x, mean_rep, 0); + cat = ggml_concat(ctx0, cat, std_rep, 0); + + // attention TDNN (3C -> attn_c) + ReLU, tanh, then 1x1 conv (attn_c -> C) + ggml_tensor * a = conv1d_same(cat, model.spk_asp_tdnn_w, model.spk_asp_tdnn_b, 1); + a = ggml_relu(ctx0, a); + a = ggml_tanh(ctx0, a); + a = conv1d_same(a, model.spk_asp_attn_w, model.spk_asp_attn_b, 1); + + // softmax over T + ggml_tensor * a_t = ggml_cont(ctx0, ggml_transpose(ctx0, a)); // [T, C] + ggml_tensor * w_t = ggml_soft_max(ctx0, a_t); + ggml_tensor * w = ggml_cont(ctx0, ggml_transpose(ctx0, w_t)); // [C, T] + + // weighted mean: sum(w * x) over T (w already sums to 1 over T, + // ggml_mean gives 1/T scaling so multiply back by T to undo it) + ggml_tensor * wx = ggml_mul(ctx0, w, x); + ggml_tensor * wx_t = ggml_cont(ctx0, ggml_transpose(ctx0, wx)); + ggml_tensor * w_mean = ggml_mean(ctx0, wx_t); + w_mean = ggml_scale(ctx0, w_mean, (float) T); + w_mean = ggml_cont(ctx0, ggml_transpose(ctx0, w_mean)); // [C, 1] + + // weighted std: sum(w * (x - w_mean)^2) over T + ggml_tensor * w_mean_rep = ggml_repeat(ctx0, w_mean, x); + ggml_tensor * dev = ggml_sub(ctx0, x, w_mean_rep); + ggml_tensor * w_var_in = ggml_mul(ctx0, w, ggml_sqr(ctx0, dev)); + ggml_tensor * w_var_t = ggml_cont(ctx0, ggml_transpose(ctx0, w_var_in)); + ggml_tensor * w_var = ggml_mean(ctx0, w_var_t); + w_var = ggml_scale(ctx0, w_var, (float) T); + w_var = ggml_cont(ctx0, ggml_transpose(ctx0, w_var)); + w_var = ggml_scale_bias(ctx0, w_var, 1.0f, 1e-12f); + ggml_tensor * w_std = ggml_sqrt(ctx0, w_var); + + return ggml_concat(ctx0, w_mean, w_std, 0); // [2C, 1] +} + +ggml_cgraph * clip_graph_qwen3tts_spkenc::build() { + // inp_raw: [T, n_mel, 1, 1] (nx=T frames, ny=n_mel bins), from the + // preprocessor's mel_spectrogram() output (mtmd_audio_preprocessor_qwen3tts_spk) + ggml_tensor * inp = build_inp_raw(1); + inp = ggml_reshape_2d(ctx0, inp, inp->ne[0], inp->ne[1]); + + // this file's convention is [C, T]; the preprocessor delivers [T, C] + ggml_tensor * mel = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [n_mel, T] + cb(mel, "mel", -1); + + // frontend conv0 TDNN k=5, dilation=1: 128 -> 512 + ggml_tensor * cur = conv1d_same(mel, model.conv1d_1_w, model.conv1d_1_b, 1); + cur = ggml_relu(ctx0, cur); + cb(cur, "frontend", -1); + + // 3 SE-Res2Net blocks at dilations 2, 3, 4 + GGML_ASSERT((int) model.layers.size() == 3); + std::vector blk_out(3); + for (int il = 0; il < 3; il++) { + cur = se_res2net_block(cur, model.layers[il], SPK_DILATIONS[il], SPK_RES2NET_SCALE); + blk_out[il] = cur; + cb(cur, "block_out", il); + } + + // multi-layer feature aggregation: cat blk[0..2] then TDNN k=1 + ReLU + ggml_tensor * cat = ggml_concat(ctx0, blk_out[0], blk_out[1], 0); + cat = ggml_concat(ctx0, cat, blk_out[2], 0); // [1536, T] + ggml_tensor * mfa = conv1d_same(cat, model.conv_out_w, model.conv_out_b, 1); + mfa = ggml_relu(ctx0, mfa); + cb(mfa, "mfa", -1); + + // attentive statistics pooling: [1536, T] -> [3072, 1] + ggml_tensor * stats = attentive_stats_pool(mfa); + cb(stats, "asp", -1); + + // final FC k=1: [3072, 1] -> [enc_dim, 1] + ggml_tensor * emb = conv1d_same(stats, model.mm_fc_w, model.mm_fc_b, 1); + + emb = ggml_reshape_1d(ctx0, emb, emb->ne[0]); + emb = ggml_cont(ctx0, emb); + cb(emb, "spk_embedding", -1); + + ggml_build_forward_expand(gf, emb); + return gf; +}