From de699957b92f490efebad149665b0dccf127eaff Mon Sep 17 00:00:00 2001 From: tc-mb <157115220+tc-mb@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:38:36 +0800 Subject: [PATCH] mtmd: add minicpmv46 downsample (#25993) * add minicpmv46 downsample Signed-off-by: tc-mb * put downsample mode inside gguf. Signed-off-by: tc-mb * build mtmd_image_preprocessor_llava_uhd Signed-off-by: tc-mb * fix code Signed-off-by: tc-mb * add convert Signed-off-by: tc-mb * add 4x ignore vit merger Signed-off-by: tc-mb --------- Signed-off-by: tc-mb --- conversion/minicpm.py | 13 +- tools/mtmd/clip.cpp | 143 ++++++++------- tools/mtmd/models/minicpmv.cpp | 315 ++++++++++++++------------------- tools/mtmd/mtmd-image.cpp | 20 +++ tools/mtmd/mtmd-image.h | 10 +- tools/mtmd/mtmd.cpp | 2 +- 6 files changed, 251 insertions(+), 252 deletions(-) diff --git a/conversion/minicpm.py b/conversion/minicpm.py index e31b26a008..bf3fa81421 100644 --- a/conversion/minicpm.py +++ b/conversion/minicpm.py @@ -137,6 +137,15 @@ class MiniCPMV4_6TextModel(Qwen3_5TextModel): class MiniCPMV4_6VisionModel(MmprojModel): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.downsample_mode = self.preprocessor_config.get("downsample_mode", "16x") + if self.downsample_mode not in {"4x", "16x"}: + raise ValueError(f"Unsupported downsample mode: {self.downsample_mode}") + if self.downsample_mode == "4x": + self.model_tensors = { + name: tensor for name, tensor in self.model_tensors.items() + if ".vit_merger." not in name + } + if self.hparams_vision is not None: # In MiniCPM-V 4.6 `vision_config.image_size` (980) describes the SigLIP # positional embedding bucket grid (70 x 70), while the per-slice processing @@ -156,8 +165,8 @@ class MiniCPMV4_6VisionModel(MmprojModel): # (mapped to PROJECTOR_TYPE_MINICPMV4_6). self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MINICPMV4_6) - # ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension; used for slice alignment - self.gguf_writer.add_vision_projector_scale_factor(4) + self.gguf_writer.add_vision_projector_scale_factor( + 2 if self.downsample_mode == "4x" else 4) # borrow wa_layer_indexes for vit_merger insertion point insert_layer_id = int(self.global_config.get( diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 5f0d00b660..c1870813fb 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1337,6 +1337,7 @@ struct clip_model_loader { // ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension hparams.n_merge = 4; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + GGML_ASSERT(hparams.n_merge == 2 || hparams.n_merge == 4); // borrow wa_layer_indexes for vit_merger insertion point std::vector wa_layer_indexes_vec; @@ -2143,24 +2144,29 @@ struct clip_model_loader { } break; case PROJECTOR_TYPE_MINICPMV4_6: { + const bool merger_required = hparams.n_merge == 4; + auto get_merger_tensor = [&](const std::string & name, bool required = true) { + return get_tensor(name, merger_required && required); + }; + // ViT merger: window self-attention - model.vit_merger_ln1_w = get_tensor(string_format(TN_VIT_MERGER_LN1, "weight")); - model.vit_merger_ln1_b = get_tensor(string_format(TN_VIT_MERGER_LN1, "bias")); - model.vit_merger_attn_q_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "weight")); - model.vit_merger_attn_q_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "bias"), false); - model.vit_merger_attn_k_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_K, "weight")); - model.vit_merger_attn_k_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_K, "bias"), false); - model.vit_merger_attn_v_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_V, "weight")); - model.vit_merger_attn_v_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_V, "bias"), false); - model.vit_merger_attn_o_w = get_tensor(string_format(TN_VIT_MERGER_ATTN_O, "weight")); - model.vit_merger_attn_o_b = get_tensor(string_format(TN_VIT_MERGER_ATTN_O, "bias"), false); + model.vit_merger_ln1_w = get_merger_tensor(string_format(TN_VIT_MERGER_LN1, "weight")); + model.vit_merger_ln1_b = get_merger_tensor(string_format(TN_VIT_MERGER_LN1, "bias")); + model.vit_merger_attn_q_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "weight")); + model.vit_merger_attn_q_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_Q, "bias"), false); + model.vit_merger_attn_k_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_K, "weight")); + model.vit_merger_attn_k_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_K, "bias"), false); + model.vit_merger_attn_v_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_V, "weight")); + model.vit_merger_attn_v_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_V, "bias"), false); + model.vit_merger_attn_o_w = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_O, "weight")); + model.vit_merger_attn_o_b = get_merger_tensor(string_format(TN_VIT_MERGER_ATTN_O, "bias"), false); // ViT merger: MLP downsample - model.vit_merger_ds_ln_w = get_tensor(string_format(TN_VIT_MERGER_DS_LN, "weight")); - model.vit_merger_ds_ln_b = get_tensor(string_format(TN_VIT_MERGER_DS_LN, "bias")); - model.vit_merger_ds_up_w = get_tensor(string_format(TN_VIT_MERGER_DS_UP, "weight")); - model.vit_merger_ds_up_b = get_tensor(string_format(TN_VIT_MERGER_DS_UP, "bias"), false); - model.vit_merger_ds_down_w = get_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "weight")); - model.vit_merger_ds_down_b = get_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "bias"), false); + model.vit_merger_ds_ln_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_LN, "weight")); + model.vit_merger_ds_ln_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_LN, "bias")); + model.vit_merger_ds_up_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_UP, "weight")); + model.vit_merger_ds_up_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_UP, "bias"), false); + model.vit_merger_ds_down_w = get_merger_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "weight")); + model.vit_merger_ds_down_b = get_merger_tensor(string_format(TN_VIT_MERGER_DS_DOWN, "bias"), false); // Final Merger (DownsampleMLP) model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B, false); @@ -3591,8 +3597,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { } break; case PROJECTOR_TYPE_MINICPMV4_6: { - // ViT merger 4x + final merger 4x = 16x total spatial downsample - n_patches = n_patches / 16; + n_patches /= params.n_merge * params.n_merge; } break; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: @@ -3974,6 +3979,8 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 } break; case PROJECTOR_TYPE_MINICPMV4_6: { + const bool is_4x = hparams.n_merge == 2; + // SigLIP position buckets (same as resampler path) std::vector positions(pos_h * pos_w); int bucket_coords_h[1024]; @@ -3994,40 +4001,6 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 const int half_h = pos_h / 2; const int half_w = pos_w / 2; - // window reorder indices for 2x2 windows - std::vector window_idx(n_pos); - std::vector inv_window_idx(n_pos); - { - int k = 0; - for (int wi = 0; wi < half_h; wi++) { - for (int wj = 0; wj < half_w; wj++) { - window_idx[k++] = (2*wi ) * pos_w + (2*wj ); - window_idx[k++] = (2*wi ) * pos_w + (2*wj + 1); - window_idx[k++] = (2*wi + 1) * pos_w + (2*wj ); - window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1); - } - } - for (int i = 0; i < n_pos; i++) { - inv_window_idx[window_idx[i]] = i; - } - } - set_input_i32("vit_merger_window_idx", window_idx); - set_input_i32("vit_merger_inv_window_idx", inv_window_idx); - - // block-diagonal attention mask: tokens in the same 4-token - // window attend to each other (mask = 0), all other positions - // are masked out (-inf). matches the window-major reorder above. - std::vector window_mask_data(n_pos * n_pos, std::numeric_limits::lowest()); - for (int wi = 0; wi < n_pos / 4; wi++) { - for (int i = 0; i < 4; i++) { - for (int j = 0; j < 4; j++) { - window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f; - } - } - } - set_input_f32("vit_merger_window_mask", window_mask_data); - - // ViT merger 2x2 downsample indices auto make_ds_idx = [](int off_r, int off_c, int ds_h, int ds_w, int stride_w) { std::vector idx(ds_h * ds_w); for (int i = 0; i < ds_h; i++) { @@ -4037,22 +4010,58 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 } return idx; }; - auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w); - auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w); - auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w); - auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w); - set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0); - set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1); - set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2); - set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3); - // final merger 2x2 downsample indices (operates on half_h x half_w grid) - const int qh = half_h / 2; - const int qw = half_w / 2; - auto m_ds_0 = make_ds_idx(0, 0, qh, qw, half_w); - auto m_ds_1 = make_ds_idx(0, 1, qh, qw, half_w); - auto m_ds_2 = make_ds_idx(1, 0, qh, qw, half_w); - auto m_ds_3 = make_ds_idx(1, 1, qh, qw, half_w); + if (!is_4x) { + // window reorder indices for 2x2 windows + std::vector window_idx(n_pos); + std::vector inv_window_idx(n_pos); + { + int k = 0; + for (int wi = 0; wi < half_h; wi++) { + for (int wj = 0; wj < half_w; wj++) { + window_idx[k++] = (2*wi ) * pos_w + (2*wj ); + window_idx[k++] = (2*wi ) * pos_w + (2*wj + 1); + window_idx[k++] = (2*wi + 1) * pos_w + (2*wj ); + window_idx[k++] = (2*wi + 1) * pos_w + (2*wj + 1); + } + } + for (int i = 0; i < n_pos; i++) { + inv_window_idx[window_idx[i]] = i; + } + } + set_input_i32("vit_merger_window_idx", window_idx); + set_input_i32("vit_merger_inv_window_idx", inv_window_idx); + + // block-diagonal attention mask: tokens in the same 4-token + // window attend to each other (mask = 0), all other positions + // are masked out (-inf). matches the window-major reorder above. + std::vector window_mask_data(n_pos * n_pos, std::numeric_limits::lowest()); + for (int wi = 0; wi < n_pos / 4; wi++) { + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + window_mask_data[(wi*4 + i) * n_pos + (wi*4 + j)] = 0.0f; + } + } + } + set_input_f32("vit_merger_window_mask", window_mask_data); + + // ViT merger 2x2 downsample indices + auto vit_merger_ds_0 = make_ds_idx(0, 0, half_h, half_w, pos_w); + auto vit_merger_ds_1 = make_ds_idx(0, 1, half_h, half_w, pos_w); + auto vit_merger_ds_2 = make_ds_idx(1, 0, half_h, half_w, pos_w); + auto vit_merger_ds_3 = make_ds_idx(1, 1, half_h, half_w, pos_w); + set_input_i32("vit_merger_ds_idx_0", vit_merger_ds_0); + set_input_i32("vit_merger_ds_idx_1", vit_merger_ds_1); + set_input_i32("vit_merger_ds_idx_2", vit_merger_ds_2); + set_input_i32("vit_merger_ds_idx_3", vit_merger_ds_3); + } + + const int merger_h = is_4x ? pos_h : half_h; + const int merger_w = is_4x ? pos_w : half_w; + auto m_ds_0 = make_ds_idx(0, 0, merger_h / 2, merger_w / 2, merger_w); + auto m_ds_1 = make_ds_idx(0, 1, merger_h / 2, merger_w / 2, merger_w); + auto m_ds_2 = make_ds_idx(1, 0, merger_h / 2, merger_w / 2, merger_w); + auto m_ds_3 = make_ds_idx(1, 1, merger_h / 2, merger_w / 2, merger_w); set_input_i32("merger_ds_idx_0", m_ds_0); set_input_i32("merger_ds_idx_1", m_ds_1); set_input_i32("merger_ds_idx_2", m_ds_2); diff --git a/tools/mtmd/models/minicpmv.cpp b/tools/mtmd/models/minicpmv.cpp index bac087ffdf..3e9c4c2a11 100644 --- a/tools/mtmd/models/minicpmv.cpp +++ b/tools/mtmd/models/minicpmv.cpp @@ -114,14 +114,12 @@ ggml_cgraph * clip_graph_minicpmv::build() { } ggml_cgraph * clip_graph_minicpmv4_6::build() { - const int insert_lid = hparams.insert_layer_id; - const int n_pos = n_patches; - const int half_h = n_patches_y / 2; - const int half_w = n_patches_x / 2; - const int n_ds = half_h * half_w; // after ViT merger 2x2 downsample - const int qh = half_h / 2; - const int qw = half_w / 2; - const int n_ds2 = qh * qw; // after final merger 2x2 downsample + const bool is_4x = hparams.n_merge == 2; + const int n_pos = n_patches; + const int half_h = n_patches_y / 2; + const int half_w = n_patches_x / 2; + const int n_ds = half_h * half_w; + const int n_out = is_4x ? n_ds : (half_h / 2) * (half_w / 2); auto add_i32_input = [&](const char * name, int n) { ggml_tensor * t = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n); @@ -134,29 +132,39 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() { ggml_tensor * positions = add_i32_input("positions", n_pos); ggml_tensor * learned_pos_embd = ggml_get_rows(ctx0, model.position_embeddings, positions); - // ViT merger window reorder indices + block-diagonal mask - // (mask layout follows qwen2vl: -inf except for 4x4 blocks on the diagonal, - // so each window-major group of 4 tokens only attends to itself) - ggml_tensor * vit_merger_window_idx = add_i32_input("vit_merger_window_idx", n_pos); - ggml_tensor * vit_merger_inv_window_idx = add_i32_input("vit_merger_inv_window_idx", n_pos); - ggml_tensor * vit_merger_window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos); - ggml_set_name(vit_merger_window_mask, "vit_merger_window_mask"); - ggml_set_input(vit_merger_window_mask); - if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { - vit_merger_window_mask = ggml_cast(ctx0, vit_merger_window_mask, GGML_TYPE_F16); + ggml_tensor * vit_merger_window_idx = nullptr; + ggml_tensor * vit_merger_inv_window_idx = nullptr; + ggml_tensor * vit_merger_window_mask = nullptr; + ggml_tensor * vit_merger_ds_idx_0 = nullptr; + ggml_tensor * vit_merger_ds_idx_1 = nullptr; + ggml_tensor * vit_merger_ds_idx_2 = nullptr; + ggml_tensor * vit_merger_ds_idx_3 = nullptr; + + if (!is_4x) { + // ViT merger window reorder indices + block-diagonal mask + // (mask layout follows qwen2vl: -inf except for 4x4 blocks on the diagonal, + // so each window-major group of 4 tokens only attends to itself) + vit_merger_window_idx = add_i32_input("vit_merger_window_idx", n_pos); + vit_merger_inv_window_idx = add_i32_input("vit_merger_inv_window_idx", n_pos); + vit_merger_window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos); + ggml_set_name(vit_merger_window_mask, "vit_merger_window_mask"); + ggml_set_input(vit_merger_window_mask); + if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { + vit_merger_window_mask = ggml_cast(ctx0, vit_merger_window_mask, GGML_TYPE_F16); + } + + // ViT merger 2x2 downsample gather indices + vit_merger_ds_idx_0 = add_i32_input("vit_merger_ds_idx_0", n_ds); + vit_merger_ds_idx_1 = add_i32_input("vit_merger_ds_idx_1", n_ds); + vit_merger_ds_idx_2 = add_i32_input("vit_merger_ds_idx_2", n_ds); + vit_merger_ds_idx_3 = add_i32_input("vit_merger_ds_idx_3", n_ds); } - // ViT merger 2x2 downsample gather indices - ggml_tensor * vit_merger_ds_idx_0 = add_i32_input("vit_merger_ds_idx_0", n_ds); - ggml_tensor * vit_merger_ds_idx_1 = add_i32_input("vit_merger_ds_idx_1", n_ds); - ggml_tensor * vit_merger_ds_idx_2 = add_i32_input("vit_merger_ds_idx_2", n_ds); - ggml_tensor * vit_merger_ds_idx_3 = add_i32_input("vit_merger_ds_idx_3", n_ds); - // final merger 2x2 downsample gather indices - ggml_tensor * merger_ds_idx_0 = add_i32_input("merger_ds_idx_0", n_ds2); - ggml_tensor * merger_ds_idx_1 = add_i32_input("merger_ds_idx_1", n_ds2); - ggml_tensor * merger_ds_idx_2 = add_i32_input("merger_ds_idx_2", n_ds2); - ggml_tensor * merger_ds_idx_3 = add_i32_input("merger_ds_idx_3", n_ds2); + ggml_tensor * merger_ds_idx_0 = add_i32_input("merger_ds_idx_0", n_out); + ggml_tensor * merger_ds_idx_1 = add_i32_input("merger_ds_idx_1", n_out); + ggml_tensor * merger_ds_idx_2 = add_i32_input("merger_ds_idx_2", n_out); + ggml_tensor * merger_ds_idx_3 = add_i32_input("merger_ds_idx_3", n_out); // patch embedding + positional embedding ggml_tensor * inp = build_inp(); @@ -169,150 +177,10 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() { cb(inpL, "pre_ln", -1); } - // ViT layers 0..insert_layer_id (inclusive) - // Mirrors the separate-qkv path of clip_graph::build_vit so the two manually - // unrolled segments around the ViT merger read like build_vit() expansions. - for (int il = 0; il <= insert_lid; il++) { - auto & layer = model.layers[il]; - ggml_tensor * cur = inpL; - - cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il); - cb(cur, "layer_inp_normed", il); - - { - ggml_tensor * Qcur = build_mm(layer.q_w, cur); - if (layer.q_b) { - Qcur = ggml_add(ctx0, Qcur, layer.q_b); - } - ggml_tensor * Kcur = build_mm(layer.k_w, cur); - if (layer.k_b) { - Kcur = ggml_add(ctx0, Kcur, layer.k_b); - } - ggml_tensor * Vcur = build_mm(layer.v_w, cur); - if (layer.v_b) { - Vcur = ggml_add(ctx0, Vcur, layer.v_b); - } - - Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos); - Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos); - Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos); - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - cur = build_attn(layer.o_w, layer.o_b, Qcur, Kcur, Vcur, nullptr, kq_scale, il); - cb(cur, "attn_out", il); - } - - if (layer.ls_1_w) { - cur = ggml_mul(ctx0, cur, layer.ls_1_w); - cb(cur, "attn_out_scaled", il); - } - cur = ggml_add(ctx0, cur, inpL); - inpL = cur; - cb(cur, "ffn_inp", il); - - cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il); - cb(cur, "ffn_inp_normed", il); - - cur = build_ffn(cur, layer.ff_up_w, layer.ff_up_b, layer.ff_gate_w, layer.ff_gate_b, - layer.ff_down_w, layer.ff_down_b, hparams.ffn_op, il); - cb(cur, "ffn_out", il); - - if (layer.ls_2_w) { - cur = ggml_mul(ctx0, cur, layer.ls_2_w); - cb(cur, "ffn_out_scaled", il); - } - cur = ggml_add(ctx0, inpL, cur); - cb(cur, "layer_out", il); - - inpL = cur; - } - - // ViT merger: window self-attention - // Tokens are reordered to window-major (4 tokens per window are contiguous), - // and a block-diagonal mask restricts attention to within each window. This - // mirrors the qwen2vl windowed-attention pattern so build_attn() can pick the - // flash-attention path when available. - { - ggml_tensor * residual = inpL; - ggml_tensor * cur = build_norm(inpL, - model.vit_merger_ln1_w, model.vit_merger_ln1_b, - NORM_TYPE_NORMAL, eps, -1); - cb(cur, "vit_merger_attn_inp_normed", -1); - - cur = ggml_get_rows(ctx0, cur, vit_merger_window_idx); - cb(cur, "vit_merger_window_reorder", -1); - - ggml_tensor * Qcur = build_mm(model.vit_merger_attn_q_w, cur); - if (model.vit_merger_attn_q_b) { - Qcur = ggml_add(ctx0, Qcur, model.vit_merger_attn_q_b); - } - ggml_tensor * Kcur = build_mm(model.vit_merger_attn_k_w, cur); - if (model.vit_merger_attn_k_b) { - Kcur = ggml_add(ctx0, Kcur, model.vit_merger_attn_k_b); - } - ggml_tensor * Vcur = build_mm(model.vit_merger_attn_v_w, cur); - if (model.vit_merger_attn_v_b) { - Vcur = ggml_add(ctx0, Vcur, model.vit_merger_attn_v_b); - } - - Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos); - Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos); - Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos); - cb(Qcur, "vit_merger_Qcur", -1); - cb(Kcur, "vit_merger_Kcur", -1); - cb(Vcur, "vit_merger_Vcur", -1); - - cur = build_attn(model.vit_merger_attn_o_w, model.vit_merger_attn_o_b, - Qcur, Kcur, Vcur, vit_merger_window_mask, kq_scale, -1); - cb(cur, "vit_merger_attn_out", -1); - - cur = ggml_get_rows(ctx0, cur, vit_merger_inv_window_idx); - inpL = ggml_add(ctx0, cur, residual); - cb(inpL, "vit_merger_attn_residual", -1); - } - - // ViT merger: 2x2 spatial downsample + MLP (4 tokens -> 1) - { - ggml_tensor * p0 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_0); - ggml_tensor * p1 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_1); - ggml_tensor * p2 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_2); - ggml_tensor * p3 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_3); - - ggml_tensor * mean_res = ggml_add(ctx0, p0, p1); - mean_res = ggml_add(ctx0, mean_res, p2); - mean_res = ggml_add(ctx0, mean_res, p3); - mean_res = ggml_scale(ctx0, mean_res, 0.25f); - cb(mean_res, "vit_merger_ds_mean_res", -1); - - ggml_tensor * cat = ggml_concat(ctx0, p0, p1, 0); - cat = ggml_concat(ctx0, cat, p2, 0); - cat = ggml_concat(ctx0, cat, p3, 0); - - ggml_tensor * cur = build_norm(cat, - model.vit_merger_ds_ln_w, model.vit_merger_ds_ln_b, - NORM_TYPE_NORMAL, eps, -1); - cb(cur, "vit_merger_ds_normed", -1); - - // ViTWindowAttentionMerger downsample MLP uses gelu_pytorch_tanh (FFN_GELU) - cur = build_ffn(cur, - model.vit_merger_ds_up_w, model.vit_merger_ds_up_b, - nullptr, nullptr, - model.vit_merger_ds_down_w, model.vit_merger_ds_down_b, - FFN_GELU, -1); - cb(cur, "vit_merger_ds_mlp_out", -1); - - inpL = ggml_add(ctx0, cur, mean_res); - cb(inpL, "vit_merger_ds_out", -1); - } - - // ViT layers (insert_layer_id+1)..n_layer-1, operating on the downsampled tokens - { - const int64_t n_pos_ds = n_ds; - for (int il = insert_lid + 1; il < n_layer; il++) { + auto build_vit_layers = [&](ggml_tensor * input, int il_begin, int il_end, int64_t n_pos_layer) { + for (int il = il_begin; il < il_end; il++) { auto & layer = model.layers[il]; - ggml_tensor * cur = inpL; + ggml_tensor * cur = input; cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il); cb(cur, "layer_inp_normed", il); @@ -331,9 +199,9 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() { Vcur = ggml_add(ctx0, Vcur, layer.v_b); } - Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos_ds); - Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos_ds); - Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos_ds); + Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos_layer); + Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos_layer); + Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos_layer); cb(Qcur, "Qcur", il); cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); @@ -346,8 +214,8 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() { cur = ggml_mul(ctx0, cur, layer.ls_1_w); cb(cur, "attn_out_scaled", il); } - cur = ggml_add(ctx0, cur, inpL); - inpL = cur; + cur = ggml_add(ctx0, cur, input); + input = cur; cb(cur, "ffn_inp", il); cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il); @@ -361,11 +229,98 @@ ggml_cgraph * clip_graph_minicpmv4_6::build() { cur = ggml_mul(ctx0, cur, layer.ls_2_w); cb(cur, "ffn_out_scaled", il); } - cur = ggml_add(ctx0, inpL, cur); - cb(cur, "layer_out", il); - - inpL = cur; + input = ggml_add(ctx0, input, cur); + cb(input, "layer_out", il); } + return input; + }; + + if (!is_4x) { + const int insert_lid = hparams.insert_layer_id; + + inpL = build_vit_layers(inpL, 0, insert_lid + 1, n_pos); + + // ViT merger: window self-attention + // Tokens are reordered to window-major (4 tokens per window are contiguous), + // and a block-diagonal mask restricts attention to within each window. This + // mirrors the qwen2vl windowed-attention pattern so build_attn() can pick the + // flash-attention path when available. + { + ggml_tensor * residual = inpL; + ggml_tensor * cur = build_norm(inpL, + model.vit_merger_ln1_w, model.vit_merger_ln1_b, + NORM_TYPE_NORMAL, eps, -1); + cb(cur, "vit_merger_attn_inp_normed", -1); + + cur = ggml_get_rows(ctx0, cur, vit_merger_window_idx); + cb(cur, "vit_merger_window_reorder", -1); + + ggml_tensor * Qcur = build_mm(model.vit_merger_attn_q_w, cur); + if (model.vit_merger_attn_q_b) { + Qcur = ggml_add(ctx0, Qcur, model.vit_merger_attn_q_b); + } + ggml_tensor * Kcur = build_mm(model.vit_merger_attn_k_w, cur); + if (model.vit_merger_attn_k_b) { + Kcur = ggml_add(ctx0, Kcur, model.vit_merger_attn_k_b); + } + ggml_tensor * Vcur = build_mm(model.vit_merger_attn_v_w, cur); + if (model.vit_merger_attn_v_b) { + Vcur = ggml_add(ctx0, Vcur, model.vit_merger_attn_v_b); + } + + Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos); + Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos); + Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos); + cb(Qcur, "vit_merger_Qcur", -1); + cb(Kcur, "vit_merger_Kcur", -1); + cb(Vcur, "vit_merger_Vcur", -1); + + cur = build_attn(model.vit_merger_attn_o_w, model.vit_merger_attn_o_b, + Qcur, Kcur, Vcur, vit_merger_window_mask, kq_scale, -1); + cb(cur, "vit_merger_attn_out", -1); + + cur = ggml_get_rows(ctx0, cur, vit_merger_inv_window_idx); + inpL = ggml_add(ctx0, cur, residual); + cb(inpL, "vit_merger_attn_residual", -1); + } + + // ViT merger: 2x2 spatial downsample + MLP (4 tokens -> 1) + { + ggml_tensor * p0 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_0); + ggml_tensor * p1 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_1); + ggml_tensor * p2 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_2); + ggml_tensor * p3 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_3); + + ggml_tensor * mean_res = ggml_add(ctx0, p0, p1); + mean_res = ggml_add(ctx0, mean_res, p2); + mean_res = ggml_add(ctx0, mean_res, p3); + mean_res = ggml_scale(ctx0, mean_res, 0.25f); + cb(mean_res, "vit_merger_ds_mean_res", -1); + + ggml_tensor * cat = ggml_concat(ctx0, p0, p1, 0); + cat = ggml_concat(ctx0, cat, p2, 0); + cat = ggml_concat(ctx0, cat, p3, 0); + + ggml_tensor * cur = build_norm(cat, + model.vit_merger_ds_ln_w, model.vit_merger_ds_ln_b, + NORM_TYPE_NORMAL, eps, -1); + cb(cur, "vit_merger_ds_normed", -1); + + // ViTWindowAttentionMerger downsample MLP uses gelu_pytorch_tanh (FFN_GELU) + cur = build_ffn(cur, + model.vit_merger_ds_up_w, model.vit_merger_ds_up_b, + nullptr, nullptr, + model.vit_merger_ds_down_w, model.vit_merger_ds_down_b, + FFN_GELU, -1); + cb(cur, "vit_merger_ds_mlp_out", -1); + + inpL = ggml_add(ctx0, cur, mean_res); + cb(inpL, "vit_merger_ds_out", -1); + } + + inpL = build_vit_layers(inpL, insert_lid + 1, n_layer, n_ds); + } else { + inpL = build_vit_layers(inpL, 0, n_layer, n_pos); } if (model.post_ln_w) { diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 72d35fce69..10cfe52f56 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -972,6 +972,26 @@ mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const cl return output; } +// +// mtmd_image_preprocessor_minicpmv +// + +mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_minicpmv::get_slice_instructions(const clip_image_size & original_size) { + if (hparams.n_merge == 2) { + const int slice_size = hparams.image_size; + const float ratio = (float)original_size.width * original_size.height / (slice_size * slice_size); + if (ratio <= 1.0f) { + mtmd_image_preprocessor_llava_uhd::slice_instructions inst; + const int patch_size = hparams.patch_size * hparams.n_merge; + inst.overview_size = get_best_resize(original_size, slice_size, patch_size, true); + inst.refined_size = clip_image_size{0, 0}; + inst.grid_size = clip_image_size{0, 0}; + return inst; + } + } + return mtmd_image_preprocessor_llava_uhd::get_slice_instructions(original_size); +} + // // mtmd_image_preprocessor_lfm2 // diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index 115cba51e8..ecb203f767 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -74,7 +74,6 @@ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor { std::vector slices; }; - // LFM2 override this function to implement its custom slicing logic virtual slice_instructions get_slice_instructions(const clip_image_size & original_size); struct slice_output { @@ -83,9 +82,10 @@ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor { }; slice_output slice_image(const clip_image_u8 & img, const slice_instructions & inst); -private: +protected: clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false); +private: clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max); /** @@ -129,6 +129,12 @@ struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor { mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; }; +// custom llava-uhd slicing logic for MiniCPM-V +struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd { + using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd; + slice_instructions get_slice_instructions(const clip_image_size & original_size) override; +}; + // custom llava-uhd slicing logic for LFM2 // ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd { diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 93ca8cbcf8..d3899f5c85 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -451,7 +451,7 @@ struct mtmd_context { tok_row_end = {lookup_token("\n")}; tok_row_end_trail = false; // no trailing end-of-row token ov_img_first = true; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(ctx_v); } break; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: