llama.cpp/tools/mtmd/clip-model.h

#pragma once

#include "ggml.h"
#include "clip.h"
#include "clip-impl.h"

#include <algorithm>
#include <array>
#include <vector>
#include <unordered_set>
#include <cstdint>
#include <cmath>

enum ffn_op_type {
    FFN_GELU,
    FFN_GELU_ERF,
    FFN_SILU,
    FFN_GELU_QUICK,
    FFN_RELU_SQR,
};

enum norm_type {
    NORM_TYPE_NORMAL,
    NORM_TYPE_RMS,
};

enum patch_merge_type {
    PATCH_MERGE_FLAT,
    PATCH_MERGE_SPATIAL_UNPAD,
};

enum resize_algo {
    RESIZE_ALGO_BILINEAR, // stretch to target resolution
    RESIZE_ALGO_BICUBIC, // center-crop when aspect ratio doesn't match
    RESIZE_ALGO_BICUBIC_PILLOW,
    // RESIZE_ALGO_LANCZOS, // TODO
};

// Padding style for img_tool::resize
//   PAD_NONE    - no padding; direct resize to target dimensions
//   PAD_CEIL    - aspect-preserving pad (default)
//   PAD_NEAREST - aspect-preserving pad with nearest-integer rounding (Pillow byte-parity)
enum pad_style {
    PAD_NONE,
    PAD_CEIL,
    PAD_NEAREST,
};

struct clip_hparams {
    int32_t image_size = 0;
    int32_t patch_size = 0;
    int32_t n_embd = 0;
    int32_t n_ff = 0;
    int32_t projection_dim = 0;
    int32_t n_head = 0;
    int32_t n_head_kv = 0;
    int32_t n_layer = 0;
    int32_t n_merge = 1; // number of patch merges **per-side**

    // for preprocessor
    int32_t image_longest_edge = 0;
    int32_t image_min_pixels = -1;
    int32_t image_max_pixels = -1;
    resize_algo image_resize_algo = RESIZE_ALGO_BICUBIC;
    pad_style image_resize_pad = PAD_CEIL; // padding style when resizing
    std::array<uint8_t, 3> image_pad_color = {0, 0, 0};

    // (preprocessor) for llava-uhd style models
    std::vector<clip_image_size> image_res_candidates;
    int32_t preproc_min_tiles = 0;
    int32_t preproc_max_tiles = 0;
    int32_t preproc_tile_size = 0; // local tile size (deepseek-ocr)
    resize_algo image_resize_algo_rf = RESIZE_ALGO_BICUBIC;
    resize_algo image_resize_algo_ov = RESIZE_ALGO_BILINEAR;
    pad_style image_pad_rf = PAD_CEIL;  // padding style for the refined image (e.g. llava-1.6)
    pad_style image_pad_ov = PAD_NONE;  // padding style for the overview image (e.g. llava-1.6)
    std::array<uint8_t, 3> image_pad_color_rf = {0, 0, 0}; // padding color for refined image
    std::array<uint8_t, 3> image_pad_color_ov = {0, 0, 0}; // padding color for overview image

    float image_mean[3];
    float image_std[3];

    // for models using dynamic image size, we need to have a smaller image size to warmup
    // otherwise, user will get OOM every time they load the model
    int32_t warmup_image_size = 0;
    int32_t warmup_audio_size = 3000;

    ffn_op_type ffn_op = FFN_GELU;

    patch_merge_type mm_patch_merge_type = PATCH_MERGE_FLAT;

    float eps = 1e-6;
    float rope_theta = 0.0;
    std::vector<int32_t> feature_layers;
    int32_t attn_window_size = 0;
    int32_t n_wa_pattern = 0;
    std::unordered_set<int32_t> wa_layer_indexes; // explicit layer indexes that use full attention (for irregular patterns like YoutuVL)
    std::vector<int32_t> wa_pattern_mode; // mimovl: per-layer window-attention mode

    // deepseek-ocr (sam)
    int32_t sam_n_layer = 0;
    int32_t sam_n_head  = 0;
    int32_t sam_n_embd  = 0;

    // Granite4 Vision
    std::vector<int32_t> proj_spatial_offsets;
    int32_t downsample_query_side;
    int32_t downsample_window_side;

    // audio
    int32_t n_mel_bins = 0; // whisper preprocessor
    int32_t proj_stack_factor = 0; // ultravox
    int32_t subsampling_factor = 0; // parakeet

    int32_t audio_chunk_size           = 0;
    int32_t audio_conv_kernel_size     = 0;
    int32_t audio_max_pos_emb          = 0;
    int32_t audio_proj_window_size     = 0;
    int32_t audio_proj_downsample_rate = 0;
    int32_t audio_proj_head_count      = 0;

    // audio-to-mel preprocessor params
    int32_t audio_chunk_len   = -1; // in seconds
    int32_t audio_sample_rate = -1;
    int32_t audio_n_fft       = -1;
    int32_t audio_window_len  = -1;
    int32_t audio_hop_len     = -1;

    // parakeet
    std::vector<float> mel_filters;
    std::vector<float> window;

    // mimo-audio-tokenizer: residual vector quantizer
    int32_t rvq_num_quantizers = 0;
    std::vector<int32_t> rvq_codebook_size; // per-quantizer bin count (ragged, e.g. 1024/1024/256/128x17)

    // mimo-v2.5: LLM-side connector (input_local_transformer)
    int32_t audio_local_n_layer = 0;
    int32_t audio_local_group_size = 0;

    // legacy
    bool has_llava_projector = false;
    int minicpmv_version = 0;
    int32_t minicpmv_query_num = 0;         // MiniCPM-V query number
    int32_t insert_layer_id   = 0;          // MiniCPM-V 4.6 ViT merger insertion layer

    // custom value provided by user, can be undefined if not set
    int32_t custom_image_min_tokens = -1;
    int32_t custom_image_max_tokens = -1;

    void set_limit_image_tokens(int n_tokens_min, int n_tokens_max) {
        const int patch_area = patch_size * patch_size * n_merge * n_merge;
        image_min_pixels = (custom_image_min_tokens > 0 ? custom_image_min_tokens : n_tokens_min) * patch_area;
        image_max_pixels = (custom_image_max_tokens > 0 ? custom_image_max_tokens : n_tokens_max) * patch_area;
        warmup_image_size = static_cast<int>(std::sqrt(image_max_pixels));
    }

    void set_warmup_n_tokens(int n_tokens) {
        int n_tok_per_side = static_cast<int>(std::sqrt(n_tokens));
        GGML_ASSERT(n_tok_per_side * n_tok_per_side == n_tokens && "n_tokens must be n*n");
        warmup_image_size = n_tok_per_side * patch_size * n_merge;
        // TODO: support warmup size for custom token numbers
    }
    // sam vit deepseek-ocr
    std::vector<int32_t> global_attn_indices() const {
        return {  2,  5,  8, 11 };
    }
    bool is_global_attn(int32_t layer) const {
        const auto indices = global_attn_indices();

        for (const auto & idx : indices) {
            if (layer == idx) {
                return true;
            }
        }

        return false;
    }

    bool is_feature_layer(int32_t layer) const {
        return std::find(feature_layers.begin(), feature_layers.end(), layer) != feature_layers.end();
    }
};

struct clip_layer {
    // layernorm 1 (or layer input norm, or pre-attention norm)
    ggml_tensor * ln_1_w = nullptr;
    ggml_tensor * ln_1_b = nullptr;

    // attention
    ggml_tensor * k_w = nullptr;
    ggml_tensor * k_b = nullptr;
    ggml_tensor * q_w = nullptr;
    ggml_tensor * q_b = nullptr;
    ggml_tensor * v_w = nullptr;
    ggml_tensor * v_b = nullptr;
    ggml_tensor * qkv_w = nullptr;
    ggml_tensor * qkv_b = nullptr;

    ggml_tensor * o_w = nullptr;
    ggml_tensor * o_b = nullptr;

    ggml_tensor * attn_sinks = nullptr;

    ggml_tensor * k_norm = nullptr;
    ggml_tensor * q_norm = nullptr;

    ggml_tensor * attn_post_norm_w = nullptr;

    ggml_tensor * ff_up_w = nullptr;
    ggml_tensor * ff_up_b = nullptr;
    ggml_tensor * ff_gate_w = nullptr;
    ggml_tensor * ff_gate_b = nullptr;
    ggml_tensor * ff_down_w = nullptr;
    ggml_tensor * ff_down_b = nullptr;

    // layernorm 2 (or pre-FFN norm)
    ggml_tensor * ln_2_w = nullptr;
    ggml_tensor * ln_2_b = nullptr;

    ggml_tensor * ff_post_norm_w = nullptr;

    // layer scale (no bias)
    ggml_tensor * ls_1_w   = nullptr;
    ggml_tensor * ls_2_w   = nullptr;
    ggml_tensor * ls_out_w = nullptr; // gemma4

    // qwen3vl deepstack merger
    ggml_tensor * deepstack_norm_w = nullptr;
    ggml_tensor * deepstack_norm_b = nullptr;
    ggml_tensor * deepstack_fc1_w = nullptr;
    ggml_tensor * deepstack_fc1_b = nullptr;
    ggml_tensor * deepstack_fc2_w = nullptr;
    ggml_tensor * deepstack_fc2_b = nullptr;

    // sam rel_pos
    ggml_tensor * rel_pos_w = nullptr;
    ggml_tensor * rel_pos_h = nullptr;
    // lfm2
    ggml_tensor * ff_norm_w     = nullptr;
    ggml_tensor * ff_norm_b     = nullptr;
    ggml_tensor * ff_norm_1_w   = nullptr;
    ggml_tensor * ff_norm_1_b   = nullptr;
    ggml_tensor * ff_up_1_w     = nullptr;
    ggml_tensor * ff_up_1_b     = nullptr;
    ggml_tensor * ff_down_1_w   = nullptr;
    ggml_tensor * ff_down_1_b   = nullptr;
    ggml_tensor * pos_bias_u    = nullptr;
    ggml_tensor * pos_bias_v    = nullptr;
    ggml_tensor * norm_conv_w   = nullptr;
    ggml_tensor * norm_conv_b   = nullptr;
    ggml_tensor * linear_pos_w  = nullptr;

    ggml_tensor * conv_norm_w    = nullptr;
    ggml_tensor * conv_norm_b    = nullptr;
    ggml_tensor * conv_norm_mean = nullptr;  // parakeet
    ggml_tensor * conv_norm_var  = nullptr;  // parakeet
    ggml_tensor * conv_dw_w      = nullptr;
    ggml_tensor * conv_dw_b      = nullptr;
    ggml_tensor * conv_pw1_w     = nullptr;
    ggml_tensor * conv_pw1_b     = nullptr;
    ggml_tensor * conv_pw2_w     = nullptr;
    ggml_tensor * conv_pw2_b     = nullptr;

    // gemma4 audio conformer per-layer
    ggml_tensor * attn_pre_norm_w   = nullptr;
    ggml_tensor * attn_k_rel_w      = nullptr;
    ggml_tensor * per_dim_scale_w   = nullptr;
    ggml_tensor * per_dim_k_scale_w = nullptr;
    ggml_tensor * ff_post_norm_1_w  = nullptr;

    // granite_speech conformer per-layer
    ggml_tensor * attn_rel_pos_emb = nullptr;

    // granite_speech qformer cross-attention
    ggml_tensor * cross_attn_q_w    = nullptr;
    ggml_tensor * cross_attn_q_b    = nullptr;
    ggml_tensor * cross_attn_k_w    = nullptr;
    ggml_tensor * cross_attn_k_b    = nullptr;
    ggml_tensor * cross_attn_v_w    = nullptr;
    ggml_tensor * cross_attn_v_b    = nullptr;
    ggml_tensor * cross_attn_o_w    = nullptr;
    ggml_tensor * cross_attn_o_b    = nullptr;
    ggml_tensor * cross_attn_norm_w = nullptr;
    ggml_tensor * cross_attn_norm_b = nullptr;

    bool has_deepstack() const {
        return deepstack_fc1_w != nullptr;
    }
};

// Expanded MobileNetV5 block structure for Gemma3n vision encoder
struct mobilenetv5_block {
    // Stage 0 (Edge Residual)
    ggml_tensor * s0_conv_exp_w = nullptr;
    ggml_tensor * s0_bn1_w      = nullptr;
    ggml_tensor * s0_conv_pwl_w = nullptr;
    ggml_tensor * s0_bn2_w      = nullptr;

    // Stage 1+ (Universal Inverted Residual)
    ggml_tensor * dw_start_w    = nullptr;
    ggml_tensor * dw_start_bn_w = nullptr;

    ggml_tensor * pw_exp_w      = nullptr;
    ggml_tensor * pw_exp_bn_w   = nullptr;

    ggml_tensor * dw_mid_w      = nullptr;
    ggml_tensor * dw_mid_bn_w   = nullptr;

    ggml_tensor * pw_proj_w     = nullptr;
    ggml_tensor * pw_proj_bn_w  = nullptr;

    ggml_tensor * layer_scale_w = nullptr;

    // Attention (MQA) components
    ggml_tensor * attn_q_w = nullptr;
    ggml_tensor * attn_k_w = nullptr;
    ggml_tensor * attn_v_w = nullptr;
    ggml_tensor * attn_o_w = nullptr;

    // Optional downsampling/norm in attention
    ggml_tensor * attn_k_dw_w   = nullptr;
    ggml_tensor * attn_k_norm_w = nullptr;
    ggml_tensor * attn_v_dw_w   = nullptr;
    ggml_tensor * attn_v_norm_w = nullptr;

    // Block norm (often present in attention blocks)
    ggml_tensor * attn_norm_w   = nullptr;
};

struct yasa2_block {
    ggml_tensor * dw_w  = nullptr;
    ggml_tensor * dw_b  = nullptr;
    ggml_tensor * ln_w  = nullptr;
    ggml_tensor * ln_b  = nullptr;
    ggml_tensor * pw1_w = nullptr;
    ggml_tensor * pw1_b = nullptr;
    ggml_tensor * grn_w = nullptr;
    ggml_tensor * grn_b = nullptr;
    ggml_tensor * pw2_w = nullptr;
    ggml_tensor * pw2_b = nullptr;
};

struct yasa2_stage {
    ggml_tensor * down_ln_w   = nullptr;
    ggml_tensor * down_ln_b   = nullptr;
    ggml_tensor * down_conv_w = nullptr;
    ggml_tensor * down_conv_b = nullptr;
    std::vector<yasa2_block> blocks;
};

// QFormer projector block for models with 1 (or more) QFormer projectors
// Granite Speech, Granite4 Vision
struct qf_block {
    ggml_tensor * qf_proj_query       = nullptr;
    ggml_tensor * qf_proj_norm_w      = nullptr;
    ggml_tensor * qf_proj_norm_b      = nullptr;
    ggml_tensor * qf_proj_linear_w    = nullptr;
    ggml_tensor * qf_proj_linear_b    = nullptr;
    ggml_tensor * qf_proj_post_norm_w = nullptr;
    ggml_tensor * qf_proj_post_norm_b = nullptr;
    ggml_tensor * qf_proj_img_pos     = nullptr; // Vision only
    std::vector<clip_layer> qf_proj_layers;
};

struct clip_model {
    clip_modality modality = CLIP_MODALITY_VISION;
    projector_type proj_type = PROJECTOR_TYPE_MLP;
    clip_hparams hparams;

    // embeddings
    ggml_tensor * class_embedding = nullptr;
    ggml_tensor * patch_embeddings_0 = nullptr;
    ggml_tensor * patch_embeddings_1 = nullptr;  // second Conv2D kernel when we decouple Conv3D along temporal dimension (Qwen2VL)
    ggml_tensor * patch_bias = nullptr;
    ggml_tensor * position_embeddings = nullptr;
    ggml_tensor * norm_embd_w = nullptr;
    ggml_tensor * norm_embd_b = nullptr;

    // "indexed" patch embedding norms
    ggml_tensor * patch_norm_1_w = nullptr;
    ggml_tensor * patch_norm_1_b = nullptr;
    ggml_tensor * patch_norm_2_w = nullptr;
    ggml_tensor * patch_norm_2_b = nullptr;
    ggml_tensor * patch_norm_3_w = nullptr;
    ggml_tensor * patch_norm_3_b = nullptr;

    ggml_tensor * pre_ln_w = nullptr;
    ggml_tensor * pre_ln_b = nullptr;

    std::vector<clip_layer> layers;

    int32_t n_deepstack_layers = 0; // used by Qwen3-VL, calculated from clip_layer

    ggml_tensor * post_ln_w;
    ggml_tensor * post_ln_b;

    ggml_tensor * mm_fc_w;
    ggml_tensor * mm_fc_b;
    ggml_tensor * mm_ffn_up_w = nullptr;
    ggml_tensor * mm_ffn_up_b = nullptr;
    ggml_tensor * mm_ffn_gate_w = nullptr;
    ggml_tensor * mm_ffn_gate_b = nullptr;
    ggml_tensor * mm_ffn_down_w = nullptr;
    ggml_tensor * mm_ffn_down_b = nullptr;
    ggml_tensor * mm_post_norm_w = nullptr;
    ggml_tensor * mm_post_norm_b = nullptr;

    // LLaVA projection
    ggml_tensor * mm_input_norm_w = nullptr;
    ggml_tensor * mm_input_norm_b = nullptr;
    ggml_tensor * mm_0_w = nullptr;
    ggml_tensor * mm_0_b = nullptr;
    ggml_tensor * mm_2_w = nullptr;
    ggml_tensor * mm_2_b = nullptr;
    ggml_tensor * mm_merger_fc1_w = nullptr;   // minimax-m3
    ggml_tensor * mm_merger_fc1_b = nullptr;
    ggml_tensor * mm_merger_fc2_w = nullptr;
    ggml_tensor * mm_merger_fc2_b = nullptr;

    ggml_tensor * image_newline = nullptr;
    ggml_tensor * view_seperator = nullptr;


    // Yi type models with mlp+normalization projection
    ggml_tensor * mm_1_w = nullptr; // Yi type models have 0, 1, 3, 4
    ggml_tensor * mm_1_b = nullptr;
    ggml_tensor * mm_3_w = nullptr;
    ggml_tensor * mm_3_b = nullptr;
    ggml_tensor * mm_4_w = nullptr;
    ggml_tensor * mm_4_b = nullptr;

    // GLMV-Edge projection
    ggml_tensor * mm_model_adapter_conv_w = nullptr;
    ggml_tensor * mm_model_adapter_conv_b = nullptr;

    // MobileVLM projection
    ggml_tensor * mm_model_mlp_1_w = nullptr;
    ggml_tensor * mm_model_mlp_1_b = nullptr;
    ggml_tensor * mm_model_mlp_3_w = nullptr;
    ggml_tensor * mm_model_mlp_3_b = nullptr;
    ggml_tensor * mm_model_block_1_block_0_0_w = nullptr;
    ggml_tensor * mm_model_block_1_block_0_1_w = nullptr;
    ggml_tensor * mm_model_block_1_block_0_1_b = nullptr;
    ggml_tensor * mm_model_block_1_block_1_fc1_w = nullptr;
    ggml_tensor * mm_model_block_1_block_1_fc1_b = nullptr;
    ggml_tensor * mm_model_block_1_block_1_fc2_w = nullptr;
    ggml_tensor * mm_model_block_1_block_1_fc2_b = nullptr;
    ggml_tensor * mm_model_block_1_block_2_0_w = nullptr;
    ggml_tensor * mm_model_block_1_block_2_1_w = nullptr;
    ggml_tensor * mm_model_block_1_block_2_1_b = nullptr;
    ggml_tensor * mm_model_block_2_block_0_0_w = nullptr;
    ggml_tensor * mm_model_block_2_block_0_1_w = nullptr;
    ggml_tensor * mm_model_block_2_block_0_1_b = nullptr;
    ggml_tensor * mm_model_block_2_block_1_fc1_w = nullptr;
    ggml_tensor * mm_model_block_2_block_1_fc1_b = nullptr;
    ggml_tensor * mm_model_block_2_block_1_fc2_w = nullptr;
    ggml_tensor * mm_model_block_2_block_1_fc2_b = nullptr;
    ggml_tensor * mm_model_block_2_block_2_0_w = nullptr;
    ggml_tensor * mm_model_block_2_block_2_1_w = nullptr;
    ggml_tensor * mm_model_block_2_block_2_1_b = nullptr;

    // MobileVLM_V2 projection
    ggml_tensor * mm_model_mlp_0_w = nullptr;
    ggml_tensor * mm_model_mlp_0_b = nullptr;
    ggml_tensor * mm_model_mlp_2_w = nullptr;
    ggml_tensor * mm_model_mlp_2_b = nullptr;
    ggml_tensor * mm_model_peg_0_w = nullptr;
    ggml_tensor * mm_model_peg_0_b = nullptr;

    // MINICPMV projection
    ggml_tensor * mm_model_pos_embed_k = nullptr;
    ggml_tensor * mm_model_query = nullptr;
    ggml_tensor * mm_model_proj   = nullptr;
    ggml_tensor * mm_model_proj_b = nullptr;
    ggml_tensor * mm_model_kv_proj = nullptr;
    ggml_tensor * mm_model_attn_q_w = nullptr;
    ggml_tensor * mm_model_attn_q_b = nullptr;
    ggml_tensor * mm_model_attn_k_w = nullptr;
    ggml_tensor * mm_model_attn_k_b = nullptr;
    ggml_tensor * mm_model_attn_v_w = nullptr;
    ggml_tensor * mm_model_attn_v_b = nullptr;
    ggml_tensor * mm_model_attn_o_w = nullptr;
    ggml_tensor * mm_model_attn_o_b = nullptr;
    ggml_tensor * mm_model_ln_q_w = nullptr;
    ggml_tensor * mm_model_ln_q_b = nullptr;
    ggml_tensor * mm_model_ln_kv_w = nullptr;
    ggml_tensor * mm_model_ln_kv_b = nullptr;
    ggml_tensor * mm_model_ln_post_w = nullptr;
    ggml_tensor * mm_model_ln_post_b = nullptr;

    // MiniCPM-V 4.6 ViT merger (window self-attention + ViT MLP downsample)
    ggml_tensor * vit_merger_ln1_w     = nullptr;
    ggml_tensor * vit_merger_ln1_b     = nullptr;
    ggml_tensor * vit_merger_attn_q_w  = nullptr;
    ggml_tensor * vit_merger_attn_q_b  = nullptr;
    ggml_tensor * vit_merger_attn_k_w  = nullptr;
    ggml_tensor * vit_merger_attn_k_b  = nullptr;
    ggml_tensor * vit_merger_attn_v_w  = nullptr;
    ggml_tensor * vit_merger_attn_v_b  = nullptr;
    ggml_tensor * vit_merger_attn_o_w  = nullptr;
    ggml_tensor * vit_merger_attn_o_b  = nullptr;
    ggml_tensor * vit_merger_ds_ln_w   = nullptr;
    ggml_tensor * vit_merger_ds_ln_b   = nullptr;
    ggml_tensor * vit_merger_ds_up_w   = nullptr;
    ggml_tensor * vit_merger_ds_up_b   = nullptr;
    ggml_tensor * vit_merger_ds_down_w = nullptr;
    ggml_tensor * vit_merger_ds_down_b = nullptr;

    // gemma3
    ggml_tensor * mm_input_proj_w = nullptr;
    ggml_tensor * mm_soft_emb_norm_w = nullptr;

    // mobilenetv5 for gemma3n
    std::vector<mobilenetv5_block> mobilenet_blocks;
    std::vector<int> mobilenet_stage_ends;
    ggml_tensor * mobilenet_stem_conv_w = nullptr;
    ggml_tensor * mobilenet_stem_conv_b = nullptr;
    ggml_tensor * mobilenet_stem_norm_w = nullptr;
    ggml_tensor * mm_post_proj_norm_w = nullptr;

    // Multi-Scale Fusion Adapter (MSFA) components
    ggml_tensor * msfa_concat_conv_w = nullptr;
    ggml_tensor * msfa_concat_norm_w = nullptr;
    ggml_tensor * msfa_ffn_expand_w = nullptr;
    ggml_tensor * msfa_ffn_project_w = nullptr;
    ggml_tensor * msfa_ffn_expand_bn = nullptr;
    ggml_tensor * msfa_ffn_project_bn = nullptr;

    // yasa2
    ggml_tensor * yasa_patch_w = nullptr;
    ggml_tensor * yasa_patch_b = nullptr;
    ggml_tensor * yasa_patch_ln_w = nullptr;
    ggml_tensor * yasa_patch_ln_b = nullptr;
    ggml_tensor * yasa_backbone_ln_w = nullptr;
    ggml_tensor * yasa_backbone_ln_b = nullptr;
    ggml_tensor * yasa_vision_pos_embed = nullptr;
    std::vector<yasa2_stage> yasa_stages;

    // pixtral, glm4v
    ggml_tensor * token_embd_img_break = nullptr;
    ggml_tensor * mm_patch_merger_w = nullptr;
    ggml_tensor * mm_patch_merger_b = nullptr;

    // ultravox / whisper encoder
    ggml_tensor * conv1d_1_w = nullptr;
    ggml_tensor * conv1d_1_b = nullptr;
    ggml_tensor * conv1d_2_w = nullptr;
    ggml_tensor * conv1d_2_b = nullptr;
    ggml_tensor * conv_out_w = nullptr;
    ggml_tensor * conv_out_b = nullptr;
    ggml_tensor * mm_norm_pre_w = nullptr;
    ggml_tensor * mm_norm_pre_b = nullptr;
    ggml_tensor * mm_norm_mid_w = nullptr;

    // mimo-audio-tokenizer: post-transformer downsample + RVQ codebook
    ggml_tensor * downsample_conv_w = nullptr; // no bias
    ggml_tensor * downsample_norm_w = nullptr;
    ggml_tensor * downsample_norm_b = nullptr;
    ggml_tensor * rvq_codebook = nullptr; // merged 3D [n_q, max_bins, dim]

    // mimo-v2.5: text-side RVQ code embedding ("text codebook")
    ggml_tensor * mm_a_code_embd = nullptr; // merged 3D [n_channels, vocab, dim]

    // mimo-v2.5: LLM-side connector (input_local_transformer, separate from the
    // audio_tokenizer's own encoder `layers`)
    std::vector<clip_layer> mm_a_local_layers;
    ggml_tensor * mm_a_local_norm_w = nullptr;

    // qwen3a
    ggml_tensor * conv2d_1_w = nullptr;
    ggml_tensor * conv2d_1_b = nullptr;
    ggml_tensor * conv2d_2_w = nullptr;
    ggml_tensor * conv2d_2_b = nullptr;
    ggml_tensor * conv2d_3_w = nullptr;
    ggml_tensor * conv2d_3_b = nullptr;

    // cogvlm
    ggml_tensor * mm_post_fc_norm_w = nullptr;
    ggml_tensor * mm_post_fc_norm_b = nullptr;
    ggml_tensor * mm_h_to_4h_w = nullptr;
    ggml_tensor * mm_gate_w = nullptr;
    ggml_tensor * mm_4h_to_h_w = nullptr;
    ggml_tensor * mm_boi = nullptr;
    ggml_tensor * mm_eoi = nullptr;

    // hunyuanvl perceiver
    ggml_tensor * mm_pre_norm_w  = nullptr;
    ggml_tensor * mm_img_begin   = nullptr;
    ggml_tensor * mm_img_end     = nullptr;

    // deepseek ocr sam
    ggml_tensor * patch_embed_proj_w = nullptr;
    ggml_tensor * patch_embed_proj_b = nullptr;
    ggml_tensor * pos_embed          = nullptr;

    ggml_tensor * neck_0_w;
    ggml_tensor * neck_1_w;
    ggml_tensor * neck_1_b;
    ggml_tensor * neck_2_w;
    ggml_tensor * neck_3_w;
    ggml_tensor * neck_3_b;
    ggml_tensor * net_2;
    ggml_tensor * net_3;

    int32_t n_sam_layers = 12; // used by deepseek-ocr sam encoder

    std::vector<clip_layer> sam_layers;

    // deepseek-ocr-2
    ggml_tensor * resample_query_768 = nullptr;
    ggml_tensor * resample_query_1024 = nullptr;

    // lfm2 audio
    std::array<ggml_tensor *, 7> pre_encode_conv_X_w = {nullptr};
    std::array<ggml_tensor *, 7> pre_encode_conv_X_b = {nullptr};
    ggml_tensor * pre_encode_out_w = nullptr;
    ggml_tensor * pre_encode_out_b = nullptr;

    // gemma4
    ggml_tensor * std_bias = nullptr;
    ggml_tensor * std_scale = nullptr;
    // Gemma4ClippableLinear
    struct clamp_info {
        float inp_max;
        float inp_min;
        float out_max;
        float out_min;
    };
    std::map<std::string, clamp_info> clamp_info_map;

    // gemma4 audio conformer
    std::array<ggml_tensor *, 2> sscp_conv_w = {nullptr};
    std::array<ggml_tensor *, 2> sscp_conv_b = {nullptr};
    std::array<ggml_tensor *, 2> sscp_norm_w = {nullptr};
    ggml_tensor * sscp_inp_proj_w = nullptr;
    ggml_tensor * sscp_inp_proj_b = nullptr;
    ggml_tensor * audio_out_proj_w = nullptr;
    ggml_tensor * audio_out_proj_b = nullptr;

    // granite_speech encoder
    ggml_tensor * inp_proj_w    = nullptr;
    ggml_tensor * inp_proj_b    = nullptr;
    ggml_tensor * ctc_out_w     = nullptr;
    ggml_tensor * ctc_out_b     = nullptr;
    ggml_tensor * ctc_out_mid_w = nullptr;
    ggml_tensor * ctc_out_mid_b = nullptr;
    // qformer projector(s)
    std::vector<qf_block> qf_proj_blocks;

    bool audio_has_avgpool() const {
        return proj_type == PROJECTOR_TYPE_QWEN2A
            || proj_type == PROJECTOR_TYPE_VOXTRAL
            || proj_type == PROJECTOR_TYPE_MUSIC_FLAMINGO;
    }

    bool audio_has_stack_frames() const {
        return proj_type == PROJECTOR_TYPE_ULTRAVOX
            || proj_type == PROJECTOR_TYPE_VOXTRAL
            || proj_type == PROJECTOR_TYPE_MERALION;
    }
};

const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx);