upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-03 11:27:25 -05:00
Code Issues Packages Projects Releases Wiki Activity

"clip" header declarations

Browse Source
This commit is contained in:
Pedro Cuenca
2026-08-04 16:32:54 +02:00
parent 10bad7ad18
commit efed93383d
2 changed files with 21 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip-impl.h
+11
View File
@@ -91,6 +91,12 @@
#define KEY_A_LOCAL_GROUP_SIZE "clip.audio.local_group_size" // mimo-v2.5: input_local_transformer grouping size
#define KEY_AUDIO_SUBSAMPLING_FACTOR "clip.audio.subsampling_factor"
#define KEY_ONYX_PATCH_TEMPORAL "clip.vision.onyx.patch_temporal"
#define KEY_ONYX_DOWNSAMPLE "clip.vision.onyx.downsample_factor"
#define KEY_ONYX_SPARSE_FACTOR "clip.vision.onyx.sparse_attention_factor"
#define KEY_ONYX_POS_GRID "clip.vision.onyx.pos_emb_grid"
#define KEY_ONYX_ROPE_THETA "clip.vision.onyx.rope_theta"
//
// tensor name constants
//
@@ -128,6 +134,9 @@
#define TN_MM_GATE "mm.gate.%s"
#define TN_MM_DOWN "mm.down.%s"
#define TN_MM_POST_NORM "mm.post_norm.%s"
#define TN_MM_ADAPTER_FC "mm.adapter_fc.%s" // onyx
#define TN_MM_ADAPTER_PROJ "mm.adapter_proj.%s" // onyx
#define TN_MM_VISION_PROJ "mm.vision_proj.%s" // onyx
#define TN_MVLM_PROJ_MLP "mm.model.mlp.%d.%s"
#define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s"
#define TN_MVLM_PROJ_PEG "mm.model.peg.%d.%s"
@@ -408,6 +417,7 @@ enum projector_type {
PROJECTOR_TYPE_MINIMAX_M3,
PROJECTOR_TYPE_GRANITE4_VISION,
PROJECTOR_TYPE_MIMO_AUDIO,
PROJECTOR_TYPE_ONYX,
PROJECTOR_TYPE_UNKNOWN,
};
@@ -465,6 +475,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"},
{ PROJECTOR_TYPE_MIMO_AUDIO, "mimo_audio"},
{ PROJECTOR_TYPE_PARAKEET, "parakeet"},
{ PROJECTOR_TYPE_ONYX, "onyx"},
};
static projector_type clip_projector_type_from_string(const std::string & str) {
tools/mtmd/clip-model.h
+10
View File
@@ -109,6 +109,11 @@ struct clip_hparams {
int32_t downsample_query_side;
int32_t downsample_window_side;
// Onyx vision (per-block sparse-window pattern, learned pos-emb, patch-temporal)
int32_t onyx_patch_temporal = 0;
int32_t onyx_sparse_factor = 0;
int32_t onyx_pos_grid = 0;
// audio
int32_t n_mel_bins = 0; // whisper preprocessor
int32_t proj_stack_factor = 0; // ultravox
@@ -408,6 +413,11 @@ struct clip_model {
ggml_tensor * mm_post_norm_w = nullptr;
ggml_tensor * mm_post_norm_b = nullptr;
// Onyx adapter + final vision projection (3-linear MLP with erf-GELU)
ggml_tensor * mm_adapter_fc = nullptr;
ggml_tensor * mm_adapter_proj = nullptr;
ggml_tensor * mm_vision_proj = nullptr;
// LLaVA projection
ggml_tensor * mm_input_norm_w = nullptr;
ggml_tensor * mm_input_norm_b = nullptr;