From efed93383d249209adcd29250815c5b70ba4a412 Mon Sep 17 00:00:00 2001
From: Pedro Cuenca <pedro@huggingface.co>
Date: Tue, 4 Aug 2026 16:32:54 +0200
Subject: [PATCH] "clip" header declarations

---
 tools/mtmd/clip-impl.h  | 11 +++++++++++
 tools/mtmd/clip-model.h | 10 ++++++++++
 2 files changed, 21 insertions(+)

diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index d42b38222c..0a2326ff6d 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -91,6 +91,12 @@
 #define KEY_A_LOCAL_GROUP_SIZE     "clip.audio.local_group_size"     // mimo-v2.5: input_local_transformer grouping size
 #define KEY_AUDIO_SUBSAMPLING_FACTOR "clip.audio.subsampling_factor"
 
+#define KEY_ONYX_PATCH_TEMPORAL "clip.vision.onyx.patch_temporal"
+#define KEY_ONYX_DOWNSAMPLE     "clip.vision.onyx.downsample_factor"
+#define KEY_ONYX_SPARSE_FACTOR  "clip.vision.onyx.sparse_attention_factor"
+#define KEY_ONYX_POS_GRID       "clip.vision.onyx.pos_emb_grid"
+#define KEY_ONYX_ROPE_THETA     "clip.vision.onyx.rope_theta"
+
 //
 // tensor name constants
 //
@@ -128,6 +134,9 @@
 #define TN_MM_GATE         "mm.gate.%s"
 #define TN_MM_DOWN         "mm.down.%s"
 #define TN_MM_POST_NORM    "mm.post_norm.%s"
+#define TN_MM_ADAPTER_FC   "mm.adapter_fc.%s"   // onyx
+#define TN_MM_ADAPTER_PROJ "mm.adapter_proj.%s" // onyx
+#define TN_MM_VISION_PROJ  "mm.vision_proj.%s"  // onyx
 #define TN_MVLM_PROJ_MLP   "mm.model.mlp.%d.%s"
 #define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s"
 #define TN_MVLM_PROJ_PEG   "mm.model.peg.%d.%s"
@@ -408,6 +417,7 @@ enum projector_type {
     PROJECTOR_TYPE_MINIMAX_M3,
     PROJECTOR_TYPE_GRANITE4_VISION,
     PROJECTOR_TYPE_MIMO_AUDIO,
+    PROJECTOR_TYPE_ONYX,
     PROJECTOR_TYPE_UNKNOWN,
 };
 
@@ -465,6 +475,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
     { PROJECTOR_TYPE_GRANITE4_VISION,   "granite4_vision"},
     { PROJECTOR_TYPE_MIMO_AUDIO,        "mimo_audio"},
     { PROJECTOR_TYPE_PARAKEET,          "parakeet"},
+    { PROJECTOR_TYPE_ONYX,              "onyx"},
 };
 
 static projector_type clip_projector_type_from_string(const std::string & str) {
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h
index 8b9db5101d..8214d3271b 100644
--- a/tools/mtmd/clip-model.h
+++ b/tools/mtmd/clip-model.h
@@ -109,6 +109,11 @@ struct clip_hparams {
     int32_t downsample_query_side;
     int32_t downsample_window_side;
 
+    // Onyx vision (per-block sparse-window pattern, learned pos-emb, patch-temporal)
+    int32_t onyx_patch_temporal = 0;
+    int32_t onyx_sparse_factor  = 0;
+    int32_t onyx_pos_grid       = 0;
+
     // audio
     int32_t n_mel_bins = 0; // whisper preprocessor
     int32_t proj_stack_factor = 0; // ultravox
@@ -408,6 +413,11 @@ struct clip_model {
     ggml_tensor * mm_post_norm_w = nullptr;
     ggml_tensor * mm_post_norm_b = nullptr;
 
+    // Onyx adapter + final vision projection (3-linear MLP with erf-GELU)
+    ggml_tensor * mm_adapter_fc   = nullptr;
+    ggml_tensor * mm_adapter_proj = nullptr;
+    ggml_tensor * mm_vision_proj  = nullptr;
+
     // LLaVA projection
     ggml_tensor * mm_input_norm_w = nullptr;
     ggml_tensor * mm_input_norm_b = nullptr;