diff --git a/conversion/__init__.py b/conversion/__init__.py index 11f24f4089..9fffe82c07 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -296,6 +296,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "MiniCPMV4_6ForConditionalGeneration": "minicpm", "Mistral3ForConditionalGeneration": "llava", "NemotronH_Nano_VL_V2": "nemotron", + "OnyxForConditionalGeneration": "onyx", "PaddleOCRVisionModel": "ernie", "Phi4ForCausalLMV": "phi", "Qwen2AudioForConditionalGeneration": "ultravox", diff --git a/conversion/onyx.py b/conversion/onyx.py index e5669cc01e..9a0f722ea2 100644 --- a/conversion/onyx.py +++ b/conversion/onyx.py @@ -1,14 +1,14 @@ from __future__ import annotations import json -from typing import Iterable, TYPE_CHECKING +from typing import Any, Iterable, TYPE_CHECKING import torch if TYPE_CHECKING: from torch import Tensor -from .base import ModelBase, TextModel, gguf, logger +from .base import MmprojModel, ModelBase, TextModel, gguf def _unpermute_for_rope(tensor: "Tensor", n_heads: int) -> "Tensor": @@ -46,13 +46,8 @@ class OnyxModel(TextModel): self.gguf_writer.add_final_logit_softcapping(hparams["final_logit_softcapping"]) self.gguf_writer.add_logit_scale(hparams["output_multiplier"]) - self.gguf_writer.add_post_norm_rms_eps(hparams["post_norm_eps"]) - - # SWA + NoPE: [SW, SW, SW, Full], NoPE used on Full layers. References: - # https://huggingface.co/someorgtoo-hf/onyx-hf-converted/blob/main/config.json#L19 - # https://huggingface.co/someorgtoo-hf/onyx-hf-converted/blob/main/config.json#L73 self.gguf_writer.add_sliding_window(hparams["sliding_window"]) - self.gguf_writer.add_sliding_window_pattern(4) + self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hparams["layer_types"]]) def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: shift = self.norm_shift(name) @@ -129,3 +124,54 @@ class OnyxAssistantModel(TextModel): # DFlash defaults to NEOX (rotate_half) rope, matching transformers HF layout for Q/K, QK-norms # no permutation needed. yield (self.map_tensor_name(name), data_torch) + + +@ModelBase.register("OnyxForConditionalGeneration") +class OnyxVisionModel(MmprojModel): + def get_vision_config(self) -> dict[str, Any] | None: + c = self.global_config.get("vision_config") + if not c: + return None + # Onyx actually uses dynamic size, initialize with nominal size + image_size = c["pos_emb_height"] * c["patch_size"] * c["merge_size"] + return {**c, "image_size": image_size} + + def set_gguf_parameters(self): + super().set_gguf_parameters() + c = self.hparams_vision # enriched vision_config from get_vision_config() + + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.ONYX) + self.gguf_writer.add_vision_attention_layernorm_eps(float(c["layer_norm_eps"])) + self.gguf_writer.add_vision_spatial_merge_size(int(c["merge_size"])) + + @classmethod + def filter_tensors(cls, item): + name, gen = item + keep = ("model.vision_tower.", "model.vision_adapter.", "model.vision_projection.") + if not any(name.startswith(k) for k in keep): + return None + return super().filter_tensors((name, gen)) + + # 3-layer projector MLP + _MM_MLP_MAP = { + "model.vision_adapter.fc1": (gguf.MODEL_TENSOR.V_MMPROJ, 0), + "model.vision_adapter.fc2": (gguf.MODEL_TENSOR.V_MMPROJ, 1), + "model.vision_projection": (gguf.MODEL_TENSOR.V_MMPROJ, 2), + } + + def modify_tensors(self, data_torch, name, bid): + if ".attn.q_proj." in name or ".attn.k_proj." in name: + n_heads = int(self.hparams_vision["num_attention_heads"]) + data_torch = _unpermute_for_rope(data_torch, n_heads) + # Lay out the pt=2 temporal slabs of the patch embedding as a conv2d for build_inp() + if name.endswith("patch_embedder.patch_embedding.weight"): + n_embd = data_torch.shape[0] + pt = int(self.hparams_vision["patch_temporal"]) + ps = int(self.hparams_vision["patch_size"]) + data_torch = data_torch.view(n_embd, pt, 3, ps, ps).sum(dim=1) # (n_embd, 3, ps, ps) + stem, _, suffix = name.rpartition(".") + if stem in self._MM_MLP_MAP: + tensor_key, idx = self._MM_MLP_MAP[stem] + yield (self.format_tensor_name(tensor_key, bid=idx, suffix="." + suffix), data_torch) + return + yield (self.map_tensor_name(name), data_torch) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 7a2ba560a8..1ae9deb3b9 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -172,7 +172,6 @@ class Keys: VALUE_LENGTH = "{arch}.attention.value_length" LAYERNORM_EPS = "{arch}.attention.layer_norm_epsilon" LAYERNORM_RMS_EPS = "{arch}.attention.layer_norm_rms_epsilon" - POST_NORM_RMS_EPS = "{arch}.attention.post_norm_rms_epsilon" GROUPNORM_EPS = "{arch}.attention.group_norm_epsilon" GROUPNORM_GROUPS = "{arch}.attention.group_norm_groups" CAUSAL = "{arch}.attention.causal" @@ -1481,8 +1480,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.V_MM_UP: "mm.up", MODEL_TENSOR.V_MM_DOWN: "mm.down", MODEL_TENSOR.V_MM_GATE: "mm.gate", - MODEL_TENSOR.V_MM_MERGER_FC1: "mm.merger.fc1", - MODEL_TENSOR.V_MM_MERGER_FC2: "mm.merger.fc2", + MODEL_TENSOR.V_MM_MERGER_FC1: "mm.merger.fc1", + MODEL_TENSOR.V_MM_MERGER_FC2: "mm.merger.fc2", MODEL_TENSOR.V_TOK_BOI: "v.boi", MODEL_TENSOR.V_TOK_EOI: "v.eoi", MODEL_TENSOR.V_MM_PRE_NORM: "mm.pre_norm", @@ -4907,6 +4906,7 @@ class VisionProjectorType: MIMOVL = "mimovl" MIMO_AUDIO = "mimo_audio" GRANITE4_VISION = "granite4_vision" + ONYX = "onyx" # Items here are (block size, type size) diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index af14e24fe4..c5905164c3 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -923,9 +923,6 @@ class GGUFWriter: def add_layer_norm_rms_eps(self, value: float) -> None: self.add_float32(Keys.Attention.LAYERNORM_RMS_EPS.format(arch=self.arch), value) - def add_post_norm_rms_eps(self, value: float) -> None: - self.add_float32(Keys.Attention.POST_NORM_RMS_EPS.format(arch=self.arch), value) - def add_group_norm_eps(self, value: float) -> None: self.add_float32(Keys.Attention.GROUPNORM_EPS.format(arch=self.arch), value) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index dead291994..28228d9f3b 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -1469,6 +1469,7 @@ class TensorNameMap: "vision_tower.patch_embed.patchifier.proj", # dots.ocr "vision_model.conv1", # Step3-VL "model.vision_embedder.patch_dense", # gemma4 unified + "model.vision_tower.patch_embedder.patch_embedding", # onyx ), MODEL_TENSOR.V_ENC_EMBD_NORM: ( @@ -1536,7 +1537,8 @@ class TensorNameMap: "siglip2.vision_model.encoder.layers.{bid}.self_attn.q_proj", # youtuvl "model.vision_model.transformer.layers.{bid}.self_attn.q_proj", # Deepseek-OCR CLIP, generated "vision_model.model.layers.{bid}.self_attn.q_proj.linear", # gemma4 - "model.qwen2_model.model.model.layers.{bid}.self_attn.q_proj" # Deepseek-OCR-2 qwen2 + "model.qwen2_model.model.model.layers.{bid}.self_attn.q_proj", # Deepseek-OCR-2 qwen2 + "model.vision_tower.layers.{bid}.attn.q_proj", # onyx ), MODEL_TENSOR.V_ENC_ATTN_Q_NORM: ( @@ -1562,7 +1564,8 @@ class TensorNameMap: "model.vision_model.transformer.layers.{bid}.self_attn.k_proj", # Deepseek-OCR CLIP, generated "siglip2.vision_model.encoder.layers.{bid}.self_attn.k_proj", "vision_model.model.layers.{bid}.self_attn.k_proj.linear", # gemma4 - "model.qwen2_model.model.model.layers.{bid}.self_attn.k_proj" # Deepseek-OCR-2 qwen2 + "model.qwen2_model.model.model.layers.{bid}.self_attn.k_proj", # Deepseek-OCR-2 qwen2 + "model.vision_tower.layers.{bid}.attn.k_proj", # onyx ), MODEL_TENSOR.V_ENC_ATTN_K_NORM: ( @@ -1588,7 +1591,8 @@ class TensorNameMap: "siglip2.vision_model.encoder.layers.{bid}.self_attn.v_proj", "model.vision_model.transformer.layers.{bid}.self_attn.v_proj", # Deepseek-OCR CLIP, generated "vision_model.model.layers.{bid}.self_attn.v_proj.linear", # gemma4 - "model.qwen2_model.model.model.layers.{bid}.self_attn.v_proj" # Deepseek-OCR-2 qwen2 + "model.qwen2_model.model.model.layers.{bid}.self_attn.v_proj", # Deepseek-OCR-2 qwen2 + "model.vision_tower.layers.{bid}.attn.v_proj", # onyx ), MODEL_TENSOR.V_ENC_INPUT_NORM: ( @@ -1612,6 +1616,7 @@ class TensorNameMap: "vision_tower.blocks.{bid}.norm1", # dots.ocr "vision_model.transformer.resblocks.{bid}.ln_1", # Step3-VL "model.qwen2_model.model.model.layers.{bid}.input_layernorm", # Deepseek-OCR-2 qwen2 + "model.vision_tower.layers.{bid}.norm1", # onyx ), MODEL_TENSOR.V_ENC_ATTN_O: ( @@ -1637,6 +1642,7 @@ class TensorNameMap: "vision_model.model.layers.{bid}.self_attn.o_proj.linear", # gemma4 "vision_tower.blocks.{bid}.attn.proj", # dots.ocr "vision_model.transformer.resblocks.{bid}.attn.out_proj", # Step3-VL + "model.vision_tower.layers.{bid}.attn.proj", # onyx ), MODEL_TENSOR.V_ENC_ATTN_SINKS: ( @@ -1665,6 +1671,7 @@ class TensorNameMap: "vision_tower.blocks.{bid}.norm2", # dots.ocr "vision_model.transformer.resblocks.{bid}.ln_2", # Step3-VL "model.qwen2_model.model.model.layers.{bid}.post_attention_layernorm", # Deepseek-OCR-2 qwen2 + "model.vision_tower.layers.{bid}.norm2", # onyx ), MODEL_TENSOR.V_ENC_FFN_UP: ( @@ -1689,6 +1696,7 @@ class TensorNameMap: "vision_model.model.layers.{bid}.mlp.up_proj", # gemma4 "vision_model.transformer.resblocks.{bid}.mlp.c_fc", # Step3-VL "model.qwen2_model.model.model.layers.{bid}.mlp.up_proj", # Deepseek-OCR-2 qwen2 + "model.vision_tower.layers.{bid}.mlp.fc1", # onyx ), MODEL_TENSOR.V_ENC_FFN_GATE: ( @@ -1721,6 +1729,7 @@ class TensorNameMap: "model.qwen2_model.model.model.layers.{bid}.mlp.down_proj" , # Deepseek-OCR-2 qwen2 "vision_model.model.layers.{bid}.mlp.down_proj", # gemma4 "vision_model.transformer.resblocks.{bid}.mlp.c_proj", # Step3-VL + "model.vision_tower.layers.{bid}.mlp.fc2", # onyx ), MODEL_TENSOR.V_ENC_ATTN_POST_NORM: ( @@ -1755,6 +1764,7 @@ class TensorNameMap: "model.vision_model.pre_layrnorm", # Deepseek-OCR CLIP "vision_tower.patch_embed.patchifier.norm", # dots.ocr "vision_model.ln_pre", # Step3-VL + "model.vision_tower.ln_pre", # onyx ), MODEL_TENSOR.V_POST_NORM: ( @@ -1768,6 +1778,7 @@ class TensorNameMap: "visual.post_layernorm", # glm4v "siglip2.vision_model.post_layernorm", "model.qwen2_model.model.model.norm", # Deepseek-OCR-2 qwen2 + "model.vision_tower.ln_post", # onyx ), MODEL_TENSOR.V_MM_POST_NORM: ( diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 661409a7ca..2ce0a7752d 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -234,7 +234,6 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ATTENTION_VALUE_LENGTH, "%s.attention.value_length" }, { LLM_KV_ATTENTION_LAYERNORM_EPS, "%s.attention.layer_norm_epsilon" }, { LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, "%s.attention.layer_norm_rms_epsilon" }, - { LLM_KV_ATTENTION_POST_NORM_RMS_EPS, "%s.attention.post_norm_rms_epsilon" }, { LLM_KV_ATTENTION_GROUPNORM_EPS, "%s.attention.group_norm_epsilon" }, { LLM_KV_ATTENTION_GROUPNORM_GROUPS, "%s.attention.group_norm_groups" }, { LLM_KV_ATTENTION_CAUSAL, "%s.attention.causal" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 7e97292b25..24d5d97c48 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -239,7 +239,6 @@ enum llm_kv { LLM_KV_ATTENTION_VALUE_LENGTH, LLM_KV_ATTENTION_LAYERNORM_EPS, LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, - LLM_KV_ATTENTION_POST_NORM_RMS_EPS, LLM_KV_ATTENTION_GROUPNORM_EPS, LLM_KV_ATTENTION_GROUPNORM_GROUPS, LLM_KV_ATTENTION_CAUSAL, diff --git a/src/llama-hparams.h b/src/llama-hparams.h index ce430d4364..fc770bf003 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -106,7 +106,6 @@ struct llama_hparams { float f_norm_eps; float f_norm_rms_eps; float f_norm_group_eps; - float f_post_norm_rms_eps; float f_attn_logit_softcapping = 50.0f; float f_router_logit_softcapping = 30.0f; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 1b9bc6f9fa..578e423953 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1793,7 +1793,6 @@ void llama_model::print_info() const { LLAMA_LOG_INFO("%s: n_embd_v_gqa = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_embd_v_gqa(il); }, hparams.n_layer_all).c_str()); LLAMA_LOG_INFO("%s: f_norm_eps = %.1e\n", __func__, hparams.f_norm_eps); LLAMA_LOG_INFO("%s: f_norm_rms_eps = %.1e\n", __func__, hparams.f_norm_rms_eps); - LLAMA_LOG_INFO("%s: f_post_norm_rms_eps = %.1e\n", __func__, hparams.f_post_norm_rms_eps); LLAMA_LOG_INFO("%s: f_clamp_kqv = %.1e\n", __func__, hparams.f_clamp_kqv); LLAMA_LOG_INFO("%s: f_max_alibi_bias = %.1e\n", __func__, hparams.f_max_alibi_bias); LLAMA_LOG_INFO("%s: f_logit_scale = %.1e\n", __func__, hparams.f_logit_scale); @@ -2623,7 +2622,6 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_STEP35: case LLM_ARCH_TALKIE: case LLM_ARCH_MELLUM: - // fallback, dflash rope is inherited from the linked target case LLM_ARCH_DFLASH: return LLAMA_ROPE_TYPE_NEOX; diff --git a/src/models/onyx.cpp b/src/models/onyx.cpp index b2897b6935..8aef05197f 100644 --- a/src/models/onyx.cpp +++ b/src/models/onyx.cpp @@ -5,25 +5,22 @@ void llama_model_onyx::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false); ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale); - ml.get_key(LLM_KV_ATTENTION_POST_NORM_RMS_EPS, hparams.f_post_norm_rms_eps); - // SWA layers share the model rope theta; they are also the only layers that use rope - // here (global layers are NoPE), so the 10000.0 default would apply to all of them. hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); - // SWA + NoPE: [SW, SW, SW, Full], NoPE used on Full layers. - if (hparams.n_swa > 0) { - hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - uint32_t swa_period = 4; - ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false); + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + uint32_t swa_period = 4; + if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) { hparams.set_swa_pattern(swa_period); - hparams.n_no_rope_layer_step = swa_period; } else { - hparams.swa_type = LLAMA_SWA_TYPE_NONE; + ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); } - type = LLM_TYPE_UNKNOWN; + switch (hparams.n_layer()) { + case 52: type = LLM_TYPE_30B; break; + default: type = LLM_TYPE_UNKNOWN; + } } void llama_model_onyx::load_arch_tensors(llama_model_loader &) { @@ -67,6 +64,9 @@ llama_model_onyx::graph::graph(const llama_model & model, const llm_graph_params const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + // Different to f_norm_rms_eps for post-attn / post-FFN norms + const float post_norm_eps = 1e-8f; + ggml_tensor * cur; ggml_tensor * inpL; @@ -89,8 +89,8 @@ llama_model_onyx::graph::graph(const llama_model & model, const llm_graph_params ggml_tensor * inpSA = inpL; - const bool use_rope = hparams.n_no_rope_layer_step > 0 && - (il + 1) % hparams.n_no_rope_layer_step != 0; + // RoPE runs on the SWA layers, NoPE on full ones. + const bool use_rope = hparams.is_swa(il); // pre-attention norm (weight+1 folded at conversion time) cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); @@ -143,8 +143,7 @@ llama_model_onyx::graph::graph(const llama_model & model, const llm_graph_params cb(cur, "attn_o_proj", il); } - // post-attention norm (uses f_post_norm_rms_eps, not the general f_norm_rms_eps). - cur = ggml_rms_norm(ctx0, cur, hparams.f_post_norm_rms_eps); + cur = ggml_rms_norm(ctx0, cur, post_norm_eps); cur = ggml_mul(ctx0, cur, model.layers[il].attn_post_norm); cb(cur, "attn_post_norm", il); @@ -169,8 +168,7 @@ llama_model_onyx::graph::graph(const llama_model & model, const llm_graph_params LLM_FFN_SILU, LLM_FFN_PAR, il); cb(cur, "ffn_out", il); - // post-FFN norm (uses f_post_norm_rms_eps, not the general f_norm_rms_eps). - cur = ggml_rms_norm(ctx0, cur, hparams.f_post_norm_rms_eps); + cur = ggml_rms_norm(ctx0, cur, post_norm_eps); cur = ggml_mul(ctx0, cur, model.layers[il].ffn_post_norm); cb(cur, "ffn_post_norm", il); diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 15040e4af5..82f7752487 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -41,6 +41,7 @@ add_library(mtmd models/kimivl.cpp models/kimik25.cpp models/nemotron-v2-vl.cpp + models/onyx.cpp models/llama4.cpp models/llava.cpp models/minicpmv.cpp diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index d42b38222c..c159e249e2 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -408,6 +408,7 @@ enum projector_type { PROJECTOR_TYPE_MINIMAX_M3, PROJECTOR_TYPE_GRANITE4_VISION, PROJECTOR_TYPE_MIMO_AUDIO, + PROJECTOR_TYPE_ONYX, PROJECTOR_TYPE_UNKNOWN, }; @@ -465,6 +466,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"}, { PROJECTOR_TYPE_MIMO_AUDIO, "mimo_audio"}, { PROJECTOR_TYPE_PARAKEET, "parakeet"}, + { PROJECTOR_TYPE_ONYX, "onyx"}, }; static projector_type clip_projector_type_from_string(const std::string & str) { diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 8b9db5101d..5dc706c26e 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -109,6 +109,10 @@ struct clip_hparams { int32_t downsample_query_side; int32_t downsample_window_side; + // Onyx vision (per-block sparse-window pattern, learned pos-emb, patch-temporal) + int32_t onyx_patch_temporal = 0; + int32_t onyx_sparse_factor = 0; + // audio int32_t n_mel_bins = 0; // whisper preprocessor int32_t proj_stack_factor = 0; // ultravox diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 5f0d00b660..a204e7fb25 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -928,6 +928,10 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_ONYX: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_STEP3VL: { builder = std::make_unique(ctx, img); @@ -1516,6 +1520,17 @@ struct clip_model_loader { hparams.set_limit_image_tokens(8, 576); hparams.set_warmup_n_tokens(16*16); } break; + case PROJECTOR_TYPE_ONYX: + { + hparams.n_merge = 2; // pixel-shuffle downsample after the ViT + hparams.image_resize_algo = RESIZE_ALGO_LANCZOS; + hparams.rope_theta = 10000.0f; + hparams.onyx_patch_temporal = 2; // Onyx-arch invariant + hparams.onyx_sparse_factor = 4; // 3 sparse layers + 1 global, repeating + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + hparams.set_limit_image_tokens(1, 4096); + hparams.set_warmup_n_tokens(32*32); + } break; case PROJECTOR_TYPE_MIMOVL: { hparams.n_merge = 2; // spatial_merge_size @@ -2218,6 +2233,13 @@ struct clip_model_loader { model.mm_merger_fc2_w = get_tensor(string_format(TN_MM_MERGER_FC2, "weight")); model.mm_merger_fc2_b = get_tensor(string_format(TN_MM_MERGER_FC2, "bias")); } break; + case PROJECTOR_TYPE_ONYX: + { + // 3-linear MLP: fc -> erf-GELU -> proj -> erf-GELU -> vision_proj (into LLM residual dim) + model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); + } break; case PROJECTOR_TYPE_STEP3VL: { model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); @@ -3498,6 +3520,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_HUNYUANVL: case PROJECTOR_TYPE_YOUTUVL: + case PROJECTOR_TYPE_ONYX: return (img->nx() / params.patch_size) / 2; case PROJECTOR_TYPE_STEP3VL: return img->nx() / (params.patch_size * params.n_merge); @@ -3523,6 +3546,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_HUNYUANVL: case PROJECTOR_TYPE_YOUTUVL: + case PROJECTOR_TYPE_ONYX: return (img->ny() / params.patch_size) / 2; case PROJECTOR_TYPE_STEP3VL: return img->ny() / (params.patch_size * params.n_merge); @@ -3602,6 +3626,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_MINIMAX_M3: case PROJECTOR_TYPE_GLM4V: case PROJECTOR_TYPE_YOUTUVL: + case PROJECTOR_TYPE_ONYX: { // dynamic size (2 conv, so double patch size) int x_patch = img->nx() / (params.patch_size * 2); @@ -3928,6 +3953,70 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 // set input per projector switch (ctx->model.proj_type) { + case PROJECTOR_TYPE_ONYX: + { + const int grid_w = pos_w; // image_size_width / patch_size + const int grid_h = pos_h; // image_size_height / patch_size + const int n_tok = grid_w * grid_h; + const int pgrid = (int) std::sqrt((double) ctx->model.position_embeddings->ne[1]); // 32 + const int f = hparams.n_merge; // downsample 2 + + // pixel patchify runs inside the graph via build_inp() (ggml_conv_2d); + // pos-emb bilinear interp via resize_position_embeddings(). + + // --- sparse window grouping (pgrid x pgrid windows) --- + const int win = pgrid; + const int nwin_h = (grid_h + win - 1) / win; + const int nwin_w = (grid_w + win - 1) / win; + std::vector sp_perm; sp_perm.reserve(n_tok); + std::vector sp_slens; + for (int wy = 0; wy < nwin_h; wy++) { + for (int wx = 0; wx < nwin_w; wx++) { + int cnt = 0; + for (int hh = 0; hh < win; hh++) { + for (int ww = 0; ww < win; ww++) { + const int gy = wy * win + hh; + const int gx = wx * win + ww; + if (gy < grid_h && gx < grid_w) { sp_perm.push_back(gy * grid_w + gx); cnt++; } + } + } + if (cnt > 0) sp_slens.push_back(cnt); + } + } + std::vector rpos_w(n_tok), rpos_h(n_tok), inv_perm(n_tok); + for (int i = 0; i < n_tok; i++) { + const int orig = sp_perm[i]; + rpos_w[i] = (orig % grid_w) + 1; // 1-indexed + rpos_h[i] = (orig / grid_w) + 1; + inv_perm[orig] = i; + } + set_input_i32("onyx_sp_perm", sp_perm); + set_input_i32("onyx_inv_perm", inv_perm); + set_input_i32("onyx_pos_w", rpos_w); + set_input_i32("onyx_pos_h", rpos_h); + + // block-diagonal window mask (permuted order) + std::vector sp_mask((size_t) n_tok * n_tok, -INFINITY); + { + int off = 0; + for (int s : sp_slens) { + for (int a = 0; a < s; a++) + for (int b = 0; b < s; b++) + sp_mask[(size_t) (off + a) * n_tok + (off + b)] = 0.0f; + off += s; + } + } + set_input_f32("onyx_sp_mask", sp_mask); + + // pixel-shuffle gather (original order): f*f spatial neighbours grouped + std::vector dsp; dsp.reserve(n_tok); + for (int oy = 0; oy < grid_h / f; oy++) + for (int ox = 0; ox < grid_w / f; ox++) + for (int ry = 0; ry < f; ry++) + for (int rx = 0; rx < f; rx++) + dsp.push_back((oy * f + ry) * grid_w + (ox * f + rx)); + set_input_i32("onyx_ds_perm", dsp); + } break; case PROJECTOR_TYPE_MINICPMV: { // inspired from siglip: @@ -4976,6 +5065,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.mm_model_mlp_3_w->ne[1]; case PROJECTOR_TYPE_MINIMAX_M3: return ctx->model.mm_merger_fc2_b->ne[0]; + case PROJECTOR_TYPE_ONYX: + return ctx->model.mm_2_w->ne[1]; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_EXAONE4_5: diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index e54366a086..614d66f507 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -254,3 +254,8 @@ private: ggml_tensor * build_newline_row(ggml_context * ctx0); ggml_tensor * append_rowwise_newlines(ggml_context * ctx0, ggml_tensor * tile_output); }; + +struct clip_graph_onyx : clip_graph { + clip_graph_onyx(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; +}; diff --git a/tools/mtmd/models/onyx.cpp b/tools/mtmd/models/onyx.cpp new file mode 100644 index 0000000000..20308c8353 --- /dev/null +++ b/tools/mtmd/models/onyx.cpp @@ -0,0 +1,88 @@ +#include "models.h" + +// Onyx vision encoder: 50-layer ViT with 2D RoPE, sparse block-diagonal +// window attention (every 4th + last layer global), pixel-shuffle downsample, then +// adapter MLP + LLM's vision_projection. +// +// Several quantities are precomputed on host and fed as named graph inputs (filled in +// clip.cpp set_input, PROJECTOR_TYPE_ONYX branch): +// onyx_pos_w/_h [n_tok] i32 : 1-indexed RoPE positions (sparse-permuted order) +// onyx_sp_perm [n_tok] i32 : window grouping permutation (applied after ln_pre) +// onyx_inv_perm [n_tok] i32 : inverse of sp_perm (applied after blocks) +// onyx_ds_perm [n_tok] i32 : pixel-shuffle gather (original order) +// onyx_sp_mask [n_tok, n_tok] f32 : block-diagonal window mask (sparse layers) +ggml_cgraph * clip_graph_onyx::build() { + const int ds = hparams.n_merge; // downsample factor (2) + const int sf = hparams.onyx_sparse_factor; // 4 + const int n_tok = n_patches; + const int n_out = (n_patches_x / ds) * (n_patches_y / ds); + const float rope_base = hparams.rope_theta; // 10000 + + auto inp_i32 = [&](const char * name, int64_t n) { + ggml_tensor * t = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n); + ggml_set_name(t, name); + ggml_set_input(t); + return t; + }; + + ggml_tensor * pos_w = inp_i32("onyx_pos_w", n_tok); + ggml_tensor * pos_h = inp_i32("onyx_pos_h", n_tok); + ggml_tensor * sp_perm = inp_i32("onyx_sp_perm", n_tok); + ggml_tensor * inv_perm = inp_i32("onyx_inv_perm", n_tok); + ggml_tensor * ds_perm = inp_i32("onyx_ds_perm", n_tok); + + ggml_tensor * sp_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tok, n_tok); + ggml_set_name(sp_mask, "onyx_sp_mask"); + ggml_set_input(sp_mask); + + // patchify via build_inp (conv2d over raw pixels) + bilinear-resized learned pos-emb + ggml_tensor * x = build_inp(); // [n_embd, n_tok, 1] + x = ggml_add(ctx0, x, resize_position_embeddings(GGML_SCALE_MODE_BILINEAR)); + cb(x, "after_posemb", -1); + + // group patches into pgrid x pgrid windows (sparse attention order) + x = ggml_get_rows(ctx0, x, sp_perm); + cb(x, "after_sp_perm", -1); + + // per-layer mask: sparse layers get sp_mask, global layers (every sf-th and last) get none + std::vector attn_mask_layers(n_layer); + for (int il = 0; il < n_layer; ++il) { + const bool is_global = (il == n_layer - 1) || ((il + 1) % sf == 0); + attn_mask_layers[il] = is_global ? nullptr : sp_mask; + } + + // 2D RoPE: first half of head_dim uses width pos, second half uses height pos + auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + return build_rope_2d(ctx0, cur, pos_w, pos_h, rope_base, false); + }; + + build_vit_opts opts; + opts.attn_mask_layers = std::move(attn_mask_layers); + + // pre_ln, per-layer transformer, post_ln (all inside build_vit); reference uses exact (erf) GELU + x = build_vit(x, n_tok, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr, add_pos, opts); + + // un-permute back to original grid order + x = ggml_get_rows(ctx0, x, inv_perm); + cb(x, "after_inv_perm", -1); + + // pixel-shuffle downsample: gather f*f spatial neighbors then concat channel-outer. + // out[c*(ds*ds)+s, o] = x[ds_perm gathered][o*(ds*ds)+s, c] + x = ggml_get_rows(ctx0, x, ds_perm); // [n_embd, n_tok], grouped + x = ggml_reshape_3d(ctx0, x, n_embd, ds * ds, n_out);// [c, s, o] + x = ggml_permute(ctx0, x, 1, 0, 2, 3); // [s, c, o] + x = ggml_cont(ctx0, x); + x = ggml_reshape_2d(ctx0, x, n_embd * ds * ds, n_out); // [6144, n_out] + cb(x, "encoder_out", -1); + + // adapter (6144->4096->4096, exact GELU each) + LLM vision_projection (4096->6656) + x = build_mm(model.mm_0_w, x); + x = ggml_gelu_erf(ctx0, x); + x = build_mm(model.mm_1_w, x); + x = ggml_gelu_erf(ctx0, x); + x = build_mm(model.mm_2_w, x); // [6656, n_out] + cb(x, "projected", -1); + + ggml_build_forward_expand(gf, x); + return gf; +} diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 72d35fce69..7c3c585ad0 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -1591,3 +1591,65 @@ mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_im } return output; } + +// +// mtmd_image_preprocessor_onyx +// + +// Replicates transformers' get_aspect_ratio_preserving_size (image_processing_onyx.py) +static clip_image_size onyx_grid_size(int img_w, int img_h, int patch_hw, int max_tokens) { + double i_nph = (double) img_h / patch_hw; + double i_npw = (double) img_w / patch_hw; + const double ratio = i_nph > 0.0 ? i_npw / i_nph : 1.0; + if (i_nph * i_npw > (double) max_tokens) { + i_nph = std::sqrt((double) max_tokens / ratio); + i_npw = i_nph * ratio; + } + const int hs[2] = { (int) std::floor(i_nph), (int) std::ceil(i_nph) }; + const int ws[2] = { (int) std::floor(i_npw), (int) std::ceil(i_npw) }; + const double target_ar = (double) img_h / (double) img_w; + int best_nph = -1; + int best_npw = -1; + double best_d = 0.0; + for (int a = 0; a < 2; ++a) { + for (int b = 0; b < 2; ++b) { + const int nph = hs[a]; + const int npw = ws[b]; + if (nph < 1 || npw < 1 || nph * npw > max_tokens) { + continue; + } + const double d = std::fabs((double) nph / (double) npw - target_ar); + const int n_tokens = nph * npw; + const int best_n_tokens = best_nph * best_npw; + if (best_nph < 0 || d < best_d || (d == best_d && n_tokens > best_n_tokens)) { + best_nph = nph; + best_npw = npw; + best_d = d; + } + } + } + if (best_nph < 0) { // no candidate fit under the cap: round and clamp + best_nph = std::max(1, (int) std::lround(i_nph)); + best_npw = std::max(1, (int) std::lround(i_npw)); + } + return clip_image_size{ best_npw * patch_hw, best_nph * patch_hw }; +} + +mtmd_image_preproc_out mtmd_image_preprocessor_onyx::preprocess(const clip_image_u8 & img) { + const int patch_hw = hparams.patch_size * hparams.n_merge; + const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge; + GGML_ASSERT(patch_area > 0 && hparams.image_max_pixels > 0); + const int max_tokens = hparams.image_max_pixels / patch_area; + + const clip_image_size original_size = img.get_size(); + const clip_image_size target_size = onyx_grid_size( + original_size.width, original_size.height, patch_hw, max_tokens); + + // PIL resizes directly to (target_w, target_h) -- a stretch, no padding. + clip_image_u8 resized_image; + img_tool::resize(img, resized_image, target_size, hparams.image_resize_algo, PAD_NONE); + + mtmd_image_preproc_out output; + output.append(hparams, resized_image, true); + return output; +} diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index 115cba51e8..26fd84d485 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -224,3 +224,9 @@ struct mtmd_image_preprocessor_granite : mtmd_image_preprocessor_llava_uhd { mtmd_image_preprocessor_granite(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; }; + +// pick the patch grid closest to the input aspect ratio under the per-image token cap, stretch-resize. +struct mtmd_image_preprocessor_onyx : mtmd_image_preprocessor { + mtmd_image_preprocessor_onyx(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; +}; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 93ca8cbcf8..bd81780538 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -470,6 +470,12 @@ struct mtmd_context { img_end = "]<]end of image[>["; image_preproc = std::make_unique(ctx_v); } break; + case PROJECTOR_TYPE_ONYX: + { + img_beg = "<|image_start|>"; + img_end = "<|image_end|>"; + image_preproc = std::make_unique(ctx_v); + } break; case PROJECTOR_TYPE_YOUTUVL: { // <|vision_start|> ... (image embeddings) ... <|vision_end|>