upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-03 03:17:32 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'onyx' into onyx-dflash

Browse Source
This commit is contained in:
Pedro Cuenca
2026-08-09 14:11:25 +02:00
parent 83dd146b75 0c5c49f5f5
commit 0dd638731c
19 changed files with 352 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/__init__.py
+1
View File
@@ -296,6 +296,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"MiniCPMV4_6ForConditionalGeneration": "minicpm",
"Mistral3ForConditionalGeneration": "llava",
"NemotronH_Nano_VL_V2": "nemotron",
"OnyxForConditionalGeneration": "onyx",
"PaddleOCRVisionModel": "ernie",
"Phi4ForCausalLMV": "phi",
"Qwen2AudioForConditionalGeneration": "ultravox",
conversion/onyx.py
+54 -8
View File
@@ -1,14 +1,14 @@
from __future__ import annotations
import json
from typing import Iterable, TYPE_CHECKING
from typing import Any, Iterable, TYPE_CHECKING
import torch
if TYPE_CHECKING:
from torch import Tensor
from .base import ModelBase, TextModel, gguf, logger
from .base import MmprojModel, ModelBase, TextModel, gguf
def _unpermute_for_rope(tensor: "Tensor", n_heads: int) -> "Tensor":
@@ -46,13 +46,8 @@ class OnyxModel(TextModel):
self.gguf_writer.add_final_logit_softcapping(hparams["final_logit_softcapping"])
self.gguf_writer.add_logit_scale(hparams["output_multiplier"])
self.gguf_writer.add_post_norm_rms_eps(hparams["post_norm_eps"])
# SWA + NoPE: [SW, SW, SW, Full], NoPE used on Full layers. References:
# https://huggingface.co/someorgtoo-hf/onyx-hf-converted/blob/main/config.json#L19
# https://huggingface.co/someorgtoo-hf/onyx-hf-converted/blob/main/config.json#L73
self.gguf_writer.add_sliding_window(hparams["sliding_window"])
self.gguf_writer.add_sliding_window_pattern(4)
self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hparams["layer_types"]])
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
shift = self.norm_shift(name)
@@ -129,3 +124,54 @@ class OnyxAssistantModel(TextModel):
# DFlash defaults to NEOX (rotate_half) rope, matching transformers HF layout for Q/K, QK-norms
# no permutation needed.
yield (self.map_tensor_name(name), data_torch)
@ModelBase.register("OnyxForConditionalGeneration")
class OnyxVisionModel(MmprojModel):
def get_vision_config(self) -> dict[str, Any] | None:
c = self.global_config.get("vision_config")
if not c:
return None
# Onyx actually uses dynamic size, initialize with nominal size
image_size = c["pos_emb_height"] * c["patch_size"] * c["merge_size"]
return {**c, "image_size": image_size}
def set_gguf_parameters(self):
super().set_gguf_parameters()
c = self.hparams_vision # enriched vision_config from get_vision_config()
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.ONYX)
self.gguf_writer.add_vision_attention_layernorm_eps(float(c["layer_norm_eps"]))
self.gguf_writer.add_vision_spatial_merge_size(int(c["merge_size"]))
@classmethod
def filter_tensors(cls, item):
name, gen = item
keep = ("model.vision_tower.", "model.vision_adapter.", "model.vision_projection.")
if not any(name.startswith(k) for k in keep):
return None
return super().filter_tensors((name, gen))
# 3-layer projector MLP
_MM_MLP_MAP = {
"model.vision_adapter.fc1": (gguf.MODEL_TENSOR.V_MMPROJ, 0),
"model.vision_adapter.fc2": (gguf.MODEL_TENSOR.V_MMPROJ, 1),
"model.vision_projection": (gguf.MODEL_TENSOR.V_MMPROJ, 2),
}
def modify_tensors(self, data_torch, name, bid):
if ".attn.q_proj." in name or ".attn.k_proj." in name:
n_heads = int(self.hparams_vision["num_attention_heads"])
data_torch = _unpermute_for_rope(data_torch, n_heads)
# Lay out the pt=2 temporal slabs of the patch embedding as a conv2d for build_inp()
if name.endswith("patch_embedder.patch_embedding.weight"):
n_embd = data_torch.shape[0]
pt = int(self.hparams_vision["patch_temporal"])
ps = int(self.hparams_vision["patch_size"])
data_torch = data_torch.view(n_embd, pt, 3, ps, ps).sum(dim=1) # (n_embd, 3, ps, ps)
stem, _, suffix = name.rpartition(".")
if stem in self._MM_MLP_MAP:
tensor_key, idx = self._MM_MLP_MAP[stem]
yield (self.format_tensor_name(tensor_key, bid=idx, suffix="." + suffix), data_torch)
return
yield (self.map_tensor_name(name), data_torch)
gguf-py/gguf/constants.py
+3 -3
View File
@@ -172,7 +172,6 @@ class Keys:
VALUE_LENGTH = "{arch}.attention.value_length"
LAYERNORM_EPS = "{arch}.attention.layer_norm_epsilon"
LAYERNORM_RMS_EPS = "{arch}.attention.layer_norm_rms_epsilon"
POST_NORM_RMS_EPS = "{arch}.attention.post_norm_rms_epsilon"
GROUPNORM_EPS = "{arch}.attention.group_norm_epsilon"
GROUPNORM_GROUPS = "{arch}.attention.group_norm_groups"
CAUSAL = "{arch}.attention.causal"
@@ -1481,8 +1480,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.V_MM_UP: "mm.up",
MODEL_TENSOR.V_MM_DOWN: "mm.down",
MODEL_TENSOR.V_MM_GATE: "mm.gate",
MODEL_TENSOR.V_MM_MERGER_FC1: "mm.merger.fc1",
MODEL_TENSOR.V_MM_MERGER_FC2: "mm.merger.fc2",
MODEL_TENSOR.V_MM_MERGER_FC1: "mm.merger.fc1",
MODEL_TENSOR.V_MM_MERGER_FC2: "mm.merger.fc2",
MODEL_TENSOR.V_TOK_BOI: "v.boi",
MODEL_TENSOR.V_TOK_EOI: "v.eoi",
MODEL_TENSOR.V_MM_PRE_NORM: "mm.pre_norm",
@@ -4907,6 +4906,7 @@ class VisionProjectorType:
MIMOVL = "mimovl"
MIMO_AUDIO = "mimo_audio"
GRANITE4_VISION = "granite4_vision"
ONYX = "onyx"
# Items here are (block size, type size)
gguf-py/gguf/gguf_writer.py
-3
View File
@@ -923,9 +923,6 @@ class GGUFWriter:
def add_layer_norm_rms_eps(self, value: float) -> None:
self.add_float32(Keys.Attention.LAYERNORM_RMS_EPS.format(arch=self.arch), value)
def add_post_norm_rms_eps(self, value: float) -> None:
self.add_float32(Keys.Attention.POST_NORM_RMS_EPS.format(arch=self.arch), value)
def add_group_norm_eps(self, value: float) -> None:
self.add_float32(Keys.Attention.GROUPNORM_EPS.format(arch=self.arch), value)
gguf-py/gguf/tensor_mapping.py
+14 -3
View File
@@ -1469,6 +1469,7 @@ class TensorNameMap:
"vision_tower.patch_embed.patchifier.proj", # dots.ocr
"vision_model.conv1", # Step3-VL
"model.vision_embedder.patch_dense", # gemma4 unified
"model.vision_tower.patch_embedder.patch_embedding", # onyx
),
MODEL_TENSOR.V_ENC_EMBD_NORM: (
@@ -1536,7 +1537,8 @@ class TensorNameMap:
"siglip2.vision_model.encoder.layers.{bid}.self_attn.q_proj", # youtuvl
"model.vision_model.transformer.layers.{bid}.self_attn.q_proj", # Deepseek-OCR CLIP, generated
"vision_model.model.layers.{bid}.self_attn.q_proj.linear", # gemma4
"model.qwen2_model.model.model.layers.{bid}.self_attn.q_proj" # Deepseek-OCR-2 qwen2
"model.qwen2_model.model.model.layers.{bid}.self_attn.q_proj", # Deepseek-OCR-2 qwen2
"model.vision_tower.layers.{bid}.attn.q_proj", # onyx
),
MODEL_TENSOR.V_ENC_ATTN_Q_NORM: (
@@ -1562,7 +1564,8 @@ class TensorNameMap:
"model.vision_model.transformer.layers.{bid}.self_attn.k_proj", # Deepseek-OCR CLIP, generated
"siglip2.vision_model.encoder.layers.{bid}.self_attn.k_proj",
"vision_model.model.layers.{bid}.self_attn.k_proj.linear", # gemma4
"model.qwen2_model.model.model.layers.{bid}.self_attn.k_proj" # Deepseek-OCR-2 qwen2
"model.qwen2_model.model.model.layers.{bid}.self_attn.k_proj", # Deepseek-OCR-2 qwen2
"model.vision_tower.layers.{bid}.attn.k_proj", # onyx
),
MODEL_TENSOR.V_ENC_ATTN_K_NORM: (
@@ -1588,7 +1591,8 @@ class TensorNameMap:
"siglip2.vision_model.encoder.layers.{bid}.self_attn.v_proj",
"model.vision_model.transformer.layers.{bid}.self_attn.v_proj", # Deepseek-OCR CLIP, generated
"vision_model.model.layers.{bid}.self_attn.v_proj.linear", # gemma4
"model.qwen2_model.model.model.layers.{bid}.self_attn.v_proj" # Deepseek-OCR-2 qwen2
"model.qwen2_model.model.model.layers.{bid}.self_attn.v_proj", # Deepseek-OCR-2 qwen2
"model.vision_tower.layers.{bid}.attn.v_proj", # onyx
),
MODEL_TENSOR.V_ENC_INPUT_NORM: (
@@ -1612,6 +1616,7 @@ class TensorNameMap:
"vision_tower.blocks.{bid}.norm1", # dots.ocr
"vision_model.transformer.resblocks.{bid}.ln_1", # Step3-VL
"model.qwen2_model.model.model.layers.{bid}.input_layernorm", # Deepseek-OCR-2 qwen2
"model.vision_tower.layers.{bid}.norm1", # onyx
),
MODEL_TENSOR.V_ENC_ATTN_O: (
@@ -1637,6 +1642,7 @@ class TensorNameMap:
"vision_model.model.layers.{bid}.self_attn.o_proj.linear", # gemma4
"vision_tower.blocks.{bid}.attn.proj", # dots.ocr
"vision_model.transformer.resblocks.{bid}.attn.out_proj", # Step3-VL
"model.vision_tower.layers.{bid}.attn.proj", # onyx
),
MODEL_TENSOR.V_ENC_ATTN_SINKS: (
@@ -1665,6 +1671,7 @@ class TensorNameMap:
"vision_tower.blocks.{bid}.norm2", # dots.ocr
"vision_model.transformer.resblocks.{bid}.ln_2", # Step3-VL
"model.qwen2_model.model.model.layers.{bid}.post_attention_layernorm", # Deepseek-OCR-2 qwen2
"model.vision_tower.layers.{bid}.norm2", # onyx
),
MODEL_TENSOR.V_ENC_FFN_UP: (
@@ -1689,6 +1696,7 @@ class TensorNameMap:
"vision_model.model.layers.{bid}.mlp.up_proj", # gemma4
"vision_model.transformer.resblocks.{bid}.mlp.c_fc", # Step3-VL
"model.qwen2_model.model.model.layers.{bid}.mlp.up_proj", # Deepseek-OCR-2 qwen2
"model.vision_tower.layers.{bid}.mlp.fc1", # onyx
),
MODEL_TENSOR.V_ENC_FFN_GATE: (
@@ -1721,6 +1729,7 @@ class TensorNameMap:
"model.qwen2_model.model.model.layers.{bid}.mlp.down_proj" , # Deepseek-OCR-2 qwen2
"vision_model.model.layers.{bid}.mlp.down_proj", # gemma4
"vision_model.transformer.resblocks.{bid}.mlp.c_proj", # Step3-VL
"model.vision_tower.layers.{bid}.mlp.fc2", # onyx
),
MODEL_TENSOR.V_ENC_ATTN_POST_NORM: (
@@ -1755,6 +1764,7 @@ class TensorNameMap:
"model.vision_model.pre_layrnorm", # Deepseek-OCR CLIP
"vision_tower.patch_embed.patchifier.norm", # dots.ocr
"vision_model.ln_pre", # Step3-VL
"model.vision_tower.ln_pre", # onyx
),
MODEL_TENSOR.V_POST_NORM: (
@@ -1768,6 +1778,7 @@ class TensorNameMap:
"visual.post_layernorm", # glm4v
"siglip2.vision_model.post_layernorm",
"model.qwen2_model.model.model.norm", # Deepseek-OCR-2 qwen2
"model.vision_tower.ln_post", # onyx
),
MODEL_TENSOR.V_MM_POST_NORM: (
src/llama-arch.cpp
-1
View File
@@ -234,7 +234,6 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_ATTENTION_VALUE_LENGTH, "%s.attention.value_length" },
{ LLM_KV_ATTENTION_LAYERNORM_EPS, "%s.attention.layer_norm_epsilon" },
{ LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, "%s.attention.layer_norm_rms_epsilon" },
{ LLM_KV_ATTENTION_POST_NORM_RMS_EPS, "%s.attention.post_norm_rms_epsilon" },
{ LLM_KV_ATTENTION_GROUPNORM_EPS, "%s.attention.group_norm_epsilon" },
{ LLM_KV_ATTENTION_GROUPNORM_GROUPS, "%s.attention.group_norm_groups" },
{ LLM_KV_ATTENTION_CAUSAL, "%s.attention.causal" },
src/llama-arch.h
-1
View File
@@ -239,7 +239,6 @@ enum llm_kv {
LLM_KV_ATTENTION_VALUE_LENGTH,
LLM_KV_ATTENTION_LAYERNORM_EPS,
LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,
LLM_KV_ATTENTION_POST_NORM_RMS_EPS,
LLM_KV_ATTENTION_GROUPNORM_EPS,
LLM_KV_ATTENTION_GROUPNORM_GROUPS,
LLM_KV_ATTENTION_CAUSAL,
src/llama-hparams.h
-1
View File
@@ -106,7 +106,6 @@ struct llama_hparams {
float f_norm_eps;
float f_norm_rms_eps;
float f_norm_group_eps;
float f_post_norm_rms_eps;
float f_attn_logit_softcapping = 50.0f;
float f_router_logit_softcapping = 30.0f;
src/llama-model.cpp
-2
View File
@@ -1793,7 +1793,6 @@ void llama_model::print_info() const {
LLAMA_LOG_INFO("%s: n_embd_v_gqa = %s\n", __func__, print_f([&](uint32_t il) { return hparams.n_embd_v_gqa(il); }, hparams.n_layer_all).c_str());
LLAMA_LOG_INFO("%s: f_norm_eps = %.1e\n", __func__, hparams.f_norm_eps);
LLAMA_LOG_INFO("%s: f_norm_rms_eps = %.1e\n", __func__, hparams.f_norm_rms_eps);
LLAMA_LOG_INFO("%s: f_post_norm_rms_eps = %.1e\n", __func__, hparams.f_post_norm_rms_eps);
LLAMA_LOG_INFO("%s: f_clamp_kqv = %.1e\n", __func__, hparams.f_clamp_kqv);
LLAMA_LOG_INFO("%s: f_max_alibi_bias = %.1e\n", __func__, hparams.f_max_alibi_bias);
LLAMA_LOG_INFO("%s: f_logit_scale = %.1e\n", __func__, hparams.f_logit_scale);
@@ -2623,7 +2622,6 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_STEP35:
case LLM_ARCH_TALKIE:
case LLM_ARCH_MELLUM:
// fallback, dflash rope is inherited from the linked target
case LLM_ARCH_DFLASH:
return LLAMA_ROPE_TYPE_NEOX;
src/models/onyx.cpp
+15 -17
View File
@@ -5,25 +5,22 @@ void llama_model_onyx::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);
ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false);
ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);
ml.get_key(LLM_KV_ATTENTION_POST_NORM_RMS_EPS, hparams.f_post_norm_rms_eps);
// SWA layers share the model rope theta; they are also the only layers that use rope
// here (global layers are NoPE), so the 10000.0 default would apply to all of them.
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
// SWA + NoPE: [SW, SW, SW, Full], NoPE used on Full layers.
if (hparams.n_swa > 0) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
uint32_t swa_period = 4;
ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
uint32_t swa_period = 4;
if (ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false)) {
hparams.set_swa_pattern(swa_period);
hparams.n_no_rope_layer_step = swa_period;
} else {
hparams.swa_type = LLAMA_SWA_TYPE_NONE;
ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());
}
type = LLM_TYPE_UNKNOWN;
switch (hparams.n_layer()) {
case 52: type = LLM_TYPE_30B; break;
default: type = LLM_TYPE_UNKNOWN;
}
}
void llama_model_onyx::load_arch_tensors(llama_model_loader &) {
@@ -67,6 +64,9 @@ llama_model_onyx::graph::graph(const llama_model & model, const llm_graph_params
const int64_t n_embd_head = hparams.n_embd_head_v();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
// Different to f_norm_rms_eps for post-attn / post-FFN norms
const float post_norm_eps = 1e-8f;
ggml_tensor * cur;
ggml_tensor * inpL;
@@ -89,8 +89,8 @@ llama_model_onyx::graph::graph(const llama_model & model, const llm_graph_params
ggml_tensor * inpSA = inpL;
const bool use_rope = hparams.n_no_rope_layer_step > 0 &&
(il + 1) % hparams.n_no_rope_layer_step != 0;
// RoPE runs on the SWA layers, NoPE on full ones.
const bool use_rope = hparams.is_swa(il);
// pre-attention norm (weight+1 folded at conversion time)
cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
@@ -143,8 +143,7 @@ llama_model_onyx::graph::graph(const llama_model & model, const llm_graph_params
cb(cur, "attn_o_proj", il);
}
// post-attention norm (uses f_post_norm_rms_eps, not the general f_norm_rms_eps).
cur = ggml_rms_norm(ctx0, cur, hparams.f_post_norm_rms_eps);
cur = ggml_rms_norm(ctx0, cur, post_norm_eps);
cur = ggml_mul(ctx0, cur, model.layers[il].attn_post_norm);
cb(cur, "attn_post_norm", il);
@@ -169,8 +168,7 @@ llama_model_onyx::graph::graph(const llama_model & model, const llm_graph_params
LLM_FFN_SILU, LLM_FFN_PAR, il);
cb(cur, "ffn_out", il);
// post-FFN norm (uses f_post_norm_rms_eps, not the general f_norm_rms_eps).
cur = ggml_rms_norm(ctx0, cur, hparams.f_post_norm_rms_eps);
cur = ggml_rms_norm(ctx0, cur, post_norm_eps);
cur = ggml_mul(ctx0, cur, model.layers[il].ffn_post_norm);
cb(cur, "ffn_post_norm", il);
tools/mtmd/CMakeLists.txt
+1
View File
@@ -41,6 +41,7 @@ add_library(mtmd
models/kimivl.cpp
models/kimik25.cpp
models/nemotron-v2-vl.cpp
models/onyx.cpp
models/llama4.cpp
models/llava.cpp
models/minicpmv.cpp
tools/mtmd/clip-impl.h
+2
View File
@@ -408,6 +408,7 @@ enum projector_type {
PROJECTOR_TYPE_MINIMAX_M3,
PROJECTOR_TYPE_GRANITE4_VISION,
PROJECTOR_TYPE_MIMO_AUDIO,
PROJECTOR_TYPE_ONYX,
PROJECTOR_TYPE_UNKNOWN,
};
@@ -465,6 +466,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"},
{ PROJECTOR_TYPE_MIMO_AUDIO, "mimo_audio"},
{ PROJECTOR_TYPE_PARAKEET, "parakeet"},
{ PROJECTOR_TYPE_ONYX, "onyx"},
};
static projector_type clip_projector_type_from_string(const std::string & str) {
tools/mtmd/clip-model.h
+4
View File
@@ -109,6 +109,10 @@ struct clip_hparams {
int32_t downsample_query_side;
int32_t downsample_window_side;
// Onyx vision (per-block sparse-window pattern, learned pos-emb, patch-temporal)
int32_t onyx_patch_temporal = 0;
int32_t onyx_sparse_factor = 0;
// audio
int32_t n_mel_bins = 0; // whisper preprocessor
int32_t proj_stack_factor = 0; // ultravox
tools/mtmd/clip.cpp
+91
View File
@@ -928,6 +928,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_minimax_m3>(ctx, img);
} break;
case PROJECTOR_TYPE_ONYX:
{
builder = std::make_unique<clip_graph_onyx>(ctx, img);
} break;
case PROJECTOR_TYPE_STEP3VL:
{
builder = std::make_unique<clip_graph_step3vl>(ctx, img);
@@ -1516,6 +1520,17 @@ struct clip_model_loader {
hparams.set_limit_image_tokens(8, 576);
hparams.set_warmup_n_tokens(16*16);
} break;
case PROJECTOR_TYPE_ONYX:
{
hparams.n_merge = 2; // pixel-shuffle downsample after the ViT
hparams.image_resize_algo = RESIZE_ALGO_LANCZOS;
hparams.rope_theta = 10000.0f;
hparams.onyx_patch_temporal = 2; // Onyx-arch invariant
hparams.onyx_sparse_factor = 4; // 3 sparse layers + 1 global, repeating
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
hparams.set_limit_image_tokens(1, 4096);
hparams.set_warmup_n_tokens(32*32);
} break;
case PROJECTOR_TYPE_MIMOVL:
{
hparams.n_merge = 2; // spatial_merge_size
@@ -2218,6 +2233,13 @@ struct clip_model_loader {
model.mm_merger_fc2_w = get_tensor(string_format(TN_MM_MERGER_FC2, "weight"));
model.mm_merger_fc2_b = get_tensor(string_format(TN_MM_MERGER_FC2, "bias"));
} break;
case PROJECTOR_TYPE_ONYX:
{
// 3-linear MLP: fc -> erf-GELU -> proj -> erf-GELU -> vision_proj (into LLM residual dim)
model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
} break;
case PROJECTOR_TYPE_STEP3VL:
{
model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
@@ -3498,6 +3520,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_HUNYUANVL:
case PROJECTOR_TYPE_YOUTUVL:
case PROJECTOR_TYPE_ONYX:
return (img->nx() / params.patch_size) / 2;
case PROJECTOR_TYPE_STEP3VL:
return img->nx() / (params.patch_size * params.n_merge);
@@ -3523,6 +3546,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_HUNYUANVL:
case PROJECTOR_TYPE_YOUTUVL:
case PROJECTOR_TYPE_ONYX:
return (img->ny() / params.patch_size) / 2;
case PROJECTOR_TYPE_STEP3VL:
return img->ny() / (params.patch_size * params.n_merge);
@@ -3602,6 +3626,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_MINIMAX_M3:
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_YOUTUVL:
case PROJECTOR_TYPE_ONYX:
{
// dynamic size (2 conv, so double patch size)
int x_patch = img->nx() / (params.patch_size * 2);
@@ -3928,6 +3953,70 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
// set input per projector
switch (ctx->model.proj_type) {
case PROJECTOR_TYPE_ONYX:
{
const int grid_w = pos_w; // image_size_width / patch_size
const int grid_h = pos_h; // image_size_height / patch_size
const int n_tok = grid_w * grid_h;
const int pgrid = (int) std::sqrt((double) ctx->model.position_embeddings->ne[1]); // 32
const int f = hparams.n_merge; // downsample 2
// pixel patchify runs inside the graph via build_inp() (ggml_conv_2d);
// pos-emb bilinear interp via resize_position_embeddings().
// --- sparse window grouping (pgrid x pgrid windows) ---
const int win = pgrid;
const int nwin_h = (grid_h + win - 1) / win;
const int nwin_w = (grid_w + win - 1) / win;
std::vector<int32_t> sp_perm; sp_perm.reserve(n_tok);
std::vector<int> sp_slens;
for (int wy = 0; wy < nwin_h; wy++) {
for (int wx = 0; wx < nwin_w; wx++) {
int cnt = 0;
for (int hh = 0; hh < win; hh++) {
for (int ww = 0; ww < win; ww++) {
const int gy = wy * win + hh;
const int gx = wx * win + ww;
if (gy < grid_h && gx < grid_w) { sp_perm.push_back(gy * grid_w + gx); cnt++; }
}
}
if (cnt > 0) sp_slens.push_back(cnt);
}
}
std::vector<int32_t> rpos_w(n_tok), rpos_h(n_tok), inv_perm(n_tok);
for (int i = 0; i < n_tok; i++) {
const int orig = sp_perm[i];
rpos_w[i] = (orig % grid_w) + 1; // 1-indexed
rpos_h[i] = (orig / grid_w) + 1;
inv_perm[orig] = i;
}
set_input_i32("onyx_sp_perm", sp_perm);
set_input_i32("onyx_inv_perm", inv_perm);
set_input_i32("onyx_pos_w", rpos_w);
set_input_i32("onyx_pos_h", rpos_h);
// block-diagonal window mask (permuted order)
std::vector<float> sp_mask((size_t) n_tok * n_tok, -INFINITY);
{
int off = 0;
for (int s : sp_slens) {
for (int a = 0; a < s; a++)
for (int b = 0; b < s; b++)
sp_mask[(size_t) (off + a) * n_tok + (off + b)] = 0.0f;
off += s;
}
}
set_input_f32("onyx_sp_mask", sp_mask);
// pixel-shuffle gather (original order): f*f spatial neighbours grouped
std::vector<int32_t> dsp; dsp.reserve(n_tok);
for (int oy = 0; oy < grid_h / f; oy++)
for (int ox = 0; ox < grid_w / f; ox++)
for (int ry = 0; ry < f; ry++)
for (int rx = 0; rx < f; rx++)
dsp.push_back((oy * f + ry) * grid_w + (ox * f + rx));
set_input_i32("onyx_ds_perm", dsp);
} break;
case PROJECTOR_TYPE_MINICPMV:
{
// inspired from siglip:
@@ -4976,6 +5065,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_model_mlp_3_w->ne[1];
case PROJECTOR_TYPE_MINIMAX_M3:
return ctx->model.mm_merger_fc2_b->ne[0];
case PROJECTOR_TYPE_ONYX:
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_EXAONE4_5:
tools/mtmd/models/models.h
+5
View File
@@ -254,3 +254,8 @@ private:
ggml_tensor * build_newline_row(ggml_context * ctx0);
ggml_tensor * append_rowwise_newlines(ggml_context * ctx0, ggml_tensor * tile_output);
};
struct clip_graph_onyx : clip_graph {
clip_graph_onyx(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
};
tools/mtmd/models/onyx.cpp
+88
View File
@@ -0,0 +1,88 @@
#include "models.h"
// Onyx vision encoder: 50-layer ViT with 2D RoPE, sparse block-diagonal
// window attention (every 4th + last layer global), pixel-shuffle downsample, then
// adapter MLP + LLM's vision_projection.
//
// Several quantities are precomputed on host and fed as named graph inputs (filled in
// clip.cpp set_input, PROJECTOR_TYPE_ONYX branch):
// onyx_pos_w/_h [n_tok] i32 : 1-indexed RoPE positions (sparse-permuted order)
// onyx_sp_perm [n_tok] i32 : window grouping permutation (applied after ln_pre)
// onyx_inv_perm [n_tok] i32 : inverse of sp_perm (applied after blocks)
// onyx_ds_perm [n_tok] i32 : pixel-shuffle gather (original order)
// onyx_sp_mask [n_tok, n_tok] f32 : block-diagonal window mask (sparse layers)
ggml_cgraph * clip_graph_onyx::build() {
const int ds = hparams.n_merge; // downsample factor (2)
const int sf = hparams.onyx_sparse_factor; // 4
const int n_tok = n_patches;
const int n_out = (n_patches_x / ds) * (n_patches_y / ds);
const float rope_base = hparams.rope_theta; // 10000
auto inp_i32 = [&](const char * name, int64_t n) {
ggml_tensor * t = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n);
ggml_set_name(t, name);
ggml_set_input(t);
return t;
};
ggml_tensor * pos_w = inp_i32("onyx_pos_w", n_tok);
ggml_tensor * pos_h = inp_i32("onyx_pos_h", n_tok);
ggml_tensor * sp_perm = inp_i32("onyx_sp_perm", n_tok);
ggml_tensor * inv_perm = inp_i32("onyx_inv_perm", n_tok);
ggml_tensor * ds_perm = inp_i32("onyx_ds_perm", n_tok);
ggml_tensor * sp_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tok, n_tok);
ggml_set_name(sp_mask, "onyx_sp_mask");
ggml_set_input(sp_mask);
// patchify via build_inp (conv2d over raw pixels) + bilinear-resized learned pos-emb
ggml_tensor * x = build_inp(); // [n_embd, n_tok, 1]
x = ggml_add(ctx0, x, resize_position_embeddings(GGML_SCALE_MODE_BILINEAR));
cb(x, "after_posemb", -1);
// group patches into pgrid x pgrid windows (sparse attention order)
x = ggml_get_rows(ctx0, x, sp_perm);
cb(x, "after_sp_perm", -1);
// per-layer mask: sparse layers get sp_mask, global layers (every sf-th and last) get none
std::vector<ggml_tensor *> attn_mask_layers(n_layer);
for (int il = 0; il < n_layer; ++il) {
const bool is_global = (il == n_layer - 1) || ((il + 1) % sf == 0);
attn_mask_layers[il] = is_global ? nullptr : sp_mask;
}
// 2D RoPE: first half of head_dim uses width pos, second half uses height pos
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
return build_rope_2d(ctx0, cur, pos_w, pos_h, rope_base, false);
};
build_vit_opts opts;
opts.attn_mask_layers = std::move(attn_mask_layers);
// pre_ln, per-layer transformer, post_ln (all inside build_vit); reference uses exact (erf) GELU
x = build_vit(x, n_tok, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr, add_pos, opts);
// un-permute back to original grid order
x = ggml_get_rows(ctx0, x, inv_perm);
cb(x, "after_inv_perm", -1);
// pixel-shuffle downsample: gather f*f spatial neighbors then concat channel-outer.
// out[c*(ds*ds)+s, o] = x[ds_perm gathered][o*(ds*ds)+s, c]
x = ggml_get_rows(ctx0, x, ds_perm); // [n_embd, n_tok], grouped
x = ggml_reshape_3d(ctx0, x, n_embd, ds * ds, n_out);// [c, s, o]
x = ggml_permute(ctx0, x, 1, 0, 2, 3); // [s, c, o]
x = ggml_cont(ctx0, x);
x = ggml_reshape_2d(ctx0, x, n_embd * ds * ds, n_out); // [6144, n_out]
cb(x, "encoder_out", -1);
// adapter (6144->4096->4096, exact GELU each) + LLM vision_projection (4096->6656)
x = build_mm(model.mm_0_w, x);
x = ggml_gelu_erf(ctx0, x);
x = build_mm(model.mm_1_w, x);
x = ggml_gelu_erf(ctx0, x);
x = build_mm(model.mm_2_w, x); // [6656, n_out]
cb(x, "projected", -1);
ggml_build_forward_expand(gf, x);
return gf;
}
tools/mtmd/mtmd-image.cpp
+62
View File
@@ -1591,3 +1591,65 @@ mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_im
}
return output;
}
//
// mtmd_image_preprocessor_onyx
//
// Replicates transformers' get_aspect_ratio_preserving_size (image_processing_onyx.py)
static clip_image_size onyx_grid_size(int img_w, int img_h, int patch_hw, int max_tokens) {
double i_nph = (double) img_h / patch_hw;
double i_npw = (double) img_w / patch_hw;
const double ratio = i_nph > 0.0 ? i_npw / i_nph : 1.0;
if (i_nph * i_npw > (double) max_tokens) {
i_nph = std::sqrt((double) max_tokens / ratio);
i_npw = i_nph * ratio;
}
const int hs[2] = { (int) std::floor(i_nph), (int) std::ceil(i_nph) };
const int ws[2] = { (int) std::floor(i_npw), (int) std::ceil(i_npw) };
const double target_ar = (double) img_h / (double) img_w;
int best_nph = -1;
int best_npw = -1;
double best_d = 0.0;
for (int a = 0; a < 2; ++a) {
for (int b = 0; b < 2; ++b) {
const int nph = hs[a];
const int npw = ws[b];
if (nph < 1 || npw < 1 || nph * npw > max_tokens) {
continue;
}
const double d = std::fabs((double) nph / (double) npw - target_ar);
const int n_tokens = nph * npw;
const int best_n_tokens = best_nph * best_npw;
if (best_nph < 0 || d < best_d || (d == best_d && n_tokens > best_n_tokens)) {
best_nph = nph;
best_npw = npw;
best_d = d;
}
}
}
if (best_nph < 0) { // no candidate fit under the cap: round and clamp
best_nph = std::max(1, (int) std::lround(i_nph));
best_npw = std::max(1, (int) std::lround(i_npw));
}
return clip_image_size{ best_npw * patch_hw, best_nph * patch_hw };
}
mtmd_image_preproc_out mtmd_image_preprocessor_onyx::preprocess(const clip_image_u8 & img) {
const int patch_hw = hparams.patch_size * hparams.n_merge;
const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
GGML_ASSERT(patch_area > 0 && hparams.image_max_pixels > 0);
const int max_tokens = hparams.image_max_pixels / patch_area;
const clip_image_size original_size = img.get_size();
const clip_image_size target_size = onyx_grid_size(
original_size.width, original_size.height, patch_hw, max_tokens);
// PIL resizes directly to (target_w, target_h) -- a stretch, no padding.
clip_image_u8 resized_image;
img_tool::resize(img, resized_image, target_size, hparams.image_resize_algo, PAD_NONE);
mtmd_image_preproc_out output;
output.append(hparams, resized_image, true);
return output;
}
tools/mtmd/mtmd-image.h
+6
View File
@@ -224,3 +224,9 @@ struct mtmd_image_preprocessor_granite : mtmd_image_preprocessor_llava_uhd {
mtmd_image_preprocessor_granite(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};
// pick the patch grid closest to the input aspect ratio under the per-image token cap, stretch-resize.
struct mtmd_image_preprocessor_onyx : mtmd_image_preprocessor {
mtmd_image_preprocessor_onyx(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override;
};
tools/mtmd/mtmd.cpp
+6
View File
@@ -470,6 +470,12 @@ struct mtmd_context {
img_end = "]<]end of image[>[";
image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
} break;
case PROJECTOR_TYPE_ONYX:
{
img_beg = "<|image_start|>";
img_end = "<|image_end|>";
image_preproc = std::make_unique<mtmd_image_preprocessor_onyx>(ctx_v);
} break;
case PROJECTOR_TYPE_YOUTUVL:
{
// <|vision_start|> ... (image embeddings) ... <|vision_end|>