From 18acf25487c455ee0b0def74acd2228a3bf8e9ef Mon Sep 17 00:00:00 2001 From: Pedro Cuenca Date: Tue, 4 Aug 2026 23:39:00 +0200 Subject: [PATCH] Prefer _size instead of independent _h and _w --- conversion/onyx.py | 5 +++-- tools/mtmd/clip-impl.h | 3 +-- tools/mtmd/clip-model.h | 3 +-- tools/mtmd/clip.cpp | 43 +++++++++++++++++++---------------------- 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/conversion/onyx.py b/conversion/onyx.py index bc95d899f2..255f347c9c 100644 --- a/conversion/onyx.py +++ b/conversion/onyx.py @@ -90,10 +90,11 @@ class OnyxVisionModel(MmprojModel): self.gguf_writer.add_vision_spatial_merge_size(int(c["merge_size"])) self.gguf_writer.add_vision_rope_theta(float(c.get("rope_parameters", {}).get("rope_theta", self.ROPE_THETA))) + pos_h, pos_w = int(c["pos_emb_height"]), int(c["pos_emb_width"]) + assert pos_h == pos_w, f"Onyx assumes square pos-emb grid; got {pos_h}x{pos_w}" self.gguf_writer.add_uint32("clip.vision.onyx.patch_temporal", int(c["patch_temporal"])) self.gguf_writer.add_uint32("clip.vision.onyx.sparse_attention_factor", int(c["sparse_attention_factor"])) - self.gguf_writer.add_uint32("clip.vision.onyx.pos_emb_height", int(c["pos_emb_height"])) - self.gguf_writer.add_uint32("clip.vision.onyx.pos_emb_width", int(c["pos_emb_width"])) + self.gguf_writer.add_uint32("clip.vision.onyx.pos_emb_size", pos_h) @classmethod def filter_tensors(cls, item): diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 3346e946fa..b66359845f 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -94,8 +94,7 @@ #define KEY_ONYX_PATCH_TEMPORAL "clip.vision.onyx.patch_temporal" #define KEY_ONYX_SPARSE_FACTOR "clip.vision.onyx.sparse_attention_factor" -#define KEY_ONYX_POS_EMB_H "clip.vision.onyx.pos_emb_height" -#define KEY_ONYX_POS_EMB_W "clip.vision.onyx.pos_emb_width" +#define KEY_ONYX_POS_EMB_SIZE "clip.vision.onyx.pos_emb_size" // // tensor name constants diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 1c71a04681..b4fe606005 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -112,8 +112,7 @@ struct clip_hparams { // Onyx vision (per-block sparse-window pattern, learned pos-emb, patch-temporal) int32_t onyx_patch_temporal = 0; int32_t onyx_sparse_factor = 0; - int32_t onyx_pos_emb_h = 0; - int32_t onyx_pos_emb_w = 0; + int32_t onyx_pos_emb_size = 0; // audio int32_t n_mel_bins = 0; // whisper preprocessor diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index ec89043077..a6aa24303f 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1529,8 +1529,7 @@ struct clip_model_loader { get_f32(KEY_V_ROPE_THETA, hparams.rope_theta, false); get_u32(KEY_ONYX_PATCH_TEMPORAL, hparams.onyx_patch_temporal, false); get_u32(KEY_ONYX_SPARSE_FACTOR, hparams.onyx_sparse_factor, false); - get_u32(KEY_ONYX_POS_EMB_H, hparams.onyx_pos_emb_h, false); - get_u32(KEY_ONYX_POS_EMB_W, hparams.onyx_pos_emb_w, false); + get_u32(KEY_ONYX_POS_EMB_SIZE, hparams.onyx_pos_emb_size, false); hparams.set_limit_image_tokens(1, 4096); hparams.set_warmup_n_tokens(32*32); } break; @@ -3967,9 +3966,8 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 const int ps = patch_size; const int nx = image_size_width; const int pt = hparams.onyx_patch_temporal; - const int pos_h = hparams.onyx_pos_emb_h; // 32 - const int pos_w = hparams.onyx_pos_emb_w; // 32 - const int nemb = hparams.n_embd; // 1536 + const int pgrid = hparams.onyx_pos_emb_size; // 32 + const int nemb = hparams.n_embd; // 1536 const int f = hparams.n_merge; // downsample 2 const int patch_dim = pt * 3 * ps * ps; const auto & buf = imgs.entries[0].get_ro_buf(); // interleaved pixels (3ch image / 6ch video) @@ -4002,9 +4000,9 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 set_input_f32("onyx_patches", patches); // --- learned pos-emb bilinear interpolation (grid_sample align_corners=False, - // zeros padding) from pos_h x pos_w to grid_h x grid_w --- - ggml_tensor * pe = ctx->model.position_embeddings; // [nemb, pos_h*pos_w] - std::vector pe_host((size_t) nemb * pos_h * pos_w); + // zeros padding) from pgrid x pgrid to grid_h x grid_w --- + ggml_tensor * pe = ctx->model.position_embeddings; // [nemb, pgrid*pgrid] + std::vector pe_host((size_t) nemb * pgrid * pgrid); { std::vector raw(ggml_nbytes(pe)); ggml_backend_tensor_get(pe, raw.data(), 0, ggml_nbytes(pe)); @@ -4012,7 +4010,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 std::memcpy(pe_host.data(), raw.data(), ggml_nbytes(pe)); } else { const auto * tt = ggml_get_type_traits(pe->type); - tt->to_float(raw.data(), pe_host.data(), (int64_t) nemb * pos_h * pos_w); + tt->to_float(raw.data(), pe_host.data(), (int64_t) nemb * pgrid * pgrid); } } // NOTE: the reference uses meshgrid(ys, xs, indexing="xy") which yields a @@ -4024,40 +4022,39 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 for (int t = 0; t < n_tok; t++) { const int i = t / grid_h; // width index (0..grid_w-1) const int j = t % grid_h; // height index (0..grid_h-1) - const float af = (i + 0.5f) * pos_h / grid_w - 0.5f; // row axis (pos_h) - const float bf = (j + 0.5f) * pos_w / grid_h - 0.5f; // col axis (pos_w) + const float af = (i + 0.5f) * pgrid / grid_w - 0.5f; // row / H axis + const float bf = (j + 0.5f) * pgrid / grid_h - 0.5f; // col / W axis const int a0 = (int) std::floor(af); const float wa = af - a0; const int b0 = (int) std::floor(bf); const float wb = bf - b0; const int as[2] = { a0, a0 + 1 }; const float was[2] = { 1.0f - wa, wa }; const int bs[2] = { b0, b0 + 1 }; const float wbs[2] = { 1.0f - wb, wb }; float * dst = pos_emb.data() + (size_t) t * nemb; for (int ka = 0; ka < 2; ka++) { - if (as[ka] < 0 || as[ka] >= pos_h) continue; + if (as[ka] < 0 || as[ka] >= pgrid) continue; for (int kb = 0; kb < 2; kb++) { - if (bs[kb] < 0 || bs[kb] >= pos_w) continue; + if (bs[kb] < 0 || bs[kb] >= pgrid) continue; const float w = was[ka] * wbs[kb]; if (w == 0.0f) continue; - const float * src = pe_host.data() + (size_t) (as[ka] * pos_w + bs[kb]) * nemb; + const float * src = pe_host.data() + (size_t) (as[ka] * pgrid + bs[kb]) * nemb; for (int c = 0; c < nemb; c++) dst[c] += w * src[c]; } } } set_input_f32("onyx_pos_emb", pos_emb); - // --- sparse window grouping (pos_h x pos_w windows; square in the trained model) --- - const int win_h = pos_h; - const int win_w = pos_w; - const int nwin_h = (grid_h + win_h - 1) / win_h; - const int nwin_w = (grid_w + win_w - 1) / win_w; + // --- sparse window grouping (pgrid x pgrid windows) --- + const int win = pgrid; + const int nwin_h = (grid_h + win - 1) / win; + const int nwin_w = (grid_w + win - 1) / win; std::vector sp_perm; sp_perm.reserve(n_tok); std::vector sp_slens; for (int wy = 0; wy < nwin_h; wy++) { for (int wx = 0; wx < nwin_w; wx++) { int cnt = 0; - for (int hh = 0; hh < win_h; hh++) { - for (int ww = 0; ww < win_w; ww++) { - const int gy = wy * win_h + hh; - const int gx = wx * win_w + ww; + for (int hh = 0; hh < win; hh++) { + for (int ww = 0; ww < win; ww++) { + const int gy = wy * win + hh; + const int gx = wx * win + ww; if (gy < grid_h && gx < grid_w) { sp_perm.push_back(gy * grid_w + gx); cnt++; } } }