upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-03 19:37:29 -05:00
Code Issues Packages Projects Releases Wiki Activity

Prefer _size instead of independent _h and _w

Browse Source
This commit is contained in:
Pedro Cuenca
2026-08-04 23:39:00 +02:00
parent 4db53f43bf
commit 18acf25487
4 changed files with 25 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/onyx.py
+3 -2
View File
@@ -90,10 +90,11 @@ class OnyxVisionModel(MmprojModel):
self.gguf_writer.add_vision_spatial_merge_size(int(c["merge_size"]))
self.gguf_writer.add_vision_rope_theta(float(c.get("rope_parameters", {}).get("rope_theta", self.ROPE_THETA)))
pos_h, pos_w = int(c["pos_emb_height"]), int(c["pos_emb_width"])
assert pos_h == pos_w, f"Onyx assumes square pos-emb grid; got {pos_h}x{pos_w}"
self.gguf_writer.add_uint32("clip.vision.onyx.patch_temporal", int(c["patch_temporal"]))
self.gguf_writer.add_uint32("clip.vision.onyx.sparse_attention_factor", int(c["sparse_attention_factor"]))
self.gguf_writer.add_uint32("clip.vision.onyx.pos_emb_height", int(c["pos_emb_height"]))
self.gguf_writer.add_uint32("clip.vision.onyx.pos_emb_width", int(c["pos_emb_width"]))
self.gguf_writer.add_uint32("clip.vision.onyx.pos_emb_size", pos_h)
@classmethod
def filter_tensors(cls, item):
tools/mtmd/clip-impl.h
+1 -2
View File
@@ -94,8 +94,7 @@
#define KEY_ONYX_PATCH_TEMPORAL "clip.vision.onyx.patch_temporal"
#define KEY_ONYX_SPARSE_FACTOR "clip.vision.onyx.sparse_attention_factor"
#define KEY_ONYX_POS_EMB_H "clip.vision.onyx.pos_emb_height"
#define KEY_ONYX_POS_EMB_W "clip.vision.onyx.pos_emb_width"
#define KEY_ONYX_POS_EMB_SIZE "clip.vision.onyx.pos_emb_size"
//
// tensor name constants
tools/mtmd/clip-model.h
+1 -2
View File
@@ -112,8 +112,7 @@ struct clip_hparams {
// Onyx vision (per-block sparse-window pattern, learned pos-emb, patch-temporal)
int32_t onyx_patch_temporal = 0;
int32_t onyx_sparse_factor = 0;
int32_t onyx_pos_emb_h = 0;
int32_t onyx_pos_emb_w = 0;
int32_t onyx_pos_emb_size = 0;
// audio
int32_t n_mel_bins = 0; // whisper preprocessor
tools/mtmd/clip.cpp
+20 -23
View File
@@ -1529,8 +1529,7 @@ struct clip_model_loader {
get_f32(KEY_V_ROPE_THETA, hparams.rope_theta, false);
get_u32(KEY_ONYX_PATCH_TEMPORAL, hparams.onyx_patch_temporal, false);
get_u32(KEY_ONYX_SPARSE_FACTOR, hparams.onyx_sparse_factor, false);
get_u32(KEY_ONYX_POS_EMB_H, hparams.onyx_pos_emb_h, false);
get_u32(KEY_ONYX_POS_EMB_W, hparams.onyx_pos_emb_w, false);
get_u32(KEY_ONYX_POS_EMB_SIZE, hparams.onyx_pos_emb_size, false);
hparams.set_limit_image_tokens(1, 4096);
hparams.set_warmup_n_tokens(32*32);
} break;
@@ -3967,9 +3966,8 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
const int ps = patch_size;
const int nx = image_size_width;
const int pt = hparams.onyx_patch_temporal;
const int pos_h = hparams.onyx_pos_emb_h; // 32
const int pos_w = hparams.onyx_pos_emb_w; // 32
const int nemb = hparams.n_embd; // 1536
const int pgrid = hparams.onyx_pos_emb_size; // 32
const int nemb = hparams.n_embd; // 1536
const int f = hparams.n_merge; // downsample 2
const int patch_dim = pt * 3 * ps * ps;
const auto & buf = imgs.entries[0].get_ro_buf(); // interleaved pixels (3ch image / 6ch video)
@@ -4002,9 +4000,9 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
set_input_f32("onyx_patches", patches);
// --- learned pos-emb bilinear interpolation (grid_sample align_corners=False,
// zeros padding) from pos_h x pos_w to grid_h x grid_w ---
ggml_tensor * pe = ctx->model.position_embeddings; // [nemb, pos_h*pos_w]
std::vector<float> pe_host((size_t) nemb * pos_h * pos_w);
// zeros padding) from pgrid x pgrid to grid_h x grid_w ---
ggml_tensor * pe = ctx->model.position_embeddings; // [nemb, pgrid*pgrid]
std::vector<float> pe_host((size_t) nemb * pgrid * pgrid);
{
std::vector<char> raw(ggml_nbytes(pe));
ggml_backend_tensor_get(pe, raw.data(), 0, ggml_nbytes(pe));
@@ -4012,7 +4010,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
std::memcpy(pe_host.data(), raw.data(), ggml_nbytes(pe));
} else {
const auto * tt = ggml_get_type_traits(pe->type);
tt->to_float(raw.data(), pe_host.data(), (int64_t) nemb * pos_h * pos_w);
tt->to_float(raw.data(), pe_host.data(), (int64_t) nemb * pgrid * pgrid);
}
}
// NOTE: the reference uses meshgrid(ys, xs, indexing="xy") which yields a
@@ -4024,40 +4022,39 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
for (int t = 0; t < n_tok; t++) {
const int i = t / grid_h; // width index (0..grid_w-1)
const int j = t % grid_h; // height index (0..grid_h-1)
const float af = (i + 0.5f) * pos_h / grid_w - 0.5f; // row axis (pos_h)
const float bf = (j + 0.5f) * pos_w / grid_h - 0.5f; // col axis (pos_w)
const float af = (i + 0.5f) * pgrid / grid_w - 0.5f; // row / H axis
const float bf = (j + 0.5f) * pgrid / grid_h - 0.5f; // col / W axis
const int a0 = (int) std::floor(af); const float wa = af - a0;
const int b0 = (int) std::floor(bf); const float wb = bf - b0;
const int as[2] = { a0, a0 + 1 }; const float was[2] = { 1.0f - wa, wa };
const int bs[2] = { b0, b0 + 1 }; const float wbs[2] = { 1.0f - wb, wb };
float * dst = pos_emb.data() + (size_t) t * nemb;
for (int ka = 0; ka < 2; ka++) {
if (as[ka] < 0 || as[ka] >= pos_h) continue;
if (as[ka] < 0 || as[ka] >= pgrid) continue;
for (int kb = 0; kb < 2; kb++) {
if (bs[kb] < 0 || bs[kb] >= pos_w) continue;
if (bs[kb] < 0 || bs[kb] >= pgrid) continue;
const float w = was[ka] * wbs[kb];
if (w == 0.0f) continue;
const float * src = pe_host.data() + (size_t) (as[ka] * pos_w + bs[kb]) * nemb;
const float * src = pe_host.data() + (size_t) (as[ka] * pgrid + bs[kb]) * nemb;
for (int c = 0; c < nemb; c++) dst[c] += w * src[c];
}
}
}
set_input_f32("onyx_pos_emb", pos_emb);
// --- sparse window grouping (pos_h x pos_w windows; square in the trained model) ---
const int win_h = pos_h;
const int win_w = pos_w;
const int nwin_h = (grid_h + win_h - 1) / win_h;
const int nwin_w = (grid_w + win_w - 1) / win_w;
// --- sparse window grouping (pgrid x pgrid windows) ---
const int win = pgrid;
const int nwin_h = (grid_h + win - 1) / win;
const int nwin_w = (grid_w + win - 1) / win;
std::vector<int32_t> sp_perm; sp_perm.reserve(n_tok);
std::vector<int> sp_slens;
for (int wy = 0; wy < nwin_h; wy++) {
for (int wx = 0; wx < nwin_w; wx++) {
int cnt = 0;
for (int hh = 0; hh < win_h; hh++) {
for (int ww = 0; ww < win_w; ww++) {
const int gy = wy * win_h + hh;
const int gx = wx * win_w + ww;
for (int hh = 0; hh < win; hh++) {
for (int ww = 0; ww < win; ww++) {
const int gy = wy * win + hh;
const int gx = wx * win + ww;
if (gy < grid_h && gx < grid_w) { sp_perm.push_back(gy * grid_w + gx); cnt++; }
}
}