upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-03 19:37:29 -05:00
Code Issues Packages Projects Releases Wiki Activity

Less params, bilinear pos-emb interpolation as a graph op instead of CPU

Browse Source
This commit is contained in:
Pedro Cuenca
2026-08-06 12:10:55 +02:00
parent 6af2931853
commit d6e2e02d75
5 changed files with 2 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/onyx.py
-3
View File
@@ -90,11 +90,8 @@ class OnyxVisionModel(MmprojModel):
self.gguf_writer.add_vision_spatial_merge_size(int(c["merge_size"]))
self.gguf_writer.add_vision_rope_theta(float(c.get("rope_parameters", {}).get("rope_theta", self.ROPE_THETA)))
pos_h, pos_w = int(c["pos_emb_height"]), int(c["pos_emb_width"])
assert pos_h == pos_w, f"Onyx assumes square pos-emb grid; got {pos_h}x{pos_w}"
self.gguf_writer.add_uint32("clip.vision.onyx.patch_temporal", int(c["patch_temporal"]))
self.gguf_writer.add_uint32("clip.vision.onyx.sparse_attention_factor", int(c["sparse_attention_factor"]))
self.gguf_writer.add_uint32("clip.vision.onyx.pos_emb_size", pos_h)
@classmethod
def filter_tensors(cls, item):
tools/mtmd/clip-impl.h
-1
View File
@@ -94,7 +94,6 @@
#define KEY_ONYX_PATCH_TEMPORAL "clip.vision.onyx.patch_temporal"
#define KEY_ONYX_SPARSE_FACTOR "clip.vision.onyx.sparse_attention_factor"
#define KEY_ONYX_POS_EMB_SIZE "clip.vision.onyx.pos_emb_size"
//
// tensor name constants
tools/mtmd/clip-model.h
-1
View File
@@ -112,7 +112,6 @@ struct clip_hparams {
// Onyx vision (per-block sparse-window pattern, learned pos-emb, patch-temporal)
int32_t onyx_patch_temporal = 0;
int32_t onyx_sparse_factor = 0;
int32_t onyx_pos_emb_size = 0;
// audio
int32_t n_mel_bins = 0; // whisper preprocessor
tools/mtmd/clip.cpp
+1 -44
View File
@@ -1529,7 +1529,6 @@ struct clip_model_loader {
get_f32(KEY_V_ROPE_THETA, hparams.rope_theta, false);
get_u32(KEY_ONYX_PATCH_TEMPORAL, hparams.onyx_patch_temporal, false);
get_u32(KEY_ONYX_SPARSE_FACTOR, hparams.onyx_sparse_factor, false);
get_u32(KEY_ONYX_POS_EMB_SIZE, hparams.onyx_pos_emb_size, false);
hparams.set_limit_image_tokens(1, 4096);
hparams.set_warmup_n_tokens(32*32);
} break;
@@ -3966,8 +3965,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
const int ps = patch_size;
const int nx = image_size_width;
const int pt = hparams.onyx_patch_temporal;
const int pgrid = hparams.onyx_pos_emb_size; // 32
const int nemb = hparams.n_embd; // 1536
const int pgrid = (int) std::sqrt((double) ctx->model.position_embeddings->ne[1]); // 32
const int f = hparams.n_merge; // downsample 2
const int patch_dim = pt * 3 * ps * ps;
const auto & buf = imgs.entries[0].get_ro_buf(); // interleaved pixels (3ch image / 6ch video)
@@ -3999,47 +3997,6 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
}
set_input_f32("onyx_patches", patches);
// --- learned pos-emb bilinear interpolation (grid_sample align_corners=False,
// zeros padding) from pgrid x pgrid to grid_h x grid_w ---
ggml_tensor * pe = ctx->model.position_embeddings; // [nemb, pgrid*pgrid]
std::vector<float> pe_host((size_t) nemb * pgrid * pgrid);
{
std::vector<char> raw(ggml_nbytes(pe));
ggml_backend_tensor_get(pe, raw.data(), 0, ggml_nbytes(pe));
if (pe->type == GGML_TYPE_F32) {
std::memcpy(pe_host.data(), raw.data(), ggml_nbytes(pe));
} else {
const auto * tt = ggml_get_type_traits(pe->type);
tt->to_float(raw.data(), pe_host.data(), (int64_t) nemb * pgrid * pgrid);
}
}
// Token layout is row-major with grid_w as the inner stride (matches the sparse
// window loop below). H axis samples pe rows scaled by grid_h; W axis samples
// pe cols scaled by grid_w. Self-cancelling on square grids.
std::vector<float> pos_emb((size_t) nemb * n_tok, 0.0f);
for (int t = 0; t < n_tok; t++) {
const int gy = t / grid_w; // height index (0..grid_h-1)
const int gx = t % grid_w; // width index (0..grid_w-1)
const float af = (gy + 0.5f) * pgrid / grid_h - 0.5f; // H axis
const float bf = (gx + 0.5f) * pgrid / grid_w - 0.5f; // W axis
const int a0 = (int) std::floor(af); const float wa = af - a0;
const int b0 = (int) std::floor(bf); const float wb = bf - b0;
const int as[2] = { a0, a0 + 1 }; const float was[2] = { 1.0f - wa, wa };
const int bs[2] = { b0, b0 + 1 }; const float wbs[2] = { 1.0f - wb, wb };
float * dst = pos_emb.data() + (size_t) t * nemb;
for (int ka = 0; ka < 2; ka++) {
if (as[ka] < 0 || as[ka] >= pgrid) continue;
for (int kb = 0; kb < 2; kb++) {
if (bs[kb] < 0 || bs[kb] >= pgrid) continue;
const float w = was[ka] * wbs[kb];
if (w == 0.0f) continue;
const float * src = pe_host.data() + (size_t) (as[ka] * pgrid + bs[kb]) * nemb;
for (int c = 0; c < nemb; c++) dst[c] += w * src[c];
}
}
}
set_input_f32("onyx_pos_emb", pos_emb);
// --- sparse window grouping (pgrid x pgrid windows) ---
const int win = pgrid;
const int nwin_h = (grid_h + win - 1) / win;
tools/mtmd/models/onyx.cpp
+1 -5
View File
@@ -9,7 +9,6 @@
// Several quantities are precomputed on host and fed as named graph inputs (filled in
// clip.cpp set_input, PROJECTOR_TYPE_ONYX branch):
// onyx_patches [patch_dim, n_tok] : patchified pixels ([pt,c,ps,ps] layout)
// onyx_pos_emb [n_embd, n_tok] : bilinear-interpolated learned pos-emb (orig order)
// onyx_pos_w/_h [n_tok] i32 : 1-indexed RoPE positions (sparse-permuted order)
// onyx_sp_perm [n_tok] i32 : window grouping permutation (applied after ln_pre)
// onyx_inv_perm [n_tok] i32 : inverse of sp_perm (applied after blocks)
@@ -34,9 +33,6 @@ ggml_cgraph * clip_graph_onyx::build() {
ggml_tensor * patches = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, patch_dim, n_tok);
ggml_set_name(patches, "onyx_patches"); ggml_set_input(patches);
ggml_tensor * pos_emb = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tok);
ggml_set_name(pos_emb, "onyx_pos_emb"); ggml_set_input(pos_emb);
ggml_tensor * pos_w = inp_i32("onyx_pos_w", n_tok);
ggml_tensor * pos_h = inp_i32("onyx_pos_h", n_tok);
ggml_tensor * sp_perm = inp_i32("onyx_sp_perm", n_tok);
@@ -48,7 +44,7 @@ ggml_cgraph * clip_graph_onyx::build() {
// patchify (conv1_linear as a matmul, no bias) + learned pos-emb
ggml_tensor * x = build_mm(model.patch_embeddings_0, patches); // [n_embd, n_tok]
x = ggml_add(ctx0, x, pos_emb);
x = ggml_add(ctx0, x, resize_position_embeddings(GGML_SCALE_MODE_BILINEAR));
cb(x, "after_posemb", -1);
// ln_pre (LayerNorm)