upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-02 16:10:48 -05:00
Code Issues Packages Projects Releases Wiki Activity

mtmd: dsocr-tiles fixes (#25481)

Browse Source 
* ds-ocr img-preproc fuse_row tile-drop fix for multi rows and columns images

* mtmd drop the duplicate redundant img_end

* deepseekocr graph simplify CLS broadcast cleanup
This commit is contained in:
Saba Fallah
2026-07-09 13:45:21 +02:00
committed by GitHub
parent 2e4cbade70
commit e5d8b4685c
3 changed files with 4 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tools/mtmd/models/deepseekocr.cpp
View File

@@ -320,9 +320,7 @@ ggml_cgraph * clip_graph_deepseekocr::build() {
// add CLS token per batch item
// inp: [n_embd, clip_n_patches, n_batch]
// class_embedding: [n_embd] -> [n_embd, 1, n_batch]
ggml_tensor * cls_embd = ggml_reshape_3d(ctx0, model.class_embedding, n_embd, 1, 1);
ggml_tensor * cls_target = ggml_new_tensor_3d(ctx0, model.class_embedding->type, n_embd, 1, n_batch);
cls_embd = ggml_repeat(ctx0, cls_embd, cls_target);
ggml_tensor * cls_embd = ggml_repeat_4d(ctx0, model.class_embedding, n_embd, 1, n_batch, 1);
inp = ggml_concat(ctx0, cls_embd, inp, 1);
// for selecting learned pos embd, used by ViT

4
tools/mtmd/mtmd-image.cpp
View File

@@ -1204,7 +1204,6 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const cli
}
}
output.append(hparams, row_img, true);
grid_w = 1; // reset grid_w to 1 since we fused the row
} else {
for (int col = 0; col < grid_w; col++) {
clip_image_u8 tile;
@@ -1213,6 +1212,9 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const cli
}
}
}
if (fuse_row) {
grid_w = 1; // each fused row is one image; a single output column
}
}
LOG_DBG("%s: grid size: %d x %d (%d tiles) + global view\n", __func__, grid_w, grid_h, grid_w * grid_h);

6
tools/mtmd/mtmd.cpp
View File

@@ -1170,12 +1170,6 @@ struct mtmd_tokenizer {
cur.entries.emplace_back(std::move(ov_chunk));
add_text(ctx->tok_ov_img_end);
}
if (!ctx->img_end.empty()) {
// add image end token (currently only used by deepseekocr)
add_text(ctx->img_end, true);
}
} else {
if (preproc_out.entries.size() == 0) {