upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-23 11:10:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

mtmd: add batching API (#24384)

Browse Source 
* mtmd: add batching API

* wip

* first working version (gemma4v)

* add arg

* nits

* wire up support_batch()

* fix 0.0 output embd

* fix audio

* nits

* refactor a bit

* nits

* fix non-batching case

* fix comment
This commit is contained in:
Xuan-Son Nguyen
2026-06-13 00:10:29 +02:00
committed by GitHub
parent f58bad4137
commit e37abd6b5f
14 changed files with 544 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tools/server/server-common.cpp
View File

@@ -344,6 +344,14 @@ const mtmd::input_chunk_ptr & server_tokens::find_chunk(size_t idx) const {
throw std::runtime_error("Chunk not found");
}
std::pair<const mtmd::input_chunk_ptr *, size_t> server_tokens::find_next_media_chunk(size_t idx) const {
auto it = map_idx_to_media.upper_bound(idx);
if (it != map_idx_to_media.end()) {
return { &it->second, it->first };
}
return { nullptr, 0 };
}
void server_tokens::push_back(llama_token tok) {
if (tok == LLAMA_TOKEN_NULL) {
throw std::runtime_error("Invalid token");