upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-03 11:27:25 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into xsn/video_args

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-27 01:11:19 +02:00
parent 9eb4e9dbb7 925e117994
commit c619e22d0f
2038 changed files with 331788 additions and 90380 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/mtmd-cli.cpp
+116 -18
View File
@@ -32,9 +32,9 @@ static volatile bool g_is_generating = false;
static volatile bool g_is_interrupted = false;
/**
* Please note that this is NOT a production-ready stuff.
* Please note that this is NOT a production-ready binary.
* It is a playground for trying multimodal support in llama.cpp.
* For contributors: please keep this code simple and easy to understand.
* For contributors: please keep this code simple and easy to understand. Do not add unnecessary complexity. The goal is to have a simple CLI for testing multimodal support.
*/
static void show_additional_info(int /*argc*/, char ** argv) {
@@ -65,6 +65,14 @@ static void sigint_handler(int signo) {
}
#endif
// this is only used by tests.sh to capture the response ; it's not meant to be used in production
static void inject_test_response_marker() {
const char * env = std::getenv("MTMD_TEST_RESPONSE_MARKER");
if (env) {
LOG("%s\n", env);
}
}
struct mtmd_cli_context {
mtmd::context_ptr ctx_vision;
common_init_result_ptr llama_init;
@@ -79,6 +87,8 @@ struct mtmd_cli_context {
mtmd::bitmaps bitmaps;
std::vector<mtmd_helper::video_ptr> videos;
mtmd::batch_ptr mbatch;
// chat template
common_chat_templates_ptr tmpls;
std::vector<common_chat_msg> chat_history;
@@ -106,6 +116,14 @@ struct mtmd_cli_context {
exit(1);
}
init_vision_context(params);
if (!mtmd_helper_model_can_chat(lctx, ctx_vision.get())) {
LOG_ERR("Model does not support chat mode\n");
LOG_ERR("Hint: for TTS models, please use llama-tts\n");
exit(1);
}
if (!llama_model_chat_template(model, nullptr) && params.chat_template.empty()) {
LOG_ERR("Model does not have chat template.\n");
LOG_ERR(" For old llava models, you may need to use '--chat-template vicuna'\n");
@@ -119,8 +137,6 @@ struct mtmd_cli_context {
chat_history.clear();
LOG_INF("%s: chat template example:\n%s\n", __func__, common_chat_format_example(tmpls.get(), params.use_jinja, params.default_template_kwargs).c_str());
init_vision_context(params);
// load antiprompt tokens for legacy templates
if (params.chat_template == "vicuna") {
antiprompt_tokens = common_tokenize(lctx, "ASSISTANT:", false, true);
@@ -138,6 +154,7 @@ struct mtmd_cli_context {
const char * clip_path = params.mmproj.path.c_str();
mtmd_context_params mparams = mtmd_context_params_default();
mparams.use_gpu = params.mmproj_use_gpu;
mparams.device = params.mmproj_device;
mparams.print_timings = true;
mparams.n_threads = params.cpuparams.n_threads;
mparams.flash_attn_type = params.flash_attn_type;
@@ -242,12 +259,15 @@ static std::string chat_add_and_format(mtmd_cli_context & ctx, common_chat_msg &
}
static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) {
inject_test_response_marker();
bool add_bos = ctx.chat_history.empty();
auto formatted_chat = chat_add_and_format(ctx, msg);
LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.c_str());
mtmd_input_text text;
text.text = formatted_chat.c_str();
text.text = formatted_chat.data();
text.text_len = formatted_chat.size();
text.add_special = add_bos;
text.parse_special = true;
@@ -268,20 +288,95 @@ static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) {
ctx.bitmaps.entries.clear();
ctx.videos.clear();
llama_pos new_n_past;
if (mtmd_helper_eval_chunks(ctx.ctx_vision.get(),
ctx.lctx, // lctx
chunks.ptr.get(), // chunks
ctx.n_past, // n_past
0, // seq_id
ctx.n_batch, // n_batch
true, // logits_last
&new_n_past)) {
LOG_ERR("Unable to eval prompt\n");
return 1;
}
// batch encode all media chunks, then decode each
size_t n_chunks = mtmd_input_chunks_size(chunks.ptr.get());
for (size_t i = 0; i < n_chunks; i++) {
auto chunk = mtmd_input_chunks_get(chunks.ptr.get(), i);
auto chunk_type = mtmd_input_chunk_get_type(chunk);
ctx.n_past = new_n_past;
if (chunk_type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
// decode text chunk
llama_pos new_n_past = ctx.n_past;
res = mtmd_helper_eval_chunk_single(ctx.ctx_vision.get(),
ctx.lctx,
chunk,
ctx.n_past,
0, // seq_id
ctx.n_batch,
i == n_chunks - 1, // logits_last
&new_n_past);
if (res != 0) {
LOG_ERR("Unable to eval text chunk %zu\n", i);
return 1;
}
ctx.n_past = new_n_past;
} else {
// media chunk: try to get embd from existing batch, or create a new batch
float * embd = nullptr;
if (ctx.mbatch) {
embd = mtmd_batch_get_output_embd(ctx.mbatch.get(), chunk);
if (embd) {
LOG_DBG("found embd for media chunk %zu in existing batch\n", i);
} else {
LOG_DBG("media chunk %zu not found in existing batch, creating new batch\n", i);
}
}
if (!embd) {
// create and encode a new batch with as many media chunks as possible
ctx.mbatch.reset(mtmd_batch_init(ctx.ctx_vision.get()));
res = mtmd_batch_add_chunk(ctx.mbatch.get(), chunk);
GGML_ASSERT(res == 0); // first chunk must always succeed
int n_added = 1;
// add as many subsequent media chunks as possible
for (size_t j = i + 1; j < n_chunks; j++) {
auto next_chunk = mtmd_input_chunks_get(chunks.ptr.get(), j);
auto next_type = mtmd_input_chunk_get_type(next_chunk);
if (next_type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
break; // text chunk splits the batch
}
res = mtmd_batch_add_chunk(ctx.mbatch.get(), next_chunk);
if (res != 0) {
break; // batch full or incompatible
}
n_added++;
}
int64_t time_start = ggml_time_ms();
LOG_INF("encoding mtmd batch, n_chunks = %d (done = %zu, total = %zu)\n", n_added, i, n_chunks);
res = mtmd_batch_encode(ctx.mbatch.get());
if (res != 0) {
LOG_ERR("Failed to encode mtmd batch, res = %d\n", res);
return 1;
}
LOG_INF("mtmd batch encoding done in %d ms\n", (int)(ggml_time_ms() - time_start));
embd = mtmd_batch_get_output_embd(ctx.mbatch.get(), chunk);
}
GGML_ASSERT(embd != nullptr);
llama_pos new_n_past = ctx.n_past;
res = mtmd_helper_decode_image_chunk(ctx.ctx_vision.get(),
ctx.lctx,
chunk,
embd,
ctx.n_past,
0, // seq_id
ctx.n_batch,
&new_n_past,
nullptr, // callback
nullptr // user_data
);
if (res != 0) {
LOG_ERR("Unable to decode media chunk %zu\n", i);
return 1;
}
ctx.n_past = new_n_past;
}
}
LOG("\n");
@@ -318,6 +413,9 @@ int main(int argc, char ** argv) {
int n_predict = params.n_predict < 0 ? INT_MAX : params.n_predict;
console::init(params.simple_io, params.use_color);
atexit([]() { console::cleanup(); });
// Ctrl+C handling
{
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))