From c8e03ce8122b7af76f836d53efde6df1ce5ec437 Mon Sep 17 00:00:00 2001 From: Pascal Date: Thu, 6 Aug 2026 00:47:59 +0200 Subject: [PATCH] mtmd/ggml: add ggml_build_forward_order (#26649) * ggml: add ggml_build_forward_order ggml_build_forward_expand marks the tensor and all its ancestors for compute, so using it as a pure ordering hint (keeping q, k and v together) defeats ggml_build_forward_select: the unselected branch is forced to run with inputs that were never uploaded. In the mtmd audio graph this makes GEN_WAV calls execute the GEN_CODE branch with a stale inp_code0, hitting the get_rows bound assert on CPU. Add ggml_build_forward_order, which inserts nodes without the compute flag; the flag is restored when the branch is actually selected. Switch the q/k/v hints in clip_graph::build_attn to it. * nit: reduce comments (AGENTS.md) --- ggml/include/ggml.h | 6 ++++++ ggml/src/ggml.c | 4 ++++ tools/mtmd/clip.cpp | 7 ++++--- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 35f0c44ec4..5cb49d0ee4 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -2788,6 +2788,12 @@ extern "C" { struct ggml_cgraph * cgraph, struct ggml_tensor * tensor); + // add the tensor and its parents to the graph without marking them for compute + // the flag is set later, when the tensor is reached from a node that computes + GGML_API void ggml_build_forward_order( + struct ggml_cgraph * cgraph, + struct ggml_tensor * tensor); + GGML_API void ggml_build_backward_expand( struct ggml_context * ctx, // context for gradient computation struct ggml_cgraph * cgraph, diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 59191c663e..da7f3a5f2e 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -7200,6 +7200,10 @@ void ggml_build_forward_expand(struct ggml_cgraph * cgraph, struct ggml_tensor * ggml_build_forward_impl(cgraph, tensor, true, true); } +void ggml_build_forward_order(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor) { + ggml_build_forward_impl(cgraph, tensor, true, false); +} + void ggml_build_backward_expand( struct ggml_context * ctx, struct ggml_cgraph * cgraph, diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index c77e7cc9d3..b1360fd7d3 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -708,9 +708,10 @@ ggml_tensor * clip_graph::build_attn( ggml_tensor * sinks) const { // these nodes are added to the graph together so that they are not reordered // by doing so, the number of splits in the graph is reduced - ggml_build_forward_expand(gf, q_cur); - ggml_build_forward_expand(gf, k_cur); - ggml_build_forward_expand(gf, v_cur); + // the order is fixed without the compute flag, so an unselected branch stays out of the compute set + ggml_build_forward_order(gf, q_cur); + ggml_build_forward_order(gf, k_cur); + ggml_build_forward_order(gf, v_cur); ggml_tensor * q = ggml_permute(ctx0, q_cur, 0, 2, 1, 3); //cb(q, "q", il);