From eb95d125bad0b8d302717a7a55e61845da9e7883 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Wed, 26 Aug 2026 00:58:41 +0000 Subject: [PATCH] qwen4exp: hash the image placeholder for multimodal batches The PLE row indices are computed host-side from ubatch->token, and set_input returned early when that was null. A multimodal ubatch is exactly that case: the mtmd layer consumes the image placeholder ids and hands llama_decode embeddings instead. The early return left the I32 index tensor uninitialised, so ggml_get_rows indexed a 320 M row table with whatever the buffer happened to contain, and aborted: GGML_ASSERT(i01 >= 0 && i01 < ne01) failed ggml_compute_forward_get_rows mtmd_helper_decode_image_chunk -> llama_decode Every image request crashed. Nothing caught it because the vision work had only ever been verified by converting an mmproj, never by running one. The reference computes the hash over input_ids, where those positions still hold the image placeholder, so carry that id through as qwen4exp.ple.image_token_id and hash it. The key is optional: a file converted before it existed falls back to the PLE EOS token, which is defined and treats the image as a segment boundary rather than crashing. Verified end to end with llama-mtmd-cli, a Q4_K_M base and the F16 mmproj, on a generated image with known content. The model names the red circle, the blue square, the inverted green triangle and reads "UNSLOTH 42", each with the right position. --- conversion/qwen4exp.py | 20 ++++++++++++++++++++ gguf-py/gguf/constants.py | 1 + gguf-py/gguf/gguf_writer.py | 3 +++ src/llama-arch.cpp | 1 + src/llama-arch.h | 1 + src/llama-hparams.h | 3 +++ src/models/qwen4exp.cpp | 31 ++++++++++++++++++++++++------- 7 files changed, 53 insertions(+), 7 deletions(-) diff --git a/conversion/qwen4exp.py b/conversion/qwen4exp.py index 70c31222cd..2a67c77013 100644 --- a/conversion/qwen4exp.py +++ b/conversion/qwen4exp.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from typing import Iterable import torch @@ -81,6 +82,12 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): self.gguf_writer.add_ple_heads_per_ngram(hp["heads_per_ngram"]) self.gguf_writer.add_ple_conv_kernel(hp["ple_conv_kernel_size"]) self.gguf_writer.add_ple_eos_token_id(self._eos_token_id()) + # The PLE hash runs over token ids, but a multimodal batch arrives as embeddings + # with the placeholder consumed. Carry it so those positions hash what the + # reference sees in input_ids instead of being undefined. + _img = self._image_token_id() + if _img is not None: + self.gguf_writer.add_ple_image_token_id(int(_img)) if self._ple_row_dim is not None: self.gguf_writer.add_embedding_length_per_layer_input(self._ple_row_dim) @@ -91,6 +98,19 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): self.gguf_writer.add_ple_head_vocab_sizes( self._read_hash_constants("ple_embedding.ngram_heads_vocab_sizes")) + def _image_token_id(self) -> int | None: + # image_token_id is top-level in config.json, not in self.hparams once that is + # narrowed to text_config, and the text model has no global_config; read the file + img = self.hparams.get("image_token_id") + if img is not None: + return int(img) + try: + with open(self.dir_model / "config.json", "r", encoding="utf-8") as f: + img = json.load(f).get("image_token_id") + except Exception: + return None + return None if img is None else int(img) + def _eos_token_id(self) -> int: eos = self.hparams.get("eos_token_id") if isinstance(eos, list): diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 2caff0b0cc..60ddeb5540 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -237,6 +237,7 @@ class Keys: HEAD_OFFSETS = "{arch}.ple.head_offsets" HEAD_VOCAB_SIZES = "{arch}.ple.head_vocab_sizes" EOS_TOKEN_ID = "{arch}.ple.eos_token_id" + IMAGE_TOKEN_ID = "{arch}.ple.image_token_id" class Rope: DIMENSION_COUNT = "{arch}.rope.dimension_count" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 363e9f751b..32b8154de5 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1060,6 +1060,9 @@ class GGUFWriter: def add_ple_eos_token_id(self, value: int) -> None: self.add_uint32(Keys.PLE.EOS_TOKEN_ID.format(arch=self.arch), value) + def add_ple_image_token_id(self, value: int) -> None: + self.add_uint32(Keys.PLE.IMAGE_TOKEN_ID.format(arch=self.arch), value) + def add_attention_scale(self, value: float) -> None: self.add_float32(Keys.Attention.SCALE.format(arch=self.arch), value) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 870b66922d..3305f26837 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -304,6 +304,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_PLE_HEAD_OFFSETS, "%s.ple.head_offsets" }, { LLM_KV_PLE_HEAD_VOCAB_SIZES, "%s.ple.head_vocab_sizes" }, { LLM_KV_PLE_EOS_TOKEN_ID, "%s.ple.eos_token_id" }, + { LLM_KV_PLE_IMAGE_TOKEN_ID, "%s.ple.image_token_id" }, { LLM_KV_HASH_LAYER_COUNT, "%s.hash_layer_count" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index bcfbc3e99b..94cf08ae16 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -309,6 +309,7 @@ enum llm_kv { LLM_KV_PLE_HEAD_OFFSETS, LLM_KV_PLE_HEAD_VOCAB_SIZES, LLM_KV_PLE_EOS_TOKEN_ID, + LLM_KV_PLE_IMAGE_TOKEN_ID, LLM_KV_HASH_LAYER_COUNT, diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 053ad1641d..a3e30754b2 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -281,6 +281,9 @@ struct llama_hparams { uint32_t ple_n_heads = 0; // (ngram_size - 1) * heads_per_ngram uint32_t ple_head_dim = 0; uint32_t ple_eos_token_id = 0; + // placeholder the PLE hash sees where an image chunk is spliced in; 0 means the + // file predates this key and the loader falls back to EOS + uint32_t ple_image_token_id = 0; std::array is_ple_impl; std::array ple_layer_multipliers; std::array ple_head_offsets; diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 0a009329dc..686bae2d05 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -45,6 +45,9 @@ void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_PLE_HEADS_PER_NGRAM, hparams.ple_heads_per_ngram); ml.get_key(LLM_KV_PLE_CONV_KERNEL, hparams.ple_conv_kernel); ml.get_key(LLM_KV_PLE_EOS_TOKEN_ID, hparams.ple_eos_token_id); + // optional: absent in files converted before multimodal batches + // were exercised, in which case the PLE hash falls back to EOS + ml.get_key(LLM_KV_PLE_IMAGE_TOKEN_ID, hparams.ple_image_token_id, false); ml.get_key(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.n_embd_per_layer); hparams.ple_n_heads = (hparams.ple_ngram_size - 1) * hparams.ple_heads_per_ngram; @@ -803,12 +806,26 @@ public: }; void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) { - if (!ubatch->token) { - return; - } - const auto & hp = pmodel.hparams; + // A multimodal ubatch arrives as embeddings: the mtmd layer has already + // consumed the image placeholder ids, so ubatch->token is null. The hash + // still has to produce a row for every position, because this input feeds + // ggml_get_rows. Returning early here left the index buffer uninitialised, + // so whatever happened to be in it indexed a 320 M row table -- an + // out-of-range gather, which aborts inside ggml_compute_forward_get_rows. + // + // The reference hashes input_ids, where those positions still hold the + // image placeholder, so use that token here. A file converted before the + // key existed falls back to EOS, which is defined and simply treats the + // image as a segment boundary. + const llama_token img_tok = hp.ple_image_token_id != 0 + ? (llama_token) hp.ple_image_token_id + : (llama_token) hp.ple_eos_token_id; + auto tok_of = [&](int64_t k) -> llama_token { + return ubatch->token ? ubatch->token[k] : img_tok; + }; + const int64_t n_tokens = ubatch->n_tokens; const int64_t n_gram = hp.ple_ngram_size; const int64_t n_heads = hp.ple_n_heads; @@ -848,7 +865,7 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) { auto prev = [&](int64_t s) -> int64_t { const int64_t j = i - s; if (j >= 0 && ubatch->seq_id[j][0] == seq && ubatch->pos[j] == pos - s) { - return ubatch->token[j]; + return tok_of(j); } // s - i positions before this ubatch started, most recent last const int64_t back = s - i; @@ -864,7 +881,7 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) { // takes the last EOS strictly *before* this position, so a segment // boundary only hides tokens from the positions that follow it. std::vector ctx(n_gram); - ctx[0] = ubatch->token[i]; + ctx[0] = tok_of(i); bool cut = false; for (int64_t s = 1; s < n_gram; ++s) { ctx[s] = cut ? eos : prev(s); @@ -887,7 +904,7 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) { } auto & h = hist_map[seq]; - h.toks.push_back(ubatch->token[i]); + h.toks.push_back(tok_of(i)); if ((int64_t) h.toks.size() > n_gram - 1) { h.toks.erase(h.toks.begin(), h.toks.end() - (n_gram - 1)); }