chat : fix Muse Spark ignoring response_format json_schema (#32017)

The hidden desktop strip relied on inherited `visibility: hidden` to keep
its content from catching clicks. Since the sidebar's sections fold
(#31936), an open section's CollapsibleRegion sets `visibility: visible`
again, and its rows' context-menu triggers carry an inline
`pointer-events: auto`, so they caught clicks meant for the page.

Make the hidden strip's content inert, which no descendant can undo, and
add a client test that hit-tests the hidden sidebar's area.

Fixes #31971

Assisted-by: Claude (Opus 5.5 Medium)

.github/workflows/python-type-check.yml

@@ -35,7 +35,7 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: "3.11"
-          pip-install: -r requirements/requirements-all.txt ty==0.0.88
+          pip-install: -r requirements/requirements-all.txt ty==0.0.100
       # - name: Type-check with Pyright
       #   uses: jakebailey/pyright-action@v2
       #   with:

README.md

@@ -1,6 +1,6 @@
 # llama.cpp
 
-![llama](https://raw.githubusercontent.com/ggml-org/llama.brand/refs/heads/master/cover/llama-cpp/cover-llama-cpp-dark.svg)
+![llama.cpp](https://raw.githubusercontent.com/ggml-org/llama.brand/refs/heads/master/logo/svg/cover/llama-cpp/cover-llama-cpp-dark.svg)
 
 <div align="center">
 

common/parsers/muse-spark.cpp

@@ -46,9 +46,10 @@ common_chat_params common_chat_params_init_muse_spark(const common_chat_template
 
     auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
 
-    auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
-    // Constrained grammar whenever tools are offered.
-    auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
+    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
+    auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
+    // Constrained grammar whenever tools are offered or a response format is requested.
+    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
 
     data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
         auto start = p.rule("start", p.literal("<|start|>assistant"));
@@ -75,6 +76,15 @@ common_chat_params common_chat_params_init_muse_spark(const common_chat_template
         auto final_msg  = p.rule("final", recipient + p.literal("<|message|>") +
                                               p.content(p.until_one_of({ "<|eot|>", "<|eom|>" })));
 
+        if (has_response_format) {
+            auto response_json   = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema));
+            auto response_format = p.rule("response-format",
+                recipient + p.literal("<|message|>") +
+                ((p.literal("```json") + p.space() + response_json + p.space() + p.literal("```")) | response_json));
+
+            return p.zero_or_more(start + analysis) + start + response_format;
+        }
+
         if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
             auto string_value = p.ac(
                 p.tool_arg_string_value(p.until("</atem:parameter>")) + p.tool_arg_close(p.literal("</atem:parameter>")),
@@ -132,7 +142,7 @@ common_chat_params common_chat_params_init_muse_spark(const common_chat_template
     });
 
     if (include_grammar) {
-        data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
+        data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
         data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
             data.parser.build_grammar(builder, data.grammar_lazy);
         });

conversion/base.py

@@ -32,10 +32,10 @@ import gguf
 from gguf.vocab import MistralTokenizerType, MistralVocab
 
 try:
-    from mistral_common.tokens.tokenizers.base import TokenizerVersion # type: ignore[import-not-found, ty:unresolved-import]
-    from mistral_common.tokens.tokenizers.multimodal import DATASET_MEAN as _MISTRAL_COMMON_DATASET_MEAN, DATASET_STD as _MISTRAL_COMMON_DATASET_STD # type: ignore[import-not-found, ty:unresolved-import]
-    from mistral_common.tokens.tokenizers.tekken import Tekkenizer # type: ignore[import-not-found, ty:unresolved-import]
-    from mistral_common.tokens.tokenizers.sentencepiece import ( # type: ignore[import-not-found, ty:unresolved-import]
+    from mistral_common.tokens.tokenizers.base import TokenizerVersion # ty: ignore[import-not-found]
+    from mistral_common.tokens.tokenizers.multimodal import DATASET_MEAN as _MISTRAL_COMMON_DATASET_MEAN, DATASET_STD as _MISTRAL_COMMON_DATASET_STD # ty: ignore[import-not-found]
+    from mistral_common.tokens.tokenizers.tekken import Tekkenizer # ty: ignore[import-not-found]
+    from mistral_common.tokens.tokenizers.sentencepiece import ( # ty: ignore[import-not-found]
         SentencePieceTokenizer,
     )
 
@@ -273,7 +273,7 @@ class ModelBase:
                     if weight_map is None or not isinstance(weight_map, dict):
                         raise ValueError(f"Can't load 'weight_map' from {index_name!r}")
                     tensor_names_from_index.update(weight_map.keys())
-                    part_dict: dict[str, None] = dict.fromkeys(weight_map.values(), None) # ty: ignore[invalid-assignment]
+                    part_dict: dict[str, None] = dict.fromkeys(weight_map.values(), None)
                     part_names = sorted(part_dict.keys())
             else:
                 weight_map = {}
@@ -2055,15 +2055,16 @@ class TextModel(ModelBase):
 
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
-        vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab))  # ty: ignore[unresolved-attribute]
-        assert max(tokenizer.vocab.values()) < vocab_size  # ty: ignore[unresolved-attribute]
+        assert tokenizer is not None
+        vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab))
+        assert max(tokenizer.vocab.values()) < vocab_size
 
         tokpre = self.get_vocab_base_pre(tokenizer)
 
-        reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}  # ty: ignore[unresolved-attribute]
-        added_vocab = tokenizer.get_added_vocab()  # ty: ignore[unresolved-attribute]
+        reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
+        added_vocab = tokenizer.get_added_vocab()
 
-        added_tokens_decoder = tokenizer.added_tokens_decoder  # ty: ignore[unresolved-attribute]
+        added_tokens_decoder = tokenizer.added_tokens_decoder
 
         for i in range(vocab_size):
             if i not in reverse_vocab:
@@ -2076,7 +2077,7 @@ class TextModel(ModelBase):
                     # To avoid unexpected issues - we make sure to normalize non-normalized tokens
                     if not added_tokens_decoder[i].normalized:
                         previous_token = token
-                        token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))  # ty: ignore[unresolved-attribute, invalid-assignment]
+                        token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))  # ty: ignore[invalid-assignment]
                         if previous_token != token:
                             logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")
 
@@ -2486,6 +2487,7 @@ class TextModel(ModelBase):
     def _set_vocab_hybriddna(self):
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+        assert tokenizer is not None
         vocab = tokenizer.get_vocab()
         vocab_size = self.hparams.get("vocab_size", len(vocab))
         assert max(vocab.values()) < vocab_size
@@ -2499,11 +2501,11 @@ class TextModel(ModelBase):
         if tokenizer.k != 6 or tail not in ("pad_a", "pad_n"):
             raise NotImplementedError(f"HybridDNATokenizer with k={tokenizer.k}, tail={tail!r}")
         for kmer in [*tokenizer.kmers, *getattr(tokenizer, "npad_tokens", [])]:
-            reverse_vocab[tokenizer.dna_token_to_id[kmer]] = kmer + "\ue000"  # ty: ignore[unresolved-attribute]
+            reverse_vocab[tokenizer.dna_token_to_id[kmer]] = kmer + "\ue000"
         # older HybridDNATokenizer versions (Carbon-500M/3B/8B) do not register the base tokenizer's added tokens (<|im_start|>, <think>, ...) as their own
         # but they are still encoded as special tokens: take their types from the base tokenizer
         base_tokenizer = getattr(tokenizer, "_base_tokenizer", tokenizer)
-        added_tokens_decoder = {**base_tokenizer.added_tokens_decoder, **tokenizer.added_tokens_decoder}  # ty: ignore[unresolved-attribute]
+        added_tokens_decoder = {**base_tokenizer.added_tokens_decoder, **tokenizer.added_tokens_decoder}
 
         tokens: list[str] = []
         toktypes: list[int] = []
@@ -2550,24 +2552,25 @@ class TextModel(ModelBase):
 
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
+        assert tokenizer is not None
         vocab_size = hparams["vocab_size"]
-        assert max(tokenizer.get_vocab().values()) < vocab_size  # ty: ignore[unresolved-attribute]
+        assert max(tokenizer.get_vocab().values()) < vocab_size
 
         tokpre = self.get_vocab_base_pre(tokenizer)
 
         merges = []
         vocab = {}
-        mergeable_ranks = tokenizer.mergeable_ranks  # ty: ignore[unresolved-attribute]
+        mergeable_ranks = tokenizer.mergeable_ranks
         for token, rank in mergeable_ranks.items():
             vocab[QwenModel.token_bytes_to_string(token)] = rank
             if len(token) == 1:
                 continue
             merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)
             assert len(merged) == 2
-            merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))
+            merges.append(str(' '.join(map(QwenModel.token_bytes_to_string, merged))))
 
         # for this kind of tokenizer, added_vocab is not a subset of vocab, so they need to be combined
-        added_vocab = tokenizer.special_tokens  # ty: ignore[unresolved-attribute]
+        added_vocab = tokenizer.special_tokens
         reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **added_vocab}.items()}
 
         for i in range(vocab_size):
@@ -2590,10 +2593,10 @@ class TextModel(ModelBase):
         special_vocab.merges = merges
         # only add special tokens when they were not already loaded from config.json
         if len(special_vocab.special_token_ids) == 0:
-            special_vocab._set_special_token("bos", tokenizer.special_tokens["<|endoftext|>"])  # ty: ignore[unresolved-attribute]
-            special_vocab._set_special_token("eos", tokenizer.special_tokens["<|endoftext|>"])  # ty: ignore[unresolved-attribute]
+            special_vocab._set_special_token("bos", tokenizer.special_tokens["<|endoftext|>"])
+            special_vocab._set_special_token("eos", tokenizer.special_tokens["<|endoftext|>"])
         # this one is usually not in config.json anyway
-        special_vocab._set_special_token("unk", tokenizer.special_tokens["<|endoftext|>"])  # ty: ignore[unresolved-attribute]
+        special_vocab._set_special_token("unk", tokenizer.special_tokens["<|endoftext|>"])
         special_vocab.add_to_gguf(self.gguf_writer)
 
     def _set_vocab_sentencepiece(self, add_to_gguf=True):
@@ -2867,21 +2870,23 @@ class TextModel(ModelBase):
     def _set_vocab_glmedge(self):
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+        assert tokenizer is not None
         special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
         tokens, toktypes, tokpre = self.get_vocab_base()
         self.gguf_writer.add_tokenizer_model("gpt2")
         self.gguf_writer.add_tokenizer_pre(tokpre)
         self.gguf_writer.add_token_list(tokens)
         self.gguf_writer.add_token_types(toktypes)
-        special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])  # ty: ignore[unresolved-attribute]
-        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])  # ty: ignore[unresolved-attribute]
-        special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])  # ty: ignore[unresolved-attribute]
-        special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"])  # ty: ignore[unresolved-attribute]
+        special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])
+        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])
+        special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])
+        special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"])
         special_vocab.add_to_gguf(self.gguf_writer)
 
     def _set_vocab_glm(self):
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+        assert tokenizer is not None
         special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
         tokens, toktypes, tokpre = self.get_vocab_base()
         self.gguf_writer.add_tokenizer_model("gpt2")
@@ -2890,10 +2895,10 @@ class TextModel(ModelBase):
         self.gguf_writer.add_token_types(toktypes)
         # Special tokens
         # Note: Using <|endoftext|> (151329) for eot causes endless generation
-        special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["[gMASK]"])  # ty: ignore[unresolved-attribute]  # 151331
-        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])  # ty: ignore[unresolved-attribute]  # 151336
-        special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])  # ty: ignore[unresolved-attribute]  # 151329
-        special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"])  # ty: ignore[unresolved-attribute]  # 151338
+        special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["[gMASK]"])  # 151331
+        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])  # 151336
+        special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])  # 151329
+        special_vocab._set_special_token("eom", tokenizer.get_added_vocab()["<|observation|>"])  # 151338
         special_vocab.add_to_gguf(self.gguf_writer)
 
     def _set_vocab_interns1(self):
@@ -2902,16 +2907,17 @@ class TextModel(ModelBase):
 
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
-        vocab = getattr(tokenizer, 'vocab', tokenizer.get_vocab())  # ty: ignore[unresolved-attribute]
+        assert tokenizer is not None
+        vocab = getattr(tokenizer, 'vocab', tokenizer.get_vocab())
         vocab_size = self.hparams.get("vocab_size", len(vocab))
         assert max(vocab.values()) < vocab_size
 
         tokpre = self.get_vocab_base_pre(tokenizer)
 
         reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab.items()}
-        added_vocab = tokenizer.get_added_vocab()  # ty: ignore[unresolved-attribute]
+        added_vocab = tokenizer.get_added_vocab()
 
-        added_tokens_decoder = tokenizer.added_tokens_decoder  # ty: ignore[unresolved-attribute]
+        added_tokens_decoder = tokenizer.added_tokens_decoder
 
         for i in range(vocab_size):
             if i not in reverse_vocab:
@@ -2924,7 +2930,7 @@ class TextModel(ModelBase):
                     # To avoid unexpected issues - we make sure to normalize non-normalized tokens
                     if not added_tokens_decoder[i].normalized:
                         previous_token = token
-                        token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))  # ty: ignore[unresolved-attribute, invalid-assignment]
+                        token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))  # ty: ignore[invalid-assignment]
                         if previous_token != token:
                             logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")
 

conversion/beam.py

@@ -60,7 +60,8 @@ class BeamModel(TextModel):
         # otherwise EOT is guessed from a hash-ordered scan
         from transformers import AutoTokenizer
         tok = AutoTokenizer.from_pretrained(self.dir_model)
-        ret_id = tok.convert_tokens_to_ids("<|return|>")  # ty: ignore[unresolved-attribute]
+        assert tok is not None
+        ret_id = tok.convert_tokens_to_ids("<|return|>")
         if isinstance(ret_id, int) and ret_id >= 0:
             self.gguf_writer.add_eot_token_id(ret_id)
 

conversion/bert.py

@@ -132,6 +132,7 @@ class BertModel(TextModel):
             from base64 import b64decode
             from transformers import AutoTokenizer
             tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+            assert tokenizer is not None
 
             with open(tokenizer_path, "r", encoding="utf-8") as fp:
                 tokenizer_json = json.load(fp)
@@ -140,13 +141,13 @@ class BertModel(TextModel):
                 with open(tokenizer_config_path, "r", encoding="utf-8") as fp:
                     tokenizer_config_json = json.load(fp)
 
-            add_prefix = tokenizer.add_prefix_space  # ty: ignore[unresolved-attribute]
-            remove_whitespaces = tokenizer.clean_up_tokenization_spaces  # ty: ignore[unresolved-attribute]
+            add_prefix = tokenizer.add_prefix_space
+            remove_whitespaces = tokenizer.clean_up_tokenization_spaces
             precompiled_charsmap = b64decode(tokenizer_json["normalizer"]["precompiled_charsmap"])
 
-            vocab_size = max(self.hparams.get("vocab_size", 0), tokenizer.vocab_size)  # ty: ignore[unresolved-attribute]
+            vocab_size = max(self.hparams.get("vocab_size", 0), tokenizer.vocab_size)
         else:
-            sentencepiece_model = model.ModelProto()  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+            sentencepiece_model = model.ModelProto()  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
             sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
             assert sentencepiece_model.trainer_spec.model_type == 1  # UNIGRAM
 
@@ -183,20 +184,20 @@ class BertModel(TextModel):
                 scores[token_id] = score
                 toktypes[token_id] = toktype
         else:
-            added_vocab = tokenizer.get_added_vocab()  # ty: ignore[unresolved-attribute]
+            added_vocab = tokenizer.get_added_vocab()
             unk_token = tokenizer_config_json.get("unk_token")
-            unk_token_id = added_vocab.get(unk_token, tokenizer_json["model"].get("unk_id", 3))  # ty: ignore[no-matching-overload]
+            unk_token_id = added_vocab.get(unk_token, tokenizer_json["model"].get("unk_id", 3))
 
-            for token_id in range(tokenizer.vocab_size):  # ty: ignore[unresolved-attribute]
-                piece = tokenizer._convert_id_to_token(token_id)  # ty: ignore[unresolved-attribute]
-                if (piece := tokenizer._convert_id_to_token(token_id)) is not None:  # ty: ignore[unresolved-attribute]
+            for token_id in range(tokenizer.vocab_size):
+                piece = tokenizer._convert_id_to_token(token_id)
+                if (piece := tokenizer._convert_id_to_token(token_id)) is not None:
                     text = piece.encode("utf-8")
                     score = tokenizer_json["model"]["vocab"][token_id][1]
 
                     toktype = SentencePieceTokenTypes.NORMAL
                     if token_id == unk_token_id:
                         toktype = SentencePieceTokenTypes.UNKNOWN
-                    elif token_id in tokenizer.all_special_ids:  # ty: ignore[unresolved-attribute]
+                    elif token_id in tokenizer.all_special_ids:
                         toktype = SentencePieceTokenTypes.CONTROL
                     elif token_id in added_vocab.values():
                         toktype = SentencePieceTokenTypes.USER_DEFINED

conversion/chatglm.py

@@ -22,12 +22,13 @@ class ChatGLMModel(TextModel):
 
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
-        vocab_size = hparams.get("padded_vocab_size", len(tokenizer.get_vocab()))  # ty: ignore[unresolved-attribute]
-        assert max(tokenizer.get_vocab().values()) < vocab_size  # ty: ignore[unresolved-attribute]
+        assert tokenizer is not None
+        vocab_size = hparams.get("padded_vocab_size", len(tokenizer.get_vocab()))
+        assert max(tokenizer.get_vocab().values()) < vocab_size
         role_special_tokens = ["<|system|>", "<|user|>", "<|assistant|>", "<|observation|>"]
         special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "sop", "eop"] + role_special_tokens
         for token_id in range(vocab_size):
-            piece = tokenizer._convert_id_to_token(token_id)  # ty: ignore[unresolved-attribute]
+            piece = tokenizer._convert_id_to_token(token_id)
             if token_id == 0:
                 piece = "<unk>"
             elif token_id == 1:
@@ -35,17 +36,18 @@ class ChatGLMModel(TextModel):
             elif token_id == 2:
                 piece = "<eos>"
 
-            text = piece.encode("utf-8")  # ty: ignore[unresolved-attribute]
+            assert isinstance(piece, str)
+            text = piece.encode("utf-8")
             score = 0.0
             # Referencing the tokenizer Python implementation(https://huggingface.co/THUDM/chatglm3-6b/blob/main/tokenization_chatglm.py),
             # it is only valid if it is less than tokenizer.tokenizer.sp_model.vocab_size()
-            if len(piece) != 0 and token_id < tokenizer.tokenizer.sp_model.vocab_size():  # ty: ignore[unresolved-attribute, invalid-argument-type]
-                score = tokenizer.tokenizer.sp_model.get_score(token_id)  # ty: ignore[unresolved-attribute]
+            if len(piece) != 0 and token_id < tokenizer.tokenizer.sp_model.vocab_size():
+                score = tokenizer.tokenizer.sp_model.get_score(token_id)
 
-            if token_id >= tokenizer.tokenizer.sp_model.vocab_size():  # ty: ignore[unresolved-attribute]
+            if token_id >= tokenizer.tokenizer.sp_model.vocab_size():
                 if piece in special_tokens:
                     toktype = SentencePieceTokenTypes.CONTROL
-                elif len(piece) == 0:  # ty: ignore[invalid-argument-type]
+                elif len(piece) == 0:
                     text = f"[PAD{token_id}]".encode("utf-8")
                     toktype = SentencePieceTokenTypes.UNUSED
                 else:
@@ -56,13 +58,13 @@ class ChatGLMModel(TextModel):
                 continue
 
             toktype = SentencePieceTokenTypes.NORMAL
-            if tokenizer.tokenizer.sp_model.is_unknown(token_id):  # ty: ignore[unresolved-attribute]
+            if tokenizer.tokenizer.sp_model.is_unknown(token_id):
                 toktype = SentencePieceTokenTypes.UNKNOWN
-            elif tokenizer.tokenizer.sp_model.is_control(token_id):  # ty: ignore[unresolved-attribute]
+            elif tokenizer.tokenizer.sp_model.is_control(token_id):
                 toktype = SentencePieceTokenTypes.CONTROL
-            elif tokenizer.tokenizer.sp_model.is_unused(token_id):  # ty: ignore[unresolved-attribute]
+            elif tokenizer.tokenizer.sp_model.is_unused(token_id):
                 toktype = SentencePieceTokenTypes.UNUSED
-            elif tokenizer.tokenizer.sp_model.is_byte(token_id):  # ty: ignore[unresolved-attribute]
+            elif tokenizer.tokenizer.sp_model.is_byte(token_id):
                 toktype = SentencePieceTokenTypes.BYTE
 
             tokens.append(text)
@@ -115,8 +117,9 @@ class ChatGLMModel(TextModel):
 
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)
+        assert tokenizer is not None
         vocab_size = hparams.get("padded_vocab_size",hparams["vocab_size"])
-        assert max(tokenizer.get_vocab().values()) < vocab_size  # ty: ignore[unresolved-attribute]
+        assert max(tokenizer.get_vocab().values()) < vocab_size
 
         tokens, toktypes, tokpre = self.get_vocab_base()
         self.gguf_writer.add_tokenizer_model("gpt2")
@@ -125,10 +128,10 @@ class ChatGLMModel(TextModel):
         self.gguf_writer.add_token_types(toktypes)
         special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
         # only add special tokens when they were not already loaded from config.json
-        special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])  # ty: ignore[unresolved-attribute]
-        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])  # ty: ignore[unresolved-attribute]
+        special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])
+        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])
         # this one is usually not in config.json anyway
-        special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])  # ty: ignore[unresolved-attribute]
+        special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])
         special_vocab.add_to_gguf(self.gguf_writer)
 
     def set_gguf_parameters(self):

conversion/deepseek.py

@@ -277,13 +277,14 @@ class DeepseekV2Model(TextModel):
 
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+        assert tokenizer is not None
         tokpre = self.get_vocab_base_pre(tokenizer)
 
         if tokpre == "kimi-k2":
             # Build merges list using the approach similar to HunYuanMoE
             merges = []
             vocab = {}
-            mergeable_ranks = tokenizer.model._mergeable_ranks  # ty: ignore[unresolved-attribute]
+            mergeable_ranks = tokenizer.model._mergeable_ranks  # ty: ignore[missing-union-attribute]
             for token, rank in mergeable_ranks.items():
                 vocab[QwenModel.token_bytes_to_string(token)] = rank
                 if len(token) == 1:
@@ -294,7 +295,7 @@ class DeepseekV2Model(TextModel):
 
             # Build token list
             vocab_size = self.hparams["vocab_size"]
-            special_tokens = tokenizer.special_tokens  # ty: ignore[unresolved-attribute]
+            special_tokens = tokenizer.special_tokens
             reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
             tokens: list[str] = []
             toktypes: list[int] = []

conversion/deepseek41.py

@@ -62,6 +62,7 @@ def engram_compressed_token_map(tokenizer_dir: Path) -> tuple[list[int], int]:
     from transformers import AutoTokenizer
 
     tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir)
+    assert tokenizer is not None
     sentinel = "\ue000"
     normalizer = normalizers.Sequence([
         normalizers.NFKC(),
@@ -80,6 +81,7 @@ def engram_compressed_token_map(tokenizer_dir: Path) -> tuple[list[int], int]:
         text = backend.decode([token_id], skip_special_tokens=False)
         if "\ufffd" in text:
             key = backend.id_to_token(token_id)
+            assert key is not None
         else:
             normalized = normalizer.normalize_str(text)
             key = normalized if normalized else text

conversion/dots3.py

@@ -75,13 +75,14 @@ class Dots3NoteModel(DeepseekV2Model):
     def set_vocab(self):
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+        assert tokenizer is not None
         special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
         tokens, toktypes, tokpre = self.get_vocab_base()
         self.gguf_writer.add_tokenizer_model("gpt2")
         self.gguf_writer.add_tokenizer_pre(tokpre)
         self.gguf_writer.add_token_list(tokens)
         self.gguf_writer.add_token_types(toktypes)
-        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endofassistant|>"])  # ty: ignore[unresolved-attribute]
+        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endofassistant|>"])
         special_vocab.add_to_gguf(self.gguf_writer)
 
     @classmethod

conversion/dream.py

@@ -19,15 +19,16 @@ class DreamModel(TextModel):
 
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+        assert tokenizer is not None
 
-        vocab_dict = tokenizer.get_vocab()  # ty: ignore[unresolved-attribute]
+        vocab_dict = tokenizer.get_vocab()
         vocab_size = self.hparams.get("vocab_size", len(vocab_dict))
         assert max(vocab_dict.values()) < vocab_size
 
         tokpre = self.get_vocab_base_pre(tokenizer)
 
         reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab_dict.items()}
-        added_vocab = tokenizer.get_added_vocab()  # ty: ignore[unresolved-attribute]
+        added_vocab = tokenizer.get_added_vocab()
 
         for i in range(vocab_size):
             if i not in reverse_vocab:

conversion/glm.py

@@ -31,6 +31,7 @@ class Glm4Model(TextModel):
     def set_vocab(self):
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+        assert tokenizer is not None
         special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
         tokens, toktypes, tokpre = self.get_vocab_base()
         self.gguf_writer.add_tokenizer_model("gpt2")
@@ -38,10 +39,10 @@ class Glm4Model(TextModel):
         self.gguf_writer.add_token_list(tokens)
         self.gguf_writer.add_token_types(toktypes)
         special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
-        special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])  # ty: ignore[unresolved-attribute]
-        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])  # ty: ignore[unresolved-attribute]
-        special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])  # ty: ignore[unresolved-attribute]
-        special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"])  # ty: ignore[unresolved-attribute]
+        special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])
+        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|user|>"])
+        special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<|endoftext|>"])
+        special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|endoftext|>"])
         special_vocab.add_to_gguf(self.gguf_writer)
 
     def set_gguf_parameters(self):
@@ -391,16 +392,17 @@ class SolarOpenModel(Glm4MoeModel):
     def set_vocab(self):
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+        assert tokenizer is not None
         special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
         tokens, toktypes, tokpre = self.get_vocab_base()
         self.gguf_writer.add_tokenizer_model("gpt2")
         self.gguf_writer.add_tokenizer_pre(tokpre)
         self.gguf_writer.add_token_list(tokens)
         self.gguf_writer.add_token_types(toktypes)
-        special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])  # ty: ignore[unresolved-attribute]
-        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endoftext|>"])  # ty: ignore[unresolved-attribute]
-        special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<unk>"])  # ty: ignore[unresolved-attribute]
-        special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|startoftext|>"])  # ty: ignore[unresolved-attribute]
+        special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["<|endoftext|>"])
+        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["<|endoftext|>"])
+        special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<unk>"])
+        special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["<|startoftext|>"])
         special_vocab.add_to_gguf(self.gguf_writer)
 
 

conversion/hunyuan.py

@@ -24,6 +24,7 @@ class HunYuanMoEModel(TextModel):
     def set_vocab(self):
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+        assert tokenizer is not None
 
         # 1. Get the pre-tokenizer identifier hash
         tokpre = self.get_vocab_base_pre(tokenizer)
@@ -31,7 +32,7 @@ class HunYuanMoEModel(TextModel):
         # 2. Reverse-engineer the merges list from mergeable_ranks
         merges = []
         vocab = {}
-        mergeable_ranks = tokenizer.mergeable_ranks  # ty: ignore[unresolved-attribute]
+        mergeable_ranks = tokenizer.mergeable_ranks
         for token, rank in mergeable_ranks.items():
             vocab[QwenModel.token_bytes_to_string(token)] = rank
             if len(token) == 1:
@@ -42,8 +43,8 @@ class HunYuanMoEModel(TextModel):
 
         # 3. Generate the tokens and toktypes lists
         vocab_size = self.hparams["vocab_size"]
-        assert tokenizer.vocab_size == vocab_size  # ty: ignore[unresolved-attribute]
-        special_tokens = tokenizer.special_tokens  # ty: ignore[unresolved-attribute]
+        assert tokenizer.vocab_size == vocab_size
+        special_tokens = tokenizer.special_tokens
         reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
         tokens: list[str] = []
         toktypes: list[int] = []
@@ -184,6 +185,7 @@ class HunYuanModel(TextModel):
         else:
             from transformers import AutoTokenizer
             tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+            assert tokenizer is not None
 
             # 1. Get the pre-tokenizer identifier hash
             tokpre = self.get_vocab_base_pre(tokenizer)
@@ -191,7 +193,7 @@ class HunYuanModel(TextModel):
             # 2. Reverse-engineer the merges list from mergeable_ranks
             merges = []
             vocab = {}
-            mergeable_ranks = tokenizer.mergeable_ranks  # ty: ignore[unresolved-attribute]
+            mergeable_ranks = tokenizer.mergeable_ranks
             for token, rank in mergeable_ranks.items():
                 vocab[QwenModel.token_bytes_to_string(token)] = rank
                 if len(token) == 1:
@@ -202,8 +204,8 @@ class HunYuanModel(TextModel):
 
             # 3. Generate the tokens and toktypes lists
             vocab_size = self.hparams["vocab_size"]
-            assert tokenizer.vocab_size == vocab_size  # ty: ignore[unresolved-attribute]
-            special_tokens = tokenizer.special_tokens  # ty: ignore[unresolved-attribute]
+            assert tokenizer.vocab_size == vocab_size
+            special_tokens = tokenizer.special_tokens
             reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
             tokens: list[str] = []
             toktypes: list[int] = []

conversion/internlm.py

@@ -36,7 +36,7 @@ class InternLM2Model(TextModel):
             logger.error(f'Error: Missing {tokenizer_path}')
             sys.exit(1)
 
-        sentencepiece_model = model.ModelProto()  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+        sentencepiece_model = model.ModelProto()  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
         sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
         add_prefix = sentencepiece_model.normalizer_spec.add_dummy_prefix
 

conversion/kimi_linear.py

@@ -29,13 +29,14 @@ class KimiLinearModel(TextModel):
 
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+        assert tokenizer is not None
         tokpre = self.get_vocab_base_pre(tokenizer)
 
         if tokpre == "kimi-k2":
             # Build merges list using the approach similar to HunYuanMoE
             merges = []
             vocab = {}
-            mergeable_ranks = tokenizer.model._mergeable_ranks  # ty: ignore[unresolved-attribute]
+            mergeable_ranks = tokenizer.model._mergeable_ranks  # ty: ignore[missing-union-attribute]
             for token, rank in mergeable_ranks.items():
                 vocab[QwenModel.token_bytes_to_string(token)] = rank
                 if len(token) == 1:
@@ -45,7 +46,7 @@ class KimiLinearModel(TextModel):
                     merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))
             # Build token list
             vocab_size = self.hparams["vocab_size"]
-            special_tokens = tokenizer.special_tokens  # ty: ignore[unresolved-attribute]
+            special_tokens = tokenizer.special_tokens
             reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
             tokens: list[str] = []
             toktypes: list[int] = []
@@ -71,7 +72,7 @@ class KimiLinearModel(TextModel):
             special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=False)
             special_vocab.add_to_gguf(self.gguf_writer)
             # override eos id in config.json with tiktoken eos id
-            self.gguf_writer.add_eos_token_id(tokenizer.eos_id)  # ty: ignore[unresolved-attribute]
+            self.gguf_writer.add_eos_token_id(tokenizer.eos_id)
         else:
             raise NotImplementedError(f"Deepseek pre-tokenizer {tokpre!r} is not supported yet!")
 

conversion/lev.py

@@ -77,10 +77,10 @@ class _DecisionLoraMixin:
             delta = self.lora_scale * (lora["B"] @ lora["A"])
             data_torch = data_torch.float() + LazyTorchTensor.from_eager(delta)
             self.lora_merged.add(name[name.index("layers."):])
-        yield from super().modify_tensors(data_torch, name, bid)  # ty: ignore[unresolved-attribute]
+        yield from super().modify_tensors(data_torch, name, bid)  # ty: ignore[missing-attribute]
 
     def prepare_tensors(self):
-        super().prepare_tensors()  # ty: ignore[unresolved-attribute]
+        super().prepare_tensors()  # ty: ignore[missing-attribute]
         if len(self.lora_merged) != len(self.lora):
             raise ValueError(f"only {len(self.lora_merged)} of {len(self.lora)} LoRA tensors were merged into the base model")
 

conversion/llada.py

@@ -22,15 +22,16 @@ class LLaDAModel(TextModel):
 
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+        assert tokenizer is not None
 
-        vocab_dict = tokenizer.get_vocab()  # ty: ignore[unresolved-attribute]
+        vocab_dict = tokenizer.get_vocab()
         vocab_size = self.hparams.get("vocab_size", len(vocab_dict))
         assert max(vocab_dict.values()) < vocab_size
 
         tokpre = self.get_vocab_base_pre(tokenizer)
 
         reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab_dict.items()}
-        added_vocab = tokenizer.get_added_vocab()  # ty: ignore[unresolved-attribute]
+        added_vocab = tokenizer.get_added_vocab()
 
         for i in range(vocab_size):
             if i not in reverse_vocab:

conversion/minicpm.py

@@ -263,6 +263,7 @@ class MiniCPMV4_7VisionModel(MiniCPMV4_6VisionModel):
             # the tile separators are text tokens; clip appends their embeddings so that one chunk holds the whole image
             from transformers import AutoTokenizer
             tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+            assert tokenizer is not None
             ids = []
             for text in self.tok_embd_sep:
                 tok = tokenizer.encode(text, add_special_tokens=False)

conversion/minimax.py

@@ -25,7 +25,8 @@ class MiniMaxText01Model(TextModel):
         # they get in the way of the token sampling process and must be suppressed
 
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
-        tokenizer_vocab_size = tokenizer.vocab_size  # ty: ignore[unresolved-attribute]
+        assert tokenizer is not None
+        tokenizer_vocab_size = tokenizer.vocab_size
 
         with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f:
             weight_map = json.load(f)["weight_map"]

conversion/mistral.py

@@ -12,9 +12,9 @@ from .deepseek import DeepseekV2Model
 from .llama import LlamaModel
 
 if _mistral_common_installed:
-    from mistral_common.tokens.tokenizers.base import TokenizerVersion  # type: ignore[import-not-found, ty:unresolved-import]
-    from mistral_common.tokens.tokenizers.tekken import Tekkenizer  # type: ignore[import-not-found, ty:unresolved-import]
-    from mistral_common.tokens.tokenizers.sentencepiece import SentencePieceTokenizer  # type: ignore[import-not-found, ty:unresolved-import]
+    from mistral_common.tokens.tokenizers.base import TokenizerVersion  # ty: ignore[import-not-found]
+    from mistral_common.tokens.tokenizers.tekken import Tekkenizer  # ty: ignore[import-not-found]
+    from mistral_common.tokens.tokenizers.sentencepiece import SentencePieceTokenizer  # ty: ignore[import-not-found]
 else:
     TokenizerVersion = None  # type: ignore[assignment]
     Tekkenizer = None  # type: ignore[assignment]

conversion/muse_glimmer.py

@@ -37,7 +37,8 @@ class MuseGlimmerModel(TextModel):
 
         from transformers import AutoTokenizer
         tok = AutoTokenizer.from_pretrained(self.dir_model)
-        eot_id = tok.convert_tokens_to_ids("<|eot|>")  # ty: ignore[unresolved-attribute]
+        assert tok is not None
+        eot_id = tok.convert_tokens_to_ids("<|eot|>")
         if isinstance(eot_id, int) and eot_id >= 0:
             self.gguf_writer.add_eot_token_id(eot_id)
 
@@ -152,7 +153,7 @@ class MuseGlimmerAssistantModel(TextModel):
             target_arch = json.load(f)["architectures"][0]
         target_cls = get_model_class(target_arch)
         if target_cls is not type(self):
-            target_cls.set_vocab(self)  # ty: ignore[unresolved-attribute]
+            target_cls.set_vocab(self)  # ty: ignore[missing-attribute]
         else:
             super().set_vocab()
 

conversion/muse_spark.py

@@ -44,7 +44,8 @@ class MuseSparkModel(TextModel):
 
         from transformers import AutoTokenizer
         tok = AutoTokenizer.from_pretrained(self.dir_model)
-        eot_id = tok.convert_tokens_to_ids("<|eot|>")  # ty: ignore[unresolved-attribute]
+        assert tok is not None
+        eot_id = tok.convert_tokens_to_ids("<|eot|>")
         if isinstance(eot_id, int) and eot_id >= 0:
             self.gguf_writer.add_eot_token_id(eot_id)
 

conversion/nemotron.py

@@ -73,7 +73,7 @@ class LiLiCorrDraftModel(DFlashModel):
         try:
             if target_arch == "NemotronHForCausalLM":
                 self.is_moe = "num_experts_per_tok" in target_text_config
-            target_cls.set_vocab(self)  # ty: ignore[unresolved-attribute]
+            target_cls.set_vocab(self)  # ty: ignore[missing-attribute]
         finally:
             self.dir_model = original_dir
         self.gguf_writer.add_mask_token_id(self.hparams["dflash_config"]["mask_token_id"])
@@ -609,6 +609,7 @@ class NemotronHModel(GraniteHybridModel):
 
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
+        assert tokenizer is not None
 
         # Pad vocab size (from Mamba2Model/GraniteHybridModel)
         self.hparams["pad_vocab_size_multiple"] = 8 # Setting this here since GraniteHybridModel.set_vocab() isn't being invoked now.
@@ -623,14 +624,14 @@ class NemotronHModel(GraniteHybridModel):
         vocab_skip = getattr(self, "_vocab_skip", ())
         user_defined_tokens = getattr(self, "_user_defined_tokens", None)
 
-        assert max(id_ for tok, id_ in tokenizer.vocab.items() if tok not in vocab_skip) < vocab_size  # ty: ignore[unresolved-attribute]
+        assert max(id_ for tok, id_ in tokenizer.vocab.items() if tok not in vocab_skip) < vocab_size
 
         tokpre = self.get_vocab_base_pre(tokenizer)
 
-        reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}  # ty: ignore[unresolved-attribute]
-        added_vocab = tokenizer.get_added_vocab()  # ty: ignore[unresolved-attribute]
+        reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
+        added_vocab = tokenizer.get_added_vocab()
 
-        added_tokens_decoder = tokenizer.added_tokens_decoder  # ty: ignore[unresolved-attribute]
+        added_tokens_decoder = tokenizer.added_tokens_decoder
 
         for i in range(vocab_size):
             if i not in reverse_vocab:
@@ -641,7 +642,7 @@ class NemotronHModel(GraniteHybridModel):
                 if token in added_vocab:
                     if not added_tokens_decoder[i].normalized:
                         previous_token = token
-                        token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))  # ty: ignore[unresolved-attribute, invalid-assignment]
+                        token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))  # ty: ignore[invalid-assignment]
                         if previous_token != token:
                             logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")
 

conversion/pockettts.py

@@ -191,7 +191,7 @@ class PocketTTSModel(TextModel):
         # unigram segmentation, so use the UGM tokenizer instead
         from sentencepiece import sentencepiece_model_pb2 as model
 
-        proto = model.ModelProto()  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+        proto = model.ModelProto()  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
         proto.ParseFromString(open(self.dir_model / "tokenizer.model", "rb").read())
         assert proto.trainer_spec.model_type == 1, "expected a unigram tokenizer"
 

conversion/qwen.py

@@ -225,12 +225,13 @@ class Qwen3Model(Qwen2Model):
     def _find_rerank_config(self):
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+        assert tokenizer is not None
 
         self.is_rerank = True
         self.is_tied_embeddings = self.hparams.get("tie_word_embeddings", False)
-        self.token_false_id = tokenizer.convert_tokens_to_ids("no")  # ty: ignore[unresolved-attribute, invalid-assignment]
-        self.token_true_id = tokenizer.convert_tokens_to_ids("yes")  # ty: ignore[unresolved-attribute, invalid-assignment]
-        self.sep_token_id = tokenizer.convert_tokens_to_ids("|")  # ty: ignore[unresolved-attribute]
+        self.token_false_id = tokenizer.convert_tokens_to_ids("no")  # ty: ignore[invalid-assignment]
+        self.token_true_id = tokenizer.convert_tokens_to_ids("yes")  # ty: ignore[invalid-assignment]
+        self.sep_token_id = tokenizer.convert_tokens_to_ids("|")
 
         assert self.token_false_id is not None and self.token_true_id is not None
 
@@ -329,7 +330,7 @@ class _QwenMtpMixin:
         key = next((k for k in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"] if k in hparams), None)
         type(self)._original_block_count = hparams.get(key)
         type(self).opt_num_mtp_layers = 0
-        return super().index_tensors(remote_hf_model_id=remote_hf_model_id)  # ty: ignore[unresolved-attribute]
+        return super().index_tensors(remote_hf_model_id=remote_hf_model_id)  # ty: ignore[missing-attribute]
 
     @classmethod
     def filter_tensors(cls, item):
@@ -366,7 +367,7 @@ class _QwenMtpMixin:
         return name, gen
 
     def set_gguf_parameters(self):
-        super().set_gguf_parameters()  # ty: ignore[unresolved-attribute]
+        super().set_gguf_parameters()  # ty: ignore[missing-attribute]
         if self.no_mtp:
             return
         if (n := self.block_count - self.hparams["num_hidden_layers"]) > 0:
@@ -374,15 +375,15 @@ class _QwenMtpMixin:
 
     def prepare_metadata(self, vocab_only: bool):
         from_dir = self.fname_out.is_dir()
-        super().prepare_metadata(vocab_only=vocab_only)  # ty: ignore[unresolved-attribute]
+        super().prepare_metadata(vocab_only=vocab_only)  # ty: ignore[missing-attribute]
 
         if not self.mtp_only or not from_dir:
             return
 
-        output_type: str = self.ftype.name.partition("_")[2]  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+        output_type: str = self.ftype.name.partition("_")[2]  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
         fname_default: str = gguf.naming_convention(
-            self.metadata.name, self.metadata.basename, self.metadata.finetune,                  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
-            self.metadata.version, size_label=None, output_type=output_type, model_type=None)    # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+            self.metadata.name, self.metadata.basename, self.metadata.finetune,                  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
+            self.metadata.version, size_label=None, output_type=output_type, model_type=None)    # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
         self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
 
 
@@ -685,7 +686,7 @@ class _Qwen35MRopeMixin:
     rope_parameters: dict
 
     def set_gguf_parameters(self):
-        super().set_gguf_parameters()  # ty: ignore[unresolved-attribute]
+        super().set_gguf_parameters()  # ty: ignore[missing-attribute]
         if "mrope_section" not in self.rope_parameters:
             self.gguf_writer.add_rope_dimension_sections(self._QWEN35_DEFAULT_MROPE_SECTION)
 
@@ -843,7 +844,7 @@ class DFlashModel(Qwen3Model):
         if target_cls is not type(self):
             if target_arch == "NemotronHForCausalLM":
                 setattr(self, "is_moe", "num_experts_per_tok" in target_hparams)
-            target_cls.set_vocab(self)  # ty: ignore[unresolved-attribute]
+            target_cls.set_vocab(self)  # ty: ignore[missing-attribute]
         else:
             super().set_vocab()
 

conversion/t5.py

@@ -40,7 +40,7 @@ class T5Model(TextModel):
         if not tokenizer_path.is_file():
             raise FileNotFoundError(f"File not found: {tokenizer_path}")
 
-        sentencepiece_model = model.ModelProto()  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+        sentencepiece_model = model.ModelProto()  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
         sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
 
         # some models like Pile-T5 family use BPE tokenizer instead of Unigram
@@ -178,7 +178,7 @@ class T5EncoderModel(TextModel):
         if not tokenizer_path.is_file():
             raise FileNotFoundError(f"File not found: {tokenizer_path}")
 
-        sentencepiece_model = model.ModelProto()  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+        sentencepiece_model = model.ModelProto()  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
         sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())
 
         # some models like Pile-T5 family use BPE tokenizer instead of Unigram

conversion/xverse.py

@@ -25,15 +25,16 @@ class XverseModel(TextModel):
 
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(dir_model)
-        vocab_size = hparams.get("vocab_size", len(tokenizer.vocab))  # ty: ignore[unresolved-attribute]
+        assert tokenizer is not None
+        vocab_size = hparams.get("vocab_size", len(tokenizer.vocab))
         # Since we are checking the maximum index, we need to ensure it's strictly less than vocab_size,
         # because vocab_size is the count of items, and indexes start at 0.
-        max_vocab_index = max(tokenizer.get_vocab().values())  # ty: ignore[unresolved-attribute]
+        max_vocab_index = max(tokenizer.get_vocab().values())
         if max_vocab_index >= vocab_size:
             raise ValueError("Vocabulary size exceeds expected maximum size.")
 
-        reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}  # ty: ignore[unresolved-attribute]
-        added_vocab = tokenizer.get_added_vocab()  # ty: ignore[unresolved-attribute]
+        reverse_vocab: dict[int, str] = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
+        added_vocab = tokenizer.get_added_vocab()
 
         for token_id in range(vocab_size):
             token_text = reverse_vocab[token_id].encode('utf-8')
@@ -44,7 +45,7 @@ class XverseModel(TextModel):
             elif re.fullmatch(br"<0x[0-9A-Fa-f]{2}>", token_text):
                 toktype = gguf.TokenType.BYTE  # special
             elif reverse_vocab[token_id] in added_vocab:
-                if tokenizer.added_tokens_decoder[token_id].special:  # ty: ignore[unresolved-attribute]
+                if tokenizer.added_tokens_decoder[token_id].special:
                     toktype = gguf.TokenType.CONTROL
                 else:
                     toktype = gguf.TokenType.USER_DEFINED

convert_hf_to_gguf_update.py

@@ -325,7 +325,8 @@ for model in [*pre_computed_hashes, *all_models]:
         except Exception as e:
             raise OSError(f"Error loading tokenizer for model {name}.") from e
 
-        chktok = tokenizer.encode(CHK_TXT)  # ty: ignore[unresolved-attribute]
+        assert tokenizer is not None
+        chktok = tokenizer.encode(CHK_TXT)
         chkhsh = sha256(str(chktok).encode()).hexdigest()
 
         logger.info(f"model: {name}")
@@ -486,6 +487,7 @@ for model in models:
         logger.error(f"Failed to load tokenizer for model {name}. Error: {e}")
         continue  # Skip this model and continue with the next one in the loop
 
+    assert tokenizer is not None
     if not os.path.exists(f"models/ggml-vocab-{name}.gguf"):
         logger.info(f"Skip vocab files for model {name}, no GGUF file found")
         continue
@@ -497,7 +499,7 @@ for model in models:
 
     with open(f"models/ggml-vocab-{name}.gguf.out", "w") as f:
         for text in tests:
-            res = tokenizer.encode(text, add_special_tokens=False)  # ty: ignore[unresolved-attribute]
+            res = tokenizer.encode(text, add_special_tokens=False)
             for r in res:
                 f.write(f" {r}")
             f.write("\n")

convert_lora_to_gguf.py

@@ -443,7 +443,8 @@ if __name__ == '__main__':
                     # the invocation string includes the "<|start_of_turn|>"
                     # token, but the adapters themselves were trained to
                     # activate _after_ that first token, so we drop it here.
-                    alora_invocation_tokens = tokenizer(invocation_string)["input_ids"][1:]  # ty: ignore[call-non-callable]
+                    assert tokenizer is not None
+                    alora_invocation_tokens = tokenizer(invocation_string)["input_ids"][1:]
                 if alora_invocation_tokens:
                     logger.debug("GGUF KV: %s = %s", gguf.Keys.Adapter.ALORA_INVOCATION_TOKENS, alora_invocation_tokens)
                     self.gguf_writer.add_key_value(

examples/model-conversion/scripts/causal/compare-logits.py

@@ -7,7 +7,7 @@ import os
 
 # Add utils directory to path for direct script execution
 sys.path.insert(0, str(Path(__file__).parent.parent / "utils"))
-from common import get_model_name_from_env_path, compare_tokens, exit_with_warning  # type: ignore[import-not-found, ty:unresolved-import]
+from common import get_model_name_from_env_path, compare_tokens, exit_with_warning  # ty: ignore[import-not-found]
 
 def quick_logits_check(pytorch_file, llamacpp_file):
     """Lightweight sanity check before NMSE"""

examples/model-conversion/scripts/causal/run-casual-gen-embeddings-org.py

@@ -33,6 +33,7 @@ print("EOS token id:     ", config.eos_token_id)
 
 print("Loading model and tokenizer using AutoTokenizer:", model_path)
 tokenizer = AutoTokenizer.from_pretrained(model_path)
+assert tokenizer is not None
 
 if unreleased_model_name:
     model_name_lower = unreleased_model_name.lower()
@@ -56,11 +57,11 @@ model_name = os.path.basename(model_path)
 print(f"Model name: {model_name}")
 
 prompt = "Hello world today"
-input_ids = tokenizer(prompt, return_tensors="pt").input_ids  # ty: ignore[call-non-callable]
+input_ids = tokenizer(prompt, return_tensors="pt").input_ids
 token_ids = input_ids[0].cpu().tolist()
 print(f"Input tokens: {input_ids}")
 print(f"Input text: {repr(prompt)}")
-print(f"Tokenized: {tokenizer.convert_ids_to_tokens(input_ids[0])}")  # ty: ignore[unresolved-attribute]
+print(f"Tokenized: {tokenizer.convert_ids_to_tokens(input_ids[0])}")
 
 with torch.no_grad():
     outputs = model(input_ids, output_hidden_states=True)
@@ -83,7 +84,7 @@ with torch.no_grad():
 
     # Print embeddings per token in the requested format
     print("\nToken embeddings:")
-    tokens = tokenizer.convert_ids_to_tokens(input_ids[0])  # ty: ignore[unresolved-attribute]
+    tokens = tokenizer.convert_ids_to_tokens(input_ids[0])
     for i, embedding in enumerate(token_embeddings):
         # Format: show first few values, ..., then last few values
         if len(embedding) > 10:

examples/model-conversion/scripts/embedding/run-original-model.py

@@ -64,7 +64,7 @@ def load_model_and_tokenizer(model_path, use_sentence_transformers=False, device
         print("Using SentenceTransformer to apply all numbered layers")
         model = SentenceTransformer(model_path)
         tokenizer = model.tokenizer
-        config = model[0].auto_model.config  # ty: ignore[unresolved-attribute]
+        config = model[0].auto_model.config  # ty: ignore[missing-union-attribute]
     else:
         tokenizer = AutoTokenizer.from_pretrained(model_path)
         config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)

examples/model-conversion/scripts/utils/check-nmse.py

@@ -5,7 +5,7 @@ import sys
 import os
 import argparse
 from pathlib import Path
-from common import get_model_name_from_env_path  # type: ignore[import-not-found, ty:unresolved-import]
+from common import get_model_name_from_env_path  # ty: ignore[import-not-found]
 
 def calculate_nmse(reference, test):
     mse = np.mean((test - reference) ** 2)

examples/model-conversion/scripts/utils/compare_tokens.py

@@ -2,7 +2,7 @@
 
 import argparse
 import sys
-from common import compare_tokens  # type: ignore[import-not-found, ty:unresolved-import]
+from common import compare_tokens  # ty: ignore[import-not-found]
 
 
 def parse_arguments():

examples/model-conversion/scripts/utils/semantic_check.py

@@ -7,7 +7,7 @@ import importlib
 from pathlib import Path
 
 from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM, AutoModel
-from common import compare_tokens, exit_with_warning  # type: ignore[import-not-found, ty:unresolved-import]
+from common import compare_tokens, exit_with_warning  # ty: ignore[import-not-found]
 
 unreleased_model_name = os.getenv('UNRELEASED_MODEL_NAME')
 
@@ -183,6 +183,7 @@ def main():
     # Load the python model to get configuration information and also to load the tokenizer.
     print("Loading model and tokenizer using AutoTokenizer:", args.model_path)
     tokenizer = AutoTokenizer.from_pretrained(args.model_path)
+    assert tokenizer is not None
     config = AutoConfig.from_pretrained(args.model_path, trust_remote_code=True)
 
     if unreleased_model_name:
@@ -207,8 +208,8 @@ def main():
         else:
             model = AutoModel.from_pretrained(args.model_path, trust_remote_code=True)
 
-    encoded = tokenizer(prompt, return_tensors="pt")  # ty: ignore[call-non-callable]
-    tokens = tokenizer.convert_ids_to_tokens(encoded['input_ids'][0])  # ty: ignore[unresolved-attribute]
+    encoded = tokenizer(prompt, return_tensors="pt")
+    tokens = tokenizer.convert_ids_to_tokens(encoded['input_ids'][0])
     n_tokens = len(tokens)
     print(f"n_tokens: {n_tokens}");
     print(f"hidden_size: {model.config.hidden_size}")

examples/pydantic_models_to_grammar.py

@@ -1285,7 +1285,7 @@ def convert_dictionary_to_pydantic_model(dictionary: dict[str, Any], model_name:
                     if items != {}:
                         array = {"properties": items}
                         array_type = convert_dictionary_to_pydantic_model(array, f"{model_name}_{field_name}_items")
-                        fields[field_name] = (list[array_type], ...)  # ty: ignore[invalid-type-form]
+                        fields[field_name] = (list[array_type], ...)
                     else:
                         fields[field_name] = (list, ...)
                 elif field_type == "object":

gguf-py/gguf/lazy.py

@@ -138,7 +138,7 @@ class LazyBase(ABC, metaclass=LazyMeta):
                     if isinstance(meta_noop, tuple):
                         dtype, shape = meta_noop
                         assert callable(shape)
-                        res = cls.meta_with_dtype_and_shape(dtype, shape(res.shape))  # ty: ignore[call-top-callable]
+                        res = cls.meta_with_dtype_and_shape(dtype, shape(res.shape))
                     else:
                         res = cls.meta_with_dtype_and_shape(meta_noop, res.shape)
 

gguf-py/gguf/vocab.py

@@ -14,12 +14,12 @@ except ImportError:
     SentencePieceProcessor: Any = None
 
 try:
-    from mistral_common.tokens.tokenizers.mistral import MistralTokenizer # type: ignore[import-not-found, ty:unresolved-import]
-    from mistral_common.tokens.tokenizers.tekken import Tekkenizer # type: ignore[import-not-found, ty:unresolved-import]
-    from mistral_common.tokens.tokenizers.utils import ( # type: ignore[import-not-found, ty:unresolved-import]
+    from mistral_common.tokens.tokenizers.mistral import MistralTokenizer # ty: ignore[import-not-found]
+    from mistral_common.tokens.tokenizers.tekken import Tekkenizer # ty: ignore[import-not-found]
+    from mistral_common.tokens.tokenizers.utils import ( # ty: ignore[import-not-found]
         _filter_valid_tokenizer_files,
     )
-    from mistral_common.tokens.tokenizers.sentencepiece import ( # type: ignore[import-not-found, ty:unresolved-import]
+    from mistral_common.tokens.tokenizers.sentencepiece import ( # ty: ignore[import-not-found]
         SentencePieceTokenizer,
     )
 except ImportError:
@@ -32,7 +32,7 @@ else:
     _mistral_common_installed = True
 
 try:
-    from mistral_common.tokens.tokenizers.utils import ( # type: ignore[import-not-found, ty:unresolved-import]
+    from mistral_common.tokens.tokenizers.utils import ( # ty: ignore[import-not-found]
         get_one_valid_tokenizer_file,
     )
 except ImportError:
@@ -629,7 +629,8 @@ class LlamaHfVocab(Vocab):
             cache_dir=base_path,
             local_files_only=True,
         )
-        assert self.tokenizer.is_fast  # assume tokenizer.json is used  # ty: ignore[unresolved-attribute]
+        assert self.tokenizer is not None
+        assert self.tokenizer.is_fast  # assume tokenizer.json is used
 
         # Initialize lists and dictionaries for added tokens
         self.added_tokens_list = []
@@ -638,30 +639,31 @@ class LlamaHfVocab(Vocab):
 
         # Process added tokens
         for tok, tokidx in sorted(
-            self.tokenizer.get_added_vocab().items(), key=lambda x: x[1]  # ty: ignore[unresolved-attribute]
+            self.tokenizer.get_added_vocab().items(), key=lambda x: x[1]
         ):
             # Only consider added tokens that are not in the base vocabulary
-            if tokidx >= self.tokenizer.vocab_size:  # ty: ignore[unresolved-attribute]
+            if tokidx >= self.tokenizer.vocab_size:
                 self.added_tokens_list.append(tok)
                 self.added_tokens_dict[tok] = tokidx
                 self.added_tokens_ids.add(tokidx)
 
         # Store special tokens and their IDs
         self.specials = {
-            tok: self.tokenizer.get_vocab()[tok]  # ty: ignore[unresolved-attribute]
-            for tok in self.tokenizer.all_special_tokens  # ty: ignore[unresolved-attribute]
+            tok: self.tokenizer.get_vocab()[tok]
+            for tok in self.tokenizer.all_special_tokens
         }
-        self.special_ids = set(self.tokenizer.all_special_ids)  # ty: ignore[unresolved-attribute]
+        self.special_ids = set(self.tokenizer.all_special_ids)
 
         # Set vocabulary sizes
-        self.vocab_size_base = self.tokenizer.vocab_size  # ty: ignore[unresolved-attribute]
+        self.vocab_size_base = self.tokenizer.vocab_size
         self.vocab_size      = self.vocab_size_base + len(self.added_tokens_list)
 
         self.fname_tokenizer = fname_tokenizer
 
     def hf_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
+        assert self.tokenizer is not None
         reverse_vocab = {
-            id: encoded_tok for encoded_tok, id in self.tokenizer.get_vocab().items()  # ty: ignore[unresolved-attribute]
+            id: encoded_tok for encoded_tok, id in self.tokenizer.get_vocab().items()
         }
 
         for token_id in range(self.vocab_size_base):
@@ -702,7 +704,8 @@ class LlamaHfVocab(Vocab):
             yield text.encode("utf-8"), score, toktype
 
     def has_newline_token(self):
-        return "<0x0A>" in self.tokenizer.vocab or "\n" in self.tokenizer.vocab  # ty: ignore[unresolved-attribute]
+        assert self.tokenizer is not None
+        return "<0x0A>" in self.tokenizer.vocab or "\n" in self.tokenizer.vocab
 
     def all_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
         yield from self.hf_tokens()

media/llama1-icon-transparent.svg

@@ -1,77 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   id="Layer_1"
-   version="1.1"
-   viewBox="0 0 250 250"
-   sodipodi:docname="llama1-icon-transparent.svg"
-   width="250"
-   height="250"
-   inkscape:version="1.4.2 (ebf0e940d0, 2025-05-08)"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg">
-  <sodipodi:namedview
-     id="namedview7"
-     pagecolor="#505050"
-     bordercolor="#ffffff"
-     borderopacity="1"
-     inkscape:showpageshadow="0"
-     inkscape:pageopacity="0"
-     inkscape:pagecheckerboard="1"
-     inkscape:deskcolor="#505050"
-     inkscape:zoom="2.48"
-     inkscape:cx="49.596774"
-     inkscape:cy="189.91935"
-     inkscape:window-width="3440"
-     inkscape:window-height="1440"
-     inkscape:window-x="0"
-     inkscape:window-y="0"
-     inkscape:window-maximized="1"
-     inkscape:current-layer="Layer_1" />
-  <!-- Generator: Adobe Illustrator 29.3.1, SVG Export Plug-In . SVG Version: 2.1.0 Build 151)  -->
-  <defs
-     id="defs1">
-    <style
-       id="style1">
-      .st0 {
-        fill: #ff8236;
-      }
-
-      .st1 {
-        fill: #fff;
-      }
-
-      .st2 {
-        fill: #1b1f20;
-      }
-    </style>
-  </defs>
-  <g
-     id="g7">
-    <g
-       id="g6"
-       transform="translate(-995.51066,-129.70875)">
-      <path
-         class="st0"
-         d="m 1163.3,226.8 -13.5,24 c -17.8,-13.7 -44.2,-15.7 -62,-1 -28.7,23.7 -26.7,78.5 18,78.8 12.5,0 23.1,-5.9 34.5,-9.8 l 6,23.9 c -10.1,4.7 -20.4,9.5 -31.5,11 -101.2,13.8 -95.4,-132.3 -3.9,-139.9 19.2,-1.6 36.1,3.4 52.5,13 z"
-         id="path4" />
-      <path
-         class="st0"
-         d="m 1093.4,203.8 c -15.4,4.6 -29.7,13.1 -40.5,25 -2,-24.2 3.4,-73.1 30.3,-82.7 4,-1.4 17.7,-4.9 17.3,2.2 -0.4,7.1 -9.9,19.3 -12.2,25.9 -4,11.6 -0.3,19.6 5.2,29.7 z"
-         id="path5" />
-      <polygon
-         class="st0"
-         points="1131.4,307.8 1116.4,307.8 1116.4,290.8 1099.4,290.8 1099.4,276.8 1114.9,276.8 1116.4,275.3 1116.4,258.8 1131.4,258.8 1131.4,276.8 1147.4,276.8 1147.4,290.8 1131.4,290.8 "
-         id="polygon5" />
-      <polygon
-         class="st0"
-         points="1186.4,290.8 1186.4,307.8 1171.4,307.8 1171.4,290.8 1155.4,290.8 1155.4,276.8 1171.4,276.8 1171.4,258.8 1186.4,258.8 1186.4,275.3 1187.9,276.8 1203.4,276.8 1203.4,290.8 "
-         id="polygon6" />
-      <path
-         class="st0"
-         d="m 1142.3,156.9 c 2,3 -9.3,15.9 -11.1,19.2 -5.2,9.8 -1.7,15.4 2.2,24.7 -11.3,-1.7 -21.8,-0.3 -33,1 2.5,-21.5 14.6,-52.8 41.9,-44.9 z"
-         id="path6" />
-    </g>
-  </g>
-</svg>

media/llama1-icon.svg

@@ -1,87 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   id="Layer_1"
-   version="1.1"
-   viewBox="0 0 250 250"
-   sodipodi:docname="llama-icon.svg"
-   width="250"
-   height="250"
-   inkscape:version="1.4.2 (ebf0e940d0, 2025-05-08)"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg">
-  <sodipodi:namedview
-     id="namedview7"
-     pagecolor="#505050"
-     bordercolor="#ffffff"
-     borderopacity="1"
-     inkscape:showpageshadow="0"
-     inkscape:pageopacity="0"
-     inkscape:pagecheckerboard="1"
-     inkscape:deskcolor="#505050"
-     inkscape:zoom="2.48"
-     inkscape:cx="146.57258"
-     inkscape:cy="189.91936"
-     inkscape:window-width="3440"
-     inkscape:window-height="1440"
-     inkscape:window-x="0"
-     inkscape:window-y="0"
-     inkscape:window-maximized="1"
-     inkscape:current-layer="g7" />
-  <!-- Generator: Adobe Illustrator 29.3.1, SVG Export Plug-In . SVG Version: 2.1.0 Build 151)  -->
-  <defs
-     id="defs1">
-    <style
-       id="style1">
-      .st0 {
-        fill: #ff8236;
-      }
-
-      .st1 {
-        fill: #fff;
-      }
-
-      .st2 {
-        fill: #1b1f20;
-      }
-    </style>
-  </defs>
-  <rect
-     class="st2"
-     width="250"
-     height="250"
-     rx="8.6857386"
-     ry="8.7008333"
-     id="rect1"
-     x="0"
-     y="0"
-     style="stroke-width:0.266071" />
-  <g
-     id="g7">
-    <g
-       id="g6"
-       transform="translate(-995.51066,-129.70875)">
-      <path
-         class="st0"
-         d="m 1163.3,226.8 -13.5,24 c -17.8,-13.7 -44.2,-15.7 -62,-1 -28.7,23.7 -26.7,78.5 18,78.8 12.5,0 23.1,-5.9 34.5,-9.8 l 6,23.9 c -10.1,4.7 -20.4,9.5 -31.5,11 -101.2,13.8 -95.4,-132.3 -3.9,-139.9 19.2,-1.6 36.1,3.4 52.5,13 z"
-         id="path4" />
-      <path
-         class="st0"
-         d="m 1093.4,203.8 c -15.4,4.6 -29.7,13.1 -40.5,25 -2,-24.2 3.4,-73.1 30.3,-82.7 4,-1.4 17.7,-4.9 17.3,2.2 -0.4,7.1 -9.9,19.3 -12.2,25.9 -4,11.6 -0.3,19.6 5.2,29.7 z"
-         id="path5" />
-      <polygon
-         class="st0"
-         points="1131.4,307.8 1116.4,307.8 1116.4,290.8 1099.4,290.8 1099.4,276.8 1114.9,276.8 1116.4,275.3 1116.4,258.8 1131.4,258.8 1131.4,276.8 1147.4,276.8 1147.4,290.8 1131.4,290.8 "
-         id="polygon5" />
-      <polygon
-         class="st0"
-         points="1186.4,290.8 1186.4,307.8 1171.4,307.8 1171.4,290.8 1155.4,290.8 1155.4,276.8 1171.4,276.8 1171.4,258.8 1186.4,258.8 1186.4,275.3 1187.9,276.8 1203.4,276.8 1203.4,290.8 "
-         id="polygon6" />
-      <path
-         class="st0"
-         d="m 1142.3,156.9 c 2,3 -9.3,15.9 -11.1,19.2 -5.2,9.8 -1.7,15.4 2.2,24.7 -11.3,-1.7 -21.8,-0.3 -33,1 2.5,-21.5 14.6,-52.8 41.9,-44.9 z"
-         id="path6" />
-    </g>
-  </g>
-</svg>

media/llama1-logo.svg

@@ -1,34 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg id="Layer_1" xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="0 0 1500 500">
-  <!-- Generator: Adobe Illustrator 29.3.1, SVG Export Plug-In . SVG Version: 2.1.0 Build 151)  -->
-  <defs>
-    <style>
-      .st0 {
-        fill: #ff8236;
-      }
-
-      .st1 {
-        fill: #fff;
-      }
-
-      .st2 {
-        fill: #1b1f20;
-      }
-    </style>
-  </defs>
-  <rect class="st2" width="1500" height="500" rx="16" ry="16"/>
-  <g>
-    <path class="st1" d="M749.4,353.8l5.4-204.1,20.4-.8,45.1,98.8,42.5-99h19l6.5,205h-38l-2-98-24.9,61.4c-1,1.3-8,1.3-9-1l-25.6-61.4-1.5,99h-38Z"/>
-    <path class="st1" d="M727.5,240.1c-10.8-27.1-53.1-24.5-75.3-14.7l3.1,28.4c9.2-1.9,30-8,37.5-1,.9.9,3.5,5.7,3.5,6.5v16.5c-31.8-17.2-54.5,6.1-54.4,38.5,0,36.5,28.4,57.3,56.4,27.5v12h32v-104.5c0-.5-2.4-8-2.8-9.2ZM696.4,327.8c-8.4,1.7-15.4,2.9-19.2-6.3-5.8-14,.6-37.9,19.2-27.2v33.5Z"/>
-    <path class="st1" d="M899.4,353.8l47.6-205.1h30.3c0,.1,47,205.1,47,205.1h-38l-7.9-33.6h-34.1l-7.9,33.6h-37ZM951.4,285.8h20l-10.5-56-9.5,56Z"/>
-    <polygon class="st1" points="490.4 148.8 490.4 317.3 491.9 318.8 534.4 318.8 534.4 353.8 451.4 353.8 451.4 150.3 452.9 148.8 490.4 148.8"/>
-    <polygon class="st1" points="589.4 148.8 589.4 318.8 633.4 318.8 633.4 353.8 550.4 353.8 550.4 148.8 589.4 148.8"/>
-    <g>
-      <path class="st0" d="M1163.3,226.8l-13.5,24c-17.8-13.7-44.2-15.7-62-1-28.7,23.7-26.7,78.5,18,78.8,12.5,0,23.1-5.9,34.5-9.8l6,23.9c-10.1,4.7-20.4,9.5-31.5,11-101.2,13.8-95.4-132.3-3.9-139.9,19.2-1.6,36.1,3.4,52.5,13Z"/>
-      <path class="st0" d="M1093.4,203.8c-15.4,4.6-29.7,13.1-40.5,25-2-24.2,3.4-73.1,30.3-82.7,4-1.4,17.7-4.9,17.3,2.2s-9.9,19.3-12.2,25.9c-4,11.6-.3,19.6,5.2,29.7Z"/>
-      <polygon class="st0" points="1131.4 258.8 1131.4 276.8 1147.4 276.8 1147.4 290.8 1131.4 290.8 1131.4 307.8 1116.4 307.8 1116.4 290.8 1099.4 290.8 1099.4 276.8 1114.9 276.8 1116.4 275.3 1116.4 258.8 1131.4 258.8"/>
-      <polygon class="st0" points="1186.4 258.8 1186.4 275.3 1187.9 276.8 1203.4 276.8 1203.4 290.8 1186.4 290.8 1186.4 307.8 1171.4 307.8 1171.4 290.8 1155.4 290.8 1155.4 276.8 1171.4 276.8 1171.4 258.8 1186.4 258.8"/>
-      <path class="st0" d="M1142.3,156.9c2,3-9.3,15.9-11.1,19.2-5.2,9.8-1.7,15.4,2.2,24.7-11.3-1.7-21.8-.3-33,1,2.5-21.5,14.6-52.8,41.9-44.9Z"/>
-    </g>
-  </g>
-</svg>

scripts/jinja/jinja-tester.py

@@ -241,10 +241,10 @@ class CodeEditor(QPlainTextEdit):
         if not self.isReadOnly():
             selection = QTextEdit.ExtraSelection()
             line_color = QColorConstants.Yellow.lighter(160)
-            selection.format.setBackground(line_color)  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
-            selection.format.setProperty(QTextFormat.Property.FullWidthSelection, True)  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
-            selection.cursor = self.textCursor()  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
-            selection.cursor.clearSelection()  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+            selection.format.setBackground(line_color)  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
+            selection.format.setProperty(QTextFormat.Property.FullWidthSelection, True)  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
+            selection.cursor = self.textCursor()  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
+            selection.cursor.clearSelection()  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
             extra_selections.append(selection)
         self.setExtraSelections(extra_selections)
 
@@ -262,8 +262,8 @@ class CodeEditor(QPlainTextEdit):
                 )
 
             extra = QTextEdit.ExtraSelection()
-            extra.format.setBackground(color.lighter(160))  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
-            extra.cursor = cursor  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+            extra.format.setBackground(color.lighter(160))  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
+            extra.cursor = cursor  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
 
             self.setExtraSelections(self.extraSelections() + [extra])
 
@@ -274,8 +274,8 @@ class CodeEditor(QPlainTextEdit):
             cursor.select(QTextCursor.SelectionType.LineUnderCursor)
 
             extra = QTextEdit.ExtraSelection()
-            extra.format.setBackground(color.lighter(160))  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
-            extra.cursor = cursor  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
+            extra.format.setBackground(color.lighter(160))  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
+            extra.cursor = cursor  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[missing-attribute]
 
             self.setExtraSelections(self.extraSelections() + [extra])
 

scripts/tool_bench.py

@@ -52,8 +52,8 @@ import typer
 
 sys.path.insert(0, Path(__file__).parent.parent.as_posix())
 if True:
-    from tools.server.tests.utils import ServerProcess  # ty: ignore[unresolved-import]
-    from tools.server.tests.unit.test_tool_call import do_test_calc_result, do_test_hello_world, do_test_weather  # ty: ignore[unresolved-import]
+    from tools.server.tests.utils import ServerProcess  # ty: ignore[import-not-found]
+    from tools.server.tests.unit.test_tool_call import do_test_calc_result, do_test_hello_world, do_test_weather  # ty: ignore[import-not-found]
 
 
 @contextmanager

tests/test-chat.cpp

@@ -7018,6 +7018,22 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
             .expect_reasoning("Thinking.")
             .expect_content("Working on it.Hello!")
             .run();
+
+        // Structured output, straight to the final answer
+        tst.test(" to=user<|message|>" R"({"amount": 123.45, "date": "2025-12-03"})")
+            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+            .json_schema(invoice_schema)
+            .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})")
+            .run();
+
+        // Structured output after a reasoning message: reasoning stays free-form
+        tst.test(" to=self<|message|>I need to output the invoice details in JSON<|eom|>"
+                 "<|start|>assistant to=user<|message|>" R"({"amount": 123.45, "date": "2025-12-03"})")
+            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+            .json_schema(invoice_schema)
+            .expect_reasoning("I need to output the invoice details in JSON")
+            .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})")
+            .run();
     }
 
     // GPT-OSS format tests

tests/test-tokenizer-0.py

@@ -12,6 +12,7 @@ dir_tokenizer = args.dir_tokenizer
 fname_tok = args.fname_tok
 
 tokenizer = AutoTokenizer.from_pretrained(dir_tokenizer)
+assert tokenizer is not None
 
 print('tokenizing file: ', fname_tok) # noqa: NP100
 fname_out = fname_tok + '.tok'
@@ -19,7 +20,7 @@ with open(fname_tok, 'r', encoding='utf-8') as f:
     lines = f.readlines()
     s = ''.join(lines)
     t_start = time.time()
-    res = tokenizer.encode(s, add_special_tokens=False)  # ty: ignore[unresolved-attribute]
+    res = tokenizer.encode(s, add_special_tokens=False)
     t_end = time.time()
     print('\nmain : tokenized in', "{:.3f}".format(1000.0 * (t_end - t_start)), 'ms (py)') # noqa: NP100
     with open(fname_out, 'w', encoding='utf-8') as f:

tools/mtmd/models/granite-speech.cpp

@@ -189,7 +189,7 @@ ggml_cgraph * mtmd_graph_granite_speech::build() {
                 } else {
                     concat_result = ggml_concat(ctx0, concat_result, cur, 0);
                 }
-                cb(concat_result, string_format("feature_layer_%d", il + 1).c_str(), il);
+                cb(concat_result, mtmd_format("feature_layer_%d", il + 1).c_str(), il);
             }
         }
 

tools/mtmd/mtmd-impl.h

@@ -1044,24 +1044,11 @@ static std::ifstream open_ifstream_binary(const std::string & fname) {
 }
 #endif
 
+std::string mtmd_format(const char * fmt, ...);
+
 // in test-mtmd-impl, we include woth common.h and this file, and these functions are duplicated
 // this is a quick fix to avoid compilation errors
 #ifndef DIRECTORY_SEPARATOR
-static std::string string_format(const char * fmt, ...) {
-    va_list ap;
-    va_list ap2;
-    va_start(ap, fmt);
-    va_copy(ap2, ap);
-    int size = vsnprintf(NULL, 0, fmt, ap);
-    GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT
-    std::vector<char> buf(size + 1);
-    int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);
-    GGML_ASSERT(size2 == size);
-    va_end(ap2);
-    va_end(ap);
-    return std::string(buf.data(), buf.size());
-}
-
 static void string_replace_all(std::string & s, const std::string & search, const std::string & replace) {
     if (search.empty()) {
         return;
@@ -1123,7 +1110,7 @@ static std::string gguf_data_to_str(enum gguf_type type, const void * data, int
         case GGUF_TYPE_FLOAT32: return std::to_string(((const float    *)data)[i]);
         case GGUF_TYPE_FLOAT64: return std::to_string(((const double   *)data)[i]);
         case GGUF_TYPE_BOOL:    return ((const int8_t *)data)[i] != 0 ? "true" : "false";
-        default:                return string_format("unknown type %d", type);
+        default:                return mtmd_format("unknown type %d", type);
     }
 }
 

tools/mtmd/mtmd-subcontext.cpp

@@ -82,7 +82,7 @@ mtmd_backends::mtmd_backends(ggml_backend_dev_t device, int n_threads, int max_n
         backend = ggml_backend_dev_init(device, nullptr);
         if (!backend) {
             ggml_backend_free(backend_cpu);
-            throw std::runtime_error(string_format("%s: failed to initialize \"%s\" backend\n",
+            throw std::runtime_error(mtmd_format("%s: failed to initialize \"%s\" backend\n",
                                                    __func__, ggml_backend_dev_name(device)));
         }
     }
@@ -164,7 +164,7 @@ mtmd_image_f32_batch mtmd_subcontext::get_dummy_batch() const {
         // so the [1, 256] bound only applies to FFT-based models.
         const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA;
         if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) {
-            throw std::runtime_error(string_format("%s: invalid n_mel_bins (%d), must be in [1, 256]\n", __func__, hparams.n_mel_bins));
+            throw std::runtime_error(mtmd_format("%s: invalid n_mel_bins (%d), must be in [1, 256]\n", __func__, hparams.n_mel_bins));
         }
         img.set_size({hparams.warmup_audio_size, hparams.n_mel_bins}, false, false);
         LOG_INF("%s: warmup with audio size = %d\n", __func__, hparams.warmup_audio_size);

tools/mtmd/mtmd.cpp

@@ -32,6 +32,21 @@
 #include <type_traits>
 #include <vector>
 
+std::string mtmd_format(const char * fmt, ...) {
+    va_list ap;
+    va_list ap2;
+    va_start(ap, fmt);
+    va_copy(ap2, ap);
+    int size = vsnprintf(NULL, 0, fmt, ap);
+    GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT
+    std::vector<char> buf(size + 1);
+    int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);
+    GGML_ASSERT(size2 == size);
+    va_end(ap2);
+    va_end(ap);
+    return std::string(buf.data(), size);
+}
+
 // remember to bump this if the serialization format changes
 #define MTMD_SERIALIZATION_VERSION 4
 
@@ -691,7 +706,7 @@ struct mtmd_context {
                         pos_type = MTMD_POS_TYPE_MROPE;
                     } break;
                 default:
-                    throw std::runtime_error(string_format("unsupported decoder rope type: %d\n", decoder_rope_type));
+                    throw std::runtime_error(mtmd_format("unsupported decoder rope type: %d\n", decoder_rope_type));
             }
         }
 
@@ -711,7 +726,7 @@ struct mtmd_context {
 
         // soprano and pocket-tts without voice cloning have no encoder
         if (!models.v && !models.a && !models.gen_a) {
-            throw std::runtime_error(string_format("Failed to load mmproj model from %s\n", mmproj_fname));
+            throw std::runtime_error(mtmd_format("Failed to load mmproj model from %s\n", mmproj_fname));
         }
 
         int max_nodes = 0;
@@ -752,7 +767,7 @@ struct mtmd_context {
             int n_embd_v = ctx_v->model.n_mmproj_embd();
             int n_embd_a = ctx_a->model.n_mmproj_embd();
             if (n_embd_v != n_embd_a) {
-                throw std::runtime_error(string_format(
+                throw std::runtime_error(mtmd_format(
                     "mismatch between vision and audio mmproj (n_embd_v = %d, n_embd_a = %d)\n",
                     n_embd_v, n_embd_a));
             }
@@ -763,7 +778,7 @@ struct mtmd_context {
         // a gen-only mmproj is validated below
         int n_embd_mmproj = ctx_v || ctx_a ? (ctx_v ? ctx_v : ctx_a)->model.n_mmproj_embd() : n_embd_text;
         if (n_embd_text > 0 && n_embd_text != n_embd_mmproj) {
-            throw std::runtime_error(string_format(
+            throw std::runtime_error(mtmd_format(
                 "mismatch between text model (n_embd = %d) and mmproj (n_embd = %d)\n"
                 "hint: you may be using wrong mmproj\n",
                 n_embd_text, n_embd_mmproj));
@@ -771,7 +786,7 @@ struct mtmd_context {
         if (ctx_gen_a) {
             int n_embd_gen = ctx_gen_a->model.n_mmproj_embd();
             if (n_embd_text > 0 && n_embd_text != n_embd_gen) {
-                throw std::runtime_error(string_format(
+                throw std::runtime_error(mtmd_format(
                     "mismatch between text model (n_embd = %d) and gen-audio mmproj (n_embd = %d)\n"
                     "hint: you may be using wrong mmproj\n",
                     n_embd_text, n_embd_gen));
@@ -836,7 +851,7 @@ struct mtmd_context {
                         ov_img_first      = true;
 
                     } else if (minicpmv_version != 0) {
-                        throw std::runtime_error(string_format("unsupported minicpmv version: %d\n", minicpmv_version));
+                        throw std::runtime_error(mtmd_format("unsupported minicpmv version: %d\n", minicpmv_version));
                     }
                     image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(models.v.get());
                 } break;
@@ -1148,7 +1163,7 @@ struct mtmd_context {
                     image_preproc = std::make_unique<mtmd_image_preprocessor_granite>(models.v.get());
                 } break;
             default:
-                throw std::runtime_error(string_format("%s: unexpected vision projector type %d\n", __func__, proj));
+                throw std::runtime_error(mtmd_format("%s: unexpected vision projector type %d\n", __func__, proj));
         }
 
         GGML_ASSERT(image_preproc != nullptr);
@@ -1274,7 +1289,7 @@ struct mtmd_context {
                     audio_preproc = std::make_unique<mtmd_audio_preprocessor_chatterbox_ref>(models.a.get(), cbx_is_mtl(*models.a));
                 } break;
             default:
-                throw std::runtime_error(string_format("%s: unexpected audio projector type %d\n", __func__, proj));
+                throw std::runtime_error(mtmd_format("%s: unexpected audio projector type %d\n", __func__, proj));
         }
 
         // initialize audio preprocessor
@@ -1436,7 +1451,7 @@ struct mtmd_tokenizer {
         for (const auto & part : parts_str) {
             if (part == ctx->media_marker) {
                 if (i_bm >= bitmaps.size()) {
-                    throw std::runtime_error(string_format("number of media markers in text (%zu) exceeds number of bitmaps (%zu)", i_bm + 1, bitmaps.size()));
+                    throw std::runtime_error(mtmd_format("number of media markers in text (%zu) exceeds number of bitmaps (%zu)", i_bm + 1, bitmaps.size()));
                 }
                 parts.push_back({"", bitmaps[i_bm++]});
             } else {
@@ -1451,7 +1466,7 @@ struct mtmd_tokenizer {
             }
         }
         if (n_markers != bitmaps.size()) {
-            throw std::runtime_error(string_format("number of media markers in text (%zu) does not match number of bitmaps (%zu)", n_markers, bitmaps.size()));
+            throw std::runtime_error(mtmd_format("number of media markers in text (%zu) does not match number of bitmaps (%zu)", n_markers, bitmaps.size()));
         }
 
         expand_lazy_bitmaps();
@@ -1496,7 +1511,7 @@ struct mtmd_tokenizer {
                                     &out_bm,
                                     &out_str);
                     if (out_bm && out_str) {
-                        throw std::runtime_error(string_format("lazy callback cannot return both bitmap and text"));
+                        throw std::runtime_error(mtmd_format("lazy callback cannot return both bitmap and text"));
                     }
                     if (res == 0) {
                         // OK, append the returned chunk; lazy part is not yet added
@@ -1520,7 +1535,7 @@ struct mtmd_tokenizer {
                         break;
                     } else if (res == -2) {
                         // error
-                        throw std::runtime_error(string_format("lazy callback returned error"));
+                        throw std::runtime_error(mtmd_format("lazy callback returned error"));
                     }
                 }
                 if (!st.pieces.empty()) {
@@ -1654,7 +1669,7 @@ struct mtmd_tokenizer {
                 add_text(ctx->img_beg, true); // add image begin token
             }
             if (ctx->has_prompt_img_size) {
-                add_text(string_format("image %ux%u<|media_content|>", bitmaps[0]->nx, bitmaps[0]->ny), true);
+                add_text(mtmd_format("image %ux%u<|media_content|>", bitmaps[0]->nx, bitmaps[0]->ny), true);
             }
 
             // TODO @ngxson : this is quite hacky because preprocessor only support batch with one single element, that need to be fixed in the future (e.g. by changing the preprocessor interface always take single input)
@@ -1764,7 +1779,7 @@ struct mtmd_tokenizer {
                             auto & curr_chunk = chunks[y * n_col + x];
                             auto & curr_batch = curr_chunk.tokens_image->batch_f32;
                             if (curr_batch.entries.size() != 1) {
-                                throw std::runtime_error(string_format("%s: expect 1 image in batch_f32", __func__));
+                                throw std::runtime_error(mtmd_format("%s: expect 1 image in batch_f32", __func__));
                             }
 
                             LOG_DBG("%s: adding slice image at row %d col %d\n", __func__, y, x);
@@ -2158,14 +2173,14 @@ struct mtmd_tokenizer {
         // overview image first
         auto & overview = preproc_out.overview;
         if (overview.nx() == 0 || overview.ny() == 0) {
-            throw std::runtime_error(string_format("%s: invalid overview image for llava-uhd style preprocessing\n", __func__));
+            throw std::runtime_error(mtmd_format("%s: invalid overview image for llava-uhd style preprocessing\n", __func__));
         }
         process_chunk(std::move(preproc_out.overview));
 
         // then, process slices
         for (auto & entry : preproc_out.entries) {
             if (entry.nx() == 0 || entry.ny() == 0) {
-                throw std::runtime_error(string_format("%s: invalid image slice for llava-uhd style preprocessing\n", __func__));
+                throw std::runtime_error(mtmd_format("%s: invalid image slice for llava-uhd style preprocessing\n", __func__));
             }
             process_chunk(std::move(entry));
         }

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetails.svelte

@@ -35,17 +35,23 @@
 		return tags.find((t) => t.startsWith('license:'))?.replace('license:', '') ?? null;
 	});
 
-	// Capabilities derived from HF metadata. Vision comes from an mmproj sidecar
-	// or a multimodal pipeline tag; tool use / reasoning from the chat template.
-	let hasMmproj = $derived(
-		files.some(
+	// Capabilities derived from HF metadata. Vision and audio come from the
+	// pipeline tag and the mmproj; tool use / reasoning from the chat template.
+	let mmprojPath = $derived(
+		files.find(
 			(f) => HuggingFaceService.extractQuantMeta(f.path)?.sidecar === ModelAuxSidecar.MMPROJ
-		)
+		)?.path
+	);
+	let modalities = $derived(
+		HuggingFaceService.getModalities(modelId, details?.pipeline_tag, mmprojPath)
 	);
-	let hasVision = $derived(hasMmproj || details?.pipeline_tag === 'image-text-to-text');
 	let hasTools = $derived(detectToolUseSupport(gguf?.chat_template ?? ''));
 	let hasReasoning = $derived(detectThinkingSupport(gguf?.chat_template ?? ''));
 
+	$effect(() => {
+		if (mmprojPath) void HuggingFaceService.getMmprojEncoders(modelId, mmprojPath);
+	});
+
 	let bitDepthRows = $derived.by<ModelBitDepthRow[]>(() => {
 		const rows = new SvelteMap<number, HfModelSibling[]>();
 
@@ -84,8 +90,8 @@
 			{gguf}
 			{hasReasoning}
 			{hasTools}
-			{hasVision}
 			{licenseTag}
+			{modalities}
 			{modelId}
 		/>
 

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHeader.svelte

@@ -8,6 +8,7 @@
 	import { Button } from '$lib/components/ui/button';
 	import { HuggingFaceService } from '$lib/services';
 	import type { HfModelDetailInfo, HfModelGguf } from '$lib/types/huggingface';
+	import type { ModelModalities } from '$lib/types/models';
 	import { orgOf } from '$lib/utils';
 
 	interface Props {
@@ -16,13 +17,21 @@
 		gguf?: HfModelGguf;
 		baseModels: string[];
 		licenseTag: string | null;
-		hasVision: boolean;
+		modalities: ModelModalities;
 		hasTools: boolean;
 		hasReasoning: boolean;
 	}
 
-	let { baseModels, details, gguf, hasReasoning, hasTools, hasVision, licenseTag, modelId }: Props =
-		$props();
+	let {
+		baseModels,
+		details,
+		gguf,
+		hasReasoning,
+		hasTools,
+		licenseTag,
+		modalities,
+		modelId
+	}: Props = $props();
 
 	// Avatar shows the base model's org (e.g. the Qwen logo for a ggml-org GGUF)
 	// with the quant org as a corner badge when they differ.
@@ -50,7 +59,7 @@
 					<ModelCapabilityIcons
 						gapClass="gap-2"
 						iconSize="h-4 w-4"
-						modalities={{ audio: false, video: false, vision: hasVision }}
+						{modalities}
 						supportsThinking={hasReasoning}
 						supportsToolUse={hasTools}
 					/>

tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItem.svelte

@@ -5,7 +5,7 @@
 	import ModelOrgAvatar from '../../ModelOrgAvatar.svelte';
 	import { Pause, Play, X } from '@lucide/svelte';
 	import { ActionIcon } from '$lib/components/app';
-	import { HF_MMPROJ_FILENAME_TOKEN, HF_MODALITY_PIPELINE_TAGS, MODEL_ID } from '$lib/constants';
+	import { HF_MMPROJ_FILENAME_TOKEN, MODEL_ID } from '$lib/constants';
 	import { ModelDraftSidecar, ModelRowDownloadState } from '$lib/enums';
 	import { HuggingFaceService } from '$lib/services';
 	import { modelsDiscoverStore, modelsStore } from '$lib/stores';
@@ -96,19 +96,19 @@
 
 	let supportsToolUse = $derived(detectToolUseSupport(model.gguf?.chat_template ?? ''));
 
-	// Modalities derived from HF metadata: vision from an mmproj sidecar or a
-	// multimodal pipeline tag, audio/video from their pipeline tags.
-	let modalities = $derived.by<ModelModalities>(() => {
-		const tag = model.pipeline_tag ?? '';
-		const vision =
-			HF_MODALITY_PIPELINE_TAGS.vision.includes(tag) ||
-			Boolean(
-				model.siblings?.some((s) => s.rfilename.toLowerCase().includes(HF_MMPROJ_FILENAME_TOKEN))
-			);
-		const audio = HF_MODALITY_PIPELINE_TAGS.audio.includes(tag);
-		const video = HF_MODALITY_PIPELINE_TAGS.video.includes(tag);
+	let mmprojPath = $derived(
+		model.siblings?.find((s) => s.rfilename.toLowerCase().includes(HF_MMPROJ_FILENAME_TOKEN))
+			?.rfilename
+	);
 
-		return { audio, video, vision };
+	// Modalities derived from HF metadata: the pipeline tag and the mmproj.
+	let modalities = $derived<ModelModalities>(
+		HuggingFaceService.getModalities(model.id, model.pipeline_tag, mmprojPath)
+	);
+
+	// the mmproj's header says whether it also hears; read once per repo
+	$effect(() => {
+		if (mmprojPath) void HuggingFaceService.getMmprojEncoders(model.id, mmprojPath);
 	});
 
 	// Draft sidecars (mtp, dflash, dspark, eagle3) the repo offers, badged with no

tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigation.svelte

@@ -648,12 +648,16 @@
 				{/if}
 			</div>
 
+			<!-- the hidden strip's content goes inert, not only invisible: descendants can opt
+			     back into visibility (an open CollapsibleRegion) and pointer events (the inline
+			     style of a context-menu trigger), and would catch clicks meant for the page -->
 			<div
 				in:fade={{ duration: SIDEBAR_CONTENT_FADE_MS }}
 				out:fade={{ duration: SIDEBAR_CONTENT_FADE_MS }}
 				class="mt-2 flex min-h-0 flex-1 flex-col gap-7 max-md:mt-6 md:gap-1 md:transition-opacity md:duration-150 md:ease-out {isStripHidden
 					? 'md:opacity-0 md:invisible'
 					: ''}"
+				inert={isStripHidden}
 			>
 				<!-- a phone has no footer: its items join these -->
 				<SidebarNavigationActions

tools/ui/src/lib/constants/huggingface.constants.ts

@@ -43,6 +43,7 @@ export const HF_MODEL_LIST_EXPAND: readonly string[] = [
 export const HF_MAIN_BRANCH = 'main';
 export const HF_README_FILENAME = 'README.md';
 export const HF_RAW_PATH = 'raw';
+export const HF_RESOLVE_PATH = 'resolve';
 export const HF_TREE_PATH = 'tree';
 
 // Pagination
@@ -55,7 +56,10 @@ export const HF_LINK_HEADER = 'Link';
 
 export const HF_RETRY_ATTEMPTS = 3;
 export const HF_RETRY_DELAY_MS = 1000;
+export const HF_HTTP_PARTIAL_CONTENT = 206;
 export const HF_HTTP_NOT_FOUND = 404;
+/** Unauthorized, forbidden, not found: a gated, private or missing file, which a retry won't change. */
+export const HF_HTTP_NO_ACCESS_STATUSES: readonly number[] = [401, 403, HF_HTTP_NOT_FOUND];
 export const HF_HTTP_SERVER_ERROR_MIN = 500;
 
 // Search limits
@@ -129,6 +133,13 @@ export const HF_MODALITY_PIPELINE_TAGS: Readonly<
 /** Filename token marking an mmproj sidecar sibling (unlocks vision / audio). */
 export const HF_MMPROJ_FILENAME_TOKEN = 'mmproj';
 
+/** Bytes read from the start of an mmproj to reach its metadata, which comes before the tensors. */
+export const HF_MMPROJ_HEADER_BYTES = 65_536;
+
+/** mmproj metadata keys saying which encoders it carries (tools/mtmd/mtmd-impl.h). */
+export const GGUF_KEY_HAS_VISION_ENCODER = 'clip.has_vision_encoder';
+export const GGUF_KEY_HAS_AUDIO_ENCODER = 'clip.has_audio_encoder';
+
 export const HF_TASK_TAGS: readonly string[] = [
 	'audio-classification',
 	'audio-to-audio',

tools/ui/src/lib/services/huggingface.service.ts

@@ -1,4 +1,9 @@
-import { PATH_SEPARATOR, SETTINGS_KEYS } from '$lib/constants';
+import {
+	GGUF_KEY_HAS_AUDIO_ENCODER,
+	GGUF_KEY_HAS_VISION_ENCODER,
+	PATH_SEPARATOR,
+	SETTINGS_KEYS
+} from '$lib/constants';
 import {
 	BYTE,
 	BYTE_LABEL,
@@ -24,19 +29,24 @@ import {
 	HF_GATED_TAG,
 	HF_GGUF_FILTER,
 	HF_GGUF_TAG,
+	HF_HTTP_NO_ACCESS_STATUSES,
 	HF_HTTP_NOT_FOUND,
+	HF_HTTP_PARTIAL_CONTENT,
 	HF_HTTP_SERVER_ERROR_MIN,
 	HF_LICENSE_TAG_PREFIX,
 	HF_LINK_HEADER,
 	HF_LINK_NEXT_REGEX,
 	HF_MAIN_BRANCH,
 	HF_MAX_LIMIT,
+	HF_MMPROJ_HEADER_BYTES,
+	HF_MODALITY_PIPELINE_TAGS,
 	HF_MODEL_LIST_EXPAND,
 	HF_PARAM_COUNT_REGEX,
 	HF_QUANT_PRECISION_REGEX,
 	HF_RAW_PATH,
 	HF_README_FILENAME,
 	HF_RECURSIVE_TREE_PARAM,
+	HF_RESOLVE_PATH,
 	HF_RETRY_ATTEMPTS,
 	HF_RETRY_DELAY_MS,
 	HF_SAFETENSORS_TAG,
@@ -69,12 +79,14 @@ import { HfEntryType, HfModelSort, SidecarForm } from '$lib/enums';
 import { settingsStore } from '$lib/stores/settings/index.svelte';
 import type {
 	HfCatalogEntry,
+	HfMmprojEncoders,
 	HfModelDetailInfo,
 	HfModelInfo,
 	HfModelSearchParams,
 	HfModelSibling
 } from '$lib/types/huggingface';
-import { sidecarFromFileToken } from '$lib/utils';
+import type { ModelModalities } from '$lib/types/models';
+import { readGgufMetadata, sidecarFromFileToken } from '$lib/utils';
 import { SvelteMap } from 'svelte/reactivity';
 
 /** Fetch failure carrying the HTTP status, so retry logic tests the code instead of the message. */
@@ -107,6 +119,11 @@ export class HuggingFaceService {
 
 	private static detailsPending = new Map<string, Promise<HfModelDetailInfo | null>>();
 
+	// Encoders an mmproj carries, keyed by repo id: its header never changes, so one read per session.
+	private static mmprojEncodersCache = new SvelteMap<string, HfMmprojEncoders>();
+
+	private static mmprojEncodersPending = new Map<string, Promise<HfMmprojEncoders | null>>();
+
 	/**
 	 * Map of quant token to its average bit-depth in bits-per-weight (bpw).
 	 */
@@ -160,8 +177,6 @@ export class HuggingFaceService {
 		return HuggingFaceService.detailsCache.get(hfRepoId);
 	}
 
-	// GGUF Model Browsing
-
 	/**
 	 * Collapse split GGUF shard sets (`-00001-of-00015.gguf`, ...) to their first
 	 * shard, summing every shard's size so the kept entry reflects the whole
@@ -200,6 +215,8 @@ export class HuggingFaceService {
 		return result;
 	}
 
+	// GGUF Model Browsing
+
 	/**
 	 * Extract the GGUF quantization token (e.g. `Q4_K_M`) and any sidecar type
 	 * (`mtp`, `dflash`, `mmproj`, ...) from a `.gguf` filename.
@@ -355,8 +372,6 @@ export class HuggingFaceService {
 		return `${Math.floor(diffDays / DAYS_PER_YEAR)} ${YEARS_AGO_LABEL}`;
 	}
 
-	// Model Details & Files
-
 	/**
 	 * Format a min-max size range with one shared unit, e.g. `19.0-28.6 GB`.
 	 */
@@ -382,6 +397,8 @@ export class HuggingFaceService {
 		return `${fmt(min)}-${fmt(max)} ${unit}`;
 	}
 
+	// Model Details & Files
+
 	/**
 	 * Avatar URL for an author (org or user). 404s when the author does not
 	 * exist, so callers should provide a fallback.
@@ -544,6 +561,97 @@ export class HuggingFaceService {
 		return promise;
 	}
 
+	/**
+	 * Which encoders a repo's mmproj carries, from the metadata at the start of the file: a ranged read of its
+	 * first bytes, so the file itself is never downloaded. The pipeline tag can't say this: omni models are
+	 * `any-to-any`, and their audio encoder lives in the same mmproj as the vision one. Null when the read fails.
+	 */
+	static getMmprojEncoders(modelId: string, mmprojPath: string): Promise<HfMmprojEncoders | null> {
+		if (!HuggingFaceService.isEnabled()) return Promise.resolve(null);
+
+		const cached = HuggingFaceService.mmprojEncodersCache.get(modelId);
+
+		if (cached) return Promise.resolve(cached);
+
+		const pending = HuggingFaceService.mmprojEncodersPending.get(modelId);
+
+		if (pending) return pending;
+
+		// Do not encode the modelId, it contains slashes for author/name
+		const url = `${HF_BASE_URL}${PATH_SEPARATOR}${modelId}${PATH_SEPARATOR}${HF_RESOLVE_PATH}${PATH_SEPARATOR}${HF_MAIN_BRANCH}${PATH_SEPARATOR}${mmprojPath.split(PATH_SEPARATOR).map(encodeURIComponent).join(PATH_SEPARATOR)}`;
+		const promise = (async () => {
+			const response = await fetch(url, {
+				headers: { Range: `bytes=0-${HF_MMPROJ_HEADER_BYTES - 1}` }
+			});
+
+			// anything but a partial response would stream the whole file, which can be gigabytes
+			if (response.status !== HF_HTTP_PARTIAL_CONTENT) {
+				void response.body?.cancel();
+
+				// a gated, private or missing file answers the same on every read: keep what the repo says
+				if (HF_HTTP_NO_ACCESS_STATUSES.includes(response.status)) {
+					const unknown: HfMmprojEncoders = { audio: false, vision: null };
+
+					HuggingFaceService.mmprojEncodersCache.set(modelId, unknown);
+
+					return unknown;
+				}
+
+				throw new Error(`Failed to read ${mmprojPath}: ${response.status}`);
+			}
+
+			const { complete, metadata } = readGgufMetadata(await response.arrayBuffer());
+
+			if (metadata.size === 0) throw new Error(`${mmprojPath} has no GGUF metadata`);
+
+			// the converter writes these keys only when true, so one missing from a header read in full means no
+			// such encoder; a header cut short leaves vision unknown
+			const hasVision = metadata.get(GGUF_KEY_HAS_VISION_ENCODER) === true;
+			const encoders: HfMmprojEncoders = {
+				audio: metadata.get(GGUF_KEY_HAS_AUDIO_ENCODER) === true,
+				vision: hasVision || (complete ? false : null)
+			};
+
+			HuggingFaceService.mmprojEncodersCache.set(modelId, encoders);
+
+			return encoders;
+		})()
+			.catch((error: unknown) => {
+				// not cached: a rate limited or failed read should retry on the next mount
+				console.error(`Error reading the mmproj of ${modelId}:`, error);
+
+				return null;
+			})
+			.finally(() => HuggingFaceService.mmprojEncodersPending.delete(modelId));
+
+		HuggingFaceService.mmprojEncodersPending.set(modelId, promise);
+
+		return promise;
+	}
+
+	/**
+	 * Modalities a repo offers, from its pipeline tag and its mmproj: vision from a vision tag or an mmproj
+	 * (unless its header has no vision encoder, as in audio-only models), audio from an audio tag or
+	 * an audio encoder in the mmproj (omni models are `any-to-any`), video from a video tag. Reactive: it
+	 * updates once {@link getMmprojEncoders} has read the mmproj.
+	 */
+	static getModalities(
+		modelId: string,
+		pipelineTag: string | null | undefined,
+		mmprojPath: string | undefined
+	): ModelModalities {
+		const tag = pipelineTag ?? '';
+		const encoders = mmprojPath ? HuggingFaceService.mmprojEncodersCache.get(modelId) : undefined;
+
+		return {
+			audio: HF_MODALITY_PIPELINE_TAGS.audio.includes(tag) || encoders?.audio === true,
+			video: HF_MODALITY_PIPELINE_TAGS.video.includes(tag),
+			vision:
+				HF_MODALITY_PIPELINE_TAGS.vision.includes(tag) ||
+				(Boolean(mmprojPath) && encoders?.vision !== false)
+		};
+	}
+
 	static getModelUrl(modelId: string): string {
 		return `${HF_BASE_URL}${PATH_SEPARATOR}${modelId}`;
 	}

tools/ui/src/lib/types/huggingface.d.ts

@@ -172,6 +172,12 @@ export interface HfModelDetailInfo extends HfModelInfo {
 }
 
 /** A single entry in a model repository's file tree (`/tree` responses) */
+/** Encoders an mmproj's header says it carries; vision is null when the header was cut short before it said. */
+export interface HfMmprojEncoders {
+	audio: boolean;
+	vision: boolean | null;
+}
+
 export interface HfModelSibling {
 	/** Relative path of the file or directory within the repo */
 	path: string;

tools/ui/src/lib/types/index.ts

@@ -47,6 +47,7 @@ export type {
 	HfModelGguf,
 	HfModelInfo,
 	HfModelSearchParams,
+	HfMmprojEncoders,
 	HfModelSibling,
 	HfModelSiblingRef
 } from './huggingface';

tools/ui/src/lib/utils/gguf-metadata.ts

@@ -0,0 +1,153 @@
+/**
+ * Reads the metadata key/value pairs at the start of a GGUF file, from as many bytes of it as are given.
+ * Format: https://github.com/ggml-org/ggml/blob/master/docs/gguf.md
+ */
+
+const GGUF_MAGIC = 0x46554747; // "GGUF", little-endian
+
+enum GgufType {
+	UINT8 = 0,
+	INT8 = 1,
+	UINT16 = 2,
+	INT16 = 3,
+	UINT32 = 4,
+	INT32 = 5,
+	FLOAT32 = 6,
+	BOOL = 7,
+	STRING = 8,
+	ARRAY = 9,
+	UINT64 = 10,
+	INT64 = 11,
+	FLOAT64 = 12
+}
+
+const SCALAR_SIZE: Partial<Record<GgufType, number>> = {
+	[GgufType.BOOL]: 1,
+	[GgufType.FLOAT32]: 4,
+	[GgufType.FLOAT64]: 8,
+	[GgufType.INT8]: 1,
+	[GgufType.INT16]: 2,
+	[GgufType.INT32]: 4,
+	[GgufType.INT64]: 8,
+	[GgufType.UINT8]: 1,
+	[GgufType.UINT16]: 2,
+	[GgufType.UINT32]: 4,
+	[GgufType.UINT64]: 8
+};
+
+export type GgufValue = boolean | number | bigint | string;
+
+export interface GgufMetadata {
+	/** True when every pair was read; false when the buffer ended first or is not GGUF. */
+	complete: boolean;
+	metadata: Map<string, GgufValue>;
+}
+
+/**
+ * The scalar and string metadata of a GGUF file (arrays are skipped). A buffer cut short returns the pairs read
+ * before the cut; a buffer that is not GGUF returns an empty map.
+ */
+export function readGgufMetadata(buffer: ArrayBuffer): GgufMetadata {
+	const view = new DataView(buffer);
+	const decoder = new TextDecoder();
+	const metadata = new Map<string, GgufValue>();
+
+	let offset = 0;
+
+	const fits = (bytes: number) => offset + bytes <= view.byteLength;
+	const readString = (): string | null => {
+		if (!fits(8)) return null;
+
+		const length = Number(view.getBigUint64(offset, true));
+
+		if (!fits(8 + length)) return null;
+
+		const text = decoder.decode(new Uint8Array(buffer, offset + 8, length));
+
+		offset += 8 + length;
+
+		return text;
+	};
+	const readScalar = (type: GgufType): GgufValue | null => {
+		const size = SCALAR_SIZE[type];
+
+		if (size === undefined || !fits(size)) return null;
+
+		const at = offset;
+
+		offset += size;
+
+		switch (type) {
+			case GgufType.UINT8:
+				return view.getUint8(at);
+			case GgufType.INT8:
+				return view.getInt8(at);
+			case GgufType.UINT16:
+				return view.getUint16(at, true);
+			case GgufType.INT16:
+				return view.getInt16(at, true);
+			case GgufType.UINT32:
+				return view.getUint32(at, true);
+			case GgufType.INT32:
+				return view.getInt32(at, true);
+			case GgufType.FLOAT32:
+				return view.getFloat32(at, true);
+			case GgufType.BOOL:
+				return view.getUint8(at) !== 0;
+			case GgufType.UINT64:
+				return view.getBigUint64(at, true);
+			case GgufType.INT64:
+				return view.getBigInt64(at, true);
+			default:
+				return view.getFloat64(at, true);
+		}
+	};
+	// moves past an array's elements; false when the buffer ends inside it
+	const skipArray = (): boolean => {
+		if (!fits(12)) return false;
+
+		const type = view.getUint32(offset, true) as GgufType;
+		const count = Number(view.getBigUint64(offset + 4, true));
+
+		offset += 12;
+
+		for (let i = 0; i < count; i++) {
+			const value = type === GgufType.STRING ? readString() : readScalar(type);
+
+			if (value === null) return false;
+		}
+
+		return true;
+	};
+
+	// magic, version, tensor count, metadata count
+	if (!fits(24) || view.getUint32(0, true) !== GGUF_MAGIC) return { complete: false, metadata };
+
+	const count = Number(view.getBigUint64(16, true));
+
+	offset = 24;
+
+	for (let i = 0; i < count; i++) {
+		const key = readString();
+
+		if (key === null || !fits(4)) return { complete: false, metadata };
+
+		const type = view.getUint32(offset, true) as GgufType;
+
+		offset += 4;
+
+		if (type === GgufType.ARRAY) {
+			if (!skipArray()) return { complete: false, metadata };
+
+			continue;
+		}
+
+		const value = type === GgufType.STRING ? readString() : readScalar(type);
+
+		if (value === null) return { complete: false, metadata };
+
+		metadata.set(key, value);
+	}
+
+	return { complete: true, metadata };
+}

tools/ui/src/lib/utils/index.ts

@@ -366,6 +366,9 @@ export { detectToolUseSupport } from './chat-template-tool-detector';
 // Model memory estimation
 export { minMemoryTierGb } from './model-compatibility';
 
+// GGUF file metadata, e.g. which encoders an mmproj carries
+export { type GgufMetadata, type GgufValue, readGgufMetadata } from './gguf-metadata';
+
 // Cryptography utilities
 
 export { uuid } from './uuid';

tools/ui/tests/client/components/SidebarNavigationHarness.svelte

@@ -0,0 +1,9 @@
+<script lang="ts">
+	import SidebarNavigation from '$lib/components/app/navigation/SidebarNavigation/SidebarNavigation.svelte';
+	import * as Tooltip from '$lib/components/ui/tooltip';
+</script>
+
+<!-- the sidebar with the tooltip provider +layout.svelte gives it -->
+<Tooltip.Provider>
+	<SidebarNavigation />
+</Tooltip.Provider>

tools/ui/tests/client/sidebar-hidden-clicks.svelte.test.ts

@@ -0,0 +1,66 @@
+// A hidden desktop sidebar keeps its full width in the layout, floated under the page's
+// left edge, so whatever it still lets the pointer hit there steals the page's clicks
+// (#31971). Hiding it by inherited `visibility` alone is not enough: an open section's
+// CollapsibleRegion sets `visibility: visible` again, and bits-ui's context-menu triggers
+// put `pointer-events: auto` inline on the rows.
+
+import '../../src/app.css';
+import SidebarNavigationHarness from './components/SidebarNavigationHarness.svelte';
+import { uiStore } from '$lib/stores';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { page } from 'vitest/browser';
+import { render } from 'vitest-browser-svelte';
+
+const GRID_STEP_PX = 8;
+
+/** Elements of the sidebar the pointer lands on over its box, the expand toggle aside. */
+function hitTargetsInSidebar(aside: HTMLElement): Element[] {
+	const toggle = aside.querySelector('[aria-label="Expand navigation"]');
+	const box = aside.getBoundingClientRect();
+	const hits = new Set<Element>();
+
+	for (let x = box.left + 1; x < box.right; x += GRID_STEP_PX) {
+		for (let y = box.top + 1; y < box.bottom; y += GRID_STEP_PX) {
+			const el = document.elementFromPoint(x, y);
+
+			if (el && aside.contains(el) && !toggle?.contains(el)) hits.add(el);
+		}
+	}
+
+	return [...hits];
+}
+
+describe('hidden sidebar', () => {
+	let wasExpanded: boolean;
+
+	beforeEach(async () => {
+		await page.viewport(1280, 900);
+		wasExpanded = uiStore.isSidebarExpanded;
+	});
+
+	afterEach(() => {
+		uiStore.isSidebarExpanded = wasExpanded;
+	});
+
+	it('lets clicks over its strip through to the page', async () => {
+		uiStore.isSidebarExpanded = false;
+		render(SidebarNavigationHarness);
+
+		const aside = await vi.waitFor(() => {
+			const el = document.querySelector<HTMLElement>('aside');
+
+			if (!el) throw new Error('sidebar not rendered');
+
+			return el;
+		});
+
+		// an open section is what used to bring the rows back to the pointer
+		await vi.waitFor(() =>
+			expect(aside.querySelector('.collapsible-region-content')).not.toBeNull()
+		);
+		// its width settles over a transition; the strip's area is what counts
+		await vi.waitFor(() => expect(aside.getBoundingClientRect().width).toBeGreaterThan(48));
+
+		expect(hitTargetsInSidebar(aside).map((el) => el.outerHTML.slice(0, 120))).toEqual([]);
+	});
+});

tools/ui/tests/unit/gguf-metadata.test.ts

@@ -0,0 +1,98 @@
+import { readGgufMetadata } from '$lib/utils/gguf-metadata';
+import { describe, expect, it } from 'vitest';
+
+type Entry =
+	| [key: string, type: 'bool', value: boolean]
+	| [key: string, type: 'u32', value: number]
+	| [key: string, type: 'string', value: string]
+	| [key: string, type: 'string[]', value: string[]]
+	| [key: string, type: 'f32[]', value: number[]];
+
+/** A GGUF header with the given metadata and no tensors. */
+function gguf(entries: Entry[]): ArrayBuffer {
+	const bytes: number[] = [];
+	const u32 = (n: number) => bytes.push(...new Uint8Array(new Uint32Array([n]).buffer));
+	const u64 = (n: number) => bytes.push(...new Uint8Array(new BigUint64Array([BigInt(n)]).buffer));
+	const str = (s: string) => {
+		const encoded = new TextEncoder().encode(s);
+
+		u64(encoded.length);
+		bytes.push(...encoded);
+	};
+
+	u32(0x46554747);
+	u32(3);
+	u64(0);
+	u64(entries.length);
+
+	for (const [key, type, value] of entries) {
+		str(key);
+
+		if (type === 'bool') {
+			u32(7);
+			bytes.push(value ? 1 : 0);
+		} else if (type === 'u32') {
+			u32(4);
+			u32(value);
+		} else if (type === 'string') {
+			u32(8);
+			str(value);
+		} else if (type === 'string[]') {
+			u32(9);
+			u32(8);
+			u64(value.length);
+			value.forEach(str);
+		} else {
+			u32(9);
+			u32(6);
+			u64(value.length);
+			value.forEach((n) => bytes.push(...new Uint8Array(new Float32Array([n]).buffer)));
+		}
+	}
+
+	return new Uint8Array(bytes).buffer;
+}
+
+const MMPROJ: Entry[] = [
+	['general.architecture', 'string', 'clip'],
+	['clip.projector_type', 'string', 'gemma4'],
+	['clip.vision.image_mean', 'f32[]', [0.5, 0.5, 0.5]],
+	['tokenizer.ggml.tokens', 'string[]', ['<a>', '<b>']],
+	['clip.has_vision_encoder', 'bool', true],
+	['clip.has_audio_encoder', 'bool', true],
+	['clip.audio.num_mel_bins', 'u32', 128]
+];
+
+describe('readGgufMetadata', () => {
+	it('reads scalars and strings, skipping arrays', () => {
+		const { complete, metadata } = readGgufMetadata(gguf(MMPROJ));
+
+		expect(complete).toBe(true);
+
+		expect(metadata.get('general.architecture')).toBe('clip');
+		expect(metadata.get('clip.has_vision_encoder')).toBe(true);
+		expect(metadata.get('clip.has_audio_encoder')).toBe(true);
+		expect(metadata.get('clip.audio.num_mel_bins')).toBe(128);
+		expect(metadata.has('clip.vision.image_mean')).toBe(false);
+	});
+
+	it('returns what it read before a buffer cut short', () => {
+		const full = gguf(MMPROJ);
+		const { complete, metadata } = readGgufMetadata(full.slice(0, full.byteLength - 20));
+
+		expect(complete).toBe(false);
+
+		expect(metadata.get('clip.has_vision_encoder')).toBe(true);
+		expect(metadata.has('clip.audio.num_mel_bins')).toBe(false);
+	});
+
+	it('returns nothing for a file that is not GGUF', () => {
+		expect(
+			readGgufMetadata(new TextEncoder().encode('not a gguf file at all...').buffer)
+		).toStrictEqual({ complete: false, metadata: new Map() });
+		expect(readGgufMetadata(new ArrayBuffer(0))).toStrictEqual({
+			complete: false,
+			metadata: new Map()
+		});
+	});
+});

tools/ui/tests/unit/hf-mmproj-modalities.test.ts

@@ -0,0 +1,148 @@
+import { HuggingFaceService } from '$lib/services/huggingface.service';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+/** A GGUF header with the given bool metadata and no tensors. */
+function mmprojHeader(entries: Record<string, boolean>): ArrayBuffer {
+	const bytes: number[] = [];
+	const u32 = (n: number) => bytes.push(...new Uint8Array(new Uint32Array([n]).buffer));
+	const u64 = (n: number) => bytes.push(...new Uint8Array(new BigUint64Array([BigInt(n)]).buffer));
+
+	u32(0x46554747);
+	u32(3);
+	u64(0);
+	u64(Object.keys(entries).length);
+
+	for (const [key, value] of Object.entries(entries)) {
+		const encoded = new TextEncoder().encode(key);
+
+		u64(encoded.length);
+		bytes.push(...encoded);
+		u32(7);
+		bytes.push(value ? 1 : 0);
+	}
+
+	return new Uint8Array(bytes).buffer;
+}
+
+function respond(body: ArrayBuffer, status: number) {
+	const fetchMock = vi.fn(async () => new Response(body, { status }));
+
+	vi.stubGlobal('fetch', fetchMock);
+
+	return fetchMock;
+}
+
+describe('HuggingFaceService mmproj modalities', () => {
+	beforeEach(() => {
+		vi.spyOn(HuggingFaceService, 'isEnabled').mockReturnValue(true);
+		vi.spyOn(console, 'error').mockImplementation(() => {});
+	});
+
+	afterEach(() => {
+		vi.restoreAllMocks();
+		vi.unstubAllGlobals();
+	});
+
+	it('takes audio from the pipeline tag and vision from the mmproj before its header is read', () => {
+		expect(HuggingFaceService.getModalities('org/unread', 'text-to-speech', 'mmproj.gguf')).toEqual(
+			{ audio: true, video: false, vision: true }
+		);
+		expect(HuggingFaceService.getModalities('org/plain', 'text-generation', undefined)).toEqual({
+			audio: false,
+			video: false,
+			vision: false
+		});
+	});
+
+	it('adds audio for an omni mmproj, with a ranged read of an encoded path', async () => {
+		const fetchMock = respond(
+			mmprojHeader({ 'clip.has_audio_encoder': true, 'clip.has_vision_encoder': true }),
+			206
+		);
+
+		await HuggingFaceService.getMmprojEncoders('org/omni', 'sub dir/mmproj#1.gguf');
+
+		expect(fetchMock).toHaveBeenCalledWith(
+			'https://huggingface.co/org/omni/resolve/main/sub%20dir/mmproj%231.gguf',
+			{ headers: { Range: 'bytes=0-65535' } }
+		);
+		expect(HuggingFaceService.getModalities('org/omni', 'any-to-any', 'mmproj.gguf')).toEqual({
+			audio: true,
+			video: false,
+			vision: true
+		});
+	});
+
+	it('drops vision for an audio-only mmproj', async () => {
+		// the converter writes the vision key only when true
+		respond(mmprojHeader({ 'clip.has_audio_encoder': true }), 206);
+
+		await HuggingFaceService.getMmprojEncoders('org/audio-only', 'mmproj.gguf');
+
+		expect(HuggingFaceService.getModalities('org/audio-only', undefined, 'mmproj.gguf')).toEqual({
+			audio: true,
+			video: false,
+			vision: false
+		});
+	});
+
+	it('keeps vision when a header cut short never said', async () => {
+		const header = mmprojHeader({
+			'clip.has_audio_encoder': true,
+			'clip.has_vision_encoder': true
+		});
+
+		respond(header.slice(0, header.byteLength - 4), 206);
+
+		expect(await HuggingFaceService.getMmprojEncoders('org/cut', 'mmproj.gguf')).toEqual({
+			audio: true,
+			vision: null
+		});
+		expect(HuggingFaceService.getModalities('org/cut', undefined, 'mmproj.gguf').vision).toBe(true);
+	});
+
+	it('keeps vision only for a vision mmproj', async () => {
+		respond(mmprojHeader({ 'clip.has_vision_encoder': true }), 206);
+
+		await HuggingFaceService.getMmprojEncoders('org/vision', 'mmproj.gguf');
+
+		expect(HuggingFaceService.getModalities('org/vision', 'image-text-to-text', 'm.gguf')).toEqual({
+			audio: false,
+			video: false,
+			vision: true
+		});
+	});
+
+	it('keeps what the repo says for a gated mmproj, without reading it again', async () => {
+		const fetchMock = respond(new ArrayBuffer(0), 401);
+
+		expect(await HuggingFaceService.getMmprojEncoders('org/gated', 'mmproj.gguf')).toEqual({
+			audio: false,
+			vision: null
+		});
+		await HuggingFaceService.getMmprojEncoders('org/gated', 'mmproj.gguf');
+
+		expect(fetchMock).toHaveBeenCalledTimes(1);
+		expect(HuggingFaceService.getModalities('org/gated', undefined, 'mmproj.gguf')).toEqual({
+			audio: false,
+			video: false,
+			vision: true
+		});
+	});
+
+	it('rejects a full response that ignored the range, and retries on the next call', async () => {
+		respond(mmprojHeader({ 'clip.has_audio_encoder': true }), 200);
+
+		expect(await HuggingFaceService.getMmprojEncoders('org/no-range', 'mmproj.gguf')).toBeNull();
+		expect(HuggingFaceService.getModalities('org/no-range', undefined, 'mmproj.gguf').audio).toBe(
+			false
+		);
+
+		respond(mmprojHeader({ 'clip.has_audio_encoder': true }), 206);
+
+		expect(await HuggingFaceService.getMmprojEncoders('org/no-range', 'mmproj.gguf')).toEqual({
+			audio: true,
+			vision: false
+		});
+	});
+});

ty.toml

@@ -19,8 +19,9 @@ include = [
 
 [overrides.rules]
 unresolved-reference = "ignore"
-unresolved-import = "ignore"
-unresolved-attribute = "ignore"
+import-not-found = "ignore"
+missing-attribute = "ignore"
+missing-union-attribute = "ignore"
 
 [[overrides]]
 include = [
@@ -30,3 +31,4 @@ include = [
 [overrides.rules]
 unsupported-operator = "ignore"
 not-subscriptable = "ignore"
+invalid-type-form = "ignore"