upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-10-03 03:17:51 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: use tokenizer-specific pre-tokenization rules (#1975)

Browse Source
This commit is contained in:
leejet
2026-09-15 02:37:52 +08:00
committed by GitHub
parent 07a85c74cb
commit 59c23bce0d
9 changed files with 92 additions and 1021 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tokenizers/gemma_tokenizer.cpp
+3 -1
View File
@@ -46,7 +46,9 @@ void GemmaTokenizer::load_from_merges(const std::string& merges_utf8_str, const
bpe_len = rank;
}
GemmaTokenizer::GemmaTokenizer(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {
GemmaTokenizer::GemmaTokenizer(const std::string& merges_utf8_str, const std::string& vocab_utf8_str)
: BPETokenizer("") {
// Gemma replaces spaces with metaspace before its literal-space Split, so no regex boundaries apply.
byte_level_bpe = false;
byte_fallback = true;
add_bos_token = true;