upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-10-03 11:27:43 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: normalize CLIP prompts before special-token splitting (#1670)

Browse Source
This commit is contained in:
leejet
2026-06-17 00:33:00 +08:00
committed by GitHub
parent 92a3b73cdb
commit 7f0e728b7d
4 changed files with 28 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tokenizers/clip_tokenizer.cpp
+4 -3
View File
@@ -22,9 +22,10 @@ CLIPTokenizer::CLIPTokenizer(int pad_token_id, const std::string& merges_utf8_st
EOS_TOKEN_ID = 49407;
PAD_TOKEN_ID = pad_token_id;
end_of_word_suffix = "</w>";
add_bos_token = true;
add_eos_token = true;
end_of_word_suffix = "</w>";
add_bos_token = true;
add_eos_token = true;
normalize_before_split = true;
if (merges_utf8_str.size() > 0) {
load_from_merges(merges_utf8_str);