upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-09-28 17:08:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: add PiD support (#1585)

Browse Source
This commit is contained in:
leejet
2026-05-31 22:38:39 +08:00
committed by GitHub
parent d2797b8667
commit 0982807139
23 changed files with 1387 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tokenizers/bpe_tokenizer.cpp
+2 -1
View File
@@ -182,7 +182,8 @@ std::vector<int> BPETokenizer::encode(const std::string& text, on_new_token_cb_t
unsigned char b = utf8_token_str[i];
char hex_buf[16];
snprintf(hex_buf, sizeof(hex_buf), "<0x%02X>", b);
iter = encoder.find(utf8_to_utf32(hex_buf));
iter = encoder.find(utf8_to_utf32(hex_buf));
token_id = iter != encoder.end() ? iter->second : UNK_TOKEN_ID;
bpe_tokens.push_back(token_id);
token_strs.push_back(hex_buf);
}