upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-09-29 17:38:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: support external Hugging Face tokenizer JSON files (#1973)

Browse Source
This commit is contained in:
leejet
2026-09-15 00:27:39 +08:00
committed by GitHub
parent 42d6c0ab92
commit 4964abdfc5
58 changed files with 97390 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tokenizers/bpe_tokenizer.cpp
+7 -2
View File
@@ -130,7 +130,11 @@ std::vector<std::u32string> BPETokenizer::bpe(const std::u32string& token) const
return word;
}
std::vector<int> BPETokenizer::encode(const std::string& text, on_new_token_cb_t on_new_token_cb) {
bool BPETokenizer::encode(const std::string& text, std::vector<int>& result, on_new_token_cb_t on_new_token_cb, std::string* error) {
result.clear();
if (error) {
error->clear();
}
std::vector<int32_t> bpe_tokens;
std::vector<std::string> token_strs;
@@ -206,7 +210,8 @@ std::vector<int> BPETokenizer::encode(const std::string& text, on_new_token_cb_t
}
ss << "]";
LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", text.c_str(), bpe_tokens.size(), ss.str().c_str());
return bpe_tokens;
result = std::move(bpe_tokens);
return true;
}
std::string BPETokenizer::decode_token(int token_id) const {