upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-08-03 00:30:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: enhance the tokenizer's handing of Unicode (#120)

Browse Source
This commit is contained in:
leejet
2023-12-21 00:22:03 +08:00
committed by GitHub
parent 9842a3f819
commit 8f6b4a39d6
6 changed files with 43830 additions and 80117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
model.cpp
View File

@@ -1192,20 +1192,9 @@ ggml_type ModelLoader::get_sd_wtype() {
return GGML_TYPE_COUNT;
}
bool ModelLoader::load_vocab(on_new_token_cb_t on_new_token_cb) {
char* vocab_buffer = reinterpret_cast<char*>(vocab_json);
nlohmann::json vocab = nlohmann::json::parse(vocab_buffer);
std::map<char, int> decoder = unicode_to_byte();
for (auto& it : vocab.items()) {
int token_id = it.value();
std::string token_str = it.key();
std::string token = "";
for (char c : token_str) {
token += decoder[c];
}
on_new_token_cb(token, token_id);
}
return true;
std::string ModelLoader::load_merges() {
std::string merges_utf8_str(reinterpret_cast<const char*>(merges_utf8_c_str), sizeof(merges_utf8_c_str));
return merges_utf8_str;
}
bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb) {