upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-08-06 02:00:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: enhance the tokenizer's handing of Unicode (#120)

Browse Source
This commit is contained in:
leejet
2023-12-21 00:22:03 +08:00
committed by GitHub
parent 9842a3f819
commit 8f6b4a39d6
6 changed files with 43830 additions and 80117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
util.cpp
View File

@@ -1,7 +1,9 @@
#include "util.h"
#include <stdarg.h>
#include <codecvt>
#include <fstream>
#include <locale>
#include <thread>
#include <unordered_set>
#include <vector>
@@ -119,6 +121,21 @@ int32_t get_num_physical_cores() {
return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4;
}
std::u32string utf8_to_utf32(const std::string& utf8_str) {
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> converter;
return converter.from_bytes(utf8_str);
}
std::string utf32_to_utf8(const std::u32string& utf32_str) {
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> converter;
return converter.to_bytes(utf32_str);
}
std::u32string unicode_value_to_utf32(int unicode_value) {
std::u32string utf32_string = {static_cast<char32_t>(unicode_value)};
return utf32_string;
}
std::string basename(const std::string& path) {
size_t pos = path.find_last_of('/');
if (pos != std::string::npos) {