Compare commits

..

1 Commits

Author SHA1 Message Date
stduhpf
14206fd488 fix: fix clip tokenizer (#383) 2024-09-02 22:31:46 +08:00

View File

@@ -388,7 +388,7 @@ public:
std::string token_str = token.str();
std::u32string utf32_token;
for (int i = 0; i < token_str.length(); i++) {
char b = token_str[i];
unsigned char b = token_str[i];
utf32_token += byte_encoder[b];
}
auto bpe_strs = bpe(utf32_token);