upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-10-03 19:37:51 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: use tokenizer-specific pre-tokenization rules (#1975)

Browse Source
This commit is contained in:
leejet
2026-09-15 02:37:52 +08:00
committed by GitHub
parent 07a85c74cb
commit 59c23bce0d
9 changed files with 92 additions and 1021 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tokenizers/clip_tokenizer.cpp
+2 -16
View File
@@ -8,10 +8,10 @@
#include "core/util.h"
#include "ggml.h"
#include "tokenize_util.h"
#include "vocab/vocab.h"
CLIPTokenizer::CLIPTokenizer(int pad_token_id, const std::string& merges_utf8_str) {
CLIPTokenizer::CLIPTokenizer(int pad_token_id, const std::string& merges_utf8_str)
: BPETokenizer(R"((?i:'s|'t|'re|'ve|'m|'ll|'d)|\p{L}+|\p{N}|[^\s\p{L}\p{N}]+)") {
UNK_TOKEN = "<|endoftext|>";
BOS_TOKEN = "<|startoftext|>";
EOS_TOKEN = "<|endoftext|>";
@@ -101,17 +101,3 @@ std::string CLIPTokenizer::normalize(const std::string& text) const {
std::transform(normalized_text.begin(), normalized_text.end(), normalized_text.begin(), [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
return normalized_text;
}
std::vector<std::string> CLIPTokenizer::token_split(const std::string& text) const {
std::regex clip_pat(R"('s|'t|'re|'ve|'m|'ll|'d|[[:alpha:]]+|[[:digit:]]|[^[:space:][:alpha:][:digit:]]+)",
std::regex::icase);
std::sregex_iterator iter(text.begin(), text.end(), clip_pat);
std::sregex_iterator end;
std::vector<std::string> result;
for (; iter != end; ++iter) {
result.emplace_back(iter->str());
}
return result;
}