mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-09-28 00:47:38 -05:00
107 lines
3.0 KiB
C++
107 lines
3.0 KiB
C++
#include "qwen2_tokenizer.h"
|
|
|
|
#include "core/util.h"
|
|
#include "vocab/vocab.h"
|
|
|
|
void Qwen2Tokenizer::load_from_merges(const std::string& merges_utf8_str) {
|
|
auto byte_unicode_pairs = bytes_to_unicode();
|
|
byte_encoder = std::map<int, std::u32string>(byte_unicode_pairs.begin(), byte_unicode_pairs.end());
|
|
for (auto& pair : byte_unicode_pairs) {
|
|
byte_decoder[pair.second] = pair.first;
|
|
}
|
|
|
|
std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
|
|
LOG_VERBOSE("merges size %zu", merges.size());
|
|
std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
|
|
for (const auto& merge : merges) {
|
|
size_t space_pos = merge.find(' ');
|
|
merge_pairs.emplace_back(merge.substr(0, space_pos), merge.substr(space_pos + 1));
|
|
}
|
|
|
|
std::vector<std::u32string> tokens;
|
|
for (const auto& pair : byte_unicode_pairs) {
|
|
tokens.push_back(pair.second);
|
|
}
|
|
for (const auto& merge : merge_pairs) {
|
|
tokens.push_back(merge.first + merge.second);
|
|
}
|
|
for (auto& special_token : special_tokens) {
|
|
tokens.push_back(utf8_to_utf32(special_token));
|
|
}
|
|
|
|
int i = 0;
|
|
for (const auto& token : tokens) {
|
|
encoder[token] = i;
|
|
decoder[i] = token;
|
|
i++;
|
|
}
|
|
encoder_len = i;
|
|
LOG_VERBOSE("vocab size: %d", encoder_len);
|
|
|
|
int rank = 0;
|
|
for (const auto& merge : merge_pairs) {
|
|
bpe_ranks[merge] = rank++;
|
|
}
|
|
bpe_len = rank;
|
|
}
|
|
|
|
static const std::vector<std::string>& qwen2_special_tokens() {
|
|
static const std::vector<std::string> tokens = {
|
|
"<|endoftext|>",
|
|
"<|im_start|>",
|
|
"<|im_end|>",
|
|
"<|object_ref_start|>",
|
|
"<|object_ref_end|>",
|
|
"<|box_start|>",
|
|
"<|box_end|>",
|
|
"<|quad_start|>",
|
|
"<|quad_end|>",
|
|
"<|vision_start|>",
|
|
"<|vision_end|>",
|
|
"<|vision_pad|>",
|
|
"<|image_pad|>",
|
|
"<|video_pad|>",
|
|
"<tool_call>",
|
|
"</tool_call>",
|
|
"<|fim_prefix|>",
|
|
"<|fim_middle|>",
|
|
"<|fim_suffix|>",
|
|
"<|fim_pad|>",
|
|
"<|repo_name|>",
|
|
"<|file_sep|>",
|
|
"<tool_response>",
|
|
"</tool_response>",
|
|
"<think>",
|
|
"</think>",
|
|
"<|boi_token|>",
|
|
"<|bor_token|>",
|
|
"<|eor_token|>",
|
|
"<|bot_token|>",
|
|
"<|tms_token|>",
|
|
};
|
|
return tokens;
|
|
}
|
|
|
|
Qwen2Tokenizer::Qwen2Tokenizer(const std::string& merges_utf8_str)
|
|
: Qwen2Tokenizer(merges_utf8_str, qwen2_special_tokens()) {
|
|
}
|
|
|
|
Qwen2Tokenizer::Qwen2Tokenizer(const std::string& merges_utf8_str,
|
|
const std::vector<std::string>& special_tokens_override) {
|
|
UNK_TOKEN = "<|endoftext|>";
|
|
EOS_TOKEN = "<|endoftext|>";
|
|
PAD_TOKEN = "<|endoftext|>";
|
|
|
|
UNK_TOKEN_ID = 151643;
|
|
EOS_TOKEN_ID = 151643;
|
|
PAD_TOKEN_ID = 151643;
|
|
|
|
special_tokens = special_tokens_override;
|
|
|
|
if (merges_utf8_str.size() > 0) {
|
|
load_from_merges(merges_utf8_str);
|
|
} else {
|
|
load_from_merges(load_qwen2_merges());
|
|
}
|
|
}
|