upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-09-29 09:28:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: add ltx2.3 support (#1463)

Browse Source 
* add GemmaTokenizer

* add basic ltx2.3 support

* change vocab file encoding

* fix ci

* fix ubuntu build

* add temporal tiling support

* add ltx audio support

* update ggml submodule url

* fix generate_video

* add i2v support

* minify bundled Gemma tokenizer vocab sources

* pass video fps into temporal rope embeddings

* fix av_ca_timestep_scale_multiplier

* add LTX2Scheduler support

* update docs

* fix ci
This commit is contained in:
leejet
2026-05-17 16:46:20 +08:00
committed by GitHub
parent 3b4d26f3d9
commit 67dda3f897
44 changed files with 6393 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tokenizers/bpe_tokenizer.cpp
+28 -4
View File
@@ -162,13 +162,37 @@ std::vector<int> BPETokenizer::encode(const std::string& text, on_new_token_cb_t
std::string token_str = token;
std::u32string utf32_token;
for (int i = 0; i < static_cast<int>(token_str.length()); i++) {
unsigned char b = token_str[i];
utf32_token += byte_encoder[b];
if (byte_level_bpe) {
for (int i = 0; i < token_str.length(); i++) {
unsigned char b = token_str[i];
utf32_token += byte_encoder[b];
}
} else {
utf32_token = utf8_to_utf32(token_str);
}
auto bpe_strs = bpe(utf32_token);
for (auto bpe_str : bpe_strs) {
bpe_tokens.push_back(encoder[bpe_str]);
int token_id;
auto iter = encoder.find(bpe_str);
if (iter != encoder.end()) {
token_id = iter->second;
} else {
if (byte_fallback) {
auto utf8_token_str = utf32_to_utf8(bpe_str);
for (int i = 0; i < utf8_token_str.length(); i++) {
unsigned char b = utf8_token_str[i];
char hex_buf[16];
snprintf(hex_buf, sizeof(hex_buf), "<0x%02X>", b);
iter = encoder.find(utf8_to_utf32(hex_buf));
bpe_tokens.push_back(token_id);
token_strs.push_back(hex_buf);
}
continue;
} else {
token_id = UNK_TOKEN_ID;
}
}
bpe_tokens.push_back(token_id);
token_strs.push_back(utf32_to_utf8(bpe_str));
}
}