upstream-archive/stable-diffusion.cpp
1
0
Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-07-29 14:20:57 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: add left padding support to tokenizers (#1424)

Browse Source
This commit is contained in:
leejet
2026-04-15 23:17:47 +08:00
committed by GitHub
parent 9ac7b672c2
commit c41c5ded7a
2 changed files with 18 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/tokenizers/tokenizer.cpp
View File

@@ -107,13 +107,24 @@ void Tokenizer::pad_tokens(std::vector<int>& tokens,
if (final_length > out_tokens.size()) {
const size_t pad_count = final_length - out_tokens.size();
out_tokens.insert(out_tokens.end(), pad_count, PAD_TOKEN_ID);
if (pad_left) {
out_tokens.insert(out_tokens.begin(), pad_count, PAD_TOKEN_ID);
if (use_weights) {
out_weights.insert(out_weights.end(), pad_count, 1.0f);
}
if (use_mask) {
out_mask.insert(out_mask.end(), pad_count, 0.0f);
if (use_weights) {
out_weights.insert(out_weights.begin(), pad_count, 1.0f);
}
if (use_mask) {
out_mask.insert(out_mask.begin(), pad_count, 0.0f);
}
} else {
out_tokens.insert(out_tokens.end(), pad_count, PAD_TOKEN_ID);
if (use_weights) {
out_weights.insert(out_weights.end(), pad_count, 1.0f);
}
if (use_mask) {
out_mask.insert(out_mask.end(), pad_count, 0.0f);
}
}
}
};