upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-29 09:27:33 -05:00
Code Issues Packages Projects Releases Wiki Activity

common/peg : handle invalid utf-8 sequences in the AST (#29161)

Browse Source 
* common/peg : handle invalid utf-8 sequences in the AST

* cont : return maximal subpart per Unicode recommendations

* cont : remove strict argument
This commit is contained in:
Aldehir Rojas
2026-09-20 06:51:40 -05:00
committed by GitHub
parent 3cf03257f2
commit 3d82ef62d4
7 changed files with 173 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/unicode.cpp
+19 -14
View File
@@ -26,16 +26,16 @@ utf8_parse_result common_parse_utf8_codepoint(std::string_view input, size_t off
// Invalid: continuation byte as first byte
if (!(input[offset] & 0x40)) {
return utf8_parse_result(utf8_parse_result::INVALID);
return utf8_parse_result(utf8_parse_result::INVALID, 0, 1);
}
// 2-byte sequence
if (!(input[offset] & 0x20)) {
if (offset + 1 >= input.size()) {
return utf8_parse_result(utf8_parse_result::INCOMPLETE);
return utf8_parse_result(utf8_parse_result::INCOMPLETE, 0, 1);
}
if ((input[offset + 1] & 0xc0) != 0x80) {
return utf8_parse_result(utf8_parse_result::INVALID);
return utf8_parse_result(utf8_parse_result::INVALID, 0, 1);
}
auto result = ((input[offset] & 0x1f) << 6) | (input[offset + 1] & 0x3f);
return utf8_parse_result(utf8_parse_result::SUCCESS, result, 2);
@@ -43,11 +43,14 @@ utf8_parse_result common_parse_utf8_codepoint(std::string_view input, size_t off
// 3-byte sequence
if (!(input[offset] & 0x10)) {
if (offset + 2 >= input.size()) {
return utf8_parse_result(utf8_parse_result::INCOMPLETE);
}
if ((input[offset + 1] & 0xc0) != 0x80 || (input[offset + 2] & 0xc0) != 0x80) {
return utf8_parse_result(utf8_parse_result::INVALID);
// Check one byte at a time so a bad byte is reported before a short input
for (size_t i = 1; i < 3; i++) {
if (offset + i >= input.size()) {
return utf8_parse_result(utf8_parse_result::INCOMPLETE, 0, i);
}
if ((input[offset + i] & 0xc0) != 0x80) {
return utf8_parse_result(utf8_parse_result::INVALID, 0, i);
}
}
auto result = ((input[offset] & 0x0f) << 12) | ((input[offset + 1] & 0x3f) << 6) | (input[offset + 2] & 0x3f);
return utf8_parse_result(utf8_parse_result::SUCCESS, result, 3);
@@ -55,18 +58,20 @@ utf8_parse_result common_parse_utf8_codepoint(std::string_view input, size_t off
// 4-byte sequence
if (!(input[offset] & 0x08)) {
if (offset + 3 >= input.size()) {
return utf8_parse_result(utf8_parse_result::INCOMPLETE);
}
if ((input[offset + 1] & 0xc0) != 0x80 || (input[offset + 2] & 0xc0) != 0x80 || (input[offset + 3] & 0xc0) != 0x80) {
return utf8_parse_result(utf8_parse_result::INVALID);
for (size_t i = 1; i < 4; i++) {
if (offset + i >= input.size()) {
return utf8_parse_result(utf8_parse_result::INCOMPLETE, 0, i);
}
if ((input[offset + i] & 0xc0) != 0x80) {
return utf8_parse_result(utf8_parse_result::INVALID, 0, i);
}
}
auto result = ((input[offset] & 0x07) << 18) | ((input[offset + 1] & 0x3f) << 12) | ((input[offset + 2] & 0x3f) << 6) | (input[offset + 3] & 0x3f);
return utf8_parse_result(utf8_parse_result::SUCCESS, result, 4);
}
// Invalid first byte
return utf8_parse_result(utf8_parse_result::INVALID);
return utf8_parse_result(utf8_parse_result::INVALID, 0, 1);
}
bool common_utf8_is_complete(const std::string & s) {