upstream-archive/whisper.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/whisper.cpp.git synced 2026-07-23 11:10:57 -05:00
Code Issues Packages Projects Releases Wiki Activity

server : merge split utf-8 token text in verbose json (#3850)

Browse Source
This commit is contained in:
Noah Lyons
2026-06-02 07:10:27 -04:00
committed by GitHub
parent ef24de1e58
commit e5d4412578
6 changed files with 94 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tests/CMakeLists.txt
View File

@@ -88,6 +88,14 @@ if (WHISPER_COMMON_FFMPEG)
set_tests_properties(${TEST_TARGET} PROPERTIES LABELS "tiny;mp3")
endif()
# UTF-8 helper unit test
set(UTF8_TEST test-common-utf8)
add_executable(${UTF8_TEST} ${UTF8_TEST}.cpp)
target_include_directories(${UTF8_TEST} PRIVATE ../examples)
target_link_libraries(${UTF8_TEST} PRIVATE common)
add_test(NAME ${UTF8_TEST} COMMAND ${UTF8_TEST})
set_tests_properties(${UTF8_TEST} PROPERTIES LABELS "unit")
# VAD test tests VAD in isolation
set(VAD_TEST test-vad)
add_executable(${VAD_TEST} ${VAD_TEST}.cpp)

34
tests/test-common-utf8.cpp Normal file
View File

@@ -0,0 +1,34 @@
#include "common-whisper.h"
#include <cstdlib>
#include <cstdio>
#include <string>
static void expect_needed(const std::string & input, int expected) {
const int actual = utf8_trailing_bytes_needed(input);
if (actual != expected) {
fprintf(stderr, "expected %d trailing UTF-8 bytes, got %d\n", expected, actual);
std::abort();
}
}
int main() {
expect_needed("", 0);
expect_needed("plain ascii", 0);
const std::string cjk = "\xE4\xBD\xA0"; // U+4F60
expect_needed(cjk.substr(0, 1), 2);
expect_needed(cjk.substr(0, 2), 1);
expect_needed(cjk, 0);
const std::string emoji = "\xF0\x9F\x98\x80"; // U+1F600
expect_needed(emoji.substr(0, 1), 3);
expect_needed(emoji.substr(0, 2), 2);
expect_needed(emoji.substr(0, 3), 1);
expect_needed(emoji, 0);
expect_needed("\x80\x80", 0);
expect_needed("\xFF", 0);
return 0;
}