feat: add SenseNova U1.5 support (#1935)

src/tokenizers/qwen2_tokenizer.cpp

@@ -45,16 +45,8 @@ void Qwen2Tokenizer::load_from_merges(const std::string& merges_utf8_str) {
     bpe_len = rank;
 }
 
-Qwen2Tokenizer::Qwen2Tokenizer(const std::string& merges_utf8_str) {
-    UNK_TOKEN = "<|endoftext|>";
-    EOS_TOKEN = "<|endoftext|>";
-    PAD_TOKEN = "<|endoftext|>";
-
-    UNK_TOKEN_ID = 151643;
-    EOS_TOKEN_ID = 151643;
-    PAD_TOKEN_ID = 151643;
-
-    special_tokens = {
+static const std::vector<std::string>& qwen2_special_tokens() {
+    static const std::vector<std::string> tokens = {
         "<|endoftext|>",
         "<|im_start|>",
         "<|im_end|>",
@@ -87,6 +79,24 @@ Qwen2Tokenizer::Qwen2Tokenizer(const std::string& merges_utf8_str) {
         "<|bot_token|>",
         "<|tms_token|>",
     };
+    return tokens;
+}
+
+Qwen2Tokenizer::Qwen2Tokenizer(const std::string& merges_utf8_str)
+    : Qwen2Tokenizer(merges_utf8_str, qwen2_special_tokens()) {
+}
+
+Qwen2Tokenizer::Qwen2Tokenizer(const std::string& merges_utf8_str,
+                               const std::vector<std::string>& special_tokens_override) {
+    UNK_TOKEN = "<|endoftext|>";
+    EOS_TOKEN = "<|endoftext|>";
+    PAD_TOKEN = "<|endoftext|>";
+
+    UNK_TOKEN_ID = 151643;
+    EOS_TOKEN_ID = 151643;
+    PAD_TOKEN_ID = 151643;
+
+    special_tokens = special_tokens_override;
 
     if (merges_utf8_str.size() > 0) {
         load_from_merges(merges_utf8_str);

src/tokenizers/qwen2_tokenizer.h

@@ -2,12 +2,15 @@
 #define __SD_TOKENIZERS_QWEN2_TOKENIZER_H__
 
 #include <string>
+#include <vector>
 
 #include "bpe_tokenizer.h"
 
 class Qwen2Tokenizer : public BPETokenizer {
 protected:
     void load_from_merges(const std::string& merges_utf8_str);
+    Qwen2Tokenizer(const std::string& merges_utf8_str,
+                   const std::vector<std::string>& special_tokens_override);
 
 public:
     explicit Qwen2Tokenizer(const std::string& merges_utf8_str = "");

src/tokenizers/sensenova_u1_tokenizer.cpp

@@ -0,0 +1,44 @@
+#include "sensenova_u1_tokenizer.h"
+
+#include <vector>
+
+static const std::vector<std::string>& sensenova_u1_special_tokens() {
+    static const std::vector<std::string> tokens = {
+        "<|endoftext|>",
+        "<|im_start|>",
+        "<|im_end|>",
+        "<|object_ref_start|>",
+        "<|object_ref_end|>",
+        "<|box_start|>",
+        "<|box_end|>",
+        "<|quad_start|>",
+        "<|quad_end|>",
+        "<|vision_start|>",
+        "<|vision_end|>",
+        "<|vision_pad|>",
+        "<|image_pad|>",
+        "<|video_pad|>",
+        "<tool_call>",
+        "</tool_call>",
+        "<|fim_prefix|>",
+        "<|fim_middle|>",
+        "<|fim_suffix|>",
+        "<|fim_pad|>",
+        "<|repo_name|>",
+        "<|file_sep|>",
+        "<tool_response>",
+        "</tool_response>",
+        "<think>",
+        "</think>",
+        "<IMG_CONTEXT>",
+        "<img>",
+        "</img>",
+    };
+    return tokens;
+}
+
+SenseNovaU1Tokenizer::SenseNovaU1Tokenizer(const std::string& merges_utf8_str)
+    : Qwen2Tokenizer(merges_utf8_str, sensenova_u1_special_tokens()) {
+    EOS_TOKEN    = "<|im_end|>";
+    EOS_TOKEN_ID = 151645;
+}

src/tokenizers/sensenova_u1_tokenizer.h

@@ -0,0 +1,13 @@
+#ifndef __SD_TOKENIZERS_SENSENOVA_U1_TOKENIZER_H__
+#define __SD_TOKENIZERS_SENSENOVA_U1_TOKENIZER_H__
+
+#include <string>
+
+#include "qwen2_tokenizer.h"
+
+class SenseNovaU1Tokenizer : public Qwen2Tokenizer {
+public:
+    explicit SenseNovaU1Tokenizer(const std::string& merges_utf8_str = "");
+};
+
+#endif  // __SD_TOKENIZERS_SENSENOVA_U1_TOKENIZER_H__