refactor: reorganize src model layout (#1615)

src/tokenizers/bpe_tokenizer.cpp

@@ -3,8 +3,8 @@
 #include <algorithm>
 #include <sstream>
 
+#include "core/util.h"
 #include "tokenize_util.h"
-#include "util.h"
 
 std::vector<std::pair<int, std::u32string>> BPETokenizer::bytes_to_unicode() {
     std::vector<std::pair<int, std::u32string>> byte_unicode_pairs;

src/tokenizers/clip_tokenizer.cpp

@@ -6,9 +6,9 @@
 #include <regex>
 #include <set>
 
+#include "core/util.h"
 #include "ggml.h"
 #include "tokenize_util.h"
-#include "util.h"
 #include "vocab/vocab.h"
 
 CLIPTokenizer::CLIPTokenizer(int pad_token_id, const std::string& merges_utf8_str) {

src/tokenizers/gemma_tokenizer.cpp

@@ -1,8 +1,8 @@
 #include "gemma_tokenizer.h"
 
+#include "core/util.h"
 #include "ggml.h"
 #include "json.hpp"
-#include "util.h"
 #include "vocab/vocab.h"
 
 std::string GemmaTokenizer::normalize(const std::string& text) const {

src/tokenizers/gpt_oss_tokenizer.cpp

@@ -1,7 +1,7 @@
 #include "gpt_oss_tokenizer.h"
 
+#include "core/util.h"
 #include "json.hpp"
-#include "util.h"
 #include "vocab/vocab.h"
 
 void GPTOSSTokenizer::load_from_merges(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {

src/tokenizers/mistral_tokenizer.cpp

@@ -1,8 +1,8 @@
 #include "mistral_tokenizer.h"
 
+#include "core/util.h"
 #include "ggml.h"
 #include "json.hpp"
-#include "util.h"
 #include "vocab/vocab.h"
 
 void MistralTokenizer::load_from_merges(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {

src/tokenizers/qwen2_tokenizer.cpp

@@ -1,6 +1,6 @@
 #include "qwen2_tokenizer.h"
 
-#include "util.h"
+#include "core/util.h"
 #include "vocab/vocab.h"
 
 void Qwen2Tokenizer::load_from_merges(const std::string& merges_utf8_str) {

src/tokenizers/t5_unigram_tokenizer.cpp

@@ -6,9 +6,9 @@
 #include <regex>
 #include <sstream>
 
+#include "core/util.h"
 #include "json.hpp"
 #include "tokenize_util.h"
-#include "util.h"
 #include "vocab/vocab.h"
 
 // Port from: https://github.com/google/sentencepiece/blob/master/src/unigram_model.h

src/tokenizers/tokenizer.cpp

@@ -4,7 +4,7 @@
 #include <cmath>
 #include <regex>
 
-#include "util.h"
+#include "core/util.h"
 
 void Tokenizer::add_special_token(const std::string& token) {
     special_tokens.push_back(token);