diff --git a/include/llama.h b/include/llama.h index ac2215dc7e..31bbf8b0d9 100644 --- a/include/llama.h +++ b/include/llama.h @@ -77,6 +77,7 @@ extern "C" { LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming + LLAMA_VOCAB_TYPE_TEST = 7, // Dummy tokenizer for testing: rolling hash of fixed-size chunks -> tokens, tokens -> hex }; enum llama_rope_type { diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 0f5155b2e4..39160a417e 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -387,13 +387,13 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_TOKENIZER_SCORES, scores); add_kv(LLM_KV_TOKENIZER_MERGES, vocab.get_bpe_merges()); // FIXME llama_token is type i32 but when reading in a GGUF file u32 is expected, not an issue for writing though - add_kv(LLM_KV_TOKENIZER_BOS_ID, uint32_t(vocab.token_bos())); - add_kv(LLM_KV_TOKENIZER_EOS_ID, uint32_t(vocab.token_eos())); - add_kv(LLM_KV_TOKENIZER_EOT_ID, uint32_t(vocab.token_eot())); - add_kv(LLM_KV_TOKENIZER_EOM_ID, uint32_t(vocab.token_eom())); - add_kv(LLM_KV_TOKENIZER_UNK_ID, uint32_t(vocab.token_unk())); - add_kv(LLM_KV_TOKENIZER_SEP_ID, uint32_t(vocab.token_sep())); - add_kv(LLM_KV_TOKENIZER_PAD_ID, uint32_t(vocab.token_pad())); + if (vocab.token_bos() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_BOS_ID, uint32_t(vocab.token_bos())); } + if (vocab.token_eos() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOS_ID, uint32_t(vocab.token_eos())); } + if (vocab.token_eot() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOT_ID, uint32_t(vocab.token_eot())); } + if (vocab.token_eom() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOM_ID, uint32_t(vocab.token_eom())); } + if (vocab.token_unk() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_UNK_ID, uint32_t(vocab.token_unk())); } + if (vocab.token_sep() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_SEP_ID, uint32_t(vocab.token_sep())); } + if (vocab.token_pad() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_PAD_ID, uint32_t(vocab.token_pad())); } // add_kv(LLM_KV_TOKENIZER_CLS_ID, uint32_t(vocab.token_bos())); // deprecated // add_kv(LLM_KV_TOKENIZER_MASK_ID, ???); add_kv(LLM_KV_TOKENIZER_ADD_BOS, vocab.get_add_bos()); @@ -404,12 +404,12 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, vocab.get_precompiled_charsmap()); // add_kv(LLM_KV_TOKENIZER_HF_JSON, ???); // add_kv(LLM_KV_TOKENIZER_RWKV, ???); - add_kv(LLM_KV_TOKENIZER_FIM_PRE_ID, uint32_t(vocab.token_fim_pre())); - add_kv(LLM_KV_TOKENIZER_FIM_SUF_ID, uint32_t(vocab.token_fim_suf())); - add_kv(LLM_KV_TOKENIZER_FIM_MID_ID, uint32_t(vocab.token_fim_mid())); - add_kv(LLM_KV_TOKENIZER_FIM_PAD_ID, uint32_t(vocab.token_fim_pad())); - add_kv(LLM_KV_TOKENIZER_FIM_REP_ID, uint32_t(vocab.token_fim_rep())); - add_kv(LLM_KV_TOKENIZER_FIM_SEP_ID, uint32_t(vocab.token_fim_sep())); + if (vocab.token_fim_pre() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_PRE_ID, uint32_t(vocab.token_fim_pre())); } + if (vocab.token_fim_suf() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_SUF_ID, uint32_t(vocab.token_fim_suf())); } + if (vocab.token_fim_mid() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_MID_ID, uint32_t(vocab.token_fim_mid())); } + if (vocab.token_fim_pad() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_PAD_ID, uint32_t(vocab.token_fim_pad())); } + if (vocab.token_fim_rep() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_REP_ID, uint32_t(vocab.token_fim_rep())); } + if (vocab.token_fim_sep() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_SEP_ID, uint32_t(vocab.token_fim_sep())); } // TODO: implement LoRA support // add_kv(LLM_KV_ADAPTER_TYPE, ???); diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 737e072756..e038637ce7 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -2087,6 +2087,16 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { special_unk_id = LLAMA_TOKEN_NULL; special_sep_id = LLAMA_TOKEN_NULL; special_pad_id = LLAMA_TOKEN_NULL; + } else if (tokenizer_model == "test") { + type = LLAMA_VOCAB_TYPE_TEST; + + // default special tokens + special_bos_id = LLAMA_TOKEN_NULL; + special_eos_id = LLAMA_TOKEN_NULL; + special_unk_id = LLAMA_TOKEN_NULL; + special_sep_id = LLAMA_TOKEN_NULL; + special_pad_id = LLAMA_TOKEN_NULL; + special_mask_id = LLAMA_TOKEN_NULL; } else if (tokenizer_model == "plamo2") { type = LLAMA_VOCAB_TYPE_PLAMO2; @@ -3134,6 +3144,7 @@ std::string llama_vocab::impl::type_name() const{ case LLAMA_VOCAB_TYPE_UGM: return "UGM"; case LLAMA_VOCAB_TYPE_RWKV: return "RWKV"; case LLAMA_VOCAB_TYPE_PLAMO2: return "PLaMo2"; + case LLAMA_VOCAB_TYPE_TEST: return "TEST"; default: return "unknown"; } } @@ -3222,6 +3233,9 @@ void llama_vocab::impl::init_tokenizer(enum llama_vocab_type type) { case LLAMA_VOCAB_TYPE_PLAMO2: tokenizer = std::make_unique(vocab); break; + case LLAMA_VOCAB_TYPE_TEST: + tokenizer = std::make_unique(); + break; default: GGML_ABORT("unsupported vocab type"); } @@ -3595,6 +3609,42 @@ std::vector llama_vocab::impl::tokenize( } } } break; + case LLAMA_VOCAB_TYPE_TEST: + { + const uint32_t n_vocab = vocab.n_tokens(); + constexpr size_t chunk_size = 5; + + // reserve output to avoid repeated reallocations + size_t n_tokens = 0; + for (const auto & fragment : fragment_buffer) { + if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { + n_tokens += (fragment.length + chunk_size - 1) / chunk_size; + } else { + ++n_tokens; + } + } + output.reserve(output.size() + n_tokens); + + for (const auto & fragment : fragment_buffer) { + if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { + const auto & text = fragment.raw_text; + const size_t begin = fragment.offset; + const size_t end = begin + fragment.length; + size_t pos = begin; + while (pos < end) { + const size_t n = std::min(chunk_size, end - pos); + uint64_t hash = 0; + for (size_t i = 0; i < n; ++i) { + hash = hash*31 + (uint8_t) text[pos + i]; + } + output.push_back((llama_token)(hash % n_vocab)); + pos += n; + } + } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN) + output.push_back(fragment.token); + } + } + } break; case LLAMA_VOCAB_TYPE_NONE: GGML_ABORT("fatal error"); } @@ -3693,6 +3743,11 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t memcpy(buf, result.data(), result.size()); return (int)result.size(); } + case LLAMA_VOCAB_TYPE_TEST: { + // tokens -> text: simply stringify the token id in hex + std::string result = format("%x", token); + return _try_copy(result.data(), result.size()); + } case LLAMA_VOCAB_TYPE_PLAMO2: { // PLaMo-2 uses similar token handling as BPE/SPM if (vocab.is_byte(token)) { @@ -3963,6 +4018,9 @@ llama_token llama_vocab::byte_to_token(uint8_t ch) const { snprintf(hex_str, sizeof(hex_str), "<0x%02X>", ch); return pimpl->token_to_id.at(hex_str); } + case LLAMA_VOCAB_TYPE_TEST: + // TEST tokens have no byte-level mapping + return LLAMA_TOKEN_NULL; default: GGML_ABORT("fatal error"); } diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 568f7234c1..f848fc139d 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -338,7 +338,19 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f); } - ms.add_kv(LLM_KV_TOKENIZER_MODEL, "no_vocab"); + // dummy tokenizer: token ids are derived from fixed-size chunks and detokenized as hex ids + { + std::vector tokenizer_list(n_vocab); + std::vector tokenizer_scores(n_vocab, 0.0f); + + ms.add_kv(LLM_KV_TOKENIZER_MODEL, "test"); + for (uint32_t i = 0; i < n_vocab; i++) { + tokenizer_list[i] = "tok_" + std::to_string(i); + } + ms.add_kv(LLM_KV_TOKENIZER_LIST, tokenizer_list); + ms.add_kv(LLM_KV_TOKENIZER_SCORES, tokenizer_scores); + } + // ms.add_kv(LLM_KV_DENSE_2_FEAT_OUT, n_embd); // ms.add_kv(LLM_KV_DENSE_3_FEAT_IN, n_embd);