diff --git a/clip.hpp b/clip.hpp index 0cb51545..d655661d 100644 --- a/clip.hpp +++ b/clip.hpp @@ -166,7 +166,7 @@ public: } encoder_len = i; - auto it = encoder.find(utf8_to_utf32("img")); + auto it = encoder.find(utf8_to_utf32("img")); if (it != encoder.end()) { printf(" trigger word img already in vocab \n"); }else{ @@ -1215,6 +1215,34 @@ struct FrozenCLIPEmbedderWithCustomWords : public GGMLModule { return tokenize(text, text_model.max_position_embeddings, padding); } + std::vector convert_token_to_id(std::string text) { + auto on_new_token_cb = [&](std::string& str, std::vector& bpe_tokens) -> bool { + size_t word_end = str.find(","); + std::string embd_name = word_end == std::string::npos ? str : str.substr(0, word_end); + embd_name = trim(embd_name); + std::string embd_path = get_full_path(text_model.embd_dir, embd_name + ".pt"); + if (embd_path.size() == 0) { + embd_path = get_full_path(text_model.embd_dir, embd_name + ".ckpt"); + } + if (embd_path.size() == 0) { + embd_path = get_full_path(text_model.embd_dir, embd_name + ".safetensors"); + } + if (embd_path.size() > 0) { + if (text_model.load_embedding(embd_name, embd_path, bpe_tokens)) { + if (word_end != std::string::npos) { + str = str.substr(word_end); + } else { + str = ""; + } + return true; + } + } + return false; + }; + std::vector curr_tokens = tokenizer.encode(text, on_new_token_cb); + return curr_tokens; + } + std::pair, std::vector> tokenize(std::string text, size_t max_length = 0, bool padding = false) { diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp index 08c516eb..c898a0e9 100644 --- a/stable-diffusion.cpp +++ b/stable-diffusion.cpp @@ -585,6 +585,12 @@ public: int height, bool force_zero_embeddings = false) { cond_stage_model.set_clip_skip(clip_skip); + auto image_tokens = cond_stage_model.convert_token_to_id(trigger_word); + printf(" length of image tokens: %lu \n", image_tokens.size()); + if(image_tokens.size() == 1){ + printf(" image token id is: %d \n", image_tokens[0]); + } + auto tokens_and_weights = cond_stage_model.tokenize(text, true); std::vector& tokens = tokens_and_weights.first; std::vector& weights = tokens_and_weights.second;