deprecate: drop fallback url to a google bucket (#726)

* deprecate: drop fallback url to a google bucket

* deprecation: add a deprecation warning when a model is loaded using cached gcs data

NOTICE

@@ -12,8 +12,8 @@ This distribution includes the following Jina AI models, each with its respectiv
 
 These models are developed by Jina (https://jina.ai/) and are subject to Jina AI's licensing terms.
 
-This distribution includes the following Google models, each with its respective license:
-- vidore/colpali-v1.3
+This distribution includes the following Gemma-based models, each with its respective license:
+- Qdrant/colpali-v1.3-fp16
     - License: gemma
 - google/embeddinggemma-300m
     - License: gemma

fastembed/common/model_management.py

@@ -5,6 +5,7 @@ import json
 import shutil
 import tarfile
 import tempfile
+import warnings
 import contextlib
 from copy import deepcopy
 from pathlib import Path, PureWindowsPath
@@ -27,6 +28,27 @@ T = TypeVar("T", bound=BaseModelDescription)
 _DOWNLOAD_CHUNK_SIZE = 256 * 1024
 
 
+def _hf_transport_errors() -> tuple[type[Exception], ...]:
+    """Network errors of huggingface_hub's HTTP library that aren't OSError.
+
+    A refused connection, a DNS failure or a timeout raises an OSError in huggingface_hub 0.x,
+    which is built on requests, but a TransportError in 1.x (httpx) and 2.x (httpx2).
+    """
+    try:
+        # huggingface_hub>=1.30 re-exports whichever of httpx and httpx2 it's built on.
+        from huggingface_hub.utils import httpx
+    except ImportError:
+        try:
+            import httpx  # huggingface_hub 1.0 to 1.29
+        except ImportError:  # huggingface_hub 0.x
+            return ()
+    return (httpx.TransportError,)
+
+
+# Errors from an HF download that download_model handles by falling back to url and retrying.
+_HF_DOWNLOAD_ERRORS = (OSError, RepositoryNotFoundError, ValueError) + _hf_transport_errors()
+
+
 class ModelManagement(Generic[T]):
     METADATA_FILE = "files_metadata.json"
 
@@ -374,6 +396,18 @@ class ModelManagement(Generic[T]):
 
         # check if the model_dir and the model files are both present for macOS
         if model_dir.exists() and len(list(model_dir.glob("*"))) > 0:
+            if deprecated_tar_struct:
+                # No built-in model is served from the bucket anymore, only copies of it remain.
+                # stacklevel points at the caller of TextEmbedding(...), the path that gets here.
+                warnings.warn(
+                    f"Loading {model_name} from {model_dir}, a copy downloaded from Google Cloud "
+                    "Storage by an older fastembed version. Support for such copies is deprecated "
+                    "and will be removed in a future release. To switch to Hugging Face, load the "
+                    "model once with network access (without `local_files_only=True` or "
+                    f"`HF_HUB_OFFLINE=1`), then delete {model_dir}.",
+                    FutureWarning,
+                    stacklevel=5,
+                )
             return model_dir
 
         if local_files_only:
@@ -440,7 +474,6 @@ class ModelManagement(Generic[T]):
                     "description": "Base English model, v1.5",
                     "size_in_GB": 0.44,
                     "sources": {
-                        "url": "https://storage.googleapis.com/qdrant-fastembed/fast-bge-base-en-v1.5.tar.gz",
                         "hf": "qdrant/bge-base-en-v1.5-onnx-q",
                     }
                 }
@@ -502,7 +535,7 @@ class ModelManagement(Generic[T]):
                             **kwargs,
                         )
                     )
-                except (EnvironmentError, RepositoryNotFoundError, ValueError) as e:
+                except _HF_DOWNLOAD_ERRORS as e:
                     if not local_files_only:
                         logger.error(
                             f"Could not download model from HuggingFace: {e} "

fastembed/image/onnx_embedding.py

@@ -33,7 +33,7 @@ supported_onnx_models: list[DenseModelDescription] = [
         model="Qdrant/Unicom-ViT-B-16",
         dim=768,
         description="Image embeddings (more detailed than Unicom-ViT-B-32), Multimodal (text&image), 2023 year",
-        license="apache-2.0",
+        license="mit",
         size_in_GB=0.82,
         sources=ModelSource(hf="Qdrant/Unicom-ViT-B-16"),
         model_file="model.onnx",
@@ -42,7 +42,7 @@ supported_onnx_models: list[DenseModelDescription] = [
         model="Qdrant/Unicom-ViT-B-32",
         dim=512,
         description="Image embeddings, Multimodal (text&image), 2023 year",
-        license="apache-2.0",
+        license="mit",
         size_in_GB=0.48,
         sources=ModelSource(hf="Qdrant/Unicom-ViT-B-32"),
         model_file="model.onnx",

fastembed/late_interaction/colbert.py

@@ -221,6 +221,8 @@ class Colbert(LateInteractionTextEmbeddingBase, OnnxTextModel[NumpyArray]):
         self.query_tokenizer, _ = load_tokenizer(model_dir=self._model_dir)
 
         assert self.tokenizer is not None
+        # load_tokenizer always enables both
+        assert self.tokenizer.padding is not None and self.tokenizer.truncation is not None
         self.mask_token_id = self.special_token_to_id[self.MASK_TOKEN]
         self.pad_token_id = self.tokenizer.padding["pad_id"]
         self.skip_list = {

fastembed/late_interaction_multimodal/colpali.py

@@ -22,7 +22,7 @@ supported_colpali_models: list[DenseModelDescription] = [
         model="Qdrant/colpali-v1.3-fp16",
         dim=128,
         description="Text embeddings, Multimodal (text&image), English, 50 tokens query length truncation, 2024.",
-        license="mit",
+        license="gemma",
         size_in_GB=6.5,
         sources=ModelSource(hf="Qdrant/colpali-v1.3-fp16"),
         additional_files=["model.onnx_data"],

fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding.py

@@ -33,7 +33,7 @@ class LateInteractionMultimodalEmbedding(LateInteractionMultimodalEmbeddingBase)
                          "model": "Qdrant/colpali-v1.3-fp16",
                          "dim": 128,
                          "description": "Text embeddings, Unimodal (text), Aligned to image latent space, ColBERT-compatible, 512 tokens max, 2024.",
-                         "license": "mit",
+                         "license": "gemma",
                          "size_in_GB": 6.06,
                          "sources": {
                             "hf": "Qdrant/colpali-v1.3-fp16",

fastembed/sparse/utils/vocab_resolver.py

@@ -25,11 +25,19 @@ class VocabTokenizer(VocabTokenizerBase):
         return np.array(self.tokenizer.encode(sentence).ids)
 
     def convert_ids_to_tokens(self, token_ids: NumpyArray) -> list[str]:
-        return [self.tokenizer.id_to_token(token_id) for token_id in token_ids]
+        tokens = []
+        for token_id in token_ids:
+            token = self.tokenizer.id_to_token(token_id)
+            if token is None:
+                raise ValueError(f"Token id {token_id} is not in the vocabulary")
+            tokens.append(token)
+        return tokens
 
 
 class VocabResolver:
-    def __init__(self, tokenizer: VocabTokenizerBase, stopwords: set[str], stemmer: SnowballStemmer):
+    def __init__(
+        self, tokenizer: VocabTokenizerBase, stopwords: set[str], stemmer: SnowballStemmer
+    ):
         # Word to id mapping
         self.vocab: dict[str, int] = {}
         # Id to word mapping
@@ -199,4 +207,3 @@ class VocabResolver:
             else:
                 counts[vocab_id] += 1
         return token_ids, counts, oov_count, forms
-

fastembed/text/onnx_embedding.py

@@ -19,7 +19,6 @@ supported_onnx_models: list[DenseModelDescription] = [
         size_in_GB=0.42,
         sources=ModelSource(
             hf="Qdrant/fast-bge-base-en",
-            url="https://storage.googleapis.com/qdrant-fastembed/fast-bge-base-en.tar.gz",
             _deprecated_tar_struct=True,
         ),
         model_file="model_optimized.onnx",
@@ -35,7 +34,6 @@ supported_onnx_models: list[DenseModelDescription] = [
         size_in_GB=0.21,
         sources=ModelSource(
             hf="Qdrant/bge-base-en-v1.5-onnx-Q",
-            url="https://storage.googleapis.com/qdrant-fastembed/fast-bge-base-en-v1.5.tar.gz",
             _deprecated_tar_struct=True,
         ),
         model_file="model_optimized.onnx",
@@ -63,7 +61,6 @@ supported_onnx_models: list[DenseModelDescription] = [
         size_in_GB=0.13,
         sources=ModelSource(
             hf="Qdrant/bge-small-en",
-            url="https://storage.googleapis.com/qdrant-fastembed/BAAI-bge-small-en.tar.gz",
             _deprecated_tar_struct=True,
         ),
         model_file="model_optimized.onnx",
@@ -91,7 +88,6 @@ supported_onnx_models: list[DenseModelDescription] = [
         size_in_GB=0.09,
         sources=ModelSource(
             hf="Qdrant/bge-small-zh-v1.5",
-            url="https://storage.googleapis.com/qdrant-fastembed/fast-bge-small-zh-v1.5.tar.gz",
             _deprecated_tar_struct=True,
         ),
         model_file="model_optimized.onnx",

fastembed/text/pooled_embedding.py

@@ -81,7 +81,6 @@ supported_pooled_models: list[DenseModelDescription] = [
         size_in_GB=2.24,
         sources=ModelSource(
             hf="qdrant/multilingual-e5-large-onnx",
-            url="https://storage.googleapis.com/qdrant-fastembed/fast-multilingual-e5-large.tar.gz",
             _deprecated_tar_struct=True,
         ),
         model_file="model.onnx",

fastembed/text/pooled_normalized_embedding.py

@@ -19,7 +19,6 @@ supported_pooled_normalized_models: list[DenseModelDescription] = [
         license="apache-2.0",
         size_in_GB=0.09,
         sources=ModelSource(
-            url="https://storage.googleapis.com/qdrant-fastembed/sentence-transformers-all-MiniLM-L6-v2.tar.gz",
             hf="qdrant/all-MiniLM-L6-v2-onnx",
             _deprecated_tar_struct=True,
         ),