draft

fastembed/config/dense_models.json

@@ -0,0 +1,16 @@
+{
+  "models": [
+    {
+      "model": "BAAI/bge-base-en",
+      "dim": 768,
+      "description": "Text embeddings, Unimodal (text), English...",
+      "license": "mit",
+      "size_in_GB": 0.42,
+      "sources": {
+        "hf": "Qdrant/fast-bge-base-en",
+        "url": "https://storage.googleapis.com/qdrant-fastembed/fast-bge-base-en.tar.gz"
+      },
+      "model_file": "model_optimized.onnx"
+    }
+  ]
+}

fastembed/image/image_embedding.py

@@ -1,4 +1,3 @@
-import warnings
 from typing import Any, Iterable, Optional, Sequence, Type, Union
 from dataclasses import asdict
 
@@ -58,12 +57,6 @@ class ImageEmbedding(ImageEmbeddingBase):
         **kwargs: Any,
     ):
         super().__init__(model_name, cache_dir, threads, **kwargs)
-        if not cuda and device_ids:
-            warnings.warn(
-                "`device_ids` are only used when `cuda` is set to True. Device ids will be ignored.",
-                UserWarning,
-                stacklevel=2,
-            )
         for EMBEDDING_MODEL_TYPE in self.EMBEDDINGS_REGISTRY:
             supported_models = EMBEDDING_MODEL_TYPE._list_supported_models()
             if any(model_name.lower() == model.model.lower() for model in supported_models):

fastembed/late_interaction/late_interaction_text_embedding.py

@@ -1,4 +1,3 @@
-import warnings
 from typing import Any, Iterable, Optional, Sequence, Type, Union
 from dataclasses import asdict
 
@@ -61,13 +60,6 @@ class LateInteractionTextEmbedding(LateInteractionTextEmbeddingBase):
         **kwargs: Any,
     ):
         super().__init__(model_name, cache_dir, threads, **kwargs)
-        if not cuda and device_ids:
-            warnings.warn(
-                "`device_ids` are only used when `cuda` is set to True. Device ids will be ignored.",
-                UserWarning,
-                stacklevel=2,
-            )
-
         for EMBEDDING_MODEL_TYPE in self.EMBEDDINGS_REGISTRY:
             supported_models = EMBEDDING_MODEL_TYPE._list_supported_models()
             if any(model_name.lower() == model.model.lower() for model in supported_models):

fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding.py

@@ -1,4 +1,3 @@
-import warnings
 from typing import Any, Iterable, Optional, Sequence, Type, Union
 from dataclasses import asdict
 
@@ -64,12 +63,6 @@ class LateInteractionMultimodalEmbedding(LateInteractionMultimodalEmbeddingBase)
         **kwargs: Any,
     ):
         super().__init__(model_name, cache_dir, threads, **kwargs)
-        if not cuda and device_ids:
-            warnings.warn(
-                "`device_ids` are only used when `cuda` is set to True. Device ids will be ignored.",
-                UserWarning,
-                stacklevel=2,
-            )
         for EMBEDDING_MODEL_TYPE in self.EMBEDDINGS_REGISTRY:
             supported_models = EMBEDDING_MODEL_TYPE._list_supported_models()
             if any(model_name.lower() == model.model.lower() for model in supported_models):

fastembed/model_loader.py

@@ -0,0 +1,16 @@
+from pathlib import Path
+import json
+from typing import Dict, List
+
+
+class ModelLoader:
+    def __init__(self):
+        self.config_dir = Path(__file__).parent / "configs"
+        self._models: Dict[str, List[Dict]] = {}
+
+    def load_models(self, model_type: str) -> List[Dict]:
+        if model_type not in self._models:
+            config_path = self.config_dir / f"{model_type}_models.json"
+            with open(config_path) as f:
+                self._models[model_type] = json.load(f)["models"]
+        return self._models[model_type]

fastembed/rerank/cross_encoder/text_cross_encoder.py

@@ -1,4 +1,3 @@
-import warnings
 from typing import Any, Iterable, Optional, Sequence, Type
 from dataclasses import asdict
 
@@ -57,12 +56,6 @@ class TextCrossEncoder(TextCrossEncoderBase):
         **kwargs: Any,
     ):
         super().__init__(model_name, cache_dir, threads, **kwargs)
-        if not cuda and device_ids:
-            warnings.warn(
-                "`device_ids` are only used when `cuda` is set to True. Device ids will be ignored.",
-                UserWarning,
-                stacklevel=2,
-            )
 
         for CROSS_ENCODER_TYPE in self.CROSS_ENCODER_REGISTRY:
             supported_models = CROSS_ENCODER_TYPE._list_supported_models()

fastembed/sparse/sparse_text_embedding.py

@@ -61,13 +61,6 @@ class SparseTextEmbedding(SparseTextEmbeddingBase):
         **kwargs: Any,
     ):
         super().__init__(model_name, cache_dir, threads, **kwargs)
-        if not cuda and device_ids:
-            warnings.warn(
-                "`device_ids` are only used when `cuda` is set to True. Device ids will be ignored.",
-                UserWarning,
-                stacklevel=2,
-            )
-
         if model_name == "prithvida/Splade_PP_en_v1":
             warnings.warn(
                 "The right spelling is prithivida/Splade_PP_en_v1. "

fastembed/text/text_embedding.py

@@ -49,12 +49,6 @@ class TextEmbedding(TextEmbeddingBase):
         **kwargs: Any,
     ):
         super().__init__(model_name, cache_dir, threads, **kwargs)
-        if not cuda and device_ids:
-            warnings.warn(
-                "`device_ids` are only used when `cuda` is set to True. Device ids will be ignored.",
-                UserWarning,
-                stacklevel=2,
-            )
         if model_name == "nomic-ai/nomic-embed-text-v1.5-Q":
             warnings.warn(
                 "The model 'nomic-ai/nomic-embed-text-v1.5-Q' has been updated on HuggingFace. "