upstream-archive/fastembed
1
0
Fork 0
mirror of https://github.com/qdrant/fastembed.git synced 2026-07-31 15:11:40 -05:00
Code Issues Packages Projects Releases Wiki Activity
Files
4e699cc9fa7af0899fafdbddffddd082ded78230
fastembed/docs/examples/Supported_Models.ipynb
Anush 0e258ab875 feat: Added jina-embeddings-v2-base-code (#301) 
* feat: Added jina-embeddings-v2-base-code

* fix: test embeddings for "hello world" not "Hello"

* docs: Updated supported models
2024-07-18 18:17:46 +05:30

667 lines
22 KiB
Plaintext
Raw Blame History

{
"cells": [
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"ExecuteTime": {
"end_time": "2024-05-31T18:13:23.806907Z",
"start_time": "2024-05-31T18:13:23.797078Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The autoreload extension is already loaded. To reload it, use:\n",
" %reload_ext autoreload\n"
]
}
],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2024-05-31T18:14:31.147674Z",
"start_time": "2024-05-31T18:14:31.134015Z"
}
},
"outputs": [],
"source": [
"import pandas as pd\n",
"\n",
"from fastembed import (\n",
" SparseTextEmbedding,\n",
" TextEmbedding,\n",
" LateInteractionTextEmbedding,\n",
" ImageEmbedding,\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Supported Text Embedding Models"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"ExecuteTime": {
"end_time": "2024-05-31T18:13:25.863008Z",
"start_time": "2024-05-31T18:13:25.837795Z"
}
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>model</th>\n",
" <th>dim</th>\n",
" <th>description</th>\n",
" <th>size_in_GB</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>BAAI/bge-small-en-v1.5</td>\n",
" <td>384</td>\n",
" <td>Fast and Default English model</td>\n",
" <td>0.067</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>BAAI/bge-small-zh-v1.5</td>\n",
" <td>512</td>\n",
" <td>Fast and recommended Chinese model</td>\n",
" <td>0.090</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>snowflake/snowflake-arctic-embed-xs</td>\n",
" <td>384</td>\n",
" <td>Based on all-MiniLM-L6-v2 model with only 22m ...</td>\n",
" <td>0.090</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>sentence-transformers/all-MiniLM-L6-v2</td>\n",
" <td>384</td>\n",
" <td>Sentence Transformer model, MiniLM-L6-v2</td>\n",
" <td>0.090</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>jinaai/jina-embeddings-v2-small-en</td>\n",
" <td>512</td>\n",
" <td>English embedding model supporting 8192 sequen...</td>\n",
" <td>0.120</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>BAAI/bge-small-en</td>\n",
" <td>384</td>\n",
" <td>Fast English model</td>\n",
" <td>0.130</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>snowflake/snowflake-arctic-embed-s</td>\n",
" <td>384</td>\n",
" <td>Based on infloat/e5-small-unsupervised, does n...</td>\n",
" <td>0.130</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>nomic-ai/nomic-embed-text-v1.5-Q</td>\n",
" <td>768</td>\n",
" <td>Quantized 8192 context length english model</td>\n",
" <td>0.130</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>BAAI/bge-base-en-v1.5</td>\n",
" <td>768</td>\n",
" <td>Base English model, v1.5</td>\n",
" <td>0.210</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>sentence-transformers/paraphrase-multilingual-...</td>\n",
" <td>384</td>\n",
" <td>Sentence Transformer model, paraphrase-multili...</td>\n",
" <td>0.220</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>Qdrant/clip-ViT-B-32-text</td>\n",
" <td>512</td>\n",
" <td>CLIP text encoder</td>\n",
" <td>0.250</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>jinaai/jina-embeddings-v2-base-de</td>\n",
" <td>768</td>\n",
" <td>German embedding model supporting 8192 sequenc...</td>\n",
" <td>0.320</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>BAAI/bge-base-en</td>\n",
" <td>768</td>\n",
" <td>Base English model</td>\n",
" <td>0.420</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>snowflake/snowflake-arctic-embed-m</td>\n",
" <td>768</td>\n",
" <td>Based on intfloat/e5-base-unsupervised model, ...</td>\n",
" <td>0.430</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>nomic-ai/nomic-embed-text-v1.5</td>\n",
" <td>768</td>\n",
" <td>8192 context length english model</td>\n",
" <td>0.520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>jinaai/jina-embeddings-v2-base-en</td>\n",
" <td>768</td>\n",
" <td>English embedding model supporting 8192 sequen...</td>\n",
" <td>0.520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>nomic-ai/nomic-embed-text-v1</td>\n",
" <td>768</td>\n",
" <td>8192 context length english model</td>\n",
" <td>0.520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>snowflake/snowflake-arctic-embed-m-long</td>\n",
" <td>768</td>\n",
" <td>Based on nomic-ai/nomic-embed-text-v1-unsuperv...</td>\n",
" <td>0.540</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>mixedbread-ai/mxbai-embed-large-v1</td>\n",
" <td>1024</td>\n",
" <td>MixedBread Base sentence embedding model, does...</td>\n",
" <td>0.640</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>jinaai/jina-embeddings-v2-base-code</td>\n",
" <td>768</td>\n",
" <td>Source code embedding model supporting 8192 se...</td>\n",
" <td>0.640</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>sentence-transformers/paraphrase-multilingual-...</td>\n",
" <td>768</td>\n",
" <td>Sentence-transformers model for tasks like clu...</td>\n",
" <td>1.000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21</th>\n",
" <td>snowflake/snowflake-arctic-embed-l</td>\n",
" <td>1024</td>\n",
" <td>Based on intfloat/e5-large-unsupervised, large...</td>\n",
" <td>1.020</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22</th>\n",
" <td>thenlper/gte-large</td>\n",
" <td>1024</td>\n",
" <td>Large general text embeddings model</td>\n",
" <td>1.200</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23</th>\n",
" <td>BAAI/bge-large-en-v1.5</td>\n",
" <td>1024</td>\n",
" <td>Large English model, v1.5</td>\n",
" <td>1.200</td>\n",
" </tr>\n",
" <tr>\n",
" <th>24</th>\n",
" <td>intfloat/multilingual-e5-large</td>\n",
" <td>1024</td>\n",
" <td>Multilingual model, e5-large. Recommend using ...</td>\n",
" <td>2.240</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" model dim \\\n",
"0 BAAI/bge-small-en-v1.5 384 \n",
"1 BAAI/bge-small-zh-v1.5 512 \n",
"2 snowflake/snowflake-arctic-embed-xs 384 \n",
"3 sentence-transformers/all-MiniLM-L6-v2 384 \n",
"4 jinaai/jina-embeddings-v2-small-en 512 \n",
"5 BAAI/bge-small-en 384 \n",
"6 snowflake/snowflake-arctic-embed-s 384 \n",
"7 nomic-ai/nomic-embed-text-v1.5-Q 768 \n",
"8 BAAI/bge-base-en-v1.5 768 \n",
"9 sentence-transformers/paraphrase-multilingual-... 384 \n",
"10 Qdrant/clip-ViT-B-32-text 512 \n",
"11 jinaai/jina-embeddings-v2-base-de 768 \n",
"12 BAAI/bge-base-en 768 \n",
"13 snowflake/snowflake-arctic-embed-m 768 \n",
"14 nomic-ai/nomic-embed-text-v1.5 768 \n",
"15 jinaai/jina-embeddings-v2-base-en 768 \n",
"16 nomic-ai/nomic-embed-text-v1 768 \n",
"17 snowflake/snowflake-arctic-embed-m-long 768 \n",
"18 mixedbread-ai/mxbai-embed-large-v1 1024 \n",
"19 jinaai/jina-embeddings-v2-base-code 768 \n",
"20 sentence-transformers/paraphrase-multilingual-... 768 \n",
"21 snowflake/snowflake-arctic-embed-l 1024 \n",
"22 thenlper/gte-large 1024 \n",
"23 BAAI/bge-large-en-v1.5 1024 \n",
"24 intfloat/multilingual-e5-large 1024 \n",
"\n",
" description size_in_GB \n",
"0 Fast and Default English model 0.067 \n",
"1 Fast and recommended Chinese model 0.090 \n",
"2 Based on all-MiniLM-L6-v2 model with only 22m ... 0.090 \n",
"3 Sentence Transformer model, MiniLM-L6-v2 0.090 \n",
"4 English embedding model supporting 8192 sequen... 0.120 \n",
"5 Fast English model 0.130 \n",
"6 Based on infloat/e5-small-unsupervised, does n... 0.130 \n",
"7 Quantized 8192 context length english model 0.130 \n",
"8 Base English model, v1.5 0.210 \n",
"9 Sentence Transformer model, paraphrase-multili... 0.220 \n",
"10 CLIP text encoder 0.250 \n",
"11 German embedding model supporting 8192 sequenc... 0.320 \n",
"12 Base English model 0.420 \n",
"13 Based on intfloat/e5-base-unsupervised model, ... 0.430 \n",
"14 8192 context length english model 0.520 \n",
"15 English embedding model supporting 8192 sequen... 0.520 \n",
"16 8192 context length english model 0.520 \n",
"17 Based on nomic-ai/nomic-embed-text-v1-unsuperv... 0.540 \n",
"18 MixedBread Base sentence embedding model, does... 0.640 \n",
"19 Source code embedding model supporting 8192 se... 0.640 \n",
"20 Sentence-transformers model for tasks like clu... 1.000 \n",
"21 Based on intfloat/e5-large-unsupervised, large... 1.020 \n",
"22 Large general text embeddings model 1.200 \n",
"23 Large English model, v1.5 1.200 \n",
"24 Multilingual model, e5-large. Recommend using ... 2.240 "
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"supported_models = (\n",
" pd.DataFrame(TextEmbedding.list_supported_models())\n",
" .sort_values(\"size_in_GB\")\n",
" .drop(columns=[\"sources\", \"model_file\", \"additional_files\"])\n",
" .reset_index(drop=True)\n",
")\n",
"supported_models"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Supported Sparse Text Embedding Models"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"ExecuteTime": {
"end_time": "2024-05-31T18:13:27.124747Z",
"start_time": "2024-05-31T18:13:27.096212Z"
}
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>model</th>\n",
" <th>vocab_size</th>\n",
" <th>description</th>\n",
" <th>size_in_GB</th>\n",
" <th>requires_idf</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Qdrant/bm25</td>\n",
" <td>NaN</td>\n",
" <td>BM25 as sparse embeddings meant to be used wit...</td>\n",
" <td>0.010</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Qdrant/bm42-all-minilm-l6-v2-attentions</td>\n",
" <td>30522.0</td>\n",
" <td>Light sparse embedding model, which assigns an...</td>\n",
" <td>0.090</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>prithvida/Splade_PP_en_v1</td>\n",
" <td>30522.0</td>\n",
" <td>Misspelled version of the model. Retained for ...</td>\n",
" <td>0.532</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>prithivida/Splade_PP_en_v1</td>\n",
" <td>30522.0</td>\n",
" <td>Independent Implementation of SPLADE++ Model f...</td>\n",
" <td>0.532</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" model vocab_size \\\n",
"0 Qdrant/bm25 NaN \n",
"1 Qdrant/bm42-all-minilm-l6-v2-attentions 30522.0 \n",
"2 prithvida/Splade_PP_en_v1 30522.0 \n",
"3 prithivida/Splade_PP_en_v1 30522.0 \n",
"\n",
" description size_in_GB requires_idf \n",
"0 BM25 as sparse embeddings meant to be used wit... 0.010 True \n",
"1 Light sparse embedding model, which assigns an... 0.090 True \n",
"2 Misspelled version of the model. Retained for ... 0.532 NaN \n",
"3 Independent Implementation of SPLADE++ Model f... 0.532 NaN "
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"(\n",
" pd.DataFrame(SparseTextEmbedding.list_supported_models())\n",
" .sort_values(\"size_in_GB\")\n",
" .drop(columns=[\"sources\", \"model_file\", \"additional_files\"])\n",
" .reset_index(drop=True)\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false
},
"source": [
"## Supported Late Interaction Text Embedding Models"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"ExecuteTime": {
"end_time": "2024-05-31T18:14:34.370252Z",
"start_time": "2024-05-31T18:14:34.354270Z"
},
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>model</th>\n",
" <th>dim</th>\n",
" <th>description</th>\n",
" <th>size_in_GB</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>colbert-ir/colbertv2.0</td>\n",
" <td>128</td>\n",
" <td>Late interaction model</td>\n",
" <td>0.44</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" model dim description size_in_GB\n",
"0 colbert-ir/colbertv2.0 128 Late interaction model 0.44"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"(\n",
" pd.DataFrame(LateInteractionTextEmbedding.list_supported_models())\n",
" .sort_values(\"size_in_GB\")\n",
" .drop(columns=[\"sources\", \"model_file\"])\n",
" .reset_index(drop=True)\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": false
},
"source": [
"## Supported Image Embedding Models"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"ExecuteTime": {
"end_time": "2024-05-31T18:14:42.501881Z",
"start_time": "2024-05-31T18:14:42.484726Z"
},
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>model</th>\n",
" <th>dim</th>\n",
" <th>description</th>\n",
" <th>size_in_GB</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Qdrant/resnet50-onnx</td>\n",
" <td>2048</td>\n",
" <td>ResNet-50 from `Deep Residual Learning for Ima...</td>\n",
" <td>0.10</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Qdrant/clip-ViT-B-32-vision</td>\n",
" <td>512</td>\n",
" <td>CLIP vision encoder based on ViT-B/32</td>\n",
" <td>0.34</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Qdrant/Unicom-ViT-B-32</td>\n",
" <td>512</td>\n",
" <td>Unicom Unicom-ViT-B-32 from open-metric-learning</td>\n",
" <td>0.48</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Qdrant/Unicom-ViT-B-16</td>\n",
" <td>768</td>\n",
" <td>Unicom Unicom-ViT-B-16 from open-metric-learning</td>\n",
" <td>0.82</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" model dim \\\n",
"0 Qdrant/resnet50-onnx 2048 \n",
"1 Qdrant/clip-ViT-B-32-vision 512 \n",
"2 Qdrant/Unicom-ViT-B-32 512 \n",
"3 Qdrant/Unicom-ViT-B-16 768 \n",
"\n",
" description size_in_GB \n",
"0 ResNet-50 from `Deep Residual Learning for Ima... 0.10 \n",
"1 CLIP vision encoder based on ViT-B/32 0.34 \n",
"2 Unicom Unicom-ViT-B-32 from open-metric-learning 0.48 \n",
"3 Unicom Unicom-ViT-B-16 from open-metric-learning 0.82 "
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"(\n",
" pd.DataFrame(ImageEmbedding.list_supported_models()).sort_values(\"size_in_GB\")\n",
" .drop(columns=[\"sources\", \"model_file\"])\n",
" .reset_index(drop=True)\n",
")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.8.18 ('base')",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.8"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "c4a27af61e455bc18dcf16f5867a2ff0402fa12b01dd0f6ce3a79ae73ad15e91"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Reference in New Issue View Git Blame Copy Permalink