mirror of
https://github.com/qdrant/fastembed.git
synced 2026-07-25 20:21:10 -05:00
cc4112d859
* feat: Snowflake models * Added snowflake/snowflake-arctic-embed-m * docs: snowflake/snowflake-arctic-embed-m
401 lines
14 KiB
Plaintext
401 lines
14 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {
|
|
"ExecuteTime": {
|
|
"end_time": "2024-03-30T11:18:52.052764Z",
|
|
"start_time": "2024-03-30T11:18:52.039616Z"
|
|
}
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"%load_ext autoreload\n",
|
|
"%autoreload 2"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"from fastembed import SparseTextEmbedding, TextEmbedding"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Supported Text Embedding Models"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>model</th>\n",
|
|
" <th>dim</th>\n",
|
|
" <th>description</th>\n",
|
|
" <th>size_in_GB</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>BAAI/bge-small-en-v1.5</td>\n",
|
|
" <td>384</td>\n",
|
|
" <td>Fast and Default English model</td>\n",
|
|
" <td>0.067</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>BAAI/bge-small-zh-v1.5</td>\n",
|
|
" <td>512</td>\n",
|
|
" <td>Fast and recommended Chinese model</td>\n",
|
|
" <td>0.090</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>sentence-transformers/all-MiniLM-L6-v2</td>\n",
|
|
" <td>384</td>\n",
|
|
" <td>Sentence Transformer model, MiniLM-L6-v2</td>\n",
|
|
" <td>0.090</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>snowflake/snowflake-arctic-embed-xs</td>\n",
|
|
" <td>384</td>\n",
|
|
" <td>Based on all-MiniLM-L6-v2 model with only 22m ...</td>\n",
|
|
" <td>0.090</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>jinaai/jina-embeddings-v2-small-en</td>\n",
|
|
" <td>512</td>\n",
|
|
" <td>English embedding model supporting 8192 sequen...</td>\n",
|
|
" <td>0.120</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5</th>\n",
|
|
" <td>snowflake/snowflake-arctic-embed-s</td>\n",
|
|
" <td>384</td>\n",
|
|
" <td>Based on infloat/e5-small-unsupervised, does n...</td>\n",
|
|
" <td>0.130</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>6</th>\n",
|
|
" <td>BAAI/bge-small-en</td>\n",
|
|
" <td>384</td>\n",
|
|
" <td>Fast English model</td>\n",
|
|
" <td>0.130</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>7</th>\n",
|
|
" <td>BAAI/bge-base-en-v1.5</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Base English model, v1.5</td>\n",
|
|
" <td>0.210</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>8</th>\n",
|
|
" <td>sentence-transformers/paraphrase-multilingual-...</td>\n",
|
|
" <td>384</td>\n",
|
|
" <td>Sentence Transformer model, paraphrase-multili...</td>\n",
|
|
" <td>0.220</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>9</th>\n",
|
|
" <td>BAAI/bge-base-en</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Base English model</td>\n",
|
|
" <td>0.420</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>10</th>\n",
|
|
" <td>snowflake/snowflake-arctic-embed-m</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Based on intfloat/e5-base-unsupervised model, ...</td>\n",
|
|
" <td>0.430</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>11</th>\n",
|
|
" <td>jinaai/jina-embeddings-v2-base-en</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>English embedding model supporting 8192 sequen...</td>\n",
|
|
" <td>0.520</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>12</th>\n",
|
|
" <td>nomic-ai/nomic-embed-text-v1</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>8192 context length english model</td>\n",
|
|
" <td>0.520</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>13</th>\n",
|
|
" <td>nomic-ai/nomic-embed-text-v1.5</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>8192 context length english model</td>\n",
|
|
" <td>0.520</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>14</th>\n",
|
|
" <td>snowflake/snowflake-arctic-embed-m-long</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Based on nomic-ai/nomic-embed-text-v1-unsuperv...</td>\n",
|
|
" <td>0.540</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>15</th>\n",
|
|
" <td>mixedbread-ai/mxbai-embed-large-v1</td>\n",
|
|
" <td>1024</td>\n",
|
|
" <td>MixedBread Base sentence embedding model, does...</td>\n",
|
|
" <td>0.640</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>16</th>\n",
|
|
" <td>sentence-transformers/paraphrase-multilingual-...</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Sentence-transformers model for tasks like clu...</td>\n",
|
|
" <td>1.000</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>17</th>\n",
|
|
" <td>snowflake/snowflake-arctic-embed-l</td>\n",
|
|
" <td>1024</td>\n",
|
|
" <td>Based on intfloat/e5-large-unsupervised, large...</td>\n",
|
|
" <td>1.020</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>18</th>\n",
|
|
" <td>BAAI/bge-large-en-v1.5</td>\n",
|
|
" <td>1024</td>\n",
|
|
" <td>Large English model, v1.5</td>\n",
|
|
" <td>1.200</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>19</th>\n",
|
|
" <td>thenlper/gte-large</td>\n",
|
|
" <td>1024</td>\n",
|
|
" <td>Large general text embeddings model</td>\n",
|
|
" <td>1.200</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>20</th>\n",
|
|
" <td>intfloat/multilingual-e5-large</td>\n",
|
|
" <td>1024</td>\n",
|
|
" <td>Multilingual model, e5-large. Recommend using ...</td>\n",
|
|
" <td>2.240</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" model dim \\\n",
|
|
"0 BAAI/bge-small-en-v1.5 384 \n",
|
|
"1 BAAI/bge-small-zh-v1.5 512 \n",
|
|
"2 sentence-transformers/all-MiniLM-L6-v2 384 \n",
|
|
"3 snowflake/snowflake-arctic-embed-xs 384 \n",
|
|
"4 jinaai/jina-embeddings-v2-small-en 512 \n",
|
|
"5 snowflake/snowflake-arctic-embed-s 384 \n",
|
|
"6 BAAI/bge-small-en 384 \n",
|
|
"7 BAAI/bge-base-en-v1.5 768 \n",
|
|
"8 sentence-transformers/paraphrase-multilingual-... 384 \n",
|
|
"9 BAAI/bge-base-en 768 \n",
|
|
"10 snowflake/snowflake-arctic-embed-m 768 \n",
|
|
"11 jinaai/jina-embeddings-v2-base-en 768 \n",
|
|
"12 nomic-ai/nomic-embed-text-v1 768 \n",
|
|
"13 nomic-ai/nomic-embed-text-v1.5 768 \n",
|
|
"14 snowflake/snowflake-arctic-embed-m-long 768 \n",
|
|
"15 mixedbread-ai/mxbai-embed-large-v1 1024 \n",
|
|
"16 sentence-transformers/paraphrase-multilingual-... 768 \n",
|
|
"17 snowflake/snowflake-arctic-embed-l 1024 \n",
|
|
"18 BAAI/bge-large-en-v1.5 1024 \n",
|
|
"19 thenlper/gte-large 1024 \n",
|
|
"20 intfloat/multilingual-e5-large 1024 \n",
|
|
"\n",
|
|
" description size_in_GB \n",
|
|
"0 Fast and Default English model 0.067 \n",
|
|
"1 Fast and recommended Chinese model 0.090 \n",
|
|
"2 Sentence Transformer model, MiniLM-L6-v2 0.090 \n",
|
|
"3 Based on all-MiniLM-L6-v2 model with only 22m ... 0.090 \n",
|
|
"4 English embedding model supporting 8192 sequen... 0.120 \n",
|
|
"5 Based on infloat/e5-small-unsupervised, does n... 0.130 \n",
|
|
"6 Fast English model 0.130 \n",
|
|
"7 Base English model, v1.5 0.210 \n",
|
|
"8 Sentence Transformer model, paraphrase-multili... 0.220 \n",
|
|
"9 Base English model 0.420 \n",
|
|
"10 Based on intfloat/e5-base-unsupervised model, ... 0.430 \n",
|
|
"11 English embedding model supporting 8192 sequen... 0.520 \n",
|
|
"12 8192 context length english model 0.520 \n",
|
|
"13 8192 context length english model 0.520 \n",
|
|
"14 Based on nomic-ai/nomic-embed-text-v1-unsuperv... 0.540 \n",
|
|
"15 MixedBread Base sentence embedding model, does... 0.640 \n",
|
|
"16 Sentence-transformers model for tasks like clu... 1.000 \n",
|
|
"17 Based on intfloat/e5-large-unsupervised, large... 1.020 \n",
|
|
"18 Large English model, v1.5 1.200 \n",
|
|
"19 Large general text embeddings model 1.200 \n",
|
|
"20 Multilingual model, e5-large. Recommend using ... 2.240 "
|
|
]
|
|
},
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"supported_models = (\n",
|
|
" pd.DataFrame(TextEmbedding.list_supported_models())\n",
|
|
" .sort_values(\"size_in_GB\")\n",
|
|
" .drop(columns=\"sources\")\n",
|
|
" .reset_index(drop=True)\n",
|
|
")\n",
|
|
"supported_models"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Supported Sparse Text Embedding Models"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {
|
|
"ExecuteTime": {
|
|
"end_time": "2024-03-30T11:19:01.564291Z",
|
|
"start_time": "2024-03-30T11:19:01.538768Z"
|
|
}
|
|
},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>model</th>\n",
|
|
" <th>vocab_size</th>\n",
|
|
" <th>description</th>\n",
|
|
" <th>size_in_GB</th>\n",
|
|
" <th>sources</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>prithvida/Splade_PP_en_v1</td>\n",
|
|
" <td>30522</td>\n",
|
|
" <td>Misspelled version of the model. Retained for ...</td>\n",
|
|
" <td>0.532</td>\n",
|
|
" <td>{'hf': 'Qdrant/SPLADE_PP_en_v1'}</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>prithivida/Splade_PP_en_v1</td>\n",
|
|
" <td>30522</td>\n",
|
|
" <td>Independent Implementation of SPLADE++ Model f...</td>\n",
|
|
" <td>0.532</td>\n",
|
|
" <td>{'hf': 'Qdrant/SPLADE_PP_en_v1'}</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" model vocab_size \\\n",
|
|
"0 prithvida/Splade_PP_en_v1 30522 \n",
|
|
"1 prithivida/Splade_PP_en_v1 30522 \n",
|
|
"\n",
|
|
" description size_in_GB \\\n",
|
|
"0 Misspelled version of the model. Retained for ... 0.532 \n",
|
|
"1 Independent Implementation of SPLADE++ Model f... 0.532 \n",
|
|
"\n",
|
|
" sources \n",
|
|
"0 {'hf': 'Qdrant/SPLADE_PP_en_v1'} \n",
|
|
"1 {'hf': 'Qdrant/SPLADE_PP_en_v1'} "
|
|
]
|
|
},
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"pd.DataFrame(SparseTextEmbedding.list_supported_models())"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3.8.18 ('base')",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.13"
|
|
},
|
|
"orig_nbformat": 4,
|
|
"vscode": {
|
|
"interpreter": {
|
|
"hash": "c4a27af61e455bc18dcf16f5867a2ff0402fa12b01dd0f6ce3a79ae73ad15e91"
|
|
}
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|