upstream-archive/fastembed
1
0
Fork 0
mirror of https://github.com/qdrant/fastembed.git synced 2026-07-26 12:41:07 -05:00
Code Issues Packages Projects Releases Wiki Activity
Files
QModels
fastembed/docs/examples/Supported_Models.ipynb

356 lines
12 KiB
Plaintext
Raw Permalink Blame History

{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2024-03-30T11:18:52.052764Z",
"start_time": "2024-03-30T11:18:52.039616Z"
}
},
"outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"\n",
"from fastembed import SparseTextEmbedding, TextEmbedding"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Supported Text Embedding Models"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>model</th>\n",
" <th>dim</th>\n",
" <th>description</th>\n",
" <th>size_in_GB</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>BAAI/bge-small-en-v1.5</td>\n",
" <td>384</td>\n",
" <td>Fast and Default English model</td>\n",
" <td>0.067</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>BAAI/bge-small-zh-v1.5</td>\n",
" <td>512</td>\n",
" <td>Fast and recommended Chinese model</td>\n",
" <td>0.090</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>sentence-transformers/all-MiniLM-L6-v2</td>\n",
" <td>384</td>\n",
" <td>Sentence Transformer model, MiniLM-L6-v2</td>\n",
" <td>0.090</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>jinaai/jina-embeddings-v2-small-en</td>\n",
" <td>512</td>\n",
" <td>English embedding model supporting 8192 sequen...</td>\n",
" <td>0.120</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>BAAI/bge-small-en</td>\n",
" <td>384</td>\n",
" <td>Fast English model</td>\n",
" <td>0.130</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>BAAI/bge-base-en-v1.5</td>\n",
" <td>768</td>\n",
" <td>Base English model, v1.5</td>\n",
" <td>0.210</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>sentence-transformers/paraphrase-multilingual-...</td>\n",
" <td>384</td>\n",
" <td>Sentence Transformer model, paraphrase-multili...</td>\n",
" <td>0.220</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>BAAI/bge-base-en</td>\n",
" <td>768</td>\n",
" <td>Base English model</td>\n",
" <td>0.420</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>nomic-ai/nomic-embed-text-v1</td>\n",
" <td>768</td>\n",
" <td>8192 context length english model</td>\n",
" <td>0.520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>nomic-ai/nomic-embed-text-v1.5</td>\n",
" <td>768</td>\n",
" <td>8192 context length english model</td>\n",
" <td>0.520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>jinaai/jina-embeddings-v2-base-en</td>\n",
" <td>768</td>\n",
" <td>English embedding model supporting 8192 sequen...</td>\n",
" <td>0.520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>mixedbread-ai/mxbai-embed-large-v1</td>\n",
" <td>1024</td>\n",
" <td>MixedBread Base sentence embedding model, does...</td>\n",
" <td>0.640</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>sentence-transformers/paraphrase-multilingual-...</td>\n",
" <td>768</td>\n",
" <td>Sentence-transformers model for tasks like clu...</td>\n",
" <td>1.000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>BAAI/bge-large-en-v1.5</td>\n",
" <td>1024</td>\n",
" <td>Large English model, v1.5</td>\n",
" <td>1.200</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>thenlper/gte-large</td>\n",
" <td>1024</td>\n",
" <td>Large general text embeddings model</td>\n",
" <td>1.200</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>intfloat/multilingual-e5-large</td>\n",
" <td>1024</td>\n",
" <td>Multilingual model, e5-large. Recommend using ...</td>\n",
" <td>2.240</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" model dim \\\n",
"0 BAAI/bge-small-en-v1.5 384 \n",
"1 BAAI/bge-small-zh-v1.5 512 \n",
"2 sentence-transformers/all-MiniLM-L6-v2 384 \n",
"3 jinaai/jina-embeddings-v2-small-en 512 \n",
"4 BAAI/bge-small-en 384 \n",
"5 BAAI/bge-base-en-v1.5 768 \n",
"6 sentence-transformers/paraphrase-multilingual-... 384 \n",
"7 BAAI/bge-base-en 768 \n",
"8 nomic-ai/nomic-embed-text-v1 768 \n",
"9 nomic-ai/nomic-embed-text-v1.5 768 \n",
"10 jinaai/jina-embeddings-v2-base-en 768 \n",
"11 mixedbread-ai/mxbai-embed-large-v1 1024 \n",
"12 sentence-transformers/paraphrase-multilingual-... 768 \n",
"13 BAAI/bge-large-en-v1.5 1024 \n",
"14 thenlper/gte-large 1024 \n",
"15 intfloat/multilingual-e5-large 1024 \n",
"\n",
" description size_in_GB \n",
"0 Fast and Default English model 0.067 \n",
"1 Fast and recommended Chinese model 0.090 \n",
"2 Sentence Transformer model, MiniLM-L6-v2 0.090 \n",
"3 English embedding model supporting 8192 sequen... 0.120 \n",
"4 Fast English model 0.130 \n",
"5 Base English model, v1.5 0.210 \n",
"6 Sentence Transformer model, paraphrase-multili... 0.220 \n",
"7 Base English model 0.420 \n",
"8 8192 context length english model 0.520 \n",
"9 8192 context length english model 0.520 \n",
"10 English embedding model supporting 8192 sequen... 0.520 \n",
"11 MixedBread Base sentence embedding model, does... 0.640 \n",
"12 Sentence-transformers model for tasks like clu... 1.000 \n",
"13 Large English model, v1.5 1.200 \n",
"14 Large general text embeddings model 1.200 \n",
"15 Multilingual model, e5-large. Recommend using ... 2.240 "
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"supported_models = (\n",
" pd.DataFrame(TextEmbedding.list_supported_models())\n",
" .sort_values(\"size_in_GB\")\n",
" .drop(columns=\"sources\")\n",
" .reset_index(drop=True)\n",
")\n",
"supported_models"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Supported Sparse Text Embedding Models"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"ExecuteTime": {
"end_time": "2024-03-30T11:19:01.564291Z",
"start_time": "2024-03-30T11:19:01.538768Z"
}
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>model</th>\n",
" <th>vocab_size</th>\n",
" <th>description</th>\n",
" <th>size_in_GB</th>\n",
" <th>sources</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>prithvida/Splade_PP_en_v1</td>\n",
" <td>30522</td>\n",
" <td>Misspelled version of the model. Retained for ...</td>\n",
" <td>0.532</td>\n",
" <td>{'hf': 'Qdrant/SPLADE_PP_en_v1'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>prithivida/Splade_PP_en_v1</td>\n",
" <td>30522</td>\n",
" <td>Independent Implementation of SPLADE++ Model f...</td>\n",
" <td>0.532</td>\n",
" <td>{'hf': 'Qdrant/SPLADE_PP_en_v1'}</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" model vocab_size \\\n",
"0 prithvida/Splade_PP_en_v1 30522 \n",
"1 prithivida/Splade_PP_en_v1 30522 \n",
"\n",
" description size_in_GB \\\n",
"0 Misspelled version of the model. Retained for ... 0.532 \n",
"1 Independent Implementation of SPLADE++ Model f... 0.532 \n",
"\n",
" sources \n",
"0 {'hf': 'Qdrant/SPLADE_PP_en_v1'} \n",
"1 {'hf': 'Qdrant/SPLADE_PP_en_v1'} "
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pd.DataFrame(SparseTextEmbedding.list_supported_models())"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.8.18 ('base')",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.13"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "c4a27af61e455bc18dcf16f5867a2ff0402fa12b01dd0f6ce3a79ae73ad15e91"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Reference in New Issue View Git Blame Copy Permalink