mirror of
https://github.com/qdrant/fastembed.git
synced 2026-07-23 11:20:51 -05:00
860e2ad691
* feat: Added jina reranker models * chore: Added jina reranker canonical score values * chore: added rounding of the output for easier reproducability * chore: Added jina reranker models in batch test * chore: remove redundant np.round * chore: test only <1gb files in local * chore: Updated docs to add rerankers * fix: recompute canonical values with fp16 * new: extend NOTICE with jina reranker v2 --------- Co-authored-by: George Panchuk <george.panchuk@qdrant.tech>
875 lines
30 KiB
Plaintext
875 lines
30 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"metadata": {
|
|
"ExecuteTime": {
|
|
"end_time": "2024-11-13T09:01:03.324551Z",
|
|
"start_time": "2024-11-13T09:01:03.234711Z"
|
|
}
|
|
},
|
|
"source": [
|
|
"%load_ext autoreload\n",
|
|
"%autoreload 2"
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"The autoreload extension is already loaded. To reload it, use:\n",
|
|
" %reload_ext autoreload\n"
|
|
]
|
|
}
|
|
],
|
|
"execution_count": 10
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"metadata": {
|
|
"ExecuteTime": {
|
|
"end_time": "2024-11-13T09:01:04.505772Z",
|
|
"start_time": "2024-11-13T09:01:04.493296Z"
|
|
}
|
|
},
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"from fastembed import (\n",
|
|
" SparseTextEmbedding,\n",
|
|
" TextEmbedding,\n",
|
|
" LateInteractionTextEmbedding,\n",
|
|
" ImageEmbedding,\n",
|
|
")\n",
|
|
"from fastembed.rerank.cross_encoder import TextCrossEncoder"
|
|
],
|
|
"outputs": [],
|
|
"execution_count": 11
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Supported Text Embedding Models"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"metadata": {
|
|
"ExecuteTime": {
|
|
"end_time": "2024-11-13T09:01:05.812271Z",
|
|
"start_time": "2024-11-13T09:01:05.795846Z"
|
|
}
|
|
},
|
|
"source": [
|
|
"supported_models = (\n",
|
|
" pd.DataFrame(TextEmbedding.list_supported_models())\n",
|
|
" .sort_values(\"size_in_GB\")\n",
|
|
" .drop(columns=[\"sources\", \"model_file\", \"additional_files\"])\n",
|
|
" .reset_index(drop=True)\n",
|
|
")\n",
|
|
"supported_models"
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
" model dim \\\n",
|
|
"0 BAAI/bge-small-en-v1.5 384 \n",
|
|
"1 BAAI/bge-small-zh-v1.5 512 \n",
|
|
"2 snowflake/snowflake-arctic-embed-xs 384 \n",
|
|
"3 sentence-transformers/all-MiniLM-L6-v2 384 \n",
|
|
"4 jinaai/jina-embeddings-v2-small-en 512 \n",
|
|
"5 BAAI/bge-small-en 384 \n",
|
|
"6 snowflake/snowflake-arctic-embed-s 384 \n",
|
|
"7 nomic-ai/nomic-embed-text-v1.5-Q 768 \n",
|
|
"8 BAAI/bge-base-en-v1.5 768 \n",
|
|
"9 sentence-transformers/paraphrase-multilingual-... 384 \n",
|
|
"10 Qdrant/clip-ViT-B-32-text 512 \n",
|
|
"11 jinaai/jina-embeddings-v2-base-de 768 \n",
|
|
"12 BAAI/bge-base-en 768 \n",
|
|
"13 snowflake/snowflake-arctic-embed-m 768 \n",
|
|
"14 nomic-ai/nomic-embed-text-v1.5 768 \n",
|
|
"15 jinaai/jina-embeddings-v2-base-en 768 \n",
|
|
"16 nomic-ai/nomic-embed-text-v1 768 \n",
|
|
"17 snowflake/snowflake-arctic-embed-m-long 768 \n",
|
|
"18 mixedbread-ai/mxbai-embed-large-v1 1024 \n",
|
|
"19 jinaai/jina-embeddings-v2-base-code 768 \n",
|
|
"20 sentence-transformers/paraphrase-multilingual-... 768 \n",
|
|
"21 snowflake/snowflake-arctic-embed-l 1024 \n",
|
|
"22 thenlper/gte-large 1024 \n",
|
|
"23 BAAI/bge-large-en-v1.5 1024 \n",
|
|
"24 intfloat/multilingual-e5-large 1024 \n",
|
|
"\n",
|
|
" description license size_in_GB \n",
|
|
"0 Text embeddings, Unimodal (text), English, 512... mit 0.067 \n",
|
|
"1 Text embeddings, Unimodal (text), Chinese, 512... mit 0.090 \n",
|
|
"2 Text embeddings, Unimodal (text), English, 512... apache-2.0 0.090 \n",
|
|
"3 Text embeddings, Unimodal (text), English, 256... apache-2.0 0.090 \n",
|
|
"4 Text embeddings, Unimodal (text), English, 819... apache-2.0 0.120 \n",
|
|
"5 Text embeddings, Unimodal (text), English, 512... mit 0.130 \n",
|
|
"6 Text embeddings, Unimodal (text), English, 512... apache-2.0 0.130 \n",
|
|
"7 Text embeddings, Multimodal (text, image), Eng... apache-2.0 0.130 \n",
|
|
"8 Text embeddings, Unimodal (text), English, 512... mit 0.210 \n",
|
|
"9 Text embeddings, Unimodal (text), Multilingual... apache-2.0 0.220 \n",
|
|
"10 Text embeddings, Multimodal (text&image), Engl... mit 0.250 \n",
|
|
"11 Text embeddings, Unimodal (text), Multilingual... apache-2.0 0.320 \n",
|
|
"12 Text embeddings, Unimodal (text), English, 512... mit 0.420 \n",
|
|
"13 Text embeddings, Unimodal (text), English, 512... apache-2.0 0.430 \n",
|
|
"14 Text embeddings, Multimodal (text, image), Eng... apache-2.0 0.520 \n",
|
|
"15 Text embeddings, Unimodal (text), English, 819... apache-2.0 0.520 \n",
|
|
"16 Text embeddings, Multimodal (text, image), Eng... apache-2.0 0.520 \n",
|
|
"17 Text embeddings, Unimodal (text), English, 204... apache-2.0 0.540 \n",
|
|
"18 Text embeddings, Unimodal (text), English, 512... apache-2.0 0.640 \n",
|
|
"19 Text embeddings, Unimodal (text), Multilingual... apache-2.0 0.640 \n",
|
|
"20 Text embeddings, Unimodal (text), Multilingual... apache-2.0 1.000 \n",
|
|
"21 Text embeddings, Unimodal (text), English, 512... apache-2.0 1.020 \n",
|
|
"22 Text embeddings, Unimodal (text), English, 512... mit 1.200 \n",
|
|
"23 Text embeddings, Unimodal (text), English, 512... mit 1.200 \n",
|
|
"24 Text embeddings, Unimodal (text), Multilingual... mit 2.240 "
|
|
],
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>model</th>\n",
|
|
" <th>dim</th>\n",
|
|
" <th>description</th>\n",
|
|
" <th>license</th>\n",
|
|
" <th>size_in_GB</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>BAAI/bge-small-en-v1.5</td>\n",
|
|
" <td>384</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), English, 512...</td>\n",
|
|
" <td>mit</td>\n",
|
|
" <td>0.067</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>BAAI/bge-small-zh-v1.5</td>\n",
|
|
" <td>512</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), Chinese, 512...</td>\n",
|
|
" <td>mit</td>\n",
|
|
" <td>0.090</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>snowflake/snowflake-arctic-embed-xs</td>\n",
|
|
" <td>384</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), English, 512...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.090</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>sentence-transformers/all-MiniLM-L6-v2</td>\n",
|
|
" <td>384</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), English, 256...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.090</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>jinaai/jina-embeddings-v2-small-en</td>\n",
|
|
" <td>512</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), English, 819...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.120</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5</th>\n",
|
|
" <td>BAAI/bge-small-en</td>\n",
|
|
" <td>384</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), English, 512...</td>\n",
|
|
" <td>mit</td>\n",
|
|
" <td>0.130</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>6</th>\n",
|
|
" <td>snowflake/snowflake-arctic-embed-s</td>\n",
|
|
" <td>384</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), English, 512...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.130</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>7</th>\n",
|
|
" <td>nomic-ai/nomic-embed-text-v1.5-Q</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Text embeddings, Multimodal (text, image), Eng...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.130</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>8</th>\n",
|
|
" <td>BAAI/bge-base-en-v1.5</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), English, 512...</td>\n",
|
|
" <td>mit</td>\n",
|
|
" <td>0.210</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>9</th>\n",
|
|
" <td>sentence-transformers/paraphrase-multilingual-...</td>\n",
|
|
" <td>384</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), Multilingual...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.220</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>10</th>\n",
|
|
" <td>Qdrant/clip-ViT-B-32-text</td>\n",
|
|
" <td>512</td>\n",
|
|
" <td>Text embeddings, Multimodal (text&image), Engl...</td>\n",
|
|
" <td>mit</td>\n",
|
|
" <td>0.250</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>11</th>\n",
|
|
" <td>jinaai/jina-embeddings-v2-base-de</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), Multilingual...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.320</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>12</th>\n",
|
|
" <td>BAAI/bge-base-en</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), English, 512...</td>\n",
|
|
" <td>mit</td>\n",
|
|
" <td>0.420</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>13</th>\n",
|
|
" <td>snowflake/snowflake-arctic-embed-m</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), English, 512...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.430</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>14</th>\n",
|
|
" <td>nomic-ai/nomic-embed-text-v1.5</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Text embeddings, Multimodal (text, image), Eng...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.520</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>15</th>\n",
|
|
" <td>jinaai/jina-embeddings-v2-base-en</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), English, 819...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.520</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>16</th>\n",
|
|
" <td>nomic-ai/nomic-embed-text-v1</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Text embeddings, Multimodal (text, image), Eng...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.520</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>17</th>\n",
|
|
" <td>snowflake/snowflake-arctic-embed-m-long</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), English, 204...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.540</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>18</th>\n",
|
|
" <td>mixedbread-ai/mxbai-embed-large-v1</td>\n",
|
|
" <td>1024</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), English, 512...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.640</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>19</th>\n",
|
|
" <td>jinaai/jina-embeddings-v2-base-code</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), Multilingual...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.640</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>20</th>\n",
|
|
" <td>sentence-transformers/paraphrase-multilingual-...</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), Multilingual...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>1.000</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>21</th>\n",
|
|
" <td>snowflake/snowflake-arctic-embed-l</td>\n",
|
|
" <td>1024</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), English, 512...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>1.020</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>22</th>\n",
|
|
" <td>thenlper/gte-large</td>\n",
|
|
" <td>1024</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), English, 512...</td>\n",
|
|
" <td>mit</td>\n",
|
|
" <td>1.200</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>23</th>\n",
|
|
" <td>BAAI/bge-large-en-v1.5</td>\n",
|
|
" <td>1024</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), English, 512...</td>\n",
|
|
" <td>mit</td>\n",
|
|
" <td>1.200</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>24</th>\n",
|
|
" <td>intfloat/multilingual-e5-large</td>\n",
|
|
" <td>1024</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), Multilingual...</td>\n",
|
|
" <td>mit</td>\n",
|
|
" <td>2.240</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
]
|
|
},
|
|
"execution_count": 12,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"execution_count": 12
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Supported Sparse Text Embedding Models"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"metadata": {
|
|
"ExecuteTime": {
|
|
"end_time": "2024-11-13T09:01:07.038954Z",
|
|
"start_time": "2024-11-13T09:01:07.019656Z"
|
|
}
|
|
},
|
|
"source": [
|
|
"(\n",
|
|
" pd.DataFrame(SparseTextEmbedding.list_supported_models())\n",
|
|
" .sort_values(\"size_in_GB\")\n",
|
|
" .drop(columns=[\"sources\", \"model_file\", \"additional_files\"])\n",
|
|
" .reset_index(drop=True)\n",
|
|
")"
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
" model vocab_size \\\n",
|
|
"0 Qdrant/bm25 NaN \n",
|
|
"1 Qdrant/bm42-all-minilm-l6-v2-attentions 30522.0 \n",
|
|
"2 prithivida/Splade_PP_en_v1 30522.0 \n",
|
|
"3 prithvida/Splade_PP_en_v1 30522.0 \n",
|
|
"\n",
|
|
" description license size_in_GB \\\n",
|
|
"0 BM25 as sparse embeddings meant to be used wit... apache-2.0 0.010 \n",
|
|
"1 Light sparse embedding model, which assigns an... apache-2.0 0.090 \n",
|
|
"2 Independent Implementation of SPLADE++ Model f... apache-2.0 0.532 \n",
|
|
"3 Independent Implementation of SPLADE++ Model f... apache-2.0 0.532 \n",
|
|
"\n",
|
|
" requires_idf \n",
|
|
"0 True \n",
|
|
"1 True \n",
|
|
"2 NaN \n",
|
|
"3 NaN "
|
|
],
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>model</th>\n",
|
|
" <th>vocab_size</th>\n",
|
|
" <th>description</th>\n",
|
|
" <th>license</th>\n",
|
|
" <th>size_in_GB</th>\n",
|
|
" <th>requires_idf</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>Qdrant/bm25</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>BM25 as sparse embeddings meant to be used wit...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.010</td>\n",
|
|
" <td>True</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>Qdrant/bm42-all-minilm-l6-v2-attentions</td>\n",
|
|
" <td>30522.0</td>\n",
|
|
" <td>Light sparse embedding model, which assigns an...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.090</td>\n",
|
|
" <td>True</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>prithivida/Splade_PP_en_v1</td>\n",
|
|
" <td>30522.0</td>\n",
|
|
" <td>Independent Implementation of SPLADE++ Model f...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.532</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>prithvida/Splade_PP_en_v1</td>\n",
|
|
" <td>30522.0</td>\n",
|
|
" <td>Independent Implementation of SPLADE++ Model f...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.532</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
]
|
|
},
|
|
"execution_count": 13,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"execution_count": 13
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"source": [
|
|
"## Supported Late Interaction Text Embedding Models"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"ExecuteTime": {
|
|
"end_time": "2024-11-13T09:01:08.074442Z",
|
|
"start_time": "2024-11-13T09:01:08.056138Z"
|
|
}
|
|
},
|
|
"source": [
|
|
"(\n",
|
|
" pd.DataFrame(LateInteractionTextEmbedding.list_supported_models())\n",
|
|
" .sort_values(\"size_in_GB\")\n",
|
|
" .drop(columns=[\"sources\", \"model_file\"])\n",
|
|
" .reset_index(drop=True)\n",
|
|
")"
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
" model dim \\\n",
|
|
"0 answerdotai/answerai-colbert-small-v1 96 \n",
|
|
"1 colbert-ir/colbertv2.0 128 \n",
|
|
"2 jinaai/jina-colbert-v2 128 \n",
|
|
"\n",
|
|
" description license \\\n",
|
|
"0 Text embeddings, Unimodal (text), Multilingual... apache-2.0 \n",
|
|
"1 Late interaction model mit \n",
|
|
"2 New model that expands capabilities of colbert... cc-by-nc-4.0 \n",
|
|
"\n",
|
|
" size_in_GB additional_files \n",
|
|
"0 0.13 NaN \n",
|
|
"1 0.44 NaN \n",
|
|
"2 2.24 [onnx/model.onnx_data] "
|
|
],
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>model</th>\n",
|
|
" <th>dim</th>\n",
|
|
" <th>description</th>\n",
|
|
" <th>license</th>\n",
|
|
" <th>size_in_GB</th>\n",
|
|
" <th>additional_files</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>answerdotai/answerai-colbert-small-v1</td>\n",
|
|
" <td>96</td>\n",
|
|
" <td>Text embeddings, Unimodal (text), Multilingual...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.13</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>colbert-ir/colbertv2.0</td>\n",
|
|
" <td>128</td>\n",
|
|
" <td>Late interaction model</td>\n",
|
|
" <td>mit</td>\n",
|
|
" <td>0.44</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>jinaai/jina-colbert-v2</td>\n",
|
|
" <td>128</td>\n",
|
|
" <td>New model that expands capabilities of colbert...</td>\n",
|
|
" <td>cc-by-nc-4.0</td>\n",
|
|
" <td>2.24</td>\n",
|
|
" <td>[onnx/model.onnx_data]</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
]
|
|
},
|
|
"execution_count": 14,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"execution_count": 14
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"collapsed": false
|
|
},
|
|
"source": [
|
|
"## Supported Image Embedding Models"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"metadata": {
|
|
"collapsed": false,
|
|
"ExecuteTime": {
|
|
"end_time": "2024-11-13T09:01:09.171647Z",
|
|
"start_time": "2024-11-13T09:01:09.150940Z"
|
|
}
|
|
},
|
|
"source": [
|
|
"(\n",
|
|
" pd.DataFrame(ImageEmbedding.list_supported_models())\n",
|
|
" .sort_values(\"size_in_GB\")\n",
|
|
" .drop(columns=[\"sources\", \"model_file\"])\n",
|
|
" .reset_index(drop=True)\n",
|
|
")"
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
" model dim \\\n",
|
|
"0 Qdrant/resnet50-onnx 2048 \n",
|
|
"1 Qdrant/clip-ViT-B-32-vision 512 \n",
|
|
"2 Qdrant/Unicom-ViT-B-32 512 \n",
|
|
"3 Qdrant/Unicom-ViT-B-16 768 \n",
|
|
"\n",
|
|
" description license size_in_GB \n",
|
|
"0 Image embeddings, Unimodal (image), 2016 year apache-2.0 0.10 \n",
|
|
"1 Image embeddings, Multimodal (text&image), 202... mit 0.34 \n",
|
|
"2 Image embeddings, Multimodal (text&image), 202... apache-2.0 0.48 \n",
|
|
"3 Image embeddings (more detailed than Unicom-Vi... apache-2.0 0.82 "
|
|
],
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>model</th>\n",
|
|
" <th>dim</th>\n",
|
|
" <th>description</th>\n",
|
|
" <th>license</th>\n",
|
|
" <th>size_in_GB</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>Qdrant/resnet50-onnx</td>\n",
|
|
" <td>2048</td>\n",
|
|
" <td>Image embeddings, Unimodal (image), 2016 year</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.10</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>Qdrant/clip-ViT-B-32-vision</td>\n",
|
|
" <td>512</td>\n",
|
|
" <td>Image embeddings, Multimodal (text&image), 202...</td>\n",
|
|
" <td>mit</td>\n",
|
|
" <td>0.34</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>Qdrant/Unicom-ViT-B-32</td>\n",
|
|
" <td>512</td>\n",
|
|
" <td>Image embeddings, Multimodal (text&image), 202...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.48</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>Qdrant/Unicom-ViT-B-16</td>\n",
|
|
" <td>768</td>\n",
|
|
" <td>Image embeddings (more detailed than Unicom-Vi...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" <td>0.82</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
]
|
|
},
|
|
"execution_count": 15,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"execution_count": 15
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Supported Rerank Cross Encoder Models"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"metadata": {
|
|
"ExecuteTime": {
|
|
"end_time": "2024-11-13T09:01:10.313943Z",
|
|
"start_time": "2024-11-13T09:01:10.298428Z"
|
|
}
|
|
},
|
|
"source": [
|
|
"(\n",
|
|
" pd.DataFrame(TextCrossEncoder.list_supported_models())\n",
|
|
" .sort_values(\"size_in_GB\")\n",
|
|
" .drop(columns=[\"sources\", \"model_file\"])\n",
|
|
" .reset_index(drop=True)\n",
|
|
")"
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
" model size_in_GB \\\n",
|
|
"0 Xenova/ms-marco-MiniLM-L-6-v2 0.08 \n",
|
|
"1 Xenova/ms-marco-MiniLM-L-12-v2 0.12 \n",
|
|
"2 jinaai/jina-reranker-v1-tiny-en 0.13 \n",
|
|
"3 jinaai/jina-reranker-v1-turbo-en 0.15 \n",
|
|
"4 BAAI/bge-reranker-base 1.04 \n",
|
|
"5 jinaai/jina-reranker-v2-base-multilingual 1.11 \n",
|
|
"\n",
|
|
" description license \n",
|
|
"0 MiniLM-L-6-v2 model optimized for re-ranking t... apache-2.0 \n",
|
|
"1 MiniLM-L-12-v2 model optimized for re-ranking ... apache-2.0 \n",
|
|
"2 Designed for blazing-fast re-ranking with 8K c... apache-2.0 \n",
|
|
"3 Designed for blazing-fast re-ranking with 8K c... apache-2.0 \n",
|
|
"4 BGE reranker base model for cross-encoder re-r... mit \n",
|
|
"5 A multi-lingual reranker model for cross-encod... cc-by-nc-4.0 "
|
|
],
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>model</th>\n",
|
|
" <th>size_in_GB</th>\n",
|
|
" <th>description</th>\n",
|
|
" <th>license</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>Xenova/ms-marco-MiniLM-L-6-v2</td>\n",
|
|
" <td>0.08</td>\n",
|
|
" <td>MiniLM-L-6-v2 model optimized for re-ranking t...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>Xenova/ms-marco-MiniLM-L-12-v2</td>\n",
|
|
" <td>0.12</td>\n",
|
|
" <td>MiniLM-L-12-v2 model optimized for re-ranking ...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>jinaai/jina-reranker-v1-tiny-en</td>\n",
|
|
" <td>0.13</td>\n",
|
|
" <td>Designed for blazing-fast re-ranking with 8K c...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>jinaai/jina-reranker-v1-turbo-en</td>\n",
|
|
" <td>0.15</td>\n",
|
|
" <td>Designed for blazing-fast re-ranking with 8K c...</td>\n",
|
|
" <td>apache-2.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>BAAI/bge-reranker-base</td>\n",
|
|
" <td>1.04</td>\n",
|
|
" <td>BGE reranker base model for cross-encoder re-r...</td>\n",
|
|
" <td>mit</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5</th>\n",
|
|
" <td>jinaai/jina-reranker-v2-base-multilingual</td>\n",
|
|
" <td>1.11</td>\n",
|
|
" <td>A multi-lingual reranker model for cross-encod...</td>\n",
|
|
" <td>cc-by-nc-4.0</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
]
|
|
},
|
|
"execution_count": 16,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"execution_count": 16
|
|
},
|
|
{
|
|
"metadata": {},
|
|
"cell_type": "code",
|
|
"outputs": [],
|
|
"execution_count": null,
|
|
"source": ""
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3.8.18 ('base')",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.8"
|
|
},
|
|
"orig_nbformat": 4,
|
|
"vscode": {
|
|
"interpreter": {
|
|
"hash": "c4a27af61e455bc18dcf16f5867a2ff0402fa12b01dd0f6ce3a79ae73ad15e91"
|
|
}
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|