{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2024-03-30T11:18:52.052764Z", "start_time": "2024-03-30T11:18:52.039616Z" } }, "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "from fastembed import SparseTextEmbedding, TextEmbedding" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Supported Text Embedding Models" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
modeldimdescriptionsize_in_GB
0BAAI/bge-small-en-v1.5384Fast and Default English model0.067
1BAAI/bge-small-zh-v1.5512Fast and recommended Chinese model0.090
2sentence-transformers/all-MiniLM-L6-v2384Sentence Transformer model, MiniLM-L6-v20.090
3jinaai/jina-embeddings-v2-small-en512English embedding model supporting 8192 sequen...0.120
4BAAI/bge-small-en384Fast English model0.130
5BAAI/bge-base-en-v1.5768Base English model, v1.50.210
6sentence-transformers/paraphrase-multilingual-...384Sentence Transformer model, paraphrase-multili...0.220
7BAAI/bge-base-en768Base English model0.420
8nomic-ai/nomic-embed-text-v17688192 context length english model0.520
9nomic-ai/nomic-embed-text-v1.57688192 context length english model0.520
10jinaai/jina-embeddings-v2-base-en768English embedding model supporting 8192 sequen...0.520
11mixedbread-ai/mxbai-embed-large-v11024MixedBread Base sentence embedding model, does...0.640
12sentence-transformers/paraphrase-multilingual-...768Sentence-transformers model for tasks like clu...1.000
13BAAI/bge-large-en-v1.51024Large English model, v1.51.200
14thenlper/gte-large1024Large general text embeddings model1.200
15intfloat/multilingual-e5-large1024Multilingual model, e5-large. Recommend using ...2.240
\n", "
" ], "text/plain": [ " model dim \\\n", "0 BAAI/bge-small-en-v1.5 384 \n", "1 BAAI/bge-small-zh-v1.5 512 \n", "2 sentence-transformers/all-MiniLM-L6-v2 384 \n", "3 jinaai/jina-embeddings-v2-small-en 512 \n", "4 BAAI/bge-small-en 384 \n", "5 BAAI/bge-base-en-v1.5 768 \n", "6 sentence-transformers/paraphrase-multilingual-... 384 \n", "7 BAAI/bge-base-en 768 \n", "8 nomic-ai/nomic-embed-text-v1 768 \n", "9 nomic-ai/nomic-embed-text-v1.5 768 \n", "10 jinaai/jina-embeddings-v2-base-en 768 \n", "11 mixedbread-ai/mxbai-embed-large-v1 1024 \n", "12 sentence-transformers/paraphrase-multilingual-... 768 \n", "13 BAAI/bge-large-en-v1.5 1024 \n", "14 thenlper/gte-large 1024 \n", "15 intfloat/multilingual-e5-large 1024 \n", "\n", " description size_in_GB \n", "0 Fast and Default English model 0.067 \n", "1 Fast and recommended Chinese model 0.090 \n", "2 Sentence Transformer model, MiniLM-L6-v2 0.090 \n", "3 English embedding model supporting 8192 sequen... 0.120 \n", "4 Fast English model 0.130 \n", "5 Base English model, v1.5 0.210 \n", "6 Sentence Transformer model, paraphrase-multili... 0.220 \n", "7 Base English model 0.420 \n", "8 8192 context length english model 0.520 \n", "9 8192 context length english model 0.520 \n", "10 English embedding model supporting 8192 sequen... 0.520 \n", "11 MixedBread Base sentence embedding model, does... 0.640 \n", "12 Sentence-transformers model for tasks like clu... 1.000 \n", "13 Large English model, v1.5 1.200 \n", "14 Large general text embeddings model 1.200 \n", "15 Multilingual model, e5-large. Recommend using ... 2.240 " ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "supported_models = (\n", " pd.DataFrame(TextEmbedding.list_supported_models())\n", " .sort_values(\"size_in_GB\")\n", " .drop(columns=\"sources\")\n", " .reset_index(drop=True)\n", ")\n", "supported_models" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Supported Sparse Text Embedding Models" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2024-03-30T11:19:01.564291Z", "start_time": "2024-03-30T11:19:01.538768Z" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
modelvocab_sizedescriptionsize_in_GBsources
0prithvida/Splade_PP_en_v130522Misspelled version of the model. Retained for ...0.532{'hf': 'Qdrant/SPLADE_PP_en_v1'}
1prithivida/Splade_PP_en_v130522Independent Implementation of SPLADE++ Model f...0.532{'hf': 'Qdrant/SPLADE_PP_en_v1'}
\n", "
" ], "text/plain": [ " model vocab_size \\\n", "0 prithvida/Splade_PP_en_v1 30522 \n", "1 prithivida/Splade_PP_en_v1 30522 \n", "\n", " description size_in_GB \\\n", "0 Misspelled version of the model. Retained for ... 0.532 \n", "1 Independent Implementation of SPLADE++ Model f... 0.532 \n", "\n", " sources \n", "0 {'hf': 'Qdrant/SPLADE_PP_en_v1'} \n", "1 {'hf': 'Qdrant/SPLADE_PP_en_v1'} " ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pd.DataFrame(SparseTextEmbedding.list_supported_models())" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3.8.18 ('base')", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.13" }, "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "c4a27af61e455bc18dcf16f5867a2ff0402fa12b01dd0f6ce3a79ae73ad15e91" } } }, "nbformat": 4, "nbformat_minor": 2 }