From 615d6ee2b6dec88b4f85752afbfc25db7eae8ea3 Mon Sep 17 00:00:00 2001 From: Nirant Date: Fri, 7 Jun 2024 00:29:27 +0530 Subject: [PATCH] Replace Data Source (#206) * Re-run of identical hardware and generate graphs * Re-run of identical hardware and generate graphs Fixes https://github.com/qdrant/fastembed/issues/174 * Change dataset source * Refactor code for better readability and maintainability * Inline outputs * Replace hard coded constants with dataset specific n_dim * fix: fix binary quant from scratch notebook * fix: fix result table, explain corner case --------- Co-authored-by: George Panchuk --- .../Binary Quantization from Scratch.ipynb | 388 ++++++++---------- .../Binary_Quantization_with_Qdrant.ipynb | 149 ++++--- 2 files changed, 267 insertions(+), 270 deletions(-) diff --git a/docs/experimental/Binary Quantization from Scratch.ipynb b/docs/experimental/Binary Quantization from Scratch.ipynb index 5a3369c..1463c49 100644 --- a/docs/experimental/Binary Quantization from Scratch.ipynb +++ b/docs/experimental/Binary Quantization from Scratch.ipynb @@ -14,23 +14,33 @@ }, { "cell_type": "code", - "execution_count": 11, - "metadata": {}, + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2024-06-06T17:00:06.460001Z", + "start_time": "2024-06-06T17:00:04.214098Z" + } + }, "outputs": [], "source": [ - "!pip install matplotlib tqdm pandas numpy --quiet" + "!pip install matplotlib tqdm pandas numpy datasets --quiet --upgrade" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 2, "metadata": { + "ExecuteTime": { + "end_time": "2024-06-06T17:00:07.041784Z", + "start_time": "2024-06-06T17:00:06.461658Z" + }, "id": "WBVTItUX4yyr" }, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", + "from datasets import load_dataset\n", "from tqdm import tqdm" ] }, @@ -52,8 +62,12 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 3, "metadata": { + "ExecuteTime": { + "end_time": "2024-06-06T17:01:09.343230Z", + "start_time": "2024-06-06T17:00:07.042526Z" + }, "colab": { "base_uri": "https://localhost:8080/", "height": 250 @@ -61,58 +75,24 @@ "id": "REJpFqkG7EG2", "outputId": "7a43c0ae-fbcc-45fe-fd58-bfe691297b22" }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 26/26 [00:10<00:00, 2.45it/s]\n" - ] - }, - { - "data": { - "text/plain": [ - "(1000000, 1536)" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "def get_openai_vectors(force_download: bool = False):\n", - " res = []\n", - " for i in tqdm(range(26)):\n", - " if force_download:\n", - " !wget https://huggingface.co/api/datasets/KShivendu/dbpedia-entities-openai-1M/parquet/KShivendu--dbpedia-entities-openai-1M/train/{i}.parquet\n", - " df = pd.read_parquet(f\"{i}.parquet\", engine=\"pyarrow\")\n", - " res.append(np.stack(df.openai))\n", - " del df\n", - "\n", - " openai_vectors = np.concatenate(res)\n", - " del res\n", - " return openai_vectors\n", - "\n", - "\n", - "openai_vectors = get_openai_vectors(force_download=False)\n", - "openai_vectors.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## ㆓ Binary Conversion\n", - "\n", - "Here, we will use 0 as the threshold for the binary conversion. All values greater than 0 will be set to 1, and others will remain 0. This is a simple and effective way to convert continuous values into binary values for OpenAI embeddings." + "# Download from Huggingface Hub\n", + "ds = load_dataset(\n", + " \"Qdrant/dbpedia-entities-openai3-text-embedding-3-large-3072-100K\", split=\"train\"\n", + ")\n", + "openai_vectors = np.array(ds[\"text-embedding-3-large-3072-embedding\"])\n", + "del ds" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 4, "metadata": { - "id": "0JM2-Bj2Jkab" + "ExecuteTime": { + "end_time": "2024-06-06T17:01:10.900963Z", + "start_time": "2024-06-06T17:01:09.344842Z" + } }, "outputs": [], "source": [ @@ -120,6 +100,30 @@ "openai_bin[openai_vectors > 0] = 1" ] }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2024-06-06T17:01:10.906827Z", + "start_time": "2024-06-06T17:01:10.901820Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": "3072" + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "n_dim = openai_vectors.shape[1]\n", + "n_dim" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -131,8 +135,12 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 6, "metadata": { + "ExecuteTime": { + "end_time": "2024-06-06T17:01:10.909730Z", + "start_time": "2024-06-06T17:01:10.908166Z" + }, "id": "FqshI-GlIERd" }, "outputs": [], @@ -141,7 +149,7 @@ " scores = np.dot(openai_vectors, openai_vectors[idx])\n", " dot_results = np.argsort(scores)[-limit:][::-1]\n", "\n", - " bin_scores = 1536 - np.logical_xor(openai_bin, openai_bin[idx]).sum(axis=1)\n", + " bin_scores = n_dim - np.logical_xor(openai_bin, openai_bin[idx]).sum(axis=1)\n", " bin_results = np.argsort(bin_scores)[-(limit * oversampling) :][::-1]\n", "\n", " return len(set(dot_results).intersection(set(bin_results))) / limit" @@ -156,8 +164,12 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 7, "metadata": { + "ExecuteTime": { + "end_time": "2024-06-06T17:01:25.206592Z", + "start_time": "2024-06-06T17:01:10.911971Z" + }, "colab": { "base_uri": "https://localhost:8080/" }, @@ -169,110 +181,128 @@ "name": "stderr", "output_type": "stream", "text": [ - " 0%| | 0/4 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
sampling_ratelimitrecall
01100.800
111000.708
22100.950
321000.877
43100.960
531000.937
65100.980
751000.977
\n", - "" - ], - "text/plain": [ - " sampling_rate limit recall\n", - "0 1 10 0.800\n", - "1 1 100 0.708\n", - "2 2 10 0.950\n", - "3 2 100 0.877\n", - "4 3 10 0.960\n", - "5 3 100 0.937\n", - "6 5 10 0.980\n", - "7 5 100 0.977" - ] + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
sampling_ratelimitmean_acc
0130.90
11100.83
2231.00
32100.97
4331.00
53100.98
6531.00
75100.99
\n
", + "text/plain": " sampling_rate limit mean_acc\n0 1 3 0.90\n1 1 10 0.83\n2 2 3 1.00\n3 2 10 0.97\n4 3 3 1.00\n5 3 10 0.98\n6 5 3 1.00\n7 5 10 0.99" }, - "execution_count": 19, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -408,22 +372,13 @@ ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "| sampling_rate | limit | accuracy |\n", - "|---------------|-------|----------|\n", - "| 1 | 10 | 0.800 |\n", - "| 1 | 100 | 0.708 |\n", - "| 2 | 10 | 0.950 |\n", - "| 2 | 100 | 0.877 |\n", - "| 4 | 10 | 0.970 |\n", - "| 4 | 100 | 0.956 |\n", - "| 8 | 10 | 0.990 |\n", - "| 8 | 100 | 0.990 |\n", - "| 16 | 10 | 1.000 |\n", - "| 16 | 100 | 0.998 |" - ] + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [] } ], "metadata": { @@ -432,7 +387,8 @@ "provenance": [] }, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", + "language": "python", "name": "python3" }, "language_info": { @@ -445,7 +401,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.17" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/qdrant/Binary_Quantization_with_Qdrant.ipynb b/docs/qdrant/Binary_Quantization_with_Qdrant.ipynb index b6d8b5c..cf1f5de 100644 --- a/docs/qdrant/Binary_Quantization_with_Qdrant.ipynb +++ b/docs/qdrant/Binary_Quantization_with_Qdrant.ipynb @@ -36,8 +36,8 @@ "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2024-04-01T16:33:03.013948Z", - "start_time": "2024-04-01T16:33:01.019043Z" + "end_time": "2024-06-06T18:31:54.357040Z", + "start_time": "2024-06-06T18:31:52.672431Z" } }, "outputs": [], @@ -50,25 +50,17 @@ "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2024-04-01T16:33:03.914729Z", - "start_time": "2024-04-01T16:33:03.015394Z" + "end_time": "2024-06-06T18:31:55.482034Z", + "start_time": "2024-06-06T18:31:54.357645Z" } }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/joein/work/qdrant/fastembed/venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "import random\n", "import time\n", "\n", + "import datasets\n", "import numpy as np\n", "import pandas as pd\n", "from qdrant_client import QdrantClient, models\n", @@ -91,8 +83,8 @@ "execution_count": 3, "metadata": { "ExecuteTime": { - "end_time": "2024-04-01T16:33:09.085853Z", - "start_time": "2024-04-01T16:33:03.912688Z" + "end_time": "2024-06-06T18:31:58.405581Z", + "start_time": "2024-06-06T18:31:55.471027Z" } }, "outputs": [ @@ -106,8 +98,6 @@ } ], "source": [ - "import datasets\n", - "\n", "dataset = datasets.load_dataset(\n", " \"Qdrant/dbpedia-entities-openai3-text-embedding-3-small-1536-100K\", split=\"train\"\n", ")\n", @@ -119,14 +109,14 @@ "execution_count": 4, "metadata": { "ExecuteTime": { - "end_time": "2024-04-01T16:33:09.176212Z", - "start_time": "2024-04-01T16:33:09.084550Z" + "end_time": "2024-06-06T18:32:24.456211Z", + "start_time": "2024-06-06T18:31:58.396924Z" } }, "outputs": [ { "data": { - "text/plain": "True" + "text/plain": "1536" }, "execution_count": 4, "metadata": {}, @@ -134,15 +124,40 @@ } ], "source": [ - "client = QdrantClient(\n", + "n_dim = len(dataset[\"text-embedding-3-small-1536-embedding\"][0])\n", + "n_dim" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2024-06-06T18:32:24.536891Z", + "start_time": "2024-06-06T18:32:24.455087Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": "True" + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "client = QdrantClient( # assumes Qdrant is launched at localhost:6333\n", " prefer_grpc=True,\n", ")\n", "\n", "collection_name = \"binary-quantization\"\n", - "client.recreate_collection(\n", + "\n", + "client.create_collection(\n", " collection_name=collection_name,\n", " vectors_config=models.VectorParams(\n", - " size=1536,\n", + " size=n_dim,\n", " distance=models.Distance.DOT,\n", " on_disk=True,\n", " ),\n", @@ -154,11 +169,11 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": { "ExecuteTime": { - "end_time": "2024-04-01T16:34:13.383986Z", - "start_time": "2024-04-01T16:33:09.175725Z" + "end_time": "2024-06-06T18:33:22.583349Z", + "start_time": "2024-06-06T18:32:24.815155Z" } }, "outputs": [], @@ -179,19 +194,19 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 13, "metadata": { "ExecuteTime": { - "end_time": "2024-04-01T16:34:13.390886Z", - "start_time": "2024-04-01T16:34:13.385961Z" + "end_time": "2024-06-06T18:35:47.748898Z", + "start_time": "2024-06-06T18:35:47.740719Z" } }, "outputs": [ { "data": { - "text/plain": "{'status': ,\n 'optimizer_status': ,\n 'vectors_count': 116640,\n 'indexed_vectors_count': 43520,\n 'points_count': 116640,\n 'segments_count': 6,\n 'config': {'params': {'vectors': {'size': 1536,\n 'distance': ,\n 'hnsw_config': None,\n 'quantization_config': None,\n 'on_disk': True},\n 'shard_number': 1,\n 'sharding_method': None,\n 'replication_factor': 1,\n 'write_consistency_factor': 1,\n 'read_fan_out_factor': None,\n 'on_disk_payload': True,\n 'sparse_vectors': None},\n 'hnsw_config': {'m': 16,\n 'ef_construct': 100,\n 'full_scan_threshold': 10000,\n 'max_indexing_threads': 0,\n 'on_disk': False,\n 'payload_m': None},\n 'optimizer_config': {'deleted_threshold': 0.2,\n 'vacuum_min_vector_number': 1000,\n 'default_segment_number': 0,\n 'max_segment_size': None,\n 'memmap_threshold': None,\n 'indexing_threshold': 20000,\n 'flush_interval_sec': 5,\n 'max_optimization_threads': None},\n 'wal_config': {'wal_capacity_mb': 32, 'wal_segments_ahead': 0},\n 'quantization_config': {'binary': {'always_ram': True}}},\n 'payload_schema': {}}" + "text/plain": "{'status': ,\n 'optimizer_status': ,\n 'vectors_count': None,\n 'indexed_vectors_count': 97760,\n 'points_count': 100000,\n 'segments_count': 7,\n 'config': {'params': {'vectors': {'size': 1536,\n 'distance': ,\n 'hnsw_config': None,\n 'quantization_config': None,\n 'on_disk': True,\n 'datatype': None},\n 'shard_number': 1,\n 'sharding_method': None,\n 'replication_factor': 1,\n 'write_consistency_factor': 1,\n 'read_fan_out_factor': None,\n 'on_disk_payload': True,\n 'sparse_vectors': None},\n 'hnsw_config': {'m': 16,\n 'ef_construct': 100,\n 'full_scan_threshold': 10000,\n 'max_indexing_threads': 0,\n 'on_disk': False,\n 'payload_m': None},\n 'optimizer_config': {'deleted_threshold': 0.2,\n 'vacuum_min_vector_number': 1000,\n 'default_segment_number': 0,\n 'max_segment_size': None,\n 'memmap_threshold': None,\n 'indexing_threshold': 20000,\n 'flush_interval_sec': 5,\n 'max_optimization_threads': None},\n 'wal_config': {'wal_capacity_mb': 32, 'wal_segments_ahead': 0},\n 'quantization_config': {'binary': {'always_ram': True}}},\n 'payload_schema': {}}" }, - "execution_count": 6, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -214,11 +229,11 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": { "ExecuteTime": { - "end_time": "2024-04-01T16:34:13.453626Z", - "start_time": "2024-04-01T16:34:13.391567Z" + "end_time": "2024-06-06T18:33:22.643845Z", + "start_time": "2024-06-06T18:33:22.589346Z" } }, "outputs": [ @@ -226,7 +241,7 @@ "data": { "text/plain": "[89391,\n 79659,\n 12006,\n 80978,\n 87219,\n 97885,\n 83155,\n 67504,\n 4645,\n 82711,\n 48395,\n 57375,\n 69208,\n 14136,\n 89515,\n 59880,\n 78730,\n 36952,\n 49620,\n 96486,\n 55473,\n 58179,\n 18926,\n 6489,\n 11931,\n 54146,\n 9850,\n 71259,\n 37825,\n 47331,\n 84964,\n 92399,\n 56669,\n 77042,\n 73744,\n 47993,\n 83780,\n 92429,\n 75114,\n 4463,\n 69030,\n 81185,\n 27950,\n 66217,\n 54652,\n 8260,\n 1151,\n 993,\n 85954,\n 66863,\n 47303,\n 8992,\n 92688,\n 76030,\n 29472,\n 3077,\n 42454,\n 46120,\n 69140,\n 20877,\n 2844,\n 95423,\n 1770,\n 28568,\n 96448,\n 94227,\n 40837,\n 91684,\n 29785,\n 66936,\n 85121,\n 39546,\n 81910,\n 5514,\n 37068,\n 35731,\n 93990,\n 26685,\n 63076,\n 18762,\n 27922,\n 34916,\n 80976,\n 83189,\n 6328,\n 57508,\n 58860,\n 13758,\n 72976,\n 85030,\n 332,\n 34963,\n 85009,\n 31344,\n 11560,\n 58108,\n 85163,\n 17064,\n 44712,\n 45962]" }, - "execution_count": 7, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -239,11 +254,11 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": { "ExecuteTime": { - "end_time": "2024-04-01T16:34:13.453928Z", - "start_time": "2024-04-01T16:34:13.452405Z" + "end_time": "2024-06-06T18:33:22.644389Z", + "start_time": "2024-06-06T18:33:22.642909Z" } }, "outputs": [], @@ -257,11 +272,11 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": { "ExecuteTime": { - "end_time": "2024-04-01T16:34:13.457839Z", - "start_time": "2024-04-01T16:34:13.455431Z" + "end_time": "2024-06-06T18:33:22.647634Z", + "start_time": "2024-06-06T18:33:22.646107Z" } }, "outputs": [], @@ -290,11 +305,11 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 14, "metadata": { "ExecuteTime": { - "end_time": "2024-04-01T16:40:48.347002Z", - "start_time": "2024-04-01T16:40:42.228551Z" + "end_time": "2024-06-06T18:36:09.037565Z", + "start_time": "2024-06-06T18:35:58.069484Z" }, "collapsed": false }, @@ -302,7 +317,7 @@ "source": [ "limit_grid = [1, 3, 10, 20, 50]\n", "oversampling_grid = [1.0, 3.0, 5.0]\n", - "rescore_grid = [False, True]\n", + "rescore_grid = [True, False]\n", "results = []\n", "\n", "for limit in limit_grid:\n", @@ -317,7 +332,7 @@ " {\n", " \"limit\": limit,\n", " \"oversampling\": oversampling,\n", - " \"candidates\": int(oversampling * limit),\n", + " \"bq_candidates\": int(oversampling * limit),\n", " \"rescore\": rescore,\n", " \"accuracy\": correct_results / 100,\n", " \"total queries\": len(query_dataset[\"text\"]),\n", @@ -328,29 +343,55 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 15, "metadata": { "ExecuteTime": { - "end_time": "2024-04-01T16:41:55.445405Z", - "start_time": "2024-04-01T16:41:55.442687Z" + "end_time": "2024-06-06T18:36:09.054593Z", + "start_time": "2024-06-06T18:36:09.048053Z" } }, "outputs": [ { "data": { - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
candidatesrescoreaccuracytime
01False0.900.221826
11True0.910.134167
23False0.880.115299
33True0.970.209320
45False0.840.154485
55True0.910.124424
63False0.990.121695
73True0.960.123257
89False0.940.119629
99True0.980.119372
1015False0.900.121621
1115True0.970.125466
1210False0.930.135910
1310True0.950.138135
1430False0.940.177928
1530True0.980.254588
1650False0.940.268659
1750True0.960.269792
1820False0.960.249941
1920True0.960.247138
2060False0.970.251301
2160True0.980.256504
22100False0.980.270049
23100True0.970.248972
2450False0.970.306356
2550True0.980.257544
26150False0.980.238811
27150True0.990.263939
28250False0.990.256558
29250True1.000.335823
\n
", - "text/plain": " candidates rescore accuracy time\n0 1 False 0.90 0.221826\n1 1 True 0.91 0.134167\n2 3 False 0.88 0.115299\n3 3 True 0.97 0.209320\n4 5 False 0.84 0.154485\n5 5 True 0.91 0.124424\n6 3 False 0.99 0.121695\n7 3 True 0.96 0.123257\n8 9 False 0.94 0.119629\n9 9 True 0.98 0.119372\n10 15 False 0.90 0.121621\n11 15 True 0.97 0.125466\n12 10 False 0.93 0.135910\n13 10 True 0.95 0.138135\n14 30 False 0.94 0.177928\n15 30 True 0.98 0.254588\n16 50 False 0.94 0.268659\n17 50 True 0.96 0.269792\n18 20 False 0.96 0.249941\n19 20 True 0.96 0.247138\n20 60 False 0.97 0.251301\n21 60 True 0.98 0.256504\n22 100 False 0.98 0.270049\n23 100 True 0.97 0.248972\n24 50 False 0.97 0.306356\n25 50 True 0.98 0.257544\n26 150 False 0.98 0.238811\n27 150 True 0.99 0.263939\n28 250 False 0.99 0.256558\n29 250 True 1.00 0.335823" + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
limitoversamplingrescoreaccuracybq_candidatestime
011.0True0.9510.300152
111.0False0.8510.244668
213.0True0.9530.124406
313.0False0.8330.171471
415.0True0.9850.118219
515.0False0.8750.111914
631.0True0.9530.121328
731.0False0.9230.267725
833.0True0.9690.416834
933.0False0.9090.410730
1035.0True0.97150.231671
1135.0False0.93150.252269
12101.0True0.96100.133462
13101.0False0.92100.285158
14103.0True0.95300.320695
15103.0False0.98300.457904
16105.0True0.96500.453204
17105.0False0.94500.450944
18201.0True0.97200.361066
19201.0False0.95200.585992
20203.0True0.96600.550389
21203.0False0.96600.618630
22205.0True1.001000.458241
23205.0False0.951000.441106
24501.0True0.98500.603967
25501.0False0.96500.514531
26503.0True1.001500.548153
27503.0False0.981500.608930
28505.0True1.002500.487522
29505.0False0.992500.313810
\n
", + "text/plain": " limit oversampling rescore accuracy bq_candidates time\n0 1 1.0 True 0.95 1 0.300152\n1 1 1.0 False 0.85 1 0.244668\n2 1 3.0 True 0.95 3 0.124406\n3 1 3.0 False 0.83 3 0.171471\n4 1 5.0 True 0.98 5 0.118219\n5 1 5.0 False 0.87 5 0.111914\n6 3 1.0 True 0.95 3 0.121328\n7 3 1.0 False 0.92 3 0.267725\n8 3 3.0 True 0.96 9 0.416834\n9 3 3.0 False 0.90 9 0.410730\n10 3 5.0 True 0.97 15 0.231671\n11 3 5.0 False 0.93 15 0.252269\n12 10 1.0 True 0.96 10 0.133462\n13 10 1.0 False 0.92 10 0.285158\n14 10 3.0 True 0.95 30 0.320695\n15 10 3.0 False 0.98 30 0.457904\n16 10 5.0 True 0.96 50 0.453204\n17 10 5.0 False 0.94 50 0.450944\n18 20 1.0 True 0.97 20 0.361066\n19 20 1.0 False 0.95 20 0.585992\n20 20 3.0 True 0.96 60 0.550389\n21 20 3.0 False 0.96 60 0.618630\n22 20 5.0 True 1.00 100 0.458241\n23 20 5.0 False 0.95 100 0.441106\n24 50 1.0 True 0.98 50 0.603967\n25 50 1.0 False 0.96 50 0.514531\n26 50 3.0 True 1.00 150 0.548153\n27 50 3.0 False 0.98 150 0.608930\n28 50 5.0 True 1.00 250 0.487522\n29 50 5.0 False 0.99 250 0.313810" }, - "execution_count": 22, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = pd.DataFrame(results)\n", - "df[[\"candidates\", \"rescore\", \"accuracy\", \"time\"]]\n", + "\n", + "df[[\"limit\", \"oversampling\", \"rescore\", \"accuracy\", \"bq_candidates\", \"time\"]]\n", "# df.to_csv(\"candidates-rescore-time.csv\", index=False)" ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [ + "#### Why results for oversampling=1.0 and limit=1 with rescore=True are better than with rescore=False? \n", + "\n", + "It might seem that with oversampling=1.0 and limit=1 Qdrant retrieves only 1 point, and it does not matter whether we rescore it or not, it should stay the same, but with a different score (from original vectors).\n", + "\n", + "But in fact, there are 2 reasons why results are different:\n", + "1) HNSW is an approximate algorithm, and it might return different results for the same query. \n", + "2) Qdrant stores points in segments. When we do a query for 1 point, Qdrant looks for this one point in each segment, and then chooses the best match. \n", + "3) In this example we had 8 segments, Qdrant found 8 points with binary scores, replaced their scores with original vectors scores, and selected the best one from them, which led to a better accuracy. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [] } ], "metadata": {