From ee7a165d9bf64fda5261a157fa5d78283bf041b5 Mon Sep 17 00:00:00 2001
From: Ivan Pleshkov <pleshkov.ivan@gmail.com>
Date: Thu, 30 Nov 2023 21:47:30 +0100
Subject: [PATCH] Optional sparse full scan threshold (#3116)

* optional sparse full scan threshold

* update openapi

* skip serializing if none

* review remarks
---
 docs/redoc/master/openapi.json                           | 6 ++----
 lib/segment/benches/sparse_index_build.rs                | 2 +-
 lib/segment/src/fixtures/sparse_fixtures.rs              | 2 +-
 .../src/index/sparse_index/sparse_index_config.rs        | 6 ++++--
 .../src/index/sparse_index/sparse_vector_index.rs        | 9 +++++++--
 lib/segment/src/types.rs                                 | 2 ++
 6 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/docs/redoc/master/openapi.json b/docs/redoc/master/openapi.json
index 0c7a4736cf..a1d7149119 100644
--- a/docs/redoc/master/openapi.json
+++ b/docs/redoc/master/openapi.json
@@ -10543,15 +10543,13 @@
       },
       "SparseIndexConfig": {
         "type": "object",
-        "required": [
-          "full_scan_threshold"
-        ],
         "properties": {
           "full_scan_threshold": {
             "description": "We prefer a full scan search upto (excluding) this number of vectors.\n\nNote: this is number of vectors, not KiloBytes.",
             "type": "integer",
             "format": "uint",
-            "minimum": 0
+            "minimum": 0,
+            "nullable": true
           },
           "on_disk": {
             "description": "Store index on disk. If set to false, the index will be stored in RAM. Default: false",
diff --git a/lib/segment/benches/sparse_index_build.rs b/lib/segment/benches/sparse_index_build.rs
index 5c32700525..81e5c34bf0 100644
--- a/lib/segment/benches/sparse_index_build.rs
+++ b/lib/segment/benches/sparse_index_build.rs
@@ -64,7 +64,7 @@ fn sparse_vector_index_build_benchmark(c: &mut Criterion) {
     drop(borrowed_storage);
 
     // save index config to disk
-    let index_config = SparseIndexConfig::new(10_000, None);
+    let index_config = SparseIndexConfig::new(Some(10_000), None);
 
     // intent: measure in-memory build time from storage
     group.bench_function("build-ram-index", |b| {
diff --git a/lib/segment/src/fixtures/sparse_fixtures.rs b/lib/segment/src/fixtures/sparse_fixtures.rs
index 84b83851bb..73c40e1918 100644
--- a/lib/segment/src/fixtures/sparse_fixtures.rs
+++ b/lib/segment/src/fixtures/sparse_fixtures.rs
@@ -46,7 +46,7 @@ pub fn fixture_open_sparse_index<I: InvertedIndex>(
     let db = open_db(storage_dir, &[DB_VECTOR_CF]).unwrap();
     let vector_storage = open_simple_sparse_vector_storage(db, DB_VECTOR_CF)?;
 
-    let sparse_index_config = SparseIndexConfig::new(full_scan_threshold, None);
+    let sparse_index_config = SparseIndexConfig::new(Some(full_scan_threshold), None);
     let sparse_vector_index: SparseVectorIndex<I> = SparseVectorIndex::open(
         sparse_index_config,
         id_tracker,
diff --git a/lib/segment/src/index/sparse_index/sparse_index_config.rs b/lib/segment/src/index/sparse_index/sparse_index_config.rs
index b020007e50..661f8cffe3 100644
--- a/lib/segment/src/index/sparse_index/sparse_index_config.rs
+++ b/lib/segment/src/index/sparse_index/sparse_index_config.rs
@@ -15,8 +15,10 @@ pub struct SparseIndexConfig {
     /// We prefer a full scan search upto (excluding) this number of vectors.
     ///
     /// Note: this is number of vectors, not KiloBytes.
-    pub full_scan_threshold: usize,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub full_scan_threshold: Option<usize>,
     /// Store index on disk. If set to false, the index will be stored in RAM. Default: false
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub on_disk: Option<bool>,
 }
 
@@ -30,7 +32,7 @@ impl Anonymize for SparseIndexConfig {
 }
 
 impl SparseIndexConfig {
-    pub fn new(full_scan_threshold: usize, on_disk: Option<bool>) -> Self {
+    pub fn new(full_scan_threshold: Option<usize>, on_disk: Option<bool>) -> Self {
         SparseIndexConfig {
             full_scan_threshold,
             on_disk,
diff --git a/lib/segment/src/index/sparse_index/sparse_vector_index.rs b/lib/segment/src/index/sparse_index/sparse_vector_index.rs
index 4e45048431..21ac4999a2 100644
--- a/lib/segment/src/index/sparse_index/sparse_vector_index.rs
+++ b/lib/segment/src/index/sparse_index/sparse_vector_index.rs
@@ -21,7 +21,7 @@ use crate::index::sparse_index::sparse_search_telemetry::SparseSearchesTelemetry
 use crate::index::struct_payload_index::StructPayloadIndex;
 use crate::index::{PayloadIndex, VectorIndex};
 use crate::telemetry::VectorIndexSearchesTelemetry;
-use crate::types::{Filter, SearchParams};
+use crate::types::{Filter, SearchParams, DEFAULT_SPARSE_FULL_SCAN_THRESHOLD};
 use crate::vector_storage::sparse_raw_scorer::sparse_check_vector;
 use crate::vector_storage::{new_stoppable_raw_scorer, VectorStorage, VectorStorageEnum};
 
@@ -198,7 +198,12 @@ impl<TInvertedIndex: InvertedIndex> VectorIndex for SparseVectorIndex<TInvertedI
                 );
 
                 // if cardinality is small - use plain search
-                if query_cardinality.max < self.config.full_scan_threshold {
+                if query_cardinality.max
+                    < self
+                        .config
+                        .full_scan_threshold
+                        .unwrap_or(DEFAULT_SPARSE_FULL_SCAN_THRESHOLD)
+                {
                     let _timer =
                         ScopeDurationMeasurer::new(&self.searches_telemetry.small_cardinality);
                     return self.search_plain(vectors, filter, top, is_stopped);
diff --git a/lib/segment/src/types.rs b/lib/segment/src/types.rs
index 3761639b55..1c1cbb44ab 100644
--- a/lib/segment/src/types.rs
+++ b/lib/segment/src/types.rs
@@ -718,6 +718,8 @@ impl VectorDataConfig {
 /// Default value based on <https://github.com/google-research/google-research/blob/master/scann/docs/algorithms.md>
 pub const DEFAULT_FULL_SCAN_THRESHOLD: usize = 20_000;
 
+pub const DEFAULT_SPARSE_FULL_SCAN_THRESHOLD: usize = 5_000;
+
 /// Persistable state of segment configuration
 #[derive(Debug, Deserialize, Serialize, Clone)]
 #[serde(rename_all = "snake_case")]