Improve description of full_scan_threshold (#7033)

* Improve description of full_scan_threshold in configuration

* Update OpenAPI spec

* edit in grpc

* coderabbit nit

---------

Co-authored-by: timvisee <tim@visee.me>
Co-authored-by: Luis Cossío <luis.cossio@outlook.com>

config/config.yaml

@@ -156,9 +156,11 @@ storage:
     # Number of neighbours to consider during the index building. Larger the value - more accurate the search, more time required to build index.
     ef_construct: 100
 
-    # Minimal size (in KiloBytes) of vectors for additional payload-based indexing.
-    # If payload chunk is smaller than `full_scan_threshold_kb` additional indexing won't be used -
-    # in this case full-scan search should be preferred by query planner and additional indexing is not required.
+    # Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search.
+    # This measures the total size of vectors being queried against.
+    # When the maximum estimated amount of points that a condition satisfies is smaller than
+    # `full_scan_threshold_kb`, the query planner will use full-scan search instead of HNSW index
+    # traversal for better performance.
     # Note: 1Kb = 1 vector of size 256
     full_scan_threshold_kb: 10000
 

docs/grpc/docs.md

@@ -937,7 +937,7 @@
 | ----- | ---- | ----- | ----------- |
 | m | [uint64](#uint64) | optional | Number of edges per node in the index graph. Larger the value - more accurate the search, more space required. |
 | ef_construct | [uint64](#uint64) | optional | Number of neighbours to consider during the index building. Larger the value - more accurate the search, more time required to build the index. |
-| full_scan_threshold | [uint64](#uint64) | optional | Minimal size (in KiloBytes) of vectors for additional payload-based indexing. If the payload chunk is smaller than `full_scan_threshold` additional indexing won't be used - in this case full-scan search should be preferred by query planner and additional indexing is not required. Note: 1 Kb = 1 vector of size 256 |
+| full_scan_threshold | [uint64](#uint64) | optional | Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search. This measures the total size of vectors being queried against. When the maximum estimated amount of points that a condition satisfies is smaller than `full_scan_threshold`, the query planner will use full-scan search instead of HNSW index traversal for better performance. Note: 1Kb = 1 vector of size 256 |
 | max_indexing_threads | [uint64](#uint64) | optional | Number of parallel threads used for background index building. If 0 - automatically select from 8 to 16. Best to keep between 8 and 16 to prevent likelihood of building broken/inefficient HNSW graphs. On small CPUs, less threads are used. |
 | on_disk | [bool](#bool) | optional | Store HNSW index on disk. If set to false, the index will be stored in RAM. |
 | payload_m | [uint64](#uint64) | optional | Number of additional payload-aware links per node in the index graph. If not set - regular M parameter will be used. |

docs/redoc/master/openapi.json

@@ -6884,7 +6884,7 @@
             "nullable": true
           },
           "full_scan_threshold": {
-            "description": "Minimal size (in kilobytes) of vectors for additional payload-based indexing. If payload chunk is smaller than `full_scan_threshold_kb` additional indexing won't be used - in this case full-scan search should be preferred by query planner and additional indexing is not required. Note: 1Kb = 1 vector of size 256",
+            "description": "Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search. This measures the total size of vectors being queried against. When the maximum estimated amount of points that a condition satisfies is smaller than `full_scan_threshold_kb`, the query planner will use full-scan search instead of HNSW index traversal for better performance. Note: 1Kb = 1 vector of size 256",
             "type": "integer",
             "format": "uint",
             "minimum": 10,
@@ -7178,7 +7178,7 @@
             "minimum": 4
           },
           "full_scan_threshold": {
-            "description": "Minimal size (in KiloBytes) of vectors for additional payload-based indexing. If payload chunk is smaller than `full_scan_threshold_kb` additional indexing won't be used - in this case full-scan search should be preferred by query planner and additional indexing is not required. Note: 1Kb = 1 vector of size 256",
+            "description": "Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search. This measures the total size of vectors being queried against. When the maximum estimated amount of points that a condition satisfies is smaller than `full_scan_threshold_kb`, the query planner will use full-scan search instead of HNSW index traversal for better performance. Note: 1Kb = 1 vector of size 256",
             "type": "integer",
             "format": "uint",
             "minimum": 0

lib/api/src/grpc/proto/collections.proto

@@ -170,10 +170,12 @@ message HnswConfigDiff {
   */
   optional uint64 ef_construct = 2;
   /*
-     Minimal size (in KiloBytes) of vectors for additional payload-based indexing.
-     If the payload chunk is smaller than `full_scan_threshold` additional indexing won't be used -
-     in this case full-scan search should be preferred by query planner and additional indexing is not required.
-     Note: 1 Kb = 1 vector of size 256
+    Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search.
+    This measures the total size of vectors being queried against.
+    When the maximum estimated amount of points that a condition satisfies is smaller than
+    `full_scan_threshold`, the query planner will use full-scan search instead of HNSW index
+    traversal for better performance.
+    Note: 1Kb = 1 vector of size 256
   */
   optional uint64 full_scan_threshold = 3;
   /*

lib/api/src/grpc/qdrant.rs

@@ -287,10 +287,12 @@ pub struct HnswConfigDiff {
     #[prost(uint64, optional, tag = "2")]
     #[validate(range(min = 4))]
     pub ef_construct: ::core::option::Option<u64>,
-    /// Minimal size (in KiloBytes) of vectors for additional payload-based indexing.
-    /// If the payload chunk is smaller than `full_scan_threshold` additional indexing won't be used -
-    /// in this case full-scan search should be preferred by query planner and additional indexing is not required.
-    /// Note: 1 Kb = 1 vector of size 256
+    /// Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search.
+    /// This measures the total size of vectors being queried against.
+    /// When the maximum estimated amount of points that a condition satisfies is smaller than
+    /// `full_scan_threshold`, the query planner will use full-scan search instead of HNSW index
+    /// traversal for better performance.
+    /// Note: 1Kb = 1 vector of size 256
     #[prost(uint64, optional, tag = "3")]
     pub full_scan_threshold: ::core::option::Option<u64>,
     /// Number of parallel threads used for background index building.

lib/collection/src/operations/config_diff.rs

@@ -65,9 +65,11 @@ pub struct HnswConfigDiff {
     #[validate(range(min = 4))]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ef_construct: Option<usize>,
-    /// Minimal size (in kilobytes) of vectors for additional payload-based indexing.
-    /// If payload chunk is smaller than `full_scan_threshold_kb` additional indexing won't be used -
-    /// in this case full-scan search should be preferred by query planner and additional indexing is not required.
+    /// Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search.
+    /// This measures the total size of vectors being queried against.
+    /// When the maximum estimated amount of points that a condition satisfies is smaller than
+    /// `full_scan_threshold_kb`, the query planner will use full-scan search instead of HNSW index
+    /// traversal for better performance.
     /// Note: 1Kb = 1 vector of size 256
     #[serde(
         alias = "full_scan_threshold_kb",

lib/segment/src/types.rs

@@ -570,9 +570,11 @@ pub struct HnswConfig {
     /// Number of neighbours to consider during the index building. Larger the value - more accurate the search, more time required to build index.
     #[validate(range(min = 4))]
     pub ef_construct: usize,
-    /// Minimal size (in KiloBytes) of vectors for additional payload-based indexing.
-    /// If payload chunk is smaller than `full_scan_threshold_kb` additional indexing won't be used -
-    /// in this case full-scan search should be preferred by query planner and additional indexing is not required.
+    /// Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search.
+    /// This measures the total size of vectors being queried against.
+    /// When the maximum estimated amount of points that a condition satisfies is smaller than
+    /// `full_scan_threshold_kb`, the query planner will use full-scan search instead of HNSW index
+    /// traversal for better performance.
     /// Note: 1Kb = 1 vector of size 256
     #[serde(alias = "full_scan_threshold_kb")]
     pub full_scan_threshold: usize,