diff --git a/config/config.yaml b/config/config.yaml index c217349c66..602a5e9608 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -156,9 +156,11 @@ storage: # Number of neighbours to consider during the index building. Larger the value - more accurate the search, more time required to build index. ef_construct: 100 - # Minimal size (in KiloBytes) of vectors for additional payload-based indexing. - # If payload chunk is smaller than `full_scan_threshold_kb` additional indexing won't be used - - # in this case full-scan search should be preferred by query planner and additional indexing is not required. + # Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search. + # This measures the total size of vectors being queried against. + # When the maximum estimated amount of points that a condition satisfies is smaller than + # `full_scan_threshold_kb`, the query planner will use full-scan search instead of HNSW index + # traversal for better performance. # Note: 1Kb = 1 vector of size 256 full_scan_threshold_kb: 10000 diff --git a/docs/grpc/docs.md b/docs/grpc/docs.md index 66c50b27cb..51903f2b80 100644 --- a/docs/grpc/docs.md +++ b/docs/grpc/docs.md @@ -937,7 +937,7 @@ | ----- | ---- | ----- | ----------- | | m | [uint64](#uint64) | optional | Number of edges per node in the index graph. Larger the value - more accurate the search, more space required. | | ef_construct | [uint64](#uint64) | optional | Number of neighbours to consider during the index building. Larger the value - more accurate the search, more time required to build the index. | -| full_scan_threshold | [uint64](#uint64) | optional | Minimal size (in KiloBytes) of vectors for additional payload-based indexing. If the payload chunk is smaller than `full_scan_threshold` additional indexing won't be used - in this case full-scan search should be preferred by query planner and additional indexing is not required. Note: 1 Kb = 1 vector of size 256 | +| full_scan_threshold | [uint64](#uint64) | optional | Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search. This measures the total size of vectors being queried against. When the maximum estimated amount of points that a condition satisfies is smaller than `full_scan_threshold`, the query planner will use full-scan search instead of HNSW index traversal for better performance. Note: 1Kb = 1 vector of size 256 | | max_indexing_threads | [uint64](#uint64) | optional | Number of parallel threads used for background index building. If 0 - automatically select from 8 to 16. Best to keep between 8 and 16 to prevent likelihood of building broken/inefficient HNSW graphs. On small CPUs, less threads are used. | | on_disk | [bool](#bool) | optional | Store HNSW index on disk. If set to false, the index will be stored in RAM. | | payload_m | [uint64](#uint64) | optional | Number of additional payload-aware links per node in the index graph. If not set - regular M parameter will be used. | diff --git a/docs/redoc/master/openapi.json b/docs/redoc/master/openapi.json index 7c976f9351..e0c65b742b 100644 --- a/docs/redoc/master/openapi.json +++ b/docs/redoc/master/openapi.json @@ -6884,7 +6884,7 @@ "nullable": true }, "full_scan_threshold": { - "description": "Minimal size (in kilobytes) of vectors for additional payload-based indexing. If payload chunk is smaller than `full_scan_threshold_kb` additional indexing won't be used - in this case full-scan search should be preferred by query planner and additional indexing is not required. Note: 1Kb = 1 vector of size 256", + "description": "Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search. This measures the total size of vectors being queried against. When the maximum estimated amount of points that a condition satisfies is smaller than `full_scan_threshold_kb`, the query planner will use full-scan search instead of HNSW index traversal for better performance. Note: 1Kb = 1 vector of size 256", "type": "integer", "format": "uint", "minimum": 10, @@ -7178,7 +7178,7 @@ "minimum": 4 }, "full_scan_threshold": { - "description": "Minimal size (in KiloBytes) of vectors for additional payload-based indexing. If payload chunk is smaller than `full_scan_threshold_kb` additional indexing won't be used - in this case full-scan search should be preferred by query planner and additional indexing is not required. Note: 1Kb = 1 vector of size 256", + "description": "Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search. This measures the total size of vectors being queried against. When the maximum estimated amount of points that a condition satisfies is smaller than `full_scan_threshold_kb`, the query planner will use full-scan search instead of HNSW index traversal for better performance. Note: 1Kb = 1 vector of size 256", "type": "integer", "format": "uint", "minimum": 0 diff --git a/lib/api/src/grpc/proto/collections.proto b/lib/api/src/grpc/proto/collections.proto index 1d1ab59cb5..44e4d84cb4 100644 --- a/lib/api/src/grpc/proto/collections.proto +++ b/lib/api/src/grpc/proto/collections.proto @@ -170,10 +170,12 @@ message HnswConfigDiff { */ optional uint64 ef_construct = 2; /* - Minimal size (in KiloBytes) of vectors for additional payload-based indexing. - If the payload chunk is smaller than `full_scan_threshold` additional indexing won't be used - - in this case full-scan search should be preferred by query planner and additional indexing is not required. - Note: 1 Kb = 1 vector of size 256 + Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search. + This measures the total size of vectors being queried against. + When the maximum estimated amount of points that a condition satisfies is smaller than + `full_scan_threshold`, the query planner will use full-scan search instead of HNSW index + traversal for better performance. + Note: 1Kb = 1 vector of size 256 */ optional uint64 full_scan_threshold = 3; /* diff --git a/lib/api/src/grpc/qdrant.rs b/lib/api/src/grpc/qdrant.rs index 33a6207bf4..b8bed8f103 100644 --- a/lib/api/src/grpc/qdrant.rs +++ b/lib/api/src/grpc/qdrant.rs @@ -287,10 +287,12 @@ pub struct HnswConfigDiff { #[prost(uint64, optional, tag = "2")] #[validate(range(min = 4))] pub ef_construct: ::core::option::Option, - /// Minimal size (in KiloBytes) of vectors for additional payload-based indexing. - /// If the payload chunk is smaller than `full_scan_threshold` additional indexing won't be used - - /// in this case full-scan search should be preferred by query planner and additional indexing is not required. - /// Note: 1 Kb = 1 vector of size 256 + /// Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search. + /// This measures the total size of vectors being queried against. + /// When the maximum estimated amount of points that a condition satisfies is smaller than + /// `full_scan_threshold`, the query planner will use full-scan search instead of HNSW index + /// traversal for better performance. + /// Note: 1Kb = 1 vector of size 256 #[prost(uint64, optional, tag = "3")] pub full_scan_threshold: ::core::option::Option, /// Number of parallel threads used for background index building. diff --git a/lib/collection/src/operations/config_diff.rs b/lib/collection/src/operations/config_diff.rs index ddd502d38c..b788802a29 100644 --- a/lib/collection/src/operations/config_diff.rs +++ b/lib/collection/src/operations/config_diff.rs @@ -65,9 +65,11 @@ pub struct HnswConfigDiff { #[validate(range(min = 4))] #[serde(skip_serializing_if = "Option::is_none")] pub ef_construct: Option, - /// Minimal size (in kilobytes) of vectors for additional payload-based indexing. - /// If payload chunk is smaller than `full_scan_threshold_kb` additional indexing won't be used - - /// in this case full-scan search should be preferred by query planner and additional indexing is not required. + /// Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search. + /// This measures the total size of vectors being queried against. + /// When the maximum estimated amount of points that a condition satisfies is smaller than + /// `full_scan_threshold_kb`, the query planner will use full-scan search instead of HNSW index + /// traversal for better performance. /// Note: 1Kb = 1 vector of size 256 #[serde( alias = "full_scan_threshold_kb", diff --git a/lib/segment/src/types.rs b/lib/segment/src/types.rs index 35a6bd44ce..dc2d425b3b 100644 --- a/lib/segment/src/types.rs +++ b/lib/segment/src/types.rs @@ -570,9 +570,11 @@ pub struct HnswConfig { /// Number of neighbours to consider during the index building. Larger the value - more accurate the search, more time required to build index. #[validate(range(min = 4))] pub ef_construct: usize, - /// Minimal size (in KiloBytes) of vectors for additional payload-based indexing. - /// If payload chunk is smaller than `full_scan_threshold_kb` additional indexing won't be used - - /// in this case full-scan search should be preferred by query planner and additional indexing is not required. + /// Minimal size threshold (in KiloBytes) below which full-scan is preferred over HNSW search. + /// This measures the total size of vectors being queried against. + /// When the maximum estimated amount of points that a condition satisfies is smaller than + /// `full_scan_threshold_kb`, the query planner will use full-scan search instead of HNSW index + /// traversal for better performance. /// Note: 1Kb = 1 vector of size 256 #[serde(alias = "full_scan_threshold_kb")] pub full_scan_threshold: usize,