Skip prefetch for small vector storages (#10420)

* Skip prefetch for small vector storages (they fit in L2)

* Update lib/common/common/src/prefetch.rs

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* Update lib/common/common/src/prefetch.rs

Co-authored-by: Andrey Vasnetsov <andrey@vasnetsov.com>

* clippy

---------

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Co-authored-by: Andrey Vasnetsov <andrey@vasnetsov.com>

lib/common/common/src/prefetch.rs

@@ -25,6 +25,17 @@ const FAR_BYTES: usize = 4096;
 /// common batch sizes (HNSW neighbor expansions) once the window is deep.
 pub const MAX_UNPREFETCHED_BATCH: usize = 2;
 
+/// Smallest per-segment quantized storage that still benefits from prefetch
+/// A storage below this size stays resident in a core's private
+/// L2 (and low L3), so its vectors are already hot when scored and the hints
+/// only add issue overhead. Measured: a cache-resident binary/128 storage
+/// (~0.4 MB per segment) regressed ~2-4% under contention across Zen 2/3 and
+/// Intel, while its ~3.2 MB per-segment 1024-dim counterpart, which overflows
+/// L2, kept the full win. 1 MiB sits above the 512 KB-1 MB L2 of common server
+/// cores and below that overflow point. Compared against the storage's own
+/// byte size, so it scales with how points are split across segments.
+pub const MIN_PREFETCH_STORAGE_BYTES: usize = 1024 * 1024; // 1Mb
+
 /// Near (L1, [`prefetch_slice`]) and far (L2, [`prefetch_slice_l2`]) prefetch
 /// window sizes in vectors, covering a constant [`NEAR_BYTES`] / [`FAR_BYTES`]
 /// of lead — bytes, not vector counts, map to the time a fetch has to

lib/segment/src/vector_storage/quantized/quantized_ram_storage.rs

@@ -5,7 +5,8 @@ use std::path::{Path, PathBuf};
 use common::counter::hardware_counter::HardwareCounterCell;
 use common::mmap::MmapFlusher;
 use common::prefetch::{
-    MAX_UNPREFETCHED_BATCH, prefetch_slice, prefetch_slice_l2, prefetch_windows,
+    MAX_UNPREFETCHED_BATCH, MIN_PREFETCH_STORAGE_BYTES, prefetch_slice, prefetch_slice_l2,
+    prefetch_windows,
 };
 use common::types::PointOffsetType;
 use common::universal_io::{CachedReadFs, OneshotFile, UniversalRead, UniversalReadFs};
@@ -130,10 +131,15 @@ impl quantization::EncodedStorage for QuantizedRamStorage {
         offsets: &[PointOffsetType],
         mut callback: impl FnMut(usize, Cow<'_, [u8]>),
     ) {
-        // Tiny batches gain nothing from hints, and dense-ascending batches
-        // stream — the hardware prefetcher already covers them and software
+        // Tiny batches gain nothing from hints, cache-resident storages have
+        // nothing to fetch, and dense-ascending batches stream — in all three
+        // the hardware prefetcher already covers the access and software
         // prefetch is pure overhead.
-        if offsets.len() <= MAX_UNPREFETCHED_BATCH || is_read_with_prefetch_efficient(offsets) {
+        let storage_bytes = self.vectors.len() * self.vectors.vector_size_bytes();
+        if offsets.len() <= MAX_UNPREFETCHED_BATCH
+            || storage_bytes < MIN_PREFETCH_STORAGE_BYTES
+            || is_read_with_prefetch_efficient(offsets)
+        {
             default_for_each_batch(self, offsets, callback);
             return;
         }

lib/segment/src/vector_storage/quantized/quantized_storage.rs

@@ -10,7 +10,8 @@ use common::generic_consts::{AccessPattern, Random, Sequential};
 use common::maybe_uninit::maybe_uninit_fill_from;
 use common::mmap::{AdviceSetting, MmapFlusher, advice};
 use common::prefetch::{
-    MAX_UNPREFETCHED_BATCH, prefetch_slice, prefetch_slice_l2, prefetch_windows,
+    MAX_UNPREFETCHED_BATCH, MIN_PREFETCH_STORAGE_BYTES, prefetch_slice, prefetch_slice_l2,
+    prefetch_windows,
 };
 use common::types::PointOffsetType;
 use common::universal_io::{
@@ -104,6 +105,12 @@ impl<S: UniversalRead> QuantizedStorage<S> {
         let mut vectors_buffer = [const { MaybeUninit::uninit() }; VECTOR_READ_BATCH_SIZE];
         let (near, far) = prefetch_windows(self.quantized_vector_size.get());
 
+        // A storage small enough to stay cache-resident has nothing to fetch,
+        // so the hints are pure overhead; skip them. On the rare chance the
+        // size lookup fails, treat the storage as large and keep prefetching.
+        let storage_bytes = self.storage.len::<u8>().unwrap_or(u64::MAX) as usize;
+        let storage_fits_cache = storage_bytes < MIN_PREFETCH_STORAGE_BYTES;
+
         for (batch_idx, keys) in keys.chunks(VECTOR_READ_BATCH_SIZE).enumerate() {
             let sequential = is_read_with_prefetch_efficient(keys);
             let vectors = if sequential {
@@ -116,9 +123,10 @@ impl<S: UniversalRead> QuantizedStorage<S> {
 
             let batch_offset = VECTOR_READ_BATCH_SIZE * batch_idx;
 
-            // Dense-ascending batches stream; the hardware prefetcher already
-            // covers them and software prefetch is pure overhead.
-            if sequential || vectors.len() <= MAX_UNPREFETCHED_BATCH {
+            // Dense-ascending batches stream and cache-resident storages are
+            // already hot; the hardware prefetcher covers both and software
+            // prefetch is pure overhead.
+            if sequential || storage_fits_cache || vectors.len() <= MAX_UNPREFETCHED_BATCH {
                 for (vector_idx, vector) in vectors.iter().enumerate() {
                     f(batch_offset + vector_idx, vector);
                 }