perf(hnsw): prefetch neighbor vectors during graph search

Issue a software prefetch for each unvisited neighbor's vector data in
search_on_level before the batch scoring pass, so the scattered-load
latency overlaps the rest of the neighbor-collection loop.

The prefetch is plumbed through default-no-op hooks (FilteredScorer::
prefetch_points -> RawScorer -> QueryScorer::prefetch_stored ->
DenseVectorStorageRead::prefetch_dense), so only dense metric scoring
over in-RAM/mmap storage is affected; multivector, sparse and quantized
paths are untouched.

It is gated by DenseVectorStorageRead::get_dense_is_borrowed(): io_uring
reads copy into an owned buffer, so prefetching there would trigger a
redundant read. The method returns false for io_uring, keeping prefetch
a no-op on that path.

Benchmark (hnsw_search_graph, 1M x 64d cosine, k=10, ef=100, single
thread, 10 runs): +20.6% on in-RAM storage and +17.4% on
page-cache-warm mmap storage; every run was faster with prefetch
(paired-t ~10.3 and ~9.0, p<0.001). Also adds a permanent mmap variant
to the benchmark.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

lib/segment/benches/fixture.rs

@@ -6,13 +6,17 @@ use fs_err as fs;
 use rand::SeedableRng as _;
 use rand::rngs::SmallRng;
 use rayon::iter::{IntoParallelIterator as _, ParallelIterator as _};
-use segment::fixtures::index_fixtures::TestRawScorerProducer;
+use segment::data_types::vectors::VectorElementType;
+use segment::fixtures::index_fixtures::{TestRawScorerProducer, random_vector};
 use segment::index::hnsw_index::HnswM;
 use segment::index::hnsw_index::graph_layers::GraphLayers;
 use segment::index::hnsw_index::graph_layers_builder::GraphLayersBuilder;
 use segment::index::hnsw_index::graph_links::{GraphLinksFormatParam, GraphLinksResidency};
 use segment::index::hnsw_index::hnsw::SINGLE_THREADED_HNSW_BUILD_THRESHOLD;
 use segment::spaces::metric::Metric;
+use segment::vector_storage::dense::dense_vector_storage::open_dense_vector_storage;
+use segment::vector_storage::{DEFAULT_STOPPED, DenseVectorStorage, VectorStorageEnum};
+use tempfile::TempDir;
 
 /// Generate vectors and HNSW graph to be used in benchmarks.
 ///
@@ -89,6 +93,50 @@ where
     (vector_holder, graph_layers)
 }
 
+/// Build an mmap-backed dense vector storage holding the same seed-42 vectors
+/// as [`make_cached_graph`], so the cached graph is valid against it.
+///
+/// Returns the [`TempDir`] (which must be kept alive for the mmap to stay open)
+/// and a [`TestRawScorerProducer`] whose `scorer` reads from the mmap storage.
+// Shared bench module: not every bench that includes `fixture` uses this.
+#[allow(dead_code)]
+pub fn make_memmap_producer<METRIC>(
+    num_vectors: usize,
+    dim: usize,
+) -> (TempDir, TestRawScorerProducer)
+where
+    METRIC: Metric<f32>,
+{
+    let distance = METRIC::distance();
+
+    // Force the plain mmap variant (not io_uring), so prefetch is exercised.
+    #[cfg(target_os = "linux")]
+    segment::vector_storage::common::set_async_scorer(false);
+
+    let tmp = tempfile::Builder::new()
+        .prefix("hnsw-mmap-bench")
+        .tempdir()
+        .unwrap();
+    let mut storage = open_dense_vector_storage(tmp.path(), dim, distance, false).unwrap();
+
+    // Regenerate the exact vectors from `make_cached_graph` (same seed + preprocess).
+    let mut rng = SmallRng::seed_from_u64(42);
+    let mut vectors = (0..num_vectors).map(|_| {
+        let v = random_vector(&mut rng, dim);
+        let v = distance.preprocess_vector::<VectorElementType>(v);
+        (std::borrow::Cow::Owned(v), false)
+    });
+    let VectorStorageEnum::DenseMemmap(mmap_storage) = &mut storage else {
+        panic!("expected DenseMemmap storage");
+    };
+    mmap_storage
+        .update_from(&mut vectors, &DEFAULT_STOPPED)
+        .unwrap();
+
+    let producer = TestRawScorerProducer::from_storage(storage, num_vectors);
+    (tmp, producer)
+}
+
 fn updated_ago(path: &Path) -> Result<String, Box<dyn std::error::Error>> {
     let elapsed = fs::metadata(path)?.modified()?.elapsed()?;
     let secs_rounded = elapsed.as_secs().next_multiple_of(60);

lib/segment/benches/hnsw_search_graph.rs

@@ -30,27 +30,32 @@ fn hnsw_benchmark(c: &mut Criterion) {
     let (vector_holder, mut graph_layers) =
         fixture::make_cached_graph::<Metric>(NUM_VECTORS, DIM, M, EF_CONSTRUCT, USE_HEURISTIC);
 
-    let mut rng = SmallRng::seed_from_u64(42);
-    group.bench_function("uncompressed", |b| {
-        b.iter(|| {
-            let query = random_vector(&mut rng, DIM);
+    let (_mmap_tmp, mmap_holder) = fixture::make_memmap_producer::<Metric>(NUM_VECTORS, DIM);
 
-            let scorer = vector_holder.scorer(query);
+    // Search the same graph over in-RAM and mmap-backed vector storage.
+    for (name, holder) in [("uncompressed", &vector_holder), ("mmap", &mmap_holder)] {
+        let mut rng = SmallRng::seed_from_u64(42);
+        group.bench_function(name, |b| {
+            b.iter(|| {
+                let query = random_vector(&mut rng, DIM);
 
-            black_box(
-                graph_layers
-                    .search(
-                        TOP,
-                        EF,
-                        SearchAlgorithm::Hnsw,
-                        scorer,
-                        None,
-                        &DEFAULT_STOPPED,
-                    )
-                    .unwrap(),
-            );
-        })
-    });
+                let scorer = holder.scorer(query);
+
+                black_box(
+                    graph_layers
+                        .search(
+                            TOP,
+                            EF,
+                            SearchAlgorithm::Hnsw,
+                            scorer,
+                            None,
+                            &DEFAULT_STOPPED,
+                        )
+                        .unwrap(),
+                );
+            })
+        });
+    }
 
     graph_layers.compress_ram();
     let mut rng = SmallRng::seed_from_u64(42);

lib/segment/src/fixtures/index_fixtures.rs

@@ -77,6 +77,16 @@ impl TestRawScorerProducer {
         }
     }
 
+    /// Wrap a pre-constructed storage (e.g. an mmap-backed one) with
+    /// `num_vectors` live points and no quantization. For tests and benchmarks.
+    pub fn from_storage(storage: VectorStorageEnum, num_vectors: usize) -> Self {
+        TestRawScorerProducer {
+            storage,
+            deleted_points: BitVec::repeat(false, num_vectors),
+            quantized_vectors: None,
+        }
+    }
+
     pub fn storage(&self) -> &VectorStorageEnum {
         &self.storage
     }

lib/segment/src/index/hnsw_index/graph_layers.rs

@@ -136,6 +136,11 @@ pub trait GraphLayersBase {
                 }
             });
 
+            // Warm the CPU cache with the candidates' vectors before the
+            // scattered-load scoring pass below. No-op for storages whose reads
+            // copy (e.g. io_uring); see `DenseVectorStorageRead::prefetch_dense`.
+            points_scorer.prefetch_points(&points_ids);
+
             points_scorer
                 .score_points(&mut points_ids, limit)
                 .for_each(|score_point| {

lib/segment/src/index/hnsw_index/point_scorer.rs

@@ -279,6 +279,14 @@ impl<'a> FilteredScorer<'a> {
         self.score_points_unfiltered(point_ids)
     }
 
+    /// Best-effort prefetch of the stored vectors for `point_ids` (experimental).
+    ///
+    /// Used to overlap the scattered-load latency of the subsequent
+    /// `score_points` pass with the rest of the neighbor collection.
+    pub fn prefetch_points(&self, point_ids: &[PointOffsetType]) {
+        self.raw_scorer.prefetch_points(point_ids);
+    }
+
     pub fn score_points_unfiltered(
         &mut self,
         point_ids: &[PointOffsetType],

lib/segment/src/vector_storage/dense/dense_vector_storage.rs

@@ -237,6 +237,12 @@ where
             .unwrap_or_else(|| panic!("vector not found: {key}"))
     }
 
+    fn get_dense_is_borrowed(&self) -> bool {
+        // io_uring reads copy into an owned buffer; prefetching would trigger a
+        // redundant read, so gate it off there. mmap reads borrow.
+        !self.vectors.as_ref().unwrap().is_io_uring()
+    }
+
     fn for_each_in_dense_batch<F: FnMut(usize, &[T])>(
         &self,
         keys: &[PointOffsetType],

lib/segment/src/vector_storage/dense/immutable_dense_vectors.rs

@@ -114,6 +114,12 @@ impl<T: PrimitiveVectorElement, S: UniversalRead> ImmutableDenseVectorData<T, S>
         Some(offset)
     }
 
+    /// Whether the backing storage reads via io_uring, which copies into an
+    /// owned buffer rather than borrowing from an mmap.
+    pub fn is_io_uring(&self) -> bool {
+        TypedStorage::<ReadOnly<S>, T>::kind() == common::universal_io::UniversalKind::IoUring
+    }
+
     /// Read one vector from storage at the given byte offset.
     fn raw_vector_offset<P: AccessPattern>(&self, byte_offset: usize) -> Cow<'_, [T]> {
         let range = ReadRange {
@@ -299,6 +305,10 @@ impl<T: PrimitiveVectorElement, S: UniversalRead> ImmutableDenseVectors<T, S> {
         self.data.for_each_in_batch(keys, f)
     }
 
+    pub fn is_io_uring(&self) -> bool {
+        self.data.is_io_uring()
+    }
+
     /// Marks the key as deleted.
     ///
     /// Returns true if the key was not deleted before, and it is now deleted.

lib/segment/src/vector_storage/query_scorer/metric_query_scorer.rs

@@ -91,6 +91,13 @@ impl<
             .expect("read vectors");
     }
 
+    #[inline]
+    fn prefetch_stored(&self, ids: &[PointOffsetType]) {
+        for &id in ids {
+            self.vector_storage.prefetch_dense(id);
+        }
+    }
+
     fn score_internal(&self, point_a: PointOffsetType, point_b: PointOffsetType) -> ScoreType {
         self.hardware_counter.cpu_counter().incr();
         let v1 = self.vector_storage.get_dense::<Random>(point_a);

lib/segment/src/vector_storage/query_scorer/mod.rs

@@ -35,6 +35,12 @@ pub trait QueryScorer {
 
     fn score_internal(&self, point_a: PointOffsetType, point_b: PointOffsetType) -> ScoreType;
 
+    /// Best-effort prefetch of the stored vectors for `ids` (experimental).
+    ///
+    /// Default is a no-op; storages that can cheaply address their vectors
+    /// override this to warm the CPU cache ahead of scoring.
+    fn prefetch_stored(&self, _ids: &[PointOffsetType]) {}
+
     type SupportsBytes: TBool;
     fn score_bytes(&self, _: Self::SupportsBytes, bytes: &[u8]) -> ScoreType;
 }

lib/segment/src/vector_storage/raw_scorer.rs

@@ -40,6 +40,11 @@ use crate::vector_storage::turbo::multi::TurboMultiVectorStorage;
 pub trait RawScorer {
     fn score_points(&self, points: &[PointOffsetType], scores: &mut [ScoreType]);
 
+    /// Best-effort prefetch of the stored vectors for `points` (experimental).
+    ///
+    /// Default is a no-op.
+    fn prefetch_points(&self, _points: &[PointOffsetType]) {}
+
     /// Score stored vector with vector under the given index
     fn score_point(&self, point: PointOffsetType) -> ScoreType;
 
@@ -569,6 +574,10 @@ impl<TQueryScorer: QueryScorer> RawScorer for RawScorerImpl<TQueryScorer> {
         self.query_scorer.score_stored_batch(points, scores);
     }
 
+    fn prefetch_points(&self, points: &[PointOffsetType]) {
+        self.query_scorer.prefetch_stored(points);
+    }
+
     fn score_point(&self, point: PointOffsetType) -> ScoreType {
         self.query_scorer.score_stored(point)
     }

lib/segment/src/vector_storage/vector_storage_base.rs

@@ -250,6 +250,32 @@ pub trait VectorStorage: VectorStorageRead {
     fn delete_vector(&mut self, key: PointOffsetType) -> OperationResult<bool>;
 }
 
+/// Best-effort software prefetch (temporal, all cache levels) of every cache
+/// line spanning `bytes`.
+///
+/// Experimental: used to overlap the scattered-load latency of HNSW batch
+/// scoring. A prefetch is only a hint and never faults, so this is always safe.
+/// No-op on non-x86_64 targets (aarch64 has no stable prefetch intrinsic).
+#[inline(always)]
+fn prefetch_slice(bytes: &[u8]) {
+    #[cfg(target_arch = "x86_64")]
+    {
+        use std::arch::x86_64::{_MM_HINT_T0, _mm_prefetch};
+
+        const CACHE_LINE: usize = 64;
+        let ptr = bytes.as_ptr();
+        let mut offset = 0;
+        while offset < bytes.len() {
+            // SAFETY: `_mm_prefetch` is a non-faulting hint; the computed address
+            // lies within (or one cache line past) a valid borrowed slice.
+            unsafe { _mm_prefetch::<_MM_HINT_T0>(ptr.add(offset).cast::<i8>()) };
+            offset += CACHE_LINE;
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    let _ = bytes;
+}
+
 /// Read-only access to a dense vector storage.
 ///
 /// Everything needed to read and score dense vectors, without the ability to
@@ -260,6 +286,29 @@ pub trait DenseVectorStorageRead<T: PrimitiveVectorElement>: VectorStorageRead {
 
     fn get_dense<P: AccessPattern>(&self, key: PointOffsetType) -> Cow<'_, [T]>;
 
+    /// Whether [`Self::get_dense`] returns a cheap borrow (in-RAM or mmap)
+    /// rather than performing I/O and allocating an owned buffer.
+    ///
+    /// Gates [`Self::prefetch_dense`]: prefetching only makes sense when the
+    /// vector data can be addressed without reading it. Storages that copy on
+    /// read (e.g. io_uring) return `false` so prefetch stays a no-op and does
+    /// not trigger a redundant read.
+    fn get_dense_is_borrowed(&self) -> bool {
+        true
+    }
+
+    /// Best-effort software prefetch of a single vector's data into cache.
+    ///
+    /// Issued while collecting HNSW neighbors so the scattered load latency is
+    /// hidden behind the rest of the collection pass. No-op for storages whose
+    /// reads copy (see [`Self::get_dense_is_borrowed`]).
+    fn prefetch_dense(&self, key: PointOffsetType) {
+        if self.get_dense_is_borrowed() && (key as usize) < self.total_vector_count() {
+            let dense = self.get_dense::<Random>(key);
+            prefetch_slice(bytemuck::cast_slice::<T, u8>(&dense));
+        }
+    }
+
     /// Call `f` with the raw bytes of the vector if it exists.
     ///
     /// Uses `bytemuck::cast_slice` on the borrowed data — zero copy, zero allocation.