[AI + manual] add preopen functions to id tracker (#9729)

lib/segment/src/id_tracker/disk_id_tracker/read_only.rs

@@ -22,7 +22,7 @@ use common::mmap::AdviceSetting;
 use common::stored_bitslice::StoredBitSlice;
 use common::types::{DeferredBehavior, PointOffsetType};
 use common::universal_io::{
-    OpenOptions, Populate, ReadRange, TypedStorage, UniversalRead, UniversalReadFs,
+    CachedReadFs, OpenOptions, Populate, ReadRange, TypedStorage, UniversalRead, UniversalReadFs,
 };
 
 use super::mappings::{DiskMappingsSource, log_lookup_err};
@@ -53,6 +53,36 @@ pub struct ReadOnlyDiskIdTracker<S: UniversalRead> {
 }
 
 impl<S: UniversalRead> ReadOnlyDiskIdTracker<S> {
+    fn open_options() -> OpenOptions {
+        OpenOptions {
+            writeable: false,
+            need_sequential: false,
+            populate: Populate::No,
+            advice: AdviceSetting::Global,
+        }
+    }
+
+    /// Schedule background prefetch of every file [`try_open`](Self::try_open)
+    /// will read
+    ///
+    /// Returns `false` (nothing scheduled) when the tracker is not in the
+    /// on-disk format.
+    pub fn try_preopen(
+        fs: &impl CachedReadFs<File = S>,
+        segment_path: &Path,
+    ) -> OperationResult<bool> {
+        if !DiskMappingReader::try_preopen(fs, segment_path)? {
+            return Ok(false);
+        }
+
+        let options = Self::open_options();
+
+        fs.schedule_prefetch(&version_mapping_path(segment_path), Some(options), None)?;
+        fs.schedule_prefetch(&deleted_path(segment_path), Some(options), None)?;
+
+        Ok(true)
+    }
+
     /// Open a read-only disk id tracker at `segment_path`. Reads only the two
     /// headers and the e2i sparse block index into RAM; all per-point data stays
     /// on the backing store.
@@ -79,12 +109,8 @@ impl<S: UniversalRead> ReadOnlyDiskIdTracker<S> {
             return Ok(None);
         };
 
-        let options = OpenOptions {
-            writeable: false,
-            need_sequential: false,
-            populate: Populate::No,
-            advice: AdviceSetting::Global,
-        };
+        let options = Self::open_options();
+
         let versions = TypedStorage::<S, SeqNumberType>::new(fs.open(
             version_mapping_path(segment_path),
             options,

lib/segment/src/id_tracker/disk_id_tracker/reader.rs

@@ -20,7 +20,8 @@ use common::bitvec::{BitSlice, BitSliceExt as _};
 use common::mmap::AdviceSetting;
 use common::types::PointOffsetType;
 use common::universal_io::{
-    OkNotFound, OpenOptions, Populate, ReadRange, UniversalRead, UniversalReadFs,
+    CachedReadFs, OkNotFound, OpenOptions, Populate, ReadRange, UniversalRead,
+    UniversalReadFileOps, UniversalReadFs,
 };
 use itertools::Itertools as _;
 use rand::distr::{Distribution as _, Uniform};
@@ -49,6 +50,38 @@ pub struct DiskMappingReader<S: UniversalRead> {
 }
 
 impl<S: UniversalRead> DiskMappingReader<S> {
+    fn open_options() -> OpenOptions {
+        OpenOptions {
+            writeable: false,
+            need_sequential: false,
+            // TODO(uio): files have headers, we should use Populate::BackgroundPartial
+            populate: Populate::No,
+            advice: AdviceSetting::Global,
+        }
+    }
+
+    /// Schedule background prefetch of the `i2e`/`e2i` handles that
+    /// [`try_open`](Self::try_open) will open, without reading any bytes.
+    ///
+    /// Returns `false` (nothing scheduled) when the mapping is
+    /// not in the on-disk format.
+    pub fn try_preopen(
+        fs: &impl CachedReadFs<File = S>,
+        segment_path: &Path,
+    ) -> OperationResult<bool> {
+        let i2e_path = i2e_path(segment_path);
+        if !UniversalReadFileOps::exists(fs, &i2e_path)? {
+            return Ok(false);
+        }
+
+        let options = Self::open_options();
+
+        fs.schedule_prefetch(&i2e_path, Some(options), None)?;
+        fs.schedule_prefetch(&e2i_path(segment_path), Some(options), None)?;
+
+        Ok(true)
+    }
+
     /// Open the `i2e`/`e2i` handles and read the headers + sparse index. No
     /// per-point data is read.
     ///
@@ -71,12 +104,7 @@ impl<S: UniversalRead> DiskMappingReader<S> {
         fs: &impl UniversalReadFs<File = S>,
         segment_path: &Path,
     ) -> OperationResult<Option<Self>> {
-        let options = OpenOptions {
-            writeable: false,
-            need_sequential: false,
-            populate: Populate::No,
-            advice: AdviceSetting::Global,
-        };
+        let options = Self::open_options();
 
         let Some(i2e) = fs
             .open(i2e_path(segment_path), options, Default::default())

lib/segment/src/id_tracker/id_tracker_base/read_only_tracker_enum.rs

@@ -2,7 +2,7 @@ use std::path::Path;
 
 use common::bitvec::BitSlice;
 use common::types::PointOffsetType;
-use common::universal_io::{UniversalRead, UniversalReadFs};
+use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
 
 use crate::common::operation_error::OperationResult;
 use crate::id_tracker::disk_id_tracker::ReadOnlyDiskIdTracker;
@@ -20,6 +20,18 @@ pub enum ReadOnlyIdTrackerEnum<S: UniversalRead> {
 }
 
 impl<S: UniversalRead> ReadOnlyIdTrackerEnum<S> {
+    /// Schedule background prefetch for whichever id-tracker format is
+    /// present, probing in the same order as [`Self::detect_and_load`].
+    pub fn preopen(fs: &impl CachedReadFs<File = S>, segment_path: &Path) -> OperationResult<()> {
+        if ReadOnlyDiskIdTracker::try_preopen(fs, segment_path)? {
+            return Ok(());
+        }
+        if ReadOnlyImmutableIdTracker::try_preopen(fs, segment_path)? {
+            return Ok(());
+        }
+        ReadOnlyAppendableIdTracker::preopen(fs, segment_path)
+    }
+
     /// Detect the persisted id-tracker format and load it, by *attempting* each
     /// format's open rather than probing file names one by one.
     ///

lib/segment/src/id_tracker/immutable_id_tracker/read_only/lifecycle.rs

@@ -6,8 +6,8 @@ use common::generic_consts::Sequential;
 use common::mmap::AdviceSetting;
 use common::stored_bitslice::StoredBitSlice;
 use common::universal_io::{
-    OpenOptions, Populate, ReadRange, TypedStorage, UniversalRead, UniversalReadFileOps,
-    UniversalReadFs,
+    CachedReadFs, OpenOptions, Populate, ReadRange, TypedStorage, UniversalRead,
+    UniversalReadFileOps, UniversalReadFs,
 };
 
 use super::ReadOnlyImmutableIdTracker;
@@ -19,6 +19,37 @@ use crate::id_tracker::immutable_id_tracker::versions_storage::version_mapping_p
 use crate::types::SeqNumberType;
 
 impl<S: UniversalRead> ReadOnlyImmutableIdTracker<S> {
+    fn open_options() -> OpenOptions {
+        OpenOptions {
+            writeable: false,
+            need_sequential: false,
+            populate: Populate::PreferBackground,
+            advice: AdviceSetting::Global,
+        }
+    }
+
+    /// Schedule background prefetch of every file [`open`](Self::open) will
+    /// read
+    ///
+    /// Returns `false` (nothing scheduled) when the segment is not in the
+    /// immutable format.
+    pub fn try_preopen(
+        fs: &impl CachedReadFs<File = S>,
+        segment_path: &Path,
+    ) -> OperationResult<bool> {
+        if !UniversalReadFileOps::exists(fs, &mappings_path(segment_path))? {
+            return Ok(false);
+        }
+
+        let options = Self::open_options();
+
+        fs.schedule_prefetch(&deleted_path(segment_path), Some(options), None)?;
+        fs.schedule_prefetch(&version_mapping_path(segment_path), Some(options), None)?;
+        fs.schedule_prefetch(&mappings_path(segment_path), Some(options), None)?;
+
+        Ok(true)
+    }
+
     /// Open a read-only view over immutable ID tracker data at `segment_path`, threading every file
     /// open through `fs`. Read-only mirror of [`ImmutableIdTracker::open`]; it never writes.
     ///
@@ -43,12 +74,7 @@ impl<S: UniversalRead> ReadOnlyImmutableIdTracker<S> {
     }
 
     pub fn open(fs: &impl UniversalReadFs<File = S>, segment_path: &Path) -> OperationResult<Self> {
-        let options = OpenOptions {
-            writeable: false,
-            need_sequential: false,
-            populate: Populate::Blocking,
-            advice: AdviceSetting::Global,
-        };
+        let options = Self::open_options();
 
         let deleted =
             StoredBitSlice::open(fs, deleted_path(segment_path), options, Default::default())?;

lib/segment/src/id_tracker/mutable_id_tracker/read_only/lifecycle.rs

@@ -3,13 +3,37 @@ use std::path::{Path, PathBuf};
 use common::mmap::Advice::Normal;
 use common::mmap::AdviceSetting;
 use common::types::PointOffsetType;
-use common::universal_io::{OkNotFound, OpenOptions, Populate, UniversalRead, UniversalReadFs};
+use common::universal_io::{
+    CachedReadFs, OkNotFound, OpenOptions, Populate, UniversalRead, UniversalReadFs,
+};
 
 use super::ReadOnlyAppendableIdTracker;
 use crate::common::operation_error::OperationResult;
+use crate::id_tracker::mutable_id_tracker::mappings_storage::mappings_path;
+use crate::id_tracker::mutable_id_tracker::versions_storage::versions_path;
 use crate::id_tracker::point_mappings::PointMappings;
 
 impl<S: UniversalRead> ReadOnlyAppendableIdTracker<S> {
+    fn open_options() -> OpenOptions {
+        OpenOptions {
+            writeable: false,
+            need_sequential: false,
+            populate: Populate::PreferBackground,
+            advice: AdviceSetting::Advice(Normal),
+        }
+    }
+
+    /// Schedule background prefetch of the mappings log and versions file that
+    /// [`open`](Self::open) reads via [`live_reload`](Self::live_reload).
+    pub fn preopen(fs: &impl CachedReadFs<File = S>, segment_path: &Path) -> OperationResult<()> {
+        let options = Self::open_options();
+
+        fs.schedule_prefetch(&mappings_path(segment_path), Some(options), None)?;
+        fs.schedule_prefetch(&versions_path(segment_path), Some(options), None)?;
+
+        Ok(())
+    }
+
     /// Open a read-only view over the appendable ID tracker data at `segment_path`, threading every
     /// file open through the filesystem handle `fs`.
     ///
@@ -70,12 +94,7 @@ impl<S: UniversalRead> ReadOnlyAppendableIdTracker<S> {
     /// first read, so a missing object can instead surface as `NotFound` from a later `len`/`read`
     /// — `live_reload` tolerates that case too.
     pub(super) fn try_open(fs: &S::Fs, path: &Path) -> OperationResult<Option<S>> {
-        let options = OpenOptions {
-            writeable: false,
-            need_sequential: false,
-            populate: Populate::No,
-            advice: AdviceSetting::Advice(Normal),
-        };
+        let options = Self::open_options();
         Ok(fs.open(path, options, Default::default()).ok_not_found()?)
     }
 }

lib/segment/src/segment/read_only/lifecycle.rs

@@ -82,6 +82,8 @@ impl<S: UniversalReadExt + 'static> ReadOnlySegment<S> {
         };
         ReadOnlyPayloadStorage::preopen(fs, segment_path.to_path_buf(), payload_populate)?;
 
+        ReadOnlyIdTrackerEnum::preopen(fs, segment_path)?;
+
         Ok(())
     }