mirror of
https://github.com/qdrant/qdrant.git
synced 2026-09-21 05:27:39 -05:00
schedule_open returns nothing (#10355)
This commit is contained in:
@@ -77,7 +77,7 @@ impl<S: UniversalRead> Pages<S> {
|
||||
if !page_files.contains(&page_path) {
|
||||
break;
|
||||
}
|
||||
fs.schedule_open(&page_path, Some(page_open_options(populate, false)), None)?;
|
||||
fs.schedule_open(&page_path, Some(page_open_options(populate, false)), None);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -465,7 +465,7 @@ impl<S: UniversalRead> Pages<S> {
|
||||
let page_path = self.page_path(page_id);
|
||||
|
||||
// Re-schedule so that unchanged files don't re-fetch.
|
||||
fs.reschedule_open(&page_path, Some(page_open_options(populate, false)), None)?;
|
||||
fs.reschedule_open(&page_path, Some(page_open_options(populate, false)), None);
|
||||
}
|
||||
|
||||
for page_id in next_page_id.. {
|
||||
@@ -473,7 +473,7 @@ impl<S: UniversalRead> Pages<S> {
|
||||
if !fs.exists(&page_path)? {
|
||||
break;
|
||||
}
|
||||
fs.schedule_open(&page_path, Some(page_open_options(populate, false)), None)?;
|
||||
fs.schedule_open(&page_path, Some(page_open_options(populate, false)), None);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -72,7 +72,7 @@ impl<V: Blob, S: UniversalRead> GridstoreReader<V, S> {
|
||||
) -> Result<()> {
|
||||
// schedule tracker
|
||||
let tracker_path = Tracker::<S>::tracker_file_name(base_path);
|
||||
Tracker::<S>::preopen(fs, &tracker_path, populate)?;
|
||||
Tracker::<S>::preopen(fs, &tracker_path, populate);
|
||||
|
||||
// schedule pages
|
||||
Pages::preopen(fs, base_path, populate)
|
||||
@@ -220,7 +220,7 @@ impl<V: Blob, S: UniversalRead> GridstoreReader<V, S> {
|
||||
}
|
||||
|
||||
pub(crate) fn live_preload<Fs: CachedReadFs<File = S>>(&self, fs: &Fs) -> Result<()> {
|
||||
self.tracker.live_preload(fs)?;
|
||||
self.tracker.live_preload(fs);
|
||||
self.pages.live_preload(fs, self.populate)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -64,7 +64,7 @@ impl<S: UniversalRead> AppendOnlyPages<S> {
|
||||
&path,
|
||||
Some(AppendOnlyPage::<S>::open_options(populate, false)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -292,7 +292,7 @@ impl<S: UniversalRead> AppendOnlyPages<S> {
|
||||
&path,
|
||||
Some(AppendOnlyPage::<S>::open_options(populate, false)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@ impl<V: Blob, S: UniversalRead> LogstoreReader<V, S> {
|
||||
base_path: &Path,
|
||||
populate: Populate,
|
||||
) -> Result<()> {
|
||||
AppendOnlyTracker::<S>::preopen(fs, base_path, populate)?;
|
||||
AppendOnlyTracker::<S>::preopen(fs, base_path, populate);
|
||||
AppendOnlyPages::<S>::preopen(fs, base_path, populate)
|
||||
}
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ impl<V: Blob, S: UniversalRead> BlobstoreReader<V, S> {
|
||||
) -> Result<()> {
|
||||
// schedule config file, so the config read in `open` is served from the prefetch pool
|
||||
let config_path = base_path.join(CONFIG_FILENAME);
|
||||
fs.schedule_open(&config_path, None, None)?;
|
||||
fs.schedule_open(&config_path, None, None);
|
||||
|
||||
// Don't read config now; instead, probe all modes and ignore not-found errors
|
||||
for mode in Mode::iter() {
|
||||
|
||||
@@ -69,17 +69,12 @@ impl<S: UniversalRead> AppendOnlyTracker<S> {
|
||||
|
||||
/// Schedule a prefetch of the tracker file, so a subsequent open is served from the prefetch
|
||||
/// pool.
|
||||
pub fn preopen<Fs: CachedReadFs<File = S>>(
|
||||
fs: &Fs,
|
||||
dir: &Path,
|
||||
populate: Populate,
|
||||
) -> Result<()> {
|
||||
pub fn preopen<Fs: CachedReadFs<File = S>>(fs: &Fs, dir: &Path, populate: Populate) {
|
||||
fs.schedule_open(
|
||||
&Self::tracker_file_name(dir),
|
||||
Some(Self::open_options(populate, false)),
|
||||
None,
|
||||
)?;
|
||||
Ok(())
|
||||
);
|
||||
}
|
||||
|
||||
/// Open the tracker file handle, mapping a missing file to a service error.
|
||||
|
||||
@@ -323,19 +323,14 @@ impl<S> Tracker<S> {
|
||||
|
||||
// Read operations -- only require UniversalRead
|
||||
impl<S: UniversalRead> Tracker<S> {
|
||||
pub fn preopen<Fs: CachedReadFs<File = S>>(
|
||||
fs: &Fs,
|
||||
tracker_path: &Path,
|
||||
populate: Populate,
|
||||
) -> Result<()> {
|
||||
pub fn preopen<Fs: CachedReadFs<File = S>>(fs: &Fs, tracker_path: &Path, populate: Populate) {
|
||||
// Default a lazy open to partially populating the header.
|
||||
let populate = populate.or_partial(0..size_of::<TrackerHeader>() as u64);
|
||||
fs.schedule_open(
|
||||
tracker_path,
|
||||
Some(tracker_open_options(populate, false)),
|
||||
None,
|
||||
)?;
|
||||
Ok(())
|
||||
);
|
||||
}
|
||||
|
||||
/// Open an existing PageTracker at the given path
|
||||
|
||||
@@ -58,9 +58,8 @@ impl<S: UniversalRead> ReadOnlyTracker<S> {
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn live_preload<Fs: CachedReadFs<File = S>>(&self, fs: &Fs) -> Result<()> {
|
||||
Tracker::preopen(fs, &self.path, self.populate)?;
|
||||
Ok(())
|
||||
pub(crate) fn live_preload<Fs: CachedReadFs<File = S>>(&self, fs: &Fs) {
|
||||
Tracker::preopen(fs, &self.path, self.populate);
|
||||
}
|
||||
|
||||
/// Refresh to the current on-disk state by opening a fresh storage handle
|
||||
|
||||
@@ -51,7 +51,7 @@ where
|
||||
fs: &Fs,
|
||||
path: impl AsRef<Path>,
|
||||
mut options: OpenOptions,
|
||||
) -> UioResult<()> {
|
||||
) {
|
||||
// Default a lazy open to partially populating the header + a slice of
|
||||
// the perfect-hash table, so the map can be opened without a full read.
|
||||
options.populate = options.populate.or_partial(0..HEADER_AND_BASIC_PHF_SIZE);
|
||||
|
||||
@@ -237,11 +237,11 @@ impl<Fs: UniversalReadFs> CachedReadFs for CachedFs<Fs> {
|
||||
path: &Path,
|
||||
open_arguments: Option<OpenOptions>,
|
||||
open_extra: Option<Fs::OpenExtra>,
|
||||
) -> UioResult<()> {
|
||||
) {
|
||||
let mut files_prefetched = self.files_prefetched.lock();
|
||||
|
||||
if files_prefetched.contains_key(path) {
|
||||
return Ok(());
|
||||
return;
|
||||
}
|
||||
|
||||
let open_options = open_arguments.unwrap_or(OpenOptions {
|
||||
@@ -270,21 +270,20 @@ impl<Fs: UniversalReadFs> CachedReadFs for CachedFs<Fs> {
|
||||
}
|
||||
});
|
||||
files_prefetched.insert(path.to_path_buf(), scheduled);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// TODO(uio): merge into `schedule_open`? might make it simpler to use
|
||||
fn reschedule_open(
|
||||
&self,
|
||||
path: &Path,
|
||||
open_arguments: Option<OpenOptions>,
|
||||
open_extra: Option<Fs::OpenExtra>,
|
||||
) -> UioResult<()> {
|
||||
) {
|
||||
{
|
||||
let mut files_prefetched = self.files_prefetched.lock();
|
||||
|
||||
if files_prefetched.contains_key(path) {
|
||||
return Ok(());
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if their file info is complete and didn't change.
|
||||
@@ -294,7 +293,7 @@ impl<Fs: UniversalReadFs> CachedReadFs for CachedFs<Fs> {
|
||||
.is_some_and(|(previous, current)| previous.full_eq(current))
|
||||
{
|
||||
files_prefetched.insert(path.to_path_buf(), ScheduledFile::Unchanged);
|
||||
return Ok(());
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ impl<S: UniversalRead> OneshotFile<S> {
|
||||
///
|
||||
/// A one-shot file is always read in full, so the prefetch populates it —
|
||||
/// the fetch overlaps whatever runs between the schedule and the open.
|
||||
pub fn preopen<Fs: CachedReadFs<File = S>>(fs: &Fs, path: impl AsRef<Path>) -> UioResult<()> {
|
||||
pub fn preopen<Fs: CachedReadFs<File = S>>(fs: &Fs, path: impl AsRef<Path>) {
|
||||
fs.schedule_open(
|
||||
path.as_ref(),
|
||||
Some(Self::open_options(Populate::PreferBackground)),
|
||||
|
||||
@@ -182,7 +182,7 @@ pub trait CachedReadFs: UniversalReadFs {
|
||||
path: &Path,
|
||||
open_arguments: Option<OpenOptions>,
|
||||
open_extra: Option<Self::OpenExtra>,
|
||||
) -> UioResult<()>;
|
||||
);
|
||||
|
||||
/// Schedule a prefetch for a file that has been opened already.
|
||||
///
|
||||
@@ -193,7 +193,7 @@ pub trait CachedReadFs: UniversalReadFs {
|
||||
path: &Path,
|
||||
open_arguments: Option<OpenOptions>,
|
||||
open_extra: Option<Self::OpenExtra>,
|
||||
) -> UioResult<()>;
|
||||
);
|
||||
|
||||
/// Return the file info from the current snapshot.
|
||||
fn cached_file_info(&self, path: &Path) -> Option<FileInfo>;
|
||||
|
||||
@@ -72,21 +72,21 @@ impl InMemoryBitvecFlags {
|
||||
&status_file(directory),
|
||||
Some(bitslice_open_options(Populate::PreferBackground)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
|
||||
// Bitslice
|
||||
fs.schedule_open(
|
||||
&directory.join(FLAGS_FILE),
|
||||
Some(bitslice_open_options(Populate::PreferBackground)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
}
|
||||
FlagsMode::Compact => {
|
||||
fs.schedule_open(
|
||||
&directory.join(COMPACT_FLAGS_FILE),
|
||||
Some(compact_open_options(Populate::PreferBackground)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -105,19 +105,19 @@ impl InMemoryBitvecFlags {
|
||||
&status_file(directory),
|
||||
Some(bitslice_open_options(Populate::PreferBackground)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
fs.reschedule_open(
|
||||
&directory.join(FLAGS_FILE),
|
||||
Some(bitslice_open_options(Populate::PreferBackground)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
}
|
||||
FlagsMode::Compact => {
|
||||
fs.reschedule_open(
|
||||
&directory.join(COMPACT_FLAGS_FILE),
|
||||
Some(compact_open_options(Populate::PreferBackground)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -72,14 +72,17 @@ impl<S: UniversalRead> ReadOnlyCompactFlags<S> {
|
||||
partial @ Populate::Partial(_) => partial,
|
||||
};
|
||||
|
||||
Ok(fs
|
||||
.schedule_open(
|
||||
&directory.join(COMPACT_FLAGS_FILE),
|
||||
Some(open_options(populate)),
|
||||
None,
|
||||
)
|
||||
.ok_not_found()?
|
||||
.is_some())
|
||||
if !fs.exists(&directory.join(COMPACT_FLAGS_FILE))? {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
fs.schedule_open(
|
||||
&directory.join(COMPACT_FLAGS_FILE),
|
||||
Some(open_options(populate)),
|
||||
None,
|
||||
);
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Open persisted flags read-only, retaining the bitmask handle for the
|
||||
@@ -129,7 +132,7 @@ impl<S: UniversalRead> ReadOnlyCompactFlags<S> {
|
||||
Ok(self.bitmap.get_or_init(|| bitmap))
|
||||
}
|
||||
|
||||
pub fn live_preload<Fs: CachedReadFs<File = S>>(&self, cached_fs: &Fs) -> OperationResult<()> {
|
||||
pub fn live_preload<Fs: CachedReadFs<File = S>>(&self, cached_fs: &Fs) {
|
||||
let directory = self.directory.as_path();
|
||||
let populate = if self.bitmap.get().is_some() {
|
||||
Populate::PreferBackground
|
||||
@@ -140,8 +143,7 @@ impl<S: UniversalRead> ReadOnlyCompactFlags<S> {
|
||||
&directory.join(COMPACT_FLAGS_FILE),
|
||||
Some(open_options(populate)),
|
||||
None,
|
||||
)?;
|
||||
Ok(())
|
||||
);
|
||||
}
|
||||
|
||||
/// Refresh to the current on-disk state.
|
||||
|
||||
@@ -80,7 +80,8 @@ impl<S: UniversalRead> LiveReload for ReadOnlyFlags<S> {
|
||||
match self {
|
||||
ReadOnlyFlags::Dynamic(dynamic) => dynamic.live_preload(cached_fs),
|
||||
ReadOnlyFlags::Compact(compact) => compact.live_preload(cached_fs),
|
||||
}
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn live_reload<Fs: UniversalReadFs<File = Self::File>>(
|
||||
|
||||
@@ -84,24 +84,22 @@ impl<S: UniversalRead> ReadOnlyRoaringFlags<S> {
|
||||
populate: Populate,
|
||||
) -> OperationResult<bool> {
|
||||
// Status file.
|
||||
if fs
|
||||
.schedule_open(
|
||||
&status_file(directory),
|
||||
Some(open_options(Populate::PreferBackground)),
|
||||
None,
|
||||
)
|
||||
.ok_not_found()?
|
||||
.is_none()
|
||||
{
|
||||
let status_path = status_file(directory);
|
||||
if !fs.exists(&status_path)? {
|
||||
return Ok(false);
|
||||
}
|
||||
fs.schedule_open(
|
||||
&status_path,
|
||||
Some(open_options(Populate::PreferBackground)),
|
||||
None,
|
||||
);
|
||||
|
||||
// Bitslice
|
||||
fs.schedule_open(
|
||||
&directory.join(FLAGS_FILE),
|
||||
Some(open_options(populate)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
@@ -160,7 +158,7 @@ impl<S: UniversalRead> ReadOnlyRoaringFlags<S> {
|
||||
Ok(self.bitmap.get_or_init(|| bitmap))
|
||||
}
|
||||
|
||||
pub fn live_preload<Fs: CachedReadFs<File = S>>(&self, cached_fs: &Fs) -> OperationResult<()> {
|
||||
pub fn live_preload<Fs: CachedReadFs<File = S>>(&self, cached_fs: &Fs) {
|
||||
let directory = self.directory.as_path();
|
||||
|
||||
// Bitslice
|
||||
@@ -174,16 +172,14 @@ impl<S: UniversalRead> ReadOnlyRoaringFlags<S> {
|
||||
&directory.join(FLAGS_FILE),
|
||||
Some(open_options(populate)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
|
||||
// Status file.
|
||||
cached_fs.reschedule_open(
|
||||
&status_file(directory),
|
||||
Some(open_options(Populate::PreferBackground)),
|
||||
None,
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
);
|
||||
}
|
||||
|
||||
/// Refresh to the current on-disk state.
|
||||
|
||||
@@ -47,12 +47,12 @@ impl<S: UniversalRead> ReadOnlyDiskIdTracker<S> {
|
||||
}
|
||||
|
||||
let options = Self::open_options();
|
||||
fs.schedule_open(&version_mapping_path(segment_path), Some(options), None)?;
|
||||
fs.schedule_open(&version_mapping_path(segment_path), Some(options), None);
|
||||
fs.schedule_open(
|
||||
&deleted_path(segment_path),
|
||||
Some(Self::deleted_open_options()),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ impl<S: UniversalRead> ReadOnlyDiskIdTracker<S> {
|
||||
&deleted_path(&self.path),
|
||||
Some(Self::deleted_open_options()),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -58,8 +58,8 @@ impl<S: UniversalRead> DiskMappingReader<S> {
|
||||
|
||||
let options = Self::open_options();
|
||||
|
||||
fs.schedule_open(&i2e_path, Some(options), None)?;
|
||||
fs.schedule_open(&e2i_path(segment_path), Some(options), None)?;
|
||||
fs.schedule_open(&i2e_path, Some(options), None);
|
||||
fs.schedule_open(&e2i_path(segment_path), Some(options), None);
|
||||
fs.schedule_open(
|
||||
&is_uuid_path(segment_path),
|
||||
Some(OpenOptions {
|
||||
@@ -69,7 +69,7 @@ impl<S: UniversalRead> DiskMappingReader<S> {
|
||||
..Self::is_uuid_open_options()
|
||||
}),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
@@ -29,7 +29,8 @@ impl<S: UniversalRead> ReadOnlyIdTrackerEnum<S> {
|
||||
if ReadOnlyImmutableIdTracker::try_preopen(fs, segment_path)? {
|
||||
return Ok(());
|
||||
}
|
||||
ReadOnlyAppendableIdTracker::preopen(fs, segment_path)
|
||||
ReadOnlyAppendableIdTracker::preopen(fs, segment_path);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Detect the persisted id-tracker format and load it, by *attempting* each
|
||||
|
||||
@@ -43,9 +43,9 @@ impl<S: UniversalRead> ReadOnlyImmutableIdTracker<S> {
|
||||
|
||||
let options = Self::open_options();
|
||||
|
||||
fs.schedule_open(&deleted_path(segment_path), Some(options), None)?;
|
||||
fs.schedule_open(&version_mapping_path(segment_path), Some(options), None)?;
|
||||
fs.schedule_open(&mappings_path(segment_path), Some(options), None)?;
|
||||
fs.schedule_open(&deleted_path(segment_path), Some(options), None);
|
||||
fs.schedule_open(&version_mapping_path(segment_path), Some(options), None);
|
||||
fs.schedule_open(&mappings_path(segment_path), Some(options), None);
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ use crate::id_tracker::mutable_id_tracker::read_only::LiveReloadResult;
|
||||
impl<S: UniversalRead> ReadOnlyImmutableIdTracker<S> {
|
||||
/// Stage the fresh deleted-bitslice handle [`live_reload`](Self::live_reload) swaps in.
|
||||
pub fn live_preload(&self, fs: &impl CachedReadFs<File = S>) -> OperationResult<()> {
|
||||
fs.reschedule_open(&deleted_path(&self.path), Some(Self::open_options()), None)?;
|
||||
fs.reschedule_open(&deleted_path(&self.path), Some(Self::open_options()), None);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -30,15 +30,11 @@ impl<S: UniversalRead> ReadOnlyAppendableIdTracker<S> {
|
||||
/// Either file may not exist yet — the writer only creates them once it
|
||||
/// flushes the first point, and [`open`](Self::open) treats a missing file
|
||||
/// as an empty storage — so absence is tolerated here too.
|
||||
pub fn preopen(fs: &impl CachedReadFs<File = S>, segment_path: &Path) -> OperationResult<()> {
|
||||
pub fn preopen(fs: &impl CachedReadFs<File = S>, segment_path: &Path) {
|
||||
let options = Self::open_options();
|
||||
|
||||
fs.schedule_open(&mappings_path(segment_path), Some(options), None)
|
||||
.ok_not_found()?;
|
||||
fs.schedule_open(&versions_path(segment_path), Some(options), None)
|
||||
.ok_not_found()?;
|
||||
|
||||
Ok(())
|
||||
fs.schedule_open(&mappings_path(segment_path), Some(options), None);
|
||||
fs.schedule_open(&versions_path(segment_path), Some(options), None);
|
||||
}
|
||||
|
||||
/// Open a read-only view over the appendable ID tracker data at `segment_path`, threading every
|
||||
|
||||
@@ -69,12 +69,13 @@ impl<S: UniversalRead> ReadOnlyAppendableIdTracker<S> {
|
||||
(&self.mappings_file, mappings_path(&self.segment_path)),
|
||||
] {
|
||||
match file {
|
||||
Some(file) => file
|
||||
.schedule_reopen(|p| fs.cached_file_info(p))
|
||||
.ok_not_found()?,
|
||||
None => fs
|
||||
.schedule_open(&path, Some(options), None)
|
||||
.ok_not_found()?,
|
||||
Some(file) => {
|
||||
file.schedule_reopen(|p| fs.cached_file_info(p))
|
||||
.ok_not_found()?;
|
||||
}
|
||||
None => {
|
||||
fs.schedule_open(&path, Some(options), None);
|
||||
}
|
||||
};
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -172,19 +172,11 @@ pub(super) fn preopen_deleted_mask<S>(
|
||||
dir: &Path,
|
||||
legacy_file: &str,
|
||||
options: OpenOptions,
|
||||
) -> OperationResult<()>
|
||||
where
|
||||
) where
|
||||
S: UniversalRead,
|
||||
{
|
||||
if fs
|
||||
.schedule_open(&deleted_mask_path(dir), Some(options), None)
|
||||
.ok_not_found()?
|
||||
.is_some()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
fs.schedule_open(&dir.join(legacy_file), Some(options), None)?;
|
||||
Ok(())
|
||||
fs.schedule_open(&deleted_mask_path(dir), Some(options), None);
|
||||
fs.schedule_open(&dir.join(legacy_file), Some(options), None);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
+14
-18
@@ -9,8 +9,8 @@ use common::mmap::{Advice, AdviceSetting, MmapSlice};
|
||||
use common::persisted_hashmap::{READ_ENTRY_OVERHEAD, UniversalHashMap, serialize_hashmap};
|
||||
use common::types::PointOffsetType;
|
||||
use common::universal_io::{
|
||||
CachedReadFs, MmapFile, OkNotFound, OpenOptions, Populate, ReadRange, TypedStorage,
|
||||
UniversalRead, UniversalReadFs, UserData,
|
||||
CachedReadFs, MmapFile, OpenOptions, Populate, ReadRange, TypedStorage, UniversalRead,
|
||||
UniversalReadFs, UserData,
|
||||
};
|
||||
use on_disk_postings::OnDiskPostings;
|
||||
use types::ZerocopyPostingValue;
|
||||
@@ -156,35 +156,31 @@ impl<S: UniversalRead> OnDiskInvertedIndex<S> {
|
||||
) -> OperationResult<bool> {
|
||||
// Postings.
|
||||
let postings_path = path.join(POSTINGS_FILE);
|
||||
if fs
|
||||
.schedule_open(
|
||||
&postings_path,
|
||||
Some(Self::open_options(
|
||||
populate,
|
||||
AdviceSetting::Advice(Advice::Normal),
|
||||
)),
|
||||
None,
|
||||
)
|
||||
.ok_not_found()?
|
||||
.is_none()
|
||||
{
|
||||
// If postings don't exist, assume the index doesn't exist on disk
|
||||
if !fs.exists(&postings_path)? {
|
||||
return Ok(false);
|
||||
}
|
||||
fs.schedule_open(
|
||||
&postings_path,
|
||||
Some(Self::open_options(
|
||||
populate,
|
||||
AdviceSetting::Advice(Advice::Normal),
|
||||
)),
|
||||
None,
|
||||
);
|
||||
|
||||
// Vocabulary
|
||||
UniversalHashMap::<str, TokenId, S>::preopen(
|
||||
fs,
|
||||
&path.join(VOCAB_FILE),
|
||||
Self::open_options(populate, AdviceSetting::Global),
|
||||
)?;
|
||||
);
|
||||
|
||||
// Point to tokens count
|
||||
fs.schedule_open(
|
||||
&path.join(POINT_TO_TOKENS_COUNT_FILE),
|
||||
Some(Self::open_options(populate, AdviceSetting::Global)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
|
||||
// "No tokens" mask
|
||||
preopen_deleted_mask(
|
||||
@@ -192,7 +188,7 @@ impl<S: UniversalRead> OnDiskInvertedIndex<S> {
|
||||
path,
|
||||
DELETED_POINTS_FILE,
|
||||
Self::open_options(Populate::PreferBackground, AdviceSetting::Global),
|
||||
)?;
|
||||
);
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
@@ -165,32 +165,25 @@ impl<S: UniversalRead> OnDiskGeoIndex<S> {
|
||||
) -> OperationResult<bool> {
|
||||
// Stats
|
||||
let stats_path = path.join(STATS_PATH);
|
||||
if fs
|
||||
.schedule_open(&stats_path, None, None)
|
||||
.ok_not_found()?
|
||||
.is_none()
|
||||
{
|
||||
if !fs.exists(&stats_path)? {
|
||||
// If stats file doesn't exist, assume the index doesn't exist on disk
|
||||
return Ok(false);
|
||||
}
|
||||
fs.schedule_open(&stats_path, None, None);
|
||||
|
||||
// Geohash counts, points map, and point-id list
|
||||
let options = Self::open_options(populate);
|
||||
fs.schedule_open(&path.join(COUNTS_PER_HASH), Some(options), None)?;
|
||||
fs.schedule_open(&path.join(POINTS_MAP), Some(options), None)?;
|
||||
fs.schedule_open(&path.join(POINTS_MAP_IDS), Some(options), None)?;
|
||||
fs.schedule_open(&path.join(COUNTS_PER_HASH), Some(options), None);
|
||||
fs.schedule_open(&path.join(POINTS_MAP), Some(options), None);
|
||||
fs.schedule_open(&path.join(POINTS_MAP_IDS), Some(options), None);
|
||||
|
||||
// Block indexes over the two sorted arrays; optional, absent on old
|
||||
// segments
|
||||
let _ = fs
|
||||
.schedule_open(&path.join(COUNTS_PER_HASH_BLOCK_INDEX), None, None)
|
||||
.ok_not_found()?;
|
||||
let _ = fs
|
||||
.schedule_open(&path.join(POINTS_MAP_BLOCK_INDEX), None, None)
|
||||
.ok_not_found()?;
|
||||
fs.schedule_open(&path.join(COUNTS_PER_HASH_BLOCK_INDEX), None, None);
|
||||
fs.schedule_open(&path.join(POINTS_MAP_BLOCK_INDEX), None, None);
|
||||
|
||||
// Point to values
|
||||
OnDiskPointToValues::<GeoPoint, S>::preopen(fs, path, populate)?;
|
||||
OnDiskPointToValues::<GeoPoint, S>::preopen(fs, path, populate);
|
||||
|
||||
// "No values" mask
|
||||
preopen_deleted_mask(
|
||||
@@ -198,7 +191,7 @@ impl<S: UniversalRead> OnDiskGeoIndex<S> {
|
||||
path,
|
||||
DELETED_PATH,
|
||||
Self::open_options(Populate::PreferBackground),
|
||||
)?;
|
||||
);
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
@@ -53,10 +53,9 @@ impl<T: Numericable + Serialize + DeserializeOwned> Histogram<T> {
|
||||
}
|
||||
|
||||
/// Schedule background prefetch of the two files [`open`](Self::open) reads.
|
||||
pub fn preopen(fs: &impl CachedReadFs, path: &Path) -> OperationResult<()> {
|
||||
fs.schedule_open(&path.join(CONFIG_PATH), None, None)?;
|
||||
fs.schedule_open(&path.join(BORDERS_PATH), None, None)?;
|
||||
Ok(())
|
||||
pub fn preopen(fs: &impl CachedReadFs, path: &Path) {
|
||||
fs.schedule_open(&path.join(CONFIG_PATH), None, None);
|
||||
fs.schedule_open(&path.join(BORDERS_PATH), None, None);
|
||||
}
|
||||
|
||||
pub fn open<Fs: UniversalReadFs>(fs: &Fs, path: &Path) -> OperationResult<Self> {
|
||||
|
||||
@@ -46,14 +46,11 @@ where
|
||||
) -> OperationResult<bool> {
|
||||
// Config
|
||||
let config_path = path.join(CONFIG_PATH);
|
||||
if fs
|
||||
.schedule_open(&config_path, None, None)
|
||||
.ok_not_found()?
|
||||
.is_none()
|
||||
{
|
||||
if !fs.exists(&config_path)? {
|
||||
// If config doesn't exist, assume the index doesn't exist on disk
|
||||
return Ok(false);
|
||||
}
|
||||
fs.schedule_open(&config_path, None, None);
|
||||
|
||||
// Value to points
|
||||
let hashmap_path = path.join(HASHMAP_PATH);
|
||||
@@ -61,10 +58,10 @@ where
|
||||
fs,
|
||||
&hashmap_path,
|
||||
Self::open_options(populate),
|
||||
)?;
|
||||
);
|
||||
|
||||
// Point to values
|
||||
OnDiskPointToValues::<N, S>::preopen(fs, path, populate)?;
|
||||
OnDiskPointToValues::<N, S>::preopen(fs, path, populate);
|
||||
|
||||
// Prefix index
|
||||
PrefixIndex::preopen(fs, path, populate)?;
|
||||
@@ -75,7 +72,7 @@ where
|
||||
path,
|
||||
DELETED_PATH,
|
||||
Self::open_options(Populate::PreferBackground),
|
||||
)?;
|
||||
);
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
@@ -78,7 +78,7 @@ impl<S: UniversalRead> PrefixIndex<S> {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
fs.schedule_open(&file_path, Some(Self::prefix_open_options(populate)), None)?;
|
||||
fs.schedule_open(&file_path, Some(Self::prefix_open_options(populate)), None);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -119,29 +119,23 @@ where
|
||||
) -> OperationResult<bool> {
|
||||
// Config
|
||||
let config_path = path.join(CONFIG_PATH);
|
||||
if fs
|
||||
.schedule_open(&config_path, None, None)
|
||||
.ok_not_found()?
|
||||
.is_none()
|
||||
{
|
||||
// If config doesn't exist, assume the index doesn't exist on disk
|
||||
if !fs.exists(&config_path)? {
|
||||
return Ok(false);
|
||||
}
|
||||
fs.schedule_open(&config_path, None, None);
|
||||
|
||||
// Histogram
|
||||
Histogram::<T>::preopen(fs, path)?;
|
||||
Histogram::<T>::preopen(fs, path);
|
||||
|
||||
// Value pairs
|
||||
let pairs_path = path.join(PAIRS_PATH);
|
||||
fs.schedule_open(&pairs_path, Some(Self::open_options(populate)), None)?;
|
||||
fs.schedule_open(&pairs_path, Some(Self::open_options(populate)), None);
|
||||
|
||||
// Block index over the value pairs; optional, absent on old segments
|
||||
let _ = fs
|
||||
.schedule_open(&path.join(PAIRS_BLOCK_INDEX_PATH), None, None)
|
||||
.ok_not_found()?;
|
||||
fs.schedule_open(&path.join(PAIRS_BLOCK_INDEX_PATH), None, None);
|
||||
|
||||
// Point to values
|
||||
OnDiskPointToValues::<T, S>::preopen(fs, path, populate)?;
|
||||
OnDiskPointToValues::<T, S>::preopen(fs, path, populate);
|
||||
|
||||
// "No values" mask
|
||||
preopen_deleted_mask(
|
||||
@@ -149,7 +143,7 @@ where
|
||||
path,
|
||||
DELETED_PATH,
|
||||
Self::open_options(Populate::PreferBackground),
|
||||
)?;
|
||||
);
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
@@ -177,16 +177,9 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn preopen(
|
||||
fs: &impl CachedReadFs<File = S>,
|
||||
dir: &Path,
|
||||
populate: Populate,
|
||||
) -> OperationResult<()> {
|
||||
pub fn preopen(fs: &impl CachedReadFs<File = S>, dir: &Path, populate: Populate) {
|
||||
let file_name = dir.join(POINT_TO_VALUES_PATH);
|
||||
|
||||
fs.schedule_open(&file_name, Some(Self::open_options(populate)), None)?;
|
||||
|
||||
Ok(())
|
||||
fs.schedule_open(&file_name, Some(Self::open_options(populate)), None);
|
||||
}
|
||||
|
||||
pub fn open(
|
||||
|
||||
@@ -116,7 +116,7 @@ impl<S: UniversalRead> HnswGraph<S> {
|
||||
dir: &Path,
|
||||
residency: GraphLinksResidency,
|
||||
) -> OperationResult<()> {
|
||||
fs.schedule_open(&GraphLayers::get_path(dir), None, None)?;
|
||||
fs.schedule_open(&GraphLayers::get_path(dir), None, None);
|
||||
let Some(format) = GraphLayers::probe_links_format(fs, dir)? else {
|
||||
return Ok(());
|
||||
};
|
||||
@@ -129,7 +129,7 @@ impl<S: UniversalRead> HnswGraph<S> {
|
||||
&GraphLayers::get_links_path(dir, format),
|
||||
Some(options),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use atomic_refcell::AtomicRefCell;
|
||||
use common::universal_io::{CachedReadFs, OkNotFound, Populate, UniversalRead, UniversalReadFs};
|
||||
use common::universal_io::{CachedReadFs, Populate, UniversalRead, UniversalReadFs};
|
||||
use once_cell::sync::OnceCell;
|
||||
|
||||
use super::read_view::HNSWIndexReadView;
|
||||
@@ -101,9 +101,8 @@ impl<S: UniversalReadExt> ReadOnlyHNSWIndex<S> {
|
||||
hnsw_config: &HnswConfig,
|
||||
populate_override: Option<Populate>,
|
||||
) -> OperationResult<()> {
|
||||
// Graph config; may legitimately be absent (`open` derives defaults).
|
||||
fs.schedule_open(&HnswGraphConfig::get_config_path(path), None, None)
|
||||
.ok_not_found()?;
|
||||
// Graph config
|
||||
fs.schedule_open(&HnswGraphConfig::get_config_path(path), None, None);
|
||||
|
||||
// Graph data and links
|
||||
let (_memory, residency) = graph_residency(hnsw_config, populate_override);
|
||||
|
||||
@@ -9,7 +9,7 @@ use common::counter::hardware_counter::HardwareCounterCell;
|
||||
use common::low_memory::low_memory_mode;
|
||||
use common::storage_version::VERSION_FILE;
|
||||
use common::types::{ScoredPointOffset, TelemetryDetail};
|
||||
use common::universal_io::{CachedReadFs, OkNotFound, Populate, UniversalReadFs};
|
||||
use common::universal_io::{CachedReadFs, Populate, UniversalReadFs};
|
||||
use half::f16;
|
||||
use sparse::common::types::{DimId, QuantizedU8};
|
||||
use sparse::index::inverted_index::inverted_index_compressed_immutable_ram::InvertedIndexCompressedImmutableRam;
|
||||
@@ -132,15 +132,8 @@ impl<S: UniversalReadExt + 'static> VectorIndexReadEnum<S> {
|
||||
SparseIndexType::ImmutableRam | SparseIndexType::Mmap => {}
|
||||
}
|
||||
|
||||
// Sparse index config; `open_sparse` reads it off the parked handle.
|
||||
let config_path = SparseIndexConfig::get_config_path(path);
|
||||
if fs
|
||||
.schedule_open(&config_path, None, None)
|
||||
.ok_not_found()?
|
||||
.is_none()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
fs.schedule_open(&config_path, None, None);
|
||||
|
||||
// The effective placement (structural index type refined by the
|
||||
// `memory` parameter, degraded by low-memory mode — which is also
|
||||
@@ -165,10 +158,10 @@ impl<S: UniversalReadExt + 'static> VectorIndexReadEnum<S> {
|
||||
inverted_index_compressed_mmap::preopen(fs, path, populate)?;
|
||||
|
||||
// Version check
|
||||
fs.schedule_open(&path.join(VERSION_FILE), None, None)?;
|
||||
fs.schedule_open(&path.join(VERSION_FILE), None, None);
|
||||
|
||||
// Indices tracker
|
||||
fs.schedule_open(&IndicesTracker::file_path(path), None, None)?;
|
||||
fs.schedule_open(&IndicesTracker::file_path(path), None, None);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -5,9 +5,7 @@ use std::sync::Arc;
|
||||
use atomic_refcell::AtomicRefCell;
|
||||
use common::storage_version::{StorageVersion, VERSION_FILE};
|
||||
use common::types::PointOffsetType;
|
||||
use common::universal_io::{
|
||||
CachedFs, CachedReadFs, OkNotFound, Populate, UniversalReadFs, read_json_via,
|
||||
};
|
||||
use common::universal_io::{CachedFs, CachedReadFs, Populate, UniversalReadFs, read_json_via};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{ReadOnlySegment, ReadOnlyVectorData};
|
||||
@@ -45,19 +43,15 @@ fn build_cached_fs<Fs: UniversalReadFs>(
|
||||
|
||||
// Absence is tolerated here: the subsequent read reports it gracefully.
|
||||
for file_name in [VERSION_FILE, SEGMENT_STATE_FILE] {
|
||||
cached_fs
|
||||
.schedule_open(&segment_path.join(file_name), None, None)
|
||||
.ok_not_found()?;
|
||||
cached_fs.schedule_open(&segment_path.join(file_name), None, None);
|
||||
}
|
||||
|
||||
// Payload index config
|
||||
cached_fs
|
||||
.schedule_open(
|
||||
&PayloadConfig::get_config_path(&get_payload_index_path(segment_path)),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.ok_not_found()?;
|
||||
cached_fs.schedule_open(
|
||||
&PayloadConfig::get_config_path(&get_payload_index_path(segment_path)),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
cached_fs.cache_file_info()?;
|
||||
|
||||
|
||||
@@ -6,8 +6,7 @@ use atomic_refcell::AtomicRefCell;
|
||||
use common::storage_version::{StorageVersion, VERSION_FILE};
|
||||
use common::types::PointOffsetType;
|
||||
use common::universal_io::{
|
||||
CachedFs, CachedReadFs, OkNotFound as _, Populate, UniversalRead, UniversalReadFs,
|
||||
read_json_via,
|
||||
CachedFs, CachedReadFs, Populate, UniversalRead, UniversalReadFs, read_json_via,
|
||||
};
|
||||
|
||||
use super::LookupSegment;
|
||||
@@ -46,9 +45,7 @@ fn build_cached_fs<Fs: UniversalReadFs>(
|
||||
|
||||
// Absence is tolerated here: the subsequent read reports it gracefully.
|
||||
for file_name in [VERSION_FILE, SEGMENT_STATE_FILE] {
|
||||
cached_fs
|
||||
.schedule_open(&segment_path.join(file_name), None, None)
|
||||
.ok_not_found()?;
|
||||
cached_fs.schedule_open(&segment_path.join(file_name), None, None);
|
||||
}
|
||||
|
||||
cached_fs.cache_file_info()?;
|
||||
|
||||
@@ -21,10 +21,10 @@ impl<T: bytemuck::Pod + Send, S: UniversalRead> ReadOnlyChunkedVectors<T, S> {
|
||||
populate: Populate,
|
||||
) -> OperationResult<()> {
|
||||
// Config file
|
||||
fs.schedule_open(&config_file(directory), None, None)?;
|
||||
fs.schedule_open(&config_file(directory), None, None);
|
||||
|
||||
// Status file
|
||||
fs.schedule_open(&status_file(directory), None, None)?;
|
||||
fs.schedule_open(&status_file(directory), None, None);
|
||||
|
||||
// Chunks
|
||||
preopen_chunks(fs, directory, advice, populate)?;
|
||||
@@ -127,7 +127,7 @@ fn preopen_chunks(
|
||||
&listed.path,
|
||||
Some(chunk_open_options(advice, populate, false)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -16,7 +16,7 @@ impl<T: bytemuck::Pod + Send, S: UniversalRead> LiveReload for ReadOnlyChunkedVe
|
||||
|
||||
fn live_preload<Fs: CachedReadFs<File = S>>(&self, fs: &Fs) -> OperationResult<()> {
|
||||
// Status is the change signal, let reload skip reloading if this didn't change.
|
||||
fs.reschedule_open(&status_file(&self.directory), None, None)?;
|
||||
fs.reschedule_open(&status_file(&self.directory), None, None);
|
||||
|
||||
let num_files = list_chunk_files(fs, &self.directory)?.len();
|
||||
|
||||
@@ -29,7 +29,7 @@ impl<T: bytemuck::Pod + Send, S: UniversalRead> LiveReload for ReadOnlyChunkedVe
|
||||
&chunk_name(&self.directory, last_chunk),
|
||||
Some(chunk_open_options(self.advice, self.populate, false)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
last_chunk + 1
|
||||
} else {
|
||||
last_chunk
|
||||
@@ -41,7 +41,7 @@ impl<T: bytemuck::Pod + Send, S: UniversalRead> LiveReload for ReadOnlyChunkedVe
|
||||
&chunk_name(&self.directory, chunk_id),
|
||||
Some(chunk_open_options(self.advice, self.populate, false)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -58,7 +58,7 @@ impl<T: PrimitiveVectorElement, S: UniversalRead> ImmutableDenseVectorData<T, S>
|
||||
populate: Populate,
|
||||
) -> OperationResult<()> {
|
||||
// Vector data
|
||||
fs.schedule_open(vectors_path, Some(Self::open_options(populate)), None)?;
|
||||
fs.schedule_open(vectors_path, Some(Self::open_options(populate)), None);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ impl<T: PrimitiveVectorElement, S: UniversalRead> ReadOnlyImmutableDenseVectorSt
|
||||
&path.join(DELETED_PATH),
|
||||
Some(bitslice_open_options(Populate::PreferBackground)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -64,7 +64,7 @@ impl MultivectorOffsetsStorageRam {
|
||||
path,
|
||||
Some(Self::open_options(Populate::PreferBackground)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -176,13 +176,8 @@ impl<S: UniversalRead> MultivectorOffsetsStorageMmap<S> {
|
||||
///
|
||||
/// The offsets are read lazily; `populate` warms the parked handle for
|
||||
/// the `cached` memory placement.
|
||||
pub fn preopen(
|
||||
fs: &impl CachedReadFs<File = S>,
|
||||
path: &Path,
|
||||
populate: Populate,
|
||||
) -> OperationResult<()> {
|
||||
fs.schedule_open(path, Some(Self::open_options(populate)), None)?;
|
||||
Ok(())
|
||||
pub fn preopen(fs: &impl CachedReadFs<File = S>, path: &Path, populate: Populate) {
|
||||
fs.schedule_open(path, Some(Self::open_options(populate)), None);
|
||||
}
|
||||
|
||||
/// Open the offsets file read-only through the provided [`UniversalRead`] filesystem.
|
||||
|
||||
@@ -29,12 +29,8 @@ impl QuantizedRamStorage {
|
||||
/// Schedule background prefetch of the data file [`Self::from_file`] reads.
|
||||
///
|
||||
/// The load reads the whole file, so the prefetch populates it.
|
||||
pub fn preopen<S: UniversalRead>(
|
||||
fs: &impl CachedReadFs<File = S>,
|
||||
path: &Path,
|
||||
) -> OperationResult<()> {
|
||||
OneshotFile::<S>::preopen(fs, path)?;
|
||||
Ok(())
|
||||
pub fn preopen<S: UniversalRead>(fs: &impl CachedReadFs<File = S>, path: &Path) {
|
||||
OneshotFile::<S>::preopen(fs, path)
|
||||
}
|
||||
|
||||
/// Load all quantized vectors into RAM through the provided [`UniversalRead`]
|
||||
|
||||
@@ -178,13 +178,8 @@ impl<S: UniversalRead> QuantizedStorage<S> {
|
||||
///
|
||||
/// The storage reads lazily through its mmap-style handle; `populate`
|
||||
/// warms the parked handle for the `cached` memory placement.
|
||||
pub fn preopen(
|
||||
fs: &impl CachedReadFs<File = S>,
|
||||
path: &Path,
|
||||
populate: Populate,
|
||||
) -> OperationResult<()> {
|
||||
fs.schedule_open(path, Some(Self::open_options(populate)), None)?;
|
||||
Ok(())
|
||||
pub fn preopen(fs: &impl CachedReadFs<File = S>, path: &Path, populate: Populate) {
|
||||
fs.schedule_open(path, Some(Self::open_options(populate)), None)
|
||||
}
|
||||
|
||||
pub fn from_file(
|
||||
|
||||
@@ -64,16 +64,13 @@ impl<S: UniversalRead> ReadOnlyQuantizedVectors<S> {
|
||||
|
||||
// Config; `open` reads it off the parked handle.
|
||||
let config_path = QuantizedVectors::get_config_path(path);
|
||||
if fs
|
||||
.schedule_open(&config_path, None, None)
|
||||
.ok_not_found()?
|
||||
.is_none()
|
||||
{
|
||||
if !fs.exists(&config_path)? {
|
||||
return Ok(());
|
||||
}
|
||||
fs.schedule_open(&config_path, None, None);
|
||||
|
||||
// Per-method metadata
|
||||
fs.schedule_open(&QuantizedVectors::get_meta_path(path), None, None)?;
|
||||
fs.schedule_open(&QuantizedVectors::get_meta_path(path), None, None);
|
||||
|
||||
let placement = QuantizedVectors::memory_placement(
|
||||
quantization_config.memory_placement(),
|
||||
@@ -119,9 +116,9 @@ impl<S: UniversalRead> ReadOnlyQuantizedVectors<S> {
|
||||
QuantizedVectorsStorageType::Immutable => {
|
||||
if fs.exists(&data_path)? {
|
||||
match placement {
|
||||
Memory::Pinned => QuantizedRamStorage::preopen(fs, &data_path)?,
|
||||
Memory::Pinned => QuantizedRamStorage::preopen(fs, &data_path),
|
||||
Memory::Cached | Memory::Cold => {
|
||||
QuantizedStorage::<S>::preopen(fs, &data_path, data_populate)?
|
||||
QuantizedStorage::<S>::preopen(fs, &data_path, data_populate)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -135,7 +132,7 @@ impl<S: UniversalRead> ReadOnlyQuantizedVectors<S> {
|
||||
fs,
|
||||
&offsets_path,
|
||||
populate,
|
||||
)?
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ impl<S: UniversalRead> ReadOnlyImmutableTurboVectorStorage<S> {
|
||||
path: &Path,
|
||||
populate: Populate,
|
||||
) -> OperationResult<()> {
|
||||
QuantizedStorage::<S>::preopen(fs, &path.join(VECTORS_PATH), populate)?;
|
||||
QuantizedStorage::<S>::preopen(fs, &path.join(VECTORS_PATH), populate);
|
||||
InMemoryBitvecFlags::preopen(fs, &path.join(DELETED_DIR_PATH))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -61,14 +61,14 @@ fn index_open_options(populate: Populate) -> OpenOptions {
|
||||
/// (mmap).
|
||||
pub fn preopen(fs: &impl CachedReadFs, path: &Path, populate: Populate) -> UioResult<()> {
|
||||
// Config
|
||||
fs.schedule_open(&index_config_file_path(path), None, None)?;
|
||||
fs.schedule_open(&index_config_file_path(path), None, None);
|
||||
|
||||
// Index data
|
||||
fs.schedule_open(
|
||||
&index_file_path(path),
|
||||
Some(index_open_options(populate)),
|
||||
None,
|
||||
)?;
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user