[LiveReload] Preload indexes (#10229)

* impl live_preload for payload indexes

enable live_preload for bool and null indexes

* (not) impl live_preload for `VectorIndexReadEnum`

* impl live_preload for `ReadOnlyPayloadStorage`
This commit is contained in:
Luis Cossío
2026-09-03 12:42:24 +02:00
committed by timvisee
parent 5f952a62d9
commit 76d18c319c
24 changed files with 175 additions and 33 deletions
@@ -1,7 +1,7 @@
use common::counter::hardware_counter::HardwareCounterCell;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::UniversalReadFs;
use common::universal_io::{CachedReadFs, UniversalReadFs};
use super::ReadOnlyBoolIndex;
use crate::common::operation_error::OperationResult;
@@ -11,6 +11,12 @@ use crate::index::field_index::LiveReload;
impl<S: UniversalReadExt> LiveReload for ReadOnlyBoolIndex<S> {
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, cached_fs: &Fs) -> OperationResult<()> {
self.storage.trues_flags.live_preload(cached_fs)?;
self.storage.falses_flags.live_preload(cached_fs)?;
Ok(())
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
fs: &Fs,
@@ -124,6 +124,7 @@ mod tests {
CachedFs, CachedReadFs, MmapFile, Populate, ReadOnly, UniversalRead, UniversalReadFileOps,
};
use itertools::Itertools as _;
use rstest::rstest;
use serde_json::json;
use tempfile::TempDir;
@@ -230,17 +231,16 @@ mod tests {
/// in-memory bitmap in place, or — when nothing has materialized it yet —
/// leave it alone and let the eventual scan read the updated file. Both must
/// land on the same state, so both are exercised.
#[test]
fn live_reload_matches_fresh_open_materialized() {
live_reload_matches_fresh_open(true);
}
#[rstest]
#[case(true, false)]
#[case(true, true)]
#[case(false, false)]
fn live_reload_matches_fresh_open(
#[case] materialize_before_reload: bool,
#[case] preload: bool,
) {
use common::universal_io::{CachedFs, CachedReadFs};
#[test]
fn live_reload_matches_fresh_open_lazy() {
live_reload_matches_fresh_open(false);
}
fn live_reload_matches_fresh_open(materialize_before_reload: bool) {
let dir = TempDir::with_prefix("read_only_bool_index_live_reload").unwrap();
let hw_counter = HardwareCounterCell::new();
@@ -289,9 +289,16 @@ mod tests {
index.add_point(1100, &[&json!(true)], &hw_counter).unwrap();
index.flusher()().unwrap();
// `CachedFs` passes opens through until a snapshot is taken, so the
// non-preload path reloads over it untouched.
let mut cached_fs = CachedFs::new(fs.clone(), dir.path()).unwrap();
if preload {
cached_fs.cache_file_info().unwrap();
reloaded.live_preload(&cached_fs).unwrap();
}
reloaded
.live_reload(
&fs,
&cached_fs,
&SortedSlice::new(&[1, 2]).unwrap(),
&SortedSlice::new(&[6, 7, 1100]).unwrap(),
&hw_counter,
@@ -7,7 +7,7 @@ use std::path::PathBuf;
use common::counter::hardware_counter::HardwareCounterCell;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::UniversalReadFs;
use common::universal_io::{CachedReadFs, UniversalReadFs};
pub(crate) use crate::common::live_reload::LiveReload;
use crate::common::operation_error::OperationResult;
@@ -276,6 +276,22 @@ impl<S: UniversalReadExt> ReadOnlyFieldIndex<S> {
impl<S: UniversalReadExt> LiveReload for ReadOnlyFieldIndex<S> {
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, fs: &Fs) -> OperationResult<()> {
match self {
ReadOnlyFieldIndex::IntIndex(index) => index.live_preload(fs),
ReadOnlyFieldIndex::DatetimeIndex(index) => index.live_preload(fs),
ReadOnlyFieldIndex::IntMapIndex(index) => index.live_preload(fs),
ReadOnlyFieldIndex::KeywordIndex(index) => index.live_preload(fs),
ReadOnlyFieldIndex::FloatIndex(index) => index.live_preload(fs),
ReadOnlyFieldIndex::GeoIndex(index) => index.live_preload(fs),
ReadOnlyFieldIndex::FullTextIndex(index) => index.live_preload(fs),
ReadOnlyFieldIndex::BoolIndex(index) => index.live_preload(fs),
ReadOnlyFieldIndex::UuidIndex(index) => index.live_preload(fs),
ReadOnlyFieldIndex::UuidMapIndex(index) => index.live_preload(fs),
ReadOnlyFieldIndex::NullIndex(index) => index.live_preload(fs),
}
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
fs: &Fs,
@@ -1,7 +1,7 @@
use common::counter::hardware_counter::HardwareCounterCell;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use super::ImmutableFullTextIndex;
use crate::common::operation_error::OperationResult;
@@ -10,6 +10,10 @@ use crate::index::field_index::LiveReload;
impl<S: UniversalRead> LiveReload for ImmutableFullTextIndex<S> {
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, _fs: &Fs) -> OperationResult<()> {
Ok(())
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
_fs: &Fs,
@@ -2,7 +2,7 @@ use common::counter::hardware_counter::HardwareCounterCell;
use common::generic_consts::Sequential;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use super::ReadOnlyAppendableFullTextIndex;
use crate::common::operation_error::{OperationError, OperationResult};
@@ -15,6 +15,10 @@ use crate::index::field_index::full_text_index::inverted_index::{
impl<S: UniversalRead> LiveReload for ReadOnlyAppendableFullTextIndex<S> {
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, fs: &Fs) -> OperationResult<()> {
Ok(self.storage.live_preload(fs)?)
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
fs: &Fs,
@@ -1,7 +1,7 @@
use common::counter::hardware_counter::HardwareCounterCell;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use super::OnDiskFullTextIndex;
use crate::common::operation_error::OperationResult;
@@ -10,6 +10,10 @@ use crate::index::field_index::LiveReload;
impl<S: UniversalRead> LiveReload for OnDiskFullTextIndex<S> {
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, _fs: &Fs) -> OperationResult<()> {
Ok(())
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
_fs: &Fs,
@@ -1,7 +1,7 @@
use common::counter::hardware_counter::HardwareCounterCell;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use super::ReadOnlyFullTextIndex;
use crate::common::operation_error::OperationResult;
@@ -10,6 +10,14 @@ use crate::index::field_index::LiveReload;
impl<S: UniversalRead> LiveReload for ReadOnlyFullTextIndex<S> {
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, fs: &Fs) -> OperationResult<()> {
match self {
Self::Appendable(index) => index.live_preload(fs),
Self::Immutable(index) => index.live_preload(fs),
Self::OnDisk(index) => index.live_preload(fs),
}
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
fs: &Fs,
@@ -1,7 +1,7 @@
use common::counter::hardware_counter::HardwareCounterCell;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use super::ImmutableGeoIndex;
use crate::common::operation_error::OperationResult;
@@ -10,6 +10,10 @@ use crate::index::field_index::LiveReload;
impl<S: UniversalRead> LiveReload for ImmutableGeoIndex<S> {
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, _fs: &Fs) -> OperationResult<()> {
Ok(())
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
_fs: &Fs,
@@ -2,7 +2,7 @@ use common::counter::hardware_counter::HardwareCounterCell;
use common::generic_consts::Sequential;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use super::ReadOnlyAppendableGeoIndex;
use crate::common::operation_error::{OperationError, OperationResult};
@@ -12,6 +12,10 @@ use crate::types::{GeoPoint, RawGeoPoint};
impl<S: UniversalRead> LiveReload for ReadOnlyAppendableGeoIndex<S> {
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, fs: &Fs) -> OperationResult<()> {
Ok(self.storage.live_preload(fs)?)
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
fs: &Fs,
@@ -1,7 +1,7 @@
use common::counter::hardware_counter::HardwareCounterCell;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use super::OnDiskGeoIndex;
use crate::common::operation_error::OperationResult;
@@ -10,6 +10,10 @@ use crate::index::field_index::LiveReload;
impl<S: UniversalRead> LiveReload for OnDiskGeoIndex<S> {
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, _fs: &Fs) -> OperationResult<()> {
Ok(())
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
_fs: &Fs,
@@ -1,7 +1,7 @@
use common::counter::hardware_counter::HardwareCounterCell;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use super::ReadOnlyGeoIndex;
use crate::common::operation_error::OperationResult;
@@ -10,6 +10,14 @@ use crate::index::field_index::LiveReload;
impl<S: UniversalRead> LiveReload for ReadOnlyGeoIndex<S> {
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, fs: &Fs) -> OperationResult<()> {
match self {
Self::Appendable(index) => index.live_preload(fs),
Self::Immutable(index) => index.live_preload(fs),
Self::OnDisk(index) => index.live_preload(fs),
}
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
fs: &Fs,
@@ -3,7 +3,7 @@ use common::counter::hardware_counter::HardwareCounterCell;
use common::persisted_hashmap::Key;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use super::ImmutableMapIndex;
use crate::common::operation_error::OperationResult;
@@ -18,6 +18,10 @@ where
{
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, _fs: &Fs) -> OperationResult<()> {
Ok(())
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
_fs: &Fs,
@@ -4,7 +4,7 @@ use common::counter::hardware_counter::HardwareCounterCell;
use common::generic_consts::Sequential;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use crate::common::operation_error::OperationResult;
use crate::index::field_index::LiveReload;
@@ -17,6 +17,10 @@ where
{
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, fs: &Fs) -> OperationResult<()> {
Ok(self.storage.live_preload(fs)?)
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
fs: &Fs,
@@ -2,7 +2,7 @@ use common::counter::hardware_counter::HardwareCounterCell;
use common::persisted_hashmap::Key;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use crate::common::operation_error::OperationResult;
use crate::index::field_index::LiveReload;
@@ -16,6 +16,10 @@ where
{
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, _fs: &Fs) -> OperationResult<()> {
Ok(())
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
_fs: &Fs,
@@ -3,7 +3,7 @@ use common::counter::hardware_counter::HardwareCounterCell;
use common::persisted_hashmap::Key;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use super::super::MapIndexKey;
use super::ReadOnlyMapIndex;
@@ -16,6 +16,14 @@ where
{
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, fs: &Fs) -> OperationResult<()> {
match self {
Self::Appendable(index) => index.live_preload(fs),
Self::Immutable(index) => index.live_preload(fs),
Self::OnDisk(index) => index.live_preload(fs),
}
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
fs: &Fs,
@@ -1,7 +1,7 @@
use common::counter::hardware_counter::HardwareCounterCell;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use super::ReadOnlyNullIndex;
use crate::common::operation_error::OperationResult;
@@ -10,6 +10,12 @@ use crate::index::field_index::LiveReload;
impl<S: UniversalRead> LiveReload for ReadOnlyNullIndex<S> {
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, cached_fs: &Fs) -> OperationResult<()> {
self.storage.has_values_flags.live_preload(cached_fs)?;
self.storage.is_null_flags.live_preload(cached_fs)?;
Ok(())
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
fs: &Fs,
@@ -2,7 +2,7 @@ use blobstore::Blob;
use common::counter::hardware_counter::HardwareCounterCell;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use super::ImmutableNumericIndex;
use crate::common::operation_error::OperationResult;
@@ -18,6 +18,10 @@ where
{
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, _fs: &Fs) -> OperationResult<()> {
Ok(())
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
_fs: &Fs,
@@ -4,7 +4,7 @@ use common::counter::hardware_counter::HardwareCounterCell;
use common::generic_consts::Sequential;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use super::ReadOnlyAppendableNumericIndex;
use crate::common::operation_error::OperationResult;
@@ -19,6 +19,10 @@ where
{
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, fs: &Fs) -> OperationResult<()> {
Ok(self.storage.live_preload(fs)?)
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
fs: &Fs,
@@ -1,7 +1,7 @@
use common::counter::hardware_counter::HardwareCounterCell;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use crate::common::operation_error::OperationResult;
use crate::index::field_index::LiveReload;
@@ -15,6 +15,10 @@ impl<T: Encodable + Numericable + Default + StoredValue + 'static, S: UniversalR
{
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, _fs: &Fs) -> OperationResult<()> {
Ok(())
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
_fs: &Fs,
@@ -2,7 +2,7 @@ use blobstore::Blob;
use common::counter::hardware_counter::HardwareCounterCell;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use super::{Encodable, ReadOnlyNumericIndex};
use crate::common::operation_error::OperationResult;
@@ -20,6 +20,10 @@ where
{
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, fs: &Fs) -> OperationResult<()> {
self.inner.live_preload(fs)
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
fs: &Fs,
@@ -2,7 +2,7 @@ use blobstore::Blob;
use common::counter::hardware_counter::HardwareCounterCell;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::{UniversalRead, UniversalReadFs};
use common::universal_io::{CachedReadFs, UniversalRead, UniversalReadFs};
use super::ReadOnlyNumericIndexInner;
use crate::common::operation_error::OperationResult;
@@ -18,6 +18,14 @@ where
{
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, fs: &Fs) -> OperationResult<()> {
match self {
Self::Appendable(index) => index.live_preload(fs),
Self::Immutable(index) => index.live_preload(fs),
Self::OnDisk(index) => index.live_preload(fs),
}
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
fs: &Fs,
@@ -1,7 +1,7 @@
use common::counter::hardware_counter::HardwareCounterCell;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::UniversalReadFs;
use common::universal_io::{CachedReadFs, UniversalReadFs};
use super::VectorIndexReadEnum;
use crate::common::live_reload::LiveReload;
@@ -11,6 +11,13 @@ use crate::index::UniversalReadExt;
impl<S: UniversalReadExt> LiveReload for VectorIndexReadEnum<S> {
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, _fs: &Fs) -> OperationResult<()> {
// Nothing to stage: the persisted variants are immutable after build,
// and the mutable-RAM sparse index ingests through the already-reloaded
// vector storage.
Ok(())
}
/// No-op for the persisted variants: read-only vector indexes are immutable —
/// the HNSW graph and the compressed sparse inverted indexes are built once,
/// and plain has no index files at all. Deletions and newly appended points
@@ -7,7 +7,7 @@ use atomic_refcell::AtomicRefCell;
use common::counter::hardware_counter::HardwareCounterCell;
use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::UniversalReadFs;
use common::universal_io::{CachedReadFs, UniversalReadFs};
use crate::common::live_reload::LiveReload;
use crate::common::operation_error::OperationResult;
@@ -74,6 +74,14 @@ impl<S: UniversalReadExt> ReadOnlyStructPayloadIndex<S> {
impl<S: UniversalReadExt> LiveReload for ReadOnlyStructPayloadIndex<S> {
type File = S;
fn live_preload<Fs: CachedReadFs<File = S>>(&self, fs: &Fs) -> OperationResult<()> {
for field_index in self.field_indexes.values().flatten() {
field_index.live_preload(fs)?;
}
Ok(())
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
fs: &Fs,
@@ -10,6 +10,14 @@ use crate::common::operation_error::OperationResult;
impl<S: UniversalRead> LiveReload for ReadOnlyPayloadStorage<S> {
type File = S;
fn live_preload<Fs: common::universal_io::CachedReadFs<File = Self::File>>(
&self,
cached_fs: &Fs,
) -> OperationResult<()> {
self.storage.live_preload(cached_fs)?;
Ok(())
}
fn live_reload<Fs: UniversalReadFs<File = S>>(
&mut self,
fs: &Fs,