diff --git a/lib/segment/src/index/field_index/null_index/read_only_null_index/live_reload.rs b/lib/segment/src/index/field_index/null_index/read_only_null_index/live_reload.rs new file mode 100644 index 0000000000..19fa10983b --- /dev/null +++ b/lib/segment/src/index/field_index/null_index/read_only_null_index/live_reload.rs @@ -0,0 +1,34 @@ +use common::counter::hardware_counter::HardwareCounterCell; +use common::types::PointOffsetType; +use common::universal_io::UniversalRead; + +use super::ReadOnlyNullIndex; +use crate::common::operation_error::OperationResult; +use crate::index::field_index::LiveReload; + +impl LiveReload for ReadOnlyNullIndex { + type Fs = S::Fs; + + fn live_reload( + &mut self, + fs: &S::Fs, + deleted_points: &[PointOffsetType], + new_points: &[PointOffsetType], + hw_counter: &HardwareCounterCell, + ) -> OperationResult<()> { + // Reload each flag set's bitmap from the changed points only. + self.storage + .has_values_flags + .live_reload(fs, deleted_points, new_points, hw_counter)?; + self.storage + .is_null_flags + .live_reload(fs, deleted_points, new_points, hw_counter)?; + + // total_point_count only grows, to cover appended offsets. + self.total_point_count = new_points + .iter() + .fold(self.total_point_count, |max, &id| max.max(id as usize + 1)); + + Ok(()) + } +} diff --git a/lib/segment/src/index/field_index/null_index/read_only_null_index/mod.rs b/lib/segment/src/index/field_index/null_index/read_only_null_index/mod.rs index 131c2eabc5..eca8fa25f7 100644 --- a/lib/segment/src/index/field_index/null_index/read_only_null_index/mod.rs +++ b/lib/segment/src/index/field_index/null_index/read_only_null_index/mod.rs @@ -6,6 +6,7 @@ use crate::common::flags::read_only_roaring_flags::ReadOnlyRoaringFlags; use crate::index::payload_config::IndexMutability; mod lifecycle; +mod live_reload; mod read_ops; /// Read-only counterpart of [`MutableNullIndex`][1] / [`ImmutableNullIndex`][2]. @@ -70,7 +71,9 @@ mod tests { use super::super::mutable_null_index::{HAS_VALUES_DIRNAME, IS_NULL_DIRNAME, MutableNullIndex}; use super::super::read_ops::NullIndexRead; use super::ReadOnlyNullIndex; - use crate::index::field_index::{FieldIndexBuilderTrait, PayloadFieldIndexRead}; + use crate::index::field_index::{ + FieldIndexBuilderTrait, LiveReload, PayloadFieldIndex, PayloadFieldIndexRead, + }; use crate::json_path::JsonPath; use crate::types::FieldCondition; @@ -168,6 +171,128 @@ mod tests { assert!(!index.values_is_null(3)); } + /// The incremental `LiveReload` path must land on exactly the same in-memory + /// state as a fresh `ReadOnlyNullIndex::open` over the post-write files. + /// + /// A writer keeps mutating the on-disk flags after the read-only view is + /// open: a point is deleted, one flips from empty to having a value, and a + /// null point is appended. `live_reload` is handed only that delta, yet its + /// bitmaps and the `total_point_count`-driven `is_empty` results must match + /// the authoritative re-open. + #[test] + fn live_reload_matches_fresh_open() { + let dir = TempDir::with_prefix("read_only_null_index_live_reload").unwrap(); + let hw_counter = HardwareCounterCell::new(); + + // Initial on-disk state: points 0..=3 (total 4). + let null_in_array = Value::Array(vec![Value::String("x".to_string()), Value::Null]); + let mut builder = MutableNullIndex::builder(dir.path(), 0).unwrap(); + builder.add_point(0, &[&Value::Null], &hw_counter).unwrap(); // null, no values + builder + .add_point(1, &[&null_in_array], &hw_counter) + .unwrap(); // null + values + builder.add_point(2, &[], &hw_counter).unwrap(); // empty + builder.add_point(3, &[&json!(true)], &hw_counter).unwrap(); // values, not null + let mut index = builder.finalize().unwrap(); + + type RoFs = as UniversalRead>::Fs; + let fs = RoFs::from_context(Default::default()).unwrap(); + + // Read-only view of points 0..=3, taken before the writer continues. + let mut reloaded = ReadOnlyNullIndex::>::open(&fs, dir.path(), 4) + .unwrap() + .unwrap(); + + // Writer's delta: drop point 1, flip point 2 (empty -> value), append a + // null point 4 (which grows `total_point_count` to 5). + index.remove_point(1).unwrap(); + index.add_point(2, &[&json!(true)], &hw_counter).unwrap(); + index.add_point(4, &[&Value::Null], &hw_counter).unwrap(); + index.flusher()().unwrap(); + let total = 5; + + reloaded + .live_reload(&fs, &[1], &[2, 4], &hw_counter) + .unwrap(); + + let fresh = ReadOnlyNullIndex::>::open(&fs, dir.path(), total) + .unwrap() + .unwrap(); + + let key = JsonPath::new("test"); + let is_null = FieldCondition::new_is_null(key.clone(), true); + let is_not_empty = FieldCondition { + key: key.clone(), + r#match: None, + range: None, + geo_bounding_box: None, + geo_radius: None, + geo_polygon: None, + values_count: None, + is_empty: Some(false), + is_null: None, + }; + let is_empty = FieldCondition { + key: key.clone(), + r#match: None, + range: None, + geo_bounding_box: None, + geo_radius: None, + geo_polygon: None, + values_count: None, + is_empty: Some(true), + is_null: None, + }; + + let reloaded_null = reloaded + .filter(&is_null, &hw_counter) + .unwrap() + .unwrap() + .collect_vec(); + let reloaded_not_empty = reloaded + .filter(&is_not_empty, &hw_counter) + .unwrap() + .unwrap() + .collect_vec(); + let reloaded_empty = reloaded + .filter(&is_empty, &hw_counter) + .unwrap() + .unwrap() + .collect_vec(); + let fresh_null = fresh + .filter(&is_null, &hw_counter) + .unwrap() + .unwrap() + .collect_vec(); + let fresh_not_empty = fresh + .filter(&is_not_empty, &hw_counter) + .unwrap() + .unwrap() + .collect_vec(); + let fresh_empty = fresh + .filter(&is_empty, &hw_counter) + .unwrap() + .unwrap() + .collect_vec(); + + // Parity with the authoritative re-open … + assert_eq!(reloaded_null, fresh_null); + assert_eq!(reloaded_not_empty, fresh_not_empty); + assert_eq!(reloaded_empty, fresh_empty); + assert_eq!( + reloaded.count_indexed_points(), + fresh.count_indexed_points() + ); + + // … and the concrete expected sets. Point 2's flip moves it from empty + // to has-values, point 1 is gone, point 4 is an appended null, and the + // grown `total_point_count` (5) drives the `is_empty` tail. + assert_eq!(reloaded_null, vec![0, 4]); + assert_eq!(reloaded_not_empty, vec![2, 3]); + assert_eq!(reloaded_empty, vec![0, 1, 4]); + assert_eq!(reloaded.count_indexed_points(), 5); + } + /// A partial on-disk layout — exactly one of the `has_values` / `is_null` /// flag directories present — is corrupt storage: `open` surfaces an error /// in either direction, rather than silently reporting a missing index.