feat: implement LiveReload for null index (#9300)

* fix: false values update * fix: linter * fix: ci/cd * chore: add comment * fix: reopen and update trues_count and falses_count * feat: implement LiveReload for null index Reload the null index incrementally through the roaring-flags LiveReload trait: reopen the has_values/is_null bitslices and re-read only the changed points, then grow total_point_count, instead of re-materializing both bitmaps via a full open. * chore: trigger ci
2026-09-21 13:37:46 -05:00 · 2026-08-04 11:16:48 +02:00
parent 80a7f31d4b
commit 845ea36253
2 changed files with 160 additions and 1 deletions
@@ -0,0 +1,34 @@
+use common::counter::hardware_counter::HardwareCounterCell;
+use common::types::PointOffsetType;
+use common::universal_io::UniversalRead;
+
+use super::ReadOnlyNullIndex;
+use crate::common::operation_error::OperationResult;
+use crate::index::field_index::LiveReload;
+
+impl<S: UniversalRead> LiveReload for ReadOnlyNullIndex<S> {
+    type Fs = S::Fs;
+
+    fn live_reload(
+        &mut self,
+        fs: &S::Fs,
+        deleted_points: &[PointOffsetType],
+        new_points: &[PointOffsetType],
+        hw_counter: &HardwareCounterCell,
+    ) -> OperationResult<()> {
+        // Reload each flag set's bitmap from the changed points only.
+        self.storage
+            .has_values_flags
+            .live_reload(fs, deleted_points, new_points, hw_counter)?;
+        self.storage
+            .is_null_flags
+            .live_reload(fs, deleted_points, new_points, hw_counter)?;
+
+        // total_point_count only grows, to cover appended offsets.
+        self.total_point_count = new_points
+            .iter()
+            .fold(self.total_point_count, |max, &id| max.max(id as usize + 1));
+
+        Ok(())
+    }
+}
@@ -6,6 +6,7 @@ use crate::common::flags::read_only_roaring_flags::ReadOnlyRoaringFlags;
 use crate::index::payload_config::IndexMutability;

 mod lifecycle;
+mod live_reload;
 mod read_ops;

 /// Read-only counterpart of [`MutableNullIndex`][1] / [`ImmutableNullIndex`][2].
@@ -70,7 +71,9 @@ mod tests {
    use super::super::mutable_null_index::{HAS_VALUES_DIRNAME, IS_NULL_DIRNAME, MutableNullIndex};
    use super::super::read_ops::NullIndexRead;
    use super::ReadOnlyNullIndex;
-    use crate::index::field_index::{FieldIndexBuilderTrait, PayloadFieldIndexRead};
+    use crate::index::field_index::{
+        FieldIndexBuilderTrait, LiveReload, PayloadFieldIndex, PayloadFieldIndexRead,
+    };
    use crate::json_path::JsonPath;
    use crate::types::FieldCondition;

@@ -168,6 +171,128 @@ mod tests {
        assert!(!index.values_is_null(3));
    }

+    /// The incremental `LiveReload` path must land on exactly the same in-memory
+    /// state as a fresh `ReadOnlyNullIndex::open` over the post-write files.
+    ///
+    /// A writer keeps mutating the on-disk flags after the read-only view is
+    /// open: a point is deleted, one flips from empty to having a value, and a
+    /// null point is appended. `live_reload` is handed only that delta, yet its
+    /// bitmaps and the `total_point_count`-driven `is_empty` results must match
+    /// the authoritative re-open.
+    #[test]
+    fn live_reload_matches_fresh_open() {
+        let dir = TempDir::with_prefix("read_only_null_index_live_reload").unwrap();
+        let hw_counter = HardwareCounterCell::new();
+
+        // Initial on-disk state: points 0..=3 (total 4).
+        let null_in_array = Value::Array(vec![Value::String("x".to_string()), Value::Null]);
+        let mut builder = MutableNullIndex::builder(dir.path(), 0).unwrap();
+        builder.add_point(0, &[&Value::Null], &hw_counter).unwrap(); // null, no values
+        builder
+            .add_point(1, &[&null_in_array], &hw_counter)
+            .unwrap(); // null + values
+        builder.add_point(2, &[], &hw_counter).unwrap(); // empty
+        builder.add_point(3, &[&json!(true)], &hw_counter).unwrap(); // values, not null
+        let mut index = builder.finalize().unwrap();
+
+        type RoFs = <ReadOnly<MmapFile> as UniversalRead>::Fs;
+        let fs = RoFs::from_context(Default::default()).unwrap();
+
+        // Read-only view of points 0..=3, taken before the writer continues.
+        let mut reloaded = ReadOnlyNullIndex::<ReadOnly<MmapFile>>::open(&fs, dir.path(), 4)
+            .unwrap()
+            .unwrap();
+
+        // Writer's delta: drop point 1, flip point 2 (empty -> value), append a
+        // null point 4 (which grows `total_point_count` to 5).
+        index.remove_point(1).unwrap();
+        index.add_point(2, &[&json!(true)], &hw_counter).unwrap();
+        index.add_point(4, &[&Value::Null], &hw_counter).unwrap();
+        index.flusher()().unwrap();
+        let total = 5;
+
+        reloaded
+            .live_reload(&fs, &[1], &[2, 4], &hw_counter)
+            .unwrap();
+
+        let fresh = ReadOnlyNullIndex::<ReadOnly<MmapFile>>::open(&fs, dir.path(), total)
+            .unwrap()
+            .unwrap();
+
+        let key = JsonPath::new("test");
+        let is_null = FieldCondition::new_is_null(key.clone(), true);
+        let is_not_empty = FieldCondition {
+            key: key.clone(),
+            r#match: None,
+            range: None,
+            geo_bounding_box: None,
+            geo_radius: None,
+            geo_polygon: None,
+            values_count: None,
+            is_empty: Some(false),
+            is_null: None,
+        };
+        let is_empty = FieldCondition {
+            key: key.clone(),
+            r#match: None,
+            range: None,
+            geo_bounding_box: None,
+            geo_radius: None,
+            geo_polygon: None,
+            values_count: None,
+            is_empty: Some(true),
+            is_null: None,
+        };
+
+        let reloaded_null = reloaded
+            .filter(&is_null, &hw_counter)
+            .unwrap()
+            .unwrap()
+            .collect_vec();
+        let reloaded_not_empty = reloaded
+            .filter(&is_not_empty, &hw_counter)
+            .unwrap()
+            .unwrap()
+            .collect_vec();
+        let reloaded_empty = reloaded
+            .filter(&is_empty, &hw_counter)
+            .unwrap()
+            .unwrap()
+            .collect_vec();
+        let fresh_null = fresh
+            .filter(&is_null, &hw_counter)
+            .unwrap()
+            .unwrap()
+            .collect_vec();
+        let fresh_not_empty = fresh
+            .filter(&is_not_empty, &hw_counter)
+            .unwrap()
+            .unwrap()
+            .collect_vec();
+        let fresh_empty = fresh
+            .filter(&is_empty, &hw_counter)
+            .unwrap()
+            .unwrap()
+            .collect_vec();
+
+        // Parity with the authoritative re-open …
+        assert_eq!(reloaded_null, fresh_null);
+        assert_eq!(reloaded_not_empty, fresh_not_empty);
+        assert_eq!(reloaded_empty, fresh_empty);
+        assert_eq!(
+            reloaded.count_indexed_points(),
+            fresh.count_indexed_points()
+        );
+
+        // … and the concrete expected sets. Point 2's flip moves it from empty
+        // to has-values, point 1 is gone, point 4 is an appended null, and the
+        // grown `total_point_count` (5) drives the `is_empty` tail.
+        assert_eq!(reloaded_null, vec![0, 4]);
+        assert_eq!(reloaded_not_empty, vec![2, 3]);
+        assert_eq!(reloaded_empty, vec![0, 1, 4]);
+        assert_eq!(reloaded.count_indexed_points(), 5);
+    }
+
    /// A partial on-disk layout — exactly one of the `has_values` / `is_null`
    /// flag directories present — is corrupt storage: `open` surfaces an error
    /// in either direction, rather than silently reporting a missing index.