feat: read-only open for ReadOnlyChunkedMultiDenseVectorStorage (#9335)

* feat: read-only open for ReadOnlyChunkedDenseVectorStorage Add ReadOnlyChunkedDenseVectorStorage::open, the read-only counterpart of open_appendable_memmap_vector_storage_impl: open the chunked vectors/ directory via ChunkedVectorsRead::open and materialize the deleted/ flags into an owned bitvec via DynamicStoredFlags::load_bitvec, threading every file open through the UniversalRead backend without writing anything. Share the vectors/ and deleted/ directory names with the writable storage by exposing them pub(crate). Covered by a round-trip test: write and delete through the writable storage, reopen read-only, and assert vectors, deletions and counts match. Part of the read-only vector-storage open constructors (#9241). * feat: read-only open for ReadOnlyChunkedMultiDenseVectorStorage Add ReadOnlyChunkedMultiDenseVectorStorage::open, the read-only counterpart of open_appendable_memmap_multi_vector_storage_impl: open the chunked vectors/ and offsets/ directories via ChunkedVectorsRead::open and materialize the deleted/ flags into an owned bitvec via DynamicStoredFlags::load_bitvec, threading every file open through the UniversalRead backend without writing anything. Share the vectors/, offsets/ and deleted/ directory names with the writable storage by exposing them pub(crate). Covered by a round-trip test: write and delete multivectors through the writable storage, reopen read-only, and assert per-point multivector contents, deletions and counts match. Part of the read-only vector-storage open constructors (#9241).
2026-09-21 13:37:46 -05:00 · 2026-08-04 11:16:48 +02:00
parent c3315154b1
commit ead9dc4cef
2 changed files with 152 additions and 13 deletions
@@ -30,9 +30,9 @@ use crate::vector_storage::{
    MultiVectorStorage, VectorOffsetType, VectorStorage, VectorStorageEnum, VectorStorageRead,
 };

-const VECTORS_DIR_PATH: &str = "vectors";
-const OFFSETS_DIR_PATH: &str = "offsets";
-const DELETED_DIR_PATH: &str = "deleted";
+pub(crate) const VECTORS_DIR_PATH: &str = "vectors";
+pub(crate) const OFFSETS_DIR_PATH: &str = "offsets";
+pub(crate) const DELETED_DIR_PATH: &str = "deleted";

 #[derive(Copy, Clone, Debug, Default, PartialEq, bytemuck::Pod, bytemuck::Zeroable)]
 #[repr(C)]
@@ -1,28 +1,68 @@
-use common::bitvec::{BitSlice, BitVec};
+use std::path::Path;
+
+use common::bitvec::BitSlice;
 use common::generic_consts::AccessPattern;
+use common::mmap::AdviceSetting;
 use common::types::PointOffsetType;
 use common::universal_io::UniversalRead;

+use crate::common::flags::in_memory_bitvec_flags::InMemoryBitvecFlags;
+use crate::common::operation_error::OperationResult;
 use crate::data_types::named_vectors::CowVector;
 use crate::data_types::primitive::PrimitiveVectorElement;
 use crate::types::{Distance, VectorStorageDatatype};
 use crate::vector_storage::VectorStorageRead;
 use crate::vector_storage::chunked_vectors::ChunkedVectorsRead;
 use crate::vector_storage::multi_dense::appendable_mmap_multi_dense_vector_storage::{
-    MultivectorMmapOffset, flattened_to_multi_vector, read_multi_vector,
+    DELETED_DIR_PATH, MultivectorMmapOffset, OFFSETS_DIR_PATH, VECTORS_DIR_PATH,
+    flattened_to_multi_vector, read_multi_vector,
 };

 #[derive(Debug)]
 pub struct ReadOnlyChunkedMultiDenseVectorStorage<T: PrimitiveVectorElement, S: UniversalRead> {
    vectors: ChunkedVectorsRead<T, S>,
    offsets: ChunkedVectorsRead<MultivectorMmapOffset, S>,
-    /// Flags marking deleted vectors
-    ///
-    /// Structure grows dynamically, but may be smaller than actual number of vectors. Must not
-    /// depend on its length.
-    deleted: BitVec,
+    /// Flags marking deleted vectors.
+    deleted: InMemoryBitvecFlags,
    distance: Distance,
-    deleted_count: usize,
+}
+
+impl<T: PrimitiveVectorElement, S: UniversalRead> ReadOnlyChunkedMultiDenseVectorStorage<T, S> {
+    /// Open the read-only counterpart of the appendable multi-dense storage at
+    /// `path`, threading every file open through `fs`; reads the existing layout
+    /// but creates and writes nothing. `populate` warms the vector and offset
+    /// chunks.
+    #[allow(dead_code)] // pending: read-only vector storage enum will use this
+    pub fn open(
+        fs: &S::Fs,
+        path: &Path,
+        dim: usize,
+        distance: Distance,
+        advice: AdviceSetting,
+        populate: bool,
+    ) -> OperationResult<Self> {
+        let vectors = ChunkedVectorsRead::open(
+            fs,
+            &path.join(VECTORS_DIR_PATH),
+            dim,
+            advice,
+            Some(populate),
+        )?;
+
+        // Offsets store one `MultivectorMmapOffset` element per point, so the
+        // chunked storage dimensionality is 1.
+        let offsets =
+            ChunkedVectorsRead::open(fs, &path.join(OFFSETS_DIR_PATH), 1, advice, Some(populate))?;
+
+        let deleted = InMemoryBitvecFlags::open::<S>(fs, &path.join(DELETED_DIR_PATH))?;
+
+        Ok(Self {
+            vectors,
+            offsets,
+            deleted,
+            distance,
+        })
+    }
 }

 impl<T: PrimitiveVectorElement, S: UniversalRead> VectorStorageRead
@@ -75,14 +115,113 @@ impl<T: PrimitiveVectorElement, S: UniversalRead> VectorStorageRead
    }

    fn is_deleted_vector(&self, key: PointOffsetType) -> bool {
-        self.deleted.get(key as usize).is_some_and(|bit| *bit)
+        self.deleted.get(key)
    }

    fn deleted_vector_count(&self) -> usize {
-        self.deleted_count
+        self.deleted.count()
    }

    fn deleted_vector_bitslice(&self) -> &BitSlice {
        self.deleted.as_bitslice()
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use common::counter::hardware_counter::HardwareCounterCell;
+    use common::generic_consts::Random;
+    use common::universal_io::{MmapFile, MmapFs};
+    use rand::rngs::StdRng;
+    use rand::{RngExt, SeedableRng};
+    use tempfile::Builder;
+
+    use super::*;
+    use crate::data_types::vectors::{
+        MultiDenseVectorInternal, TypedMultiDenseVectorRef, VectorElementType, VectorRef,
+    };
+    use crate::types::MultiVectorConfig;
+    use crate::vector_storage::VectorStorage;
+    use crate::vector_storage::multi_dense::appendable_mmap_multi_dense_vector_storage::open_appendable_memmap_multi_vector_storage_impl;
+
+    /// Write multivectors (deleting ~10%) through the writable appendable
+    /// storage, then reopen the same directory read-only and assert it mirrors
+    /// the state — including per-point multivector contents, which exercises the
+    /// offsets storage.
+    #[test]
+    fn read_only_chunked_multi_dense_round_trip() {
+        const POINT_COUNT: PointOffsetType = 1000;
+        const DIM: usize = 128;
+
+        let dir = Builder::new().prefix("ro_multi_dense").tempdir().unwrap();
+        let mut rng = StdRng::seed_from_u64(42);
+        let hw = HardwareCounterCell::disposable();
+
+        let multivectors: Vec<MultiDenseVectorInternal> = (0..POINT_COUNT)
+            .map(|_| {
+                let inner = rng.random_range(1..=4);
+                let vectors = std::iter::repeat_with(|| {
+                    std::iter::repeat_with(|| rng.random_range(-1.0..1.0))
+                        .take(DIM)
+                        .collect()
+                })
+                .take(inner)
+                .collect::<Vec<Vec<VectorElementType>>>();
+                MultiDenseVectorInternal::try_from(vectors).unwrap()
+            })
+            .collect();
+
+        let mut deleted_ids = Vec::new();
+        {
+            let mut storage =
+                open_appendable_memmap_multi_vector_storage_impl::<VectorElementType>(
+                    dir.path(),
+                    DIM,
+                    Distance::Dot,
+                    MultiVectorConfig::default(),
+                    AdviceSetting::Global,
+                    false,
+                )
+                .unwrap();
+            for (id, multivec) in multivectors.iter().enumerate() {
+                storage
+                    .insert_vector(id as PointOffsetType, VectorRef::from(multivec), &hw)
+                    .unwrap();
+            }
+            for id in 0..POINT_COUNT {
+                if rng.random_bool(0.1) {
+                    storage.delete_vector(id).unwrap();
+                    deleted_ids.push(id);
+                }
+            }
+            storage.flusher()().unwrap();
+        }
+
+        let storage = ReadOnlyChunkedMultiDenseVectorStorage::<VectorElementType, MmapFile>::open(
+            &MmapFs,
+            dir.path(),
+            DIM,
+            Distance::Dot,
+            AdviceSetting::Global,
+            false,
+        )
+        .unwrap();
+
+        assert_eq!(storage.total_vector_count(), POINT_COUNT as usize);
+        assert_eq!(storage.distance(), Distance::Dot);
+        assert_eq!(storage.deleted_vector_count(), deleted_ids.len());
+
+        for id in 0..POINT_COUNT {
+            assert_eq!(storage.is_deleted_vector(id), deleted_ids.contains(&id));
+
+            let stored = storage.get_vector::<Random>(id);
+            let multi: TypedMultiDenseVectorRef<VectorElementType> =
+                stored.as_vec_ref().try_into().unwrap();
+            assert_eq!(
+                multi.to_owned(),
+                multivectors[id as usize],
+                "vector {id} mismatch",
+            );
+        }
+    }
+}