diff --git a/lib/segment/src/vector_storage/chunked_vectors/chunks.rs b/lib/segment/src/vector_storage/chunked_vectors/chunks.rs index 3ef9f72588..803cdcabc4 100644 --- a/lib/segment/src/vector_storage/chunked_vectors/chunks.rs +++ b/lib/segment/src/vector_storage/chunked_vectors/chunks.rs @@ -18,7 +18,7 @@ fn check_mmap_file_name_pattern(file_name: &str) -> Option { .and_then(|file_name| file_name.parse::().ok()) } -pub fn read_chunks>( +pub fn read_chunks>( directory: &Path, advice: AdviceSetting, populate: bool, diff --git a/lib/segment/src/vector_storage/chunked_vectors/config.rs b/lib/segment/src/vector_storage/chunked_vectors/config.rs index d3ecd0abbc..e988b76a09 100644 --- a/lib/segment/src/vector_storage/chunked_vectors/config.rs +++ b/lib/segment/src/vector_storage/chunked_vectors/config.rs @@ -7,6 +7,7 @@ pub(super) const MMAP_CHUNKS_PATTERN_START: &str = "chunk_"; // TODO: rename for other storages? pub(super) const MMAP_CHUNKS_PATTERN_END: &str = ".mmap"; +#[derive(Debug, Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] #[repr(C)] pub struct Status { pub len: usize, diff --git a/lib/segment/src/vector_storage/chunked_vectors/read.rs b/lib/segment/src/vector_storage/chunked_vectors/read.rs index ae85e8ed02..5f7b4d128f 100644 --- a/lib/segment/src/vector_storage/chunked_vectors/read.rs +++ b/lib/segment/src/vector_storage/chunked_vectors/read.rs @@ -25,7 +25,7 @@ use crate::vector_storage::{VectorOffset, VectorOffsetType}; /// not refreshed afterwards. Mutating storage uses [`super::ChunkedVectors`] /// which wraps this and adds a writable status mmap. #[derive(Debug)] -pub struct ChunkedVectorsRead> { +pub struct ChunkedVectorsRead> { pub(super) config: ChunkedVectorsConfig, /// Number of vectors currently stored. Snapshot for read-only mode; for /// [`super::ChunkedVectors`] this is kept in sync with the writable status @@ -35,7 +35,7 @@ pub struct ChunkedVectorsRead> { pub(super) directory: PathBuf, } -impl> ChunkedVectorsRead { +impl> ChunkedVectorsRead { pub(super) fn config_file(directory: &Path) -> PathBuf { directory.join(CONFIG_FILE_NAME) } diff --git a/lib/segment/src/vector_storage/chunked_vectors/write.rs b/lib/segment/src/vector_storage/chunked_vectors/write.rs index 2e8c0f2c38..eca578a133 100644 --- a/lib/segment/src/vector_storage/chunked_vectors/write.rs +++ b/lib/segment/src/vector_storage/chunked_vectors/write.rs @@ -1,13 +1,12 @@ use std::cmp::max; use std::ops::Deref; -use std::path::Path; +use std::path::{Path, PathBuf}; use common::counter::hardware_counter::HardwareCounterCell; use common::fs::atomic_save_json; -use common::mmap::{Advice, AdviceSetting, MmapType, open_write_mmap}; -use common::universal_io::UniversalWrite; +use common::mmap::AdviceSetting; +use common::universal_io::{OpenOptions, StoredStruct, UniversalWrite}; use fs_err as fs; -use memmap2::MmapMut; use num_traits::AsPrimitive; use super::chunks::{create_chunk, read_chunks}; @@ -19,12 +18,20 @@ use crate::vector_storage::VectorOffsetType; use crate::vector_storage::common::CHUNK_SIZE; #[derive(Debug)] -pub struct ChunkedVectors> { +pub struct ChunkedVectors +where + T: Copy + 'static, + S: UniversalWrite + UniversalWrite + Send + 'static, +{ inner: ChunkedVectorsRead, - status: MmapType, + status: StoredStruct, } -impl> Deref for ChunkedVectors { +impl Deref for ChunkedVectors +where + T: Copy + 'static, + S: UniversalWrite + UniversalWrite + Send + 'static, +{ type Target = ChunkedVectorsRead; fn deref(&self) -> &Self::Target { @@ -32,20 +39,21 @@ impl> Deref for ChunkedVectors> ChunkedVectors { - pub fn ensure_status_file(directory: &Path) -> OperationResult { +impl ChunkedVectors +where + T: Copy + 'static, + S: UniversalWrite + UniversalWrite + Send + 'static, +{ + pub fn ensure_status_file(directory: &Path) -> OperationResult { let status_file = ChunkedVectorsRead::::status_file(directory); - if !status_file.exists() { + if !S::exists(&status_file)? { { - let length = std::mem::size_of::() as u64; - common::mmap::create_and_ensure_length(&status_file, length as usize)?; + let length = std::mem::size_of::(); + // TODO(uio): migrate when UniversalWriteFileOps is available + common::mmap::create_and_ensure_length(&status_file, length)?; } } - Ok(open_write_mmap( - &status_file, - AdviceSetting::from(Advice::Normal), - false, // Status file is write-only - )?) + Ok(status_file) } fn ensure_config( @@ -107,8 +115,19 @@ impl> ChunkedVectors { populate: Option, ) -> OperationResult { fs::create_dir_all(directory)?; - let status_mmap = Self::ensure_status_file(directory)?; - let status: MmapType = unsafe { MmapType::from(status_mmap) }; + let status_path = Self::ensure_status_file(directory)?; + + let status: StoredStruct = StoredStruct::open( + status_path, + OpenOptions { + writeable: true, + need_sequential: false, + disk_parallel: None, + populate, + advice: None, + prevent_caching: None, + }, + )?; let config = Self::ensure_config(directory, dim, populate)?; let chunks = read_chunks(directory, advice, populate.unwrap_or_default(), true)?;