From dcbbdeba8e9740ace0d91cdeac992fa45e49f84c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Coss=C3=ADo?= Date: Fri, 8 May 2026 07:51:48 -0400 Subject: [PATCH] [UIO] Genericize `ChunkedVectors.status` (#8928) * genericize ChunkedVectors.status, remove `Sized` bound * check exists with `UniversalReadFileOps` * Inline UioChunkedVectors bound, drop the alias (#8952) The empty trait + blanket impl was a stable-Rust trait-alias workaround that hid a fairly short bound (UniversalWrite + UniversalWrite + Send + 'static) at the cost of an indirection readers had to mentally unwind. Spelling it out at the three sites that need it is shorter overall and immediately tells the reader what is required. Co-authored-by: Claude Opus 4.7 (1M context) --------- Co-authored-by: Andrey Vasnetsov Co-authored-by: Claude Opus 4.7 (1M context) --- .../vector_storage/chunked_vectors/chunks.rs | 2 +- .../vector_storage/chunked_vectors/config.rs | 1 + .../vector_storage/chunked_vectors/read.rs | 4 +- .../vector_storage/chunked_vectors/write.rs | 57 ++++++++++++------- 4 files changed, 42 insertions(+), 22 deletions(-) diff --git a/lib/segment/src/vector_storage/chunked_vectors/chunks.rs b/lib/segment/src/vector_storage/chunked_vectors/chunks.rs index 3ef9f72588..803cdcabc4 100644 --- a/lib/segment/src/vector_storage/chunked_vectors/chunks.rs +++ b/lib/segment/src/vector_storage/chunked_vectors/chunks.rs @@ -18,7 +18,7 @@ fn check_mmap_file_name_pattern(file_name: &str) -> Option { .and_then(|file_name| file_name.parse::().ok()) } -pub fn read_chunks>( +pub fn read_chunks>( directory: &Path, advice: AdviceSetting, populate: bool, diff --git a/lib/segment/src/vector_storage/chunked_vectors/config.rs b/lib/segment/src/vector_storage/chunked_vectors/config.rs index d3ecd0abbc..e988b76a09 100644 --- a/lib/segment/src/vector_storage/chunked_vectors/config.rs +++ b/lib/segment/src/vector_storage/chunked_vectors/config.rs @@ -7,6 +7,7 @@ pub(super) const MMAP_CHUNKS_PATTERN_START: &str = "chunk_"; // TODO: rename for other storages? pub(super) const MMAP_CHUNKS_PATTERN_END: &str = ".mmap"; +#[derive(Debug, Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] #[repr(C)] pub struct Status { pub len: usize, diff --git a/lib/segment/src/vector_storage/chunked_vectors/read.rs b/lib/segment/src/vector_storage/chunked_vectors/read.rs index ae85e8ed02..5f7b4d128f 100644 --- a/lib/segment/src/vector_storage/chunked_vectors/read.rs +++ b/lib/segment/src/vector_storage/chunked_vectors/read.rs @@ -25,7 +25,7 @@ use crate::vector_storage::{VectorOffset, VectorOffsetType}; /// not refreshed afterwards. Mutating storage uses [`super::ChunkedVectors`] /// which wraps this and adds a writable status mmap. #[derive(Debug)] -pub struct ChunkedVectorsRead> { +pub struct ChunkedVectorsRead> { pub(super) config: ChunkedVectorsConfig, /// Number of vectors currently stored. Snapshot for read-only mode; for /// [`super::ChunkedVectors`] this is kept in sync with the writable status @@ -35,7 +35,7 @@ pub struct ChunkedVectorsRead> { pub(super) directory: PathBuf, } -impl> ChunkedVectorsRead { +impl> ChunkedVectorsRead { pub(super) fn config_file(directory: &Path) -> PathBuf { directory.join(CONFIG_FILE_NAME) } diff --git a/lib/segment/src/vector_storage/chunked_vectors/write.rs b/lib/segment/src/vector_storage/chunked_vectors/write.rs index 2e8c0f2c38..eca578a133 100644 --- a/lib/segment/src/vector_storage/chunked_vectors/write.rs +++ b/lib/segment/src/vector_storage/chunked_vectors/write.rs @@ -1,13 +1,12 @@ use std::cmp::max; use std::ops::Deref; -use std::path::Path; +use std::path::{Path, PathBuf}; use common::counter::hardware_counter::HardwareCounterCell; use common::fs::atomic_save_json; -use common::mmap::{Advice, AdviceSetting, MmapType, open_write_mmap}; -use common::universal_io::UniversalWrite; +use common::mmap::AdviceSetting; +use common::universal_io::{OpenOptions, StoredStruct, UniversalWrite}; use fs_err as fs; -use memmap2::MmapMut; use num_traits::AsPrimitive; use super::chunks::{create_chunk, read_chunks}; @@ -19,12 +18,20 @@ use crate::vector_storage::VectorOffsetType; use crate::vector_storage::common::CHUNK_SIZE; #[derive(Debug)] -pub struct ChunkedVectors> { +pub struct ChunkedVectors +where + T: Copy + 'static, + S: UniversalWrite + UniversalWrite + Send + 'static, +{ inner: ChunkedVectorsRead, - status: MmapType, + status: StoredStruct, } -impl> Deref for ChunkedVectors { +impl Deref for ChunkedVectors +where + T: Copy + 'static, + S: UniversalWrite + UniversalWrite + Send + 'static, +{ type Target = ChunkedVectorsRead; fn deref(&self) -> &Self::Target { @@ -32,20 +39,21 @@ impl> Deref for ChunkedVectors> ChunkedVectors { - pub fn ensure_status_file(directory: &Path) -> OperationResult { +impl ChunkedVectors +where + T: Copy + 'static, + S: UniversalWrite + UniversalWrite + Send + 'static, +{ + pub fn ensure_status_file(directory: &Path) -> OperationResult { let status_file = ChunkedVectorsRead::::status_file(directory); - if !status_file.exists() { + if !S::exists(&status_file)? { { - let length = std::mem::size_of::() as u64; - common::mmap::create_and_ensure_length(&status_file, length as usize)?; + let length = std::mem::size_of::(); + // TODO(uio): migrate when UniversalWriteFileOps is available + common::mmap::create_and_ensure_length(&status_file, length)?; } } - Ok(open_write_mmap( - &status_file, - AdviceSetting::from(Advice::Normal), - false, // Status file is write-only - )?) + Ok(status_file) } fn ensure_config( @@ -107,8 +115,19 @@ impl> ChunkedVectors { populate: Option, ) -> OperationResult { fs::create_dir_all(directory)?; - let status_mmap = Self::ensure_status_file(directory)?; - let status: MmapType = unsafe { MmapType::from(status_mmap) }; + let status_path = Self::ensure_status_file(directory)?; + + let status: StoredStruct = StoredStruct::open( + status_path, + OpenOptions { + writeable: true, + need_sequential: false, + disk_parallel: None, + populate, + advice: None, + prevent_caching: None, + }, + )?; let config = Self::ensure_config(directory, dim, populate)?; let chunks = read_chunks(directory, advice, populate.unwrap_or_default(), true)?;