upstream-archive/qdrant
1
0
Fork 0
mirror of https://github.com/qdrant/qdrant.git synced 2026-07-23 11:11:00 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: add open for read only map index (#9223)

Browse Source 
* feat: add open methods for map

* fix: review comments

* feat: detect not-found via error instead of path.exists in map index open

Replace the path.exists() pre-check in the read-only appendable map index
open path with error-driven detection through ok_not_found(). Generalize
OkNotFound over an IsNotFound trait, implemented for io::Error, mmap::Error,
UniversalIoError, and GridstoreError so a NotFound surfacing through any
layer (including mmap's inner io::Error) maps to Ok(None).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* review: revert internal error conversion

---------

Co-authored-by: generall <andrey@vasnetsov.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Daniel Boros
2026-06-02 13:56:02 +02:00
committed by timvisee
parent 4efb78545d
commit 4bf579b1b4
10 changed files with 310 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
lib/common/common/src/mmap/mmap_rw.rs
View File

@@ -469,6 +469,16 @@ pub enum Error {
MissingFile(String),
}
impl crate::universal_io::IsNotFound for Error {
fn is_not_found(&self) -> bool {
match self {
Self::Io(err) => err.is_not_found(),
Self::MissingFile(_) => true,
Self::SizeExact(..) | Self::SizeLess(..) | Self::SizeMultiple(..) => false,
}
}
}
/// Get a second mutable reference for type `T` from the given mmap
///
/// # Warning

44
lib/common/common/src/universal_io/error.rs
View File

@@ -4,17 +4,29 @@ use std::path::PathBuf;
/// `Result` extension for treating `NotFound` as `Ok(None)`
pub trait OkNotFound {
type Ok;
type Error;
/// Treat the not found error as `Ok(None)`
fn ok_not_found(self) -> Result<Option<Self::Ok>, Self::Error>;
}
impl<T> OkNotFound for Result<T, UniversalIoError> {
type Ok = T;
type Error = UniversalIoError;
pub trait IsNotFound {
fn is_not_found(&self) -> bool;
}
fn ok_not_found(self) -> Result<Option<T>, UniversalIoError> {
impl IsNotFound for io::Error {
fn is_not_found(&self) -> bool {
self.kind() == io::ErrorKind::NotFound
}
}
impl<T, E: IsNotFound> OkNotFound for Result<T, E> {
type Ok = T;
type Error = E;
fn ok_not_found(self) -> Result<Option<T>, E> {
match self {
Ok(t) => Ok(Some(t)),
Err(err) if err.is_not_found() => Ok(None),
@@ -23,6 +35,26 @@ impl<T> OkNotFound for Result<T, UniversalIoError> {
}
}
impl IsNotFound for UniversalIoError {
fn is_not_found(&self) -> bool {
match self {
Self::NotFound { .. } => true,
Self::Io(err) | Self::IoUringNotSupported(err) => err.is_not_found(),
Self::Mmap(err) => err.is_not_found(),
Self::Bincode(_)
| Self::BytemuckCast(_)
| Self::ZerocopySize(_)
| Self::OutOfBounds { .. }
| Self::InvalidFileIndex { .. }
| Self::Uninitialized { .. }
| Self::QueueIsFull
| Self::S3(_)
| Self::S3Config { .. }
| Self::TaskPanicked(_) => false,
}
}
}
#[derive(Debug, thiserror::Error)]
pub enum UniversalIoError {
#[error(transparent)]
@@ -84,10 +116,6 @@ impl UniversalIoError {
}
}
pub fn is_not_found(&self) -> bool {
matches!(self, Self::NotFound { .. })
}
pub fn uninitialized(description: impl Into<String>) -> Self {
Self::Uninitialized {
description: description.into(),

2
lib/common/common/src/universal_io/mod.rs
View File

@@ -11,7 +11,7 @@ mod traits;
mod types;
mod wrappers;
pub use self::error::{OkNotFound, UniversalIoError};
pub use self::error::{IsNotFound, OkNotFound, UniversalIoError};
#[cfg(target_os = "linux")]
pub use self::io_uring::{IoUringFile, IoUringFs, IoUringOpenExtra};
pub use self::mmap::{MmapFile, MmapFs};

18
lib/gridstore/src/error.rs
View File

@@ -1,5 +1,5 @@
use common::mmap;
use common::universal_io::UniversalIoError;
use common::universal_io::{IsNotFound, UniversalIoError};
use crate::tracker::{PageId, PointOffset};
@@ -38,3 +38,19 @@ impl GridstoreError {
}
}
}
impl IsNotFound for GridstoreError {
fn is_not_found(&self) -> bool {
match self {
GridstoreError::UniversalIo(err) => err.is_not_found(),
GridstoreError::Io(err) => err.is_not_found(),
GridstoreError::Mmap(err) => err.is_not_found(),
GridstoreError::SerdeJson(_)
| GridstoreError::ServiceError { .. }
| GridstoreError::FlushCancelled
| GridstoreError::ValidationError { .. }
| GridstoreError::PageNotFound { .. }
| GridstoreError::ValueNotFound { .. } => false,
}
}
}

6
lib/gridstore/src/gridstore/reader.rs
View File

@@ -171,11 +171,7 @@ where
{
let config_path = base_path.join(CONFIG_FILENAME);
let config: StorageConfig =
read_json_via::<Fs, StorageConfig>(fs, &config_path).map_err(|err| {
GridstoreError::service_error(format!(
"Failed to read config from '{config_path:?}': {err}"
))
})?;
read_json_via::<Fs, StorageConfig>(fs, &config_path).map_err(GridstoreError::from)?;
let tracker = Tracker::<S>::open(fs, base_path, writeable)?;

2
lib/gridstore/src/tracker.rs
View File

@@ -291,6 +291,8 @@ impl<S: UniversalRead> Tracker<S> {
fn open_storage(fs: &S::Fs, path: &Path, writeable: bool) -> Result<S> {
let storage = match fs.open(path, tracker_open_options(writeable), Default::default()) {
Err(UniversalIoError::NotFound { .. }) => {
// If config exists and storage doesn't,
// it should be treated as inconsistent storage rather than a missing one
return Err(GridstoreError::service_error(format!(
"Tracker file does not exist: {}",
path.display()

55
lib/segment/src/index/field_index/map_index/mutable_map_index/read_only/lifecycle.rs Normal file
View File

@@ -0,0 +1,55 @@
use std::path::PathBuf;
use common::counter::hardware_counter::HardwareCounterCell;
use common::universal_io::{OkNotFound, UniversalRead};
use gridstore::error::GridstoreError;
use gridstore::{Blob, GridstoreReader};
use super::super::MapIndexKey;
use super::super::inner::MutableMapIndexInner;
use super::ReadOnlyAppendableMapIndex;
use crate::common::operation_error::OperationResult;
impl<N: MapIndexKey + ?Sized, S: UniversalRead> ReadOnlyAppendableMapIndex<N, S>
where
Vec<<N as MapIndexKey>::Owned>: Blob + Send + Sync,
{
/// Open the appendable (Gridstore) map index read-only, threading every
/// file open through the filesystem handle `fs`.
///
/// Opens a [`GridstoreReader`] over the generic filesystem object, then
/// rebuilds the in-memory state by feeding every stored value through
/// [`MutableMapIndexInner::ingest`] — the exact reconstruction the writable
/// [`MutableMapIndex::open_gridstore`][1] performs over a writable
/// [`gridstore::Gridstore`]. No write path; the reader is retained for
/// later `files` / `clear_cache` use.
///
/// Returns [`Ok(None)`] when the on-disk directory doesn't exist, matching
/// the `create_if_missing == false` branch of the writable counterpart —
/// the read path never creates.
///
/// [1]: super::super::MutableMapIndex::open_gridstore
pub fn open(fs: &S::Fs, path: PathBuf) -> OperationResult<Option<Self>> {
let Some(storage) =
GridstoreReader::<Vec<<N as MapIndexKey>::Owned>, S>::open(fs, path).ok_not_found()?
else {
// Files don't exist, cannot load
return Ok(None);
};
let mut inner = MutableMapIndexInner::<N>::empty();
let hw_counter = HardwareCounterCell::disposable();
storage.iter::<_, GridstoreError>(
storage.max_point_offset(),
|idx, values: Vec<_>| {
for value in values {
inner.ingest(idx, value);
}
Ok(true)
},
hw_counter.ref_payload_index_io_write_counter(),
)?;
Ok(Some(Self { inner, storage }))
}
}

12
lib/segment/src/index/field_index/map_index/mutable_map_index/read_only/mod.rs
View File

@@ -4,6 +4,7 @@ use gridstore::{Blob, GridstoreReader};
use super::super::MapIndexKey;
use super::inner::MutableMapIndexInner;
mod lifecycle;
mod read_ops;
/// Read-only counterpart to [`super::MutableMapIndex`].
@@ -14,15 +15,18 @@ mod read_ops;
/// [`super::super::read_ops::MapIndexRead`] by forwarding to the inner;
/// provides no mutation surface.
///
/// Loading / lifecycle (constructor, `files`, `populate`, `clear_cache`, …)
/// will be added in a follow-up.
/// Constructed via [`Self::open`] (see [`lifecycle`]); the parent
/// [`super::super::read_only::ReadOnlyMapIndex`] dispatches into this type
/// through [`super::super::read_only::ReadOnlyMapIndex::open_appendable`].
pub struct ReadOnlyAppendableMapIndex<N: MapIndexKey + ?Sized, S: UniversalRead>
where
Vec<<N as MapIndexKey>::Owned>: Blob + Send + Sync,
{
pub(super) inner: MutableMapIndexInner<N>,
// Read once the lifecycle layer lands; until then the field is held to
// pin the on-disk layout of the type.
/// Backing Gridstore reader, populated by [`Self::open`]. Held to keep the
/// storage mapped; the `files` / `populate` / `clear_cache` wiring that
/// reads it lands with the parent dispatcher (it isn't part of the
/// [`MapIndexRead`](super::super::read_ops::MapIndexRead) surface).
#[allow(dead_code)]
pub(super) storage: GridstoreReader<Vec<<N as MapIndexKey>::Owned>, S>,
}

77
lib/segment/src/index/field_index/map_index/read_only/lifecycle.rs Normal file
View File

@@ -0,0 +1,77 @@
use std::path::{Path, PathBuf};
use common::bitvec::BitSlice;
use common::universal_io::UniversalRead;
use gridstore::Blob;
use super::super::MapIndexKey;
use super::super::mutable_map_index::read_only::ReadOnlyAppendableMapIndex;
use super::super::universal_map_index::UniversalMapIndex;
use super::ReadOnlyMapIndex;
use crate::common::operation_error::OperationResult;
use crate::index::payload_config::IndexMutability;
impl<N: MapIndexKey + ?Sized, S: UniversalRead> ReadOnlyMapIndex<N, S>
where
Vec<<N as MapIndexKey>::Owned>: Blob + Send + Sync,
{
/// Read-only mirror of [`MapIndex::new_gridstore`][1]: open the appendable
/// (Gridstore-backed) map index read-only, threading every file open
/// through the filesystem handle `fs`.
///
/// Thin dispatcher over [`ReadOnlyAppendableMapIndex::open`] — wraps the
/// leaf in [`Self::Appendable`] so callers can hold the parent enum
/// uniformly. No `create_if_missing`: the read path never creates;
/// [`Ok(None)`] propagates from the leaf when the on-disk directory
/// doesn't exist.
///
/// [1]: super::super::MapIndex::new_gridstore
pub fn open_appendable(fs: &S::Fs, dir: PathBuf) -> OperationResult<Option<Self>> {
Ok(ReadOnlyAppendableMapIndex::open(fs, dir)?.map(Self::Appendable))
}
/// Read-only mirror of [`MapIndex::new_mmap`][1]: open the immutable
/// (mmap-format) map index read-only through [`UniversalMapIndex::open`],
/// threading every file open through the filesystem handle `fs`.
///
/// The writable enum has two mmap variants (`Immutable` for in-RAM with
/// mmap backing, `Mmap` for on-disk lazy); the read-only side collapses
/// to a single [`Self::Immutable`] arm because `is_on_disk` (→ populate)
/// already covers the lazy/eager distinction inside [`UniversalMapIndex`].
/// `Ok(None)` propagates from the leaf when the on-disk index doesn't
/// exist.
///
/// [1]: super::super::MapIndex::new_mmap
pub fn open_immutable(
fs: &S::Fs,
path: &Path,
is_on_disk: bool,
deleted_points: &BitSlice,
) -> OperationResult<Option<Self>> {
let effective_is_on_disk =
is_on_disk || common::low_memory::low_memory_mode().prefer_disk();
Ok(
UniversalMapIndex::open(fs, path, effective_is_on_disk, deleted_points)?
.map(Self::Immutable),
)
}
/// Reports the on-disk format's mutability, mirroring
/// [`MapIndex::get_mutability_type`][1].
///
/// The read-only enum has two variants where the writable side has three:
/// `Appendable` corresponds to the writable `Mutable` arm, `Immutable`
/// covers both writable `Immutable` (in-RAM with mmap backing) and
/// writable `Mmap` (on-disk lazy) — both already report
/// [`IndexMutability::Immutable`] on the writable side, so the read-only
/// label matches even after the collapse.
///
/// [1]: super::super::MapIndex::get_mutability_type
pub fn get_mutability_type(&self) -> IndexMutability {
match self {
Self::Appendable(_) => IndexMutability::Mutable,
Self::Immutable(_) => IndexMutability::Immutable,
}
}
}

103
lib/segment/src/index/field_index/map_index/read_only/mod.rs
View File

@@ -5,8 +5,27 @@ use crate::index::field_index::map_index::MapIndexKey;
use crate::index::field_index::map_index::mutable_map_index::read_only::ReadOnlyAppendableMapIndex;
use crate::index::field_index::map_index::universal_map_index::UniversalMapIndex;
mod lifecycle;
mod read_ops;
/// Read-only counterpart of [`MapIndex`][1], parameterised by a
/// [`UniversalRead`] storage.
///
/// Dispatches the [`MapIndexRead`][2] / [`PayloadFieldIndexRead`][3] read
/// surface to one of two backing formats:
/// - [`Appendable`][Self::Appendable] — the in-RAM appendable index loaded
/// from the gridstore (write) format;
/// - [`Immutable`][Self::Immutable] — reads directly from the immutable mmap
/// format.
///
/// Constructed via [`Self::open_appendable`] and [`Self::open_immutable`]
/// (both generic over `S`); the upstream [`ReadOnlyFieldIndex`][4] wiring
/// follows in a separate PR.
///
/// [1]: super::MapIndex
/// [2]: super::read_ops::MapIndexRead
/// [3]: crate::index::field_index::PayloadFieldIndexRead
/// [4]: crate::index::field_index::field_index_base::read_only::ReadOnlyFieldIndex
pub enum ReadOnlyMapIndex<N: MapIndexKey + ?Sized, S: UniversalRead>
where
Vec<<N as MapIndexKey>::Owned>: Blob + Send + Sync,
@@ -16,3 +35,87 @@ where
/// Directly reads from storage in immutable format
Immutable(UniversalMapIndex<N, S>),
}
#[cfg(test)]
mod tests {
use common::counter::hardware_counter::HardwareCounterCell;
use common::types::PointOffsetType;
use common::universal_io::{MmapFile, ReadOnly, UniversalRead, UniversalReadFileOps};
use itertools::Itertools as _;
use serde_json::Value;
use tempfile::TempDir;
use super::super::MapIndex;
use super::ReadOnlyMapIndex;
use crate::index::field_index::{FieldIndexBuilderTrait, PayloadFieldIndexRead};
use crate::json_path::JsonPath;
use crate::types::{FieldCondition, Match};
/// Build an appendable (Gridstore) string map index on disk, then open it
/// via the parent enum's [`ReadOnlyMapIndex::open_appendable`] over the
/// write-enforced `ReadOnly<MmapFile>` backend. Verifies the dispatcher
/// wraps into [`ReadOnlyMapIndex::Appendable`] and that the trait
/// forwarders deliver the same hit set as the values inserted.
#[test]
fn parent_open_appendable_round_trip() {
let dir = TempDir::with_prefix("ro_map_parent_gridstore").unwrap();
let hw_counter = HardwareCounterCell::new();
// Build via the writable gridstore builder (matches the existing map
// tests' `IndexType::MutableGridstore` path).
{
let mut builder = MapIndex::<str>::builder_gridstore(dir.path().to_path_buf());
builder.init().unwrap();
let entries: &[(PointOffsetType, &[&str])] = &[
(0, &["red", "green"]),
(1, &["green"]),
(2, &["blue", "red"]),
];
for (idx, values) in entries {
let values: Vec<Value> = values.iter().map(|v| Value::from(*v)).collect();
let values_ref: Vec<_> = values.iter().collect();
builder.add_point(*idx, &values_ref, &hw_counter).unwrap();
}
builder.finalize().unwrap();
}
// `S = ReadOnly<MmapFile>` → `S::Fs = ReadOnlyFs<MmapFs>`, named via the
// associated-type projection since the wrapper type isn't exported.
// The read-only filesystem context is `Default`.
type RoFs = <ReadOnly<MmapFile> as UniversalRead>::Fs;
let fs = RoFs::from_context(Default::default()).unwrap();
let index: ReadOnlyMapIndex<str, ReadOnly<MmapFile>> =
ReadOnlyMapIndex::open_appendable(&fs, dir.path().to_path_buf())
.unwrap()
.unwrap();
// Dispatcher wraps the leaf into the right variant.
assert!(matches!(index, ReadOnlyMapIndex::Appendable(_)));
// Trait dispatch on the parent enum forwards into the leaf:
// every point with at least one value is counted, and `red` matches
// the two that contain it while `blue` matches only the third.
assert_eq!(index.count_indexed_points(), 3);
let key = JsonPath::new("color");
let red = FieldCondition::new_match(key.clone(), Match::from("red".to_string()));
let blue = FieldCondition::new_match(key, Match::from("blue".to_string()));
assert_eq!(
index
.filter(&red, &hw_counter)
.unwrap()
.unwrap()
.collect_vec(),
vec![0, 2],
);
assert_eq!(
index
.filter(&blue, &hw_counter)
.unwrap()
.unwrap()
.collect_vec(),
vec![2],
);
}
}