mirror of
https://github.com/qdrant/qdrant.git
synced 2026-07-23 11:11:00 -05:00
feat: wire up read only field indexes (#9224)
* feat: wire up read only field indexes * feat: add missing indexes * feat: add try_from impl for TextIndexParams * refactor: unify read-only field index open and drop RocksDB storage Merge ReadOnlyFieldIndex::open_gridstore/open_mmap into a single `open` that picks the appendable vs immutable path from the stored FullPayloadIndexType::storage_type. The choice is modeled as a ReadMode (Appendable/Immutable) rather than a concrete backend, since the read-only stack is generic over UniversalRead and mmap is now just one implementation of it. Also remove the unsupported RocksDb variant from payload_config's StorageType and its now-dead match arms. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix: linter * fix: linter --------- Co-authored-by: generall <andrey@vasnetsov.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,211 @@
|
||||
use std::path::Path;
|
||||
|
||||
use common::bitvec::BitSlice;
|
||||
use common::universal_io::UniversalRead;
|
||||
|
||||
use super::ReadOnlyFieldIndex;
|
||||
use crate::common::operation_error::OperationResult;
|
||||
use crate::data_types::index::TextIndexParams;
|
||||
use crate::index::field_index::bool_index::ReadOnlyBoolIndex;
|
||||
use crate::index::field_index::full_text_index::read_only::ReadOnlyFullTextIndex;
|
||||
use crate::index::field_index::geo_index::ReadOnlyGeoMapIndex;
|
||||
use crate::index::field_index::index_selector::{
|
||||
bool_dir, map_dir, null_dir, numeric_dir, text_dir,
|
||||
};
|
||||
use crate::index::field_index::map_index::read_only::ReadOnlyMapIndex;
|
||||
use crate::index::field_index::null_index::ReadOnlyNullIndex;
|
||||
use crate::index::field_index::numeric_index::ReadOnlyNumericIndex;
|
||||
use crate::index::payload_config::{FullPayloadIndexType, PayloadIndexType, StorageType};
|
||||
use crate::json_path::JsonPath;
|
||||
use crate::types::{
|
||||
DateTimePayloadType, FloatPayloadType, IntPayloadType, PayloadFieldSchema, UuidIntType,
|
||||
};
|
||||
|
||||
/// Which read-only open path a leaf index should take, derived from the stored
|
||||
/// [`StorageType`]. Phrased as a mutability distinction (matching the
|
||||
/// `open_appendable` / `open_immutable` leaf methods) rather than a concrete
|
||||
/// backend: the read-only stack is generic over [`UniversalRead`], so the
|
||||
/// on-disk-vs-in-memory choice is just the `is_on_disk` flag on the immutable
|
||||
/// variant, picked from the index type rather than passed in by the caller.
|
||||
#[derive(Clone, Copy)]
|
||||
enum ReadMode {
|
||||
/// Appendable on-disk format — opened via `open_appendable`.
|
||||
Appendable,
|
||||
/// Immutable on-disk format — opened via `open_immutable`. `is_on_disk`
|
||||
/// selects keeping the data on disk versus loading it into memory.
|
||||
Immutable { is_on_disk: bool },
|
||||
}
|
||||
|
||||
impl<S: UniversalRead> ReadOnlyFieldIndex<S> {
|
||||
/// Read-only mirror of [`IndexSelector::new_index_with_type`][1]: dispatches
|
||||
/// on [`FullPayloadIndexType::index_type`] and forwards to each per-index
|
||||
/// parent's open, wrapping the leaf in the matching variant.
|
||||
///
|
||||
/// The open path (appendable vs immutable) is picked from the stored
|
||||
/// [`FullPayloadIndexType::storage_type`]; `is_on_disk` rides on the
|
||||
/// immutable mode. Generic over `S`: every per-index open threads the
|
||||
/// [`UniversalRead`] handle `fs` (the map, numeric, geo and full-text leaves
|
||||
/// are all fs-generic), so the dispatcher needn't fix a concrete backend.
|
||||
///
|
||||
/// `payload_schema` is consulted only by the full-text arm (it carries the
|
||||
/// [`TextIndexParams`] the leaf open needs) and `total_point_count` only by
|
||||
/// the null arm (it is segment-wide, not recoverable from the index files);
|
||||
/// the other arms ignore both, mirroring the writable selector.
|
||||
/// `deleted_points` reaches the immutable-only leaves; the roaring-flag bool
|
||||
/// and null leaves ignore it (a single `open` serves both modes).
|
||||
///
|
||||
/// [1]: crate::index::field_index::index_selector::IndexSelector::new_index_with_type
|
||||
#[allow(dead_code)] // no caller in the lib yet
|
||||
pub fn open(
|
||||
fs: &S::Fs,
|
||||
dir: &Path,
|
||||
field: &JsonPath,
|
||||
payload_schema: &PayloadFieldSchema,
|
||||
index_type: &FullPayloadIndexType,
|
||||
total_point_count: usize,
|
||||
deleted_points: &BitSlice,
|
||||
) -> OperationResult<Option<Self>> {
|
||||
let mode = match index_type.storage_type {
|
||||
StorageType::Gridstore => ReadMode::Appendable,
|
||||
StorageType::Mmap { is_on_disk } => ReadMode::Immutable { is_on_disk },
|
||||
};
|
||||
|
||||
let index = match index_type.index_type {
|
||||
PayloadIndexType::KeywordIndex => match mode {
|
||||
ReadMode::Appendable => {
|
||||
ReadOnlyMapIndex::<str, S>::open_appendable(fs, map_dir(dir, field))?
|
||||
}
|
||||
ReadMode::Immutable { is_on_disk } => ReadOnlyMapIndex::<str, S>::open_immutable(
|
||||
fs,
|
||||
&map_dir(dir, field),
|
||||
is_on_disk,
|
||||
deleted_points,
|
||||
)?,
|
||||
}
|
||||
.map(Self::KeywordIndex),
|
||||
PayloadIndexType::IntMapIndex => match mode {
|
||||
ReadMode::Appendable => {
|
||||
ReadOnlyMapIndex::<IntPayloadType, S>::open_appendable(fs, map_dir(dir, field))?
|
||||
}
|
||||
ReadMode::Immutable { is_on_disk } => {
|
||||
ReadOnlyMapIndex::<IntPayloadType, S>::open_immutable(
|
||||
fs,
|
||||
&map_dir(dir, field),
|
||||
is_on_disk,
|
||||
deleted_points,
|
||||
)?
|
||||
}
|
||||
}
|
||||
.map(Self::IntMapIndex),
|
||||
// Matches the writable selector's `(PayloadIndexType::UuidIndex,
|
||||
// PayloadSchemaParams::Uuid(_))` arm, which constructs a
|
||||
// `MapIndex<UuidIntType>` and wraps it in `FieldIndex::UuidMapIndex`
|
||||
// — the `UuidIndex` discriminant is historically map-backed.
|
||||
PayloadIndexType::UuidIndex | PayloadIndexType::UuidMapIndex => match mode {
|
||||
ReadMode::Appendable => {
|
||||
ReadOnlyMapIndex::<UuidIntType, S>::open_appendable(fs, map_dir(dir, field))?
|
||||
}
|
||||
ReadMode::Immutable { is_on_disk } => {
|
||||
ReadOnlyMapIndex::<UuidIntType, S>::open_immutable(
|
||||
fs,
|
||||
&map_dir(dir, field),
|
||||
is_on_disk,
|
||||
deleted_points,
|
||||
)?
|
||||
}
|
||||
}
|
||||
.map(Self::UuidMapIndex),
|
||||
PayloadIndexType::IntIndex => match mode {
|
||||
ReadMode::Appendable => {
|
||||
ReadOnlyNumericIndex::<IntPayloadType, IntPayloadType, S>::open_appendable(
|
||||
fs,
|
||||
numeric_dir(dir, field),
|
||||
)?
|
||||
}
|
||||
ReadMode::Immutable { is_on_disk } => {
|
||||
ReadOnlyNumericIndex::<IntPayloadType, IntPayloadType, S>::open_immutable(
|
||||
fs,
|
||||
&numeric_dir(dir, field),
|
||||
is_on_disk,
|
||||
deleted_points,
|
||||
)?
|
||||
}
|
||||
}
|
||||
.map(Self::IntIndex),
|
||||
PayloadIndexType::DatetimeIndex => match mode {
|
||||
ReadMode::Appendable => {
|
||||
ReadOnlyNumericIndex::<IntPayloadType, DateTimePayloadType, S>::open_appendable(
|
||||
fs,
|
||||
numeric_dir(dir, field),
|
||||
)?
|
||||
}
|
||||
ReadMode::Immutable { is_on_disk } => {
|
||||
ReadOnlyNumericIndex::<IntPayloadType, DateTimePayloadType, S>::open_immutable(
|
||||
fs,
|
||||
&numeric_dir(dir, field),
|
||||
is_on_disk,
|
||||
deleted_points,
|
||||
)?
|
||||
}
|
||||
}
|
||||
.map(Self::DatetimeIndex),
|
||||
PayloadIndexType::FloatIndex => match mode {
|
||||
ReadMode::Appendable => {
|
||||
ReadOnlyNumericIndex::<FloatPayloadType, FloatPayloadType, S>::open_appendable(
|
||||
fs,
|
||||
numeric_dir(dir, field),
|
||||
)?
|
||||
}
|
||||
ReadMode::Immutable { is_on_disk } => {
|
||||
ReadOnlyNumericIndex::<FloatPayloadType, FloatPayloadType, S>::open_immutable(
|
||||
fs,
|
||||
&numeric_dir(dir, field),
|
||||
is_on_disk,
|
||||
deleted_points,
|
||||
)?
|
||||
}
|
||||
}
|
||||
.map(Self::FloatIndex),
|
||||
// Geo reuses the writable selector's `map_dir` (`-map` suffix).
|
||||
PayloadIndexType::GeoIndex => match mode {
|
||||
ReadMode::Appendable => {
|
||||
ReadOnlyGeoMapIndex::open_gridstore(fs, map_dir(dir, field))?
|
||||
}
|
||||
ReadMode::Immutable { is_on_disk } => ReadOnlyGeoMapIndex::open_mmap(
|
||||
fs,
|
||||
&map_dir(dir, field),
|
||||
is_on_disk,
|
||||
deleted_points,
|
||||
)?,
|
||||
}
|
||||
.map(Self::GeoIndex),
|
||||
PayloadIndexType::FullTextIndex => {
|
||||
let config = TextIndexParams::try_from(payload_schema)?;
|
||||
match mode {
|
||||
ReadMode::Appendable => {
|
||||
ReadOnlyFullTextIndex::open_appendable(fs, text_dir(dir, field), config)?
|
||||
}
|
||||
ReadMode::Immutable { is_on_disk } => ReadOnlyFullTextIndex::open_immutable(
|
||||
fs,
|
||||
text_dir(dir, field),
|
||||
config,
|
||||
is_on_disk,
|
||||
deleted_points,
|
||||
)?,
|
||||
}
|
||||
.map(Self::FullTextIndex)
|
||||
}
|
||||
// Bool and null are roaring-flag backed: a single read-only `open`
|
||||
// serves both modes (neither consumes the immutable-only
|
||||
// `is_on_disk` / `deleted_points`).
|
||||
PayloadIndexType::BoolIndex => {
|
||||
ReadOnlyBoolIndex::open::<S>(fs, &bool_dir(dir, field))?.map(Self::BoolIndex)
|
||||
}
|
||||
PayloadIndexType::NullIndex => {
|
||||
ReadOnlyNullIndex::open::<S>(fs, &null_dir(dir, field), total_point_count)?
|
||||
.map(Self::NullIndex)
|
||||
}
|
||||
};
|
||||
Ok(index)
|
||||
}
|
||||
}
|
||||
@@ -1,20 +1,28 @@
|
||||
mod lifecycle;
|
||||
mod read_ops;
|
||||
|
||||
use std::fmt::{Debug, Formatter};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use common::universal_io::UniversalRead;
|
||||
|
||||
use crate::common::operation_error::OperationResult;
|
||||
use crate::index::field_index::bool_index::ReadOnlyBoolIndex;
|
||||
use crate::index::field_index::full_text_index::read_only::ReadOnlyFullTextIndex;
|
||||
use crate::index::field_index::geo_index::ReadOnlyGeoMapIndex;
|
||||
use crate::index::field_index::map_index::read_only::ReadOnlyMapIndex;
|
||||
use crate::index::field_index::null_index::ReadOnlyNullIndex;
|
||||
use crate::index::field_index::numeric_index::ReadOnlyNumericIndex;
|
||||
use crate::index::payload_config::{
|
||||
FullPayloadIndexType, IndexMutability, PayloadIndexType, StorageType,
|
||||
};
|
||||
use crate::types::{
|
||||
DateTimePayloadType, FloatPayloadType, IntPayloadType, UuidIntType, UuidPayloadType,
|
||||
};
|
||||
|
||||
// Not yet wired into the broader read-path; lifecycle and constructors land
|
||||
// in a follow-up. Variants intentionally share the `*Index` postfix to mirror
|
||||
// `FieldIndex`.
|
||||
// `lifecycle::open_gridstore` / `open_mmap` construct every variant, but they
|
||||
// have no in-lib caller yet, so the variants would trip `dead_code`. Allow at
|
||||
// the enum level until a read-only segment wires the opens in.
|
||||
#[allow(dead_code, clippy::enum_variant_names)]
|
||||
pub enum ReadOnlyFieldIndex<S: UniversalRead> {
|
||||
IntIndex(ReadOnlyNumericIndex<IntPayloadType, IntPayloadType, S>),
|
||||
@@ -29,3 +37,316 @@ pub enum ReadOnlyFieldIndex<S: UniversalRead> {
|
||||
UuidMapIndex(ReadOnlyMapIndex<UuidIntType, S>),
|
||||
NullIndex(ReadOnlyNullIndex),
|
||||
}
|
||||
|
||||
/// Mirrors [`impl Debug for FieldIndex`][1] one-for-one: each arm prints
|
||||
/// the variant's discriminant. No payload is rendered (matches the
|
||||
/// writable side, where the underlying typed index is also not formatted).
|
||||
///
|
||||
/// [1]: crate::index::field_index::FieldIndex
|
||||
impl<S: UniversalRead> Debug for ReadOnlyFieldIndex<S> {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
ReadOnlyFieldIndex::IntIndex(_) => write!(f, "IntIndex"),
|
||||
ReadOnlyFieldIndex::DatetimeIndex(_) => write!(f, "DatetimeIndex"),
|
||||
ReadOnlyFieldIndex::IntMapIndex(_) => write!(f, "IntMapIndex"),
|
||||
ReadOnlyFieldIndex::KeywordIndex(_) => write!(f, "KeywordIndex"),
|
||||
ReadOnlyFieldIndex::FloatIndex(_) => write!(f, "FloatIndex"),
|
||||
ReadOnlyFieldIndex::GeoIndex(_) => write!(f, "GeoIndex"),
|
||||
ReadOnlyFieldIndex::BoolIndex(_) => write!(f, "BoolIndex"),
|
||||
ReadOnlyFieldIndex::FullTextIndex(_) => write!(f, "FullTextIndex"),
|
||||
ReadOnlyFieldIndex::UuidIndex(_) => write!(f, "UuidIndex"),
|
||||
ReadOnlyFieldIndex::UuidMapIndex(_) => write!(f, "UuidMapIndex"),
|
||||
ReadOnlyFieldIndex::NullIndex(_) => write!(f, "NullIndex"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read-only mirror of the lifecycle / introspection surface on the
|
||||
/// writable [`FieldIndex`][1].
|
||||
///
|
||||
/// Skipped from the writable surface:
|
||||
/// - `wipe(self)`, `flusher`, `add_point`, `remove_point` — write-only;
|
||||
/// they never make sense on the read-only wrapper.
|
||||
///
|
||||
/// Mirrored here:
|
||||
/// - [`Self::files`] / [`Self::immutable_files`] — file enumeration for
|
||||
/// cache management.
|
||||
/// - [`Self::is_on_disk`] / [`Self::ram_usage_bytes`] — telemetry /
|
||||
/// placement queries.
|
||||
/// - [`Self::populate`] / [`Self::clear_cache`] — OS page-cache control.
|
||||
/// - [`Self::get_full_index_type`] / [`Self::get_mutability_type`] /
|
||||
/// [`Self::get_storage_type`] — payload-config round-tripping.
|
||||
///
|
||||
/// **Skeleton state.** Most arms are still `todo!` placeholders because the
|
||||
/// per-index parent enums don't yet expose the matching inherent methods
|
||||
/// in the branches that supply their `open_*`:
|
||||
/// - [`ReadOnlyMapIndex`] only has `get_mutability_type` (added with the
|
||||
/// map open PR), so the map-backed variants of `get_mutability_type` /
|
||||
/// `get_full_index_type` are wired here.
|
||||
/// - Every other lifecycle method (`populate`, `files`, ...) needs a small
|
||||
/// follow-up that adds the corresponding method on each
|
||||
/// `ReadOnly*Index` parent, dispatching to the leaf variant. That
|
||||
/// follow-up lifts every arm in this block out of `todo!`.
|
||||
///
|
||||
/// The `todo!` arms are intentional skeleton placeholders, not soft
|
||||
/// failures — they panic if hit at runtime so the missing wiring surfaces
|
||||
/// immediately rather than degrading silently.
|
||||
///
|
||||
/// [1]: crate::index::field_index::FieldIndex
|
||||
#[allow(dead_code)] // skeleton: no caller in the lib yet; surface is here for follow-ups
|
||||
impl<S: UniversalRead> ReadOnlyFieldIndex<S> {
|
||||
pub fn files(&self) -> Vec<PathBuf> {
|
||||
match self {
|
||||
ReadOnlyFieldIndex::IntMapIndex(_)
|
||||
| ReadOnlyFieldIndex::KeywordIndex(_)
|
||||
| ReadOnlyFieldIndex::UuidMapIndex(_) => {
|
||||
todo!("follow-up: add `ReadOnlyMapIndex::files` and dispatch")
|
||||
}
|
||||
ReadOnlyFieldIndex::IntIndex(_)
|
||||
| ReadOnlyFieldIndex::DatetimeIndex(_)
|
||||
| ReadOnlyFieldIndex::FloatIndex(_)
|
||||
| ReadOnlyFieldIndex::UuidIndex(_) => {
|
||||
todo!("blocked on #9213 (numeric parent) + `files` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::GeoIndex(_) => {
|
||||
todo!("blocked on #9211 (geo parent) + `files` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::FullTextIndex(_) => {
|
||||
todo!("blocked on #9222 (full-text parent) + `files` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::BoolIndex(_) => {
|
||||
todo!("blocked on #9200 (bool parent) + `files` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::NullIndex(_) => {
|
||||
todo!("blocked on #9197 (null parent) + `files` on it")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn immutable_files(&self) -> Vec<PathBuf> {
|
||||
match self {
|
||||
ReadOnlyFieldIndex::IntMapIndex(_)
|
||||
| ReadOnlyFieldIndex::KeywordIndex(_)
|
||||
| ReadOnlyFieldIndex::UuidMapIndex(_) => {
|
||||
todo!("follow-up: add `ReadOnlyMapIndex::immutable_files` and dispatch")
|
||||
}
|
||||
ReadOnlyFieldIndex::IntIndex(_)
|
||||
| ReadOnlyFieldIndex::DatetimeIndex(_)
|
||||
| ReadOnlyFieldIndex::FloatIndex(_)
|
||||
| ReadOnlyFieldIndex::UuidIndex(_) => {
|
||||
todo!("blocked on #9213 (numeric parent) + `immutable_files` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::GeoIndex(_) => {
|
||||
todo!("blocked on #9211 (geo parent) + `immutable_files` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::FullTextIndex(_) => {
|
||||
todo!("blocked on #9222 (full-text parent) + `immutable_files` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::BoolIndex(_) => {
|
||||
todo!("blocked on #9200 (bool parent) + `immutable_files` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::NullIndex(_) => {
|
||||
todo!("blocked on #9197 (null parent) + `immutable_files` on it")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn ram_usage_bytes(&self) -> usize {
|
||||
match self {
|
||||
ReadOnlyFieldIndex::IntMapIndex(_)
|
||||
| ReadOnlyFieldIndex::KeywordIndex(_)
|
||||
| ReadOnlyFieldIndex::UuidMapIndex(_) => {
|
||||
todo!("follow-up: add `ReadOnlyMapIndex::ram_usage_bytes` and dispatch")
|
||||
}
|
||||
ReadOnlyFieldIndex::IntIndex(_)
|
||||
| ReadOnlyFieldIndex::DatetimeIndex(_)
|
||||
| ReadOnlyFieldIndex::FloatIndex(_)
|
||||
| ReadOnlyFieldIndex::UuidIndex(_) => {
|
||||
todo!("blocked on #9213 (numeric parent) + `ram_usage_bytes` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::GeoIndex(_) => {
|
||||
todo!("blocked on #9211 (geo parent) + `ram_usage_bytes` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::FullTextIndex(_) => {
|
||||
todo!("blocked on #9222 (full-text parent) + `ram_usage_bytes` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::BoolIndex(_) => {
|
||||
todo!("blocked on #9200 (bool parent) + `ram_usage_bytes` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::NullIndex(_) => {
|
||||
todo!("blocked on #9197 (null parent) + `ram_usage_bytes` on it")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_on_disk(&self) -> bool {
|
||||
match self {
|
||||
ReadOnlyFieldIndex::IntMapIndex(_)
|
||||
| ReadOnlyFieldIndex::KeywordIndex(_)
|
||||
| ReadOnlyFieldIndex::UuidMapIndex(_) => {
|
||||
todo!("follow-up: add `ReadOnlyMapIndex::is_on_disk` and dispatch")
|
||||
}
|
||||
ReadOnlyFieldIndex::IntIndex(_)
|
||||
| ReadOnlyFieldIndex::DatetimeIndex(_)
|
||||
| ReadOnlyFieldIndex::FloatIndex(_)
|
||||
| ReadOnlyFieldIndex::UuidIndex(_) => {
|
||||
todo!("blocked on #9213 (numeric parent) + `is_on_disk` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::GeoIndex(_) => {
|
||||
todo!("blocked on #9211 (geo parent) + `is_on_disk` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::FullTextIndex(_) => {
|
||||
todo!("blocked on #9222 (full-text parent) + `is_on_disk` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::BoolIndex(_) => {
|
||||
todo!("blocked on #9200 (bool parent) + `is_on_disk` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::NullIndex(_) => {
|
||||
todo!("blocked on #9197 (null parent) + `is_on_disk` on it")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Populate all pages in the mmap. Block until all pages are populated.
|
||||
pub fn populate(&self) -> OperationResult<()> {
|
||||
match self {
|
||||
ReadOnlyFieldIndex::IntMapIndex(_)
|
||||
| ReadOnlyFieldIndex::KeywordIndex(_)
|
||||
| ReadOnlyFieldIndex::UuidMapIndex(_) => {
|
||||
todo!("follow-up: add `ReadOnlyMapIndex::populate` and dispatch")
|
||||
}
|
||||
ReadOnlyFieldIndex::IntIndex(_)
|
||||
| ReadOnlyFieldIndex::DatetimeIndex(_)
|
||||
| ReadOnlyFieldIndex::FloatIndex(_)
|
||||
| ReadOnlyFieldIndex::UuidIndex(_) => {
|
||||
todo!("blocked on #9213 (numeric parent) + `populate` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::GeoIndex(_) => {
|
||||
todo!("blocked on #9211 (geo parent) + `populate` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::FullTextIndex(_) => {
|
||||
todo!("blocked on #9222 (full-text parent) + `populate` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::BoolIndex(_) => {
|
||||
todo!("blocked on #9200 (bool parent) + `populate` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::NullIndex(_) => {
|
||||
todo!("blocked on #9197 (null parent) + `populate` on it")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop disk cache.
|
||||
pub fn clear_cache(&self) -> OperationResult<()> {
|
||||
match self {
|
||||
ReadOnlyFieldIndex::IntMapIndex(_)
|
||||
| ReadOnlyFieldIndex::KeywordIndex(_)
|
||||
| ReadOnlyFieldIndex::UuidMapIndex(_) => {
|
||||
todo!("follow-up: add `ReadOnlyMapIndex::clear_cache` and dispatch")
|
||||
}
|
||||
ReadOnlyFieldIndex::IntIndex(_)
|
||||
| ReadOnlyFieldIndex::DatetimeIndex(_)
|
||||
| ReadOnlyFieldIndex::FloatIndex(_)
|
||||
| ReadOnlyFieldIndex::UuidIndex(_) => {
|
||||
todo!("blocked on #9213 (numeric parent) + `clear_cache` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::GeoIndex(_) => {
|
||||
todo!("blocked on #9211 (geo parent) + `clear_cache` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::FullTextIndex(_) => {
|
||||
todo!("blocked on #9222 (full-text parent) + `clear_cache` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::BoolIndex(_) => {
|
||||
todo!("blocked on #9200 (bool parent) + `clear_cache` on it")
|
||||
}
|
||||
ReadOnlyFieldIndex::NullIndex(_) => {
|
||||
todo!("blocked on #9197 (null parent) + `clear_cache` on it")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Composes [`FullPayloadIndexType`] from the discriminant + the per-arm
|
||||
/// `get_mutability_type` / `get_storage_type` — same shape as the
|
||||
/// writable side.
|
||||
pub fn get_full_index_type(&self) -> FullPayloadIndexType {
|
||||
let index_type = match self {
|
||||
ReadOnlyFieldIndex::IntIndex(_) => PayloadIndexType::IntIndex,
|
||||
ReadOnlyFieldIndex::DatetimeIndex(_) => PayloadIndexType::DatetimeIndex,
|
||||
ReadOnlyFieldIndex::IntMapIndex(_) => PayloadIndexType::IntMapIndex,
|
||||
ReadOnlyFieldIndex::KeywordIndex(_) => PayloadIndexType::KeywordIndex,
|
||||
ReadOnlyFieldIndex::FloatIndex(_) => PayloadIndexType::FloatIndex,
|
||||
ReadOnlyFieldIndex::GeoIndex(_) => PayloadIndexType::GeoIndex,
|
||||
ReadOnlyFieldIndex::FullTextIndex(_) => PayloadIndexType::FullTextIndex,
|
||||
ReadOnlyFieldIndex::BoolIndex(_) => PayloadIndexType::BoolIndex,
|
||||
ReadOnlyFieldIndex::UuidIndex(_) => PayloadIndexType::UuidIndex,
|
||||
ReadOnlyFieldIndex::UuidMapIndex(_) => PayloadIndexType::UuidMapIndex,
|
||||
ReadOnlyFieldIndex::NullIndex(_) => PayloadIndexType::NullIndex,
|
||||
};
|
||||
FullPayloadIndexType {
|
||||
index_type,
|
||||
mutability: self.get_mutability_type(),
|
||||
storage_type: self.get_storage_type(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Map-backed variants dispatch to the wired
|
||||
/// [`ReadOnlyMapIndex::get_mutability_type`][1]; the rest are blocked
|
||||
/// on the per-index parent `get_mutability_type`, added with each
|
||||
/// per-index PR listed in [`Self::populate`].
|
||||
///
|
||||
/// [1]: crate::index::field_index::map_index::read_only::ReadOnlyMapIndex::get_mutability_type
|
||||
fn get_mutability_type(&self) -> IndexMutability {
|
||||
match self {
|
||||
ReadOnlyFieldIndex::IntMapIndex(index) => index.get_mutability_type(),
|
||||
ReadOnlyFieldIndex::KeywordIndex(index) => index.get_mutability_type(),
|
||||
ReadOnlyFieldIndex::UuidMapIndex(index) => index.get_mutability_type(),
|
||||
|
||||
ReadOnlyFieldIndex::IntIndex(_)
|
||||
| ReadOnlyFieldIndex::DatetimeIndex(_)
|
||||
| ReadOnlyFieldIndex::FloatIndex(_)
|
||||
| ReadOnlyFieldIndex::UuidIndex(_) => {
|
||||
todo!("blocked on #9213 (numeric `get_mutability_type`)")
|
||||
}
|
||||
ReadOnlyFieldIndex::GeoIndex(_) => {
|
||||
todo!("blocked on #9211 (geo `get_mutability_type`)")
|
||||
}
|
||||
ReadOnlyFieldIndex::FullTextIndex(_) => {
|
||||
todo!("blocked on #9222 (full-text `get_mutability_type`)")
|
||||
}
|
||||
ReadOnlyFieldIndex::BoolIndex(_) => {
|
||||
todo!("blocked on #9200 (bool `get_mutability_type`)")
|
||||
}
|
||||
ReadOnlyFieldIndex::NullIndex(_) => {
|
||||
todo!("blocked on #9197 (null `get_mutability_type`)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_storage_type(&self) -> StorageType {
|
||||
match self {
|
||||
ReadOnlyFieldIndex::IntMapIndex(_)
|
||||
| ReadOnlyFieldIndex::KeywordIndex(_)
|
||||
| ReadOnlyFieldIndex::UuidMapIndex(_) => {
|
||||
todo!("follow-up: add `ReadOnlyMapIndex::get_storage_type` and dispatch")
|
||||
}
|
||||
ReadOnlyFieldIndex::IntIndex(_)
|
||||
| ReadOnlyFieldIndex::DatetimeIndex(_)
|
||||
| ReadOnlyFieldIndex::FloatIndex(_)
|
||||
| ReadOnlyFieldIndex::UuidIndex(_) => {
|
||||
todo!("blocked on #9213 (numeric `get_storage_type`)")
|
||||
}
|
||||
ReadOnlyFieldIndex::GeoIndex(_) => {
|
||||
todo!("blocked on #9211 (geo `get_storage_type`)")
|
||||
}
|
||||
ReadOnlyFieldIndex::FullTextIndex(_) => {
|
||||
todo!("blocked on #9222 (full-text `get_storage_type`)")
|
||||
}
|
||||
ReadOnlyFieldIndex::BoolIndex(_) => {
|
||||
todo!("blocked on #9200 (bool `get_storage_type`)")
|
||||
}
|
||||
ReadOnlyFieldIndex::NullIndex(_) => {
|
||||
todo!("blocked on #9197 (null `get_storage_type`)")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -561,22 +561,22 @@ impl IndexSelector<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
fn map_dir(dir: &Path, field: &JsonPath) -> PathBuf {
|
||||
pub(crate) fn map_dir(dir: &Path, field: &JsonPath) -> PathBuf {
|
||||
dir.join(format!("{}-map", &field.filename()))
|
||||
}
|
||||
|
||||
fn numeric_dir(dir: &Path, field: &JsonPath) -> PathBuf {
|
||||
pub(crate) fn numeric_dir(dir: &Path, field: &JsonPath) -> PathBuf {
|
||||
dir.join(format!("{}-numeric", &field.filename()))
|
||||
}
|
||||
|
||||
fn text_dir(dir: &Path, field: &JsonPath) -> PathBuf {
|
||||
pub(crate) fn text_dir(dir: &Path, field: &JsonPath) -> PathBuf {
|
||||
dir.join(format!("{}-text", &field.filename()))
|
||||
}
|
||||
|
||||
fn bool_dir(dir: &Path, field: &JsonPath) -> PathBuf {
|
||||
pub(crate) fn bool_dir(dir: &Path, field: &JsonPath) -> PathBuf {
|
||||
dir.join(format!("{}-bool", &field.filename()))
|
||||
}
|
||||
|
||||
fn null_dir(dir: &Path, field: &JsonPath) -> PathBuf {
|
||||
pub(crate) fn null_dir(dir: &Path, field: &JsonPath) -> PathBuf {
|
||||
dir.join(format!("{}-null", &field.filename()))
|
||||
}
|
||||
|
||||
@@ -172,7 +172,6 @@ pub enum IndexMutability {
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum StorageType {
|
||||
Gridstore,
|
||||
RocksDb,
|
||||
Mmap { is_on_disk: bool },
|
||||
}
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ use super::field_index::index_selector::{
|
||||
IndexSelector, IndexSelectorGridstore, IndexSelectorMmap,
|
||||
};
|
||||
use super::payload_config::{FullPayloadIndexType, PayloadFieldSchemaWithIndexType};
|
||||
use crate::common::operation_error::{OperationError, OperationResult};
|
||||
use crate::common::operation_error::OperationResult;
|
||||
use crate::common::utils::IndexesMap;
|
||||
use crate::id_tracker::{IdTrackerEnum, IdTrackerRead};
|
||||
use crate::index::payload_config::{self, PayloadConfig};
|
||||
@@ -147,16 +147,15 @@ impl StructPayloadIndex {
|
||||
.iter()
|
||||
// Load each index
|
||||
.map(|index| {
|
||||
self.selector_with_type(index).and_then(|selector| {
|
||||
selector.new_index_with_type(
|
||||
field,
|
||||
&payload_schema.schema,
|
||||
index,
|
||||
create_if_missing,
|
||||
&id_tracker_borrow,
|
||||
deleted_points,
|
||||
)
|
||||
})
|
||||
let selector = self.selector_with_type(index);
|
||||
selector.new_index_with_type(
|
||||
field,
|
||||
&payload_schema.schema,
|
||||
index,
|
||||
create_if_missing,
|
||||
&id_tracker_borrow,
|
||||
deleted_points,
|
||||
)
|
||||
})
|
||||
// Interrupt loading indices if one fails to load
|
||||
// Set rebuild flag if any index fails to load
|
||||
@@ -287,28 +286,18 @@ impl StructPayloadIndex {
|
||||
}
|
||||
}
|
||||
|
||||
fn selector_with_type(
|
||||
&self,
|
||||
index_type: &FullPayloadIndexType,
|
||||
) -> OperationResult<IndexSelector<'_>> {
|
||||
let selector = match index_type.storage_type {
|
||||
fn selector_with_type(&self, index_type: &FullPayloadIndexType) -> IndexSelector<'_> {
|
||||
match index_type.storage_type {
|
||||
payload_config::StorageType::Gridstore => {
|
||||
IndexSelector::Gridstore(IndexSelectorGridstore { dir: &self.path })
|
||||
}
|
||||
payload_config::StorageType::RocksDb => {
|
||||
return Err(OperationError::service_error(
|
||||
"Loading payload index failed: Index is RocksDB but RocksDB feature is disabled.",
|
||||
));
|
||||
}
|
||||
payload_config::StorageType::Mmap { is_on_disk } => {
|
||||
IndexSelector::Mmap(IndexSelectorMmap {
|
||||
dir: &self.path,
|
||||
is_on_disk,
|
||||
})
|
||||
}
|
||||
};
|
||||
|
||||
Ok(selector)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn populate(&self) -> OperationResult<()> {
|
||||
|
||||
@@ -2364,6 +2364,24 @@ impl Display for PayloadFieldSchema {
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<&PayloadFieldSchema> for TextIndexParams {
|
||||
type Error = OperationError;
|
||||
|
||||
/// Extracts the full-text tokenizer params from a payload schema — used by
|
||||
/// the read-only full-text index `open`, the only index whose read
|
||||
/// behavior depends on its build-time config. Errors if the schema is not
|
||||
/// a text index.
|
||||
fn try_from(schema: &PayloadFieldSchema) -> Result<Self, Self::Error> {
|
||||
let expanded = schema.expand();
|
||||
let PayloadSchemaParams::Text(config) = expanded.as_ref() else {
|
||||
return Err(OperationError::service_error(
|
||||
"expected a text payload schema for a full-text index",
|
||||
));
|
||||
};
|
||||
Ok(config.clone())
|
||||
}
|
||||
}
|
||||
|
||||
impl PayloadFieldSchema {
|
||||
pub fn expand(&self) -> Cow<'_, PayloadSchemaParams> {
|
||||
match self {
|
||||
|
||||
Reference in New Issue
Block a user