upstream-archive/qdrant
1
0
Fork 0
mirror of https://github.com/qdrant/qdrant.git synced 2026-09-21 21:47:39 -05:00
Code Issues Packages Projects Releases Wiki Activity

Standardize full text index (#9387)

Browse Source 
* rename to OnDiskFullTextIndex

* rename module to `on_disk_text_index`

* rename to OnDiskPostings

* rename module to `on_disk_inverted_index`

* rename to OnDiskInvertedIndex

* remove `Box`ing

* get rid of `is_on_disk` field, propagate Populate

* add separate OnDisk and Immutable readonly variants

* rename in tests too

* nits
This commit is contained in:
Luis Cossío
2026-08-04 11:16:49 +02:00
committed by generall
parent 8beb3a08cf
commit 84ccdb37fc
23 changed files with 196 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/segment/src/index/field_index/field_index_base/builder.rs
+1 -1
View File
@@ -8,7 +8,7 @@ use crate::index::field_index::bool_index::BoolIndex;
use crate::index::field_index::bool_index::immutable_bool_index::ImmutableBoolIndexBuilder;
use crate::index::field_index::bool_index::mutable_bool_index::MutableBoolIndexBuilder;
use crate::index::field_index::full_text_index::FullTextGridstoreIndexBuilder;
use crate::index::field_index::full_text_index::mmap_text_index::FullTextMmapIndexBuilder;
use crate::index::field_index::full_text_index::on_disk_text_index::FullTextMmapIndexBuilder;
use crate::index::field_index::geo_index::{GeoIndexGridstoreBuilder, GeoIndexMmapBuilder};
use crate::index::field_index::map_index::{MapIndexGridstoreBuilder, MapIndexMmapBuilder};
use crate::index::field_index::null_index::NullIndex;
lib/segment/src/index/field_index/full_text_index/immutable_text_index/lifecycle.rs
+7 -8
View File
@@ -1,24 +1,23 @@
use std::path::PathBuf;
use common::types::PointOffsetType;
use common::universal_io::MmapFile;
use common::universal_io::UniversalRead;
use super::super::inverted_index::InvertedIndex;
use super::super::inverted_index::immutable_inverted_index::ImmutableInvertedIndex;
use super::super::mmap_text_index::MmapFullTextIndex;
use super::super::on_disk_text_index::OnDiskFullTextIndex;
use super::ImmutableFullTextIndex;
use crate::common::Flusher;
use crate::common::operation_error::{OperationError, OperationResult};
impl ImmutableFullTextIndex {
/// Open and load the immutable full text index from mmap storage.
pub fn open_mmap(index: MmapFullTextIndex<MmapFile>) -> OperationResult<Self> {
let index = Box::new(index);
impl<S: UniversalRead> ImmutableFullTextIndex<S> {
/// Open and load the immutable full text index from on-disk storage.
pub fn load_from_on_disk(index: OnDiskFullTextIndex<S>) -> OperationResult<Self> {
let inverted_index = ImmutableInvertedIndex::try_from(&index.inverted_index)?;
// Index is now loaded into memory, clear cache of backing mmap storage
// Index is now loaded into memory, clear cache of backing on-disk storage
if let Err(err) = index.inverted_index.clear_cache() {
log::warn!("Failed to clear mmap cache of ram mmap full text index: {err}");
log::warn!("Failed to clear cache of on-disk full text index: {err}");
}
let mut result = Self {
lib/segment/src/index/field_index/full_text_index/immutable_text_index/mod.rs
+4 -4
View File
@@ -1,15 +1,15 @@
use common::universal_io::MmapFile;
use common::universal_io::{MmapFile, UniversalRead};
use super::inverted_index::immutable_inverted_index::ImmutableInvertedIndex;
use super::mmap_text_index::MmapFullTextIndex;
use super::on_disk_text_index::OnDiskFullTextIndex;
mod lifecycle;
mod read_ops;
pub struct ImmutableFullTextIndex {
pub struct ImmutableFullTextIndex<S: UniversalRead = MmapFile> {
pub(super) inverted_index: ImmutableInvertedIndex,
/// Backing mmap storage; source of state, persists deletions.
pub(super) storage: Box<MmapFullTextIndex<MmapFile>>,
pub(super) storage: OnDiskFullTextIndex<S>,
/// Snapshot of approximate RAM usage at construction time.
/// Not refreshed on `remove_point`.
pub(super) cached_ram_usage_bytes: usize,
lib/segment/src/index/field_index/full_text_index/immutable_text_index/read_ops.rs
+3 -5
View File
@@ -1,6 +1,6 @@
use common::counter::hardware_counter::HardwareCounterCell;
use common::types::PointOffsetType;
use common::universal_io::UserData;
use common::universal_io::{UniversalRead, UserData};
use super::super::full_text_index_read::FullTextIndexRead;
use super::super::inverted_index::{InvertedIndex, ParsedQuery, TokenId};
@@ -11,7 +11,7 @@ use crate::index::field_index::{CardinalityEstimation, PayloadBlockCondition};
use crate::index::payload_config::StorageType;
use crate::types::{FieldCondition, PayloadKeyType};
impl FullTextIndexRead for ImmutableFullTextIndex {
impl<S: UniversalRead> FullTextIndexRead for ImmutableFullTextIndex<S> {
fn tokenizer(&self) -> &Tokenizer {
&self.storage.tokenizer
}
@@ -74,9 +74,7 @@ impl FullTextIndexRead for ImmutableFullTextIndex {
}
fn get_storage_type(&self) -> StorageType {
StorageType::Mmap {
is_on_disk: self.storage.is_on_disk(),
}
StorageType::Mmap { is_on_disk: false }
}
fn ram_usage_bytes(&self) -> usize {
lib/segment/src/index/field_index/full_text_index/inverted_index/immutable_inverted_index.rs
+6 -6
View File
@@ -10,9 +10,9 @@ use itertools::Either;
use posting_list::{PostingBuilder, PostingList, PostingListView, PostingValue};
use super::immutable_postings_enum::ImmutablePostings;
use super::mmap_inverted_index::MmapInvertedIndex;
use super::mmap_inverted_index::mmap_postings_enum::MmapPostingsEnum;
use super::mutable_inverted_index::MutableInvertedIndex;
use super::on_disk_inverted_index::OnDiskInvertedIndex;
use super::on_disk_inverted_index::on_disk_postings_enum::OnDiskPostingsEnum;
use super::positions::Positions;
use super::postings_iterator::{
intersect_compressed_postings_iterator, merge_compressed_postings_iterator,
@@ -495,15 +495,15 @@ fn create_compressed_postings_with_positions(
.collect()
}
impl<S: common::universal_io::UniversalRead> TryFrom<&MmapInvertedIndex<S>>
impl<S: common::universal_io::UniversalRead> TryFrom<&OnDiskInvertedIndex<S>>
for ImmutableInvertedIndex
{
type Error = OperationError;
fn try_from(index: &MmapInvertedIndex<S>) -> OperationResult<Self> {
fn try_from(index: &OnDiskInvertedIndex<S>) -> OperationResult<Self> {
let postings = match &index.storage.postings {
MmapPostingsEnum::Ids(postings) => ImmutablePostings::Ids(postings.all_postings()?),
MmapPostingsEnum::WithPositions(postings) => {
OnDiskPostingsEnum::Ids(postings) => ImmutablePostings::Ids(postings.all_postings()?),
OnDiskPostingsEnum::WithPositions(postings) => {
ImmutablePostings::WithPositions(postings.all_postings()?)
}
};
lib/segment/src/index/field_index/full_text_index/inverted_index/mod.rs
+10 -10
View File
@@ -1,8 +1,8 @@
pub(super) mod immutable_inverted_index;
pub mod immutable_postings_enum;
pub(super) mod mmap_inverted_index;
pub(super) mod mutable_inverted_index;
pub(super) mod mutable_inverted_index_builder;
pub(super) mod on_disk_inverted_index;
mod positions;
mod posting_list;
mod postings_iterator;
@@ -414,15 +414,15 @@ mod tests {
use common::bitvec::{BitSliceExt, BitVec};
use common::counter::hardware_counter::HardwareCounterCell;
use common::universal_io::MmapFs;
use common::universal_io::{MmapFs, Populate};
use rand::RngExt;
use rand::seq::SliceRandom;
use rstest::rstest;
use super::{Document, InvertedIndex, ParsedQuery, TokenId, TokenSet};
use crate::index::field_index::full_text_index::inverted_index::immutable_inverted_index::ImmutableInvertedIndex;
use crate::index::field_index::full_text_index::inverted_index::mmap_inverted_index::MmapInvertedIndex;
use crate::index::field_index::full_text_index::inverted_index::mutable_inverted_index::MutableInvertedIndex;
use crate::index::field_index::full_text_index::inverted_index::on_disk_inverted_index::OnDiskInvertedIndex;
fn generate_word() -> String {
let mut rng = rand::rng();
@@ -560,12 +560,12 @@ mod tests {
let hw_counter = HardwareCounterCell::new();
MmapInvertedIndex::create(mmap_dir.path().into(), &immutable).unwrap();
OnDiskInvertedIndex::create(mmap_dir.path().into(), &immutable).unwrap();
let empty_deleted = BitVec::new();
let mmap: MmapInvertedIndex = MmapInvertedIndex::open(
let mmap: OnDiskInvertedIndex = OnDiskInvertedIndex::open(
&MmapFs,
mmap_dir.path().into(),
false,
Populate::No,
phrase_matching,
&empty_deleted,
)
@@ -641,12 +641,12 @@ mod tests {
let mut mut_index = mutable_inverted_index(indexed_count, deleted_count, phrase_matching);
let immutable = ImmutableInvertedIndex::from(mut_index.clone());
MmapInvertedIndex::create(mmap_dir.path().into(), &immutable).unwrap();
OnDiskInvertedIndex::create(mmap_dir.path().into(), &immutable).unwrap();
let empty_deleted = BitVec::new();
let mut mmap_index = MmapInvertedIndex::open(
let mut mmap_index = OnDiskInvertedIndex::open(
&MmapFs,
mmap_dir.path().into(),
false,
Populate::No,
phrase_matching,
&empty_deleted,
)
@@ -698,7 +698,7 @@ mod tests {
mmap_parsed_queries: &[Option<ParsedQuery>],
imm_mmap_parsed_queries: &[Option<ParsedQuery>],
mut_index: &MutableInvertedIndex,
mmap_index: &MmapInvertedIndex,
mmap_index: &OnDiskInvertedIndex,
imm_mmap_index: &ImmutableInvertedIndex,
hw_counter: &HardwareCounterCell,
) {
lib/segment/src/index/field_index/full_text_index/inverted_index/mmap_inverted_index/create_postings.rs → lib/segment/src/index/field_index/full_text_index/inverted_index/on_disk_inverted_index/create_postings.rs
+1 -1
View File
@@ -7,7 +7,7 @@ use fs_err::File;
use posting_list::{PostingList, PostingListComponents};
use zerocopy::IntoBytes;
use crate::index::field_index::full_text_index::inverted_index::mmap_inverted_index::types::{
use crate::index::field_index::full_text_index::inverted_index::on_disk_inverted_index::types::{
ALIGNMENT, PostingListHeader, PostingsHeader, ZerocopyPostingValue,
};
lib/segment/src/index/field_index/full_text_index/inverted_index/mmap_inverted_index/mod.rs → lib/segment/src/index/field_index/full_text_index/inverted_index/on_disk_inverted_index/mod.rs
+40 -48
View File
@@ -13,13 +13,13 @@ use common::types::PointOffsetType;
use common::universal_io::{
MmapFile, MmapFs, OpenOptions, Populate, ReadRange, TypedStorage, UniversalRead, UserData,
};
use on_disk_postings::OnDiskPostings;
use types::ZerocopyPostingValue;
use uio_postings::UniversalPostings;
use self::create_postings::create_postings_file;
use super::immutable_inverted_index::ImmutableInvertedIndex;
use super::immutable_postings_enum::ImmutablePostings;
use super::mmap_inverted_index::mmap_postings_enum::MmapPostingsEnum;
use super::on_disk_inverted_index::on_disk_postings_enum::OnDiskPostingsEnum;
use super::positions::Positions;
use super::postings_iterator::{
intersect_compressed_postings_iterator, merge_compressed_postings_iterator,
@@ -33,10 +33,10 @@ use crate::index::field_index::full_text_index::inverted_index::postings_iterato
};
mod create_postings;
pub mod mmap_postings_enum;
mod on_disk_postings;
pub mod on_disk_postings_enum;
mod raw_posting_list;
pub(in crate::index::field_index::full_text_index) mod types;
mod uio_postings;
const POSTINGS_FILE: &str = "postings.dat";
const VOCAB_FILE: &str = "vocab.dat";
@@ -55,16 +55,15 @@ const DELETED_POINTS_FILE: &str = "deleted_points.dat";
/// only updates the in-memory bitvec. Callers must re-supply the authoritative
/// deletion set (typically `id_tracker.deleted_point_bitslice()`) via the
/// `deleted_points` argument to [`Self::open`] on reload.
pub struct MmapInvertedIndex<S: UniversalRead = MmapFile> {
pub struct OnDiskInvertedIndex<S: UniversalRead = MmapFile> {
pub(in crate::index::field_index::full_text_index) path: PathBuf,
pub(in crate::index::field_index::full_text_index) storage: Storage<S>,
/// Number of points which are not deleted
pub(in crate::index::field_index::full_text_index) active_points_count: usize,
pub(in crate::index::field_index::full_text_index) is_on_disk: bool,
}
pub(in crate::index::field_index::full_text_index) struct Storage<S: UniversalRead = MmapFile> {
pub(in crate::index::field_index::full_text_index) postings: MmapPostingsEnum<S>,
pub(in crate::index::field_index::full_text_index) postings: OnDiskPostingsEnum<S>,
pub(in crate::index::field_index::full_text_index) vocab: UniversalHashMap<str, TokenId, S>,
pub(in crate::index::field_index::full_text_index) point_to_tokens_count:
TypedStorage<S, usize>,
@@ -84,7 +83,7 @@ impl<S: UniversalRead> Storage<S> {
}
}
impl MmapInvertedIndex<MmapFile> {
impl OnDiskInvertedIndex<MmapFile> {
pub fn create(path: PathBuf, inverted_index: &ImmutableInvertedIndex) -> OperationResult<()> {
let ImmutableInvertedIndex {
postings,
@@ -152,11 +151,11 @@ impl MmapInvertedIndex<MmapFile> {
}
}
impl<S: UniversalRead> MmapInvertedIndex<S> {
impl<S: UniversalRead> OnDiskInvertedIndex<S> {
pub fn open(
fs: &S::Fs,
path: PathBuf,
populate: bool,
populate: Populate,
has_positions: bool,
deleted_points: &BitSlice,
) -> OperationResult<Option<Self>> {
@@ -168,25 +167,25 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
let postings_open_options = OpenOptions {
writeable: false,
need_sequential: false,
populate: Populate::from(populate),
populate,
advice: AdviceSetting::Advice(Advice::Normal),
};
let Some(postings) = (match has_positions {
false => UniversalPostings::<(), S>::open(
false => OnDiskPostings::<(), S>::open(
fs,
&postings_path,
postings_open_options,
Default::default(),
)?
.map(MmapPostingsEnum::Ids),
true => UniversalPostings::<Positions, S>::open(
.map(OnDiskPostingsEnum::Ids),
true => OnDiskPostings::<Positions, S>::open(
fs,
&postings_path,
postings_open_options,
Default::default(),
)?
.map(MmapPostingsEnum::WithPositions),
.map(OnDiskPostingsEnum::WithPositions),
}) else {
// If postings don't exist, assume the index doesn't exist on disk
return Ok(None);
@@ -197,7 +196,7 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
OpenOptions {
writeable: false,
need_sequential: false,
populate: Populate::from(populate),
populate,
advice: AdviceSetting::Global,
},
Default::default(),
@@ -209,7 +208,7 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
OpenOptions {
writeable: false,
need_sequential: false,
populate: Populate::from(populate),
populate,
advice: AdviceSetting::Global,
},
Default::default(),
@@ -221,7 +220,7 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
OpenOptions {
writeable: true,
need_sequential: false,
populate: Populate::from(populate),
populate,
advice: AdviceSetting::Global,
},
Default::default(),
@@ -251,7 +250,6 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
deleted_points: deleted,
},
active_points_count: points_count,
is_on_disk: !populate,
}))
}
@@ -284,7 +282,7 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
let filter = move |idx| self.is_active(idx);
fn intersection<V: ZerocopyPostingValue, S: UniversalRead>(
postings: &UniversalPostings<V, S>,
postings: &OnDiskPostings<V, S>,
tokens: TokenSet,
filter: impl Fn(PointOffsetType) -> bool,
) -> OperationResult<Vec<PointOffsetType>> {
@@ -304,8 +302,8 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
}
match &self.storage.postings {
MmapPostingsEnum::Ids(postings) => intersection(postings, tokens, filter),
MmapPostingsEnum::WithPositions(postings) => intersection(postings, tokens, filter),
OnDiskPostingsEnum::Ids(postings) => intersection(postings, tokens, filter),
OnDiskPostingsEnum::WithPositions(postings) => intersection(postings, tokens, filter),
}
}
@@ -315,7 +313,7 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
let is_active = move |idx| self.is_active(idx);
fn merge<V: ZerocopyPostingValue, S: UniversalRead>(
postings: &UniversalPostings<V, S>,
postings: &OnDiskPostings<V, S>,
tokens: TokenSet,
is_active: impl Fn(PointOffsetType) -> bool,
) -> OperationResult<Vec<PointOffsetType>> {
@@ -332,8 +330,8 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
}
match &self.storage.postings {
MmapPostingsEnum::Ids(postings) => merge(postings, tokens, is_active),
MmapPostingsEnum::WithPositions(postings) => merge(postings, tokens, is_active),
OnDiskPostingsEnum::Ids(postings) => merge(postings, tokens, is_active),
OnDiskPostingsEnum::WithPositions(postings) => merge(postings, tokens, is_active),
}
}
@@ -353,7 +351,7 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
}
fn check_intersection<V: ZerocopyPostingValue, S: UniversalRead>(
postings: &UniversalPostings<V, S>,
postings: &OnDiskPostings<V, S>,
tokens: &TokenSet,
point_id: PointOffsetType,
) -> OperationResult<bool> {
@@ -367,8 +365,8 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
}
match &self.storage.postings {
MmapPostingsEnum::Ids(postings) => check_intersection(postings, tokens, point_id),
MmapPostingsEnum::WithPositions(postings) => {
OnDiskPostingsEnum::Ids(postings) => check_intersection(postings, tokens, point_id),
OnDiskPostingsEnum::WithPositions(postings) => {
check_intersection(postings, tokens, point_id)
}
}
@@ -385,7 +383,7 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
}
fn check_any<V: ZerocopyPostingValue, S: UniversalRead>(
postings: &UniversalPostings<V, S>,
postings: &OnDiskPostings<V, S>,
tokens: &TokenSet,
point_id: PointOffsetType,
) -> OperationResult<bool> {
@@ -397,8 +395,8 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
}
match &self.storage.postings {
MmapPostingsEnum::Ids(postings) => check_any(postings, tokens, point_id),
MmapPostingsEnum::WithPositions(postings) => check_any(postings, tokens, point_id),
OnDiskPostingsEnum::Ids(postings) => check_any(postings, tokens, point_id),
OnDiskPostingsEnum::WithPositions(postings) => check_any(postings, tokens, point_id),
}
}
@@ -408,7 +406,7 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
let is_active = move |idx| self.is_active(idx);
match &self.storage.postings {
MmapPostingsEnum::WithPositions(postings) => {
OnDiskPostingsEnum::WithPositions(postings) => {
// Deduplicate phrase tokens: repeated tokens (e.g. "zn zn") must
// not fetch the same posting list twice, otherwise positions get
// added twice in `phrase_in_all_postings`.
@@ -428,7 +426,7 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
Ok(result.unwrap_or_default())
}
// cannot do phrase matching if there's no positional information
MmapPostingsEnum::Ids(_postings) => Ok(Vec::new()),
OnDiskPostingsEnum::Ids(_postings) => Ok(Vec::new()),
}
}
@@ -443,7 +441,7 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
}
match &self.storage.postings {
MmapPostingsEnum::WithPositions(postings) => {
OnDiskPostingsEnum::WithPositions(postings) => {
let unique_tokens = phrase.to_token_set();
let result = postings.with_all_or_none_postings(
unique_tokens.tokens(),
@@ -459,7 +457,7 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
Ok(result.unwrap_or(false))
}
// cannot do phrase matching if there's no positional information
MmapPostingsEnum::Ids(_postings) => Ok(false),
OnDiskPostingsEnum::Ids(_postings) => Ok(false),
}
}
@@ -482,7 +480,7 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
}
/// No-op flusher: the on-disk state is build-time only. See the type-level
/// docs on [`MmapInvertedIndex`] for the deletion durability contract.
/// docs on [`OnDiskInvertedIndex`] for the deletion durability contract.
#[allow(clippy::unused_self)]
pub fn flusher(&self) -> Flusher {
Box::new(|| Ok(()))
@@ -492,10 +490,6 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
self.storage.ram_usage_bytes()
}
pub fn is_on_disk(&self) -> bool {
self.is_on_disk
}
/// Populate all pages in the mmap.
/// Block until all pages are populated.
pub fn populate(&self) -> OperationResult<()> {
@@ -511,7 +505,6 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
path,
storage,
active_points_count: _,
is_on_disk: _,
} = self;
let Storage {
postings,
@@ -527,9 +520,9 @@ impl<S: UniversalRead> MmapInvertedIndex<S> {
}
}
impl<S: UniversalRead> InvertedIndex for MmapInvertedIndex<S> {
impl<S: UniversalRead> InvertedIndex for OnDiskInvertedIndex<S> {
fn get_vocab_mut(&mut self) -> &mut HashMap<String, TokenId> {
unreachable!("MmapInvertedIndex does not support mutable operations")
unreachable!("OnDiskInvertedIndex does not support mutable operations")
}
fn index_tokens(
@@ -656,11 +649,10 @@ impl<S: UniversalRead> InvertedIndex for MmapInvertedIndex<S> {
self.storage
.vocab
.for_each_entry_in_iter(tokens, |user_data, token_ids| {
if self.is_on_disk {
hw_counter.payload_index_io_read_counter().incr_delta(
READ_ENTRY_OVERHEAD + size_of::<TokenId>(), // Avoid check overhead and assume token is always read
);
}
hw_counter.payload_index_io_read_counter().incr_delta(
READ_ENTRY_OVERHEAD + size_of::<TokenId>(), // Avoid check overhead and assume token is always read
);
f(user_data, token_ids.map(unwrap_token));
Ok(())
})
lib/segment/src/index/field_index/full_text_index/inverted_index/mmap_inverted_index/uio_postings.rs → lib/segment/src/index/field_index/full_text_index/inverted_index/on_disk_inverted_index/on_disk_postings.rs
+4 -4
View File
@@ -11,8 +11,8 @@ use zerocopy::FromBytes;
use crate::common::operation_error::OperationResult;
use crate::index::field_index::full_text_index::inverted_index::TokenId;
use crate::index::field_index::full_text_index::inverted_index::mmap_inverted_index::raw_posting_list::RawPostingList;
use crate::index::field_index::full_text_index::inverted_index::mmap_inverted_index::types::{
use crate::index::field_index::full_text_index::inverted_index::on_disk_inverted_index::raw_posting_list::RawPostingList;
use crate::index::field_index::full_text_index::inverted_index::on_disk_inverted_index::types::{
PostingListHeader, PostingsHeader, ZerocopyPostingValue,
};
@@ -25,7 +25,7 @@ use crate::index::field_index::full_text_index::inverted_index::mmap_inverted_in
/// `size_of::<PostingsHeader>() + token_id * size_of::<PostingListHeader>()`.
/// Each [`PostingListHeader`] then points (via absolute `offset`) into the
/// posting-data region.
pub struct UniversalPostings<V: ZerocopyPostingValue, S: UniversalRead> {
pub struct OnDiskPostings<V: ZerocopyPostingValue, S: UniversalRead> {
_path: PathBuf,
storage: S,
header: PostingsHeader,
@@ -42,7 +42,7 @@ struct HeadersBatch<'a> {
missing: Vec<TokenId>,
}
impl<V: ZerocopyPostingValue, S: UniversalRead> UniversalPostings<V, S> {
impl<V: ZerocopyPostingValue, S: UniversalRead> OnDiskPostings<V, S> {
/// Open the postings file at `path` via the `S` storage backend.
///
/// Returns `Ok(None)` if the file is not found
lib/segment/src/index/field_index/full_text_index/inverted_index/mmap_inverted_index/mmap_postings_enum.rs → lib/segment/src/index/field_index/full_text_index/inverted_index/on_disk_inverted_index/on_disk_postings_enum.rs
+13 -13
View File
@@ -5,32 +5,32 @@ use common::universal_io::UniversalRead;
use super::super::positions::Positions;
use crate::common::operation_error::OperationResult;
use crate::index::field_index::full_text_index::inverted_index::TokenId;
use crate::index::field_index::full_text_index::inverted_index::mmap_inverted_index::uio_postings::UniversalPostings;
use crate::index::field_index::full_text_index::inverted_index::on_disk_inverted_index::on_disk_postings::OnDiskPostings;
pub enum MmapPostingsEnum<S: UniversalRead> {
Ids(UniversalPostings<(), S>),
WithPositions(UniversalPostings<Positions, S>),
pub enum OnDiskPostingsEnum<S: UniversalRead> {
Ids(OnDiskPostings<(), S>),
WithPositions(OnDiskPostings<Positions, S>),
}
impl<S: UniversalRead> MmapPostingsEnum<S> {
impl<S: UniversalRead> OnDiskPostingsEnum<S> {
pub fn populate(&self) -> OperationResult<()> {
match self {
MmapPostingsEnum::Ids(postings) => postings.populate(),
MmapPostingsEnum::WithPositions(postings) => postings.populate(),
OnDiskPostingsEnum::Ids(postings) => postings.populate(),
OnDiskPostingsEnum::WithPositions(postings) => postings.populate(),
}
}
pub fn clear_cache(&self) -> OperationResult<()> {
match self {
MmapPostingsEnum::Ids(postings) => postings.clear_cache(),
MmapPostingsEnum::WithPositions(postings) => postings.clear_cache(),
OnDiskPostingsEnum::Ids(postings) => postings.clear_cache(),
OnDiskPostingsEnum::WithPositions(postings) => postings.clear_cache(),
}
}
pub fn posting_len(&self, token_id: TokenId) -> OperationResult<Option<usize>> {
match self {
MmapPostingsEnum::Ids(postings) => postings.posting_len(token_id),
MmapPostingsEnum::WithPositions(postings) => postings.posting_len(token_id),
OnDiskPostingsEnum::Ids(postings) => postings.posting_len(token_id),
OnDiskPostingsEnum::WithPositions(postings) => postings.posting_len(token_id),
}
}
@@ -42,12 +42,12 @@ impl<S: UniversalRead> MmapPostingsEnum<S> {
// Collect ids upfront so the borrowed `RawPostingList` bytes don't have
// to outlive this call. Acceptable because UniversalPostings is on disk.
let ids: Vec<PointOffsetType> = match self {
MmapPostingsEnum::Ids(postings) => {
OnDiskPostingsEnum::Ids(postings) => {
let raw = postings.get(token_id).unwrap()?;
let view = raw.as_view::<()>().unwrap();
view.into_iter().map(|elem| elem.id).collect()
}
MmapPostingsEnum::WithPositions(postings) => {
OnDiskPostingsEnum::WithPositions(postings) => {
let raw = postings.get(token_id).unwrap()?;
let view = raw.as_view::<Positions>().unwrap();
view.into_iter().map(|elem| elem.id).collect()
lib/segment/src/index/field_index/full_text_index/inverted_index/mmap_inverted_index/raw_posting_list.rs → lib/segment/src/index/field_index/full_text_index/inverted_index/on_disk_inverted_index/raw_posting_list.rs
+1 -1
View File
@@ -5,7 +5,7 @@ use posting_list::{PostingChunk, PostingListView, RemainderPosting, SizedTypeFor
use zerocopy::FromBytes;
use crate::common::operation_error::OperationResult;
use crate::index::field_index::full_text_index::inverted_index::mmap_inverted_index::types::{
use crate::index::field_index::full_text_index::inverted_index::on_disk_inverted_index::types::{
PostingListHeader, ZerocopyPostingValue,
};
lib/segment/src/index/field_index/full_text_index/inverted_index/mmap_inverted_index/types.rs → lib/segment/src/index/field_index/full_text_index/inverted_index/on_disk_inverted_index/types.rs
View File
lib/segment/src/index/field_index/full_text_index/lifecycle.rs
+8 -9
View File
@@ -4,13 +4,13 @@ use std::path::PathBuf;
use common::bitvec::BitSlice;
use common::counter::hardware_counter::HardwareCounterCell;
use common::types::PointOffsetType;
use common::universal_io::MmapFs;
use common::universal_io::{MmapFs, Populate};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use super::immutable_text_index::ImmutableFullTextIndex;
use super::mmap_text_index::{FullTextMmapIndexBuilder, MmapFullTextIndex};
use super::mutable_text_index::MutableFullTextIndex;
use super::on_disk_text_index::{FullTextMmapIndexBuilder, OnDiskFullTextIndex};
use super::{FullTextGridstoreIndexBuilder, FullTextIndex};
use crate::common::Flusher;
use crate::common::operation_error::{OperationError, OperationResult};
@@ -31,22 +31,21 @@ impl FullTextIndex {
let effective_is_on_disk =
is_on_disk || common::low_memory::low_memory_mode().prefer_disk();
let Some(mmap_index) =
MmapFullTextIndex::open(&MmapFs, path, config, effective_is_on_disk, deleted_points)?
let populate = Populate::from(!effective_is_on_disk);
let Some(on_disk_index) =
OnDiskFullTextIndex::open(&MmapFs, path, config, populate, deleted_points)?
else {
return Ok(None);
};
let index = if effective_is_on_disk {
// Use on-disk directly
Some(Self::OnDisk(Box::new(mmap_index)))
Self::OnDisk(on_disk_index)
} else {
// Load into RAM, use mmap as backing storage
Some(Self::Immutable(ImmutableFullTextIndex::open_mmap(
mmap_index,
)?))
Self::Immutable(ImmutableFullTextIndex::load_from_on_disk(on_disk_index)?)
};
Ok(index)
Ok(Some(index))
}
pub fn new_gridstore(
lib/segment/src/index/field_index/full_text_index/mod.rs
+3 -4
View File
@@ -3,16 +3,16 @@ use std::path::PathBuf;
use common::universal_io::MmapFile;
use self::immutable_text_index::ImmutableFullTextIndex;
use self::mmap_text_index::MmapFullTextIndex;
use self::mutable_text_index::MutableFullTextIndex;
use self::on_disk_text_index::OnDiskFullTextIndex;
use crate::data_types::index::TextIndexParams;
pub mod full_text_index_read;
mod immutable_text_index;
mod inverted_index;
mod lifecycle;
pub mod mmap_text_index;
mod mutable_text_index;
pub mod on_disk_text_index;
pub mod read_only;
mod read_ops;
pub mod stop_words;
@@ -21,11 +21,10 @@ pub mod tokenizers;
#[cfg(test)]
mod tests;
#[allow(clippy::large_enum_variant)]
pub enum FullTextIndex {
Mutable(MutableFullTextIndex),
Immutable(ImmutableFullTextIndex),
OnDisk(Box<MmapFullTextIndex<MmapFile>>),
OnDisk(OnDiskFullTextIndex<MmapFile>),
}
pub struct FullTextGridstoreIndexBuilder {
lib/segment/src/index/field_index/full_text_index/mmap_text_index/lifecycle.rs → lib/segment/src/index/field_index/full_text_index/on_disk_text_index/lifecycle.rs
+13 -15
View File
@@ -3,38 +3,36 @@ use std::path::PathBuf;
use common::bitvec::BitSlice;
use common::counter::hardware_counter::HardwareCounterCell;
use common::types::PointOffsetType;
use common::universal_io::{MmapFs, UniversalRead};
use common::universal_io::{MmapFs, Populate, UniversalRead};
use fs_err as fs;
use serde_json::Value;
use super::super::FullTextIndex;
use super::super::immutable_text_index::ImmutableFullTextIndex;
use super::super::inverted_index::immutable_inverted_index::ImmutableInvertedIndex;
use super::super::inverted_index::mmap_inverted_index::MmapInvertedIndex;
use super::super::inverted_index::mutable_inverted_index::MutableInvertedIndex;
use super::super::inverted_index::on_disk_inverted_index::OnDiskInvertedIndex;
use super::super::inverted_index::{ARRAY_BOUNDARY_SENTINEL, Document, InvertedIndex, TokenSet};
use super::super::tokenizers::Tokenizer;
use super::{FullTextMmapIndexBuilder, MmapFullTextIndex};
use super::{FullTextMmapIndexBuilder, OnDiskFullTextIndex};
use crate::common::Flusher;
use crate::common::operation_error::{OperationError, OperationResult};
use crate::data_types::index::TextIndexParams;
use crate::index::field_index::{FieldIndexBuilderTrait, ValueIndexer};
impl<S: UniversalRead> MmapFullTextIndex<S> {
impl<S: UniversalRead> OnDiskFullTextIndex<S> {
pub fn open(
fs: &S::Fs,
path: PathBuf,
config: TextIndexParams,
is_on_disk: bool,
populate: Populate,
deleted_points: &BitSlice,
) -> OperationResult<Option<Self>> {
let populate = !is_on_disk;
let has_positions = config.phrase_matching == Some(true);
let tokenizer = Tokenizer::new_from_text_index_params(&config);
let inverted_index =
MmapInvertedIndex::<S>::open(fs, path, populate, has_positions, deleted_points)?;
OnDiskInvertedIndex::<S>::open(fs, path, populate, has_positions, deleted_points)?;
Ok(inverted_index.map(|inverted_index| Self {
inverted_index,
tokenizer,
@@ -189,27 +187,27 @@ impl FieldIndexBuilderTrait for FullTextMmapIndexBuilder {
fs::create_dir_all(path.as_path())?;
MmapInvertedIndex::create(path.clone(), &immutable)?;
OnDiskInvertedIndex::create(path.clone(), &immutable)?;
let populate = !is_on_disk;
let populate = Populate::from(!is_on_disk);
let has_positions = config.phrase_matching.unwrap_or_default();
let inverted_index =
MmapInvertedIndex::open(&MmapFs, path, populate, has_positions, &deleted_points)?
OnDiskInvertedIndex::open(&MmapFs, path, populate, has_positions, &deleted_points)?
.ok_or_else(|| {
OperationError::service_error(
"Failed to open MmapInvertedIndex that was just created",
"Failed to open OnDiskInvertedIndex that was just created",
)
})?;
let mmap_index = MmapFullTextIndex {
let on_disk_index = OnDiskFullTextIndex {
inverted_index,
tokenizer,
};
let text_index = if is_on_disk {
FullTextIndex::OnDisk(Box::new(mmap_index))
FullTextIndex::OnDisk(on_disk_index)
} else {
FullTextIndex::Immutable(ImmutableFullTextIndex::open_mmap(mmap_index)?)
FullTextIndex::Immutable(ImmutableFullTextIndex::load_from_on_disk(on_disk_index)?)
};
Ok(text_index)
lib/segment/src/index/field_index/full_text_index/mmap_text_index/live_reload.rs → lib/segment/src/index/field_index/full_text_index/on_disk_text_index/live_reload.rs
+2 -2
View File
@@ -3,11 +3,11 @@ use common::sorted_slice::SortedSlice;
use common::types::PointOffsetType;
use common::universal_io::UniversalRead;
use super::MmapFullTextIndex;
use super::OnDiskFullTextIndex;
use crate::common::operation_error::OperationResult;
use crate::index::field_index::LiveReload;
impl<S: UniversalRead> LiveReload for MmapFullTextIndex<S> {
impl<S: UniversalRead> LiveReload for OnDiskFullTextIndex<S> {
type Fs = S::Fs;
fn live_reload(
lib/segment/src/index/field_index/full_text_index/mmap_text_index/mod.rs → lib/segment/src/index/field_index/full_text_index/on_disk_text_index/mod.rs
+3 -3
View File
@@ -3,8 +3,8 @@ use std::path::PathBuf;
use common::bitvec::BitVec;
use common::universal_io::{MmapFile, UniversalRead};
use super::inverted_index::mmap_inverted_index::MmapInvertedIndex;
use super::inverted_index::mutable_inverted_index::MutableInvertedIndex;
use super::inverted_index::on_disk_inverted_index::OnDiskInvertedIndex;
use super::tokenizers::Tokenizer;
use crate::data_types::index::TextIndexParams;
@@ -12,8 +12,8 @@ mod lifecycle;
mod live_reload;
mod read_ops;
pub struct MmapFullTextIndex<S: UniversalRead = MmapFile> {
pub(in super::super) inverted_index: MmapInvertedIndex<S>,
pub struct OnDiskFullTextIndex<S: UniversalRead = MmapFile> {
pub(in super::super) inverted_index: OnDiskInvertedIndex<S>,
pub(in super::super) tokenizer: Tokenizer,
}
lib/segment/src/index/field_index/full_text_index/mmap_text_index/read_ops.rs → lib/segment/src/index/field_index/full_text_index/on_disk_text_index/read_ops.rs
+4 -6
View File
@@ -5,13 +5,13 @@ use common::universal_io::{UniversalRead, UserData};
use super::super::full_text_index_read::FullTextIndexRead;
use super::super::inverted_index::{InvertedIndex, ParsedQuery, TokenId};
use super::super::tokenizers::Tokenizer;
use super::MmapFullTextIndex;
use super::OnDiskFullTextIndex;
use crate::common::operation_error::OperationResult;
use crate::index::field_index::{CardinalityEstimation, PayloadBlockCondition};
use crate::index::payload_config::StorageType;
use crate::types::{FieldCondition, PayloadKeyType};
impl<S: UniversalRead> FullTextIndexRead for MmapFullTextIndex<S> {
impl<S: UniversalRead> FullTextIndexRead for OnDiskFullTextIndex<S> {
fn tokenizer(&self) -> &Tokenizer {
&self.tokenizer
}
@@ -74,9 +74,7 @@ impl<S: UniversalRead> FullTextIndexRead for MmapFullTextIndex<S> {
}
fn get_storage_type(&self) -> StorageType {
StorageType::Mmap {
is_on_disk: self.inverted_index.is_on_disk(),
}
StorageType::Mmap { is_on_disk: true }
}
fn ram_usage_bytes(&self) -> usize {
@@ -84,6 +82,6 @@ impl<S: UniversalRead> FullTextIndexRead for MmapFullTextIndex<S> {
}
fn is_on_disk(&self) -> bool {
self.inverted_index.is_on_disk()
true
}
}
lib/segment/src/index/field_index/full_text_index/read_only/lifecycle.rs
+19 -6
View File
@@ -1,13 +1,14 @@
use std::path::PathBuf;
use common::bitvec::BitSlice;
use common::universal_io::UniversalRead;
use common::universal_io::{Populate, UniversalRead};
use super::super::mmap_text_index::MmapFullTextIndex;
use super::super::mutable_text_index::read_only::ReadOnlyAppendableFullTextIndex;
use super::super::on_disk_text_index::OnDiskFullTextIndex;
use super::ReadOnlyFullTextIndex;
use crate::common::operation_error::OperationResult;
use crate::data_types::index::TextIndexParams;
use crate::index::field_index::full_text_index::immutable_text_index::ImmutableFullTextIndex;
use crate::index::payload_config::IndexMutability;
impl<S: UniversalRead> ReadOnlyFullTextIndex<S> {
@@ -52,10 +53,21 @@ impl<S: UniversalRead> ReadOnlyFullTextIndex<S> {
let effective_is_on_disk =
is_on_disk || common::low_memory::low_memory_mode().prefer_disk();
Ok(
MmapFullTextIndex::open(fs, path, config, effective_is_on_disk, deleted_points)?
.map(Self::Immutable),
)
let populate = Populate::from(!effective_is_on_disk);
let Some(on_disk_index) =
OnDiskFullTextIndex::open(fs, path, config, populate, deleted_points)?
else {
return Ok(None);
};
let index = if effective_is_on_disk {
Self::OnDisk(on_disk_index)
} else {
Self::Immutable(ImmutableFullTextIndex::load_from_on_disk(on_disk_index)?)
};
Ok(Some(index))
}
/// Reports the on-disk format's mutability, mirroring
@@ -72,6 +84,7 @@ impl<S: UniversalRead> ReadOnlyFullTextIndex<S> {
pub fn get_mutability_type(&self) -> IndexMutability {
match self {
Self::Appendable(_) => IndexMutability::Mutable,
Self::OnDisk(_) => IndexMutability::Immutable,
Self::Immutable(_) => IndexMutability::Immutable,
}
}
lib/segment/src/index/field_index/full_text_index/read_only/mod.rs
+5 -2
View File
@@ -1,7 +1,8 @@
use common::universal_io::UniversalRead;
use super::mmap_text_index::MmapFullTextIndex;
use super::mutable_text_index::read_only::ReadOnlyAppendableFullTextIndex;
use super::on_disk_text_index::OnDiskFullTextIndex;
use crate::index::field_index::full_text_index::immutable_text_index::ImmutableFullTextIndex;
mod lifecycle;
mod read_ops;
@@ -33,8 +34,10 @@ mod read_ops;
pub enum ReadOnlyFullTextIndex<S: UniversalRead> {
/// Loads into RAM from appendable storage format
Appendable(ReadOnlyAppendableFullTextIndex<S>),
/// Loads into RAM from immutable format
Immutable(ImmutableFullTextIndex<S>),
/// Directly reads from storage in immutable format
Immutable(MmapFullTextIndex<S>),
OnDisk(OnDiskFullTextIndex<S>),
}
#[cfg(test)]
lib/segment/src/index/field_index/full_text_index/read_only/read_ops.rs
+17
View File
@@ -24,6 +24,7 @@ impl<S: UniversalRead> FullTextIndexRead for ReadOnlyFullTextIndex<S> {
fn tokenizer(&self) -> &Tokenizer {
match self {
ReadOnlyFullTextIndex::Appendable(index) => index.tokenizer(),
ReadOnlyFullTextIndex::OnDisk(index) => index.tokenizer(),
ReadOnlyFullTextIndex::Immutable(index) => index.tokenizer(),
}
}
@@ -31,6 +32,7 @@ impl<S: UniversalRead> FullTextIndexRead for ReadOnlyFullTextIndex<S> {
fn telemetry_index_type(&self) -> &'static str {
match self {
ReadOnlyFullTextIndex::Appendable(index) => index.telemetry_index_type(),
ReadOnlyFullTextIndex::OnDisk(index) => index.telemetry_index_type(),
ReadOnlyFullTextIndex::Immutable(index) => index.telemetry_index_type(),
}
}
@@ -38,6 +40,7 @@ impl<S: UniversalRead> FullTextIndexRead for ReadOnlyFullTextIndex<S> {
fn points_count(&self) -> usize {
match self {
ReadOnlyFullTextIndex::Appendable(index) => index.points_count(),
ReadOnlyFullTextIndex::OnDisk(index) => index.points_count(),
ReadOnlyFullTextIndex::Immutable(index) => index.points_count(),
}
}
@@ -45,6 +48,7 @@ impl<S: UniversalRead> FullTextIndexRead for ReadOnlyFullTextIndex<S> {
fn values_count(&self, point_id: PointOffsetType) -> usize {
match self {
ReadOnlyFullTextIndex::Appendable(index) => index.values_count(point_id),
ReadOnlyFullTextIndex::OnDisk(index) => index.values_count(point_id),
ReadOnlyFullTextIndex::Immutable(index) => index.values_count(point_id),
}
}
@@ -52,6 +56,7 @@ impl<S: UniversalRead> FullTextIndexRead for ReadOnlyFullTextIndex<S> {
fn values_is_empty(&self, point_id: PointOffsetType) -> bool {
match self {
ReadOnlyFullTextIndex::Appendable(index) => index.values_is_empty(point_id),
ReadOnlyFullTextIndex::OnDisk(index) => index.values_is_empty(point_id),
ReadOnlyFullTextIndex::Immutable(index) => index.values_is_empty(point_id),
}
}
@@ -66,6 +71,7 @@ impl<S: UniversalRead> FullTextIndexRead for ReadOnlyFullTextIndex<S> {
ReadOnlyFullTextIndex::Appendable(index) => {
index.for_each_token_id(iter, hw_counter, f)
}
ReadOnlyFullTextIndex::OnDisk(index) => index.for_each_token_id(iter, hw_counter, f),
ReadOnlyFullTextIndex::Immutable(index) => index.for_each_token_id(iter, hw_counter, f),
}
}
@@ -77,6 +83,7 @@ impl<S: UniversalRead> FullTextIndexRead for ReadOnlyFullTextIndex<S> {
) -> OperationResult<Box<dyn Iterator<Item = PointOffsetType> + 'a>> {
match self {
ReadOnlyFullTextIndex::Appendable(index) => index.filter_query(query, hw_counter),
ReadOnlyFullTextIndex::OnDisk(index) => index.filter_query(query, hw_counter),
ReadOnlyFullTextIndex::Immutable(index) => index.filter_query(query, hw_counter),
}
}
@@ -91,6 +98,9 @@ impl<S: UniversalRead> FullTextIndexRead for ReadOnlyFullTextIndex<S> {
ReadOnlyFullTextIndex::Appendable(index) => {
index.estimate_query_cardinality(query, condition, hw_counter)
}
ReadOnlyFullTextIndex::OnDisk(index) => {
index.estimate_query_cardinality(query, condition, hw_counter)
}
ReadOnlyFullTextIndex::Immutable(index) => {
index.estimate_query_cardinality(query, condition, hw_counter)
}
@@ -100,6 +110,7 @@ impl<S: UniversalRead> FullTextIndexRead for ReadOnlyFullTextIndex<S> {
fn check_match(&self, query: &ParsedQuery, point_id: PointOffsetType) -> OperationResult<bool> {
match self {
ReadOnlyFullTextIndex::Appendable(index) => index.check_match(query, point_id),
ReadOnlyFullTextIndex::OnDisk(index) => index.check_match(query, point_id),
ReadOnlyFullTextIndex::Immutable(index) => index.check_match(query, point_id),
}
}
@@ -114,6 +125,9 @@ impl<S: UniversalRead> FullTextIndexRead for ReadOnlyFullTextIndex<S> {
ReadOnlyFullTextIndex::Appendable(index) => {
index.for_each_payload_block_inner(threshold, key, f)
}
ReadOnlyFullTextIndex::OnDisk(index) => {
index.for_each_payload_block_inner(threshold, key, f)
}
ReadOnlyFullTextIndex::Immutable(index) => {
index.for_each_payload_block_inner(threshold, key, f)
}
@@ -123,6 +137,7 @@ impl<S: UniversalRead> FullTextIndexRead for ReadOnlyFullTextIndex<S> {
fn get_storage_type(&self) -> StorageType {
match self {
ReadOnlyFullTextIndex::Appendable(index) => index.get_storage_type(),
ReadOnlyFullTextIndex::OnDisk(index) => index.get_storage_type(),
ReadOnlyFullTextIndex::Immutable(index) => index.get_storage_type(),
}
}
@@ -130,6 +145,7 @@ impl<S: UniversalRead> FullTextIndexRead for ReadOnlyFullTextIndex<S> {
fn ram_usage_bytes(&self) -> usize {
match self {
ReadOnlyFullTextIndex::Appendable(index) => index.ram_usage_bytes(),
ReadOnlyFullTextIndex::OnDisk(index) => index.ram_usage_bytes(),
ReadOnlyFullTextIndex::Immutable(index) => index.ram_usage_bytes(),
}
}
@@ -137,6 +153,7 @@ impl<S: UniversalRead> FullTextIndexRead for ReadOnlyFullTextIndex<S> {
fn is_on_disk(&self) -> bool {
match self {
ReadOnlyFullTextIndex::Appendable(index) => index.is_on_disk(),
ReadOnlyFullTextIndex::OnDisk(index) => index.is_on_disk(),
ReadOnlyFullTextIndex::Immutable(index) => index.is_on_disk(),
}
}
lib/segment/src/index/field_index/full_text_index/read_ops.rs
+3 -3
View File
@@ -125,7 +125,7 @@ impl FullTextIndexRead for FullTextIndex {
match self {
Self::Mutable(index) => FullTextIndexRead::get_storage_type(index),
Self::Immutable(index) => FullTextIndexRead::get_storage_type(index),
Self::OnDisk(index) => FullTextIndexRead::get_storage_type(index.as_ref()),
Self::OnDisk(index) => FullTextIndexRead::get_storage_type(index),
}
}
@@ -133,7 +133,7 @@ impl FullTextIndexRead for FullTextIndex {
match self {
Self::Mutable(index) => FullTextIndexRead::ram_usage_bytes(index),
Self::Immutable(index) => FullTextIndexRead::ram_usage_bytes(index),
Self::OnDisk(index) => FullTextIndexRead::ram_usage_bytes(index.as_ref()),
Self::OnDisk(index) => FullTextIndexRead::ram_usage_bytes(index),
}
}
@@ -141,7 +141,7 @@ impl FullTextIndexRead for FullTextIndex {
match self {
Self::Mutable(index) => FullTextIndexRead::is_on_disk(index),
Self::Immutable(index) => FullTextIndexRead::is_on_disk(index),
Self::OnDisk(index) => FullTextIndexRead::is_on_disk(index.as_ref()),
Self::OnDisk(index) => FullTextIndexRead::is_on_disk(index),
}
}
}
lib/segment/src/index/field_index/full_text_index/tests/test_congruence.rs
+29 -33
View File
@@ -17,8 +17,8 @@ use crate::index::field_index::full_text_index::full_text_index_read::FullTextIn
use crate::index::field_index::full_text_index::inverted_index::{
ARRAY_BOUNDARY_SENTINEL, Document, ParsedQuery, TokenId, TokenSet,
};
use crate::index::field_index::full_text_index::mmap_text_index::FullTextMmapIndexBuilder;
use crate::index::field_index::full_text_index::mutable_text_index::MutableFullTextIndex;
use crate::index::field_index::full_text_index::on_disk_text_index::FullTextMmapIndexBuilder;
use crate::index::field_index::full_text_index::{FullTextGridstoreIndexBuilder, FullTextIndex};
use crate::index::field_index::{FieldIndexBuilderTrait, ValueIndexer};
use crate::json_path::JsonPath;
@@ -27,23 +27,20 @@ use crate::types::{FieldCondition, ValuesCount};
type Database = ();
const FIELD_NAME: &str = "test";
const TYPES: &[IndexType] = &[
IndexType::MutableGridstore,
IndexType::ImmMmap,
IndexType::ImmRamMmap,
];
const TYPES: &[IndexType] = &[IndexType::Mutable, IndexType::OnDisk, IndexType::Immutable];
#[derive(Clone, Copy, PartialEq, Debug)]
enum IndexType {
MutableGridstore,
ImmMmap,
ImmRamMmap,
Mutable,
OnDisk,
Immutable,
}
#[expect(clippy::large_enum_variant)]
enum IndexBuilder {
MutableGridstore(FullTextGridstoreIndexBuilder),
ImmMmap(FullTextMmapIndexBuilder),
ImmRamMmap(FullTextMmapIndexBuilder),
Mutable(FullTextGridstoreIndexBuilder),
OnDisk(FullTextMmapIndexBuilder),
Immutable(FullTextMmapIndexBuilder),
}
impl IndexBuilder {
@@ -54,13 +51,13 @@ impl IndexBuilder {
hw_counter: &HardwareCounterCell,
) -> OperationResult<()> {
match self {
IndexBuilder::MutableGridstore(builder) => {
IndexBuilder::Mutable(builder) => {
FieldIndexBuilderTrait::add_point(builder, id, payload, hw_counter)
}
IndexBuilder::ImmMmap(builder) => {
IndexBuilder::OnDisk(builder) => {
FieldIndexBuilderTrait::add_point(builder, id, payload, hw_counter)
}
IndexBuilder::ImmRamMmap(builder) => {
IndexBuilder::Immutable(builder) => {
FieldIndexBuilderTrait::add_point(builder, id, payload, hw_counter)
}
}
@@ -68,9 +65,9 @@ impl IndexBuilder {
fn finalize(self) -> OperationResult<FullTextIndex> {
match self {
IndexBuilder::MutableGridstore(builder) => builder.finalize(),
IndexBuilder::ImmMmap(builder) => builder.finalize(),
IndexBuilder::ImmRamMmap(builder) => builder.finalize(),
IndexBuilder::Mutable(builder) => builder.finalize(),
IndexBuilder::OnDisk(builder) => builder.finalize(),
IndexBuilder::Immutable(builder) => builder.finalize(),
}
}
}
@@ -89,16 +86,17 @@ fn create_builder(
let empty_deleted = BitVec::new();
let mut builder = match index_type {
IndexType::MutableGridstore => IndexBuilder::MutableGridstore(
FullTextIndex::builder_gridstore(temp_dir.path().to_path_buf(), config),
),
IndexType::ImmMmap => IndexBuilder::ImmMmap(FullTextIndex::builder_mmap(
IndexType::Mutable => IndexBuilder::Mutable(FullTextIndex::builder_gridstore(
temp_dir.path().to_path_buf(),
config,
)),
IndexType::OnDisk => IndexBuilder::OnDisk(FullTextIndex::builder_mmap(
temp_dir.path().to_path_buf(),
config,
true,
&empty_deleted,
)),
IndexType::ImmRamMmap => IndexBuilder::ImmRamMmap(FullTextIndex::builder_mmap(
IndexType::Immutable => IndexBuilder::Immutable(FullTextIndex::builder_mmap(
temp_dir.path().to_path_buf(),
config,
false,
@@ -106,9 +104,9 @@ fn create_builder(
)),
};
match &mut builder {
IndexBuilder::MutableGridstore(builder) => builder.init().unwrap(),
IndexBuilder::ImmMmap(builder) => builder.init().unwrap(),
IndexBuilder::ImmRamMmap(builder) => builder.init().unwrap(),
IndexBuilder::Mutable(builder) => builder.init().unwrap(),
IndexBuilder::OnDisk(builder) => builder.init().unwrap(),
IndexBuilder::Immutable(builder) => builder.init().unwrap(),
}
(builder, temp_dir, db)
}
@@ -142,18 +140,18 @@ fn reopen_index(
// Reopen based on index type
match index_type {
IndexType::MutableGridstore => {
IndexType::Mutable => {
FullTextIndex::new_gridstore(temp_dir.path().to_path_buf(), config, false)
.unwrap()
.expect("Failed to reopen MutableGridstore index")
}
IndexType::ImmMmap => {
IndexType::OnDisk => {
// Reopen with is_on_disk = true (mmap directly)
FullTextIndex::new_mmap(temp_dir.path().to_path_buf(), config, true, &deleted)
.unwrap()
.expect("Failed to reopen ImmMmap index")
}
IndexType::ImmRamMmap => {
IndexType::Immutable => {
// Reopen with is_on_disk = false (load into RAM)
// This is the path that will call ImmutableFullTextIndex::open_mmap
FullTextIndex::new_mmap(temp_dir.path().to_path_buf(), config, false, &deleted)
@@ -518,8 +516,7 @@ fn check_phrase<const KEYWORD_COUNT: usize>(
/// full-text index is enabled.
#[rstest]
fn test_phrase_matching_respects_array_boundaries(
#[values(IndexType::MutableGridstore, IndexType::ImmMmap, IndexType::ImmRamMmap)]
index_type: IndexType,
#[values(IndexType::Mutable, IndexType::OnDisk, IndexType::Immutable)] index_type: IndexType,
) {
let hw = HardwareCounterCell::new();
let (mut builder, _temp_dir, _db) = create_builder(index_type, true);
@@ -586,8 +583,7 @@ fn test_phrase_matching_respects_array_boundaries(
/// Single-element arrays and plain strings should still work normally.
#[rstest]
fn test_phrase_matching_single_element_array(
#[values(IndexType::MutableGridstore, IndexType::ImmMmap, IndexType::ImmRamMmap)]
index_type: IndexType,
#[values(IndexType::Mutable, IndexType::OnDisk, IndexType::Immutable)] index_type: IndexType,
) {
let hw = HardwareCounterCell::new();
let (mut builder, _temp_dir, _db) = create_builder(index_type, true);