upstream-archive/qdrant
1
0
Fork 0
mirror of https://github.com/qdrant/qdrant.git synced 2026-09-21 13:37:46 -05:00
Code Issues Packages Projects Releases Wiki Activity

Open and load bool payload indices in single stage (#7039)

Browse Source 
* Open and load simple bool index in single stage

* Open and load mutable bool index in single stage

* Open and load mutable null index in single stage

* Refactor index selector, don't return early
This commit is contained in:
Tim Visée
2025-08-14 14:30:26 +02:00
committed by timvisee
parent 9218d74985
commit 68825bfe6a
6 changed files with 140 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/segment/src/index/field_index/bool_index/mod.rs
+5 -1
View File
@@ -161,6 +161,7 @@ impl PayloadFieldIndex for BoolIndex {
}
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load(&mut self) -> crate::common::operation_error::OperationResult<bool> {
match self {
#[cfg(feature = "rocksdb")]
@@ -334,7 +335,9 @@ mod tests {
impl OpenIndex for SimpleBoolIndex {
fn open_at(path: &Path) -> BoolIndex {
let db = open_db_with_existing_cf(path).unwrap();
let mut index = SimpleBoolIndex::new(db.clone(), FIELD_NAME);
let mut index = SimpleBoolIndex::new(db.clone(), FIELD_NAME, true)
.unwrap()
.unwrap();
// Try to load if it exists
if index.load().unwrap() {
return BoolIndex::Simple(index);
@@ -343,6 +346,7 @@ mod tests {
// Otherwise create a new one
SimpleBoolIndex::builder(db, FIELD_NAME)
.unwrap()
.make_empty()
.unwrap()
}
lib/segment/src/index/field_index/bool_index/mutable_bool_index.rs
+24 -51
View File
@@ -37,30 +37,28 @@ struct Storage {
impl MutableBoolIndex {
pub fn builder(path: &Path) -> OperationResult<MutableBoolIndexBuilder> {
Ok(MutableBoolIndexBuilder(Self::open(path, true)?))
Ok(MutableBoolIndexBuilder(
Self::open(path, true)?.ok_or_else(|| {
OperationError::service_error("Failed to create and open MutableBoolIndex")
})?,
))
}
/// Open or create a boolean index at the given path.
/// Open and load or create a boolean index at the given path.
///
/// # Arguments
/// - `path` - The directory where the index files should live, must be exclusive to this index.
/// - `is_on_disk` - If the index should be kept on disk. Memory will be populated if false.
/// - `create_if_missing` - If true, creates the index if it doesn't exist.
pub fn open(path: &Path, create_if_missing: bool) -> OperationResult<Self> {
pub fn open(path: &Path, create_if_missing: bool) -> OperationResult<Option<Self>> {
let falses_dir = path.join(FALSES_DIRNAME);
// If falses directory doesn't exist, assume the index doesn't exist on disk
if !falses_dir.is_dir() && !create_if_missing {
return Ok(Self {
base_dir: path.to_path_buf(),
storage: None,
indexed_count: 0,
trues_count: 0,
falses_count: 0,
});
return Ok(None);
}
Self::open_or_create(path)
Ok(Some(Self::open_or_create(path)?))
}
fn open_or_create(path: &Path) -> OperationResult<Self> {
@@ -80,16 +78,23 @@ impl MutableBoolIndex {
let falses_slice = DynamicMmapFlags::open(&falses_path, false)?;
let falses_flags = RoaringFlags::new(falses_slice);
let trues_count = trues_flags.count_trues();
let falses_count = falses_flags.count_trues();
let indexed_count = {
let trues = trues_flags.get_bitmap();
let falses = falses_flags.get_bitmap();
trues.union_len(falses) as usize
};
Ok(Self {
base_dir: path.to_path_buf(),
storage: Some(Storage {
trues_flags,
falses_flags,
}),
// loading is done after opening during `PayloadFieldIndex::load()`
indexed_count: 0,
trues_count: 0,
falses_count: 0,
trues_count,
falses_count,
indexed_count,
})
}
@@ -165,17 +170,6 @@ impl MutableBoolIndex {
}
}
/// Calculates the count of true values of the union of both slices.
fn calculate_indexed_count(&self) -> u32 {
if let Some(storage) = &self.storage {
let trues = storage.trues_flags.get_bitmap();
let falses = storage.falses_flags.get_bitmap();
trues.union_len(falses) as u32
} else {
0
}
}
pub fn get_telemetry_data(&self) -> PayloadIndexTelemetry {
PayloadIndexTelemetry {
field_name: None,
@@ -359,32 +353,11 @@ impl PayloadFieldIndex for MutableBoolIndex {
self.indexed_count
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load(&mut self) -> OperationResult<bool> {
// Failed to load
if self.storage.is_none() {
return Ok(false);
}
// Note: this structure is now loaded on open
let calculated_indexed_count = self.calculate_indexed_count();
// Destructure to not forget any fields
let Self {
base_dir: _,
indexed_count,
storage,
trues_count,
falses_count,
} = self;
let Storage {
trues_flags,
falses_flags,
} = storage.as_ref().unwrap();
*indexed_count = calculated_indexed_count as usize;
*trues_count = trues_flags.count_trues();
*falses_count = falses_flags.count_trues();
Ok(true)
Ok(self.storage.is_some())
}
fn cleanup(self) -> OperationResult<()> {
@@ -530,7 +503,7 @@ mod tests {
#[test]
fn test_files() {
let dir = TempDir::with_prefix("test_mmap_bool_index").unwrap();
let index = MutableBoolIndex::open(dir.path(), true).unwrap();
let index = MutableBoolIndex::open(dir.path(), true).unwrap().unwrap();
let reported = index.files().into_iter().collect::<HashSet<_>>();
lib/segment/src/index/field_index/bool_index/simple_bool_index.rs
+38 -19
View File
@@ -9,7 +9,7 @@ use serde_json::Value;
use self::memory::{BoolMemory, BooleanItem};
use super::BoolIndex;
use crate::common::operation_error::OperationResult;
use crate::common::operation_error::{OperationError, OperationResult};
use crate::common::rocksdb_buffered_delete_wrapper::DatabaseColumnScheduledDeleteWrapper;
use crate::common::rocksdb_wrapper::DatabaseColumnWrapper;
use crate::index::field_index::map_index::IdIter;
@@ -187,20 +187,48 @@ pub struct SimpleBoolIndex {
}
impl SimpleBoolIndex {
pub fn new(db: Arc<RwLock<DB>>, field_name: &str) -> SimpleBoolIndex {
pub fn new(
db: Arc<RwLock<DB>>,
field_name: &str,
create_if_missing: bool,
) -> OperationResult<Option<SimpleBoolIndex>> {
let store_cf_name = Self::storage_cf_name(field_name);
let db_wrapper = DatabaseColumnScheduledDeleteWrapper::new(DatabaseColumnWrapper::new(
db,
&store_cf_name,
));
Self {
memory: BoolMemory::new(),
db_wrapper,
if !db_wrapper.has_column_family()? {
if create_if_missing {
db_wrapper.recreate_column_family()?;
} else {
// Column family doesn't exist, cannot load
return Ok(None);
}
};
// Load in-memory index from RocksDB
let mut memory = BoolMemory::new();
for (key, value) in db_wrapper.lock_db().iter()? {
let idx = PointOffsetType::from_be_bytes(key.as_ref().try_into().unwrap());
debug_assert_eq!(value.len(), 1);
let item = BooleanItem::from(value[0]);
memory.set_or_insert(idx, &item);
}
Ok(Some(Self { memory, db_wrapper }))
}
pub fn builder(db: Arc<RwLock<DB>>, field_name: &str) -> BoolIndexBuilder {
BoolIndexBuilder(Self::new(db, field_name))
pub fn builder(db: Arc<RwLock<DB>>, field_name: &str) -> OperationResult<BoolIndexBuilder> {
Ok(BoolIndexBuilder(
Self::new(db, field_name, true)?.ok_or_else(|| {
OperationError::service_error(format!(
"Failed to create and open SimpleBoolIndex for field: {field_name}",
))
})?,
))
}
fn storage_cf_name(field: &str) -> String {
@@ -305,20 +333,11 @@ impl FieldIndexBuilderTrait for BoolIndexBuilder {
}
impl PayloadFieldIndex for SimpleBoolIndex {
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load(&mut self) -> OperationResult<bool> {
if !self.db_wrapper.has_column_family()? {
return Ok(false);
}
// Note: this structure is now loaded on open
for (key, value) in self.db_wrapper.lock_db().iter()? {
let idx = PointOffsetType::from_be_bytes(key.as_ref().try_into().unwrap());
debug_assert_eq!(value.len(), 1);
let item = BooleanItem::from(value[0]);
self.memory.set_or_insert(idx, &item);
}
Ok(true)
self.db_wrapper.has_column_family()
}
fn cleanup(self) -> OperationResult<()> {
lib/segment/src/index/field_index/index_selector.rs
+53 -74
View File
@@ -80,9 +80,8 @@ impl IndexSelector<'_> {
);
}
return Ok(self
.numeric_new(field, create_if_missing)?
.map(FieldIndex::IntIndex));
self.numeric_new(field, create_if_missing)?
.map(FieldIndex::IntIndex)
}
(PayloadIndexType::IntMapIndex, PayloadSchemaParams::Integer(params)) => {
// IntMapIndex only gets created if `lookup` is true. This will only throw an error if storage is corrupt.
@@ -95,64 +94,47 @@ impl IndexSelector<'_> {
);
}
return Ok(self
.map_new(field, create_if_missing)?
.map(FieldIndex::IntMapIndex));
}
(PayloadIndexType::DatetimeIndex, PayloadSchemaParams::Datetime(_)) => {
return Ok(self
.numeric_new(field, create_if_missing)?
.map(FieldIndex::DatetimeIndex));
self.map_new(field, create_if_missing)?
.map(FieldIndex::IntMapIndex)
}
(PayloadIndexType::DatetimeIndex, PayloadSchemaParams::Datetime(_)) => self
.numeric_new(field, create_if_missing)?
.map(FieldIndex::DatetimeIndex),
(PayloadIndexType::KeywordIndex, PayloadSchemaParams::Keyword(_)) => {
return Ok(self
.map_new(field, create_if_missing)?
.map(FieldIndex::KeywordIndex));
}
(PayloadIndexType::KeywordIndex, PayloadSchemaParams::Keyword(_)) => self
.map_new(field, create_if_missing)?
.map(FieldIndex::KeywordIndex),
(PayloadIndexType::FloatIndex, PayloadSchemaParams::Float(_)) => {
return Ok(self
.numeric_new(field, create_if_missing)?
.map(FieldIndex::FloatIndex));
}
(PayloadIndexType::FloatIndex, PayloadSchemaParams::Float(_)) => self
.numeric_new(field, create_if_missing)?
.map(FieldIndex::FloatIndex),
(PayloadIndexType::GeoIndex, PayloadSchemaParams::Geo(_)) => {
return Ok(self
.geo_new(field, create_if_missing)?
.map(FieldIndex::GeoIndex));
}
(PayloadIndexType::GeoIndex, PayloadSchemaParams::Geo(_)) => self
.geo_new(field, create_if_missing)?
.map(FieldIndex::GeoIndex),
(PayloadIndexType::FullTextIndex, PayloadSchemaParams::Text(params)) => {
return Ok(self
.text_new(field, params.clone(), create_if_missing)?
.map(FieldIndex::FullTextIndex));
}
(PayloadIndexType::FullTextIndex, PayloadSchemaParams::Text(params)) => self
.text_new(field, params.clone(), create_if_missing)?
.map(FieldIndex::FullTextIndex),
(PayloadIndexType::BoolIndex, PayloadSchemaParams::Bool(_)) => {
self.bool_new(field, create_if_missing)?
}
(PayloadIndexType::BoolIndex, PayloadSchemaParams::Bool(_)) => self
.bool_new(field, create_if_missing)?
.map(FieldIndex::BoolIndex),
(PayloadIndexType::UuidIndex, PayloadSchemaParams::Uuid(_)) => {
return Ok(self
.map_new(field, create_if_missing)?
.map(FieldIndex::UuidMapIndex));
}
(PayloadIndexType::UuidIndex, PayloadSchemaParams::Uuid(_)) => self
.map_new(field, create_if_missing)?
.map(FieldIndex::UuidMapIndex),
(PayloadIndexType::UuidMapIndex, PayloadSchemaParams::Uuid(_)) => {
return Ok(self
.map_new(field, create_if_missing)?
.map(FieldIndex::UuidMapIndex));
}
(PayloadIndexType::UuidMapIndex, PayloadSchemaParams::Uuid(_)) => self
.map_new(field, create_if_missing)?
.map(FieldIndex::UuidMapIndex),
(PayloadIndexType::NullIndex, _) => {
let null_index = MutableNullIndex::open(
&null_dir(path, field),
total_point_count,
create_if_missing,
)?;
FieldIndex::NullIndex(null_index)
}
(PayloadIndexType::NullIndex, _) => MutableNullIndex::open(
&null_dir(path, field),
total_point_count,
create_if_missing,
)?
.map(FieldIndex::NullIndex),
// Storage inconsistency. Should never happen.
(index_type, schema) => {
@@ -162,7 +144,7 @@ impl IndexSelector<'_> {
}
};
Ok(Some(index))
Ok(index)
}
/// Selects index type based on field type.
@@ -208,7 +190,9 @@ impl IndexSelector<'_> {
PayloadSchemaParams::Text(text_index_params) => self
.text_new(field, text_index_params.clone(), create_if_missing)?
.map(|index| vec![FieldIndex::FullTextIndex(index)]),
PayloadSchemaParams::Bool(_) => Some(vec![self.bool_new(field, create_if_missing)?]),
PayloadSchemaParams::Bool(_) => self
.bool_new(field, create_if_missing)?
.map(|index| vec![FieldIndex::BoolIndex(index)]),
PayloadSchemaParams::Datetime(_) => self
.numeric_new(field, create_if_missing)?
.map(|index| vec![FieldIndex::DatetimeIndex(index)]),
@@ -483,13 +467,12 @@ impl IndexSelector<'_> {
field: &JsonPath,
total_point_count: usize,
create_if_missing: bool,
) -> OperationResult<FieldIndex> {
) -> OperationResult<Option<FieldIndex>> {
// null index is always on disk and is appendable
Ok(FieldIndex::NullIndex(MutableNullIndex::open(
&null_dir(dir, field),
total_point_count,
create_if_missing,
)?))
Ok(
MutableNullIndex::open(&null_dir(dir, field), total_point_count, create_if_missing)?
.map(FieldIndex::NullIndex),
)
}
fn text_new(
@@ -558,7 +541,7 @@ impl IndexSelector<'_> {
}) => Ok(FieldIndexBuilder::BoolIndex(SimpleBoolIndex::builder(
Arc::clone(db),
&field.to_string(),
))),
)?)),
IndexSelector::Mmap(IndexSelectorMmap { dir, is_on_disk: _ }) => {
let dir = bool_dir(dir, field);
Ok(FieldIndexBuilder::BoolMmapIndex(MutableBoolIndex::builder(
@@ -575,30 +558,26 @@ impl IndexSelector<'_> {
}
}
fn bool_new(&self, field: &JsonPath, create_if_missing: bool) -> OperationResult<FieldIndex> {
fn bool_new(
&self,
field: &JsonPath,
create_if_missing: bool,
) -> OperationResult<Option<BoolIndex>> {
Ok(match self {
#[cfg(feature = "rocksdb")]
IndexSelector::RocksDb(IndexSelectorRocksDb {
db,
is_appendable: _,
}) => FieldIndex::BoolIndex(BoolIndex::Simple(SimpleBoolIndex::new(
Arc::clone(db),
&field.to_string(),
))),
}) => SimpleBoolIndex::new(Arc::clone(db), &field.to_string(), create_if_missing)?
.map(BoolIndex::Simple),
IndexSelector::Mmap(IndexSelectorMmap { dir, is_on_disk: _ }) => {
let dir = bool_dir(dir, field);
FieldIndex::BoolIndex(BoolIndex::Mmap(MutableBoolIndex::open(
&dir,
create_if_missing,
)?))
MutableBoolIndex::open(&dir, create_if_missing)?.map(BoolIndex::Mmap)
}
// Skip Gridstore for boolean index, mmap index is simpler and is also mutable
IndexSelector::Gridstore(IndexSelectorGridstore { dir }) => {
let dir = bool_dir(dir, field);
FieldIndex::BoolIndex(BoolIndex::Mmap(MutableBoolIndex::open(
&dir,
create_if_missing,
)?))
MutableBoolIndex::open(&dir, create_if_missing)?.map(BoolIndex::Mmap)
}
})
}
lib/segment/src/index/field_index/null_index/mutable_null_index.rs
+16 -11
View File
@@ -36,10 +36,17 @@ struct Storage {
impl MutableNullIndex {
pub fn builder(path: &Path) -> OperationResult<MutableNullIndexBuilder> {
Ok(MutableNullIndexBuilder(Self::open(path, 0, true)?))
Ok(MutableNullIndexBuilder(
Self::open(path, 0, true)?.ok_or_else(|| {
OperationError::service_error(format!(
"Failed to create and open mutable null index at path: {}",
path.display(),
))
})?,
))
}
/// Open or create a mutable null index at the given path.
/// Open and load or create a mutable null index at the given path.
///
/// # Arguments
/// - `path` - The directory where the index files should live, must be exclusive to this index.
@@ -49,19 +56,15 @@ impl MutableNullIndex {
path: &Path,
total_point_count: usize,
create_if_missing: bool,
) -> OperationResult<Self> {
) -> OperationResult<Option<Self>> {
let has_values_dir = path.join(HAS_VALUES_DIRNAME);
// If has values directory doesn't exist, assume the index doesn't exist on disk
if !has_values_dir.is_dir() && !create_if_missing {
return Ok(Self {
base_dir: path.to_path_buf(),
storage: None,
total_point_count,
});
return Ok(None);
}
Self::open_or_create(path, total_point_count)
Ok(Some(Self::open_or_create(path, total_point_count)?))
}
fn open_or_create(path: &Path, total_point_count: usize) -> OperationResult<Self> {
@@ -240,9 +243,11 @@ impl PayloadFieldIndex for MutableNullIndex {
.map_or(0, |storage| storage.has_values_flags.len())
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load(&mut self) -> OperationResult<bool> {
let is_loaded = self.storage.is_some();
Ok(is_loaded)
// Note: this structure is now loaded on open
Ok(self.storage.is_some())
}
fn cleanup(self) -> OperationResult<()> {
lib/segment/src/index/struct_payload_index.rs
+4 -4
View File
@@ -199,14 +199,14 @@ impl StructPayloadIndex {
);
// Special null index complements every index.
let null_index = IndexSelector::new_null_index(
if let Some(null_index) = IndexSelector::new_null_index(
&self.path,
field,
total_point_count,
create_if_missing,
)?;
indexes.push(null_index);
)? {
indexes.push(null_index);
}
// Persist exact payload index types
is_dirty = true;