upstream-archive/qdrant
1
0
Fork 0
mirror of https://github.com/qdrant/qdrant.git synced 2026-09-21 13:37:46 -05:00
Code Issues Packages Projects Releases Wiki Activity

Remove load function in payload indices (#7040)

Browse Source 
* Remove load functions from numeric index

* Remove load functions from map index

* Remove unnecessary result and option

* Remove load functions from full text index

* Remove load functions from geo index

* Remove unnecessary result and option

* Remove load functions from bool index

* Remove load functions from null index

* Don't invoke load when loading payload indices, remove load from trait
This commit is contained in:
Tim Visée
2025-08-14 14:30:39 +02:00
committed by timvisee
parent 68825bfe6a
commit a48e559990
24 changed files with 32 additions and 549 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/segment/src/index/field_index/bool_index/mod.rs
+3 -23
View File
@@ -161,15 +161,6 @@ impl PayloadFieldIndex for BoolIndex {
}
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load(&mut self) -> crate::common::operation_error::OperationResult<bool> {
match self {
#[cfg(feature = "rocksdb")]
BoolIndex::Simple(index) => index.load(),
BoolIndex::Mmap(index) => index.load(),
}
}
fn cleanup(self) -> crate::common::operation_error::OperationResult<()> {
match self {
#[cfg(feature = "rocksdb")]
@@ -335,20 +326,10 @@ mod tests {
impl OpenIndex for SimpleBoolIndex {
fn open_at(path: &Path) -> BoolIndex {
let db = open_db_with_existing_cf(path).unwrap();
let mut index = SimpleBoolIndex::new(db.clone(), FIELD_NAME, true)
let index = SimpleBoolIndex::new(db.clone(), FIELD_NAME, true)
.unwrap()
.unwrap();
// Try to load if it exists
if index.load().unwrap() {
return BoolIndex::Simple(index);
}
drop(index);
// Otherwise create a new one
SimpleBoolIndex::builder(db, FIELD_NAME)
.unwrap()
.make_empty()
.unwrap()
BoolIndex::Simple(index)
}
}
@@ -459,8 +440,7 @@ mod tests {
drop(index);
let mut new_index = I::open_at(tmp_dir.path());
assert!(new_index.load().unwrap());
let new_index = I::open_at(tmp_dir.path());
let hw_acc = HwMeasurementAcc::new();
let hw_counter = hw_acc.get_counter_cell();
lib/segment/src/index/field_index/bool_index/mutable_bool_index.rs
-7
View File
@@ -353,13 +353,6 @@ impl PayloadFieldIndex for MutableBoolIndex {
self.indexed_count
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load(&mut self) -> OperationResult<bool> {
// Note: this structure is now loaded on open
Ok(self.storage.is_some())
}
fn cleanup(self) -> OperationResult<()> {
if self.base_dir.is_dir() {
std::fs::remove_dir_all(self.base_dir)?;
lib/segment/src/index/field_index/bool_index/simple_bool_index.rs
-7
View File
@@ -333,13 +333,6 @@ impl FieldIndexBuilderTrait for BoolIndexBuilder {
}
impl PayloadFieldIndex for SimpleBoolIndex {
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load(&mut self) -> OperationResult<bool> {
// Note: this structure is now loaded on open
self.db_wrapper.has_column_family()
}
fn cleanup(self) -> OperationResult<()> {
self.db_wrapper.remove_column_family()
}
lib/segment/src/index/field_index/field_index_base.rs
-19
View File
@@ -40,9 +40,6 @@ pub trait PayloadFieldIndex {
/// Return number of points with at least one value indexed in here
fn count_indexed_points(&self) -> usize;
/// Load index from disk.
fn load(&mut self) -> OperationResult<bool>;
/// Remove db content or files of the current payload index
fn cleanup(self) -> OperationResult<()>;
@@ -216,22 +213,6 @@ impl FieldIndex {
}
}
pub fn load(&mut self) -> OperationResult<bool> {
match self {
FieldIndex::IntIndex(payload_field_index) => payload_field_index.load(),
FieldIndex::DatetimeIndex(payload_field_index) => payload_field_index.load(),
FieldIndex::IntMapIndex(payload_field_index) => payload_field_index.load(),
FieldIndex::KeywordIndex(payload_field_index) => payload_field_index.load(),
FieldIndex::FloatIndex(payload_field_index) => payload_field_index.load(),
FieldIndex::GeoIndex(payload_field_index) => payload_field_index.load(),
FieldIndex::BoolIndex(payload_field_index) => payload_field_index.load(),
FieldIndex::FullTextIndex(payload_field_index) => payload_field_index.load(),
FieldIndex::UuidIndex(payload_field_index) => payload_field_index.load(),
FieldIndex::UuidMapIndex(payload_field_index) => payload_field_index.load(),
FieldIndex::NullIndex(payload_field_index) => payload_field_index.load(),
}
}
pub fn cleanup(self) -> OperationResult<()> {
match self {
FieldIndex::IntIndex(index) => index.cleanup(),
lib/segment/src/index/field_index/full_text_index/immutable_text_index.rs
-40
View File
@@ -92,46 +92,6 @@ impl ImmutableFullTextIndex {
})
}
/// Load storage
///
/// Loads in-memory index from backing RocksDB or mmap storage.
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
pub fn load(&self) -> OperationResult<bool> {
match self.storage {
#[cfg(feature = "rocksdb")]
Storage::RocksDb(_) => self.load_rocksdb(),
Storage::Mmap(_) => self.load_mmap(),
}
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
#[cfg(feature = "rocksdb")]
fn load_rocksdb(&self) -> OperationResult<bool> {
let Storage::RocksDb(db_wrapper) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from RocksDB, using different storage backend",
));
};
// Note: this structure is now loaded on open
db_wrapper.has_column_family()
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load_mmap(&self) -> OperationResult<bool> {
#[cfg_attr(not(feature = "rocksdb"), expect(irrefutable_let_patterns))]
let Storage::Mmap(_) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from mmap, using different storage backend",
));
};
// Note: this structure is now loaded on open
Ok(true)
}
#[cfg_attr(not(feature = "rocksdb"), expect(clippy::unnecessary_wraps))]
pub fn remove_point(&mut self, id: PointOffsetType) -> OperationResult<()> {
if self.inverted_index.remove(id) {
lib/segment/src/index/field_index/full_text_index/inverted_index/mmap_inverted_index/mod.rs
-7
View File
@@ -150,13 +150,6 @@ impl MmapInvertedIndex {
}))
}
// TODO(payload-index-non-optional-storage): remove this method when single stage open/load is implemented
pub fn load(&self) -> bool {
// Note: this structure is now loaded on open
self.storage.is_some()
}
// TODO(payload-index-non-optional-storage): remove Either, just return pure iterator
pub(super) fn iter_vocab(&self) -> impl Iterator<Item = (&str, &TokenId)> + '_ {
let Some(storage) = &self.storage else {
lib/segment/src/index/field_index/full_text_index/mmap_text_index.rs
-4
View File
@@ -42,10 +42,6 @@ impl MmapFullTextIndex {
}))
}
pub fn load(&self) -> bool {
self.inverted_index.load()
}
pub fn files(&self) -> Vec<PathBuf> {
self.inverted_index.files()
}
lib/segment/src/index/field_index/full_text_index/mutable_text_index.rs
-44
View File
@@ -136,46 +136,6 @@ impl MutableFullTextIndex {
}))
}
/// Load storage
///
/// Loads in-memory index from backing RocksDB or Gridstore storage.
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
pub(super) fn load(&self) -> OperationResult<bool> {
match self.storage {
#[cfg(feature = "rocksdb")]
Storage::RocksDb(_) => self.load_rocksdb(),
Storage::Gridstore(Some(_)) => self.load_gridstore(),
Storage::Gridstore(None) => Ok(false),
}
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
#[cfg(feature = "rocksdb")]
fn load_rocksdb(&self) -> OperationResult<bool> {
let Storage::RocksDb(db_wrapper) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from RocksDB, using different storage backend",
));
};
// Note: this structure is now loaded on open
db_wrapper.has_column_family()
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load_gridstore(&self) -> OperationResult<bool> {
let Storage::Gridstore(Some(_)) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from Gridstore, using different storage backend",
));
};
// Note: this structure is now loaded on open
Ok(true)
}
#[inline]
pub(super) fn init(&self) -> OperationResult<()> {
match &self.storage {
@@ -480,8 +440,6 @@ mod tests {
FullTextIndex::new_gridstore(temp_dir.path().join("test_db"), config.clone(), true)
.unwrap()
.unwrap();
let loaded = index.load().unwrap();
assert!(loaded);
let hw_cell = HardwareCounterCell::new();
@@ -552,8 +510,6 @@ mod tests {
FullTextIndex::new_gridstore(temp_dir.path().join("test_db"), config, true)
.unwrap()
.unwrap();
let loaded = index.load().unwrap();
assert!(loaded);
assert_eq!(index.count_indexed_points(), 4);
lib/segment/src/index/field_index/full_text_index/text_index.rs
-8
View File
@@ -526,14 +526,6 @@ impl PayloadFieldIndex for FullTextIndex {
self.points_count()
}
fn load(&mut self) -> OperationResult<bool> {
match self {
Self::Mutable(index) => index.load(),
Self::Immutable(index) => index.load(),
Self::Mmap(index) => Ok(index.load()),
}
}
fn cleanup(self) -> OperationResult<()> {
match self {
Self::Mutable(index) => index.wipe(),
lib/segment/src/index/field_index/geo_index/immutable_geo_index.rs
+2 -50
View File
@@ -86,10 +86,6 @@ impl ImmutableGeoMapIndex {
// Column family doesn't exist, cannot load
return Ok(None);
};
// TODO(payload-index-remove-load): remove load when single stage open/load is implemented
if !mutable.load()? {
return Ok(None);
}
let InMemoryGeoMapIndex {
points_per_hash,
@@ -140,11 +136,7 @@ impl ImmutableGeoMapIndex {
}
/// Open and load immutable geo index from mmap storage
pub fn open_mmap(index: MmapGeoMapIndex) -> OperationResult<Option<Self>> {
// TODO(payload-index-remove-load): remove load when single stage open/load is implemented
if !index.load()? {
return Ok(None);
}
pub fn open_mmap(index: MmapGeoMapIndex) -> Self {
let index_storage = index.storage.as_ref().unwrap();
let counts_per_hash = index_storage
@@ -237,47 +229,7 @@ impl ImmutableGeoMapIndex {
index.decrement_hash_point_counts(&removed_geo_hashes);
}
Ok(Some(index))
}
/// Load storage
///
/// Loads in-memory index from backing RocksDB or mmap storage.
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
pub fn load(&self) -> OperationResult<bool> {
match self.storage {
#[cfg(feature = "rocksdb")]
Storage::RocksDb(_) => self.load_rocksdb(),
Storage::Mmap(_) => self.load_mmap(),
}
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
#[cfg(feature = "rocksdb")]
fn load_rocksdb(&self) -> OperationResult<bool> {
let Storage::RocksDb(db_wrapper) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from RocksDB, using different storage backend",
));
};
// Note: this structure is now loaded on open
db_wrapper.has_column_family()
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load_mmap(&self) -> OperationResult<bool> {
#[allow(irrefutable_let_patterns)]
let Storage::Mmap(_) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from mmap, using different storage backend",
));
};
// Note: this structure is now loaded on open
Ok(true)
index
}
#[cfg(all(test, feature = "rocksdb"))]
lib/segment/src/index/field_index/geo_index/mmap_geo_index.rs
-8
View File
@@ -258,14 +258,6 @@ impl MmapGeoMapIndex {
}))
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
pub fn load(&self) -> OperationResult<bool> {
// Note: this structure is now loaded on open
let is_loaded = self.storage.is_some();
Ok(is_loaded)
}
pub fn check_values_any(
&self,
idx: PointOffsetType,
lib/segment/src/index/field_index/geo_index/mod.rs
+17 -40
View File
@@ -76,12 +76,12 @@ impl GeoMapIndex {
};
let index = if is_on_disk {
Some(GeoMapIndex::Mmap(Box::new(mmap_index)))
GeoMapIndex::Mmap(Box::new(mmap_index))
} else {
ImmutableGeoMapIndex::open_mmap(mmap_index)?.map(GeoMapIndex::Immutable)
GeoMapIndex::Immutable(ImmutableGeoMapIndex::open_mmap(mmap_index))
};
Ok(index)
Ok(Some(index))
}
pub fn new_gridstore(dir: PathBuf, create_if_missing: bool) -> OperationResult<Option<Self>> {
@@ -522,12 +522,10 @@ impl FieldIndexBuilderTrait for GeoMapImmutableIndexBuilder {
fn finalize(self) -> OperationResult<Self::FieldIndexType> {
drop(self.index);
let mut immutable_index = GeoMapIndex::new_memory(self.db, &self.field, false, false)?
let immutable_index = GeoMapIndex::new_memory(self.db, &self.field, false, false)?
.ok_or_else(|| {
OperationError::service_error("Failed to open GeoMapIndex after creating it")
})?;
// TODO(payload-index-remove-load): remove load when single stage open/load is implemented
immutable_index.load()?;
Ok(immutable_index)
}
}
@@ -672,14 +670,6 @@ impl PayloadFieldIndex for GeoMapIndex {
self.points_count()
}
fn load(&mut self) -> OperationResult<bool> {
match self {
GeoMapIndex::Mutable(index) => index.load(),
GeoMapIndex::Immutable(index) => index.load(),
GeoMapIndex::Mmap(index) => index.load(),
}
}
fn cleanup(self) -> OperationResult<()> {
match self {
GeoMapIndex::Mutable(index) => index.wipe(),
@@ -901,10 +891,7 @@ mod tests {
};
// Load index from mmap
let mut index = GeoMapIndex::Immutable(
ImmutableGeoMapIndex::open_mmap(*index).unwrap().unwrap(),
);
index.load()?;
let index = GeoMapIndex::Immutable(ImmutableGeoMapIndex::open_mmap(*index));
Ok(index)
}
}
@@ -1516,7 +1503,7 @@ mod tests {
#[cfg(feature = "rocksdb")]
let db = open_db_with_existing_cf(&temp_dir.path().join("test_db")).unwrap();
let mut new_index = match index_type {
let new_index = match index_type {
#[cfg(feature = "rocksdb")]
IndexType::Mutable => GeoMapIndex::new_memory(db, FIELD_NAME, true, true)
.unwrap()
@@ -1533,17 +1520,12 @@ mod tests {
IndexType::Mmap => GeoMapIndex::new_mmap(temp_dir.path(), false)
.unwrap()
.unwrap(),
IndexType::RamMmap => GeoMapIndex::Immutable(
ImmutableGeoMapIndex::open_mmap(
MmapGeoMapIndex::open(temp_dir.path(), false)
.unwrap()
.unwrap(),
)
.unwrap()
.unwrap(),
),
IndexType::RamMmap => GeoMapIndex::Immutable(ImmutableGeoMapIndex::open_mmap(
MmapGeoMapIndex::open(temp_dir.path(), false)
.unwrap()
.unwrap(),
)),
};
new_index.load().unwrap();
let berlin_geo_radius = GeoRadius {
center: BERLIN,
@@ -1611,7 +1593,7 @@ mod tests {
#[cfg(feature = "rocksdb")]
let db = open_db_with_existing_cf(&temp_dir.path().join("test_db")).unwrap();
let mut new_index = match index_type {
let new_index = match index_type {
#[cfg(feature = "rocksdb")]
IndexType::Mutable => GeoMapIndex::new_memory(db, FIELD_NAME, true, true)
.unwrap()
@@ -1628,17 +1610,12 @@ mod tests {
IndexType::Mmap => GeoMapIndex::new_mmap(temp_dir.path(), false)
.unwrap()
.unwrap(),
IndexType::RamMmap => GeoMapIndex::Immutable(
ImmutableGeoMapIndex::open_mmap(
MmapGeoMapIndex::open(temp_dir.path(), false)
.unwrap()
.unwrap(),
)
.unwrap()
.unwrap(),
),
IndexType::RamMmap => GeoMapIndex::Immutable(ImmutableGeoMapIndex::open_mmap(
MmapGeoMapIndex::open(temp_dir.path(), false)
.unwrap()
.unwrap(),
)),
};
new_index.load().unwrap();
assert_eq!(new_index.points_count(), 1);
if index_type != IndexType::Mmap {
assert_eq!(new_index.points_values_count(), 2);
lib/segment/src/index/field_index/geo_index/mutable_geo_index.rs
-40
View File
@@ -229,46 +229,6 @@ impl MutableGeoMapIndex {
}))
}
/// Load storage
///
/// Loads in-memory index from backing RocksDB or Gridstore storage.
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
pub(super) fn load(&self) -> OperationResult<bool> {
match self.storage {
#[cfg(feature = "rocksdb")]
Storage::RocksDb(_) => self.load_rocksdb(),
Storage::Gridstore(Some(_)) => self.load_gridstore(),
Storage::Gridstore(None) => Ok(false),
}
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
#[cfg(feature = "rocksdb")]
fn load_rocksdb(&self) -> OperationResult<bool> {
let Storage::RocksDb(db_wrapper) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from RocksDB, using different storage backend",
));
};
// Note: this structure is now loaded on open
db_wrapper.has_column_family()
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load_gridstore(&self) -> OperationResult<bool> {
let Storage::Gridstore(Some(_)) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from Gridstore, using different storage backend",
));
};
// Note: this structure is now loaded on open
Ok(true)
}
#[cfg_attr(not(feature = "rocksdb"), expect(dead_code))]
#[inline]
pub(super) fn clear(&self) -> OperationResult<()> {
lib/segment/src/index/field_index/map_index/immutable_map_index.rs
+3 -50
View File
@@ -20,7 +20,7 @@ use super::MapIndex;
use super::mmap_map_index::MmapMapIndex;
use super::{IdIter, MapIndexKey};
use crate::common::Flusher;
use crate::common::operation_error::{OperationError, OperationResult};
use crate::common::operation_error::OperationResult;
#[cfg(feature = "rocksdb")]
use crate::common::rocksdb_buffered_delete_wrapper::DatabaseColumnScheduledDeleteWrapper;
#[cfg(feature = "rocksdb")]
@@ -80,8 +80,6 @@ where
// Column family doesn't exist, cannot load
return Ok(None);
};
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
mutable.load()?;
let MutableMapIndex::<N> {
map,
point_to_values,
@@ -140,12 +138,7 @@ where
}
/// Open and load immutable numeric index from mmap storage
pub(super) fn open_mmap(index: MmapMapIndex<N>) -> OperationResult<Option<Self>> {
// TODO(payload-index-remove-load): remove load when single stage open/load is implemented
if !index.load()? {
return Ok(None);
}
pub(super) fn open_mmap(index: MmapMapIndex<N>) -> Self {
let index_storage = index.storage.as_ref().unwrap();
// Construct intermediate values to points map from backing storage
@@ -226,7 +219,7 @@ where
log::warn!("Failed to clear mmap cache of ram mmap map index: {err}");
}
Ok(Some(Self {
Self {
value_to_points,
value_to_points_container,
deleted_value_to_points_container: BitVec::new(),
@@ -234,49 +227,9 @@ where
indexed_points,
values_count,
storage: Storage::Mmap(Box::new(index)),
}))
}
/// Load storage
///
/// Loads in-memory index from backing RocksDB or mmap storage.
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
pub fn load(&self) -> OperationResult<bool> {
match self.storage {
#[cfg(feature = "rocksdb")]
Storage::RocksDb(_) => self.load_rocksdb(),
Storage::Mmap(_) => self.load_mmap(),
}
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
#[cfg(feature = "rocksdb")]
fn load_rocksdb(&self) -> OperationResult<bool> {
let Storage::RocksDb(db_wrapper) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from RocksDB, using different storage backend",
));
};
// Note: this structure is now loaded on open
db_wrapper.has_column_family()
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load_mmap(&self) -> OperationResult<bool> {
#[allow(irrefutable_let_patterns)]
let Storage::Mmap(_) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from mmap, using different storage backend",
));
};
// Note: this structure is now loaded on open
Ok(true)
}
/// Return mutable slice of a container which holds point_ids for given value.
///
/// The returned slice is sorted and does contain deleted values.
lib/segment/src/index/field_index/map_index/mmap_map_index.rs
-8
View File
@@ -87,14 +87,6 @@ impl<N: MapIndexKey + Key + ?Sized> MmapMapIndex<N> {
}))
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
pub fn load(&self) -> OperationResult<bool> {
// Note: this structure is now loaded on open
let is_loaded = self.storage.is_some();
Ok(is_loaded)
}
pub fn build(
path: &Path,
point_to_values: Vec<Vec<N::Owned>>,
lib/segment/src/index/field_index/map_index/mod.rs
+4 -25
View File
@@ -132,12 +132,12 @@ where
let index = if is_on_disk {
// Use on mmap directly
Some(MapIndex::Mmap(Box::new(mmap_index)))
MapIndex::Mmap(Box::new(mmap_index))
} else {
// Load into RAM, use mmap as backing storage
ImmutableMapIndex::open_mmap(mmap_index)?.map(MapIndex::Immutable)
MapIndex::Immutable(ImmutableMapIndex::open_mmap(mmap_index))
};
Ok(index)
Ok(Some(index))
}
pub fn new_gridstore(dir: PathBuf, create_if_missing: bool) -> OperationResult<Option<Self>> {
@@ -172,14 +172,6 @@ where
MapIndexGridstoreBuilder::new(dir)
}
fn load(&mut self) -> OperationResult<bool> {
match self {
MapIndex::Mutable(index) => index.load(),
MapIndex::Immutable(index) => index.load(),
MapIndex::Mmap(index) => index.load(),
}
}
pub fn check_values_any(
&self,
idx: PointOffsetType,
@@ -736,10 +728,6 @@ impl PayloadFieldIndex for MapIndex<str> {
self.get_indexed_points()
}
fn load(&mut self) -> OperationResult<bool> {
self.load()
}
fn cleanup(self) -> OperationResult<()> {
self.wipe()
}
@@ -888,10 +876,6 @@ impl PayloadFieldIndex for MapIndex<UuidIntType> {
self.get_indexed_points()
}
fn load(&mut self) -> OperationResult<bool> {
self.load()
}
fn cleanup(self) -> OperationResult<()> {
self.wipe()
}
@@ -1081,10 +1065,6 @@ impl PayloadFieldIndex for MapIndex<IntPayloadType> {
self.get_indexed_points()
}
fn load(&mut self) -> OperationResult<bool> {
self.load()
}
fn cleanup(self) -> OperationResult<()> {
self.wipe()
}
@@ -1450,7 +1430,7 @@ mod tests {
where
Vec<N::Owned>: Blob + Send + Sync,
{
let mut index = match index_type {
let index = match index_type {
#[cfg(feature = "rocksdb")]
IndexType::Mutable => MapIndex::<N>::new_rocksdb(
open_db_with_existing_cf(path).unwrap(),
@@ -1475,7 +1455,6 @@ mod tests {
IndexType::Mmap => MapIndex::<N>::new_mmap(path, true).unwrap().unwrap(),
IndexType::RamMmap => MapIndex::<N>::new_mmap(path, false).unwrap().unwrap(),
};
index.load().unwrap();
for (idx, values) in data.iter().enumerate() {
let index_values: HashSet<N::Owned> = index
.get_values(idx as PointOffsetType)
lib/segment/src/index/field_index/map_index/mutable_map_index.rs
-40
View File
@@ -189,46 +189,6 @@ where
}))
}
/// Load storage
///
/// Loads in-memory index from backing RocksDB or Gridstore storage.
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
pub(super) fn load(&self) -> OperationResult<bool> {
match self.storage {
#[cfg(feature = "rocksdb")]
Storage::RocksDb(_) => self.load_rocksdb(),
Storage::Gridstore(Some(_)) => self.load_gridstore(),
Storage::Gridstore(None) => Ok(false),
}
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
#[cfg(feature = "rocksdb")]
fn load_rocksdb(&self) -> OperationResult<bool> {
let Storage::RocksDb(db_wrapper) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from RocksDB, using different storage backend",
));
};
// Note: this structure is now loaded on open
db_wrapper.has_column_family()
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load_gridstore(&self) -> OperationResult<bool> {
let Storage::Gridstore(Some(_)) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from Gridstore, using different storage backend",
));
};
// Note: this structure is now loaded on open
Ok(true)
}
pub fn add_many_to_map<Q>(
&mut self,
idx: PointOffsetType,
lib/segment/src/index/field_index/null_index/mutable_null_index.rs
-7
View File
@@ -243,13 +243,6 @@ impl PayloadFieldIndex for MutableNullIndex {
.map_or(0, |storage| storage.has_values_flags.len())
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load(&mut self) -> OperationResult<bool> {
// Note: this structure is now loaded on open
Ok(self.storage.is_some())
}
fn cleanup(self) -> OperationResult<()> {
if self.base_dir.is_dir() {
std::fs::remove_dir_all(&self.base_dir)?;
lib/segment/src/index/field_index/numeric_index/immutable_numeric_index.rs
+1 -43
View File
@@ -17,7 +17,7 @@ use super::Encodable;
use super::mmap_numeric_index::MmapNumericIndex;
use super::mutable_numeric_index::InMemoryNumericIndex;
use crate::common::Flusher;
use crate::common::operation_error::{OperationError, OperationResult};
use crate::common::operation_error::OperationResult;
#[cfg(feature = "rocksdb")]
use crate::common::rocksdb_buffered_delete_wrapper::DatabaseColumnScheduledDeleteWrapper;
#[cfg(feature = "rocksdb")]
@@ -179,8 +179,6 @@ where
// Column family doesn't exist, cannot load
return Ok(None);
};
// TODO(payload-index-remove-load): remove load when single stage open/load is implemented
mutable.load()?;
let InMemoryNumericIndex {
map,
@@ -226,46 +224,6 @@ where
}
}
/// Load storage
///
/// Loads in-memory index from backing RocksDB or mmap storage.
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
pub(super) fn load(&self) -> OperationResult<bool> {
match self.storage {
#[cfg(feature = "rocksdb")]
Storage::RocksDb(_) => self.load_rocksdb(),
Storage::Mmap(_) => self.load_mmap(),
}
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
#[cfg(feature = "rocksdb")]
fn load_rocksdb(&self) -> OperationResult<bool> {
let Storage::RocksDb(_) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from RocksDB, using different storage backend",
));
};
// Note: this structure is now loaded on open
Ok(true)
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load_mmap(&self) -> OperationResult<bool> {
#[allow(irrefutable_let_patterns)]
let Storage::Mmap(_) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from mmap, using different storage backend",
));
};
// Note: this structure is now loaded on open
Ok(true)
}
#[cfg(all(test, feature = "rocksdb"))]
pub(super) fn db_wrapper(&self) -> Option<&DatabaseColumnScheduledDeleteWrapper> {
match &self.storage {
lib/segment/src/index/field_index/numeric_index/mmap_numeric_index.rs
-8
View File
@@ -198,14 +198,6 @@ impl<T: Encodable + Numericable + Default + MmapValue> MmapNumericIndex<T> {
}))
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
pub fn load(&self) -> OperationResult<bool> {
// Note: this structure is now loaded on open
let is_loaded = self.storage.is_some();
Ok(is_loaded)
}
pub fn wipe(self) -> OperationResult<()> {
let files = self.files();
let Self { path, .. } = self;
lib/segment/src/index/field_index/numeric_index/mod.rs
+1 -15
View File
@@ -217,14 +217,6 @@ where
.map(NumericIndexInner::Mutable))
}
pub fn load(&mut self) -> OperationResult<bool> {
match self {
NumericIndexInner::Mutable(index) => index.load(),
NumericIndexInner::Immutable(index) => index.load(),
NumericIndexInner::Mmap(index) => index.load(),
}
}
fn get_histogram(&self) -> &Histogram<T> {
match self {
NumericIndexInner::Mutable(index) => index.get_histogram(),
@@ -647,7 +639,6 @@ where
pub fn check_values_any(&self, idx: PointOffsetType, check_fn: impl Fn(&T) -> bool, hw_counter: &HardwareCounterCell) -> bool;
pub fn cleanup(self) -> OperationResult<()>;
pub fn get_telemetry_data(&self) -> PayloadIndexTelemetry;
pub fn load(&mut self) -> OperationResult<bool>;
pub fn values_count(&self, idx: PointOffsetType) -> usize;
pub fn get_values(&self, idx: PointOffsetType) -> Option<Box<dyn Iterator<Item = T> + '_>>;
pub fn values_is_empty(&self, idx: PointOffsetType) -> bool;
@@ -745,11 +736,10 @@ where
fn finalize(self) -> OperationResult<Self::FieldIndexType> {
self.index.inner.flusher()()?;
drop(self.index);
let mut inner: NumericIndexInner<T> =
let inner: NumericIndexInner<T> =
NumericIndexInner::new_rocksdb(self.db, &self.field, false, false)?
// unwrap safety: only used in testing
.unwrap();
inner.load()?;
Ok(NumericIndex {
inner,
_phantom: PhantomData,
@@ -892,10 +882,6 @@ where
self.get_points_count()
}
fn load(&mut self) -> OperationResult<bool> {
NumericIndexInner::load(self)
}
fn cleanup(self) -> OperationResult<()> {
match self {
NumericIndexInner::Mutable(index) => index.wipe(),
lib/segment/src/index/field_index/numeric_index/mutable_numeric_index.rs
-41
View File
@@ -346,47 +346,6 @@ where
}))
}
/// Load storage
///
/// Loads in-memory index from backing RocksDB or Gridstore storage.
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
pub(super) fn load(&self) -> OperationResult<bool> {
match self.storage {
#[cfg(feature = "rocksdb")]
Storage::RocksDb(_) => self.load_rocksdb(),
Storage::Gridstore(Some(_)) => self.load_gridstore(),
Storage::Gridstore(None) => Ok(false),
}
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
#[cfg(feature = "rocksdb")]
fn load_rocksdb(&self) -> OperationResult<bool> {
let Storage::RocksDb(db_wrapper) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from RocksDB, using different storage backend",
));
};
// Note: this structure is now loaded on open
db_wrapper.has_column_family()
}
// TODO(payload-index-remove-load): remove method when single stage open/load is implemented
fn load_gridstore(&self) -> OperationResult<bool> {
#[allow(irrefutable_let_patterns)]
let Storage::Gridstore(store) = &self.storage else {
return Err(OperationError::service_error(
"Failed to load index from Gridstore, using different storage backend",
));
};
// Note: this structure is now loaded on open
Ok(store.is_some())
}
pub fn into_in_memory_index(self) -> InMemoryNumericIndex<T> {
self.in_memory_index
}
lib/segment/src/index/field_index/numeric_index/tests.rs
+1 -2
View File
@@ -384,7 +384,7 @@ fn test_numeric_index_load_from_disk(#[case] index_type: IndexType) {
};
drop(index);
let mut new_index = match index_type {
let new_index = match index_type {
#[cfg(feature = "rocksdb")]
IndexType::Mutable => {
NumericIndexInner::<FloatPayloadType>::new_rocksdb(db.unwrap(), COLUMN_NAME, true, true)
@@ -415,7 +415,6 @@ fn test_numeric_index_load_from_disk(#[case] index_type: IndexType) {
.unwrap()
}
};
new_index.load().unwrap();
test_cond(
&new_index,
lib/segment/src/index/struct_payload_index.rs
-13
View File
@@ -280,19 +280,6 @@ impl StructPayloadIndex {
}
}
// Load all indices, trigger rebuild if load is not successful
if !rebuild {
for ref mut index in indexes.iter_mut() {
if !index.load()? {
rebuild = true;
log::debug!(
"Payload index for field `{field}` was not loaded, triggering rebuild",
);
break;
}
}
}
// If index is not properly loaded or when migrating, rebuild indices
if rebuild {
log::debug!("Rebuilding payload index for field `{field}`...");