From 614cedae4698e02bb2b5637dfbd29b1e233d98f2 Mon Sep 17 00:00:00 2001 From: Andrey Vasnetsov Date: Wed, 29 Apr 2026 09:20:45 +0200 Subject: [PATCH] Split IdTracker trait into IdTrackerRead and IdTracker (#8826) Read-only methods now live on a separate IdTrackerRead trait, with the mutating IdTracker trait extending it. This lets read-only call sites depend only on the read API. Co-authored-by: Claude Opus 4.7 (1M context) --- .../optimizers/vacuum_optimizer.rs | 2 +- lib/segment/benches/hnsw_incremental_build.rs | 2 +- lib/segment/benches/vector_search.rs | 2 +- .../src/id_tracker/id_tracker_base/mod.rs | 2 +- .../id_tracker_base/tracker_enum.rs | 142 +++++++++-------- .../id_tracker/id_tracker_base/trait_def.rs | 150 +++++++++--------- .../src/id_tracker/immutable_id_tracker.rs | 90 ++++++----- .../src/id_tracker/in_memory_id_tracker.rs | 118 +++++++------- .../src/id_tracker/mutable_id_tracker/mod.rs | 100 ++++++------ .../id_tracker/mutable_id_tracker/tests.rs | 2 +- .../src/index/field_index/index_selector.rs | 2 +- lib/segment/src/index/hnsw_index/hnsw.rs | 2 +- lib/segment/src/index/plain_payload_index.rs | 2 +- lib/segment/src/index/plain_vector_index.rs | 2 +- .../query_optimization/condition_converter.rs | 2 +- .../index/sparse_index/sparse_vector_index.rs | 2 +- lib/segment/src/index/struct_payload_index.rs | 2 +- .../src/payload_storage/query_checker.rs | 2 +- lib/segment/src/segment/entry.rs | 2 +- lib/segment/src/segment/facet.rs | 2 +- lib/segment/src/segment/order_by.rs | 2 +- lib/segment/src/segment/sampling.rs | 2 +- lib/segment/src/segment/scroll.rs | 2 +- lib/segment/src/segment/search.rs | 2 +- lib/segment/src/segment/segment_ops.rs | 2 +- lib/segment/src/segment/vector_name_ops.rs | 2 +- lib/segment/src/segment/vectors.rs | 2 +- .../segment_constructor/segment_builder.rs | 2 +- .../segment_constructor_base.rs | 2 +- .../dense/dense_vector_storage.rs | 2 +- .../vector_storage/tests/async_raw_scorer.rs | 2 +- .../tests/custom_query_scorer_equivalency.rs | 2 +- .../test_appendable_dense_vector_storage.rs | 2 +- ...t_appendable_multi_dense_vector_storage.rs | 2 +- .../test_appendable_sparse_vector_storage.rs | 2 +- .../integration/multivector_hnsw_test.rs | 2 +- .../tests/integration/payload_index_test.rs | 2 +- .../tests/integration/segment_builder_test.rs | 2 +- .../sparse_vector_index_search_tests.rs | 2 +- 39 files changed, 339 insertions(+), 329 deletions(-) diff --git a/lib/collection/src/collection_manager/optimizers/vacuum_optimizer.rs b/lib/collection/src/collection_manager/optimizers/vacuum_optimizer.rs index 9d51063f67..1df9193e75 100644 --- a/lib/collection/src/collection_manager/optimizers/vacuum_optimizer.rs +++ b/lib/collection/src/collection_manager/optimizers/vacuum_optimizer.rs @@ -14,7 +14,7 @@ mod tests { use common::counter::hardware_counter::HardwareCounterCell; use itertools::Itertools; use segment::entry::{NonAppendableSegmentEntry as _, ReadSegmentEntry as _}; - use segment::id_tracker::IdTracker; + use segment::id_tracker::IdTrackerRead; use segment::index::VectorIndex; use segment::payload_json; use segment::types::{ diff --git a/lib/segment/benches/hnsw_incremental_build.rs b/lib/segment/benches/hnsw_incremental_build.rs index 65854c2257..a8f46451a8 100644 --- a/lib/segment/benches/hnsw_incremental_build.rs +++ b/lib/segment/benches/hnsw_incremental_build.rs @@ -30,7 +30,7 @@ use segment::data_types::vectors::{ }; use segment::entry::{SegmentEntry as _, StorageSegmentEntry as _}; use segment::fixtures::index_fixtures::random_vector; -use segment::id_tracker::{IdTracker, IdTrackerEnum}; +use segment::id_tracker::{IdTrackerEnum, IdTrackerRead}; use segment::index::hnsw_index::get_num_indexing_threads; use segment::index::hnsw_index::hnsw::{HNSWIndex, HnswIndexOpenArgs}; use segment::index::{VectorIndex as _, VectorIndexEnum}; diff --git a/lib/segment/benches/vector_search.rs b/lib/segment/benches/vector_search.rs index f09c910225..3ab588c347 100644 --- a/lib/segment/benches/vector_search.rs +++ b/lib/segment/benches/vector_search.rs @@ -9,7 +9,7 @@ use rand::distr::StandardUniform; use segment::data_types::named_vectors::CowVector; use segment::data_types::vectors::{DenseVector, QueryVector}; use segment::fixtures::payload_context_fixture::create_id_tracker_fixture; -use segment::id_tracker::IdTracker; +use segment::id_tracker::IdTrackerRead; use segment::index::hnsw_index::point_scorer::BatchFilteredSearcher; use segment::types::Distance; use segment::vector_storage::dense::dense_vector_storage::open_dense_vector_storage; diff --git a/lib/segment/src/id_tracker/id_tracker_base/mod.rs b/lib/segment/src/id_tracker/id_tracker_base/mod.rs index d34a364dd6..0ce0647c42 100644 --- a/lib/segment/src/id_tracker/id_tracker_base/mod.rs +++ b/lib/segment/src/id_tracker/id_tracker_base/mod.rs @@ -4,4 +4,4 @@ mod trait_def; pub use point_mappings_ref::{PointMappingsGuard, PointMappingsRefEnum}; pub use tracker_enum::IdTrackerEnum; -pub use trait_def::{DELETED_POINT_VERSION, IdTracker}; +pub use trait_def::{DELETED_POINT_VERSION, IdTracker, IdTrackerRead}; diff --git a/lib/segment/src/id_tracker/id_tracker_base/tracker_enum.rs b/lib/segment/src/id_tracker/id_tracker_base/tracker_enum.rs index c8005e5c04..d0eb1422c0 100644 --- a/lib/segment/src/id_tracker/id_tracker_base/tracker_enum.rs +++ b/lib/segment/src/id_tracker/id_tracker_base/tracker_enum.rs @@ -4,7 +4,7 @@ use common::bitvec::BitSlice; use common::types::PointOffsetType; use super::point_mappings_ref::PointMappingsRefEnum; -use super::trait_def::IdTracker; +use super::trait_def::{IdTracker, IdTrackerRead}; use crate::common::Flusher; use crate::common::operation_error::OperationResult; use crate::id_tracker::immutable_id_tracker::ImmutableIdTracker; @@ -20,7 +20,7 @@ pub enum IdTrackerEnum { InMemoryIdTracker(InMemoryIdTracker), } -impl IdTracker for IdTrackerEnum { +impl IdTrackerRead for IdTrackerEnum { fn internal_version(&self, internal_id: PointOffsetType) -> Option { match self { IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.internal_version(internal_id), @@ -33,24 +33,6 @@ impl IdTracker for IdTrackerEnum { } } - fn set_internal_version( - &mut self, - internal_id: PointOffsetType, - version: SeqNumberType, - ) -> OperationResult<()> { - match self { - IdTrackerEnum::MutableIdTracker(id_tracker) => { - id_tracker.set_internal_version(internal_id, version) - } - IdTrackerEnum::ImmutableIdTracker(id_tracker) => { - id_tracker.set_internal_version(internal_id, version) - } - IdTrackerEnum::InMemoryIdTracker(id_tracker) => { - id_tracker.set_internal_version(internal_id, version) - } - } - } - fn internal_id(&self, external_id: PointIdType) -> Option { match self { IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.internal_id(external_id), @@ -67,40 +49,6 @@ impl IdTracker for IdTrackerEnum { } } - fn set_link( - &mut self, - external_id: PointIdType, - internal_id: PointOffsetType, - ) -> OperationResult<()> { - match self { - IdTrackerEnum::MutableIdTracker(id_tracker) => { - id_tracker.set_link(external_id, internal_id) - } - IdTrackerEnum::ImmutableIdTracker(id_tracker) => { - id_tracker.set_link(external_id, internal_id) - } - IdTrackerEnum::InMemoryIdTracker(id_tracker) => { - id_tracker.set_link(external_id, internal_id) - } - } - } - - fn drop(&mut self, external_id: PointIdType) -> OperationResult<()> { - match self { - IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.drop(external_id), - IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.drop(external_id), - IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.drop(external_id), - } - } - - fn drop_internal(&mut self, internal_id: PointOffsetType) -> OperationResult<()> { - match self { - IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.drop_internal(internal_id), - IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.drop_internal(internal_id), - IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.drop_internal(internal_id), - } - } - fn point_mappings(&self) -> PointMappingsRefEnum<'_> { match self { IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.point_mappings(), @@ -109,22 +57,6 @@ impl IdTracker for IdTrackerEnum { } } - fn mapping_flusher(&self) -> Flusher { - match self { - IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.mapping_flusher(), - IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.mapping_flusher(), - IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.mapping_flusher(), - } - } - - fn versions_flusher(&self) -> Flusher { - match self { - IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.versions_flusher(), - IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.versions_flusher(), - IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.versions_flusher(), - } - } - fn total_point_count(&self) -> usize { match self { IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.total_point_count(), @@ -178,6 +110,76 @@ impl IdTracker for IdTrackerEnum { IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.iter_internal_versions(), } } +} + +impl IdTracker for IdTrackerEnum { + fn set_internal_version( + &mut self, + internal_id: PointOffsetType, + version: SeqNumberType, + ) -> OperationResult<()> { + match self { + IdTrackerEnum::MutableIdTracker(id_tracker) => { + id_tracker.set_internal_version(internal_id, version) + } + IdTrackerEnum::ImmutableIdTracker(id_tracker) => { + id_tracker.set_internal_version(internal_id, version) + } + IdTrackerEnum::InMemoryIdTracker(id_tracker) => { + id_tracker.set_internal_version(internal_id, version) + } + } + } + + fn set_link( + &mut self, + external_id: PointIdType, + internal_id: PointOffsetType, + ) -> OperationResult<()> { + match self { + IdTrackerEnum::MutableIdTracker(id_tracker) => { + id_tracker.set_link(external_id, internal_id) + } + IdTrackerEnum::ImmutableIdTracker(id_tracker) => { + id_tracker.set_link(external_id, internal_id) + } + IdTrackerEnum::InMemoryIdTracker(id_tracker) => { + id_tracker.set_link(external_id, internal_id) + } + } + } + + fn drop(&mut self, external_id: PointIdType) -> OperationResult<()> { + match self { + IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.drop(external_id), + IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.drop(external_id), + IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.drop(external_id), + } + } + + fn drop_internal(&mut self, internal_id: PointOffsetType) -> OperationResult<()> { + match self { + IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.drop_internal(internal_id), + IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.drop_internal(internal_id), + IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.drop_internal(internal_id), + } + } + + fn mapping_flusher(&self) -> Flusher { + match self { + IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.mapping_flusher(), + IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.mapping_flusher(), + IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.mapping_flusher(), + } + } + + fn versions_flusher(&self) -> Flusher { + match self { + IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.versions_flusher(), + IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.versions_flusher(), + IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.versions_flusher(), + } + } fn fix_inconsistencies(&mut self) -> OperationResult> { match self { diff --git a/lib/segment/src/id_tracker/id_tracker_base/trait_def.rs b/lib/segment/src/id_tracker/id_tracker_base/trait_def.rs index a0d6a2b068..197aeb1525 100644 --- a/lib/segment/src/id_tracker/id_tracker_base/trait_def.rs +++ b/lib/segment/src/id_tracker/id_tracker_base/trait_def.rs @@ -26,25 +26,13 @@ pub const DELETED_POINT_VERSION: SeqNumberType = 0; /// This tracker is used to convert external (i.e. user-facing) point id into internal point id /// as well as for keeping track on point version /// Internal ids are useful for contiguous-ness -pub trait IdTracker: fmt::Debug { - fn internal_version(&self, internal_id: PointOffsetType) -> Option; - +pub trait IdTracker: IdTrackerRead + fmt::Debug { fn set_internal_version( &mut self, internal_id: PointOffsetType, version: SeqNumberType, ) -> OperationResult<()>; - /// Returns internal ID of the point, which is used inside this segment - /// - /// Excludes soft deleted points. - fn internal_id(&self, external_id: PointIdType) -> Option; - - /// Return external ID for internal point, defined by user - /// - /// Excludes soft deleted points. - fn external_id(&self, internal_id: PointOffsetType) -> Option; - /// Set mapping fn set_link( &mut self, @@ -59,73 +47,12 @@ pub trait IdTracker: fmt::Debug { /// If mapping doesn't exist, still removes( unsets ) version. fn drop_internal(&mut self, internal_id: PointOffsetType) -> OperationResult<()>; - /// Get a reference to the point mappings, which provides iteration methods. - fn point_mappings(&self) -> PointMappingsRefEnum<'_>; - /// Flush id mapping to disk fn mapping_flusher(&self) -> Flusher; /// Flush points versions to disk fn versions_flusher(&self) -> Flusher; - /// Number of total points - /// - /// - includes soft deleted points - fn total_point_count(&self) -> usize; - - /// Number of available points - /// - /// - excludes soft deleted points - fn available_point_count(&self) -> usize { - self.total_point_count() - self.deleted_point_count() - } - - /// Number of deleted points - fn deleted_point_count(&self) -> usize; - - /// Get [`BitSlice`] representation for deleted points with deletion flags - /// - /// The size of this slice is not guaranteed. It may be smaller/larger than the number of - /// vectors in this segment. - fn deleted_point_bitslice(&self) -> &BitSlice; - - /// Check whether the given point is soft deleted - fn is_deleted_point(&self, internal_id: PointOffsetType) -> bool; - - fn name(&self) -> &'static str; - - /// Iterator over `n` random IDs which are not deleted - /// - /// A [`BitSlice`] of deleted vectors may optionally be given to also consider deleted named - /// vectors. - fn sample_ids<'a>( - &'a self, - deleted_vector_bitslice: Option<&'a BitSlice>, - ) -> Box + 'a> { - // Use seeded randomness, prevents 'inconsistencies' in search results with sampling - let mut rng = StdRng::seed_from_u64(SEED); - - let total = self.total_point_count() as PointOffsetType; - Box::new( - (0..total) - .map(move |_| rng.random_range(0..total)) - .filter(move |&x| { - // Check for deleted vector first, as that is more likely - !deleted_vector_bitslice - .and_then(|d| d.get_bit(x as usize)) - .unwrap_or(false) - // Also check point deletion for integrity - && !self.is_deleted_point(x) - }), - ) - } - - /// Iterate over all stored internal versions, even if they were deleted - /// Required for cleanup on segment open - fn iter_internal_versions( - &self, - ) -> Box + '_>; - /// Finds inconsistencies between id mapping and versions storage. /// It might happen that point doesn't have version due to un-flushed WAL. /// This method makes those points usable again. @@ -173,3 +100,78 @@ pub trait IdTracker: fmt::Debug { Vec::new() } } + +pub trait IdTrackerRead { + /// Get a reference to the point mappings, which provides iteration methods. + fn point_mappings(&self) -> PointMappingsRefEnum<'_>; + + fn internal_version(&self, internal_id: PointOffsetType) -> Option; + + /// Returns internal ID of the point, which is used inside this segment + /// + /// Excludes soft deleted points. + fn internal_id(&self, external_id: PointIdType) -> Option; + + /// Return external ID for internal point, defined by user + /// + /// Excludes soft deleted points. + fn external_id(&self, internal_id: PointOffsetType) -> Option; + + /// Number of total points + /// + /// - includes soft deleted points + fn total_point_count(&self) -> usize; + + /// Number of available points + /// + /// - excludes soft deleted points + fn available_point_count(&self) -> usize { + self.total_point_count() - self.deleted_point_count() + } + + /// Number of deleted points + fn deleted_point_count(&self) -> usize; + + /// Get [`BitSlice`] representation for deleted points with deletion flags + /// + /// The size of this slice is not guaranteed. It may be smaller/larger than the number of + /// vectors in this segment. + fn deleted_point_bitslice(&self) -> &BitSlice; + + /// Check whether the given point is soft deleted + fn is_deleted_point(&self, internal_id: PointOffsetType) -> bool; + + fn name(&self) -> &'static str; + + /// Iterator over `n` random IDs which are not deleted + /// + /// A [`BitSlice`] of deleted vectors may optionally be given to also consider deleted named + /// vectors. + fn sample_ids<'a>( + &'a self, + deleted_vector_bitslice: Option<&'a BitSlice>, + ) -> Box + 'a> { + // Use seeded randomness, prevents 'inconsistencies' in search results with sampling + let mut rng = StdRng::seed_from_u64(SEED); + + let total = self.total_point_count() as PointOffsetType; + Box::new( + (0..total) + .map(move |_| rng.random_range(0..total)) + .filter(move |&x| { + // Check for deleted vector first, as that is more likely + !deleted_vector_bitslice + .and_then(|d| d.get_bit(x as usize)) + .unwrap_or(false) + // Also check point deletion for integrity + && !self.is_deleted_point(x) + }), + ) + } + + /// Iterate over all stored internal versions, even if they were deleted + /// Required for cleanup on segment open + fn iter_internal_versions( + &self, + ) -> Box + '_>; +} diff --git a/lib/segment/src/id_tracker/immutable_id_tracker.rs b/lib/segment/src/id_tracker/immutable_id_tracker.rs index 8dc80f2433..592acc11a9 100644 --- a/lib/segment/src/id_tracker/immutable_id_tracker.rs +++ b/lib/segment/src/id_tracker/immutable_id_tracker.rs @@ -22,7 +22,7 @@ use crate::id_tracker::compressed::internal_to_external::CompressedInternalToExt use crate::id_tracker::compressed::versions_store::CompressedVersions; use crate::id_tracker::in_memory_id_tracker::InMemoryIdTracker; use crate::id_tracker::point_mappings::FileEndianess; -use crate::id_tracker::{DELETED_POINT_VERSION, IdTracker, PointMappingsRefEnum}; +use crate::id_tracker::{DELETED_POINT_VERSION, IdTracker, IdTrackerRead, PointMappingsRefEnum}; use crate::types::{ExtendedPointId, PointIdType, SeqNumberType}; pub const DELETED_FILE_NAME: &str = "id_tracker.deleted"; @@ -414,11 +414,55 @@ fn mmap_size(len: usize) -> usize { len.div_ceil(item_width) * item_width // Make it a multiple of usize-width. } -impl IdTracker for ImmutableIdTracker { +impl IdTrackerRead for ImmutableIdTracker { fn internal_version(&self, internal_id: PointOffsetType) -> Option { self.internal_to_version.get(internal_id) } + fn internal_id(&self, external_id: PointIdType) -> Option { + self.mappings.internal_id(&external_id) + } + + fn external_id(&self, internal_id: PointOffsetType) -> Option { + self.mappings.external_id(internal_id) + } + + fn point_mappings(&self) -> PointMappingsRefEnum<'_> { + PointMappingsRefEnum::Compressed(&self.mappings) + } + + fn total_point_count(&self) -> usize { + self.mappings.total_point_count() + } + + fn available_point_count(&self) -> usize { + self.mappings.available_point_count() + } + + fn deleted_point_count(&self) -> usize { + self.total_point_count() - self.available_point_count() + } + + fn deleted_point_bitslice(&self) -> &BitSlice { + self.mappings.deleted() + } + + fn is_deleted_point(&self, key: PointOffsetType) -> bool { + self.mappings.is_deleted_point(key) + } + + fn name(&self) -> &'static str { + "immutable id tracker" + } + + fn iter_internal_versions( + &self, + ) -> Box + '_> { + Box::new(self.internal_to_version.iter()) + } +} + +impl IdTracker for ImmutableIdTracker { fn set_internal_version( &mut self, internal_id: PointOffsetType, @@ -437,14 +481,6 @@ impl IdTracker for ImmutableIdTracker { Ok(()) } - fn internal_id(&self, external_id: PointIdType) -> Option { - self.mappings.internal_id(&external_id) - } - - fn external_id(&self, internal_id: PointOffsetType) -> Option { - self.mappings.external_id(internal_id) - } - fn set_link( &mut self, _external_id: PointIdType, @@ -475,10 +511,6 @@ impl IdTracker for ImmutableIdTracker { Ok(()) } - fn point_mappings(&self) -> PointMappingsRefEnum<'_> { - PointMappingsRefEnum::Compressed(&self.mappings) - } - /// Creates a flusher function, that writes the deleted points bitvec to disk. fn mapping_flusher(&self) -> Flusher { // Only flush deletions because mappings are immutable @@ -491,36 +523,6 @@ impl IdTracker for ImmutableIdTracker { Box::new(move || flusher().map_err(OperationError::from)) } - fn total_point_count(&self) -> usize { - self.mappings.total_point_count() - } - - fn available_point_count(&self) -> usize { - self.mappings.available_point_count() - } - - fn deleted_point_count(&self) -> usize { - self.total_point_count() - self.available_point_count() - } - - fn deleted_point_bitslice(&self) -> &BitSlice { - self.mappings.deleted() - } - - fn is_deleted_point(&self, key: PointOffsetType) -> bool { - self.mappings.is_deleted_point(key) - } - - fn name(&self) -> &'static str { - "immutable id tracker" - } - - fn iter_internal_versions( - &self, - ) -> Box + '_> { - Box::new(self.internal_to_version.iter()) - } - fn files(&self) -> Vec { vec![ Self::deleted_file_path(&self.path), diff --git a/lib/segment/src/id_tracker/in_memory_id_tracker.rs b/lib/segment/src/id_tracker/in_memory_id_tracker.rs index c47e566f16..ff64da774e 100644 --- a/lib/segment/src/id_tracker/in_memory_id_tracker.rs +++ b/lib/segment/src/id_tracker/in_memory_id_tracker.rs @@ -10,7 +10,7 @@ use rand::rngs::StdRng; use crate::common::Flusher; use crate::common::operation_error::OperationResult; use crate::id_tracker::point_mappings::PointMappings; -use crate::id_tracker::{DELETED_POINT_VERSION, IdTracker, PointMappingsRefEnum}; +use crate::id_tracker::{DELETED_POINT_VERSION, IdTracker, IdTrackerRead, PointMappingsRefEnum}; use crate::types::{PointIdType, SeqNumberType}; /// A non-persistent ID tracker for faster and more efficient building of `ImmutableIdTracker`. @@ -59,28 +59,11 @@ impl InMemoryIdTracker { } } -impl IdTracker for InMemoryIdTracker { +impl IdTrackerRead for InMemoryIdTracker { fn internal_version(&self, internal_id: PointOffsetType) -> Option { self.internal_to_version.get(internal_id as usize).copied() } - fn set_internal_version( - &mut self, - internal_id: PointOffsetType, - version: SeqNumberType, - ) -> OperationResult<()> { - if self.external_id(internal_id).is_some() { - if let Some(old_version) = self.internal_to_version.get_mut(internal_id as usize) { - *old_version = version; - } else { - self.internal_to_version.resize(internal_id as usize + 1, 0); - self.internal_to_version[internal_id as usize] = version; - } - } - - Ok(()) - } - fn internal_id(&self, external_id: PointIdType) -> Option { self.mappings.internal_id(&external_id) } @@ -89,49 +72,10 @@ impl IdTracker for InMemoryIdTracker { self.mappings.external_id(internal_id) } - fn set_link( - &mut self, - external_id: PointIdType, - internal_id: PointOffsetType, - ) -> OperationResult<()> { - let _replaced_internal_id = self.mappings.set_link(external_id, internal_id); - Ok(()) - } - - fn drop(&mut self, external_id: PointIdType) -> OperationResult<()> { - // Unset version first because it still requires the mapping to exist - if let Some(internal_id) = self.internal_id(external_id) { - self.set_internal_version(internal_id, DELETED_POINT_VERSION)?; - } - self.mappings.drop(external_id); - Ok(()) - } - - fn drop_internal(&mut self, internal_id: PointOffsetType) -> OperationResult<()> { - // Unset version first because it still requires the mapping to exist - self.set_internal_version(internal_id, DELETED_POINT_VERSION)?; - if let Some(external_id) = self.mappings.external_id(internal_id) { - self.mappings.drop(external_id); - } - Ok(()) - } - fn point_mappings(&self) -> PointMappingsRefEnum<'_> { PointMappingsRefEnum::Plain(&self.mappings) } - /// Creates a flusher function, that writes the deleted points bitvec to disk. - fn mapping_flusher(&self) -> Flusher { - debug_assert!(false, "InMemoryIdTracker should not be flushed"); - Box::new(|| Ok(())) - } - - /// Creates a flusher function, that writes the points versions to disk. - fn versions_flusher(&self) -> Flusher { - debug_assert!(false, "InMemoryIdTracker should not be flushed"); - Box::new(|| Ok(())) - } - fn total_point_count(&self) -> usize { self.mappings.total_point_count() } @@ -166,6 +110,64 @@ impl IdTracker for InMemoryIdTracker { .map(|(i, version)| (i as PointOffsetType, *version)), ) } +} + +impl IdTracker for InMemoryIdTracker { + fn set_internal_version( + &mut self, + internal_id: PointOffsetType, + version: SeqNumberType, + ) -> OperationResult<()> { + if self.external_id(internal_id).is_some() { + if let Some(old_version) = self.internal_to_version.get_mut(internal_id as usize) { + *old_version = version; + } else { + self.internal_to_version.resize(internal_id as usize + 1, 0); + self.internal_to_version[internal_id as usize] = version; + } + } + + Ok(()) + } + + fn set_link( + &mut self, + external_id: PointIdType, + internal_id: PointOffsetType, + ) -> OperationResult<()> { + let _replaced_internal_id = self.mappings.set_link(external_id, internal_id); + Ok(()) + } + + fn drop(&mut self, external_id: PointIdType) -> OperationResult<()> { + // Unset version first because it still requires the mapping to exist + if let Some(internal_id) = self.internal_id(external_id) { + self.set_internal_version(internal_id, DELETED_POINT_VERSION)?; + } + self.mappings.drop(external_id); + Ok(()) + } + + fn drop_internal(&mut self, internal_id: PointOffsetType) -> OperationResult<()> { + // Unset version first because it still requires the mapping to exist + self.set_internal_version(internal_id, DELETED_POINT_VERSION)?; + if let Some(external_id) = self.mappings.external_id(internal_id) { + self.mappings.drop(external_id); + } + Ok(()) + } + + /// Creates a flusher function, that writes the deleted points bitvec to disk. + fn mapping_flusher(&self) -> Flusher { + debug_assert!(false, "InMemoryIdTracker should not be flushed"); + Box::new(|| Ok(())) + } + + /// Creates a flusher function, that writes the points versions to disk. + fn versions_flusher(&self) -> Flusher { + debug_assert!(false, "InMemoryIdTracker should not be flushed"); + Box::new(|| Ok(())) + } fn files(&self) -> Vec { debug_assert!(false, "InMemoryIdTracker should not be persisted"); diff --git a/lib/segment/src/id_tracker/mutable_id_tracker/mod.rs b/lib/segment/src/id_tracker/mutable_id_tracker/mod.rs index f129314da5..2bc4f45232 100644 --- a/lib/segment/src/id_tracker/mutable_id_tracker/mod.rs +++ b/lib/segment/src/id_tracker/mutable_id_tracker/mod.rs @@ -26,7 +26,7 @@ use self::versions_storage::{ use crate::common::Flusher; use crate::common::operation_error::{OperationError, OperationResult}; use crate::id_tracker::point_mappings::PointMappings; -use crate::id_tracker::{DELETED_POINT_VERSION, IdTracker, PointMappingsRefEnum}; +use crate::id_tracker::{DELETED_POINT_VERSION, IdTracker, IdTrackerRead, PointMappingsRefEnum}; use crate::types::{PointIdType, SeqNumberType}; /// Mutable in-memory ID tracker with simple file based backing storage @@ -156,11 +156,60 @@ impl MutableIdTracker { } } -impl IdTracker for MutableIdTracker { +impl IdTrackerRead for MutableIdTracker { fn internal_version(&self, internal_id: PointOffsetType) -> Option { self.internal_to_version.get(internal_id as usize).copied() } + fn internal_id(&self, external_id: PointIdType) -> Option { + self.mappings.internal_id(&external_id) + } + + fn external_id(&self, internal_id: PointOffsetType) -> Option { + self.mappings.external_id(internal_id) + } + + fn point_mappings(&self) -> PointMappingsRefEnum<'_> { + PointMappingsRefEnum::Plain(&self.mappings) + } + + fn total_point_count(&self) -> usize { + self.mappings.total_point_count() + } + + fn available_point_count(&self) -> usize { + self.mappings.available_point_count() + } + + fn deleted_point_count(&self) -> usize { + self.total_point_count() - self.available_point_count() + } + + fn is_deleted_point(&self, key: PointOffsetType) -> bool { + self.mappings.is_deleted_point(key) + } + + fn deleted_point_bitslice(&self) -> &BitSlice { + self.mappings.deleted() + } + + fn iter_internal_versions( + &self, + ) -> Box + '_> { + Box::new( + self.internal_to_version + .iter() + .enumerate() + .map(|(i, version)| (i as PointOffsetType, *version)), + ) + } + + fn name(&self) -> &'static str { + "mutable id tracker" + } +} + +impl IdTracker for MutableIdTracker { fn set_internal_version( &mut self, internal_id: PointOffsetType, @@ -184,14 +233,6 @@ impl IdTracker for MutableIdTracker { Ok(()) } - fn internal_id(&self, external_id: PointIdType) -> Option { - self.mappings.internal_id(&external_id) - } - - fn external_id(&self, internal_id: PointOffsetType) -> Option { - self.mappings.external_id(internal_id) - } - fn set_link( &mut self, external_id: PointIdType, @@ -228,22 +269,6 @@ impl IdTracker for MutableIdTracker { Ok(()) } - fn point_mappings(&self) -> PointMappingsRefEnum<'_> { - PointMappingsRefEnum::Plain(&self.mappings) - } - - fn total_point_count(&self) -> usize { - self.mappings.total_point_count() - } - - fn available_point_count(&self) -> usize { - self.mappings.available_point_count() - } - - fn deleted_point_count(&self) -> usize { - self.total_point_count() - self.available_point_count() - } - /// Creates a flusher function, that persists the removed points in the mapping database /// and flushes the mapping to disk. /// This function should be called _before_ flushing the version database. @@ -332,29 +357,6 @@ impl IdTracker for MutableIdTracker { }) } - fn is_deleted_point(&self, key: PointOffsetType) -> bool { - self.mappings.is_deleted_point(key) - } - - fn deleted_point_bitslice(&self) -> &BitSlice { - self.mappings.deleted() - } - - fn iter_internal_versions( - &self, - ) -> Box + '_> { - Box::new( - self.internal_to_version - .iter() - .enumerate() - .map(|(i, version)| (i as PointOffsetType, *version)), - ) - } - - fn name(&self) -> &'static str { - "mutable id tracker" - } - #[inline] fn files(&self) -> Vec { Self::segment_files(&self.segment_path) diff --git a/lib/segment/src/id_tracker/mutable_id_tracker/tests.rs b/lib/segment/src/id_tracker/mutable_id_tracker/tests.rs index a08ad346eb..15d582dac7 100644 --- a/lib/segment/src/id_tracker/mutable_id_tracker/tests.rs +++ b/lib/segment/src/id_tracker/mutable_id_tracker/tests.rs @@ -15,9 +15,9 @@ use super::mappings_storage::{load_mappings, mappings_path, read_mappings}; use super::versions_storage::{ VERSION_ELEMENT_SIZE, load_versions, store_version_changes, versions_path, }; -use crate::id_tracker::IdTracker; use crate::id_tracker::compressed::compressed_point_mappings::CompressedPointMappings; use crate::id_tracker::in_memory_id_tracker::InMemoryIdTracker; +use crate::id_tracker::{IdTracker, IdTrackerRead}; use crate::types::{PointIdType, SeqNumberType}; const RAND_SEED: u64 = 42; diff --git a/lib/segment/src/index/field_index/index_selector.rs b/lib/segment/src/index/field_index/index_selector.rs index b2e99eeee4..1faf004820 100644 --- a/lib/segment/src/index/field_index/index_selector.rs +++ b/lib/segment/src/index/field_index/index_selector.rs @@ -16,7 +16,7 @@ use super::stored_point_to_values::StoredValue; use super::{FieldIndexBuilder, ValueIndexer}; use crate::common::operation_error::{OperationError, OperationResult}; use crate::data_types::index::TextIndexParams; -use crate::id_tracker::{IdTracker, IdTrackerEnum}; +use crate::id_tracker::{IdTrackerEnum, IdTrackerRead}; use crate::index::field_index::FieldIndex; use crate::index::field_index::full_text_index::text_index::FullTextIndex; use crate::index::field_index::geo_index::GeoMapIndex; diff --git a/lib/segment/src/index/hnsw_index/hnsw.rs b/lib/segment/src/index/hnsw_index/hnsw.rs index 441b344ba0..7a1670bcb0 100644 --- a/lib/segment/src/index/hnsw_index/hnsw.rs +++ b/lib/segment/src/index/hnsw_index/hnsw.rs @@ -34,7 +34,7 @@ use crate::common::operation_time_statistics::{ }; use crate::data_types::query_context::VectorQueryContext; use crate::data_types::vectors::{QueryVector, VectorInternal, VectorRef}; -use crate::id_tracker::{IdTracker, IdTrackerEnum}; +use crate::id_tracker::{IdTrackerEnum, IdTrackerRead}; use crate::index::field_index::PayloadBlockCondition; use crate::index::hnsw_index::HnswM; use crate::index::hnsw_index::build_condition_checker::BuildConditionChecker; diff --git a/lib/segment/src/index/plain_payload_index.rs b/lib/segment/src/index/plain_payload_index.rs index 7d67328dbe..97ef377957 100644 --- a/lib/segment/src/index/plain_payload_index.rs +++ b/lib/segment/src/index/plain_payload_index.rs @@ -14,7 +14,7 @@ use super::field_index::FieldIndex; use super::payload_config::PayloadFieldSchemaWithIndexType; use crate::common::Flusher; use crate::common::operation_error::OperationResult; -use crate::id_tracker::{IdTracker, IdTrackerEnum}; +use crate::id_tracker::{IdTrackerEnum, IdTrackerRead}; use crate::index::field_index::{CardinalityEstimation, PayloadBlockCondition}; use crate::index::payload_config::PayloadConfig; use crate::index::{BuildIndexResult, PayloadIndex}; diff --git a/lib/segment/src/index/plain_vector_index.rs b/lib/segment/src/index/plain_vector_index.rs index db02df016b..574d0fa931 100644 --- a/lib/segment/src/index/plain_vector_index.rs +++ b/lib/segment/src/index/plain_vector_index.rs @@ -14,7 +14,7 @@ use crate::common::operation_time_statistics::{ }; use crate::data_types::query_context::VectorQueryContext; use crate::data_types::vectors::{QueryVector, VectorRef}; -use crate::id_tracker::{IdTracker, IdTrackerEnum}; +use crate::id_tracker::{IdTrackerEnum, IdTrackerRead}; use crate::index::struct_payload_index::StructPayloadIndex; use crate::index::vector_index_search_common::{ get_oversampled_top, is_quantized_search, postprocess_search_result, diff --git a/lib/segment/src/index/query_optimization/condition_converter.rs b/lib/segment/src/index/query_optimization/condition_converter.rs index cc65f3a83a..86c48747f9 100644 --- a/lib/segment/src/index/query_optimization/condition_converter.rs +++ b/lib/segment/src/index/query_optimization/condition_converter.rs @@ -8,7 +8,7 @@ use match_converter::get_match_checkers; use ordered_float::OrderedFloat; use serde_json::Value; -use crate::id_tracker::IdTracker; +use crate::id_tracker::IdTrackerRead; use crate::index::field_index::FieldIndex; use crate::index::field_index::null_index::NullIndex; use crate::index::query_optimization::optimized_filter::ConditionCheckerFn; diff --git a/lib/segment/src/index/sparse_index/sparse_vector_index.rs b/lib/segment/src/index/sparse_index/sparse_vector_index.rs index 92fbe23292..04e6f7ffb4 100644 --- a/lib/segment/src/index/sparse_index/sparse_vector_index.rs +++ b/lib/segment/src/index/sparse_index/sparse_vector_index.rs @@ -26,7 +26,7 @@ use crate::common::operation_time_statistics::ScopeDurationMeasurer; use crate::data_types::named_vectors::CowVector; use crate::data_types::query_context::VectorQueryContext; use crate::data_types::vectors::{QueryVector, VectorInternal, VectorRef}; -use crate::id_tracker::{IdTracker, IdTrackerEnum}; +use crate::id_tracker::{IdTrackerEnum, IdTrackerRead}; use crate::index::field_index::CardinalityEstimation; use crate::index::hnsw_index::point_scorer::BatchFilteredSearcher; use crate::index::query_estimator::adjust_to_available_vectors; diff --git a/lib/segment/src/index/struct_payload_index.rs b/lib/segment/src/index/struct_payload_index.rs index 3208b0423d..8862c249d6 100644 --- a/lib/segment/src/index/struct_payload_index.rs +++ b/lib/segment/src/index/struct_payload_index.rs @@ -23,7 +23,7 @@ use super::payload_config::{FullPayloadIndexType, PayloadFieldSchemaWithIndexTyp use crate::common::Flusher; use crate::common::operation_error::{OperationError, OperationResult}; use crate::common::utils::IndexesMap; -use crate::id_tracker::{IdTracker, IdTrackerEnum, PointMappingsRefEnum}; +use crate::id_tracker::{IdTrackerEnum, IdTrackerRead, PointMappingsRefEnum}; use crate::index::field_index::{ CardinalityEstimation, FieldIndex, PayloadBlockCondition, PrimaryCondition, }; diff --git a/lib/segment/src/payload_storage/query_checker.rs b/lib/segment/src/payload_storage/query_checker.rs index 2fb068c20a..6c751cce9a 100644 --- a/lib/segment/src/payload_storage/query_checker.rs +++ b/lib/segment/src/payload_storage/query_checker.rs @@ -10,7 +10,7 @@ use common::counter::hardware_counter::HardwareCounterCell; use common::types::PointOffsetType; use crate::common::utils::{IndexesMap, check_is_empty, check_is_null}; -use crate::id_tracker::{IdTracker, IdTrackerEnum}; +use crate::id_tracker::{IdTrackerEnum, IdTrackerRead}; use crate::index::field_index::FieldIndex; use crate::payload_storage::condition_checker::ValueChecker; use crate::payload_storage::payload_storage_enum::PayloadStorageEnum; diff --git a/lib/segment/src/segment/entry.rs b/lib/segment/src/segment/entry.rs index ba02378f8b..7e3995eda2 100644 --- a/lib/segment/src/segment/entry.rs +++ b/lib/segment/src/segment/entry.rs @@ -27,7 +27,7 @@ use crate::data_types::vectors::{QueryVector, VectorInternal}; use crate::entry::entry_point::{ NonAppendableSegmentEntry, ReadSegmentEntry, SegmentEntry, StorageSegmentEntry, }; -use crate::id_tracker::{IdTracker, PointMappingsGuard}; +use crate::id_tracker::{IdTracker, IdTrackerRead, PointMappingsGuard}; use crate::index::field_index::{CardinalityEstimation, FieldIndex}; use crate::index::query_estimator::adjust_for_deferred_points; use crate::index::{BuildIndexResult, PayloadIndex, VectorIndex}; diff --git a/lib/segment/src/segment/facet.rs b/lib/segment/src/segment/facet.rs index f17c7406e8..9d78309305 100644 --- a/lib/segment/src/segment/facet.rs +++ b/lib/segment/src/segment/facet.rs @@ -9,7 +9,7 @@ use super::Segment; use crate::common::operation_error::{OperationResult, check_process_stopped}; use crate::data_types::facets::{FacetParams, FacetValue}; use crate::entry::ReadSegmentEntry; -use crate::id_tracker::IdTracker; +use crate::id_tracker::IdTrackerRead; use crate::index::PayloadIndex; use crate::json_path::JsonPath; use crate::payload_storage::FilterContext; diff --git a/lib/segment/src/segment/order_by.rs b/lib/segment/src/segment/order_by.rs index 50c0a2ff72..05f0c88a36 100644 --- a/lib/segment/src/segment/order_by.rs +++ b/lib/segment/src/segment/order_by.rs @@ -9,7 +9,7 @@ use itertools::Either; use super::Segment; use crate::common::operation_error::{OperationError, OperationResult}; use crate::data_types::order_by::{Direction, OrderBy, OrderValue}; -use crate::id_tracker::IdTracker; +use crate::id_tracker::IdTrackerRead; use crate::index::PayloadIndex; use crate::index::field_index::numeric_index::StreamRange; use crate::spaces::tools::{peek_top_largest_iterable, peek_top_smallest_iterable}; diff --git a/lib/segment/src/segment/sampling.rs b/lib/segment/src/segment/sampling.rs index 52c3f3160f..1d713c1238 100644 --- a/lib/segment/src/segment/sampling.rs +++ b/lib/segment/src/segment/sampling.rs @@ -6,7 +6,7 @@ use rand::seq::{IteratorRandom, SliceRandom}; use super::Segment; use crate::common::operation_error::OperationResult; -use crate::id_tracker::IdTracker; +use crate::id_tracker::IdTrackerRead; use crate::index::PayloadIndex; use crate::types::{Filter, PointIdType}; diff --git a/lib/segment/src/segment/scroll.rs b/lib/segment/src/segment/scroll.rs index 96e41d0e3d..59e6c47cf0 100644 --- a/lib/segment/src/segment/scroll.rs +++ b/lib/segment/src/segment/scroll.rs @@ -7,7 +7,7 @@ use common::types::DeferredBehavior; use super::Segment; use crate::common::operation_error::OperationResult; use crate::entry::ReadSegmentEntry; -use crate::id_tracker::IdTracker; +use crate::id_tracker::IdTrackerRead; use crate::index::PayloadIndex; use crate::spaces::tools::peek_top_smallest_iterable; use crate::types::{Filter, PointIdType}; diff --git a/lib/segment/src/segment/search.rs b/lib/segment/src/segment/search.rs index 93e3a5219a..01acef8e13 100644 --- a/lib/segment/src/segment/search.rs +++ b/lib/segment/src/segment/search.rs @@ -12,7 +12,7 @@ use crate::data_types::segment_record::SegmentRecord; use crate::data_types::vectors::QueryVector; use crate::data_types::vectors::VectorStructInternal; use crate::entry::ReadSegmentEntry; -use crate::id_tracker::IdTracker; +use crate::id_tracker::IdTrackerRead; #[cfg(feature = "testing")] use crate::types::VectorName; #[cfg(feature = "testing")] diff --git a/lib/segment/src/segment/segment_ops.rs b/lib/segment/src/segment/segment_ops.rs index 39e15ba82c..5f416056ef 100644 --- a/lib/segment/src/segment/segment_ops.rs +++ b/lib/segment/src/segment/segment_ops.rs @@ -22,7 +22,7 @@ use crate::data_types::named_vectors::NamedVectors; use crate::data_types::vectors::VectorInternal; use crate::entry::entry_point::StorageSegmentEntry as _; use crate::entry::{NonAppendableSegmentEntry as _, ReadSegmentEntry}; -use crate::id_tracker::IdTracker; +use crate::id_tracker::{IdTracker, IdTrackerRead}; use crate::index::{PayloadIndex, VectorIndex}; use crate::types::{ Payload, PayloadFieldSchema, PayloadKeyType, PointIdType, SegmentState, SeqNumberType, diff --git a/lib/segment/src/segment/vector_name_ops.rs b/lib/segment/src/segment/vector_name_ops.rs index 9d5e225954..2ca4cba0c8 100644 --- a/lib/segment/src/segment/vector_name_ops.rs +++ b/lib/segment/src/segment/vector_name_ops.rs @@ -6,7 +6,7 @@ use atomic_refcell::AtomicRefCell; use super::Segment; use crate::common::operation_error::OperationResult; use crate::data_types::vector_name_config::VectorNameConfig; -use crate::id_tracker::IdTracker as _; +use crate::id_tracker::IdTrackerRead as _; use crate::index::VectorIndexEnum; use crate::index::plain_vector_index::PlainVectorIndex; use crate::index::sparse_index::sparse_index_config::SparseIndexType; diff --git a/lib/segment/src/segment/vectors.rs b/lib/segment/src/segment/vectors.rs index 940e56fe3d..57a4a4652e 100644 --- a/lib/segment/src/segment/vectors.rs +++ b/lib/segment/src/segment/vectors.rs @@ -5,7 +5,7 @@ use common::iterator_ext::IteratorExt; use crate::common::operation_error::OperationResult; use crate::data_types::vectors::VectorInternal; -use crate::id_tracker::IdTracker; +use crate::id_tracker::IdTrackerRead; use crate::segment::Segment; use crate::types::{PointIdType, VectorName}; diff --git a/lib/segment/src/segment_constructor/segment_builder.rs b/lib/segment/src/segment_constructor/segment_builder.rs index 69000dc067..2936feea8d 100644 --- a/lib/segment/src/segment_constructor/segment_builder.rs +++ b/lib/segment/src/segment_constructor/segment_builder.rs @@ -34,7 +34,7 @@ use crate::entry::ReadSegmentEntry; use crate::id_tracker::compressed::compressed_point_mappings::CompressedPointMappings; use crate::id_tracker::immutable_id_tracker::ImmutableIdTracker; use crate::id_tracker::in_memory_id_tracker::InMemoryIdTracker; -use crate::id_tracker::{IdTracker, IdTrackerEnum, for_each_unique_point}; +use crate::id_tracker::{IdTracker, IdTrackerEnum, IdTrackerRead, for_each_unique_point}; use crate::index::field_index::FieldIndex; use crate::index::sparse_index::sparse_vector_index::SparseVectorIndexOpenArgs; use crate::index::struct_payload_index::StructPayloadIndex; diff --git a/lib/segment/src/segment_constructor/segment_constructor_base.rs b/lib/segment/src/segment_constructor/segment_constructor_base.rs index 6581a30038..0fdd06b0fc 100644 --- a/lib/segment/src/segment_constructor/segment_constructor_base.rs +++ b/lib/segment/src/segment_constructor/segment_constructor_base.rs @@ -27,7 +27,7 @@ use crate::common::operation_error::{OperationError, OperationResult, check_proc use crate::data_types::vectors::DEFAULT_VECTOR_NAME; use crate::id_tracker::immutable_id_tracker::ImmutableIdTracker; use crate::id_tracker::mutable_id_tracker::MutableIdTracker; -use crate::id_tracker::{IdTracker, IdTrackerEnum}; +use crate::id_tracker::{IdTrackerEnum, IdTrackerRead}; use crate::index::VectorIndexEnum; use crate::index::hnsw_index::gpu::gpu_devices_manager::LockedGpuDevice; use crate::index::hnsw_index::hnsw::{HNSWIndex, HnswIndexOpenArgs}; diff --git a/lib/segment/src/vector_storage/dense/dense_vector_storage.rs b/lib/segment/src/vector_storage/dense/dense_vector_storage.rs index c309177888..8927533cb3 100644 --- a/lib/segment/src/vector_storage/dense/dense_vector_storage.rs +++ b/lib/segment/src/vector_storage/dense/dense_vector_storage.rs @@ -389,7 +389,7 @@ mod tests { use super::*; use crate::data_types::vectors::{DenseVector, QueryVector, VectorElementType}; use crate::fixtures::payload_context_fixture::create_id_tracker_fixture; - use crate::id_tracker::IdTracker; + use crate::id_tracker::{IdTracker, IdTrackerRead}; use crate::index::hnsw_index::point_scorer::{BatchFilteredSearcher, FilteredScorer}; use crate::types::{PointIdType, QuantizationConfig, ScalarQuantizationConfig}; use crate::vector_storage::dense::volatile_dense_vector_storage::new_volatile_dense_vector_storage; diff --git a/lib/segment/src/vector_storage/tests/async_raw_scorer.rs b/lib/segment/src/vector_storage/tests/async_raw_scorer.rs index 4683232aed..b0b03ad950 100644 --- a/lib/segment/src/vector_storage/tests/async_raw_scorer.rs +++ b/lib/segment/src/vector_storage/tests/async_raw_scorer.rs @@ -9,7 +9,7 @@ use rand::seq::IteratorRandom as _; use super::utils::{Result, delete_random_vectors, insert_distributed_vectors, sampler}; use crate::data_types::vectors::QueryVector; use crate::fixtures::payload_context_fixture::create_id_tracker_fixture; -use crate::id_tracker::IdTracker; +use crate::id_tracker::IdTrackerRead; use crate::index::hnsw_index::point_scorer::FilteredScorer; use crate::types::Distance; use crate::vector_storage::VectorStorageEnum; diff --git a/lib/segment/src/vector_storage/tests/custom_query_scorer_equivalency.rs b/lib/segment/src/vector_storage/tests/custom_query_scorer_equivalency.rs index 1f751b7852..2ff22d0a97 100644 --- a/lib/segment/src/vector_storage/tests/custom_query_scorer_equivalency.rs +++ b/lib/segment/src/vector_storage/tests/custom_query_scorer_equivalency.rs @@ -16,7 +16,7 @@ use super::utils::sampler; use crate::data_types::vectors::{QueryVector, VectorElementType}; use crate::fixtures::payload_context_fixture::create_id_tracker_fixture; use crate::fixtures::query_fixtures::QueryVariant; -use crate::id_tracker::IdTracker; +use crate::id_tracker::IdTrackerRead; use crate::index::hnsw_index::point_scorer::FilteredScorer; use crate::types::{ BinaryQuantizationConfig, Distance, ProductQuantizationConfig, QuantizationConfig, diff --git a/lib/segment/src/vector_storage/tests/test_appendable_dense_vector_storage.rs b/lib/segment/src/vector_storage/tests/test_appendable_dense_vector_storage.rs index 9c2f0d0eec..e7ba69284b 100644 --- a/lib/segment/src/vector_storage/tests/test_appendable_dense_vector_storage.rs +++ b/lib/segment/src/vector_storage/tests/test_appendable_dense_vector_storage.rs @@ -9,7 +9,7 @@ use tempfile::Builder; use crate::data_types::vectors::QueryVector; use crate::fixtures::payload_context_fixture::create_id_tracker_fixture; -use crate::id_tracker::IdTracker; +use crate::id_tracker::{IdTracker, IdTrackerRead}; use crate::index::hnsw_index::point_scorer::{BatchFilteredSearcher, FilteredScorer}; use crate::types::{Distance, PointIdType, QuantizationConfig, ScalarQuantizationConfig}; use crate::vector_storage::dense::appendable_dense_vector_storage::open_appendable_memmap_vector_storage_full; diff --git a/lib/segment/src/vector_storage/tests/test_appendable_multi_dense_vector_storage.rs b/lib/segment/src/vector_storage/tests/test_appendable_multi_dense_vector_storage.rs index 760ef533f6..34aa1076e1 100644 --- a/lib/segment/src/vector_storage/tests/test_appendable_multi_dense_vector_storage.rs +++ b/lib/segment/src/vector_storage/tests/test_appendable_multi_dense_vector_storage.rs @@ -12,7 +12,7 @@ use crate::data_types::vectors::{ MultiDenseVectorInternal, QueryVector, TypedMultiDenseVectorRef, VectorElementType, VectorRef, }; use crate::fixtures::payload_context_fixture::create_id_tracker_fixture; -use crate::id_tracker::IdTracker; +use crate::id_tracker::IdTrackerRead; use crate::index::hnsw_index::point_scorer::BatchFilteredSearcher; use crate::types::{Distance, MultiVectorConfig}; use crate::vector_storage::common::CHUNK_SIZE; diff --git a/lib/segment/src/vector_storage/tests/test_appendable_sparse_vector_storage.rs b/lib/segment/src/vector_storage/tests/test_appendable_sparse_vector_storage.rs index 915d9dd492..b8cad6867e 100644 --- a/lib/segment/src/vector_storage/tests/test_appendable_sparse_vector_storage.rs +++ b/lib/segment/src/vector_storage/tests/test_appendable_sparse_vector_storage.rs @@ -9,7 +9,7 @@ use tempfile::Builder; use crate::data_types::vectors::QueryVector; use crate::fixtures::payload_context_fixture::create_id_tracker_fixture; -use crate::id_tracker::IdTracker; +use crate::id_tracker::IdTrackerRead; use crate::index::hnsw_index::point_scorer::BatchFilteredSearcher; use crate::vector_storage::query::RecoQuery; use crate::vector_storage::sparse::mmap_sparse_vector_storage::MmapSparseVectorStorage; diff --git a/lib/segment/tests/integration/multivector_hnsw_test.rs b/lib/segment/tests/integration/multivector_hnsw_test.rs index e21301512c..94e6136e18 100644 --- a/lib/segment/tests/integration/multivector_hnsw_test.rs +++ b/lib/segment/tests/integration/multivector_hnsw_test.rs @@ -16,7 +16,7 @@ use segment::data_types::vectors::{ use segment::entry::{NonAppendableSegmentEntry, SegmentEntry}; use segment::fixtures::index_fixtures::random_vector; use segment::fixtures::payload_fixtures::random_int_payload; -use segment::id_tracker::IdTracker; +use segment::id_tracker::IdTrackerRead; use segment::index::VectorIndex; use segment::index::hnsw_index::hnsw::{HNSWIndex, HnswIndexOpenArgs}; use segment::json_path::JsonPath; diff --git a/lib/segment/tests/integration/payload_index_test.rs b/lib/segment/tests/integration/payload_index_test.rs index 7dc7c49518..4b3aa7b8df 100644 --- a/lib/segment/tests/integration/payload_index_test.rs +++ b/lib/segment/tests/integration/payload_index_test.rs @@ -27,7 +27,7 @@ use segment::fixtures::payload_fixtures::{ STR_PROJ_KEY, STR_ROOT_PROJ_KEY, TEXT_KEY, generate_diverse_nested_payload, generate_diverse_payload, random_filter, random_nested_filter, random_vector, }; -use segment::id_tracker::IdTracker; +use segment::id_tracker::IdTrackerRead; use segment::index::PayloadIndex; use segment::index::field_index::{FieldIndex, PrimaryCondition}; use segment::index::struct_payload_index::StructPayloadIndex; diff --git a/lib/segment/tests/integration/segment_builder_test.rs b/lib/segment/tests/integration/segment_builder_test.rs index ab5e7d9eff..019ad867bd 100644 --- a/lib/segment/tests/integration/segment_builder_test.rs +++ b/lib/segment/tests/integration/segment_builder_test.rs @@ -13,7 +13,7 @@ use segment::common::operation_error::OperationError; use segment::data_types::named_vectors::NamedVectors; use segment::data_types::vectors::{DEFAULT_VECTOR_NAME, VectorRef, only_default_vector}; use segment::entry::entry_point::{NonAppendableSegmentEntry, ReadSegmentEntry, SegmentEntry}; -use segment::id_tracker::IdTracker; +use segment::id_tracker::IdTrackerRead; use segment::index::hnsw_index::get_num_indexing_threads; use segment::json_path::JsonPath; use segment::segment::Segment; diff --git a/lib/segment/tests/integration/sparse_vector_index_search_tests.rs b/lib/segment/tests/integration/sparse_vector_index_search_tests.rs index 0653101f9c..25b8599edf 100644 --- a/lib/segment/tests/integration/sparse_vector_index_search_tests.rs +++ b/lib/segment/tests/integration/sparse_vector_index_search_tests.rs @@ -16,7 +16,7 @@ use segment::data_types::vectors::{QueryVector, VectorInternal}; use segment::entry::{SegmentEntry, StorageSegmentEntry as _}; use segment::fixtures::payload_fixtures::STR_KEY; use segment::fixtures::sparse_fixtures::{fixture_sparse_index, fixture_sparse_index_from_iter}; -use segment::id_tracker::IdTracker; +use segment::id_tracker::{IdTracker, IdTrackerRead}; use segment::index::sparse_index::sparse_index_config::{SparseIndexConfig, SparseIndexType}; use segment::index::sparse_index::sparse_vector_index::{ SparseVectorIndex, SparseVectorIndexOpenArgs,