upstream-archive/qdrant
1
0
Fork 0
mirror of https://github.com/qdrant/qdrant.git synced 2026-08-04 17:10:59 -05:00
Code Issues Packages Projects Releases Wiki Activity

Split IdTracker trait into IdTrackerRead and IdTracker (#8826)

Browse Source 
Read-only methods now live on a separate IdTrackerRead trait, with the
mutating IdTracker trait extending it. This lets read-only call sites
depend only on the read API.

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Andrey Vasnetsov
2026-04-29 09:20:45 +02:00
committed by timvisee
parent bde40eb44a
commit 614cedae46
39 changed files with 339 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/collection/src/collection_manager/optimizers/vacuum_optimizer.rs
View File

@@ -14,7 +14,7 @@ mod tests {
use common::counter::hardware_counter::HardwareCounterCell;
use itertools::Itertools;
use segment::entry::{NonAppendableSegmentEntry as _, ReadSegmentEntry as _};
use segment::id_tracker::IdTracker;
use segment::id_tracker::IdTrackerRead;
use segment::index::VectorIndex;
use segment::payload_json;
use segment::types::{

2
lib/segment/benches/hnsw_incremental_build.rs
View File

@@ -30,7 +30,7 @@ use segment::data_types::vectors::{
};
use segment::entry::{SegmentEntry as _, StorageSegmentEntry as _};
use segment::fixtures::index_fixtures::random_vector;
use segment::id_tracker::{IdTracker, IdTrackerEnum};
use segment::id_tracker::{IdTrackerEnum, IdTrackerRead};
use segment::index::hnsw_index::get_num_indexing_threads;
use segment::index::hnsw_index::hnsw::{HNSWIndex, HnswIndexOpenArgs};
use segment::index::{VectorIndex as _, VectorIndexEnum};

2
lib/segment/benches/vector_search.rs
View File

@@ -9,7 +9,7 @@ use rand::distr::StandardUniform;
use segment::data_types::named_vectors::CowVector;
use segment::data_types::vectors::{DenseVector, QueryVector};
use segment::fixtures::payload_context_fixture::create_id_tracker_fixture;
use segment::id_tracker::IdTracker;
use segment::id_tracker::IdTrackerRead;
use segment::index::hnsw_index::point_scorer::BatchFilteredSearcher;
use segment::types::Distance;
use segment::vector_storage::dense::dense_vector_storage::open_dense_vector_storage;

2
lib/segment/src/id_tracker/id_tracker_base/mod.rs
View File

@@ -4,4 +4,4 @@ mod trait_def;
pub use point_mappings_ref::{PointMappingsGuard, PointMappingsRefEnum};
pub use tracker_enum::IdTrackerEnum;
pub use trait_def::{DELETED_POINT_VERSION, IdTracker};
pub use trait_def::{DELETED_POINT_VERSION, IdTracker, IdTrackerRead};

142
lib/segment/src/id_tracker/id_tracker_base/tracker_enum.rs
View File

@@ -4,7 +4,7 @@ use common::bitvec::BitSlice;
use common::types::PointOffsetType;
use super::point_mappings_ref::PointMappingsRefEnum;
use super::trait_def::IdTracker;
use super::trait_def::{IdTracker, IdTrackerRead};
use crate::common::Flusher;
use crate::common::operation_error::OperationResult;
use crate::id_tracker::immutable_id_tracker::ImmutableIdTracker;
@@ -20,7 +20,7 @@ pub enum IdTrackerEnum {
InMemoryIdTracker(InMemoryIdTracker),
}
impl IdTracker for IdTrackerEnum {
impl IdTrackerRead for IdTrackerEnum {
fn internal_version(&self, internal_id: PointOffsetType) -> Option<SeqNumberType> {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.internal_version(internal_id),
@@ -33,24 +33,6 @@ impl IdTracker for IdTrackerEnum {
}
}
fn set_internal_version(
&mut self,
internal_id: PointOffsetType,
version: SeqNumberType,
) -> OperationResult<()> {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => {
id_tracker.set_internal_version(internal_id, version)
}
IdTrackerEnum::ImmutableIdTracker(id_tracker) => {
id_tracker.set_internal_version(internal_id, version)
}
IdTrackerEnum::InMemoryIdTracker(id_tracker) => {
id_tracker.set_internal_version(internal_id, version)
}
}
}
fn internal_id(&self, external_id: PointIdType) -> Option<PointOffsetType> {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.internal_id(external_id),
@@ -67,40 +49,6 @@ impl IdTracker for IdTrackerEnum {
}
}
fn set_link(
&mut self,
external_id: PointIdType,
internal_id: PointOffsetType,
) -> OperationResult<()> {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => {
id_tracker.set_link(external_id, internal_id)
}
IdTrackerEnum::ImmutableIdTracker(id_tracker) => {
id_tracker.set_link(external_id, internal_id)
}
IdTrackerEnum::InMemoryIdTracker(id_tracker) => {
id_tracker.set_link(external_id, internal_id)
}
}
}
fn drop(&mut self, external_id: PointIdType) -> OperationResult<()> {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.drop(external_id),
IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.drop(external_id),
IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.drop(external_id),
}
}
fn drop_internal(&mut self, internal_id: PointOffsetType) -> OperationResult<()> {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.drop_internal(internal_id),
IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.drop_internal(internal_id),
IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.drop_internal(internal_id),
}
}
fn point_mappings(&self) -> PointMappingsRefEnum<'_> {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.point_mappings(),
@@ -109,22 +57,6 @@ impl IdTracker for IdTrackerEnum {
}
}
fn mapping_flusher(&self) -> Flusher {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.mapping_flusher(),
IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.mapping_flusher(),
IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.mapping_flusher(),
}
}
fn versions_flusher(&self) -> Flusher {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.versions_flusher(),
IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.versions_flusher(),
IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.versions_flusher(),
}
}
fn total_point_count(&self) -> usize {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.total_point_count(),
@@ -178,6 +110,76 @@ impl IdTracker for IdTrackerEnum {
IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.iter_internal_versions(),
}
}
}
impl IdTracker for IdTrackerEnum {
fn set_internal_version(
&mut self,
internal_id: PointOffsetType,
version: SeqNumberType,
) -> OperationResult<()> {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => {
id_tracker.set_internal_version(internal_id, version)
}
IdTrackerEnum::ImmutableIdTracker(id_tracker) => {
id_tracker.set_internal_version(internal_id, version)
}
IdTrackerEnum::InMemoryIdTracker(id_tracker) => {
id_tracker.set_internal_version(internal_id, version)
}
}
}
fn set_link(
&mut self,
external_id: PointIdType,
internal_id: PointOffsetType,
) -> OperationResult<()> {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => {
id_tracker.set_link(external_id, internal_id)
}
IdTrackerEnum::ImmutableIdTracker(id_tracker) => {
id_tracker.set_link(external_id, internal_id)
}
IdTrackerEnum::InMemoryIdTracker(id_tracker) => {
id_tracker.set_link(external_id, internal_id)
}
}
}
fn drop(&mut self, external_id: PointIdType) -> OperationResult<()> {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.drop(external_id),
IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.drop(external_id),
IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.drop(external_id),
}
}
fn drop_internal(&mut self, internal_id: PointOffsetType) -> OperationResult<()> {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.drop_internal(internal_id),
IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.drop_internal(internal_id),
IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.drop_internal(internal_id),
}
}
fn mapping_flusher(&self) -> Flusher {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.mapping_flusher(),
IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.mapping_flusher(),
IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.mapping_flusher(),
}
}
fn versions_flusher(&self) -> Flusher {
match self {
IdTrackerEnum::MutableIdTracker(id_tracker) => id_tracker.versions_flusher(),
IdTrackerEnum::ImmutableIdTracker(id_tracker) => id_tracker.versions_flusher(),
IdTrackerEnum::InMemoryIdTracker(id_tracker) => id_tracker.versions_flusher(),
}
}
fn fix_inconsistencies(&mut self) -> OperationResult<Vec<PointOffsetType>> {
match self {

150
lib/segment/src/id_tracker/id_tracker_base/trait_def.rs
View File

@@ -26,25 +26,13 @@ pub const DELETED_POINT_VERSION: SeqNumberType = 0;
/// This tracker is used to convert external (i.e. user-facing) point id into internal point id
/// as well as for keeping track on point version
/// Internal ids are useful for contiguous-ness
pub trait IdTracker: fmt::Debug {
fn internal_version(&self, internal_id: PointOffsetType) -> Option<SeqNumberType>;
pub trait IdTracker: IdTrackerRead + fmt::Debug {
fn set_internal_version(
&mut self,
internal_id: PointOffsetType,
version: SeqNumberType,
) -> OperationResult<()>;
/// Returns internal ID of the point, which is used inside this segment
///
/// Excludes soft deleted points.
fn internal_id(&self, external_id: PointIdType) -> Option<PointOffsetType>;
/// Return external ID for internal point, defined by user
///
/// Excludes soft deleted points.
fn external_id(&self, internal_id: PointOffsetType) -> Option<PointIdType>;
/// Set mapping
fn set_link(
&mut self,
@@ -59,73 +47,12 @@ pub trait IdTracker: fmt::Debug {
/// If mapping doesn't exist, still removes( unsets ) version.
fn drop_internal(&mut self, internal_id: PointOffsetType) -> OperationResult<()>;
/// Get a reference to the point mappings, which provides iteration methods.
fn point_mappings(&self) -> PointMappingsRefEnum<'_>;
/// Flush id mapping to disk
fn mapping_flusher(&self) -> Flusher;
/// Flush points versions to disk
fn versions_flusher(&self) -> Flusher;
/// Number of total points
///
/// - includes soft deleted points
fn total_point_count(&self) -> usize;
/// Number of available points
///
/// - excludes soft deleted points
fn available_point_count(&self) -> usize {
self.total_point_count() - self.deleted_point_count()
}
/// Number of deleted points
fn deleted_point_count(&self) -> usize;
/// Get [`BitSlice`] representation for deleted points with deletion flags
///
/// The size of this slice is not guaranteed. It may be smaller/larger than the number of
/// vectors in this segment.
fn deleted_point_bitslice(&self) -> &BitSlice;
/// Check whether the given point is soft deleted
fn is_deleted_point(&self, internal_id: PointOffsetType) -> bool;
fn name(&self) -> &'static str;
/// Iterator over `n` random IDs which are not deleted
///
/// A [`BitSlice`] of deleted vectors may optionally be given to also consider deleted named
/// vectors.
fn sample_ids<'a>(
&'a self,
deleted_vector_bitslice: Option<&'a BitSlice>,
) -> Box<dyn Iterator<Item = PointOffsetType> + 'a> {
// Use seeded randomness, prevents 'inconsistencies' in search results with sampling
let mut rng = StdRng::seed_from_u64(SEED);
let total = self.total_point_count() as PointOffsetType;
Box::new(
(0..total)
.map(move |_| rng.random_range(0..total))
.filter(move |&x| {
// Check for deleted vector first, as that is more likely
!deleted_vector_bitslice
.and_then(|d| d.get_bit(x as usize))
.unwrap_or(false)
// Also check point deletion for integrity
&& !self.is_deleted_point(x)
}),
)
}
/// Iterate over all stored internal versions, even if they were deleted
/// Required for cleanup on segment open
fn iter_internal_versions(
&self,
) -> Box<dyn Iterator<Item = (PointOffsetType, SeqNumberType)> + '_>;
/// Finds inconsistencies between id mapping and versions storage.
/// It might happen that point doesn't have version due to un-flushed WAL.
/// This method makes those points usable again.
@@ -173,3 +100,78 @@ pub trait IdTracker: fmt::Debug {
Vec::new()
}
}
pub trait IdTrackerRead {
/// Get a reference to the point mappings, which provides iteration methods.
fn point_mappings(&self) -> PointMappingsRefEnum<'_>;
fn internal_version(&self, internal_id: PointOffsetType) -> Option<SeqNumberType>;
/// Returns internal ID of the point, which is used inside this segment
///
/// Excludes soft deleted points.
fn internal_id(&self, external_id: PointIdType) -> Option<PointOffsetType>;
/// Return external ID for internal point, defined by user
///
/// Excludes soft deleted points.
fn external_id(&self, internal_id: PointOffsetType) -> Option<PointIdType>;
/// Number of total points
///
/// - includes soft deleted points
fn total_point_count(&self) -> usize;
/// Number of available points
///
/// - excludes soft deleted points
fn available_point_count(&self) -> usize {
self.total_point_count() - self.deleted_point_count()
}
/// Number of deleted points
fn deleted_point_count(&self) -> usize;
/// Get [`BitSlice`] representation for deleted points with deletion flags
///
/// The size of this slice is not guaranteed. It may be smaller/larger than the number of
/// vectors in this segment.
fn deleted_point_bitslice(&self) -> &BitSlice;
/// Check whether the given point is soft deleted
fn is_deleted_point(&self, internal_id: PointOffsetType) -> bool;
fn name(&self) -> &'static str;
/// Iterator over `n` random IDs which are not deleted
///
/// A [`BitSlice`] of deleted vectors may optionally be given to also consider deleted named
/// vectors.
fn sample_ids<'a>(
&'a self,
deleted_vector_bitslice: Option<&'a BitSlice>,
) -> Box<dyn Iterator<Item = PointOffsetType> + 'a> {
// Use seeded randomness, prevents 'inconsistencies' in search results with sampling
let mut rng = StdRng::seed_from_u64(SEED);
let total = self.total_point_count() as PointOffsetType;
Box::new(
(0..total)
.map(move |_| rng.random_range(0..total))
.filter(move |&x| {
// Check for deleted vector first, as that is more likely
!deleted_vector_bitslice
.and_then(|d| d.get_bit(x as usize))
.unwrap_or(false)
// Also check point deletion for integrity
&& !self.is_deleted_point(x)
}),
)
}
/// Iterate over all stored internal versions, even if they were deleted
/// Required for cleanup on segment open
fn iter_internal_versions(
&self,
) -> Box<dyn Iterator<Item = (PointOffsetType, SeqNumberType)> + '_>;
}

90
lib/segment/src/id_tracker/immutable_id_tracker.rs
View File

@@ -22,7 +22,7 @@ use crate::id_tracker::compressed::internal_to_external::CompressedInternalToExt
use crate::id_tracker::compressed::versions_store::CompressedVersions;
use crate::id_tracker::in_memory_id_tracker::InMemoryIdTracker;
use crate::id_tracker::point_mappings::FileEndianess;
use crate::id_tracker::{DELETED_POINT_VERSION, IdTracker, PointMappingsRefEnum};
use crate::id_tracker::{DELETED_POINT_VERSION, IdTracker, IdTrackerRead, PointMappingsRefEnum};
use crate::types::{ExtendedPointId, PointIdType, SeqNumberType};
pub const DELETED_FILE_NAME: &str = "id_tracker.deleted";
@@ -414,11 +414,55 @@ fn mmap_size<T>(len: usize) -> usize {
len.div_ceil(item_width) * item_width // Make it a multiple of usize-width.
}
impl IdTracker for ImmutableIdTracker {
impl IdTrackerRead for ImmutableIdTracker {
fn internal_version(&self, internal_id: PointOffsetType) -> Option<SeqNumberType> {
self.internal_to_version.get(internal_id)
}
fn internal_id(&self, external_id: PointIdType) -> Option<PointOffsetType> {
self.mappings.internal_id(&external_id)
}
fn external_id(&self, internal_id: PointOffsetType) -> Option<PointIdType> {
self.mappings.external_id(internal_id)
}
fn point_mappings(&self) -> PointMappingsRefEnum<'_> {
PointMappingsRefEnum::Compressed(&self.mappings)
}
fn total_point_count(&self) -> usize {
self.mappings.total_point_count()
}
fn available_point_count(&self) -> usize {
self.mappings.available_point_count()
}
fn deleted_point_count(&self) -> usize {
self.total_point_count() - self.available_point_count()
}
fn deleted_point_bitslice(&self) -> &BitSlice {
self.mappings.deleted()
}
fn is_deleted_point(&self, key: PointOffsetType) -> bool {
self.mappings.is_deleted_point(key)
}
fn name(&self) -> &'static str {
"immutable id tracker"
}
fn iter_internal_versions(
&self,
) -> Box<dyn Iterator<Item = (PointOffsetType, SeqNumberType)> + '_> {
Box::new(self.internal_to_version.iter())
}
}
impl IdTracker for ImmutableIdTracker {
fn set_internal_version(
&mut self,
internal_id: PointOffsetType,
@@ -437,14 +481,6 @@ impl IdTracker for ImmutableIdTracker {
Ok(())
}
fn internal_id(&self, external_id: PointIdType) -> Option<PointOffsetType> {
self.mappings.internal_id(&external_id)
}
fn external_id(&self, internal_id: PointOffsetType) -> Option<PointIdType> {
self.mappings.external_id(internal_id)
}
fn set_link(
&mut self,
_external_id: PointIdType,
@@ -475,10 +511,6 @@ impl IdTracker for ImmutableIdTracker {
Ok(())
}
fn point_mappings(&self) -> PointMappingsRefEnum<'_> {
PointMappingsRefEnum::Compressed(&self.mappings)
}
/// Creates a flusher function, that writes the deleted points bitvec to disk.
fn mapping_flusher(&self) -> Flusher {
// Only flush deletions because mappings are immutable
@@ -491,36 +523,6 @@ impl IdTracker for ImmutableIdTracker {
Box::new(move || flusher().map_err(OperationError::from))
}
fn total_point_count(&self) -> usize {
self.mappings.total_point_count()
}
fn available_point_count(&self) -> usize {
self.mappings.available_point_count()
}
fn deleted_point_count(&self) -> usize {
self.total_point_count() - self.available_point_count()
}
fn deleted_point_bitslice(&self) -> &BitSlice {
self.mappings.deleted()
}
fn is_deleted_point(&self, key: PointOffsetType) -> bool {
self.mappings.is_deleted_point(key)
}
fn name(&self) -> &'static str {
"immutable id tracker"
}
fn iter_internal_versions(
&self,
) -> Box<dyn Iterator<Item = (PointOffsetType, SeqNumberType)> + '_> {
Box::new(self.internal_to_version.iter())
}
fn files(&self) -> Vec<PathBuf> {
vec![
Self::deleted_file_path(&self.path),

118
lib/segment/src/id_tracker/in_memory_id_tracker.rs
View File

@@ -10,7 +10,7 @@ use rand::rngs::StdRng;
use crate::common::Flusher;
use crate::common::operation_error::OperationResult;
use crate::id_tracker::point_mappings::PointMappings;
use crate::id_tracker::{DELETED_POINT_VERSION, IdTracker, PointMappingsRefEnum};
use crate::id_tracker::{DELETED_POINT_VERSION, IdTracker, IdTrackerRead, PointMappingsRefEnum};
use crate::types::{PointIdType, SeqNumberType};
/// A non-persistent ID tracker for faster and more efficient building of `ImmutableIdTracker`.
@@ -59,28 +59,11 @@ impl InMemoryIdTracker {
}
}
impl IdTracker for InMemoryIdTracker {
impl IdTrackerRead for InMemoryIdTracker {
fn internal_version(&self, internal_id: PointOffsetType) -> Option<SeqNumberType> {
self.internal_to_version.get(internal_id as usize).copied()
}
fn set_internal_version(
&mut self,
internal_id: PointOffsetType,
version: SeqNumberType,
) -> OperationResult<()> {
if self.external_id(internal_id).is_some() {
if let Some(old_version) = self.internal_to_version.get_mut(internal_id as usize) {
*old_version = version;
} else {
self.internal_to_version.resize(internal_id as usize + 1, 0);
self.internal_to_version[internal_id as usize] = version;
}
}
Ok(())
}
fn internal_id(&self, external_id: PointIdType) -> Option<PointOffsetType> {
self.mappings.internal_id(&external_id)
}
@@ -89,49 +72,10 @@ impl IdTracker for InMemoryIdTracker {
self.mappings.external_id(internal_id)
}
fn set_link(
&mut self,
external_id: PointIdType,
internal_id: PointOffsetType,
) -> OperationResult<()> {
let _replaced_internal_id = self.mappings.set_link(external_id, internal_id);
Ok(())
}
fn drop(&mut self, external_id: PointIdType) -> OperationResult<()> {
// Unset version first because it still requires the mapping to exist
if let Some(internal_id) = self.internal_id(external_id) {
self.set_internal_version(internal_id, DELETED_POINT_VERSION)?;
}
self.mappings.drop(external_id);
Ok(())
}
fn drop_internal(&mut self, internal_id: PointOffsetType) -> OperationResult<()> {
// Unset version first because it still requires the mapping to exist
self.set_internal_version(internal_id, DELETED_POINT_VERSION)?;
if let Some(external_id) = self.mappings.external_id(internal_id) {
self.mappings.drop(external_id);
}
Ok(())
}
fn point_mappings(&self) -> PointMappingsRefEnum<'_> {
PointMappingsRefEnum::Plain(&self.mappings)
}
/// Creates a flusher function, that writes the deleted points bitvec to disk.
fn mapping_flusher(&self) -> Flusher {
debug_assert!(false, "InMemoryIdTracker should not be flushed");
Box::new(|| Ok(()))
}
/// Creates a flusher function, that writes the points versions to disk.
fn versions_flusher(&self) -> Flusher {
debug_assert!(false, "InMemoryIdTracker should not be flushed");
Box::new(|| Ok(()))
}
fn total_point_count(&self) -> usize {
self.mappings.total_point_count()
}
@@ -166,6 +110,64 @@ impl IdTracker for InMemoryIdTracker {
.map(|(i, version)| (i as PointOffsetType, *version)),
)
}
}
impl IdTracker for InMemoryIdTracker {
fn set_internal_version(
&mut self,
internal_id: PointOffsetType,
version: SeqNumberType,
) -> OperationResult<()> {
if self.external_id(internal_id).is_some() {
if let Some(old_version) = self.internal_to_version.get_mut(internal_id as usize) {
*old_version = version;
} else {
self.internal_to_version.resize(internal_id as usize + 1, 0);
self.internal_to_version[internal_id as usize] = version;
}
}
Ok(())
}
fn set_link(
&mut self,
external_id: PointIdType,
internal_id: PointOffsetType,
) -> OperationResult<()> {
let _replaced_internal_id = self.mappings.set_link(external_id, internal_id);
Ok(())
}
fn drop(&mut self, external_id: PointIdType) -> OperationResult<()> {
// Unset version first because it still requires the mapping to exist
if let Some(internal_id) = self.internal_id(external_id) {
self.set_internal_version(internal_id, DELETED_POINT_VERSION)?;
}
self.mappings.drop(external_id);
Ok(())
}
fn drop_internal(&mut self, internal_id: PointOffsetType) -> OperationResult<()> {
// Unset version first because it still requires the mapping to exist
self.set_internal_version(internal_id, DELETED_POINT_VERSION)?;
if let Some(external_id) = self.mappings.external_id(internal_id) {
self.mappings.drop(external_id);
}
Ok(())
}
/// Creates a flusher function, that writes the deleted points bitvec to disk.
fn mapping_flusher(&self) -> Flusher {
debug_assert!(false, "InMemoryIdTracker should not be flushed");
Box::new(|| Ok(()))
}
/// Creates a flusher function, that writes the points versions to disk.
fn versions_flusher(&self) -> Flusher {
debug_assert!(false, "InMemoryIdTracker should not be flushed");
Box::new(|| Ok(()))
}
fn files(&self) -> Vec<PathBuf> {
debug_assert!(false, "InMemoryIdTracker should not be persisted");

100
lib/segment/src/id_tracker/mutable_id_tracker/mod.rs
View File

@@ -26,7 +26,7 @@ use self::versions_storage::{
use crate::common::Flusher;
use crate::common::operation_error::{OperationError, OperationResult};
use crate::id_tracker::point_mappings::PointMappings;
use crate::id_tracker::{DELETED_POINT_VERSION, IdTracker, PointMappingsRefEnum};
use crate::id_tracker::{DELETED_POINT_VERSION, IdTracker, IdTrackerRead, PointMappingsRefEnum};
use crate::types::{PointIdType, SeqNumberType};
/// Mutable in-memory ID tracker with simple file based backing storage
@@ -156,11 +156,60 @@ impl MutableIdTracker {
}
}
impl IdTracker for MutableIdTracker {
impl IdTrackerRead for MutableIdTracker {
fn internal_version(&self, internal_id: PointOffsetType) -> Option<SeqNumberType> {
self.internal_to_version.get(internal_id as usize).copied()
}
fn internal_id(&self, external_id: PointIdType) -> Option<PointOffsetType> {
self.mappings.internal_id(&external_id)
}
fn external_id(&self, internal_id: PointOffsetType) -> Option<PointIdType> {
self.mappings.external_id(internal_id)
}
fn point_mappings(&self) -> PointMappingsRefEnum<'_> {
PointMappingsRefEnum::Plain(&self.mappings)
}
fn total_point_count(&self) -> usize {
self.mappings.total_point_count()
}
fn available_point_count(&self) -> usize {
self.mappings.available_point_count()
}
fn deleted_point_count(&self) -> usize {
self.total_point_count() - self.available_point_count()
}
fn is_deleted_point(&self, key: PointOffsetType) -> bool {
self.mappings.is_deleted_point(key)
}
fn deleted_point_bitslice(&self) -> &BitSlice {
self.mappings.deleted()
}
fn iter_internal_versions(
&self,
) -> Box<dyn Iterator<Item = (PointOffsetType, SeqNumberType)> + '_> {
Box::new(
self.internal_to_version
.iter()
.enumerate()
.map(|(i, version)| (i as PointOffsetType, *version)),
)
}
fn name(&self) -> &'static str {
"mutable id tracker"
}
}
impl IdTracker for MutableIdTracker {
fn set_internal_version(
&mut self,
internal_id: PointOffsetType,
@@ -184,14 +233,6 @@ impl IdTracker for MutableIdTracker {
Ok(())
}
fn internal_id(&self, external_id: PointIdType) -> Option<PointOffsetType> {
self.mappings.internal_id(&external_id)
}
fn external_id(&self, internal_id: PointOffsetType) -> Option<PointIdType> {
self.mappings.external_id(internal_id)
}
fn set_link(
&mut self,
external_id: PointIdType,
@@ -228,22 +269,6 @@ impl IdTracker for MutableIdTracker {
Ok(())
}
fn point_mappings(&self) -> PointMappingsRefEnum<'_> {
PointMappingsRefEnum::Plain(&self.mappings)
}
fn total_point_count(&self) -> usize {
self.mappings.total_point_count()
}
fn available_point_count(&self) -> usize {
self.mappings.available_point_count()
}
fn deleted_point_count(&self) -> usize {
self.total_point_count() - self.available_point_count()
}
/// Creates a flusher function, that persists the removed points in the mapping database
/// and flushes the mapping to disk.
/// This function should be called _before_ flushing the version database.
@@ -332,29 +357,6 @@ impl IdTracker for MutableIdTracker {
})
}
fn is_deleted_point(&self, key: PointOffsetType) -> bool {
self.mappings.is_deleted_point(key)
}
fn deleted_point_bitslice(&self) -> &BitSlice {
self.mappings.deleted()
}
fn iter_internal_versions(
&self,
) -> Box<dyn Iterator<Item = (PointOffsetType, SeqNumberType)> + '_> {
Box::new(
self.internal_to_version
.iter()
.enumerate()
.map(|(i, version)| (i as PointOffsetType, *version)),
)
}
fn name(&self) -> &'static str {
"mutable id tracker"
}
#[inline]
fn files(&self) -> Vec<PathBuf> {
Self::segment_files(&self.segment_path)

2
lib/segment/src/id_tracker/mutable_id_tracker/tests.rs
View File

@@ -15,9 +15,9 @@ use super::mappings_storage::{load_mappings, mappings_path, read_mappings};
use super::versions_storage::{
VERSION_ELEMENT_SIZE, load_versions, store_version_changes, versions_path,
};
use crate::id_tracker::IdTracker;
use crate::id_tracker::compressed::compressed_point_mappings::CompressedPointMappings;
use crate::id_tracker::in_memory_id_tracker::InMemoryIdTracker;
use crate::id_tracker::{IdTracker, IdTrackerRead};
use crate::types::{PointIdType, SeqNumberType};
const RAND_SEED: u64 = 42;

2
lib/segment/src/index/field_index/index_selector.rs
View File

@@ -16,7 +16,7 @@ use super::stored_point_to_values::StoredValue;
use super::{FieldIndexBuilder, ValueIndexer};
use crate::common::operation_error::{OperationError, OperationResult};
use crate::data_types::index::TextIndexParams;
use crate::id_tracker::{IdTracker, IdTrackerEnum};
use crate::id_tracker::{IdTrackerEnum, IdTrackerRead};
use crate::index::field_index::FieldIndex;
use crate::index::field_index::full_text_index::text_index::FullTextIndex;
use crate::index::field_index::geo_index::GeoMapIndex;

2
lib/segment/src/index/hnsw_index/hnsw.rs
View File

@@ -34,7 +34,7 @@ use crate::common::operation_time_statistics::{
};
use crate::data_types::query_context::VectorQueryContext;
use crate::data_types::vectors::{QueryVector, VectorInternal, VectorRef};
use crate::id_tracker::{IdTracker, IdTrackerEnum};
use crate::id_tracker::{IdTrackerEnum, IdTrackerRead};
use crate::index::field_index::PayloadBlockCondition;
use crate::index::hnsw_index::HnswM;
use crate::index::hnsw_index::build_condition_checker::BuildConditionChecker;

2
lib/segment/src/index/plain_payload_index.rs
View File

@@ -14,7 +14,7 @@ use super::field_index::FieldIndex;
use super::payload_config::PayloadFieldSchemaWithIndexType;
use crate::common::Flusher;
use crate::common::operation_error::OperationResult;
use crate::id_tracker::{IdTracker, IdTrackerEnum};
use crate::id_tracker::{IdTrackerEnum, IdTrackerRead};
use crate::index::field_index::{CardinalityEstimation, PayloadBlockCondition};
use crate::index::payload_config::PayloadConfig;
use crate::index::{BuildIndexResult, PayloadIndex};

2
lib/segment/src/index/plain_vector_index.rs
View File

@@ -14,7 +14,7 @@ use crate::common::operation_time_statistics::{
};
use crate::data_types::query_context::VectorQueryContext;
use crate::data_types::vectors::{QueryVector, VectorRef};
use crate::id_tracker::{IdTracker, IdTrackerEnum};
use crate::id_tracker::{IdTrackerEnum, IdTrackerRead};
use crate::index::struct_payload_index::StructPayloadIndex;
use crate::index::vector_index_search_common::{
get_oversampled_top, is_quantized_search, postprocess_search_result,

2
lib/segment/src/index/query_optimization/condition_converter.rs
View File

@@ -8,7 +8,7 @@ use match_converter::get_match_checkers;
use ordered_float::OrderedFloat;
use serde_json::Value;
use crate::id_tracker::IdTracker;
use crate::id_tracker::IdTrackerRead;
use crate::index::field_index::FieldIndex;
use crate::index::field_index::null_index::NullIndex;
use crate::index::query_optimization::optimized_filter::ConditionCheckerFn;

2
lib/segment/src/index/sparse_index/sparse_vector_index.rs
View File

@@ -26,7 +26,7 @@ use crate::common::operation_time_statistics::ScopeDurationMeasurer;
use crate::data_types::named_vectors::CowVector;
use crate::data_types::query_context::VectorQueryContext;
use crate::data_types::vectors::{QueryVector, VectorInternal, VectorRef};
use crate::id_tracker::{IdTracker, IdTrackerEnum};
use crate::id_tracker::{IdTrackerEnum, IdTrackerRead};
use crate::index::field_index::CardinalityEstimation;
use crate::index::hnsw_index::point_scorer::BatchFilteredSearcher;
use crate::index::query_estimator::adjust_to_available_vectors;

2
lib/segment/src/index/struct_payload_index.rs
View File

@@ -23,7 +23,7 @@ use super::payload_config::{FullPayloadIndexType, PayloadFieldSchemaWithIndexTyp
use crate::common::Flusher;
use crate::common::operation_error::{OperationError, OperationResult};
use crate::common::utils::IndexesMap;
use crate::id_tracker::{IdTracker, IdTrackerEnum, PointMappingsRefEnum};
use crate::id_tracker::{IdTrackerEnum, IdTrackerRead, PointMappingsRefEnum};
use crate::index::field_index::{
CardinalityEstimation, FieldIndex, PayloadBlockCondition, PrimaryCondition,
};

2
lib/segment/src/payload_storage/query_checker.rs
View File

@@ -10,7 +10,7 @@ use common::counter::hardware_counter::HardwareCounterCell;
use common::types::PointOffsetType;
use crate::common::utils::{IndexesMap, check_is_empty, check_is_null};
use crate::id_tracker::{IdTracker, IdTrackerEnum};
use crate::id_tracker::{IdTrackerEnum, IdTrackerRead};
use crate::index::field_index::FieldIndex;
use crate::payload_storage::condition_checker::ValueChecker;
use crate::payload_storage::payload_storage_enum::PayloadStorageEnum;

2
lib/segment/src/segment/entry.rs
View File

@@ -27,7 +27,7 @@ use crate::data_types::vectors::{QueryVector, VectorInternal};
use crate::entry::entry_point::{
NonAppendableSegmentEntry, ReadSegmentEntry, SegmentEntry, StorageSegmentEntry,
};
use crate::id_tracker::{IdTracker, PointMappingsGuard};
use crate::id_tracker::{IdTracker, IdTrackerRead, PointMappingsGuard};
use crate::index::field_index::{CardinalityEstimation, FieldIndex};
use crate::index::query_estimator::adjust_for_deferred_points;
use crate::index::{BuildIndexResult, PayloadIndex, VectorIndex};

2
lib/segment/src/segment/facet.rs
View File

@@ -9,7 +9,7 @@ use super::Segment;
use crate::common::operation_error::{OperationResult, check_process_stopped};
use crate::data_types::facets::{FacetParams, FacetValue};
use crate::entry::ReadSegmentEntry;
use crate::id_tracker::IdTracker;
use crate::id_tracker::IdTrackerRead;
use crate::index::PayloadIndex;
use crate::json_path::JsonPath;
use crate::payload_storage::FilterContext;

2
lib/segment/src/segment/order_by.rs
View File

@@ -9,7 +9,7 @@ use itertools::Either;
use super::Segment;
use crate::common::operation_error::{OperationError, OperationResult};
use crate::data_types::order_by::{Direction, OrderBy, OrderValue};
use crate::id_tracker::IdTracker;
use crate::id_tracker::IdTrackerRead;
use crate::index::PayloadIndex;
use crate::index::field_index::numeric_index::StreamRange;
use crate::spaces::tools::{peek_top_largest_iterable, peek_top_smallest_iterable};

2
lib/segment/src/segment/sampling.rs
View File

@@ -6,7 +6,7 @@ use rand::seq::{IteratorRandom, SliceRandom};
use super::Segment;
use crate::common::operation_error::OperationResult;
use crate::id_tracker::IdTracker;
use crate::id_tracker::IdTrackerRead;
use crate::index::PayloadIndex;
use crate::types::{Filter, PointIdType};

2
lib/segment/src/segment/scroll.rs
View File

@@ -7,7 +7,7 @@ use common::types::DeferredBehavior;
use super::Segment;
use crate::common::operation_error::OperationResult;
use crate::entry::ReadSegmentEntry;
use crate::id_tracker::IdTracker;
use crate::id_tracker::IdTrackerRead;
use crate::index::PayloadIndex;
use crate::spaces::tools::peek_top_smallest_iterable;
use crate::types::{Filter, PointIdType};

2
lib/segment/src/segment/search.rs
View File

@@ -12,7 +12,7 @@ use crate::data_types::segment_record::SegmentRecord;
use crate::data_types::vectors::QueryVector;
use crate::data_types::vectors::VectorStructInternal;
use crate::entry::ReadSegmentEntry;
use crate::id_tracker::IdTracker;
use crate::id_tracker::IdTrackerRead;
#[cfg(feature = "testing")]
use crate::types::VectorName;
#[cfg(feature = "testing")]

2
lib/segment/src/segment/segment_ops.rs
View File

@@ -22,7 +22,7 @@ use crate::data_types::named_vectors::NamedVectors;
use crate::data_types::vectors::VectorInternal;
use crate::entry::entry_point::StorageSegmentEntry as _;
use crate::entry::{NonAppendableSegmentEntry as _, ReadSegmentEntry};
use crate::id_tracker::IdTracker;
use crate::id_tracker::{IdTracker, IdTrackerRead};
use crate::index::{PayloadIndex, VectorIndex};
use crate::types::{
Payload, PayloadFieldSchema, PayloadKeyType, PointIdType, SegmentState, SeqNumberType,

2
lib/segment/src/segment/vector_name_ops.rs
View File

@@ -6,7 +6,7 @@ use atomic_refcell::AtomicRefCell;
use super::Segment;
use crate::common::operation_error::OperationResult;
use crate::data_types::vector_name_config::VectorNameConfig;
use crate::id_tracker::IdTracker as _;
use crate::id_tracker::IdTrackerRead as _;
use crate::index::VectorIndexEnum;
use crate::index::plain_vector_index::PlainVectorIndex;
use crate::index::sparse_index::sparse_index_config::SparseIndexType;

2
lib/segment/src/segment/vectors.rs
View File

@@ -5,7 +5,7 @@ use common::iterator_ext::IteratorExt;
use crate::common::operation_error::OperationResult;
use crate::data_types::vectors::VectorInternal;
use crate::id_tracker::IdTracker;
use crate::id_tracker::IdTrackerRead;
use crate::segment::Segment;
use crate::types::{PointIdType, VectorName};

2
lib/segment/src/segment_constructor/segment_builder.rs
View File

@@ -34,7 +34,7 @@ use crate::entry::ReadSegmentEntry;
use crate::id_tracker::compressed::compressed_point_mappings::CompressedPointMappings;
use crate::id_tracker::immutable_id_tracker::ImmutableIdTracker;
use crate::id_tracker::in_memory_id_tracker::InMemoryIdTracker;
use crate::id_tracker::{IdTracker, IdTrackerEnum, for_each_unique_point};
use crate::id_tracker::{IdTracker, IdTrackerEnum, IdTrackerRead, for_each_unique_point};
use crate::index::field_index::FieldIndex;
use crate::index::sparse_index::sparse_vector_index::SparseVectorIndexOpenArgs;
use crate::index::struct_payload_index::StructPayloadIndex;

2
lib/segment/src/segment_constructor/segment_constructor_base.rs
View File

@@ -27,7 +27,7 @@ use crate::common::operation_error::{OperationError, OperationResult, check_proc
use crate::data_types::vectors::DEFAULT_VECTOR_NAME;
use crate::id_tracker::immutable_id_tracker::ImmutableIdTracker;
use crate::id_tracker::mutable_id_tracker::MutableIdTracker;
use crate::id_tracker::{IdTracker, IdTrackerEnum};
use crate::id_tracker::{IdTrackerEnum, IdTrackerRead};
use crate::index::VectorIndexEnum;
use crate::index::hnsw_index::gpu::gpu_devices_manager::LockedGpuDevice;
use crate::index::hnsw_index::hnsw::{HNSWIndex, HnswIndexOpenArgs};

2
lib/segment/src/vector_storage/dense/dense_vector_storage.rs
View File

@@ -389,7 +389,7 @@ mod tests {
use super::*;
use crate::data_types::vectors::{DenseVector, QueryVector, VectorElementType};
use crate::fixtures::payload_context_fixture::create_id_tracker_fixture;
use crate::id_tracker::IdTracker;
use crate::id_tracker::{IdTracker, IdTrackerRead};
use crate::index::hnsw_index::point_scorer::{BatchFilteredSearcher, FilteredScorer};
use crate::types::{PointIdType, QuantizationConfig, ScalarQuantizationConfig};
use crate::vector_storage::dense::volatile_dense_vector_storage::new_volatile_dense_vector_storage;

2
lib/segment/src/vector_storage/tests/async_raw_scorer.rs
View File

@@ -9,7 +9,7 @@ use rand::seq::IteratorRandom as _;
use super::utils::{Result, delete_random_vectors, insert_distributed_vectors, sampler};
use crate::data_types::vectors::QueryVector;
use crate::fixtures::payload_context_fixture::create_id_tracker_fixture;
use crate::id_tracker::IdTracker;
use crate::id_tracker::IdTrackerRead;
use crate::index::hnsw_index::point_scorer::FilteredScorer;
use crate::types::Distance;
use crate::vector_storage::VectorStorageEnum;

2
lib/segment/src/vector_storage/tests/custom_query_scorer_equivalency.rs
View File

@@ -16,7 +16,7 @@ use super::utils::sampler;
use crate::data_types::vectors::{QueryVector, VectorElementType};
use crate::fixtures::payload_context_fixture::create_id_tracker_fixture;
use crate::fixtures::query_fixtures::QueryVariant;
use crate::id_tracker::IdTracker;
use crate::id_tracker::IdTrackerRead;
use crate::index::hnsw_index::point_scorer::FilteredScorer;
use crate::types::{
BinaryQuantizationConfig, Distance, ProductQuantizationConfig, QuantizationConfig,

2
lib/segment/src/vector_storage/tests/test_appendable_dense_vector_storage.rs
View File

@@ -9,7 +9,7 @@ use tempfile::Builder;
use crate::data_types::vectors::QueryVector;
use crate::fixtures::payload_context_fixture::create_id_tracker_fixture;
use crate::id_tracker::IdTracker;
use crate::id_tracker::{IdTracker, IdTrackerRead};
use crate::index::hnsw_index::point_scorer::{BatchFilteredSearcher, FilteredScorer};
use crate::types::{Distance, PointIdType, QuantizationConfig, ScalarQuantizationConfig};
use crate::vector_storage::dense::appendable_dense_vector_storage::open_appendable_memmap_vector_storage_full;

2
lib/segment/src/vector_storage/tests/test_appendable_multi_dense_vector_storage.rs
View File

@@ -12,7 +12,7 @@ use crate::data_types::vectors::{
MultiDenseVectorInternal, QueryVector, TypedMultiDenseVectorRef, VectorElementType, VectorRef,
};
use crate::fixtures::payload_context_fixture::create_id_tracker_fixture;
use crate::id_tracker::IdTracker;
use crate::id_tracker::IdTrackerRead;
use crate::index::hnsw_index::point_scorer::BatchFilteredSearcher;
use crate::types::{Distance, MultiVectorConfig};
use crate::vector_storage::common::CHUNK_SIZE;

2
lib/segment/src/vector_storage/tests/test_appendable_sparse_vector_storage.rs
View File

@@ -9,7 +9,7 @@ use tempfile::Builder;
use crate::data_types::vectors::QueryVector;
use crate::fixtures::payload_context_fixture::create_id_tracker_fixture;
use crate::id_tracker::IdTracker;
use crate::id_tracker::IdTrackerRead;
use crate::index::hnsw_index::point_scorer::BatchFilteredSearcher;
use crate::vector_storage::query::RecoQuery;
use crate::vector_storage::sparse::mmap_sparse_vector_storage::MmapSparseVectorStorage;

2
lib/segment/tests/integration/multivector_hnsw_test.rs
View File

@@ -16,7 +16,7 @@ use segment::data_types::vectors::{
use segment::entry::{NonAppendableSegmentEntry, SegmentEntry};
use segment::fixtures::index_fixtures::random_vector;
use segment::fixtures::payload_fixtures::random_int_payload;
use segment::id_tracker::IdTracker;
use segment::id_tracker::IdTrackerRead;
use segment::index::VectorIndex;
use segment::index::hnsw_index::hnsw::{HNSWIndex, HnswIndexOpenArgs};
use segment::json_path::JsonPath;

2
lib/segment/tests/integration/payload_index_test.rs
View File

@@ -27,7 +27,7 @@ use segment::fixtures::payload_fixtures::{
STR_PROJ_KEY, STR_ROOT_PROJ_KEY, TEXT_KEY, generate_diverse_nested_payload,
generate_diverse_payload, random_filter, random_nested_filter, random_vector,
};
use segment::id_tracker::IdTracker;
use segment::id_tracker::IdTrackerRead;
use segment::index::PayloadIndex;
use segment::index::field_index::{FieldIndex, PrimaryCondition};
use segment::index::struct_payload_index::StructPayloadIndex;

2
lib/segment/tests/integration/segment_builder_test.rs
View File

@@ -13,7 +13,7 @@ use segment::common::operation_error::OperationError;
use segment::data_types::named_vectors::NamedVectors;
use segment::data_types::vectors::{DEFAULT_VECTOR_NAME, VectorRef, only_default_vector};
use segment::entry::entry_point::{NonAppendableSegmentEntry, ReadSegmentEntry, SegmentEntry};
use segment::id_tracker::IdTracker;
use segment::id_tracker::IdTrackerRead;
use segment::index::hnsw_index::get_num_indexing_threads;
use segment::json_path::JsonPath;
use segment::segment::Segment;

2
lib/segment/tests/integration/sparse_vector_index_search_tests.rs
View File

@@ -16,7 +16,7 @@ use segment::data_types::vectors::{QueryVector, VectorInternal};
use segment::entry::{SegmentEntry, StorageSegmentEntry as _};
use segment::fixtures::payload_fixtures::STR_KEY;
use segment::fixtures::sparse_fixtures::{fixture_sparse_index, fixture_sparse_index_from_iter};
use segment::id_tracker::IdTracker;
use segment::id_tracker::{IdTracker, IdTrackerRead};
use segment::index::sparse_index::sparse_index_config::{SparseIndexConfig, SparseIndexType};
use segment::index::sparse_index::sparse_vector_index::{
SparseVectorIndex, SparseVectorIndexOpenArgs,