mirror of
https://github.com/qdrant/qdrant.git
synced 2026-08-05 17:40:55 -05:00
852ca200b5
* feat: feature-flagged segment manifest for LocalShard
Add an on-disk segment manifest (`segments/manifest.json`) that lists a
shard's segments and their state, so out-of-process readers (e.g. a
read-only follower, possibly over object storage) can discover segments
without scanning the filesystem. Gated by the new `write_segment_manifest`
feature flag (off by default).
shard: define the structure + helpers (`SegmentsManifest`,
`SegmentManifestState`, `from_segment_holder`) in a new `segment_manifest`
module, plus the `SEGMENT_MANIFEST_FILE` constant and path helper. The
manifest is a flat `{ "<uuid>": "<state>" }` map; only `active` is written
today, with `under_construction`/`retiring` defined so the format can be
extended without breaking compatibility.
collection: LocalShard owns the writing logic. The manifest is persisted
via `SaveOnDisk<SegmentsManifest>`, initialized from the live segment set
on load/build and refreshed by the optimization worker whenever the
segment set changes (the helper re-derives from the holder and no-ops when
unchanged). No changes to `lib/shard/src/optimize.rs` internals.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Emj8TFxdtrf3K32eWhGgor
* feat: make append_only_mutations a proper feature flag
Replace the debug-only `QDRANT_APPEND_ONLY_MUTATIONS=1` env-var escape
hatch in segment construction with a `FeatureFlags::append_only_mutations`
flag, so it works in release builds and is configurable like the other
flags (config / `QDRANT__FEATURE_FLAGS__APPEND_ONLY_MUTATIONS`).
Deliberately left out of `FeatureFlags::all()`: it changes mutation
semantics and `all` is enabled in dev and e2e configs, so it stays
explicit opt-in.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Emj8TFxdtrf3K32eWhGgor
* upd openapi schema
* feat: register segments in manifest via must-use token, holder builder
Wire segment-manifest maintenance through the segment lifecycle so a
newly created segment is registered as soon as it exists on disk, and
construction can't silently skip it:
- build_segment now returns a #[must_use] NewSegmentToken carrying the
new segment UUID; the lint forces callers to register or drop it.
- SegmentHolder owns the manifest and reconciles it on sync; new
segments are registered ASAP (even before being added to the holder)
via the token, before they can receive writes.
- SegmentHolderBuilder is the only way to obtain a shard's holder; its
build() wires up the manifest, so it can't be forgotten. init/set
manifest helpers are now private / test-only.
- Optimization registers the optimized segment before dropping the
superseded segments' data; deletion is intentionally lenient.
- Document the consistency assumptions on SegmentsManifest: it is a
superset-biased view that may list not-yet-finalized or already-deleted
segments, which readers must tolerate.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---------
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
444 lines
14 KiB
Rust
444 lines
14 KiB
Rust
use std::collections::HashMap;
|
|
use std::path::Path;
|
|
|
|
use common::counter::hardware_counter::HardwareCounterCell;
|
|
use segment::data_types::named_vectors::NamedVectors;
|
|
use segment::data_types::vectors::{DenseVector, VectorRef, only_default_vector};
|
|
use segment::entry::entry_point::SegmentEntry;
|
|
use segment::index::sparse_index::sparse_index_config::{SparseIndexConfig, SparseIndexType};
|
|
use segment::payload_json;
|
|
use segment::segment::Segment;
|
|
use segment::segment_constructor::build_segment;
|
|
use segment::segment_constructor::simple_segment_constructor::build_simple_segment;
|
|
use segment::types::{
|
|
Distance, Indexes, SegmentConfig, SparseVectorDataConfig, SparseVectorStorageType,
|
|
VectorDataConfig, VectorName, VectorStorageType,
|
|
};
|
|
use sparse::common::sparse_vector::SparseVector;
|
|
|
|
pub fn empty_segment(path: &Path) -> Segment {
|
|
build_simple_segment(path, 4, Distance::Dot).unwrap()
|
|
}
|
|
|
|
pub const PAYLOAD_KEY: &str = "color";
|
|
pub const SPARSE_VECTOR_NAME: &VectorName = "sparse";
|
|
|
|
pub fn build_segment_1(path: &Path) -> Segment {
|
|
let mut segment1 = empty_segment(path);
|
|
|
|
let vec1 = vec![1.0, 0.0, 1.0, 1.0];
|
|
let vec2 = vec![1.0, 0.0, 1.0, 0.0];
|
|
let vec3 = vec![1.0, 1.0, 1.0, 1.0];
|
|
let vec4 = vec![1.0, 1.0, 0.0, 1.0];
|
|
let vec5 = vec![1.0, 0.0, 0.0, 0.0];
|
|
|
|
let hw_counter = HardwareCounterCell::new();
|
|
|
|
segment1
|
|
.upsert_point(1, 1.into(), only_default_vector(&vec1), &hw_counter)
|
|
.unwrap();
|
|
segment1
|
|
.upsert_point(2, 2.into(), only_default_vector(&vec2), &hw_counter)
|
|
.unwrap();
|
|
segment1
|
|
.upsert_point(3, 3.into(), only_default_vector(&vec3), &hw_counter)
|
|
.unwrap();
|
|
segment1
|
|
.upsert_point(4, 4.into(), only_default_vector(&vec4), &hw_counter)
|
|
.unwrap();
|
|
segment1
|
|
.upsert_point(5, 5.into(), only_default_vector(&vec5), &hw_counter)
|
|
.unwrap();
|
|
|
|
let payload_key = PAYLOAD_KEY;
|
|
|
|
let payload_option1 = payload_json! {payload_key: vec!["red".to_owned()]};
|
|
let payload_option2 = payload_json! {payload_key: vec!["red".to_owned(), "blue".to_owned()]};
|
|
let payload_option3 = payload_json! {payload_key: vec!["blue".to_owned()]};
|
|
|
|
segment1
|
|
.set_payload(6, 1.into(), &payload_option1, &None, &hw_counter)
|
|
.unwrap();
|
|
segment1
|
|
.set_payload(6, 2.into(), &payload_option1, &None, &hw_counter)
|
|
.unwrap();
|
|
segment1
|
|
.set_payload(6, 3.into(), &payload_option3, &None, &hw_counter)
|
|
.unwrap();
|
|
segment1
|
|
.set_payload(6, 4.into(), &payload_option2, &None, &hw_counter)
|
|
.unwrap();
|
|
segment1
|
|
.set_payload(6, 5.into(), &payload_option2, &None, &hw_counter)
|
|
.unwrap();
|
|
|
|
segment1
|
|
}
|
|
|
|
pub fn build_segment_2(path: &Path) -> Segment {
|
|
let mut segment2 = empty_segment(path);
|
|
|
|
let vec1 = vec![-1.0, 0.0, 1.0, 1.0];
|
|
let vec2 = vec![-1.0, 0.0, 1.0, 0.0];
|
|
let vec3 = vec![-1.0, 1.0, 1.0, 1.0];
|
|
let vec4 = vec![-1.0, 1.0, 0.0, 1.0];
|
|
let vec5 = vec![-1.0, 0.0, 0.0, 0.0];
|
|
|
|
let hw_counter = HardwareCounterCell::new();
|
|
|
|
segment2
|
|
.upsert_point(11, 11.into(), only_default_vector(&vec1), &hw_counter)
|
|
.unwrap();
|
|
segment2
|
|
.upsert_point(12, 12.into(), only_default_vector(&vec2), &hw_counter)
|
|
.unwrap();
|
|
segment2
|
|
.upsert_point(13, 13.into(), only_default_vector(&vec3), &hw_counter)
|
|
.unwrap();
|
|
segment2
|
|
.upsert_point(14, 14.into(), only_default_vector(&vec4), &hw_counter)
|
|
.unwrap();
|
|
segment2
|
|
.upsert_point(15, 15.into(), only_default_vector(&vec5), &hw_counter)
|
|
.unwrap();
|
|
|
|
let payload_key = PAYLOAD_KEY;
|
|
|
|
let payload_option1 = payload_json! {payload_key: vec!["red".to_owned()]};
|
|
let payload_option2 = payload_json! {payload_key: vec!["red".to_owned(), "blue".to_owned()]};
|
|
let payload_option3 = payload_json! {payload_key: vec!["blue".to_owned()]};
|
|
|
|
segment2
|
|
.set_payload(16, 11.into(), &payload_option1, &None, &hw_counter)
|
|
.unwrap();
|
|
segment2
|
|
.set_payload(16, 12.into(), &payload_option1, &None, &hw_counter)
|
|
.unwrap();
|
|
segment2
|
|
.set_payload(16, 13.into(), &payload_option3, &None, &hw_counter)
|
|
.unwrap();
|
|
segment2
|
|
.set_payload(16, 14.into(), &payload_option2, &None, &hw_counter)
|
|
.unwrap();
|
|
segment2
|
|
.set_payload(16, 15.into(), &payload_option2, &None, &hw_counter)
|
|
.unwrap();
|
|
|
|
segment2
|
|
}
|
|
|
|
pub fn build_segment_3(path: &Path) -> Segment {
|
|
let (mut segment3, _) = build_segment(
|
|
path,
|
|
&SegmentConfig {
|
|
vector_data: HashMap::from([
|
|
(
|
|
"vector1".into(),
|
|
VectorDataConfig {
|
|
size: 4,
|
|
distance: Distance::Dot,
|
|
storage_type: VectorStorageType::default(),
|
|
index: Indexes::Plain {},
|
|
quantization_config: None,
|
|
multivector_config: None,
|
|
datatype: None,
|
|
},
|
|
),
|
|
(
|
|
"vector2".into(),
|
|
VectorDataConfig {
|
|
size: 1,
|
|
distance: Distance::Dot,
|
|
storage_type: VectorStorageType::default(),
|
|
index: Indexes::Plain {},
|
|
quantization_config: None,
|
|
multivector_config: None,
|
|
datatype: None,
|
|
},
|
|
),
|
|
(
|
|
"vector3".into(),
|
|
VectorDataConfig {
|
|
size: 4,
|
|
distance: Distance::Euclid,
|
|
storage_type: VectorStorageType::default(),
|
|
index: Indexes::Plain {},
|
|
quantization_config: None,
|
|
multivector_config: None,
|
|
datatype: None,
|
|
},
|
|
),
|
|
]),
|
|
sparse_vector_data: Default::default(),
|
|
payload_storage_type: Default::default(),
|
|
},
|
|
None,
|
|
true,
|
|
)
|
|
.unwrap();
|
|
|
|
let collect_points_data = |vectors: &[DenseVector]| {
|
|
NamedVectors::from_pairs([
|
|
("vector1".into(), vectors[0].clone()),
|
|
("vector2".into(), vectors[1].clone()),
|
|
("vector3".into(), vectors[2].clone()),
|
|
])
|
|
};
|
|
|
|
let vec1 = [
|
|
vec![1.0, 0.0, 1.0, 1.0],
|
|
vec![0.0],
|
|
vec![-1.0, 0.0, 1.0, 1.0],
|
|
];
|
|
let vec2 = [
|
|
vec![1.0, 0.0, 1.0, 0.0],
|
|
vec![1.0],
|
|
vec![-1.0, 0.0, 1.0, 1.0],
|
|
];
|
|
let vec3 = [
|
|
vec![1.0, 1.0, 1.0, 1.0],
|
|
vec![2.0],
|
|
vec![-1.0, 0.0, 1.0, 1.0],
|
|
];
|
|
let vec4 = [
|
|
vec![1.0, 1.0, 0.0, 1.0],
|
|
vec![3.0],
|
|
vec![-1.0, 0.0, 1.0, 1.0],
|
|
];
|
|
let vec5 = [
|
|
vec![1.0, 0.0, 0.0, 0.0],
|
|
vec![4.0],
|
|
vec![-1.0, 0.0, 1.0, 1.0],
|
|
];
|
|
|
|
let hw_counter = HardwareCounterCell::new();
|
|
|
|
segment3
|
|
.upsert_point(1, 1.into(), collect_points_data(&vec1), &hw_counter)
|
|
.unwrap();
|
|
segment3
|
|
.upsert_point(2, 2.into(), collect_points_data(&vec2), &hw_counter)
|
|
.unwrap();
|
|
segment3
|
|
.upsert_point(3, 3.into(), collect_points_data(&vec3), &hw_counter)
|
|
.unwrap();
|
|
segment3
|
|
.upsert_point(4, 4.into(), collect_points_data(&vec4), &hw_counter)
|
|
.unwrap();
|
|
segment3
|
|
.upsert_point(5, 5.into(), collect_points_data(&vec5), &hw_counter)
|
|
.unwrap();
|
|
|
|
let payload_key = PAYLOAD_KEY;
|
|
|
|
let payload_option1 = payload_json! {payload_key: vec!["red".to_owned()]};
|
|
let payload_option2 = payload_json! {payload_key: vec!["red".to_owned(), "blue".to_owned()]};
|
|
let payload_option3 = payload_json! {payload_key: vec!["blue".to_owned()]};
|
|
|
|
segment3
|
|
.set_payload(6, 1.into(), &payload_option1, &None, &hw_counter)
|
|
.unwrap();
|
|
segment3
|
|
.set_payload(6, 2.into(), &payload_option1, &None, &hw_counter)
|
|
.unwrap();
|
|
segment3
|
|
.set_payload(6, 3.into(), &payload_option3, &None, &hw_counter)
|
|
.unwrap();
|
|
segment3
|
|
.set_payload(6, 4.into(), &payload_option2, &None, &hw_counter)
|
|
.unwrap();
|
|
segment3
|
|
.set_payload(6, 5.into(), &payload_option2, &None, &hw_counter)
|
|
.unwrap();
|
|
|
|
segment3
|
|
}
|
|
|
|
pub fn build_segment_sparse_1(path: &Path) -> Segment {
|
|
let (mut segment1, _) = build_segment(
|
|
path,
|
|
&SegmentConfig {
|
|
vector_data: Default::default(),
|
|
sparse_vector_data: HashMap::from([(
|
|
SPARSE_VECTOR_NAME.to_owned(),
|
|
SparseVectorDataConfig {
|
|
index: SparseIndexConfig::new(None, SparseIndexType::MutableRam, None),
|
|
storage_type: SparseVectorStorageType::default(),
|
|
modifier: None,
|
|
},
|
|
)]),
|
|
payload_storage_type: Default::default(),
|
|
},
|
|
None,
|
|
true,
|
|
)
|
|
.unwrap();
|
|
|
|
let vec1 = SparseVector::new(vec![0, 1, 2, 3], vec![1.0, 0.0, 1.0, 1.0]).unwrap();
|
|
let vec2 = SparseVector::new(vec![0, 1, 2, 3], vec![1.0, 0.0, 1.0, 0.0]).unwrap();
|
|
let vec3 = SparseVector::new(vec![0, 1, 2, 3], vec![1.0, 1.0, 1.0, 1.0]).unwrap();
|
|
let vec4 = SparseVector::new(vec![0, 1, 2, 3], vec![1.0, 1.0, 0.0, 1.0]).unwrap();
|
|
let vec5 = SparseVector::new(vec![0, 1, 2, 3], vec![1.0, 0.0, 0.0, 0.0]).unwrap();
|
|
|
|
let hw_counter = HardwareCounterCell::new();
|
|
|
|
segment1
|
|
.upsert_point(
|
|
1,
|
|
1.into(),
|
|
NamedVectors::from_ref(SPARSE_VECTOR_NAME, VectorRef::Sparse(&vec1)),
|
|
&hw_counter,
|
|
)
|
|
.unwrap();
|
|
segment1
|
|
.upsert_point(
|
|
2,
|
|
2.into(),
|
|
NamedVectors::from_ref(SPARSE_VECTOR_NAME, VectorRef::Sparse(&vec2)),
|
|
&hw_counter,
|
|
)
|
|
.unwrap();
|
|
segment1
|
|
.upsert_point(
|
|
3,
|
|
3.into(),
|
|
NamedVectors::from_ref(SPARSE_VECTOR_NAME, VectorRef::Sparse(&vec3)),
|
|
&hw_counter,
|
|
)
|
|
.unwrap();
|
|
segment1
|
|
.upsert_point(
|
|
4,
|
|
4.into(),
|
|
NamedVectors::from_ref(SPARSE_VECTOR_NAME, VectorRef::Sparse(&vec4)),
|
|
&hw_counter,
|
|
)
|
|
.unwrap();
|
|
segment1
|
|
.upsert_point(
|
|
5,
|
|
5.into(),
|
|
NamedVectors::from_ref(SPARSE_VECTOR_NAME, VectorRef::Sparse(&vec5)),
|
|
&hw_counter,
|
|
)
|
|
.unwrap();
|
|
|
|
let payload_key = PAYLOAD_KEY;
|
|
|
|
let payload_option1 = payload_json! {payload_key: vec!["red".to_owned()]};
|
|
let payload_option2 = payload_json! {payload_key: vec!["red".to_owned(), "blue".to_owned()]};
|
|
let payload_option3 = payload_json! {payload_key: vec!["blue".to_owned()]};
|
|
|
|
segment1
|
|
.set_payload(6, 1.into(), &payload_option1, &None, &hw_counter)
|
|
.unwrap();
|
|
segment1
|
|
.set_payload(6, 2.into(), &payload_option1, &None, &hw_counter)
|
|
.unwrap();
|
|
segment1
|
|
.set_payload(6, 3.into(), &payload_option3, &None, &hw_counter)
|
|
.unwrap();
|
|
segment1
|
|
.set_payload(6, 4.into(), &payload_option2, &None, &hw_counter)
|
|
.unwrap();
|
|
segment1
|
|
.set_payload(6, 5.into(), &payload_option2, &None, &hw_counter)
|
|
.unwrap();
|
|
|
|
segment1
|
|
}
|
|
|
|
pub fn build_segment_sparse_2(path: &Path) -> Segment {
|
|
let (mut segment2, _) = build_segment(
|
|
path,
|
|
&SegmentConfig {
|
|
vector_data: Default::default(),
|
|
sparse_vector_data: HashMap::from([(
|
|
SPARSE_VECTOR_NAME.to_owned(),
|
|
SparseVectorDataConfig {
|
|
index: SparseIndexConfig::new(None, SparseIndexType::MutableRam, None),
|
|
storage_type: SparseVectorStorageType::default(),
|
|
modifier: None,
|
|
},
|
|
)]),
|
|
payload_storage_type: Default::default(),
|
|
},
|
|
None,
|
|
true,
|
|
)
|
|
.unwrap();
|
|
|
|
let hw_counter = HardwareCounterCell::new();
|
|
|
|
let vec1 = SparseVector::new(vec![0, 1, 2, 3], vec![-1.0, 0.0, 1.0, 1.0]).unwrap();
|
|
let vec2 = SparseVector::new(vec![0, 1, 2, 3], vec![-1.0, 0.0, 1.0, 0.0]).unwrap();
|
|
let vec3 = SparseVector::new(vec![0, 1, 2, 3], vec![-1.0, 1.0, 1.0, 1.0]).unwrap();
|
|
let vec4 = SparseVector::new(vec![0, 1, 2, 3], vec![-1.0, 1.0, 0.0, 1.0]).unwrap();
|
|
let vec5 = SparseVector::new(vec![0, 1, 2, 3], vec![-1.0, 0.0, 0.0, 0.0]).unwrap();
|
|
|
|
segment2
|
|
.upsert_point(
|
|
11,
|
|
11.into(),
|
|
NamedVectors::from_ref(SPARSE_VECTOR_NAME, VectorRef::Sparse(&vec1)),
|
|
&hw_counter,
|
|
)
|
|
.unwrap();
|
|
segment2
|
|
.upsert_point(
|
|
12,
|
|
12.into(),
|
|
NamedVectors::from_ref(SPARSE_VECTOR_NAME, VectorRef::Sparse(&vec2)),
|
|
&hw_counter,
|
|
)
|
|
.unwrap();
|
|
segment2
|
|
.upsert_point(
|
|
13,
|
|
13.into(),
|
|
NamedVectors::from_ref(SPARSE_VECTOR_NAME, VectorRef::Sparse(&vec3)),
|
|
&hw_counter,
|
|
)
|
|
.unwrap();
|
|
segment2
|
|
.upsert_point(
|
|
14,
|
|
14.into(),
|
|
NamedVectors::from_ref(SPARSE_VECTOR_NAME, VectorRef::Sparse(&vec4)),
|
|
&hw_counter,
|
|
)
|
|
.unwrap();
|
|
segment2
|
|
.upsert_point(
|
|
15,
|
|
15.into(),
|
|
NamedVectors::from_ref(SPARSE_VECTOR_NAME, VectorRef::Sparse(&vec5)),
|
|
&hw_counter,
|
|
)
|
|
.unwrap();
|
|
|
|
let payload_key = PAYLOAD_KEY;
|
|
|
|
let payload_option1 = payload_json! {payload_key: vec!["red".to_owned()]};
|
|
let payload_option2 = payload_json! {payload_key: vec!["red".to_owned(), "blue".to_owned()]};
|
|
let payload_option3 = payload_json! {payload_key: vec!["blue".to_owned()]};
|
|
|
|
segment2
|
|
.set_payload(16, 11.into(), &payload_option1, &None, &hw_counter)
|
|
.unwrap();
|
|
segment2
|
|
.set_payload(16, 12.into(), &payload_option1, &None, &hw_counter)
|
|
.unwrap();
|
|
segment2
|
|
.set_payload(16, 13.into(), &payload_option3, &None, &hw_counter)
|
|
.unwrap();
|
|
segment2
|
|
.set_payload(16, 14.into(), &payload_option2, &None, &hw_counter)
|
|
.unwrap();
|
|
segment2
|
|
.set_payload(16, 15.into(), &payload_option2, &None, &hw_counter)
|
|
.unwrap();
|
|
|
|
segment2
|
|
}
|