upstream-archive/qdrant
1
0
Fork 0
mirror of https://github.com/qdrant/qdrant.git synced 2026-09-21 13:37:46 -05:00
Code Issues Packages Projects Releases Wiki Activity

crud named vectors (#8605)

Browse Source 
* Add empty placeholder vector storage types for named vector CRUD

Introduce EmptyDenseVectorStorage and EmptySparseVectorStorage as
placeholder storages for newly created named vectors on immutable
segments. These report all vectors as deleted, consume no disk space,
and are reconstructed from segment config on load via the new
VectorStorageType::Empty and SparseVectorStorageType::Empty variants.

Key design decisions:
- is_on_disk is derived from original user config, not hardcoded
- MultiVectorConfig is preserved for multi-vector support
- Config mismatch optimizer skips Empty storage to avoid false rebuilds
- Quantization delegates normally (handles 0 vectors gracefully)
- get_vector includes debug_assert to catch unexpected access

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* [AI] segment-level operations for creating and deleting anmed vectors

* [AI] implement named vector creation and deleting in proxy segment

* [AI] Step 3: Proxy Segment Handling for Named Vector Operations

* [AI] implement for Edge

* [AI] implement consensus operations for named vector operations

* [AI] refactor VectorNameConfig, remove VectorNameConfigInternal

* [AI] handle vector schema inconsistency in raft snapshot recovery

* [AI] rest + grpc API

* [AI] clippy

* [AI] generate openAPI schema

* fmt

* ci fixes

* [AI] fix jwt access test

* [AI] nop operation for awaiting of consensus-commited update ops

* [AI] move vector name operations into points service

* [AI] implement internal api for vector name operations

* [AI] change collection-level config along with segment level operation

* [AI] vector schema reconceliation instead of error

* fmt

* missing compile-time option

* [AI] integration test

* [AI] fix missing JWT tests

* [AI] remove NOP

* [AI] openapi test

* [AI] fix initialization of mutable segment

* [AI] more simple integration tests

* fmt

* [AI] make cluster test a bit harder

* [AI] make test less flacky

* [AI] rabbit comments

* [AI] check params compatibility before writing vector config

* [AI] make sure to register vector storages in structure payload index

* [AI] vector name validation

* lower vector length validation to 200 chars to account for prefix in filename

* [AI] proxy segment: prevent stale data leak through optimization

* fmt

* [AI] filter out removed vectors from proxy response

* [AI] handle vector name in proxy

* fmt

* adjust proxy info based on dropped vectors

* [AI] proxy segment: update filters to correct has_vector condition

* fmt

* clippy

* Fix consensus snapshot applicaiton for vector schema

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Andrey Vasnetsov
2026-04-10 14:45:18 +02:00
committed by GitHub
co-authored by Claude Opus 4.6
parent b8a78c18eb
commit acfb6503b1
82 changed files with 5894 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/redoc/master/openapi.json
+358
View File
@@ -1777,6 +1777,246 @@
}
}
},
"/collections/{collection_name}/vectors/{vector_name}": {
"put": {
"tags": [
"Collections"
],
"summary": "Create named vector",
"description": "Create a new named vector on an existing collection",
"operationId": "create_vector_name",
"parameters": [
{
"name": "collection_name",
"in": "path",
"description": "Name of the collection",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "vector_name",
"in": "path",
"description": "Name of the vector to create",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "wait",
"in": "query",
"description": "If true, wait for changes to actually happen",
"required": false,
"schema": {
"type": "boolean"
}
},
{
"name": "ordering",
"in": "query",
"description": "define ordering guarantees for the operation",
"required": false,
"schema": {
"$ref": "#/components/schemas/WriteOrdering"
}
},
{
"name": "timeout",
"in": "query",
"description": "Timeout for the operation",
"required": false,
"schema": {
"type": "integer",
"minimum": 1
}
}
],
"requestBody": {
"description": "Vector configuration - dense or sparse",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/VectorNameConfig"
}
}
}
},
"responses": {
"default": {
"description": "error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
},
"4XX": {
"description": "error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
},
"200": {
"description": "successful operation",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"usage": {
"default": null,
"anyOf": [
{
"$ref": "#/components/schemas/Usage"
},
{
"nullable": true
}
]
},
"time": {
"type": "number",
"format": "float",
"description": "Time spent to process this request",
"example": 0.002
},
"status": {
"type": "string",
"example": "ok"
},
"result": {
"type": "boolean"
}
}
}
}
}
}
}
},
"delete": {
"tags": [
"Collections"
],
"summary": "Delete named vector",
"description": "Delete a named vector from a collection",
"operationId": "delete_vector_name",
"parameters": [
{
"name": "collection_name",
"in": "path",
"description": "Name of the collection",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "vector_name",
"in": "path",
"description": "Name of the vector to delete",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "wait",
"in": "query",
"description": "If true, wait for changes to actually happen",
"required": false,
"schema": {
"type": "boolean"
}
},
{
"name": "ordering",
"in": "query",
"description": "define ordering guarantees for the operation",
"required": false,
"schema": {
"$ref": "#/components/schemas/WriteOrdering"
}
},
{
"name": "timeout",
"in": "query",
"description": "Timeout for the operation",
"required": false,
"schema": {
"type": "integer",
"minimum": 1
}
}
],
"responses": {
"default": {
"description": "error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
},
"4XX": {
"description": "error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ErrorResponse"
}
}
}
},
"200": {
"description": "successful operation",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"usage": {
"default": null,
"anyOf": [
{
"$ref": "#/components/schemas/Usage"
},
{
"nullable": true
}
]
},
"time": {
"type": "number",
"format": "float",
"description": "Time spent to process this request",
"example": 0.002
},
"status": {
"type": "string",
"example": "ok"
},
"result": {
"type": "boolean"
}
}
}
}
}
}
}
}
},
"/collections/{collection_name}/cluster": {
"get": {
"tags": [
@@ -13058,6 +13298,13 @@
"enum": [
"InRamMmap"
]
},
{
"description": "Placeholder storage: contains no data, all vectors reported as deleted. Used for newly created named vectors on immutable segments. No files on disk, reconstructed from config on load.",
"type": "string",
"enum": [
"Empty"
]
}
]
},
@@ -13203,6 +13450,13 @@
"enum": [
"mmap"
]
},
{
"description": "Placeholder storage: contains no data, all vectors reported as deleted. Used for newly created sparse named vectors on immutable segments.",
"type": "string",
"enum": [
"empty"
]
}
]
},
@@ -17334,6 +17588,110 @@
"$ref": "#/components/schemas/ConsensusThreadStatus"
}
}
},
"VectorNameConfig": {
"description": "Configuration for creating a new named vector.\n\nContains only the immutable properties that define a vector space. Storage type, index, and quantization are determined automatically based on the segment type and can be configured separately later.\n\nExample JSON for a dense vector: ```json { \"dense\": { \"size\": 768, \"distance\": \"Cosine\" } } ```\n\nExample JSON for a sparse vector: ```json { \"sparse\": { \"modifier\": \"Idf\" } } ```",
"anyOf": [
{
"$ref": "#/components/schemas/DenseVectorNameConfig"
},
{
"$ref": "#/components/schemas/SparseVectorNameConfig"
}
]
},
"DenseVectorNameConfig": {
"description": "Wrapper for dense vector creation config.",
"type": "object",
"required": [
"dense"
],
"properties": {
"dense": {
"$ref": "#/components/schemas/DenseVectorConfig"
}
}
},
"DenseVectorConfig": {
"description": "Configuration for creating a new dense named vector.\n\nOnly includes properties that define the vector space and cannot be changed after creation. Storage type, index type, and quantization are inferred.",
"type": "object",
"required": [
"distance",
"size"
],
"properties": {
"size": {
"description": "Dimensionality of the vectors",
"type": "integer",
"format": "uint",
"minimum": 0
},
"distance": {
"$ref": "#/components/schemas/Distance"
},
"multivector_config": {
"description": "Configuration for multi-vector points (e.g., ColBERT)",
"anyOf": [
{
"$ref": "#/components/schemas/MultiVectorConfig"
},
{
"nullable": true
}
]
},
"datatype": {
"description": "Element storage type (Float32, Float16, Uint8)",
"anyOf": [
{
"$ref": "#/components/schemas/VectorStorageDatatype"
},
{
"nullable": true
}
]
}
}
},
"SparseVectorNameConfig": {
"description": "Wrapper for sparse vector creation config.",
"type": "object",
"required": [
"sparse"
],
"properties": {
"sparse": {
"$ref": "#/components/schemas/SparseVectorConfig"
}
}
},
"SparseVectorConfig": {
"description": "Configuration for creating a new sparse named vector.\n\nOnly includes properties that define the vector space and cannot be changed after creation.",
"type": "object",
"properties": {
"modifier": {
"description": "Value modifier for sparse vectors (e.g., IDF)",
"anyOf": [
{
"$ref": "#/components/schemas/Modifier"
},
{
"nullable": true
}
]
},
"datatype": {
"description": "Datatype used to store weights in the index",
"anyOf": [
{
"$ref": "#/components/schemas/VectorStorageDatatype"
},
{
"nullable": true
}
]
}
}
}
}
}
lib/api/build.rs
+4
View File
@@ -227,6 +227,8 @@ fn configure_validation(builder: Builder) -> Builder {
("PayloadIndexParams.index_params", ""),
("DeleteFieldIndexCollection.collection_name", "length(min = 1, max = 255), custom(function = \"common::validation::validate_collection_name_legacy\")"),
("DeleteFieldIndexCollection.field_name", "length(min = 1)"),
("CreateVectorNameRequest.collection_name", "length(min = 1, max = 255), custom(function = \"common::validation::validate_collection_name_legacy\")"),
("DeleteVectorNameRequest.collection_name", "length(min = 1, max = 255), custom(function = \"common::validation::validate_collection_name_legacy\")"),
("SearchPoints.collection_name", "length(min = 1, max = 255), custom(function = \"common::validation::validate_collection_name_legacy\")"),
("SearchPoints.filter", ""),
("SearchPoints.limit", "range(min = 1)"),
@@ -368,6 +370,8 @@ fn configure_validation(builder: Builder) -> Builder {
("ClearPayloadPointsInternal.clear_payload_points", ""),
("CreateFieldIndexCollectionInternal.create_field_index_collection", ""),
("DeleteFieldIndexCollectionInternal.delete_field_index_collection", ""),
("CreateVectorNameInternal.create_vector_name", ""),
("DeleteVectorNameInternal.delete_vector_name", ""),
("UpdateOperation.update", ""),
("UpdateBatchInternal.operations", ""),
("SearchPointsInternal.search_points", ""),
lib/api/src/grpc/conversions.rs
+123 -1
View File
@@ -20,7 +20,9 @@ use segment::data_types::{facets as segment_facets, vectors as segment_vectors};
use segment::index::query_optimization::rescore_formula::parsed_formula::{
DatetimeExpression, DecayKind, ParsedExpression, ParsedFormula,
};
use segment::types::{DateTimePayloadType, FloatPayloadType, default_quantization_ignore_value};
use segment::types::{
DateTimePayloadType, FloatPayloadType, VectorStorageDatatype, default_quantization_ignore_value,
};
use segment::vector_storage::query::{self as segment_query, NaiveFeedbackCoefficients};
use sparse::common::sparse_vector::validate_sparse_vector_impl;
use tonic::Status;
@@ -3451,3 +3453,123 @@ impl From<Modifier> for grpc::Modifier {
}
}
}
impl TryFrom<grpc::create_vector_name_request::VectorConfig>
for segment::data_types::vector_name_config::VectorNameConfig
{
type Error = Status;
fn try_from(
config: grpc::create_vector_name_request::VectorConfig,
) -> Result<Self, Self::Error> {
use segment::data_types::vector_name_config::{
DenseVectorConfig, SparseVectorConfig, VectorNameConfig,
};
match config {
grpc::create_vector_name_request::VectorConfig::DenseConfig(p) => {
let grpc::DenseVectorCreationConfig {
size,
distance,
multivector_config,
datatype,
} = p;
Ok(VectorNameConfig::dense(DenseVectorConfig {
size: size as usize,
distance: from_grpc_dist(distance)?,
multivector_config: multivector_config.map(|c| c.try_into()).transpose()?,
datatype: convert_datatype_from_proto(datatype)?,
}))
}
grpc::create_vector_name_request::VectorConfig::SparseConfig(p) => {
let grpc::SparseVectorCreationConfig { modifier, datatype } = p;
Ok(VectorNameConfig::sparse(SparseVectorConfig {
modifier: modifier
.map(|m| {
grpc::Modifier::try_from(m).map_err(|_| {
Status::invalid_argument(format!(
"Cannot convert sparse modifier: {m}"
))
})
})
.transpose()?
.map(Modifier::from),
datatype: convert_datatype_from_proto(datatype)?,
}))
}
}
}
}
fn convert_datatype_from_proto(
datatype: Option<i32>,
) -> Result<Option<VectorStorageDatatype>, Status> {
let Some(dt) = datatype else {
return Ok(None);
};
let grpc_dt = grpc::Datatype::try_from(dt)
.map_err(|_| Status::invalid_argument(format!("Cannot convert datatype: {dt}")))?;
match grpc_dt {
grpc::Datatype::Default => Ok(None),
grpc::Datatype::Float32 => Ok(Some(VectorStorageDatatype::Float32)),
grpc::Datatype::Float16 => Ok(Some(VectorStorageDatatype::Float16)),
grpc::Datatype::Uint8 => Ok(Some(VectorStorageDatatype::Uint8)),
}
}
impl From<segment::data_types::vector_name_config::VectorNameConfig>
for grpc::create_vector_name_request::VectorConfig
{
fn from(config: segment::data_types::vector_name_config::VectorNameConfig) -> Self {
use segment::data_types::vector_name_config::{
DenseVectorConfig, DenseVectorNameConfig, SparseVectorConfig, SparseVectorNameConfig,
VectorNameConfig,
};
use segment::types::Distance;
match config {
VectorNameConfig::Dense(DenseVectorNameConfig {
dense:
DenseVectorConfig {
size,
distance,
multivector_config,
datatype,
},
}) => {
let distance = match distance {
Distance::Cosine => grpc::Distance::Cosine,
Distance::Euclid => grpc::Distance::Euclid,
Distance::Dot => grpc::Distance::Dot,
Distance::Manhattan => grpc::Distance::Manhattan,
};
grpc::create_vector_name_request::VectorConfig::DenseConfig(
grpc::DenseVectorCreationConfig {
size: size as u64,
distance: i32::from(distance),
multivector_config: multivector_config.map(grpc::MultiVectorConfig::from),
datatype: datatype.map(|dt| i32::from(datatype_to_grpc(dt))),
},
)
}
VectorNameConfig::Sparse(SparseVectorNameConfig {
sparse: SparseVectorConfig { modifier, datatype },
}) => grpc::create_vector_name_request::VectorConfig::SparseConfig(
grpc::SparseVectorCreationConfig {
modifier: modifier.map(|m| i32::from(grpc::Modifier::from(m))),
datatype: datatype.map(|dt| i32::from(datatype_to_grpc(dt))),
},
),
}
}
}
fn datatype_to_grpc(dt: VectorStorageDatatype) -> grpc::Datatype {
match dt {
VectorStorageDatatype::Float32 => grpc::Datatype::Float32,
VectorStorageDatatype::Float16 => grpc::Datatype::Float16,
VectorStorageDatatype::Uint8 => grpc::Datatype::Uint8,
}
}
lib/api/src/grpc/proto/points.proto
+52
View File
@@ -352,6 +352,58 @@ message DeleteFieldIndexCollection {
optional uint64 timeout = 5;
}
// Dense vector creation parameters.
// Only includes immutable properties that define the vector space.
// Storage type, index, and quantization are configured separately.
message DenseVectorCreationConfig {
// Size/dimensionality of the vectors
uint64 size = 1;
// Distance function used for comparing vectors
Distance distance = 2;
// Configuration for multi-vector search (e.g., ColBERT)
optional MultiVectorConfig multivector_config = 3;
// Data type of the vectors (Float32, Float16, Uint8)
optional Datatype datatype = 4;
}
// Sparse vector creation parameters.
// Only includes immutable properties that define the vector space.
message SparseVectorCreationConfig {
// If set - apply modifier to the vector values (e.g., IDF)
optional Modifier modifier = 1;
// Data type used to store weights in the index
optional Datatype datatype = 2;
}
message CreateVectorNameRequest {
// Name of the collection
string collection_name = 1;
// Wait until the changes have been applied?
optional bool wait = 2;
// Name of the new vector
string vector_name = 3;
// Configuration for the new vector - either dense or sparse
oneof vector_config {
// Dense vector parameters
DenseVectorCreationConfig dense_config = 4;
// Sparse vector parameters
SparseVectorCreationConfig sparse_config = 5;
}
// If set, overrides global timeout setting for this request. Unit is seconds.
optional uint64 timeout = 6;
}
message DeleteVectorNameRequest {
// Name of the collection
string collection_name = 1;
// Wait until the changes have been applied?
optional bool wait = 2;
// Name of the vector to delete
string vector_name = 3;
// If set, overrides global timeout setting for this request. Unit is seconds.
optional uint64 timeout = 4;
}
message PayloadIncludeSelector {
// List of payload keys to include into result
repeated string fields = 1;
lib/api/src/grpc/proto/points_internal_service.proto
+22
View File
@@ -26,6 +26,12 @@ service PointsInternal {
returns (PointsOperationResponseInternal) {}
rpc DeleteFieldIndex(DeleteFieldIndexCollectionInternal)
returns (PointsOperationResponseInternal) {}
rpc CreateVectorName(CreateVectorNameInternal)
returns (PointsOperationResponseInternal) {}
rpc DeleteVectorName(DeleteVectorNameInternal)
returns (PointsOperationResponseInternal) {}
rpc UpdateBatch(UpdateBatchInternal)
returns (PointsOperationResponseInternal) {}
rpc CoreSearchBatch(CoreSearchBatchPointsInternal)
@@ -144,6 +150,20 @@ message DeleteFieldIndexCollectionInternal {
optional WaitUntil wait_override = 4;
}
message CreateVectorNameInternal {
CreateVectorNameRequest create_vector_name = 1;
optional uint32 shard_id = 2;
optional ClockTag clock_tag = 3;
optional WaitUntil wait_override = 4;
}
message DeleteVectorNameInternal {
DeleteVectorNameRequest delete_vector_name = 1;
optional uint32 shard_id = 2;
optional ClockTag clock_tag = 3;
optional WaitUntil wait_override = 4;
}
message UpdateOperation {
oneof update {
SyncPointsInternal sync = 1;
@@ -157,6 +177,8 @@ message UpdateOperation {
ClearPayloadPointsInternal clear_payload = 9;
CreateFieldIndexCollectionInternal create_field_index = 10;
DeleteFieldIndexCollectionInternal delete_field_index = 11;
CreateVectorNameInternal create_vector_name = 12;
DeleteVectorNameInternal delete_vector_name = 13;
}
}
lib/api/src/grpc/proto/points_service.proto
+4
View File
@@ -31,6 +31,10 @@ service Points {
// Delete field index for collection
rpc DeleteFieldIndex(DeleteFieldIndexCollection)
returns (PointsOperationResponse) {}
// Create a new named vector on the collection
rpc CreateVectorName(CreateVectorNameRequest) returns (PointsOperationResponse) {}
// Delete a named vector from the collection
rpc DeleteVectorName(DeleteVectorNameRequest) returns (PointsOperationResponse) {}
// Retrieve closest points based on vector similarity and given filtering
// conditions
rpc Search(SearchPoints) returns (SearchResponse) {}
lib/api/src/grpc/qdrant.rs
+448 -1
View File
@@ -5399,6 +5399,101 @@ pub struct DeleteFieldIndexCollection {
#[prost(uint64, optional, tag = "5")]
pub timeout: ::core::option::Option<u64>,
}
/// Dense vector creation parameters.
/// Only includes immutable properties that define the vector space.
/// Storage type, index, and quantization are configured separately.
#[derive(serde::Serialize)]
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct DenseVectorCreationConfig {
/// Size/dimensionality of the vectors
#[prost(uint64, tag = "1")]
pub size: u64,
/// Distance function used for comparing vectors
#[prost(enumeration = "Distance", tag = "2")]
pub distance: i32,
/// Configuration for multi-vector search (e.g., ColBERT)
#[prost(message, optional, tag = "3")]
pub multivector_config: ::core::option::Option<MultiVectorConfig>,
/// Data type of the vectors (Float32, Float16, Uint8)
#[prost(enumeration = "Datatype", optional, tag = "4")]
pub datatype: ::core::option::Option<i32>,
}
/// Sparse vector creation parameters.
/// Only includes immutable properties that define the vector space.
#[derive(serde::Serialize)]
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct SparseVectorCreationConfig {
/// If set - apply modifier to the vector values (e.g., IDF)
#[prost(enumeration = "Modifier", optional, tag = "1")]
pub modifier: ::core::option::Option<i32>,
/// Data type used to store weights in the index
#[prost(enumeration = "Datatype", optional, tag = "2")]
pub datatype: ::core::option::Option<i32>,
}
#[derive(validator::Validate)]
#[derive(serde::Serialize)]
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct CreateVectorNameRequest {
/// Name of the collection
#[prost(string, tag = "1")]
#[validate(
length(min = 1, max = 255),
custom(function = "common::validation::validate_collection_name_legacy")
)]
pub collection_name: ::prost::alloc::string::String,
/// Wait until the changes have been applied?
#[prost(bool, optional, tag = "2")]
pub wait: ::core::option::Option<bool>,
/// Name of the new vector
#[prost(string, tag = "3")]
pub vector_name: ::prost::alloc::string::String,
/// If set, overrides global timeout setting for this request. Unit is seconds.
#[prost(uint64, optional, tag = "6")]
pub timeout: ::core::option::Option<u64>,
/// Configuration for the new vector - either dense or sparse
#[prost(oneof = "create_vector_name_request::VectorConfig", tags = "4, 5")]
pub vector_config: ::core::option::Option<create_vector_name_request::VectorConfig>,
}
/// Nested message and enum types in `CreateVectorNameRequest`.
pub mod create_vector_name_request {
/// Configuration for the new vector - either dense or sparse
#[derive(serde::Serialize)]
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum VectorConfig {
/// Dense vector parameters
#[prost(message, tag = "4")]
DenseConfig(super::DenseVectorCreationConfig),
/// Sparse vector parameters
#[prost(message, tag = "5")]
SparseConfig(super::SparseVectorCreationConfig),
}
}
#[derive(validator::Validate)]
#[derive(serde::Serialize)]
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct DeleteVectorNameRequest {
/// Name of the collection
#[prost(string, tag = "1")]
#[validate(
length(min = 1, max = 255),
custom(function = "common::validation::validate_collection_name_legacy")
)]
pub collection_name: ::prost::alloc::string::String,
/// Wait until the changes have been applied?
#[prost(bool, optional, tag = "2")]
pub wait: ::core::option::Option<bool>,
/// Name of the vector to delete
#[prost(string, tag = "3")]
pub vector_name: ::prost::alloc::string::String,
/// If set, overrides global timeout setting for this request. Unit is seconds.
#[prost(uint64, optional, tag = "4")]
pub timeout: ::core::option::Option<u64>,
}
#[derive(serde::Serialize)]
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
@@ -8209,6 +8304,58 @@ pub mod points_client {
.insert(GrpcMethod::new("qdrant.Points", "DeleteFieldIndex"));
self.inner.unary(req, path, codec).await
}
/// Create a new named vector on the collection
pub async fn create_vector_name(
&mut self,
request: impl tonic::IntoRequest<super::CreateVectorNameRequest>,
) -> std::result::Result<
tonic::Response<super::PointsOperationResponse>,
tonic::Status,
> {
self.inner
.ready()
.await
.map_err(|e| {
tonic::Status::new(
tonic::Code::Unknown,
format!("Service was not ready: {}", e.into()),
)
})?;
let codec = tonic::codec::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/qdrant.Points/CreateVectorName",
);
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("qdrant.Points", "CreateVectorName"));
self.inner.unary(req, path, codec).await
}
/// Delete a named vector from the collection
pub async fn delete_vector_name(
&mut self,
request: impl tonic::IntoRequest<super::DeleteVectorNameRequest>,
) -> std::result::Result<
tonic::Response<super::PointsOperationResponse>,
tonic::Status,
> {
self.inner
.ready()
.await
.map_err(|e| {
tonic::Status::new(
tonic::Code::Unknown,
format!("Service was not ready: {}", e.into()),
)
})?;
let codec = tonic::codec::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/qdrant.Points/DeleteVectorName",
);
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("qdrant.Points", "DeleteVectorName"));
self.inner.unary(req, path, codec).await
}
/// Retrieve closest points based on vector similarity and given filtering
/// conditions
pub async fn search(
@@ -8736,6 +8883,22 @@ pub mod points_server {
tonic::Response<super::PointsOperationResponse>,
tonic::Status,
>;
/// Create a new named vector on the collection
async fn create_vector_name(
&self,
request: tonic::Request<super::CreateVectorNameRequest>,
) -> std::result::Result<
tonic::Response<super::PointsOperationResponse>,
tonic::Status,
>;
/// Delete a named vector from the collection
async fn delete_vector_name(
&self,
request: tonic::Request<super::DeleteVectorNameRequest>,
) -> std::result::Result<
tonic::Response<super::PointsOperationResponse>,
tonic::Status,
>;
/// Retrieve closest points based on vector similarity and given filtering
/// conditions
async fn search(
@@ -9465,6 +9628,98 @@ pub mod points_server {
};
Box::pin(fut)
}
"/qdrant.Points/CreateVectorName" => {
#[allow(non_camel_case_types)]
struct CreateVectorNameSvc<T: Points>(pub Arc<T>);
impl<
T: Points,
> tonic::server::UnaryService<super::CreateVectorNameRequest>
for CreateVectorNameSvc<T> {
type Response = super::PointsOperationResponse;
type Future = BoxFuture<
tonic::Response<Self::Response>,
tonic::Status,
>;
fn call(
&mut self,
request: tonic::Request<super::CreateVectorNameRequest>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as Points>::create_vector_name(&inner, request).await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let inner = inner.0;
let method = CreateVectorNameSvc(inner);
let codec = tonic::codec::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
send_compression_encodings,
)
.apply_max_message_size_config(
max_decoding_message_size,
max_encoding_message_size,
);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/qdrant.Points/DeleteVectorName" => {
#[allow(non_camel_case_types)]
struct DeleteVectorNameSvc<T: Points>(pub Arc<T>);
impl<
T: Points,
> tonic::server::UnaryService<super::DeleteVectorNameRequest>
for DeleteVectorNameSvc<T> {
type Response = super::PointsOperationResponse;
type Future = BoxFuture<
tonic::Response<Self::Response>,
tonic::Status,
>;
fn call(
&mut self,
request: tonic::Request<super::DeleteVectorNameRequest>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as Points>::delete_vector_name(&inner, request).await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let inner = inner.0;
let method = DeleteVectorNameSvc(inner);
let codec = tonic::codec::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
send_compression_encodings,
)
.apply_max_message_size_config(
max_decoding_message_size,
max_encoding_message_size,
);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/qdrant.Points/Search" => {
#[allow(non_camel_case_types)]
struct SearchSvc<T: Points>(pub Arc<T>);
@@ -10471,10 +10726,40 @@ pub struct DeleteFieldIndexCollectionInternal {
#[derive(validator::Validate)]
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct CreateVectorNameInternal {
#[prost(message, optional, tag = "1")]
#[validate(nested)]
pub create_vector_name: ::core::option::Option<CreateVectorNameRequest>,
#[prost(uint32, optional, tag = "2")]
pub shard_id: ::core::option::Option<u32>,
#[prost(message, optional, tag = "3")]
pub clock_tag: ::core::option::Option<ClockTag>,
#[prost(enumeration = "WaitUntil", optional, tag = "4")]
pub wait_override: ::core::option::Option<i32>,
}
#[derive(serde::Serialize)]
#[derive(validator::Validate)]
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct DeleteVectorNameInternal {
#[prost(message, optional, tag = "1")]
#[validate(nested)]
pub delete_vector_name: ::core::option::Option<DeleteVectorNameRequest>,
#[prost(uint32, optional, tag = "2")]
pub shard_id: ::core::option::Option<u32>,
#[prost(message, optional, tag = "3")]
pub clock_tag: ::core::option::Option<ClockTag>,
#[prost(enumeration = "WaitUntil", optional, tag = "4")]
pub wait_override: ::core::option::Option<i32>,
}
#[derive(serde::Serialize)]
#[derive(validator::Validate)]
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct UpdateOperation {
#[prost(
oneof = "update_operation::Update",
tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11"
tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13"
)]
#[validate(nested)]
pub update: ::core::option::Option<update_operation::Update>,
@@ -10507,6 +10792,10 @@ pub mod update_operation {
CreateFieldIndex(super::CreateFieldIndexCollectionInternal),
#[prost(message, tag = "11")]
DeleteFieldIndex(super::DeleteFieldIndexCollectionInternal),
#[prost(message, tag = "12")]
CreateVectorName(super::CreateVectorNameInternal),
#[prost(message, tag = "13")]
DeleteVectorName(super::DeleteVectorNameInternal),
}
}
#[derive(serde::Serialize)]
@@ -11493,6 +11782,56 @@ pub mod points_internal_client {
.insert(GrpcMethod::new("qdrant.PointsInternal", "DeleteFieldIndex"));
self.inner.unary(req, path, codec).await
}
pub async fn create_vector_name(
&mut self,
request: impl tonic::IntoRequest<super::CreateVectorNameInternal>,
) -> std::result::Result<
tonic::Response<super::PointsOperationResponseInternal>,
tonic::Status,
> {
self.inner
.ready()
.await
.map_err(|e| {
tonic::Status::new(
tonic::Code::Unknown,
format!("Service was not ready: {}", e.into()),
)
})?;
let codec = tonic::codec::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/qdrant.PointsInternal/CreateVectorName",
);
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("qdrant.PointsInternal", "CreateVectorName"));
self.inner.unary(req, path, codec).await
}
pub async fn delete_vector_name(
&mut self,
request: impl tonic::IntoRequest<super::DeleteVectorNameInternal>,
) -> std::result::Result<
tonic::Response<super::PointsOperationResponseInternal>,
tonic::Status,
> {
self.inner
.ready()
.await
.map_err(|e| {
tonic::Status::new(
tonic::Code::Unknown,
format!("Service was not ready: {}", e.into()),
)
})?;
let codec = tonic::codec::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/qdrant.PointsInternal/DeleteVectorName",
);
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("qdrant.PointsInternal", "DeleteVectorName"));
self.inner.unary(req, path, codec).await
}
pub async fn update_batch(
&mut self,
request: impl tonic::IntoRequest<super::UpdateBatchInternal>,
@@ -11769,6 +12108,20 @@ pub mod points_internal_server {
tonic::Response<super::PointsOperationResponseInternal>,
tonic::Status,
>;
async fn create_vector_name(
&self,
request: tonic::Request<super::CreateVectorNameInternal>,
) -> std::result::Result<
tonic::Response<super::PointsOperationResponseInternal>,
tonic::Status,
>;
async fn delete_vector_name(
&self,
request: tonic::Request<super::DeleteVectorNameInternal>,
) -> std::result::Result<
tonic::Response<super::PointsOperationResponseInternal>,
tonic::Status,
>;
async fn update_batch(
&self,
request: tonic::Request<super::UpdateBatchInternal>,
@@ -12411,6 +12764,100 @@ pub mod points_internal_server {
};
Box::pin(fut)
}
"/qdrant.PointsInternal/CreateVectorName" => {
#[allow(non_camel_case_types)]
struct CreateVectorNameSvc<T: PointsInternal>(pub Arc<T>);
impl<
T: PointsInternal,
> tonic::server::UnaryService<super::CreateVectorNameInternal>
for CreateVectorNameSvc<T> {
type Response = super::PointsOperationResponseInternal;
type Future = BoxFuture<
tonic::Response<Self::Response>,
tonic::Status,
>;
fn call(
&mut self,
request: tonic::Request<super::CreateVectorNameInternal>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as PointsInternal>::create_vector_name(&inner, request)
.await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let inner = inner.0;
let method = CreateVectorNameSvc(inner);
let codec = tonic::codec::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
send_compression_encodings,
)
.apply_max_message_size_config(
max_decoding_message_size,
max_encoding_message_size,
);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/qdrant.PointsInternal/DeleteVectorName" => {
#[allow(non_camel_case_types)]
struct DeleteVectorNameSvc<T: PointsInternal>(pub Arc<T>);
impl<
T: PointsInternal,
> tonic::server::UnaryService<super::DeleteVectorNameInternal>
for DeleteVectorNameSvc<T> {
type Response = super::PointsOperationResponseInternal;
type Future = BoxFuture<
tonic::Response<Self::Response>,
tonic::Status,
>;
fn call(
&mut self,
request: tonic::Request<super::DeleteVectorNameInternal>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as PointsInternal>::delete_vector_name(&inner, request)
.await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let inner = inner.0;
let method = DeleteVectorNameSvc(inner);
let codec = tonic::codec::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
send_compression_encodings,
)
.apply_max_message_size_config(
max_decoding_message_size,
max_encoding_message_size,
);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/qdrant.PointsInternal/UpdateBatch" => {
#[allow(non_camel_case_types)]
struct UpdateBatchSvc<T: PointsInternal>(pub Arc<T>);
lib/api/src/grpc/validate.rs
+2
View File
@@ -232,6 +232,8 @@ impl Validate for grpc::update_operation::Update {
Update::ClearPayload(op) => op.validate(),
Update::CreateFieldIndex(op) => op.validate(),
Update::DeleteFieldIndex(op) => op.validate(),
Update::CreateVectorName(op) => op.validate(),
Update::DeleteVectorName(op) => op.validate(),
}
}
}
lib/collection/src/collection/mod.rs
+1
View File
@@ -13,6 +13,7 @@ mod sharding_keys;
mod snapshots;
mod state_management;
mod telemetry;
mod vector_name_schema;
use std::collections::HashMap;
use std::ops::Deref;
lib/collection/src/collection/payload_index_schema.rs
+2 -1
View File
@@ -64,7 +64,7 @@ impl Collection {
}),
);
self.update_all_local(create_index_operation, WaitUntil::from(wait), hw_acc)
self.update_all_local(create_index_operation, WaitUntil::from(wait), hw_acc, false)
.await
}
@@ -85,6 +85,7 @@ impl Collection {
delete_index_operation,
WaitUntil::from(false),
HwMeasurementAcc::disposable(), // Unmeasured API
false,
)
.await?;
lib/collection/src/collection/point_ops.rs
+2 -1
View File
@@ -34,6 +34,7 @@ impl Collection {
operation: CollectionUpdateOperations,
wait: WaitUntil,
hw_measurement_acc: HwMeasurementAcc,
force: bool,
) -> CollectionResult<Option<UpdateResult>> {
let shard_holder = self.shards_holder.clone().read_owned().await;
@@ -59,7 +60,7 @@ impl Collection {
wait,
None,
hw_measurement_acc.clone(),
false,
force,
)
})
.collect();
lib/collection/src/collection/state_management.rs
+141 -1
View File
@@ -8,7 +8,7 @@ use futures::stream::FuturesUnordered;
use crate::collection::Collection;
use crate::collection::payload_index_schema::PayloadIndexSchema;
use crate::collection_state::{ShardInfo, State};
use crate::config::CollectionConfigInternal;
use crate::config::{CollectionConfigInternal, CollectionParams};
use crate::operations::types::{CollectionError, CollectionResult};
use crate::shards::replica_set::ShardReplicaSet;
use crate::shards::resharding::ReshardState;
@@ -44,6 +44,11 @@ impl Collection {
payload_index_schema,
} = state;
// Used to detect which named vectors have changed after applying new config
let old_collection_config = self.collection_config.read().await.params.clone();
// Apply config first — this updates the collection-level vector definitions
let new_config = config.clone();
self.apply_config(config).await?;
self.apply_shard_transfers(transfers, this_peer_id, abort_transfer)
.await?;
@@ -51,6 +56,10 @@ impl Collection {
self.apply_shard_info(shards, shards_key_mapping).await?;
self.apply_payload_index_schema(payload_index_schema)
.await?;
// Reconcile named vectors at the segment level to match the new config.
// This ensures segments have the correct vector storages after a Raft snapshot.
self.apply_vector_name_schema(old_collection_config, &new_config)
.await?;
Ok(())
}
@@ -255,6 +264,137 @@ impl Collection {
Ok(())
}
/// Reconcile named vectors at the segment level to match the given config.
///
/// This is called during state application (Raft snapshot recovery) to ensure
/// that segments have the correct vector storages, even if the node missed the
/// original create/delete vector name operations.
async fn apply_vector_name_schema(
&self,
old_collection_params: CollectionParams,
target_config: &CollectionConfigInternal,
) -> CollectionResult<()> {
use segment::data_types::vector_name_config::{
DenseVectorConfig, SparseVectorConfig, VectorNameConfig,
};
use segment::types::VectorStorageDatatype;
let mut to_create: Vec<(segment::types::VectorNameBuf, VectorNameConfig)> = Vec::new();
let mut to_delete: Vec<segment::types::VectorNameBuf> = Vec::new();
// Dense vectors: compare target vs current
for (vector_name, target_params) in target_config.params.vectors.params_iter() {
let target_dense = DenseVectorConfig {
size: target_params.size.get() as usize,
distance: target_params.distance,
multivector_config: target_params.multivector_config,
datatype: target_params.datatype.map(VectorStorageDatatype::from),
};
match old_collection_params.vectors.get_params(vector_name) {
None => {
// New vector — create it
to_create.push((
vector_name.to_owned(),
VectorNameConfig::dense(target_dense),
));
}
Some(current_params) => {
// Exists in both — check if parameters changed (delete+recreate)
let current_dense = DenseVectorConfig {
size: current_params.size.get() as usize,
distance: current_params.distance,
multivector_config: current_params.multivector_config,
datatype: current_params.datatype.map(VectorStorageDatatype::from),
};
if current_dense != target_dense {
to_delete.push(vector_name.to_owned());
to_create.push((
vector_name.to_owned(),
VectorNameConfig::dense(target_dense),
));
}
}
}
}
// Dense vectors: delete those in current but not in target
for (vector_name, _) in old_collection_params.vectors.params_iter() {
if target_config
.params
.vectors
.get_params(vector_name)
.is_none()
{
to_delete.push(vector_name.to_owned());
}
}
// Sparse vectors: compare target vs current
if let Some(target_sparse) = &target_config.params.sparse_vectors {
let current_sparse = old_collection_params.sparse_vectors.as_ref();
for (vector_name, target_params) in target_sparse {
let target_sparse_cfg = SparseVectorConfig {
modifier: target_params.modifier,
datatype: target_params
.index
.as_ref()
.and_then(|idx| idx.datatype)
.map(VectorStorageDatatype::from),
};
let current_exists = current_sparse.and_then(|c| c.get(vector_name));
match current_exists {
None => {
to_create.push((
vector_name.clone(),
VectorNameConfig::sparse(target_sparse_cfg),
));
}
Some(current_params) => {
let current_sparse_cfg = SparseVectorConfig {
modifier: current_params.modifier,
datatype: current_params
.index
.as_ref()
.and_then(|idx| idx.datatype)
.map(VectorStorageDatatype::from),
};
if current_sparse_cfg != target_sparse_cfg {
to_delete.push(vector_name.clone());
to_create.push((
vector_name.clone(),
VectorNameConfig::sparse(target_sparse_cfg),
));
}
}
}
}
}
// Sparse vectors: delete those in current but not in target
if let Some(current_sparse) = &old_collection_params.sparse_vectors {
let target_sparse = target_config.params.sparse_vectors.as_ref();
for vector_name in current_sparse.keys() {
if !target_sparse.is_some_and(|t| t.contains_key(vector_name)) {
to_delete.push(vector_name.clone());
}
}
}
// Delete first (includes changed vectors), then create
for vector_name in to_delete {
self.delete_named_vector(vector_name).await?;
}
for (vector_name, config) in to_create {
self.create_named_vector(vector_name, config, HwMeasurementAcc::disposable())
.await?;
}
Ok(())
}
/// Truncate unapplied WAL records for all local shards in the collection.
/// Returns amount of removed records.
pub async fn truncate_unapplied_wal(&self) -> CollectionResult<usize> {
lib/collection/src/collection/vector_name_schema.rs
+230
View File
@@ -0,0 +1,230 @@
use std::num::NonZeroU64;
use common::counter::hardware_accumulator::HwMeasurementAcc;
use segment::data_types::vector_name_config::{
DenseVectorConfig, SparseVectorConfig, VectorNameConfig,
};
use segment::types::VectorNameBuf;
use shard::operations::{
CollectionUpdateOperations, CreateVectorName, DeleteVectorName, VectorNameOperations,
};
use crate::collection::Collection;
use crate::operations::types::{
CollectionError, CollectionResult, SparseVectorParams, VectorParams,
};
use crate::shards::shard_trait::WaitUntil;
impl Collection {
pub async fn create_named_vector(
&self,
vector_name: VectorNameBuf,
config: VectorNameConfig,
hw_acc: HwMeasurementAcc,
) -> CollectionResult<()> {
self.update_collection_vector_config(|params| {
add_vector_to_config(params, &vector_name, &config)
})
.await?;
let operation = CollectionUpdateOperations::VectorNameOperation(
VectorNameOperations::CreateVectorName(CreateVectorName {
vector_name,
config,
}),
);
self.update_all_local(operation, WaitUntil::from(false), hw_acc, true)
.await?;
Ok(())
}
pub async fn delete_named_vector(&self, vector_name: VectorNameBuf) -> CollectionResult<()> {
self.update_collection_vector_config(|params| {
remove_vector_from_config(params, &vector_name);
Ok(())
})
.await?;
let operation = CollectionUpdateOperations::VectorNameOperation(
VectorNameOperations::DeleteVectorName(DeleteVectorName { vector_name }),
);
self.update_all_local(
operation,
WaitUntil::from(true),
HwMeasurementAcc::disposable(),
true, // Delete even in dead shards
)
.await?;
Ok(())
}
/// Apply a mutation to collection params and persist.
async fn update_collection_vector_config(
&self,
mutate: impl FnOnce(&mut crate::config::CollectionParams) -> CollectionResult<()>,
) -> CollectionResult<()> {
let mut config = self.collection_config.write().await;
mutate(&mut config.params)?;
config.save(&self.path)?;
Ok(())
}
}
fn dense_config_to_params(config: &DenseVectorConfig) -> VectorParams {
let DenseVectorConfig {
size,
distance,
multivector_config,
datatype,
} = config;
VectorParams {
size: NonZeroU64::new(*size as u64).unwrap_or(NonZeroU64::MIN),
distance: *distance,
hnsw_config: None,
quantization_config: None,
on_disk: None,
datatype: datatype.map(storage_datatype_to_collection),
multivector_config: *multivector_config,
}
}
fn sparse_config_to_params(config: &SparseVectorConfig) -> SparseVectorParams {
let SparseVectorConfig { modifier, datatype } = config;
SparseVectorParams {
index: datatype.map(|dt| crate::operations::types::SparseIndexParams {
full_scan_threshold: None,
on_disk: None,
datatype: Some(storage_datatype_to_collection(dt)),
}),
modifier: *modifier,
}
}
/// Insert a new named vector into the stored collection params.
///
/// Returns `Ok(true)` if the vector was newly added, `Ok(false)` if a vector with the
/// same name and an identical config already exists (idempotent no-op), or
/// `Err(BadInput)` if a vector with the same name exists with a conflicting config or
/// a conflicting kind (dense vs. sparse). This guards against silently overwriting the
/// stored schema with a different one while shards keep the old one — shard-level
/// `CreateVectorName` is idempotent and will not re-apply the new config.
fn add_vector_to_config(
params: &mut crate::config::CollectionParams,
vector_name: &VectorNameBuf,
config: &VectorNameConfig,
) -> CollectionResult<()> {
match config {
VectorNameConfig::Dense(wrapper) => {
if let Some(existing) = params.vectors.get_params(vector_name.as_str()) {
if !dense_schema_matches(existing, &wrapper.dense) {
return Err(CollectionError::bad_input(format!(
"Vector `{vector_name}` already exists with a different configuration",
)));
}
return Ok(());
}
if params
.sparse_vectors
.as_ref()
.is_some_and(|sv| sv.contains_key(vector_name))
{
return Err(CollectionError::bad_input(format!(
"Vector `{vector_name}` already exists as a sparse vector",
)));
}
params
.vectors
.insert(vector_name.clone(), dense_config_to_params(&wrapper.dense));
Ok(())
}
VectorNameConfig::Sparse(wrapper) => {
if let Some(existing) = params
.sparse_vectors
.as_ref()
.and_then(|sv| sv.get(vector_name))
{
if !sparse_schema_matches(existing, &wrapper.sparse) {
return Err(CollectionError::bad_input(format!(
"Sparse vector `{vector_name}` already exists with a different configuration",
)));
}
return Ok(());
}
if params.vectors.get_params(vector_name.as_str()).is_some() {
return Err(CollectionError::bad_input(format!(
"Vector `{vector_name}` already exists as a dense vector",
)));
}
params
.sparse_vectors
.get_or_insert_with(Default::default)
.insert(
vector_name.clone(),
sparse_config_to_params(&wrapper.sparse),
);
Ok(())
}
}
}
/// Compare the schema-defining subset of `VectorParams` against a `DenseVectorConfig`.
///
/// `VectorParams` carries fields that can be tuned independently after creation
/// (`hnsw_config`, `quantization_config`, `on_disk`); those must be ignored here so
/// repeated `create_named_vector` calls don't spuriously conflict with a vector whose
/// HNSW/quantization/on_disk settings were updated since.
fn dense_schema_matches(existing: &VectorParams, config: &DenseVectorConfig) -> bool {
let DenseVectorConfig {
size,
distance,
multivector_config,
datatype,
} = config;
existing.size.get() == *size as u64
&& existing.distance == *distance
&& existing.multivector_config == *multivector_config
&& existing.datatype == datatype.map(storage_datatype_to_collection)
}
/// Compare the schema-defining subset of `SparseVectorParams` against a
/// `SparseVectorConfig`. Index tuning fields (`full_scan_threshold`, `on_disk`) are
/// ignored for the same reason as in [`dense_schema_matches`].
fn sparse_schema_matches(existing: &SparseVectorParams, config: &SparseVectorConfig) -> bool {
let SparseVectorConfig { modifier, datatype } = config;
let existing_datatype = existing.index.as_ref().and_then(|index| index.datatype);
existing.modifier == *modifier
&& existing_datatype == datatype.map(storage_datatype_to_collection)
}
fn remove_vector_from_config(
params: &mut crate::config::CollectionParams,
vector_name: &VectorNameBuf,
) {
params.vectors.remove(vector_name);
if let Some(sparse) = &mut params.sparse_vectors {
sparse.remove(vector_name);
}
}
fn storage_datatype_to_collection(
dt: segment::types::VectorStorageDatatype,
) -> crate::operations::types::Datatype {
match dt {
segment::types::VectorStorageDatatype::Float32 => {
crate::operations::types::Datatype::Float32
}
segment::types::VectorStorageDatatype::Float16 => {
crate::operations::types::Datatype::Float16
}
segment::types::VectorStorageDatatype::Uint8 => crate::operations::types::Datatype::Uint8,
}
}
lib/collection/src/collection_manager/collection_updater.rs
+3
View File
@@ -83,6 +83,9 @@ impl CollectionUpdater {
hw_counter,
)
}
CollectionUpdateOperations::VectorNameOperation(vector_name_operation) => {
process_vector_name_operation(&segments_guard, op_num, &vector_name_operation)
}
#[cfg(feature = "staging")]
CollectionUpdateOperations::StagingOperation(staging_operation) => {
shard::update::process_staging_operation(
lib/collection/src/config.rs
+3 -25
View File
@@ -1,4 +1,4 @@
use std::collections::{BTreeMap, HashMap, HashSet};
use std::collections::{BTreeMap, HashMap};
use std::io::{Read, Write as _};
use std::num::{NonZeroU32, NonZeroUsize};
use std::path::Path;
@@ -146,7 +146,7 @@ impl CollectionParams {
pub fn check_compatible(&self, other: &CollectionParams) -> CollectionResult<()> {
let CollectionParams {
vectors,
vectors: _, // May be changed
shard_number: _, // Maybe be updated by resharding, assume local shards needs to be dropped
sharding_method, // Not changeable
replication_factor: _, // May be changed
@@ -154,31 +154,9 @@ impl CollectionParams {
read_fan_out_factor: _, // May be changed
read_fan_out_delay_ms: _, // May be changed,
on_disk_payload: _, // May be changed
sparse_vectors, // Parameters may be changes, but not the structure
sparse_vectors: _, // Sets may differ via named vector CRUD
} = other;
self.vectors.check_compatible(vectors)?;
let this_sparse_vectors: HashSet<_> = if let Some(sparse_vectors) = &self.sparse_vectors {
sparse_vectors.keys().collect()
} else {
HashSet::new()
};
let other_sparse_vectors: HashSet<_> = if let Some(sparse_vectors) = sparse_vectors {
sparse_vectors.keys().collect()
} else {
HashSet::new()
};
if this_sparse_vectors != other_sparse_vectors {
return Err(CollectionError::bad_input(format!(
"sparse vectors are incompatible: \
origin sparse vectors: {this_sparse_vectors:?}, \
while other sparse vectors: {other_sparse_vectors:?}",
)));
}
let this_sharding_method = self.sharding_method.unwrap_or_default();
let other_sharding_method = sharding_method.unwrap_or_default();
lib/collection/src/operations/conversions.rs
+1 -1
View File
@@ -747,7 +747,7 @@ impl TryFrom<api::grpc::qdrant::VectorParams> for VectorParams {
}
}
fn convert_datatype_from_proto(datatype: Option<i32>) -> Result<Option<Datatype>, Status> {
pub fn convert_datatype_from_proto(datatype: Option<i32>) -> Result<Option<Datatype>, Status> {
if let Some(datatype_int) = datatype {
let grpc_datatype = api::grpc::qdrant::Datatype::try_from(datatype_int);
if let Ok(grpc_datatype) = grpc_datatype {
lib/collection/src/operations/generalizer/update_persisted.rs
+10 -1
View File
@@ -8,7 +8,7 @@ use shard::operations::point_ops::{
VectorPersisted, VectorStructPersisted,
};
use shard::operations::vector_ops::{PointVectorsPersisted, UpdateVectorsOp, VectorOperations};
use shard::operations::{CollectionUpdateOperations, FieldIndexOperations};
use shard::operations::{CollectionUpdateOperations, FieldIndexOperations, VectorNameOperations};
use sparse::common::sparse_vector::SparseVector;
use sparse::common::types::DimId;
@@ -40,6 +40,9 @@ impl Generalizer for CollectionUpdateOperations {
CollectionUpdateOperations::FieldIndexOperation(field_operation) => {
CollectionUpdateOperations::FieldIndexOperation(field_operation.remove_details())
}
CollectionUpdateOperations::VectorNameOperation(op) => {
CollectionUpdateOperations::VectorNameOperation(op.remove_details())
}
#[cfg(feature = "staging")]
CollectionUpdateOperations::StagingOperation(op) => {
CollectionUpdateOperations::StagingOperation(op.clone())
@@ -302,3 +305,9 @@ impl Generalizer for FieldIndexOperations {
self.clone()
}
}
impl Generalizer for VectorNameOperations {
fn remove_details(&self) -> Self {
self.clone()
}
}
lib/collection/src/operations/mod.rs
+3
View File
@@ -53,6 +53,9 @@ impl SplitByShard for CollectionUpdateOperations {
operation @ CollectionUpdateOperations::FieldIndexOperation(_) => {
OperationToShard::to_all(operation)
}
operation @ CollectionUpdateOperations::VectorNameOperation(_) => {
OperationToShard::to_all(operation)
}
#[cfg(feature = "staging")]
operation @ CollectionUpdateOperations::StagingOperation(_) => {
OperationToShard::to_all(operation)
lib/collection/src/operations/operation_effect.rs
+1
View File
@@ -40,6 +40,7 @@ impl EstimateOperationEffectArea for CollectionUpdateOperations {
payload_operation.estimate_effect_area()
}
CollectionUpdateOperations::FieldIndexOperation(_) => OperationEffectArea::Empty,
CollectionUpdateOperations::VectorNameOperation(_) => OperationEffectArea::Empty,
#[cfg(feature = "staging")]
CollectionUpdateOperations::StagingOperation(_) => OperationEffectArea::Empty,
}
lib/collection/src/operations/types.rs
+32 -24
View File
@@ -1515,6 +1515,38 @@ impl VectorsConfig {
}
}
/// Insert or replace a named vector. Converts `Single` to `Multi` if needed.
pub fn insert(&mut self, name: VectorNameBuf, params: VectorParams) {
match self {
VectorsConfig::Single(_) => {
let mut multi = BTreeMap::new();
if let VectorsConfig::Single(existing) =
std::mem::replace(self, VectorsConfig::empty())
{
multi.insert(DEFAULT_VECTOR_NAME.to_owned(), existing);
}
multi.insert(name, params);
*self = VectorsConfig::Multi(multi);
}
VectorsConfig::Multi(vectors) => {
vectors.insert(name, params);
}
}
}
/// Remove a named vector. Converts `Single` to empty `Multi` if name matches.
pub fn remove(&mut self, name: &VectorName) {
match self {
VectorsConfig::Single(_) if name == DEFAULT_VECTOR_NAME => {
*self = VectorsConfig::Multi(Default::default());
}
VectorsConfig::Single(_) => {}
VectorsConfig::Multi(vectors) => {
vectors.remove(name);
}
}
}
pub fn get_params_mut(&mut self, name: &VectorName) -> Option<&mut VectorParams> {
match self {
VectorsConfig::Single(params) => (name == DEFAULT_VECTOR_NAME).then_some(params),
@@ -1534,30 +1566,6 @@ impl VectorsConfig {
}
}
// TODO: Further unify `check_compatible` and `check_compatible_with_segment_config`?
pub fn check_compatible(&self, other: &Self) -> CollectionResult<()> {
match (self, other) {
(Self::Single(_), Self::Single(_)) | (Self::Multi(_), Self::Multi(_)) => (),
_ => {
return Err(incompatible_vectors_error(
self.params_iter().map(|(name, _)| name),
other.params_iter().map(|(name, _)| name),
));
}
};
for (vector_name, this) in self.params_iter() {
let Some(other) = other.get_params(vector_name) else {
return Err(missing_vector_error(vector_name));
};
VectorParamsBase::from(this).check_compatibility(&other.into(), vector_name)?;
}
Ok(())
}
// TODO: Further unify `check_compatible` and `check_compatible_with_segment_config`?
pub fn check_compatible_with_segment_config(
&self,
other: &HashMap<VectorNameBuf, segment::types::VectorDataConfig>,
lib/collection/src/shards/conversions.rs
+43
View File
@@ -496,6 +496,49 @@ pub fn internal_delete_index(
}
}
pub fn internal_create_vector_name(
shard_id: Option<ShardId>,
clock_tag: Option<ClockTag>,
collection_name: String,
create: shard::operations::CreateVectorName,
wait: WaitUntil,
wait_timeout: Option<u64>,
) -> api::grpc::qdrant::CreateVectorNameInternal {
api::grpc::qdrant::CreateVectorNameInternal {
shard_id,
clock_tag: clock_tag.map(Into::into),
wait_override: wait_override_to_proto(wait),
create_vector_name: Some(api::grpc::qdrant::CreateVectorNameRequest {
collection_name,
wait: Some(wait.needs_callback()),
vector_name: create.vector_name,
vector_config: Some(create.config.into()),
timeout: wait_timeout,
}),
}
}
pub fn internal_delete_vector_name(
shard_id: Option<ShardId>,
clock_tag: Option<ClockTag>,
collection_name: String,
delete: shard::operations::DeleteVectorName,
wait: WaitUntil,
wait_timeout: Option<u64>,
) -> api::grpc::qdrant::DeleteVectorNameInternal {
api::grpc::qdrant::DeleteVectorNameInternal {
shard_id,
clock_tag: clock_tag.map(Into::into),
wait_override: wait_override_to_proto(wait),
delete_vector_name: Some(api::grpc::qdrant::DeleteVectorNameRequest {
collection_name,
wait: Some(wait.needs_callback()),
vector_name: delete.vector_name,
timeout: wait_timeout,
}),
}
}
pub fn try_scored_point_from_grpc(
point: api::grpc::qdrant::ScoredPoint,
with_payload: bool,
lib/collection/src/shards/dummy_shard.rs
+4 -3
View File
@@ -107,9 +107,10 @@ impl ShardOperation for DummyShard {
CollectionUpdateOperations::VectorOperation(_) => self.dummy("Update Vectors"),
CollectionUpdateOperations::PayloadOperation(_) => self.dummy("Update Payloads"),
// Allow (and ignore) field index operations. Field index schema is stored in collection
// config, and indices will be created (if needed) when dummy shard is recovered.
CollectionUpdateOperations::FieldIndexOperation(_) => Ok(UpdateResult {
// Allow (and ignore) field index and vector name operations.
// These schemas are stored in collection config and will be recreated when recovered.
CollectionUpdateOperations::FieldIndexOperation(_)
| CollectionUpdateOperations::VectorNameOperation(_) => Ok(UpdateResult {
operation_id: None,
status: UpdateStatus::Acknowledged,
clock_tag: None,
lib/collection/src/shards/local_shard/mod.rs
+53 -18
View File
@@ -77,7 +77,7 @@ use crate::operations::OperationWithClockTag;
use crate::operations::shared_storage_config::SharedStorageConfig;
use crate::operations::types::{
CollectionError, CollectionResult, OptimizersStatus, ShardInfoInternal, ShardStatus,
ShardUpdateQueueInfo, check_sparse_compatible_with_segment_config,
ShardUpdateQueueInfo,
};
use crate::optimizers_builder::{OptimizersConfig, build_optimizers, clear_temp_segments};
use crate::shards::CollectionId;
@@ -406,9 +406,13 @@ impl LocalShard {
})
.map(|entry| entry.path());
// Build desired vector names from collection config for segment reconciliation
let desired_vector_names = desired_vector_names_from_config(&collection_config_read.params);
let mut segment_stream = futures::stream::iter(segment_paths)
.map(|segment_path| {
let payload_index_schema = Arc::clone(&payload_index_schema);
let desired_vectors = desired_vector_names.clone();
let handle = tokio::task::spawn_blocking(move || {
let Some((segment_path, uuid)) = normalize_segment_dir(&segment_path)? else {
return CollectionResult::Ok(None);
@@ -428,6 +432,10 @@ impl LocalShard {
)?;
}
// Reconcile named vectors: create vectors that exist in collection config
// but are missing from the segment (e.g. after crash between config update and shard update)
segment.update_all_vector_names(&desired_vectors)?;
CollectionResult::Ok(Some(segment))
});
AbortOnDropHandle::new(handle)
@@ -446,23 +454,6 @@ impl LocalShard {
continue;
};
collection_config_read
.params
.vectors
.check_compatible_with_segment_config(&segment.config().vector_data, true)?;
collection_config_read
.params
.sparse_vectors
.as_ref()
.map(|sparse_vectors| {
check_sparse_compatible_with_segment_config(
sparse_vectors,
&segment.config().sparse_vector_data,
true,
)
})
.unwrap_or(Ok(()))?;
segment_holder.add_new(segment);
}
drop(segment_stream); // release `payload_index_schema` from borrow checker
@@ -1432,3 +1423,47 @@ impl LocalShardClocks {
shard::files::oldest_clocks_path(shard_path)
}
}
/// Build a list of desired vector names from collection params for segment reconciliation.
fn desired_vector_names_from_config(
params: &crate::config::CollectionParams,
) -> Vec<(
segment::types::VectorNameBuf,
segment::data_types::vector_name_config::VectorNameConfig,
)> {
use segment::data_types::vector_name_config::{
DenseVectorConfig, SparseVectorConfig, VectorNameConfig,
};
let mut desired = Vec::new();
for (name, vp) in params.vectors.params_iter() {
desired.push((
name.to_owned(),
VectorNameConfig::dense(DenseVectorConfig {
size: vp.size.get() as usize,
distance: vp.distance,
multivector_config: vp.multivector_config,
datatype: vp.datatype.map(segment::types::VectorStorageDatatype::from),
}),
));
}
if let Some(sparse) = &params.sparse_vectors {
for (name, sp) in sparse {
desired.push((
name.clone(),
VectorNameConfig::sparse(SparseVectorConfig {
modifier: sp.modifier,
datatype: sp
.index
.as_ref()
.and_then(|idx| idx.datatype)
.map(segment::types::VectorStorageDatatype::from),
}),
));
}
}
desired
}
lib/collection/src/shards/remote_shard.rs
+70 -4
View File
@@ -60,14 +60,17 @@ use crate::operations::types::{
};
use crate::operations::universal_query::shard_query::{ShardQueryRequest, ShardQueryResponse};
use crate::operations::vector_ops::VectorOperations;
use crate::operations::{CollectionUpdateOperations, FieldIndexOperations, OperationWithClockTag};
use crate::operations::{
CollectionUpdateOperations, FieldIndexOperations, OperationWithClockTag, VectorNameOperations,
};
use crate::shards::CollectionId;
use crate::shards::channel_service::ChannelService;
use crate::shards::conversions::{
internal_clear_payload, internal_clear_payload_by_filter, internal_create_index,
internal_delete_index, internal_delete_payload, internal_delete_points,
internal_delete_points_by_filter, internal_set_payload, internal_sync_points,
internal_upsert_points, try_scored_point_from_grpc, wait_override_to_proto,
internal_create_vector_name, internal_delete_index, internal_delete_payload,
internal_delete_points, internal_delete_points_by_filter, internal_delete_vector_name,
internal_set_payload, internal_sync_points, internal_upsert_points, try_scored_point_from_grpc,
wait_override_to_proto,
};
use crate::shards::replica_set::replica_set_state::ReplicaState;
use crate::shards::shard::{PeerId, ShardId};
@@ -444,6 +447,32 @@ impl RemoteShard {
}
}
}
CollectionUpdateOperations::VectorNameOperation(vector_name_op) => {
match vector_name_op {
VectorNameOperations::CreateVectorName(create) => {
let request = internal_create_vector_name(
shard_id,
operation.clock_tag,
collection_name.clone(),
create,
wait,
timeout,
);
Update::CreateVectorName(request)
}
VectorNameOperations::DeleteVectorName(delete) => {
let request = internal_delete_vector_name(
shard_id,
operation.clock_tag,
collection_name.clone(),
delete,
wait,
timeout,
);
Update::DeleteVectorName(request)
}
}
}
#[cfg(feature = "staging")]
CollectionUpdateOperations::StagingOperation(_) => {
// Staging operations should not be forwarded to remote shards
@@ -804,6 +833,43 @@ impl RemoteShard {
.into_inner()
}
},
CollectionUpdateOperations::VectorNameOperation(vector_name_op) => match vector_name_op
{
VectorNameOperations::CreateVectorName(create) => {
let request = &internal_create_vector_name(
shard_id,
operation.clock_tag,
collection_name,
create,
wait,
timeout,
);
self.with_points_client(|mut client| async move {
client
.create_vector_name(tonic::Request::new(request.clone()))
.await
})
.await?
.into_inner()
}
VectorNameOperations::DeleteVectorName(delete) => {
let request = &internal_delete_vector_name(
shard_id,
operation.clock_tag,
collection_name,
delete,
wait,
timeout,
);
self.with_points_client(|mut client| async move {
client
.delete_vector_name(tonic::Request::new(request.clone()))
.await
})
.await?
.into_inner()
}
},
#[cfg(feature = "staging")]
CollectionUpdateOperations::StagingOperation(staging_op) => {
// TODO: Add gRPC support to forward staging operations to remote shards
lib/collection/src/shards/shard_holder/resharding.rs
+2 -1
View File
@@ -610,7 +610,8 @@ impl OperationsByMode {
},
CollectionUpdateOperations::VectorOperation(_)
| CollectionUpdateOperations::PayloadOperation(_)
| CollectionUpdateOperations::FieldIndexOperation(_) => {
| CollectionUpdateOperations::FieldIndexOperation(_)
| CollectionUpdateOperations::VectorNameOperation(_) => {
vec![operation]
}
#[cfg(feature = "staging")]
lib/common/common/src/validation.rs
+43 -11
View File
@@ -57,23 +57,55 @@ pub fn validate_not_empty(value: &str) -> Result<(), ValidationError> {
}
}
/// Filesystem-unsafe characters rejected for both collection and vector names.
///
/// These end up as path components on disk (collection directories,
/// per-vector storage subdirectories — see
/// `segment_constructor::get_vector_storage_path`), so they must be safe on both
/// Linux and Windows filesystems.
const INVALID_NAME_CHARS: [char; 11] =
['<', '>', ':', '"', '/', '\\', '|', '?', '*', '\0', '\u{1F}'];
/// Reject any character from [`INVALID_NAME_CHARS`] in `value`. The `kind`
/// argument is interpolated into the error message ("collection name" /
/// "vector name") so callers get a context-appropriate error.
fn check_invalid_name_chars(value: &str, kind: &str) -> Result<(), ValidationError> {
let Some(c) = INVALID_NAME_CHARS.into_iter().find(|c| value.contains(*c)) else {
return Ok(());
};
let mut err = ValidationError::new("does_not_contain");
err.add_param(Cow::from("pattern"), &c);
err.message
.replace(format!("{kind} cannot contain \"{c}\" char").into());
Err(err)
}
/// Validate the collection name contains no illegal characters
///
/// This does not check the length of the name.
pub fn validate_collection_name(value: &str) -> Result<(), ValidationError> {
const INVALID_CHARS: [char; 11] =
['<', '>', ':', '"', '/', '\\', '|', '?', '*', '\0', '\u{1F}'];
check_invalid_name_chars(value, "collection name")
}
match INVALID_CHARS.into_iter().find(|c| value.contains(*c)) {
Some(c) => {
let mut err = ValidationError::new("does_not_contain");
err.add_param(Cow::from("pattern"), &c);
err.message
.replace(format!("collection name cannot contain \"{c}\" char").into());
Err(err)
}
None => Ok(()),
/// Validate a named vector identifier.
///
/// Vector names become directory components on disk (see
/// `segment_constructor::get_vector_storage_path`), so they are subject to the same
/// rules as collection names: at most 200 bytes, and free of the
/// filesystem-unsafe characters listed in [`INVALID_NAME_CHARS`].
pub fn validate_vector_name(value: &str) -> Result<(), ValidationError> {
const MAX_LEN: usize = 200;
if value.len() > MAX_LEN {
let mut err = ValidationError::new("length");
err.add_param(Cow::from("max"), &MAX_LEN);
err.add_param(Cow::from("actual"), &value.len());
err.message
.replace(format!("vector name must be at most {MAX_LEN} bytes long").into());
return Err(err);
}
check_invalid_name_chars(value, "vector name")
}
/// Validate the collection name contains no illegal characters, legacy edition
lib/edge/python/examples/add-named-vector.py
+160
View File
@@ -0,0 +1,160 @@
#!/usr/bin/env python3
"""
Example: Adding a named vector to an existing Edge shard.
Demonstrates the workflow for migrating to a new embedding model
or adding hybrid search by creating a new named vector after the
shard is already populated with data.
"""
import os, shutil
from pathlib import Path
from qdrant_edge import (
Distance,
EdgeConfig,
EdgeShard,
EdgeVectorParams,
Modifier,
Point,
Query,
QueryRequest,
SparseVector,
UpdateOperation,
VectorStorageDatatype,
)
DATA_DIR = Path(__file__).parent.parent.parent / "data"
TMP_DIR = DATA_DIR / "tmp"
path = TMP_DIR / "qdrant_edge_add_vector"
shutil.rmtree(path, ignore_errors=True)
os.makedirs(path)
# ------------------------------------------------------------------
# 1. Create shard with a single dense vector and populate it
# ------------------------------------------------------------------
print("---- Create shard with initial vector ----")
config = EdgeConfig(
vectors=EdgeVectorParams(size=4, distance=Distance.Cosine),
)
shard = EdgeShard.create(path, config)
shard.update(UpdateOperation.upsert_points([
Point(1, [0.1, 0.2, 0.3, 0.4], {"text": "first document"}),
Point(2, [0.5, 0.6, 0.7, 0.8], {"text": "second document"}),
Point(3, [0.9, 0.1, 0.2, 0.3], {"text": "third document"}),
]))
print(f"Points after initial insert: {shard.info().points_count}")
# Verify search works on the default vector
results = shard.query(QueryRequest(
query=Query.Nearest([0.1, 0.2, 0.3, 0.4]),
limit=3,
with_payload=True,
))
print(f"Search on default vector: {len(results)} results")
assert len(results) == 3
# ------------------------------------------------------------------
# 2. Add a new dense vector (e.g., a new embedding model)
# ------------------------------------------------------------------
print("\n---- Add new dense vector 'v2' ----")
shard.update(UpdateOperation.create_dense_vector(
vector_name="v2",
size=8,
distance=Distance.Dot,
))
# Existing points don't have 'v2' yet - insert new points with both vectors
shard.update(UpdateOperation.upsert_points([
Point(4, {"": [0.4, 0.3, 0.2, 0.1], "v2": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]}, {"text": "fourth document"}),
Point(5, {"": [0.8, 0.7, 0.6, 0.5], "v2": [8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0]}, {"text": "fifth document"}),
]))
print(f"Points after adding v2: {shard.info().points_count}")
# Search on the new vector - only points 4 and 5 have 'v2'
results = shard.query(QueryRequest(
query=Query.Nearest([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0], using="v2"),
limit=5,
with_payload=True,
))
print(f"Search on 'v2': {len(results)} results")
assert len(results) >= 1, "Should find at least point 4 or 5"
# ------------------------------------------------------------------
# 3. Add a sparse vector (e.g., for hybrid search with BM25/SPLADE)
# ------------------------------------------------------------------
print("\n---- Add sparse vector 'keywords' ----")
shard.update(UpdateOperation.create_sparse_vector(
vector_name="keywords",
modifier=Modifier.Idf,
))
# Insert a point with sparse keyword data
shard.update(UpdateOperation.upsert_points([
Point(6, {
"": [0.3, 0.3, 0.3, 0.3],
"keywords": SparseVector(indices=[10, 25, 42], values=[0.8, 0.5, 0.3]),
}, {"text": "sixth document with keywords"}),
]))
print(f"Points after adding keywords: {shard.info().points_count}")
# Search on sparse vector
results = shard.query(QueryRequest(
query=Query.Nearest(SparseVector(indices=[10, 25], values=[1.0, 0.5]), using="keywords"),
limit=5,
with_payload=True,
))
print(f"Search on 'keywords': {len(results)} results")
assert len(results) >= 1
# ------------------------------------------------------------------
# 4. Delete a named vector
# ------------------------------------------------------------------
print("\n---- Delete vector 'v2' ----")
shard.update(UpdateOperation.delete_vector_name("v2"))
print(f"Points after deleting v2: {shard.info().points_count}")
# Search on the default vector still works
results = shard.query(QueryRequest(
query=Query.Nearest([0.1, 0.2, 0.3, 0.4]),
limit=5,
with_payload=True,
))
print(f"Search on default vector after deleting v2: {len(results)} results")
assert len(results) >= 3
# ------------------------------------------------------------------
# 5. Verify persistence - close and reopen
# ------------------------------------------------------------------
print("\n---- Close and reopen ----")
shard.close()
shard = EdgeShard.load(path)
info = shard.info()
print(f"Reopened shard: {info.points_count} points")
results = shard.query(QueryRequest(
query=Query.Nearest(SparseVector(indices=[10, 25], values=[1.0, 0.5]), using="keywords"),
limit=5,
with_payload=True,
))
print(f"Search on 'keywords' after reopen: {len(results)} results")
assert len(results) >= 1
print("\nDone!")
lib/edge/python/qdrant_edge.pyi
+46
View File
@@ -3098,3 +3098,49 @@ class UpdateOperation:
field_name: Path to the payload field.
"""
...
@staticmethod
def create_dense_vector(
vector_name: str,
size: int,
distance: Distance,
multivector_config: Optional[MultiVectorConfig] = None,
datatype: Optional[VectorStorageDatatype] = None,
) -> "UpdateOperation":
"""
Create a new dense named vector on the collection.
Args:
vector_name: Name for the new vector.
size: Dimensionality of the vectors.
distance: Distance function (Cosine, Euclid, Dot, Manhattan).
multivector_config: Optional multi-vector configuration (e.g., for ColBERT).
datatype: Optional element storage type (Float32, Float16, Uint8).
"""
...
@staticmethod
def create_sparse_vector(
vector_name: str,
modifier: Optional[Modifier] = None,
datatype: Optional[VectorStorageDatatype] = None,
) -> "UpdateOperation":
"""
Create a new sparse named vector on the collection.
Args:
vector_name: Name for the new sparse vector.
modifier: Optional value modifier (e.g., Modifier.Idf).
datatype: Optional datatype for storing weights in the index.
"""
...
@staticmethod
def delete_vector_name(vector_name: str) -> "UpdateOperation":
"""
Delete a named vector from the collection.
Args:
vector_name: Name of the vector to delete.
"""
...
lib/edge/python/src/update.rs
+54 -1
View File
@@ -1,8 +1,11 @@
use bytemuck::TransparentWrapperAlloc as _;
use derive_more::Into;
use pyo3::prelude::*;
use segment::data_types::modifier::Modifier;
use segment::json_path::JsonPath;
use segment::types::{Filter, Payload, VectorNameBuf};
use segment::types::{
Distance, Filter, MultiVectorConfig, Payload, VectorNameBuf, VectorStorageDatatype,
};
use shard::operations::point_ops::{PointIdsList, PointInsertOperationsInternal, UpdateMode};
use shard::operations::*;
@@ -211,6 +214,56 @@ impl PyUpdateOperation {
let operation = FieldIndexOperations::DeleteIndex(JsonPath::from(field_name));
Self(CollectionUpdateOperations::FieldIndexOperation(operation))
}
/// Create a new dense named vector on the collection.
#[staticmethod]
#[pyo3(signature = (vector_name, size, distance, multivector_config=None, datatype=None))]
pub fn create_dense_vector(
vector_name: String,
size: usize,
distance: PyDistance,
multivector_config: Option<PyMultiVectorConfig>,
datatype: Option<PyVectorStorageDatatype>,
) -> Self {
let config = vector_name_ops::VectorNameConfig::dense(vector_name_ops::DenseVectorConfig {
size,
distance: Distance::from(distance),
multivector_config: multivector_config.map(MultiVectorConfig::from),
datatype: datatype.map(VectorStorageDatatype::from),
});
let operation = VectorNameOperations::CreateVectorName(CreateVectorName {
vector_name,
config,
});
Self(CollectionUpdateOperations::VectorNameOperation(operation))
}
/// Create a new sparse named vector on the collection.
#[staticmethod]
#[pyo3(signature = (vector_name, modifier=None, datatype=None))]
pub fn create_sparse_vector(
vector_name: String,
modifier: Option<PyModifier>,
datatype: Option<PyVectorStorageDatatype>,
) -> Self {
let config =
vector_name_ops::VectorNameConfig::sparse(vector_name_ops::SparseVectorConfig {
modifier: modifier.map(Modifier::from),
datatype: datatype.map(VectorStorageDatatype::from),
});
let operation = VectorNameOperations::CreateVectorName(CreateVectorName {
vector_name,
config,
});
Self(CollectionUpdateOperations::VectorNameOperation(operation))
}
/// Delete a named vector from the collection.
#[staticmethod]
pub fn delete_vector_name(vector_name: String) -> Self {
let operation = VectorNameOperations::DeleteVectorName(DeleteVectorName { vector_name });
Self(CollectionUpdateOperations::VectorNameOperation(operation))
}
}
/// Defines the mode of the upsert operation
lib/edge/src/update.rs
+61 -1
View File
@@ -2,11 +2,13 @@ use std::fmt;
use common::counter::hardware_counter::HardwareCounterCell;
use segment::common::operation_error::{OperationError, OperationResult};
use shard::operations::CollectionUpdateOperations;
use shard::operations::vector_name_ops::VectorNameConfig;
use shard::operations::{CollectionUpdateOperations, VectorNameOperations};
use shard::update::*;
use shard::wal::WalRawRecord;
use crate::EdgeShard;
use crate::config::vectors::{EdgeSparseVectorParams, EdgeVectorParams};
impl EdgeShard {
pub fn update(&self, operation: CollectionUpdateOperations) -> OperationResult<()> {
@@ -48,6 +50,18 @@ impl EdgeShard {
&hw_counter,
)
}
CollectionUpdateOperations::VectorNameOperation(ref vector_name_operation) => {
let result = process_vector_name_operation(
&segments_guard,
operation_id,
vector_name_operation,
);
// Also update the edge shard config so queries can resolve the vector name
if result.is_ok() {
self.apply_vector_name_to_config(vector_name_operation)?;
}
result
}
#[cfg(feature = "staging")]
CollectionUpdateOperations::StagingOperation(staging_operation) => {
shard::update::process_staging_operation(
@@ -60,6 +74,52 @@ impl EdgeShard {
result.map(|_| ())
}
/// Update the edge shard config to reflect a vector name create/delete operation.
fn apply_vector_name_to_config(&self, operation: &VectorNameOperations) -> OperationResult<()> {
match operation {
VectorNameOperations::CreateVectorName(create) => {
self.config
.write(|config| match &create.config {
VectorNameConfig::Dense(wrapper) => {
config.vectors.insert(
create.vector_name.clone(),
EdgeVectorParams {
size: wrapper.dense.size,
distance: wrapper.dense.distance,
on_disk: None,
multivector_config: wrapper.dense.multivector_config,
datatype: wrapper.dense.datatype,
quantization_config: None,
hnsw_config: None,
},
);
}
VectorNameConfig::Sparse(wrapper) => {
config.sparse_vectors.insert(
create.vector_name.clone(),
EdgeSparseVectorParams {
full_scan_threshold: None,
on_disk: None,
modifier: wrapper.sparse.modifier,
datatype: wrapper.sparse.datatype,
},
);
}
})
.map_err(service_error)?;
}
VectorNameOperations::DeleteVectorName(delete) => {
self.config
.write(|config| {
config.vectors.remove(&delete.vector_name);
config.sparse_vectors.remove(&delete.vector_name);
})
.map_err(service_error)?;
}
}
Ok(())
}
}
fn service_error(err: impl fmt::Display) -> OperationError {
lib/segment/src/data_types/mod.rs
+1
View File
@@ -11,4 +11,5 @@ pub mod primitive;
pub mod query_context;
pub mod segment_record;
pub mod tiny_map;
pub mod vector_name_config;
pub mod vectors;
lib/segment/src/data_types/vector_name_config.rs
+143
View File
@@ -0,0 +1,143 @@
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use validator::Validate;
use crate::data_types::modifier::Modifier;
use crate::index::sparse_index::sparse_index_config::SparseIndexConfig;
use crate::types::{
Distance, Indexes, MultiVectorConfig, SparseVectorDataConfig, VectorDataConfig,
VectorStorageDatatype, VectorStorageType,
};
/// Configuration for creating a new named vector.
///
/// Contains only the immutable properties that define a vector space.
/// Storage type, index, and quantization are determined automatically
/// based on the segment type and can be configured separately later.
///
/// Example JSON for a dense vector:
/// ```json
/// { "dense": { "size": 768, "distance": "Cosine" } }
/// ```
///
/// Example JSON for a sparse vector:
/// ```json
/// { "sparse": { "modifier": "Idf" } }
/// ```
#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
#[serde(untagged)]
pub enum VectorNameConfig {
Dense(DenseVectorNameConfig),
Sparse(SparseVectorNameConfig),
}
/// Wrapper for dense vector creation config.
#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub struct DenseVectorNameConfig {
/// Dense vector parameters
pub dense: DenseVectorConfig,
}
/// Wrapper for sparse vector creation config.
#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub struct SparseVectorNameConfig {
/// Sparse vector parameters
pub sparse: SparseVectorConfig,
}
/// Configuration for creating a new dense named vector.
///
/// Only includes properties that define the vector space and cannot be changed
/// after creation. Storage type, index type, and quantization are inferred.
#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub struct DenseVectorConfig {
/// Dimensionality of the vectors
pub size: usize,
/// Distance function used for measuring distance between vectors
pub distance: Distance,
/// Configuration for multi-vector points (e.g., ColBERT)
#[serde(default, skip_serializing_if = "Option::is_none")]
pub multivector_config: Option<MultiVectorConfig>,
/// Element storage type (Float32, Float16, Uint8)
#[serde(default, skip_serializing_if = "Option::is_none")]
pub datatype: Option<VectorStorageDatatype>,
}
/// Configuration for creating a new sparse named vector.
///
/// Only includes properties that define the vector space and cannot be changed
/// after creation.
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub struct SparseVectorConfig {
/// Value modifier for sparse vectors (e.g., IDF)
#[serde(default, skip_serializing_if = "Option::is_none")]
pub modifier: Option<Modifier>,
/// Datatype used to store weights in the index
#[serde(default, skip_serializing_if = "Option::is_none")]
pub datatype: Option<VectorStorageDatatype>,
}
impl Validate for VectorNameConfig {
fn validate(&self) -> Result<(), validator::ValidationErrors> {
Ok(())
}
}
// Convenience constructors for creating VectorNameConfig without wrappers
impl VectorNameConfig {
pub fn dense(config: DenseVectorConfig) -> Self {
VectorNameConfig::Dense(DenseVectorNameConfig { dense: config })
}
pub fn sparse(config: SparseVectorConfig) -> Self {
VectorNameConfig::Sparse(SparseVectorNameConfig { sparse: config })
}
}
impl DenseVectorConfig {
/// Convert to internal VectorDataConfig with appropriate defaults for a new vector.
///
/// - Storage type is inferred from `on_disk` preference
/// - Index is always Plain (HNSW can be built later by optimizer)
/// - Quantization is None (can be configured separately)
pub fn to_internal(&self, on_disk: bool) -> VectorDataConfig {
let Self {
size,
distance,
multivector_config,
datatype,
} = self;
VectorDataConfig {
size: *size,
distance: *distance,
storage_type: VectorStorageType::from_on_disk(on_disk),
index: Indexes::Plain {},
quantization_config: None,
multivector_config: *multivector_config,
datatype: *datatype,
}
}
}
impl SparseVectorConfig {
/// Convert to internal SparseVectorDataConfig with appropriate defaults.
pub fn to_internal(&self) -> SparseVectorDataConfig {
let Self { modifier, datatype } = self;
SparseVectorDataConfig {
index: SparseIndexConfig {
datatype: *datatype,
..Default::default()
},
storage_type: Default::default(),
modifier: *modifier,
}
}
}
lib/segment/src/entry/entry_point.rs
+22
View File
@@ -16,6 +16,7 @@ use crate::data_types::named_vectors::NamedVectors;
use crate::data_types::order_by::{OrderBy, OrderValue};
use crate::data_types::query_context::{FormulaContext, QueryContext, SegmentQueryContext};
use crate::data_types::segment_record::SegmentRecord;
use crate::data_types::vector_name_config::VectorNameConfig;
use crate::data_types::vectors::{QueryVector, VectorInternal};
use crate::entry::snapshot_entry::SnapshotEntry;
use crate::index::field_index::{CardinalityEstimation, FieldIndex};
@@ -371,6 +372,27 @@ pub trait NonAppendableSegmentEntry: StorageSegmentEntry {
self.apply_field_index(op_num, key.to_owned(), schema, indexes)
}
/// Create a new named vector in the segment.
/// For appendable segments: creates a real, writable vector storage + plain index.
/// For immutable segments: creates a placeholder (empty) vector storage.
/// Returns Ok(false) if the vector already exists (idempotent).
fn create_vector_name(
&mut self,
op_num: SeqNumberType,
vector_name: &VectorName,
vector_config: &VectorNameConfig,
) -> OperationResult<bool>;
/// Delete a named vector from the segment.
/// Removes vector storage, index, and quantization data.
/// Removes the vector from segment config.
/// Returns Ok(false) if the vector does not exist (idempotent).
fn delete_vector_name(
&mut self,
op_num: SeqNumberType,
vector_name: &VectorName,
) -> OperationResult<bool>;
}
/// Define mutable operations which can be performed with Segment or Segment-like entity.
lib/segment/src/index/hnsw_index/gpu/gpu_vector_storage/mod.rs
+5
View File
@@ -466,6 +466,11 @@ impl GpuVectorStorage {
VectorStorageEnum::MultiDenseAppendableMemmapHalf(vector_storage) => {
Self::new_multi_f16(device, vector_storage.as_ref(), stopped)
}
VectorStorageEnum::EmptyDense(_) | VectorStorageEnum::EmptySparse(_) => {
Err(OperationError::service_error(
"Cannot create GPU vector storage for empty vector storage",
))
}
}
}
lib/segment/src/index/struct_payload_index.rs
+22
View File
@@ -291,6 +291,28 @@ impl StructPayloadIndex {
Ok(index)
}
/// Register a vector storage for the `has_vector` filtering condition.
///
/// Must be called whenever a new named vector is added to the segment after the
/// payload index has been opened, otherwise `has_vector` queries will see stale
/// state (no matches for the new vector) until the segment is reloaded.
pub fn register_vector_storage(
&mut self,
vector_name: VectorNameBuf,
vector_storage: Arc<AtomicRefCell<VectorStorageEnum>>,
) {
self.vector_storages.insert(vector_name, vector_storage);
}
/// Drop a vector storage from the `has_vector` lookup map.
///
/// Must be called whenever a named vector is removed from the segment, otherwise
/// `has_vector` queries will keep matching points against the deleted storage
/// until the segment is reloaded.
pub fn unregister_vector_storage(&mut self, vector_name: &str) {
self.vector_storages.remove(vector_name);
}
pub fn build_field_indexes(
&self,
field: PayloadKeyTypeRef,
lib/segment/src/segment/entry.rs
+22
View File
@@ -22,6 +22,7 @@ use crate::data_types::query_context::{
FormulaContext, QueryContext, QueryIdfStats, SegmentQueryContext,
};
use crate::data_types::segment_record::{NamedVectorsOwned, SegmentRecord};
use crate::data_types::vector_name_config::VectorNameConfig;
use crate::data_types::vectors::{QueryVector, VectorInternal};
use crate::entry::entry_point::{
NonAppendableSegmentEntry, ReadSegmentEntry, SegmentEntry, StorageSegmentEntry,
@@ -971,6 +972,27 @@ impl NonAppendableSegmentEntry for Segment {
Ok(true)
})
}
fn create_vector_name(
&mut self,
op_num: SeqNumberType,
vector_name: &VectorName,
vector_config: &VectorNameConfig,
) -> OperationResult<bool> {
self.handle_segment_version_and_failure(op_num, |segment| {
segment.create_vector_name_impl(op_num, vector_name, vector_config)
})
}
fn delete_vector_name(
&mut self,
op_num: SeqNumberType,
vector_name: &VectorName,
) -> OperationResult<bool> {
self.handle_segment_version_and_failure(op_num, |segment| {
segment.delete_vector_name_impl(op_num, vector_name)
})
}
}
impl SegmentEntry for Segment {
lib/segment/src/segment/mod.rs
+1
View File
@@ -6,6 +6,7 @@ mod sampling;
mod scroll;
mod search;
mod segment_ops;
mod vector_name_ops;
mod version_tracker;
pub mod snapshot;
lib/segment/src/segment/tests.rs → lib/segment/src/segment/tests/mod.rs
+2
View File
@@ -1,3 +1,5 @@
mod test_vector_name_ops;
use std::sync::atomic::AtomicBool;
use ahash::AHashSet;
lib/segment/src/segment/tests/test_vector_name_ops.rs
+485
View File
@@ -0,0 +1,485 @@
use std::collections::HashMap;
use std::sync::atomic::AtomicBool;
use common::counter::hardware_counter::HardwareCounterCell;
use sparse::common::sparse_vector::SparseVector;
use tempfile::Builder;
use crate::data_types::named_vectors::NamedVectors;
use crate::data_types::vector_name_config::{
DenseVectorConfig, SparseVectorConfig, VectorNameConfig,
};
use crate::data_types::vectors::DEFAULT_VECTOR_NAME;
use crate::entry::entry_point::{
NonAppendableSegmentEntry as _, ReadSegmentEntry as _, SegmentEntry as _,
StorageSegmentEntry as _,
};
use crate::segment::Segment;
use crate::segment_constructor::segment_builder::SegmentBuilder;
use crate::segment_constructor::{build_segment, load_segment};
use crate::types::{
Distance, HnswGlobalConfig, Indexes, SegmentConfig, VectorDataConfig, VectorStorageType,
};
use crate::vector_storage::VectorStorage as _;
const DIM: usize = 4;
const NUM_POINTS: usize = 5;
fn default_dense_config(dim: usize) -> VectorDataConfig {
VectorDataConfig {
size: dim,
distance: Distance::Dot,
storage_type: VectorStorageType::default(),
index: Indexes::Plain {},
quantization_config: None,
multivector_config: None,
datatype: None,
}
}
fn mmap_dense_config(dim: usize) -> VectorDataConfig {
VectorDataConfig {
storage_type: VectorStorageType::Mmap,
..default_dense_config(dim)
}
}
fn dense_vector_name_config(dim: usize) -> VectorNameConfig {
VectorNameConfig::dense(DenseVectorConfig {
size: dim,
distance: Distance::Dot,
multivector_config: None,
datatype: None,
})
}
fn sparse_vector_name_config() -> VectorNameConfig {
VectorNameConfig::sparse(SparseVectorConfig {
modifier: None,
datatype: None,
})
}
fn hw() -> HardwareCounterCell {
HardwareCounterCell::new()
}
/// Build an appendable segment with NUM_POINTS points on the default vector.
fn build_appendable_segment_with_data(path: &std::path::Path) -> Segment {
let mut segment = build_segment(
path,
&SegmentConfig {
vector_data: HashMap::from([(
DEFAULT_VECTOR_NAME.to_owned(),
default_dense_config(DIM),
)]),
sparse_vector_data: Default::default(),
payload_storage_type: Default::default(),
},
None,
true,
)
.unwrap();
let hw = hw();
for i in 0..NUM_POINTS {
let vec = vec![(i + 1) as f32; DIM];
let vectors = NamedVectors::from_ref(DEFAULT_VECTOR_NAME, vec.as_slice().into());
segment
.upsert_point((i + 1) as u64, (i as u64 + 1).into(), vectors, &hw)
.unwrap();
}
assert_eq!(segment.available_point_count(), NUM_POINTS);
segment
}
/// Build a proper immutable segment by optimizing an appendable segment through SegmentBuilder.
fn build_immutable_segment_with_data(
segments_path: &std::path::Path,
temp_path: &std::path::Path,
) -> Segment {
let source = build_appendable_segment_with_data(segments_path);
// Target config with Mmap storage -> non-appendable
let target_config = SegmentConfig {
vector_data: HashMap::from([(DEFAULT_VECTOR_NAME.to_owned(), mmap_dense_config(DIM))]),
sparse_vector_data: Default::default(),
payload_storage_type: Default::default(),
};
assert!(!target_config.is_appendable());
let mut builder =
SegmentBuilder::new(temp_path, &target_config, &HnswGlobalConfig::default()).unwrap();
let stopped = AtomicBool::new(false);
builder.update(&[&source], &stopped, &hw()).unwrap();
let segment = builder.build_for_test(segments_path);
assert!(!segment.appendable_flag);
assert_eq!(segment.available_point_count(), NUM_POINTS);
segment
}
// ---------------------------------------------------------------------------
// Creating dense vectors
// ---------------------------------------------------------------------------
#[test]
fn test_create_dense_vector_on_appendable_segment() {
let dir = Builder::new().prefix("segment_dir").tempdir().unwrap();
let mut segment = build_appendable_segment_with_data(dir.path());
let hw = hw();
assert_eq!(segment.available_point_count(), NUM_POINTS);
// Create a new dense vector
let new_dim = 8;
let result = segment
.create_vector_name(100, "v2", &dense_vector_name_config(new_dim))
.unwrap();
assert!(result);
assert_eq!(segment.segment_config.vector_data["v2"].size, new_dim);
assert!(segment.vector_data.contains_key("v2"));
// Insert a new point with both vectors
let mut vectors = NamedVectors::default();
vectors.insert(DEFAULT_VECTOR_NAME.to_owned(), vec![9.0f32; DIM].into());
vectors.insert("v2".to_owned(), vec![1.0f32; new_dim].into());
segment
.upsert_point(101, (NUM_POINTS as u64 + 1).into(), vectors, &hw)
.unwrap();
assert_eq!(segment.available_point_count(), NUM_POINTS + 1);
// The new point has both vectors
let all_vecs = segment
.all_vectors((NUM_POINTS as u64 + 1).into(), &hw)
.unwrap();
assert!(all_vecs.contains_key(DEFAULT_VECTOR_NAME));
assert!(all_vecs.contains_key("v2"));
// Can read the new vector back
let v2_vec = segment
.vector("v2", (NUM_POINTS as u64 + 1).into(), &hw)
.unwrap();
assert!(v2_vec.is_some());
}
#[test]
fn test_create_sparse_vector_on_appendable_segment() {
let dir = Builder::new().prefix("segment_dir").tempdir().unwrap();
let mut segment = build_appendable_segment_with_data(dir.path());
let hw = hw();
let result = segment
.create_vector_name(100, "sparse1", &sparse_vector_name_config())
.unwrap();
assert!(result);
assert!(
segment
.segment_config
.sparse_vector_data
.contains_key("sparse1")
);
// Insert a point with sparse data
let sparse_vec = SparseVector::new(vec![0, 2, 5], vec![1.0, 0.5, 0.3]).unwrap();
let mut vectors = NamedVectors::default();
vectors.insert(DEFAULT_VECTOR_NAME.to_owned(), vec![7.0f32; DIM].into());
vectors.insert("sparse1".to_owned(), sparse_vec.into());
segment
.upsert_point(101, (NUM_POINTS as u64 + 1).into(), vectors, &hw)
.unwrap();
// Verify the sparse vector is retrievable
let all_vecs = segment
.all_vectors((NUM_POINTS as u64 + 1).into(), &hw)
.unwrap();
assert!(all_vecs.contains_key("sparse1"));
}
#[test]
fn test_create_dense_vector_on_immutable_segment() {
let dir = Builder::new().prefix("segment_dir").tempdir().unwrap();
let temp_dir = Builder::new().prefix("segment_temp").tempdir().unwrap();
let mut segment = build_immutable_segment_with_data(dir.path(), temp_dir.path());
let hw = hw();
assert!(!segment.appendable_flag);
assert_eq!(segment.available_point_count(), NUM_POINTS);
// Create a new dense vector on immutable segment -> empty placeholder
let new_dim = 8;
let result = segment
.create_vector_name(100, "v2", &dense_vector_name_config(new_dim))
.unwrap();
assert!(result);
assert!(segment.segment_config.vector_data.contains_key("v2"));
// Empty placeholder: all existing points have the vector deleted
let storage = segment.vector_data["v2"].vector_storage.borrow();
assert_eq!(storage.total_vector_count(), NUM_POINTS);
assert_eq!(storage.available_vector_count(), 0);
assert_eq!(storage.deleted_vector_count(), NUM_POINTS);
drop(storage);
// Existing points should not have the new vector
for i in 1..=NUM_POINTS as u64 {
let v2_vec = segment.vector("v2", i.into(), &hw).unwrap();
assert!(v2_vec.is_none(), "point {i} should not have v2");
}
// Original default vector is still readable for all points
for i in 1..=NUM_POINTS as u64 {
let default_vec = segment.vector(DEFAULT_VECTOR_NAME, i.into(), &hw).unwrap();
assert!(
default_vec.is_some(),
"point {i} should have default vector"
);
}
}
#[test]
fn test_create_sparse_vector_on_immutable_segment() {
let dir = Builder::new().prefix("segment_dir").tempdir().unwrap();
let temp_dir = Builder::new().prefix("segment_temp").tempdir().unwrap();
let mut segment = build_immutable_segment_with_data(dir.path(), temp_dir.path());
let hw = hw();
assert!(!segment.appendable_flag);
assert_eq!(segment.available_point_count(), NUM_POINTS);
// Create a sparse vector on immutable segment with MutableRam config.
// The implementation should upgrade to a non-appendable index type (Mmap).
let result = segment
.create_vector_name(100, "sp", &sparse_vector_name_config())
.unwrap();
assert!(result);
assert!(segment.segment_config.sparse_vector_data.contains_key("sp"));
// Verify the index type was upgraded from MutableRam to Mmap
let stored_index_type = segment.segment_config.sparse_vector_data["sp"]
.index
.index_type;
assert!(
stored_index_type.is_immutable(),
"expected immutable sparse index type on immutable segment, got {stored_index_type:?}"
);
// Empty placeholder: all existing points have the vector deleted
let storage = segment.vector_data["sp"].vector_storage.borrow();
assert_eq!(storage.total_vector_count(), NUM_POINTS);
assert_eq!(storage.available_vector_count(), 0);
assert_eq!(storage.deleted_vector_count(), NUM_POINTS);
drop(storage);
// Existing points should not have the sparse vector
for i in 1..=NUM_POINTS as u64 {
let sp_vec = segment.vector("sp", i.into(), &hw).unwrap();
assert!(sp_vec.is_none(), "point {i} should not have sp");
}
// Original default vector is still readable
for i in 1..=NUM_POINTS as u64 {
let default_vec = segment.vector(DEFAULT_VECTOR_NAME, i.into(), &hw).unwrap();
assert!(
default_vec.is_some(),
"point {i} should have default vector"
);
}
}
// ---------------------------------------------------------------------------
// Idempotency
// ---------------------------------------------------------------------------
#[test]
fn test_create_vector_idempotent() {
let dir = Builder::new().prefix("segment_dir").tempdir().unwrap();
let mut segment = build_appendable_segment_with_data(dir.path());
let config = dense_vector_name_config(8);
assert!(segment.create_vector_name(100, "v2", &config).unwrap());
// Second call returns false (already exists)
assert!(!segment.create_vector_name(101, "v2", &config).unwrap());
}
#[test]
fn test_delete_vector_idempotent() {
let dir = Builder::new().prefix("segment_dir").tempdir().unwrap();
let mut segment = build_appendable_segment_with_data(dir.path());
assert!(!segment.delete_vector_name(100, "nonexistent").unwrap());
}
// ---------------------------------------------------------------------------
// Deleting named vectors
// ---------------------------------------------------------------------------
#[test]
fn test_delete_dense_vector_with_data() {
let dir = Builder::new().prefix("segment_dir").tempdir().unwrap();
let mut segment = build_appendable_segment_with_data(dir.path());
let hw = hw();
// Create vector and insert data
let new_dim = 8;
segment
.create_vector_name(100, "to_delete", &dense_vector_name_config(new_dim))
.unwrap();
let mut vectors = NamedVectors::default();
vectors.insert(DEFAULT_VECTOR_NAME.to_owned(), vec![5.0f32; DIM].into());
vectors.insert("to_delete".to_owned(), vec![2.0f32; new_dim].into());
segment
.upsert_point(101, (NUM_POINTS as u64 + 1).into(), vectors, &hw)
.unwrap();
// Confirm the new vector is there
let all_vecs = segment
.all_vectors((NUM_POINTS as u64 + 1).into(), &hw)
.unwrap();
assert!(all_vecs.contains_key("to_delete"));
// Delete the vector
assert!(segment.delete_vector_name(102, "to_delete").unwrap());
assert!(!segment.vector_data.contains_key("to_delete"));
assert!(!segment.segment_config.vector_data.contains_key("to_delete"));
// Original points and the default vector are still intact
assert_eq!(segment.available_point_count(), NUM_POINTS + 1);
for i in 1..=NUM_POINTS as u64 {
let default_vec = segment.vector(DEFAULT_VECTOR_NAME, i.into(), &hw).unwrap();
assert!(default_vec.is_some());
}
}
#[test]
fn test_delete_sparse_vector_with_data() {
let dir = Builder::new().prefix("segment_dir").tempdir().unwrap();
let mut segment = build_appendable_segment_with_data(dir.path());
let hw = hw();
segment
.create_vector_name(100, "sp", &sparse_vector_name_config())
.unwrap();
// Insert a point with sparse data
let sparse_vec = SparseVector::new(vec![1, 3], vec![0.5, 0.8]).unwrap();
let mut vectors = NamedVectors::default();
vectors.insert(DEFAULT_VECTOR_NAME.to_owned(), vec![4.0f32; DIM].into());
vectors.insert("sp".to_owned(), sparse_vec.into());
segment
.upsert_point(101, (NUM_POINTS as u64 + 1).into(), vectors, &hw)
.unwrap();
// Verify data is there
let all_vecs = segment
.all_vectors((NUM_POINTS as u64 + 1).into(), &hw)
.unwrap();
assert!(all_vecs.contains_key("sp"));
// Delete
assert!(segment.delete_vector_name(102, "sp").unwrap());
assert!(!segment.segment_config.sparse_vector_data.contains_key("sp"));
assert!(!segment.vector_data.contains_key("sp"));
}
// ---------------------------------------------------------------------------
// Persistence (save/reload)
// ---------------------------------------------------------------------------
#[test]
fn test_persistence_after_create_with_data() {
let dir = Builder::new().prefix("segment_dir").tempdir().unwrap();
let mut segment = build_appendable_segment_with_data(dir.path());
let hw = hw();
let new_dim = 6;
segment
.create_vector_name(100, "persisted", &dense_vector_name_config(new_dim))
.unwrap();
// Insert data into the new vector
let mut vectors = NamedVectors::default();
vectors.insert(DEFAULT_VECTOR_NAME.to_owned(), vec![8.0f32; DIM].into());
vectors.insert("persisted".to_owned(), vec![1.5f32; new_dim].into());
segment
.upsert_point(101, (NUM_POINTS as u64 + 1).into(), vectors, &hw)
.unwrap();
// Save, drop, reload
let segment_path = segment.data_path();
let segment_uuid = segment.uuid;
segment.flush(true).unwrap();
drop(segment);
let stopped = AtomicBool::new(false);
let loaded = load_segment(&segment_path, segment_uuid, None, &stopped).unwrap();
// Config persisted
assert_eq!(loaded.segment_config.vector_data["persisted"].size, new_dim);
assert_eq!(loaded.available_point_count(), NUM_POINTS + 1);
// Data persisted - vector is readable
let vec = loaded
.vector("persisted", (NUM_POINTS as u64 + 1).into(), &hw)
.unwrap();
assert!(vec.is_some());
// Original data intact
for i in 1..=NUM_POINTS as u64 {
let original = loaded.vector(DEFAULT_VECTOR_NAME, i.into(), &hw).unwrap();
assert!(
original.is_some(),
"point {i} should have default vector after reload"
);
}
}
#[test]
fn test_persistence_after_delete_with_data() {
let dir = Builder::new().prefix("segment_dir").tempdir().unwrap();
let mut segment = build_appendable_segment_with_data(dir.path());
let hw = hw();
let new_dim = 8;
segment
.create_vector_name(100, "temp", &dense_vector_name_config(new_dim))
.unwrap();
// Insert data, then delete the vector
let mut vectors = NamedVectors::default();
vectors.insert(DEFAULT_VECTOR_NAME.to_owned(), vec![6.0f32; DIM].into());
vectors.insert("temp".to_owned(), vec![2.0f32; new_dim].into());
segment
.upsert_point(101, (NUM_POINTS as u64 + 1).into(), vectors, &hw)
.unwrap();
segment.delete_vector_name(102, "temp").unwrap();
// Save, drop, reload
let segment_path = segment.data_path();
let segment_uuid = segment.uuid;
segment.flush(true).unwrap();
drop(segment);
let stopped = AtomicBool::new(false);
let loaded = load_segment(&segment_path, segment_uuid, None, &stopped).unwrap();
assert!(!loaded.segment_config.vector_data.contains_key("temp"));
assert_eq!(loaded.available_point_count(), NUM_POINTS + 1);
// Original data still intact
for i in 1..=NUM_POINTS as u64 {
let original = loaded.vector(DEFAULT_VECTOR_NAME, i.into(), &hw).unwrap();
assert!(
original.is_some(),
"point {i} should have default vector after reload"
);
}
}
lib/segment/src/segment/vector_name_ops.rs
+273
View File
@@ -0,0 +1,273 @@
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use atomic_refcell::AtomicRefCell;
use super::Segment;
use crate::common::operation_error::OperationResult;
use crate::data_types::vector_name_config::VectorNameConfig;
use crate::id_tracker::IdTracker as _;
use crate::index::VectorIndexEnum;
use crate::index::plain_vector_index::PlainVectorIndex;
use crate::index::sparse_index::sparse_index_config::SparseIndexType;
use crate::index::sparse_index::sparse_vector_index::SparseVectorIndexOpenArgs;
use crate::segment::VectorData;
use crate::segment_constructor::{
create_sparse_vector_index, create_sparse_vector_storage, get_vector_index_path,
get_vector_storage_path, open_vector_storage,
};
use crate::types::{
SeqNumberType, SparseVectorDataConfig, VectorDataConfig, VectorName, VectorStorageType,
};
use crate::vector_storage::dense::empty_dense_vector_storage::new_empty_dense_vector_storage;
use crate::vector_storage::sparse::empty_sparse_vector_storage::new_empty_sparse_vector_storage;
impl Segment {
/// Core logic for creating a new named vector.
/// Called from the `NonAppendableSegmentEntry::create_vector_name` trait impl in entry.rs.
pub(super) fn create_vector_name_impl(
&mut self,
op_num: SeqNumberType,
vector_name: &VectorName,
config: &VectorNameConfig,
) -> OperationResult<bool> {
// Idempotent: if vector already exists, return false
if self.vector_data.contains_key(vector_name) {
return Ok(false);
}
match config {
VectorNameConfig::Dense(wrapper) => {
let internal = wrapper.dense.to_internal(false);
self.create_dense_vector(vector_name, &internal)
}
VectorNameConfig::Sparse(wrapper) => {
let internal = wrapper.sparse.to_internal();
self.create_sparse_vector(vector_name, &internal)
}
}?;
// Persist and track version
Segment::save_state(&self.get_state(), &self.segment_path)?;
self.version_tracker
.set_vector_names_schema(vector_name, Some(op_num));
Ok(true)
}
fn create_dense_vector(
&mut self,
vector_name: &VectorName,
config: &VectorDataConfig,
) -> OperationResult<()> {
let num_points = self.id_tracker.borrow().total_point_count();
let mut vector_storage = if self.appendable_flag {
// Appendable segment: create real writable storage
let storage_path = get_vector_storage_path(&self.segment_path, vector_name);
// Use the configured storage type for appendable segments
let config_for_open = VectorDataConfig {
// Override storage type to appendable chunked mmap
storage_type: VectorStorageType::from_on_disk(config.storage_type.is_on_disk()),
// Use plain index for new vectors
index: crate::types::Indexes::Plain {},
..config.clone()
};
open_vector_storage(&config_for_open, &storage_path)?
} else {
// Immutable segment: create empty placeholder
new_empty_dense_vector_storage(
config.size,
config.distance,
config.datatype.unwrap_or_default(),
config.storage_type.is_on_disk(),
config.multivector_config,
num_points,
)
};
// Fill storage with deleted entries for all existing points so that
// total_vector_count matches the segment's point count.
vector_storage.prefill_deleted_entries(num_points)?;
let vector_storage = Arc::new(AtomicRefCell::new(vector_storage));
let quantized_vectors = Arc::new(AtomicRefCell::new(None));
// Create plain index for the new storage
let vector_index = VectorIndexEnum::Plain(PlainVectorIndex::new(
self.id_tracker.clone(),
vector_storage.clone(),
quantized_vectors.clone(),
self.payload_index.clone(),
));
// Register the new storage with the payload index so `has_vector`
// filtering sees it immediately, not just after a restart.
self.payload_index
.borrow_mut()
.register_vector_storage(vector_name.to_owned(), vector_storage.clone());
self.vector_data.insert(
vector_name.to_owned(),
VectorData {
vector_index: Arc::new(AtomicRefCell::new(vector_index)),
vector_storage,
quantized_vectors,
},
);
self.segment_config
.vector_data
.insert(vector_name.to_owned(), config.clone());
Ok(())
}
fn create_sparse_vector(
&mut self,
vector_name: &VectorName,
config: &SparseVectorDataConfig,
) -> OperationResult<()> {
let num_points = self.id_tracker.borrow().total_point_count();
// Choose appropriate index type based on segment appendability
let index_type = if self.appendable_flag {
SparseIndexType::MutableRam
} else if config.index.index_type.is_appendable() {
// Immutable segment can't use MutableRam, upgrade to Mmap
SparseIndexType::Mmap
} else {
config.index.index_type
};
let mut effective_config = *config;
effective_config.index.index_type = index_type;
let mut vector_storage = if self.appendable_flag {
// Appendable: create real sparse mmap storage
let storage_path = get_vector_storage_path(&self.segment_path, vector_name);
create_sparse_vector_storage(&storage_path, &effective_config.storage_type)?
} else {
// Immutable: empty placeholder
new_empty_sparse_vector_storage(num_points)
};
// Fill storage with deleted entries for all existing points so that
// total_vector_count matches the segment's point count.
vector_storage.prefill_deleted_entries(num_points)?;
let vector_storage = Arc::new(AtomicRefCell::new(vector_storage));
let quantized_vectors = Arc::new(AtomicRefCell::new(None));
// Create sparse vector index
let vector_index_path = get_vector_index_path(&self.segment_path, vector_name);
let stopped = AtomicBool::new(false);
let vector_index = create_sparse_vector_index(SparseVectorIndexOpenArgs {
config: effective_config.index,
id_tracker: self.id_tracker.clone(),
vector_storage: vector_storage.clone(),
payload_index: self.payload_index.clone(),
path: &vector_index_path,
stopped: &stopped,
tick_progress: || (),
deferred_internal_id: None,
})?;
// Register the new storage with the payload index so `has_vector`
// filtering sees it immediately, not just after a restart.
self.payload_index
.borrow_mut()
.register_vector_storage(vector_name.to_owned(), vector_storage.clone());
self.vector_data.insert(
vector_name.to_owned(),
VectorData {
vector_index: Arc::new(AtomicRefCell::new(vector_index)),
vector_storage,
quantized_vectors,
},
);
self.segment_config
.sparse_vector_data
.insert(vector_name.to_owned(), effective_config);
Ok(())
}
/// Core logic for deleting a named vector.
/// Called from the `NonAppendableSegmentEntry::delete_vector_name` trait impl in entry.rs.
pub(super) fn delete_vector_name_impl(
&mut self,
_op_num: SeqNumberType,
vector_name: &VectorName,
) -> OperationResult<bool> {
// Idempotent: if vector doesn't exist, return false
if !self.vector_data.contains_key(vector_name) {
return Ok(false);
}
// Drop the storage from the payload index lookup so `has_vector`
// filtering stops matching against the deleted vector immediately.
self.payload_index
.borrow_mut()
.unregister_vector_storage(vector_name);
// Remove from runtime data (drops VectorData - releases storage/index/quantized)
self.vector_data.remove(vector_name);
// Remove from config (could be either dense or sparse)
self.segment_config.vector_data.remove(vector_name);
self.segment_config.sparse_vector_data.remove(vector_name);
// Update version tracker
self.version_tracker
.set_vector_names_schema(vector_name, None);
// Persist state
Segment::save_state(&self.get_state(), &self.segment_path)?;
// Clean up disk files (best-effort)
let storage_path = get_vector_storage_path(&self.segment_path, vector_name);
let index_path = get_vector_index_path(&self.segment_path, vector_name);
if storage_path.exists()
&& let Err(e) = fs_err::remove_dir_all(&storage_path)
{
log::warn!(
"Failed to remove vector storage at {}: {e}",
storage_path.display()
);
}
if index_path.exists()
&& let Err(e) = fs_err::remove_dir_all(&index_path)
{
log::warn!(
"Failed to remove vector index at {}: {e}",
index_path.display()
);
}
Ok(true)
}
/// Reconcile named vectors in this segment to match the desired configuration.
///
/// Creates vectors that are in `desired` but missing from the segment.
/// Does NOT delete extra vectors (segment may have vectors from ongoing operations).
///
/// This is similar to `update_all_field_indices` for payload indexes.
pub fn update_all_vector_names(
&mut self,
desired: &[(crate::types::VectorNameBuf, VectorNameConfig)],
) -> OperationResult<()> {
let version = self.version.unwrap_or(0);
for (name, config) in desired {
if self.vector_data.contains_key(name) {
continue;
}
log::warn!("Segment is missing vector '{name}', creating it now");
self.create_vector_name_impl(version, name, config)?;
}
Ok(())
}
}
lib/segment/src/segment/version_tracker.rs
+13
View File
@@ -27,6 +27,11 @@ pub struct VersionTracker {
/// E.g., payload index can be created *after* immutable segment is created, and so we have to
/// track payload index schema version separately from `Segment::initial_version`.
payload_index_schema: HashMap<JsonPath, SeqNumberType>,
/// Tracks version of named vector schema changes (create/delete named vectors).
/// Similar to `payload_index_schema`, named vectors can be added or removed on
/// immutable segments, so their schema changes need to be tracked separately.
vector_names_schema: HashMap<VectorNameBuf, SeqNumberType>,
}
impl VersionTracker {
@@ -53,6 +58,14 @@ impl VersionTracker {
pub fn set_payload_index_schema(&mut self, field: &JsonPath, version: Option<SeqNumberType>) {
bump_key(&mut self.payload_index_schema, field, version)
}
pub fn get_vector_names_schema(&self, vector_name: &str) -> Option<SeqNumberType> {
self.vector_names_schema.get(vector_name).copied()
}
pub fn set_vector_names_schema(&mut self, vector_name: &str, version: Option<SeqNumberType>) {
bump_key(&mut self.vector_names_schema, vector_name, version)
}
}
fn bump(current: Option<SeqNumberType>, new: Option<SeqNumberType>) -> Option<SeqNumberType> {
lib/segment/src/segment_constructor/segment_constructor_base.rs
+29 -4
View File
@@ -191,6 +191,19 @@ pub(crate) fn open_vector_storage(
AdviceSetting::from(Advice::Normal),
true,
),
// Empty placeholder storage, no files on disk
VectorStorageType::Empty => {
use crate::vector_storage::dense::empty_dense_vector_storage::new_empty_dense_vector_storage;
Ok(new_empty_dense_vector_storage(
vector_config.size,
vector_config.distance,
vector_config.datatype.unwrap_or_default(),
vector_config.storage_type.is_on_disk(),
vector_config.multivector_config,
0, // num_points set after id_tracker is loaded
))
}
}
}
@@ -371,6 +384,10 @@ pub(crate) fn create_sparse_vector_storage(
let mmap_storage = MmapSparseVectorStorage::open_or_create(path)?;
Ok(VectorStorageEnum::SparseMmap(mmap_storage))
}
SparseVectorStorageType::Empty => {
use crate::vector_storage::sparse::empty_sparse_vector_storage::new_empty_sparse_vector_storage;
Ok(new_empty_sparse_vector_storage(0))
}
}
}
@@ -451,14 +468,18 @@ fn create_segment(
let vector_storage = vector_storages.remove(vector_name).unwrap();
let vector_index_path = get_vector_index_path(segment_path, vector_name);
// Warn when number of points between ID tracker and storage differs
// Ensure vector storage is sized to match the id_tracker's point count.
// This can be out of sync when a named vector was added to an existing segment.
let point_count = id_tracker.borrow().total_point_count();
let vector_count = vector_storage.borrow().total_vector_count();
if vector_count != point_count {
log::debug!(
"Mismatch of point and vector counts ({point_count} != {vector_count}, storage: {})",
"Mismatch of point and vector counts ({point_count} != {vector_count}, storage: {}), pre-filling deleted entries",
vector_storage_path.display(),
);
vector_storage
.borrow_mut()
.prefill_deleted_entries(point_count)?;
}
let started = Instant::now();
@@ -515,14 +536,18 @@ fn create_segment(
let vector_index_path = get_vector_index_path(segment_path, vector_name);
let vector_storage = vector_storages.remove(vector_name).unwrap();
// Warn when number of points between ID tracker and storage differs
// Ensure vector storage is sized to match the id_tracker's point count.
// This can be out of sync when a named vector was added to an existing segment.
let point_count = id_tracker.borrow().total_point_count();
let vector_count = vector_storage.borrow().total_vector_count();
if vector_count != point_count {
log::debug!(
"Mismatch of point and vector counts ({point_count} != {vector_count}, storage: {})",
"Mismatch of point and vector counts ({point_count} != {vector_count}, storage: {}), pre-filling deleted entries",
vector_storage_path.display(),
);
vector_storage
.borrow_mut()
.prefill_deleted_entries(point_count)?;
}
let started = Instant::now();
lib/segment/src/types.rs
+20 -2
View File
@@ -1544,6 +1544,10 @@ pub enum VectorStorageType {
/// Storage in a single mmap file, not appendable
/// Pre-fetched into RAM on load
InRamMmap,
/// Placeholder storage: contains no data, all vectors reported as deleted.
/// Used for newly created named vectors on immutable segments.
/// No files on disk, reconstructed from config on load.
Empty,
}
#[cfg(any(test, feature = "testing"))]
@@ -1556,7 +1560,7 @@ impl Default for VectorStorageType {
/// Storage types for vectors
#[derive(
Default, Debug, Deserialize, Serialize, JsonSchema, Anonymize, Eq, PartialEq, Copy, Clone,
Default, Debug, Deserialize, Serialize, JsonSchema, Anonymize, Eq, PartialEq, Copy, Clone, Hash,
)]
#[serde(rename_all = "snake_case")]
pub enum VectorStorageDatatype {
@@ -1619,8 +1623,18 @@ impl VectorStorageType {
match self {
Self::Memory | Self::InRamChunkedMmap | Self::InRamMmap => false,
Self::Mmap | Self::ChunkedMmap => true,
// Empty storage has no actual data; report based on what the
// runtime EmptyDenseVectorStorage was configured with.
// This fallback returns true to be safe, but callers that need
// the real on-disk status should check the storage instance.
Self::Empty => true,
}
}
/// Whether this is a placeholder empty storage type
pub fn is_empty(&self) -> bool {
matches!(self, Self::Empty)
}
}
/// Config of single vector data storage
@@ -1660,6 +1674,7 @@ impl VectorDataConfig {
VectorStorageType::ChunkedMmap => true,
VectorStorageType::InRamChunkedMmap => true,
VectorStorageType::InRamMmap => false,
VectorStorageType::Empty => false,
};
is_index_appendable && is_storage_appendable
}
@@ -1726,6 +1741,9 @@ pub enum SparseVectorStorageType {
/// Storage in memory maps (gridstore storage)
#[default]
Mmap,
/// Placeholder storage: contains no data, all vectors reported as deleted.
/// Used for newly created sparse named vectors on immutable segments.
Empty,
}
impl SparseVectorStorageType {
@@ -1734,7 +1752,7 @@ impl SparseVectorStorageType {
match self {
// Both options are on disk, but we keep it explicit for the case if someone adds a new
// storage type in the future
Self::Mmap => true,
Self::Mmap | Self::Empty => true,
}
}
}
lib/segment/src/vector_storage/dense/empty_dense_vector_storage.rs
+299
View File
@@ -0,0 +1,299 @@
use std::borrow::Cow;
use std::ops::Range;
use std::path::PathBuf;
use std::sync::atomic::AtomicBool;
use common::bitvec::{BitSlice, BitVec};
use common::counter::hardware_counter::HardwareCounterCell;
use common::generic_consts::AccessPattern;
use common::types::PointOffsetType;
use crate::common::Flusher;
use crate::common::operation_error::{OperationError, OperationResult};
use crate::data_types::named_vectors::CowVector;
use crate::data_types::vectors::{VectorElementType, VectorRef};
use crate::types::{Distance, MultiVectorConfig, VectorStorageDatatype};
use crate::vector_storage::{DenseVectorStorage, VectorStorage, VectorStorageEnum};
/// Placeholder vector storage that contains no data.
///
/// All vectors are reported as deleted. Used for newly created named vectors
/// on immutable segments where no actual vector data exists yet.
/// Reconstructed from config on segment load (no files on disk).
#[derive(Debug)]
pub struct EmptyDenseVectorStorage {
distance: Distance,
dim: usize,
datatype: VectorStorageDatatype,
is_on_disk: bool,
multi_vector_config: Option<MultiVectorConfig>,
/// Number of points in this storage (all reported as deleted)
num_points: usize,
/// All-ones bitvec indicating every vector is deleted
deleted_bitvec: BitVec,
}
impl EmptyDenseVectorStorage {
pub fn new(
dim: usize,
distance: Distance,
datatype: VectorStorageDatatype,
is_on_disk: bool,
multi_vector_config: Option<MultiVectorConfig>,
num_points: usize,
) -> Self {
Self {
distance,
dim,
datatype,
is_on_disk,
multi_vector_config,
num_points,
deleted_bitvec: BitVec::repeat(true, num_points),
}
}
/// Update the number of points. All points are marked as deleted.
pub fn set_num_points(&mut self, num_points: usize) {
self.num_points = num_points;
self.deleted_bitvec.resize(num_points, true);
}
pub fn multi_vector_config(&self) -> Option<&MultiVectorConfig> {
self.multi_vector_config.as_ref()
}
}
pub fn new_empty_dense_vector_storage(
dim: usize,
distance: Distance,
datatype: VectorStorageDatatype,
is_on_disk: bool,
multi_vector_config: Option<MultiVectorConfig>,
num_points: usize,
) -> VectorStorageEnum {
VectorStorageEnum::EmptyDense(EmptyDenseVectorStorage::new(
dim,
distance,
datatype,
is_on_disk,
multi_vector_config,
num_points,
))
}
impl DenseVectorStorage<VectorElementType> for EmptyDenseVectorStorage {
fn vector_dim(&self) -> usize {
self.dim
}
fn get_dense<P: AccessPattern>(&self, _key: PointOffsetType) -> Cow<'_, [VectorElementType]> {
debug_assert!(false, "get_dense called on EmptyDenseVectorStorage");
Cow::Owned(vec![0.0; self.dim])
}
}
impl VectorStorage for EmptyDenseVectorStorage {
fn distance(&self) -> Distance {
self.distance
}
fn datatype(&self) -> VectorStorageDatatype {
self.datatype
}
fn is_on_disk(&self) -> bool {
self.is_on_disk
}
fn total_vector_count(&self) -> usize {
self.num_points
}
fn get_vector<P: AccessPattern>(&self, _key: PointOffsetType) -> CowVector<'_> {
debug_assert!(false, "get_vector called on EmptyDenseVectorStorage");
CowVector::from(vec![0.0; self.dim])
}
fn get_vector_opt<P: AccessPattern>(&self, _key: PointOffsetType) -> Option<CowVector<'_>> {
None
}
fn insert_vector(
&mut self,
_key: PointOffsetType,
_vector: VectorRef,
_hw_counter: &HardwareCounterCell,
) -> OperationResult<()> {
Err(OperationError::service_error(
"Cannot insert into empty vector storage",
))
}
fn update_from<'a>(
&mut self,
_other_vectors: &'a mut impl Iterator<Item = (CowVector<'a>, bool)>,
_stopped: &AtomicBool,
) -> OperationResult<Range<PointOffsetType>> {
Err(OperationError::service_error(
"Cannot update empty vector storage",
))
}
fn flusher(&self) -> Flusher {
Box::new(|| Ok(()))
}
fn files(&self) -> Vec<PathBuf> {
vec![]
}
fn delete_vector(&mut self, _key: PointOffsetType) -> OperationResult<bool> {
Ok(false) // Already deleted
}
fn is_deleted_vector(&self, _key: PointOffsetType) -> bool {
true
}
fn deleted_vector_count(&self) -> usize {
self.num_points
}
fn deleted_vector_bitslice(&self) -> &BitSlice {
self.deleted_bitvec.as_bitslice()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_empty_dense_basic_contract() {
let storage = EmptyDenseVectorStorage::new(
128,
Distance::Cosine,
VectorStorageDatatype::Float32,
true,
None,
1000,
);
assert_eq!(storage.distance(), Distance::Cosine);
assert_eq!(storage.datatype(), VectorStorageDatatype::Float32);
assert!(storage.is_on_disk());
assert_eq!(storage.total_vector_count(), 1000);
assert_eq!(storage.available_vector_count(), 0);
assert_eq!(storage.deleted_vector_count(), 1000);
assert_eq!(storage.deleted_vector_bitslice().len(), 1000);
assert!(storage.is_deleted_vector(0));
assert!(storage.is_deleted_vector(999));
assert_eq!(storage.vector_dim(), 128);
assert!(storage.files().is_empty());
assert!(storage.multi_vector_config().is_none());
// get_vector_opt always returns None
assert!(
storage
.get_vector_opt::<common::generic_consts::Random>(0)
.is_none()
);
assert!(
storage
.get_vector_opt::<common::generic_consts::Random>(500)
.is_none()
);
}
#[test]
fn test_empty_dense_respects_on_disk_flag() {
let storage_on_disk = EmptyDenseVectorStorage::new(
64,
Distance::Dot,
VectorStorageDatatype::Float32,
true,
None,
0,
);
assert!(storage_on_disk.is_on_disk());
let storage_in_ram = EmptyDenseVectorStorage::new(
64,
Distance::Dot,
VectorStorageDatatype::Float32,
false,
None,
0,
);
assert!(!storage_in_ram.is_on_disk());
}
#[test]
fn test_empty_dense_multi_vector_config() {
let multi_cfg = MultiVectorConfig {
comparator: crate::types::MultiVectorComparator::MaxSim,
};
let storage = EmptyDenseVectorStorage::new(
64,
Distance::Cosine,
VectorStorageDatatype::Float32,
true,
Some(multi_cfg),
0,
);
assert!(storage.multi_vector_config().is_some());
}
#[test]
fn test_empty_dense_set_num_points() {
let mut storage = EmptyDenseVectorStorage::new(
64,
Distance::Dot,
VectorStorageDatatype::Float32,
true,
None,
0,
);
assert_eq!(storage.total_vector_count(), 0);
assert_eq!(storage.deleted_vector_count(), 0);
storage.set_num_points(500);
assert_eq!(storage.total_vector_count(), 500);
assert_eq!(storage.deleted_vector_count(), 500);
assert_eq!(storage.deleted_vector_bitslice().len(), 500);
}
#[test]
fn test_empty_dense_insert_errors() {
let mut storage = EmptyDenseVectorStorage::new(
4,
Distance::Cosine,
VectorStorageDatatype::Float32,
true,
None,
10,
);
let vector = vec![1.0, 2.0, 3.0, 4.0];
let result = storage.insert_vector(
0,
VectorRef::from(&vector),
&HardwareCounterCell::disposable(),
);
assert!(result.is_err());
}
#[test]
fn test_empty_dense_delete_is_noop() {
let mut storage = EmptyDenseVectorStorage::new(
4,
Distance::Cosine,
VectorStorageDatatype::Float32,
true,
None,
10,
);
// delete_vector returns false because it was already deleted
assert!(!storage.delete_vector(0).unwrap());
}
}
lib/segment/src/vector_storage/dense/mod.rs
+1
View File
@@ -1,4 +1,5 @@
pub mod appendable_dense_vector_storage;
pub mod dense_vector_storage;
pub mod empty_dense_vector_storage;
pub mod immutable_dense_vectors;
pub mod volatile_dense_vector_storage;
lib/segment/src/vector_storage/mod.rs
+1
View File
@@ -4,6 +4,7 @@ mod chunked_vectors;
pub mod common;
pub mod dense;
pub mod multi_dense;
mod prefill_deleted;
pub mod quantized;
pub mod query;
pub mod query_scorer;
lib/segment/src/vector_storage/prefill_deleted.rs
+75
View File
@@ -0,0 +1,75 @@
use std::sync::atomic::AtomicBool;
use super::VectorStorage as _;
use super::vector_storage_base::VectorStorageEnum;
use crate::common::operation_error::OperationResult;
use crate::data_types::named_vectors::CowVector;
impl VectorStorageEnum {
/// Ensure the storage has at least `num_points` entries, with any newly
/// created entries marked as deleted.
///
/// This is needed when a new named vector is added to a segment that
/// already contains points — the storage must be sized to match the
/// id_tracker's point count.
pub fn prefill_deleted_entries(&mut self, num_points: usize) -> OperationResult<()> {
if num_points == 0 || self.total_vector_count() >= num_points {
return Ok(());
}
let entries_to_add = num_points - self.total_vector_count();
match self {
// Empty storages can adjust their size directly.
VectorStorageEnum::EmptyDense(v) => {
v.set_num_points(num_points);
return Ok(());
}
VectorStorageEnum::EmptySparse(v) => {
v.set_num_points(num_points);
return Ok(());
}
// All other storages need to be extended via update_from.
// We just need to know which default vector to use.
VectorStorageEnum::DenseVolatile(_)
| VectorStorageEnum::DenseMemmap(_)
| VectorStorageEnum::DenseMemmapByte(_)
| VectorStorageEnum::DenseMemmapHalf(_)
| VectorStorageEnum::DenseAppendableMemmap(_)
| VectorStorageEnum::DenseAppendableMemmapByte(_)
| VectorStorageEnum::DenseAppendableMemmapHalf(_)
| VectorStorageEnum::MultiDenseVolatile(_)
| VectorStorageEnum::MultiDenseAppendableMemmap(_)
| VectorStorageEnum::MultiDenseAppendableMemmapByte(_)
| VectorStorageEnum::MultiDenseAppendableMemmapHalf(_) => {}
#[cfg(test)]
VectorStorageEnum::DenseVolatileByte(_)
| VectorStorageEnum::DenseVolatileHalf(_)
| VectorStorageEnum::MultiDenseVolatileByte(_)
| VectorStorageEnum::MultiDenseVolatileHalf(_) => {}
#[cfg(target_os = "linux")]
VectorStorageEnum::DenseUring(_)
| VectorStorageEnum::DenseUringByte(_)
| VectorStorageEnum::DenseUringHalf(_) => {}
VectorStorageEnum::SparseVolatile(_) | VectorStorageEnum::SparseMmap(_) => {
let stopped = AtomicBool::new(false);
let mut iter =
std::iter::repeat_n((CowVector::default_sparse(), true), entries_to_add);
self.update_from(&mut iter, &stopped)?;
return Ok(());
}
}
// Dense / multi-dense: fill with zero vectors of the appropriate dimension.
let default_vector = CowVector::from(self.default_vector());
let stopped = AtomicBool::new(false);
let mut iter = std::iter::repeat_n((default_vector, true), entries_to_add);
self.update_from(&mut iter, &stopped)?;
Ok(())
}
}
lib/segment/src/vector_storage/quantized/quantized_vectors.rs
+9
View File
@@ -614,6 +614,15 @@ impl QuantizedVectors {
max_threads,
stopped,
),
VectorStorageEnum::EmptyDense(v) => Self::create_impl(
v,
quantization_config,
storage_type,
path,
max_threads,
stopped,
),
VectorStorageEnum::EmptySparse(_) => Err(OperationError::WrongSparse),
}
}
lib/segment/src/vector_storage/raw_scorer.rs
+2
View File
@@ -92,6 +92,8 @@ pub fn new_raw_scorer<'a>(
VectorStorageEnum::MultiDenseAppendableMemmapHalf(vs) => {
raw_multi_scorer_impl(query, vs.as_ref(), hc)
}
VectorStorageEnum::EmptyDense(vs) => raw_scorer_impl(query, vs, hc),
VectorStorageEnum::EmptySparse(vs) => raw_sparse_scorer_impl(query, vs, hc),
}
}
lib/segment/src/vector_storage/sparse/empty_sparse_vector_storage.rs
+177
View File
@@ -0,0 +1,177 @@
use std::ops::Range;
use std::path::PathBuf;
use std::sync::atomic::AtomicBool;
use common::bitvec::{BitSlice, BitVec};
use common::counter::hardware_counter::HardwareCounterCell;
use common::generic_consts::AccessPattern;
use common::types::PointOffsetType;
use sparse::common::sparse_vector::SparseVector;
use crate::common::Flusher;
use crate::common::operation_error::{OperationError, OperationResult};
use crate::data_types::named_vectors::CowVector;
use crate::data_types::vectors::VectorRef;
use crate::types::{Distance, VectorStorageDatatype};
use crate::vector_storage::sparse::SPARSE_VECTOR_DISTANCE;
use crate::vector_storage::{SparseVectorStorage, VectorStorage, VectorStorageEnum};
/// Placeholder sparse vector storage that contains no data.
///
/// All vectors are reported as deleted. Used for newly created sparse named vectors
/// on immutable segments where no actual vector data exists yet.
/// Reconstructed from config on segment load (no files on disk).
#[derive(Debug)]
pub struct EmptySparseVectorStorage {
/// Number of points in this storage (all reported as deleted)
num_points: usize,
/// All-ones bitvec indicating every vector is deleted
deleted_bitvec: BitVec,
}
impl EmptySparseVectorStorage {
pub fn new(num_points: usize) -> Self {
Self {
num_points,
deleted_bitvec: BitVec::repeat(true, num_points),
}
}
/// Update the number of points. All points are marked as deleted.
pub fn set_num_points(&mut self, num_points: usize) {
self.num_points = num_points;
self.deleted_bitvec.resize(num_points, true);
}
}
pub fn new_empty_sparse_vector_storage(num_points: usize) -> VectorStorageEnum {
VectorStorageEnum::EmptySparse(EmptySparseVectorStorage::new(num_points))
}
impl SparseVectorStorage for EmptySparseVectorStorage {
fn get_sparse<P: AccessPattern>(&self, _key: PointOffsetType) -> OperationResult<SparseVector> {
Ok(SparseVector::default())
}
fn get_sparse_opt<P: AccessPattern>(
&self,
_key: PointOffsetType,
) -> OperationResult<Option<SparseVector>> {
Ok(None)
}
}
impl VectorStorage for EmptySparseVectorStorage {
fn distance(&self) -> Distance {
SPARSE_VECTOR_DISTANCE
}
fn datatype(&self) -> VectorStorageDatatype {
VectorStorageDatatype::Float32
}
fn is_on_disk(&self) -> bool {
true
}
fn total_vector_count(&self) -> usize {
self.num_points
}
fn get_vector<P: AccessPattern>(&self, _key: PointOffsetType) -> CowVector<'_> {
debug_assert!(false, "get_vector called on EmptySparseVectorStorage");
CowVector::default_sparse()
}
fn get_vector_opt<P: AccessPattern>(&self, _key: PointOffsetType) -> Option<CowVector<'_>> {
None
}
fn insert_vector(
&mut self,
_key: PointOffsetType,
_vector: VectorRef,
_hw_counter: &HardwareCounterCell,
) -> OperationResult<()> {
Err(OperationError::service_error(
"Cannot insert into empty sparse vector storage",
))
}
fn update_from<'a>(
&mut self,
_other_vectors: &'a mut impl Iterator<Item = (CowVector<'a>, bool)>,
_stopped: &AtomicBool,
) -> OperationResult<Range<PointOffsetType>> {
Err(OperationError::service_error(
"Cannot update empty sparse vector storage",
))
}
fn flusher(&self) -> Flusher {
Box::new(|| Ok(()))
}
fn files(&self) -> Vec<PathBuf> {
vec![]
}
fn delete_vector(&mut self, _key: PointOffsetType) -> OperationResult<bool> {
Ok(false) // Already deleted
}
fn is_deleted_vector(&self, _key: PointOffsetType) -> bool {
true
}
fn deleted_vector_count(&self) -> usize {
self.num_points
}
fn deleted_vector_bitslice(&self) -> &BitSlice {
self.deleted_bitvec.as_bitslice()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_empty_sparse_basic_contract() {
let storage = EmptySparseVectorStorage::new(500);
assert_eq!(storage.distance(), Distance::Dot);
assert_eq!(storage.datatype(), VectorStorageDatatype::Float32);
assert!(storage.is_on_disk());
assert_eq!(storage.total_vector_count(), 500);
assert_eq!(storage.available_vector_count(), 0);
assert_eq!(storage.deleted_vector_count(), 500);
assert_eq!(storage.deleted_vector_bitslice().len(), 500);
assert!(storage.is_deleted_vector(0));
assert!(storage.is_deleted_vector(499));
assert!(storage.files().is_empty());
assert!(
storage
.get_vector_opt::<common::generic_consts::Random>(0)
.is_none()
);
assert!(
storage
.get_sparse_opt::<common::generic_consts::Random>(0)
.unwrap()
.is_none()
);
}
#[test]
fn test_empty_sparse_set_num_points() {
let mut storage = EmptySparseVectorStorage::new(0);
assert_eq!(storage.total_vector_count(), 0);
storage.set_num_points(1000);
assert_eq!(storage.total_vector_count(), 1000);
assert_eq!(storage.deleted_vector_count(), 1000);
}
}
lib/segment/src/vector_storage/sparse/mod.rs
+1
View File
@@ -1,3 +1,4 @@
pub mod empty_sparse_vector_storage;
pub mod mmap_sparse_vector_storage;
mod stored_sparse_vectors;
pub mod volatile_sparse_vector_storage;
lib/segment/src/vector_storage/tests/test_appendable_multi_dense_vector_storage.rs
+3
View File
@@ -101,6 +101,9 @@ fn do_test_delete_points(vector_dim: usize, vec_count: usize, storage: &mut Vect
}
VectorStorageEnum::MultiDenseAppendableMemmapByte(_)
| VectorStorageEnum::MultiDenseAppendableMemmapHalf(_) => unreachable!(),
VectorStorageEnum::EmptyDense(_) | VectorStorageEnum::EmptySparse(_) => {
unreachable!()
}
};
}
lib/segment/src/vector_storage/vector_storage_base.rs
+50
View File
@@ -16,9 +16,11 @@ use common::universal_io::IoUringFile;
use sparse::common::sparse_vector::SparseVector;
use super::dense::dense_vector_storage::DenseVectorStorageImpl;
use super::dense::empty_dense_vector_storage::EmptyDenseVectorStorage;
use super::dense::volatile_dense_vector_storage::VolatileDenseVectorStorage;
use super::multi_dense::appendable_mmap_multi_dense_vector_storage::AppendableMmapMultiDenseVectorStorage;
use super::multi_dense::volatile_multi_dense_vector_storage::VolatileMultiDenseVectorStorage;
use super::sparse::empty_sparse_vector_storage::EmptySparseVectorStorage;
use super::sparse::mmap_sparse_vector_storage::MmapSparseVectorStorage;
use super::sparse::volatile_sparse_vector_storage::VolatileSparseVectorStorage;
use crate::common::Flusher;
@@ -264,6 +266,8 @@ pub enum VectorStorageEnum {
MultiDenseAppendableMemmapHalf(
Box<AppendableMmapMultiDenseVectorStorage<VectorElementTypeHalf>>,
),
EmptyDense(EmptyDenseVectorStorage),
EmptySparse(EmptySparseVectorStorage),
}
impl VectorStorageEnum {
@@ -298,6 +302,8 @@ impl VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(s) => Some(s.multi_vector_config()),
VectorStorageEnum::MultiDenseAppendableMemmapByte(s) => Some(s.multi_vector_config()),
VectorStorageEnum::MultiDenseAppendableMemmapHalf(s) => Some(s.multi_vector_config()),
VectorStorageEnum::EmptyDense(s) => s.multi_vector_config(),
VectorStorageEnum::EmptySparse(_) => None,
}
}
@@ -358,6 +364,8 @@ impl VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => {
VectorInternal::from(MultiDenseVectorInternal::placeholder(v.vector_dim()))
}
VectorStorageEnum::EmptyDense(v) => VectorInternal::from(vec![1.0; v.vector_dim()]),
VectorStorageEnum::EmptySparse(_) => VectorInternal::from(SparseVector::default()),
}
}
@@ -406,6 +414,8 @@ impl VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => {
v.size_of_available_vectors_in_bytes()
}
VectorStorageEnum::EmptyDense(_) => 0,
VectorStorageEnum::EmptySparse(_) => 0,
}
}
@@ -440,6 +450,8 @@ impl VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(vs) => vs.populate()?,
VectorStorageEnum::MultiDenseAppendableMemmapByte(vs) => vs.populate()?,
VectorStorageEnum::MultiDenseAppendableMemmapHalf(vs) => vs.populate()?,
VectorStorageEnum::EmptyDense(_) => {}
VectorStorageEnum::EmptySparse(_) => {}
}
Ok(())
}
@@ -475,6 +487,8 @@ impl VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(vs) => vs.clear_cache()?,
VectorStorageEnum::MultiDenseAppendableMemmapByte(vs) => vs.clear_cache()?,
VectorStorageEnum::MultiDenseAppendableMemmapHalf(vs) => vs.clear_cache()?,
VectorStorageEnum::EmptyDense(_) => {}
VectorStorageEnum::EmptySparse(_) => {}
}
Ok(())
}
@@ -519,6 +533,8 @@ impl VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(_) => None,
VectorStorageEnum::MultiDenseAppendableMemmapByte(_) => None,
VectorStorageEnum::MultiDenseAppendableMemmapHalf(_) => None,
VectorStorageEnum::EmptyDense(_) => None,
VectorStorageEnum::EmptySparse(_) => None,
}
}
@@ -554,6 +570,8 @@ impl VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(_) => {}
VectorStorageEnum::MultiDenseAppendableMemmapByte(_) => {}
VectorStorageEnum::MultiDenseAppendableMemmapHalf(_) => {}
VectorStorageEnum::EmptyDense(_) => {}
VectorStorageEnum::EmptySparse(_) => {}
}
Err(OperationError::service_error(
"Vector layout is not implemented for this storage",
@@ -593,6 +611,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(v) => v.distance(),
VectorStorageEnum::MultiDenseAppendableMemmapByte(v) => v.distance(),
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => v.distance(),
VectorStorageEnum::EmptyDense(v) => v.distance(),
VectorStorageEnum::EmptySparse(v) => v.distance(),
}
}
@@ -627,6 +647,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(v) => v.datatype(),
VectorStorageEnum::MultiDenseAppendableMemmapByte(v) => v.datatype(),
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => v.datatype(),
VectorStorageEnum::EmptyDense(v) => v.datatype(),
VectorStorageEnum::EmptySparse(v) => v.datatype(),
}
}
@@ -663,6 +685,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(v) => v.is_on_disk(),
VectorStorageEnum::MultiDenseAppendableMemmapByte(v) => v.is_on_disk(),
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => v.is_on_disk(),
VectorStorageEnum::EmptyDense(v) => v.is_on_disk(),
VectorStorageEnum::EmptySparse(v) => v.is_on_disk(),
}
}
@@ -697,6 +721,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(v) => v.total_vector_count(),
VectorStorageEnum::MultiDenseAppendableMemmapByte(v) => v.total_vector_count(),
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => v.total_vector_count(),
VectorStorageEnum::EmptyDense(v) => v.total_vector_count(),
VectorStorageEnum::EmptySparse(v) => v.total_vector_count(),
}
}
@@ -731,6 +757,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(v) => v.get_vector::<P>(key),
VectorStorageEnum::MultiDenseAppendableMemmapByte(v) => v.get_vector::<P>(key),
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => v.get_vector::<P>(key),
VectorStorageEnum::EmptyDense(v) => v.get_vector::<P>(key),
VectorStorageEnum::EmptySparse(v) => v.get_vector::<P>(key),
}
}
@@ -773,6 +801,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => {
v.read_vectors::<P>(keys, callback)
}
VectorStorageEnum::EmptyDense(v) => v.read_vectors::<P>(keys, callback),
VectorStorageEnum::EmptySparse(v) => v.read_vectors::<P>(keys, callback),
}
}
@@ -807,6 +837,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(v) => v.get_vector_opt::<P>(key),
VectorStorageEnum::MultiDenseAppendableMemmapByte(v) => v.get_vector_opt::<P>(key),
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => v.get_vector_opt::<P>(key),
VectorStorageEnum::EmptyDense(v) => v.get_vector_opt::<P>(key),
VectorStorageEnum::EmptySparse(v) => v.get_vector_opt::<P>(key),
}
}
@@ -860,6 +892,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => {
v.insert_vector(key, vector, hw_counter)
}
VectorStorageEnum::EmptyDense(v) => v.insert_vector(key, vector, hw_counter),
VectorStorageEnum::EmptySparse(v) => v.insert_vector(key, vector, hw_counter),
}
}
@@ -908,6 +942,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => {
v.update_from(other_vectors, stopped)
}
VectorStorageEnum::EmptyDense(v) => v.update_from(other_vectors, stopped),
VectorStorageEnum::EmptySparse(v) => v.update_from(other_vectors, stopped),
}
}
@@ -942,6 +978,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(v) => v.flusher(),
VectorStorageEnum::MultiDenseAppendableMemmapByte(v) => v.flusher(),
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => v.flusher(),
VectorStorageEnum::EmptyDense(v) => v.flusher(),
VectorStorageEnum::EmptySparse(v) => v.flusher(),
}
}
@@ -976,6 +1014,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(v) => v.files(),
VectorStorageEnum::MultiDenseAppendableMemmapByte(v) => v.files(),
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => v.files(),
VectorStorageEnum::EmptyDense(v) => v.files(),
VectorStorageEnum::EmptySparse(v) => v.files(),
}
}
@@ -1010,6 +1050,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(v) => v.immutable_files(),
VectorStorageEnum::MultiDenseAppendableMemmapByte(v) => v.immutable_files(),
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => v.immutable_files(),
VectorStorageEnum::EmptyDense(v) => v.immutable_files(),
VectorStorageEnum::EmptySparse(v) => v.immutable_files(),
}
}
@@ -1044,6 +1086,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(v) => v.delete_vector(key),
VectorStorageEnum::MultiDenseAppendableMemmapByte(v) => v.delete_vector(key),
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => v.delete_vector(key),
VectorStorageEnum::EmptyDense(v) => v.delete_vector(key),
VectorStorageEnum::EmptySparse(v) => v.delete_vector(key),
}
}
@@ -1078,6 +1122,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(v) => v.is_deleted_vector(key),
VectorStorageEnum::MultiDenseAppendableMemmapByte(v) => v.is_deleted_vector(key),
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => v.is_deleted_vector(key),
VectorStorageEnum::EmptyDense(v) => v.is_deleted_vector(key),
VectorStorageEnum::EmptySparse(v) => v.is_deleted_vector(key),
}
}
@@ -1112,6 +1158,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(v) => v.deleted_vector_count(),
VectorStorageEnum::MultiDenseAppendableMemmapByte(v) => v.deleted_vector_count(),
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => v.deleted_vector_count(),
VectorStorageEnum::EmptyDense(v) => v.deleted_vector_count(),
VectorStorageEnum::EmptySparse(v) => v.deleted_vector_count(),
}
}
@@ -1146,6 +1194,8 @@ impl VectorStorage for VectorStorageEnum {
VectorStorageEnum::MultiDenseAppendableMemmap(v) => v.deleted_vector_bitslice(),
VectorStorageEnum::MultiDenseAppendableMemmapByte(v) => v.deleted_vector_bitslice(),
VectorStorageEnum::MultiDenseAppendableMemmapHalf(v) => v.deleted_vector_bitslice(),
VectorStorageEnum::EmptyDense(v) => v.deleted_vector_bitslice(),
VectorStorageEnum::EmptySparse(v) => v.deleted_vector_bitslice(),
}
}
}
lib/shard/src/operations/mod.rs
+44
View File
@@ -3,6 +3,7 @@ pub mod payload_ops;
pub mod point_ops;
#[cfg(feature = "staging")]
pub mod staging;
pub mod vector_name_ops;
pub mod vector_ops;
use segment::json_path::JsonPath;
@@ -10,6 +11,9 @@ use segment::types::{PayloadFieldSchema, PointIdType};
use serde::{Deserialize, Serialize};
use strum::{EnumDiscriminants, EnumIter};
pub use self::vector_name_ops::{
CreateVectorName, DeleteVectorName, VectorNameConfig, VectorNameOperations,
};
use crate::PeerId;
use crate::operations::point_ops::PointOperations;
@@ -21,6 +25,7 @@ pub enum CollectionUpdateOperations {
VectorOperation(vector_ops::VectorOperations),
PayloadOperation(payload_ops::PayloadOps),
FieldIndexOperation(FieldIndexOperations),
VectorNameOperation(VectorNameOperations),
/// Staging-only operations for testing and debugging purposes
#[cfg(feature = "staging")]
StagingOperation(staging::StagingOperations),
@@ -47,6 +52,7 @@ impl CollectionUpdateOperations {
Self::VectorOperation(op) => op.point_ids(),
Self::PayloadOperation(op) => op.point_ids(),
Self::FieldIndexOperation(_) => None,
Self::VectorNameOperation(_) => None,
#[cfg(feature = "staging")]
Self::StagingOperation(_) => None,
}
@@ -68,6 +74,7 @@ impl CollectionUpdateOperations {
Self::VectorOperation(_) => None,
Self::PayloadOperation(_) => None,
Self::FieldIndexOperation(_) => None,
Self::VectorNameOperation(_) => None,
#[cfg(feature = "staging")]
Self::StagingOperation(_) => None,
}
@@ -82,6 +89,7 @@ impl CollectionUpdateOperations {
Self::VectorOperation(op) => op.retain_point_ids(filter),
Self::PayloadOperation(op) => op.retain_point_ids(filter),
Self::FieldIndexOperation(_) => (),
Self::VectorNameOperation(_) => (),
#[cfg(feature = "staging")]
Self::StagingOperation(_) => (),
}
@@ -290,6 +298,7 @@ mod tests {
any::<vector_ops::VectorOperations>().prop_map(Self::VectorOperation),
any::<payload_ops::PayloadOps>().prop_map(Self::PayloadOperation),
any::<FieldIndexOperations>().prop_map(Self::FieldIndexOperation),
any::<VectorNameOperations>().prop_map(Self::VectorNameOperation),
]
.boxed()
}
@@ -420,6 +429,41 @@ mod tests {
}
}
impl Arbitrary for VectorNameOperations {
type Parameters = ();
type Strategy = BoxedStrategy<Self>;
fn arbitrary_with(_: Self::Parameters) -> Self::Strategy {
use crate::operations::vector_name_ops::{
self as vnops, DenseVectorConfig, SparseVectorConfig,
};
let create_dense = Self::CreateVectorName(CreateVectorName {
vector_name: "test_vector".into(),
config: vnops::VectorNameConfig::dense(DenseVectorConfig {
size: 4,
distance: Distance::Cosine,
multivector_config: None,
datatype: None,
}),
});
let create_sparse = Self::CreateVectorName(CreateVectorName {
vector_name: "sparse_test".into(),
config: vnops::VectorNameConfig::sparse(SparseVectorConfig {
modifier: None,
datatype: None,
}),
});
let delete = Self::DeleteVectorName(DeleteVectorName {
vector_name: "test_vector".into(),
});
prop_oneof![Just(create_dense), Just(create_sparse), Just(delete),].boxed()
}
}
#[test]
fn test_delete_by_filter_with_has_id_uuids_cbor_roundtrip() {
let uuids: Vec<PointIdType> = vec![ExtendedPointId::Uuid(
lib/shard/src/operations/vector_name_ops.rs
+32
View File
@@ -0,0 +1,32 @@
// Re-export VectorNameConfig types from segment crate
pub use segment::data_types::vector_name_config::{
DenseVectorConfig, SparseVectorConfig, VectorNameConfig,
};
use segment::types::VectorNameBuf;
use serde::{Deserialize, Serialize};
use strum::{EnumDiscriminants, EnumIter};
/// Operations for creating and deleting named vectors at the shard level.
/// Serialized into WAL.
#[derive(Clone, Debug, PartialEq, Deserialize, Serialize, EnumDiscriminants, Hash)]
#[strum_discriminants(derive(EnumIter))]
#[serde(rename_all = "snake_case")]
pub enum VectorNameOperations {
/// Create a new named vector in the shard
CreateVectorName(CreateVectorName),
/// Delete a named vector from the shard
DeleteVectorName(DeleteVectorName),
}
#[derive(Clone, Debug, PartialEq, Deserialize, Serialize, Hash)]
#[serde(rename_all = "snake_case")]
pub struct CreateVectorName {
pub vector_name: VectorNameBuf,
pub config: VectorNameConfig,
}
#[derive(Clone, Debug, PartialEq, Deserialize, Serialize, Hash)]
#[serde(rename_all = "snake_case")]
pub struct DeleteVectorName {
pub vector_name: VectorNameBuf,
}
lib/shard/src/optimize.rs
+58 -1
View File
@@ -33,7 +33,10 @@ use segment::types::PointIdType;
use uuid::Uuid;
use crate::locked_segment::LockedSegment;
use crate::proxy_segment::{DeletedPoints, ProxyIndexChange, ProxyIndexChanges, ProxySegment};
use crate::proxy_segment::{
DeletedPoints, IntendedVector, ProxyIndexChange, ProxyIndexChanges, ProxySegment,
ProxyVectorNameChanges,
};
use crate::segment_holder::SegmentId;
use crate::segment_holder::locked::LockedSegmentHolder;
@@ -149,6 +152,28 @@ pub fn proxy_index_changes(proxies: &[LockedSegment]) -> ProxyIndexChanges {
index_changes
}
/// Accumulates vector name changes made in a given set of proxies
///
/// This list is not synchronized (if not externally enforced),
/// but guarantees that it contains at least all vector name changes made in the proxies
/// before the call to this function.
pub fn proxy_vector_name_changes(proxies: &[LockedSegment]) -> ProxyVectorNameChanges {
let mut changes = ProxyVectorNameChanges::default();
for proxy_segment in proxies {
match proxy_segment {
LockedSegment::Original(_) => {
log::error!("Reading raw segment, while proxy expected");
debug_assert!(false, "Reading raw segment, while proxy expected");
}
LockedSegment::Proxy(proxy) => {
let proxy_read = proxy.read();
changes.merge(proxy_read.get_vector_name_changes())
}
}
}
changes
}
/// Function to wrap slow part of optimization. Performs proxy rollback in case of cancellation.
/// Warn: this function might be _VERY_ CPU intensive,
/// so it is necessary to avoid any locks inside this part of the code
@@ -421,6 +446,38 @@ fn finish_optimization(
// This mutex prevents update operations, which could create inconsistency during transition.
let update_guard = segment_holder.acquire_updates_lock();
// Apply vector name changes before index and point changes
// New named vectors must exist before indexes or points reference them
let old_optimized_segment_version = optimized_segment.version();
let vector_name_changes = proxy_vector_name_changes(&locked_proxies);
for (vector_name, intent) in vector_name_changes.iter_ordered() {
debug_assert!(
intent.version() >= old_optimized_segment_version,
"proxied vector name change should have newer version than segment",
);
match intent {
IntendedVector::Absent { version } => {
optimized_segment.delete_vector_name(*version, vector_name)?;
}
IntendedVector::Present {
config,
version,
supersedes_wrapped,
} => {
if *supersedes_wrapped {
// The optimised segment was built from the wrapped data,
// so it currently carries the *old* schema for this name.
// `create_vector_name_impl` is idempotent and would
// silently keep that old storage; clear it first so the
// new schema actually takes effect.
optimized_segment.delete_vector_name(*version, vector_name)?;
}
optimized_segment.create_vector_name(*version, vector_name, config)?;
}
}
check_process_stopped(stopped)?;
}
let index_changes = proxy_index_changes(&locked_proxies);
// Apply index changes before point deletions
lib/shard/src/optimizers/config_mismatch_optimizer.rs
+3 -1
View File
@@ -100,7 +100,9 @@ impl ConfigMismatchOptimizer {
}
}
if let Some(is_required_on_disk) = self.check_if_vectors_on_disk(vector_name)
if !vector_data.storage_type.is_empty()
&& let Some(is_required_on_disk) =
self.check_if_vectors_on_disk(vector_name)
&& is_required_on_disk != vector_data.storage_type.is_on_disk()
{
return true;
lib/shard/src/proxy_segment/mod.rs
+51 -5
View File
@@ -1,9 +1,12 @@
pub mod segment_entry;
pub mod snapshot_entry;
mod vector_name_changes;
#[cfg(test)]
mod tests;
use std::borrow::Cow;
use ahash::AHashMap;
use common::bitvec::BitVec;
use common::counter::hardware_counter::HardwareCounterCell;
@@ -12,6 +15,7 @@ use itertools::Itertools as _;
use segment::common::operation_error::OperationResult;
use segment::types::*;
pub use self::vector_name_changes::{IntendedVector, ProxyVectorNameChanges};
use crate::locked_segment::LockedSegment;
pub type DeletedPoints = AHashMap<PointIdType, ProxyDeletedPoint>;
@@ -28,6 +32,7 @@ pub struct ProxySegment {
/// Used for faster deletion checks
deleted_mask: Option<BitVec>,
changed_indexes: ProxyIndexChanges,
changed_vector_names: ProxyVectorNameChanges,
/// Points which should no longer used from wrapped_segment
/// May contain points which are not in wrapped_segment,
/// because the set is shared among all proxy segments
@@ -63,6 +68,7 @@ impl ProxySegment {
wrapped_segment: segment,
deleted_mask,
changed_indexes: ProxyIndexChanges::default(),
changed_vector_names: ProxyVectorNameChanges::default(),
deleted_points: AHashMap::new(),
deleted_deferred_count: 0,
wrapped_config,
@@ -129,8 +135,12 @@ impl ProxySegment {
}
}
/// Build a filter that excludes the given deleted points. Accepts
/// `Option<Cow<Filter>>` so that a filter already owned by the caller
/// (e.g. from [`ProxyVectorNameChanges::redact_filter`]) is reused
/// without an extra clone.
fn add_deleted_points_condition_to_filter(
filter: Option<&Filter>,
filter: Option<Cow<'_, Filter>>,
deleted_points: impl IntoIterator<Item = PointIdType>,
) -> Filter {
#[allow(clippy::from_iter_instead_of_collect)]
@@ -138,10 +148,8 @@ impl ProxySegment {
match filter {
None => Filter::new_must_not(wrapper_condition),
Some(f) => {
let mut new_filter = f.clone();
let must_not = new_filter.must_not;
let new_must_not = match must_not {
let mut new_filter = f.into_owned();
let new_must_not = match new_filter.must_not {
None => Some(vec![wrapper_condition]),
Some(mut conditions) => {
conditions.push(wrapper_condition);
@@ -211,6 +219,40 @@ impl ProxySegment {
}
}
// Propagate vector name changes (between index changes and point deletions)
{
if !self.changed_vector_names.is_empty() {
wrapped_segment.with_upgraded(|wrapped_segment| {
for (vector_name, intent) in self.changed_vector_names.iter_ordered() {
match intent {
IntendedVector::Absent { version } => {
wrapped_segment.delete_vector_name(*version, vector_name)?;
}
IntendedVector::Present {
config,
version,
supersedes_wrapped,
} => {
if *supersedes_wrapped {
// `create_vector_name_impl` is idempotent and would
// silently keep the wrapped's stale storage. Clear it
// first so the new schema actually takes effect.
wrapped_segment.delete_vector_name(*version, vector_name)?;
}
wrapped_segment.create_vector_name(
*version,
vector_name,
config,
)?;
}
}
}
OperationResult::Ok(())
})?;
self.changed_vector_names.clear();
}
}
// Propagate deleted points
// Lock ordering is important here and must match the flush function to prevent a deadlock
{
@@ -250,6 +292,10 @@ impl ProxySegment {
pub fn get_index_changes(&self) -> &ProxyIndexChanges {
&self.changed_indexes
}
pub fn get_vector_name_changes(&self) -> &ProxyVectorNameChanges {
&self.changed_vector_names
}
}
/// Point persion information of points to delete from a wrapped proxy segment.
lib/shard/src/proxy_segment/segment_entry.rs
+172 -31
View File
@@ -15,6 +15,7 @@ use segment::data_types::named_vectors::NamedVectors;
use segment::data_types::order_by::OrderValue;
use segment::data_types::query_context::{FormulaContext, QueryContext, SegmentQueryContext};
use segment::data_types::segment_record::SegmentRecord;
use segment::data_types::vector_name_config::VectorNameConfig;
use segment::data_types::vectors::{QueryVector, VectorInternal};
use segment::entry::StorageSegmentEntry;
use segment::entry::entry_point::{NonAppendableSegmentEntry, ReadSegmentEntry, SegmentEntry};
@@ -66,6 +67,25 @@ impl ReadSegmentEntry for ProxySegment {
params: Option<&SearchParams>,
query_context: &SegmentQueryContext,
) -> OperationResult<Vec<Vec<ScoredPoint>>> {
// If the search target is a vector that the proxy queues for deletion
// or schema replacement, the wrapped segment's storage is stale (or
// would error on a dimensionality mismatch). Short-circuit to an
// empty result-per-query batch — semantically the new vector has no
// points indexed in this segment yet.
if self.changed_vector_names.is_wrapped_data_stale(vector_name) {
return Ok(vec![Vec::new(); vectors.len()]);
}
// Strip any vector names that the proxy intends to delete or replace
// with a different schema, so the wrapped segment doesn't return
// stale data for them. `Cow::Borrowed` in the common case.
let with_vector = self
.changed_vector_names
.redact_with_vector(with_vector, &self.wrapped_config);
let with_vector = with_vector.as_ref();
let filter = filter.map(|f| self.changed_vector_names.redact_filter(f));
// Some point might be deleted after temporary segment creation
// We need to prevent them from being found by search request
// That is why we need to pass additional filter for deleted points
@@ -79,18 +99,16 @@ impl ReadSegmentEntry for ProxySegment {
let query_context_with_deleted =
query_context.fork().with_deleted_points(deleted_points);
let res = self.wrapped_segment.get().read().search_batch(
self.wrapped_segment.get().read().search_batch(
vector_name,
vectors,
with_payload,
with_vector,
filter,
filter.as_deref(),
top,
params,
&query_context_with_deleted,
);
res?
)?
} else {
let wrapped_filter = Self::add_deleted_points_condition_to_filter(
filter,
@@ -114,7 +132,7 @@ impl ReadSegmentEntry for ProxySegment {
vectors,
with_payload,
with_vector,
filter,
filter.as_deref(),
top,
params,
query_context,
@@ -155,6 +173,15 @@ impl ReadSegmentEntry for ProxySegment {
point_id: PointIdType,
hw_counter: &HardwareCounterCell,
) -> OperationResult<Option<VectorInternal>> {
// The proxy queues a delete or schema-superseding create for this
// vector — the wrapped's stored data is no longer authoritative.
// Treat the lookup as if the point had no value for this vector.
// `all_vectors` enumerates names and skips `None`s, so this also
// hides stale entries from the per-point vector dump.
if self.changed_vector_names.is_wrapped_data_stale(vector_name) {
return Ok(None);
}
if self.deleted_points.contains_key(&point_id) {
Ok(None)
} else {
@@ -174,6 +201,8 @@ impl ReadSegmentEntry for ProxySegment {
let wrapped = self.wrapped_segment.get();
let wrapped_guard = wrapped.read();
let config = wrapped_guard.config();
// Tip: self.vector already handles dropped vector names
let vector_names: Vec<_> = config
.vector_data
.keys()
@@ -216,6 +245,13 @@ impl ReadSegmentEntry for ProxySegment {
is_stopped: &AtomicBool,
deferred_behavior: DeferredBehavior,
) -> OperationResult<AHashMap<ExtendedPointId, SegmentRecord>> {
// Strip any vector names that the proxy intends to delete or replace
// with a different schema before delegating to the wrapped segment.
let with_vector = self
.changed_vector_names
.redact_with_vector(with_vector, &self.wrapped_config);
let with_vector = with_vector.as_ref();
let filtered_point_ids: Vec<PointIdType> = point_ids
.iter()
.copied()
@@ -247,11 +283,13 @@ impl ReadSegmentEntry for ProxySegment {
hw_counter: &HardwareCounterCell,
deferred_behavior: DeferredBehavior,
) -> OperationResult<Vec<PointIdType>> {
let filter = filter.map(|f| self.changed_vector_names.redact_filter(f));
if self.deleted_points.is_empty() {
self.wrapped_segment.get().read().read_filtered(
offset,
limit,
filter,
filter.as_deref(),
is_stopped,
hw_counter,
deferred_behavior,
@@ -281,10 +319,12 @@ impl ReadSegmentEntry for ProxySegment {
hw_counter: &HardwareCounterCell,
deferred_behavior: DeferredBehavior,
) -> OperationResult<Vec<(OrderValue, PointIdType)>> {
let filter = filter.map(|f| self.changed_vector_names.redact_filter(f));
let read_points = if self.deleted_points.is_empty() {
self.wrapped_segment.get().read().read_ordered_filtered(
limit,
filter,
filter.as_deref(),
order_by,
is_stopped,
hw_counter,
@@ -314,11 +354,15 @@ impl ReadSegmentEntry for ProxySegment {
is_stopped: &AtomicBool,
hw_counter: &HardwareCounterCell,
) -> OperationResult<Vec<PointIdType>> {
let filter = filter.map(|f| self.changed_vector_names.redact_filter(f));
if self.deleted_points.is_empty() {
self.wrapped_segment
.get()
.read()
.read_random_filtered(limit, filter, is_stopped, hw_counter)
self.wrapped_segment.get().read().read_random_filtered(
limit,
filter.as_deref(),
is_stopped,
hw_counter,
)
} else {
let wrapped_filter = Self::add_deleted_points_condition_to_filter(
filter,
@@ -353,11 +397,13 @@ impl ReadSegmentEntry for ProxySegment {
is_stopped: &AtomicBool,
hw_counter: &HardwareCounterCell,
) -> OperationResult<BTreeSet<FacetValue>> {
let values = self
.wrapped_segment
.get()
.read()
.unique_values(key, filter, is_stopped, hw_counter)?;
let filter = filter.map(|f| self.changed_vector_names.redact_filter(f));
let values = self.wrapped_segment.get().read().unique_values(
key,
filter.as_deref(),
is_stopped,
hw_counter,
)?;
Ok(values)
}
@@ -367,14 +413,34 @@ impl ReadSegmentEntry for ProxySegment {
is_stopped: &AtomicBool,
hw_counter: &HardwareCounterCell,
) -> OperationResult<HashMap<FacetValue, usize>> {
let filter = request
.filter
.as_ref()
.map(|f| self.changed_vector_names.redact_filter(f));
let hits = if self.deleted_points.is_empty() {
self.wrapped_segment
.get()
.read()
.facet(request, is_stopped, hw_counter)?
match filter {
// No filter, or filter unchanged — use original request as-is.
None | Some(std::borrow::Cow::Borrowed(_)) => self
.wrapped_segment
.get()
.read()
.facet(request, is_stopped, hw_counter)?,
// Filter was redacted — build a new request with the owned filter.
Some(std::borrow::Cow::Owned(f)) => {
let new_request = FacetParams {
filter: Some(f),
..request.clone()
};
self.wrapped_segment
.get()
.read()
.facet(&new_request, is_stopped, hw_counter)?
}
}
} else {
let wrapped_filter = Self::add_deleted_points_condition_to_filter(
request.filter.as_ref(),
filter,
self.deleted_points.keys().copied(),
);
let new_request = FacetParams {
@@ -426,6 +492,15 @@ impl ReadSegmentEntry for ProxySegment {
}
fn available_vectors_size_in_bytes(&self, vector_name: &VectorName) -> OperationResult<usize> {
// Stale vectors contribute zero bytes to the size estimate: the
// wrapped's storage is doomed to be discarded by the optimiser, and
// any new schema has no points indexed in this segment yet. Also
// avoids calling into the wrapped with a name whose query may now
// mean a different shape.
if self.changed_vector_names.is_wrapped_data_stale(vector_name) {
return Ok(0);
}
let wrapped_segment = self.wrapped_segment.get();
let wrapped_segment_guard = wrapped_segment.read();
let wrapped_size = wrapped_segment_guard.available_vectors_size_in_bytes(vector_name)?;
@@ -451,6 +526,8 @@ impl ReadSegmentEntry for ProxySegment {
filter: Option<&'a Filter>,
hw_counter: &HardwareCounterCell,
) -> OperationResult<CardinalityEstimation> {
let filter = filter.map(|f| self.changed_vector_names.redact_filter(f));
let deleted_point_count = self
.deleted_points
.len()
@@ -460,7 +537,7 @@ impl ReadSegmentEntry for ProxySegment {
let wrapped_segment = self.wrapped_segment.get();
let wrapped_segment_guard = wrapped_segment.read();
(
wrapped_segment_guard.estimate_point_count(filter, hw_counter)?,
wrapped_segment_guard.estimate_point_count(filter.as_deref(), hw_counter)?,
wrapped_segment_guard.available_point_count_without_deferred(),
)
};
@@ -503,25 +580,48 @@ impl ReadSegmentEntry for ProxySegment {
fn info(&self) -> SegmentInfo {
let wrapped_info = self.wrapped_segment.get().read().info();
let vector_name_count =
self.config().vector_data.len() + self.config().sparse_vector_data.len();
// Remove vector-data entries for names that the proxy has deleted or
// superseded with a different schema — their counts and size reflect
// the old, stale storage and should not be surfaced.
let mut vector_data = wrapped_info.vector_data;
let mut removed_num_vectors = 0usize;
let mut removed_num_indexed = 0usize;
let mut removed_num_deleted = 0usize;
vector_data.retain(|name, info| {
if self.changed_vector_names.is_wrapped_data_stale(name) {
removed_num_vectors += info.num_vectors;
removed_num_indexed += info.num_indexed_vectors;
removed_num_deleted += info.num_deleted_vectors;
false
} else {
true
}
});
let vector_name_count = vector_data.len();
let deleted_points_count = self.deleted_points.len();
// This is a best estimate
// Best estimate: start from wrapped aggregate, subtract what we just
// removed (stale vectors) and what the proxy deleted (per-point
// deletions × remaining vector names).
let num_vectors = wrapped_info
.num_vectors
.saturating_sub(removed_num_vectors)
.saturating_sub(deleted_points_count * vector_name_count);
let num_indexed_vectors = if wrapped_info.segment_type == SegmentType::Indexed {
wrapped_info
.num_vectors
.num_indexed_vectors
.saturating_sub(removed_num_indexed)
.saturating_sub(deleted_points_count * vector_name_count)
} else {
0
};
let vector_data = wrapped_info.vector_data;
let num_deleted_vectors = wrapped_info
.num_deleted_vectors
.saturating_sub(removed_num_deleted)
+ deleted_points_count * vector_name_count;
SegmentInfo {
uuid: wrapped_info.uuid,
@@ -535,9 +635,8 @@ impl ReadSegmentEntry for ProxySegment {
num_deleted_deferred_points.saturating_add(self.deleted_deferred_count)
},
),
num_deleted_vectors: wrapped_info.num_deleted_vectors
+ deleted_points_count * vector_name_count,
vectors_size_bytes: wrapped_info.vectors_size_bytes, // + write_info.vectors_size_bytes,
num_deleted_vectors,
vectors_size_bytes: wrapped_info.vectors_size_bytes,
payloads_size_bytes: wrapped_info.payloads_size_bytes,
ram_usage_bytes: wrapped_info.ram_usage_bytes,
disk_usage_bytes: wrapped_info.disk_usage_bytes,
@@ -887,4 +986,46 @@ impl NonAppendableSegmentEntry for ProxySegment {
Ok(true)
}
fn create_vector_name(
&mut self,
op_num: SeqNumberType,
vector_name: &VectorName,
vector_config: &VectorNameConfig,
) -> OperationResult<bool> {
if self.version() > op_num {
return Ok(false);
}
self.version = cmp::max(self.version, op_num);
// `record_create` consults `wrapped_config` (and any earlier intent
// recorded for this name) to compute `supersedes_wrapped`, so the
// optimiser/propagator can clear stale wrapped storage when needed.
self.changed_vector_names.record_create(
vector_name.to_owned(),
vector_config.clone(),
op_num,
&self.wrapped_config,
);
Ok(true)
}
fn delete_vector_name(
&mut self,
op_num: SeqNumberType,
vector_name: &VectorName,
) -> OperationResult<bool> {
if self.version() > op_num {
return Ok(false);
}
self.version = cmp::max(self.version, op_num);
self.changed_vector_names
.record_delete(vector_name.to_owned(), op_num);
Ok(true)
}
}
lib/shard/src/proxy_segment/vector_name_changes.rs
+452
View File
@@ -0,0 +1,452 @@
//! Pending vector-schema changes recorded in a [`super::ProxySegment`].
//!
//! See [`IntendedVector`] for the rationale behind the intent representation
//! and [`ProxyVectorNameChanges`] for the per-proxy buffer.
use std::borrow::Cow;
use std::sync::Arc;
use ahash::{AHashMap, AHashSet};
use itertools::Itertools as _;
use segment::data_types::vector_name_config::{
DenseVectorConfig, SparseVectorConfig, VectorNameConfig,
};
use segment::index::field_index::CardinalityEstimation;
use segment::types::{
Condition, CustomIdCheckerCondition, ExtendedPointId, Filter, SegmentConfig, SeqNumberType,
SparseVectorDataConfig, VectorDataConfig, VectorName, VectorNameBuf, WithVector,
};
/// A [`CustomIdCheckerCondition`] that never matches any point. Used to
/// replace `HasVector` conditions that reference a vector the proxy has
/// deleted or superseded — the wrapped segment's storage for that vector is
/// stale, so the condition must evaluate to `false` for every point.
#[derive(Debug)]
struct AlwaysFalseChecker;
impl CustomIdCheckerCondition for AlwaysFalseChecker {
fn estimate_cardinality(&self, _points: usize) -> CardinalityEstimation {
CardinalityEstimation::exact(0)
}
fn check(&self, _point_id: ExtendedPointId) -> bool {
false
}
}
/// Desired end-state of a single named vector inside a [`super::ProxySegment`].
///
/// `ProxyVectorNameChanges` records, for each name touched in the proxy, the
/// final state we want the wrapped/optimised segment to converge to once the
/// proxy is drained. The previous representation was a Create-or-Delete pair
/// stored in a HashMap, which silently collapsed `Delete v then Create v with
/// a different schema` into a plain `Create v` and let the optimiser keep the
/// stale storage from the wrapped segment. The intent representation keeps
/// enough information that the apply path can clear that stale storage before
/// installing the new schema.
#[derive(Debug, Clone)]
pub enum IntendedVector {
/// The vector should exist with this configuration once the proxy drains.
Present {
config: VectorNameConfig,
version: SeqNumberType,
/// `true` iff the wrapped segment carries this name with a *different*
/// schema (or as the wrong kind), or has been touched by an earlier
/// `Absent` intent in this proxy. When this flag is set, any code that
/// re-applies the change must clear the existing vector data before
/// installing the new config — otherwise the idempotent
/// `create_vector_name_impl` will silently keep the old storage.
supersedes_wrapped: bool,
},
/// The vector has been deleted in the proxy.
Absent { version: SeqNumberType },
}
impl IntendedVector {
pub fn version(&self) -> SeqNumberType {
match self {
IntendedVector::Present { version, .. } => *version,
IntendedVector::Absent { version } => *version,
}
}
/// Whether the wrapped segment's data for this vector is no longer
/// authoritative and must not be migrated as-is by the optimiser.
fn taints_wrapped(&self) -> bool {
matches!(
self,
IntendedVector::Absent { .. }
| IntendedVector::Present {
supersedes_wrapped: true,
..
}
)
}
}
#[derive(Debug, Default)]
pub struct ProxyVectorNameChanges {
intent: AHashMap<VectorNameBuf, IntendedVector>,
}
impl ProxyVectorNameChanges {
/// Record a `Create` intent for `vector_name`.
///
/// `wrapped_config` is the segment config the proxy is wrapping; it is used
/// to decide whether the wrapped segment already carries this name with a
/// matching schema (in which case the new entry can leave
/// `supersedes_wrapped = false` and the optimiser's idempotent create is
/// fine) or with a stale schema / wrong kind (in which case the flag is
/// set so the apply path knows to delete first).
pub fn record_create(
&mut self,
vector_name: VectorNameBuf,
config: VectorNameConfig,
version: SeqNumberType,
wrapped_config: &SegmentConfig,
) {
// Carry forward an earlier "tainted" flag: a previous `Absent` (or a
// previous `Present { supersedes_wrapped: true }`) means the wrapped
// data has already been logically discarded by this proxy, so even a
// same-schema re-create cannot resurrect it.
let previous_taints = self
.intent
.get(&vector_name)
.is_some_and(IntendedVector::taints_wrapped);
let supersedes_wrapped =
previous_taints || wrapped_carries_stale_schema(wrapped_config, &vector_name, &config);
self.intent.insert(
vector_name,
IntendedVector::Present {
config,
version,
supersedes_wrapped,
},
);
}
/// Record a `Delete` intent for `vector_name`.
pub fn record_delete(&mut self, vector_name: VectorNameBuf, version: SeqNumberType) {
self.intent
.insert(vector_name, IntendedVector::Absent { version });
}
pub fn is_empty(&self) -> bool {
self.intent.is_empty()
}
pub fn clear(&mut self) {
self.intent.clear();
}
/// Iterate over proxied vector name intents in order of version.
///
/// Intents must be applied in version order: an intent with a stale version
/// will be silently rejected by the target segment.
pub fn iter_ordered(&self) -> impl Iterator<Item = (&VectorNameBuf, &IntendedVector)> {
self.intent
.iter()
.sorted_by_key(|(_, intent)| intent.version())
}
/// Whether the wrapped segment's data for `vector_name` is no longer
/// authoritative — i.e. the proxy queues a `Delete` (`Absent`) or a
/// `Create` that supersedes the wrapped's existing schema. Read paths
/// in `ProxySegment` use this to short-circuit lookups against the
/// wrapped segment and return empty results instead of stale data.
///
/// Returns `false` for names not touched by the proxy at all (the
/// wrapped is the source of truth) and for `Present { supersedes_wrapped:
/// false }` (a same-schema re-create or a brand-new name where wrapped
/// has nothing to be stale about).
pub fn is_wrapped_data_stale(&self, vector_name: &VectorName) -> bool {
self.intent
.get(vector_name)
.is_some_and(IntendedVector::taints_wrapped)
}
/// Drop any vector names from `with_vector` whose data the wrapped segment
/// can no longer be trusted to serve — either because the proxy intends
/// to delete them outright (`Absent`) or because it intends to replace
/// them with a different schema (`Present { supersedes_wrapped: true }`).
/// Read paths in `ProxySegment` use this to rewrite the parameter before
/// delegating to the wrapped segment, so a request like
/// `WithVector::Selector(["v_dropped"])` doesn't bring back stale data.
///
/// `wrapped_config` is the config of the segment this proxy is wrapping;
/// it is the source of truth for "what does `Bool(true)` (= all vectors)
/// expand to?". It's passed in rather than stored on the buffer because
/// (a) `ProxySegment` already owns the canonical copy at
/// `self.wrapped_config`, (b) `ProxyVectorNameChanges` is a pure delta
/// buffer and adding base state to it would muddy its role and break
/// `merge` across proxies that wrap different segments, and (c) every
/// call site is inside `impl ProxySegment` and already has the config in
/// scope.
///
/// Returns [`Cow::Borrowed`] when nothing needs to change (no tainted
/// names, or the request is `Bool(false)`, or none of the explicitly
/// requested names are tainted) and [`Cow::Owned`] only when at least
/// one name was actually dropped or `Bool(true)` had to be expanded to a
/// `Selector`.
pub fn redact_with_vector<'a>(
&self,
with_vector: &'a WithVector,
wrapped_config: &SegmentConfig,
) -> Cow<'a, WithVector> {
let tainted: AHashSet<&str> = self
.intent
.iter()
.filter(|(_, intent)| intent.taints_wrapped())
.map(|(name, _)| name.as_str())
.collect();
if tainted.is_empty() {
return Cow::Borrowed(with_vector);
}
match with_vector {
// Nothing requested — nothing to redact.
WithVector::Bool(false) => Cow::Borrowed(with_vector),
// "All vectors" expands to whatever the wrapped knows minus the
// tainted set. Pending Creates of brand-new names are deliberately
// excluded: the wrapped has no data for them anyway, and asking
// it for them would error out.
WithVector::Bool(true) => {
let kept: Vec<VectorNameBuf> = wrapped_config
.vector_data
.keys()
.chain(wrapped_config.sparse_vector_data.keys())
.filter(|name| !tainted.contains(name.as_str()))
.cloned()
.collect();
Cow::Owned(WithVector::Selector(kept))
}
// Filter the explicit list. Skip the allocation if no requested
// name is actually tainted (the common case for an unrelated
// schema change happening in the background).
WithVector::Selector(requested) => {
let needs_redact = requested.iter().any(|name| tainted.contains(name.as_str()));
if !needs_redact {
return Cow::Borrowed(with_vector);
}
let kept: Vec<VectorNameBuf> = requested
.iter()
.filter(|name| !tainted.contains(name.as_str()))
.cloned()
.collect();
Cow::Owned(WithVector::Selector(kept))
}
}
}
/// Rewrite a [`Filter`] so that any `HasVector` condition that references
/// a deleted or superseded vector name is replaced with a
/// [`Condition::CustomIdChecker`] that always returns `false`. This makes
/// queries like `must: [{ has_vector: "v_dropped" }]` correctly match
/// zero points in the proxy, instead of leaking through to the wrapped
/// segment's stale storage.
///
/// Returns [`Cow::Borrowed`] when no `HasVector` in the filter tree
/// references a tainted name (the common case) and [`Cow::Owned`] only
/// when at least one condition was rewritten. The scan is a cheap
/// read-only pass; cloning only happens if a rewrite is actually needed.
pub fn redact_filter<'a>(&self, filter: &'a Filter) -> Cow<'a, Filter> {
if !self.filter_has_stale_has_vector(filter) {
return Cow::Borrowed(filter);
}
let mut owned = filter.clone();
self.redact_filter_inplace(&mut owned);
Cow::Owned(owned)
}
/// Recursive read-only scan: does the filter tree contain at least one
/// `HasVector` whose name is tainted?
fn filter_has_stale_has_vector(&self, filter: &Filter) -> bool {
let Filter {
should,
min_should,
must,
must_not,
} = filter;
let conditions = should
.iter()
.flatten()
.chain(must.iter().flatten())
.chain(must_not.iter().flatten())
.chain(min_should.iter().flat_map(|ms| ms.conditions.iter()));
for cond in conditions {
match cond {
Condition::HasVector(hv) => {
if self.is_wrapped_data_stale(&hv.has_vector) {
return true;
}
}
Condition::Nested(nested) => {
if self.filter_has_stale_has_vector(&nested.nested.filter) {
return true;
}
}
Condition::Filter(inner) => {
if self.filter_has_stale_has_vector(inner) {
return true;
}
}
Condition::Field(_) => {}
Condition::IsEmpty(_) => {}
Condition::IsNull(_) => {}
Condition::HasId(_) => {}
Condition::CustomIdChecker(_) => {}
}
}
false
}
/// Walk a cloned filter tree in-place, replacing tainted `HasVector`
/// conditions with an always-false checker.
fn redact_filter_inplace(&self, filter: &mut Filter) {
let Filter {
should,
min_should,
must,
must_not,
} = filter;
if let Some(conds) = should {
self.redact_conditions_inplace(conds);
}
if let Some(conds) = must {
self.redact_conditions_inplace(conds);
}
if let Some(conds) = must_not {
self.redact_conditions_inplace(conds);
}
if let Some(ms) = min_should {
self.redact_conditions_inplace(&mut ms.conditions);
}
}
fn redact_conditions_inplace(&self, conditions: &mut [Condition]) {
for cond in conditions.iter_mut() {
match cond {
Condition::HasVector(hv) => {
if self.is_wrapped_data_stale(&hv.has_vector) {
*cond = Condition::new_custom(Arc::new(AlwaysFalseChecker));
}
}
Condition::Nested(nested) => {
self.redact_filter_inplace(&mut nested.nested.filter);
}
Condition::Filter(inner) => {
self.redact_filter_inplace(inner);
}
Condition::Field(_) => {}
Condition::IsEmpty(_) => {}
Condition::IsNull(_) => {}
Condition::HasId(_) => {}
Condition::CustomIdChecker(_) => {}
}
}
}
/// Merge intents from another proxy into this one. Per name, the higher
/// version wins; if either side carried a `taints_wrapped` flag, it is
/// propagated onto the winner so the apply path errs on the side of
/// clearing stale data.
pub fn merge(&mut self, other: &Self) {
for (name, other_intent) in &other.intent {
let other_taints = other_intent.taints_wrapped();
match self.intent.get_mut(name) {
None => {
self.intent.insert(name.clone(), other_intent.clone());
}
Some(self_intent) => {
if other_intent.version() > self_intent.version() {
let self_taints = self_intent.taints_wrapped();
let mut winner = other_intent.clone();
if self_taints
&& let IntendedVector::Present {
supersedes_wrapped, ..
} = &mut winner
{
*supersedes_wrapped = true;
}
*self_intent = winner;
} else if other_taints
&& let IntendedVector::Present {
supersedes_wrapped, ..
} = self_intent
{
*supersedes_wrapped = true;
}
}
}
}
}
}
/// Returns `true` iff the wrapped segment carries `vector_name` with a schema
/// that does **not** match the requested `new_config` — including the
/// kind-mismatch case (wrapped has it as dense, new is sparse, or vice versa).
///
/// The check intentionally only looks at user-controlled, immutable schema
/// fields (`size` / `distance` / `multivector_config` / `datatype` for dense,
/// `modifier` / `index.datatype` for sparse) and ignores tunable fields like
/// HNSW or quantization config that can be updated independently after
/// creation. A wrapped segment that doesn't have the name at all has nothing
/// stale to carry, so the function returns `false` in that case.
fn wrapped_carries_stale_schema(
wrapped_config: &SegmentConfig,
vector_name: &VectorName,
new_config: &VectorNameConfig,
) -> bool {
match new_config {
VectorNameConfig::Dense(wrapper) => {
// Wrong kind — wrapped stores it as a sparse vector.
if wrapped_config.sparse_vector_data.contains_key(vector_name) {
return true;
}
let Some(existing) = wrapped_config.vector_data.get(vector_name) else {
return false;
};
!wrapped_dense_schema_matches(existing, &wrapper.dense)
}
VectorNameConfig::Sparse(wrapper) => {
// Wrong kind — wrapped stores it as a dense vector.
if wrapped_config.vector_data.contains_key(vector_name) {
return true;
}
let Some(existing) = wrapped_config.sparse_vector_data.get(vector_name) else {
return false;
};
!wrapped_sparse_schema_matches(existing, &wrapper.sparse)
}
}
}
fn wrapped_dense_schema_matches(
existing: &VectorDataConfig,
new_config: &DenseVectorConfig,
) -> bool {
let DenseVectorConfig {
size,
distance,
multivector_config,
datatype,
} = new_config;
existing.size == *size
&& existing.distance == *distance
&& existing.multivector_config == *multivector_config
&& existing.datatype == *datatype
}
fn wrapped_sparse_schema_matches(
existing: &SparseVectorDataConfig,
new_config: &SparseVectorConfig,
) -> bool {
let SparseVectorConfig { modifier, datatype } = new_config;
existing.modifier == *modifier && existing.index.datatype == *datatype
}
lib/shard/src/update.rs
+31 -1
View File
@@ -16,12 +16,14 @@ use segment::types::{
SeqNumberType, VectorNameBuf, WithPayload, WithVector,
};
use crate::operations::FieldIndexOperations;
use crate::operations::payload_ops::PayloadOps;
use crate::operations::point_ops::{
ConditionalInsertOperationInternal, PointOperations, PointStructPersisted, UpdateMode,
};
use crate::operations::vector_ops::{PointVectorsPersisted, UpdateVectorsOp, VectorOperations};
use crate::operations::{
CreateVectorName, DeleteVectorName, FieldIndexOperations, VectorNameOperations,
};
use crate::segment_holder::{SegmentHolder, SegmentId};
pub fn process_point_operation(
@@ -955,6 +957,34 @@ pub fn delete_field_index(
})
}
pub fn process_vector_name_operation(
segments: &SegmentHolder,
op_num: SeqNumberType,
vector_name_operation: &VectorNameOperations,
) -> OperationResult<usize> {
match vector_name_operation {
VectorNameOperations::CreateVectorName(create_data) => {
let CreateVectorName {
vector_name,
config,
} = create_data;
segments.apply_segments(|write_segment| {
write_segment.with_upgraded(|segment| {
segment.create_vector_name(op_num, vector_name, config)
})
})
}
VectorNameOperations::DeleteVectorName(delete_data) => {
let DeleteVectorName { vector_name } = delete_data;
segments.apply_segments(|write_segment| {
write_segment
.with_upgraded(|segment| segment.delete_vector_name(op_num, vector_name))
})
}
}
}
fn select_excluded_by_filter_ids(
segments: &SegmentHolder,
point_ids: impl IntoIterator<Item = PointIdType>,
lib/storage/src/content_manager/collection_meta_ops.rs
+46
View File
@@ -194,6 +194,37 @@ impl CreateCollectionOperation {
collection_name: String,
create_collection: CreateCollection,
) -> StorageResult<Self> {
// Apply the same vector-name validation that the
// `PUT /collections/{name}/vectors/{vector_name}` endpoint enforces
// (length 0..=200, no filesystem-unsafe characters), so both creation
// paths reject the same set of bad names. The `Validate` derive on
// `CreateCollection` only walks `BTreeMap` *values*, never keys, so this
// has to run imperatively here.
//
// The unnamed slot used by `VectorsConfig::Single` is exempt: its
// implicit key is the empty `DEFAULT_VECTOR_NAME` constant and a
// `Single` config has no user-supplied name to validate.
if let collection::operations::types::VectorsConfig::Multi(multi) =
&create_collection.vectors
{
for vector_name in multi.keys() {
common::validation::validate_vector_name(vector_name).map_err(|err| {
StorageError::bad_input(format!(
"Invalid dense vector name `{vector_name}`: {err}",
))
})?;
}
}
if let Some(sparse_config) = &create_collection.sparse_vectors {
for vector_name in sparse_config.keys() {
common::validation::validate_vector_name(vector_name).map_err(|err| {
StorageError::bad_input(format!(
"Invalid sparse vector name `{vector_name}`: {err}",
))
})?;
}
}
// validate vector names are unique between dense and sparse vectors
if let Some(sparse_config) = &create_collection.sparse_vectors {
if sparse_config.contains_key(DEFAULT_VECTOR_NAME) {
@@ -405,6 +436,19 @@ pub struct DropPayloadIndex {
pub field_name: PayloadKeyType,
}
#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Hash, Clone)]
pub struct CreateNamedVector {
pub collection_name: String,
pub vector_name: segment::types::VectorNameBuf,
pub config: shard::operations::vector_name_ops::VectorNameConfig,
}
#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Hash, Clone)]
pub struct DeleteNamedVector {
pub collection_name: String,
pub vector_name: segment::types::VectorNameBuf,
}
/// Enumeration of all possible collection update operations
#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Hash, Clone)]
#[serde(rename_all = "snake_case")]
@@ -420,6 +464,8 @@ pub enum CollectionMetaOperations {
DropShardKey(DropShardKey),
CreatePayloadIndex(CreatePayloadIndex),
DropPayloadIndex(DropPayloadIndex),
CreateNamedVector(CreateNamedVector),
DeleteNamedVector(DeleteNamedVector),
Nop {
token: usize,
}, // Empty operation
lib/storage/src/content_manager/toc/collection_container.rs
+16 -16
View File
@@ -130,11 +130,11 @@ impl TableOfContent {
data: consensus_manager::CollectionsSnapshot,
) -> Result<(), StorageError> {
self.general_runtime.block_on(async {
let mut collections = self.collections.write().await;
let mut existing_collections = self.collections.write().await;
for (id, state) in &data.collections {
if let Some(collection) = collections.get(id) {
let collection_uuid = collection.uuid().await;
if let Some(existing_collection) = existing_collections.get(id) {
let collection_uuid = existing_collection.uuid().await;
let recreate_collection = if collection_uuid != state.config.uuid {
log::warn!(
@@ -145,7 +145,7 @@ impl TableOfContent {
);
true
} else if let Err(err) = collection.check_config_compatible(&state.config).await {
} else if let Err(err) = existing_collection.check_config_compatible(&state.config).await {
log::warn!(
"Recreating collection {id}, because collection config is incompatible: \
{err}",
@@ -158,17 +158,17 @@ impl TableOfContent {
if recreate_collection {
// Drop `collections` lock
drop(collections);
drop(existing_collections);
// Delete collection
self.delete_collection(id).await?;
// Re-acquire `collections` lock 🙄
collections = self.collections.write().await;
existing_collections = self.collections.write().await;
}
}
let collection_exists = collections.contains_key(id);
let collection_exists = existing_collections.contains_key(id);
// Create collection if not present locally
if !collection_exists {
@@ -207,16 +207,16 @@ impl TableOfContent {
self.storage_config.optimizers_overwrite.clone(),
)
.await?;
collections.validate_collection_not_exists(id)?;
collections.insert(id.clone(), Arc::new(collection));
existing_collections.validate_collection_not_exists(id)?;
existing_collections.insert(id.clone(), Arc::new(collection));
}
let Some(collection) = collections.get(id) else {
let Some(existing_collection) = existing_collections.get(id) else {
unreachable!()
};
// Update collection state
if &collection.state().await != state {
if &existing_collection.state().await != state {
if let Some(proposal_sender) = self.consensus_proposal_sender.clone() {
// In some cases on state application it might be needed to abort the transfer
let abort_transfer = |transfer| {
@@ -232,7 +232,7 @@ impl TableOfContent {
)
};
};
collection
existing_collection
.apply_state(state.clone(), self.this_peer_id(), abort_transfer)
.await?;
} else {
@@ -243,8 +243,8 @@ impl TableOfContent {
// Mark local shards as dead (to initiate shard transfer),
// if collection has been created during snapshot application
if !collection_exists {
for shard_id in collection.get_local_shards().await {
let shard_holder = collection.shards_holder().read_owned().await;
for shard_id in existing_collection.get_local_shards().await {
let shard_holder = existing_collection.shards_holder().read_owned().await;
let Some(replica_set) = shard_holder.get_shard(shard_id) else {
continue;
@@ -258,10 +258,10 @@ impl TableOfContent {
}
// Collect names of collections that are present locally
let collection_names: Vec<_> = collections.keys().cloned().collect();
let collection_names: Vec<_> = existing_collections.keys().cloned().collect();
// Drop `collections` lock
drop(collections);
drop(existing_collections);
// Remove collections that are present locally, but are not in the snapshot state
for collection_name in &collection_names {
lib/storage/src/content_manager/toc/collection_meta_ops.rs
+34
View File
@@ -122,6 +122,18 @@ impl TableOfContent {
.await
.map(|()| true)
}
CollectionMetaOperations::CreateNamedVector(create_named_vector) => {
log::debug!("Create named vector {create_named_vector:?}");
self.create_named_vector(create_named_vector)
.await
.map(|()| true)
}
CollectionMetaOperations::DeleteNamedVector(delete_named_vector) => {
log::debug!("Delete named vector {delete_named_vector:?}");
self.delete_named_vector(delete_named_vector)
.await
.map(|()| true)
}
#[cfg(feature = "staging")]
CollectionMetaOperations::TestSlowDown(test_slow_down) => {
test_slow_down.execute(self.this_peer_id).await;
@@ -714,4 +726,26 @@ impl TableOfContent {
.await?;
Ok(())
}
async fn create_named_vector(&self, operation: CreateNamedVector) -> Result<(), StorageError> {
let collection_hw_acc = HwMeasurementAcc::new_with_metrics_drain(
self.get_collection_hw_metrics(operation.collection_name.clone()),
);
self.get_collection_unchecked(&operation.collection_name)
.await?
.create_named_vector(operation.vector_name, operation.config, collection_hw_acc)
.await?;
Ok(())
}
async fn delete_named_vector(&self, operation: DeleteNamedVector) -> Result<(), StorageError> {
self.get_collection_unchecked(&operation.collection_name)
.await?
.delete_named_vector(operation.vector_name)
.await?;
Ok(())
}
}
lib/storage/src/dispatcher.rs
+2
View File
@@ -178,6 +178,8 @@ impl Dispatcher {
| CollectionMetaOperations::DropShardKey(_)
| CollectionMetaOperations::CreatePayloadIndex(_)
| CollectionMetaOperations::DropPayloadIndex(_)
| CollectionMetaOperations::CreateNamedVector(_)
| CollectionMetaOperations::DeleteNamedVector(_)
| CollectionMetaOperations::Nop { .. } => false,
#[cfg(feature = "staging")]
lib/storage/src/rbac/auditable_operation.rs
+7 -1
View File
@@ -1,7 +1,7 @@
use shard::operations::payload_ops::PayloadOps;
use shard::operations::point_ops::PointOperations;
use shard::operations::vector_ops::VectorOperations;
use shard::operations::{CollectionUpdateOperations, FieldIndexOperations};
use shard::operations::{CollectionUpdateOperations, FieldIndexOperations, VectorNameOperations};
use crate::content_manager::collection_meta_ops::CollectionMetaOperations;
@@ -35,6 +35,10 @@ impl AuditableOperation for CollectionUpdateOperations {
FieldIndexOperations::CreateIndex(_) => "create_field_index",
FieldIndexOperations::DeleteIndex(_) => "delete_field_index",
},
CollectionUpdateOperations::VectorNameOperation(op) => match op {
VectorNameOperations::CreateVectorName(_) => "create_vector_name",
VectorNameOperations::DeleteVectorName(_) => "delete_vector_name",
},
#[cfg(feature = "staging")]
CollectionUpdateOperations::StagingOperation(_) => "debug",
}
@@ -55,6 +59,8 @@ impl AuditableOperation for CollectionMetaOperations {
CollectionMetaOperations::DropShardKey(_) => "drop_shard_key",
CollectionMetaOperations::CreatePayloadIndex(_) => "create_payload_index",
CollectionMetaOperations::DropPayloadIndex(_) => "drop_payload_index",
CollectionMetaOperations::CreateNamedVector(_) => "create_named_vector",
CollectionMetaOperations::DeleteNamedVector(_) => "delete_named_vector",
CollectionMetaOperations::Nop { .. } => "nop",
#[cfg(feature = "staging")]
CollectionMetaOperations::TestSlowDown(_) => "debug",
lib/storage/src/rbac/ops_checks.rs
+56 -1
View File
@@ -67,6 +67,18 @@ impl Access {
AccessRequirements::new().write().extras(),
)?;
}
CollectionMetaOperations::CreateNamedVector(op) => {
self.check_collection_access(
&op.collection_name,
AccessRequirements::new().write().extras(),
)?;
}
CollectionMetaOperations::DeleteNamedVector(op) => {
self.check_collection_access(
&op.collection_name,
AccessRequirements::new().write().extras(),
)?;
}
CollectionMetaOperations::Nop { token: _ } => (),
#[cfg(feature = "staging")]
CollectionMetaOperations::TestSlowDown(_) => {
@@ -295,7 +307,8 @@ impl CheckableCollectionOperation for CollectionUpdateOperations {
manage: false,
extras: false,
},
CollectionUpdateOperations::FieldIndexOperation(_) => AccessRequirements {
CollectionUpdateOperations::FieldIndexOperation(_)
| CollectionUpdateOperations::VectorNameOperation(_) => AccessRequirements {
write: true,
manage: true,
extras: true,
@@ -687,6 +700,9 @@ mod tests_ops {
CollectionUpdateOperationsDiscriminants::FieldIndexOperation => {
check_collection_update_operations_field_index()
}
CollectionUpdateOperationsDiscriminants::VectorNameOperation => {
check_collection_update_operations_vector_name()
}
#[cfg(feature = "staging")]
CollectionUpdateOperationsDiscriminants::StagingOperation => {
use shard::operations::staging::{StagingOperations, TestDelayOperation};
@@ -901,4 +917,43 @@ mod tests_ops {
);
}
}
/// Tests for [`CollectionUpdateOperations::VectorNameOperation`].
fn check_collection_update_operations_vector_name() {
use segment::types::Distance;
use shard::operations::vector_name_ops::{
DenseVectorConfig, VectorNameConfig, VectorNameOperationsDiscriminants,
};
use shard::operations::{CreateVectorName, DeleteVectorName, VectorNameOperations};
for discr in VectorNameOperationsDiscriminants::iter() {
let inner = match discr {
VectorNameOperationsDiscriminants::CreateVectorName => {
VectorNameOperations::CreateVectorName(CreateVectorName {
vector_name: "test".into(),
config: VectorNameConfig::dense(DenseVectorConfig {
size: 4,
distance: Distance::Cosine,
multivector_config: None,
datatype: None,
}),
})
}
VectorNameOperationsDiscriminants::DeleteVectorName => {
VectorNameOperations::DeleteVectorName(DeleteVectorName {
vector_name: "test".into(),
})
}
};
let op = CollectionUpdateOperations::VectorNameOperation(inner);
assert_allowed(&op, &Access::Global(GlobalAccessMode::Manage));
assert_forbidden(&op, &Access::Global(GlobalAccessMode::Read));
assert_forbidden(&op, &AccessCollectionBuilder::new().add("col", true).into());
assert_forbidden(
&op,
&AccessCollectionBuilder::new().add("col", false).into(),
);
}
}
}
openapi/openapi-collections.ytt.yaml
+87
View File
@@ -231,6 +231,93 @@ paths:
minimum: 1
responses: #@ response(reference("UpdateResult"))
/collections/{collection_name}/vectors/{vector_name}:
put:
tags:
- Collections
summary: Create named vector
description: Create a new named vector on an existing collection
operationId: create_vector_name
parameters:
- name: collection_name
in: path
description: Name of the collection
required: true
schema:
type: string
- name: vector_name
in: path
description: Name of the vector to create
required: true
schema:
type: string
- name: wait
in: query
description: "If true, wait for changes to actually happen"
required: false
schema:
type: boolean
- name: ordering
in: query
description: "define ordering guarantees for the operation"
required: false
schema:
$ref: "#/components/schemas/WriteOrdering"
- name: timeout
in: query
description: "Timeout for the operation"
required: false
schema:
type: integer
minimum: 1
requestBody:
description: Vector configuration - dense or sparse
content:
application/json:
schema:
$ref: "#/components/schemas/VectorNameConfig"
responses: #@ response(type("boolean"))
delete:
tags:
- Collections
summary: Delete named vector
description: Delete a named vector from a collection
operationId: delete_vector_name
parameters:
- name: collection_name
in: path
description: Name of the collection
required: true
schema:
type: string
- name: vector_name
in: path
description: Name of the vector to delete
required: true
schema:
type: string
- name: wait
in: query
description: "If true, wait for changes to actually happen"
required: false
schema:
type: boolean
- name: ordering
in: query
description: "define ordering guarantees for the operation"
required: false
schema:
$ref: "#/components/schemas/WriteOrdering"
- name: timeout
in: query
description: "Timeout for the operation"
required: false
schema:
type: integer
minimum: 1
responses: #@ response(type("boolean"))
/collections/{collection_name}/cluster:
get:
tags:
src/actix/api/mod.rs
+1
View File
@@ -22,6 +22,7 @@ pub mod service_api;
pub mod shards_api;
pub mod snapshot_api;
pub mod update_api;
pub mod vector_name_api;
/// A collection path with stricter validation
///
src/actix/api/vector_name_api.rs
+91
View File
@@ -0,0 +1,91 @@
use actix_web::rt::time::Instant;
use actix_web::{Responder, delete, put, web};
use actix_web_validator::{Json, Path, Query};
use common::validation::validate_vector_name;
use serde::Deserialize;
use storage::dispatcher::Dispatcher;
use validator::Validate;
use crate::actix::auth::ActixAuth;
use crate::actix::helpers::{get_request_hardware_counter, process_response};
use crate::common::update::{InternalUpdateParams, UpdateParams};
use crate::settings::ServiceConfig;
#[derive(Deserialize, Validate)]
struct VectorNamePath {
#[validate(length(min = 1, max = 255))]
collection_name: String,
#[validate(custom(function = "validate_vector_name"))]
vector_name: String,
}
#[put("/collections/{collection_name}/vectors/{vector_name}")]
async fn create_vector_name(
dispatcher: web::Data<Dispatcher>,
path: Path<VectorNamePath>,
body: Json<segment::data_types::vector_name_config::VectorNameConfig>,
params: Query<UpdateParams>,
ActixAuth(auth): ActixAuth,
service_config: web::Data<ServiceConfig>,
) -> impl Responder {
let timing = Instant::now();
let path = path.into_inner();
let config = body.into_inner();
let request_hw_counter = get_request_hardware_counter(
&dispatcher,
path.collection_name.clone(),
service_config.hardware_reporting(),
Some(params.wait),
);
let response = crate::common::update::do_create_vector_name(
dispatcher.into_inner(),
path.collection_name,
path.vector_name,
config,
InternalUpdateParams::default(),
params.into_inner(),
auth,
request_hw_counter.get_counter(),
)
.await;
process_response(response, timing, None)
}
#[delete("/collections/{collection_name}/vectors/{vector_name}")]
async fn delete_vector_name(
dispatcher: web::Data<Dispatcher>,
path: Path<VectorNamePath>,
params: Query<UpdateParams>,
ActixAuth(auth): ActixAuth,
service_config: web::Data<ServiceConfig>,
) -> impl Responder {
let timing = Instant::now();
let path = path.into_inner();
let request_hw_counter = get_request_hardware_counter(
&dispatcher,
path.collection_name.clone(),
service_config.hardware_reporting(),
Some(params.wait),
);
let response = crate::common::update::do_delete_vector_name(
dispatcher.into_inner(),
path.collection_name,
path.vector_name,
InternalUpdateParams::default(),
params.into_inner(),
auth,
request_hw_counter.get_counter(),
)
.await;
process_response(response, timing, None)
}
pub fn config_vector_name_api(cfg: &mut web::ServiceConfig) {
cfg.service(create_vector_name).service(delete_vector_name);
}
src/actix/mod.rs
+2
View File
@@ -42,6 +42,7 @@ use crate::actix::api::service_api::config_service_api;
use crate::actix::api::shards_api::config_shards_api;
use crate::actix::api::snapshot_api::config_snapshots_api;
use crate::actix::api::update_api::config_update_api;
use crate::actix::api::vector_name_api::config_vector_name_api;
use crate::actix::auth::{AuthTransform, WhitelistItem};
use crate::actix::web_ui::{WEB_UI_PATH, web_ui_factory, web_ui_folder};
use crate::common::auth::AuthKeys;
@@ -147,6 +148,7 @@ pub fn init(
.app_data(audit_config_data.clone())
.service(index)
.configure(config_collections_api)
.configure(config_vector_name_api)
.configure(config_snapshots_api)
.configure(config_update_api)
.configure(config_cluster_api)
src/common/metrics.rs
+3
View File
@@ -52,6 +52,7 @@ const REST_ENDPOINT_WHITELIST: &[&str] = &[
"/collections/{collection_name}/points/search/matrix/pairs",
"/collections/{collection_name}/points/vectors",
"/collections/{collection_name}/points/vectors/delete",
"/collections/{collection_name}/vectors/{vector_name}",
];
/// Whitelist for GRPC endpoints in metrics output.
@@ -62,8 +63,10 @@ const REST_ENDPOINT_WHITELIST: &[&str] = &[
const GRPC_ENDPOINT_WHITELIST: &[&str] = &[
"/qdrant.Points/ClearPayload",
"/qdrant.Points/Count",
"/qdrant.Points/CreateVectorName",
"/qdrant.Points/Delete",
"/qdrant.Points/DeletePayload",
"/qdrant.Points/DeleteVectorName",
"/qdrant.Points/Discover",
"/qdrant.Points/DiscoverBatch",
"/qdrant.Points/Facet",
src/common/update.rs
+154 -7
View File
@@ -918,10 +918,6 @@ pub async fn do_create_index(
.submit_collection_meta_op(consensus_op, auth, params.timeout)
.await?;
// This function is required as long as we want to maintain interface compatibility
// for `wait` parameter and return type.
// The idea is to migrate from the point-like interface to consensus-like interface in the next few versions
do_create_index_internal(
toc,
collection_name,
@@ -1030,6 +1026,146 @@ pub async fn do_delete_index_internal(
.await
}
#[expect(clippy::too_many_arguments)]
pub async fn do_create_vector_name(
dispatcher: Arc<Dispatcher>,
collection_name: String,
vector_name: String,
config: VectorNameConfig,
internal_params: InternalUpdateParams,
params: UpdateParams,
auth: Auth,
hw_measurement_acc: HwMeasurementAcc,
) -> Result<UpdateResult, StorageError> {
use collection::operations::verification::new_unchecked_verification_pass;
// Validate the vector name once at the single chokepoint that both REST and
// gRPC entrypoints land in (REST also runs the same check via `VectorNamePath`).
common::validation::validate_vector_name(&vector_name).map_err(|err| {
StorageError::bad_input(format!("Invalid vector name `{vector_name}`: {err}"))
})?;
let consensus_op = CreateNamedVector {
collection_name: collection_name.clone(),
vector_name: vector_name.clone(),
config: config.clone(),
};
let pass = new_unchecked_verification_pass();
let toc = dispatcher.toc(&auth, &pass).clone();
dispatcher
.submit_collection_meta_op(
CollectionMetaOperations::CreateNamedVector(consensus_op),
auth,
params.timeout,
)
.await?;
do_create_vector_name_internal(
toc,
collection_name,
vector_name,
config,
internal_params,
params,
hw_measurement_acc,
)
.await
}
pub async fn do_create_vector_name_internal(
toc: Arc<TableOfContent>,
collection_name: String,
vector_name: String,
config: segment::data_types::vector_name_config::VectorNameConfig,
internal_params: InternalUpdateParams,
params: UpdateParams,
hw_measurement_acc: HwMeasurementAcc,
) -> Result<UpdateResult, StorageError> {
let operation = CollectionUpdateOperations::VectorNameOperation(
VectorNameOperations::CreateVectorName(CreateVectorName {
vector_name,
config,
}),
);
update(
&toc,
&collection_name,
operation,
internal_params,
params,
None,
Auth::new_internal(Access::full("Internal API")),
hw_measurement_acc,
)
.await
}
pub async fn do_delete_vector_name(
dispatcher: Arc<Dispatcher>,
collection_name: String,
vector_name: String,
internal_params: InternalUpdateParams,
params: UpdateParams,
auth: Auth,
hw_measurement_acc: HwMeasurementAcc,
) -> Result<UpdateResult, StorageError> {
use collection::operations::verification::new_unchecked_verification_pass;
let consensus_op = DeleteNamedVector {
collection_name: collection_name.clone(),
vector_name: vector_name.clone(),
};
let pass = new_unchecked_verification_pass();
let toc = dispatcher.toc(&auth, &pass).clone();
dispatcher
.submit_collection_meta_op(
CollectionMetaOperations::DeleteNamedVector(consensus_op),
auth,
params.timeout,
)
.await?;
do_delete_vector_name_internal(
toc,
collection_name,
vector_name,
internal_params,
params,
hw_measurement_acc,
)
.await
}
pub async fn do_delete_vector_name_internal(
toc: Arc<TableOfContent>,
collection_name: String,
vector_name: String,
internal_params: InternalUpdateParams,
params: UpdateParams,
hw_measurement_acc: HwMeasurementAcc,
) -> Result<UpdateResult, StorageError> {
let operation = CollectionUpdateOperations::VectorNameOperation(
VectorNameOperations::DeleteVectorName(DeleteVectorName { vector_name }),
);
update(
&toc,
&collection_name,
operation,
internal_params,
params,
None,
Auth::new_internal(Access::full("Internal API")),
hw_measurement_acc,
)
.await
}
#[expect(clippy::too_many_arguments)]
pub async fn update(
toc: &TableOfContent,
@@ -1073,7 +1209,8 @@ pub async fn update(
}
}
CollectionUpdateOperations::FieldIndexOperation(_) => {
CollectionUpdateOperations::FieldIndexOperation(_)
| CollectionUpdateOperations::VectorNameOperation(_) => {
debug_assert_eq!(
shard_key, None,
"Field index operations can't specify shard key"
@@ -1084,8 +1221,18 @@ pub async fn update(
None => ShardSelectorInternal::All,
}
}
_ => get_shard_selector_for_update(shard_id, shard_key),
CollectionUpdateOperations::VectorOperation(_)
| CollectionUpdateOperations::PointOperation(PointOperations::UpsertPoints(_))
| CollectionUpdateOperations::PointOperation(PointOperations::UpsertPointsConditional(_))
| CollectionUpdateOperations::PointOperation(PointOperations::DeletePoints { .. })
| CollectionUpdateOperations::PointOperation(PointOperations::DeletePointsByFilter(_))
| CollectionUpdateOperations::PayloadOperation(_) => {
get_shard_selector_for_update(shard_id, shard_key)
}
#[cfg(feature = "staging")]
CollectionUpdateOperations::StagingOperation(_) => {
get_shard_selector_for_update(shard_id, shard_key)
}
};
toc.update(
src/schema_generator.rs
+1
View File
@@ -101,6 +101,7 @@ struct AllDefinitions {
bo: ShardKeysResponse,
bp: OptimizationsResponse,
bq: DistributedTelemetryData,
br: segment::data_types::vector_name_config::VectorNameConfig,
}
fn save_schema<T: JsonSchema>() {
src/tonic/api/points_api.rs
+52 -10
View File
@@ -4,16 +4,16 @@ use std::time::{Duration, Instant};
use api::grpc::qdrant::points_server::Points;
use api::grpc::qdrant::{
ClearPayloadPoints, CountPoints, CountResponse, CreateFieldIndexCollection,
DeleteFieldIndexCollection, DeletePayloadPoints, DeletePointVectors, DeletePoints,
DiscoverBatchPoints, DiscoverBatchResponse, DiscoverPoints, DiscoverResponse, FacetCounts,
FacetResponse, GetPoints, GetResponse, PointsOperationResponse, QueryBatchPoints,
QueryBatchResponse, QueryGroupsResponse, QueryPointGroups, QueryPoints, QueryResponse,
RecommendBatchPoints, RecommendBatchResponse, RecommendGroupsResponse, RecommendPointGroups,
RecommendPoints, RecommendResponse, ScrollPoints, ScrollResponse, SearchBatchPoints,
SearchBatchResponse, SearchGroupsResponse, SearchMatrixOffsets, SearchMatrixOffsetsResponse,
SearchMatrixPairs, SearchMatrixPairsResponse, SearchMatrixPoints, SearchPointGroups,
SearchPoints, SearchResponse, SetPayloadPoints, UpdateBatchPoints, UpdateBatchResponse,
UpdatePointVectors, UpsertPoints,
CreateVectorNameRequest, DeleteFieldIndexCollection, DeletePayloadPoints, DeletePointVectors,
DeletePoints, DeleteVectorNameRequest, DiscoverBatchPoints, DiscoverBatchResponse,
DiscoverPoints, DiscoverResponse, FacetCounts, FacetResponse, GetPoints, GetResponse,
PointsOperationResponse, QueryBatchPoints, QueryBatchResponse, QueryGroupsResponse,
QueryPointGroups, QueryPoints, QueryResponse, RecommendBatchPoints, RecommendBatchResponse,
RecommendGroupsResponse, RecommendPointGroups, RecommendPoints, RecommendResponse,
ScrollPoints, ScrollResponse, SearchBatchPoints, SearchBatchResponse, SearchGroupsResponse,
SearchMatrixOffsets, SearchMatrixOffsetsResponse, SearchMatrixPairs, SearchMatrixPairsResponse,
SearchMatrixPoints, SearchPointGroups, SearchPoints, SearchResponse, SetPayloadPoints,
UpdateBatchPoints, UpdateBatchResponse, UpdatePointVectors, UpsertPoints,
};
use api::grpc::{PointsOperationResponseInternal, Usage};
use collection::operations::types::CoreSearchRequest;
@@ -337,6 +337,48 @@ impl Points for PointsService {
.map(|resp| resp.map(PointsOperationResponseInternal::into))
}
async fn create_vector_name(
&self,
mut request: Request<CreateVectorNameRequest>,
) -> Result<Response<PointsOperationResponse>, Status> {
validate(request.get_ref())?;
let auth = extract_auth(&mut request);
let collection_name = request.get_ref().collection_name.clone();
let wait = Some(request.get_ref().wait.unwrap_or(false));
let hw_metrics = self.get_request_collection_hw_usage_counter(collection_name, wait);
super::update_common::create_vector_name(
self.dispatcher.clone(),
request.into_inner(),
InternalUpdateParams::default(),
auth,
hw_metrics,
)
.await
.map(|resp| resp.map(Into::into))
}
async fn delete_vector_name(
&self,
mut request: Request<DeleteVectorNameRequest>,
) -> Result<Response<PointsOperationResponse>, Status> {
validate(request.get_ref())?;
let auth = extract_auth(&mut request);
let collection_name = request.get_ref().collection_name.clone();
let wait = Some(request.get_ref().wait.unwrap_or(false));
let hw_metrics = self.get_request_collection_hw_usage_counter(collection_name, wait);
super::update_common::delete_vector_name(
self.dispatcher.clone(),
request.into_inner(),
InternalUpdateParams::default(),
auth,
hw_metrics,
)
.await
.map(|resp| resp.map(Into::into))
}
async fn search(
&self,
mut request: Request<SearchPoints>,
src/tonic/api/points_internal_api.rs
+72 -7
View File
@@ -7,13 +7,14 @@ use api::grpc::HardwareUsage;
use api::grpc::qdrant::points_internal_server::PointsInternal;
use api::grpc::qdrant::{
ClearPayloadPointsInternal, CoreSearchBatchPointsInternal, CountPointsInternal, CountResponse,
CreateFieldIndexCollectionInternal, DeleteFieldIndexCollectionInternal,
DeletePayloadPointsInternal, DeletePointsInternal, DeleteVectorsInternal, FacetCountsInternal,
FacetResponseInternal, GetPointsInternal, GetResponse, IntermediateResult,
PointsOperationResponseInternal, QueryBatchPointsInternal, QueryBatchResponseInternal,
QueryResultInternal, QueryShardPoints, RecommendPointsInternal, RecommendResponse,
ScrollPointsInternal, ScrollResponse, SearchBatchResponse, SetPayloadPointsInternal,
SyncPointsInternal, UpdateBatchInternal, UpdateVectorsInternal, UpsertPointsInternal,
CreateFieldIndexCollectionInternal, CreateVectorNameInternal,
DeleteFieldIndexCollectionInternal, DeletePayloadPointsInternal, DeletePointsInternal,
DeleteVectorNameInternal, DeleteVectorsInternal, FacetCountsInternal, FacetResponseInternal,
GetPointsInternal, GetResponse, IntermediateResult, PointsOperationResponseInternal,
QueryBatchPointsInternal, QueryBatchResponseInternal, QueryResultInternal, QueryShardPoints,
RecommendPointsInternal, RecommendResponse, ScrollPointsInternal, ScrollResponse,
SearchBatchResponse, SetPayloadPointsInternal, SyncPointsInternal, UpdateBatchInternal,
UpdateVectorsInternal, UpsertPointsInternal,
};
use api::grpc::update_operation::Update;
use collection::operations::shard_selector_internal::ShardSelectorInternal;
@@ -346,6 +347,44 @@ impl PointsInternalService {
)
.await
}
async fn create_vector_name_internal(
&self,
request: CreateVectorNameInternal,
) -> Result<Response<PointsOperationResponseInternal>, Status> {
let CreateVectorNameInternal {
create_vector_name,
shard_id,
clock_tag,
wait_override,
} = request;
create_vector_name_internal(
self.toc.clone(),
extract_internal_request(create_vector_name)?,
InternalUpdateParams::from_grpc(shard_id, clock_tag, wait_override),
)
.await
}
async fn delete_vector_name_internal(
&self,
request: DeleteVectorNameInternal,
) -> Result<Response<PointsOperationResponseInternal>, Status> {
let DeleteVectorNameInternal {
delete_vector_name,
shard_id,
clock_tag,
wait_override,
} = request;
delete_vector_name_internal(
self.toc.clone(),
extract_internal_request(delete_vector_name)?,
InternalUpdateParams::from_grpc(shard_id, clock_tag, wait_override),
)
.await
}
}
pub async fn query_batch_internal(
@@ -566,6 +605,24 @@ impl PointsInternal for PointsInternalService {
self.delete_field_index_internal(request.into_inner()).await
}
async fn create_vector_name(
&self,
request: Request<CreateVectorNameInternal>,
) -> Result<Response<PointsOperationResponseInternal>, Status> {
validate_and_log(request.get_ref());
self.create_vector_name_internal(request.into_inner()).await
}
async fn delete_vector_name(
&self,
request: Request<DeleteVectorNameInternal>,
) -> Result<Response<PointsOperationResponseInternal>, Status> {
validate_and_log(request.get_ref());
self.delete_vector_name_internal(request.into_inner()).await
}
async fn update_batch(
&self,
request: Request<UpdateBatchInternal>,
@@ -624,6 +681,12 @@ impl PointsInternal for PointsInternalService {
Update::DeleteFieldIndex(inner) => {
inner.wait_override.get_or_insert(batch_wo);
}
Update::CreateVectorName(inner) => {
inner.wait_override.get_or_insert(batch_wo);
}
Update::DeleteVectorName(inner) => {
inner.wait_override.get_or_insert(batch_wo);
}
}
}
@@ -665,6 +728,8 @@ impl PointsInternal for PointsInternalService {
Update::DeleteFieldIndex(delete_field_index) => {
self.delete_field_index_internal(delete_field_index).await?
}
Update::CreateVectorName(op) => self.create_vector_name_internal(op).await?,
Update::DeleteVectorName(op) => self.delete_vector_name_internal(op).await?,
},
};
let mut response = result.into_inner();
src/tonic/api/telemetry_wrapper.rs
+35 -13
View File
@@ -10,19 +10,19 @@ use api::grpc::qdrant::snapshots_server::Snapshots;
use api::grpc::qdrant::{
ClearPayloadPoints, CountPoints, CountResponse, CreateFieldIndexCollection,
CreateFullSnapshotRequest, CreateShardSnapshotRequest, CreateSnapshotRequest,
CreateSnapshotResponse, DeleteFieldIndexCollection, DeleteFullSnapshotRequest,
DeletePayloadPoints, DeletePointVectors, DeletePoints, DeleteShardSnapshotRequest,
DeleteSnapshotRequest, DeleteSnapshotResponse, DiscoverBatchPoints, DiscoverBatchResponse,
DiscoverPoints, DiscoverResponse, FacetCounts, FacetResponse, GetPoints, GetResponse,
ListFullSnapshotsRequest, ListShardSnapshotsRequest, ListSnapshotsRequest,
ListSnapshotsResponse, PointsOperationResponse, QueryBatchPoints, QueryBatchResponse,
QueryGroupsResponse, QueryPointGroups, QueryPoints, QueryResponse, RecommendBatchPoints,
RecommendBatchResponse, RecommendGroupsResponse, RecommendPointGroups, RecommendPoints,
RecommendResponse, RecoverShardSnapshotRequest, RecoverSnapshotResponse, ScrollPoints,
ScrollResponse, SearchBatchPoints, SearchBatchResponse, SearchGroupsResponse,
SearchMatrixOffsetsResponse, SearchMatrixPairsResponse, SearchMatrixPoints, SearchPointGroups,
SearchPoints, SearchResponse, SetPayloadPoints, UpdateBatchPoints, UpdateBatchResponse,
UpdatePointVectors, UpsertPoints,
CreateSnapshotResponse, CreateVectorNameRequest, DeleteFieldIndexCollection,
DeleteFullSnapshotRequest, DeletePayloadPoints, DeletePointVectors, DeletePoints,
DeleteShardSnapshotRequest, DeleteSnapshotRequest, DeleteSnapshotResponse,
DeleteVectorNameRequest, DiscoverBatchPoints, DiscoverBatchResponse, DiscoverPoints,
DiscoverResponse, FacetCounts, FacetResponse, GetPoints, GetResponse, ListFullSnapshotsRequest,
ListShardSnapshotsRequest, ListSnapshotsRequest, ListSnapshotsResponse,
PointsOperationResponse, QueryBatchPoints, QueryBatchResponse, QueryGroupsResponse,
QueryPointGroups, QueryPoints, QueryResponse, RecommendBatchPoints, RecommendBatchResponse,
RecommendGroupsResponse, RecommendPointGroups, RecommendPoints, RecommendResponse,
RecoverShardSnapshotRequest, RecoverSnapshotResponse, ScrollPoints, ScrollResponse,
SearchBatchPoints, SearchBatchResponse, SearchGroupsResponse, SearchMatrixOffsetsResponse,
SearchMatrixPairsResponse, SearchMatrixPoints, SearchPointGroups, SearchPoints, SearchResponse,
SetPayloadPoints, UpdateBatchPoints, UpdateBatchResponse, UpdatePointVectors, UpsertPoints,
};
use tonic::{Request, Response, Status};
@@ -158,6 +158,26 @@ impl<T: Points> Points for PointsTelemetryWrapper<T> {
Ok(resp)
}
async fn create_vector_name(
&self,
request: Request<CreateVectorNameRequest>,
) -> Result<Response<PointsOperationResponse>, Status> {
let cn = request.get_ref().collection_name.clone();
let mut resp = self.inner.create_vector_name(request).await?;
resp.extensions_mut().insert(CollectionName(cn));
Ok(resp)
}
async fn delete_vector_name(
&self,
request: Request<DeleteVectorNameRequest>,
) -> Result<Response<PointsOperationResponse>, Status> {
let cn = request.get_ref().collection_name.clone();
let mut resp = self.inner.delete_vector_name(request).await?;
resp.extensions_mut().insert(CollectionName(cn));
Ok(resp)
}
async fn search(
&self,
request: Request<SearchPoints>,
@@ -498,6 +518,8 @@ mod tests {
update_batch(UpdateBatchPoints) -> UpdateBatchResponse,
create_field_index(CreateFieldIndexCollection) -> PointsOperationResponse,
delete_field_index(DeleteFieldIndexCollection) -> PointsOperationResponse,
create_vector_name(CreateVectorNameRequest) -> PointsOperationResponse,
delete_vector_name(DeleteVectorNameRequest) -> PointsOperationResponse,
search(SearchPoints) -> SearchResponse,
search_batch(SearchBatchPoints) -> SearchBatchResponse,
search_groups(SearchPointGroups) -> SearchGroupsResponse,
src/tonic/api/update_common.rs
+149 -1
View File
@@ -28,7 +28,7 @@ use segment::types::{
use storage::content_manager::toc::TableOfContent;
use storage::content_manager::toc::request_hw_counter::RequestHwCounter;
use storage::dispatcher::Dispatcher;
use storage::rbac::Auth;
use storage::rbac::{Access, Auth};
use tonic::{Response, Status};
use crate::common::inference::params::InferenceParams;
@@ -1019,3 +1019,151 @@ fn convert_field_type(
Ok(field_schema)
}
pub async fn create_vector_name(
dispatcher: Arc<Dispatcher>,
request: api::grpc::qdrant::CreateVectorNameRequest,
internal_params: InternalUpdateParams,
auth: Auth,
request_hw_counter: RequestHwCounter,
) -> Result<Response<PointsOperationResponseInternal>, Status> {
let api::grpc::qdrant::CreateVectorNameRequest {
collection_name,
wait,
vector_name,
vector_config,
timeout,
} = request;
let config = segment::data_types::vector_name_config::VectorNameConfig::try_from(
vector_config.ok_or_else(|| {
Status::invalid_argument("vector_config is required (dense_config or sparse_config)")
})?,
)?;
let timing = Instant::now();
let result = do_create_vector_name(
dispatcher,
collection_name,
vector_name,
config,
internal_params,
UpdateParams::from_grpc(wait, None, timeout)?,
auth,
request_hw_counter.get_counter(),
)
.await?;
let response = points_operation_response_internal(timing, result, None);
Ok(Response::new(response))
}
pub async fn create_vector_name_internal(
toc: Arc<TableOfContent>,
request: api::grpc::qdrant::CreateVectorNameRequest,
internal_params: InternalUpdateParams,
) -> Result<Response<PointsOperationResponseInternal>, Status> {
let api::grpc::qdrant::CreateVectorNameRequest {
collection_name,
wait,
vector_name,
vector_config,
timeout,
} = request;
let config = segment::data_types::vector_name_config::VectorNameConfig::try_from(
vector_config.ok_or_else(|| {
Status::invalid_argument("vector_config is required (dense_config or sparse_config)")
})?,
)?;
let operation = CollectionUpdateOperations::VectorNameOperation(
shard::operations::VectorNameOperations::CreateVectorName(
shard::operations::CreateVectorName {
vector_name,
config,
},
),
);
let timing = Instant::now();
let result = update(
&toc,
&collection_name,
operation,
internal_params,
UpdateParams::from_grpc(wait, None, timeout)?,
None,
Auth::new_internal(Access::full("Internal API")),
HwMeasurementAcc::disposable(),
)
.await?;
let response = points_operation_response_internal(timing, result, None);
Ok(Response::new(response))
}
pub async fn delete_vector_name_internal(
toc: Arc<TableOfContent>,
request: api::grpc::qdrant::DeleteVectorNameRequest,
internal_params: InternalUpdateParams,
) -> Result<Response<PointsOperationResponseInternal>, Status> {
let api::grpc::qdrant::DeleteVectorNameRequest {
collection_name,
wait,
vector_name,
timeout,
} = request;
let operation = CollectionUpdateOperations::VectorNameOperation(
shard::operations::VectorNameOperations::DeleteVectorName(
shard::operations::DeleteVectorName { vector_name },
),
);
let timing = Instant::now();
let result = update(
&toc,
&collection_name,
operation,
internal_params,
UpdateParams::from_grpc(wait, None, timeout)?,
None,
Auth::new_internal(Access::full("Internal API")),
HwMeasurementAcc::disposable(),
)
.await?;
let response = points_operation_response_internal(timing, result, None);
Ok(Response::new(response))
}
pub async fn delete_vector_name(
dispatcher: Arc<Dispatcher>,
request: api::grpc::qdrant::DeleteVectorNameRequest,
internal_params: InternalUpdateParams,
auth: Auth,
request_hw_counter: RequestHwCounter,
) -> Result<Response<PointsOperationResponseInternal>, Status> {
let api::grpc::qdrant::DeleteVectorNameRequest {
collection_name,
wait,
vector_name,
timeout,
} = request;
let timing = Instant::now();
let result = do_delete_vector_name(
dispatcher,
collection_name,
vector_name,
internal_params,
UpdateParams::from_grpc(wait, None, timeout)?,
auth,
request_hw_counter.get_counter(),
)
.await?;
let response = points_operation_response_internal(timing, result, None);
Ok(Response::new(response))
}
tests/consensus_tests/auth_tests/test_jwt_access.py
+37
View File
@@ -286,6 +286,22 @@ ACTION_ACCESS = {
"qdrant.Points/DeleteFieldIndex",
coll_prw=False,
),
"create_vector_name": EndpointAccess(
False,
True,
True,
"PUT /collections/{collection_name}/vectors/{vector_name}",
"qdrant.Points/CreateVectorName",
coll_prw=False,
),
"delete_vector_name": EndpointAccess(
False,
True,
True,
"DELETE /collections/{collection_name}/vectors/{vector_name}",
"qdrant.Points/DeleteVectorName",
coll_prw=False,
),
### Collection Snapshots ###
"list_collection_snapshots": EndpointAccess(
True,
@@ -1280,6 +1296,27 @@ def test_delete_index():
)
def test_create_vector_name():
check_access(
"create_vector_name",
rest_request={"dense": {"size": 4, "distance": "Cosine"}},
path_params={"collection_name": COLL_NAME, "vector_name": "new_vec"},
grpc_request={
"collection_name": COLL_NAME,
"vector_name": "new_vec",
"dense_config": {"size": 4, "distance": 1},
},
)
def test_delete_vector_name():
check_access(
"delete_vector_name",
path_params={"collection_name": COLL_NAME, "vector_name": "fake_vector_name"},
grpc_request={"collection_name": COLL_NAME, "vector_name": "fake_vector_name"},
)
def test_list_collection_snapshots():
check_access(
"list_collection_snapshots",
tests/consensus_tests/test_named_vector_crud.py
+340
View File
@@ -0,0 +1,340 @@
import pathlib
from time import sleep
import requests
from .fixtures import create_collection, upsert_random_points
from .test_shard_transfer_deferred import VECTOR_DIM
from .utils import *
N_PEERS = 3
COLLECTION_NAME = "test_vector_crud"
def create_vector_name(peer_url, collection_name, vector_name, config, timeout=10, wait=True):
"""Create a named vector via PUT /collections/{name}/vectors/{vector_name}"""
r = requests.put(
f"{peer_url}/collections/{collection_name}/vectors/{vector_name}?timeout={timeout}&wait={'true' if wait else 'false'}",
json=config,
)
assert_http_ok(r)
return r.json()
def delete_vector_name(peer_url, collection_name, vector_name, timeout=10, wait=True):
"""Delete a named vector via DELETE /collections/{name}/vectors/{vector_name}"""
r = requests.delete(
f"{peer_url}/collections/{collection_name}/vectors/{vector_name}?timeout={timeout}&wait={'true' if wait else 'false'}",
)
assert_http_ok(r)
return r.json()
def get_collection_vectors_config(peer_url, collection_name):
"""Get the vectors configuration from collection info."""
info = get_collection_info(peer_url, collection_name)
return info.get("config", {}).get("params", {}).get("vectors", {})
def get_collection_sparse_vectors_config(peer_url, collection_name):
"""Get the sparse vectors configuration from collection info."""
info = get_collection_info(peer_url, collection_name)
return info.get("config", {}).get("params", {}).get("sparse_vectors", {})
def wait_collection_vector_config(peer_url, collection_name, vector_name, expected_size):
"""Wait until a peer's collection config contains the given vector with the expected size."""
def check():
vectors = get_collection_vectors_config(peer_url, collection_name)
return vector_name in vectors and vectors[vector_name].get("size") == expected_size
wait_for(check)
def get_optimizer_status(peer_url, collection_name):
"""Get optimizer status from collection info."""
info = get_collection_info(peer_url, collection_name)
return info.get("status", {})
def test_create_vector_no_optimization(tmp_path: pathlib.Path):
"""
Test that creating named vectors does not trigger segment optimization.
1. Create cluster, create collection, upload 1000 points.
2. Set indexing threshold low to trigger indexing, wait for green.
3. Record segment count.
4. Create a new dense named vector.
5. Assert segment count unchanged (no optimization triggered).
6. Create a new sparse named vector.
7. Assert segment count unchanged (no optimization triggered).
"""
VECTOR_DIM = 64
VECTOR_DIM2 = 99
assert_project_root()
peer_api_uris, peer_dirs, bootstrap_uri = start_cluster(tmp_path, N_PEERS, port_seed=10000)
# Create collection with low indexing threshold to trigger indexing
r = requests.put(
f"{peer_api_uris[0]}/collections/{COLLECTION_NAME}?timeout=30",
json={
"vectors": {"default": {"size": VECTOR_DIM, "distance": "Cosine"}},
"shard_number": 1,
"replication_factor": 1,
"optimizers_config": {
"indexing_threshold": 100,
},
},
)
assert_http_ok(r)
wait_collection_exists_and_active_on_all_peers(
collection_name=COLLECTION_NAME, peer_api_uris=peer_api_uris
)
# Upload 1000 points
for i in range(10):
points = [
{
"id": i * 100 + j,
"vector": {"default": [float(x) / 1000 for x in range(VECTOR_DIM)] },
"payload": {"idx": i * 100 + j},
}
for j in range(100)
]
r = requests.put(
f"{peer_api_uris[0]}/collections/{COLLECTION_NAME}/points?wait=true",
json={"points": points},
)
assert_http_ok(r)
# Wait for indexing to complete (collection goes green)
wait_collection_green(peer_api_uris[0], COLLECTION_NAME)
# Record segment state after indexing
status_before_creation = get_optimizer_status(peer_api_uris[0], COLLECTION_NAME)
print(f"Segments after indexing: {status_before_creation}")
# Create a new dense named vector
create_vector_name(
peer_api_uris[0],
COLLECTION_NAME,
"new_dense",
{"dense": {"size": VECTOR_DIM2, "distance": "Dot"}},
)
# Verify no optimization was triggered - segment count should be unchanged
status_after_creation = get_optimizer_status(peer_api_uris[0], COLLECTION_NAME)
print(f"Segments after creating dense vector: {status_after_creation}")
assert status_after_creation == status_before_creation, (
f"Segment count changed after creating dense vector: {status_before_creation} -> {status_after_creation}"
)
# Verify the new vector exists in collection config on all peers
for uri in peer_api_uris:
vectors = get_collection_vectors_config(uri, COLLECTION_NAME)
assert "new_dense" in vectors, f"new_dense not in vectors config on {uri}"
assert vectors["new_dense"]["size"] == VECTOR_DIM2
# Create a new sparse named vector
create_vector_name(
peer_api_uris[0],
COLLECTION_NAME,
"new_sparse",
{"sparse": {}},
)
# Verify no optimization was triggered - segment count should be unchanged
status_after_creation = get_optimizer_status(peer_api_uris[0], COLLECTION_NAME)
print(f"Segments after creating sparse vector: {status_after_creation}")
assert status_after_creation == status_before_creation, (
f"Segment count changed after creating sparse vector: {status_before_creation} -> {status_after_creation}"
)
# Verify sparse vector exists on all peers
for uri in peer_api_uris:
sparse = get_collection_sparse_vectors_config(uri, COLLECTION_NAME)
assert "new_sparse" in sparse, f"new_sparse not in sparse vectors config on {uri}"
def test_vector_crud_with_consensus_snapshot(tmp_path: pathlib.Path):
"""
Test that named vector create/delete survives consensus snapshot recovery.
1. Create cluster with aggressive WAL compaction (forces consensus snapshots).
2. Create collection, upload 1000 points.
3. Kill one node.
4. Delete the original vector, create a new one with different dimensions.
5. Restart the killed node — it must recover via consensus snapshot.
6. Verify the restarted node has the new vector config.
"""
assert_project_root()
VECTOR_NAME = "v1"
VECTOR_DIM = 64
VECTOR_DIM2 = 78
env = {
# Force consensus snapshot by aggressively compacting WAL
"QDRANT__CLUSTER__CONSENSUS__COMPACT_WAL_ENTRIES": "1",
}
peer_api_uris, peer_dirs, bootstrap_uri = start_cluster(
tmp_path, N_PEERS, port_seed=11000, extra_env=env
)
# Create collection with a named dense vector VECTOR_NAME
r = requests.put(
f"{peer_api_uris[0]}/collections/{COLLECTION_NAME}?timeout=30",
json={
"vectors": {
VECTOR_NAME: {"size": VECTOR_DIM, "distance": "Cosine"},
},
"shard_number": 1,
"replication_factor": 3,
},
)
assert_http_ok(r)
wait_collection_exists_and_active_on_all_peers(
collection_name=COLLECTION_NAME, peer_api_uris=peer_api_uris
)
# Upload 1000 points with v1
for i in range(10):
points = [
{
"id": i * 100 + j,
"vector": {VECTOR_NAME: [float(x) / 1000 for x in range(VECTOR_DIM)]},
"payload": {"idx": i * 100 + j},
}
for j in range(100)
]
r = requests.put(
f"{peer_api_uris[0]}/collections/{COLLECTION_NAME}/points?wait=true",
json={"points": points},
)
assert_http_ok(r)
# Verify all 1000 points on all peers
for uri in peer_api_uris:
wait_collection_points_count(uri, COLLECTION_NAME, 1000)
# Kill the last peer
killed_peer = processes.pop()
killed_peer.kill()
print(f"Killed peer at port {killed_peer.http_port}")
# Perform some consensus operations to trigger WAL compaction + snapshot
# Delete vector v1 and create v2 with different dimensions
# Use wait=False because a peer is down — await_consensus_sync can't reach all peers
delete_vector_name(peer_api_uris[0], COLLECTION_NAME, VECTOR_NAME, wait=False)
sleep(1) # Give consensus time to propagate to surviving peers
# Verify v1 is gone on surviving peers
for uri in peer_api_uris[:-1]:
vectors = get_collection_vectors_config(uri, COLLECTION_NAME)
assert VECTOR_NAME not in vectors, f"{VECTOR_NAME} should be deleted on {uri}"
# Create VECTOR_NAME with different dimensions
create_vector_name(
peer_api_uris[0],
COLLECTION_NAME,
VECTOR_NAME,
{"dense": {"size": VECTOR_DIM2, "distance": "Dot"}},
)
sleep(1) # Give consensus time to propagate to surviving peers
# Verify v2 exists on surviving peers
for uri in peer_api_uris[:-1]:
vectors = get_collection_vectors_config(uri, COLLECTION_NAME)
assert VECTOR_NAME in vectors, f"{VECTOR_NAME} not in config on {uri}"
assert vectors[VECTOR_NAME]["size"] == VECTOR_DIM2
# Do a few more consensus operations to ensure WAL compaction triggers snapshot
for _ in range(5):
create_vector_name(
peer_api_uris[0], COLLECTION_NAME, "tmp_vec",
{"dense": {"size": 2, "distance": "Cosine"}},
)
delete_vector_name(peer_api_uris[0], COLLECTION_NAME, "tmp_vec")
# Upload 200 points with v1
for i in range(2):
points = [
{
"id": i * 100 + j,
"vector": {
VECTOR_NAME: [float(x) / 1000 for x in range(VECTOR_DIM2)]
},
"payload": {"idx": i * 100 + j},
}
for j in range(100)
]
r = requests.put(
f"{peer_api_uris[0]}/collections/{COLLECTION_NAME}/points?wait=true",
json={"points": points},
)
assert_http_ok(r)
# Restart the killed peer — it should recover via consensus snapshot
new_url = start_peer(
peer_dirs[-1], "peer_restarted.log", bootstrap_uri, port=21000, extra_env=env
)
peer_api_uris[-1] = new_url
wait_all_peers_up([new_url])
wait_collection_exists_and_active_on_all_peers(
collection_name=COLLECTION_NAME, peer_api_uris=[new_url]
)
# Wait for the restarted node to sync the correct vector config via consensus
wait_collection_vector_config(new_url, COLLECTION_NAME, VECTOR_NAME, VECTOR_DIM2)
# Verify point count is still correct
wait_collection_points_count(new_url, COLLECTION_NAME, 1000)
# Verify the restarted peer actually stores vectors with the new schema.
# Points 0–199 were upserted with VECTOR_DIM2 after the delete+create;
# their vectors must be retrievable with the correct dimensionality.
r = requests.post(
f"{new_url}/collections/{COLLECTION_NAME}/points/scroll",
json={
"limit": 10,
"with_vector": [VECTOR_NAME],
"filter": {"must": [{"key": "idx", "range": {"lte": 9}}]},
},
)
assert_http_ok(r)
scroll_result = r.json()["result"]["points"]
assert len(scroll_result) > 0, "Expected at least one point from scroll"
for point in scroll_result:
vec = point.get("vector", {}).get(VECTOR_NAME)
assert vec is not None, (
f"Point {point['id']} on restarted peer has no vector '{VECTOR_NAME}'"
)
assert len(vec) == VECTOR_DIM2, (
f"Point {point['id']}: expected dim {VECTOR_DIM2}, got {len(vec)}"
)
# Also verify that a search with the new dimensionality works on the restarted peer
r = requests.post(
f"{new_url}/collections/{COLLECTION_NAME}/points/search",
json={
"vector": {
"name": VECTOR_NAME,
"vector": [0.1] * VECTOR_DIM2,
},
"limit": 5,
},
)
assert_http_ok(r)
search_result = r.json()["result"]
assert len(search_result) > 0, (
"Search with new vector schema returned no results on restarted peer"
)
tests/openapi/test_named_vector_crud.py
+231
View File
@@ -0,0 +1,231 @@
import pytest
import time
import requests
from .helpers.helpers import request_with_validation
from .helpers.settings import QDRANT_HOST
VECTOR_SIZE1 = 4
VECTOR_SIZE2 = 8
@pytest.fixture(autouse=True, scope="module")
def setup(collection_name):
# Drop if exists
requests.delete(f"{QDRANT_HOST}/collections/{collection_name}")
# Create collection with two named vectors
response = request_with_validation(
api='/collections/{collection_name}',
method="PUT",
path_params={'collection_name': collection_name},
body={
"vectors": {
"vec_a": {"size": VECTOR_SIZE1, "distance": "Cosine"},
"vec_b": {"size": VECTOR_SIZE2, "distance": "Cosine"},
},
"optimizers_config": {
"indexing_threshold": 1,
},
},
)
assert response.ok
yield
requests.delete(f"{QDRANT_HOST}/collections/{collection_name}")
def test_delete_recreate_vector_scroll(collection_name):
"""
Deleting a named vector and recreating it must not break scroll.
Reproduces a bug where EmptyDenseVectorStorage on immutable segments
has total_vector_count=0 while points exist, causing a panic
in vectors_by_offsets during scroll/retrieve.
"""
# Upsert points with both vectors
points = [
{
"id": i,
"vector": {
"vec_a": [float(x) / 100 for x in range(VECTOR_SIZE1)],
"vec_b": [float(x) / 100 for x in range(VECTOR_SIZE2)],
},
"payload": {"idx": i},
}
for i in range(200)
]
response = request_with_validation(
api='/collections/{collection_name}/points',
method="PUT",
path_params={'collection_name': collection_name},
query_params={'wait': 'true'},
body={"points": points},
)
assert response.ok
# Delete vec_a (use plain requests — OpenAPI response schema is mismatched)
response = requests.delete(
f"{QDRANT_HOST}/collections/{collection_name}/vectors/vec_a",
params={'wait': 'true'},
)
assert response.ok
# Recreate vec_a with different dimensions
response = requests.put(
f"{QDRANT_HOST}/collections/{collection_name}/vectors/vec_a",
params={'wait': 'true'},
json={"dense": {"size": 6, "distance": "Dot"}},
)
assert response.ok
# Scroll with vectors — must not panic
response = request_with_validation(
api='/collections/{collection_name}/points/scroll',
method="POST",
path_params={'collection_name': collection_name},
body={"limit": 10, "with_vector": True},
)
assert response.ok
result = response.json()['result']
assert len(result['points']) == 10
for point in result['points']:
assert 'vec_b' in point['vector'], f"vec_b missing from point {point['id']}"
assert len(point['vector']['vec_b']) == VECTOR_SIZE2
def test_delete_recreate_indexed_vector_scroll():
"""
Delete and recreate a named vector on indexed segments (HNSW built),
then scroll, upsert new data, and scroll again.
"""
_run_delete_recreate_scroll(wait_for_indexing=True)
def test_delete_recreate_unindexed_vector_scroll():
"""
Delete and recreate a named vector without waiting for indexing,
then scroll, upsert new data, and scroll again.
"""
_run_delete_recreate_scroll(wait_for_indexing=False)
def _run_delete_recreate_scroll(wait_for_indexing: bool):
coll = "test_named_vec_idx" if wait_for_indexing else "test_named_vec_noidx"
dim_a = 128
dim_b = 64
dim_a_new = 96
num_points = 200
# Setup
requests.delete(f"{QDRANT_HOST}/collections/{coll}")
response = requests.put(
f"{QDRANT_HOST}/collections/{coll}",
json={
"vectors": {
"vec_a": {"size": dim_a, "distance": "Cosine"},
"vec_b": {"size": dim_b, "distance": "Cosine"},
},
"optimizers_config": {
"indexing_threshold": 10 if wait_for_indexing else 10_000,
},
},
)
assert response.ok
# Upsert points
points = [
{
"id": i,
"vector": {
"vec_a": [float(i * x % 97) / 100 for x in range(dim_a)],
"vec_b": [float(i * x % 53) / 100 for x in range(dim_b)],
},
}
for i in range(num_points)
]
response = requests.put(
f"{QDRANT_HOST}/collections/{coll}/points?wait=true",
json={"points": points},
)
assert response.ok
if wait_for_indexing:
wait_collection_green(coll)
# Delete vec_a
response = requests.delete(
f"{QDRANT_HOST}/collections/{coll}/vectors/vec_a?wait=true",
)
assert response.ok
# Recreate vec_a with different dimensions
response = requests.put(
f"{QDRANT_HOST}/collections/{coll}/vectors/vec_a?wait=true",
json={"dense": {"size": dim_a_new, "distance": "Dot"}},
)
assert response.ok
# First scroll — must not panic
response = requests.post(
f"{QDRANT_HOST}/collections/{coll}/points/scroll",
json={"limit": 10, "with_vector": True},
)
assert response.ok
result = response.json()['result']
assert len(result['points']) == 10
for point in result['points']:
assert 'vec_b' in point['vector']
assert len(point['vector']['vec_b']) == dim_b
# Upsert some points with data for the new vec_a
updated_points = [
{
"id": i,
"vector": {
"vec_a": [float(i * x % 41) / 100 for x in range(dim_a_new)],
"vec_b": [float(i * x % 53) / 100 for x in range(dim_b)],
},
}
for i in range(50)
]
response = requests.put(
f"{QDRANT_HOST}/collections/{coll}/points?wait=true",
json={"points": updated_points},
)
assert response.ok
# Second scroll — verify updated points have vec_a data
response = requests.post(
f"{QDRANT_HOST}/collections/{coll}/points/scroll",
json={"limit": num_points, "with_vector": True},
)
assert response.ok
result = response.json()['result']
assert len(result['points']) == num_points
updated_ids = set(range(50))
for point in result['points']:
assert 'vec_b' in point['vector']
assert len(point['vector']['vec_b']) == dim_b
if point['id'] in updated_ids:
assert 'vec_a' in point['vector'], f"vec_a missing from updated point {point['id']}"
assert len(point['vector']['vec_a']) == dim_a_new
# Cleanup
requests.delete(f"{QDRANT_HOST}/collections/{coll}")
def wait_collection_green(collection_name, timeout=30):
"""Poll collection status until optimizer is idle."""
start = time.time()
while time.time() - start < timeout:
r = requests.get(f"{QDRANT_HOST}/collections/{collection_name}")
assert r.ok
if r.json()['result']['status'] == 'green':
return
time.sleep(0.5)
raise TimeoutError(f"Collection {collection_name} did not turn green within {timeout}s")
tests/openapi_consistency_check.sh
+1 -1
View File
@@ -37,7 +37,7 @@ rm -f ./docs/redoc/master/.diff.openapi.json
NUMBER_OF_APIS=$(cat ./docs/redoc/master/openapi.json | jq '[.paths[] | length] | add')
EXPECTED_NUMBER_OF_APIS=73
EXPECTED_NUMBER_OF_APIS=75
if [ "$NUMBER_OF_APIS" -ne "$EXPECTED_NUMBER_OF_APIS" ]; then
echo "ERROR: It looks like the total number of APIs has changed."