mirror of
https://github.com/qdrant/qdrant.git
synced 2026-09-28 17:07:49 -05:00
* Add empty placeholder vector storage types for named vector CRUD Introduce EmptyDenseVectorStorage and EmptySparseVectorStorage as placeholder storages for newly created named vectors on immutable segments. These report all vectors as deleted, consume no disk space, and are reconstructed from segment config on load via the new VectorStorageType::Empty and SparseVectorStorageType::Empty variants. Key design decisions: - is_on_disk is derived from original user config, not hardcoded - MultiVectorConfig is preserved for multi-vector support - Config mismatch optimizer skips Empty storage to avoid false rebuilds - Quantization delegates normally (handles 0 vectors gracefully) - get_vector includes debug_assert to catch unexpected access Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * [AI] segment-level operations for creating and deleting anmed vectors * [AI] implement named vector creation and deleting in proxy segment * [AI] Step 3: Proxy Segment Handling for Named Vector Operations * [AI] implement for Edge * [AI] implement consensus operations for named vector operations * [AI] refactor VectorNameConfig, remove VectorNameConfigInternal * [AI] handle vector schema inconsistency in raft snapshot recovery * [AI] rest + grpc API * [AI] clippy * [AI] generate openAPI schema * fmt * ci fixes * [AI] fix jwt access test * [AI] nop operation for awaiting of consensus-commited update ops * [AI] move vector name operations into points service * [AI] implement internal api for vector name operations * [AI] change collection-level config along with segment level operation * [AI] vector schema reconceliation instead of error * fmt * missing compile-time option * [AI] integration test * [AI] fix missing JWT tests * [AI] remove NOP * [AI] openapi test * [AI] fix initialization of mutable segment * [AI] more simple integration tests * fmt * [AI] make cluster test a bit harder * [AI] make test less flacky * [AI] rabbit comments * [AI] check params compatibility before writing vector config * [AI] make sure to register vector storages in structure payload index * [AI] vector name validation * lower vector length validation to 200 chars to account for prefix in filename * [AI] proxy segment: prevent stale data leak through optimization * fmt * [AI] filter out removed vectors from proxy response * [AI] handle vector name in proxy * fmt * adjust proxy info based on dropped vectors * [AI] proxy segment: update filters to correct has_vector condition * fmt * clippy * Fix consensus snapshot applicaiton for vector schema --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
291 lines
10 KiB
Rust
291 lines
10 KiB
Rust
use bytemuck::TransparentWrapperAlloc as _;
|
|
use derive_more::Into;
|
|
use pyo3::prelude::*;
|
|
use segment::data_types::modifier::Modifier;
|
|
use segment::json_path::JsonPath;
|
|
use segment::types::{
|
|
Distance, Filter, MultiVectorConfig, Payload, VectorNameBuf, VectorStorageDatatype,
|
|
};
|
|
use shard::operations::point_ops::{PointIdsList, PointInsertOperationsInternal, UpdateMode};
|
|
use shard::operations::*;
|
|
|
|
use crate::*;
|
|
|
|
#[pyclass(name = "UpdateOperation", from_py_object)]
|
|
#[derive(Clone, Debug, Into)]
|
|
pub struct PyUpdateOperation(CollectionUpdateOperations);
|
|
|
|
#[pymethods]
|
|
impl PyUpdateOperation {
|
|
#[staticmethod]
|
|
#[pyo3(signature = (points, condition=None, update_mode=None))]
|
|
pub fn upsert_points(
|
|
points: Vec<PyPoint>,
|
|
condition: Option<PyFilter>,
|
|
update_mode: Option<PyUpdateMode>,
|
|
) -> Self {
|
|
let points = PointInsertOperationsInternal::PointsList(PyPoint::peel_vec(points));
|
|
let update_mode = update_mode.map(UpdateMode::from);
|
|
|
|
let operation = match (condition, update_mode) {
|
|
// If condition or non-default update_mode is provided, use conditional upsert
|
|
(Some(condition), mode) => point_ops::PointOperations::UpsertPointsConditional(
|
|
point_ops::ConditionalInsertOperationInternal {
|
|
points_op: points,
|
|
condition: Filter::from(condition),
|
|
update_mode: mode,
|
|
},
|
|
),
|
|
(None, Some(mode)) => point_ops::PointOperations::UpsertPointsConditional(
|
|
point_ops::ConditionalInsertOperationInternal {
|
|
points_op: points,
|
|
condition: Filter::default(),
|
|
update_mode: Some(mode),
|
|
},
|
|
),
|
|
// Default case: regular upsert
|
|
(None, None) => point_ops::PointOperations::UpsertPoints(points),
|
|
};
|
|
|
|
Self(CollectionUpdateOperations::PointOperation(operation))
|
|
}
|
|
|
|
#[staticmethod]
|
|
pub fn delete_points(point_ids: Vec<PyPointId>) -> Self {
|
|
let operation = point_ops::PointOperations::DeletePoints {
|
|
ids: PyPointId::peel_vec(point_ids),
|
|
};
|
|
|
|
Self(CollectionUpdateOperations::PointOperation(operation))
|
|
}
|
|
|
|
#[staticmethod]
|
|
pub fn delete_points_by_filter(filter: PyFilter) -> Self {
|
|
let operation = point_ops::PointOperations::DeletePointsByFilter(Filter::from(filter));
|
|
Self(CollectionUpdateOperations::PointOperation(operation))
|
|
}
|
|
|
|
#[staticmethod]
|
|
#[pyo3(signature = (point_vectors, condition=None))]
|
|
pub fn update_vectors(point_vectors: Vec<PyPointVectors>, condition: Option<PyFilter>) -> Self {
|
|
let operation = vector_ops::VectorOperations::UpdateVectors(vector_ops::UpdateVectorsOp {
|
|
points: PyPointVectors::peel_vec(point_vectors),
|
|
update_filter: condition.map(Filter::from),
|
|
});
|
|
|
|
Self(CollectionUpdateOperations::VectorOperation(operation))
|
|
}
|
|
|
|
#[staticmethod]
|
|
pub fn delete_vectors(point_ids: Vec<PyPointId>, vector_names: Vec<VectorNameBuf>) -> Self {
|
|
let operation = vector_ops::VectorOperations::DeleteVectors(
|
|
PointIdsList::from(PyPointId::peel_vec(point_ids)),
|
|
vector_names,
|
|
);
|
|
|
|
Self(CollectionUpdateOperations::VectorOperation(operation))
|
|
}
|
|
|
|
#[staticmethod]
|
|
pub fn delete_vectors_by_filter(filter: PyFilter, vector_names: Vec<VectorNameBuf>) -> Self {
|
|
let operation =
|
|
vector_ops::VectorOperations::DeleteVectorsByFilter(Filter::from(filter), vector_names);
|
|
|
|
Self(CollectionUpdateOperations::VectorOperation(operation))
|
|
}
|
|
|
|
#[staticmethod]
|
|
#[pyo3(signature = (point_ids, payload, key=None))]
|
|
pub fn set_payload(
|
|
point_ids: Vec<PyPointId>,
|
|
payload: PyPayload,
|
|
key: Option<PyJsonPath>,
|
|
) -> Self {
|
|
let operation = payload_ops::PayloadOps::SetPayload(payload_ops::SetPayloadOp {
|
|
payload: Payload::from(payload),
|
|
points: Some(PyPointId::peel_vec(point_ids)),
|
|
filter: None,
|
|
key: key.map(JsonPath::from),
|
|
});
|
|
|
|
Self(CollectionUpdateOperations::PayloadOperation(operation))
|
|
}
|
|
|
|
#[staticmethod]
|
|
#[pyo3(signature = (filter, payload, key=None))]
|
|
pub fn set_payload_by_filter(
|
|
filter: PyFilter,
|
|
payload: PyPayload,
|
|
key: Option<PyJsonPath>,
|
|
) -> Self {
|
|
let operation = payload_ops::PayloadOps::SetPayload(payload_ops::SetPayloadOp {
|
|
payload: Payload::from(payload),
|
|
points: None,
|
|
filter: Some(Filter::from(filter)),
|
|
key: key.map(JsonPath::from),
|
|
});
|
|
|
|
Self(CollectionUpdateOperations::PayloadOperation(operation))
|
|
}
|
|
|
|
#[staticmethod]
|
|
pub fn delete_payload(point_ids: Vec<PyPointId>, keys: Vec<PyJsonPath>) -> Self {
|
|
let operation = payload_ops::PayloadOps::DeletePayload(payload_ops::DeletePayloadOp {
|
|
keys: PyJsonPath::peel_vec(keys),
|
|
points: Some(PyPointId::peel_vec(point_ids)),
|
|
filter: None,
|
|
});
|
|
|
|
Self(CollectionUpdateOperations::PayloadOperation(operation))
|
|
}
|
|
|
|
#[staticmethod]
|
|
pub fn delete_payload_by_filter(filter: PyFilter, keys: Vec<PyJsonPath>) -> Self {
|
|
let operation = payload_ops::PayloadOps::DeletePayload(payload_ops::DeletePayloadOp {
|
|
keys: PyJsonPath::peel_vec(keys),
|
|
points: None,
|
|
filter: Some(Filter::from(filter)),
|
|
});
|
|
|
|
Self(CollectionUpdateOperations::PayloadOperation(operation))
|
|
}
|
|
|
|
#[staticmethod]
|
|
pub fn clear_payload(point_ids: Vec<PyPointId>) -> Self {
|
|
let operation = payload_ops::PayloadOps::ClearPayload {
|
|
points: PyPointId::peel_vec(point_ids),
|
|
};
|
|
|
|
Self(CollectionUpdateOperations::PayloadOperation(operation))
|
|
}
|
|
|
|
#[staticmethod]
|
|
pub fn clear_payload_by_filter(filter: PyFilter) -> Self {
|
|
let operation = payload_ops::PayloadOps::ClearPayloadByFilter(Filter::from(filter));
|
|
Self(CollectionUpdateOperations::PayloadOperation(operation))
|
|
}
|
|
|
|
#[staticmethod]
|
|
#[pyo3(signature = (point_ids, payload, key=None))]
|
|
pub fn overwrite_payload(
|
|
point_ids: Vec<PyPointId>,
|
|
payload: PyPayload,
|
|
key: Option<PyJsonPath>,
|
|
) -> Self {
|
|
let operation = payload_ops::PayloadOps::OverwritePayload(payload_ops::SetPayloadOp {
|
|
payload: Payload::from(payload),
|
|
points: Some(PyPointId::peel_vec(point_ids)),
|
|
filter: None,
|
|
key: key.map(JsonPath::from),
|
|
});
|
|
|
|
Self(CollectionUpdateOperations::PayloadOperation(operation))
|
|
}
|
|
|
|
#[staticmethod]
|
|
#[pyo3(signature = (filter, payload, key=None))]
|
|
pub fn overwrite_payload_by_filter(
|
|
filter: PyFilter,
|
|
payload: PyPayload,
|
|
key: Option<PyJsonPath>,
|
|
) -> Self {
|
|
let operation = payload_ops::PayloadOps::OverwritePayload(payload_ops::SetPayloadOp {
|
|
payload: Payload::from(payload),
|
|
points: None,
|
|
filter: Some(Filter::from(filter)),
|
|
key: key.map(JsonPath::from),
|
|
});
|
|
|
|
Self(CollectionUpdateOperations::PayloadOperation(operation))
|
|
}
|
|
|
|
#[staticmethod]
|
|
pub fn create_field_index(field_name: PyJsonPath, schema: PyPayloadFieldSchema) -> Self {
|
|
let operation = FieldIndexOperations::CreateIndex(CreateIndex {
|
|
field_name: JsonPath::from(field_name),
|
|
field_schema: Some(PayloadFieldSchema::from(schema)),
|
|
});
|
|
|
|
Self(CollectionUpdateOperations::FieldIndexOperation(operation))
|
|
}
|
|
|
|
#[staticmethod]
|
|
pub fn delete_field_index(field_name: PyJsonPath) -> Self {
|
|
let operation = FieldIndexOperations::DeleteIndex(JsonPath::from(field_name));
|
|
Self(CollectionUpdateOperations::FieldIndexOperation(operation))
|
|
}
|
|
|
|
/// Create a new dense named vector on the collection.
|
|
#[staticmethod]
|
|
#[pyo3(signature = (vector_name, size, distance, multivector_config=None, datatype=None))]
|
|
pub fn create_dense_vector(
|
|
vector_name: String,
|
|
size: usize,
|
|
distance: PyDistance,
|
|
multivector_config: Option<PyMultiVectorConfig>,
|
|
datatype: Option<PyVectorStorageDatatype>,
|
|
) -> Self {
|
|
let config = vector_name_ops::VectorNameConfig::dense(vector_name_ops::DenseVectorConfig {
|
|
size,
|
|
distance: Distance::from(distance),
|
|
multivector_config: multivector_config.map(MultiVectorConfig::from),
|
|
datatype: datatype.map(VectorStorageDatatype::from),
|
|
});
|
|
let operation = VectorNameOperations::CreateVectorName(CreateVectorName {
|
|
vector_name,
|
|
config,
|
|
});
|
|
Self(CollectionUpdateOperations::VectorNameOperation(operation))
|
|
}
|
|
|
|
/// Create a new sparse named vector on the collection.
|
|
#[staticmethod]
|
|
#[pyo3(signature = (vector_name, modifier=None, datatype=None))]
|
|
pub fn create_sparse_vector(
|
|
vector_name: String,
|
|
modifier: Option<PyModifier>,
|
|
datatype: Option<PyVectorStorageDatatype>,
|
|
) -> Self {
|
|
let config =
|
|
vector_name_ops::VectorNameConfig::sparse(vector_name_ops::SparseVectorConfig {
|
|
modifier: modifier.map(Modifier::from),
|
|
datatype: datatype.map(VectorStorageDatatype::from),
|
|
});
|
|
let operation = VectorNameOperations::CreateVectorName(CreateVectorName {
|
|
vector_name,
|
|
config,
|
|
});
|
|
Self(CollectionUpdateOperations::VectorNameOperation(operation))
|
|
}
|
|
|
|
/// Delete a named vector from the collection.
|
|
#[staticmethod]
|
|
pub fn delete_vector_name(vector_name: String) -> Self {
|
|
let operation = VectorNameOperations::DeleteVectorName(DeleteVectorName { vector_name });
|
|
Self(CollectionUpdateOperations::VectorNameOperation(operation))
|
|
}
|
|
}
|
|
|
|
/// Defines the mode of the upsert operation
|
|
#[pyclass(name = "UpdateMode", eq, eq_int, from_py_object)]
|
|
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
|
pub enum PyUpdateMode {
|
|
/// Default mode - insert new points, update existing points
|
|
#[default]
|
|
Upsert = 0,
|
|
/// Only insert new points, do not update existing points
|
|
InsertOnly = 1,
|
|
/// Only update existing points, do not insert new points
|
|
UpdateOnly = 2,
|
|
}
|
|
|
|
impl From<PyUpdateMode> for UpdateMode {
|
|
fn from(mode: PyUpdateMode) -> Self {
|
|
match mode {
|
|
PyUpdateMode::Upsert => UpdateMode::Upsert,
|
|
PyUpdateMode::InsertOnly => UpdateMode::InsertOnly,
|
|
PyUpdateMode::UpdateOnly => UpdateMode::UpdateOnly,
|
|
}
|
|
}
|
|
}
|