diff --git a/Cargo.lock b/Cargo.lock index 37596d82ab..00dc05c4c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1943,6 +1943,7 @@ dependencies = [ "bytemuck", "derive_more 2.0.1", "edge", + "edge-py-codegen", "fnv", "indexmap 2.12.1", "ordered-float 5.1.0", @@ -1954,6 +1955,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "edge-py-codegen" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "either" version = "1.13.0" diff --git a/Cargo.toml b/Cargo.toml index 29e3a1c250..a4dfd8bcbe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -329,6 +329,7 @@ members = [ "lib/common/*", "lib/edge", "lib/edge/python", + "lib/edge/python/codegen", "lib/gridstore", "lib/macros", "lib/posting_list", diff --git a/lib/edge/python/Cargo.toml b/lib/edge/python/Cargo.toml index dcc9527f45..a99a34038c 100644 --- a/lib/edge/python/Cargo.toml +++ b/lib/edge/python/Cargo.toml @@ -5,16 +5,17 @@ authors = ["Qdrant Team "] license = "Apache-2.0" edition = "2024" -[lints] -workspace = true - [lib] name = "qdrant_edge" crate-type = ["cdylib"] +[lints] +workspace = true + [dependencies] edge = { path = ".." } +edge-py-codegen = { path = "./codegen" } segment = { path = "../../segment", default-features = false } shard = { path = "../../shard" } sparse = { path = "../../sparse" } diff --git a/lib/edge/python/codegen/Cargo.toml b/lib/edge/python/codegen/Cargo.toml new file mode 100644 index 0000000000..151c955038 --- /dev/null +++ b/lib/edge/python/codegen/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "edge-py-codegen" +version = "0.1.0" +authors = ["Qdrant Team "] +license = "Apache-2.0" +edition = "2024" + +[lib] +proc-macro = true + +[lints] +workspace = true + + +[dependencies] +proc-macro2 = "1.0" +syn = { version = "2.0", features = ["full"] } +quote = "1.0" diff --git a/lib/edge/python/codegen/src/lib.rs b/lib/edge/python/codegen/src/lib.rs new file mode 100644 index 0000000000..4e54d5776c --- /dev/null +++ b/lib/edge/python/codegen/src/lib.rs @@ -0,0 +1,12 @@ +mod pyclass_repr; + +#[proc_macro_attribute] +pub fn pyclass_repr( + _attributes: proc_macro::TokenStream, + input: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + match pyclass_repr::pyclass_repr(input.into()) { + Ok(output) => output.into(), + Err(error) => error.to_compile_error().into(), + } +} diff --git a/lib/edge/python/codegen/src/pyclass_repr.rs b/lib/edge/python/codegen/src/pyclass_repr.rs new file mode 100644 index 0000000000..31fd71abd8 --- /dev/null +++ b/lib/edge/python/codegen/src/pyclass_repr.rs @@ -0,0 +1,34 @@ +pub fn pyclass_repr(input: proc_macro2::TokenStream) -> syn::Result { + let impl_block: syn::ItemImpl = syn::parse2(input)?; + + let type_name = &impl_block.self_ty; + let mut fields = Vec::new(); + + for item in &impl_block.items { + let syn::ImplItem::Fn(func) = item else { + continue; + }; + + if !func.attrs.iter().any(|attr| attr.path().is_ident("getter")) { + continue; + } + + fields.push(&func.sig.ident); + } + + let output = quote::quote! { + #impl_block + + impl crate::repr::Repr for #type_name { + fn fmt(&self, f: &mut crate::repr::Formatter<'_>) -> std::fmt::Result { + use crate::repr::WriteExt as _; + + f.class::(&[ + #( (stringify!(#fields), &self.#fields()) ),* + ]) + } + } + }; + + Ok(output) +} diff --git a/lib/edge/python/examples/fusion-query.py b/lib/edge/python/examples/fusion-query.py index 0d4acb34e3..55bc077d83 100644 --- a/lib/edge/python/examples/fusion-query.py +++ b/lib/edge/python/examples/fusion-query.py @@ -2,11 +2,9 @@ from qdrant_edge import * from common import * - shard = load_new_shard() fill_dummy_data(shard) - search_filter = Filter( must=[ FieldCondition( @@ -16,7 +14,6 @@ search_filter = Filter( ] ) - result = shard.query(QueryRequest( prefetches = [ Prefetch( @@ -46,9 +43,5 @@ result = shard.query(QueryRequest( with_payload = True, )) - for point in result: - print(f"Point: {point.id}, vector: {point.vector}, payload: {point.payload}, score: {point.score}") - - - + print(point) diff --git a/lib/edge/python/examples/mmr-query.py b/lib/edge/python/examples/mmr-query.py index 011d1b5fe1..d81019797c 100644 --- a/lib/edge/python/examples/mmr-query.py +++ b/lib/edge/python/examples/mmr-query.py @@ -5,8 +5,6 @@ from common import * shard = load_new_shard() fill_dummy_data(shard) - - result = shard.query(QueryRequest( prefetches = [], query = Mmr([6.0, 9.0, 4.0, 2.0], None, 0.9, 100), @@ -19,9 +17,5 @@ result = shard.query(QueryRequest( with_payload = True, )) - for point in result: - print(f"Point: {point.id}, vector: {point.vector}, payload: {point.payload}, score: {point.score}") - - - + print(point) diff --git a/lib/edge/python/examples/qdrant-edge.py b/lib/edge/python/examples/qdrant-edge.py index 82671815aa..a35361dc76 100755 --- a/lib/edge/python/examples/qdrant-edge.py +++ b/lib/edge/python/examples/qdrant-edge.py @@ -13,12 +13,11 @@ points = [ # Test points conversion into internal representation and back for point in points: - print(f"Point: {point.id}, vector: {point.vector}, payload: {point.payload}") + print(point) print("---- Load shard ----") - shard = load_new_shard() @@ -82,7 +81,7 @@ result = shard.query(QueryRequest( )) for point in result: - print(f"Point: {point.id}, vector: {point.vector}, payload: {point.payload}, score: {point.score}") + print(point) print("---- Search ----") @@ -99,7 +98,7 @@ points = shard.search(SearchRequest( )) for point in points: - print(f"Point: {point.id}, vector: {point.vector}, payload: {point.payload}, score: {point.score}") + print(point) print("---- Search + Filter ----") @@ -129,7 +128,7 @@ points = shard.search(SearchRequest( )) for point in points: - print(f"Point: {point.id}, vector: {point.vector}, payload: {point.payload}, score: {point.score}") + print(point) print("---- Retrieve ----") @@ -137,4 +136,4 @@ print("---- Retrieve ----") points = shard.retrieve(point_ids=[1], with_vector=True, with_payload=True) for point in points: - print(f"Point: {point.id}, vector: {point.vector}, payload: {point.payload}") + print(point) diff --git a/lib/edge/python/examples/repr.py b/lib/edge/python/examples/repr.py new file mode 100755 index 0000000000..67fd857d2a --- /dev/null +++ b/lib/edge/python/examples/repr.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +from qdrant_edge import * + +config = SegmentConfig( + vector_data = { + "": VectorDataConfig( + size = 128, + distance = Distance.Cosine, + storage_type = VectorStorageType.ChunkedMmap, + index = HnswIndexConfig( + m = 16, + ef_construct = 16, + full_scan_threshold = 1024, + on_disk = False, + payload_m = 16, + inline_storage = False, + ), + ) + }, + sparse_vector_data = { + "sparse": SparseVectorDataConfig( + index = SparseIndexConfig( + full_scan_threshold = 1024, + index_type = SparseIndexType.MutableRam, + datatype = VectorStorageDatatype.Float32, + ), + storage_type = SparseVectorStorageType.Mmap, + modifier = Modifier.Idf, + ) + }, + payload_storage_type = PayloadStorageType.Mmap, +) + +print(config) diff --git a/lib/edge/python/src/config/mod.rs b/lib/edge/python/src/config/mod.rs index 23c7528569..1d6b2d237e 100644 --- a/lib/edge/python/src/config/mod.rs +++ b/lib/edge/python/src/config/mod.rs @@ -3,6 +3,7 @@ pub mod sparse_vector_data; pub mod vector_data; use std::collections::HashMap; +use std::fmt; use bytemuck::TransparentWrapper; use derive_more::Into; @@ -12,12 +13,14 @@ use segment::types::*; pub use self::quantization::*; pub use self::sparse_vector_data::*; pub use self::vector_data::*; +use crate::repr::*; #[pyclass(name = "SegmentConfig")] #[derive(Clone, Debug, Into, TransparentWrapper)] #[repr(transparent)] pub struct PySegmentConfig(SegmentConfig); +#[pyclass_repr] #[pymethods] impl PySegmentConfig { #[new] @@ -47,6 +50,21 @@ impl PySegmentConfig { pub fn payload_storage_type(&self) -> PyPayloadStorageType { PyPayloadStorageType::from(self.0.payload_storage_type) } + + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl PySegmentConfig { + fn _getters(self) { + // Every field should have a getter method + let SegmentConfig { + vector_data: _, + sparse_vector_data: _, + payload_storage_type: _, + } = self.0; + } } #[pyclass(name = "PayloadStorageType")] @@ -56,6 +74,24 @@ pub enum PyPayloadStorageType { InRamMmap, } +#[pymethods] +impl PyPayloadStorageType { + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl Repr for PyPayloadStorageType { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let repr = match self { + PyPayloadStorageType::Mmap => "Mmap", + PyPayloadStorageType::InRamMmap => "InRamMmap", + }; + + f.simple_enum::(repr) + } +} + impl From for PyPayloadStorageType { fn from(storage_type: PayloadStorageType) -> Self { #[allow(unreachable_patterns)] diff --git a/lib/edge/python/src/config/quantization.rs b/lib/edge/python/src/config/quantization.rs index 8d7a99c42d..07034201dd 100644 --- a/lib/edge/python/src/config/quantization.rs +++ b/lib/edge/python/src/config/quantization.rs @@ -1,9 +1,15 @@ +use std::fmt; + +use bytemuck::TransparentWrapper; use derive_more::Into; use pyo3::IntoPyObjectExt as _; use pyo3::prelude::*; use segment::types::*; -#[derive(Clone, Debug, Into)] +use crate::repr::*; + +#[derive(Clone, Debug, Into, TransparentWrapper)] +#[repr(transparent)] pub struct PyQuantizationConfig(pub QuantizationConfig); impl FromPyObject<'_, '_> for PyQuantizationConfig { @@ -53,10 +59,28 @@ impl<'py> IntoPyObject<'py> for PyQuantizationConfig { } } +impl Repr for PyQuantizationConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match &self.0 { + QuantizationConfig::Scalar(scalar) => { + PyScalarQuantizationConfig::wrap_ref(&scalar.scalar).fmt(f) + } + QuantizationConfig::Product(product) => { + PyProductQuantizationConfig::wrap_ref(&product.product).fmt(f) + } + QuantizationConfig::Binary(binary) => { + PyBinaryQuantizationConfig::wrap_ref(&binary.binary).fmt(f) + } + } + } +} + #[pyclass(name = "ScalarQuantizationConfig")] -#[derive(Clone, Debug, Into)] +#[derive(Clone, Debug, Into, TransparentWrapper)] +#[repr(transparent)] pub struct PyScalarQuantizationConfig(ScalarQuantizationConfig); +#[pyclass_repr] #[pymethods] impl PyScalarQuantizationConfig { #[new] @@ -83,6 +107,21 @@ impl PyScalarQuantizationConfig { pub fn always_ram(&self) -> Option { self.0.always_ram } + + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl PyScalarQuantizationConfig { + fn _getters(self) { + // Every field should have a getter method + let ScalarQuantizationConfig { + r#type: _, + quantile: _, + always_ram: _, + } = self.0; + } } #[pyclass(name = "ScalarType")] @@ -91,6 +130,23 @@ pub enum PyScalarType { Int8, } +#[pymethods] +impl PyScalarType { + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl Repr for PyScalarType { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let repr = match self { + Self::Int8 => "Int8", + }; + + f.simple_enum::(repr) + } +} + impl From for PyScalarType { fn from(scalar_type: ScalarType) -> Self { match scalar_type { @@ -108,9 +164,11 @@ impl From for ScalarType { } #[pyclass(name = "ProductQuantizationConfig")] -#[derive(Clone, Debug, Into)] +#[derive(Clone, Debug, Into, TransparentWrapper)] +#[repr(transparent)] pub struct PyProductQuantizationConfig(ProductQuantizationConfig); +#[pyclass_repr] #[pymethods] impl PyProductQuantizationConfig { #[new] @@ -131,6 +189,20 @@ impl PyProductQuantizationConfig { pub fn always_ram(&self) -> Option { self.0.always_ram } + + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl PyProductQuantizationConfig { + fn _getters(self) { + // Every field should have a getter method + let ProductQuantizationConfig { + compression: _, + always_ram: _, + } = self.0; + } } #[pyclass(name = "CompressionRatio")] @@ -143,6 +215,27 @@ pub enum PyCompressionRatio { X64, } +#[pymethods] +impl PyCompressionRatio { + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl Repr for PyCompressionRatio { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let repr = match self { + Self::X4 => "X4", + Self::X8 => "X8", + Self::X16 => "X16", + Self::X32 => "X32", + Self::X64 => "X64", + }; + + f.simple_enum::(repr) + } +} + impl From for PyCompressionRatio { fn from(compression: CompressionRatio) -> Self { match compression { @@ -168,9 +261,11 @@ impl From for CompressionRatio { } #[pyclass(name = "BinaryQuantizationConfig")] -#[derive(Clone, Debug, Into)] +#[derive(Clone, Debug, Into, TransparentWrapper)] +#[repr(transparent)] pub struct PyBinaryQuantizationConfig(BinaryQuantizationConfig); +#[pyclass_repr] #[pymethods] impl PyBinaryQuantizationConfig { #[new] @@ -203,6 +298,21 @@ impl PyBinaryQuantizationConfig { .query_encoding .map(PyBinaryQuantizationQueryEncoding::from) } + + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl PyBinaryQuantizationConfig { + fn _getters(self) { + // Every field should have a getter method + let BinaryQuantizationConfig { + always_ram: _, + encoding: _, + query_encoding: _, + } = self.0; + } } #[pyclass(name = "BinaryQuantizationEncoding")] @@ -213,6 +323,25 @@ pub enum PyBinaryQuantizationEncoding { OneAndHalfBits, } +#[pymethods] +impl PyBinaryQuantizationEncoding { + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl Repr for PyBinaryQuantizationEncoding { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let repr = match self { + Self::OneBit => "OneBit", + Self::TwoBits => "TwoBits", + Self::OneAndHalfBits => "OneAndHalfBits", + }; + + f.simple_enum::(repr) + } +} + impl From for PyBinaryQuantizationEncoding { fn from(encoding: BinaryQuantizationEncoding) -> Self { match encoding { @@ -246,6 +375,26 @@ pub enum PyBinaryQuantizationQueryEncoding { Scalar8Bits, } +#[pymethods] +impl PyBinaryQuantizationQueryEncoding { + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl Repr for PyBinaryQuantizationQueryEncoding { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let repr = match self { + Self::Default => "Default", + Self::Binary => "Binary", + Self::Scalar4Bits => "Scalar4Bits", + Self::Scalar8Bits => "Scalar8Bits", + }; + + f.simple_enum::(repr) + } +} + impl From for PyBinaryQuantizationQueryEncoding { fn from(encoding: BinaryQuantizationQueryEncoding) -> Self { match encoding { diff --git a/lib/edge/python/src/config/sparse_vector_data.rs b/lib/edge/python/src/config/sparse_vector_data.rs index bcd7d4c2bb..3c5a4dcf15 100644 --- a/lib/edge/python/src/config/sparse_vector_data.rs +++ b/lib/edge/python/src/config/sparse_vector_data.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::mem; +use std::{fmt, mem}; use bytemuck::TransparentWrapper; use derive_more::Into; @@ -9,6 +9,7 @@ use segment::index::sparse_index::sparse_index_config::{SparseIndexConfig, Spars use segment::types::*; use super::vector_data::*; +use crate::repr::*; #[pyclass(name = "SparseVectorDataConfig")] #[derive(Copy, Clone, Debug, Into, TransparentWrapper)] @@ -31,6 +32,7 @@ impl PySparseVectorDataConfig { } } +#[pyclass_repr] #[pymethods] impl PySparseVectorDataConfig { #[new] @@ -60,6 +62,21 @@ impl PySparseVectorDataConfig { pub fn modifier(&self) -> Option { self.0.modifier.map(PyModifier::from) } + + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl PySparseVectorDataConfig { + fn _getters(self) { + // Every field should have a getter method + let SparseVectorDataConfig { + index: _, + storage_type: _, + modifier: _, + } = self.0; + } } impl<'py> IntoPyObject<'py> for &PySparseVectorDataConfig { @@ -73,9 +90,11 @@ impl<'py> IntoPyObject<'py> for &PySparseVectorDataConfig { } #[pyclass(name = "SparseIndexConfig")] -#[derive(Copy, Clone, Debug, Into)] +#[derive(Copy, Clone, Debug, Into, TransparentWrapper)] +#[repr(transparent)] pub struct PySparseIndexConfig(SparseIndexConfig); +#[pyclass_repr] #[pymethods] impl PySparseIndexConfig { #[new] @@ -105,6 +124,21 @@ impl PySparseIndexConfig { pub fn datatype(&self) -> Option { self.0.datatype.map(PyVectorStorageDatatype::from) } + + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl PySparseIndexConfig { + fn _getters(self) { + // Every field should have a getter method + let SparseIndexConfig { + full_scan_threshold: _, + index_type: _, + datatype: _, + } = self.0; + } } #[pyclass(name = "SparseIndexType")] @@ -115,6 +149,25 @@ pub enum PySparseIndexType { Mmap, } +#[pymethods] +impl PySparseIndexType { + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl Repr for PySparseIndexType { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let repr = match self { + Self::MutableRam => "MutableRam", + Self::ImmutableRam => "ImmutableRam", + Self::Mmap => "Mmap", + }; + + f.simple_enum::(repr) + } +} + impl From for PySparseIndexType { fn from(index_type: SparseIndexType) -> Self { match index_type { @@ -141,6 +194,23 @@ pub enum PySparseVectorStorageType { Mmap, } +#[pymethods] +impl PySparseVectorStorageType { + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl Repr for PySparseVectorStorageType { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let repr = match self { + Self::Mmap => "Mmap", + }; + + f.simple_enum::(repr) + } +} + impl From for PySparseVectorStorageType { fn from(storage_type: SparseVectorStorageType) -> Self { #[allow(unreachable_patterns)] @@ -167,6 +237,24 @@ pub enum PyModifier { Idf, } +#[pymethods] +impl PyModifier { + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl Repr for PyModifier { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let repr = match self { + Self::None => "None", + Self::Idf => "Idf", + }; + + f.simple_enum::(repr) + } +} + impl From for PyModifier { fn from(modifier: Modifier) -> Self { match modifier { diff --git a/lib/edge/python/src/config/vector_data.rs b/lib/edge/python/src/config/vector_data.rs index 67a862e5ce..477bad83cc 100644 --- a/lib/edge/python/src/config/vector_data.rs +++ b/lib/edge/python/src/config/vector_data.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::mem; +use std::{fmt, mem}; use bytemuck::TransparentWrapper; use derive_more::Into; @@ -8,6 +8,7 @@ use pyo3::prelude::*; use segment::types::*; use super::quantization::*; +use crate::repr::*; #[pyclass(name = "VectorDataConfig")] #[derive(Clone, Debug, Into, TransparentWrapper)] @@ -30,9 +31,11 @@ impl PyVectorDataConfig { } } +#[pyclass_repr] #[pymethods] impl PyVectorDataConfig { #[new] + #[pyo3(signature = (size, distance, storage_type, index, quantization_config=None, multivector_config=None, datatype=None))] pub fn new( size: usize, distance: PyDistance, @@ -87,6 +90,25 @@ impl PyVectorDataConfig { pub fn datatype(&self) -> Option { self.0.datatype.map(PyVectorStorageDatatype::from) } + + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl PyVectorDataConfig { + fn _getters(self) { + // Every field should have a getter method + let VectorDataConfig { + size: _, + distance: _, + storage_type: _, + index: _, + quantization_config: _, + multivector_config: _, + datatype: _, + } = self.0; + } } impl<'py> IntoPyObject<'py> for &PyVectorDataConfig { @@ -108,6 +130,26 @@ pub enum PyDistance { Manhattan, } +#[pymethods] +impl PyDistance { + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl Repr for PyDistance { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let repr = match self { + Self::Cosine => "Cosine", + Self::Euclid => "Euclid", + Self::Dot => "Dot", + Self::Manhattan => "Manhattan", + }; + + f.simple_enum::(repr) + } +} + impl From for PyDistance { fn from(distance: Distance) -> Self { match distance { @@ -139,6 +181,26 @@ pub enum PyVectorStorageType { InRamChunkedMmap, } +#[pymethods] +impl PyVectorStorageType { + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl Repr for PyVectorStorageType { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let repr = match self { + Self::Memory => "Memory", + Self::Mmap => "Mmap", + Self::ChunkedMmap => "ChunkedMmap", + Self::InRamChunkedMmap => "InRamChunkedMmap", + }; + + f.simple_enum::(repr) + } +} + impl From for PyVectorStorageType { fn from(storage_type: VectorStorageType) -> Self { match storage_type { @@ -161,7 +223,8 @@ impl From for VectorStorageType { } } -#[derive(Clone, Debug, Into)] +#[derive(Clone, Debug, Into, TransparentWrapper)] +#[repr(transparent)] pub struct PyIndexes(Indexes); impl FromPyObject<'_, '_> for PyIndexes { @@ -203,22 +266,38 @@ impl<'py> IntoPyObject<'py> for PyIndexes { } } +impl Repr for PyIndexes { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match &self.0 { + Indexes::Plain {} => PyPlainIndexConfig.fmt(f), + Indexes::Hnsw(hnsw) => PyHnswIndexConfig::wrap_ref(hnsw).fmt(f), + } + } +} + #[pyclass(name = "PlainIndexConfig")] #[derive(Copy, Clone, Debug, Default, Into)] pub struct PyPlainIndexConfig; +#[pyclass_repr] #[pymethods] impl PyPlainIndexConfig { #[new] pub fn new() -> Self { Self } + + pub fn __repr__(&self) -> String { + self.repr() + } } #[pyclass(name = "HnswIndexConfig")] -#[derive(Copy, Clone, Debug, Into)] +#[derive(Copy, Clone, Debug, Into, TransparentWrapper)] +#[repr(transparent)] pub struct PyHnswIndexConfig(HnswConfig); +#[pyclass_repr] #[pymethods] impl PyHnswIndexConfig { #[new] @@ -271,12 +350,33 @@ impl PyHnswIndexConfig { pub fn inline_storage(&self) -> Option { self.0.inline_storage } + + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl PyHnswIndexConfig { + fn _getters(self) { + // Every field should have a getter method + let HnswConfig { + m: _, + ef_construct: _, + full_scan_threshold: _, + max_indexing_threads: _, // not relevant for Qdrant Edge + on_disk: _, + payload_m: _, + inline_storage: _, + } = self.0; + } } #[pyclass(name = "MultiVectorConfig")] -#[derive(Copy, Clone, Debug, Into)] +#[derive(Copy, Clone, Debug, Into, TransparentWrapper)] +#[repr(transparent)] pub struct PyMultiVectorConfig(MultiVectorConfig); +#[pyclass_repr] #[pymethods] impl PyMultiVectorConfig { #[new] @@ -290,6 +390,17 @@ impl PyMultiVectorConfig { pub fn comparator(&self) -> PyMultiVectorComparator { PyMultiVectorComparator::from(self.0.comparator) } + + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl PyMultiVectorConfig { + fn _getters(self) { + // Every field should have a getter method + let MultiVectorConfig { comparator: _ } = self.0; + } } #[pyclass(name = "MultiVectorComparator")] @@ -298,6 +409,23 @@ pub enum PyMultiVectorComparator { MaxSim, } +#[pymethods] +impl PyMultiVectorComparator { + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl Repr for PyMultiVectorComparator { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let repr = match self { + Self::MaxSim => "MaxSim", + }; + + f.simple_enum::(repr) + } +} + impl From for PyMultiVectorComparator { fn from(comparator: MultiVectorComparator) -> Self { match comparator { @@ -322,6 +450,25 @@ pub enum PyVectorStorageDatatype { Uint8, } +#[pymethods] +impl PyVectorStorageDatatype { + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl Repr for PyVectorStorageDatatype { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let repr = match self { + Self::Float32 => "Float32", + Self::Float16 => "Float16", + Self::Uint8 => "Uint8", + }; + + f.simple_enum::(repr) + } +} + impl From for PyVectorStorageDatatype { fn from(datatype: VectorStorageDatatype) -> Self { match datatype { diff --git a/lib/edge/python/src/lib.rs b/lib/edge/python/src/lib.rs index 8b33763064..31d4bafadf 100644 --- a/lib/edge/python/src/lib.rs +++ b/lib/edge/python/src/lib.rs @@ -1,5 +1,6 @@ pub mod config; pub mod query; +pub mod repr; pub mod search; pub mod types; pub mod update; @@ -46,8 +47,7 @@ mod qdrant_edge { }; #[pymodule_export] use super::search::{ - PyAcornSearchParams, PyQuantizationSearchParams, PyScoredPoint, PySearchParams, - PySearchRequest, + PyAcornSearchParams, PyQuantizationSearchParams, PySearchParams, PySearchRequest, }; #[pymodule_export] use super::types::filter::{ @@ -64,7 +64,7 @@ mod qdrant_edge { PyQueryInterface, PyRecommendQuery, PySimpleFeedbackStrategy, }; #[pymodule_export] - use super::types::{PyPoint, PyPointVectors, PyRecord, PySparseVector}; + use super::types::{PyPoint, PyPointVectors, PyRecord, PyScoredPoint, PySparseVector}; #[pymodule_export] use super::update::PyUpdateOperation; } diff --git a/lib/edge/python/src/repr.rs b/lib/edge/python/src/repr.rs new file mode 100644 index 0000000000..e8186b0c73 --- /dev/null +++ b/lib/edge/python/src/repr.rs @@ -0,0 +1,248 @@ +use std::collections::HashMap; +use std::fmt; + +pub use edge_py_codegen::pyclass_repr; +use pyo3::PyTypeInfo; + +pub trait Repr { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result; + + fn repr(&self) -> String { + let mut repr = String::new(); + self.fmt(&mut repr).expect("infallible"); + repr + } +} + +pub type Formatter<'a> = dyn fmt::Write + 'a; + +impl Repr for &T { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + Repr::fmt(*self, f) + } +} + +impl Repr for bool { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", if *self { "True" } else { "False" }) + } +} + +impl Repr for u32 { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{self}") + } +} + +impl Repr for u64 { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{self}") + } +} + +impl Repr for i64 { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{self}") + } +} + +impl Repr for usize { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{self}") + } +} + +impl Repr for f32 { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{self}") + } +} + +impl Repr for f64 { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{self}") + } +} + +impl Repr for str { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{self:?}") + } +} + +impl Repr for String { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + self.as_str().fmt(f) + } +} + +impl Repr for [T] { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.list(self) + } +} + +impl Repr for Vec { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.list(self) + } +} + +impl, V: Repr, S> Repr for HashMap { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.map(self) + } +} + +impl Repr for Option { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + Some(value) => value.fmt(f), + None => write!(f, "None"), + } + } +} + +impl Repr for uuid::Uuid { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "\"{self}\"") + } +} + +impl Repr for serde_json::Value { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + serde_json::Value::Null => write!(f, "None"), + serde_json::Value::Bool(bool) => bool.fmt(f), + serde_json::Value::Number(num) => num.fmt(f), + serde_json::Value::String(str) => str.fmt(f), + serde_json::Value::Array(array) => array.fmt(f), + serde_json::Value::Object(object) => object.fmt(f), + } + } +} + +impl Repr for serde_json::Number { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{self}") + } +} + +impl Repr for serde_json::Map { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.map(self) + } +} + +pub trait WriteExt: fmt::Write { + fn class(&mut self, fields: &[(&str, &dyn Repr)]) -> fmt::Result { + write!(self, "{}(", T::NAME)?; + + let mut separator = ""; + for (field, value) in fields { + write!(self, "{separator}{field}={}", ReprFmt(value))?; + separator = ", "; + } + + write!(self, ")")?; + + Ok(()) + } + + fn complex_enum( + &mut self, + variant: &str, + fields: &[(&str, &dyn Repr)], + ) -> fmt::Result { + write!(self, "{}.{}(", T::NAME, variant)?; + + let mut separator = ""; + for (field, value) in fields { + write!(self, "{separator}{field}={}", ReprFmt(value))?; + separator = ", "; + } + + write!(self, ")")?; + + Ok(()) + } + + fn simple_enum(&mut self, variant: &str) -> fmt::Result { + write!(self, "{}.{}", T::NAME, variant) + } + + fn list(&mut self, list: impl IntoIterator) -> fmt::Result { + write!(self, "[")?; + + let mut separator = ""; + for value in list { + write!(self, "{separator}{}", ReprFmt(value))?; + separator = ", "; + } + + write!(self, "]")?; + + Ok(()) + } + + fn map(&mut self, map: impl IntoIterator) -> fmt::Result + where + K: AsRef, + V: Repr, + { + write!(self, "{{")?; + + let mut separator = ""; + for (key, value) in map { + write!( + self, + "{separator}{}: {}", + ReprFmt(key.as_ref()), + ReprFmt(value) + )?; + + separator = ", "; + } + + write!(self, "}}")?; + + Ok(()) + } + + fn set(&mut self, set: impl IntoIterator) -> fmt::Result { + let mut set = set.into_iter().peekable(); + + if set.peek().is_none() { + self.write_str("set()")?; + return Ok(()); + } + + write!(self, "{{")?; + + let mut separator = ""; + for value in set { + write!(self, "{separator}{}", ReprFmt(value))?; + separator = ", "; + } + + write!(self, "}}")?; + + Ok(()) + } + + fn unimplemented(&mut self) -> fmt::Result { + self.write_str("UNIMPLEMENTED") + } +} + +impl WriteExt for W {} +impl<'a> WriteExt for dyn fmt::Write + 'a {} + +#[derive(Copy, Clone)] +struct ReprFmt(pub T); + +impl fmt::Display for ReprFmt { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + Repr::fmt(&self.0, f) + } +} diff --git a/lib/edge/python/src/search.rs b/lib/edge/python/src/search.rs index d7e68c8b0f..d63a54cac6 100644 --- a/lib/edge/python/src/search.rs +++ b/lib/edge/python/src/search.rs @@ -310,36 +310,3 @@ pub enum PyPayloadSelectorInterface { Include(Vec), Exclude(Vec), } - -#[pyclass(name = "ScoredPoint")] -#[derive(Clone, Debug, Into, TransparentWrapper)] -#[repr(transparent)] -pub struct PyScoredPoint(pub ScoredPoint); - -#[pymethods] -impl PyScoredPoint { - #[getter] - pub fn id(&self) -> PyPointId { - PyPointId(self.0.id) - } - - #[getter] - pub fn version(&self) -> u64 { - self.0.version - } - - #[getter] - pub fn score(&self) -> f32 { - self.0.score - } - - #[getter] - pub fn vector(&self) -> Option<&PyVectorInternal> { - self.0.vector.as_ref().map(PyVectorInternal::wrap_ref) - } - - #[getter] - pub fn payload(&self) -> Option<&PyPayload> { - self.0.payload.as_ref().map(PyPayload::wrap_ref) - } -} diff --git a/lib/edge/python/src/types/mod.rs b/lib/edge/python/src/types/mod.rs index 13c62a919b..fc489b8935 100644 --- a/lib/edge/python/src/types/mod.rs +++ b/lib/edge/python/src/types/mod.rs @@ -1,12 +1,14 @@ pub mod filter; pub mod formula; pub mod json_path; +pub mod order_value; pub mod payload; pub mod point; pub mod point_id; pub mod point_vectors; pub mod query; pub mod record; +pub mod scored_point; pub mod value; pub mod vector; pub mod vector_internal; @@ -14,12 +16,14 @@ pub mod vector_internal; pub use self::filter::*; pub use self::formula::*; pub use self::json_path::*; +pub use self::order_value::*; pub use self::payload::*; pub use self::point::*; pub use self::point_id::*; pub use self::point_vectors::*; pub use self::query::*; pub use self::record::*; +pub use self::scored_point::*; pub use self::value::*; pub use self::vector::*; pub use self::vector_internal::*; diff --git a/lib/edge/python/src/types/order_value.rs b/lib/edge/python/src/types/order_value.rs new file mode 100644 index 0000000000..1cbc62ee54 --- /dev/null +++ b/lib/edge/python/src/types/order_value.rs @@ -0,0 +1,30 @@ +use std::fmt; + +use pyo3::prelude::*; +use segment::data_types::order_by::OrderValue; + +use crate::repr::*; + +#[derive(IntoPyObject)] +pub enum PyOrderValue { + Int(i64), + Float(f64), +} + +impl Repr for PyOrderValue { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::Int(int) => int.fmt(f), + Self::Float(float) => float.fmt(f), + } + } +} + +impl From for PyOrderValue { + fn from(value: OrderValue) -> Self { + match value { + OrderValue::Int(int) => Self::Int(int), + OrderValue::Float(float) => Self::Float(float), + } + } +} diff --git a/lib/edge/python/src/types/payload.rs b/lib/edge/python/src/types/payload.rs index 2c8d868ae6..2b44ae965b 100644 --- a/lib/edge/python/src/types/payload.rs +++ b/lib/edge/python/src/types/payload.rs @@ -1,9 +1,12 @@ +use std::fmt; + use bytemuck::TransparentWrapper; use derive_more::Into; use pyo3::prelude::*; use segment::types::*; use super::value::*; +use crate::repr::*; #[derive(Clone, Debug, Into, TransparentWrapper)] #[repr(transparent)] @@ -37,3 +40,9 @@ impl<'py> IntoPyObject<'py> for &PyPayload { value_map_into_py(&self.0.0, py) } } + +impl Repr for PyPayload { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + self.0.0.fmt(f) + } +} diff --git a/lib/edge/python/src/types/point.rs b/lib/edge/python/src/types/point.rs index 477457bd82..3f9e5dd345 100644 --- a/lib/edge/python/src/types/point.rs +++ b/lib/edge/python/src/types/point.rs @@ -1,26 +1,29 @@ -use bytemuck::TransparentWrapper as _; +use bytemuck::TransparentWrapper; use derive_more::Into; use pyo3::prelude::*; use segment::types::{Payload, PointIdType}; use shard::operations::point_ops::{PointStructPersisted, VectorStructPersisted}; +use crate::repr::*; use crate::{PyPayload, PyPointId, PyVector}; #[pyclass(name = "Point")] -#[derive(Clone, Debug, Into)] +#[derive(Clone, Debug, Into, TransparentWrapper)] +#[repr(transparent)] pub struct PyPoint(PointStructPersisted); +#[pyclass_repr] #[pymethods] impl PyPoint { #[new] - pub fn new(id: PyPointId, vector: PyVector, payload: Option) -> Result { + pub fn new(id: PyPointId, vector: PyVector, payload: Option) -> Self { let point = PointStructPersisted { id: PointIdType::from(id), vector: VectorStructPersisted::from(vector), payload: payload.map(Payload::from), }; - Ok(Self(point)) + Self(point) } #[getter] @@ -37,4 +40,19 @@ impl PyPoint { pub fn payload(&self) -> Option<&PyPayload> { self.0.payload.as_ref().map(PyPayload::wrap_ref) } + + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl PyPoint { + fn _getters(self) { + // Every field should have a getter method + let PointStructPersisted { + id: _, + vector: _, + payload: _, + } = self.0; + } } diff --git a/lib/edge/python/src/types/point_id.rs b/lib/edge/python/src/types/point_id.rs index e0e76bb430..c32b68d21a 100644 --- a/lib/edge/python/src/types/point_id.rs +++ b/lib/edge/python/src/types/point_id.rs @@ -1,4 +1,4 @@ -use std::mem; +use std::{fmt, mem}; use bytemuck::TransparentWrapper; use derive_more::Into; @@ -8,6 +8,8 @@ use pyo3::prelude::*; use segment::types::PointIdType; use uuid::Uuid; +use crate::repr::*; + #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Into, TransparentWrapper)] #[repr(transparent)] pub struct PyPointId(pub PointIdType); @@ -84,3 +86,12 @@ impl<'py> IntoPyObject<'py> for &PyPointId { } } } + +impl Repr for PyPointId { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match &self.0 { + PointIdType::NumId(id) => id.fmt(f), + PointIdType::Uuid(uuid) => uuid.fmt(f), + } + } +} diff --git a/lib/edge/python/src/types/point_vectors.rs b/lib/edge/python/src/types/point_vectors.rs index 245168dd16..0c0fb65416 100644 --- a/lib/edge/python/src/types/point_vectors.rs +++ b/lib/edge/python/src/types/point_vectors.rs @@ -1,16 +1,19 @@ -use bytemuck::TransparentWrapper as _; +use bytemuck::TransparentWrapper; use derive_more::Into; use pyo3::{pyclass, pymethods}; use segment::types::PointIdType; use shard::operations::point_ops::VectorStructPersisted; use shard::operations::vector_ops::PointVectorsPersisted; +use crate::repr::*; use crate::types::{PyPointId, PyVector}; #[pyclass(name = "PointVectors")] -#[derive(Clone, Debug, Into)] +#[derive(Clone, Debug, Into, TransparentWrapper)] +#[repr(transparent)] pub struct PyPointVectors(pub PointVectorsPersisted); +#[pyclass_repr] #[pymethods] impl PyPointVectors { #[new] @@ -22,12 +25,23 @@ impl PyPointVectors { } #[getter] - fn id(&self) -> PyPointId { + pub fn id(&self) -> PyPointId { PyPointId(self.0.id) } #[getter] - fn vector(&self) -> &PyVector { + pub fn vector(&self) -> &PyVector { PyVector::wrap_ref(&self.0.vector) } + + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl PyPointVectors { + fn _getters(self) { + // Every field should have a getter method + let PointVectorsPersisted { id: _, vector: _ } = self.0; + } } diff --git a/lib/edge/python/src/types/record.rs b/lib/edge/python/src/types/record.rs index f7acf645dc..157000c184 100644 --- a/lib/edge/python/src/types/record.rs +++ b/lib/edge/python/src/types/record.rs @@ -1,9 +1,9 @@ use bytemuck::TransparentWrapper; use derive_more::Into; use pyo3::prelude::*; -use segment::data_types::order_by::OrderValue; use shard::retrieve::record_internal::RecordInternal; +use crate::repr::*; use crate::*; #[pyclass(name = "Record")] @@ -11,6 +11,7 @@ use crate::*; #[repr(transparent)] pub struct PyRecord(pub RecordInternal); +#[pyclass_repr] #[pymethods] impl PyRecord { #[getter] @@ -32,19 +33,21 @@ impl PyRecord { pub fn order_value(&self) -> Option { self.0.order_value.map(PyOrderValue::from) } -} -#[derive(IntoPyObject)] -pub enum PyOrderValue { - Int(i64), - Float(f64), -} - -impl From for PyOrderValue { - fn from(value: OrderValue) -> Self { - match value { - OrderValue::Int(int) => Self::Int(int), - OrderValue::Float(float) => Self::Float(float), - } + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl PyRecord { + fn _getters(self) { + // Every field should have a getter method + let RecordInternal { + id: _, + payload: _, + vector: _, + shard_key: _, // not relevant for Qdrant Edge + order_value: _, + } = self.0; } } diff --git a/lib/edge/python/src/types/scored_point.rs b/lib/edge/python/src/types/scored_point.rs new file mode 100644 index 0000000000..c43f2c1056 --- /dev/null +++ b/lib/edge/python/src/types/scored_point.rs @@ -0,0 +1,66 @@ +use bytemuck::TransparentWrapper; +use derive_more::Into; +use pyo3::prelude::*; +use segment::types::ScoredPoint; + +use super::PyOrderValue; +use crate::repr::*; +use crate::{PyPayload, PyPointId, PyVectorInternal}; + +#[pyclass(name = "ScoredPoint")] +#[derive(Clone, Debug, Into, TransparentWrapper)] +#[repr(transparent)] +pub struct PyScoredPoint(pub ScoredPoint); + +#[pyclass_repr] +#[pymethods] +impl PyScoredPoint { + #[getter] + pub fn id(&self) -> PyPointId { + PyPointId(self.0.id) + } + + #[getter] + pub fn version(&self) -> u64 { + self.0.version + } + + #[getter] + pub fn score(&self) -> f32 { + self.0.score + } + + #[getter] + pub fn vector(&self) -> Option<&PyVectorInternal> { + self.0.vector.as_ref().map(PyVectorInternal::wrap_ref) + } + + #[getter] + pub fn payload(&self) -> Option<&PyPayload> { + self.0.payload.as_ref().map(PyPayload::wrap_ref) + } + + #[getter] + pub fn order_value(&self) -> Option { + self.0.order_value.map(PyOrderValue::from) + } + + pub fn __repr__(&self) -> String { + self.repr() + } +} + +impl PyScoredPoint { + fn _getters(self) { + // Every field should have a getter method + let ScoredPoint { + id: _, + version: _, + score: _, + vector: _, + payload: _, + shard_key: _, // not relevant for Qdrant Edge + order_value: _, + } = self.0; + } +} diff --git a/lib/edge/python/src/types/value.rs b/lib/edge/python/src/types/value.rs index 89f8f14d94..190bd9ab88 100644 --- a/lib/edge/python/src/types/value.rs +++ b/lib/edge/python/src/types/value.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::mem; +use std::{fmt, mem}; use bytemuck::{TransparentWrapper, TransparentWrapperAlloc as _}; use derive_more::Into; @@ -8,6 +8,8 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::{PyDict, PyString}; +use crate::repr::*; + #[derive(Clone, Debug, Into, TransparentWrapper)] #[repr(transparent)] pub struct PyValue(serde_json::Value); @@ -99,6 +101,12 @@ impl<'py> IntoPyObject<'py> for &PyValue { } } +impl Repr for PyValue { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + pub type ValueMap = serde_json::Map; pub fn value_map_from_py(dict: &Bound<'_, PyAny>) -> PyResult { diff --git a/lib/edge/python/src/types/vector.rs b/lib/edge/python/src/types/vector.rs index 3f2b6d8b03..a464b7f8e2 100644 --- a/lib/edge/python/src/types/vector.rs +++ b/lib/edge/python/src/types/vector.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::mem; +use std::{fmt, mem}; use bytemuck::TransparentWrapper; use derive_more::Into; @@ -10,6 +10,8 @@ use shard::operations::point_ops::{VectorPersisted, VectorStructPersisted}; use sparse::common::sparse_vector::SparseVector; use sparse::common::types::{DimId, DimWeight}; +use crate::repr::*; + #[derive(Clone, Debug, Into, TransparentWrapper)] #[repr(transparent)] pub struct PyVector(VectorStructPersisted); @@ -69,6 +71,16 @@ impl<'py> IntoPyObject<'py> for &PyVector { } } +impl Repr for PyVector { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match &self.0 { + VectorStructPersisted::Single(single) => single.fmt(f), + VectorStructPersisted::MultiDense(multi) => multi.fmt(f), + VectorStructPersisted::Named(named) => PyNamedVector::wrap_map_ref(named).fmt(f), + } + } +} + #[derive(Clone, Debug, Into, TransparentWrapper)] #[repr(transparent)] pub struct PyNamedVector(VectorPersisted); @@ -142,10 +154,22 @@ impl<'py> IntoPyObject<'py> for &PyNamedVector { } } +impl Repr for PyNamedVector { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match &self.0 { + VectorPersisted::Dense(dense) => dense.fmt(f), + VectorPersisted::Sparse(sparse) => PySparseVector::wrap_ref(sparse).fmt(f), + VectorPersisted::MultiDense(multi) => multi.fmt(f), + } + } +} + #[pyclass(name = "SparseVector")] -#[derive(Clone, Debug, Into)] +#[derive(Clone, Debug, Into, TransparentWrapper)] +#[repr(transparent)] pub struct PySparseVector(pub SparseVector); +#[pyclass_repr] #[pymethods] impl PySparseVector { #[new] @@ -164,10 +188,16 @@ impl PySparseVector { } fn __repr__(&self) -> String { - format!( - "SparseVector(indices={:?}, values={:?})", - self.indices(), - self.values() - ) + self.repr() + } +} + +impl PySparseVector { + fn _getters(self) { + // Every field should have a getter method + let SparseVector { + indices: _, + values: _, + } = self.0; } } diff --git a/lib/edge/python/src/types/vector_internal.rs b/lib/edge/python/src/types/vector_internal.rs index daed1fef87..60fc74b451 100644 --- a/lib/edge/python/src/types/vector_internal.rs +++ b/lib/edge/python/src/types/vector_internal.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::mem; +use std::{fmt, mem}; use bytemuck::TransparentWrapper; use derive_more::Into; @@ -12,6 +12,7 @@ use segment::types::VectorNameBuf; use sparse::common::sparse_vector::SparseVector; use super::vector::PySparseVector; +use crate::repr::*; #[derive(Clone, Debug, Into, TransparentWrapper)] #[repr(transparent)] @@ -76,6 +77,16 @@ impl<'py> IntoPyObject<'py> for &PyVectorInternal { } } +impl Repr for PyVectorInternal { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match &self.0 { + VectorStructInternal::Single(single) => single.fmt(f), + VectorStructInternal::MultiDense(multi) => f.list(multi.multi_vectors()), + VectorStructInternal::Named(named) => PyNamedVectorInternal::wrap_map_ref(named).fmt(f), + } + } +} + #[derive(Clone, Debug, Into, TransparentWrapper)] #[repr(transparent)] pub struct PyNamedVectorInternal(pub VectorInternal); @@ -143,6 +154,16 @@ impl<'py> IntoPyObject<'py> for &PyNamedVectorInternal { } } +impl Repr for PyNamedVectorInternal { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match &self.0 { + VectorInternal::Dense(dense) => dense.fmt(f), + VectorInternal::Sparse(sparse) => PySparseVector::wrap_ref(sparse).fmt(f), + VectorInternal::MultiDense(multi) => f.list(multi.multi_vectors()), + } + } +} + type MultiDenseVector = TypedMultiDenseVector; fn multi_dense_from_py(matrix: &Bound<'_, PyAny>) -> PyResult { diff --git a/lib/edge/python/src/update.rs b/lib/edge/python/src/update.rs index c1df281ce3..605ddbb8c2 100644 --- a/lib/edge/python/src/update.rs +++ b/lib/edge/python/src/update.rs @@ -1,160 +1,97 @@ -use std::str::FromStr; - use bytemuck::TransparentWrapperAlloc as _; use derive_more::Into; -use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use segment::json_path::JsonPath; use segment::types::{Filter, Payload, VectorNameBuf}; -use shard::operations::point_ops::{ - PointIdsList, PointInsertOperationsInternal, PointStructPersisted, -}; -use shard::operations::vector_ops::PointVectorsPersisted; +use shard::operations::point_ops::{PointIdsList, PointInsertOperationsInternal}; use shard::operations::{CollectionUpdateOperations, payload_ops, point_ops, vector_ops}; -use crate::types::{PyFilter, PyPayload, PyPoint, PyPointId, PyPointVectors}; +use crate::*; #[pyclass(name = "UpdateOperation")] #[derive(Clone, Debug, Into)] pub struct PyUpdateOperation(CollectionUpdateOperations); -#[pymethods] // Can't split impl block due to pyo3 limitations, so all constructors go here +#[pymethods] impl PyUpdateOperation { #[staticmethod] - pub fn upsert_points(points: Vec) -> Self { - let points = points.into_iter().map(PointStructPersisted::from).collect(); + #[pyo3(signature = (points, condition=None))] + pub fn upsert_points(points: Vec, condition: Option) -> Self { + let points = PointInsertOperationsInternal::PointsList(PyPoint::peel_vec(points)); - let operation = - CollectionUpdateOperations::PointOperation(point_ops::PointOperations::UpsertPoints( - PointInsertOperationsInternal::PointsList(points), - )); - - Self(operation) - } - - #[staticmethod] - pub fn upsert_points_conditional(points: Vec, condition: PyFilter) -> Self { - let points = points.into_iter().map(PointStructPersisted::from).collect(); - let points_op = PointInsertOperationsInternal::PointsList(points); - - let condition = Filter::from(condition); - - let operation = CollectionUpdateOperations::PointOperation( - point_ops::PointOperations::UpsertPointsConditional( + let operation = match condition { + Some(condition) => point_ops::PointOperations::UpsertPointsConditional( point_ops::ConditionalInsertOperationInternal { - points_op, - condition, + points_op: points, + condition: Filter::from(condition), }, ), - ); - Self(operation) + None => point_ops::PointOperations::UpsertPoints(points), + }; + + Self(CollectionUpdateOperations::PointOperation(operation)) } #[staticmethod] - pub fn delete_points(ids: Vec) -> Self { - let point_ids = PyPointId::peel_vec(ids); + pub fn delete_points(point_ids: Vec) -> Self { + let operation = point_ops::PointOperations::DeletePoints { + ids: PyPointId::peel_vec(point_ids), + }; - let operation = - CollectionUpdateOperations::PointOperation(point_ops::PointOperations::DeletePoints { - ids: point_ids, - }); - - Self(operation) + Self(CollectionUpdateOperations::PointOperation(operation)) } #[staticmethod] pub fn delete_points_by_filter(filter: PyFilter) -> Self { - let filter = Filter::from(filter); - - let operation = CollectionUpdateOperations::PointOperation( - point_ops::PointOperations::DeletePointsByFilter(filter), - ); - - Self(operation) + let operation = point_ops::PointOperations::DeletePointsByFilter(Filter::from(filter)); + Self(CollectionUpdateOperations::PointOperation(operation)) } #[staticmethod] - pub fn update_vectors(point_vectors: Vec) -> Self { - let points = point_vectors - .into_iter() - .map(PointVectorsPersisted::from) - .collect(); + #[pyo3(signature = (point_vectors, condition=None))] + pub fn update_vectors(point_vectors: Vec, condition: Option) -> Self { + let operation = vector_ops::VectorOperations::UpdateVectors(vector_ops::UpdateVectorsOp { + points: PyPointVectors::peel_vec(point_vectors), + update_filter: condition.map(Filter::from), + }); - let operation = CollectionUpdateOperations::VectorOperation( - vector_ops::VectorOperations::UpdateVectors(vector_ops::UpdateVectorsOp { - points, - update_filter: None, - }), - ); - - Self(operation) + Self(CollectionUpdateOperations::VectorOperation(operation)) } #[staticmethod] - pub fn update_vectors_conditional( - point_vectors: Vec, - filter: PyFilter, - ) -> Self { - let points = point_vectors - .into_iter() - .map(PointVectorsPersisted::from) - .collect(); - let filter = Filter::from(filter); - let operation = CollectionUpdateOperations::VectorOperation( - vector_ops::VectorOperations::UpdateVectors(vector_ops::UpdateVectorsOp { - points, - update_filter: Some(filter), - }), + pub fn delete_vectors(point_ids: Vec, vector_names: Vec) -> Self { + let operation = vector_ops::VectorOperations::DeleteVectors( + PointIdsList::from(PyPointId::peel_vec(point_ids)), + vector_names, ); - Self(operation) - } - #[staticmethod] - pub fn delete_vectors(ids: Vec, vector_names: Vec) -> Self { - let point_ids = PyPointId::peel_vec(ids); - let operation = CollectionUpdateOperations::VectorOperation( - vector_ops::VectorOperations::DeleteVectors( - PointIdsList::from(point_ids), - vector_names, - ), - ); - Self(operation) + Self(CollectionUpdateOperations::VectorOperation(operation)) } #[staticmethod] pub fn delete_vectors_by_filter(filter: PyFilter, vector_names: Vec) -> Self { - let filter = Filter::from(filter); - let operation = CollectionUpdateOperations::VectorOperation( - vector_ops::VectorOperations::DeleteVectorsByFilter(filter, vector_names), - ); - Self(operation) + let operation = + vector_ops::VectorOperations::DeleteVectorsByFilter(Filter::from(filter), vector_names); + + Self(CollectionUpdateOperations::VectorOperation(operation)) } #[staticmethod] - #[pyo3(signature = (ids, payload, key=None))] + #[pyo3(signature = (point_ids, payload, key=None))] pub fn set_payload( - ids: Vec, + point_ids: Vec, payload: PyPayload, - key: Option, - ) -> Result { - let point_ids = PyPointId::peel_vec(ids); - let payload = Payload::from(payload); + key: Option, + ) -> Self { + let operation = payload_ops::PayloadOps::SetPayload(payload_ops::SetPayloadOp { + payload: Payload::from(payload), + points: Some(PyPointId::peel_vec(point_ids)), + filter: None, + key: key.map(JsonPath::from), + }); - let key = key - .map(|k| JsonPath::from_str(&k).map_err(|_| PyErr::new::(k))) - .transpose()?; - - let operation = CollectionUpdateOperations::PayloadOperation( - payload_ops::PayloadOps::SetPayload(payload_ops::SetPayloadOp { - payload, - points: Some(point_ids), - filter: None, - key, - }), - ); - - Ok(Self(operation)) + Self(CollectionUpdateOperations::PayloadOperation(operation)) } #[staticmethod] @@ -162,108 +99,70 @@ impl PyUpdateOperation { pub fn set_payload_by_filter( filter: PyFilter, payload: PyPayload, - key: Option, - ) -> Result { - let filter = Filter::from(filter); - let payload = Payload::from(payload); + key: Option, + ) -> Self { + let operation = payload_ops::PayloadOps::SetPayload(payload_ops::SetPayloadOp { + payload: Payload::from(payload), + points: None, + filter: Some(Filter::from(filter)), + key: key.map(JsonPath::from), + }); - let key = key - .map(|k| JsonPath::from_str(&k).map_err(|_| PyErr::new::(k))) - .transpose()?; - - let operation = CollectionUpdateOperations::PayloadOperation( - payload_ops::PayloadOps::SetPayload(payload_ops::SetPayloadOp { - payload, - points: None, - filter: Some(filter), - key, - }), - ); - Ok(Self(operation)) + Self(CollectionUpdateOperations::PayloadOperation(operation)) } #[staticmethod] - #[pyo3(signature = (ids, keys))] - pub fn delete_payload(ids: Vec, keys: Vec) -> Result { - let point_ids = PyPointId::peel_vec(ids); + pub fn delete_payload(point_ids: Vec, keys: Vec) -> Self { + let operation = payload_ops::PayloadOps::DeletePayload(payload_ops::DeletePayloadOp { + keys: PyJsonPath::peel_vec(keys), + points: Some(PyPointId::peel_vec(point_ids)), + filter: None, + }); - let keys: Vec<_> = keys - .into_iter() - .map(|k| JsonPath::from_str(&k).map_err(|_| PyErr::new::(k))) - .collect::>()?; - - let operation = CollectionUpdateOperations::PayloadOperation( - payload_ops::PayloadOps::DeletePayload(payload_ops::DeletePayloadOp { - keys, - points: Some(point_ids), - filter: None, - }), - ); - Ok(Self(operation)) + Self(CollectionUpdateOperations::PayloadOperation(operation)) } #[staticmethod] - #[pyo3(signature = (filter, keys))] - pub fn delete_payload_by_filter(filter: PyFilter, keys: Vec) -> Result { - let filter = Filter::from(filter); - let keys: Vec<_> = keys - .into_iter() - .map(|k| JsonPath::from_str(&k).map_err(|_| PyErr::new::(k))) - .collect::>()?; + pub fn delete_payload_by_filter(filter: PyFilter, keys: Vec) -> Self { + let operation = payload_ops::PayloadOps::DeletePayload(payload_ops::DeletePayloadOp { + keys: PyJsonPath::peel_vec(keys), + points: None, + filter: Some(Filter::from(filter)), + }); - let operation = CollectionUpdateOperations::PayloadOperation( - payload_ops::PayloadOps::DeletePayload(payload_ops::DeletePayloadOp { - keys, - points: None, - filter: Some(filter), - }), - ); - Ok(Self(operation)) + Self(CollectionUpdateOperations::PayloadOperation(operation)) } #[staticmethod] - pub fn clear_payload(ids: Vec) -> Self { - let point_ids = PyPointId::peel_vec(ids); - let operation = - CollectionUpdateOperations::PayloadOperation(payload_ops::PayloadOps::ClearPayload { - points: point_ids, - }); - Self(operation) + pub fn clear_payload(point_ids: Vec) -> Self { + let operation = payload_ops::PayloadOps::ClearPayload { + points: PyPointId::peel_vec(point_ids), + }; + + Self(CollectionUpdateOperations::PayloadOperation(operation)) } #[staticmethod] pub fn clear_payload_by_filter(filter: PyFilter) -> Self { - let filter = Filter::from(filter); - let operation = CollectionUpdateOperations::PayloadOperation( - payload_ops::PayloadOps::ClearPayloadByFilter(filter), - ); - Self(operation) + let operation = payload_ops::PayloadOps::ClearPayloadByFilter(Filter::from(filter)); + Self(CollectionUpdateOperations::PayloadOperation(operation)) } #[staticmethod] - #[pyo3(signature = (ids, payload, key=None))] + #[pyo3(signature = (point_ids, payload, key=None))] pub fn overwrite_payload( - ids: Vec, + point_ids: Vec, payload: PyPayload, - key: Option, - ) -> Result { - let point_ids = PyPointId::peel_vec(ids); - let payload = Payload::from(payload); + key: Option, + ) -> Self { + let operation = payload_ops::PayloadOps::OverwritePayload(payload_ops::SetPayloadOp { + payload: Payload::from(payload), + points: Some(PyPointId::peel_vec(point_ids)), + filter: None, + key: key.map(JsonPath::from), + }); - let key = key - .map(|k| JsonPath::from_str(&k).map_err(|_| PyErr::new::(k))) - .transpose()?; - - let operation = CollectionUpdateOperations::PayloadOperation( - payload_ops::PayloadOps::OverwritePayload(payload_ops::SetPayloadOp { - payload, - points: Some(point_ids), - filter: None, - key, - }), - ); - - Ok(Self(operation)) + Self(CollectionUpdateOperations::PayloadOperation(operation)) } #[staticmethod] @@ -271,23 +170,15 @@ impl PyUpdateOperation { pub fn overwrite_payload_by_filter( filter: PyFilter, payload: PyPayload, - key: Option, - ) -> Result { - let filter = Filter::from(filter); - let payload = Payload::from(payload); + key: Option, + ) -> Self { + let operation = payload_ops::PayloadOps::OverwritePayload(payload_ops::SetPayloadOp { + payload: Payload::from(payload), + points: None, + filter: Some(Filter::from(filter)), + key: key.map(JsonPath::from), + }); - let key = key - .map(|k| JsonPath::from_str(&k).map_err(|_| PyErr::new::(k))) - .transpose()?; - - let operation = CollectionUpdateOperations::PayloadOperation( - payload_ops::PayloadOps::OverwritePayload(payload_ops::SetPayloadOp { - payload, - points: None, - filter: Some(filter), - key, - }), - ); - Ok(Self(operation)) + Self(CollectionUpdateOperations::PayloadOperation(operation)) } } diff --git a/lib/segment/src/types.rs b/lib/segment/src/types.rs index 3d56bf1141..bc1677681b 100644 --- a/lib/segment/src/types.rs +++ b/lib/segment/src/types.rs @@ -739,7 +739,7 @@ pub enum ScalarType { Int8, } -#[derive(Debug, Deserialize, Serialize, JsonSchema, Validate, Clone, PartialEq)] +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize, JsonSchema, Validate)] #[serde(rename_all = "snake_case")] pub struct ScalarQuantizationConfig { /// Type of quantization to use @@ -765,13 +765,13 @@ impl ScalarQuantizationConfig { } } -#[derive(Debug, Deserialize, Serialize, JsonSchema, Validate, Clone, PartialEq, Eq, Hash)] +#[derive(Clone, Debug, Eq, PartialEq, Hash, Deserialize, Serialize, JsonSchema, Validate)] pub struct ScalarQuantization { #[validate(nested)] pub scalar: ScalarQuantizationConfig, } -#[derive(Debug, Deserialize, Serialize, JsonSchema, Validate, Clone, PartialEq, Eq, Hash)] +#[derive(Clone, Debug, Eq, PartialEq, Hash, Deserialize, Serialize, JsonSchema, Validate)] #[serde(rename_all = "snake_case")] pub struct ProductQuantizationConfig { pub compression: CompressionRatio, @@ -791,7 +791,7 @@ impl ProductQuantizationConfig { } } -#[derive(Debug, Deserialize, Serialize, JsonSchema, Validate, Clone, PartialEq, Eq, Hash)] +#[derive(Clone, Debug, Eq, PartialEq, Hash, Deserialize, Serialize, JsonSchema, Validate)] pub struct ProductQuantization { #[validate(nested)] pub product: ProductQuantizationConfig, @@ -821,7 +821,7 @@ impl BinaryQuantizationEncoding { } } -#[derive(Debug, Deserialize, Serialize, JsonSchema, Validate, Clone, PartialEq, Eq, Hash)] +#[derive(Clone, Debug, Eq, PartialEq, Hash, Deserialize, Serialize, JsonSchema, Validate)] #[serde(rename_all = "snake_case")] pub struct BinaryQuantizationConfig { #[serde(skip_serializing_if = "Option::is_none")] @@ -837,13 +837,13 @@ pub struct BinaryQuantizationConfig { pub query_encoding: Option, } -#[derive(Debug, Deserialize, Serialize, JsonSchema, Validate, Clone, PartialEq, Eq, Hash)] +#[derive(Clone, Debug, Eq, PartialEq, Hash, Deserialize, Serialize, JsonSchema, Validate)] pub struct BinaryQuantization { #[validate(nested)] pub binary: BinaryQuantizationConfig, } -#[derive(Debug, Deserialize, Serialize, JsonSchema, Anonymize, Clone, PartialEq, Eq, Hash)] +#[derive(Clone, Debug, Eq, PartialEq, Hash, Deserialize, Serialize, JsonSchema, Anonymize)] #[serde(untagged, rename_all = "snake_case")] #[anonymize(false)] pub enum QuantizationConfig {