[TQDT] API (#9172)

* [ai] TQDT in the API * [ai] unify new sparse error for TQDT * Rename to `turbo4` * Add `Turbo4` to comments and doc strings. * Also validate named sparse vector creation
2026-07-23 11:11:00 -05:00 · 2026-05-29 15:26:39 +02:00
parent 9b87cd369c
commit e58be0c380
22 changed files with 129 additions and 9 deletions
--- a/docs/redoc/master/openapi.json
+++ b/docs/redoc/master/openapi.json
@@ -7515,7 +7515,7 @@
            "nullable": true
          },
          "datatype": {
-            "description": "Defines which datatype should be used to represent vectors in the storage. Choosing different datatypes allows to optimize memory usage and performance vs accuracy.\n\n- For `float32` datatype - vectors are stored as single-precision floating point numbers, 4 bytes. - For `float16` datatype - vectors are stored as half-precision floating point numbers, 2 bytes. - For `uint8` datatype - vectors are stored as unsigned 8-bit integers, 1 byte. It expects vector elements to be in range `[0, 255]`.",
+            "description": "Defines which datatype should be used to represent vectors in the storage. Choosing different datatypes allows to optimize memory usage and performance vs accuracy.\n\n- For `float32` datatype - vectors are stored as single-precision floating point numbers, 4 bytes. - For `float16` datatype - vectors are stored as half-precision floating point numbers, 2 bytes. - For `uint8` datatype - vectors are stored as unsigned 8-bit integers, 1 byte. It expects vector elements to be in range `[0, 255]`. - For `turbo4` datatype - vectors are quantized to 4 bits per element using the TurboQuant algorithm.",
            "anyOf": [
              {
                "$ref": "#/components/schemas/Datatype"
@@ -7792,7 +7792,8 @@
        "enum": [
          "float32",
          "uint8",
-          "float16"
+          "float16",
+          "turbo4"
        ]
      },
      "MultiVectorConfig": {
@@ -13469,7 +13470,8 @@
        "enum": [
          "float32",
          "float16",
-          "uint8"
+          "uint8",
+          "turbo4"
        ]
      },
      "SparseVectorDataConfig": {
@@ -17781,7 +17783,7 @@
            ]
          },
          "datatype": {
-            "description": "Element storage type (Float32, Float16, Uint8)",
+            "description": "Element storage type (Float32, Float16, Uint8, Turbo4)",
            "anyOf": [
              {
                "$ref": "#/components/schemas/VectorStorageDatatype"
--- a/lib/api/build.rs
+++ b/lib/api/build.rs
@@ -184,6 +184,7 @@ fn configure_validation(builder: Builder) -> Builder {
            ("QuantizationConfig.quantization", ""),
            ("QuantizationConfigDiff.quantization", ""),
            ("ScalarQuantization.quantile", "range(min = 0.5, max = 1.0)"),
+            ("SparseIndexConfig.datatype", "custom(function = \"crate::grpc::validate::validate_sparse_datatype\")"),
            ("UpdateCollectionClusterSetupRequest.timeout", "range(min = 1)"),
            ("UpdateCollectionClusterSetupRequest.operation", ""),
            ("StrictModeConfig.max_query_limit", "range(min = 1)"),
@@ -256,6 +257,7 @@ fn configure_validation(builder: Builder) -> Builder {
            ("CreateVectorNameRequest.collection_name", "length(min = 1, max = 255), custom(function = \"common::validation::validate_collection_name_legacy\")"),
            ("CreateVectorNameRequest.vector_config", ""),
            ("DenseVectorCreationConfig.size", "range(min = 1, max = 65536)"),
+            ("SparseVectorCreationConfig.datatype", "custom(function = \"crate::grpc::validate::validate_sparse_datatype\")"),
            ("DeleteVectorNameRequest.collection_name", "length(min = 1, max = 255), custom(function = \"common::validation::validate_collection_name_legacy\")"),
            ("SearchPoints.collection_name", "length(min = 1, max = 255), custom(function = \"common::validation::validate_collection_name_legacy\")"),
            ("SearchPoints.filter", ""),
--- a/lib/api/src/grpc/conversions.rs
+++ b/lib/api/src/grpc/conversions.rs
@@ -3572,6 +3572,7 @@ fn convert_datatype_from_proto(
        grpc::Datatype::Float32 => Ok(Some(VectorStorageDatatype::Float32)),
        grpc::Datatype::Float16 => Ok(Some(VectorStorageDatatype::Float16)),
        grpc::Datatype::Uint8 => Ok(Some(VectorStorageDatatype::Uint8)),
+        grpc::Datatype::Turbo4 => Ok(Some(VectorStorageDatatype::Turbo4)),
    }
 }

@@ -3627,5 +3628,6 @@ fn datatype_to_grpc(dt: VectorStorageDatatype) -> grpc::Datatype {
        VectorStorageDatatype::Float32 => grpc::Datatype::Float32,
        VectorStorageDatatype::Float16 => grpc::Datatype::Float16,
        VectorStorageDatatype::Uint8 => grpc::Datatype::Uint8,
+        VectorStorageDatatype::Turbo4 => grpc::Datatype::Turbo4,
    }
 }
--- a/lib/api/src/grpc/proto/collections.proto
+++ b/lib/api/src/grpc/proto/collections.proto
@@ -11,6 +11,7 @@ enum Datatype {
  Float32 = 1;
  Uint8 = 2;
  Float16 = 3;
+  Turbo4 = 4;
 }

 // ---------------------------------------------
--- a/lib/api/src/grpc/proto/points.proto
+++ b/lib/api/src/grpc/proto/points.proto
@@ -362,7 +362,7 @@ message DenseVectorCreationConfig {
  Distance distance = 2;
  // Configuration for multi-vector search (e.g., ColBERT)
  optional MultiVectorConfig multivector_config = 3;
-  // Data type of the vectors (Float32, Float16, Uint8)
+  // Data type of the vectors (Float32, Float16, Uint8, Turbo4)
  optional Datatype datatype = 4;
 }

--- a/lib/api/src/grpc/qdrant.rs
+++ b/lib/api/src/grpc/qdrant.rs
@@ -700,6 +700,7 @@ pub struct HnswConfigDiff {
    #[prost(bool, optional, tag = "7")]
    pub inline_storage: ::core::option::Option<bool>,
 }
+#[derive(validator::Validate)]
 #[derive(serde::Serialize)]
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct SparseIndexConfig {
@@ -712,6 +713,7 @@ pub struct SparseIndexConfig {
    pub on_disk: ::core::option::Option<bool>,
    /// Datatype used to store weights in the index.
    #[prost(enumeration = "Datatype", optional, tag = "3")]
+    #[validate(custom(function = "crate::grpc::validate::validate_sparse_datatype"))]
    pub datatype: ::core::option::Option<i32>,
 }
 #[derive(validator::Validate)]
@@ -2021,6 +2023,7 @@ pub enum Datatype {
    Float32 = 1,
    Uint8 = 2,
    Float16 = 3,
+    Turbo4 = 4,
 }
 impl Datatype {
    /// String value of the enum field names used in the ProtoBuf definition.
@@ -2033,6 +2036,7 @@ impl Datatype {
            Self::Float32 => "Float32",
            Self::Uint8 => "Uint8",
            Self::Float16 => "Float16",
+            Self::Turbo4 => "Turbo4",
        }
    }
    /// Creates an enum from field names used in the ProtoBuf definition.
@@ -2042,6 +2046,7 @@ impl Datatype {
            "Float32" => Some(Self::Float32),
            "Uint8" => Some(Self::Uint8),
            "Float16" => Some(Self::Float16),
+            "Turbo4" => Some(Self::Turbo4),
            _ => None,
        }
    }
@@ -5387,7 +5392,7 @@ pub struct DenseVectorCreationConfig {
    /// Configuration for multi-vector search (e.g., ColBERT)
    #[prost(message, optional, tag = "3")]
    pub multivector_config: ::core::option::Option<MultiVectorConfig>,
-    /// Data type of the vectors (Float32, Float16, Uint8)
+    /// Data type of the vectors (Float32, Float16, Uint8, Turbo4)
    #[prost(enumeration = "Datatype", optional, tag = "4")]
    pub datatype: ::core::option::Option<i32>,
 }
@@ -5402,6 +5407,7 @@ pub struct SparseVectorCreationConfig {
    pub modifier: ::core::option::Option<i32>,
    /// Data type used to store weights in the index
    #[prost(enumeration = "Datatype", optional, tag = "2")]
+    #[validate(custom(function = "crate::grpc::validate::validate_sparse_datatype"))]
    pub datatype: ::core::option::Option<i32>,
 }
 #[derive(validator::Validate)]
--- a/lib/api/src/grpc/validate.rs
+++ b/lib/api/src/grpc/validate.rs
@@ -484,6 +484,15 @@ pub fn validate_geo_polygon_interiors(
    Ok(())
 }

+/// Reject the `Turbo4` datatype on sparse vector configs.
+/// `validator` unwraps `Option<i32>` before calling, so we receive `&i32`.
+pub fn validate_sparse_datatype(datatype: &i32) -> Result<(), ValidationError> {
+    if *datatype == grpc::Datatype::Turbo4 as i32 {
+        return Err(common::validation::sparse_turbo4_unsupported_error());
+    }
+    Ok(())
+}
+
 /// Validate that the timestamp is within the range specified in the protobuf docs.
 /// <https://protobuf.dev/reference/protobuf/google.protobuf/#timestamp>
 pub fn validate_timestamp(ts: &prost_wkt_types::Timestamp) -> Result<(), ValidationError> {
--- a/lib/collection/src/collection/vector_name_schema.rs
+++ b/lib/collection/src/collection/vector_name_schema.rs
@@ -243,5 +243,6 @@ fn storage_datatype_to_collection(
            crate::operations::types::Datatype::Float16
        }
        segment::types::VectorStorageDatatype::Uint8 => crate::operations::types::Datatype::Uint8,
+        segment::types::VectorStorageDatatype::Turbo4 => crate::operations::types::Datatype::Turbo4,
    }
 }
--- a/lib/collection/src/config.rs
+++ b/lib/collection/src/config.rs
@@ -195,6 +195,9 @@ impl CollectionParams {
                let element_bytes = match params.datatype {
                    Some(Datatype::Float16) => 2,
                    Some(Datatype::Uint8) => 1,
+                    // Placeholder: Turbo4 is ~0.5 byte/dim + per-row scale.
+                    // Mirroring Uint8 (1 byte) until accurate accounting is implemented.
+                    Some(Datatype::Turbo4) => 1,
                    Some(Datatype::Float32) | None => 4,
                };

--- a/lib/collection/src/operations/conversions.rs
+++ b/lib/collection/src/operations/conversions.rs
@@ -756,6 +756,7 @@ pub fn convert_datatype_from_proto(datatype: Option<i32>) -> Result<Option<Datat
                api::grpc::qdrant::Datatype::Uint8 => Ok(Some(Datatype::Uint8)),
                api::grpc::qdrant::Datatype::Float32 => Ok(Some(Datatype::Float32)),
                api::grpc::qdrant::Datatype::Float16 => Ok(Some(Datatype::Float16)),
+                api::grpc::qdrant::Datatype::Turbo4 => Ok(Some(Datatype::Turbo4)),
                api::grpc::qdrant::Datatype::Default => Ok(None),
            }
        } else {
@@ -1425,6 +1426,7 @@ impl From<Datatype> for api::grpc::qdrant::Datatype {
            Datatype::Float32 => api::grpc::qdrant::Datatype::Float32,
            Datatype::Uint8 => api::grpc::qdrant::Datatype::Uint8,
            Datatype::Float16 => api::grpc::qdrant::Datatype::Float16,
+            Datatype::Turbo4 => api::grpc::qdrant::Datatype::Turbo4,
        }
    }
 }
--- a/lib/collection/src/operations/types.rs
+++ b/lib/collection/src/operations/types.rs
@@ -1319,6 +1319,7 @@ pub enum Datatype {
    Float32,
    Uint8,
    Float16,
+    Turbo4,
 }

 impl From<Datatype> for VectorStorageDatatype {
@@ -1327,6 +1328,7 @@ impl From<Datatype> for VectorStorageDatatype {
            Datatype::Float32 => VectorStorageDatatype::Float32,
            Datatype::Uint8 => VectorStorageDatatype::Uint8,
            Datatype::Float16 => VectorStorageDatatype::Float16,
+            Datatype::Turbo4 => VectorStorageDatatype::Turbo4,
        }
    }
 }
@@ -1370,6 +1372,8 @@ pub struct VectorParams {
    ///   2 bytes.
    /// - For `uint8` datatype - vectors are stored as unsigned 8-bit integers, 1 byte.
    ///   It expects vector elements to be in range `[0, 255]`.
+    /// - For `turbo4` datatype - vectors are quantized to 4 bits per element using the
+    ///   TurboQuant algorithm.
    pub datatype: Option<Datatype>,

    #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -1383,6 +1387,15 @@ pub fn validate_nonzerou64_range_min_1_max_65536(
    validate_range_generic(value.get(), Some(1), Some(65536))
 }

+/// Reject the `Turbo4` datatype on sparse vector configs.
+/// `validator` unwraps `Option<Datatype>` before calling, so we receive `&Datatype`.
+fn validate_sparse_datatype(datatype: &Datatype) -> Result<(), ValidationError> {
+    if matches!(datatype, Datatype::Turbo4) {
+        return Err(common::validation::sparse_turbo4_unsupported_error());
+    }
+    Ok(())
+}
+
 /// Is considered empty if `None` or if diff has no field specified
 fn is_hnsw_diff_empty(hnsw_config: &Option<HnswConfigDiff>) -> bool {
    hnsw_config.is_none() || *hnsw_config == Some(HnswConfigDiff::default())
@@ -1396,6 +1409,7 @@ fn is_hnsw_diff_empty(hnsw_config: &Option<HnswConfigDiff>) -> bool {
 pub struct SparseVectorParams {
    /// Custom params for index. If none - values from collection configuration are used.
    #[serde(default, skip_serializing_if = "Option::is_none")]
+    #[validate(nested)]
    pub index: Option<SparseIndexParams>,

    /// Configures addition value modifications for sparse vectors.
@@ -1412,7 +1426,18 @@ impl SparseVectorParams {

 /// Configuration for sparse inverted index.
 #[derive(
-    Debug, Hash, Deserialize, Serialize, JsonSchema, Anonymize, Copy, Clone, PartialEq, Eq, Default,
+    Debug,
+    Hash,
+    Deserialize,
+    Serialize,
+    JsonSchema,
+    Validate,
+    Anonymize,
+    Copy,
+    Clone,
+    PartialEq,
+    Eq,
+    Default,
 )]
 #[serde(rename_all = "snake_case")]
 pub struct SparseIndexParams {
@@ -1436,6 +1461,7 @@ pub struct SparseIndexParams {
    ///   Quantization to fit byte range `[0, 255]` happens during indexing automatically, so the
    ///   actual vector data does not need to conform to this range.
    #[serde(default, skip_serializing_if = "Option::is_none")]
+    #[validate(custom(function = "validate_sparse_datatype"))]
    pub datatype: Option<Datatype>,
 }

--- a/lib/common/common/src/validation.rs
+++ b/lib/common/common/src/validation.rs
@@ -65,6 +65,16 @@ where
    Err(err)
 }

+/// Build the `ValidationError` for a sparse vector configured with the
+/// `Turbo4` datatype. Shared between REST and gRPC validators.
+pub fn sparse_turbo4_unsupported_error() -> ValidationError {
+    let mut err = ValidationError::new("unsupported_sparse_datatype");
+    err.message = Some(Cow::Borrowed(
+        "sparse vectors do not support the `turbo4` datatype",
+    ));
+    err
+}
+
 /// Validate that `value` is a non-empty string.
 pub fn validate_not_empty(value: &str) -> Result<(), ValidationError> {
    if value.is_empty() {
--- a/lib/edge/python/src/config/vector_data.rs
+++ b/lib/edge/python/src/config/vector_data.rs
@@ -292,6 +292,7 @@ pub enum PyVectorStorageDatatype {
    Float32,
    Float16,
    Uint8,
+    Turbo4,
 }

 #[pymethods]
@@ -307,6 +308,7 @@ impl Repr for PyVectorStorageDatatype {
            Self::Float32 => "Float32",
            Self::Float16 => "Float16",
            Self::Uint8 => "Uint8",
+            Self::Turbo4 => "Turbo4",
        };

        f.simple_enum::<Self>(repr)
@@ -319,6 +321,7 @@ impl From<VectorStorageDatatype> for PyVectorStorageDatatype {
            VectorStorageDatatype::Float32 => PyVectorStorageDatatype::Float32,
            VectorStorageDatatype::Float16 => PyVectorStorageDatatype::Float16,
            VectorStorageDatatype::Uint8 => PyVectorStorageDatatype::Uint8,
+            VectorStorageDatatype::Turbo4 => PyVectorStorageDatatype::Turbo4,
        }
    }
 }
@@ -329,6 +332,7 @@ impl From<PyVectorStorageDatatype> for VectorStorageDatatype {
            PyVectorStorageDatatype::Float32 => VectorStorageDatatype::Float32,
            PyVectorStorageDatatype::Float16 => VectorStorageDatatype::Float16,
            PyVectorStorageDatatype::Uint8 => VectorStorageDatatype::Uint8,
+            PyVectorStorageDatatype::Turbo4 => VectorStorageDatatype::Turbo4,
        }
    }
 }
--- a/lib/segment/src/data_types/named_vectors.rs
+++ b/lib/segment/src/data_types/named_vectors.rs
@@ -346,6 +346,9 @@ impl<'a> NamedVectors<'a> {
            Some(VectorStorageDatatype::Float16) => config
                .distance
                .preprocess_vector::<VectorElementTypeHalf>(dense_vector),
+            Some(VectorStorageDatatype::Turbo4) => {
+                unimplemented!("turbo4 datatype storage not yet wired up")
+            }
        }
    }
 }
--- a/lib/segment/src/data_types/vector_name_config.rs
+++ b/lib/segment/src/data_types/vector_name_config.rs
@@ -1,6 +1,6 @@
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
-use validator::Validate;
+use validator::{Validate, ValidationError};

 use crate::data_types::modifier::Modifier;
 use crate::index::sparse_index::sparse_index_config::SparseIndexConfig;
@@ -65,7 +65,7 @@ pub struct DenseVectorConfig {
    /// Configuration for multi-vector points (e.g., ColBERT)
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub multivector_config: Option<MultiVectorConfig>,
-    /// Element storage type (Float32, Float16, Uint8)
+    /// Element storage type (Float32, Float16, Uint8, Turbo4)
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub datatype: Option<VectorStorageDatatype>,
 }
@@ -82,9 +82,19 @@ pub struct SparseVectorConfig {
    pub modifier: Option<Modifier>,
    /// Datatype used to store weights in the index
    #[serde(default, skip_serializing_if = "Option::is_none")]
+    #[validate(custom(function = "validate_sparse_datatype"))]
    pub datatype: Option<VectorStorageDatatype>,
 }

+/// Reject the `Turbo4` datatype on sparse vector configs.
+/// `validator` unwraps `Option<VectorStorageDatatype>` before calling, so we receive `&VectorStorageDatatype`.
+fn validate_sparse_datatype(datatype: &VectorStorageDatatype) -> Result<(), ValidationError> {
+    if matches!(datatype, VectorStorageDatatype::Turbo4) {
+        return Err(common::validation::sparse_turbo4_unsupported_error());
+    }
+    Ok(())
+}
+
 impl Validate for VectorNameConfig {
    fn validate(&self) -> Result<(), validator::ValidationErrors> {
        match self {
--- a/lib/segment/src/index/hnsw_index/gpu/gpu_vector_storage/mod.rs
+++ b/lib/segment/src/index/hnsw_index/gpu/gpu_vector_storage/mod.rs
@@ -96,6 +96,9 @@ impl ShaderBuilderParameters for GpuVectorStorage {
            VectorStorageDatatype::Uint8 => {
                defines.insert("VECTOR_STORAGE_ELEMENT_UINT8".to_owned(), None);
            }
+            VectorStorageDatatype::Turbo4 => {
+                unimplemented!("turbo4 datatype storage not yet wired up")
+            }
        }

        match self.distance {
--- a/lib/segment/src/segment_constructor/segment_constructor_base.rs
+++ b/lib/segment/src/segment_constructor/segment_constructor_base.rs
@@ -121,6 +121,9 @@ fn open_mmap_vector_storage(
                vector_config.distance,
                populate,
            ),
+            VectorStorageDatatype::Turbo4 => {
+                unimplemented!("turbo4 datatype storage not yet wired up")
+            }
        }
    }
 }
@@ -369,6 +372,9 @@ pub(crate) fn create_sparse_vector_index(
        (SparseIndexType::Mmap, VectorStorageDatatype::Uint8) => {
            VectorIndexEnum::SparseCompressedMmapU8(SparseVectorIndex::open(args)?)
        }
+        (_, VectorStorageDatatype::Turbo4) => {
+            unimplemented!("turbo4 datatype storage not yet wired up")
+        }
    };

    Ok(vector_index)
--- a/lib/segment/src/types.rs
+++ b/lib/segment/src/types.rs
@@ -1610,6 +1610,8 @@ pub enum VectorStorageDatatype {
    Float16,
    // Unsigned 8-bit integer
    Uint8,
+    // TurboQuant 4-bit compressed storage
+    Turbo4,
 }

 #[derive(
--- a/lib/segment/src/vector_storage/multi_dense/appendable_mmap_multi_dense_vector_storage.rs
+++ b/lib/segment/src/vector_storage/multi_dense/appendable_mmap_multi_dense_vector_storage.rs
@@ -409,6 +409,9 @@ pub fn open_appendable_memmap_vector_storage(
            madvise,
            populate,
        ),
+        VectorStorageDatatype::Turbo4 => {
+            unimplemented!("turbo4 datatype storage not yet wired up")
+        }
    }
 }

@@ -446,6 +449,9 @@ pub fn open_appendable_memmap_multi_vector_storage(
            madvise,
            populate,
        ),
+        VectorStorageDatatype::Turbo4 => {
+            unimplemented!("turbo4 datatype storage not yet wired up")
+        }
    }
 }

--- a/lib/segment/src/vector_storage/quantized/quantized_scorer_builder.rs
+++ b/lib/segment/src/vector_storage/quantized/quantized_scorer_builder.rs
@@ -85,6 +85,9 @@ impl<'a> QuantizedScorerBuilder<'a> {
                    self.build_with_metric::<VectorElementTypeHalf, ManhattanMetric>()
                }
            },
+            VectorStorageDatatype::Turbo4 => {
+                unimplemented!("turbo4 datatype storage not yet wired up")
+            }
        }
    }

--- a/lib/segment/tests/integration/byte_storage_quantization_test.rs
+++ b/lib/segment/tests/integration/byte_storage_quantization_test.rs
@@ -50,6 +50,7 @@ where
            vector
        }
        VectorStorageDatatype::Uint8 => random_dense_byte_vector(rnd_gen, dim),
+        VectorStorageDatatype::Turbo4 => unreachable!(),
    }
 }

--- a/tests/openapi/test_sparse_vector_validations.py
+++ b/tests/openapi/test_sparse_vector_validations.py
@@ -117,6 +117,24 @@ def test_sparse_vector_validations(collection_name):
    assert 'points[0].vector.?.indices: Validation error: must be unique [{}]' in response.json()["status"]["error"]


+def test_sparse_vector_turbo4_datatype_rejected(collection_name):
+    # Use a distinct name so the autouse setup's collection is untouched.
+    bad_collection = f"{collection_name}_turbo4_rejected"
+
+    response = request_with_validation(
+        api='/collections/{collection_name}',
+        method="PUT",
+        path_params={'collection_name': bad_collection},
+        body={
+            "sparse_vectors": {
+                "text": {"index": {"datatype": "turbo4"}}
+            }
+        },
+    )
+    assert not response.ok
+    assert "sparse vectors do not support" in response.json()["status"]["error"]
+
+
 def test_sorted_sparse_vector(collection_name):
    response = request_with_validation(
        api='/collections/{collection_name}/points/query',