Files
qdrant/lib/edge/python/src/search.rs
Andrey Vasnetsov 1d4d6f02da Per-query IDF corpus for sparse vector search (#9661)
* Add per-query IDF corpus for sparse vector search

Let the caller choose, per query, which population sparse IDF statistics
are computed over. `params.idf` is either `"global"` (default, unchanged
behavior) or `{"corpus": <filter>}`, where the corpus filter is
independent of - and usually broader than - the retrieval filter.
Decoupling the two keeps the score scale stable when the retrieval
filter tightens: term importance is measured against a population the
user names, not against whatever subset the filter happens to select.

Design decisions:
- Corpus grammar is restricted to a conjunction (`must`) of `match`
  conditions on payload fields; loosening later is backward compatible.
- Strict mode validates the corpus filter like a read filter
  (unindexed fields rejected).
- `idf` on a vector without the IDF modifier is a validation error,
  never silently ignored.
- An empty corpus yields degenerate but corpus-scoped scores (smoothed
  IDF over N=0), never a fallback to global statistics - in multi-tenant
  collections a fallback would leak term statistics across tenants.

Implementation:
- QueryContext IDF stats are keyed by corpus, so one batch can mix
  requests with different corpora.
- Statistics come from the sparse index: df(term) is counted over the
  query terms' posting lists only, never by scanning stored vectors.
  Small corpora (under ~1/32 of the segment, by cardinality estimate)
  are kept as a sorted id list galloping through posting lists via
  skip_to; large ones as a dense membership mask filled streaming from
  the filtered-points iterator. A misestimated small corpus degrades
  into the mask.
- Exposed uniformly: REST (`params.idf`), gRPC (`IdfParams` message),
  edge python bindings; OpenAPI schema regenerated.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* Apply rustfmt

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* Fix clippy manual_is_multiple_of in sparse IDF corpus test.

Co-authored-by: Cursor <cursoragent@cursor.com>

* Allow any filter as IDF corpus

Drop the must+match grammar restriction on the corpus filter. A
restriction enforced only as a validation step over the full Filter
type buys nothing; if a narrower corpus syntax is ever wanted, it
should be a dedicated API-level type instead.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* Fix build: add memory field to SparseIndexConfig in idf corpus test

Co-authored-by: Cursor <cursoragent@cursor.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: root <111755117+qdrant-cloud-bot@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-14 11:16:47 +02:00

315 lines
7.3 KiB
Rust

use bytemuck::TransparentWrapper;
use derive_more::Into;
use ordered_float::OrderedFloat;
use pyo3::prelude::*;
use shard::query::query_enum::QueryEnum;
use shard::search::CoreSearchRequest;
use crate::repr::*;
use crate::*;
#[pyclass(name = "SearchRequest", from_py_object)]
#[derive(Clone, Debug, Into)]
pub struct PySearchRequest(CoreSearchRequest);
#[pyclass_repr]
#[pymethods]
impl PySearchRequest {
#[new]
#[pyo3(signature = (
query,
limit,
offset = None,
filter = None,
params = None,
with_vector = None,
with_payload = None,
score_threshold = None,
))]
#[allow(clippy::too_many_arguments)]
pub fn new(
query: PyQuery,
limit: usize,
offset: Option<usize>,
filter: Option<PyFilter>,
params: Option<PySearchParams>,
with_vector: Option<PyWithVector>,
with_payload: Option<PyWithPayload>,
score_threshold: Option<f32>,
) -> Self {
Self(CoreSearchRequest {
query: QueryEnum::from(query),
limit,
offset: offset.unwrap_or(0),
filter: filter.map(Filter::from),
params: params.map(SearchParams::from),
with_vector: with_vector.map(WithVector::from),
with_payload: with_payload.map(WithPayloadInterface::from),
score_threshold,
})
}
#[getter]
pub fn query(&self) -> &PyQuery {
PyQuery::wrap_ref(&self.0.query)
}
#[getter]
pub fn filter(&self) -> Option<&PyFilter> {
self.0.filter.as_ref().map(PyFilter::wrap_ref)
}
#[getter]
pub fn params(&self) -> Option<PySearchParams> {
self.0.params.clone().map(PySearchParams)
}
#[getter]
pub fn limit(&self) -> usize {
self.0.limit
}
#[getter]
pub fn offset(&self) -> usize {
self.0.offset
}
#[getter]
pub fn with_vector(&self) -> Option<&PyWithVector> {
self.0.with_vector.as_ref().map(PyWithVector::wrap_ref)
}
#[getter]
pub fn with_payload(&self) -> Option<&PyWithPayload> {
self.0.with_payload.as_ref().map(PyWithPayload::wrap_ref)
}
#[getter]
pub fn score_threshold(&self) -> Option<f32> {
self.0.score_threshold
}
pub fn __repr__(&self) -> String {
self.repr()
}
}
impl PySearchRequest {
fn _getters(self) {
// Every field should have a getter method
let CoreSearchRequest {
query: _,
filter: _,
params: _,
limit: _,
offset: _,
with_vector: _,
with_payload: _,
score_threshold: _,
} = self.0;
}
}
#[pyclass(name = "SearchParams", from_py_object)]
#[derive(Clone, Debug, Into)]
pub struct PySearchParams(pub SearchParams);
#[pyclass_repr]
#[pymethods]
impl PySearchParams {
#[new]
#[pyo3(signature = (
hnsw_ef = None,
exact = false,
quantization = None,
indexed_only = false,
acorn = None,
idf = None,
))]
pub fn new(
hnsw_ef: Option<usize>,
exact: bool,
quantization: Option<PyQuantizationSearchParams>,
indexed_only: bool,
acorn: Option<PyAcornSearchParams>,
idf: Option<PyIdfParams>,
) -> Self {
Self(SearchParams {
hnsw_ef,
exact,
quantization: quantization.map(QuantizationSearchParams::from),
indexed_only,
acorn: acorn.map(AcornSearchParams::from),
idf: idf.map(IdfParams::from),
})
}
#[getter]
pub fn hnsw_ef(&self) -> Option<usize> {
self.0.hnsw_ef
}
#[getter]
pub fn exact(&self) -> bool {
self.0.exact
}
#[getter]
pub fn quantization(&self) -> Option<PyQuantizationSearchParams> {
self.0.quantization.map(PyQuantizationSearchParams)
}
#[getter]
pub fn indexed_only(&self) -> bool {
self.0.indexed_only
}
#[getter]
pub fn acorn(&self) -> Option<PyAcornSearchParams> {
self.0.acorn.map(PyAcornSearchParams)
}
#[getter]
pub fn idf(&self) -> Option<PyIdfParams> {
self.0.idf.clone().map(PyIdfParams)
}
pub fn __repr__(&self) -> String {
self.repr()
}
}
impl PySearchParams {
fn _getters(self) {
// Every field should have a getter method
let SearchParams {
hnsw_ef: _,
exact: _,
quantization: _,
indexed_only: _,
acorn: _,
idf: _,
} = self.0;
}
}
#[pyclass(name = "IdfParams", from_py_object)]
#[derive(Clone, Debug, Into)]
pub struct PyIdfParams(pub IdfParams);
#[pyclass_repr]
#[pymethods]
impl PyIdfParams {
#[new]
#[pyo3(signature = (corpus = None))]
pub fn new(corpus: Option<PyFilter>) -> Self {
Self(match corpus {
// No corpus filter: collection-wide (global) statistics.
None => IdfParams::Scope(IdfScope::Global),
Some(corpus) => IdfParams::Corpus(IdfCorpusParams {
corpus: Filter::from(corpus),
}),
})
}
#[getter]
pub fn corpus(&self) -> Option<&PyFilter> {
self.0.corpus().map(PyFilter::wrap_ref)
}
pub fn __repr__(&self) -> String {
self.repr()
}
}
#[pyclass(name = "QuantizationSearchParams", from_py_object)]
#[derive(Copy, Clone, Debug, Into)]
pub struct PyQuantizationSearchParams(QuantizationSearchParams);
#[pyclass_repr]
#[pymethods]
impl PyQuantizationSearchParams {
#[new]
#[pyo3(signature = (ignore = false, rescore = None, oversampling = None))]
pub fn new(ignore: bool, rescore: Option<bool>, oversampling: Option<f64>) -> Self {
Self(QuantizationSearchParams {
ignore,
rescore,
oversampling,
})
}
#[getter]
pub fn ignore(&self) -> bool {
self.0.ignore
}
#[getter]
pub fn rescore(&self) -> Option<bool> {
self.0.rescore
}
#[getter]
pub fn oversampling(&self) -> Option<f64> {
self.0.oversampling
}
pub fn __repr__(&self) -> String {
self.repr()
}
}
impl PyQuantizationSearchParams {
fn _getters(self) {
// Every field should have a getter method
let QuantizationSearchParams {
ignore: _,
rescore: _,
oversampling: _,
} = self.0;
}
}
#[pyclass(name = "AcornSearchParams", from_py_object)]
#[derive(Copy, Clone, Debug, Into)]
pub struct PyAcornSearchParams(AcornSearchParams);
#[pyclass_repr]
#[pymethods]
impl PyAcornSearchParams {
#[new]
#[pyo3(signature = (enable = false, max_selectivity = None))]
pub fn new(enable: bool, max_selectivity: Option<f64>) -> Self {
Self(AcornSearchParams {
enable,
max_selectivity: max_selectivity.map(OrderedFloat),
})
}
#[getter]
pub fn enable(&self) -> bool {
self.0.enable
}
#[getter]
pub fn max_selectivity(&self) -> Option<f64> {
self.0
.max_selectivity
.map(|selectivity| selectivity.into_inner())
}
pub fn __repr__(&self) -> String {
self.repr()
}
}
impl PyAcornSearchParams {
fn _getters(self) {
// Every field should have a getter method
let AcornSearchParams {
enable: _,
max_selectivity: _,
} = self.0;
}
}