mirror of
https://github.com/qdrant/qdrant.git
synced 2026-08-06 18:10:58 -05:00
* Add per-query IDF corpus for sparse vector search
Let the caller choose, per query, which population sparse IDF statistics
are computed over. `params.idf` is either `"global"` (default, unchanged
behavior) or `{"corpus": <filter>}`, where the corpus filter is
independent of - and usually broader than - the retrieval filter.
Decoupling the two keeps the score scale stable when the retrieval
filter tightens: term importance is measured against a population the
user names, not against whatever subset the filter happens to select.
Design decisions:
- Corpus grammar is restricted to a conjunction (`must`) of `match`
conditions on payload fields; loosening later is backward compatible.
- Strict mode validates the corpus filter like a read filter
(unindexed fields rejected).
- `idf` on a vector without the IDF modifier is a validation error,
never silently ignored.
- An empty corpus yields degenerate but corpus-scoped scores (smoothed
IDF over N=0), never a fallback to global statistics - in multi-tenant
collections a fallback would leak term statistics across tenants.
Implementation:
- QueryContext IDF stats are keyed by corpus, so one batch can mix
requests with different corpora.
- Statistics come from the sparse index: df(term) is counted over the
query terms' posting lists only, never by scanning stored vectors.
Small corpora (under ~1/32 of the segment, by cardinality estimate)
are kept as a sorted id list galloping through posting lists via
skip_to; large ones as a dense membership mask filled streaming from
the filtered-points iterator. A misestimated small corpus degrades
into the mask.
- Exposed uniformly: REST (`params.idf`), gRPC (`IdfParams` message),
edge python bindings; OpenAPI schema regenerated.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
* Apply rustfmt
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
* Fix clippy manual_is_multiple_of in sparse IDF corpus test.
Co-authored-by: Cursor <cursoragent@cursor.com>
* Allow any filter as IDF corpus
Drop the must+match grammar restriction on the corpus filter. A
restriction enforced only as a validation step over the full Filter
type buys nothing; if a narrower corpus syntax is ever wanted, it
should be a dedicated API-level type instead.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
* Fix build: add memory field to SparseIndexConfig in idf corpus test
Co-authored-by: Cursor <cursoragent@cursor.com>
---------
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: root <111755117+qdrant-cloud-bot@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
315 lines
7.3 KiB
Rust
315 lines
7.3 KiB
Rust
use bytemuck::TransparentWrapper;
|
|
use derive_more::Into;
|
|
use ordered_float::OrderedFloat;
|
|
use pyo3::prelude::*;
|
|
use shard::query::query_enum::QueryEnum;
|
|
use shard::search::CoreSearchRequest;
|
|
|
|
use crate::repr::*;
|
|
use crate::*;
|
|
|
|
#[pyclass(name = "SearchRequest", from_py_object)]
|
|
#[derive(Clone, Debug, Into)]
|
|
pub struct PySearchRequest(CoreSearchRequest);
|
|
|
|
#[pyclass_repr]
|
|
#[pymethods]
|
|
impl PySearchRequest {
|
|
#[new]
|
|
#[pyo3(signature = (
|
|
query,
|
|
limit,
|
|
offset = None,
|
|
filter = None,
|
|
params = None,
|
|
with_vector = None,
|
|
with_payload = None,
|
|
score_threshold = None,
|
|
))]
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub fn new(
|
|
query: PyQuery,
|
|
limit: usize,
|
|
offset: Option<usize>,
|
|
filter: Option<PyFilter>,
|
|
params: Option<PySearchParams>,
|
|
with_vector: Option<PyWithVector>,
|
|
with_payload: Option<PyWithPayload>,
|
|
score_threshold: Option<f32>,
|
|
) -> Self {
|
|
Self(CoreSearchRequest {
|
|
query: QueryEnum::from(query),
|
|
limit,
|
|
offset: offset.unwrap_or(0),
|
|
filter: filter.map(Filter::from),
|
|
params: params.map(SearchParams::from),
|
|
with_vector: with_vector.map(WithVector::from),
|
|
with_payload: with_payload.map(WithPayloadInterface::from),
|
|
score_threshold,
|
|
})
|
|
}
|
|
|
|
#[getter]
|
|
pub fn query(&self) -> &PyQuery {
|
|
PyQuery::wrap_ref(&self.0.query)
|
|
}
|
|
|
|
#[getter]
|
|
pub fn filter(&self) -> Option<&PyFilter> {
|
|
self.0.filter.as_ref().map(PyFilter::wrap_ref)
|
|
}
|
|
|
|
#[getter]
|
|
pub fn params(&self) -> Option<PySearchParams> {
|
|
self.0.params.clone().map(PySearchParams)
|
|
}
|
|
|
|
#[getter]
|
|
pub fn limit(&self) -> usize {
|
|
self.0.limit
|
|
}
|
|
|
|
#[getter]
|
|
pub fn offset(&self) -> usize {
|
|
self.0.offset
|
|
}
|
|
|
|
#[getter]
|
|
pub fn with_vector(&self) -> Option<&PyWithVector> {
|
|
self.0.with_vector.as_ref().map(PyWithVector::wrap_ref)
|
|
}
|
|
|
|
#[getter]
|
|
pub fn with_payload(&self) -> Option<&PyWithPayload> {
|
|
self.0.with_payload.as_ref().map(PyWithPayload::wrap_ref)
|
|
}
|
|
|
|
#[getter]
|
|
pub fn score_threshold(&self) -> Option<f32> {
|
|
self.0.score_threshold
|
|
}
|
|
|
|
pub fn __repr__(&self) -> String {
|
|
self.repr()
|
|
}
|
|
}
|
|
|
|
impl PySearchRequest {
|
|
fn _getters(self) {
|
|
// Every field should have a getter method
|
|
let CoreSearchRequest {
|
|
query: _,
|
|
filter: _,
|
|
params: _,
|
|
limit: _,
|
|
offset: _,
|
|
with_vector: _,
|
|
with_payload: _,
|
|
score_threshold: _,
|
|
} = self.0;
|
|
}
|
|
}
|
|
|
|
#[pyclass(name = "SearchParams", from_py_object)]
|
|
#[derive(Clone, Debug, Into)]
|
|
pub struct PySearchParams(pub SearchParams);
|
|
|
|
#[pyclass_repr]
|
|
#[pymethods]
|
|
impl PySearchParams {
|
|
#[new]
|
|
#[pyo3(signature = (
|
|
hnsw_ef = None,
|
|
exact = false,
|
|
quantization = None,
|
|
indexed_only = false,
|
|
acorn = None,
|
|
idf = None,
|
|
))]
|
|
pub fn new(
|
|
hnsw_ef: Option<usize>,
|
|
exact: bool,
|
|
quantization: Option<PyQuantizationSearchParams>,
|
|
indexed_only: bool,
|
|
acorn: Option<PyAcornSearchParams>,
|
|
idf: Option<PyIdfParams>,
|
|
) -> Self {
|
|
Self(SearchParams {
|
|
hnsw_ef,
|
|
exact,
|
|
quantization: quantization.map(QuantizationSearchParams::from),
|
|
indexed_only,
|
|
acorn: acorn.map(AcornSearchParams::from),
|
|
idf: idf.map(IdfParams::from),
|
|
})
|
|
}
|
|
|
|
#[getter]
|
|
pub fn hnsw_ef(&self) -> Option<usize> {
|
|
self.0.hnsw_ef
|
|
}
|
|
|
|
#[getter]
|
|
pub fn exact(&self) -> bool {
|
|
self.0.exact
|
|
}
|
|
|
|
#[getter]
|
|
pub fn quantization(&self) -> Option<PyQuantizationSearchParams> {
|
|
self.0.quantization.map(PyQuantizationSearchParams)
|
|
}
|
|
|
|
#[getter]
|
|
pub fn indexed_only(&self) -> bool {
|
|
self.0.indexed_only
|
|
}
|
|
|
|
#[getter]
|
|
pub fn acorn(&self) -> Option<PyAcornSearchParams> {
|
|
self.0.acorn.map(PyAcornSearchParams)
|
|
}
|
|
|
|
#[getter]
|
|
pub fn idf(&self) -> Option<PyIdfParams> {
|
|
self.0.idf.clone().map(PyIdfParams)
|
|
}
|
|
|
|
pub fn __repr__(&self) -> String {
|
|
self.repr()
|
|
}
|
|
}
|
|
|
|
impl PySearchParams {
|
|
fn _getters(self) {
|
|
// Every field should have a getter method
|
|
let SearchParams {
|
|
hnsw_ef: _,
|
|
exact: _,
|
|
quantization: _,
|
|
indexed_only: _,
|
|
acorn: _,
|
|
idf: _,
|
|
} = self.0;
|
|
}
|
|
}
|
|
|
|
#[pyclass(name = "IdfParams", from_py_object)]
|
|
#[derive(Clone, Debug, Into)]
|
|
pub struct PyIdfParams(pub IdfParams);
|
|
|
|
#[pyclass_repr]
|
|
#[pymethods]
|
|
impl PyIdfParams {
|
|
#[new]
|
|
#[pyo3(signature = (corpus = None))]
|
|
pub fn new(corpus: Option<PyFilter>) -> Self {
|
|
Self(match corpus {
|
|
// No corpus filter: collection-wide (global) statistics.
|
|
None => IdfParams::Scope(IdfScope::Global),
|
|
Some(corpus) => IdfParams::Corpus(IdfCorpusParams {
|
|
corpus: Filter::from(corpus),
|
|
}),
|
|
})
|
|
}
|
|
|
|
#[getter]
|
|
pub fn corpus(&self) -> Option<&PyFilter> {
|
|
self.0.corpus().map(PyFilter::wrap_ref)
|
|
}
|
|
|
|
pub fn __repr__(&self) -> String {
|
|
self.repr()
|
|
}
|
|
}
|
|
|
|
#[pyclass(name = "QuantizationSearchParams", from_py_object)]
|
|
#[derive(Copy, Clone, Debug, Into)]
|
|
pub struct PyQuantizationSearchParams(QuantizationSearchParams);
|
|
|
|
#[pyclass_repr]
|
|
#[pymethods]
|
|
impl PyQuantizationSearchParams {
|
|
#[new]
|
|
#[pyo3(signature = (ignore = false, rescore = None, oversampling = None))]
|
|
pub fn new(ignore: bool, rescore: Option<bool>, oversampling: Option<f64>) -> Self {
|
|
Self(QuantizationSearchParams {
|
|
ignore,
|
|
rescore,
|
|
oversampling,
|
|
})
|
|
}
|
|
|
|
#[getter]
|
|
pub fn ignore(&self) -> bool {
|
|
self.0.ignore
|
|
}
|
|
|
|
#[getter]
|
|
pub fn rescore(&self) -> Option<bool> {
|
|
self.0.rescore
|
|
}
|
|
|
|
#[getter]
|
|
pub fn oversampling(&self) -> Option<f64> {
|
|
self.0.oversampling
|
|
}
|
|
|
|
pub fn __repr__(&self) -> String {
|
|
self.repr()
|
|
}
|
|
}
|
|
|
|
impl PyQuantizationSearchParams {
|
|
fn _getters(self) {
|
|
// Every field should have a getter method
|
|
let QuantizationSearchParams {
|
|
ignore: _,
|
|
rescore: _,
|
|
oversampling: _,
|
|
} = self.0;
|
|
}
|
|
}
|
|
|
|
#[pyclass(name = "AcornSearchParams", from_py_object)]
|
|
#[derive(Copy, Clone, Debug, Into)]
|
|
pub struct PyAcornSearchParams(AcornSearchParams);
|
|
|
|
#[pyclass_repr]
|
|
#[pymethods]
|
|
impl PyAcornSearchParams {
|
|
#[new]
|
|
#[pyo3(signature = (enable = false, max_selectivity = None))]
|
|
pub fn new(enable: bool, max_selectivity: Option<f64>) -> Self {
|
|
Self(AcornSearchParams {
|
|
enable,
|
|
max_selectivity: max_selectivity.map(OrderedFloat),
|
|
})
|
|
}
|
|
|
|
#[getter]
|
|
pub fn enable(&self) -> bool {
|
|
self.0.enable
|
|
}
|
|
|
|
#[getter]
|
|
pub fn max_selectivity(&self) -> Option<f64> {
|
|
self.0
|
|
.max_selectivity
|
|
.map(|selectivity| selectivity.into_inner())
|
|
}
|
|
|
|
pub fn __repr__(&self) -> String {
|
|
self.repr()
|
|
}
|
|
}
|
|
|
|
impl PyAcornSearchParams {
|
|
fn _getters(self) {
|
|
// Every field should have a getter method
|
|
let AcornSearchParams {
|
|
enable: _,
|
|
max_selectivity: _,
|
|
} = self.0;
|
|
}
|
|
}
|