upstream-archive/qdrant
1
0
Fork 0
mirror of https://github.com/qdrant/qdrant.git synced 2026-07-23 11:11:00 -05:00
Code Issues Packages Projects Releases Wiki Activity

filter text any (#7100)

Browse Source 
* internal implementation for filter any

* implement api for match `text_any`

* allow naive implementation of text match any

* dedup after kmerge + renames

* improve `expected_should_estimation`

* congruence test

* openapi test

* setup index in each test

---------

Co-authored-by: Luis Cossío <luis.cossio@outlook.com>
This commit is contained in:
Andrey Vasnetsov
2025-08-24 12:09:05 +02:00
committed by timvisee
parent f040579646
commit 0ce667eee5
18 changed files with 526 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
docs/grpc/docs.md
View File

@@ -3368,6 +3368,7 @@ Additionally, the first and last points of each GeoLineString must be the same.
| except_integers | [RepeatedIntegers](#qdrant-RepeatedIntegers) | | Match any other value except those integers |
| except_keywords | [RepeatedStrings](#qdrant-RepeatedStrings) | | Match any other value except those keywords |
| phrase | [string](#string) | | Match phrase text |
| text_any | [string](#string) | | Match any word in the text |

15
docs/redoc/master/openapi.json
View File

@@ -8625,6 +8625,9 @@
{
"$ref": "#/components/schemas/MatchText"
},
{
"$ref": "#/components/schemas/MatchTextAny"
},
{
"$ref": "#/components/schemas/MatchPhrase"
},
@@ -8674,6 +8677,18 @@
}
}
},
"MatchTextAny": {
"description": "Full-text match of at least one token of the string.",
"type": "object",
"required": [
"text_any"
],
"properties": {
"text_any": {
"type": "string"
}
}
},
"MatchPhrase": {
"description": "Full-text phrase match of the string.",
"type": "object",

6
lib/api/src/grpc/conversions.rs
View File

@@ -1895,6 +1895,9 @@ impl TryFrom<Match> for segment::types::Match {
MatchValue::ExceptKeywords(ints) => {
segment::types::Match::Except(ints.strings.into())
}
MatchValue::TextAny(text_any) => {
segment::types::Match::TextAny(segment::types::MatchTextAny { text_any })
}
}),
_ => Err(Status::invalid_argument("Malformed Match condition")),
}
@@ -1935,6 +1938,9 @@ impl From<segment::types::Match> for Match {
MatchValue::ExceptIntegers(RepeatedIntegers { integers })
}
},
segment::types::Match::TextAny(segment::types::MatchTextAny { text_any }) => {
MatchValue::TextAny(text_any)
}
};
Self {
match_value: Some(match_value),

1
lib/api/src/grpc/proto/points.proto
View File

@@ -1120,6 +1120,7 @@ message Match {
RepeatedIntegers except_integers = 7; // Match any other value except those integers
RepeatedStrings except_keywords = 8; // Match any other value except those keywords
string phrase = 9; // Match phrase text
string text_any = 10; // Match any word in the text
}
}

5
lib/api/src/grpc/qdrant.rs
View File

@@ -6699,7 +6699,7 @@ pub struct FieldCondition {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Match {
#[prost(oneof = "r#match::MatchValue", tags = "1, 2, 3, 4, 5, 6, 7, 8, 9")]
#[prost(oneof = "r#match::MatchValue", tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10")]
pub match_value: ::core::option::Option<r#match::MatchValue>,
}
/// Nested message and enum types in `Match`.
@@ -6735,6 +6735,9 @@ pub mod r#match {
/// Match phrase text
#[prost(string, tag = "9")]
Phrase(::prost::alloc::string::String),
/// Match any word in the text
#[prost(string, tag = "10")]
TextAny(::prost::alloc::string::String),
}
}
#[derive(serde::Serialize)]

1
lib/collection/src/problems/unindexed_field.rs
View File

@@ -232,6 +232,7 @@ fn infer_index_from_field_condition(field_condition: &FieldCondition) -> Vec<Fie
Match::Phrase(_match_text) => vec![FieldIndexType::TextPhrase],
Match::Any(match_any) => infer_index_from_any_variants(&match_any.any),
Match::Except(match_except) => infer_index_from_any_variants(&match_except.except),
Match::TextAny(_match_text_any) => vec![FieldIndexType::Text],
})
}
if let Some(range_interface) = range {

109
lib/segment/src/index/field_index/full_text_index/inverted_index/immutable_inverted_index.rs
View File

@@ -12,7 +12,9 @@ use super::mmap_inverted_index::MmapInvertedIndex;
use super::mmap_inverted_index::mmap_postings_enum::MmapPostingsEnum;
use super::mutable_inverted_index::MutableInvertedIndex;
use super::positions::Positions;
use super::postings_iterator::intersect_compressed_postings_iterator;
use super::postings_iterator::{
intersect_compressed_postings_iterator, merge_compressed_postings_iterator,
};
use super::{Document, InvertedIndex, ParsedQuery, TokenId, TokenSet};
use crate::common::operation_error::{OperationError, OperationResult};
use crate::index::field_index::full_text_index::inverted_index::postings_iterator::{
@@ -48,7 +50,7 @@ impl ImmutableInvertedIndex {
}
/// Iterate over point ids whose documents contain all given tokens
fn filter_has_subset<'a>(
fn filter_has_all<'a>(
&'a self,
tokens: TokenSet,
) -> impl Iterator<Item = PointOffsetType> + 'a {
@@ -93,6 +95,45 @@ impl ImmutableInvertedIndex {
}
}
/// Iterate over point ids whose documents contain at least one of the given tokens
fn filter_has_any<'a>(
&'a self,
tokens: TokenSet,
) -> impl Iterator<Item = PointOffsetType> + 'a {
// in case of immutable index, deleted documents are still in the postings
let is_active = move |idx| {
self.point_to_tokens_count
.get(idx as usize)
.is_some_and(|x| *x > 0)
};
fn merge<'a, V: PostingValue>(
postings: &'a [PostingList<V>],
tokens: TokenSet,
is_active: impl Fn(PointOffsetType) -> bool + 'a,
) -> impl Iterator<Item = PointOffsetType> + 'a {
let postings: Vec<_> = tokens
.tokens()
.iter()
.filter_map(|&token_id| postings.get(token_id as usize).map(PostingList::view))
.collect();
// Query must not be empty
if postings.is_empty() {
return Either::Left(std::iter::empty());
};
Either::Right(merge_compressed_postings_iterator(postings, is_active))
}
match &self.postings {
ImmutablePostings::Ids(postings) => Either::Left(merge(postings, tokens, is_active)),
ImmutablePostings::WithPositions(postings) => {
Either::Right(merge(postings, tokens, is_active))
}
}
}
fn check_has_subset(&self, tokens: &TokenSet, point_id: PointOffsetType) -> bool {
if tokens.is_empty() {
return false;
@@ -123,6 +164,34 @@ impl ImmutableInvertedIndex {
}
}
fn check_has_any(&self, tokens: &TokenSet, point_id: PointOffsetType) -> bool {
if tokens.is_empty() {
return false;
}
// check presence of the document
if self.values_is_empty(point_id) {
return false;
}
fn check_any<V: PostingValue>(
postings: &[PostingList<V>],
tokens: &TokenSet,
point_id: PointOffsetType,
) -> bool {
// Check that at least one token is in document
tokens.tokens().iter().any(|token_id| {
let posting_list = &postings[*token_id as usize];
posting_list.visitor().contains(point_id)
})
}
match &self.postings {
ImmutablePostings::Ids(postings) => check_any(postings, tokens, point_id),
ImmutablePostings::WithPositions(postings) => check_any(postings, tokens, point_id),
}
}
/// Iterate over point ids whose documents contain all given tokens in the same order they are provided
pub fn filter_has_phrase<'a>(
&'a self,
@@ -213,8 +282,9 @@ impl InvertedIndex for ImmutableInvertedIndex {
_hw_counter: &'a HardwareCounterCell,
) -> Box<dyn Iterator<Item = PointOffsetType> + 'a> {
match query {
ParsedQuery::Tokens(tokens) => Box::new(self.filter_has_subset(tokens)),
ParsedQuery::AllTokens(tokens) => Box::new(self.filter_has_all(tokens)),
ParsedQuery::Phrase(tokens) => Box::new(self.filter_has_phrase(tokens)),
ParsedQuery::AnyTokens(tokens) => Box::new(self.filter_has_any(tokens)),
}
}
@@ -232,8 +302,9 @@ impl InvertedIndex for ImmutableInvertedIndex {
fn check_match(&self, parsed_query: &ParsedQuery, point_id: PointOffsetType) -> bool {
match parsed_query {
ParsedQuery::Tokens(tokens) => self.check_has_subset(tokens, point_id),
ParsedQuery::AllTokens(tokens) => self.check_has_subset(tokens, point_id),
ParsedQuery::Phrase(phrase) => self.check_has_phrase(phrase, point_id),
ParsedQuery::AnyTokens(tokens) => self.check_has_any(tokens, point_id),
}
}
@@ -376,21 +447,21 @@ fn create_compressed_postings_with_positions(
.collect::<Vec<_>>();
(0u32..)
.zip(postings)
.map(|(token, posting)| {
posting
.iter()
.map(|id| {
let positions = point_to_tokens_positions[id as usize]
.remove(&token)
.expect(
"If id is this token's posting list, it should have at least one position",
);
(id, positions)
})
.collect()
})
.collect()
.zip(postings)
.map(|(token, posting)| {
posting
.iter()
.map(|id| {
let positions = point_to_tokens_positions[id as usize]
.remove(&token)
.expect(
"If id is this token's posting list, it should have at least one position",
);
(id, positions)
})
.collect()
})
.collect()
}
impl From<&MmapInvertedIndex> for ImmutableInvertedIndex {

75
lib/segment/src/index/field_index/full_text_index/inverted_index/mmap_inverted_index/mod.rs
View File

@@ -16,7 +16,9 @@ use super::immutable_inverted_index::ImmutableInvertedIndex;
use super::immutable_postings_enum::ImmutablePostings;
use super::mmap_inverted_index::mmap_postings_enum::MmapPostingsEnum;
use super::positions::Positions;
use super::postings_iterator::intersect_compressed_postings_iterator;
use super::postings_iterator::{
intersect_compressed_postings_iterator, merge_compressed_postings_iterator,
};
use super::{InvertedIndex, ParsedQuery, TokenId, TokenSet};
use crate::common::Flusher;
use crate::common::mmap_bitslice_buffered_update_wrapper::MmapBitSliceBufferedUpdateWrapper;
@@ -169,7 +171,7 @@ impl MmapInvertedIndex {
}
/// Iterate over point ids whose documents contain all given tokens
pub fn filter_has_subset<'a>(
pub fn filter_has_all<'a>(
&'a self,
tokens: TokenSet,
) -> Box<dyn Iterator<Item = PointOffsetType> + 'a> {
@@ -209,6 +211,41 @@ impl MmapInvertedIndex {
}
}
/// Iterate over point ids whose documents contain at least one of the given tokens
fn filter_has_any<'a>(
&'a self,
tokens: TokenSet,
) -> impl Iterator<Item = PointOffsetType> + 'a {
// in case of immutable index, deleted documents are still in the postings
let is_active = move |idx| self.is_active(idx);
fn merge<'a, V: MmapPostingValue>(
postings: &'a MmapPostings<V>,
tokens: TokenSet,
is_active: impl Fn(PointOffsetType) -> bool + 'a,
) -> impl Iterator<Item = PointOffsetType> + 'a {
let postings: Vec<_> = tokens
.tokens()
.iter()
.filter_map(|&token_id| postings.get(token_id))
.collect();
// Query must not be empty
if postings.is_empty() {
return Either::Left(std::iter::empty());
};
Either::Right(merge_compressed_postings_iterator(postings, is_active))
}
match &self.storage.postings {
MmapPostingsEnum::Ids(postings) => Either::Left(merge(postings, tokens, is_active)),
MmapPostingsEnum::WithPositions(postings) => {
Either::Right(merge(postings, tokens, is_active))
}
}
}
fn check_has_subset(&self, tokens: &TokenSet, point_id: PointOffsetType) -> bool {
// check non-empty query
if tokens.is_empty() {
@@ -244,6 +281,34 @@ impl MmapInvertedIndex {
}
}
fn check_has_any(&self, tokens: &TokenSet, point_id: PointOffsetType) -> bool {
if tokens.is_empty() {
return false;
}
// check presence of the document
if self.values_is_empty(point_id) {
return false;
}
fn check_any<V: MmapPostingValue>(
postings: &MmapPostings<V>,
tokens: &TokenSet,
point_id: PointOffsetType,
) -> bool {
// Check that at least one token is in document
tokens.tokens().iter().any(|token_id| {
let posting_list = postings.get(*token_id).unwrap();
posting_list.visitor().contains(point_id)
})
}
match &self.storage.postings {
MmapPostingsEnum::Ids(postings) => check_any(postings, tokens, point_id),
MmapPostingsEnum::WithPositions(postings) => check_any(postings, tokens, point_id),
}
}
/// Iterate over point ids whose documents contain all given tokens in the same order they are provided
pub fn filter_has_phrase<'a>(
&'a self,
@@ -381,8 +446,9 @@ impl InvertedIndex for MmapInvertedIndex {
_hw_counter: &HardwareCounterCell,
) -> Box<dyn Iterator<Item = PointOffsetType> + 'a> {
match query {
ParsedQuery::Tokens(tokens) => self.filter_has_subset(tokens),
ParsedQuery::AllTokens(tokens) => self.filter_has_all(tokens),
ParsedQuery::Phrase(phrase) => Box::new(self.filter_has_phrase(phrase)),
ParsedQuery::AnyTokens(tokens) => Box::new(self.filter_has_any(tokens)),
}
}
@@ -405,8 +471,9 @@ impl InvertedIndex for MmapInvertedIndex {
fn check_match(&self, parsed_query: &ParsedQuery, point_id: PointOffsetType) -> bool {
match parsed_query {
ParsedQuery::Tokens(tokens) => self.check_has_subset(tokens, point_id),
ParsedQuery::AllTokens(tokens) => self.check_has_subset(tokens, point_id),
ParsedQuery::Phrase(phrase) => self.check_has_phrase(phrase, point_id),
ParsedQuery::AnyTokens(tokens) => self.check_has_any(tokens, point_id),
}
}

105
lib/segment/src/index/field_index/full_text_index/inverted_index/mod.rs
View File

@@ -7,6 +7,7 @@ mod positions;
mod posting_list;
mod postings_iterator;
use std::cmp::min;
use std::collections::HashMap;
use ahash::AHashSet;
@@ -16,6 +17,7 @@ use itertools::Itertools;
use crate::common::operation_error::OperationResult;
use crate::index::field_index::{CardinalityEstimation, PayloadBlockCondition, PrimaryCondition};
use crate::index::query_estimator::expected_should_estimation;
use crate::types::{FieldCondition, Match, PayloadKeyType};
pub type TokenId = u32;
@@ -56,6 +58,15 @@ impl TokenSet {
}
subset.0.iter().all(|token| self.contains(token))
}
/// Checks if the current set contains any of the given tokens.
/// Returns false if the subset is empty
pub fn has_any(&self, subset: &TokenSet) -> bool {
if subset.is_empty() {
return false;
}
subset.0.iter().any(|token| self.contains(token))
}
}
impl From<AHashSet<TokenId>> for TokenSet {
@@ -143,7 +154,10 @@ pub enum ParsedQuery {
/// All these tokens must be present in the document, regardless of order.
///
/// In other words this should be a subset of the document's token set.
Tokens(TokenSet),
AllTokens(TokenSet),
/// At least one of these tokens must be present in the document.
AnyTokens(TokenSet),
/// All these tokens must be present in the document, in the same order as this query.
Phrase(Document),
@@ -212,12 +226,15 @@ pub trait InvertedIndex {
hw_counter: &HardwareCounterCell,
) -> CardinalityEstimation {
match query {
ParsedQuery::Tokens(tokens) => {
ParsedQuery::AllTokens(tokens) => {
self.estimate_has_subset_cardinality(tokens, condition, hw_counter)
}
ParsedQuery::Phrase(phrase) => {
self.estimate_has_phrase_cardinality(phrase, condition, hw_counter)
}
ParsedQuery::AnyTokens(tokens) => {
self.estimate_has_any_cardinality(tokens, condition, hw_counter)
}
}
}
@@ -266,6 +283,46 @@ pub trait InvertedIndex {
}
}
fn estimate_has_any_cardinality(
&self,
tokens: &TokenSet,
condition: &FieldCondition,
hw_counter: &HardwareCounterCell,
) -> CardinalityEstimation {
let points_count = self.points_count();
let posting_lengths: Vec<_> = tokens
.tokens()
.iter()
.filter_map(|&vocab_idx| self.get_posting_len(vocab_idx, hw_counter))
.collect();
if posting_lengths.is_empty() {
// Empty request -> no matches
return CardinalityEstimation::exact(0)
.with_primary_clause(PrimaryCondition::Condition(Box::new(condition.clone())));
}
// At least one posting is the largest possible cardinality
let largest_posting = posting_lengths.iter().max().copied().unwrap();
if posting_lengths.len() == 1 {
return CardinalityEstimation::exact(largest_posting)
.with_primary_clause(PrimaryCondition::Condition(Box::new(condition.clone())));
}
let sum: usize = posting_lengths.iter().sum();
let exp = expected_should_estimation(posting_lengths.into_iter(), points_count);
CardinalityEstimation {
primary_clauses: vec![PrimaryCondition::Condition(Box::new(condition.clone()))],
min: largest_posting,
exp,
max: min(sum, points_count),
}
}
fn estimate_has_phrase_cardinality(
&self,
phrase: &Document,
@@ -367,7 +424,18 @@ mod tests {
.into_iter()
.map(token_to_id)
.collect::<Option<TokenSet>>()?;
Some(ParsedQuery::Tokens(tokens))
Some(ParsedQuery::AllTokens(tokens))
}
fn to_parsed_query_any(
query: Vec<String>,
token_to_id: impl Fn(String) -> Option<TokenId>,
) -> Option<ParsedQuery> {
let tokens = query
.into_iter()
.map(token_to_id)
.collect::<Option<TokenSet>>()?;
Some(ParsedQuery::AnyTokens(tokens))
}
fn mutable_inverted_index(
@@ -536,21 +604,38 @@ mod tests {
let mut_parsed_queries: Vec<_> = queries
.iter()
.cloned()
.map(|query| to_parsed_query(query, |token| mut_index.vocab.get(&token).copied()))
.flat_map(|query| {
vec![
to_parsed_query(query.clone(), |token| mut_index.vocab.get(&token).copied()),
to_parsed_query_any(query, |token| mut_index.vocab.get(&token).copied()),
]
})
.collect();
let mmap_parsed_queries: Vec<_> = queries
.iter()
.cloned()
.map(|query| {
to_parsed_query(query, |token| mmap_index.get_token_id(&token, &hw_counter))
.flat_map(|query| {
vec![
to_parsed_query(query.clone(), |token| {
mmap_index.get_token_id(&token, &hw_counter)
}),
to_parsed_query_any(query, |token| {
mmap_index.get_token_id(&token, &hw_counter)
}),
]
})
.collect();
let imm_mmap_parsed_queries: Vec<_> = queries
.into_iter()
.map(|query| {
to_parsed_query(query, |token| {
imm_mmap_index.get_token_id(&token, &hw_counter)
})
.flat_map(|query| {
vec![
to_parsed_query(query.clone(), |token| {
imm_mmap_index.get_token_id(&token, &hw_counter)
}),
to_parsed_query_any(query, |token| {
imm_mmap_index.get_token_id(&token, &hw_counter)
}),
]
})
.collect();

38
lib/segment/src/index/field_index/full_text_index/inverted_index/mutable_inverted_index.rs
View File

@@ -5,7 +5,7 @@ use common::types::PointOffsetType;
use itertools::Either;
use super::posting_list::PostingList;
use super::postings_iterator::intersect_postings_iterator;
use super::postings_iterator::{intersect_postings_iterator, merge_postings_iterator};
use super::{Document, InvertedIndex, ParsedQuery, TokenId, TokenSet};
use crate::common::operation_error::OperationResult;
@@ -55,7 +55,7 @@ impl MutableInvertedIndex {
}
/// Iterate over point ids whose documents contain all given tokens
fn filter_has_subset(&self, tokens: TokenSet) -> impl Iterator<Item = PointOffsetType> + '_ {
fn filter_has_all(&self, tokens: TokenSet) -> impl Iterator<Item = PointOffsetType> + '_ {
let postings_opt: Option<Vec<_>> = tokens
.tokens()
.iter()
@@ -79,6 +79,25 @@ impl MutableInvertedIndex {
Either::Right(intersect_postings_iterator(postings))
}
fn filter_has_any(&self, tokens: TokenSet) -> impl Iterator<Item = PointOffsetType> + '_ {
let postings_opt: Vec<_> = tokens
.tokens()
.iter()
.filter_map(|&token_id| {
// if a ParsedQuery token was given an index, then it must exist in the vocabulary
// dictionary. Posting list entry can be None but it exists.
self.postings.get(token_id as usize)
})
.collect();
if postings_opt.is_empty() {
// Empty request -> no matches
return Either::Left(std::iter::empty());
}
Either::Right(merge_postings_iterator(postings_opt))
}
pub fn filter_has_phrase(
&self,
phrase: Document,
@@ -89,7 +108,7 @@ impl MutableInvertedIndex {
};
let iter = self
.filter_has_subset(phrase.to_token_set())
.filter_has_all(phrase.to_token_set())
.filter(move |id| {
let doc = point_to_doc[*id as usize]
.as_ref()
@@ -209,8 +228,9 @@ impl InvertedIndex for MutableInvertedIndex {
_hw_counter: &HardwareCounterCell,
) -> Box<dyn Iterator<Item = PointOffsetType> + '_> {
match query {
ParsedQuery::Tokens(tokens) => Box::new(self.filter_has_subset(tokens)),
ParsedQuery::AllTokens(tokens) => Box::new(self.filter_has_all(tokens)),
ParsedQuery::Phrase(phrase) => self.filter_has_phrase(phrase),
ParsedQuery::AnyTokens(tokens) => Box::new(self.filter_has_any(tokens)),
}
}
@@ -228,7 +248,7 @@ impl InvertedIndex for MutableInvertedIndex {
fn check_match(&self, parsed_query: &ParsedQuery, point_id: PointOffsetType) -> bool {
match parsed_query {
ParsedQuery::Tokens(query) => {
ParsedQuery::AllTokens(query) => {
let Some(doc) = self.get_tokens(point_id) else {
return false;
};
@@ -244,6 +264,14 @@ impl InvertedIndex for MutableInvertedIndex {
// Check that all tokens are in document, in order
doc.has_phrase(document)
}
ParsedQuery::AnyTokens(query) => {
let Some(doc) = self.get_tokens(point_id) else {
return false;
};
// Check that at least one token is in document
doc.has_any(query)
}
}
}

28
lib/segment/src/index/field_index/full_text_index/inverted_index/postings_iterator.rs
View File

@@ -1,5 +1,5 @@
use common::types::PointOffsetType;
use itertools::Either;
use itertools::{Either, Itertools};
use posting_list::{PostingIterator, PostingListView, PostingValue};
use super::posting_list::PostingList;
@@ -24,6 +24,16 @@ pub fn intersect_postings_iterator<'a>(
.filter(move |doc_id| postings.iter().all(|posting| posting.contains(*doc_id)))
}
pub fn merge_postings_iterator<'a>(
postings: Vec<&'a PostingList>,
) -> impl Iterator<Item = PointOffsetType> + 'a {
postings
.into_iter()
.map(PostingList::iter)
.kmerge_by(|a, b| a < b)
.dedup()
}
pub fn intersect_compressed_postings_iterator<'a, V: PostingValue + 'a>(
mut postings: Vec<PostingListView<'a, V>>,
is_active: impl Fn(PointOffsetType) -> bool + 'a,
@@ -52,12 +62,28 @@ pub fn intersect_compressed_postings_iterator<'a, V: PostingValue + 'a>(
//
// This means that the other iterators can remember the last id they returned to avoid extra work
posting_iterator
// potential optimization: Make posting iterator of just ids, without values (a.k.a. positions).
// We are discarding them here, thus unnecessarily reading them from the tails of the posting lists.
.advance_until_greater_or_equal(*id)
.is_some_and(|elem| elem.id == *id)
})
})
}
pub fn merge_compressed_postings_iterator<'a, V: PostingValue + 'a>(
postings: Vec<PostingListView<'a, V>>,
is_active: impl Fn(PointOffsetType) -> bool + 'a,
) -> impl Iterator<Item = PointOffsetType> + 'a {
postings
.into_iter()
// potential optimization: Make posting iterator of just ids, without values (a.k.a. positions).
// We are discarding them here, thus unnecessarily reading them from the tails of the posting lists.
.map(|view| view.into_iter().map(|elem| elem.id))
.kmerge_by(|a, b| a < b)
.dedup()
.filter(move |id| is_active(*id))
}
/// Returns an iterator over the points that match the given phrase query.
pub fn intersect_compressed_postings_phrase_iterator<'a>(
phrase: Document,

2
lib/segment/src/index/field_index/full_text_index/tests/test_congruence.rs
View File

@@ -203,7 +203,7 @@ pub fn to_parsed_query(
let tokens = query.iter().map(|token| token_to_id(token.as_str()));
let parsed = match is_phrase {
false => ParsedQuery::Tokens(tokens.collect::<Option<TokenSet>>()?),
false => ParsedQuery::AllTokens(tokens.collect::<Option<TokenSet>>()?),
true => ParsedQuery::Phrase(tokens.collect::<Option<Document>>()?),
};

23
lib/segment/src/index/field_index/full_text_index/text_index.rs
View File

@@ -303,7 +303,22 @@ impl FullTextIndex {
tokens.insert(self.get_token(token.as_ref(), hw_counter));
});
let tokens = tokens.into_iter().collect::<Option<TokenSet>>()?;
Some(ParsedQuery::Tokens(tokens))
Some(ParsedQuery::AllTokens(tokens))
}
pub fn parse_text_any_query(
&self,
text: &str,
hw_counter: &HardwareCounterCell,
) -> Option<ParsedQuery> {
let mut tokens = AHashSet::new();
self.get_tokenizer().tokenize_query(text, |token| {
if let Some(token_id) = self.get_token(token.as_ref(), hw_counter) {
tokens.insert(token_id);
}
});
let tokens = tokens.into_iter().collect::<TokenSet>();
Some(ParsedQuery::AnyTokens(tokens))
}
pub fn parse_tokenset(&self, text: &str, hw_counter: &HardwareCounterCell) -> TokenSet {
@@ -358,7 +373,7 @@ impl FullTextIndex {
FullTextIndex::get_values(payload_value)
.iter()
.any(|value| match &query {
ParsedQuery::Tokens(query) => {
ParsedQuery::AllTokens(query) => {
let tokenset = self.parse_tokenset(value, hw_counter);
tokenset.has_subset(query)
}
@@ -366,6 +381,10 @@ impl FullTextIndex {
let document = self.parse_document(value, hw_counter);
document.has_phrase(query)
}
ParsedQuery::AnyTokens(query) => {
let tokenset = self.parse_tokenset(value, hw_counter);
tokenset.has_any(query)
}
})
}

29
lib/segment/src/index/query_estimator.rs
View File

@@ -64,6 +64,28 @@ pub fn adjust_to_available_vectors(
}
}
/// Combine cardinality of multiple estimations in an OR fashion by using the complement rule.
/// Assumes that the estimations are independent.
///
/// Formula is `(1 - ∏(1-pᵢ)) * total`:
/// * For each condition, it calculates the probability that an item does not match it: `1 - (x / total)`.
/// * It multiplies these probabilities to get the probability that an item matches none of the conditions.
/// * Subtracts this from 1 to get the probability that an item matches at least one condition.
/// * Multiplies this probability by the total number of items and rounds to get the expected count.
pub fn expected_should_estimation(estimations: impl Iterator<Item = usize>, total: usize) -> usize {
if total == 0 {
return 0;
}
let element_not_hit_prob: f64 = estimations
.map(|x| 1.0 - (x as f64 / total as f64))
.product();
let element_hit_prob = 1.0 - element_not_hit_prob;
(element_hit_prob * (total as f64)).round() as usize
}
pub fn combine_should_estimations(
estimations: &[CardinalityEstimation],
total: usize,
@@ -78,12 +100,7 @@ pub fn combine_should_estimations(
}
clauses.append(&mut estimation.primary_clauses.clone());
}
let element_not_hit_prob: f64 = estimations
.iter()
.map(|x| (total - x.exp) as f64 / (total as f64))
.product();
let element_hit_prob = 1.0 - element_not_hit_prob;
let expected_count = (element_hit_prob * (total as f64)).round() as usize;
let expected_count = expected_should_estimation(estimations.iter().map(|x| x.exp), total);
CardinalityEstimation {
primary_clauses: clauses,
min: estimations.iter().map(|x| x.min).max().unwrap_or(0),

29
lib/segment/src/index/query_optimization/condition_converter/match_converter.rs
View File

@@ -7,7 +7,8 @@ use crate::index::field_index::FieldIndex;
use crate::index::query_optimization::optimized_filter::ConditionCheckerFn;
use crate::payload_storage::condition_checker::INDEXSET_ITER_THRESHOLD;
use crate::types::{
AnyVariants, Match, MatchAny, MatchExcept, MatchPhrase, MatchText, MatchValue, ValueVariants,
AnyVariants, Match, MatchAny, MatchExcept, MatchPhrase, MatchText, MatchTextAny, MatchValue,
ValueVariants,
};
pub fn get_match_checkers(
@@ -17,9 +18,14 @@ pub fn get_match_checkers(
) -> Option<ConditionCheckerFn<'_>> {
match cond_match {
Match::Value(MatchValue { value }) => get_match_value_checker(value, index, hw_acc),
Match::Text(MatchText { text }) => get_match_text_checker::<false>(text, index, hw_acc),
Match::Text(MatchText { text }) => {
get_match_text_checker(text, TextQueryType::Text, index, hw_acc)
}
Match::TextAny(MatchTextAny { text_any }) => {
get_match_text_checker(text_any, TextQueryType::TextAny, index, hw_acc)
}
Match::Phrase(MatchPhrase { phrase }) => {
get_match_text_checker::<true>(phrase, index, hw_acc)
get_match_text_checker(phrase, TextQueryType::Phrase, index, hw_acc)
}
Match::Any(MatchAny { any }) => get_match_any_checker(any, index, hw_acc),
Match::Except(MatchExcept { except }) => get_match_except_checker(except, index, hw_acc),
@@ -249,18 +255,25 @@ fn get_match_except_checker(
checker
}
fn get_match_text_checker<const IS_PHRASE: bool>(
enum TextQueryType {
Phrase,
Text,
TextAny,
}
fn get_match_text_checker(
text: String,
query_type: TextQueryType,
index: &FieldIndex,
hw_acc: HwMeasurementAcc,
) -> Option<ConditionCheckerFn<'_>> {
let hw_counter = hw_acc.get_counter_cell();
match index {
FieldIndex::FullTextIndex(full_text_index) => {
let query_opt = if IS_PHRASE {
full_text_index.parse_phrase_query(&text, &hw_counter)
} else {
full_text_index.parse_text_query(&text, &hw_counter)
let query_opt = match query_type {
TextQueryType::Phrase => full_text_index.parse_phrase_query(&text, &hw_counter),
TextQueryType::Text => full_text_index.parse_text_query(&text, &hw_counter),
TextQueryType::TextAny => full_text_index.parse_text_any_query(&text, &hw_counter),
};
let Some(parsed_query) = query_opt else {

10
lib/segment/src/payload_storage/condition_checker.rs
View File

@@ -6,8 +6,8 @@ use serde_json::Value;
use crate::types::{
AnyVariants, DateTimePayloadType, FieldCondition, FloatPayloadType, GeoBoundingBox, GeoPoint,
GeoPolygon, GeoRadius, Match, MatchAny, MatchExcept, MatchPhrase, MatchText, MatchValue, Range,
RangeInterface, ValueVariants, ValuesCount,
GeoPolygon, GeoRadius, Match, MatchAny, MatchExcept, MatchPhrase, MatchText, MatchTextAny,
MatchValue, Range, RangeInterface, ValueVariants, ValuesCount,
};
/// Threshold representing the point to which iterating through an IndexSet is more efficient than using hashing.
@@ -165,6 +165,12 @@ impl ValueChecker for Match {
_ => false,
}
}
Match::TextAny(MatchTextAny { text_any }) => match payload {
Value::String(stored) => text_any
.split_whitespace()
.any(|token| stored.contains(token)),
_ => false,
},
Match::Any(MatchAny { any }) => match (payload, any) {
(Value::String(stored), AnyVariants::Strings(list)) => {
if list.len() < INDEXSET_ITER_THRESHOLD {

13
lib/segment/src/types.rs
View File

@@ -2166,6 +2166,13 @@ pub struct MatchText {
pub text: String,
}
/// Full-text match of at least one token of the string.
#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub struct MatchTextAny {
pub text_any: String,
}
impl<S: Into<String>> From<S> for MatchText {
fn from(text: S) -> Self {
MatchText { text: text.into() }
@@ -2207,6 +2214,7 @@ pub struct MatchExcept {
pub enum MatchInterface {
Value(MatchValue),
Text(MatchText),
TextAny(MatchTextAny),
Phrase(MatchPhrase),
Any(MatchAny),
Except(MatchExcept),
@@ -2218,6 +2226,7 @@ pub enum MatchInterface {
pub enum Match {
Value(MatchValue),
Text(MatchText),
TextAny(MatchTextAny),
Phrase(MatchPhrase),
Any(MatchAny),
Except(MatchExcept),
@@ -2258,6 +2267,9 @@ impl From<MatchInterface> for Match {
match value {
MatchInterface::Value(value) => Self::Value(MatchValue { value: value.value }),
MatchInterface::Text(text) => Self::Text(MatchText { text: text.text }),
MatchInterface::TextAny(text_any) => Self::TextAny(MatchTextAny {
text_any: text_any.text_any,
}),
MatchInterface::Any(any) => Self::Any(MatchAny { any: any.any }),
MatchInterface::Except(except) => Self::Except(MatchExcept {
except: except.except,
@@ -2774,6 +2786,7 @@ impl FieldCondition {
Match::Value(_) => 0,
Match::Text(_) => 0,
Match::Phrase(_) => 0,
Match::TextAny(_) => 0,
}
}
}

110
tests/openapi/test_fts.py
View File

@@ -175,21 +175,6 @@ def basic_collection_setup(
)
assert response.ok
# Create index
response = request_with_validation(
api='/collections/{collection_name}/index',
method="PUT",
path_params={'collection_name': collection_name},
query_params={'wait': 'true'},
body={
"field_name": "title",
"field_schema": {
"type": "text",
"tokenizer": "prefix",
}
}
)
assert response.ok
response = request_with_validation(
api='/collections/{collection_name}',
@@ -222,8 +207,103 @@ def setup(on_disk_vectors, on_disk_payload, collection_name):
yield
drop_collection(collection_name=collection_name)
def test_match_any(collection_name):
# Create index
response = request_with_validation(
api='/collections/{collection_name}/index',
method="PUT",
path_params={'collection_name': collection_name},
query_params={'wait': 'true'},
body={
"field_name": "title",
"field_schema": {
"type": "text",
"tokenizer": "word",
"lowercase": True,
"stopwords": "english"
}
}
)
assert response.ok
response_any = request_with_validation(
api='/collections/{collection_name}/points/scroll',
method="POST",
path_params={'collection_name': collection_name},
body={
"offset": None,
"limit": 10,
"with_payload": True,
"with_vector": False,
"filter": {
"must": {
"key": "title",
"match": {
"text_any": "robot of the star",
}
}
}
}
)
assert response_any.ok
response_any_ids = {point['id'] for point in response_any.json()['result']['points']}
# Same request, but manually split into words
response_split = request_with_validation(
api='/collections/{collection_name}/points/scroll',
method="POST",
path_params={'collection_name': collection_name},
body={
"offset": None,
"limit": 10,
"with_payload": True,
"with_vector": False,
"filter": {
"should": [
{
"key": "title",
"match": {
"text": "robot",
}
},
{
"key": "title",
"match": {
"text": "star",
}
}
]
}
}
)
assert response_split.ok
response_split_ids = {point['id'] for point in response_split.json()['result']['points']}
assert response_any_ids == response_split_ids
def test_scroll_with_prefix(collection_name):
# Create index
response = request_with_validation(
api='/collections/{collection_name}/index',
method="PUT",
path_params={'collection_name': collection_name},
query_params={'wait': 'true'},
body={
"field_name": "title",
"field_schema": {
"type": "text",
"tokenizer": "prefix",
}
}
)
assert response.ok
response = request_with_validation(
api='/collections/{collection_name}/points/scroll',
method="POST",