diff --git a/docs/grpc/docs.md b/docs/grpc/docs.md index 5df0ae1e78..4ce006bfeb 100644 --- a/docs/grpc/docs.md +++ b/docs/grpc/docs.md @@ -3368,6 +3368,7 @@ Additionally, the first and last points of each GeoLineString must be the same. | except_integers | [RepeatedIntegers](#qdrant-RepeatedIntegers) | | Match any other value except those integers | | except_keywords | [RepeatedStrings](#qdrant-RepeatedStrings) | | Match any other value except those keywords | | phrase | [string](#string) | | Match phrase text | +| text_any | [string](#string) | | Match any word in the text | diff --git a/docs/redoc/master/openapi.json b/docs/redoc/master/openapi.json index e5f11a30c1..7c681c25c0 100644 --- a/docs/redoc/master/openapi.json +++ b/docs/redoc/master/openapi.json @@ -8625,6 +8625,9 @@ { "$ref": "#/components/schemas/MatchText" }, + { + "$ref": "#/components/schemas/MatchTextAny" + }, { "$ref": "#/components/schemas/MatchPhrase" }, @@ -8674,6 +8677,18 @@ } } }, + "MatchTextAny": { + "description": "Full-text match of at least one token of the string.", + "type": "object", + "required": [ + "text_any" + ], + "properties": { + "text_any": { + "type": "string" + } + } + }, "MatchPhrase": { "description": "Full-text phrase match of the string.", "type": "object", diff --git a/lib/api/src/grpc/conversions.rs b/lib/api/src/grpc/conversions.rs index d75c9f7fb1..309b49e3bf 100644 --- a/lib/api/src/grpc/conversions.rs +++ b/lib/api/src/grpc/conversions.rs @@ -1895,6 +1895,9 @@ impl TryFrom for segment::types::Match { MatchValue::ExceptKeywords(ints) => { segment::types::Match::Except(ints.strings.into()) } + MatchValue::TextAny(text_any) => { + segment::types::Match::TextAny(segment::types::MatchTextAny { text_any }) + } }), _ => Err(Status::invalid_argument("Malformed Match condition")), } @@ -1935,6 +1938,9 @@ impl From for Match { MatchValue::ExceptIntegers(RepeatedIntegers { integers }) } }, + segment::types::Match::TextAny(segment::types::MatchTextAny { text_any }) => { + MatchValue::TextAny(text_any) + } }; Self { match_value: Some(match_value), diff --git a/lib/api/src/grpc/proto/points.proto b/lib/api/src/grpc/proto/points.proto index 576e357322..a4bb1689b5 100644 --- a/lib/api/src/grpc/proto/points.proto +++ b/lib/api/src/grpc/proto/points.proto @@ -1120,6 +1120,7 @@ message Match { RepeatedIntegers except_integers = 7; // Match any other value except those integers RepeatedStrings except_keywords = 8; // Match any other value except those keywords string phrase = 9; // Match phrase text + string text_any = 10; // Match any word in the text } } diff --git a/lib/api/src/grpc/qdrant.rs b/lib/api/src/grpc/qdrant.rs index 2995693077..32d5cb62bb 100644 --- a/lib/api/src/grpc/qdrant.rs +++ b/lib/api/src/grpc/qdrant.rs @@ -6699,7 +6699,7 @@ pub struct FieldCondition { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Match { - #[prost(oneof = "r#match::MatchValue", tags = "1, 2, 3, 4, 5, 6, 7, 8, 9")] + #[prost(oneof = "r#match::MatchValue", tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10")] pub match_value: ::core::option::Option, } /// Nested message and enum types in `Match`. @@ -6735,6 +6735,9 @@ pub mod r#match { /// Match phrase text #[prost(string, tag = "9")] Phrase(::prost::alloc::string::String), + /// Match any word in the text + #[prost(string, tag = "10")] + TextAny(::prost::alloc::string::String), } } #[derive(serde::Serialize)] diff --git a/lib/collection/src/problems/unindexed_field.rs b/lib/collection/src/problems/unindexed_field.rs index 7b96bd9035..2518774a56 100644 --- a/lib/collection/src/problems/unindexed_field.rs +++ b/lib/collection/src/problems/unindexed_field.rs @@ -232,6 +232,7 @@ fn infer_index_from_field_condition(field_condition: &FieldCondition) -> Vec vec![FieldIndexType::TextPhrase], Match::Any(match_any) => infer_index_from_any_variants(&match_any.any), Match::Except(match_except) => infer_index_from_any_variants(&match_except.except), + Match::TextAny(_match_text_any) => vec![FieldIndexType::Text], }) } if let Some(range_interface) = range { diff --git a/lib/segment/src/index/field_index/full_text_index/inverted_index/immutable_inverted_index.rs b/lib/segment/src/index/field_index/full_text_index/inverted_index/immutable_inverted_index.rs index a1333f843e..8a5c61c4a7 100644 --- a/lib/segment/src/index/field_index/full_text_index/inverted_index/immutable_inverted_index.rs +++ b/lib/segment/src/index/field_index/full_text_index/inverted_index/immutable_inverted_index.rs @@ -12,7 +12,9 @@ use super::mmap_inverted_index::MmapInvertedIndex; use super::mmap_inverted_index::mmap_postings_enum::MmapPostingsEnum; use super::mutable_inverted_index::MutableInvertedIndex; use super::positions::Positions; -use super::postings_iterator::intersect_compressed_postings_iterator; +use super::postings_iterator::{ + intersect_compressed_postings_iterator, merge_compressed_postings_iterator, +}; use super::{Document, InvertedIndex, ParsedQuery, TokenId, TokenSet}; use crate::common::operation_error::{OperationError, OperationResult}; use crate::index::field_index::full_text_index::inverted_index::postings_iterator::{ @@ -48,7 +50,7 @@ impl ImmutableInvertedIndex { } /// Iterate over point ids whose documents contain all given tokens - fn filter_has_subset<'a>( + fn filter_has_all<'a>( &'a self, tokens: TokenSet, ) -> impl Iterator + 'a { @@ -93,6 +95,45 @@ impl ImmutableInvertedIndex { } } + /// Iterate over point ids whose documents contain at least one of the given tokens + fn filter_has_any<'a>( + &'a self, + tokens: TokenSet, + ) -> impl Iterator + 'a { + // in case of immutable index, deleted documents are still in the postings + let is_active = move |idx| { + self.point_to_tokens_count + .get(idx as usize) + .is_some_and(|x| *x > 0) + }; + + fn merge<'a, V: PostingValue>( + postings: &'a [PostingList], + tokens: TokenSet, + is_active: impl Fn(PointOffsetType) -> bool + 'a, + ) -> impl Iterator + 'a { + let postings: Vec<_> = tokens + .tokens() + .iter() + .filter_map(|&token_id| postings.get(token_id as usize).map(PostingList::view)) + .collect(); + + // Query must not be empty + if postings.is_empty() { + return Either::Left(std::iter::empty()); + }; + + Either::Right(merge_compressed_postings_iterator(postings, is_active)) + } + + match &self.postings { + ImmutablePostings::Ids(postings) => Either::Left(merge(postings, tokens, is_active)), + ImmutablePostings::WithPositions(postings) => { + Either::Right(merge(postings, tokens, is_active)) + } + } + } + fn check_has_subset(&self, tokens: &TokenSet, point_id: PointOffsetType) -> bool { if tokens.is_empty() { return false; @@ -123,6 +164,34 @@ impl ImmutableInvertedIndex { } } + fn check_has_any(&self, tokens: &TokenSet, point_id: PointOffsetType) -> bool { + if tokens.is_empty() { + return false; + } + + // check presence of the document + if self.values_is_empty(point_id) { + return false; + } + + fn check_any( + postings: &[PostingList], + tokens: &TokenSet, + point_id: PointOffsetType, + ) -> bool { + // Check that at least one token is in document + tokens.tokens().iter().any(|token_id| { + let posting_list = &postings[*token_id as usize]; + posting_list.visitor().contains(point_id) + }) + } + + match &self.postings { + ImmutablePostings::Ids(postings) => check_any(postings, tokens, point_id), + ImmutablePostings::WithPositions(postings) => check_any(postings, tokens, point_id), + } + } + /// Iterate over point ids whose documents contain all given tokens in the same order they are provided pub fn filter_has_phrase<'a>( &'a self, @@ -213,8 +282,9 @@ impl InvertedIndex for ImmutableInvertedIndex { _hw_counter: &'a HardwareCounterCell, ) -> Box + 'a> { match query { - ParsedQuery::Tokens(tokens) => Box::new(self.filter_has_subset(tokens)), + ParsedQuery::AllTokens(tokens) => Box::new(self.filter_has_all(tokens)), ParsedQuery::Phrase(tokens) => Box::new(self.filter_has_phrase(tokens)), + ParsedQuery::AnyTokens(tokens) => Box::new(self.filter_has_any(tokens)), } } @@ -232,8 +302,9 @@ impl InvertedIndex for ImmutableInvertedIndex { fn check_match(&self, parsed_query: &ParsedQuery, point_id: PointOffsetType) -> bool { match parsed_query { - ParsedQuery::Tokens(tokens) => self.check_has_subset(tokens, point_id), + ParsedQuery::AllTokens(tokens) => self.check_has_subset(tokens, point_id), ParsedQuery::Phrase(phrase) => self.check_has_phrase(phrase, point_id), + ParsedQuery::AnyTokens(tokens) => self.check_has_any(tokens, point_id), } } @@ -376,21 +447,21 @@ fn create_compressed_postings_with_positions( .collect::>(); (0u32..) - .zip(postings) - .map(|(token, posting)| { - posting - .iter() - .map(|id| { - let positions = point_to_tokens_positions[id as usize] - .remove(&token) - .expect( - "If id is this token's posting list, it should have at least one position", - ); - (id, positions) - }) - .collect() - }) - .collect() + .zip(postings) + .map(|(token, posting)| { + posting + .iter() + .map(|id| { + let positions = point_to_tokens_positions[id as usize] + .remove(&token) + .expect( + "If id is this token's posting list, it should have at least one position", + ); + (id, positions) + }) + .collect() + }) + .collect() } impl From<&MmapInvertedIndex> for ImmutableInvertedIndex { diff --git a/lib/segment/src/index/field_index/full_text_index/inverted_index/mmap_inverted_index/mod.rs b/lib/segment/src/index/field_index/full_text_index/inverted_index/mmap_inverted_index/mod.rs index f22fe7a027..fee6a615ef 100644 --- a/lib/segment/src/index/field_index/full_text_index/inverted_index/mmap_inverted_index/mod.rs +++ b/lib/segment/src/index/field_index/full_text_index/inverted_index/mmap_inverted_index/mod.rs @@ -16,7 +16,9 @@ use super::immutable_inverted_index::ImmutableInvertedIndex; use super::immutable_postings_enum::ImmutablePostings; use super::mmap_inverted_index::mmap_postings_enum::MmapPostingsEnum; use super::positions::Positions; -use super::postings_iterator::intersect_compressed_postings_iterator; +use super::postings_iterator::{ + intersect_compressed_postings_iterator, merge_compressed_postings_iterator, +}; use super::{InvertedIndex, ParsedQuery, TokenId, TokenSet}; use crate::common::Flusher; use crate::common::mmap_bitslice_buffered_update_wrapper::MmapBitSliceBufferedUpdateWrapper; @@ -169,7 +171,7 @@ impl MmapInvertedIndex { } /// Iterate over point ids whose documents contain all given tokens - pub fn filter_has_subset<'a>( + pub fn filter_has_all<'a>( &'a self, tokens: TokenSet, ) -> Box + 'a> { @@ -209,6 +211,41 @@ impl MmapInvertedIndex { } } + /// Iterate over point ids whose documents contain at least one of the given tokens + fn filter_has_any<'a>( + &'a self, + tokens: TokenSet, + ) -> impl Iterator + 'a { + // in case of immutable index, deleted documents are still in the postings + let is_active = move |idx| self.is_active(idx); + + fn merge<'a, V: MmapPostingValue>( + postings: &'a MmapPostings, + tokens: TokenSet, + is_active: impl Fn(PointOffsetType) -> bool + 'a, + ) -> impl Iterator + 'a { + let postings: Vec<_> = tokens + .tokens() + .iter() + .filter_map(|&token_id| postings.get(token_id)) + .collect(); + + // Query must not be empty + if postings.is_empty() { + return Either::Left(std::iter::empty()); + }; + + Either::Right(merge_compressed_postings_iterator(postings, is_active)) + } + + match &self.storage.postings { + MmapPostingsEnum::Ids(postings) => Either::Left(merge(postings, tokens, is_active)), + MmapPostingsEnum::WithPositions(postings) => { + Either::Right(merge(postings, tokens, is_active)) + } + } + } + fn check_has_subset(&self, tokens: &TokenSet, point_id: PointOffsetType) -> bool { // check non-empty query if tokens.is_empty() { @@ -244,6 +281,34 @@ impl MmapInvertedIndex { } } + fn check_has_any(&self, tokens: &TokenSet, point_id: PointOffsetType) -> bool { + if tokens.is_empty() { + return false; + } + + // check presence of the document + if self.values_is_empty(point_id) { + return false; + } + + fn check_any( + postings: &MmapPostings, + tokens: &TokenSet, + point_id: PointOffsetType, + ) -> bool { + // Check that at least one token is in document + tokens.tokens().iter().any(|token_id| { + let posting_list = postings.get(*token_id).unwrap(); + posting_list.visitor().contains(point_id) + }) + } + + match &self.storage.postings { + MmapPostingsEnum::Ids(postings) => check_any(postings, tokens, point_id), + MmapPostingsEnum::WithPositions(postings) => check_any(postings, tokens, point_id), + } + } + /// Iterate over point ids whose documents contain all given tokens in the same order they are provided pub fn filter_has_phrase<'a>( &'a self, @@ -381,8 +446,9 @@ impl InvertedIndex for MmapInvertedIndex { _hw_counter: &HardwareCounterCell, ) -> Box + 'a> { match query { - ParsedQuery::Tokens(tokens) => self.filter_has_subset(tokens), + ParsedQuery::AllTokens(tokens) => self.filter_has_all(tokens), ParsedQuery::Phrase(phrase) => Box::new(self.filter_has_phrase(phrase)), + ParsedQuery::AnyTokens(tokens) => Box::new(self.filter_has_any(tokens)), } } @@ -405,8 +471,9 @@ impl InvertedIndex for MmapInvertedIndex { fn check_match(&self, parsed_query: &ParsedQuery, point_id: PointOffsetType) -> bool { match parsed_query { - ParsedQuery::Tokens(tokens) => self.check_has_subset(tokens, point_id), + ParsedQuery::AllTokens(tokens) => self.check_has_subset(tokens, point_id), ParsedQuery::Phrase(phrase) => self.check_has_phrase(phrase, point_id), + ParsedQuery::AnyTokens(tokens) => self.check_has_any(tokens, point_id), } } diff --git a/lib/segment/src/index/field_index/full_text_index/inverted_index/mod.rs b/lib/segment/src/index/field_index/full_text_index/inverted_index/mod.rs index ee8211a29e..9f0d910eac 100644 --- a/lib/segment/src/index/field_index/full_text_index/inverted_index/mod.rs +++ b/lib/segment/src/index/field_index/full_text_index/inverted_index/mod.rs @@ -7,6 +7,7 @@ mod positions; mod posting_list; mod postings_iterator; +use std::cmp::min; use std::collections::HashMap; use ahash::AHashSet; @@ -16,6 +17,7 @@ use itertools::Itertools; use crate::common::operation_error::OperationResult; use crate::index::field_index::{CardinalityEstimation, PayloadBlockCondition, PrimaryCondition}; +use crate::index::query_estimator::expected_should_estimation; use crate::types::{FieldCondition, Match, PayloadKeyType}; pub type TokenId = u32; @@ -56,6 +58,15 @@ impl TokenSet { } subset.0.iter().all(|token| self.contains(token)) } + + /// Checks if the current set contains any of the given tokens. + /// Returns false if the subset is empty + pub fn has_any(&self, subset: &TokenSet) -> bool { + if subset.is_empty() { + return false; + } + subset.0.iter().any(|token| self.contains(token)) + } } impl From> for TokenSet { @@ -143,7 +154,10 @@ pub enum ParsedQuery { /// All these tokens must be present in the document, regardless of order. /// /// In other words this should be a subset of the document's token set. - Tokens(TokenSet), + AllTokens(TokenSet), + + /// At least one of these tokens must be present in the document. + AnyTokens(TokenSet), /// All these tokens must be present in the document, in the same order as this query. Phrase(Document), @@ -212,12 +226,15 @@ pub trait InvertedIndex { hw_counter: &HardwareCounterCell, ) -> CardinalityEstimation { match query { - ParsedQuery::Tokens(tokens) => { + ParsedQuery::AllTokens(tokens) => { self.estimate_has_subset_cardinality(tokens, condition, hw_counter) } ParsedQuery::Phrase(phrase) => { self.estimate_has_phrase_cardinality(phrase, condition, hw_counter) } + ParsedQuery::AnyTokens(tokens) => { + self.estimate_has_any_cardinality(tokens, condition, hw_counter) + } } } @@ -266,6 +283,46 @@ pub trait InvertedIndex { } } + fn estimate_has_any_cardinality( + &self, + tokens: &TokenSet, + condition: &FieldCondition, + hw_counter: &HardwareCounterCell, + ) -> CardinalityEstimation { + let points_count = self.points_count(); + + let posting_lengths: Vec<_> = tokens + .tokens() + .iter() + .filter_map(|&vocab_idx| self.get_posting_len(vocab_idx, hw_counter)) + .collect(); + + if posting_lengths.is_empty() { + // Empty request -> no matches + return CardinalityEstimation::exact(0) + .with_primary_clause(PrimaryCondition::Condition(Box::new(condition.clone()))); + } + + // At least one posting is the largest possible cardinality + let largest_posting = posting_lengths.iter().max().copied().unwrap(); + + if posting_lengths.len() == 1 { + return CardinalityEstimation::exact(largest_posting) + .with_primary_clause(PrimaryCondition::Condition(Box::new(condition.clone()))); + } + + let sum: usize = posting_lengths.iter().sum(); + + let exp = expected_should_estimation(posting_lengths.into_iter(), points_count); + + CardinalityEstimation { + primary_clauses: vec![PrimaryCondition::Condition(Box::new(condition.clone()))], + min: largest_posting, + exp, + max: min(sum, points_count), + } + } + fn estimate_has_phrase_cardinality( &self, phrase: &Document, @@ -367,7 +424,18 @@ mod tests { .into_iter() .map(token_to_id) .collect::>()?; - Some(ParsedQuery::Tokens(tokens)) + Some(ParsedQuery::AllTokens(tokens)) + } + + fn to_parsed_query_any( + query: Vec, + token_to_id: impl Fn(String) -> Option, + ) -> Option { + let tokens = query + .into_iter() + .map(token_to_id) + .collect::>()?; + Some(ParsedQuery::AnyTokens(tokens)) } fn mutable_inverted_index( @@ -536,21 +604,38 @@ mod tests { let mut_parsed_queries: Vec<_> = queries .iter() .cloned() - .map(|query| to_parsed_query(query, |token| mut_index.vocab.get(&token).copied())) + .flat_map(|query| { + vec![ + to_parsed_query(query.clone(), |token| mut_index.vocab.get(&token).copied()), + to_parsed_query_any(query, |token| mut_index.vocab.get(&token).copied()), + ] + }) .collect(); let mmap_parsed_queries: Vec<_> = queries .iter() .cloned() - .map(|query| { - to_parsed_query(query, |token| mmap_index.get_token_id(&token, &hw_counter)) + .flat_map(|query| { + vec![ + to_parsed_query(query.clone(), |token| { + mmap_index.get_token_id(&token, &hw_counter) + }), + to_parsed_query_any(query, |token| { + mmap_index.get_token_id(&token, &hw_counter) + }), + ] }) .collect(); let imm_mmap_parsed_queries: Vec<_> = queries .into_iter() - .map(|query| { - to_parsed_query(query, |token| { - imm_mmap_index.get_token_id(&token, &hw_counter) - }) + .flat_map(|query| { + vec![ + to_parsed_query(query.clone(), |token| { + imm_mmap_index.get_token_id(&token, &hw_counter) + }), + to_parsed_query_any(query, |token| { + imm_mmap_index.get_token_id(&token, &hw_counter) + }), + ] }) .collect(); diff --git a/lib/segment/src/index/field_index/full_text_index/inverted_index/mutable_inverted_index.rs b/lib/segment/src/index/field_index/full_text_index/inverted_index/mutable_inverted_index.rs index b49d75b866..42b4f64319 100644 --- a/lib/segment/src/index/field_index/full_text_index/inverted_index/mutable_inverted_index.rs +++ b/lib/segment/src/index/field_index/full_text_index/inverted_index/mutable_inverted_index.rs @@ -5,7 +5,7 @@ use common::types::PointOffsetType; use itertools::Either; use super::posting_list::PostingList; -use super::postings_iterator::intersect_postings_iterator; +use super::postings_iterator::{intersect_postings_iterator, merge_postings_iterator}; use super::{Document, InvertedIndex, ParsedQuery, TokenId, TokenSet}; use crate::common::operation_error::OperationResult; @@ -55,7 +55,7 @@ impl MutableInvertedIndex { } /// Iterate over point ids whose documents contain all given tokens - fn filter_has_subset(&self, tokens: TokenSet) -> impl Iterator + '_ { + fn filter_has_all(&self, tokens: TokenSet) -> impl Iterator + '_ { let postings_opt: Option> = tokens .tokens() .iter() @@ -79,6 +79,25 @@ impl MutableInvertedIndex { Either::Right(intersect_postings_iterator(postings)) } + fn filter_has_any(&self, tokens: TokenSet) -> impl Iterator + '_ { + let postings_opt: Vec<_> = tokens + .tokens() + .iter() + .filter_map(|&token_id| { + // if a ParsedQuery token was given an index, then it must exist in the vocabulary + // dictionary. Posting list entry can be None but it exists. + self.postings.get(token_id as usize) + }) + .collect(); + + if postings_opt.is_empty() { + // Empty request -> no matches + return Either::Left(std::iter::empty()); + } + + Either::Right(merge_postings_iterator(postings_opt)) + } + pub fn filter_has_phrase( &self, phrase: Document, @@ -89,7 +108,7 @@ impl MutableInvertedIndex { }; let iter = self - .filter_has_subset(phrase.to_token_set()) + .filter_has_all(phrase.to_token_set()) .filter(move |id| { let doc = point_to_doc[*id as usize] .as_ref() @@ -209,8 +228,9 @@ impl InvertedIndex for MutableInvertedIndex { _hw_counter: &HardwareCounterCell, ) -> Box + '_> { match query { - ParsedQuery::Tokens(tokens) => Box::new(self.filter_has_subset(tokens)), + ParsedQuery::AllTokens(tokens) => Box::new(self.filter_has_all(tokens)), ParsedQuery::Phrase(phrase) => self.filter_has_phrase(phrase), + ParsedQuery::AnyTokens(tokens) => Box::new(self.filter_has_any(tokens)), } } @@ -228,7 +248,7 @@ impl InvertedIndex for MutableInvertedIndex { fn check_match(&self, parsed_query: &ParsedQuery, point_id: PointOffsetType) -> bool { match parsed_query { - ParsedQuery::Tokens(query) => { + ParsedQuery::AllTokens(query) => { let Some(doc) = self.get_tokens(point_id) else { return false; }; @@ -244,6 +264,14 @@ impl InvertedIndex for MutableInvertedIndex { // Check that all tokens are in document, in order doc.has_phrase(document) } + ParsedQuery::AnyTokens(query) => { + let Some(doc) = self.get_tokens(point_id) else { + return false; + }; + + // Check that at least one token is in document + doc.has_any(query) + } } } diff --git a/lib/segment/src/index/field_index/full_text_index/inverted_index/postings_iterator.rs b/lib/segment/src/index/field_index/full_text_index/inverted_index/postings_iterator.rs index 12c8ee86c4..0ce08436d9 100644 --- a/lib/segment/src/index/field_index/full_text_index/inverted_index/postings_iterator.rs +++ b/lib/segment/src/index/field_index/full_text_index/inverted_index/postings_iterator.rs @@ -1,5 +1,5 @@ use common::types::PointOffsetType; -use itertools::Either; +use itertools::{Either, Itertools}; use posting_list::{PostingIterator, PostingListView, PostingValue}; use super::posting_list::PostingList; @@ -24,6 +24,16 @@ pub fn intersect_postings_iterator<'a>( .filter(move |doc_id| postings.iter().all(|posting| posting.contains(*doc_id))) } +pub fn merge_postings_iterator<'a>( + postings: Vec<&'a PostingList>, +) -> impl Iterator + 'a { + postings + .into_iter() + .map(PostingList::iter) + .kmerge_by(|a, b| a < b) + .dedup() +} + pub fn intersect_compressed_postings_iterator<'a, V: PostingValue + 'a>( mut postings: Vec>, is_active: impl Fn(PointOffsetType) -> bool + 'a, @@ -52,12 +62,28 @@ pub fn intersect_compressed_postings_iterator<'a, V: PostingValue + 'a>( // // This means that the other iterators can remember the last id they returned to avoid extra work posting_iterator + // potential optimization: Make posting iterator of just ids, without values (a.k.a. positions). + // We are discarding them here, thus unnecessarily reading them from the tails of the posting lists. .advance_until_greater_or_equal(*id) .is_some_and(|elem| elem.id == *id) }) }) } +pub fn merge_compressed_postings_iterator<'a, V: PostingValue + 'a>( + postings: Vec>, + is_active: impl Fn(PointOffsetType) -> bool + 'a, +) -> impl Iterator + 'a { + postings + .into_iter() + // potential optimization: Make posting iterator of just ids, without values (a.k.a. positions). + // We are discarding them here, thus unnecessarily reading them from the tails of the posting lists. + .map(|view| view.into_iter().map(|elem| elem.id)) + .kmerge_by(|a, b| a < b) + .dedup() + .filter(move |id| is_active(*id)) +} + /// Returns an iterator over the points that match the given phrase query. pub fn intersect_compressed_postings_phrase_iterator<'a>( phrase: Document, diff --git a/lib/segment/src/index/field_index/full_text_index/tests/test_congruence.rs b/lib/segment/src/index/field_index/full_text_index/tests/test_congruence.rs index 9122563e61..851761dda0 100644 --- a/lib/segment/src/index/field_index/full_text_index/tests/test_congruence.rs +++ b/lib/segment/src/index/field_index/full_text_index/tests/test_congruence.rs @@ -203,7 +203,7 @@ pub fn to_parsed_query( let tokens = query.iter().map(|token| token_to_id(token.as_str())); let parsed = match is_phrase { - false => ParsedQuery::Tokens(tokens.collect::>()?), + false => ParsedQuery::AllTokens(tokens.collect::>()?), true => ParsedQuery::Phrase(tokens.collect::>()?), }; diff --git a/lib/segment/src/index/field_index/full_text_index/text_index.rs b/lib/segment/src/index/field_index/full_text_index/text_index.rs index 9cae1e2f06..43510fe4bd 100644 --- a/lib/segment/src/index/field_index/full_text_index/text_index.rs +++ b/lib/segment/src/index/field_index/full_text_index/text_index.rs @@ -303,7 +303,22 @@ impl FullTextIndex { tokens.insert(self.get_token(token.as_ref(), hw_counter)); }); let tokens = tokens.into_iter().collect::>()?; - Some(ParsedQuery::Tokens(tokens)) + Some(ParsedQuery::AllTokens(tokens)) + } + + pub fn parse_text_any_query( + &self, + text: &str, + hw_counter: &HardwareCounterCell, + ) -> Option { + let mut tokens = AHashSet::new(); + self.get_tokenizer().tokenize_query(text, |token| { + if let Some(token_id) = self.get_token(token.as_ref(), hw_counter) { + tokens.insert(token_id); + } + }); + let tokens = tokens.into_iter().collect::(); + Some(ParsedQuery::AnyTokens(tokens)) } pub fn parse_tokenset(&self, text: &str, hw_counter: &HardwareCounterCell) -> TokenSet { @@ -358,7 +373,7 @@ impl FullTextIndex { FullTextIndex::get_values(payload_value) .iter() .any(|value| match &query { - ParsedQuery::Tokens(query) => { + ParsedQuery::AllTokens(query) => { let tokenset = self.parse_tokenset(value, hw_counter); tokenset.has_subset(query) } @@ -366,6 +381,10 @@ impl FullTextIndex { let document = self.parse_document(value, hw_counter); document.has_phrase(query) } + ParsedQuery::AnyTokens(query) => { + let tokenset = self.parse_tokenset(value, hw_counter); + tokenset.has_any(query) + } }) } diff --git a/lib/segment/src/index/query_estimator.rs b/lib/segment/src/index/query_estimator.rs index 18c71408b4..fad6463618 100644 --- a/lib/segment/src/index/query_estimator.rs +++ b/lib/segment/src/index/query_estimator.rs @@ -64,6 +64,28 @@ pub fn adjust_to_available_vectors( } } +/// Combine cardinality of multiple estimations in an OR fashion by using the complement rule. +/// Assumes that the estimations are independent. +/// +/// Formula is `(1 - ∏(1-pᵢ)) * total`: +/// * For each condition, it calculates the probability that an item does not match it: `1 - (x / total)`. +/// * It multiplies these probabilities to get the probability that an item matches none of the conditions. +/// * Subtracts this from 1 to get the probability that an item matches at least one condition. +/// * Multiplies this probability by the total number of items and rounds to get the expected count. +pub fn expected_should_estimation(estimations: impl Iterator, total: usize) -> usize { + if total == 0 { + return 0; + } + + let element_not_hit_prob: f64 = estimations + .map(|x| 1.0 - (x as f64 / total as f64)) + .product(); + + let element_hit_prob = 1.0 - element_not_hit_prob; + + (element_hit_prob * (total as f64)).round() as usize +} + pub fn combine_should_estimations( estimations: &[CardinalityEstimation], total: usize, @@ -78,12 +100,7 @@ pub fn combine_should_estimations( } clauses.append(&mut estimation.primary_clauses.clone()); } - let element_not_hit_prob: f64 = estimations - .iter() - .map(|x| (total - x.exp) as f64 / (total as f64)) - .product(); - let element_hit_prob = 1.0 - element_not_hit_prob; - let expected_count = (element_hit_prob * (total as f64)).round() as usize; + let expected_count = expected_should_estimation(estimations.iter().map(|x| x.exp), total); CardinalityEstimation { primary_clauses: clauses, min: estimations.iter().map(|x| x.min).max().unwrap_or(0), diff --git a/lib/segment/src/index/query_optimization/condition_converter/match_converter.rs b/lib/segment/src/index/query_optimization/condition_converter/match_converter.rs index 03f94d64da..5b55d22a9a 100644 --- a/lib/segment/src/index/query_optimization/condition_converter/match_converter.rs +++ b/lib/segment/src/index/query_optimization/condition_converter/match_converter.rs @@ -7,7 +7,8 @@ use crate::index::field_index::FieldIndex; use crate::index::query_optimization::optimized_filter::ConditionCheckerFn; use crate::payload_storage::condition_checker::INDEXSET_ITER_THRESHOLD; use crate::types::{ - AnyVariants, Match, MatchAny, MatchExcept, MatchPhrase, MatchText, MatchValue, ValueVariants, + AnyVariants, Match, MatchAny, MatchExcept, MatchPhrase, MatchText, MatchTextAny, MatchValue, + ValueVariants, }; pub fn get_match_checkers( @@ -17,9 +18,14 @@ pub fn get_match_checkers( ) -> Option> { match cond_match { Match::Value(MatchValue { value }) => get_match_value_checker(value, index, hw_acc), - Match::Text(MatchText { text }) => get_match_text_checker::(text, index, hw_acc), + Match::Text(MatchText { text }) => { + get_match_text_checker(text, TextQueryType::Text, index, hw_acc) + } + Match::TextAny(MatchTextAny { text_any }) => { + get_match_text_checker(text_any, TextQueryType::TextAny, index, hw_acc) + } Match::Phrase(MatchPhrase { phrase }) => { - get_match_text_checker::(phrase, index, hw_acc) + get_match_text_checker(phrase, TextQueryType::Phrase, index, hw_acc) } Match::Any(MatchAny { any }) => get_match_any_checker(any, index, hw_acc), Match::Except(MatchExcept { except }) => get_match_except_checker(except, index, hw_acc), @@ -249,18 +255,25 @@ fn get_match_except_checker( checker } -fn get_match_text_checker( +enum TextQueryType { + Phrase, + Text, + TextAny, +} + +fn get_match_text_checker( text: String, + query_type: TextQueryType, index: &FieldIndex, hw_acc: HwMeasurementAcc, ) -> Option> { let hw_counter = hw_acc.get_counter_cell(); match index { FieldIndex::FullTextIndex(full_text_index) => { - let query_opt = if IS_PHRASE { - full_text_index.parse_phrase_query(&text, &hw_counter) - } else { - full_text_index.parse_text_query(&text, &hw_counter) + let query_opt = match query_type { + TextQueryType::Phrase => full_text_index.parse_phrase_query(&text, &hw_counter), + TextQueryType::Text => full_text_index.parse_text_query(&text, &hw_counter), + TextQueryType::TextAny => full_text_index.parse_text_any_query(&text, &hw_counter), }; let Some(parsed_query) = query_opt else { diff --git a/lib/segment/src/payload_storage/condition_checker.rs b/lib/segment/src/payload_storage/condition_checker.rs index e8260605c5..5825e11f60 100644 --- a/lib/segment/src/payload_storage/condition_checker.rs +++ b/lib/segment/src/payload_storage/condition_checker.rs @@ -6,8 +6,8 @@ use serde_json::Value; use crate::types::{ AnyVariants, DateTimePayloadType, FieldCondition, FloatPayloadType, GeoBoundingBox, GeoPoint, - GeoPolygon, GeoRadius, Match, MatchAny, MatchExcept, MatchPhrase, MatchText, MatchValue, Range, - RangeInterface, ValueVariants, ValuesCount, + GeoPolygon, GeoRadius, Match, MatchAny, MatchExcept, MatchPhrase, MatchText, MatchTextAny, + MatchValue, Range, RangeInterface, ValueVariants, ValuesCount, }; /// Threshold representing the point to which iterating through an IndexSet is more efficient than using hashing. @@ -165,6 +165,12 @@ impl ValueChecker for Match { _ => false, } } + Match::TextAny(MatchTextAny { text_any }) => match payload { + Value::String(stored) => text_any + .split_whitespace() + .any(|token| stored.contains(token)), + _ => false, + }, Match::Any(MatchAny { any }) => match (payload, any) { (Value::String(stored), AnyVariants::Strings(list)) => { if list.len() < INDEXSET_ITER_THRESHOLD { diff --git a/lib/segment/src/types.rs b/lib/segment/src/types.rs index 72a83d991b..8de0aa8c98 100644 --- a/lib/segment/src/types.rs +++ b/lib/segment/src/types.rs @@ -2166,6 +2166,13 @@ pub struct MatchText { pub text: String, } +/// Full-text match of at least one token of the string. +#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub struct MatchTextAny { + pub text_any: String, +} + impl> From for MatchText { fn from(text: S) -> Self { MatchText { text: text.into() } @@ -2207,6 +2214,7 @@ pub struct MatchExcept { pub enum MatchInterface { Value(MatchValue), Text(MatchText), + TextAny(MatchTextAny), Phrase(MatchPhrase), Any(MatchAny), Except(MatchExcept), @@ -2218,6 +2226,7 @@ pub enum MatchInterface { pub enum Match { Value(MatchValue), Text(MatchText), + TextAny(MatchTextAny), Phrase(MatchPhrase), Any(MatchAny), Except(MatchExcept), @@ -2258,6 +2267,9 @@ impl From for Match { match value { MatchInterface::Value(value) => Self::Value(MatchValue { value: value.value }), MatchInterface::Text(text) => Self::Text(MatchText { text: text.text }), + MatchInterface::TextAny(text_any) => Self::TextAny(MatchTextAny { + text_any: text_any.text_any, + }), MatchInterface::Any(any) => Self::Any(MatchAny { any: any.any }), MatchInterface::Except(except) => Self::Except(MatchExcept { except: except.except, @@ -2774,6 +2786,7 @@ impl FieldCondition { Match::Value(_) => 0, Match::Text(_) => 0, Match::Phrase(_) => 0, + Match::TextAny(_) => 0, } } } diff --git a/tests/openapi/test_fts.py b/tests/openapi/test_fts.py index 10158b349b..3eae3fa16e 100644 --- a/tests/openapi/test_fts.py +++ b/tests/openapi/test_fts.py @@ -175,21 +175,6 @@ def basic_collection_setup( ) assert response.ok - # Create index - response = request_with_validation( - api='/collections/{collection_name}/index', - method="PUT", - path_params={'collection_name': collection_name}, - query_params={'wait': 'true'}, - body={ - "field_name": "title", - "field_schema": { - "type": "text", - "tokenizer": "prefix", - } - } - ) - assert response.ok response = request_with_validation( api='/collections/{collection_name}', @@ -222,8 +207,103 @@ def setup(on_disk_vectors, on_disk_payload, collection_name): yield drop_collection(collection_name=collection_name) +def test_match_any(collection_name): + + # Create index + response = request_with_validation( + api='/collections/{collection_name}/index', + method="PUT", + path_params={'collection_name': collection_name}, + query_params={'wait': 'true'}, + body={ + "field_name": "title", + "field_schema": { + "type": "text", + "tokenizer": "word", + "lowercase": True, + "stopwords": "english" + } + } + ) + assert response.ok + + + response_any = request_with_validation( + api='/collections/{collection_name}/points/scroll', + method="POST", + path_params={'collection_name': collection_name}, + body={ + "offset": None, + "limit": 10, + "with_payload": True, + "with_vector": False, + "filter": { + "must": { + "key": "title", + "match": { + "text_any": "robot of the star", + } + } + } + } + ) + + assert response_any.ok + + response_any_ids = {point['id'] for point in response_any.json()['result']['points']} + + # Same request, but manually split into words + response_split = request_with_validation( + api='/collections/{collection_name}/points/scroll', + method="POST", + path_params={'collection_name': collection_name}, + body={ + "offset": None, + "limit": 10, + "with_payload": True, + "with_vector": False, + "filter": { + "should": [ + { + "key": "title", + "match": { + "text": "robot", + } + }, + { + "key": "title", + "match": { + "text": "star", + } + } + ] + } + } + ) + + assert response_split.ok + response_split_ids = {point['id'] for point in response_split.json()['result']['points']} + + assert response_any_ids == response_split_ids + def test_scroll_with_prefix(collection_name): + # Create index + response = request_with_validation( + api='/collections/{collection_name}/index', + method="PUT", + path_params={'collection_name': collection_name}, + query_params={'wait': 'true'}, + body={ + "field_name": "title", + "field_schema": { + "type": "text", + "tokenizer": "prefix", + } + } + ) + assert response.ok + response = request_with_validation( api='/collections/{collection_name}/points/scroll', method="POST",