upstream-archive/qdrant
1
0
Fork 0
mirror of https://github.com/qdrant/qdrant.git synced 2026-07-23 11:11:00 -05:00
Code Issues Packages Projects Releases Wiki Activity

Full-Text Index ASCII Folding (Normalization) (#7408)

Browse Source 
* Add ASCII folding to tokenization process

Introduced an optional ASCII folding feature within the `TokensProcessor` to normalize non-ASCII characters to their ASCII equivalents. Updated tests and documentation to reflect the changes.

* Refactor tokenization code for improved readability and maintainability

Reorganized and reformatted the tokenization module, including `TokensProcessor` initialization and ASCII folding mappings for better clarity. Updated tests to align with the changes.

* Update test cases to reflect optional tokenizer settings changes

Adjusted `ascii_folding`, `lowercase`, and `phrase_matching` settings in tests to `None` where applicable, aligning with updates in tokenizer configuration defaults.

* address review remarks

* fix codespell

* thx coderabbit

* Don't copy tokens that are already ASCII

* Shrink folded string to fit

---------

Co-authored-by: Luis Cossío <luis.cossio@outlook.com>
Co-authored-by: timvisee <tim@visee.me>
This commit is contained in:
eltu
2025-10-16 08:53:01 -03:00
committed by timvisee
parent 9da05521ab
commit d6d52d4b81
15 changed files with 747 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
docs/grpc/docs.md
View File

@@ -1729,6 +1729,7 @@ Note: 1kB = 1 vector of size 256. |
| stopwords | [StopwordsSet](#qdrant-StopwordsSet) | optional | Stopwords for the text index |
| phrase_matching | [bool](#bool) | optional | If true - support phrase matching. |
| stemmer | [StemmingAlgorithm](#qdrant-StemmingAlgorithm) | optional | Set an algorithm for stemming. |
| ascii_folding | [bool](#bool) | optional | If true, normalize tokens by folding accented characters to ASCII (e.g., &#34;ação&#34; -&gt; &#34;acao&#34;). Default: false. |

10
docs/redoc/master/openapi.json
View File

@@ -7748,6 +7748,11 @@
"type": "boolean",
"nullable": true
},
"ascii_folding": {
"description": "If true, normalize tokens by folding accented characters to ASCII (e.g., \"ação\" -> \"acao\"). Default: false.",
"type": "boolean",
"nullable": true
},
"phrase_matching": {
"description": "If true, support phrase matching. Default: false.",
"type": "boolean",
@@ -10700,6 +10705,11 @@
"type": "boolean",
"nullable": true
},
"ascii_folding": {
"description": "If true, normalize tokens by folding accented characters to ASCII (e.g., \"ação\" -> \"acao\"). Default is `false`.",
"type": "boolean",
"nullable": true
},
"stopwords": {
"description": "Configuration of the stopwords filter. Supports list of pre-defined languages and custom stopwords. Default: initialized for specified `language` or English if not specified.",
"anyOf": [

4
lib/api/src/grpc/conversions.rs
View File

@@ -215,6 +215,7 @@ impl From<segment::data_types::index::TextIndexParams> for PayloadIndexParams {
min_token_len,
max_token_len,
lowercase,
ascii_folding,
phrase_matching,
on_disk,
stopwords,
@@ -231,6 +232,7 @@ impl From<segment::data_types::index::TextIndexParams> for PayloadIndexParams {
index_params: Some(IndexParams::TextIndexParams(TextIndexParams {
tokenizer: tokenizer as i32,
lowercase,
ascii_folding,
min_token_len: min_token_len.map(|x| x as u64),
max_token_len: max_token_len.map(|x| x as u64),
phrase_matching,
@@ -503,6 +505,7 @@ impl TryFrom<TextIndexParams> for segment::data_types::index::TextIndexParams {
let TextIndexParams {
tokenizer,
lowercase,
ascii_folding,
min_token_len,
max_token_len,
phrase_matching,
@@ -531,6 +534,7 @@ impl TryFrom<TextIndexParams> for segment::data_types::index::TextIndexParams {
.map(|x| x.try_into())
.unwrap_or_else(|_| Err(Status::invalid_argument("unknown tokenizer type")))?,
lowercase,
ascii_folding,
min_token_len: min_token_len.map(|x| x as usize),
max_token_len: max_token_len.map(|x| x as usize),
phrase_matching,

1
lib/api/src/grpc/proto/collections.proto
View File

@@ -520,6 +520,7 @@ message TextIndexParams {
optional StopwordsSet stopwords = 6; // Stopwords for the text index
optional bool phrase_matching = 7; // If true - support phrase matching.
optional StemmingAlgorithm stemmer = 8; // Set an algorithm for stemming.
optional bool ascii_folding = 9; // If true, normalize tokens by folding accented characters to ASCII (e.g., "ação" -> "acao"). Default: false.
}
message StemmingAlgorithm {

3
lib/api/src/grpc/qdrant.rs
View File

@@ -1097,6 +1097,9 @@ pub struct TextIndexParams {
/// Set an algorithm for stemming.
#[prost(message, optional, tag = "8")]
pub stemmer: ::core::option::Option<StemmingAlgorithm>,
/// If true, normalize tokens by folding accented characters to ASCII (e.g., "ação" -> "acao"). Default: false.
#[prost(bool, optional, tag = "9")]
pub ascii_folding: ::core::option::Option<bool>,
}
#[derive(serde::Serialize)]
#[allow(clippy::derive_partial_eq_without_eq)]

4
lib/api/src/rest/schema.rs
View File

@@ -223,6 +223,10 @@ pub struct TextPreprocessingConfig {
/// Default is `true`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub lowercase: Option<bool>,
/// If true, normalize tokens by folding accented characters to ASCII (e.g., "ação" -> "acao").
/// Default is `false`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ascii_folding: Option<bool>,
/// Configuration of the stopwords filter. Supports list of pre-defined languages and custom stopwords.
/// Default: initialized for specified `language` or English if not specified.
#[serde(default, skip_serializing_if = "Option::is_none")]

4
lib/segment/src/data_types/index.rs
View File

@@ -199,6 +199,10 @@ pub struct TextIndexParams {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub lowercase: Option<bool>,
/// If true, normalize tokens by folding accented characters to ASCII (e.g., "ação" -> "acao"). Default: false.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ascii_folding: Option<bool>,
/// If true, support phrase matching. Default: false.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub phrase_matching: Option<bool>,

1
lib/segment/src/index/field_index/full_text_index/mutable_text_index.rs
View File

@@ -413,6 +413,7 @@ mod tests {
on_disk: None,
stopwords: None,
stemmer: None,
ascii_folding: None,
};
{

90
lib/segment/src/index/field_index/full_text_index/tests/mod.rs
View File

@@ -164,6 +164,7 @@ fn test_prefix_search() {
stopwords: None,
on_disk: None,
stemmer: None,
ascii_folding: None,
};
let mut index =
@@ -213,6 +214,7 @@ fn test_phrase_matching() {
phrase_matching: Some(true), // Enable phrase matching
stopwords: None,
stemmer: None,
ascii_folding: None,
};
let mut mutable_index =
@@ -305,3 +307,91 @@ fn test_phrase_matching() {
check_matching(mutable_index);
check_matching(mmap_index);
}
#[test]
fn test_ascii_folding_in_full_text_index_word() {
let hw_counter = HardwareCounterCell::default();
let temp_dir = Builder::new().prefix("test_dir").tempdir().unwrap();
let config_enabled = TextIndexParams {
r#type: TextIndexType::Text,
tokenizer: TokenizerType::Word,
min_token_len: None,
max_token_len: None,
lowercase: None,
on_disk: None,
phrase_matching: None,
stopwords: None,
stemmer: None,
ascii_folding: Some(true),
};
let config_disabled = TextIndexParams {
ascii_folding: Some(false),
..config_enabled.clone()
};
// Index with folding enabled
let mut index_enabled =
FullTextIndex::new_gridstore(temp_dir.path().to_path_buf(), config_enabled.clone(), true)
.unwrap()
.unwrap();
// Index with folding disabled (separate storage path)
let temp_dir2 = Builder::new().prefix("test_dir").tempdir().unwrap();
let mut index_disabled = FullTextIndex::new_gridstore(
temp_dir2.path().to_path_buf(),
config_disabled.clone(),
true,
)
.unwrap()
.unwrap();
// Documents containing accents
let docs = vec![
(0, "ação no coração".to_string()),
(1, "café com leite".to_string()),
];
for (id, text) in &docs {
index_enabled
.add_many(*id as PointOffsetType, vec![text.clone()], &hw_counter)
.unwrap();
index_disabled
.add_many(*id as PointOffsetType, vec![text.clone()], &hw_counter)
.unwrap();
}
// ASCII-only queries should match only when folding is enabled
let query_enabled = index_enabled.parse_text_query("acao", &hw_counter).unwrap();
assert!(index_enabled.check_match(&query_enabled, 0));
let results_enabled: Vec<_> = index_enabled
.filter_query(query_enabled, &hw_counter)
.collect();
assert!(results_enabled.contains(&0));
let query_disabled_opt = index_disabled.parse_text_query("acao", &hw_counter);
// Query might still parse, but should not match anything
if let Some(query_disabled) = query_disabled_opt {
let results_disabled: Vec<_> = index_disabled
.filter_query(query_disabled, &hw_counter)
.collect();
assert!(!results_disabled.contains(&0));
}
// Non-folded query must work in both
let query_acento = index_enabled.parse_text_query("ação", &hw_counter).unwrap();
assert!(index_enabled.check_match(&query_acento, 0));
let results_acento: Vec<_> = index_enabled
.filter_query(query_acento, &hw_counter)
.collect();
assert!(results_acento.contains(&0));
let query_acento2 = index_disabled
.parse_text_query("ação", &hw_counter)
.unwrap();
let results_acento2: Vec<_> = index_disabled
.filter_query(query_acento2, &hw_counter)
.collect();
assert!(results_acento2.contains(&0));
}

486
lib/segment/src/index/field_index/full_text_index/tokenizers/ascii_folding.rs Normal file
View File

@@ -0,0 +1,486 @@
// ASCII folding implementation inspired by Lucene's ASCIIFoldingFilter.
// Converts non-ASCII Latin letters and various symbols to ASCII equivalents.
// This aims to be compatible with Lucene's mapping list used in ASCIIFoldingFilter.
use std::borrow::Cow;
pub fn fold_to_ascii_cow<'a>(input: Cow<'a, str>) -> Cow<'a, str> {
// Cheap check if already ASCII
if input.is_ascii() {
return input;
}
Cow::Owned(fold_to_ascii(input.as_ref()))
}
pub fn fold_to_ascii(input: &str) -> String {
let mut out = String::with_capacity(input.len());
for ch in input.chars() {
if ch.is_ascii() {
out.push(ch);
continue;
}
match fold_char(ch) {
Some(repl) => out.push_str(repl),
None => out.push(ch),
}
}
out.shrink_to_fit();
out
}
#[inline]
fn fold_char(c: char) -> Option<&'static str> {
// Quick test
if c < '\u{0080}' {
return None;
}
// Main mapping translated from Lucene's ASCIIFoldingFilter
let s = match c {
// A/a
'\u{00C0}' | '\u{00C1}' | '\u{00C2}' | '\u{00C3}' | '\u{00C4}' | '\u{00C5}'
| '\u{0100}' | '\u{0102}' | '\u{0104}' | '\u{01CD}' | '\u{01DE}' | '\u{01E0}'
| '\u{01FA}' | '\u{0200}' | '\u{0202}' | '\u{0226}' | '\u{023A}' | '\u{1D00}'
| '\u{1E00}' | '\u{1EA0}' | '\u{1EA2}' | '\u{1EA4}' | '\u{1EA6}' | '\u{1EA8}'
| '\u{1EAA}' | '\u{1EAC}' | '\u{1EAE}' | '\u{1EB0}' | '\u{1EB2}' | '\u{1EB4}'
| '\u{1EB6}' | '\u{24B6}' | '\u{FF21}' => "A",
'\u{00E0}' | '\u{00E1}' | '\u{00E2}' | '\u{00E3}' | '\u{00E4}' | '\u{00E5}'
| '\u{0101}' | '\u{0103}' | '\u{0105}' | '\u{01CE}' | '\u{01DF}' | '\u{01E1}'
| '\u{01FB}' | '\u{0201}' | '\u{0203}' | '\u{0227}' | '\u{0250}' | '\u{0259}'
| '\u{025A}' | '\u{1D8F}' | '\u{1D95}' | '\u{1E01}' | '\u{1E9A}' | '\u{1EA1}'
| '\u{1EA3}' | '\u{1EA5}' | '\u{1EA7}' | '\u{1EA9}' | '\u{1EAB}' | '\u{1EAD}'
| '\u{1EAF}' | '\u{1EB1}' | '\u{1EB3}' | '\u{1EB5}' | '\u{1EB7}' | '\u{2090}'
| '\u{2094}' | '\u{24D0}' | '\u{2C65}' | '\u{2C6F}' | '\u{FF41}' => "a",
'\u{A732}' => "AA", // Ꜳ
'\u{00C6}' | '\u{01E2}' | '\u{01FC}' | '\u{1D01}' => "AE",
'\u{A734}' => "AO",
'\u{A736}' => "AU",
'\u{A738}' | '\u{A73A}' => "AV",
'\u{A73C}' => "AY",
'\u{249C}' => "(a)",
'\u{A733}' => "aa",
'\u{00E6}' | '\u{01E3}' | '\u{01FD}' | '\u{1D02}' => "ae",
'\u{A735}' => "ao",
'\u{A737}' => "au",
'\u{A739}' | '\u{A73B}' => "av",
'\u{A73D}' => "ay",
// B/b
'\u{0181}' | '\u{0182}' | '\u{0243}' | '\u{0299}' | '\u{1D03}' | '\u{1E02}'
| '\u{1E04}' | '\u{1E06}' | '\u{24B7}' | '\u{FF22}' => "B",
'\u{0180}' | '\u{0183}' | '\u{0253}' | '\u{1D6C}' | '\u{1D80}' | '\u{1E03}'
| '\u{1E05}' | '\u{1E07}' | '\u{24D1}' | '\u{FF42}' => "b",
'\u{249D}' => "(b)",
// C/c
'\u{00C7}' | '\u{0106}' | '\u{0108}' | '\u{010A}' | '\u{010C}' | '\u{0187}'
| '\u{023B}' | '\u{0297}' | '\u{1D04}' | '\u{1E08}' | '\u{24B8}' | '\u{FF23}' => "C",
'\u{00E7}' | '\u{0107}' | '\u{0109}' | '\u{010B}' | '\u{010D}' | '\u{0188}'
| '\u{023C}' | '\u{0255}' | '\u{1E09}' | '\u{2184}' | '\u{24D2}' | '\u{A73E}'
| '\u{A73F}' | '\u{FF43}' => "c",
'\u{249E}' => "(c)",
// D/d
'\u{00D0}' | '\u{010E}' | '\u{0110}' | '\u{0189}' | '\u{018A}' | '\u{018B}'
| '\u{1D05}' | '\u{1D06}' | '\u{1E0A}' | '\u{1E0C}' | '\u{1E0E}' | '\u{1E10}'
| '\u{1E12}' | '\u{24B9}' | '\u{A779}' | '\u{FF24}' => "D",
'\u{00F0}' | '\u{010F}' | '\u{0111}' | '\u{018C}' | '\u{0221}' | '\u{0256}'
| '\u{0257}' | '\u{1D6D}' | '\u{1D81}' | '\u{1D91}' | '\u{1E0B}' | '\u{1E0D}'
| '\u{1E0F}' | '\u{1E11}' | '\u{1E13}' | '\u{24D3}' | '\u{A77A}' | '\u{FF44}' => "d",
'\u{01C4}' | '\u{01F1}' => "DZ",
'\u{01C5}' | '\u{01F2}' => "Dz",
'\u{249F}' => "(d)",
'\u{0238}' => "db",
'\u{01C6}' | '\u{01F3}' | '\u{02A3}' | '\u{02A5}' => "dz",
// E/e
'\u{00C8}' | '\u{00C9}' | '\u{00CA}' | '\u{00CB}' | '\u{0112}' | '\u{0114}'
| '\u{0116}' | '\u{0118}' | '\u{011A}' | '\u{018E}' | '\u{0190}' | '\u{0204}'
| '\u{0206}' | '\u{0228}' | '\u{0246}' | '\u{1D07}' | '\u{1E14}' | '\u{1E16}'
| '\u{1E18}' | '\u{1E1A}' | '\u{1E1C}' | '\u{1EB8}' | '\u{1EBA}' | '\u{1EBC}'
| '\u{1EBE}' | '\u{1EC0}' | '\u{1EC2}' | '\u{1EC4}' | '\u{1EC6}' | '\u{24BA}'
| '\u{2C7B}' | '\u{FF25}' => "E",
'\u{00E8}' | '\u{00E9}' | '\u{00EA}' | '\u{00EB}' | '\u{0113}' | '\u{0115}'
| '\u{0117}' | '\u{0119}' | '\u{011B}' | '\u{01DD}' | '\u{0205}' | '\u{0207}'
| '\u{0229}' | '\u{0247}' | '\u{0258}' | '\u{025B}' | '\u{025C}' | '\u{025D}'
| '\u{025E}' | '\u{029A}' | '\u{1D08}' | '\u{1D92}' | '\u{1D93}' | '\u{1D94}'
| '\u{1E15}' | '\u{1E17}' | '\u{1E19}' | '\u{1E1B}' | '\u{1E1D}' | '\u{1EB9}'
| '\u{1EBB}' | '\u{1EBD}' | '\u{1EBF}' | '\u{1EC1}' | '\u{1EC3}' | '\u{1EC5}'
| '\u{1EC7}' | '\u{2091}' | '\u{24D4}' | '\u{2C78}' | '\u{FF45}' => "e",
'\u{24A0}' => "(e)",
// F/f and ligatures
'\u{0191}' | '\u{1E1E}' | '\u{24BB}' | '\u{A730}' | '\u{A77B}' | '\u{A7FB}'
| '\u{FF26}' => "F",
'\u{0192}' | '\u{1D6E}' | '\u{1D82}' | '\u{1E1F}' | '\u{1E9B}' | '\u{24D5}'
| '\u{A77C}' | '\u{FF46}' => "f",
'\u{24A1}' => "(f)",
'\u{FB00}' => "ff",
'\u{FB03}' => "ffi",
'\u{FB04}' => "ffl",
'\u{FB01}' => "fi",
'\u{FB02}' => "fl",
// G/g
'\u{011C}' | '\u{011E}' | '\u{0120}' | '\u{0122}' | '\u{0193}' | '\u{01E4}'
| '\u{01E6}' | '\u{01F4}' | '\u{0262}' | '\u{029B}' | '\u{1E20}' | '\u{24BC}'
| '\u{A77D}' | '\u{A77E}' | '\u{FF27}' => "G",
'\u{011D}' | '\u{011F}' | '\u{0121}' | '\u{0123}' | '\u{01E5}' | '\u{01E7}'
| '\u{01F5}' | '\u{0260}' | '\u{0261}' | '\u{1D77}' | '\u{1D79}' | '\u{1D83}'
| '\u{1E21}' | '\u{24D6}' | '\u{A77F}' | '\u{FF47}' => "g",
'\u{24A2}' => "(g)",
// H/h, HV
'\u{0124}' | '\u{0126}' | '\u{021E}' | '\u{029C}' | '\u{1E22}' | '\u{1E24}'
| '\u{1E26}' | '\u{1E28}' | '\u{1E2A}' | '\u{24BD}' | '\u{2C67}' | '\u{2C75}'
| '\u{FF28}' => "H",
'\u{0125}' | '\u{0127}' | '\u{021F}' | '\u{0265}' | '\u{0266}' | '\u{02AE}'
| '\u{02AF}' | '\u{1E23}' | '\u{1E25}' | '\u{1E27}' | '\u{1E29}' | '\u{1E2B}'
| '\u{1E96}' | '\u{24D7}' | '\u{2C68}' | '\u{2C76}' | '\u{FF48}' => "h",
'\u{01F6}' => "HV",
'\u{24A3}' => "(h)",
'\u{0195}' => "hv",
// I/i, IJ
'\u{00CC}' | '\u{00CD}' | '\u{00CE}' | '\u{00CF}' | '\u{0128}' | '\u{012A}'
| '\u{012C}' | '\u{012E}' | '\u{0130}' | '\u{0196}' | '\u{0197}' | '\u{01CF}'
| '\u{0208}' | '\u{020A}' | '\u{026A}' | '\u{1D7B}' | '\u{1E2C}' | '\u{1E2E}'
| '\u{1EC8}' | '\u{1ECA}' | '\u{24BE}' | '\u{A7FE}' | '\u{FF29}' => "I",
'\u{00EC}' | '\u{00ED}' | '\u{00EE}' | '\u{00EF}' | '\u{0129}' | '\u{012B}'
| '\u{012D}' | '\u{012F}' | '\u{0131}' | '\u{01D0}' | '\u{0209}' | '\u{020B}'
| '\u{0268}' | '\u{1D09}' | '\u{1D62}' | '\u{1D7C}' | '\u{1D96}' | '\u{1E2D}'
| '\u{1E2F}' | '\u{1EC9}' | '\u{1ECB}' | '\u{2071}' | '\u{24D8}' | '\u{FF49}' => "i",
'\u{0132}' => "IJ",
'\u{24A4}' => "(i)",
'\u{0133}' => "ij",
// J/j
'\u{0134}' | '\u{0248}' | '\u{1D0A}' | '\u{24BF}' | '\u{FF2A}' => "J",
'\u{0135}' | '\u{01F0}' | '\u{0237}' | '\u{0249}' | '\u{025F}' | '\u{0284}'
| '\u{029D}' | '\u{24D9}' | '\u{2C7C}' | '\u{FF4A}' => "j",
'\u{24A5}' => "(j)",
// K/k
'\u{0136}' | '\u{0198}' | '\u{01E8}' | '\u{1D0B}' | '\u{1E30}' | '\u{1E32}'
| '\u{1E34}' | '\u{24C0}' | '\u{2C69}' | '\u{A740}' | '\u{A742}' | '\u{A744}'
| '\u{FF2B}' => "K",
'\u{0137}' | '\u{0199}' | '\u{01E9}' | '\u{029E}' | '\u{1D84}' | '\u{1E31}'
| '\u{1E33}' | '\u{1E35}' | '\u{24DA}' | '\u{2C6A}' | '\u{A741}' | '\u{A743}'
| '\u{A745}' | '\u{FF4B}' => "k",
'\u{24A6}' => "(k)",
// L/l, LJ/ll etc.
'\u{0139}' | '\u{013B}' | '\u{013D}' | '\u{013F}' | '\u{0141}' | '\u{023D}'
| '\u{029F}' | '\u{1D0C}' | '\u{1E36}' | '\u{1E38}' | '\u{1E3A}' | '\u{1E3C}'
| '\u{24C1}' | '\u{2C60}' | '\u{2C62}' | '\u{A746}' | '\u{A748}' | '\u{A780}'
| '\u{FF2C}' => "L",
'\u{013A}' | '\u{013C}' | '\u{013E}' | '\u{0140}' | '\u{0142}' | '\u{019A}'
| '\u{0234}' | '\u{026B}' | '\u{026C}' | '\u{026D}' | '\u{1D85}' | '\u{1E37}'
| '\u{1E39}' | '\u{1E3B}' | '\u{1E3D}' | '\u{24DB}' | '\u{2C61}' | '\u{A747}'
| '\u{A749}' | '\u{A781}' | '\u{FF4C}' => "l",
'\u{01C7}' => "LJ",
'\u{1EFA}' => "LL",
'\u{01C8}' => "Lj",
'\u{24A7}' => "(l)",
'\u{01C9}' => "lj",
'\u{1EFB}' => "ll",
'\u{02AA}' => "ls",
'\u{02AB}' => "lz",
// M/m
'\u{019C}' | '\u{1D0D}' | '\u{1E3E}' | '\u{1E40}' | '\u{1E42}' | '\u{24C2}'
| '\u{2C6E}' | '\u{A7FD}' | '\u{A7FF}' | '\u{FF2D}' => "M",
'\u{026F}' | '\u{0270}' | '\u{0271}' | '\u{1D6F}' | '\u{1D86}' | '\u{1E3F}'
| '\u{1E41}' | '\u{1E43}' | '\u{24DC}' | '\u{FF4D}' => "m",
'\u{24A8}' => "(m)",
// N/n, NJ
'\u{00D1}' | '\u{0143}' | '\u{0145}' | '\u{0147}' | '\u{014A}' | '\u{019D}'
| '\u{01F8}' | '\u{0220}' | '\u{0274}' | '\u{1D0E}' | '\u{1E44}' | '\u{1E46}'
| '\u{1E48}' | '\u{1E4A}' | '\u{24C3}' | '\u{FF2E}' => "N",
'\u{00F1}' | '\u{0144}' | '\u{0146}' | '\u{0148}' | '\u{0149}' | '\u{014B}'
| '\u{019E}' | '\u{01F9}' | '\u{0235}' | '\u{0272}' | '\u{0273}' | '\u{1D70}'
| '\u{1D87}' | '\u{1E45}' | '\u{1E47}' | '\u{1E49}' | '\u{1E4B}' | '\u{207F}'
| '\u{24DD}' | '\u{FF4E}' => "n",
'\u{01CA}' => "NJ",
'\u{01CB}' => "Nj",
'\u{24A9}' => "(n)",
'\u{01CC}' => "nj",
// O/o, OE/OO/OU
'\u{00D2}' | '\u{00D3}' | '\u{00D4}' | '\u{00D5}' | '\u{00D6}' | '\u{00D8}'
| '\u{014C}' | '\u{014E}' | '\u{0150}' | '\u{0186}' | '\u{019F}' | '\u{01A0}'
| '\u{01D1}' | '\u{01EA}' | '\u{01EC}' | '\u{01FE}' | '\u{020C}' | '\u{020E}'
| '\u{022A}' | '\u{022C}' | '\u{022E}' | '\u{0230}' | '\u{1D0F}' | '\u{1D10}'
| '\u{1E4C}' | '\u{1E4E}' | '\u{1E50}' | '\u{1E52}' | '\u{1ECC}' | '\u{1ECE}'
| '\u{1ED0}' | '\u{1ED2}' | '\u{1ED4}' | '\u{1ED6}' | '\u{1ED8}' | '\u{1EDA}'
| '\u{1EDC}' | '\u{1EDE}' | '\u{1EE0}' | '\u{1EE2}' | '\u{24C4}' | '\u{A74A}'
| '\u{A74C}' | '\u{FF2F}' => "O",
'\u{00F2}' | '\u{00F3}' | '\u{00F4}' | '\u{00F5}' | '\u{00F6}' | '\u{00F8}'
| '\u{014D}' | '\u{014F}' | '\u{0151}' | '\u{01A1}' | '\u{01D2}' | '\u{01EB}'
| '\u{01ED}' | '\u{01FF}' | '\u{020D}' | '\u{020F}' | '\u{022B}' | '\u{022D}'
| '\u{022F}' | '\u{0231}' | '\u{0254}' | '\u{0275}' | '\u{1D16}' | '\u{1D17}'
| '\u{1D97}' | '\u{1E4D}' | '\u{1E4F}' | '\u{1E51}' | '\u{1E53}' | '\u{1ECD}'
| '\u{1ECF}' | '\u{1ED1}' | '\u{1ED3}' | '\u{1ED5}' | '\u{1ED7}' | '\u{1ED9}'
| '\u{1EDB}' | '\u{1EDD}' | '\u{1EDF}' | '\u{1EE1}' | '\u{1EE3}' | '\u{2092}'
| '\u{24DE}' | '\u{2C7A}' | '\u{A74B}' | '\u{A74D}' | '\u{FF4F}' => "o",
'\u{0152}' | '\u{0276}' => "OE",
'\u{A74E}' => "OO",
'\u{0222}' | '\u{1D15}' => "OU",
'\u{24AA}' => "(o)",
'\u{0153}' | '\u{1D14}' => "oe",
'\u{A74F}' => "oo",
'\u{0223}' => "ou",
// P/p
'\u{01A4}' | '\u{1D18}' | '\u{1E54}' | '\u{1E56}' | '\u{24C5}' | '\u{2C63}'
| '\u{A750}' | '\u{A752}' | '\u{A754}' | '\u{FF30}' => "P",
'\u{01A5}' | '\u{1D71}' | '\u{1D7D}' | '\u{1D88}' | '\u{1E55}' | '\u{1E57}'
| '\u{24DF}' | '\u{A751}' | '\u{A753}' | '\u{A755}' | '\u{A7FC}' | '\u{FF50}' => "p",
'\u{24AB}' => "(p)",
// Q/q
'\u{024A}' | '\u{24C6}' | '\u{A756}' | '\u{A758}' | '\u{FF31}' => "Q",
'\u{0138}' | '\u{024B}' | '\u{02A0}' | '\u{24E0}' | '\u{A757}' | '\u{A759}'
| '\u{FF51}' => "q",
'\u{24AC}' => "(q)",
'\u{0239}' => "qp",
// R/r
'\u{0154}' | '\u{0156}' | '\u{0158}' | '\u{0210}' | '\u{0212}' | '\u{024C}'
| '\u{0280}' | '\u{0281}' | '\u{1D19}' | '\u{1D1A}' | '\u{1E58}' | '\u{1E5A}'
| '\u{1E5C}' | '\u{1E5E}' | '\u{24C7}' | '\u{2C64}' | '\u{A75A}' | '\u{A782}'
| '\u{FF32}' => "R",
'\u{0155}' | '\u{0157}' | '\u{0159}' | '\u{0211}' | '\u{0213}' | '\u{024D}'
| '\u{027C}' | '\u{027D}' | '\u{027E}' | '\u{027F}' | '\u{1D63}' | '\u{1D72}'
| '\u{1D73}' | '\u{1D89}' | '\u{1E59}' | '\u{1E5B}' | '\u{1E5D}' | '\u{1E5F}'
| '\u{24E1}' | '\u{A75B}' | '\u{A783}' | '\u{FF52}' => "r",
'\u{24AD}' => "(r)",
// S/s and ß/ẞ and st
'\u{015A}' | '\u{015C}' | '\u{015E}' | '\u{0160}' | '\u{0218}' | '\u{1E60}'
| '\u{1E62}' | '\u{1E64}' | '\u{1E66}' | '\u{1E68}' | '\u{24C8}' | '\u{A731}'
| '\u{A785}' | '\u{FF33}' => "S",
'\u{015B}' | '\u{015D}' | '\u{015F}' | '\u{0161}' | '\u{017F}' | '\u{0219}'
| '\u{023F}' | '\u{0282}' | '\u{1D74}' | '\u{1D8A}' | '\u{1E61}' | '\u{1E63}'
| '\u{1E65}' | '\u{1E67}' | '\u{1E69}' | '\u{1E9C}' | '\u{1E9D}' | '\u{24E2}'
| '\u{A784}' | '\u{FF53}' => "s",
'\u{1E9E}' => "SS",
'\u{24AE}' => "(s)",
'\u{00DF}' => "ss",
'\u{FB06}' => "st",
// T/t, TH/th, TZ
'\u{0162}' | '\u{0164}' | '\u{0166}' | '\u{01AC}' | '\u{01AE}' | '\u{021A}'
| '\u{023E}' | '\u{1D1B}' | '\u{1E6A}' | '\u{1E6C}' | '\u{1E6E}' | '\u{1E70}'
| '\u{24C9}' | '\u{A786}' | '\u{FF34}' => "T",
'\u{0163}' | '\u{0165}' | '\u{0167}' | '\u{01AB}' | '\u{01AD}' | '\u{021B}'
| '\u{0236}' | '\u{0287}' | '\u{0288}' | '\u{1D75}' | '\u{1E6B}' | '\u{1E6D}'
| '\u{1E6F}' | '\u{1E71}' | '\u{1E97}' | '\u{24E3}' | '\u{2C66}' | '\u{FF54}' => "t",
'\u{00DE}' | '\u{A766}' => "TH",
'\u{A728}' => "TZ",
'\u{24AF}' => "(t)",
'\u{02A8}' => "tc",
'\u{00FE}' | '\u{1D7A}' | '\u{A767}' => "th",
'\u{02A6}' => "ts",
'\u{A729}' => "tz",
// U/u
'\u{00D9}' | '\u{00DA}' | '\u{00DB}' | '\u{00DC}' | '\u{0168}' | '\u{016A}'
| '\u{016C}' | '\u{016E}' | '\u{0170}' | '\u{0172}' | '\u{01AF}' | '\u{01D3}'
| '\u{01D5}' | '\u{01D7}' | '\u{01D9}' | '\u{01DB}' | '\u{0214}' | '\u{0216}'
| '\u{0244}' | '\u{1D1C}' | '\u{1D7E}' | '\u{1E72}' | '\u{1E74}' | '\u{1E76}'
| '\u{1E78}' | '\u{1E7A}' | '\u{1EE4}' | '\u{1EE6}' | '\u{1EE8}' | '\u{1EEA}'
| '\u{1EEC}' | '\u{1EEE}' | '\u{1EF0}' | '\u{24CA}' | '\u{FF35}' => "U",
'\u{00F9}' | '\u{00FA}' | '\u{00FB}' | '\u{00FC}' | '\u{0169}' | '\u{016B}'
| '\u{016D}' | '\u{016F}' | '\u{0171}' | '\u{0173}' | '\u{01B0}' | '\u{01D4}'
| '\u{01D6}' | '\u{01D8}' | '\u{01DA}' | '\u{01DC}' | '\u{0215}' | '\u{0217}'
| '\u{0289}' | '\u{1D64}' | '\u{1D99}' | '\u{1E73}' | '\u{1E75}' | '\u{1E77}'
| '\u{1E79}' | '\u{1E7B}' | '\u{1EE5}' | '\u{1EE7}' | '\u{1EE9}' | '\u{1EEB}'
| '\u{1EED}' | '\u{1EEF}' | '\u{1EF1}' | '\u{24E4}' | '\u{FF55}' => "u",
'\u{24B0}' => "(u)",
'\u{1D6B}' => "ue",
// V/v
'\u{01B2}' | '\u{0245}' | '\u{1D20}' | '\u{1E7C}' | '\u{1E7E}' | '\u{1EFC}'
| '\u{24CB}' | '\u{A75E}' | '\u{A768}' | '\u{FF36}' => "V",
'\u{028B}' | '\u{028C}' | '\u{1D65}' | '\u{1D8C}' | '\u{1E7D}' | '\u{1E7F}'
| '\u{24E5}' | '\u{2C71}' | '\u{2C74}' | '\u{A75F}' | '\u{FF56}' => "v",
'\u{A760}' => "VY",
'\u{24B1}' => "(v)",
'\u{A761}' => "vy",
// W/w
'\u{0174}' | '\u{01F7}' | '\u{1D21}' | '\u{1E80}' | '\u{1E82}' | '\u{1E84}'
| '\u{1E86}' | '\u{1E88}' | '\u{24CC}' | '\u{2C72}' | '\u{FF37}' => "W",
'\u{0175}' | '\u{01BF}' | '\u{028D}' | '\u{1E81}' | '\u{1E83}' | '\u{1E85}'
| '\u{1E87}' | '\u{1E89}' | '\u{1E98}' | '\u{24E6}' | '\u{2C73}' | '\u{FF57}' => "w",
'\u{24B2}' => "(w)",
// X/x
'\u{1E8A}' | '\u{1E8C}' | '\u{24CD}' | '\u{FF38}' => "X",
'\u{1D8D}' | '\u{1E8B}' | '\u{1E8D}' | '\u{2093}' | '\u{24E7}' | '\u{FF58}' => "x",
'\u{24B3}' => "(x)",
// Y/y
'\u{00DD}' | '\u{0176}' | '\u{0178}' | '\u{01B3}' | '\u{0232}' | '\u{024E}'
| '\u{028F}' | '\u{1E8E}' | '\u{1EF2}' | '\u{1EF4}' | '\u{1EF6}' | '\u{1EF8}'
| '\u{1EFE}' | '\u{24CE}' | '\u{FF39}' => "Y",
'\u{00FD}' | '\u{00FF}' | '\u{0177}' | '\u{01B4}' | '\u{0233}' | '\u{024F}'
| '\u{028E}' | '\u{1E8F}' | '\u{1E99}' | '\u{1EF3}' | '\u{1EF5}' | '\u{1EF7}'
| '\u{1EF9}' | '\u{1EFF}' | '\u{24E8}' | '\u{FF59}' => "y",
'\u{24B4}' => "(y)",
// Z/z
'\u{0179}' | '\u{017B}' | '\u{017D}' | '\u{01B5}' | '\u{021C}' | '\u{0224}'
| '\u{1D22}' | '\u{1E90}' | '\u{1E92}' | '\u{1E94}' | '\u{24CF}' | '\u{2C6B}'
| '\u{A762}' | '\u{FF3A}' => "Z",
'\u{017A}' | '\u{017C}' | '\u{017E}' | '\u{01B6}' | '\u{021D}' | '\u{0225}'
| '\u{0240}' | '\u{0290}' | '\u{0291}' | '\u{1D76}' | '\u{1D8E}' | '\u{1E91}'
| '\u{1E93}' | '\u{1E95}' | '\u{24E9}' | '\u{2C6C}' | '\u{A763}' | '\u{FF5A}' => "z",
'\u{24B5}' => "(z)",
// Digits and circled/parenthesized forms
'\u{2070}' | '\u{2080}' | '\u{24EA}' | '\u{24FF}' | '\u{FF10}' => "0",
'\u{00B9}' | '\u{2081}' | '\u{2460}' | '\u{24F5}' | '\u{2776}' | '\u{2780}'
| '\u{278A}' | '\u{FF11}' => "1",
'\u{2488}' => "1.",
'\u{2474}' => "(1)",
'\u{00B2}' | '\u{2082}' | '\u{2461}' | '\u{24F6}' | '\u{2777}' | '\u{2781}'
| '\u{278B}' | '\u{FF12}' => "2",
'\u{2489}' => "2.",
'\u{2475}' => "(2)",
'\u{00B3}' | '\u{2083}' | '\u{2462}' | '\u{24F7}' | '\u{2778}' | '\u{2782}'
| '\u{278C}' | '\u{FF13}' => "3",
'\u{248A}' => "3.",
'\u{2476}' => "(3)",
'\u{2074}' | '\u{2084}' | '\u{2463}' | '\u{24F8}' | '\u{2779}' | '\u{2783}'
| '\u{278D}' | '\u{FF14}' => "4",
'\u{248B}' => "4.",
'\u{2477}' => "(4)",
'\u{2075}' | '\u{2085}' | '\u{2464}' | '\u{24F9}' | '\u{277A}' | '\u{2784}'
| '\u{278E}' | '\u{FF15}' => "5",
'\u{248C}' => "5.",
'\u{2478}' => "(5)",
'\u{2076}' | '\u{2086}' | '\u{2465}' | '\u{24FA}' | '\u{277B}' | '\u{2785}'
| '\u{278F}' | '\u{FF16}' => "6",
'\u{248D}' => "6.",
'\u{2479}' => "(6)",
'\u{2077}' | '\u{2087}' | '\u{2466}' | '\u{24FB}' | '\u{277C}' | '\u{2786}'
| '\u{2790}' | '\u{FF17}' => "7",
'\u{248E}' => "7.",
'\u{247A}' => "(7)",
'\u{2078}' | '\u{2088}' | '\u{2467}' | '\u{24FC}' | '\u{277D}' | '\u{2787}'
| '\u{2791}' | '\u{FF18}' => "8",
'\u{248F}' => "8.",
'\u{247B}' => "(8)",
'\u{2079}' | '\u{2089}' | '\u{2468}' | '\u{24FD}' | '\u{277E}' | '\u{2788}'
| '\u{2792}' | '\u{FF19}' => "9",
'\u{2490}' => "9.",
'\u{247C}' => "(9)",
'\u{2469}' | '\u{24FE}' | '\u{277F}' | '\u{2789}' | '\u{2793}' => "10",
'\u{2491}' => "10.",
'\u{247D}' => "(10)",
'\u{246A}' | '\u{24EB}' => "11",
'\u{2492}' => "11.",
'\u{247E}' => "(11)",
'\u{246B}' | '\u{24EC}' => "12",
'\u{2493}' => "12.",
'\u{247F}' => "(12)",
'\u{246C}' | '\u{24ED}' => "13",
'\u{2494}' => "13.",
'\u{2480}' => "(13)",
'\u{246D}' | '\u{24EE}' => "14",
'\u{2495}' => "14.",
'\u{2481}' => "(14)",
'\u{246E}' | '\u{24EF}' => "15",
'\u{2496}' => "15.",
'\u{2482}' => "(15)",
'\u{246F}' | '\u{24F0}' => "16",
'\u{2497}' => "16.",
'\u{2483}' => "(16)",
'\u{2470}' | '\u{24F1}' => "17",
'\u{2498}' => "17.",
'\u{2484}' => "(17)",
'\u{2471}' | '\u{24F2}' => "18",
'\u{2499}' => "18.",
'\u{2485}' => "(18)",
'\u{2472}' | '\u{24F3}' => "19",
'\u{249A}' => "19.",
'\u{2486}' => "(19)",
'\u{2473}' | '\u{24F4}' => "20",
'\u{249B}' => "20.",
'\u{2487}' => "(20)",
// Quotes/dashes and punctuation
'\u{00AB}' | '\u{00BB}' | '\u{201C}' | '\u{201D}' | '\u{201E}' | '\u{2033}'
| '\u{2036}' | '\u{275D}' | '\u{275E}' | '\u{276E}' | '\u{276F}' | '\u{FF02}' => "\"",
'\u{2018}' | '\u{2019}' | '\u{201A}' | '\u{201B}' | '\u{2032}' | '\u{2035}'
| '\u{2039}' | '\u{203A}' | '\u{275B}' | '\u{275C}' | '\u{FF07}' => "'",
'\u{2010}' | '\u{2011}' | '\u{2012}' | '\u{2013}' | '\u{2014}' | '\u{207B}'
| '\u{208B}' | '\u{FF0D}' => "-",
'\u{2045}' | '\u{2772}' | '\u{FF3B}' => "[",
'\u{2046}' | '\u{2773}' | '\u{FF3D}' => "]",
'\u{207D}' | '\u{208D}' | '\u{2768}' | '\u{276A}' | '\u{FF08}' => "(",
'\u{2E28}' => "((",
'\u{207E}' | '\u{208E}' | '\u{2769}' | '\u{276B}' | '\u{FF09}' => ")",
'\u{2E29}' => "))",
'\u{276C}' | '\u{2770}' | '\u{FF1C}' => "<",
'\u{276D}' | '\u{2771}' | '\u{FF1E}' => ">",
'\u{2774}' | '\u{FF5B}' => "{",
'\u{2775}' | '\u{FF5D}' => "}",
'\u{207A}' | '\u{208A}' | '\u{FF0B}' => "+",
'\u{207C}' | '\u{208C}' | '\u{FF1D}' => "=",
'\u{FF01}' => "!",
'\u{203C}' => "!!",
'\u{2049}' => "!?",
'\u{FF03}' => "#",
'\u{FF04}' => "$",
'\u{2052}' | '\u{FF05}' => "%",
'\u{FF06}' => "&",
'\u{204E}' | '\u{FF0A}' => "*",
'\u{FF0C}' => ",",
'\u{FF0E}' => ".",
'\u{2044}' | '\u{FF0F}' => "/",
'\u{FF1A}' => ":",
'\u{204F}' | '\u{FF1B}' => ";",
'\u{FF1F}' => "?",
'\u{2047}' => "??",
'\u{2048}' => "?!",
'\u{FF20}' => "@",
'\u{FF3C}' => "\\",
'\u{2038}' | '\u{FF3E}' => "^",
'\u{FF3F}' => "_",
'\u{2053}' | '\u{FF5E}' => "~",
_ => return None,
};
Some(s)
}
#[cfg(test)]
mod tests {
use super::fold_to_ascii;
#[test]
fn basic_examples() {
assert_eq!(
fold_to_ascii("ação café jalapeño Über"),
"acao cafe jalapeno Uber"
);
assert_eq!(fold_to_ascii("Æsop and Œuvre"), "AEsop and OEuvre");
assert_eq!(fold_to_ascii("straße"), "strasse");
assert_eq!(fold_to_ascii("Łódź"), "Lodz");
assert_eq!(
fold_to_ascii("“quote” — test １②⒊"),
"\"quote\" - test 123."
);
// japanese characters are unchanged
assert_eq!(
fold_to_ascii("日本語のテキストです。Qdrantのコードで単体テストで使用されています。"),
"日本語のテキストです。Qdrantのコードで単体テストで使用されています。"
);
}
}

123
lib/segment/src/index/field_index/full_text_index/tokenizers/mod.rs
View File

@@ -1,5 +1,6 @@
use std::borrow::Cow;
use std::sync::Arc;
mod ascii_folding;
mod japanese;
mod multilingual;
mod stemmer;
@@ -103,11 +104,13 @@ impl PrefixTokenizer {
text.split(|c| !char::is_alphanumeric(c))
.filter(|token| !token.is_empty())
.for_each(|word| {
let word_cow = if tokens_processor.lowercase {
Cow::Owned(word.to_lowercase())
} else {
Cow::Borrowed(word)
};
// Apply ASCII folding if enabled
let mut word_cow = tokens_processor.fold_if_enabled(Cow::Borrowed(word));
// Handle lowercase
if tokens_processor.lowercase {
word_cow = Cow::Owned(word_cow.to_lowercase());
}
let word_cow = tokens_processor.stem_if_enabled(word_cow);
@@ -164,6 +167,7 @@ impl Tokenizer {
min_token_len,
max_token_len,
lowercase,
ascii_folding,
on_disk: _,
phrase_matching: _,
stopwords,
@@ -171,10 +175,12 @@ impl Tokenizer {
} = params;
let lowercase = lowercase.unwrap_or(true);
let ascii_folding = ascii_folding.unwrap_or(false);
let stopwords_filter = Arc::new(StopwordsFilter::new(stopwords, lowercase));
let tokens_processor = TokensProcessor::new(
lowercase,
ascii_folding,
stopwords_filter,
stemmer.as_ref().map(Stemmer::from_algorithm),
*min_token_len,
@@ -226,6 +232,8 @@ impl Tokenizer {
mod tests {
use std::default::Default;
use itertools::Itertools;
use super::*;
use crate::data_types::index::{
Language, Snowball, SnowballLanguage, SnowballParams, StemmingAlgorithm,
@@ -278,7 +286,7 @@ mod tests {
fn test_prefix_tokenizer() {
let text = "hello, мир!";
let tokens_processor =
TokensProcessor::new(true, Default::default(), None, Some(1), Some(4));
TokensProcessor::new(true, false, Default::default(), None, Some(1), Some(4));
let mut tokens = Vec::new();
PrefixTokenizer::tokenize(text, &tokens_processor, |token| tokens.push(token));
@@ -296,7 +304,8 @@ mod tests {
#[test]
fn test_prefix_query_tokenizer() {
let text = "hello, мир!";
let tokens_processor = TokensProcessor::new(true, Default::default(), None, None, Some(4));
let tokens_processor =
TokensProcessor::new(true, false, Default::default(), None, None, Some(4));
let mut tokens = Vec::new();
PrefixTokenizer::tokenize_query(text, &tokens_processor, |token| tokens.push(token));
@@ -324,7 +333,8 @@ mod tests {
// Test stopwords getting applied
let filter =
StopwordsFilter::new(&Some(StopwordsInterface::new_custom(&["の", "は"])), false);
let tokens_processor = TokensProcessor::new(true, Arc::new(filter), None, None, None);
let tokens_processor =
TokensProcessor::new(true, false, Arc::new(filter), None, None, None);
MultilingualTokenizer::tokenize(text, &tokens_processor, |token| tokens.push(token));
eprintln!("tokens = {tokens:#?}");
assert_eq!(tokens.len(), 2);
@@ -349,7 +359,8 @@ mod tests {
// Test stopwords getting applied
let filter = StopwordsFilter::new(&Some(StopwordsInterface::new_custom(&["是"])), false);
let tokens_processor = TokensProcessor::new(true, Arc::new(filter), None, None, None);
let tokens_processor =
TokensProcessor::new(true, false, Arc::new(filter), None, None, None);
MultilingualTokenizer::tokenize(text, &tokens_processor, |token| tokens.push(token));
eprintln!("tokens = {tokens:#?}");
assert_eq!(tokens.len(), 3);
@@ -397,6 +408,7 @@ mod tests {
min_token_len: Some(1),
max_token_len: Some(4),
lowercase: Some(true),
ascii_folding: None,
on_disk: None,
phrase_matching: None,
stopwords: None,
@@ -428,6 +440,7 @@ mod tests {
min_token_len: None,
max_token_len: None,
lowercase: Some(true),
ascii_folding: None,
on_disk: None,
phrase_matching: None,
stopwords: Some(StopwordsInterface::Language(Language::English)),
@@ -470,6 +483,7 @@ mod tests {
min_token_len: None,
max_token_len: None,
lowercase: Some(true),
ascii_folding: None,
on_disk: None,
phrase_matching: None,
stopwords: Some(StopwordsInterface::Language(Language::English)),
@@ -502,6 +516,7 @@ mod tests {
min_token_len: None,
max_token_len: None,
lowercase: Some(true),
ascii_folding: None,
on_disk: None,
phrase_matching: None,
stopwords: Some(StopwordsInterface::new_set(
@@ -540,6 +555,7 @@ mod tests {
min_token_len: None,
max_token_len: None,
lowercase: Some(true),
ascii_folding: None,
on_disk: None,
phrase_matching: None,
stopwords: Some(StopwordsInterface::new_custom(&["as", "the", "a"])),
@@ -578,6 +594,7 @@ mod tests {
min_token_len: None,
max_token_len: None,
lowercase: Some(true),
ascii_folding: None,
on_disk: None,
phrase_matching: None,
stopwords: Some(StopwordsInterface::Language(Language::English)),
@@ -613,6 +630,7 @@ mod tests {
min_token_len: None,
max_token_len: None,
lowercase: Some(true),
ascii_folding: None,
on_disk: None,
phrase_matching: None,
stopwords: Some(StopwordsInterface::new_set(
@@ -657,6 +675,7 @@ mod tests {
min_token_len: None,
max_token_len: None,
lowercase: Some(false), // Case sensitivity is enabled
ascii_folding: None,
on_disk: None,
phrase_matching: None,
stopwords: Some(StopwordsInterface::new_custom(&["the", "The", "LAZY"])),
@@ -684,11 +703,97 @@ mod tests {
assert!(tokens.contains(&Cow::Borrowed("dog")));
}
#[test]
fn test_ascii_folding_word_tokenizer_on_off() {
let text = "ação café jalapeño Über";
let expected_disabled = ["ação", "café", "jalapeño", "über"]
.into_iter()
.map(str::to_string)
.collect_vec();
let expected_enabled = ["acao", "cafe", "jalapeno", "uber"]
.into_iter()
.map(str::to_string)
.collect_vec();
// ascii_folding disabled (default)
let params_disabled = TextIndexParams {
r#type: TextIndexType::Text,
tokenizer: TokenizerType::Word,
min_token_len: None,
max_token_len: None,
lowercase: Some(true),
ascii_folding: Some(false),
on_disk: None,
phrase_matching: None,
stopwords: None,
stemmer: None,
};
let tokenizer_disabled = Tokenizer::new_from_text_index_params(&params_disabled);
let mut tokens_disabled = Vec::new();
tokenizer_disabled.tokenize_doc(text, |token| tokens_disabled.push(token.to_string()));
assert_eq!(tokens_disabled, expected_disabled);
// ascii_folding enabled
let params_enabled = TextIndexParams {
r#type: TextIndexType::Text,
tokenizer: TokenizerType::Word,
min_token_len: None,
max_token_len: None,
lowercase: Some(true),
ascii_folding: Some(true),
on_disk: None,
phrase_matching: None,
stopwords: None,
stemmer: None,
};
let tokenizer_enabled = Tokenizer::new_from_text_index_params(&params_enabled);
let mut tokens_enabled = Vec::new();
tokenizer_enabled.tokenize_doc(text, |token| tokens_enabled.push(token.to_string()));
assert_eq!(tokens_enabled, expected_enabled);
}
#[test]
fn test_ascii_folding_prefix_tokenizer() {
let text = "ação";
// With folding disabled: prefixes should preserve accents
let tokens_processor_disabled =
TokensProcessor::new(true, false, Default::default(), None, Some(1), Some(4));
let mut tokens_disabled = Vec::new();
PrefixTokenizer::tokenize(text, &tokens_processor_disabled, |t| {
tokens_disabled.push(t.to_string())
});
assert!(
tokens_disabled.contains(&"a".to_string())
|| tokens_disabled.contains(&"a".to_string())
);
// Because the first char is 'a', but next prefixes should include accented letters
assert!(
tokens_disabled.iter().any(|t| t.starts_with("aç"))
|| tokens_disabled.iter().any(|t| t.contains('ç'))
);
// With folding enabled: prefixes should be ASCII-only (acao, acao prefixes)
let tokens_processor_enabled =
TokensProcessor::new(true, true, Default::default(), None, Some(1), Some(4));
let mut tokens_enabled = Vec::new();
PrefixTokenizer::tokenize(text, &tokens_processor_enabled, |t| {
tokens_enabled.push(t.to_string())
});
// We expect prefixes like a, ac, aca, acao
assert!(tokens_enabled.contains(&"a".to_string()));
assert!(tokens_enabled.contains(&"ac".to_string()));
assert!(tokens_enabled.contains(&"aca".to_string()));
assert!(tokens_enabled.contains(&"acao".to_string()));
assert!(tokens_enabled.iter().all(|t| t.is_ascii()));
}
#[test]
fn test_stemming_snowball() {
let input = "interestingly proceeding living";
let mut tokens_processor = TokensProcessor::new(
true,
false,
Default::default(),
Some(make_stemmer(SnowballLanguage::English)),
None,

1
lib/segment/src/index/field_index/full_text_index/tokenizers/multilingual.rs
View File

@@ -181,6 +181,7 @@ mod test {
fn test_multilingual_stemming() {
let tokens_processor = TokensProcessor::new(
true,
false,
Default::default(),
Some(Stemmer::from_algorithm(&StemmingAlgorithm::Snowball(
SnowballParams {

30
lib/segment/src/index/field_index/full_text_index/tokenizers/tokens_processor.rs
View File

@@ -8,6 +8,7 @@ use crate::index::field_index::full_text_index::stop_words::StopwordsFilter;
#[derive(Debug, Clone, Default)]
pub struct TokensProcessor {
pub lowercase: bool,
pub ascii_folding: bool,
stopwords_filter: Arc<StopwordsFilter>, // TDOO(rocksdb): Remove once rocksdb has been removed!
stemmer: Option<Stemmer>,
pub min_token_len: Option<usize>,
@@ -17,6 +18,7 @@ pub struct TokensProcessor {
impl TokensProcessor {
pub fn new(
lowercase: bool,
ascii_folding: bool,
stopwords_filter: Arc<StopwordsFilter>,
stemmer: Option<Stemmer>,
min_token_len: Option<usize>,
@@ -24,6 +26,7 @@ impl TokensProcessor {
) -> Self {
Self {
lowercase,
ascii_folding,
stopwords_filter,
stemmer,
min_token_len,
@@ -46,13 +49,22 @@ impl TokensProcessor {
stemmer.stem(input)
}
/// Applies ASCII folding if enabled. Converts accented characters to their ASCII equivalents.
pub fn fold_if_enabled<'a>(&self, input: Cow<'a, str>) -> Cow<'a, str> {
if self.ascii_folding {
super::ascii_folding::fold_to_ascii_cow(input)
} else {
input
}
}
pub fn is_stopword(&self, token: &str) -> bool {
self.stopwords_filter.is_stopword(token)
}
pub fn process_token_cow<'a>(
&self,
token: Cow<'a, str>,
mut token_cow: Cow<'a, str>,
check_max_len: bool,
) -> Option<Cow<'a, str>> {
let Self {
@@ -61,18 +73,22 @@ impl TokensProcessor {
stemmer,
min_token_len,
max_token_len,
ascii_folding,
} = self;
if token.is_empty() {
if token_cow.is_empty() {
return None;
}
// Handle ASCII folding (normalize accents)
if *ascii_folding {
token_cow = super::ascii_folding::fold_to_ascii_cow(token_cow);
}
// Handle lowercase
let mut token_cow = if *lowercase {
Cow::Owned(token.to_lowercase())
} else {
token
};
if *lowercase {
token_cow = Cow::Owned(token_cow.to_lowercase());
}
// Handle stopwords
if stopwords_filter.is_stopword(&token_cow) {

4
src/common/inference/bm25.rs
View File

@@ -102,6 +102,7 @@ fn new_token_processor_from_config(value: TextPreprocessingConfig) -> TokensProc
let TextPreprocessingConfig {
language,
lowercase,
ascii_folding,
stopwords,
stemmer,
min_token_len,
@@ -110,6 +111,8 @@ fn new_token_processor_from_config(value: TextPreprocessingConfig) -> TokensProc
let lowercase = lowercase.unwrap_or(true);
let ascii_folding = ascii_folding.unwrap_or(false);
let language = language.unwrap_or_else(|| DEFAULT_LANGUAGE.to_string());
let stemmer = match stemmer {
@@ -129,6 +132,7 @@ fn new_token_processor_from_config(value: TextPreprocessingConfig) -> TokensProc
TokensProcessor::new(
lowercase,
ascii_folding,
Arc::new(StopwordsFilter::new(&stopwords_config, lowercase)),
stemmer,
min_token_len,

1
tools/codespell.toml
View File

@@ -5,6 +5,7 @@
skip = [
"**/stop_words/*.rs",
"./lib/segment/src/index/field_index/full_text_index/tokenizers/ascii_folding.rs", # contains character mappings
"./static",
"./tests/e2e_tests/test_data/cert/key.pem",
"openapi.json", # openapi.json is typically a released version and could carry old typos