From d6d52d4b81287b95d5a8bb99ebe8bbaa562b1fb3 Mon Sep 17 00:00:00 2001 From: eltu Date: Thu, 16 Oct 2025 08:53:01 -0300 Subject: [PATCH] Full-Text Index ASCII Folding (Normalization) (#7408) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add ASCII folding to tokenization process Introduced an optional ASCII folding feature within the `TokensProcessor` to normalize non-ASCII characters to their ASCII equivalents. Updated tests and documentation to reflect the changes. * Refactor tokenization code for improved readability and maintainability Reorganized and reformatted the tokenization module, including `TokensProcessor` initialization and ASCII folding mappings for better clarity. Updated tests to align with the changes. * Update test cases to reflect optional tokenizer settings changes Adjusted `ascii_folding`, `lowercase`, and `phrase_matching` settings in tests to `None` where applicable, aligning with updates in tokenizer configuration defaults. * address review remarks * fix codespell * thx coderabbit * Don't copy tokens that are already ASCII * Shrink folded string to fit --------- Co-authored-by: Luis Cossío Co-authored-by: timvisee --- docs/grpc/docs.md | 1 + docs/redoc/master/openapi.json | 10 + lib/api/src/grpc/conversions.rs | 4 + lib/api/src/grpc/proto/collections.proto | 1 + lib/api/src/grpc/qdrant.rs | 3 + lib/api/src/rest/schema.rs | 4 + lib/segment/src/data_types/index.rs | 4 + .../full_text_index/mutable_text_index.rs | 1 + .../field_index/full_text_index/tests/mod.rs | 90 ++++ .../tokenizers/ascii_folding.rs | 486 ++++++++++++++++++ .../full_text_index/tokenizers/mod.rs | 123 ++++- .../tokenizers/multilingual.rs | 1 + .../tokenizers/tokens_processor.rs | 30 +- src/common/inference/bm25.rs | 4 + tools/codespell.toml | 1 + 15 files changed, 747 insertions(+), 16 deletions(-) create mode 100644 lib/segment/src/index/field_index/full_text_index/tokenizers/ascii_folding.rs diff --git a/docs/grpc/docs.md b/docs/grpc/docs.md index a7ec3beb7e..d5946aa284 100644 --- a/docs/grpc/docs.md +++ b/docs/grpc/docs.md @@ -1729,6 +1729,7 @@ Note: 1kB = 1 vector of size 256. | | stopwords | [StopwordsSet](#qdrant-StopwordsSet) | optional | Stopwords for the text index | | phrase_matching | [bool](#bool) | optional | If true - support phrase matching. | | stemmer | [StemmingAlgorithm](#qdrant-StemmingAlgorithm) | optional | Set an algorithm for stemming. | +| ascii_folding | [bool](#bool) | optional | If true, normalize tokens by folding accented characters to ASCII (e.g., "ação" -> "acao"). Default: false. | diff --git a/docs/redoc/master/openapi.json b/docs/redoc/master/openapi.json index 966df4a7c4..29014c7d47 100644 --- a/docs/redoc/master/openapi.json +++ b/docs/redoc/master/openapi.json @@ -7748,6 +7748,11 @@ "type": "boolean", "nullable": true }, + "ascii_folding": { + "description": "If true, normalize tokens by folding accented characters to ASCII (e.g., \"ação\" -> \"acao\"). Default: false.", + "type": "boolean", + "nullable": true + }, "phrase_matching": { "description": "If true, support phrase matching. Default: false.", "type": "boolean", @@ -10700,6 +10705,11 @@ "type": "boolean", "nullable": true }, + "ascii_folding": { + "description": "If true, normalize tokens by folding accented characters to ASCII (e.g., \"ação\" -> \"acao\"). Default is `false`.", + "type": "boolean", + "nullable": true + }, "stopwords": { "description": "Configuration of the stopwords filter. Supports list of pre-defined languages and custom stopwords. Default: initialized for specified `language` or English if not specified.", "anyOf": [ diff --git a/lib/api/src/grpc/conversions.rs b/lib/api/src/grpc/conversions.rs index 48e9f49471..68beb2a807 100644 --- a/lib/api/src/grpc/conversions.rs +++ b/lib/api/src/grpc/conversions.rs @@ -215,6 +215,7 @@ impl From for PayloadIndexParams { min_token_len, max_token_len, lowercase, + ascii_folding, phrase_matching, on_disk, stopwords, @@ -231,6 +232,7 @@ impl From for PayloadIndexParams { index_params: Some(IndexParams::TextIndexParams(TextIndexParams { tokenizer: tokenizer as i32, lowercase, + ascii_folding, min_token_len: min_token_len.map(|x| x as u64), max_token_len: max_token_len.map(|x| x as u64), phrase_matching, @@ -503,6 +505,7 @@ impl TryFrom for segment::data_types::index::TextIndexParams { let TextIndexParams { tokenizer, lowercase, + ascii_folding, min_token_len, max_token_len, phrase_matching, @@ -531,6 +534,7 @@ impl TryFrom for segment::data_types::index::TextIndexParams { .map(|x| x.try_into()) .unwrap_or_else(|_| Err(Status::invalid_argument("unknown tokenizer type")))?, lowercase, + ascii_folding, min_token_len: min_token_len.map(|x| x as usize), max_token_len: max_token_len.map(|x| x as usize), phrase_matching, diff --git a/lib/api/src/grpc/proto/collections.proto b/lib/api/src/grpc/proto/collections.proto index 98e9f28440..d6f4fdea1d 100644 --- a/lib/api/src/grpc/proto/collections.proto +++ b/lib/api/src/grpc/proto/collections.proto @@ -520,6 +520,7 @@ message TextIndexParams { optional StopwordsSet stopwords = 6; // Stopwords for the text index optional bool phrase_matching = 7; // If true - support phrase matching. optional StemmingAlgorithm stemmer = 8; // Set an algorithm for stemming. + optional bool ascii_folding = 9; // If true, normalize tokens by folding accented characters to ASCII (e.g., "ação" -> "acao"). Default: false. } message StemmingAlgorithm { diff --git a/lib/api/src/grpc/qdrant.rs b/lib/api/src/grpc/qdrant.rs index a089269386..41c3589e46 100644 --- a/lib/api/src/grpc/qdrant.rs +++ b/lib/api/src/grpc/qdrant.rs @@ -1097,6 +1097,9 @@ pub struct TextIndexParams { /// Set an algorithm for stemming. #[prost(message, optional, tag = "8")] pub stemmer: ::core::option::Option, + /// If true, normalize tokens by folding accented characters to ASCII (e.g., "ação" -> "acao"). Default: false. + #[prost(bool, optional, tag = "9")] + pub ascii_folding: ::core::option::Option, } #[derive(serde::Serialize)] #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/lib/api/src/rest/schema.rs b/lib/api/src/rest/schema.rs index 70a08e42bf..3112a1dff9 100644 --- a/lib/api/src/rest/schema.rs +++ b/lib/api/src/rest/schema.rs @@ -223,6 +223,10 @@ pub struct TextPreprocessingConfig { /// Default is `true`. #[serde(default, skip_serializing_if = "Option::is_none")] pub lowercase: Option, + /// If true, normalize tokens by folding accented characters to ASCII (e.g., "ação" -> "acao"). + /// Default is `false`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ascii_folding: Option, /// Configuration of the stopwords filter. Supports list of pre-defined languages and custom stopwords. /// Default: initialized for specified `language` or English if not specified. #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/lib/segment/src/data_types/index.rs b/lib/segment/src/data_types/index.rs index b3fb924b0f..1ae9b6031a 100644 --- a/lib/segment/src/data_types/index.rs +++ b/lib/segment/src/data_types/index.rs @@ -199,6 +199,10 @@ pub struct TextIndexParams { #[serde(default, skip_serializing_if = "Option::is_none")] pub lowercase: Option, + /// If true, normalize tokens by folding accented characters to ASCII (e.g., "ação" -> "acao"). Default: false. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ascii_folding: Option, + /// If true, support phrase matching. Default: false. #[serde(default, skip_serializing_if = "Option::is_none")] pub phrase_matching: Option, diff --git a/lib/segment/src/index/field_index/full_text_index/mutable_text_index.rs b/lib/segment/src/index/field_index/full_text_index/mutable_text_index.rs index da33513ca2..59efcc4f72 100644 --- a/lib/segment/src/index/field_index/full_text_index/mutable_text_index.rs +++ b/lib/segment/src/index/field_index/full_text_index/mutable_text_index.rs @@ -413,6 +413,7 @@ mod tests { on_disk: None, stopwords: None, stemmer: None, + ascii_folding: None, }; { diff --git a/lib/segment/src/index/field_index/full_text_index/tests/mod.rs b/lib/segment/src/index/field_index/full_text_index/tests/mod.rs index bf7a812a34..871c797d9a 100644 --- a/lib/segment/src/index/field_index/full_text_index/tests/mod.rs +++ b/lib/segment/src/index/field_index/full_text_index/tests/mod.rs @@ -164,6 +164,7 @@ fn test_prefix_search() { stopwords: None, on_disk: None, stemmer: None, + ascii_folding: None, }; let mut index = @@ -213,6 +214,7 @@ fn test_phrase_matching() { phrase_matching: Some(true), // Enable phrase matching stopwords: None, stemmer: None, + ascii_folding: None, }; let mut mutable_index = @@ -305,3 +307,91 @@ fn test_phrase_matching() { check_matching(mutable_index); check_matching(mmap_index); } + +#[test] +fn test_ascii_folding_in_full_text_index_word() { + let hw_counter = HardwareCounterCell::default(); + + let temp_dir = Builder::new().prefix("test_dir").tempdir().unwrap(); + let config_enabled = TextIndexParams { + r#type: TextIndexType::Text, + tokenizer: TokenizerType::Word, + min_token_len: None, + max_token_len: None, + lowercase: None, + on_disk: None, + phrase_matching: None, + stopwords: None, + stemmer: None, + ascii_folding: Some(true), + }; + let config_disabled = TextIndexParams { + ascii_folding: Some(false), + ..config_enabled.clone() + }; + + // Index with folding enabled + let mut index_enabled = + FullTextIndex::new_gridstore(temp_dir.path().to_path_buf(), config_enabled.clone(), true) + .unwrap() + .unwrap(); + + // Index with folding disabled (separate storage path) + let temp_dir2 = Builder::new().prefix("test_dir").tempdir().unwrap(); + let mut index_disabled = FullTextIndex::new_gridstore( + temp_dir2.path().to_path_buf(), + config_disabled.clone(), + true, + ) + .unwrap() + .unwrap(); + + // Documents containing accents + let docs = vec![ + (0, "ação no coração".to_string()), + (1, "café com leite".to_string()), + ]; + + for (id, text) in &docs { + index_enabled + .add_many(*id as PointOffsetType, vec![text.clone()], &hw_counter) + .unwrap(); + index_disabled + .add_many(*id as PointOffsetType, vec![text.clone()], &hw_counter) + .unwrap(); + } + + // ASCII-only queries should match only when folding is enabled + let query_enabled = index_enabled.parse_text_query("acao", &hw_counter).unwrap(); + assert!(index_enabled.check_match(&query_enabled, 0)); + + let results_enabled: Vec<_> = index_enabled + .filter_query(query_enabled, &hw_counter) + .collect(); + assert!(results_enabled.contains(&0)); + + let query_disabled_opt = index_disabled.parse_text_query("acao", &hw_counter); + // Query might still parse, but should not match anything + if let Some(query_disabled) = query_disabled_opt { + let results_disabled: Vec<_> = index_disabled + .filter_query(query_disabled, &hw_counter) + .collect(); + assert!(!results_disabled.contains(&0)); + } + + // Non-folded query must work in both + let query_acento = index_enabled.parse_text_query("ação", &hw_counter).unwrap(); + assert!(index_enabled.check_match(&query_acento, 0)); + let results_acento: Vec<_> = index_enabled + .filter_query(query_acento, &hw_counter) + .collect(); + assert!(results_acento.contains(&0)); + + let query_acento2 = index_disabled + .parse_text_query("ação", &hw_counter) + .unwrap(); + let results_acento2: Vec<_> = index_disabled + .filter_query(query_acento2, &hw_counter) + .collect(); + assert!(results_acento2.contains(&0)); +} diff --git a/lib/segment/src/index/field_index/full_text_index/tokenizers/ascii_folding.rs b/lib/segment/src/index/field_index/full_text_index/tokenizers/ascii_folding.rs new file mode 100644 index 0000000000..45c1da9e8b --- /dev/null +++ b/lib/segment/src/index/field_index/full_text_index/tokenizers/ascii_folding.rs @@ -0,0 +1,486 @@ +// ASCII folding implementation inspired by Lucene's ASCIIFoldingFilter. +// Converts non-ASCII Latin letters and various symbols to ASCII equivalents. +// This aims to be compatible with Lucene's mapping list used in ASCIIFoldingFilter. + +use std::borrow::Cow; + +pub fn fold_to_ascii_cow<'a>(input: Cow<'a, str>) -> Cow<'a, str> { + // Cheap check if already ASCII + if input.is_ascii() { + return input; + } + + Cow::Owned(fold_to_ascii(input.as_ref())) +} + +pub fn fold_to_ascii(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + for ch in input.chars() { + if ch.is_ascii() { + out.push(ch); + continue; + } + match fold_char(ch) { + Some(repl) => out.push_str(repl), + None => out.push(ch), + } + } + out.shrink_to_fit(); + out +} + +#[inline] +fn fold_char(c: char) -> Option<&'static str> { + // Quick test + if c < '\u{0080}' { + return None; + } + + // Main mapping translated from Lucene's ASCIIFoldingFilter + let s = match c { + // A/a + '\u{00C0}' | '\u{00C1}' | '\u{00C2}' | '\u{00C3}' | '\u{00C4}' | '\u{00C5}' + | '\u{0100}' | '\u{0102}' | '\u{0104}' | '\u{01CD}' | '\u{01DE}' | '\u{01E0}' + | '\u{01FA}' | '\u{0200}' | '\u{0202}' | '\u{0226}' | '\u{023A}' | '\u{1D00}' + | '\u{1E00}' | '\u{1EA0}' | '\u{1EA2}' | '\u{1EA4}' | '\u{1EA6}' | '\u{1EA8}' + | '\u{1EAA}' | '\u{1EAC}' | '\u{1EAE}' | '\u{1EB0}' | '\u{1EB2}' | '\u{1EB4}' + | '\u{1EB6}' | '\u{24B6}' | '\u{FF21}' => "A", + '\u{00E0}' | '\u{00E1}' | '\u{00E2}' | '\u{00E3}' | '\u{00E4}' | '\u{00E5}' + | '\u{0101}' | '\u{0103}' | '\u{0105}' | '\u{01CE}' | '\u{01DF}' | '\u{01E1}' + | '\u{01FB}' | '\u{0201}' | '\u{0203}' | '\u{0227}' | '\u{0250}' | '\u{0259}' + | '\u{025A}' | '\u{1D8F}' | '\u{1D95}' | '\u{1E01}' | '\u{1E9A}' | '\u{1EA1}' + | '\u{1EA3}' | '\u{1EA5}' | '\u{1EA7}' | '\u{1EA9}' | '\u{1EAB}' | '\u{1EAD}' + | '\u{1EAF}' | '\u{1EB1}' | '\u{1EB3}' | '\u{1EB5}' | '\u{1EB7}' | '\u{2090}' + | '\u{2094}' | '\u{24D0}' | '\u{2C65}' | '\u{2C6F}' | '\u{FF41}' => "a", + '\u{A732}' => "AA", // Ꜳ + '\u{00C6}' | '\u{01E2}' | '\u{01FC}' | '\u{1D01}' => "AE", + '\u{A734}' => "AO", + '\u{A736}' => "AU", + '\u{A738}' | '\u{A73A}' => "AV", + '\u{A73C}' => "AY", + '\u{249C}' => "(a)", + '\u{A733}' => "aa", + '\u{00E6}' | '\u{01E3}' | '\u{01FD}' | '\u{1D02}' => "ae", + '\u{A735}' => "ao", + '\u{A737}' => "au", + '\u{A739}' | '\u{A73B}' => "av", + '\u{A73D}' => "ay", + + // B/b + '\u{0181}' | '\u{0182}' | '\u{0243}' | '\u{0299}' | '\u{1D03}' | '\u{1E02}' + | '\u{1E04}' | '\u{1E06}' | '\u{24B7}' | '\u{FF22}' => "B", + '\u{0180}' | '\u{0183}' | '\u{0253}' | '\u{1D6C}' | '\u{1D80}' | '\u{1E03}' + | '\u{1E05}' | '\u{1E07}' | '\u{24D1}' | '\u{FF42}' => "b", + '\u{249D}' => "(b)", + + // C/c + '\u{00C7}' | '\u{0106}' | '\u{0108}' | '\u{010A}' | '\u{010C}' | '\u{0187}' + | '\u{023B}' | '\u{0297}' | '\u{1D04}' | '\u{1E08}' | '\u{24B8}' | '\u{FF23}' => "C", + '\u{00E7}' | '\u{0107}' | '\u{0109}' | '\u{010B}' | '\u{010D}' | '\u{0188}' + | '\u{023C}' | '\u{0255}' | '\u{1E09}' | '\u{2184}' | '\u{24D2}' | '\u{A73E}' + | '\u{A73F}' | '\u{FF43}' => "c", + '\u{249E}' => "(c)", + + // D/d + '\u{00D0}' | '\u{010E}' | '\u{0110}' | '\u{0189}' | '\u{018A}' | '\u{018B}' + | '\u{1D05}' | '\u{1D06}' | '\u{1E0A}' | '\u{1E0C}' | '\u{1E0E}' | '\u{1E10}' + | '\u{1E12}' | '\u{24B9}' | '\u{A779}' | '\u{FF24}' => "D", + '\u{00F0}' | '\u{010F}' | '\u{0111}' | '\u{018C}' | '\u{0221}' | '\u{0256}' + | '\u{0257}' | '\u{1D6D}' | '\u{1D81}' | '\u{1D91}' | '\u{1E0B}' | '\u{1E0D}' + | '\u{1E0F}' | '\u{1E11}' | '\u{1E13}' | '\u{24D3}' | '\u{A77A}' | '\u{FF44}' => "d", + '\u{01C4}' | '\u{01F1}' => "DZ", + '\u{01C5}' | '\u{01F2}' => "Dz", + '\u{249F}' => "(d)", + '\u{0238}' => "db", + '\u{01C6}' | '\u{01F3}' | '\u{02A3}' | '\u{02A5}' => "dz", + + // E/e + '\u{00C8}' | '\u{00C9}' | '\u{00CA}' | '\u{00CB}' | '\u{0112}' | '\u{0114}' + | '\u{0116}' | '\u{0118}' | '\u{011A}' | '\u{018E}' | '\u{0190}' | '\u{0204}' + | '\u{0206}' | '\u{0228}' | '\u{0246}' | '\u{1D07}' | '\u{1E14}' | '\u{1E16}' + | '\u{1E18}' | '\u{1E1A}' | '\u{1E1C}' | '\u{1EB8}' | '\u{1EBA}' | '\u{1EBC}' + | '\u{1EBE}' | '\u{1EC0}' | '\u{1EC2}' | '\u{1EC4}' | '\u{1EC6}' | '\u{24BA}' + | '\u{2C7B}' | '\u{FF25}' => "E", + '\u{00E8}' | '\u{00E9}' | '\u{00EA}' | '\u{00EB}' | '\u{0113}' | '\u{0115}' + | '\u{0117}' | '\u{0119}' | '\u{011B}' | '\u{01DD}' | '\u{0205}' | '\u{0207}' + | '\u{0229}' | '\u{0247}' | '\u{0258}' | '\u{025B}' | '\u{025C}' | '\u{025D}' + | '\u{025E}' | '\u{029A}' | '\u{1D08}' | '\u{1D92}' | '\u{1D93}' | '\u{1D94}' + | '\u{1E15}' | '\u{1E17}' | '\u{1E19}' | '\u{1E1B}' | '\u{1E1D}' | '\u{1EB9}' + | '\u{1EBB}' | '\u{1EBD}' | '\u{1EBF}' | '\u{1EC1}' | '\u{1EC3}' | '\u{1EC5}' + | '\u{1EC7}' | '\u{2091}' | '\u{24D4}' | '\u{2C78}' | '\u{FF45}' => "e", + '\u{24A0}' => "(e)", + + // F/f and ligatures + '\u{0191}' | '\u{1E1E}' | '\u{24BB}' | '\u{A730}' | '\u{A77B}' | '\u{A7FB}' + | '\u{FF26}' => "F", + '\u{0192}' | '\u{1D6E}' | '\u{1D82}' | '\u{1E1F}' | '\u{1E9B}' | '\u{24D5}' + | '\u{A77C}' | '\u{FF46}' => "f", + '\u{24A1}' => "(f)", + '\u{FB00}' => "ff", + '\u{FB03}' => "ffi", + '\u{FB04}' => "ffl", + '\u{FB01}' => "fi", + '\u{FB02}' => "fl", + + // G/g + '\u{011C}' | '\u{011E}' | '\u{0120}' | '\u{0122}' | '\u{0193}' | '\u{01E4}' + | '\u{01E6}' | '\u{01F4}' | '\u{0262}' | '\u{029B}' | '\u{1E20}' | '\u{24BC}' + | '\u{A77D}' | '\u{A77E}' | '\u{FF27}' => "G", + '\u{011D}' | '\u{011F}' | '\u{0121}' | '\u{0123}' | '\u{01E5}' | '\u{01E7}' + | '\u{01F5}' | '\u{0260}' | '\u{0261}' | '\u{1D77}' | '\u{1D79}' | '\u{1D83}' + | '\u{1E21}' | '\u{24D6}' | '\u{A77F}' | '\u{FF47}' => "g", + '\u{24A2}' => "(g)", + + // H/h, HV + '\u{0124}' | '\u{0126}' | '\u{021E}' | '\u{029C}' | '\u{1E22}' | '\u{1E24}' + | '\u{1E26}' | '\u{1E28}' | '\u{1E2A}' | '\u{24BD}' | '\u{2C67}' | '\u{2C75}' + | '\u{FF28}' => "H", + '\u{0125}' | '\u{0127}' | '\u{021F}' | '\u{0265}' | '\u{0266}' | '\u{02AE}' + | '\u{02AF}' | '\u{1E23}' | '\u{1E25}' | '\u{1E27}' | '\u{1E29}' | '\u{1E2B}' + | '\u{1E96}' | '\u{24D7}' | '\u{2C68}' | '\u{2C76}' | '\u{FF48}' => "h", + '\u{01F6}' => "HV", + '\u{24A3}' => "(h)", + '\u{0195}' => "hv", + + // I/i, IJ + '\u{00CC}' | '\u{00CD}' | '\u{00CE}' | '\u{00CF}' | '\u{0128}' | '\u{012A}' + | '\u{012C}' | '\u{012E}' | '\u{0130}' | '\u{0196}' | '\u{0197}' | '\u{01CF}' + | '\u{0208}' | '\u{020A}' | '\u{026A}' | '\u{1D7B}' | '\u{1E2C}' | '\u{1E2E}' + | '\u{1EC8}' | '\u{1ECA}' | '\u{24BE}' | '\u{A7FE}' | '\u{FF29}' => "I", + '\u{00EC}' | '\u{00ED}' | '\u{00EE}' | '\u{00EF}' | '\u{0129}' | '\u{012B}' + | '\u{012D}' | '\u{012F}' | '\u{0131}' | '\u{01D0}' | '\u{0209}' | '\u{020B}' + | '\u{0268}' | '\u{1D09}' | '\u{1D62}' | '\u{1D7C}' | '\u{1D96}' | '\u{1E2D}' + | '\u{1E2F}' | '\u{1EC9}' | '\u{1ECB}' | '\u{2071}' | '\u{24D8}' | '\u{FF49}' => "i", + '\u{0132}' => "IJ", + '\u{24A4}' => "(i)", + '\u{0133}' => "ij", + + // J/j + '\u{0134}' | '\u{0248}' | '\u{1D0A}' | '\u{24BF}' | '\u{FF2A}' => "J", + '\u{0135}' | '\u{01F0}' | '\u{0237}' | '\u{0249}' | '\u{025F}' | '\u{0284}' + | '\u{029D}' | '\u{24D9}' | '\u{2C7C}' | '\u{FF4A}' => "j", + '\u{24A5}' => "(j)", + + // K/k + '\u{0136}' | '\u{0198}' | '\u{01E8}' | '\u{1D0B}' | '\u{1E30}' | '\u{1E32}' + | '\u{1E34}' | '\u{24C0}' | '\u{2C69}' | '\u{A740}' | '\u{A742}' | '\u{A744}' + | '\u{FF2B}' => "K", + '\u{0137}' | '\u{0199}' | '\u{01E9}' | '\u{029E}' | '\u{1D84}' | '\u{1E31}' + | '\u{1E33}' | '\u{1E35}' | '\u{24DA}' | '\u{2C6A}' | '\u{A741}' | '\u{A743}' + | '\u{A745}' | '\u{FF4B}' => "k", + '\u{24A6}' => "(k)", + + // L/l, LJ/ll etc. + '\u{0139}' | '\u{013B}' | '\u{013D}' | '\u{013F}' | '\u{0141}' | '\u{023D}' + | '\u{029F}' | '\u{1D0C}' | '\u{1E36}' | '\u{1E38}' | '\u{1E3A}' | '\u{1E3C}' + | '\u{24C1}' | '\u{2C60}' | '\u{2C62}' | '\u{A746}' | '\u{A748}' | '\u{A780}' + | '\u{FF2C}' => "L", + '\u{013A}' | '\u{013C}' | '\u{013E}' | '\u{0140}' | '\u{0142}' | '\u{019A}' + | '\u{0234}' | '\u{026B}' | '\u{026C}' | '\u{026D}' | '\u{1D85}' | '\u{1E37}' + | '\u{1E39}' | '\u{1E3B}' | '\u{1E3D}' | '\u{24DB}' | '\u{2C61}' | '\u{A747}' + | '\u{A749}' | '\u{A781}' | '\u{FF4C}' => "l", + '\u{01C7}' => "LJ", + '\u{1EFA}' => "LL", + '\u{01C8}' => "Lj", + '\u{24A7}' => "(l)", + '\u{01C9}' => "lj", + '\u{1EFB}' => "ll", + '\u{02AA}' => "ls", + '\u{02AB}' => "lz", + + // M/m + '\u{019C}' | '\u{1D0D}' | '\u{1E3E}' | '\u{1E40}' | '\u{1E42}' | '\u{24C2}' + | '\u{2C6E}' | '\u{A7FD}' | '\u{A7FF}' | '\u{FF2D}' => "M", + '\u{026F}' | '\u{0270}' | '\u{0271}' | '\u{1D6F}' | '\u{1D86}' | '\u{1E3F}' + | '\u{1E41}' | '\u{1E43}' | '\u{24DC}' | '\u{FF4D}' => "m", + '\u{24A8}' => "(m)", + + // N/n, NJ + '\u{00D1}' | '\u{0143}' | '\u{0145}' | '\u{0147}' | '\u{014A}' | '\u{019D}' + | '\u{01F8}' | '\u{0220}' | '\u{0274}' | '\u{1D0E}' | '\u{1E44}' | '\u{1E46}' + | '\u{1E48}' | '\u{1E4A}' | '\u{24C3}' | '\u{FF2E}' => "N", + '\u{00F1}' | '\u{0144}' | '\u{0146}' | '\u{0148}' | '\u{0149}' | '\u{014B}' + | '\u{019E}' | '\u{01F9}' | '\u{0235}' | '\u{0272}' | '\u{0273}' | '\u{1D70}' + | '\u{1D87}' | '\u{1E45}' | '\u{1E47}' | '\u{1E49}' | '\u{1E4B}' | '\u{207F}' + | '\u{24DD}' | '\u{FF4E}' => "n", + '\u{01CA}' => "NJ", + '\u{01CB}' => "Nj", + '\u{24A9}' => "(n)", + '\u{01CC}' => "nj", + + // O/o, OE/OO/OU + '\u{00D2}' | '\u{00D3}' | '\u{00D4}' | '\u{00D5}' | '\u{00D6}' | '\u{00D8}' + | '\u{014C}' | '\u{014E}' | '\u{0150}' | '\u{0186}' | '\u{019F}' | '\u{01A0}' + | '\u{01D1}' | '\u{01EA}' | '\u{01EC}' | '\u{01FE}' | '\u{020C}' | '\u{020E}' + | '\u{022A}' | '\u{022C}' | '\u{022E}' | '\u{0230}' | '\u{1D0F}' | '\u{1D10}' + | '\u{1E4C}' | '\u{1E4E}' | '\u{1E50}' | '\u{1E52}' | '\u{1ECC}' | '\u{1ECE}' + | '\u{1ED0}' | '\u{1ED2}' | '\u{1ED4}' | '\u{1ED6}' | '\u{1ED8}' | '\u{1EDA}' + | '\u{1EDC}' | '\u{1EDE}' | '\u{1EE0}' | '\u{1EE2}' | '\u{24C4}' | '\u{A74A}' + | '\u{A74C}' | '\u{FF2F}' => "O", + '\u{00F2}' | '\u{00F3}' | '\u{00F4}' | '\u{00F5}' | '\u{00F6}' | '\u{00F8}' + | '\u{014D}' | '\u{014F}' | '\u{0151}' | '\u{01A1}' | '\u{01D2}' | '\u{01EB}' + | '\u{01ED}' | '\u{01FF}' | '\u{020D}' | '\u{020F}' | '\u{022B}' | '\u{022D}' + | '\u{022F}' | '\u{0231}' | '\u{0254}' | '\u{0275}' | '\u{1D16}' | '\u{1D17}' + | '\u{1D97}' | '\u{1E4D}' | '\u{1E4F}' | '\u{1E51}' | '\u{1E53}' | '\u{1ECD}' + | '\u{1ECF}' | '\u{1ED1}' | '\u{1ED3}' | '\u{1ED5}' | '\u{1ED7}' | '\u{1ED9}' + | '\u{1EDB}' | '\u{1EDD}' | '\u{1EDF}' | '\u{1EE1}' | '\u{1EE3}' | '\u{2092}' + | '\u{24DE}' | '\u{2C7A}' | '\u{A74B}' | '\u{A74D}' | '\u{FF4F}' => "o", + '\u{0152}' | '\u{0276}' => "OE", + '\u{A74E}' => "OO", + '\u{0222}' | '\u{1D15}' => "OU", + '\u{24AA}' => "(o)", + '\u{0153}' | '\u{1D14}' => "oe", + '\u{A74F}' => "oo", + '\u{0223}' => "ou", + + // P/p + '\u{01A4}' | '\u{1D18}' | '\u{1E54}' | '\u{1E56}' | '\u{24C5}' | '\u{2C63}' + | '\u{A750}' | '\u{A752}' | '\u{A754}' | '\u{FF30}' => "P", + '\u{01A5}' | '\u{1D71}' | '\u{1D7D}' | '\u{1D88}' | '\u{1E55}' | '\u{1E57}' + | '\u{24DF}' | '\u{A751}' | '\u{A753}' | '\u{A755}' | '\u{A7FC}' | '\u{FF50}' => "p", + '\u{24AB}' => "(p)", + + // Q/q + '\u{024A}' | '\u{24C6}' | '\u{A756}' | '\u{A758}' | '\u{FF31}' => "Q", + '\u{0138}' | '\u{024B}' | '\u{02A0}' | '\u{24E0}' | '\u{A757}' | '\u{A759}' + | '\u{FF51}' => "q", + '\u{24AC}' => "(q)", + '\u{0239}' => "qp", + + // R/r + '\u{0154}' | '\u{0156}' | '\u{0158}' | '\u{0210}' | '\u{0212}' | '\u{024C}' + | '\u{0280}' | '\u{0281}' | '\u{1D19}' | '\u{1D1A}' | '\u{1E58}' | '\u{1E5A}' + | '\u{1E5C}' | '\u{1E5E}' | '\u{24C7}' | '\u{2C64}' | '\u{A75A}' | '\u{A782}' + | '\u{FF32}' => "R", + '\u{0155}' | '\u{0157}' | '\u{0159}' | '\u{0211}' | '\u{0213}' | '\u{024D}' + | '\u{027C}' | '\u{027D}' | '\u{027E}' | '\u{027F}' | '\u{1D63}' | '\u{1D72}' + | '\u{1D73}' | '\u{1D89}' | '\u{1E59}' | '\u{1E5B}' | '\u{1E5D}' | '\u{1E5F}' + | '\u{24E1}' | '\u{A75B}' | '\u{A783}' | '\u{FF52}' => "r", + '\u{24AD}' => "(r)", + + // S/s and ß/ẞ and st + '\u{015A}' | '\u{015C}' | '\u{015E}' | '\u{0160}' | '\u{0218}' | '\u{1E60}' + | '\u{1E62}' | '\u{1E64}' | '\u{1E66}' | '\u{1E68}' | '\u{24C8}' | '\u{A731}' + | '\u{A785}' | '\u{FF33}' => "S", + '\u{015B}' | '\u{015D}' | '\u{015F}' | '\u{0161}' | '\u{017F}' | '\u{0219}' + | '\u{023F}' | '\u{0282}' | '\u{1D74}' | '\u{1D8A}' | '\u{1E61}' | '\u{1E63}' + | '\u{1E65}' | '\u{1E67}' | '\u{1E69}' | '\u{1E9C}' | '\u{1E9D}' | '\u{24E2}' + | '\u{A784}' | '\u{FF53}' => "s", + '\u{1E9E}' => "SS", + '\u{24AE}' => "(s)", + '\u{00DF}' => "ss", + '\u{FB06}' => "st", + + // T/t, TH/th, TZ + '\u{0162}' | '\u{0164}' | '\u{0166}' | '\u{01AC}' | '\u{01AE}' | '\u{021A}' + | '\u{023E}' | '\u{1D1B}' | '\u{1E6A}' | '\u{1E6C}' | '\u{1E6E}' | '\u{1E70}' + | '\u{24C9}' | '\u{A786}' | '\u{FF34}' => "T", + '\u{0163}' | '\u{0165}' | '\u{0167}' | '\u{01AB}' | '\u{01AD}' | '\u{021B}' + | '\u{0236}' | '\u{0287}' | '\u{0288}' | '\u{1D75}' | '\u{1E6B}' | '\u{1E6D}' + | '\u{1E6F}' | '\u{1E71}' | '\u{1E97}' | '\u{24E3}' | '\u{2C66}' | '\u{FF54}' => "t", + '\u{00DE}' | '\u{A766}' => "TH", + '\u{A728}' => "TZ", + '\u{24AF}' => "(t)", + '\u{02A8}' => "tc", + '\u{00FE}' | '\u{1D7A}' | '\u{A767}' => "th", + '\u{02A6}' => "ts", + '\u{A729}' => "tz", + + // U/u + '\u{00D9}' | '\u{00DA}' | '\u{00DB}' | '\u{00DC}' | '\u{0168}' | '\u{016A}' + | '\u{016C}' | '\u{016E}' | '\u{0170}' | '\u{0172}' | '\u{01AF}' | '\u{01D3}' + | '\u{01D5}' | '\u{01D7}' | '\u{01D9}' | '\u{01DB}' | '\u{0214}' | '\u{0216}' + | '\u{0244}' | '\u{1D1C}' | '\u{1D7E}' | '\u{1E72}' | '\u{1E74}' | '\u{1E76}' + | '\u{1E78}' | '\u{1E7A}' | '\u{1EE4}' | '\u{1EE6}' | '\u{1EE8}' | '\u{1EEA}' + | '\u{1EEC}' | '\u{1EEE}' | '\u{1EF0}' | '\u{24CA}' | '\u{FF35}' => "U", + '\u{00F9}' | '\u{00FA}' | '\u{00FB}' | '\u{00FC}' | '\u{0169}' | '\u{016B}' + | '\u{016D}' | '\u{016F}' | '\u{0171}' | '\u{0173}' | '\u{01B0}' | '\u{01D4}' + | '\u{01D6}' | '\u{01D8}' | '\u{01DA}' | '\u{01DC}' | '\u{0215}' | '\u{0217}' + | '\u{0289}' | '\u{1D64}' | '\u{1D99}' | '\u{1E73}' | '\u{1E75}' | '\u{1E77}' + | '\u{1E79}' | '\u{1E7B}' | '\u{1EE5}' | '\u{1EE7}' | '\u{1EE9}' | '\u{1EEB}' + | '\u{1EED}' | '\u{1EEF}' | '\u{1EF1}' | '\u{24E4}' | '\u{FF55}' => "u", + '\u{24B0}' => "(u)", + '\u{1D6B}' => "ue", + + // V/v + '\u{01B2}' | '\u{0245}' | '\u{1D20}' | '\u{1E7C}' | '\u{1E7E}' | '\u{1EFC}' + | '\u{24CB}' | '\u{A75E}' | '\u{A768}' | '\u{FF36}' => "V", + '\u{028B}' | '\u{028C}' | '\u{1D65}' | '\u{1D8C}' | '\u{1E7D}' | '\u{1E7F}' + | '\u{24E5}' | '\u{2C71}' | '\u{2C74}' | '\u{A75F}' | '\u{FF56}' => "v", + '\u{A760}' => "VY", + '\u{24B1}' => "(v)", + '\u{A761}' => "vy", + + // W/w + '\u{0174}' | '\u{01F7}' | '\u{1D21}' | '\u{1E80}' | '\u{1E82}' | '\u{1E84}' + | '\u{1E86}' | '\u{1E88}' | '\u{24CC}' | '\u{2C72}' | '\u{FF37}' => "W", + '\u{0175}' | '\u{01BF}' | '\u{028D}' | '\u{1E81}' | '\u{1E83}' | '\u{1E85}' + | '\u{1E87}' | '\u{1E89}' | '\u{1E98}' | '\u{24E6}' | '\u{2C73}' | '\u{FF57}' => "w", + '\u{24B2}' => "(w)", + + // X/x + '\u{1E8A}' | '\u{1E8C}' | '\u{24CD}' | '\u{FF38}' => "X", + '\u{1D8D}' | '\u{1E8B}' | '\u{1E8D}' | '\u{2093}' | '\u{24E7}' | '\u{FF58}' => "x", + '\u{24B3}' => "(x)", + + // Y/y + '\u{00DD}' | '\u{0176}' | '\u{0178}' | '\u{01B3}' | '\u{0232}' | '\u{024E}' + | '\u{028F}' | '\u{1E8E}' | '\u{1EF2}' | '\u{1EF4}' | '\u{1EF6}' | '\u{1EF8}' + | '\u{1EFE}' | '\u{24CE}' | '\u{FF39}' => "Y", + '\u{00FD}' | '\u{00FF}' | '\u{0177}' | '\u{01B4}' | '\u{0233}' | '\u{024F}' + | '\u{028E}' | '\u{1E8F}' | '\u{1E99}' | '\u{1EF3}' | '\u{1EF5}' | '\u{1EF7}' + | '\u{1EF9}' | '\u{1EFF}' | '\u{24E8}' | '\u{FF59}' => "y", + '\u{24B4}' => "(y)", + + // Z/z + '\u{0179}' | '\u{017B}' | '\u{017D}' | '\u{01B5}' | '\u{021C}' | '\u{0224}' + | '\u{1D22}' | '\u{1E90}' | '\u{1E92}' | '\u{1E94}' | '\u{24CF}' | '\u{2C6B}' + | '\u{A762}' | '\u{FF3A}' => "Z", + '\u{017A}' | '\u{017C}' | '\u{017E}' | '\u{01B6}' | '\u{021D}' | '\u{0225}' + | '\u{0240}' | '\u{0290}' | '\u{0291}' | '\u{1D76}' | '\u{1D8E}' | '\u{1E91}' + | '\u{1E93}' | '\u{1E95}' | '\u{24E9}' | '\u{2C6C}' | '\u{A763}' | '\u{FF5A}' => "z", + '\u{24B5}' => "(z)", + + // Digits and circled/parenthesized forms + '\u{2070}' | '\u{2080}' | '\u{24EA}' | '\u{24FF}' | '\u{FF10}' => "0", + '\u{00B9}' | '\u{2081}' | '\u{2460}' | '\u{24F5}' | '\u{2776}' | '\u{2780}' + | '\u{278A}' | '\u{FF11}' => "1", + '\u{2488}' => "1.", + '\u{2474}' => "(1)", + '\u{00B2}' | '\u{2082}' | '\u{2461}' | '\u{24F6}' | '\u{2777}' | '\u{2781}' + | '\u{278B}' | '\u{FF12}' => "2", + '\u{2489}' => "2.", + '\u{2475}' => "(2)", + '\u{00B3}' | '\u{2083}' | '\u{2462}' | '\u{24F7}' | '\u{2778}' | '\u{2782}' + | '\u{278C}' | '\u{FF13}' => "3", + '\u{248A}' => "3.", + '\u{2476}' => "(3)", + '\u{2074}' | '\u{2084}' | '\u{2463}' | '\u{24F8}' | '\u{2779}' | '\u{2783}' + | '\u{278D}' | '\u{FF14}' => "4", + '\u{248B}' => "4.", + '\u{2477}' => "(4)", + '\u{2075}' | '\u{2085}' | '\u{2464}' | '\u{24F9}' | '\u{277A}' | '\u{2784}' + | '\u{278E}' | '\u{FF15}' => "5", + '\u{248C}' => "5.", + '\u{2478}' => "(5)", + '\u{2076}' | '\u{2086}' | '\u{2465}' | '\u{24FA}' | '\u{277B}' | '\u{2785}' + | '\u{278F}' | '\u{FF16}' => "6", + '\u{248D}' => "6.", + '\u{2479}' => "(6)", + '\u{2077}' | '\u{2087}' | '\u{2466}' | '\u{24FB}' | '\u{277C}' | '\u{2786}' + | '\u{2790}' | '\u{FF17}' => "7", + '\u{248E}' => "7.", + '\u{247A}' => "(7)", + '\u{2078}' | '\u{2088}' | '\u{2467}' | '\u{24FC}' | '\u{277D}' | '\u{2787}' + | '\u{2791}' | '\u{FF18}' => "8", + '\u{248F}' => "8.", + '\u{247B}' => "(8)", + '\u{2079}' | '\u{2089}' | '\u{2468}' | '\u{24FD}' | '\u{277E}' | '\u{2788}' + | '\u{2792}' | '\u{FF19}' => "9", + '\u{2490}' => "9.", + '\u{247C}' => "(9)", + '\u{2469}' | '\u{24FE}' | '\u{277F}' | '\u{2789}' | '\u{2793}' => "10", + '\u{2491}' => "10.", + '\u{247D}' => "(10)", + '\u{246A}' | '\u{24EB}' => "11", + '\u{2492}' => "11.", + '\u{247E}' => "(11)", + '\u{246B}' | '\u{24EC}' => "12", + '\u{2493}' => "12.", + '\u{247F}' => "(12)", + '\u{246C}' | '\u{24ED}' => "13", + '\u{2494}' => "13.", + '\u{2480}' => "(13)", + '\u{246D}' | '\u{24EE}' => "14", + '\u{2495}' => "14.", + '\u{2481}' => "(14)", + '\u{246E}' | '\u{24EF}' => "15", + '\u{2496}' => "15.", + '\u{2482}' => "(15)", + '\u{246F}' | '\u{24F0}' => "16", + '\u{2497}' => "16.", + '\u{2483}' => "(16)", + '\u{2470}' | '\u{24F1}' => "17", + '\u{2498}' => "17.", + '\u{2484}' => "(17)", + '\u{2471}' | '\u{24F2}' => "18", + '\u{2499}' => "18.", + '\u{2485}' => "(18)", + '\u{2472}' | '\u{24F3}' => "19", + '\u{249A}' => "19.", + '\u{2486}' => "(19)", + '\u{2473}' | '\u{24F4}' => "20", + '\u{249B}' => "20.", + '\u{2487}' => "(20)", + + // Quotes/dashes and punctuation + '\u{00AB}' | '\u{00BB}' | '\u{201C}' | '\u{201D}' | '\u{201E}' | '\u{2033}' + | '\u{2036}' | '\u{275D}' | '\u{275E}' | '\u{276E}' | '\u{276F}' | '\u{FF02}' => "\"", + '\u{2018}' | '\u{2019}' | '\u{201A}' | '\u{201B}' | '\u{2032}' | '\u{2035}' + | '\u{2039}' | '\u{203A}' | '\u{275B}' | '\u{275C}' | '\u{FF07}' => "'", + '\u{2010}' | '\u{2011}' | '\u{2012}' | '\u{2013}' | '\u{2014}' | '\u{207B}' + | '\u{208B}' | '\u{FF0D}' => "-", + '\u{2045}' | '\u{2772}' | '\u{FF3B}' => "[", + '\u{2046}' | '\u{2773}' | '\u{FF3D}' => "]", + '\u{207D}' | '\u{208D}' | '\u{2768}' | '\u{276A}' | '\u{FF08}' => "(", + '\u{2E28}' => "((", + '\u{207E}' | '\u{208E}' | '\u{2769}' | '\u{276B}' | '\u{FF09}' => ")", + '\u{2E29}' => "))", + '\u{276C}' | '\u{2770}' | '\u{FF1C}' => "<", + '\u{276D}' | '\u{2771}' | '\u{FF1E}' => ">", + '\u{2774}' | '\u{FF5B}' => "{", + '\u{2775}' | '\u{FF5D}' => "}", + '\u{207A}' | '\u{208A}' | '\u{FF0B}' => "+", + '\u{207C}' | '\u{208C}' | '\u{FF1D}' => "=", + '\u{FF01}' => "!", + '\u{203C}' => "!!", + '\u{2049}' => "!?", + '\u{FF03}' => "#", + '\u{FF04}' => "$", + '\u{2052}' | '\u{FF05}' => "%", + '\u{FF06}' => "&", + '\u{204E}' | '\u{FF0A}' => "*", + '\u{FF0C}' => ",", + '\u{FF0E}' => ".", + '\u{2044}' | '\u{FF0F}' => "/", + '\u{FF1A}' => ":", + '\u{204F}' | '\u{FF1B}' => ";", + '\u{FF1F}' => "?", + '\u{2047}' => "??", + '\u{2048}' => "?!", + '\u{FF20}' => "@", + '\u{FF3C}' => "\\", + '\u{2038}' | '\u{FF3E}' => "^", + '\u{FF3F}' => "_", + '\u{2053}' | '\u{FF5E}' => "~", + + _ => return None, + }; + Some(s) +} + +#[cfg(test)] +mod tests { + use super::fold_to_ascii; + + #[test] + fn basic_examples() { + assert_eq!( + fold_to_ascii("ação café jalapeño Über"), + "acao cafe jalapeno Uber" + ); + assert_eq!(fold_to_ascii("Æsop and Œuvre"), "AEsop and OEuvre"); + assert_eq!(fold_to_ascii("straße"), "strasse"); + assert_eq!(fold_to_ascii("Łódź"), "Lodz"); + assert_eq!( + fold_to_ascii("“quote” — test １②⒊"), + "\"quote\" - test 123." + ); + + // japanese characters are unchanged + assert_eq!( + fold_to_ascii("日本語のテキストです。Qdrantのコードで単体テストで使用されています。"), + "日本語のテキストです。Qdrantのコードで単体テストで使用されています。" + ); + } +} diff --git a/lib/segment/src/index/field_index/full_text_index/tokenizers/mod.rs b/lib/segment/src/index/field_index/full_text_index/tokenizers/mod.rs index af806d96f7..8a70807381 100644 --- a/lib/segment/src/index/field_index/full_text_index/tokenizers/mod.rs +++ b/lib/segment/src/index/field_index/full_text_index/tokenizers/mod.rs @@ -1,5 +1,6 @@ use std::borrow::Cow; use std::sync::Arc; +mod ascii_folding; mod japanese; mod multilingual; mod stemmer; @@ -103,11 +104,13 @@ impl PrefixTokenizer { text.split(|c| !char::is_alphanumeric(c)) .filter(|token| !token.is_empty()) .for_each(|word| { - let word_cow = if tokens_processor.lowercase { - Cow::Owned(word.to_lowercase()) - } else { - Cow::Borrowed(word) - }; + // Apply ASCII folding if enabled + let mut word_cow = tokens_processor.fold_if_enabled(Cow::Borrowed(word)); + + // Handle lowercase + if tokens_processor.lowercase { + word_cow = Cow::Owned(word_cow.to_lowercase()); + } let word_cow = tokens_processor.stem_if_enabled(word_cow); @@ -164,6 +167,7 @@ impl Tokenizer { min_token_len, max_token_len, lowercase, + ascii_folding, on_disk: _, phrase_matching: _, stopwords, @@ -171,10 +175,12 @@ impl Tokenizer { } = params; let lowercase = lowercase.unwrap_or(true); + let ascii_folding = ascii_folding.unwrap_or(false); let stopwords_filter = Arc::new(StopwordsFilter::new(stopwords, lowercase)); let tokens_processor = TokensProcessor::new( lowercase, + ascii_folding, stopwords_filter, stemmer.as_ref().map(Stemmer::from_algorithm), *min_token_len, @@ -226,6 +232,8 @@ impl Tokenizer { mod tests { use std::default::Default; + use itertools::Itertools; + use super::*; use crate::data_types::index::{ Language, Snowball, SnowballLanguage, SnowballParams, StemmingAlgorithm, @@ -278,7 +286,7 @@ mod tests { fn test_prefix_tokenizer() { let text = "hello, мир!"; let tokens_processor = - TokensProcessor::new(true, Default::default(), None, Some(1), Some(4)); + TokensProcessor::new(true, false, Default::default(), None, Some(1), Some(4)); let mut tokens = Vec::new(); PrefixTokenizer::tokenize(text, &tokens_processor, |token| tokens.push(token)); @@ -296,7 +304,8 @@ mod tests { #[test] fn test_prefix_query_tokenizer() { let text = "hello, мир!"; - let tokens_processor = TokensProcessor::new(true, Default::default(), None, None, Some(4)); + let tokens_processor = + TokensProcessor::new(true, false, Default::default(), None, None, Some(4)); let mut tokens = Vec::new(); PrefixTokenizer::tokenize_query(text, &tokens_processor, |token| tokens.push(token)); @@ -324,7 +333,8 @@ mod tests { // Test stopwords getting applied let filter = StopwordsFilter::new(&Some(StopwordsInterface::new_custom(&["の", "は"])), false); - let tokens_processor = TokensProcessor::new(true, Arc::new(filter), None, None, None); + let tokens_processor = + TokensProcessor::new(true, false, Arc::new(filter), None, None, None); MultilingualTokenizer::tokenize(text, &tokens_processor, |token| tokens.push(token)); eprintln!("tokens = {tokens:#?}"); assert_eq!(tokens.len(), 2); @@ -349,7 +359,8 @@ mod tests { // Test stopwords getting applied let filter = StopwordsFilter::new(&Some(StopwordsInterface::new_custom(&["是"])), false); - let tokens_processor = TokensProcessor::new(true, Arc::new(filter), None, None, None); + let tokens_processor = + TokensProcessor::new(true, false, Arc::new(filter), None, None, None); MultilingualTokenizer::tokenize(text, &tokens_processor, |token| tokens.push(token)); eprintln!("tokens = {tokens:#?}"); assert_eq!(tokens.len(), 3); @@ -397,6 +408,7 @@ mod tests { min_token_len: Some(1), max_token_len: Some(4), lowercase: Some(true), + ascii_folding: None, on_disk: None, phrase_matching: None, stopwords: None, @@ -428,6 +440,7 @@ mod tests { min_token_len: None, max_token_len: None, lowercase: Some(true), + ascii_folding: None, on_disk: None, phrase_matching: None, stopwords: Some(StopwordsInterface::Language(Language::English)), @@ -470,6 +483,7 @@ mod tests { min_token_len: None, max_token_len: None, lowercase: Some(true), + ascii_folding: None, on_disk: None, phrase_matching: None, stopwords: Some(StopwordsInterface::Language(Language::English)), @@ -502,6 +516,7 @@ mod tests { min_token_len: None, max_token_len: None, lowercase: Some(true), + ascii_folding: None, on_disk: None, phrase_matching: None, stopwords: Some(StopwordsInterface::new_set( @@ -540,6 +555,7 @@ mod tests { min_token_len: None, max_token_len: None, lowercase: Some(true), + ascii_folding: None, on_disk: None, phrase_matching: None, stopwords: Some(StopwordsInterface::new_custom(&["as", "the", "a"])), @@ -578,6 +594,7 @@ mod tests { min_token_len: None, max_token_len: None, lowercase: Some(true), + ascii_folding: None, on_disk: None, phrase_matching: None, stopwords: Some(StopwordsInterface::Language(Language::English)), @@ -613,6 +630,7 @@ mod tests { min_token_len: None, max_token_len: None, lowercase: Some(true), + ascii_folding: None, on_disk: None, phrase_matching: None, stopwords: Some(StopwordsInterface::new_set( @@ -657,6 +675,7 @@ mod tests { min_token_len: None, max_token_len: None, lowercase: Some(false), // Case sensitivity is enabled + ascii_folding: None, on_disk: None, phrase_matching: None, stopwords: Some(StopwordsInterface::new_custom(&["the", "The", "LAZY"])), @@ -684,11 +703,97 @@ mod tests { assert!(tokens.contains(&Cow::Borrowed("dog"))); } + #[test] + fn test_ascii_folding_word_tokenizer_on_off() { + let text = "ação café jalapeño Über"; + + let expected_disabled = ["ação", "café", "jalapeño", "über"] + .into_iter() + .map(str::to_string) + .collect_vec(); + let expected_enabled = ["acao", "cafe", "jalapeno", "uber"] + .into_iter() + .map(str::to_string) + .collect_vec(); + + // ascii_folding disabled (default) + let params_disabled = TextIndexParams { + r#type: TextIndexType::Text, + tokenizer: TokenizerType::Word, + min_token_len: None, + max_token_len: None, + lowercase: Some(true), + ascii_folding: Some(false), + on_disk: None, + phrase_matching: None, + stopwords: None, + stemmer: None, + }; + let tokenizer_disabled = Tokenizer::new_from_text_index_params(¶ms_disabled); + let mut tokens_disabled = Vec::new(); + tokenizer_disabled.tokenize_doc(text, |token| tokens_disabled.push(token.to_string())); + assert_eq!(tokens_disabled, expected_disabled); + + // ascii_folding enabled + let params_enabled = TextIndexParams { + r#type: TextIndexType::Text, + tokenizer: TokenizerType::Word, + min_token_len: None, + max_token_len: None, + lowercase: Some(true), + ascii_folding: Some(true), + on_disk: None, + phrase_matching: None, + stopwords: None, + stemmer: None, + }; + let tokenizer_enabled = Tokenizer::new_from_text_index_params(¶ms_enabled); + let mut tokens_enabled = Vec::new(); + tokenizer_enabled.tokenize_doc(text, |token| tokens_enabled.push(token.to_string())); + assert_eq!(tokens_enabled, expected_enabled); + } + + #[test] + fn test_ascii_folding_prefix_tokenizer() { + let text = "ação"; + // With folding disabled: prefixes should preserve accents + let tokens_processor_disabled = + TokensProcessor::new(true, false, Default::default(), None, Some(1), Some(4)); + let mut tokens_disabled = Vec::new(); + PrefixTokenizer::tokenize(text, &tokens_processor_disabled, |t| { + tokens_disabled.push(t.to_string()) + }); + assert!( + tokens_disabled.contains(&"a".to_string()) + || tokens_disabled.contains(&"a".to_string()) + ); + // Because the first char is 'a', but next prefixes should include accented letters + assert!( + tokens_disabled.iter().any(|t| t.starts_with("aç")) + || tokens_disabled.iter().any(|t| t.contains('ç')) + ); + + // With folding enabled: prefixes should be ASCII-only (acao, acao prefixes) + let tokens_processor_enabled = + TokensProcessor::new(true, true, Default::default(), None, Some(1), Some(4)); + let mut tokens_enabled = Vec::new(); + PrefixTokenizer::tokenize(text, &tokens_processor_enabled, |t| { + tokens_enabled.push(t.to_string()) + }); + // We expect prefixes like a, ac, aca, acao + assert!(tokens_enabled.contains(&"a".to_string())); + assert!(tokens_enabled.contains(&"ac".to_string())); + assert!(tokens_enabled.contains(&"aca".to_string())); + assert!(tokens_enabled.contains(&"acao".to_string())); + assert!(tokens_enabled.iter().all(|t| t.is_ascii())); + } + #[test] fn test_stemming_snowball() { let input = "interestingly proceeding living"; let mut tokens_processor = TokensProcessor::new( true, + false, Default::default(), Some(make_stemmer(SnowballLanguage::English)), None, diff --git a/lib/segment/src/index/field_index/full_text_index/tokenizers/multilingual.rs b/lib/segment/src/index/field_index/full_text_index/tokenizers/multilingual.rs index 870b7a6b2a..784a7c6e6e 100644 --- a/lib/segment/src/index/field_index/full_text_index/tokenizers/multilingual.rs +++ b/lib/segment/src/index/field_index/full_text_index/tokenizers/multilingual.rs @@ -181,6 +181,7 @@ mod test { fn test_multilingual_stemming() { let tokens_processor = TokensProcessor::new( true, + false, Default::default(), Some(Stemmer::from_algorithm(&StemmingAlgorithm::Snowball( SnowballParams { diff --git a/lib/segment/src/index/field_index/full_text_index/tokenizers/tokens_processor.rs b/lib/segment/src/index/field_index/full_text_index/tokenizers/tokens_processor.rs index 75eea8ace6..e3d804ea48 100644 --- a/lib/segment/src/index/field_index/full_text_index/tokenizers/tokens_processor.rs +++ b/lib/segment/src/index/field_index/full_text_index/tokenizers/tokens_processor.rs @@ -8,6 +8,7 @@ use crate::index::field_index::full_text_index::stop_words::StopwordsFilter; #[derive(Debug, Clone, Default)] pub struct TokensProcessor { pub lowercase: bool, + pub ascii_folding: bool, stopwords_filter: Arc, // TDOO(rocksdb): Remove once rocksdb has been removed! stemmer: Option, pub min_token_len: Option, @@ -17,6 +18,7 @@ pub struct TokensProcessor { impl TokensProcessor { pub fn new( lowercase: bool, + ascii_folding: bool, stopwords_filter: Arc, stemmer: Option, min_token_len: Option, @@ -24,6 +26,7 @@ impl TokensProcessor { ) -> Self { Self { lowercase, + ascii_folding, stopwords_filter, stemmer, min_token_len, @@ -46,13 +49,22 @@ impl TokensProcessor { stemmer.stem(input) } + /// Applies ASCII folding if enabled. Converts accented characters to their ASCII equivalents. + pub fn fold_if_enabled<'a>(&self, input: Cow<'a, str>) -> Cow<'a, str> { + if self.ascii_folding { + super::ascii_folding::fold_to_ascii_cow(input) + } else { + input + } + } + pub fn is_stopword(&self, token: &str) -> bool { self.stopwords_filter.is_stopword(token) } pub fn process_token_cow<'a>( &self, - token: Cow<'a, str>, + mut token_cow: Cow<'a, str>, check_max_len: bool, ) -> Option> { let Self { @@ -61,18 +73,22 @@ impl TokensProcessor { stemmer, min_token_len, max_token_len, + ascii_folding, } = self; - if token.is_empty() { + if token_cow.is_empty() { return None; } + // Handle ASCII folding (normalize accents) + if *ascii_folding { + token_cow = super::ascii_folding::fold_to_ascii_cow(token_cow); + } + // Handle lowercase - let mut token_cow = if *lowercase { - Cow::Owned(token.to_lowercase()) - } else { - token - }; + if *lowercase { + token_cow = Cow::Owned(token_cow.to_lowercase()); + } // Handle stopwords if stopwords_filter.is_stopword(&token_cow) { diff --git a/src/common/inference/bm25.rs b/src/common/inference/bm25.rs index ec6dc566ee..0fe163d2c8 100644 --- a/src/common/inference/bm25.rs +++ b/src/common/inference/bm25.rs @@ -102,6 +102,7 @@ fn new_token_processor_from_config(value: TextPreprocessingConfig) -> TokensProc let TextPreprocessingConfig { language, lowercase, + ascii_folding, stopwords, stemmer, min_token_len, @@ -110,6 +111,8 @@ fn new_token_processor_from_config(value: TextPreprocessingConfig) -> TokensProc let lowercase = lowercase.unwrap_or(true); + let ascii_folding = ascii_folding.unwrap_or(false); + let language = language.unwrap_or_else(|| DEFAULT_LANGUAGE.to_string()); let stemmer = match stemmer { @@ -129,6 +132,7 @@ fn new_token_processor_from_config(value: TextPreprocessingConfig) -> TokensProc TokensProcessor::new( lowercase, + ascii_folding, Arc::new(StopwordsFilter::new(&stopwords_config, lowercase)), stemmer, min_token_len, diff --git a/tools/codespell.toml b/tools/codespell.toml index 41a7af826e..a2ceb03467 100644 --- a/tools/codespell.toml +++ b/tools/codespell.toml @@ -5,6 +5,7 @@ skip = [ "**/stop_words/*.rs", + "./lib/segment/src/index/field_index/full_text_index/tokenizers/ascii_folding.rs", # contains character mappings "./static", "./tests/e2e_tests/test_data/cert/key.pem", "openapi.json", # openapi.json is typically a released version and could carry old typos