mirror of
https://github.com/qdrant/qdrant.git
synced 2026-07-29 14:11:32 -05:00
aa0f49798e
* Add ASCII folding to tokenization process Introduced an optional ASCII folding feature within the `TokensProcessor` to normalize non-ASCII characters to their ASCII equivalents. Updated tests and documentation to reflect the changes. * Refactor tokenization code for improved readability and maintainability Reorganized and reformatted the tokenization module, including `TokensProcessor` initialization and ASCII folding mappings for better clarity. Updated tests to align with the changes. * Update test cases to reflect optional tokenizer settings changes Adjusted `ascii_folding`, `lowercase`, and `phrase_matching` settings in tests to `None` where applicable, aligning with updates in tokenizer configuration defaults. * address review remarks * fix codespell * thx coderabbit * Don't copy tokens that are already ASCII * Shrink folded string to fit --------- Co-authored-by: Luis Cossío <luis.cossio@outlook.com> Co-authored-by: timvisee <tim@visee.me>
22 lines
691 B
TOML
22 lines
691 B
TOML
# You can run the following command in the repository root:
|
|
# uv tool run codespell --toml tools/codespell.toml
|
|
|
|
[tool.codespell]
|
|
|
|
skip = [
|
|
"**/stop_words/*.rs",
|
|
"./lib/segment/src/index/field_index/full_text_index/tokenizers/ascii_folding.rs", # contains character mappings
|
|
"./static",
|
|
"./tests/e2e_tests/test_data/cert/key.pem",
|
|
"openapi.json", # openapi.json is typically a released version and could carry old typos
|
|
"target",
|
|
]
|
|
|
|
ignore-words-list = [
|
|
"ABD", # Geohash example
|
|
"generall", # The boss
|
|
"hel", # Query token
|
|
"mmapped", # Memory mapped
|
|
"ser", # Serde package
|
|
]
|