mirror of
https://github.com/qdrant/qdrant.git
synced 2026-07-26 12:41:04 -05:00
446d140c2d
* feat: slice filtering condition for sliced scroll and deterministic sampling
Add a `slice` filter condition selecting points where
`stable_hash(point_id) % total == index`. The hash is SipHash-2-4 with a
zero key over canonical id bytes (8 LE bytes for numeric ids, 16 RFC 4122
bytes for UUIDs) — a frozen public contract, independent of the internal
resharding ring hash, reproducible by clients to predict membership.
For a fixed `total`, slices are disjoint and cover all points, enabling
parallel scroll streams (ES sliced-scroll style) and reproducible sampling
that composes with any other filter condition.
- REST: `{"slice": {"total": N, "index": R}}`; gRPC: `SliceCondition` in
the condition oneof (tag 8)
- Evaluated per point via id_tracker external-id lookup; no payload index
needed; cardinality estimated as `points / total` with no primary clause
- `total >= 1` enforced by NonZeroU32 at parse time, `index < total` by
validation in both REST and gRPC paths
- Hash contract locked by test vectors independently reproduced with a
reference SipHash-2-4 implementation
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
* tests: minimal OpenAPI test for slice filter condition
Scrolls all slices of a fixed total over numeric + UUID ids asserting
disjointness and full coverage, checks must_not inversion, and pins the
two rejection paths (422 for index >= total, 400 for total = 0). Requests
and responses are validated against the regenerated OpenAPI spec by the
test harness.
Note: the spec cannot itself reject total = 0 client-side — the Condition
anyOf falls through to the permissive Filter schema, as with any invalid
condition — so rejection is asserted via the server response.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---------
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
229 lines
4.9 KiB
TOML
229 lines
4.9 KiB
TOML
[package]
|
|
name = "segment"
|
|
version = "0.6.0"
|
|
authors = [
|
|
"Andrey Vasnetsov <vasnetsov93@gmail.com>",
|
|
"Qdrant Team <info@qdrant.tech>",
|
|
]
|
|
license = "Apache-2.0"
|
|
edition = "2024"
|
|
|
|
[lints]
|
|
workspace = true
|
|
|
|
[features]
|
|
default = []
|
|
testing = ["common/testing", "sparse/testing", "gpu/testing", "quantization/testing"]
|
|
gpu = ["gpu/gpu"]
|
|
|
|
[build-dependencies]
|
|
cc = { workspace = true }
|
|
|
|
[dev-dependencies]
|
|
clap = { workspace = true }
|
|
criterion = { workspace = true }
|
|
dataset = { path = "../common/dataset" }
|
|
env_logger = { workspace = true }
|
|
fs-err = { workspace = true, features = ["debug"] }
|
|
humantime = { workspace = true }
|
|
indicatif = { workspace = true }
|
|
ndarray = "0.17.2"
|
|
ndarray-npy = { version = "0.10.0", default-features = false }
|
|
rmp-serde = { workspace = true }
|
|
rand_distr = { workspace = true }
|
|
walkdir = { workspace = true }
|
|
rstest = { workspace = true }
|
|
segment = { path = ".", default-features = false, features = ["testing"] }
|
|
proptest = { workspace = true }
|
|
anyhow = { workspace = true }
|
|
object_store = { workspace = true }
|
|
bytes = { workspace = true }
|
|
tokio = { workspace = true }
|
|
|
|
[target.'cfg(not(target_os = "windows"))'.dev-dependencies]
|
|
pprof = { workspace = true }
|
|
|
|
[dependencies]
|
|
io_bridge_object_store = { path = "../common/io_bridge_object_store" }
|
|
blink-alloc = { workspace = true }
|
|
bytemuck = { workspace = true }
|
|
data-encoding = { workspace = true }
|
|
fs-err = { workspace = true }
|
|
integer-encoding = { workspace = true }
|
|
tempfile = { workspace = true }
|
|
parking_lot = { workspace = true }
|
|
rayon = { workspace = true }
|
|
itertools = { workspace = true }
|
|
uuid = { workspace = true }
|
|
bincode = { workspace = true }
|
|
serde = { workspace = true }
|
|
serde_json = { workspace = true }
|
|
serde_cbor = { workspace = true }
|
|
serde-value = "0.7"
|
|
serde_variant = { workspace = true }
|
|
serde-untagged = "0.1.9"
|
|
ordered-float = { workspace = true }
|
|
thiserror = { workspace = true }
|
|
atomic_refcell = { workspace = true }
|
|
atomicwrites = { workspace = true }
|
|
memmap2 = { workspace = true }
|
|
schemars = { workspace = true }
|
|
log = { workspace = true }
|
|
geo = "0.33.1"
|
|
geohash = "0.13.2"
|
|
num-traits = { workspace = true }
|
|
num-derive = "0.4.2"
|
|
num-cmp = "0.1.0"
|
|
once_cell = "1.21"
|
|
rand = { workspace = true }
|
|
bitvec = { workspace = true }
|
|
fs_extra = { workspace = true }
|
|
tinyvec = { workspace = true }
|
|
validator = { workspace = true }
|
|
chrono = { workspace = true }
|
|
ecow = { workspace = true }
|
|
fnv = { workspace = true }
|
|
indexmap = { workspace = true }
|
|
ahash = { workspace = true }
|
|
self_cell.workspace = true
|
|
sha2 = { workspace = true }
|
|
siphasher = "1.0.3"
|
|
smallvec = { workspace = true }
|
|
strum = { workspace = true }
|
|
byteorder = { workspace = true }
|
|
tap = { workspace = true }
|
|
zerocopy = { workspace = true }
|
|
vaporetto = { version = "0.6.5" }
|
|
rust-stemmers = { package = "qdrant-rust-stemmers", version = "1.2.2" }
|
|
sysinfo = "0.38"
|
|
charabia = { version = "0.9.9", default-features = false, features = [
|
|
"greek",
|
|
"hebrew",
|
|
"thai",
|
|
"chinese-segmentation",
|
|
"chinese-normalization",
|
|
] }
|
|
|
|
gridstore = { path = "../gridstore" }
|
|
|
|
common = { path = "../common/common" }
|
|
macros = { path = "../macros" }
|
|
posting_list = { path = "../posting_list" }
|
|
quantization = { path = "../quantization" }
|
|
sparse = { path = "../sparse" }
|
|
gpu = { path = "../gpu" }
|
|
|
|
tracing = { workspace = true, optional = true }
|
|
macro_rules_attribute = "0.2.2"
|
|
nom = "8.0.0"
|
|
half = { workspace = true }
|
|
roaring = { workspace = true }
|
|
duplicate = "2.0.1"
|
|
|
|
[target.'cfg(target_os = "linux")'.dependencies]
|
|
cgroups-rs = "0.5"
|
|
procfs = { version = "0.18", default-features = false }
|
|
io-uring = "0.7.12"
|
|
|
|
[[bench]]
|
|
name = "vector_search"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "turbo_vector_search"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "read_vectors"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "read_payloads"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "hnsw_build_graph"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "hnsw_search_graph"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "conditional_search"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "dynamic_mmap_flags"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "hnsw_build_asymptotic"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "serde_formats"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "id_type_benchmark"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "map_benchmark"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "boolean_filtering"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "numeric_index_check_values"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "sparse_index_search"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "sparse_index_build"
|
|
harness = false
|
|
bench = false # TODO(rocksdb): fix and re-enable (or just remove)
|
|
|
|
[[bench]]
|
|
name = "sparse_vector_storage"
|
|
harness = false
|
|
bench = false # TODO(rocksdb): fix and re-enable (or just remove)
|
|
|
|
[[bench]]
|
|
name = "multi_vector_search"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "metrics"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "range_filtering"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "in_memory_id_tracker"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "buffered_update_bitslice"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "segment_info"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "hnsw_incremental_build"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "facets"
|
|
harness = false
|