mirror of
https://github.com/qdrant/qdrant.git
synced 2026-07-27 05:01:08 -05:00
be543561e5
* Gridstore: introduce storage operating mode in config Add a mode field to the gridstore config, selecting between the dynamic mode (current behavior, the default) and the upcoming serverless mode. The mode is specified through StorageOptions on creation, persisted in config.json, and read back first when opening so the correct variant can be selected automatically. Configs written before this field existed deserialize as dynamic. For now, selecting the serverless mode returns an error; the variant itself is added in follow-up commits. * Gridstore: move dynamic implementation into dedicated module Mechanical move of the current Gridstore implementation into gridstore/dynamic.rs as DynamicGridstore. The public Gridstore struct becomes a thin wrapper holding a mode variant enum, propagating every call into the selected variant. For now the enum only has the dynamic variant; the serverless variant is added in follow-up commits. No logic changes to the dynamic implementation itself: only visibility, the config parameter now passed into open (the wrapper reads it first to select the mode), and open_or_create staying on the wrapper. * Gridstore: add serverless tracker Add the append-only mapping tracker for the serverless storage mode. The tracker file is a plain array of 16-byte mapping entries without any header: the number of mappings is defined by the exact file length, and the entry index is the point offset. The file starts empty and only ever grows by appending, existing bytes are never rewritten. Mappings must be set in monotonically increasing point offset order; skipped offsets are backfilled as zeroed entries which decode as None. New mappings are buffered in memory and appended with a single write per flush. A flush with a stale target is a no-op so bytes are never written twice. A torn trailing entry (file length not a multiple of the entry size) is ignored when reading and truncated away when opening writable. Unlike the dynamic tracker, the file is read and written directly with positional file IO instead of memory mapping, as serverless environments do not handle memory mapped files well. * Gridstore: add serverless storage variant Add the append-only gridstore variant for serverless deployments, which restrict IO to appending to files: existing bytes can never be rewritten, and IO is expensive so as few files as possible are used. The variant stores all value data in a single page file next to the serverless tracker and the storage config, three files in total. Both data files start empty and only ever grow by appending; there is no preallocation, no used-block bitmask and no gap/region bookkeeping. Values are appended at put time at the next block aligned offset, with the zero padding included in the write so it lands exactly at the end of the file. Mappings are buffered and appended to the tracker with a single write per flush, after the page file is synced, so a mapping on disk never points at data that is not durable. Values cannot be updated or deleted, and must be put at monotonically increasing point offsets; violations are rejected before any data is written. Files are read and written directly, never memory mapped. The mode is selected through StorageOptions on creation and picked up automatically from the persisted config when opening. * Gridstore: serverless support in reader and view Extend the read-only GridstoreReader and the GridstoreView with the serverless mode, keeping both public types unchanged: like the writable Gridstore they now hold a mode variant internally, selected automatically from the persisted config when opening. The serverless reader holds the tracker and page directly and reads the files positionally, without memory mapping. A live reload re-reads the mapping count from the exact tracker file length (there is no size header), ignoring a torn trailing entry, and never truncates as it is read-only. Value reads always go directly to the file, so newly appended data is readable without remapping anything. * Gridstore: document storage operating modes * Gridstore: review fixes for the serverless mode Hardening and cleanup from a review pass over the new serverless storage variant: - Batch the reader side iteration like the writer already did, instead of materializing tracker mappings for the full range in one go, which could transiently allocate gigabytes on large storages. - Recover the append cursors when a positional write fails partway: truncate the file back to the tracked length so a retried append or flush never rewrites bytes that already landed in the file. - Validate page addressability before appending value data, a rejected put must not grow the page file. - Cross-check tracker and page consistency when opening: mappings that reference value data past the end of the page file (e.g. after a partial copy or restore) now fail fast instead of surfacing as opaque read errors per point. - Reject value pointers into any page other than page 0 on the serverless read path with PageNotFound, matching the dynamic mode contract, instead of silently reading from a wrong location. - Refresh the reported storage size on reader live reload even when no new mappings were flushed, unflushed value data may have grown the page file already. - Validate configs read from disk: a corrupt config with zero sized blocks, pages or regions is now rejected when opening instead of panicking on a division by zero later. - Classify rejected serverless puts as UnsupportedOperation, consistent with rejected deletes, so they don't surface as user-facing validation errors at the segment level. - Deduplicate the compression dispatch into Compression::compress and Compression::decompress, and the serverless file create/open patterns into shared direct IO helpers, so the two modes and files can't silently drift apart. * Gridstore: cover both operating modes in mode-agnostic tests Parameterize the gridstore tests that exercise mode-agnostic behavior over both the dynamic and serverless mode with rstest, using a single and bulk put/get roundtrips, storage files, basic persistence, corrupt config rejection, batched read congruence, reader live reload, and the different block sizes. Mode specific expectations branch inside the tests: expected file names, storage size semantics (whole blocks vs exactly packed bytes), value pointer layout (page spill over vs a single packed page), and gaps (created by deletes in dynamic mode, by skipped puts in serverless mode). Dynamic-only internals assertions are kept behind a mode check. Tests around updates, deletes, page spanning, block reuse and other dynamic-only behavior intentionally stay dynamic; the serverless specific format invariants remain covered by the dedicated serverless tests. * Gridstore: port serverless specific tests from sibling branch Source the serverless specific test cases that the serverless-gridstore-updates branch added, adapted to the dedicated variant implemented here (distinct file names, headerless tracker with 16 byte entries, a single packed page without trailing padding, and rejected re-puts): - writes only ever append: tracker and page files only grow and previously written bytes stay byte-for-byte untouched - new mappings land exactly at the end of the tracker file, which always covers the exact number of mappings - mapping gaps are zero-padded on disk and survive reopening - values are packed back to back at block aligned offsets, the page file ends exactly at the last value - serverless mode never creates nor reports block flag files - a flusher persists exactly the mappings that existed at its creation, later puts stay pending - a config claiming the wrong mode fails loudly in both directions instead of loading the incompatible file format of the other mode Tests around their mode switching, page spanning and tolerated deletes don't apply to this design and are intentionally not ported. * Gridstore: test serverless production risk scenarios Add tests for the operational aspects that matter before serverless mode goes to production, each covering a scenario that wasn't evaluated yet: - Replayed puts of already persisted offsets (a WAL redo after a crash where the flush completed but was never acknowledged) are rejected without appending anything, and max_point_offset is the exact offset a replay must resume at. - The accepted crash case of a tracker file extended with zeroed bytes: the entries count as permanent None mappings, can never be put again, and the storage stays consistent and writable past them. - The read-only reader never modifies the files: opening over a torn tracker tail, reading, iterating and live reloading leave both files byte-for-byte untouched. - A multi-round put/flush/reopen cycle always exposes exactly the flushed prefix, with the mapping count matching the exact tracker file length and unflushed offsets reusable. - An append beyond the maximum addressable block offset is rejected before writing anything, keeping retried puts from growing the page file unboundedly. * Gridstore: rename serverless mode to append-only, split into module Rename the mode after its defining characteristic instead of its deployment target: files only ever grow, existing bytes are never rewritten. Renames Mode::Serverless to Mode::AppendOnly (persisted as "mode": "append_only") and the on-disk file names to append_only_tracker.dat and append_only_page_0.dat. The serverless deployment motivation stays in the documentation. Also split the single 2300 line serverless.rs into an append_only module with dedicated files for the storage, page, view, reader and tests. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Use universal IO in Gridstore * Include upstream preopen logic in new Gridstore variant * Gridstore: buffer append-only value writes until flush In append-only mode, put previously wrote the value data to the page file right away, one write operation per put, while mappings were already buffered and batch persisted on flush. Buffer value writes the same way: both the value and its mapping now only land on disk once a flush cycle executes. This batches all new value data into a single write operation per flush, which is significantly more efficient on S3 based storage where every write is a costly operation. A flush now performs exactly two writes: one appending all buffered value data to the page file, one appending all pending mappings to the tracker file, in that order, so a mapping on disk never points at value data that is not durable. The page mirrors the tracker's pending mechanism: an in-memory buffer that is byte for byte the next append (zero padding between block aligned values included), a watermark captured at flusher creation so puts made during a flush stay buffered, a stale-flush no-op guard so appended bytes are never written twice, and truncate-back recovery on failed writes. Reads transparently serve buffered values from memory. As a side effect, a crash between flushes now leaves nothing on disk at all, where the write-through approach left orphaned value bytes in the page file. The buffered data is held in memory until the next flush, bounded by the flush cadence. Universal IO filesystem handles are now required to be Send + Sync, so the flusher closure can carry one to grow the page file at flush time; all existing backends already satisfied this. * Gridstore: rename inner DynamicGridstore to Gridstore The dynamic variant keeps the Gridstore name; the outer dispatching type will be renamed to Blobstore in a follow-up. Until then the inner type is referred to as dynamic::Gridstore to distinguish it from the outer type. * Gridstore: rename append-only variant to Arenastore The append-only variant stores all value data in a single ever-growing page, allocating space by appending, hence: arena store. * Gridstore: rename outer storage type to Blobstore The outer type dispatching between the two storage variants is now called Blobstore, being more generic than Gridstore. This frees up the Gridstore name, which now exclusively refers to the dynamic mode variant, next to Arenastore for the append-only variant. Storage components keep using the outer type, so they now use Blobstore. The gridstore crate name, GridstoreError, and the persisted names (config.json mode, payload config storage_type) are unchanged. * Gridstore: split Gridstore and Arenastore into dedicated modules The outer module is now blobstore, matching the Blobstore type it defines. The two storage variants each get their own submodule: the dynamic Gridstore moves from dynamic.rs into gridstore/ with its reader and view extracted from the shared files, mirroring the arenastore/ module (previously append_only/) which already had this layout. * Rename gridstore crate to blobstore The crate is named after the outer Blobstore storage type it provides. The gridstore name lives on in the dynamic mode variant. GridstoreError and the persisted names (config.json mode, payload config storage_type) are unchanged. * Arenastore: pack values back to back across multiple pages Drop the block alignment from the append-only mode: values are packed byte to byte, without blocks, and the tracker offset is now a plain byte offset within the page. Blocks and regions are dynamic mode concepts; their page size constraints no longer apply to append-only configs. Bring back support for multiple pages. Once appending a value would grow the current page beyond the configured page size, a new page is started, bounding the size of and the number of appends to each file: object stores like S3 Express limit the number of appends per object. A value larger than the page size gets a page of its own; values never span pages. A rollover creates the new, empty page file at put time; the value data itself stays buffered until the next flush, which appends to each touched page with a single write, using per-page watermarks captured at flusher creation. The reader scans for consecutively numbered page files when opening, validates the most recent mappings against them, and adopts pages created since on a live reload. * Blobstore: rename dynamic mode to mutable Rename Mode::Dynamic to Mode::Mutable, and the persisted config value with it: config.json now writes "mode": "mutable". There is no compatibility alias for "dynamic", released versions never wrote the mode field (a missing field still defaults to mutable), only unreleased storages did. The Gridstore type and module names for the mutable variant are unchanged. * Fix Edge compilation due to package rename * Review remarks * Extract Gridstore preopen into module * Rename Arenastore files * Use universal IO for append operations * Rename GridstoreError to BlobstoreError The error type belongs to the Blobstore crate and is shared by both the Gridstore and Arenastore variants, so it follows the crate naming. Also update the user-facing error messages that referred to the old name. * Split config into per-variant types * Rename Arenastore to Logstore Rename the Arenastore type to Logstore, including the reader, view, config, module and variant names. The storage file names follow: log_page_{n}.dat and log_tracker.dat. The persisted mode tag stays "append_only". * Move bitmask module into the Gridstore variant The bitmask tracks free blocks, which only exists in the mutable mode. Move the module from the crate root into the Gridstore variant that owns it. It stays re-exported at the crate root because the bitmask benchmark needs a public path. * Move pages module into the Gridstore variant Like the bitmask, the block based pages module is only used by the mutable mode. Move it from the crate root into the Gridstore variant that owns it. The Logstore variant has its own page implementation. * Use universal IO for every Logstore operation Replace the direct_io module with universal IO in the append-only tracker, making the whole Logstore go through a universal IO backend bounded by UniversalRead and UniversalAppend: - The tracker is generic over the backend now. Reads go through UniversalRead with the caller's access pattern, flushes land as one atomic append with the same offset compare-and-swap recovery as the pages: a retried append after a lost acknowledgement is adopted instead of appended twice. A torn trailing entry is still truncated away on writable open, through a fresh handle since shrinking is not supported through an open one. - The reader now schedules a prefetch for the tracker file too, it no longer bypasses the backend. - The config write, clear and wipe use the backend file operations instead of local filesystem calls, matching the Gridstore variant. * Batch reads in Logstore read_values Apply the same batching logic as the Gridstore variant: resolve all mappings first, then fetch the value data, both through the backend's read pipeline so async backends can serve the reads in parallel. The tracker gains a batched lookup mirroring the mutable tracker's iter, serving pending mappings and out of range point offsets directly from memory. The pages gain a batched value read; unflushed values are served from the in-memory buffers, and since values never span pages each value is a single read without reassembly. Like in the Gridstore variant, the callback may now be invoked in a different order than the requested point offsets. * Better describe logstore live reload ordering * use enum for options, swap `*Options`<->`*Config` naming * don't wrap enum in struct * ditch unused `StorageConfig`, make deserialization more ergonomic * rename `*Options`->`*Config` * make `preopen` non-blocking * fixup! ditch unused `StorageConfig`, make deserialization more ergonomic * fixup! use enum for options, swap `*Options`<->`*Config` naming * fixup! don't wrap enum in struct * fix rebase * use `populate` param in Logstore * test: failing repro of stale page after live reload across rollover A reader that live-reloads between a page rollover and the following flush adopts the new, still empty page. The previous page is then no longer the last one and is never reloaded again, so the tail that the next flush appends to it stays invisible to the reader forever: value pointer at byte 100 with length 100 is out of range AppendOnlyPages::live_reload only reloads the last held page, assuming earlier pages never change once a newer page exists. But the rollover creates the new page file eagerly at put time, while the previous page's buffered tail only lands at the next flush (see test_rollover_writes_no_value_data_before_flush), so a page can keep growing on disk after its successor exists. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix: reload all pages that grew * use Fs in `open_or_create` * fix: publish tracker mappings only after the pages reload `AppendOnlyTracker::live_reload` observed the mapping count and made it visible in one step, before `LogstoreReader::live_reload` reloaded the pages. Every failure path in the page reload -- `list_files`, reopening a grown page, opening an adopted one, the truncation check -- therefore left the reader with mappings referencing value data it never loaded, so reads in the new offset range fail until a later reload happens to succeed. The edge refresh loop keeps a segment whose reload failed, expecting it to keep serving its pre-refresh state, which it then does not. Split observing from publishing: `reload_count` refreshes the handle and returns the count as a `PendingReload` token, `commit_reload` publishes it. The reader still observes the tracker first, as the writer persists pages before the mappings referencing them, but only commits once the pages are loaded. Reopening without committing is harmless: reads stay bounded by the unchanged count, and the bytes below it never change. A partial failure inside the page reload needs no unwinding, pages running ahead of the tracker is the safe direction. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * perf: batch the value reads in Logstore iteration `LogstoreView::iter_range`, the path behind `Logstore::iter` and `LogstoreReader::iter`, fetched the mappings for the whole range with a single read but then read the values themselves one at a time, serially. Gridstore routes its `iter` through `read_values` and pipelines both stages, so a full scan of an append-only storage was the one read path without batching -- one blocking round trip per value on the object store backends this variant exists for. It is reached by payload storage iteration and by the payload index build, which scans every payload. Feed the pointers into `read_batch_values` instead, keeping the single contiguous tracker read, which is better than the per-offset pipeline scheduling Gridstore does on that side. Values are now delivered through the read pipeline, so the callback may be invoked out of order, as it already could be for Gridstore's `iter` and for `read_values` in both variants. Both segment callers are order independent. Tests that happened to rely on the mmap backend completing reads in scheduling order now sort before comparing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * test: don't run the failed-page-reload test on Windows The test shrinks a page file out of band to make the page reload fail, but Windows refuses to resize a file while the reader holds it mapped, which it does by construction here: "the requested operation cannot be performed on a file with a user-mapped section open". The panic is on the injection itself, the code under test never runs. There is no portable injection. Truncating a page the reader holds is what the check under test detects, so the mapping cannot be avoided; failing the adopted page open instead needs a listed but unopenable file, and `local_list_files` descends into matching directories rather than listing them; failing the directory listing needs the storage directory removed, which Windows also refuses while pages are mapped. The storage itself is fine on Windows, its append path grows mapped pages there and every other Logstore test passes. The logic under test is platform independent and stays covered elsewhere, with the tracker half of the guarantee pinned by `test_live_reload`, which runs on every target. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: generall <andrey@vasnetsov.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Luis Cossío <luis.cossio@outlook.com>
466 lines
13 KiB
TOML
466 lines
13 KiB
TOML
[package]
|
|
name = "qdrant"
|
|
version = "1.18.3-dev"
|
|
description = "Qdrant - Vector Search engine"
|
|
authors = [
|
|
"Andrey Vasnetsov <andrey@vasnetsov.com>",
|
|
"Qdrant Team <info@qdrant.tech>",
|
|
]
|
|
readme = "README.md"
|
|
homepage = "https://qdrant.tech/"
|
|
repository = "https://github.com/qdrant/qdrant"
|
|
license = "Apache-2.0"
|
|
edition = "2024"
|
|
rust-version = "1.97"
|
|
default-run = "qdrant"
|
|
|
|
[lints]
|
|
workspace = true
|
|
|
|
[features]
|
|
default = []
|
|
service_debug = ["parking_lot/deadlock_detection", "collection/testing"]
|
|
tracing = [
|
|
"api/tracing",
|
|
"collection/tracing",
|
|
"segment/tracing",
|
|
"storage/tracing",
|
|
]
|
|
console = ["console-subscriber"]
|
|
console-subscriber = ["tracing", "dep:console-subscriber"]
|
|
tracy = ["tracing-tracy"]
|
|
tracing-tracy = ["tracing", "dep:tracing-tracy"]
|
|
tokio-tracing = ["tokio/tracing"]
|
|
stacktrace = ["rstack-self"]
|
|
chaos-testing = []
|
|
data-consistency-check = ["collection/data-consistency-check"]
|
|
gpu = ["gpu/gpu", "segment/gpu"]
|
|
deb = []
|
|
staging = ["collection/staging", "storage/staging", "shard/staging"]
|
|
|
|
[dev-dependencies]
|
|
serde_urlencoded = "0.7"
|
|
sealed_test = "1.1.0"
|
|
|
|
mockito = { workspace = true }
|
|
tempfile = { workspace = true }
|
|
rusty-hook = "^0.11.2"
|
|
nix = { workspace = true, features = ["process"] }
|
|
fs-err = { workspace = true, features = ["debug", "debug_tokio", "tokio"] } # for nicer error messages
|
|
|
|
|
|
[dependencies]
|
|
parking_lot = { workspace = true }
|
|
|
|
fs-err = { workspace = true }
|
|
thiserror = { workspace = true }
|
|
log = { workspace = true }
|
|
colored = "3"
|
|
serde = { workspace = true }
|
|
serde_json = { workspace = true }
|
|
serde_with = { workspace = true }
|
|
chrono = { workspace = true }
|
|
rand = { workspace = true }
|
|
schemars = { workspace = true }
|
|
itertools = { workspace = true }
|
|
anyhow = "1.0.98"
|
|
futures = { workspace = true }
|
|
futures-util = { workspace = true }
|
|
clap = { workspace = true }
|
|
ctrlc = "3.5.2"
|
|
env_logger = { workspace = true }
|
|
serde_cbor = { workspace = true }
|
|
uuid = { workspace = true }
|
|
sys-info = "0.9.1"
|
|
ordered-float = { workspace = true }
|
|
ahash = { workspace = true }
|
|
urlencoding = { workspace = true }
|
|
|
|
config = { version = "0.15.22", default-features = false, features = ["yaml"] }
|
|
|
|
tokio = { workspace = true }
|
|
tokio-util = { workspace = true }
|
|
|
|
actix-cors = "0.7.1"
|
|
actix-web-validator = { workspace = true }
|
|
actix-web = { workspace = true }
|
|
actix-files = { workspace = true }
|
|
tonic = { workspace = true }
|
|
tonic-reflection = { workspace = true }
|
|
tower = { version = "0.5.3", features = ["util"] }
|
|
tower-layer = "0.3.3"
|
|
http = { workspace = true }
|
|
reqwest = { workspace = true }
|
|
# rustls minor version must be synced with actix-web
|
|
rustls = { version = "0.23.37", default-features = false, features = [
|
|
"logging",
|
|
"std",
|
|
"tls12",
|
|
"ring",
|
|
] }
|
|
rustls-pki-types = "1.14.0"
|
|
rustls-pemfile = "2.2.0"
|
|
prometheus = { version = "0.14.0", default-features = false }
|
|
validator = { workspace = true }
|
|
jsonwebtoken = { version = "10.0", features = ["rust_crypto"] }
|
|
tempfile = { workspace = true }
|
|
|
|
# Consensus related crates
|
|
raft = { workspace = true }
|
|
slog = { version = "2.8.2", features = [
|
|
"max_level_trace",
|
|
"release_max_level_debug",
|
|
] }
|
|
slog-stdlog = "4.1.1"
|
|
prost-for-raft = { workspace = true }
|
|
raft-proto = { git = "https://github.com/tikv/raft-rs", rev = "aafb07c7bab439c6139926a77dfafc5b10e9bc84", features = [
|
|
"prost-codec",
|
|
], default-features = false }
|
|
|
|
common = { path = "lib/common/common" }
|
|
cancel = { path = "lib/common/cancel" }
|
|
issues = { path = "lib/common/issues" }
|
|
segment = { path = "lib/segment", default-features = false }
|
|
shard = { path = "lib/shard", default-features = false }
|
|
collection = { path = "lib/collection" }
|
|
storage = { path = "lib/storage" }
|
|
api = { path = "lib/api" }
|
|
bm25 = { path = "lib/bm25" }
|
|
gpu = { path = "lib/gpu" }
|
|
wal = { path = "lib/wal" }
|
|
|
|
actix-multipart = "0.8.0"
|
|
constant_time_eq = "0.5.0"
|
|
|
|
# Profiling
|
|
tracing = { workspace = true }
|
|
tracing-subscriber = { version = "0.3", features = [
|
|
"env-filter",
|
|
"fmt",
|
|
"json",
|
|
] }
|
|
tracing-log = { version = "0.2", default-features = false, features = [
|
|
"log-tracer",
|
|
"std",
|
|
] }
|
|
console-subscriber = { version = "0.5.0", default-features = false, features = [
|
|
"parking_lot",
|
|
], optional = true }
|
|
tracing-tracy = { version = "0.11.4", features = ["ondemand"], optional = true }
|
|
actix-web-extras = "0.1.0"
|
|
|
|
[target.'cfg(target_os = "linux")'.dependencies]
|
|
procfs = { version = "0.18.0", default-features = false }
|
|
pyroscope = { version = "2.1.0", features = ["backend-pprof-rs", "backend-jemalloc"] }
|
|
# Backtrace
|
|
rstack-self = { version = "0.3.0", optional = true }
|
|
|
|
# Jemalloc
|
|
[target.'cfg(all(not(target_env = "msvc"), any(target_arch = "x86_64", target_arch = "aarch64")))'.dependencies]
|
|
tikv-jemallocator = { version = "0.7.0", features = [
|
|
"stats",
|
|
"unprefixed_malloc_on_supported_platforms",
|
|
"background_threads",
|
|
"profiling",
|
|
] }
|
|
tikv-jemalloc-ctl = { version = "0.7.0", features = ["stats"] }
|
|
|
|
[workspace.lints.clippy]
|
|
cast_lossless = "warn"
|
|
doc_link_with_quotes = "warn"
|
|
enum_glob_use = "warn"
|
|
explicit_into_iter_loop = "warn"
|
|
filter_map_next = "warn"
|
|
flat_map_option = "warn"
|
|
implicit_clone = "warn"
|
|
inconsistent_struct_constructor = "warn"
|
|
inefficient_to_string = "warn"
|
|
manual_is_variant_and = "warn"
|
|
manual_let_else = "warn"
|
|
needless_continue = "warn"
|
|
needless_raw_string_hashes = "warn"
|
|
ptr_as_ptr = "warn"
|
|
ref_option_ref = "warn"
|
|
uninlined_format_args = "warn"
|
|
unnecessary_wraps = "warn"
|
|
unused_self = "warn"
|
|
used_underscore_binding = "warn"
|
|
match_wildcard_for_single_variants = "warn"
|
|
needless_pass_by_ref_mut = "warn"
|
|
unused_async = "warn"
|
|
wildcard_enum_match_arm = "warn"
|
|
|
|
[workspace.lints.rust]
|
|
# https://blog.rust-lang.org/2024/05/06/check-cfg.html#expecting-custom-cfgs
|
|
unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] }
|
|
|
|
[workspace.lints.rustdoc]
|
|
private_intra_doc_links = "allow"
|
|
|
|
[workspace.dependencies]
|
|
actix-web-validator = "7.0.0"
|
|
actix-web = { version = "4.13.0", features = ["rustls-0_23", "actix-tls"] }
|
|
actix-files = "0.6.10"
|
|
ahash = { version = "0.8.12", features = ["serde"] }
|
|
anyhow = "1.0.102"
|
|
async-trait = "0.1.89"
|
|
atomicwrites = "0.4.4"
|
|
bincode = "1.3.3" # no upgrade because 2.0.x is much slower https://github.com/qdrant/qdrant/pull/6134
|
|
bitpacking = "0.9.3"
|
|
blink-alloc = "0.4.0"
|
|
bytemuck = { version = "1.25.0", features = [
|
|
"derive",
|
|
"extern_crate_alloc",
|
|
"must_cast",
|
|
"transparentwrapper_extra",
|
|
] }
|
|
bytes = "1.11.1"
|
|
cc = "1.2"
|
|
chrono = { version = "0.4.44", features = ["serde"] }
|
|
clap = { version = "4.6.1", features = ["derive", "env"] }
|
|
criterion = "0.8.2"
|
|
data-encoding = "2.11.0"
|
|
ecow = { version = "0.3.0", features = ["serde"] }
|
|
env_logger = "0.11"
|
|
fnv = "1.0"
|
|
fs4 = "1.1.0"
|
|
fs-err = { version = "3.3.0", features = ["tokio"] }
|
|
fs_extra = "1.3.0"
|
|
futures = "0.3.32"
|
|
futures-util = "0.3.32"
|
|
generic-tests = "0.1.3"
|
|
half = { version = "2.7.1", features = [
|
|
"alloc",
|
|
"bytemuck",
|
|
"serde",
|
|
"num-traits",
|
|
] }
|
|
http = "1.4.1"
|
|
humantime = "2.3.0"
|
|
indexmap = { version = "2", features = ["serde"] }
|
|
url = { version = "2.5.8", features = ["serde"] }
|
|
urlencoding = "2.1.3"
|
|
indicatif = { version = "0.18.4", features = ["rayon"] }
|
|
integer-encoding = "4.1.0"
|
|
itertools = "0.15.0"
|
|
log = "0.4.31"
|
|
memmap2 = "0.9.10"
|
|
mockito = "1.7"
|
|
murmur3_32 = "0.1"
|
|
nix = { version = "0.31", features = ["fs", "feature"] }
|
|
num-traits = "0.2.19"
|
|
num_cpus = "1.17"
|
|
object_store = { version = "0.14.0", features = ["aws", "gcp", "azure"] }
|
|
ordered-float = { version = "5.3.0", features = ["serde", "schemars", "bytemuck"] }
|
|
rayon = "1.12.0"
|
|
parking_lot = { version = "0.12.5", features = ["arc_lock", "deadlock_detection", "serde"] }
|
|
ph = "0.8.5"
|
|
pprof = { version = "0.15.0", features = ["flamegraph", "prost-codec"] }
|
|
proptest = { version = "1.11.0", default-features = false, features = ["std"] }
|
|
prost = "0.14.0"
|
|
prost-build = { version = "0.14.0", features = ["cleanup-markdown"] }
|
|
prost-types = "0.14.0"
|
|
prost-wkt-types = "0.7"
|
|
prost-for-raft = { package = "prost", version = "=0.11.9" } # version of prost used by raft
|
|
raft = { git = "https://github.com/tikv/raft-rs", rev = "aafb07c7bab439c6139926a77dfafc5b10e9bc84" ,features = ["prost-codec"], default-features = false }
|
|
rand = "0.10.1"
|
|
rand_distr = "0.6.0"
|
|
rmp-serde = "~1.3"
|
|
roaring = "0.11.4"
|
|
reqwest = { version = "0.13.4", default-features = false, features = [
|
|
"json",
|
|
"http2",
|
|
"stream",
|
|
"rustls",
|
|
"blocking",
|
|
] }
|
|
rstest = { version = "0.26.1", default-features = false }
|
|
schemars = { version = "0.8.22", features = [
|
|
"uuid1",
|
|
"preserve_order",
|
|
"chrono",
|
|
"url",
|
|
"indexmap2",
|
|
] }
|
|
semver = { version = "1.0", features = ["serde"] }
|
|
self_cell = "1.2.2"
|
|
serde = { version = "~1.0", features = ["derive", "rc"] }
|
|
serde_cbor = "0.11.2"
|
|
serde_variant = "0.1.3"
|
|
serde_with = "3.21.0"
|
|
sha2 = "0.11.0"
|
|
serde_json = { version = "~1.0", features = ["preserve_order"] }
|
|
slab = "0.4.12"
|
|
strum = { version = "0.28.0", features = ["derive"] }
|
|
tap = "1.0.1"
|
|
tar = "0.4.46"
|
|
tempfile = "3.27.0"
|
|
tinyvec = { version = "1.11.0", features = ["alloc", "latest_stable_rust"] }
|
|
tokio = { version = "1.52.3", features = ["full"] }
|
|
tokio-util = { version = "0.7", features = ["io", "io-util", "rt"] }
|
|
tonic = { version = "0.14.6", features = ["gzip", "tls-ring"] }
|
|
tonic-prost = { version = "0.14.6" }
|
|
tonic-prost-build = { version = "0.14.6" }
|
|
tonic-reflection = "0.14.6"
|
|
tracing = { version = "0.1", features = ["async-await"] }
|
|
uuid = { version = "1.23", features = ["v4", "serde"] }
|
|
validator = { version = "0.20.0", features = ["derive"] }
|
|
zerocopy = { version = "0.8.50", features = ["derive"] }
|
|
atomic_refcell = "0.1.14"
|
|
byteorder = "1.5.0"
|
|
thiserror = "2.0.18"
|
|
bitvec = "1.0.1"
|
|
smallvec = { version = "1.15.1", features = ["write"] }
|
|
dashmap = "6.2"
|
|
walkdir = "2.5.0"
|
|
|
|
[patch.crates-io]
|
|
tonic = { git = "https://github.com/qdrant/tonic", branch = "v0.14.6-qdrant" }
|
|
|
|
[[bin]]
|
|
name = "schema_generator"
|
|
path = "src/schema_generator.rs"
|
|
test = false
|
|
bench = false
|
|
required-features = ["service_debug"]
|
|
|
|
[[bin]]
|
|
name = "wal_inspector"
|
|
path = "src/wal_inspector.rs"
|
|
test = false
|
|
bench = false
|
|
required-features = ["service_debug"]
|
|
|
|
[[bin]]
|
|
name = "wal_pop"
|
|
path = "src/wal_pop.rs"
|
|
test = false
|
|
bench = false
|
|
required-features = ["service_debug"]
|
|
|
|
[[bin]]
|
|
name = "segment_inspector"
|
|
path = "src/segment_inspector.rs"
|
|
test = false
|
|
bench = false
|
|
required-features = ["service_debug"]
|
|
|
|
[[bin]]
|
|
name = "model_testing"
|
|
path = "src/model_testing.rs"
|
|
test = false
|
|
bench = false
|
|
required-features = ["service_debug"]
|
|
|
|
|
|
[workspace]
|
|
members = [
|
|
"lib/api", "lib/bm25",
|
|
"lib/collection",
|
|
"lib/common/*",
|
|
"lib/edge",
|
|
"lib/edge/python",
|
|
"lib/edge/python/codegen",
|
|
"lib/edge/ffi",
|
|
"lib/edge/ffi/bindgen",
|
|
"lib/edge/tools/shard_query",
|
|
"lib/blobstore",
|
|
"lib/macros",
|
|
"lib/posting_list",
|
|
"lib/segment",
|
|
"lib/shard",
|
|
"lib/sparse",
|
|
"lib/storage",
|
|
"lib/trififo",
|
|
"lib/uio-grpc-client",
|
|
"lib/wal",
|
|
]
|
|
|
|
[profile.release]
|
|
lto = "fat"
|
|
codegen-units = 1
|
|
|
|
# Profile for mobile XCFramework / AAR builds.
|
|
# Trades a bit of runtime perf for much faster link times and smaller binaries.
|
|
# Uses panic = "unwind" (NOT abort) so UniFFI's catch_unwind keeps working: a Rust
|
|
# panic degrades to a catchable error on the Swift/Kotlin side instead of aborting the
|
|
# host process — important for an on-device database (abort risks WAL/segment
|
|
# consistency). Measured cost vs abort: ~+181 KB in the linked binary.
|
|
[profile.release-mobile]
|
|
inherits = "release"
|
|
lto = "thin"
|
|
codegen-units = 1
|
|
strip = "symbols"
|
|
panic = "unwind"
|
|
|
|
# Inherit from release, because we are not rebuilding often,
|
|
# and we don't want the huge binary sizes from debug builds.
|
|
[profile.ci]
|
|
inherits = "release"
|
|
debug-assertions = true
|
|
lto = false
|
|
opt-level = 0
|
|
|
|
# Inherits by default from release
|
|
[profile.bench]
|
|
lto = false
|
|
debug = true
|
|
codegen-units = 256 # restore default value for faster compilation
|
|
|
|
# Profile for performance testing, which is faster to build than release.
|
|
[profile.perf]
|
|
inherits = "release"
|
|
lto = false
|
|
opt-level = 3
|
|
codegen-units = 256 # restore default value for faster compilation
|
|
|
|
[profile.dev]
|
|
debug = "line-tables-only"
|
|
|
|
# Override for fast sha256 hashing in dev builds
|
|
[profile.dev.package.sha2]
|
|
opt-level = 3
|
|
|
|
# Override for fast sha256 hashing in ci builds
|
|
[profile.ci.package.sha2]
|
|
opt-level = 3
|
|
|
|
[package.metadata.deb]
|
|
maintainer = "Qdrant Team <team@qdrant.com>"
|
|
depends = "$auto"
|
|
default-features = true
|
|
features = ["deb"]
|
|
license-file = "LICENSE"
|
|
section = "database"
|
|
priority = "optional"
|
|
extended-description = """\
|
|
Qdrant is a high-performance vector search engine. It is designed to index and search large collections of high-dimensional vectors.\
|
|
"""
|
|
assets = [
|
|
[
|
|
"target/release/qdrant",
|
|
"usr/bin/",
|
|
"755",
|
|
],
|
|
[
|
|
"README.md",
|
|
"usr/share/doc/qdrant/README",
|
|
"644",
|
|
],
|
|
[
|
|
"static/*",
|
|
"var/lib/qdrant/static/",
|
|
"644",
|
|
],
|
|
[
|
|
"static/assets/*",
|
|
"var/lib/qdrant/static/assets/",
|
|
"644",
|
|
],
|
|
[
|
|
"config/deb.yaml",
|
|
"etc/qdrant/config.yaml",
|
|
"644",
|
|
],
|
|
]
|