mirror of
https://github.com/qdrant/qdrant.git
synced 2026-07-23 11:11:00 -05:00
d91e8db5f9
* download tar
* compute sha256 for stream download
* wip: propagate unpacking into down to the logic, todo: validation
* unpacked snapshot validation
* Minor tweaks
* Fix typo
* validation during unpack
* cancellation token
* update docstring
* remove redundant dep
* Rearrange unpack functions
- Rename `safe_unpack.rs` into `tar_unpack.rs` so it would be listed
near `tar_ext.rs` in IDEs.
- Replace calls like `ar = open_snapshot_archive(…); safe_unpack(ar, …);`
with a single call to `tar_unpack_file(…)`.
- Put calls to `Archive::new(); Archive::set_overwrite(false);` inside
`tar_unpack_reader` (was `safe_unpack`). So, now it is the only place
that does `set_overwrite`.
* Let clippy complain if tar::Archive::unpack used
* Mock snapshot download URL
Instead of downloading from storage.googleapis.com every time the test
runs, put small snapshot file to the repo.
The snapshot file is created using this command:
curl -s \
https://storage.googleapis.com/qdrant-benchmark-snapshots/test-shard.snapshot \
| tar \
--delete segments/4ea958d8-0b64-4312-9a53-0cd857e93535.tar \
--delete segments/65ac6276-8cca-4f5c-b767-9722190cee8b.tar \
> lib/storage/src/content_manager/snapshots/test-shard.snapshot
File contents:
$ tar tf lib/storage/src/content_manager/snapshots/test-shard.snapshot
wal/
wal/closed-255
newest_clocks.json
replica_state.json
shard_config.json
$ du -sh lib/storage/src/content_manager/snapshots/test-shard.snapshot
12K lib/storage/src/content_manager/snapshots/test-shard.snapshot
$ sha256sum < lib/storage/src/content_manager/snapshots/test-shard.snapshot
5d94eac5c1ede3994a28bc406120046c37370d5d45b489a0d2252531b4e3e1f2 -
---------
Co-authored-by: timvisee <tim@visee.me>
Co-authored-by: xzfc <xzfcpw@gmail.com>
74 lines
2.0 KiB
TOML
74 lines
2.0 KiB
TOML
[package]
|
|
name = "storage"
|
|
version = "0.2.0"
|
|
authors = [
|
|
"Andrey Vasnetsov <vasnetsov93@gmail.com>",
|
|
"Qdrant Team <info@qdrant.tech>",
|
|
]
|
|
license = "Apache-2.0"
|
|
edition = "2024"
|
|
|
|
[lints]
|
|
workspace = true
|
|
|
|
[features]
|
|
tracing = ["dep:tracing", "api/tracing", "collection/tracing", "segment/tracing"]
|
|
staging = ["collection/staging"]
|
|
|
|
[dev-dependencies]
|
|
fs-err = { workspace = true, features = ["debug"] }
|
|
proptest = { workspace = true }
|
|
mockito = { workspace = true }
|
|
env_logger = "0.11"
|
|
|
|
[dependencies]
|
|
ahash = { workspace = true }
|
|
fs-err = { workspace = true }
|
|
thiserror = { workspace = true }
|
|
rand = { workspace = true }
|
|
wal = { workspace = true }
|
|
tokio = { workspace = true }
|
|
tokio-util = { workspace = true }
|
|
serde = { workspace = true }
|
|
serde_json = { workspace = true }
|
|
schemars = { workspace = true }
|
|
itertools = { workspace = true }
|
|
log = { workspace = true }
|
|
tonic = { workspace = true }
|
|
http = "0.2.12" # version of http used by tonic 0.11.x
|
|
parking_lot = { workspace = true }
|
|
strum = { workspace = true }
|
|
tap = { workspace = true }
|
|
tar = { workspace = true }
|
|
chrono = { workspace = true }
|
|
validator = { workspace = true }
|
|
dashmap = { workspace = true }
|
|
semver = { workspace = true }
|
|
sha2 = { workspace = true }
|
|
|
|
# Consensus related
|
|
atomicwrites = { workspace = true }
|
|
raft = { workspace = true }
|
|
prost-for-raft = { workspace = true }
|
|
protobuf = "2.28.0" # version of protobuf used by raft
|
|
serde_cbor = { workspace = true }
|
|
|
|
common = { path = "../common/common" }
|
|
cancel = { path = "../common/cancel" }
|
|
io = { path = "../common/io" }
|
|
issues = { path = "../common/issues" }
|
|
memory = { path = "../common/memory" }
|
|
segment = { path = "../segment", default-features = false }
|
|
collection = { path = "../collection" }
|
|
shard = { path = "../shard" }
|
|
api = { path = "../api" }
|
|
futures = { workspace = true }
|
|
anyhow = "1.0.100"
|
|
uuid = { workspace = true }
|
|
url = "2.5.8"
|
|
reqwest = { workspace = true }
|
|
tempfile = { workspace = true }
|
|
async-trait = "0.1.89"
|
|
|
|
tracing = { workspace = true, optional = true }
|