mirror of
https://github.com/qdrant/qdrant.git
synced 2026-07-23 11:11:00 -05:00
f403f6d00d
* download tar
* compute sha256 for stream download
* wip: propagate unpacking into down to the logic, todo: validation
* unpacked snapshot validation
* Minor tweaks
* Fix typo
* validation during unpack
* cancellation token
* update docstring
* remove redundant dep
* Rearrange unpack functions
- Rename `safe_unpack.rs` into `tar_unpack.rs` so it would be listed
near `tar_ext.rs` in IDEs.
- Replace calls like `ar = open_snapshot_archive(…); safe_unpack(ar, …);`
with a single call to `tar_unpack_file(…)`.
- Put calls to `Archive::new(); Archive::set_overwrite(false);` inside
`tar_unpack_reader` (was `safe_unpack`). So, now it is the only place
that does `set_overwrite`.
* Let clippy complain if tar::Archive::unpack used
* Mock snapshot download URL
Instead of downloading from storage.googleapis.com every time the test
runs, put small snapshot file to the repo.
The snapshot file is created using this command:
curl -s \
https://storage.googleapis.com/qdrant-benchmark-snapshots/test-shard.snapshot \
| tar \
--delete segments/4ea958d8-0b64-4312-9a53-0cd857e93535.tar \
--delete segments/65ac6276-8cca-4f5c-b767-9722190cee8b.tar \
> lib/storage/src/content_manager/snapshots/test-shard.snapshot
File contents:
$ tar tf lib/storage/src/content_manager/snapshots/test-shard.snapshot
wal/
wal/closed-255
newest_clocks.json
replica_state.json
shard_config.json
$ du -sh lib/storage/src/content_manager/snapshots/test-shard.snapshot
12K lib/storage/src/content_manager/snapshots/test-shard.snapshot
$ sha256sum < lib/storage/src/content_manager/snapshots/test-shard.snapshot
5d94eac5c1ede3994a28bc406120046c37370d5d45b489a0d2252531b4e3e1f2 -
---------
Co-authored-by: timvisee <tim@visee.me>
Co-authored-by: xzfc <xzfcpw@gmail.com>
99 lines
7.2 KiB
TOML
99 lines
7.2 KiB
TOML
# For Rust 1.87 until fixed: <https://github.com/hyperium/tonic/issues/2253>
|
|
large-error-threshold = 256
|
|
|
|
disallowed-types = [
|
|
# Use fs_err instead of std::fs and tokio::fs
|
|
{ path = "std::fs::DirEntry", replacement = "fs_err::DirEntry" },
|
|
{ path = "std::fs::File", replacement = "fs_err::File" },
|
|
{ path = "std::fs::OpenOptions", replacement = "fs_err::OpenOptions" },
|
|
{ path = "std::fs::ReadDir", replacement = "fs_err::ReadDir" },
|
|
{ path = "tokio::fs::DirBuilder", replacement = "fs_err::tokio::DirBuilder" },
|
|
{ path = "tokio::fs::DirEntry", replacement = "fs_err::tokio::DirEntry" },
|
|
{ path = "tokio::fs::File", replacement = "fs_err::tokio::File" },
|
|
{ path = "tokio::fs::OpenOptions", replacement = "fs_err::tokio::OpenOptions" },
|
|
{ path = "tokio::fs::ReadDir", replacement = "fs_err::tokio::ReadDir" },
|
|
]
|
|
|
|
disallowed-methods = [
|
|
{ path = "tar::Archive::unpack", replacement = "common::tar_unpack", reason = "Use safe unpackers" },
|
|
|
|
# Use fs_err instead of std::fs and tokio::fs
|
|
{ path = "std::fs::DirEntry::file_type", replacement = "fs_err::DirEntry::file_type" },
|
|
{ path = "std::fs::DirEntry::metadata", replacement = "fs_err::DirEntry::metadata" },
|
|
{ path = "std::fs::File::create", replacement = "fs_err::File::create" },
|
|
{ path = "std::fs::File::create_new", replacement = "fs_err::File::create_new" },
|
|
{ path = "std::fs::File::lock", replacement = "fs_err::File::lock" },
|
|
{ path = "std::fs::File::lock_shared", replacement = "fs_err::File::lock_shared" },
|
|
{ path = "std::fs::File::metadata", replacement = "fs_err::File::metadata" },
|
|
{ path = "std::fs::File::open", replacement = "fs_err::File::open" },
|
|
{ path = "std::fs::File::options", replacement = "fs_err::File::options" },
|
|
{ path = "std::fs::File::set_len", replacement = "fs_err::File::set_len" },
|
|
{ path = "std::fs::File::set_permissions", replacement = "fs_err::File::set_permissions" },
|
|
{ path = "std::fs::File::sync_all", replacement = "fs_err::File::sync_all" },
|
|
{ path = "std::fs::File::sync_data", replacement = "fs_err::File::sync_data" },
|
|
{ path = "std::fs::File::try_clone", replacement = "fs_err::File::try_clone" },
|
|
{ path = "std::fs::File::try_lock", replacement = "fs_err::File::try_lock" },
|
|
{ path = "std::fs::File::try_lock_shared", replacement = "fs_err::File::try_lock_shared" },
|
|
{ path = "std::fs::File::unlock", replacement = "fs_err::File::unlock" },
|
|
{ path = "std::fs::canonicalize", replacement = "fs_err::canonicalize" },
|
|
{ path = "std::fs::copy", replacement = "fs_err::copy" },
|
|
{ path = "std::fs::create_dir", replacement = "fs_err::create_dir" },
|
|
{ path = "std::fs::create_dir_all", replacement = "fs_err::create_dir_all" },
|
|
{ path = "std::fs::hard_link", replacement = "fs_err::hard_link" },
|
|
{ path = "std::fs::metadata", replacement = "fs_err::metadata" },
|
|
{ path = "std::fs::read", replacement = "fs_err::read" },
|
|
{ path = "std::fs::read_dir", replacement = "fs_err::read_dir" },
|
|
{ path = "std::fs::read_link", replacement = "fs_err::read_link" },
|
|
{ path = "std::fs::read_to_string", replacement = "fs_err::read_to_string" },
|
|
{ path = "std::fs::remove_dir", replacement = "fs_err::remove_dir" },
|
|
{ path = "std::fs::remove_dir_all", replacement = "fs_err::remove_dir_all" },
|
|
{ path = "std::fs::remove_file", replacement = "fs_err::remove_file" },
|
|
{ path = "std::fs::rename", replacement = "fs_err::rename" },
|
|
{ path = "std::fs::set_permissions", replacement = "fs_err::set_permissions" },
|
|
{ path = "std::fs::soft_link", replacement = "fs_err::soft_link" },
|
|
{ path = "std::fs::symlink_metadata", replacement = "fs_err::symlink_metadata" },
|
|
{ path = "std::fs::write", replacement = "fs_err::write" },
|
|
{ path = "std::os::unix::fs::OpenOptionsExt::custom_flags", replacement = "fs_err::os::unix::fs::OpenOptionsExt::custom_flags", allow-invalid = true },
|
|
{ path = "std::os::unix::fs::OpenOptionsExt::mode", replacement = "fs_err::os::unix::fs::OpenOptionsExt::mode", allow-invalid = true },
|
|
{ path = "std::path::Path::canonicalize", replacement = "fs_err::canonicalize" },
|
|
{ path = "std::path::Path::metadata", replacement = "fs_err::metadata" },
|
|
{ path = "std::path::Path::read_dir", replacement = "fs_err::read_dir" },
|
|
{ path = "std::path::Path::read_link", replacement = "fs_err::read_link" },
|
|
{ path = "std::path::Path::symlink_metadata", replacement = "fs_err::symlink_metadata" },
|
|
{ path = "std::path::Path::try_exists", replacement = "fs_err::exists" },
|
|
{ path = "tokio::fs::File::create", replacement = "fs_err::tokio::File::create" },
|
|
{ path = "tokio::fs::File::create_new", replacement = "fs_err::tokio::File::create_new" },
|
|
{ path = "tokio::fs::File::from_std", replacement = "fs_err::tokio::File::from_std" },
|
|
{ path = "tokio::fs::File::into_std", replacement = "fs_err::tokio::File::into_std" },
|
|
{ path = "tokio::fs::File::metadata", replacement = "fs_err::tokio::File::metadata" },
|
|
{ path = "tokio::fs::File::open", replacement = "fs_err::tokio::File::open" },
|
|
{ path = "tokio::fs::File::options", replacement = "fs_err::tokio::File::options" },
|
|
{ path = "tokio::fs::File::set_len", replacement = "fs_err::tokio::File::set_len" },
|
|
{ path = "tokio::fs::File::set_permissions", replacement = "fs_err::tokio::File::set_permissions" },
|
|
{ path = "tokio::fs::File::sync_all", replacement = "fs_err::tokio::File::sync_all" },
|
|
{ path = "tokio::fs::File::sync_data", replacement = "fs_err::tokio::File::sync_data" },
|
|
{ path = "tokio::fs::File::try_clone", replacement = "fs_err::tokio::File::try_clone" },
|
|
{ path = "tokio::fs::File::try_into_std", replacement = "fs_err::tokio::File::try_into_std" },
|
|
{ path = "tokio::fs::canonicalize", replacement = "fs_err::tokio::canonicalize" },
|
|
{ path = "tokio::fs::copy", replacement = "fs_err::tokio::copy" },
|
|
{ path = "tokio::fs::create_dir", replacement = "fs_err::tokio::create_dir" },
|
|
{ path = "tokio::fs::create_dir_all", replacement = "fs_err::tokio::create_dir_all" },
|
|
{ path = "tokio::fs::hard_link", replacement = "fs_err::tokio::hard_link" },
|
|
{ path = "tokio::fs::metadata", replacement = "fs_err::tokio::metadata" },
|
|
{ path = "tokio::fs::read", replacement = "fs_err::tokio::read" },
|
|
{ path = "tokio::fs::read_dir", replacement = "fs_err::tokio::read_dir" },
|
|
{ path = "tokio::fs::read_link", replacement = "fs_err::tokio::read_link" },
|
|
{ path = "tokio::fs::read_to_string", replacement = "fs_err::tokio::read_to_string" },
|
|
{ path = "tokio::fs::remove_dir", replacement = "fs_err::tokio::remove_dir" },
|
|
{ path = "tokio::fs::remove_dir_all", replacement = "fs_err::tokio::remove_dir_all" },
|
|
{ path = "tokio::fs::remove_file", replacement = "fs_err::tokio::remove_file" },
|
|
{ path = "tokio::fs::rename", replacement = "fs_err::tokio::rename" },
|
|
{ path = "tokio::fs::set_permissions", replacement = "fs_err::tokio::set_permissions" },
|
|
{ path = "tokio::fs::symlink", replacement = "fs_err::tokio::symlink", allow-invalid = true },
|
|
{ path = "tokio::fs::symlink_dir", replacement = "fs_err::tokio::symlink_dir", allow-invalid = true },
|
|
{ path = "tokio::fs::symlink_file", replacement = "fs_err::tokio::symlink_file", allow-invalid = true },
|
|
{ path = "tokio::fs::symlink_metadata", replacement = "fs_err::tokio::symlink_metadata" },
|
|
{ path = "tokio::fs::try_exists", replacement = "fs_err::tokio::try_exists" },
|
|
{ path = "tokio::fs::write", replacement = "fs_err::tokio::write" },
|
|
]
|