mirror of
https://github.com/qdrant/qdrant.git
synced 2026-07-26 04:31:02 -05:00
2ccbd9ed82
* refactor(edge): split EdgeShard load into new() and load() - new(path, config): create edge shard only when path has no existing segments - load(path, config?): load from existing files; config optional (load from edge_config.json or infer from segments) - Helpers: has_existing_segments, ensure_dirs_and_open_wal, resolve_initial_config, load_segments, ensure_appendable_segment - Python: EdgeShard.create_new(path, config); __init__ still calls load() - Examples use new() for creation and load(path, None) for reopen - Error instead of panic on config mismatch; explicit load/create in Python - fix: use OptimizerThresholds.deferred_internal_id for dev compatibility Made-with: Cursor * rollback threshold changes * fix(edge): address dancixx review comments - Persist inferred config in load() when edge_config.json does not exist (SaveOnDisk::new only saves when file exists) - Python: add #[new] delegating to load() for backward compatibility - qdrant_edge.pyi: Optional return types for deleted_threshold, vacuum_min_vector_number, default_segment_number; add __init__ doc Made-with: Cursor * Revert "fix(edge): address dancixx review comments" This reverts commit 370e21a6c74c41210e1658f89814395cb86ed81c. * fix: optional returns * fix: save config it is not exist * fix: save data on disk always * fix: python examples * chore: remove edge.close() * fix: wal lock * fix: rename of EdgeShardConfig -> EdgeConfig --------- Co-authored-by: Cursor Agent <agent@cursor.com> Co-authored-by: Andrey Vasnetsov <andrey@vasnetsov.com> Co-authored-by: Daniel Boros <dancixx@gmail.com>
68 lines
2.0 KiB
Python
Executable File
68 lines
2.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# See lib/edge/publish/examples/src/bin/restore-snapshot.rs for the equivalent Rust example.
|
|
|
|
import os
|
|
import requests
|
|
import shutil
|
|
import json
|
|
|
|
from common import DATA_DIR
|
|
|
|
from qdrant_edge import *
|
|
|
|
SNAPSHOT_URL = 'https://storage.googleapis.com/qdrant-benchmark-snapshots/test-shard.snapshot'
|
|
|
|
# Obtained via download_partial.py and manually inserting a new point with ID 100500
|
|
PARTIAL_SNAPSHOT_URL = 'https://storage.googleapis.com/qdrant-benchmark-snapshots/partial.snapshot'
|
|
|
|
|
|
# Download the snapshot file into data directory
|
|
def download_snapshot(url: str, dest_folder: str):
|
|
if not os.path.exists(dest_folder):
|
|
os.makedirs(dest_folder)
|
|
|
|
local_filename = os.path.join(dest_folder, url.split('/')[-1])
|
|
if os.path.exists(local_filename):
|
|
print(f"Snapshot already exists at: {local_filename}")
|
|
return local_filename
|
|
|
|
with requests.get(url, stream=True, timeout = 10) as r:
|
|
r.raise_for_status()
|
|
with open(local_filename, 'wb') as f:
|
|
for chunk in r.iter_content(chunk_size=8192):
|
|
f.write(chunk)
|
|
return local_filename
|
|
|
|
snapshot_path = download_snapshot(SNAPSHOT_URL, DATA_DIR)
|
|
print(f"Snapshot downloaded to: {snapshot_path}")
|
|
|
|
recovered_path = os.path.join(DATA_DIR, "restored_shard")
|
|
print(f"Restoring shard from snapshot to: {recovered_path}")
|
|
if os.path.exists(recovered_path):
|
|
print("Removing existing recovered shard directory...")
|
|
shutil.rmtree(recovered_path)
|
|
|
|
EdgeShard.unpack_snapshot(snapshot_path, recovered_path)
|
|
|
|
shard = EdgeShard.load(recovered_path)
|
|
|
|
points = shard.retrieve(point_ids=[1, 2, 3], with_vector=False, with_payload=True)
|
|
|
|
for point in points:
|
|
print(point)
|
|
|
|
print("Manifest of restored shard:", json.dumps(shard.snapshot_manifest(), indent=2))
|
|
|
|
partial_snapshot_path = download_snapshot(PARTIAL_SNAPSHOT_URL, DATA_DIR)
|
|
|
|
shard.update_from_snapshot(partial_snapshot_path)
|
|
|
|
points = shard.retrieve(point_ids=[100500], with_vector=False, with_payload=True)
|
|
|
|
for point in points:
|
|
print(point)
|
|
|
|
info = shard.info()
|
|
|
|
print(info)
|