use std::cmp::max;
use std::collections::HashMap;
use std::path::Path;

use common::bitvec::BitVec;
use common::counter::hardware_counter::HardwareCounterCell;
use common::fs::{atomic_save_json, read_json};
use common::generic_consts::Random;
use common::tar_unpack::tar_unpack_file;
use common::types::{DeferredBehavior, PointOffsetType};
use fs_err as fs;

use super::{
    DEPRECATED_PAYLOAD_ROCKSDB_BACKUP_PATH, DEPRECATED_ROCKSDB_BACKUP_PATH, SEGMENT_STATE_FILE,
    SNAPSHOT_FILES_PATH, SNAPSHOT_PATH, Segment,
};
use crate::common::operation_error::{
    OperationError, OperationResult, SegmentFailedState, get_service_error,
};
use crate::common::{check_named_vectors, check_vector_name};
use crate::data_types::named_vectors::NamedVectors;
use crate::data_types::tiny_map::TinyMap;
use crate::entry::entry_point::StorageSegmentEntry as _;
use crate::entry::{NonAppendableSegmentEntry as _, ReadSegmentEntry};
use crate::id_tracker::{IdTracker, IdTrackerRead};
use crate::index::{PayloadIndex, PayloadIndexRead, VectorIndex};
use crate::types::{
    Payload, PayloadFieldSchema, PayloadKeyType, PointIdType, SegmentState, SeqNumberType,
    SnapshotFormat, VectorName, VectorNameBuf,
};
use crate::utils;
use crate::vector_storage::VectorStorageRead;

/// Look up a named vector in the `retrieve_raw`-shaped `(name, bytes)` list.
fn find_raw_vector<'a>(
    vectors: &'a [(VectorNameBuf, Vec<u8>)],
    vector_name: &VectorName,
) -> Option<&'a [u8]> {
    vectors
        .iter()
        .find(|(name, _)| name == vector_name)
        .map(|(_, bytes)| bytes.as_slice())
}

impl Segment {
    /// Replace vectors in-place
    ///
    /// This replaces all named vectors for this point with the given set of named vectors.
    ///
    /// - new named vectors are inserted
    /// - existing named vectors are replaced
    /// - existing named vectors not specified are deleted
    ///
    /// This differs with [`Segment::update_vectors`], because this deletes unspecified vectors.
    ///
    /// # Warning
    ///
    /// Available for appendable segments only.
    pub(super) fn replace_all_vectors(
        &mut self,
        internal_id: PointOffsetType,
        op_num: SeqNumberType,
        vectors: &NamedVectors,
        hw_counter: &HardwareCounterCell,
    ) -> OperationResult<()> {
        debug_assert!(self.is_appendable());
        check_named_vectors(vectors, &self.segment_config)?;
        for (vector_name, vector_data) in self.vector_data.iter_mut() {
            let vector = vectors.get(vector_name);
            let mut vector_index = vector_data.vector_index.borrow_mut();
            vector_index.update_vector(internal_id, vector, hw_counter)?;
            self.version_tracker.set_vector(vector_name, Some(op_num));
        }
        Ok(())
    }

    /// Update vectors in-place
    ///
    /// This updates all specified named vectors for this point with the given set of named vectors, leaving unspecified vectors untouched.
    ///
    /// - new named vectors are inserted
    /// - existing named vectors are replaced
    /// - existing named vectors not specified are untouched and kept as-is
    ///
    /// This differs with [`Segment::replace_all_vectors`], because this keeps unspecified vectors as-is.
    ///
    /// # Warning
    ///
    /// Available for appendable segments only.
    pub(super) fn update_vectors(
        &mut self,
        internal_id: PointOffsetType,
        op_num: SeqNumberType,
        vectors: NamedVectors,
        hw_counter: &HardwareCounterCell,
    ) -> OperationResult<()> {
        debug_assert!(self.is_appendable());
        check_named_vectors(&vectors, &self.segment_config)?;
        for (vector_name, new_vector) in vectors {
            let vector_data = &self.vector_data[vector_name.as_ref()];
            let mut vector_index = vector_data.vector_index.borrow_mut();
            vector_index.update_vector(internal_id, Some(new_vector.as_vec_ref()), hw_counter)?;
            self.version_tracker.set_vector(&vector_name, Some(op_num));
        }
        Ok(())
    }

    /// Insert new vectors into the segment
    ///
    /// # Warning
    ///
    /// Available for appendable segments only.
    pub(super) fn insert_new_vectors(
        &mut self,
        point_id: PointIdType,
        op_num: SeqNumberType,
        vectors: &NamedVectors,
        hw_counter: &HardwareCounterCell,
    ) -> OperationResult<PointOffsetType> {
        debug_assert!(self.is_appendable());
        check_named_vectors(vectors, &self.segment_config)?;
        let new_index = self.id_tracker.borrow().total_point_count() as PointOffsetType;
        for (vector_name, vector_data) in self.vector_data.iter_mut() {
            let vector_opt = vectors.get(vector_name);
            let mut vector_index = vector_data.vector_index.borrow_mut();
            vector_index.update_vector(new_index, vector_opt, hw_counter)?;
            self.version_tracker.set_vector(vector_name, Some(op_num));
        }
        self.id_tracker.borrow_mut().set_link(point_id, new_index)?;
        Ok(new_index)
    }

    /// Byte-blob analogue of [`Segment::replace_all_vectors`]: vector values
    /// are storage-native bytes (the `retrieve_raw` form). Semantics are the
    /// same — named vectors not present in `vectors` are deleted.
    ///
    /// Unlike its decoded twin, `internal_id` may also be a fresh id one past
    /// the current end: the raw insert paths reuse this loop and the storages
    /// grow as needed.
    ///
    /// # Warning
    ///
    /// Available for appendable segments only.
    pub(super) fn replace_all_vectors_raw(
        &mut self,
        internal_id: PointOffsetType,
        op_num: SeqNumberType,
        vectors: &[(VectorNameBuf, Vec<u8>)],
        hw_counter: &HardwareCounterCell,
    ) -> OperationResult<()> {
        debug_assert!(self.is_appendable());
        for (vector_name, vector_data) in self.vector_data.iter_mut() {
            let bytes = find_raw_vector(vectors, vector_name);
            let mut vector_index = vector_data.vector_index.borrow_mut();
            vector_index.update_vector_raw(internal_id, bytes, hw_counter)?;
            self.version_tracker.set_vector(vector_name, Some(op_num));
        }
        Ok(())
    }

    /// Byte-blob analogue of [`Segment::insert_new_vectors`]: vector values
    /// are storage-native bytes (the `retrieve_raw` form).
    ///
    /// # Warning
    ///
    /// Available for appendable segments only.
    pub(super) fn insert_new_vectors_raw(
        &mut self,
        point_id: PointIdType,
        op_num: SeqNumberType,
        vectors: &[(VectorNameBuf, Vec<u8>)],
        hw_counter: &HardwareCounterCell,
    ) -> OperationResult<PointOffsetType> {
        debug_assert!(self.is_appendable());
        let new_index = self.id_tracker.borrow().total_point_count() as PointOffsetType;
        self.replace_all_vectors_raw(new_index, op_num, vectors, hw_counter)?;
        self.id_tracker.borrow_mut().set_link(point_id, new_index)?;
        Ok(new_index)
    }

    /// Append-only counterpart of [`Segment::replace_all_vectors_raw`]: write
    /// the raw vectors at a fresh internal id and repoint the id tracker,
    /// tombstoning `old_id` — the raw twin of [`Segment::clone_and_mutate_point`]
    /// specialized to full-vector replacement.
    ///
    /// Unlike `clone_and_mutate_point`, no vector snapshot is taken: upsert
    /// discards every old named vector anyway, so reading them (a lossy
    /// decode for TurboQuant storages) is unnecessary. Only the payload
    /// survives the move; it is rewritten at the new id — always, even when
    /// empty, for the same null-index `total_point_count` bump reason.
    ///
    /// Step order matches `clone_and_mutate_point`: vectors, then payload,
    /// then `set_link` last — so a mid-way failure leaves the id tracker
    /// still pointing at the intact `old_id`.
    ///
    /// # Warning
    ///
    /// Available for appendable segments only.
    pub(super) fn clone_and_replace_point_raw(
        &mut self,
        op_num: SeqNumberType,
        point_id: PointIdType,
        old_id: PointOffsetType,
        vectors: &[(VectorNameBuf, Vec<u8>)],
        hw_counter: &HardwareCounterCell,
    ) -> OperationResult<PointOffsetType> {
        debug_assert!(self.is_appendable());
        let payload = self
            .payload_index
            .borrow()
            .with_view(|view| view.get_payload(old_id, hw_counter))?;
        let new_id = self.id_tracker.borrow().total_point_count() as PointOffsetType;
        self.replace_all_vectors_raw(new_id, op_num, vectors, hw_counter)?;
        self.payload_index
            .borrow_mut()
            .overwrite_payload(new_id, &payload, hw_counter)?;
        // The payload content is unchanged, but writing it at `new_id` still
        // mutates payload storage: stamp it so partial snapshots re-upload
        // the changed files.
        self.version_tracker.set_payload(Some(op_num));
        self.id_tracker.borrow_mut().set_link(point_id, new_id)?;
        Ok(new_id)
    }

    /// Write a complete point at `internal_id` in one pass: vectors from a
    /// mix of storage-native bytes and decoded values, then the payload.
    ///
    /// Every configured named vector storage gets touched: a name present in
    /// `updated_vectors` is written from its decoded value, a name present
    /// only in `raw_vectors` is written from its bytes, and an absent name is
    /// written as `None` (the slot is grown and marked deleted), matching
    /// [`Segment::replace_all_vectors_raw`]'s contract. The payload is
    /// written last — always, even when empty, for the same null-index
    /// `total_point_count` bump reason as [`Segment::clone_and_mutate_point`].
    ///
    /// `internal_id` may be a fresh id one past the current end: the raw
    /// write paths grow the storages as needed.
    ///
    /// # Warning
    ///
    /// Available for appendable segments only.
    pub(super) fn write_point_parts(
        &mut self,
        internal_id: PointOffsetType,
        op_num: SeqNumberType,
        raw_vectors: &[(VectorNameBuf, Vec<u8>)],
        updated_vectors: &NamedVectors,
        payload: &Payload,
        hw_counter: &HardwareCounterCell,
    ) -> OperationResult<()> {
        debug_assert!(self.is_appendable());
        for (vector_name, vector_data) in self.vector_data.iter_mut() {
            let mut vector_index = vector_data.vector_index.borrow_mut();
            match updated_vectors.get(vector_name) {
                Some(vector) => {
                    vector_index.update_vector(internal_id, Some(vector), hw_counter)?;
                }
                None => {
                    let bytes = find_raw_vector(raw_vectors, vector_name);
                    vector_index.update_vector_raw(internal_id, bytes, hw_counter)?;
                }
            }
            self.version_tracker.set_vector(vector_name, Some(op_num));
        }
        self.payload_index
            .borrow_mut()
            .overwrite_payload(internal_id, payload, hw_counter)?;
        // The overwrite mutated payload storage: stamp it so partial
        // snapshots re-upload it (the old CoW path bumped this via
        // `set_full_payload`).
        self.version_tracker.set_payload(Some(op_num));
        Ok(())
    }

    /// Append-only update: snapshot the point at `old_id` into owned raw
    /// vectors and payload, hand them to `mutate` for in-memory modification,
    /// then write the result at a fresh internal id and repoint the id
    /// tracker.
    ///
    /// Step order:
    ///
    /// 1. Read all named vectors at `old_id` as storage-native bytes into an
    ///    owned name → bytes map, and the full payload at `old_id` into an
    ///    owned `Payload`.
    /// 2. Call `mutate(&mut raw_vectors, &mut updated_vectors, &mut payload)`
    ///    to apply the op-specific change in memory: drop raw entries to
    ///    delete names, insert decoded vectors into the (initially empty)
    ///    `updated_vectors` overlay to overwrite names. The return value is
    ///    propagated to the caller alongside the new internal id.
    /// 3. Allocate `new_id = total_point_count()`.
    /// 4. Write every configured named vector at `new_id`, overlay first:
    ///    overlaid names are written decoded, remaining raw entries are
    ///    ingested verbatim — lossless for requantizing storages
    ///    (TurboQuant-as-datatype) — and names in neither are inserted as
    ///    `None` (so the slot is grown and marked deleted, matching
    ///    `insert_new_vectors`'s behavior).
    /// 5. Write the mutated payload at `new_id` — always, even when empty,
    ///    so every field index covers `new_id` (see the body comment).
    /// 6. `set_link(point_id, new_id)` — auto-tombstones `old_id` in the id
    ///    tracker so it becomes invisible to queries.
    ///
    /// `old_id`'s vector slots and payload row are intentionally left in
    /// place. Appendable storages don't support physical removal; the
    /// tombstoned slot is reclaimed later by segment optimization. Stale
    /// field-index postings keyed by `old_id` likewise stay; readers filter
    /// them via the id tracker's deleted bitslice.
    ///
    /// Returns the closure's return value together with the freshly
    /// allocated internal id. The new id should be handed to
    /// [`Segment::handle_point_version_and_failure`] so the point version is
    /// recorded against the new slot.
    ///
    /// # Warning
    ///
    /// Available for appendable segments only. Callers route into this
    /// helper from the `SegmentEntry` mutation paths when
    /// [`Segment::is_append_only`] is true — except for slots written by the
    /// current operation, which are mutated in place instead (see
    /// [`Segment::handle_point_mutate`]).
    pub(super) fn clone_and_mutate_point<F, R>(
        &mut self,
        op_num: SeqNumberType,
        point_id: PointIdType,
        old_id: PointOffsetType,
        hw_counter: &HardwareCounterCell,
        mutate: F,
    ) -> OperationResult<(R, PointOffsetType)>
    where
        F: FnOnce(
            &mut TinyMap<VectorNameBuf, Vec<u8>>,
            &mut NamedVectors<'static>,
            &mut Payload,
        ) -> OperationResult<R>,
    {
        debug_assert!(self.is_appendable());

        // 1. Snapshot vectors and payload at old_id into owned containers,
        //    dropping all storage borrows before we start writing. Vectors
        //    are read as storage-native bytes: names the operation does not
        //    overwrite travel to new_id verbatim, avoiding the lossy
        //    dequantize→requantize round-trip of TurboQuant-as-datatype
        //    storages. Slots that are marked deleted in the per-vector
        //    bitslice carry leftover default-vector bytes from the original
        //    `update_vector(_, None, _)` insert — including those would
        //    promote phantom data into the fresh slot, so we skip them
        //    and write `None` at new_id (re-tombstoning the slot). `TinyMap`
        //    keeps the container inline (no allocation), like the decoded
        //    `NamedVectors` snapshot this replaced.
        let mut raw_vectors: TinyMap<VectorNameBuf, Vec<u8>> = TinyMap::new();
        for (vector_name, vector_data) in self.vector_data.iter() {
            let storage = vector_data.vector_storage.borrow();
            if storage.is_deleted_vector(old_id) {
                continue;
            }
            if let Some(bytes) = storage.vector_bytes_opt::<Random>(old_id)? {
                raw_vectors.insert(vector_name.clone(), bytes);
            }
        }
        let mut updated_vectors: NamedVectors<'static> = NamedVectors::default();
        let mut payload = self
            .payload_index
            .borrow()
            .with_view(|view| view.get_payload(old_id, hw_counter))?;

        // 2. Let the caller apply the op-specific change in memory.
        let mutate_result = mutate(&mut raw_vectors, &mut updated_vectors, &mut payload)?;

        // 3. Allocate the fresh internal id.
        let new_id = self.id_tracker.borrow().total_point_count() as PointOffsetType;

        // 4. Write every configured named vector at new_id, overlay first:
        //    names the closure overwrote are written decoded, the rest are
        //    ingested as their snapshotted bytes. Names in neither container
        //    are written as None so the storage slot grows in lockstep and
        //    is marked deleted, matching insert_new_vectors's contract.
        for (vector_name, vector_data) in self.vector_data.iter_mut() {
            let mut vector_index = vector_data.vector_index.borrow_mut();
            match updated_vectors.get(vector_name) {
                Some(vector) => vector_index.update_vector(new_id, Some(vector), hw_counter)?,
                None => vector_index.update_vector_raw(
                    new_id,
                    raw_vectors.get(vector_name).map(Vec::as_slice),
                    hw_counter,
                )?,
            }
            self.version_tracker.set_vector(vector_name, Some(op_num));
        }

        // 5. Write the payload at new_id — always, even when empty. Each
        //    configured field index needs either an `add_point` (for
        //    present values) or a `remove_point` (for absent ones) so
        //    `total_point_count` bumps up to cover new_id; without that
        //    bump the null index's `is_empty` / `is_null` checks never
        //    see new_id and the point disappears from filter results.
        self.payload_index
            .borrow_mut()
            .overwrite_payload(new_id, &payload, hw_counter)?;

        // Writing the payload row at new_id mutated payload storage — even
        // for a vectors-only mutation whose entry point never touches the
        // payload version. Stamp it here so partial snapshots re-upload the
        // changed files. The payload entry points deliberately do not bump
        // again on this path (a same-version double bump would collapse the
        // tracked version to `None`, degrading the stamp to the segment
        // version): they bump inside their in-place closures instead.
        self.version_tracker.set_payload(Some(op_num));

        // 6. Repoint the id tracker. `set_link` auto-tombstones old_id in
        //    the deleted bitslice when external_id was previously mapped to
        //    a different internal id.
        self.id_tracker.borrow_mut().set_link(point_id, new_id)?;

        Ok((mutate_result, new_id))
    }

    /// Mutating-op wrapper that picks the right path for the segment's
    /// current mode.
    ///
    /// - On a normal segment (or when `existing_internal_id` would not
    ///   benefit from being moved), runs `in_place(&mut Segment, old_id)`.
    /// - On an [`Segment::is_append_only`] segment, routes through
    ///   [`Segment::clone_and_mutate_point`] with `snapshot_mutate`, which
    ///   sees the point's vectors as a raw-bytes snapshot plus an empty
    ///   decoded overlay, and the payload as an owned snapshot, and modifies
    ///   them in memory; the helper writes the result at a fresh internal id
    ///   and tombstones the old one. Exception: a slot written by the
    ///   current operation (its version equals `op_num`) is mutated in
    ///   place — it is not durable yet, so cloning it would only chain
    ///   dead slots for multi-step point writes.
    ///
    /// Both closures return the op-specific result bool (e.g. "was anything
    /// deleted"). The version-recording offset is chosen automatically:
    /// the existing id for the in-place path, the freshly allocated id for
    /// the append-only path.
    ///
    /// Callers must resolve `existing_internal_id` from the id tracker
    /// themselves and handle the missing-pid case before calling this.
    pub(super) fn handle_point_mutate<InPlace, SnapshotMutate>(
        &mut self,
        op_num: SeqNumberType,
        point_id: PointIdType,
        existing_internal_id: PointOffsetType,
        hw_counter: &HardwareCounterCell,
        in_place: InPlace,
        snapshot_mutate: SnapshotMutate,
    ) -> OperationResult<bool>
    where
        InPlace: FnOnce(&mut Segment, PointOffsetType) -> OperationResult<bool>,
        SnapshotMutate: FnOnce(
            &mut TinyMap<VectorNameBuf, Vec<u8>>,
            &mut NamedVectors<'static>,
            &mut Payload,
        ) -> OperationResult<bool>,
    {
        // A slot whose version already equals `op_num` was written by the
        // current operation (an earlier step of a multi-step point write,
        // e.g. the upsert preceding this set_full_payload). It cannot be
        // durable yet: the segment write lock is held across the whole
        // operation, so no flush — and hence no read-only follower — can
        // have observed it, and a crash discards it (versions flush last,
        // WAL replay re-applies the whole operation). Mutating it in place
        // is therefore invisible to readers and avoids cloning the point
        // once per step.
        let same_op_slot = self
            .id_tracker
            .borrow()
            .internal_version(existing_internal_id)
            .is_some_and(|slot_version| slot_version == op_num);
        let append_only = self.is_append_only() && !same_op_slot;
        self.handle_point_version_and_failure(
            op_num,
            point_id,
            Some(existing_internal_id),
            |segment| {
                if append_only {
                    let (op_result, new_id) = segment.clone_and_mutate_point(
                        op_num,
                        point_id,
                        existing_internal_id,
                        hw_counter,
                        snapshot_mutate,
                    )?;
                    Ok((op_result, Some(new_id)))
                } else {
                    let op_result = in_place(segment, existing_internal_id)?;
                    Ok((op_result, Some(existing_internal_id)))
                }
            },
        )
    }

    /// Operation wrapped, which handles previous and new errors in the segment, automatically
    /// updates versions and skips operations if the segment version is too old
    ///
    /// # Arguments
    ///
    /// * `op_num` - sequential operation of the current operation
    /// * `op` - operation to be wrapped. Should return `OperationResult` of bool (which is returned outside)
    ///   and optionally new offset of the changed point.
    ///
    /// # Result
    ///
    /// Propagates `OperationResult` of bool (which is returned in the `op` closure)
    pub(super) fn handle_segment_version_and_failure<F>(
        &mut self,
        op_num: SeqNumberType,
        operation: F,
    ) -> OperationResult<bool>
    where
        F: FnOnce(&mut Segment) -> OperationResult<bool>,
    {
        if let Some(SegmentFailedState {
            version: failed_version,
            point_id: _failed_point_id,
            error,
        }) = &self.error_status
        {
            // Failed operations should not be skipped,
            // fail if newer operation is attempted before proper recovery
            if *failed_version < op_num {
                return Err(OperationError::service_error(format!(
                    "Not recovered from previous error: {error}"
                )));
            } // else: Re-try operation
        }

        let res = self.handle_segment_version(op_num, operation);

        if let Some(error) = get_service_error(&res) {
            // ToDo: Recover previous segment state
            log::error!(
                "Segment {:?} operation error: {error}",
                self.segment_path.as_path(),
            );
            self.error_status = Some(SegmentFailedState {
                version: op_num,
                point_id: None,
                error,
            });
        }
        res
    }

    /// Operation wrapped, which handles previous and new errors in the segment, automatically
    /// updates versions and skips operations if the point version is too old
    ///
    /// # Arguments
    ///
    /// * `op_num` - sequential operation of the current operation
    /// * `point_id` - external id of the point the operation targets; used for error correlation.
    /// * `op_point_offset` - If point offset is specified, handler will use point version for comparison.
    ///   Otherwise, it will be applied without version checks.
    /// * `op` - operation to be wrapped. Should return `OperationResult` of bool (which is returned outside) and optionally new offset of the changed point.
    ///
    /// # Result
    ///
    /// Propagates `OperationResult` of bool (which is returned in the `op` closure)
    pub(super) fn handle_point_version_and_failure<F>(
        &mut self,
        op_num: SeqNumberType,
        point_id: PointIdType,
        op_point_offset: Option<PointOffsetType>,
        operation: F,
    ) -> OperationResult<bool>
    where
        F: FnOnce(&mut Segment) -> OperationResult<(bool, Option<PointOffsetType>)>,
    {
        if let Some(SegmentFailedState {
            version: failed_version,
            point_id: _failed_point_id,
            error,
        }) = &self.error_status
        {
            // Failed operations should not be skipped,
            // fail if newer operation is attempted before proper recovery
            if *failed_version < op_num {
                return Err(OperationError::service_error(format!(
                    "Not recovered from previous error: {error}"
                )));
            } // else: Re-try operation
        }

        let res = self.handle_point_version(op_num, op_point_offset, operation);

        match get_service_error(&res) {
            None => {
                // Recover error state
                match &self.error_status {
                    None => {} // all good
                    Some(error) if error.point_id == Some(point_id) => {
                        // Fixed
                        log::info!("Recovered from error: {}", error.error);
                        self.error_status = None;
                    }
                    Some(_) => {}
                }
            }
            Some(error) => {
                // ToDo: Recover previous segment state
                log::error!(
                    "Segment {:?} operation error: {error}",
                    self.segment_path.as_path(),
                );
                self.error_status = Some(SegmentFailedState {
                    version: op_num,
                    point_id: Some(point_id),
                    error,
                });
            }
        }
        res
    }

    /// Manage segment version checking, for segment level operations
    ///
    /// If current version is higher than operation version - do not perform the operation
    /// Update current version if operation successfully executed
    fn handle_segment_version<F>(
        &mut self,
        op_num: SeqNumberType,
        operation: F,
    ) -> OperationResult<bool>
    where
        F: FnOnce(&mut Segment) -> OperationResult<bool>,
    {
        // Global version to check if operation has already been applied, then skip without execution
        if self.version.unwrap_or(0) > op_num {
            return Ok(false);
        }

        let applied = operation(self)?;
        self.bump_segment_version(op_num);
        Ok(applied)
    }

    /// Manage point version checking inside this segment, for point level operations
    ///
    /// If current version is higher than operation version - do not perform the operation
    /// Update current version if operation successfully executed
    pub(super) fn handle_point_version<F>(
        &mut self,
        op_num: SeqNumberType,
        op_point_offset: Option<PointOffsetType>,
        operation: F,
    ) -> OperationResult<bool>
    where
        F: FnOnce(&mut Segment) -> OperationResult<(bool, Option<PointOffsetType>)>,
    {
        // If point exist and has higher version, ignore operation
        if let Some(point_offset) = op_point_offset
            && self
                .id_tracker
                .borrow()
                .internal_version(point_offset)
                .is_some_and(|current_version| current_version > op_num)
        {
            return Ok(false);
        }

        let (applied, internal_id) = operation(self)?;

        self.bump_segment_version(op_num);
        if let Some(internal_id) = internal_id {
            self.id_tracker
                .borrow_mut()
                .set_internal_version(internal_id, op_num)?;
        }

        Ok(applied)
    }

    pub fn delete_point_internal(
        &mut self,
        internal_id: PointOffsetType,
        hw_counter: &HardwareCounterCell,
    ) -> OperationResult<()> {
        // Mark point as deleted, drop mapping
        self.payload_index
            .borrow_mut()
            .clear_payload(internal_id, hw_counter)?;

        let mut id_tracker = self.id_tracker.borrow_mut();

        // `drop_internal` updates the id tracker's deferred-deleted counter when
        // the point sits at or above the deferred threshold, with double-delete
        // protection inside `PointMappings::drop`.
        id_tracker.drop_internal(internal_id)?;

        // Before, we propagated point deletions to also delete its vectors. This turns
        // out to be problematic because this sometimes makes us lose vector data
        // because we cannot control the order of segment flushes.
        // Disabled until we properly fix it or find a better way to clean up old
        // vectors.
        //
        // // Propagate point deletion to all its vectors
        // for vector_data in segment.vector_data.values() {
        //     let mut vector_storage = vector_data.vector_storage.borrow_mut();
        //     vector_storage.delete_vector(internal_id)?;
        // }

        Ok(())
    }

    /// Append-only counterpart to [`Segment::delete_point_internal`]: only
    /// touches the id tracker, leaving the payload row and field-index
    /// postings at `internal_id` in place. Readers filter the tombstone
    /// via the id tracker's deleted bitslice (see
    /// `PointMappingsRefEnum::filter_deferred_and_deleted`).
    pub fn delete_point_tombstone_only(
        &mut self,
        internal_id: PointOffsetType,
    ) -> OperationResult<()> {
        self.id_tracker.borrow_mut().drop_internal(internal_id)
    }

    fn bump_segment_version(&mut self, op_num: SeqNumberType) {
        self.version.replace(max(op_num, self.version.unwrap_or(0)));
    }

    pub fn get_internal_id(&self, point_id: PointIdType) -> Option<PointOffsetType> {
        // Proxy/transfer callers pair this with `point_is_deferred`; they need
        // the latest head, so resolve with deferred preference.
        self.id_tracker
            .borrow()
            .internal_id_with_behavior(point_id, DeferredBehavior::WithDeferred)
    }

    pub fn get_deleted_points_bitvec(&self) -> BitVec {
        BitVec::from(self.id_tracker.borrow().deleted_point_bitslice())
    }

    pub(super) fn get_state(&self) -> SegmentState {
        SegmentState {
            initial_version: self.initial_version,
            version: self.version,
            config: self.segment_config.clone(),
        }
    }

    pub fn save_state(state: &SegmentState, segment_path: &Path) -> OperationResult<()> {
        let state_path = segment_path.join(SEGMENT_STATE_FILE);
        Ok(atomic_save_json(&state_path, state)?)
    }

    pub fn load_state(segment_path: &Path) -> OperationResult<SegmentState> {
        let state_path = segment_path.join(SEGMENT_STATE_FILE);
        read_json(&state_path).map_err(|err| {
            OperationError::service_error(format!(
                "Failed to read segment state {} error: {}",
                segment_path.display(),
                err
            ))
        })
    }

    pub fn save_current_state(&self) -> OperationResult<()> {
        Self::save_state(&self.get_state(), &self.segment_path)
    }

    /// Unpacks and restores the segment snapshot in-place. The original
    /// snapshot is destroyed in the process.
    ///
    /// Both of the following calls would result in a directory
    /// `foo/bar/segment-id/` with the segment data:
    ///
    /// - `segment.restore_snapshot("foo/bar/segment-id.tar")`  (tar archive)
    /// - `segment.restore_snapshot("foo/bar/segment-id")`      (directory)
    pub fn restore_snapshot_in_place(snapshot_path: &Path) -> OperationResult<()> {
        restore_snapshot_in_place(snapshot_path).map_err(|err| {
            OperationError::service_error(format!(
                "Failed to restore snapshot from {snapshot_path:?}: {err}",
            ))
        })
    }

    /// Check consistency of the segment's data and repair it if possible.
    /// Removes partially persisted points.
    pub fn check_consistency_and_repair(&mut self) -> OperationResult<()> {
        // Get rid of mappingless points.
        let ids_to_clean = self.fix_id_tracker_inconsistencies()?;

        // There are some leftovers to clean from segment.
        // After that we need to set internal version to 0, so that
        // we won't need to clean them again.

        // This is internal operation, no hw measurement needed
        let disposable_hw_counter = HardwareCounterCell::disposable();
        if !ids_to_clean.is_empty() {
            log::debug!(
                "Cleaning up {} points with version but no mapping in segment {:?}",
                ids_to_clean.len(),
                self.data_path()
            );

            for internal_id in ids_to_clean {
                self.delete_point_internal(internal_id, &disposable_hw_counter)?;
            }

            self.flush(true)?;

            // We do not drop version here, because it is already not loaded into memory.
            // There are no explicit mapping between internal ID and version, so all dangling
            // versions will be ignored automatically.
            // Those versions could be overwritten by new points, but it is not a problem.
            // They will also be deleted by the next optimization.
        }

        Ok(())
    }

    /// Update all payload/field indices to match `desired_schemas`
    ///
    /// Missing payload indices are created. Incorrectly configured payload indices are recreated.
    /// Extra payload indices are NOT deleted.
    ///
    /// This does nothing if the current payload indices state matches `desired_schemas` exactly.
    pub fn update_all_field_indices(
        &mut self,
        desired_schemas: &HashMap<PayloadKeyType, PayloadFieldSchema>,
    ) -> OperationResult<()> {
        let schema_applied = self
            .payload_index
            .borrow()
            .with_view(|v| v.indexed_fields());
        let schema_config = desired_schemas;

        // Create or update payload indices if they don't match configuration
        for (key, schema) in schema_config {
            match schema_applied.get(key) {
                Some(existing_schema) if existing_schema == schema => continue,
                Some(existing_schema) => log::warn!(
                    "Segment has incorrect payload index for {key}, recreating it now (current: {:?}, configured: {:?})",
                    existing_schema.name(),
                    schema.name(),
                ),
                None => log::warn!(
                    "Segment is missing a {} payload index for {key}, creating it now",
                    schema.name(),
                ),
            }

            let created = self.create_field_index(
                self.version(),
                key,
                Some(schema),
                &HardwareCounterCell::disposable(), // This function is only used in Segment::load which is unmeasured.
            )?;
            if !created {
                log::warn!("Failed to create payload index for {key} in segment");
            }
        }

        // Do not delete extra payload indices, because collection-level information about
        // the payload indices might be incomplete due to migrations from older versions.

        Ok(())
    }

    /// Check data consistency of the segment on its own
    /// - internal id without external id
    /// - external id without internal
    /// - internal id without version
    /// - internal id without vector
    ///
    /// A shard can still be consistent with an inconsistent segment as points are merged based on their version.
    ///
    /// Returns an error if any inconsistency is found
    pub fn check_data_consistency(&self) -> OperationResult<()> {
        let id_tracker = self.id_tracker.borrow();

        // dangling internal ids
        let mut has_dangling_internal_ids = false;
        for internal_id in id_tracker.point_mappings().iter_internal() {
            if id_tracker.external_id(internal_id).is_none() {
                log::error!("Internal id {internal_id} without external id");
                has_dangling_internal_ids = true
            }
        }

        // dangling external ids
        let mut has_dangling_external_ids = false;
        for external_id in id_tracker.point_mappings().iter_external() {
            // Dangling check: a point is fine as long as *some* head resolves,
            // so prefer the deferred head and fall back to active.
            if id_tracker
                .internal_id_with_behavior(external_id, DeferredBehavior::WithDeferred)
                .is_none()
            {
                log::error!("External id {external_id} without internal id");
                has_dangling_external_ids = true;
            }
        }

        // checking internal id without version
        let mut has_internal_ids_without_version = false;
        for internal_id in id_tracker.point_mappings().iter_internal() {
            if id_tracker.internal_version(internal_id).is_none() {
                log::error!("Internal id {internal_id} without version");
                has_internal_ids_without_version = true;
            }
        }

        // check that non deleted points exist in vector storage
        let mut has_internal_ids_without_vector = false;
        for internal_id in id_tracker.point_mappings().iter_internal() {
            for (vector_name, vector_data) in &self.vector_data {
                let vector_storage = vector_data.vector_storage.borrow();
                let is_vector_deleted_storage = vector_storage.is_deleted_vector(internal_id);
                let is_vector_deleted_tracker = id_tracker.is_deleted_point(internal_id);
                let vector_stored = vector_storage.get_vector_opt::<Random>(internal_id);
                if !is_vector_deleted_storage
                    && !is_vector_deleted_tracker
                    && vector_stored.is_none()
                {
                    let point_id = id_tracker.external_id(internal_id);
                    let point_version = id_tracker.internal_version(internal_id);
                    // ignoring initial version because the WAL replay can resurrect un-flushed points by assigning them a new initial version
                    // those points will be deleted by the next deduplication process
                    if point_version != Some(0) {
                        log::error!(
                            "Vector storage '{vector_name}' is missing point {point_id:?} point_offset: {internal_id} version: {point_version:?}",
                        );
                        has_internal_ids_without_vector = true;
                    }
                }
            }
        }

        let is_inconsistent = has_dangling_internal_ids
            || has_dangling_external_ids
            || has_internal_ids_without_version
            || has_internal_ids_without_vector;

        if is_inconsistent {
            Err(OperationError::service_error(
                "Inconsistent segment data detected",
            ))
        } else {
            Ok(())
        }
    }

    pub fn available_vector_count(&self, vector_name: &VectorName) -> OperationResult<usize> {
        check_vector_name(vector_name, &self.segment_config)?;
        Ok(self
            .vector_data
            .get(vector_name)
            .ok_or_else(|| OperationError::vector_name_not_exists(vector_name))?
            .vector_storage
            .borrow()
            .available_vector_count())
    }

    pub fn total_point_count(&self) -> usize {
        self.id_tracker.borrow().total_point_count()
    }

    /// Fixes inconsistencies in the ID tracker, if any.
    /// Returns list of IDs without mappings which should be removed from segment
    pub fn fix_id_tracker_inconsistencies(&mut self) -> OperationResult<Vec<PointOffsetType>> {
        self.id_tracker.borrow_mut().fix_inconsistencies()
    }
}

fn restore_snapshot_in_place(snapshot_path: &Path) -> OperationResult<()> {
    let segments_dir = snapshot_path
        .parent()
        .ok_or_else(|| OperationError::service_error("Cannot extract parent path"))?;

    let file_name = snapshot_path
        .file_name()
        .and_then(|name| name.to_str())
        .ok_or_else(|| {
            OperationError::service_error("Cannot extract segment ID from snapshot path")
        })?;

    let meta = fs::metadata(snapshot_path)?;
    let (segment_id, is_tar) = match file_name.split_once('.') {
        Some((segment_id, "tar")) if meta.is_file() => (segment_id, true),
        None if meta.is_dir() => (file_name, false),
        _ => {
            return Err(OperationError::service_error(
                "Invalid snapshot path, expected either a directory or a .tar file",
            ));
        }
    };

    if !is_tar {
        log::debug!(
            "Extracting segment {} from {:?} snapshot",
            segment_id,
            SnapshotFormat::Streamable
        );
        unpack_snapshot(snapshot_path)?;
    } else {
        let segment_path = segments_dir.join(segment_id);
        tar_unpack_file(snapshot_path, &segment_path)?;

        let inner_path = segment_path.join(SNAPSHOT_PATH);
        if inner_path.is_dir() {
            log::debug!(
                "Extracting segment {} from {:?} snapshot",
                segment_id,
                SnapshotFormat::Regular
            );
            unpack_snapshot(&inner_path)?;
            utils::fs::move_all(&inner_path, &segment_path)?;
            fs::remove_dir(&inner_path)?;
        } else {
            log::debug!(
                "Extracting segment {} from {:?} snapshot",
                segment_id,
                SnapshotFormat::Ancient
            );
            // Do nothing, this format is just a plain archive.
        }

        fs::remove_file(snapshot_path)?;
    }

    Ok(())
}

fn unpack_snapshot(segment_path: &Path) -> OperationResult<()> {
    let db_backup_path = segment_path.join(DEPRECATED_ROCKSDB_BACKUP_PATH);
    if db_backup_path.is_dir() {
        log::warn!(
            "RocksDB is no longer supported, and {DEPRECATED_ROCKSDB_BACKUP_PATH} will be ignored"
        );
        fs::remove_dir_all(&db_backup_path)?;
    }
    let payload_index_db_backup = segment_path.join(DEPRECATED_PAYLOAD_ROCKSDB_BACKUP_PATH);
    if payload_index_db_backup.is_dir() {
        log::warn!(
            "RocksDB is no longer supported, and {DEPRECATED_PAYLOAD_ROCKSDB_BACKUP_PATH} will be ignored"
        );
        fs::remove_dir_all(&payload_index_db_backup)?;
    }

    // Hoist the segment files out of the nested `files/` directory into the segment directory.
    //
    // Streamable snapshots historically wrap all segment files in a nested `files/` sub-directory.
    // To allow recovering snapshots that place the files directly in the segment directory (without
    // the `files/` wrapper), only perform the hoisting when the `files/` directory is actually
    // present. When it is absent, the files are assumed to already be in their final location.
    let files_path = segment_path.join(SNAPSHOT_FILES_PATH);
    if files_path.is_dir() {
        utils::fs::move_all(&files_path, segment_path)?;
        fs::remove_dir(&files_path)?;
    }

    Ok(())
}
