mirror of
https://github.com/qdrant/qdrant.git
synced 2026-09-21 13:37:46 -05:00
[combined-storage] Integrate VectorStorageType::GraphInline reading (#10515)
* Placement accessors on VectorDataConfig * Wire up the GraphInline storage type * Let the HNSW index reuse the storage's links handle
This commit is contained in:
@@ -12716,6 +12716,13 @@
|
||||
"enum": [
|
||||
"InRamMmap"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "Vectors are inlined in the HNSW links file, not in a dedicated storage. Not appendable.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"GraphInline"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -888,9 +888,7 @@ mod tests {
|
||||
.filter(|segment| segment.total_point_count() > 0)
|
||||
.for_each(|segment| {
|
||||
assert!(
|
||||
!segment.config().vector_data[DEFAULT_VECTOR_NAME]
|
||||
.storage_type
|
||||
.is_on_disk(),
|
||||
!segment.config().vector_data[DEFAULT_VECTOR_NAME].is_on_disk(),
|
||||
"segment must not be on disk with mmap",
|
||||
);
|
||||
});
|
||||
@@ -950,9 +948,7 @@ mod tests {
|
||||
.filter(|segment| segment.total_point_count() > 0)
|
||||
.for_each(|segment| {
|
||||
assert!(
|
||||
segment.config().vector_data[DEFAULT_VECTOR_NAME]
|
||||
.storage_type
|
||||
.is_on_disk(),
|
||||
segment.config().vector_data[DEFAULT_VECTOR_NAME].is_on_disk(),
|
||||
"segment must be on disk with mmap",
|
||||
);
|
||||
});
|
||||
|
||||
@@ -75,7 +75,7 @@ impl EdgeVectorParams {
|
||||
let VectorDataConfig {
|
||||
size,
|
||||
distance,
|
||||
storage_type,
|
||||
storage_type: _,
|
||||
index,
|
||||
quantization_config, // edge uses global only
|
||||
multivector_config,
|
||||
@@ -84,7 +84,7 @@ impl EdgeVectorParams {
|
||||
Self {
|
||||
size: *size,
|
||||
distance: *distance,
|
||||
on_disk: Some(storage_type.is_on_disk()),
|
||||
on_disk: Some(v.is_on_disk()),
|
||||
multivector_config: *multivector_config,
|
||||
datatype: *datatype,
|
||||
quantization_config: quantization_config.clone(),
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
use common::ext::aligned_vec::ACow;
|
||||
use common::flags::feature_flags;
|
||||
use common::types::{PointOffsetType, ScoredPointOffset};
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
use common::universal_io::MmapFile;
|
||||
@@ -17,9 +18,11 @@ use super::entry_points::{EntryPoint, EntryPoints};
|
||||
use super::graph_layers::{GraphLayers, SearchAlgorithm};
|
||||
use super::graph_layers_batched::GraphLayersBatched;
|
||||
use super::graph_links::{GraphLinks, GraphLinksFile, GraphLinksFormat, GraphLinksResidency};
|
||||
use super::hnsw::{LINK_COMPRESSION_CONVERT_EXISTING, graph_residency};
|
||||
use super::point_scorer::{FilteredScorer, ScorerFilters};
|
||||
use crate::common::io_uring::{IoUringFallback, use_io_uring};
|
||||
use crate::common::operation_error::{OperationError, OperationResult};
|
||||
use crate::types::IoBackend;
|
||||
use crate::types::{IoBackend, Memory};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum HnswGraph<S: UniversalRead> {
|
||||
@@ -60,12 +63,14 @@ pub enum SearchScorers<'a> {
|
||||
}
|
||||
|
||||
impl HnswGraph<HnswLinksStorage> {
|
||||
pub fn open(
|
||||
dir: &Path,
|
||||
residency: GraphLinksResidency,
|
||||
do_convert: bool,
|
||||
with_uring: bool,
|
||||
) -> OperationResult<Self> {
|
||||
pub fn open(dir: &Path, memory: Memory) -> OperationResult<Self> {
|
||||
let (memory, residency) = graph_residency(memory, None);
|
||||
let with_uring = use_io_uring(
|
||||
IoUringFallback::Mmap,
|
||||
memory,
|
||||
feature_flags().async_hnsw_graph,
|
||||
);
|
||||
|
||||
if with_uring {
|
||||
#[cfg(target_os = "linux")]
|
||||
if Self::is_batched(&IoUringFs, dir, residency)? {
|
||||
@@ -74,7 +79,7 @@ impl HnswGraph<HnswLinksStorage> {
|
||||
}
|
||||
}
|
||||
|
||||
let graph = GraphLayers::load(dir, residency, do_convert)?;
|
||||
let graph = GraphLayers::load(dir, residency, LINK_COMPRESSION_CONVERT_EXISTING)?;
|
||||
Ok(HnswGraph::Direct(Arc::new(graph)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,12 +2,10 @@ use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use atomic_refcell::AtomicRefCell;
|
||||
use common::flags::feature_flags;
|
||||
use common::universal_io::{MmapFs, Populate, UniversalReadFs};
|
||||
|
||||
use self::telemetry::HNSWSearchesTelemetry;
|
||||
use crate::common::BYTES_IN_KB;
|
||||
use crate::common::io_uring::{IoUringFallback, use_io_uring};
|
||||
use crate::common::operation_error::OperationResult;
|
||||
use crate::id_tracker::IdTrackerEnum;
|
||||
use crate::index::hnsw_index::config::HnswGraphConfig;
|
||||
@@ -41,7 +39,7 @@ pub const SINGLE_THREADED_HNSW_BUILD_THRESHOLD: usize = 32;
|
||||
#[cfg(not(debug_assertions))]
|
||||
pub const SINGLE_THREADED_HNSW_BUILD_THRESHOLD: usize = 256;
|
||||
|
||||
const LINK_COMPRESSION_CONVERT_EXISTING: bool = false;
|
||||
pub(super) const LINK_COMPRESSION_CONVERT_EXISTING: bool = false;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct HNSWIndex {
|
||||
@@ -78,17 +76,10 @@ impl HNSWIndex {
|
||||
|
||||
let config = load_or_derive_config(&MmapFs, path, &hnsw_config, &vector_storage)?;
|
||||
|
||||
let do_convert = LINK_COMPRESSION_CONVERT_EXISTING;
|
||||
|
||||
let (memory, residency) = graph_residency(&hnsw_config, None);
|
||||
let is_on_disk = memory.is_on_disk();
|
||||
|
||||
let with_uring = use_io_uring(
|
||||
IoUringFallback::Mmap,
|
||||
memory,
|
||||
feature_flags().async_hnsw_graph,
|
||||
);
|
||||
let graph = HnswGraph::open(path, residency, do_convert, with_uring)?;
|
||||
let graph = match vector_storage.borrow().hnsw_graph() {
|
||||
Some(graph) => HnswGraph::clone(graph),
|
||||
None => HnswGraph::open(path, hnsw_config.memory_placement())?,
|
||||
};
|
||||
|
||||
Ok(HNSWIndex {
|
||||
id_tracker,
|
||||
@@ -97,9 +88,9 @@ impl HNSWIndex {
|
||||
payload_index,
|
||||
config,
|
||||
path: path.to_owned(),
|
||||
is_on_disk: graph.is_on_disk(),
|
||||
graph,
|
||||
searches_telemetry: HNSWSearchesTelemetry::new(),
|
||||
is_on_disk,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -212,11 +203,11 @@ fn load_or_derive_config(
|
||||
/// every residency over the same files, so even a `pinned` graph can be demoted
|
||||
/// to a lazy cold view. The returned [`Memory`] stays config-derived: it
|
||||
/// describes the configuration, not the per-open placement.
|
||||
fn graph_residency(
|
||||
hnsw_config: &HnswConfig,
|
||||
pub(crate) fn graph_residency(
|
||||
memory: Memory,
|
||||
populate_override: Option<Populate>,
|
||||
) -> (Memory, GraphLinksResidency) {
|
||||
let memory = hnsw_config.memory_placement().clamp_to_low_memory();
|
||||
let memory = memory.clamp_to_low_memory();
|
||||
|
||||
let residency = match memory.with_populate_override(populate_override) {
|
||||
// Keep the links cold: lazily loaded from disk, cached with usage
|
||||
|
||||
@@ -105,7 +105,8 @@ impl<S: UniversalReadExt> ReadOnlyHNSWIndex<S> {
|
||||
fs.schedule_open(&HnswGraphConfig::get_config_path(path), None, None);
|
||||
|
||||
// Graph data and links
|
||||
let (_memory, residency) = graph_residency(hnsw_config, populate_override);
|
||||
let (_memory, residency) =
|
||||
graph_residency(hnsw_config.memory_placement(), populate_override);
|
||||
if !graph_deferred(fs, path, populate_override, residency)? {
|
||||
HnswGraph::preopen_universal(fs, path, residency)?;
|
||||
}
|
||||
@@ -143,12 +144,14 @@ impl<S: UniversalReadExt> ReadOnlyHNSWIndex<S> {
|
||||
{
|
||||
let config = load_or_derive_config(fs, path, &hnsw_config, &vector_storage)?;
|
||||
|
||||
let (memory, residency) = graph_residency(&hnsw_config, populate_override);
|
||||
let (memory, residency) =
|
||||
graph_residency(hnsw_config.memory_placement(), populate_override);
|
||||
let is_on_disk = memory.is_on_disk();
|
||||
let graph = if graph_deferred(fs, path, populate_override, residency)? {
|
||||
OnceCell::new()
|
||||
} else {
|
||||
OnceCell::with_value(HnswGraph::open_universal(fs, path, residency)?)
|
||||
|
||||
let graph = match vector_storage.borrow().hnsw_graph() {
|
||||
Some(graph) => OnceCell::with_value(graph),
|
||||
None if graph_deferred(fs, path, populate_override, residency)? => OnceCell::new(),
|
||||
None => OnceCell::with_value(HnswGraph::open_universal(fs, path, residency)?),
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
|
||||
@@ -85,7 +85,7 @@ impl Segment {
|
||||
// Use the configured storage type for appendable segments
|
||||
let config_for_open = VectorDataConfig {
|
||||
// Override storage type to appendable chunked mmap
|
||||
storage_type: VectorStorageType::from_on_disk(config.storage_type.is_on_disk()),
|
||||
storage_type: VectorStorageType::from_on_disk(config.is_on_disk()),
|
||||
// Use plain index for new vectors
|
||||
index: crate::types::Indexes::Plain {},
|
||||
..config.clone()
|
||||
@@ -98,7 +98,7 @@ impl Segment {
|
||||
config.size,
|
||||
config.distance,
|
||||
config.datatype.unwrap_or_default(),
|
||||
config.storage_type.is_on_disk(),
|
||||
config.is_on_disk(),
|
||||
config.multivector_config,
|
||||
num_points,
|
||||
)
|
||||
|
||||
@@ -10,6 +10,7 @@ use crate::vector_storage::VectorStorageEnum;
|
||||
use crate::vector_storage::dense::dense_vector_storage::{
|
||||
open_dense_vector_storage, open_dense_vector_storage_byte, open_dense_vector_storage_half,
|
||||
};
|
||||
use crate::vector_storage::graph_inline::open_graph_inline_vector_storage;
|
||||
use crate::vector_storage::multi_dense::appendable_mmap_multi_dense_vector_storage::{
|
||||
open_appendable_memmap_multi_vector_storage, open_appendable_memmap_vector_storage,
|
||||
};
|
||||
@@ -98,7 +99,7 @@ fn open_chunked_mmap_vector_storage(
|
||||
pub(crate) fn open_vector_storage(
|
||||
vector_config: &VectorDataConfig,
|
||||
vector_storage_path: &Path,
|
||||
_vector_index_path: &Path,
|
||||
vector_index_path: &Path,
|
||||
) -> OperationResult<VectorStorageEnum> {
|
||||
match vector_config.storage_type {
|
||||
VectorStorageType::Memory => Err(OperationError::service_error(
|
||||
@@ -132,6 +133,11 @@ pub(crate) fn open_vector_storage(
|
||||
AdviceSetting::from(Advice::Normal),
|
||||
true,
|
||||
),
|
||||
|
||||
// Vectors inlined into the HNSW links file
|
||||
VectorStorageType::GraphInline => {
|
||||
open_graph_inline_vector_storage(vector_storage_path, vector_index_path, vector_config)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+24
-15
@@ -1754,9 +1754,7 @@ impl SegmentConfig {
|
||||
|
||||
/// Check if any vector storage is on-disk
|
||||
pub fn is_any_on_disk(&self) -> bool {
|
||||
self.vector_data
|
||||
.values()
|
||||
.any(|config| config.storage_type.is_on_disk())
|
||||
self.vector_data.values().any(|config| config.is_on_disk())
|
||||
|| self
|
||||
.sparse_vector_data
|
||||
.values()
|
||||
@@ -2019,6 +2017,8 @@ pub enum VectorStorageType {
|
||||
/// Storage in a single mmap file, not appendable
|
||||
/// Pre-fetched into RAM on load
|
||||
InRamMmap,
|
||||
/// Vectors are inlined in the HNSW links file, not in a dedicated storage. Not appendable.
|
||||
GraphInline,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "testing"))]
|
||||
@@ -2114,20 +2114,15 @@ impl VectorStorageType {
|
||||
}
|
||||
|
||||
/// Memory placement this storage type provides.
|
||||
pub fn memory(&self) -> Memory {
|
||||
///
|
||||
/// `None` (for GraphInline) means no placement is applicable.
|
||||
pub fn memory(&self) -> Option<Memory> {
|
||||
match self {
|
||||
// Legacy true-heap storage: pinned by construction
|
||||
Self::Memory => Memory::Pinned,
|
||||
Self::Mmap | Self::ChunkedMmap => Memory::Cold,
|
||||
Self::InRamChunkedMmap | Self::InRamMmap => Memory::Cached,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this storage type is a mmap on disk
|
||||
pub fn is_on_disk(&self) -> bool {
|
||||
match self {
|
||||
Self::Memory | Self::InRamChunkedMmap | Self::InRamMmap => false,
|
||||
Self::Mmap | Self::ChunkedMmap => true,
|
||||
Self::Memory => Some(Memory::Pinned),
|
||||
Self::Mmap | Self::ChunkedMmap => Some(Memory::Cold),
|
||||
Self::InRamChunkedMmap | Self::InRamMmap => Some(Memory::Cached),
|
||||
Self::GraphInline => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2169,6 +2164,7 @@ impl VectorDataConfig {
|
||||
VectorStorageType::ChunkedMmap => true,
|
||||
VectorStorageType::InRamChunkedMmap => true,
|
||||
VectorStorageType::InRamMmap => false,
|
||||
VectorStorageType::GraphInline => false,
|
||||
};
|
||||
is_index_appendable && is_storage_appendable
|
||||
}
|
||||
@@ -2258,6 +2254,19 @@ impl VectorDataConfig {
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub fn storage_memory(&self) -> Memory {
|
||||
match (self.storage_type.memory(), &self.index) {
|
||||
(Some(memory), _) => memory,
|
||||
(None, Indexes::Hnsw(hnsw_config)) => hnsw_config.memory_placement(),
|
||||
// Invalid config: no graph to follow
|
||||
(None, Indexes::Plain {}) => Memory::Cold,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_on_disk(&self) -> bool {
|
||||
self.storage_memory().is_on_disk()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(
|
||||
|
||||
@@ -57,6 +57,10 @@ impl<T: PrimitiveVectorElement, S: UniversalRead> GraphInlineDenseVectorStorage<
|
||||
})
|
||||
}
|
||||
|
||||
pub fn hnsw_graph(&self) -> &HnswGraph<S> {
|
||||
self.vectors.graph()
|
||||
}
|
||||
|
||||
pub fn populate(&self) {
|
||||
self.vectors.populate();
|
||||
}
|
||||
|
||||
@@ -92,6 +92,10 @@ impl<T: PrimitiveVectorElement, S: UniversalRead>
|
||||
distance,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn hnsw_graph(&self) -> HnswGraph<S> {
|
||||
self.vectors.graph().clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the immutable storage's `deleted.dat` through `fs` into an in-memory flag
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
use std::path::Path;
|
||||
|
||||
use crate::common::operation_error::OperationResult;
|
||||
use crate::index::hnsw_index::HnswGraph;
|
||||
use crate::types::{VectorDataConfig, VectorStorageDatatype};
|
||||
use crate::vector_storage::VectorStorageEnum;
|
||||
use crate::vector_storage::dense::graph_inline_dense_vector_storage::GraphInlineDenseVectorStorage;
|
||||
use crate::vector_storage::turbo::TurboVectorStorageImpl;
|
||||
|
||||
pub(crate) fn open_graph_inline_vector_storage(
|
||||
path: &Path,
|
||||
index_path: &Path,
|
||||
vector_config: &VectorDataConfig,
|
||||
) -> OperationResult<VectorStorageEnum> {
|
||||
use VectorStorageDatatype::{Float16, Float32, Turbo4, Uint8};
|
||||
|
||||
let graph = HnswGraph::open(index_path, vector_config.storage_memory())?;
|
||||
let dim = vector_config.size;
|
||||
let distance = vector_config.distance;
|
||||
Ok(match vector_config.datatype.unwrap_or_default() {
|
||||
Float32 => VectorStorageEnum::DenseGraphInline(
|
||||
GraphInlineDenseVectorStorage::open(graph, path, dim, distance).map(Box::new)?,
|
||||
),
|
||||
Float16 => VectorStorageEnum::DenseGraphInlineHalf(
|
||||
GraphInlineDenseVectorStorage::open(graph, path, dim, distance).map(Box::new)?,
|
||||
),
|
||||
Uint8 => VectorStorageEnum::DenseGraphInlineByte(
|
||||
GraphInlineDenseVectorStorage::open(graph, path, dim, distance).map(Box::new)?,
|
||||
),
|
||||
Turbo4 => VectorStorageEnum::DenseTurboGraphInline(
|
||||
TurboVectorStorageImpl::open_graph(graph, path, dim, distance).map(Box::new)?,
|
||||
),
|
||||
})
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
mod chunked_vectors;
|
||||
pub mod common;
|
||||
pub mod dense;
|
||||
pub mod graph_inline;
|
||||
pub mod graph_vectors;
|
||||
mod memory_reporter;
|
||||
pub mod multi_dense;
|
||||
|
||||
@@ -60,7 +60,7 @@ impl<S: UniversalRead> ReadOnlyQuantizedVectors<S> {
|
||||
return Ok(());
|
||||
};
|
||||
let multivector = vector_config.multivector_config.is_some();
|
||||
let on_disk_vector_storage = vector_config.storage_type.is_on_disk();
|
||||
let on_disk_vector_storage = vector_config.storage_memory().is_on_disk();
|
||||
|
||||
// Config; `open` reads it off the parked handle.
|
||||
let config_path = QuantizedVectors::get_config_path(path);
|
||||
|
||||
@@ -4,9 +4,13 @@ use common::mmap::{Advice, AdviceSetting};
|
||||
use common::universal_io::{CachedReadFs, Populate, UniversalRead, UniversalReadFs};
|
||||
|
||||
use super::VectorStorageReadEnum;
|
||||
use crate::common::flags::in_memory_bitvec_flags::InMemoryBitvecFlags;
|
||||
use crate::common::operation_error::OperationResult;
|
||||
use crate::data_types::vectors::{VectorElementType, VectorElementTypeByte, VectorElementTypeHalf};
|
||||
use crate::index::hnsw_index::HnswGraph;
|
||||
use crate::index::hnsw_index::hnsw::graph_residency;
|
||||
use crate::types::{VectorDataConfig, VectorStorageDatatype, VectorStorageType};
|
||||
use crate::vector_storage::dense::appendable_dense_vector_storage::DELETED_DIR_PATH;
|
||||
use crate::vector_storage::dense::immutable_dense_vectors::ImmutableDenseVectorData;
|
||||
use crate::vector_storage::dense::read_only::{
|
||||
ReadOnlyChunkedDenseVectorStorage, ReadOnlyImmutableDenseVectorStorage,
|
||||
@@ -17,17 +21,28 @@ use crate::vector_storage::turbo::read_only::{
|
||||
ReadOnlyChunkedTurboVectorStorage, ReadOnlyImmutableTurboVectorStorage,
|
||||
};
|
||||
|
||||
/// How the [`VectorStorageType`] maps onto the read-only open path: mmap
|
||||
/// advice, whether the storage is populated on open, and whether it uses the
|
||||
/// appendable chunked layout. `None` for the storage types with no on-disk
|
||||
/// data to open.
|
||||
fn storage_type_params(storage_type: VectorStorageType) -> Option<(AdviceSetting, Populate, bool)> {
|
||||
/// How the [`VectorStorageType`] maps onto the read-only open path.
|
||||
enum ReadOnlyLayout {
|
||||
/// No on-disk data to open.
|
||||
None,
|
||||
/// Dedicated vector files.
|
||||
Files {
|
||||
advice: AdviceSetting,
|
||||
populate: Populate,
|
||||
chunked: bool,
|
||||
},
|
||||
/// Vectors inlined in the HNSW links file.
|
||||
GraphInline,
|
||||
}
|
||||
|
||||
fn storage_type_layout(storage_type: VectorStorageType) -> ReadOnlyLayout {
|
||||
let (advice, populate, chunked) = match storage_type {
|
||||
VectorStorageType::Mmap => (AdviceSetting::Global, false, false),
|
||||
VectorStorageType::InRamMmap => (AdviceSetting::from(Advice::Normal), true, false),
|
||||
VectorStorageType::ChunkedMmap => (AdviceSetting::Global, false, true),
|
||||
VectorStorageType::InRamChunkedMmap => (AdviceSetting::from(Advice::Normal), true, true),
|
||||
VectorStorageType::Memory => return None,
|
||||
VectorStorageType::Memory => return ReadOnlyLayout::None,
|
||||
VectorStorageType::GraphInline => return ReadOnlyLayout::GraphInline,
|
||||
};
|
||||
|
||||
let populate = match populate {
|
||||
@@ -35,7 +50,11 @@ fn storage_type_params(storage_type: VectorStorageType) -> Option<(AdviceSetting
|
||||
false => Populate::No,
|
||||
};
|
||||
|
||||
Some((advice, populate, chunked))
|
||||
ReadOnlyLayout::Files {
|
||||
advice,
|
||||
populate,
|
||||
chunked,
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: UniversalRead> VectorStorageReadEnum<S> {
|
||||
@@ -53,15 +72,24 @@ impl<S: UniversalRead> VectorStorageReadEnum<S> {
|
||||
fs: &impl CachedReadFs<File = S>,
|
||||
vector_config: &VectorDataConfig,
|
||||
path: &Path,
|
||||
_vector_index_path: &Path,
|
||||
vector_index_path: &Path,
|
||||
populate_override: Option<Populate>,
|
||||
) -> OperationResult<()> {
|
||||
let datatype = vector_config.datatype.unwrap_or_default();
|
||||
|
||||
let Some((advice, populate, chunked)) = storage_type_params(vector_config.storage_type)
|
||||
else {
|
||||
// No on-disk data to prefetch for these storage types: no-op.
|
||||
return Ok(());
|
||||
let (advice, populate, chunked) = match storage_type_layout(vector_config.storage_type) {
|
||||
ReadOnlyLayout::None => return Ok(()),
|
||||
ReadOnlyLayout::GraphInline => {
|
||||
let (_memory, residency) =
|
||||
graph_residency(vector_config.storage_memory(), populate_override);
|
||||
HnswGraph::preopen_universal(fs, vector_index_path, residency)?;
|
||||
return InMemoryBitvecFlags::preopen(fs, &path.join(DELETED_DIR_PATH));
|
||||
}
|
||||
ReadOnlyLayout::Files {
|
||||
advice,
|
||||
populate,
|
||||
chunked,
|
||||
} => (advice, populate, chunked),
|
||||
};
|
||||
let populate = populate_override.unwrap_or(populate);
|
||||
|
||||
@@ -136,17 +164,51 @@ impl<S: UniversalRead> VectorStorageReadEnum<S> {
|
||||
fs: &impl UniversalReadFs<File = S>,
|
||||
vector_config: &VectorDataConfig,
|
||||
path: &Path,
|
||||
_vector_index_path: &Path,
|
||||
vector_index_path: &Path,
|
||||
populate_override: Option<Populate>,
|
||||
) -> OperationResult<Option<Self>> {
|
||||
) -> OperationResult<Option<Self>>
|
||||
where
|
||||
S: 'static,
|
||||
{
|
||||
let dim = vector_config.size;
|
||||
let distance = vector_config.distance;
|
||||
let datatype = vector_config.datatype.unwrap_or_default();
|
||||
|
||||
// No on-disk data to open for these storage types: no-op.
|
||||
let Some((advice, populate, chunked)) = storage_type_params(vector_config.storage_type)
|
||||
else {
|
||||
return Ok(None);
|
||||
let (advice, populate, chunked) = match storage_type_layout(vector_config.storage_type) {
|
||||
ReadOnlyLayout::None => return Ok(None),
|
||||
ReadOnlyLayout::GraphInline => {
|
||||
let (_memory, residency) =
|
||||
graph_residency(vector_config.storage_memory(), populate_override);
|
||||
let graph = HnswGraph::open_universal(fs, vector_index_path, residency)?;
|
||||
|
||||
return Ok(Some(match datatype {
|
||||
VectorStorageDatatype::Float32 => Self::DenseGraphInline(Box::new(
|
||||
ReadOnlyImmutableDenseVectorStorage::open_graph(
|
||||
fs, path, graph, dim, distance,
|
||||
)?,
|
||||
)),
|
||||
VectorStorageDatatype::Uint8 => Self::DenseGraphInlineByte(Box::new(
|
||||
ReadOnlyImmutableDenseVectorStorage::open_graph(
|
||||
fs, path, graph, dim, distance,
|
||||
)?,
|
||||
)),
|
||||
VectorStorageDatatype::Float16 => Self::DenseGraphInlineHalf(Box::new(
|
||||
ReadOnlyImmutableDenseVectorStorage::open_graph(
|
||||
fs, path, graph, dim, distance,
|
||||
)?,
|
||||
)),
|
||||
VectorStorageDatatype::Turbo4 => Self::DenseTurboGraphInline(Box::new(
|
||||
ReadOnlyImmutableTurboVectorStorage::open_graph(
|
||||
fs, path, graph, dim, distance,
|
||||
)?,
|
||||
)),
|
||||
}));
|
||||
}
|
||||
ReadOnlyLayout::Files {
|
||||
advice,
|
||||
populate,
|
||||
chunked,
|
||||
} => (advice, populate, chunked),
|
||||
};
|
||||
let populate = populate_override.unwrap_or(populate);
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ use crate::common::operation_error::OperationResult;
|
||||
use crate::data_types::vectors::{
|
||||
QueryVector, VectorElementType, VectorElementTypeByte, VectorElementTypeHalf,
|
||||
};
|
||||
use crate::index::hnsw_index::HnswGraph;
|
||||
use crate::vector_storage::dense::immutable_dense_vectors::ImmutableDenseVectorData;
|
||||
use crate::vector_storage::dense::read_only::{
|
||||
ReadOnlyChunkedDenseVectorStorage, ReadOnlyImmutableDenseVectorStorage,
|
||||
@@ -54,6 +55,31 @@ pub enum VectorStorageReadEnum<S: UniversalRead> {
|
||||
Sparse(Box<ReadOnlySparseVectorStorage<S>>),
|
||||
}
|
||||
|
||||
impl<S: UniversalRead> VectorStorageReadEnum<S> {
|
||||
/// See [crate::vector_storage::VectorStorageEnum::hnsw_graph].
|
||||
pub fn hnsw_graph(&self) -> Option<HnswGraph<S>> {
|
||||
match self {
|
||||
VectorStorageReadEnum::Dense(_) => None,
|
||||
VectorStorageReadEnum::DenseByte(_) => None,
|
||||
VectorStorageReadEnum::DenseHalf(_) => None,
|
||||
VectorStorageReadEnum::DenseGraphInline(s) => Some(s.hnsw_graph()),
|
||||
VectorStorageReadEnum::DenseGraphInlineByte(s) => Some(s.hnsw_graph()),
|
||||
VectorStorageReadEnum::DenseGraphInlineHalf(s) => Some(s.hnsw_graph()),
|
||||
VectorStorageReadEnum::DenseChunked(_) => None,
|
||||
VectorStorageReadEnum::DenseChunkedByte(_) => None,
|
||||
VectorStorageReadEnum::DenseChunkedHalf(_) => None,
|
||||
VectorStorageReadEnum::MultiDenseChunked(_) => None,
|
||||
VectorStorageReadEnum::MultiDenseChunkedByte(_) => None,
|
||||
VectorStorageReadEnum::MultiDenseChunkedHalf(_) => None,
|
||||
VectorStorageReadEnum::DenseTurbo(_) => None,
|
||||
VectorStorageReadEnum::DenseTurboGraphInline(s) => Some(s.hnsw_graph()),
|
||||
VectorStorageReadEnum::DenseTurboChunked(_) => None,
|
||||
VectorStorageReadEnum::MultiDenseTurbo(_) => None,
|
||||
VectorStorageReadEnum::Sparse(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: UniversalRead> RawScorerBuilder for VectorStorageReadEnum<S> {
|
||||
fn build_raw_scorer<'a>(
|
||||
&'a self,
|
||||
@@ -706,7 +732,8 @@ mod tests {
|
||||
}
|
||||
VectorStorageType::InRamMmap
|
||||
| VectorStorageType::InRamChunkedMmap
|
||||
| VectorStorageType::Memory => {
|
||||
| VectorStorageType::Memory
|
||||
| VectorStorageType::GraphInline => {
|
||||
unreachable!("unexpected storage type {storage_type:?}")
|
||||
}
|
||||
}
|
||||
@@ -728,7 +755,8 @@ mod tests {
|
||||
}
|
||||
VectorStorageType::Memory
|
||||
| VectorStorageType::InRamChunkedMmap
|
||||
| VectorStorageType::InRamMmap => false,
|
||||
| VectorStorageType::InRamMmap
|
||||
| VectorStorageType::GraphInline => false,
|
||||
};
|
||||
assert!(
|
||||
routed,
|
||||
|
||||
@@ -80,4 +80,8 @@ impl<S: UniversalRead> ReadOnlyImmutableTurboVectorStorage<GraphVectors<u8, S>>
|
||||
pub fn io_backend(&self) -> Option<IoBackend> {
|
||||
self.storage.graph().io_backend()
|
||||
}
|
||||
|
||||
pub fn hnsw_graph(&self) -> HnswGraph<S> {
|
||||
self.storage.graph().clone()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,6 +168,10 @@ impl<S: UniversalRead> TurboVectorStorageImpl<GraphVectors<u8, S>> {
|
||||
pub fn io_backend(&self) -> Option<IoBackend> {
|
||||
self.storage.graph().io_backend()
|
||||
}
|
||||
|
||||
pub fn hnsw_graph(&self) -> &HnswGraph<S> {
|
||||
self.storage.graph()
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: TurboVectorBlob> TurboVectorStorageImpl<B> {
|
||||
|
||||
@@ -89,7 +89,8 @@ impl<S: UniversalAppend + 'static> UpdateOnlyVectorStorage<S> {
|
||||
VectorStorageType::ChunkedMmap | VectorStorageType::InRamChunkedMmap => {}
|
||||
storage_type @ (VectorStorageType::Mmap
|
||||
| VectorStorageType::InRamMmap
|
||||
| VectorStorageType::Memory) => {
|
||||
| VectorStorageType::Memory
|
||||
| VectorStorageType::GraphInline) => {
|
||||
return Err(OperationError::service_error(format!(
|
||||
"Cannot open a {storage_type:?} vector storage for appending: it is not an \
|
||||
appendable storage type",
|
||||
|
||||
@@ -37,7 +37,7 @@ use crate::data_types::vectors::{
|
||||
DenseVector, MultiDenseVectorInternal, TypedMultiDenseVector, TypedMultiDenseVectorRef,
|
||||
VectorElementType, VectorElementTypeByte, VectorElementTypeHalf, VectorInternal, VectorRef,
|
||||
};
|
||||
use crate::index::hnsw_index::HnswLinksStorage;
|
||||
use crate::index::hnsw_index::{HnswGraph, HnswLinksStorage};
|
||||
use crate::types::{Distance, IoBackend, MultiVectorConfig, VectorStorageDatatype};
|
||||
use crate::vector_storage::dense::appendable_dense_vector_storage::AppendableMmapDenseVectorStorage;
|
||||
use crate::vector_storage::dense::graph_inline_dense_vector_storage::GraphInlineDenseVectorStorage;
|
||||
@@ -673,6 +673,49 @@ pub enum VectorStorageEnum {
|
||||
}
|
||||
|
||||
impl VectorStorageEnum {
|
||||
pub fn hnsw_graph(&self) -> Option<&HnswGraph<HnswLinksStorage>> {
|
||||
match self {
|
||||
VectorStorageEnum::DenseVolatile(_) => None,
|
||||
#[cfg(test)]
|
||||
VectorStorageEnum::DenseVolatileByte(_) => None,
|
||||
#[cfg(test)]
|
||||
VectorStorageEnum::DenseVolatileHalf(_) => None,
|
||||
VectorStorageEnum::DenseMemmap(_) => None,
|
||||
VectorStorageEnum::DenseMemmapByte(_) => None,
|
||||
VectorStorageEnum::DenseMemmapHalf(_) => None,
|
||||
VectorStorageEnum::DenseGraphInline(v) => Some(v.hnsw_graph()),
|
||||
VectorStorageEnum::DenseGraphInlineByte(v) => Some(v.hnsw_graph()),
|
||||
VectorStorageEnum::DenseGraphInlineHalf(v) => Some(v.hnsw_graph()),
|
||||
#[cfg(target_os = "linux")]
|
||||
VectorStorageEnum::DenseUring(_) => None,
|
||||
#[cfg(target_os = "linux")]
|
||||
VectorStorageEnum::DenseUringByte(_) => None,
|
||||
#[cfg(target_os = "linux")]
|
||||
VectorStorageEnum::DenseUringHalf(_) => None,
|
||||
VectorStorageEnum::DenseAppendableMemmap(_) => None,
|
||||
VectorStorageEnum::DenseAppendableMemmapByte(_) => None,
|
||||
VectorStorageEnum::DenseAppendableMemmapHalf(_) => None,
|
||||
VectorStorageEnum::DenseTurboMemmap(_) => None,
|
||||
VectorStorageEnum::DenseTurboGraphInline(v) => Some(v.hnsw_graph()),
|
||||
#[cfg(target_os = "linux")]
|
||||
VectorStorageEnum::DenseTurboUring(_) => None,
|
||||
VectorStorageEnum::DenseTurboAppendableMemmap(_) => None,
|
||||
VectorStorageEnum::SparseVolatile(_) => None,
|
||||
VectorStorageEnum::SparseMmap(_) => None,
|
||||
VectorStorageEnum::MultiDenseVolatile(_) => None,
|
||||
#[cfg(test)]
|
||||
VectorStorageEnum::MultiDenseVolatileByte(_) => None,
|
||||
#[cfg(test)]
|
||||
VectorStorageEnum::MultiDenseVolatileHalf(_) => None,
|
||||
VectorStorageEnum::MultiDenseAppendableMemmap(_) => None,
|
||||
VectorStorageEnum::MultiDenseAppendableMemmapByte(_) => None,
|
||||
VectorStorageEnum::MultiDenseAppendableMemmapHalf(_) => None,
|
||||
VectorStorageEnum::MultiDenseTurbo(_) => None,
|
||||
VectorStorageEnum::EmptyDense(_) => None,
|
||||
VectorStorageEnum::EmptySparse(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn try_multi_vector_config(&self) -> Option<&MultiVectorConfig> {
|
||||
match self {
|
||||
VectorStorageEnum::DenseVolatile(_) => None,
|
||||
|
||||
@@ -101,7 +101,8 @@ impl ConfigMismatchOptimizer {
|
||||
}
|
||||
|
||||
if let Some(required_memory) = self.requested_vectors_memory(vector_name)
|
||||
&& required_memory.is_on_disk() != vector_data.storage_type.is_on_disk()
|
||||
&& let Some(memory) = vector_data.storage_type.memory()
|
||||
&& required_memory.is_on_disk() != memory.is_on_disk()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -66,7 +66,6 @@ impl IndexingOptimizer {
|
||||
for (vector_name, vector_cfg) in &self.segment_optimizer_config.dense_vectors {
|
||||
if let Some(vector_data) = segment_data_config.vector_data.get(vector_name) {
|
||||
let is_indexed = vector_data.index.is_indexed();
|
||||
let is_on_disk = vector_data.storage_type.is_on_disk();
|
||||
let storage_size_bytes = segment
|
||||
.available_vectors_size_in_bytes(vector_name)
|
||||
.unwrap_or_default();
|
||||
@@ -75,10 +74,16 @@ impl IndexingOptimizer {
|
||||
let is_big_for_mmap = storage_size_bytes >= mmap_threshold_bytes;
|
||||
|
||||
let optimize_for_index = is_big_for_index && !is_indexed;
|
||||
let optimize_for_mmap = if let Some(on_disk_config) = vector_cfg.on_disk {
|
||||
on_disk_config && !is_on_disk
|
||||
} else {
|
||||
is_big_for_mmap && !is_on_disk
|
||||
let optimize_for_mmap = match vector_data.storage_type.memory() {
|
||||
Some(memory) => {
|
||||
let is_on_disk = memory.is_on_disk();
|
||||
if let Some(on_disk_config) = vector_cfg.on_disk {
|
||||
on_disk_config && !is_on_disk
|
||||
} else {
|
||||
is_big_for_mmap && !is_on_disk
|
||||
}
|
||||
}
|
||||
None => false,
|
||||
};
|
||||
|
||||
if optimize_for_index || optimize_for_mmap || has_deferred_points {
|
||||
|
||||
@@ -282,7 +282,7 @@ pub trait SegmentOptimizer: Sync {
|
||||
// If we explicitly configure the placement, but the segment storage type uses
|
||||
// something that doesn't match, warn about it
|
||||
if let Some(config_memory) = config_memory
|
||||
&& config_memory.is_on_disk() != config.storage_type.is_on_disk()
|
||||
&& config_memory.is_on_disk() != config.is_on_disk()
|
||||
{
|
||||
log::warn!(
|
||||
"Collection config for vector {vector_name} has memory placement {config_memory:?} configured, but storage type for segment doesn't match it"
|
||||
|
||||
Reference in New Issue
Block a user