upstream-archive/qdrant
1
0
Fork 0
mirror of https://github.com/qdrant/qdrant.git synced 2026-08-06 18:10:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

WIP: struct payload index texts

Browse Source
This commit is contained in:
Andrey Vasnetsov
2021-03-14 01:01:53 +01:00
parent 328f445439
commit 5dd3c1935e
5 changed files with 121 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
lib/segment/src/index/plain_payload_index.rs
View File

@@ -38,7 +38,12 @@ impl PlainPayloadIndex {
path: &Path,
) -> OperationResult<Self> {
let config_path = PayloadConfig::get_config_path(path);
let config = PayloadConfig::load(&config_path)?;
let config = if config_path.exists() {
PayloadConfig::load(&config_path)?
} else {
PayloadConfig::default()
};
let index = PlainPayloadIndex {
condition_checker,
@@ -47,26 +52,9 @@ impl PlainPayloadIndex {
path: path.to_owned()
};
Ok(index)
}
pub fn new(
condition_checker: Arc<AtomicRefCell<dyn ConditionChecker>>,
vector_storage: Arc<AtomicRefCell<dyn VectorStorage>>,
path: &Path,
config: Option<PayloadConfig>
) -> OperationResult<Self> {
create_dir_all(path)?;
let payload_config = config.unwrap_or_default();
let index = PlainPayloadIndex {
condition_checker,
vector_storage,
config: payload_config,
path: path.to_owned()
};
index.save_config()?;
if !index.config_path().exists() {
index.save_config()?
}
Ok(index)
}

68
lib/segment/src/index/struct_payload_index.rs
View File

@@ -32,7 +32,6 @@ pub struct StructPayloadIndex {
field_indexes: IndexesMap,
config: PayloadConfig,
path: PathBuf,
total_points: usize,
}
impl StructPayloadIndex {
@@ -122,10 +121,13 @@ impl StructPayloadIndex {
payload: Arc<AtomicRefCell<dyn PayloadStorage>>,
id_mapper: Arc<AtomicRefCell<dyn IdMapper>>,
path: &Path,
total_points: usize,
) -> OperationResult<Self> {
let config_path = PayloadConfig::get_config_path(path);
let config = PayloadConfig::load(&config_path)?;
let config = if config_path.exists() {
PayloadConfig::load(&config_path)?
} else {
PayloadConfig::default()
};
let mut index = StructPayloadIndex {
condition_checker,
@@ -134,10 +136,14 @@ impl StructPayloadIndex {
id_mapper,
field_indexes: Default::default(),
config,
path: path.to_owned(),
total_points,
path: path.to_owned()
};
if !index.config_path().exists() {
// Save default config
index.save_config()?
}
index.load_all_fields()?;
Ok(index)
@@ -206,33 +212,6 @@ impl StructPayloadIndex {
Ok(())
}
pub fn new(
condition_checker: Arc<AtomicRefCell<dyn ConditionChecker>>,
vector_storage: Arc<AtomicRefCell<dyn VectorStorage>>,
payload: Arc<AtomicRefCell<dyn PayloadStorage>>,
id_mapper: Arc<AtomicRefCell<dyn IdMapper>>,
path: &Path,
config: Option<PayloadConfig>,
total_points: usize,
) -> OperationResult<Self> {
create_dir_all(path)?;
let payload_config = config.unwrap_or_default();
let mut payload_index = Self {
condition_checker,
vector_storage,
payload,
id_mapper,
field_indexes: Default::default(),
config: payload_config,
path: path.to_owned(),
total_points,
};
payload_index.build_all_fields()?;
Ok(payload_index)
}
fn save(&self) -> OperationResult<()> {
let file = File::create(self.path.as_path())?;
serde_cbor::to_writer(file, &self.field_indexes)
@@ -241,7 +220,7 @@ impl StructPayloadIndex {
}
pub fn total_points(&self) -> usize {
self.total_points
self.vector_storage.borrow().vector_count()
}
}
@@ -332,31 +311,10 @@ impl PayloadIndex for StructPayloadIndex {
})
.flat_map(|x| x)
.collect();
let matched_points = preselected.into_iter()
let matched_points = preselected.into_iter()
.filter(|i| condition_checker.check(*i, query))
.collect_vec();
Box::new(matched_points.into_iter())
};
}
}
#[cfg(test)]
mod tests {
use tempdir::TempDir;
use crate::payload_storage::simple_payload_storage::SimplePayloadStorage;
use super::*;
#[test]
fn test_index_save_and_load() {
let dir = TempDir::new("storage_dir").unwrap();
let mut storage = SimplePayloadStorage::open(dir.path()).unwrap();
}
// #[test]
// fn test_flat_map() {
// let a = vec![vec![1,2,3], vec![4,5,6], vec![7,7,7]];
// a.iter().flat_map(|x| x.iter()).for_each(|x| println!("{}", x))
// }
}

16
lib/segment/src/segment_constructor/segment_constructor.rs
View File

@@ -4,7 +4,7 @@ use crate::vector_storage::simple_vector_storage::SimpleVectorStorage;
use crate::payload_storage::simple_payload_storage::SimplePayloadStorage;
use crate::index::plain_payload_index::{PlainPayloadIndex, PlainIndex};
use crate::query_planner::simple_query_planner::SimpleQueryPlanner;
use crate::types::{SegmentType, SegmentConfig, Indexes, SegmentState, SeqNumberType, StorageType};
use crate::types::{SegmentType, SegmentConfig, Indexes, SegmentState, SeqNumberType, StorageType, PayloadIndexType};
use std::sync::{Arc, Mutex};
use atomic_refcell::AtomicRefCell;
use crate::payload_storage::query_checker::SimpleConditionChecker;
@@ -15,6 +15,8 @@ use crate::entry::entry_point::{OperationResult, OperationError};
use std::io::Read;
use crate::vector_storage::memmap_vector_storage::MemmapVectorStorage;
use crate::vector_storage::vector_storage::VectorStorage;
use crate::index::struct_payload_index::StructPayloadIndex;
use crate::index::index::PayloadIndex;
fn sp<T>(t: T) -> Arc<AtomicRefCell<T>> { Arc::new(AtomicRefCell::new(t)) }
@@ -42,9 +44,15 @@ fn create_segment(version: SeqNumberType, segment_path: &Path, config: &SegmentC
id_mapper.clone(),
));
let payload_index = sp(PlainPayloadIndex::new(
condition_checker, vector_storage.clone(), &payload_index_path, None
)?);
let payload_index: Arc<AtomicRefCell<dyn PayloadIndex>> = match config.payload_index.unwrap_or_default() {
PayloadIndexType::Plain => sp(PlainPayloadIndex::open(condition_checker, vector_storage.clone(), &payload_index_path)?),
PayloadIndexType::Struct => sp(StructPayloadIndex::open(
condition_checker,
vector_storage.clone(),
payload_storage.clone(),
id_mapper.clone(),
&payload_index_path)?),
};
let index = sp(match config.index {
Indexes::Plain { .. } => PlainIndex::new(vector_storage.clone(), payload_index, config.distance),

4
lib/segment/src/types.rs
View File

@@ -133,9 +133,9 @@ impl Default for Indexes {
#[serde(tag = "type", content = "options")]
/// Type of payload index
pub enum PayloadIndexType {
/// Store vectors in memory and use persistence storage only if vectors are changed
/// Do not index anything, just keep of what should be indexed later
Plain,
/// Use memmap to store vectors, a little slower than `InMemory`, but requires little RAM
/// Build payload index. Index is saved on disc, but index itself is in RAM
Struct,
}

85
lib/segment/tests/payload_index_test.rs Normal file
View File

@@ -0,0 +1,85 @@
#[cfg(test)]
mod tests {
use super::*;
use rand::prelude::ThreadRng;
use rand::seq::SliceRandom;
use segment::types::{PayloadType, VectorElementType, SegmentConfig, Indexes, PayloadIndexType, Distance, StorageType};
use rand::Rng;
use tempdir::TempDir;
use segment::segment_constructor::segment_constructor::build_segment;
const ADJECTIVE: Vec<String> = vec![
"jobless".to_string(),
"rightful".to_string(),
"breakable".to_string(),
"impartial".to_string(),
"shocking".to_string(),
"faded".to_string(),
"phobic".to_string(),
"overt".to_string(),
"like".to_string(),
"wide-eyed".to_string(),
"broad".to_string(),
];
const NOUN: Vec<String> = vec![
"territory".to_string(),
"jam".to_string(),
"neck".to_string(),
"chicken".to_string(),
"cap".to_string(),
"kiss".to_string(),
"veil".to_string(),
"trail".to_string(),
"size".to_string(),
"digestion".to_string(),
"rod".to_string(),
"seed".to_string(),
];
fn random_keyword(rnd_gen: &mut ThreadRng) -> String {
let random_adj = ADJECTIVE.choose(rnd_gen).unwrap();
let random_noun = NOUN.choose(rnd_gen).unwrap();
format!("{} {}", random_adj, random_noun)
}
fn random_keyword_payload(rnd_gen: &mut ThreadRng) -> PayloadType {
PayloadType::Keyword(vec![random_keyword(rnd_gen)])
}
fn random_int_payload(rnd_gen: &mut ThreadRng) -> PayloadType {
let val1: i64 = rnd_gen.gen_range(0..500);
let val2: i64 = rnd_gen.gen_range(0..500);
let val3: i64 = rnd_gen.gen_range(0..500);
PayloadType::Integer(vec![val1, val2, val3])
}
fn random_vector(rnd_gen: &mut ThreadRng, size: usize) -> Vec<VectorElementType> {
(0..size).map(|_| rnd_gen.gen()).collect()
}
#[test]
fn test_struct_payload_index() {
// Compare search with plain and struct indexes
let dir = TempDir::new("segment_dir").unwrap();
let dim = 5;
let mut config = SegmentConfig {
vector_size: dim,
index: Indexes::Plain {},
payload_index: Some(PayloadIndexType::Plain),
storage_type: StorageType::InMemory,
distance: Distance::Dot,
};
let mut plain_segment = build_segment(path, &config).unwrap();
config.payload_index = Some(PayloadIndexType::Struct);
let mut struct_segment = build_segment(path, &config).unwrap();
// ToDo: Init both segments with same data
// ToDo: Compare indexed and un-indexed search results
}
}