Spaces:
Build error
Build error
File size: 2,535 Bytes
84d2a97 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 |
use std::path::{Path, PathBuf};
use io::file_operations::{atomic_save_json, read_json};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use crate::common::anonymize::Anonymize;
use crate::common::operation_error::OperationResult;
use crate::types::VectorStorageDatatype;
pub const SPARSE_INDEX_CONFIG_FILE: &str = "sparse_index_config.json";
/// Sparse index types
#[derive(Default, Hash, Debug, Deserialize, Serialize, JsonSchema, Eq, PartialEq, Copy, Clone)]
pub enum SparseIndexType {
/// Mutable RAM sparse index
#[default]
MutableRam,
/// Immutable RAM sparse index
ImmutableRam,
/// Mmap sparse index
Mmap,
}
impl SparseIndexType {
pub fn is_appendable(self) -> bool {
self == Self::MutableRam
}
pub fn is_immutable(self) -> bool {
self != Self::MutableRam
}
pub fn is_on_disk(self) -> bool {
self == Self::Mmap
}
pub fn is_persisted(self) -> bool {
self == Self::Mmap || self == Self::ImmutableRam
}
}
/// Configuration for sparse inverted index.
#[derive(Debug, Deserialize, Serialize, JsonSchema, Copy, Clone, PartialEq, Eq, Default)]
#[serde(rename_all = "snake_case")]
pub struct SparseIndexConfig {
/// We prefer a full scan search upto (excluding) this number of vectors.
///
/// Note: this is number of vectors, not KiloBytes.
pub full_scan_threshold: Option<usize>,
/// Type of sparse index
pub index_type: SparseIndexType,
/// Datatype used to store weights in the index.
#[serde(default)]
#[serde(skip_serializing_if = "Option::is_none")]
pub datatype: Option<VectorStorageDatatype>,
}
impl Anonymize for SparseIndexConfig {
fn anonymize(&self) -> Self {
SparseIndexConfig {
full_scan_threshold: self.full_scan_threshold,
index_type: self.index_type,
datatype: self.datatype,
}
}
}
impl SparseIndexConfig {
pub fn new(
full_scan_threshold: Option<usize>,
index_type: SparseIndexType,
datatype: Option<VectorStorageDatatype>,
) -> Self {
SparseIndexConfig {
full_scan_threshold,
index_type,
datatype,
}
}
pub fn get_config_path(path: &Path) -> PathBuf {
path.join(SPARSE_INDEX_CONFIG_FILE)
}
pub fn load(path: &Path) -> OperationResult<Self> {
Ok(read_json(path)?)
}
pub fn save(&self, path: &Path) -> OperationResult<()> {
Ok(atomic_save_json(path, self)?)
}
}
|