mod collection;
mod compact;
mod identity;
mod loader;
mod partition;
mod query;
mod reader;
mod record;
mod schema;
mod schema_doc;
mod space;
mod writer;
use std::path::Path;
use std::sync::Arc;
use arc_swap::ArcSwap;
use parking_lot::Mutex;
use crate::concurrency::database::Database;
use crate::error::Result;
use self::identity::IdentityIndex;
use self::space::{CollectionRecord, SchemaRegistry, check_redeclare, reject_reserved_name};
pub use self::collection::{Collection, CollectionInfo};
pub use self::compact::{CompactOptions, CompactReport};
pub use self::loader::Bulk;
pub use self::partition::{Partition, Partitioned, View, Window};
pub use self::query::{Fusion, Hit, Recency, Rerank, Search, SearchResult, TextScorer, TfMode};
pub use self::reader::Reader;
pub use self::record::{Key, Record, Stored, Value};
pub use self::schema::{Calibrate, Codec, FieldSpec, Schema, Text, UpqDesign, Vector};
pub use self::space::{Lang, Metric, Space, SpaceInfo, SpaceKind, SpaceRef};
pub use self::writer::{OwnedWriter, Writer};
pub use crate::io::Durability;
#[derive(Clone, Debug)]
pub struct StoreOptions {
pub durability: Durability,
}
impl Default for StoreOptions {
fn default() -> Self {
Self {
durability: Durability::Buffered,
}
}
}
pub(crate) struct Published {
pub(crate) db: Arc<Database>,
pub(crate) schema: Arc<Mutex<SchemaRegistry>>,
pub(crate) identity: Arc<Mutex<IdentityIndex>>,
}
#[derive(Clone)]
pub(crate) struct Handles {
pub(crate) published: Arc<ArcSwap<Published>>,
pub(crate) writer_lock: Arc<Mutex<()>>,
}
impl Handles {
pub(crate) fn current(&self) -> Arc<Published> {
self.published.load_full()
}
pub(crate) fn writer(&self) -> Writer<'_> {
let guard = self.writer_lock.lock();
let p = self.published.load_full();
Writer::new(
guard,
Arc::clone(&p.db),
Arc::clone(&p.schema),
Arc::clone(&p.identity),
)
}
pub(crate) fn writer_owned(&self) -> OwnedWriter {
let guard = self.writer_lock.lock_arc();
let p = self.published.load_full();
OwnedWriter::new(
guard,
Arc::clone(&p.db),
Arc::clone(&p.schema),
Arc::clone(&p.identity),
)
}
pub(crate) fn try_writer_owned(&self) -> Option<OwnedWriter> {
let guard = self.writer_lock.try_lock_arc()?;
let p = self.published.load_full();
Some(OwnedWriter::new(
guard,
Arc::clone(&p.db),
Arc::clone(&p.schema),
Arc::clone(&p.identity),
))
}
pub(crate) fn reader(&self) -> Reader {
Reader::new(Arc::clone(&self.published))
}
}
pub struct Store {
handles: Handles,
}
impl Store {
pub fn open(path: impl AsRef<Path>) -> Result<Store> {
Self::open_with(path, StoreOptions::default())
}
pub fn open_with(path: impl AsRef<Path>, opts: StoreOptions) -> Result<Store> {
let path = path.as_ref();
if path.exists() {
let db = Database::open_read_write(path)?;
Self::from_db(db, opts, true)
} else {
let db = Database::create(path)?;
Self::from_db(db, opts, false)
}
}
pub fn create(path: impl AsRef<Path>) -> Result<Store> {
Self::create_with(path, StoreOptions::default())
}
pub fn create_with(path: impl AsRef<Path>, opts: StoreOptions) -> Result<Store> {
let db = Database::create(path)?;
Self::from_db(db, opts, false)
}
fn from_db(db: Arc<Database>, opts: StoreOptions, rebuild: bool) -> Result<Store> {
db.set_durability(opts.durability);
let mut schema = SchemaRegistry::default();
let mut identity = IdentityIndex::default();
if rebuild {
let valise = db.inner.read();
schema.rebuild(&valise)?;
identity.rebuild(&valise)?;
}
Ok(Store {
handles: Handles {
published: Arc::new(ArcSwap::from_pointee(Published {
db,
schema: Arc::new(Mutex::new(schema)),
identity: Arc::new(Mutex::new(identity)),
})),
writer_lock: Arc::new(Mutex::new(())),
},
})
}
pub fn define_space(&self, name: &str, spec: impl Into<FieldSpec>) -> Result<Space> {
reject_reserved_name("define_space: space", name)?;
let spec = spec.into();
let p = self.handles.current();
let mut schema = p.schema.lock();
let mut valise = p.db.inner.write();
schema.define_space(&mut valise, name, &spec)
}
#[must_use]
pub fn space(&self, name: &str) -> Option<Space> {
let p = self.handles.current();
let schema = p.schema.lock();
if let Some(e) = schema.vector_spaces.get(name) {
Some(Space::vector(name, e.dim))
} else if schema.text_spaces.contains_key(name) {
Some(Space::text(name))
} else {
None
}
}
#[must_use]
pub fn spaces(&self) -> Vec<SpaceInfo> {
let p = self.handles.current();
let schema = p.schema.lock();
schema.space_infos()
}
pub fn collection(&self, name: &str, schema: Schema) -> Result<Collection> {
reject_reserved_name("collection: collection", name)?;
let p = self.handles.current();
let mut reg = p.schema.lock();
let doc = schema_doc::SchemaDoc::from_schema(name, &schema)?;
if let Some(rec) = reg.collections.get(name) {
if let Some(prev) = &rec.doc {
schema_doc::check_compatible(name, prev, &doc)?;
} else if let Some(existing) = &rec.shape {
check_redeclare(name, existing, &schema)?;
}
}
let plan = {
let valise = p.db.inner.read();
schema_doc::plan_sync(&valise, name, &doc)?
};
let shape = {
let mut valise = p.db.inner.write();
reg.bind_schema(&mut valise, name, &schema)?
};
let collection_id = ensure_collection(&p.db, &mut reg, name)?;
{
let mut valise = p.db.inner.write();
schema_doc::apply_sync(&mut valise, name, &doc, &plan)?;
}
reg.collections.insert(
name.to_owned(),
CollectionRecord {
collection_id,
shape: Some(shape),
doc: Some(doc),
},
);
Ok(Collection {
id: collection_id,
name: name.to_owned(),
})
}
#[must_use]
pub fn collections(&self) -> Vec<CollectionInfo> {
let p = self.handles.current();
let schema = p.schema.lock();
schema
.collections
.iter()
.map(|(name, rec)| CollectionInfo {
name: name.clone(),
id: rec.collection_id,
})
.collect()
}
pub fn partitioned(&self, base: &str, schema: Schema, by: Partition) -> Result<Partitioned> {
reject_reserved_name("partitioned: collection", base)?;
let p = self.handles.current();
let mut reg = p.schema.lock();
let shape = {
let mut valise = p.db.inner.write();
reg.bind_schema(&mut valise, base, &schema)?
};
Ok(Partitioned::new(
self.handles.clone(),
base,
schema,
shape,
by,
))
}
#[must_use]
pub fn writer(&self) -> Writer<'_> {
self.handles.writer()
}
#[must_use]
pub fn writer_owned(&self) -> OwnedWriter {
self.handles.writer_owned()
}
#[must_use]
pub fn try_writer_owned(&self) -> Option<OwnedWriter> {
self.handles.try_writer_owned()
}
#[must_use]
pub fn reader(&self) -> Reader {
self.handles.reader()
}
pub fn get(&self, coll: &str, key: impl Into<Key>) -> Result<Option<Stored>> {
self.reader().get(coll, key)
}
pub fn search(&self, coll: &str, q: Search) -> Result<SearchResult> {
self.reader().search(coll, q)
}
pub fn search_view(&self, view: &View, q: Search) -> Result<SearchResult> {
self.reader().search_view(view, q)
}
#[must_use]
pub fn raw(&self) -> Arc<Database> {
Arc::clone(&self.handles.current().db)
}
pub fn compact(&self, opts: CompactOptions) -> Result<CompactReport> {
compact::compact(&self.handles, &opts)
}
#[must_use]
pub fn stats(&self) -> StoreStats {
let p = self.handles.current();
let valise = p.db.inner.read();
let reserved = schema_doc::schema_collection_id(&valise);
let is_user = |cid: crate::format::CollectionId| Some(cid) != reserved;
let frames_total = valise
.frames()
.iter()
.filter(|f| is_user(f.collection_id))
.count();
let frames_active = valise
.active_frames()
.filter(|f| is_user(f.collection_id))
.count();
let vectors_total = valise.vectors().len();
let vectors_active = valise
.vectors()
.iter()
.filter(|v| v.status == crate::format::catalog::VectorStatus::Active)
.count();
let file_bytes = std::fs::metadata(valise.path())
.map(|m| m.len())
.unwrap_or(0);
StoreStats {
frames_total,
frames_active,
vectors_total,
vectors_active,
file_bytes,
}
}
}
#[derive(Clone, Copy, Debug)]
pub struct StoreStats {
pub frames_total: usize,
pub frames_active: usize,
pub vectors_total: usize,
pub vectors_active: usize,
pub file_bytes: u64,
}
impl StoreStats {
#[must_use]
pub fn tombstone_ratio(&self) -> f32 {
if self.frames_total == 0 {
0.0
} else {
(self.frames_total - self.frames_active) as f32 / self.frames_total as f32
}
}
#[must_use]
pub fn needs_compaction(&self, threshold: f32) -> bool {
self.tombstone_ratio() >= threshold
}
}
pub(crate) fn ensure_collection(
db: &Arc<Database>,
schema: &mut SchemaRegistry,
name: &str,
) -> Result<crate::format::CollectionId> {
if let Some(rec) = schema.collections.get(name) {
return Ok(rec.collection_id);
}
let id = db.inner.write().create_collection(name)?;
Ok(id)
}