mod lexical;
mod outbox;
mod vector;
pub use usearch::ScalarKind;
pub use vector::EMBEDDING_DIM;
use std::path::{Path, PathBuf};
use anyhow::{Result, anyhow};
use uuid::Uuid;
use crate::sqlite_store::SqliteStore;
use lexical::LexicalIndex;
use vector::VectorIndex;
const VECTOR_FILE: &str = "vectors.usearch";
const CHUNK_FILE: &str = "chunks.usearch";
const CHUNK_SCALAR_KIND: ScalarKind = ScalarKind::F16;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IndexesMode {
ReadWrite,
ReadOnly,
}
pub struct Indexes {
vector: VectorIndex,
chunks: VectorIndex,
lexical: LexicalIndex,
dir: PathBuf,
mode: IndexesMode,
}
impl Indexes {
pub fn open(dir: &Path) -> Result<Self> {
Self::open_with_kind(dir, ScalarKind::F32)
}
pub fn open_with_kind(dir: &Path, scalar_kind: ScalarKind) -> Result<Self> {
std::fs::create_dir_all(dir)?;
let vector = VectorIndex::open_with_kind(&dir.join(VECTOR_FILE), scalar_kind)?;
let chunks = VectorIndex::open_with_kind(&dir.join(CHUNK_FILE), CHUNK_SCALAR_KIND)?;
let lexical = LexicalIndex::open(&dir.join("tantivy"))?;
Ok(Self {
vector,
chunks,
lexical,
dir: dir.to_path_buf(),
mode: IndexesMode::ReadWrite,
})
}
pub fn open_readonly(dir: &Path) -> Result<Self> {
Self::open_readonly_with_kind(dir, ScalarKind::F32)
}
pub fn open_readonly_with_kind(dir: &Path, scalar_kind: ScalarKind) -> Result<Self> {
let vector = VectorIndex::open_view_with_kind(&dir.join(VECTOR_FILE), scalar_kind)?;
let chunk_path = dir.join(CHUNK_FILE);
let chunks = if chunk_path.exists() {
VectorIndex::open_view_with_kind(&chunk_path, CHUNK_SCALAR_KIND)?
} else {
VectorIndex::empty_view_with_kind(&chunk_path, CHUNK_SCALAR_KIND)?
};
let lexical = LexicalIndex::open_reader(&dir.join("tantivy"))?;
Ok(Self {
vector,
chunks,
lexical,
dir: dir.to_path_buf(),
mode: IndexesMode::ReadOnly,
})
}
pub fn on_disk_scalar_kind(dir: &Path) -> Result<Option<ScalarKind>> {
VectorIndex::on_disk_kind(&dir.join(VECTOR_FILE))
}
pub fn on_disk_chunk_scalar_kind(dir: &Path) -> Result<Option<ScalarKind>> {
VectorIndex::on_disk_kind(&dir.join(CHUNK_FILE))
}
pub fn mode(&self) -> IndexesMode {
self.mode
}
pub fn scalar_kind(&self) -> ScalarKind {
self.vector.scalar_kind()
}
pub(crate) fn vector_mut(&mut self) -> &mut VectorIndex {
&mut self.vector
}
pub(crate) fn chunks_mut(&mut self) -> &mut VectorIndex {
&mut self.chunks
}
pub(crate) fn lexical_mut(&mut self) -> &mut LexicalIndex {
&mut self.lexical
}
pub(crate) fn require_writable(&self) -> Result<()> {
match self.mode {
IndexesMode::ReadWrite => Ok(()),
IndexesMode::ReadOnly => Err(anyhow!(
"indexes at {} are open read-only; mutation is not permitted",
self.dir.display()
)),
}
}
pub fn vector_search(
&self,
store: &SqliteStore,
embedding: &[f32],
k: usize,
) -> Result<Vec<(Uuid, f32)>> {
let hits = self.vector.search(embedding, k)?;
let mut out = Vec::with_capacity(hits.len());
for (key, similarity) in hits {
if let Some(id) = store.memory_id_for_vec_key(key)? {
out.push((id, similarity));
}
}
Ok(out)
}
pub fn chunk_search(
&self,
store: &SqliteStore,
query: &[f32],
k: usize,
) -> Result<Vec<(Uuid, f32, usize)>> {
if self.chunks.len() == 0 {
return Ok(Vec::new());
}
let mut seen = std::collections::HashSet::new();
let mut out = Vec::new();
for (key, similarity) in self.chunks.search(query, k)? {
if let Some((id, ord)) = store.chunk_for_vec_key(key)?
&& seen.insert(id)
{
out.push((id, similarity, ord));
}
}
Ok(out)
}
pub fn bm25_search(&self, query: &str, k: usize) -> Result<Vec<(Uuid, f32)>> {
self.lexical.search(query, k)
}
pub fn vector_len(&self) -> usize {
self.vector.len()
}
pub fn chunk_vector_len(&self) -> usize {
self.chunks.len()
}
pub fn lexical_len(&self) -> u64 {
self.lexical.num_docs()
}
pub fn dir(&self) -> &Path {
&self.dir
}
pub fn consistent_with(&self, store: &SqliteStore) -> Result<bool> {
let indexable = store.indexable_count()?;
let embedded = store.vector_indexable_count(EMBEDDING_DIM)?;
let embedded_chunks = store.embedded_chunk_count(EMBEDDING_DIM)?;
Ok(self.vector.len() == embedded
&& self.chunks.len() == embedded_chunks
&& self.lexical.num_docs() as usize == indexable)
}
pub fn persist(&mut self) -> Result<()> {
self.require_writable()?;
self.lexical.commit()?;
self.vector.save()?;
self.chunks.save()
}
pub fn reload_view(&mut self) -> Result<()> {
self.vector.reload_view()?;
if self.dir.join(CHUNK_FILE).exists() {
self.chunks.reload_view()?;
}
self.lexical.reload_reader()
}
pub fn resident_bytes(&self) -> usize {
self.vector.memory_usage()
+ self.chunks.memory_usage()
+ dir_size(&self.dir.join("tantivy")) as usize
}
}
fn dir_size(dir: &Path) -> u64 {
let Ok(entries) = std::fs::read_dir(dir) else {
return 0;
};
let mut total = 0u64;
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
total += dir_size(&path);
} else if let Ok(metadata) = entry.metadata() {
total += metadata.len();
}
}
total
}