second-brain-core 0.9.0

Core library for second-brain: KuzuDB graph storage, BGE embeddings, and weighted query engine
Documentation
mod lexical;
mod outbox;
mod vector;

pub use usearch::ScalarKind;
pub use vector::EMBEDDING_DIM;

use std::path::{Path, PathBuf};

use anyhow::{Result, anyhow};
use uuid::Uuid;

use crate::sqlite_store::SqliteStore;
use lexical::LexicalIndex;
use vector::VectorIndex;

const VECTOR_FILE: &str = "vectors.usearch";

/// `ReadWrite` indexes RAM-load usearch and may construct a tantivy writer.
/// `ReadOnly` indexes mmap usearch via `view()` and open tantivy purely for
/// search, so a multi-tenant service can hold thousands of them open without
/// paying either cost. See `Indexes::open_readonly`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IndexesMode {
    ReadWrite,
    ReadOnly,
}

/// The retrieval indexes beside brain.db: usearch for dense vectors, tantivy
/// for BM25. Fed exclusively through the store's index_outbox (see
/// `Indexes::drain`), so SQLite remains the source of truth and both indexes
/// can always be rebuilt from it without re-embedding.
pub struct Indexes {
    vector: VectorIndex,
    lexical: LexicalIndex,
    dir: PathBuf,
    mode: IndexesMode,
}

impl Indexes {
    pub fn open(dir: &Path) -> Result<Self> {
        Self::open_with_kind(dir, ScalarKind::F32)
    }

    /// Same as `open`, with the scalar kind the vector index quantizes to
    /// made explicit. An existing vector file must have been written as that
    /// kind or this errors; `on_disk_scalar_kind` says which kind it was.
    pub fn open_with_kind(dir: &Path, scalar_kind: ScalarKind) -> Result<Self> {
        std::fs::create_dir_all(dir)?;
        let vector = VectorIndex::open_with_kind(&dir.join(VECTOR_FILE), scalar_kind)?;
        let lexical = LexicalIndex::open(&dir.join("tantivy"))?;
        Ok(Self {
            vector,
            lexical,
            dir: dir.to_path_buf(),
            mode: IndexesMode::ReadWrite,
        })
    }

    /// Opens both indexes read-only: usearch via mmap `view()`, tantivy with
    /// no `IndexWriter` ever constructed. Errors if either index is missing
    /// instead of creating one, and creates no files or directories, so an
    /// admission path can probe a tenant without side effects.
    pub fn open_readonly(dir: &Path) -> Result<Self> {
        Self::open_readonly_with_kind(dir, ScalarKind::F32)
    }

    /// Same as `open_readonly`, with the vector index's scalar kind made
    /// explicit; a vector file written as another kind is an error.
    pub fn open_readonly_with_kind(dir: &Path, scalar_kind: ScalarKind) -> Result<Self> {
        let vector = VectorIndex::open_view_with_kind(&dir.join(VECTOR_FILE), scalar_kind)?;
        let lexical = LexicalIndex::open_reader(&dir.join("tantivy"))?;
        Ok(Self {
            vector,
            lexical,
            dir: dir.to_path_buf(),
            mode: IndexesMode::ReadOnly,
        })
    }

    /// The scalar kind the vector index in `dir` was written as, read from
    /// the usearch file header alone. `None` when no vector index has been
    /// persisted there yet, which is the normal state of a new tenant.
    pub fn on_disk_scalar_kind(dir: &Path) -> Result<Option<ScalarKind>> {
        VectorIndex::on_disk_kind(&dir.join(VECTOR_FILE))
    }

    pub fn mode(&self) -> IndexesMode {
        self.mode
    }

    pub fn scalar_kind(&self) -> ScalarKind {
        self.vector.scalar_kind()
    }

    pub(crate) fn vector_mut(&mut self) -> &mut VectorIndex {
        &mut self.vector
    }

    pub(crate) fn lexical_mut(&mut self) -> &mut LexicalIndex {
        &mut self.lexical
    }

    pub(crate) fn require_writable(&self) -> Result<()> {
        match self.mode {
            IndexesMode::ReadWrite => Ok(()),
            IndexesMode::ReadOnly => Err(anyhow!(
                "indexes at {} are open read-only; mutation is not permitted",
                self.dir.display()
            )),
        }
    }

    /// Returns (memory id, cosine similarity) pairs, best first. The store
    /// resolves usearch's u64 keys back to memory ids via vec_keys.
    pub fn vector_search(
        &self,
        store: &SqliteStore,
        embedding: &[f32],
        k: usize,
    ) -> Result<Vec<(Uuid, f32)>> {
        let hits = self.vector.search(embedding, k)?;
        let mut out = Vec::with_capacity(hits.len());
        for (key, similarity) in hits {
            if let Some(id) = store.memory_id_for_vec_key(key)? {
                out.push((id, similarity));
            }
        }
        Ok(out)
    }

    /// Returns (memory id, BM25 score) pairs, best first.
    pub fn bm25_search(&self, query: &str, k: usize) -> Result<Vec<(Uuid, f32)>> {
        self.lexical.search(query, k)
    }

    pub fn vector_len(&self) -> usize {
        self.vector.len()
    }

    pub fn lexical_len(&self) -> u64 {
        self.lexical.num_docs()
    }

    pub fn dir(&self) -> &Path {
        &self.dir
    }

    /// Index counts must match the store's indexable rows; drift means a
    /// missed outbox row or lost index file and calls for `rebuild`.
    pub fn consistent_with(&self, store: &SqliteStore) -> Result<bool> {
        let indexable = store.indexable_count()?;
        let embedded = store.vector_indexable_count(EMBEDDING_DIM)?;
        Ok(self.vector.len() == embedded && self.lexical.num_docs() as usize == indexable)
    }

    pub fn persist(&mut self) -> Result<()> {
        self.require_writable()?;
        self.lexical.commit()?;
        self.vector.save()
    }

    /// Re-opens on-disk state so a `ReadOnly` handle observes what a writer
    /// elsewhere has since persisted. A no-op on `ReadWrite`: see
    /// `VectorIndex::reload_view` for why a mutable handle has nothing to
    /// pick up from disk.
    pub fn reload_view(&mut self) -> Result<()> {
        self.vector.reload_view()?;
        self.lexical.reload_reader()
    }

    /// Bytes this tenant's indexes cost to keep open: usearch's own RAM
    /// accounting plus the on-disk footprint of the tantivy directory, which
    /// `memory_usage` never sees. Admission control budgets in bytes, not in
    /// a count of open tenants.
    pub fn resident_bytes(&self) -> usize {
        self.vector.memory_usage() + dir_size(&self.dir.join("tantivy")) as usize
    }
}

/// A missing directory contributes zero bytes rather than erroring: for a
/// read-only handle beside an index that has never persisted anything,
/// "does not exist yet" and "zero bytes on disk" are the same fact. Entries
/// that vanish mid-walk (a concurrent writer mid-rename) are skipped for the
/// same reason `du` skips them: this is a best-effort accounting walk, not a
/// correctness-load-bearing read.
fn dir_size(dir: &Path) -> u64 {
    let Ok(entries) = std::fs::read_dir(dir) else {
        return 0;
    };
    let mut total = 0u64;
    for entry in entries.flatten() {
        let path = entry.path();
        if path.is_dir() {
            total += dir_size(&path);
        } else if let Ok(metadata) = entry.metadata() {
            total += metadata.len();
        }
    }
    total
}