second-brain-core 0.11.0

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
mod lexical;
mod outbox;
mod vector;

pub use usearch::ScalarKind;
pub use vector::EMBEDDING_DIM;

use std::path::{Path, PathBuf};

use anyhow::{Result, anyhow};
use uuid::Uuid;

use crate::sqlite_store::SqliteStore;
use lexical::LexicalIndex;
use vector::VectorIndex;

const VECTOR_FILE: &str = "vectors.usearch";
const CHUNK_FILE: &str = "chunks.usearch";

/// Chunk vectors outnumber memory vectors several times over, and F16 halves
/// their RAM against F32 at no measurable cost to cosine ranking.
const CHUNK_SCALAR_KIND: ScalarKind = ScalarKind::F16;

/// `ReadWrite` indexes RAM-load usearch and may construct a tantivy writer.
/// `ReadOnly` indexes mmap usearch via `view()` and open tantivy purely for
/// search, so a multi-tenant service can hold thousands of them open without
/// paying either cost. See `Indexes::open_readonly`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IndexesMode {
    ReadWrite,
    ReadOnly,
}

/// The retrieval indexes beside brain.db: usearch for dense vectors (one
/// file for whole memories, one for their sentence chunks), tantivy for
/// BM25. Fed exclusively through the store's index_outbox (see
/// `Indexes::drain`), so SQLite remains the source of truth and both indexes
/// can always be rebuilt from it without re-embedding.
pub struct Indexes {
    vector: VectorIndex,
    chunks: VectorIndex,
    lexical: LexicalIndex,
    dir: PathBuf,
    mode: IndexesMode,
}

impl Indexes {
    pub fn open(dir: &Path) -> Result<Self> {
        Self::open_with_kind(dir, ScalarKind::F32)
    }

    /// Same as `open`, with the scalar kind the vector index quantizes to
    /// made explicit. An existing vector file must have been written as that
    /// kind or this errors; `on_disk_scalar_kind` says which kind it was.
    pub fn open_with_kind(dir: &Path, scalar_kind: ScalarKind) -> Result<Self> {
        std::fs::create_dir_all(dir)?;
        let vector = VectorIndex::open_with_kind(&dir.join(VECTOR_FILE), scalar_kind)?;
        let chunks = VectorIndex::open_with_kind(&dir.join(CHUNK_FILE), CHUNK_SCALAR_KIND)?;
        let lexical = LexicalIndex::open(&dir.join("tantivy"))?;
        Ok(Self {
            vector,
            chunks,
            lexical,
            dir: dir.to_path_buf(),
            mode: IndexesMode::ReadWrite,
        })
    }

    /// Opens both indexes read-only: usearch via mmap `view()`, tantivy with
    /// no `IndexWriter` ever constructed. Errors if either index is missing
    /// instead of creating one, and creates no files or directories, so an
    /// admission path can probe a tenant without side effects. The chunk
    /// index is the exception: an install that predates it has no chunk
    /// file until its writer first persists, so a missing one opens empty.
    pub fn open_readonly(dir: &Path) -> Result<Self> {
        Self::open_readonly_with_kind(dir, ScalarKind::F32)
    }

    /// Same as `open_readonly`, with the vector index's scalar kind made
    /// explicit; a vector file written as another kind is an error.
    pub fn open_readonly_with_kind(dir: &Path, scalar_kind: ScalarKind) -> Result<Self> {
        let vector = VectorIndex::open_view_with_kind(&dir.join(VECTOR_FILE), scalar_kind)?;
        let chunk_path = dir.join(CHUNK_FILE);
        let chunks = if chunk_path.exists() {
            VectorIndex::open_view_with_kind(&chunk_path, CHUNK_SCALAR_KIND)?
        } else {
            VectorIndex::empty_view_with_kind(&chunk_path, CHUNK_SCALAR_KIND)?
        };
        let lexical = LexicalIndex::open_reader(&dir.join("tantivy"))?;
        Ok(Self {
            vector,
            chunks,
            lexical,
            dir: dir.to_path_buf(),
            mode: IndexesMode::ReadOnly,
        })
    }

    /// The scalar kind the vector index in `dir` was written as, read from
    /// the usearch file header alone. `None` when no vector index has been
    /// persisted there yet, which is the normal state of a new tenant.
    pub fn on_disk_scalar_kind(dir: &Path) -> Result<Option<ScalarKind>> {
        VectorIndex::on_disk_kind(&dir.join(VECTOR_FILE))
    }

    pub fn on_disk_chunk_scalar_kind(dir: &Path) -> Result<Option<ScalarKind>> {
        VectorIndex::on_disk_kind(&dir.join(CHUNK_FILE))
    }

    pub fn mode(&self) -> IndexesMode {
        self.mode
    }

    pub fn scalar_kind(&self) -> ScalarKind {
        self.vector.scalar_kind()
    }

    pub(crate) fn vector_mut(&mut self) -> &mut VectorIndex {
        &mut self.vector
    }

    pub(crate) fn chunks_mut(&mut self) -> &mut VectorIndex {
        &mut self.chunks
    }

    pub(crate) fn lexical_mut(&mut self) -> &mut LexicalIndex {
        &mut self.lexical
    }

    pub(crate) fn require_writable(&self) -> Result<()> {
        match self.mode {
            IndexesMode::ReadWrite => Ok(()),
            IndexesMode::ReadOnly => Err(anyhow!(
                "indexes at {} are open read-only; mutation is not permitted",
                self.dir.display()
            )),
        }
    }

    /// Returns (memory id, cosine similarity) pairs, best first. The store
    /// resolves usearch's u64 keys back to memory ids via vec_keys.
    pub fn vector_search(
        &self,
        store: &SqliteStore,
        embedding: &[f32],
        k: usize,
    ) -> Result<Vec<(Uuid, f32)>> {
        let hits = self.vector.search(embedding, k)?;
        let mut out = Vec::with_capacity(hits.len());
        for (key, similarity) in hits {
            if let Some(id) = store.memory_id_for_vec_key(key)? {
                out.push((id, similarity));
            }
        }
        Ok(out)
    }

    /// Returns (memory id, cosine similarity, chunk ord) for the best chunk
    /// of each memory among the top `k` chunk hits, best first, so fewer
    /// than `k` memories come back when several of one memory's chunks hit.
    pub fn chunk_search(
        &self,
        store: &SqliteStore,
        query: &[f32],
        k: usize,
    ) -> Result<Vec<(Uuid, f32, usize)>> {
        if self.chunks.len() == 0 {
            return Ok(Vec::new());
        }
        let mut seen = std::collections::HashSet::new();
        let mut out = Vec::new();
        for (key, similarity) in self.chunks.search(query, k)? {
            if let Some((id, ord)) = store.chunk_for_vec_key(key)?
                && seen.insert(id)
            {
                out.push((id, similarity, ord));
            }
        }
        Ok(out)
    }

    /// Returns (memory id, BM25 score) pairs, best first.
    pub fn bm25_search(&self, query: &str, k: usize) -> Result<Vec<(Uuid, f32)>> {
        self.lexical.search(query, k)
    }

    pub fn vector_len(&self) -> usize {
        self.vector.len()
    }

    pub fn chunk_vector_len(&self) -> usize {
        self.chunks.len()
    }

    pub fn lexical_len(&self) -> u64 {
        self.lexical.num_docs()
    }

    pub fn dir(&self) -> &Path {
        &self.dir
    }

    /// Index counts must match the store's indexable rows; drift means a
    /// missed outbox row or lost index file and calls for `rebuild`.
    pub fn consistent_with(&self, store: &SqliteStore) -> Result<bool> {
        let indexable = store.indexable_count()?;
        let embedded = store.vector_indexable_count(EMBEDDING_DIM)?;
        let embedded_chunks = store.embedded_chunk_count(EMBEDDING_DIM)?;
        Ok(self.vector.len() == embedded
            && self.chunks.len() == embedded_chunks
            && self.lexical.num_docs() as usize == indexable)
    }

    pub fn persist(&mut self) -> Result<()> {
        self.require_writable()?;
        self.lexical.commit()?;
        self.vector.save()?;
        self.chunks.save()
    }

    /// Rewrites the lexical index into fresh segments so text removed from
    /// it no longer sits in its files; the vector files hold no text.
    pub fn compact(&mut self) -> Result<()> {
        self.require_writable()?;
        self.lexical.compact()
    }

    /// Re-opens on-disk state so a `ReadOnly` handle observes what a writer
    /// elsewhere has since persisted. A no-op on `ReadWrite`: see
    /// `VectorIndex::reload_view` for why a mutable handle has nothing to
    /// pick up from disk.
    pub fn reload_view(&mut self) -> Result<()> {
        self.vector.reload_view()?;
        if self.dir.join(CHUNK_FILE).exists() {
            self.chunks.reload_view()?;
        }
        self.lexical.reload_reader()
    }

    /// Bytes this tenant's indexes cost to keep open: usearch's own RAM
    /// accounting plus the on-disk footprint of the tantivy directory, which
    /// `memory_usage` never sees. Admission control budgets in bytes, not in
    /// a count of open tenants.
    pub fn resident_bytes(&self) -> usize {
        self.vector.memory_usage()
            + self.chunks.memory_usage()
            + dir_size(&self.dir.join("tantivy")) as usize
    }
}

/// A missing directory contributes zero bytes rather than erroring: for a
/// read-only handle beside an index that has never persisted anything,
/// "does not exist yet" and "zero bytes on disk" are the same fact. Entries
/// that vanish mid-walk (a concurrent writer mid-rename) are skipped for the
/// same reason `du` skips them: this is a best-effort accounting walk, not a
/// correctness-load-bearing read.
fn dir_size(dir: &Path) -> u64 {
    let Ok(entries) = std::fs::read_dir(dir) else {
        return 0;
    };
    let mut total = 0u64;
    for entry in entries.flatten() {
        let path = entry.path();
        if path.is_dir() {
            total += dir_size(&path);
        } else if let Ok(metadata) = entry.metadata() {
            total += metadata.len();
        }
    }
    total
}