second-brain-core 0.9.0

Core library for second-brain: KuzuDB graph storage, BGE embeddings, and weighted query engine
Documentation
use std::path::{Path, PathBuf};

use anyhow::{Context, Result, anyhow};
use usearch::{Index, IndexOptions, MetricKind, ScalarKind};

/// BGE-small output width; must match the FLOAT[384] the store schema fixes.
pub const EMBEDDING_DIM: usize = 384;

const INITIAL_CAPACITY: usize = 1024;

/// `ReadWrite` indexes are RAM-loaded via `load()` and mutable. `View`
/// indexes are mmapped read-only via usearch's `view()`, so opening
/// thousands of tenant indexes for search never pays the RAM-load cost.
#[derive(Clone, Copy, PartialEq, Eq)]
enum VectorMode {
    ReadWrite,
    View,
}

pub struct VectorIndex {
    index: Index,
    path: PathBuf,
    mode: VectorMode,
    scalar_kind: ScalarKind,
}

fn new_index(scalar_kind: ScalarKind) -> Result<Index> {
    let options = IndexOptions {
        dimensions: EMBEDDING_DIM,
        metric: MetricKind::Cos,
        quantization: scalar_kind,
        ..IndexOptions::default()
    };
    Index::new(&options).map_err(|e| anyhow!("creating usearch index: {e}"))
}

/// usearch reinterprets an index file's stored bytes at whatever width the
/// caller's options declare rather than refusing a mismatch, so opening an
/// I8-quantized file with F32 options would silently produce garbage
/// distances instead of an error. The on-disk header carries the true kind
/// (`Index::metadata` reads only that header), so it is checked against what
/// the caller asked for before `load`/`view` ever runs.
fn check_on_disk_kind(path: &Path, path_str: &str, expected: ScalarKind) -> Result<()> {
    let actual = read_on_disk_kind(path, path_str)?;
    if actual != expected {
        return Err(anyhow!(
            "vector index at {} was written as {:?} but is being opened as {:?}",
            path.display(),
            actual,
            expected
        ));
    }
    Ok(())
}

fn read_on_disk_kind(path: &Path, path_str: &str) -> Result<ScalarKind> {
    Ok(Index::metadata(path_str)
        .map_err(|e| anyhow!("reading usearch header at {}: {e}", path.display()))?
        .quantization)
}

impl VectorIndex {
    pub fn on_disk_kind(path: &Path) -> Result<Option<ScalarKind>> {
        if !path.exists() {
            return Ok(None);
        }
        let path_str = path.to_str().context("vector index path is not utf-8")?;
        read_on_disk_kind(path, path_str).map(Some)
    }

    /// Every on-disk index that predates the scalar-kind option was written
    /// as `F32`, which is exactly what `Indexes::open` still passes.
    pub fn open_with_kind(path: &Path, scalar_kind: ScalarKind) -> Result<Self> {
        let path_str = path.to_str().context("vector index path is not utf-8")?;
        let index = new_index(scalar_kind)?;
        if path.exists() {
            check_on_disk_kind(path, path_str, scalar_kind)?;
            index
                .load(path_str)
                .map_err(|e| anyhow!("loading usearch index from {path_str}: {e}"))?;
        } else {
            index
                .reserve(INITIAL_CAPACITY)
                .map_err(|e| anyhow!("reserving usearch capacity: {e}"))?;
        }
        Ok(Self {
            index,
            path: path.to_path_buf(),
            mode: VectorMode::ReadWrite,
            scalar_kind,
        })
    }

    /// Mmaps the index file read-only via usearch's `view()`: pages are
    /// faulted in on search instead of the whole file loading up front, so a
    /// multi-tenant reader can hold thousands of these open at once. Errors
    /// rather than creating a file, because a reader that can create a
    /// tenant's index is a reader that can corrupt one.
    pub fn open_view_with_kind(path: &Path, scalar_kind: ScalarKind) -> Result<Self> {
        if !path.exists() {
            return Err(anyhow!(
                "vector index view: {} does not exist",
                path.display()
            ));
        }
        let path_str = path.to_str().context("vector index path is not utf-8")?;
        check_on_disk_kind(path, path_str, scalar_kind)?;
        let index = new_index(scalar_kind)?;
        index
            .view(path_str)
            .map_err(|e| anyhow!("viewing usearch index at {path_str}: {e}"))?;
        Ok(Self {
            index,
            path: path.to_path_buf(),
            mode: VectorMode::View,
            scalar_kind,
        })
    }

    pub fn scalar_kind(&self) -> ScalarKind {
        self.scalar_kind
    }

    fn require_writable(&self) -> Result<()> {
        match self.mode {
            VectorMode::ReadWrite => Ok(()),
            VectorMode::View => Err(anyhow!(
                "vector index at {} is a read-only view; mutation is not permitted",
                self.path.display()
            )),
        }
    }

    /// Replace-then-add so a replayed outbox row cannot duplicate a key.
    pub fn upsert(&mut self, key: u64, embedding: &[f32]) -> Result<()> {
        self.require_writable()?;
        self.ensure_capacity(1)?;
        self.index
            .remove(key)
            .map_err(|e| anyhow!("removing key {key}: {e}"))?;
        self.index
            .add(key, embedding)
            .map_err(|e| anyhow!("adding key {key}: {e}"))?;
        Ok(())
    }

    pub fn remove(&mut self, key: u64) -> Result<()> {
        self.require_writable()?;
        self.index
            .remove(key)
            .map_err(|e| anyhow!("removing key {key}: {e}"))?;
        Ok(())
    }

    /// Returns (key, cosine similarity) pairs, best first.
    pub fn search(&self, query: &[f32], k: usize) -> Result<Vec<(u64, f32)>> {
        let matches = self
            .index
            .search(query, k)
            .map_err(|e| anyhow!("usearch search: {e}"))?;
        Ok(matches
            .keys
            .iter()
            .zip(matches.distances.iter())
            .map(|(key, distance)| (*key, 1.0 - *distance))
            .collect())
    }

    pub fn len(&self) -> usize {
        self.index.size()
    }

    /// A relatively accurate lower bound on usearch's own RAM usage. A
    /// `View` index reports far less than the same data `ReadWrite`-loaded,
    /// since its vectors and graph stay mmapped rather than resident.
    pub fn memory_usage(&self) -> usize {
        self.index.memory_usage()
    }

    /// Writes to a sibling `.tmp` file, fsyncs it, then renames it onto the
    /// live path. Rename is atomic on the same filesystem, so a `View`
    /// reader with the old file already mmapped never observes a
    /// half-written index: it keeps its existing mapping until
    /// `reload_view` re-views the (now new) file whole.
    pub fn save(&self) -> Result<()> {
        self.require_writable()?;
        let tmp_path = self.path.with_extension("usearch.tmp");
        let tmp_str = tmp_path
            .to_str()
            .context("vector index tmp path is not utf-8")?;
        self.index
            .save(tmp_str)
            .map_err(|e| anyhow!("saving usearch index to {tmp_str}: {e}"))?;
        let tmp_file = std::fs::File::open(&tmp_path)
            .with_context(|| format!("reopening {} to fsync", tmp_path.display()))?;
        tmp_file
            .sync_all()
            .with_context(|| format!("fsyncing {}", tmp_path.display()))?;
        drop(tmp_file);
        std::fs::rename(&tmp_path, &self.path).with_context(|| {
            format!("renaming {} to {}", tmp_path.display(), self.path.display())
        })?;
        Ok(())
    }

    /// Re-mmaps the live file so a `View` index picks up whatever a writer
    /// last saved. A no-op on `ReadWrite`: that index already holds the
    /// authoritative in-memory copy, so there is nothing on disk to pick up.
    pub fn reload_view(&mut self) -> Result<()> {
        if self.mode != VectorMode::View {
            return Ok(());
        }
        let path_str = self
            .path
            .to_str()
            .context("vector index path is not utf-8")?;
        self.index
            .view(path_str)
            .map_err(|e| anyhow!("reloading usearch view at {path_str}: {e}"))?;
        Ok(())
    }

    /// Drop every vector; used by full rebuild.
    pub fn reset(&mut self) -> Result<()> {
        self.require_writable()?;
        self.index = new_index(self.scalar_kind)?;
        self.index
            .reserve(INITIAL_CAPACITY)
            .map_err(|e| anyhow!("reserving usearch capacity: {e}"))?;
        Ok(())
    }

    fn ensure_capacity(&self, extra: usize) -> Result<()> {
        let needed = self.index.size() + extra;
        if needed > self.index.capacity() {
            self.index
                .reserve((needed * 2).max(INITIAL_CAPACITY))
                .map_err(|e| anyhow!("growing usearch capacity: {e}"))?;
        }
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use tempfile::tempdir;

    fn unit_vector(hot: usize) -> Vec<f32> {
        let mut v = vec![0.0f32; EMBEDDING_DIM];
        v[hot] = 1.0;
        v
    }

    #[test]
    fn f32_index_writes_and_reads_back() {
        let dir = tempdir().unwrap();
        let path = dir.path().join("vectors.usearch");

        let mut idx = VectorIndex::open_with_kind(&path, ScalarKind::F32).unwrap();
        assert_eq!(idx.scalar_kind(), ScalarKind::F32);
        for i in 0..5u64 {
            idx.upsert(i, &unit_vector(i as usize)).unwrap();
        }
        idx.save().unwrap();

        let reopened = VectorIndex::open_with_kind(&path, ScalarKind::F32).unwrap();
        assert_eq!(reopened.scalar_kind(), ScalarKind::F32);
        let hits = reopened.search(&unit_vector(2), 1).unwrap();
        assert_eq!(hits[0].0, 2);

        let viewed = VectorIndex::open_view_with_kind(&path, ScalarKind::F32).unwrap();
        let hits = viewed.search(&unit_vector(3), 1).unwrap();
        assert_eq!(hits[0].0, 3);
    }

    #[test]
    fn opening_f32_file_as_a_different_kind_errors_instead_of_corrupting() {
        let dir = tempdir().unwrap();
        let path = dir.path().join("vectors.usearch");

        let mut idx = VectorIndex::open_with_kind(&path, ScalarKind::F32).unwrap();
        idx.upsert(0, &unit_vector(0)).unwrap();
        idx.save().unwrap();

        let err = VectorIndex::open_with_kind(&path, ScalarKind::I8)
            .err()
            .expect("opening an F32 file as I8 must error");
        assert!(err.to_string().contains("F32"), "error was: {err}");

        let err = VectorIndex::open_view_with_kind(&path, ScalarKind::F16)
            .err()
            .expect("viewing an F32 file as F16 must error");
        assert!(err.to_string().contains("F32"), "error was: {err}");
    }

    #[test]
    fn i8_and_f16_kinds_round_trip() {
        for kind in [ScalarKind::I8, ScalarKind::F16] {
            let dir = tempdir().unwrap();
            let path = dir.path().join("vectors.usearch");

            let mut idx = VectorIndex::open_with_kind(&path, kind).unwrap();
            for i in 0..5u64 {
                idx.upsert(i, &unit_vector(i as usize)).unwrap();
            }
            idx.save().unwrap();

            let viewed = VectorIndex::open_view_with_kind(&path, kind).unwrap();
            let hits = viewed.search(&unit_vector(4), 1).unwrap();
            assert_eq!(hits[0].0, 4, "kind {kind:?} lost the nearest match");
        }
    }
}