second-brain-core 0.13.3

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, mpsc};
use std::time::Duration;

use super::*;
use crate::keys::KeyGuard;
use crate::schema::{Memory, MemoryType};
use crate::sealed::{SealError, SealKey, SealKeyId, SealKeys, seal_error, sealed_key_id};
use crate::store::Store;

const MARKER: &str = "zanzibarquokka";

fn key(byte: u8) -> SealKey {
    SealKey::from_bytes(KeyGuard::new([byte; 32])).unwrap()
}

fn keys(byte: u8) -> SealKeys {
    SealKeys::single(key(byte))
}

fn store_in(dir: &Path) -> SqliteStore {
    SqliteStore::open(&dir.join("brain.db"), "test-machine".to_string()).unwrap()
}

fn memory(i: usize) -> Memory {
    let content = format!("{} note {i}", format!("{MARKER} ").repeat(i + 1));
    let mut m = Memory::new(
        content,
        MemoryType::Semantic,
        "test".to_string(),
        String::new(),
    );
    let mut e = vec![0.0; EMBEDDING_DIM];
    e[i % EMBEDDING_DIM] = 1.0;
    e[EMBEDDING_DIM - 1] = 0.25;
    m.embedding = e;
    m
}

fn seed(store: &SqliteStore, range: std::ops::Range<usize>) -> Vec<Memory> {
    range
        .map(|i| {
            let m = memory(i);
            store.store_memory(&m).unwrap();
            m
        })
        .collect()
}

fn query(hot: usize) -> Vec<f32> {
    let mut e = vec![0.0; EMBEDDING_DIM];
    e[hot] = 1.0;
    e
}

type TopK = (Vec<Uuid>, Vec<Uuid>);

fn top_k(idx: &Indexes, store: &SqliteStore) -> TopK {
    let vector = idx
        .vector_search(store, &query(2), 5)
        .unwrap()
        .into_iter()
        .map(|(id, _)| id)
        .collect();
    let lexical = idx
        .bm25_search(MARKER, 5)
        .unwrap()
        .into_iter()
        .map(|(id, _)| id)
        .collect();
    (vector, lexical)
}

fn index_files(dir: &Path) -> Vec<PathBuf> {
    let mut out = Vec::new();
    for entry in std::fs::read_dir(dir).unwrap() {
        let path = entry.unwrap().path();
        let name = path.file_name().unwrap().to_str().unwrap().to_string();
        if path.is_dir() {
            out.extend(index_files(&path));
        } else if !name.ends_with(".lock") {
            out.push(path);
        }
    }
    out
}

fn key_ids(dir: &Path) -> Vec<(PathBuf, Option<SealKeyId>)> {
    index_files(dir)
        .into_iter()
        .map(|p| {
            let id = sealed_key_id(&p).unwrap();
            (p, id)
        })
        .collect()
}

fn assert_all_sealed_under(dir: &Path, id: SealKeyId) {
    let files = key_ids(dir);
    assert!(files.len() >= 4, "too few index files: {files:?}");
    for (path, sealed_under) in files {
        assert_eq!(
            sealed_under,
            Some(id),
            "{} is not sealed under the current key",
            path.display()
        );
    }
}

fn drained_plain(dir: &Path, store: &SqliteStore) -> TopK {
    let mut idx = Indexes::open(dir).unwrap();
    idx.drain(store).unwrap();
    let before = top_k(&idx, store);
    idx.wait_for_merges().unwrap();
    before
}

#[test]
fn sealed_indexes_keep_the_recall_top_k() {
    let dir = tempfile::tempdir().unwrap();
    let store = store_in(dir.path());
    let memories = seed(&store, 0..12);
    let baseline = drained_plain(&dir.path().join("plain"), &store);
    assert_eq!(baseline.0.first(), Some(&memories[2].id));
    assert_eq!(baseline.1.len(), 5);

    let sealed_dir = dir.path().join("sealed");
    let mut idx = Indexes::open_keyed(&sealed_dir, &keys(1)).unwrap();
    idx.rebuild(&store).unwrap();
    assert_eq!(top_k(&idx, &store), baseline);
    idx.wait_for_merges().unwrap();
    assert_all_sealed_under(&sealed_dir, keys(1).current_id());

    let reopened = Indexes::open_keyed(&sealed_dir, &keys(1)).unwrap();
    assert_eq!(
        reopened.seal_keys().unwrap().current_id(),
        keys(1).current_id()
    );
    assert_eq!(top_k(&reopened, &store), baseline);
    let readonly =
        Indexes::open_readonly_with_kind_keyed(&sealed_dir, ScalarKind::F32, &keys(1)).unwrap();
    assert_eq!(readonly.mode(), IndexesMode::ReadOnly);
    assert_eq!(top_k(&readonly, &store), baseline);
}

#[test]
fn sealed_indexes_do_not_open_without_their_key() {
    let dir = tempfile::tempdir().unwrap();
    let store = store_in(dir.path());
    seed(&store, 0..4);
    let idx_dir = dir.path().join("idx");
    let mut idx = Indexes::open_keyed(&idx_dir, &keys(1)).unwrap();
    idx.drain(&store).unwrap();
    idx.wait_for_merges().unwrap();

    for err in [
        Indexes::open(&idx_dir).err(),
        Indexes::open_readonly(&idx_dir).err(),
        Indexes::on_disk_scalar_kind(&idx_dir).err(),
        LexicalIndex::open(&idx_dir.join("tantivy")).err(),
        LexicalIndex::open_reader(&idx_dir.join("tantivy")).err(),
    ] {
        err.expect("a sealed index must not open without its key");
    }
    let err = Indexes::open(&idx_dir).err().unwrap();
    assert_eq!(seal_error(&err), Some(&SealError::KeyRequired), "{err:#}");

    for err in [
        Indexes::open_keyed(&idx_dir, &keys(2)).err(),
        Indexes::open_readonly_with_kind_keyed(&idx_dir, ScalarKind::F32, &keys(2)).err(),
    ] {
        let err = err.expect("a sealed index must not open under another key");
        assert!(
            matches!(seal_error(&err), Some(SealError::UnknownKey(_))),
            "{err:#}"
        );
    }
}

#[test]
fn no_memory_text_reaches_a_sealed_index_file() {
    let dir = tempfile::tempdir().unwrap();
    let store = store_in(dir.path());
    let memories = seed(&store, 0..8);
    drained_plain(&dir.path().join("plain"), &store);
    let mut needles: Vec<Vec<u8>> =
        vec![MARKER.as_bytes().to_vec(), MARKER.bytes().rev().collect()];
    needles.extend(memories.iter().map(|m| m.id.to_string().into_bytes()));
    let leaks = |dir: &Path| {
        index_files(dir)
            .into_iter()
            .filter(|p| {
                let bytes = std::fs::read(p).unwrap();
                needles
                    .iter()
                    .any(|n| bytes.windows(n.len()).any(|w| w == n.as_slice()))
            })
            .collect::<Vec<_>>()
    };
    assert!(
        !leaks(&dir.path().join("plain")).is_empty(),
        "no needle is found even in plaintext files, so this test proves nothing"
    );

    let sealed_dir = dir.path().join("sealed");
    let mut idx = Indexes::open_keyed(&sealed_dir, &keys(1)).unwrap();
    idx.rebuild(&store).unwrap();
    idx.compact().unwrap();
    idx.wait_for_merges().unwrap();
    assert_eq!(leaks(&sealed_dir), Vec::<PathBuf>::new());
}

#[test]
fn a_plaintext_index_opened_keyed_searches_and_reseals_whole() {
    let dir = tempfile::tempdir().unwrap();
    let store = store_in(dir.path());
    seed(&store, 0..10);
    let idx_dir = dir.path().join("idx");
    let baseline = drained_plain(&idx_dir, &store);
    std::fs::write(idx_dir.join("vectors.usearch.tmp"), b"plaintext leftover").unwrap();

    let mut idx = Indexes::open_keyed(&idx_dir, &keys(1)).unwrap();
    assert_eq!(top_k(&idx, &store), baseline);
    assert!(
        key_ids(&idx_dir).iter().any(|(_, id)| id.is_none()),
        "the adopted index should still be plaintext before reseal"
    );
    idx.reseal().unwrap();
    assert_all_sealed_under(&idx_dir, keys(1).current_id());
    assert_eq!(top_k(&idx, &store), baseline);
    drop(idx);

    let reopened = Indexes::open_keyed(&idx_dir, &keys(1)).unwrap();
    assert_eq!(top_k(&reopened, &store), baseline);
}

#[test]
fn a_rotation_reseals_every_file_under_the_new_key() {
    let dir = tempfile::tempdir().unwrap();
    let store = store_in(dir.path());
    seed(&store, 0..10);
    let idx_dir = dir.path().join("idx");
    let mut idx = Indexes::open_keyed(&idx_dir, &keys(1)).unwrap();
    idx.drain(&store).unwrap();
    let baseline = top_k(&idx, &store);
    idx.wait_for_merges().unwrap();

    let rotated = SealKeys::new(key(2), vec![key(1)]);
    let mut idx = Indexes::open_keyed(&idx_dir, &rotated).unwrap();
    assert_eq!(top_k(&idx, &store), baseline);
    idx.reseal().unwrap();
    assert_all_sealed_under(&idx_dir, rotated.current_id());
    seed(&store, 10..12);
    idx.drain(&store).unwrap();
    idx.wait_for_merges().unwrap();
    assert_all_sealed_under(&idx_dir, rotated.current_id());

    let reopened = Indexes::open_keyed(&idx_dir, &keys(2)).unwrap();
    assert_eq!(reopened.vector_len(), 12);
    assert_eq!(reopened.lexical_len(), 12);
}

#[test]
fn set_seal_keys_rotates_an_open_handle() {
    let dir = tempfile::tempdir().unwrap();
    let store = store_in(dir.path());
    seed(&store, 0..6);
    let idx_dir = dir.path().join("idx");
    let mut idx = Indexes::open_keyed(&idx_dir, &keys(1)).unwrap();
    idx.drain(&store).unwrap();
    let baseline = top_k(&idx, &store);

    let rotated = SealKeys::new(key(2), vec![key(1)]);
    idx.set_seal_keys(rotated.clone()).unwrap();
    idx.reseal().unwrap();
    assert_all_sealed_under(&idx_dir, rotated.current_id());
    assert_eq!(top_k(&idx, &store), baseline);
    idx.wait_for_merges().unwrap();
    let reopened = Indexes::open_keyed(&idx_dir, &keys(2)).unwrap();
    assert_eq!(top_k(&reopened, &store), baseline);

    let mut plain = Indexes::open(&dir.path().join("plain")).unwrap();
    assert!(plain.set_seal_keys(keys(1)).is_err());
    assert!(plain.reseal().is_err());
}

#[test]
fn a_keyed_readonly_handle_sees_what_the_writer_persists_after_reload() {
    let dir = tempfile::tempdir().unwrap();
    let store = store_in(dir.path());
    seed(&store, 0..3);
    let idx_dir = dir.path().join("idx");
    let mut writer = Indexes::open_keyed(&idx_dir, &keys(1)).unwrap();
    writer.drain(&store).unwrap();
    let mut reader =
        Indexes::open_readonly_with_kind_keyed(&idx_dir, ScalarKind::F32, &keys(1)).unwrap();
    assert_eq!(reader.vector_len(), 3);

    seed(&store, 3..5);
    writer.drain(&store).unwrap();
    assert_eq!(reader.vector_len(), 3);
    reader.reload_view().unwrap();
    assert_eq!(reader.vector_len(), 5);
    assert_eq!(reader.lexical_len(), 5);
    assert!(reader.drain(&store).is_err());
}

#[test]
fn a_shared_reader_sees_a_drain_after_reload_view() {
    let dir = tempfile::tempdir().unwrap();
    let store = store_in(dir.path());
    seed(&store, 0..3);
    let mut writer = Indexes::open_keyed(&dir.path().join("idx"), &keys(1)).unwrap();
    writer.drain(&store).unwrap();
    let mut reader = writer.shared_reader().unwrap();
    assert_eq!(reader.mode(), IndexesMode::ReadOnly);
    assert_eq!(reader.vector_len(), 3);
    assert_eq!(reader.lexical_len(), 3);
    assert_eq!(reader.memory().vectors, 0);
    assert!(writer.memory().vectors > 0);

    let added = seed(&store, 3..5);
    writer.drain(&store).unwrap();
    reader.reload_view().unwrap();
    assert_eq!(reader.vector_len(), 5);
    assert_eq!(reader.lexical_len(), 5);
    let hits = reader.vector_search(&store, &query(4), 1).unwrap();
    assert_eq!(hits[0].0, added[1].id);
    assert!(reader.drain(&store).is_err());
    assert!(reader.shared_reader().is_err());
}

#[test]
fn a_shared_reader_never_sees_a_half_done_rebuild() {
    const MEMORIES: usize = 300;
    let dir = tempfile::tempdir().unwrap();
    let store = store_in(dir.path());
    seed(&store, 0..MEMORIES);
    let mut writer = Indexes::open_keyed(&dir.path().join("idx"), &keys(1)).unwrap();
    writer.drain(&store).unwrap();
    let mut reader = writer.shared_reader().unwrap();

    let stop = Arc::new(AtomicBool::new(false));
    let (done_tx, done_rx) = mpsc::channel();
    let watcher = {
        let stop = Arc::clone(&stop);
        std::thread::spawn(move || {
            let mut looks = 0u64;
            let mut seen = Vec::new();
            while !stop.load(Ordering::Acquire) {
                if let Err(e) = reader.reload_view() {
                    seen.push(format!("reload failed: {e:#}"));
                    break;
                }
                let counts = (reader.vector_len(), reader.lexical_len());
                if counts != (MEMORIES, MEMORIES as u64) {
                    seen.push(format!("{counts:?}"));
                }
                looks += 1;
            }
            let _ = done_tx.send((seen, looks));
        })
    };

    for _ in 0..3 {
        assert_eq!(writer.rebuild(&store).unwrap(), MEMORIES);
    }
    stop.store(true, Ordering::Release);
    let (seen, looks) = done_rx
        .recv_timeout(Duration::from_secs(60))
        .expect("the watching thread did not stop within 60 s");
    watcher.join().unwrap();
    assert!(looks > 0);
    assert_eq!(
        seen,
        Vec::<String>::new(),
        "the reader saw a part-built index"
    );
    assert_eq!(writer.vector_len(), MEMORIES);
}

#[test]
fn only_a_handle_with_a_live_shared_reader_rebuilds_beside_it() {
    let dir = tempfile::tempdir().unwrap();
    let store = store_in(dir.path());
    seed(&store, 0..20);
    let mut writer = Indexes::open(&dir.path().join("indexes")).unwrap();
    assert!(!writer.vector.has_shared_readers());
    assert_eq!(writer.rebuild(&store).unwrap(), 20);
    let reader = writer.shared_reader().unwrap();
    assert!(writer.vector.has_shared_readers() && writer.chunks.has_shared_readers());
    assert_eq!(writer.rebuild(&store).unwrap(), 20);
    assert_eq!(reader.vector_len(), 20);
    drop(reader);
    assert!(!writer.vector.has_shared_readers());
    assert_eq!(writer.rebuild(&store).unwrap(), 20);
    assert_eq!(writer.vector_len(), 20);
}