use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, mpsc};
use std::time::Duration;
use super::*;
use crate::keys::KeyGuard;
use crate::schema::{Memory, MemoryType};
use crate::sealed::{SealError, SealKey, SealKeyId, SealKeys, seal_error, sealed_key_id};
use crate::store::Store;
const MARKER: &str = "zanzibarquokka";
fn key(byte: u8) -> SealKey {
SealKey::from_bytes(KeyGuard::new([byte; 32])).unwrap()
}
fn keys(byte: u8) -> SealKeys {
SealKeys::single(key(byte))
}
fn store_in(dir: &Path) -> SqliteStore {
SqliteStore::open(&dir.join("brain.db"), "test-machine".to_string()).unwrap()
}
fn memory(i: usize) -> Memory {
let content = format!("{} note {i}", format!("{MARKER} ").repeat(i + 1));
let mut m = Memory::new(
content,
MemoryType::Semantic,
"test".to_string(),
String::new(),
);
let mut e = vec![0.0; EMBEDDING_DIM];
e[i % EMBEDDING_DIM] = 1.0;
e[EMBEDDING_DIM - 1] = 0.25;
m.embedding = e;
m
}
fn seed(store: &SqliteStore, range: std::ops::Range<usize>) -> Vec<Memory> {
range
.map(|i| {
let m = memory(i);
store.store_memory(&m).unwrap();
m
})
.collect()
}
fn query(hot: usize) -> Vec<f32> {
let mut e = vec![0.0; EMBEDDING_DIM];
e[hot] = 1.0;
e
}
type TopK = (Vec<Uuid>, Vec<Uuid>);
fn top_k(idx: &Indexes, store: &SqliteStore) -> TopK {
let vector = idx
.vector_search(store, &query(2), 5)
.unwrap()
.into_iter()
.map(|(id, _)| id)
.collect();
let lexical = idx
.bm25_search(MARKER, 5)
.unwrap()
.into_iter()
.map(|(id, _)| id)
.collect();
(vector, lexical)
}
fn index_files(dir: &Path) -> Vec<PathBuf> {
let mut out = Vec::new();
for entry in std::fs::read_dir(dir).unwrap() {
let path = entry.unwrap().path();
let name = path.file_name().unwrap().to_str().unwrap().to_string();
if path.is_dir() {
out.extend(index_files(&path));
} else if !name.ends_with(".lock") {
out.push(path);
}
}
out
}
fn key_ids(dir: &Path) -> Vec<(PathBuf, Option<SealKeyId>)> {
index_files(dir)
.into_iter()
.map(|p| {
let id = sealed_key_id(&p).unwrap();
(p, id)
})
.collect()
}
fn assert_all_sealed_under(dir: &Path, id: SealKeyId) {
let files = key_ids(dir);
assert!(files.len() >= 4, "too few index files: {files:?}");
for (path, sealed_under) in files {
assert_eq!(
sealed_under,
Some(id),
"{} is not sealed under the current key",
path.display()
);
}
}
fn drained_plain(dir: &Path, store: &SqliteStore) -> TopK {
let mut idx = Indexes::open(dir).unwrap();
idx.drain(store).unwrap();
let before = top_k(&idx, store);
idx.wait_for_merges().unwrap();
before
}
#[test]
fn sealed_indexes_keep_the_recall_top_k() {
let dir = tempfile::tempdir().unwrap();
let store = store_in(dir.path());
let memories = seed(&store, 0..12);
let baseline = drained_plain(&dir.path().join("plain"), &store);
assert_eq!(baseline.0.first(), Some(&memories[2].id));
assert_eq!(baseline.1.len(), 5);
let sealed_dir = dir.path().join("sealed");
let mut idx = Indexes::open_keyed(&sealed_dir, &keys(1)).unwrap();
idx.rebuild(&store).unwrap();
assert_eq!(top_k(&idx, &store), baseline);
idx.wait_for_merges().unwrap();
assert_all_sealed_under(&sealed_dir, keys(1).current_id());
let reopened = Indexes::open_keyed(&sealed_dir, &keys(1)).unwrap();
assert_eq!(
reopened.seal_keys().unwrap().current_id(),
keys(1).current_id()
);
assert_eq!(top_k(&reopened, &store), baseline);
let readonly =
Indexes::open_readonly_with_kind_keyed(&sealed_dir, ScalarKind::F32, &keys(1)).unwrap();
assert_eq!(readonly.mode(), IndexesMode::ReadOnly);
assert_eq!(top_k(&readonly, &store), baseline);
}
#[test]
fn sealed_indexes_do_not_open_without_their_key() {
let dir = tempfile::tempdir().unwrap();
let store = store_in(dir.path());
seed(&store, 0..4);
let idx_dir = dir.path().join("idx");
let mut idx = Indexes::open_keyed(&idx_dir, &keys(1)).unwrap();
idx.drain(&store).unwrap();
idx.wait_for_merges().unwrap();
for err in [
Indexes::open(&idx_dir).err(),
Indexes::open_readonly(&idx_dir).err(),
Indexes::on_disk_scalar_kind(&idx_dir).err(),
LexicalIndex::open(&idx_dir.join("tantivy")).err(),
LexicalIndex::open_reader(&idx_dir.join("tantivy")).err(),
] {
err.expect("a sealed index must not open without its key");
}
let err = Indexes::open(&idx_dir).err().unwrap();
assert_eq!(seal_error(&err), Some(&SealError::KeyRequired), "{err:#}");
for err in [
Indexes::open_keyed(&idx_dir, &keys(2)).err(),
Indexes::open_readonly_with_kind_keyed(&idx_dir, ScalarKind::F32, &keys(2)).err(),
] {
let err = err.expect("a sealed index must not open under another key");
assert!(
matches!(seal_error(&err), Some(SealError::UnknownKey(_))),
"{err:#}"
);
}
}
#[test]
fn no_memory_text_reaches_a_sealed_index_file() {
let dir = tempfile::tempdir().unwrap();
let store = store_in(dir.path());
let memories = seed(&store, 0..8);
drained_plain(&dir.path().join("plain"), &store);
let mut needles: Vec<Vec<u8>> =
vec![MARKER.as_bytes().to_vec(), MARKER.bytes().rev().collect()];
needles.extend(memories.iter().map(|m| m.id.to_string().into_bytes()));
let leaks = |dir: &Path| {
index_files(dir)
.into_iter()
.filter(|p| {
let bytes = std::fs::read(p).unwrap();
needles
.iter()
.any(|n| bytes.windows(n.len()).any(|w| w == n.as_slice()))
})
.collect::<Vec<_>>()
};
assert!(
!leaks(&dir.path().join("plain")).is_empty(),
"no needle is found even in plaintext files, so this test proves nothing"
);
let sealed_dir = dir.path().join("sealed");
let mut idx = Indexes::open_keyed(&sealed_dir, &keys(1)).unwrap();
idx.rebuild(&store).unwrap();
idx.compact().unwrap();
idx.wait_for_merges().unwrap();
assert_eq!(leaks(&sealed_dir), Vec::<PathBuf>::new());
}
#[test]
fn a_plaintext_index_opened_keyed_searches_and_reseals_whole() {
let dir = tempfile::tempdir().unwrap();
let store = store_in(dir.path());
seed(&store, 0..10);
let idx_dir = dir.path().join("idx");
let baseline = drained_plain(&idx_dir, &store);
std::fs::write(idx_dir.join("vectors.usearch.tmp"), b"plaintext leftover").unwrap();
let mut idx = Indexes::open_keyed(&idx_dir, &keys(1)).unwrap();
assert_eq!(top_k(&idx, &store), baseline);
assert!(
key_ids(&idx_dir).iter().any(|(_, id)| id.is_none()),
"the adopted index should still be plaintext before reseal"
);
idx.reseal().unwrap();
assert_all_sealed_under(&idx_dir, keys(1).current_id());
assert_eq!(top_k(&idx, &store), baseline);
drop(idx);
let reopened = Indexes::open_keyed(&idx_dir, &keys(1)).unwrap();
assert_eq!(top_k(&reopened, &store), baseline);
}
#[test]
fn a_rotation_reseals_every_file_under_the_new_key() {
let dir = tempfile::tempdir().unwrap();
let store = store_in(dir.path());
seed(&store, 0..10);
let idx_dir = dir.path().join("idx");
let mut idx = Indexes::open_keyed(&idx_dir, &keys(1)).unwrap();
idx.drain(&store).unwrap();
let baseline = top_k(&idx, &store);
idx.wait_for_merges().unwrap();
let rotated = SealKeys::new(key(2), vec![key(1)]);
let mut idx = Indexes::open_keyed(&idx_dir, &rotated).unwrap();
assert_eq!(top_k(&idx, &store), baseline);
idx.reseal().unwrap();
assert_all_sealed_under(&idx_dir, rotated.current_id());
seed(&store, 10..12);
idx.drain(&store).unwrap();
idx.wait_for_merges().unwrap();
assert_all_sealed_under(&idx_dir, rotated.current_id());
let reopened = Indexes::open_keyed(&idx_dir, &keys(2)).unwrap();
assert_eq!(reopened.vector_len(), 12);
assert_eq!(reopened.lexical_len(), 12);
}
#[test]
fn set_seal_keys_rotates_an_open_handle() {
let dir = tempfile::tempdir().unwrap();
let store = store_in(dir.path());
seed(&store, 0..6);
let idx_dir = dir.path().join("idx");
let mut idx = Indexes::open_keyed(&idx_dir, &keys(1)).unwrap();
idx.drain(&store).unwrap();
let baseline = top_k(&idx, &store);
let rotated = SealKeys::new(key(2), vec![key(1)]);
idx.set_seal_keys(rotated.clone()).unwrap();
idx.reseal().unwrap();
assert_all_sealed_under(&idx_dir, rotated.current_id());
assert_eq!(top_k(&idx, &store), baseline);
idx.wait_for_merges().unwrap();
let reopened = Indexes::open_keyed(&idx_dir, &keys(2)).unwrap();
assert_eq!(top_k(&reopened, &store), baseline);
let mut plain = Indexes::open(&dir.path().join("plain")).unwrap();
assert!(plain.set_seal_keys(keys(1)).is_err());
assert!(plain.reseal().is_err());
}
#[test]
fn a_keyed_readonly_handle_sees_what_the_writer_persists_after_reload() {
let dir = tempfile::tempdir().unwrap();
let store = store_in(dir.path());
seed(&store, 0..3);
let idx_dir = dir.path().join("idx");
let mut writer = Indexes::open_keyed(&idx_dir, &keys(1)).unwrap();
writer.drain(&store).unwrap();
let mut reader =
Indexes::open_readonly_with_kind_keyed(&idx_dir, ScalarKind::F32, &keys(1)).unwrap();
assert_eq!(reader.vector_len(), 3);
seed(&store, 3..5);
writer.drain(&store).unwrap();
assert_eq!(reader.vector_len(), 3);
reader.reload_view().unwrap();
assert_eq!(reader.vector_len(), 5);
assert_eq!(reader.lexical_len(), 5);
assert!(reader.drain(&store).is_err());
}
#[test]
fn a_shared_reader_sees_a_drain_after_reload_view() {
let dir = tempfile::tempdir().unwrap();
let store = store_in(dir.path());
seed(&store, 0..3);
let mut writer = Indexes::open_keyed(&dir.path().join("idx"), &keys(1)).unwrap();
writer.drain(&store).unwrap();
let mut reader = writer.shared_reader().unwrap();
assert_eq!(reader.mode(), IndexesMode::ReadOnly);
assert_eq!(reader.vector_len(), 3);
assert_eq!(reader.lexical_len(), 3);
assert_eq!(reader.memory().vectors, 0);
assert!(writer.memory().vectors > 0);
let added = seed(&store, 3..5);
writer.drain(&store).unwrap();
reader.reload_view().unwrap();
assert_eq!(reader.vector_len(), 5);
assert_eq!(reader.lexical_len(), 5);
let hits = reader.vector_search(&store, &query(4), 1).unwrap();
assert_eq!(hits[0].0, added[1].id);
assert!(reader.drain(&store).is_err());
assert!(reader.shared_reader().is_err());
}
#[test]
fn a_shared_reader_never_sees_a_half_done_rebuild() {
const MEMORIES: usize = 300;
let dir = tempfile::tempdir().unwrap();
let store = store_in(dir.path());
seed(&store, 0..MEMORIES);
let mut writer = Indexes::open_keyed(&dir.path().join("idx"), &keys(1)).unwrap();
writer.drain(&store).unwrap();
let mut reader = writer.shared_reader().unwrap();
let stop = Arc::new(AtomicBool::new(false));
let (done_tx, done_rx) = mpsc::channel();
let watcher = {
let stop = Arc::clone(&stop);
std::thread::spawn(move || {
let mut looks = 0u64;
let mut seen = Vec::new();
while !stop.load(Ordering::Acquire) {
if let Err(e) = reader.reload_view() {
seen.push(format!("reload failed: {e:#}"));
break;
}
let counts = (reader.vector_len(), reader.lexical_len());
if counts != (MEMORIES, MEMORIES as u64) {
seen.push(format!("{counts:?}"));
}
looks += 1;
}
let _ = done_tx.send((seen, looks));
})
};
for _ in 0..3 {
assert_eq!(writer.rebuild(&store).unwrap(), MEMORIES);
}
stop.store(true, Ordering::Release);
let (seen, looks) = done_rx
.recv_timeout(Duration::from_secs(60))
.expect("the watching thread did not stop within 60 s");
watcher.join().unwrap();
assert!(looks > 0);
assert_eq!(
seen,
Vec::<String>::new(),
"the reader saw a part-built index"
);
assert_eq!(writer.vector_len(), MEMORIES);
}
#[test]
fn only_a_handle_with_a_live_shared_reader_rebuilds_beside_it() {
let dir = tempfile::tempdir().unwrap();
let store = store_in(dir.path());
seed(&store, 0..20);
let mut writer = Indexes::open(&dir.path().join("indexes")).unwrap();
assert!(!writer.vector.has_shared_readers());
assert_eq!(writer.rebuild(&store).unwrap(), 20);
let reader = writer.shared_reader().unwrap();
assert!(writer.vector.has_shared_readers() && writer.chunks.has_shared_readers());
assert_eq!(writer.rebuild(&store).unwrap(), 20);
assert_eq!(reader.vector_len(), 20);
drop(reader);
assert!(!writer.vector.has_shared_readers());
assert_eq!(writer.rebuild(&store).unwrap(), 20);
assert_eq!(writer.vector_len(), 20);
}