use mnemo_engine::{KdfParams, Memory, MemoryType, Mnemo, MnemoConfig, MnemoError, Ulid};
use tempfile::tempdir;
fn cfg() -> MnemoConfig {
MnemoConfig {
dimensions: 8,
kdf: KdfParams::fast(),
format_version: 9,
..Default::default()
}
}
fn mem(i: usize) -> Memory {
let v: Vec<f32> = (0..8)
.map(|k| ((i * 7 + k * 3) % 11) as f32 + 0.5)
.collect();
Memory::new(format!("memory {i}"), MemoryType::Semantic, v)
}
fn file_len(p: &std::path::Path) -> u64 {
std::fs::metadata(p).unwrap().len()
}
#[test]
fn a_big_catalog_does_not_grow_the_wal() {
let dir = tempdir().unwrap();
let path = dir.path().join("t.mnemo");
let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
for i in 0..3000 {
db.remember(mem(i)).unwrap();
}
db.flush().unwrap();
assert_eq!(
db.stats().unwrap().wal_pages,
8,
"the WAL stayed at its default"
);
db.remember(mem(9999)).unwrap();
db.close().unwrap();
drop(db);
let mut db = Mnemo::open(&path, "pw").unwrap();
assert_eq!(db.verify().unwrap(), 3001);
}
#[test]
fn the_file_shrinks_when_its_tail_is_free() {
let dir = tempdir().unwrap();
let path = dir.path().join("t.mnemo");
let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
db.set_max_snapshots(1);
let small: Vec<Ulid> = (0..10).map(|i| db.remember(mem(i)).unwrap()).collect();
db.flush().unwrap();
let big: Vec<Ulid> = (10..4000)
.map(|i| {
let mut m = mem(i);
m.content = format!("{} {}", m.content, "b".repeat(2_000));
db.remember(m).unwrap()
})
.collect();
db.flush().unwrap();
let grown = file_len(&path);
for id in &big {
db.delete(id).unwrap();
}
db.flush().unwrap();
for i in 0..3 {
db.remember(mem(5000 + i)).unwrap();
db.flush().unwrap();
}
let shrunk = file_len(&path);
assert!(
shrunk * 2 < grown,
"the file shrank: {grown} -> {shrunk} bytes"
);
for id in &small {
db.get(id).unwrap();
}
assert_eq!(db.verify().unwrap(), 13);
db.close().unwrap();
drop(db);
let mut db = Mnemo::open(&path, "pw").unwrap();
assert_eq!(db.verify().unwrap(), 13);
}
#[test]
fn retention_bounds_pinned_space_under_churn() {
let dir = tempdir().unwrap();
let path = dir.path().join("t.mnemo");
let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
let keep: Vec<Ulid> = (0..200).map(|i| db.remember(mem(i)).unwrap()).collect();
db.flush().unwrap();
let first = db.snapshots()[0].txn_id;
for round in 0..40 {
let tmp: Vec<Ulid> = (0..200)
.map(|i| db.remember(mem(1000 + round * 200 + i)).unwrap())
.collect();
db.flush().unwrap();
for id in &tmp {
db.delete(id).unwrap();
}
db.flush().unwrap();
}
let snaps = db.snapshots();
assert!(
snaps.len() < 80,
"{} snapshots retained after 80 commits",
snaps.len()
);
let oldest = snaps[0].txn_id;
let (runs, _) = db.__free_list();
let free: u64 = runs.iter().map(|r| r.2).sum();
let pinned: u64 = runs.iter().filter(|r| r.0 > oldest).map(|r| r.2).sum();
let live = db.__next_page() - free;
assert!(
pinned <= (live / 4).max(128),
"{pinned} pinned of {live} live pages"
);
match db.restore_to(first) {
Err(MnemoError::NotFound(msg)) => assert!(msg.contains("retention window"), "{msg}"),
Err(e) => panic!("expected NotFound, got {e:?}"),
Ok(_) => panic!("txn {first} should be past the retention window"),
}
for id in &keep {
db.get(id).unwrap();
}
assert_eq!(db.verify().unwrap(), 200);
}
#[cfg(feature = "failpoints")]
#[test]
fn a_crash_during_out_of_wal_runs_is_pre_or_post() {
let dir = tempdir().unwrap();
let base = dir.path().join("base.mnemo");
{
let mut db = Mnemo::create(&base, "pw", cfg()).unwrap();
for i in 0..3000 {
db.remember(mem(i)).unwrap();
}
db.close().unwrap();
}
let state = |p: &std::path::Path| {
let mut db = Mnemo::open(p, "pw").unwrap();
(db.len(), db.verify().unwrap())
};
let ops = |db: &mut Mnemo| -> mnemo_engine::Result<()> {
for i in 0..50 {
db.remember(mem(10_000 + i))?;
}
db.flush()
};
let pre = state(&base);
let mut saw_pre = false;
let mut saw_post = false;
for fail_at in (0..90).step_by(3) {
let p = dir.path().join(format!("c{fail_at}.mnemo"));
std::fs::copy(&base, &p).unwrap();
let mut db = Mnemo::open(&p, "pw").unwrap();
mnemo_engine::__failpoints::set_writes_until_fail(fail_at);
let res = ops(&mut db);
mnemo_engine::__failpoints::set_writes_until_fail(-1);
drop(db);
let got = state(&p);
assert!(
got == pre || got == (pre.0 + 50, pre.1 + 50),
"fail_at {fail_at}: {got:?}"
);
if got == pre {
saw_pre = true;
} else {
saw_post = true;
}
if res.is_ok() {
break;
}
}
assert!(saw_pre && saw_post);
}
#[test]
fn small_commits_append_catalog_deltas_and_reopen() {
let dir = tempdir().unwrap();
let path = dir.path().join("t.mnemo");
let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
for i in 0..1000 {
db.remember(mem(i)).unwrap();
}
db.flush().unwrap();
let (_, chain, base) = db.__catalog_shape();
assert_eq!(chain, 0, "the first commit writes a base");
let ids: Vec<Ulid> = (0..20)
.map(|i| {
let id = db.remember(mem(2000 + i)).unwrap();
db.flush().unwrap();
id
})
.collect();
let (rows, chain, base_after) = db.__catalog_shape();
assert_eq!(
(rows, chain, base_after),
(1020, 20, base),
"one delta per commit"
);
db.close().unwrap();
drop(db);
let mut db = Mnemo::open(&path, "pw").unwrap();
assert_eq!(db.__catalog_shape().0, 1020);
for (i, id) in ids.iter().enumerate() {
assert_eq!(db.get(id).unwrap().content, format!("memory {}", 2000 + i));
}
assert_eq!(db.verify().unwrap(), 1020);
}
#[test]
fn the_chain_folds_into_a_new_base() {
let dir = tempdir().unwrap();
let path = dir.path().join("t.mnemo");
let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
for i in 0..100 {
db.remember(mem(i)).unwrap();
}
db.flush().unwrap();
for i in 0..40 {
db.remember(mem(500 + i)).unwrap();
db.flush().unwrap();
}
let (rows, chain, _) = db.__catalog_shape();
assert_eq!(rows, 140);
assert!(chain < 40, "the chain was folded ({chain} deltas)");
db.close().unwrap();
drop(db);
let mut db = Mnemo::open(&path, "pw").unwrap();
assert_eq!(db.verify().unwrap(), 140);
}
#[test]
fn deleted_rows_are_dropped_from_a_new_base() {
let dir = tempdir().unwrap();
let path = dir.path().join("t.mnemo");
let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
let ids: Vec<Ulid> = (0..100).map(|i| db.remember(mem(i)).unwrap()).collect();
db.flush().unwrap();
for id in &ids[..60] {
db.delete(id).unwrap();
}
db.flush().unwrap();
assert_eq!(
db.__catalog_shape().0,
40,
"tombstones dropped with the base"
);
for (i, id) in ids.iter().enumerate().skip(60) {
assert_eq!(db.get(id).unwrap().content, format!("memory {i}"));
}
assert!(db.get(&ids[0]).is_err());
db.close().unwrap();
drop(db);
let mut db = Mnemo::open(&path, "pw").unwrap();
assert_eq!(db.len(), 40);
assert_eq!(db.verify().unwrap(), 40);
let hits = db
.recall(
&mnemo_engine::RecallRequest::new(mem(70).vector)
.top_k(1)
.track_access(false),
)
.unwrap();
assert_eq!(hits[0].memory.id, ids[70]);
}
#[test]
fn restore_replays_the_snapshots_delta_chain() {
let dir = tempdir().unwrap();
let path = dir.path().join("t.mnemo");
let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
for i in 0..2000 {
db.remember(mem(i)).unwrap();
}
db.flush().unwrap();
let mut marks = Vec::new();
for i in 0..5 {
db.remember(mem(1000 + i)).unwrap();
db.flush().unwrap();
marks.push(db.snapshots().last().unwrap().txn_id);
}
assert!(db.__catalog_shape().1 >= 5);
db.restore_to(marks[2]).unwrap();
assert_eq!(db.len(), 2003);
assert_eq!(db.__catalog_shape().1, 0, "a restore writes a base");
db.close().unwrap();
drop(db);
let mut db = Mnemo::open(&path, "pw").unwrap();
assert_eq!(db.len(), 2003);
assert_eq!(db.verify().unwrap(), 2003);
}
#[test]
fn index_changes_persist_as_deltas_every_commit() {
std::env::set_var("MNEMO_DEV_ANN_MIN_FACTS", "0");
let dir = tempdir().unwrap();
let path = dir.path().join("t.mnemo");
let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
for i in 0..2000 {
db.remember(mem(i)).unwrap();
}
db.flush().unwrap();
let info = db.build_index().unwrap();
assert!(info.partitions > 0);
db.flush().unwrap();
let (chain, base) = db.__index_shape();
assert_eq!(chain, 0);
let fresh: Vec<Ulid> = (0..5)
.map(|i| {
let id = db.remember(mem(100_000 + i)).unwrap();
db.flush().unwrap();
id
})
.collect();
assert_eq!(db.__index_shape(), (5, base), "one index delta per commit");
drop(db);
let mut db = Mnemo::open(&path, "pw").unwrap();
for (i, id) in fresh.iter().enumerate() {
let q = mem(100_000 + i).vector;
let shortlist = db.__ann_shortlist(&q, Some(1_000), Some(4_000)).unwrap();
assert!(shortlist.contains(id), "delta {i} replayed into the index");
}
assert_eq!(db.__index_shape().0, 5);
assert_eq!(db.verify().unwrap(), 2005);
}