mnemo-engine 0.12.3

Encrypted, single-file, portable agent memory engine with multi-signal recall, sessions, snapshots, and an IVF+PQ index
Documentation
//! FMT-9c: big runs written outside the WAL, the file shrinking from its
//! tail, and snapshot retention. Every test asks for v9 explicitly.

use mnemo_engine::{KdfParams, Memory, MemoryType, Mnemo, MnemoConfig, MnemoError, Ulid};
use tempfile::tempdir;

fn cfg() -> MnemoConfig {
    MnemoConfig {
        dimensions: 8,
        kdf: KdfParams::fast(),
        format_version: 9,
        ..Default::default()
    }
}

fn mem(i: usize) -> Memory {
    let v: Vec<f32> = (0..8)
        .map(|k| ((i * 7 + k * 3) % 11) as f32 + 0.5)
        .collect();
    Memory::new(format!("memory {i}"), MemoryType::Semantic, v)
}

fn file_len(p: &std::path::Path) -> u64 {
    std::fs::metadata(p).unwrap().len()
}

#[test]
fn a_big_catalog_does_not_grow_the_wal() {
    let dir = tempdir().unwrap();
    let path = dir.path().join("t.mnemo");
    let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
    for i in 0..3000 {
        db.remember(mem(i)).unwrap();
    }
    db.flush().unwrap();
    // The catalog is tens of pages; it went straight to its home pages.
    assert_eq!(
        db.stats().unwrap().wal_pages,
        8,
        "the WAL stayed at its default"
    );
    db.remember(mem(9999)).unwrap();
    db.close().unwrap();
    let mut db = Mnemo::open(&path, "pw").unwrap();
    assert_eq!(db.verify().unwrap(), 3001);
}

#[test]
fn the_file_shrinks_when_its_tail_is_free() {
    let dir = tempdir().unwrap();
    let path = dir.path().join("t.mnemo");
    let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
    db.set_max_snapshots(1);
    let small: Vec<Ulid> = (0..10).map(|i| db.remember(mem(i)).unwrap()).collect();
    db.flush().unwrap();
    // 2 KB records: their pages dwarf the catalog (whose tombstones stay).
    let big: Vec<Ulid> = (10..4000)
        .map(|i| {
            let mut m = mem(i);
            m.content = format!("{} {}", m.content, "b".repeat(2_000));
            db.remember(m).unwrap()
        })
        .collect();
    db.flush().unwrap();
    let grown = file_len(&path);
    for id in &big {
        db.delete(id).unwrap();
    }
    db.flush().unwrap();
    // Two more small commits let the frees clear quarantine and the tail
    // come off.
    for i in 0..3 {
        db.remember(mem(5000 + i)).unwrap();
        db.flush().unwrap();
    }
    let shrunk = file_len(&path);
    assert!(
        shrunk * 2 < grown,
        "the file shrank: {grown} -> {shrunk} bytes"
    );
    for id in &small {
        db.get(id).unwrap();
    }
    assert_eq!(db.verify().unwrap(), 13);
    db.close().unwrap();
    let mut db = Mnemo::open(&path, "pw").unwrap();
    assert_eq!(db.verify().unwrap(), 13);
}

#[test]
fn retention_bounds_pinned_space_under_churn() {
    let dir = tempdir().unwrap();
    let path = dir.path().join("t.mnemo");
    let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
    let keep: Vec<Ulid> = (0..200).map(|i| db.remember(mem(i)).unwrap()).collect();
    db.flush().unwrap();
    let first = db.snapshots()[0].txn_id;
    // Churn: every round adds and then deletes 200 records. Snapshots are
    // all younger than 24 hours, so only the 25% pin cap prunes them.
    for round in 0..40 {
        let tmp: Vec<Ulid> = (0..200)
            .map(|i| db.remember(mem(1000 + round * 200 + i)).unwrap())
            .collect();
        db.flush().unwrap();
        for id in &tmp {
            db.delete(id).unwrap();
        }
        db.flush().unwrap();
    }
    let snaps = db.snapshots();
    assert!(
        snaps.len() < 80,
        "{} snapshots retained after 80 commits",
        snaps.len()
    );
    // What the retained snapshots keep from reuse stays within the cap: a
    // quarter of the live pages, or 128 pages for a small file.
    let oldest = snaps[0].txn_id;
    let (runs, _) = db.__free_list();
    let free: u64 = runs.iter().map(|r| r.2).sum();
    let pinned: u64 = runs.iter().filter(|r| r.0 > oldest).map(|r| r.2).sum();
    let live = db.__next_page() - free;
    assert!(
        pinned <= (live / 4).max(128),
        "{pinned} pinned of {live} live pages"
    );
    // Restoring past the window is a clear error.
    match db.restore_to(first) {
        Err(MnemoError::NotFound(msg)) => assert!(msg.contains("retention window"), "{msg}"),
        Err(e) => panic!("expected NotFound, got {e:?}"),
        Ok(_) => panic!("txn {first} should be past the retention window"),
    }
    for id in &keep {
        db.get(id).unwrap();
    }
    assert_eq!(db.verify().unwrap(), 200);
}

/// The big runs are written before the commit, outside the WAL: a crash at
/// any write leaves the pre- or the post-commit state.
#[cfg(feature = "failpoints")]
#[test]
fn a_crash_during_out_of_wal_runs_is_pre_or_post() {
    let dir = tempdir().unwrap();
    let base = dir.path().join("base.mnemo");
    {
        let mut db = Mnemo::create(&base, "pw", cfg()).unwrap();
        for i in 0..3000 {
            db.remember(mem(i)).unwrap();
        }
        db.close().unwrap();
    }
    let state = |p: &std::path::Path| {
        let mut db = Mnemo::open(p, "pw").unwrap();
        (db.len(), db.verify().unwrap())
    };
    let ops = |db: &mut Mnemo| -> mnemo_engine::Result<()> {
        for i in 0..50 {
            db.remember(mem(10_000 + i))?;
        }
        db.flush()
    };
    let pre = state(&base);
    let mut saw_pre = false;
    let mut saw_post = false;
    for fail_at in (0..90).step_by(3) {
        let p = dir.path().join(format!("c{fail_at}.mnemo"));
        std::fs::copy(&base, &p).unwrap();
        let mut db = Mnemo::open(&p, "pw").unwrap();
        mnemo_engine::__failpoints::set_writes_until_fail(fail_at);
        let res = ops(&mut db);
        mnemo_engine::__failpoints::set_writes_until_fail(-1);
        drop(db);
        let got = state(&p);
        assert!(
            got == pre || got == (pre.0 + 50, pre.1 + 50),
            "fail_at {fail_at}: {got:?}"
        );
        if got == pre {
            saw_pre = true;
        } else {
            saw_post = true;
        }
        if res.is_ok() {
            break;
        }
    }
    assert!(saw_pre && saw_post);
}

// --- FMT-9c.2: catalog deltas ----------------------------------------------

#[test]
fn small_commits_append_catalog_deltas_and_reopen() {
    let dir = tempdir().unwrap();
    let path = dir.path().join("t.mnemo");
    let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
    for i in 0..1000 {
        db.remember(mem(i)).unwrap();
    }
    db.flush().unwrap();
    let (_, chain, base) = db.__catalog_shape();
    assert_eq!(chain, 0, "the first commit writes a base");
    let ids: Vec<Ulid> = (0..20)
        .map(|i| {
            let id = db.remember(mem(2000 + i)).unwrap();
            db.flush().unwrap();
            id
        })
        .collect();
    let (rows, chain, base_after) = db.__catalog_shape();
    assert_eq!(
        (rows, chain, base_after),
        (1020, 20, base),
        "one delta per commit"
    );
    db.close().unwrap();
    let mut db = Mnemo::open(&path, "pw").unwrap();
    assert_eq!(db.__catalog_shape().0, 1020);
    for (i, id) in ids.iter().enumerate() {
        assert_eq!(db.get(id).unwrap().content, format!("memory {}", 2000 + i));
    }
    assert_eq!(db.verify().unwrap(), 1020);
}

#[test]
fn the_chain_folds_into_a_new_base() {
    let dir = tempdir().unwrap();
    let path = dir.path().join("t.mnemo");
    let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
    for i in 0..100 {
        db.remember(mem(i)).unwrap();
    }
    db.flush().unwrap();
    for i in 0..40 {
        db.remember(mem(500 + i)).unwrap();
        db.flush().unwrap();
    }
    let (rows, chain, _) = db.__catalog_shape();
    assert_eq!(rows, 140);
    assert!(chain < 40, "the chain was folded ({chain} deltas)");
    db.close().unwrap();
    let mut db = Mnemo::open(&path, "pw").unwrap();
    assert_eq!(db.verify().unwrap(), 140);
}

#[test]
fn deleted_rows_are_dropped_from_a_new_base() {
    let dir = tempdir().unwrap();
    let path = dir.path().join("t.mnemo");
    let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
    let ids: Vec<Ulid> = (0..100).map(|i| db.remember(mem(i)).unwrap()).collect();
    db.flush().unwrap();
    for id in &ids[..60] {
        db.delete(id).unwrap();
    }
    db.flush().unwrap();
    assert_eq!(
        db.__catalog_shape().0,
        40,
        "tombstones dropped with the base"
    );
    for (i, id) in ids.iter().enumerate().skip(60) {
        assert_eq!(db.get(id).unwrap().content, format!("memory {i}"));
    }
    assert!(db.get(&ids[0]).is_err());
    db.close().unwrap();
    let mut db = Mnemo::open(&path, "pw").unwrap();
    assert_eq!(db.len(), 40);
    assert_eq!(db.verify().unwrap(), 40);
    let hits = db
        .recall(
            &mnemo_engine::RecallRequest::new(mem(70).vector)
                .top_k(1)
                .track_access(false),
        )
        .unwrap();
    assert_eq!(hits[0].memory.id, ids[70]);
}

#[test]
fn restore_replays_the_snapshots_delta_chain() {
    let dir = tempdir().unwrap();
    let path = dir.path().join("t.mnemo");
    let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
    // Big enough that five small commits (each also moving the rows of a
    // partly filled page) stay under the fold threshold.
    for i in 0..2000 {
        db.remember(mem(i)).unwrap();
    }
    db.flush().unwrap();
    let mut marks = Vec::new();
    for i in 0..5 {
        db.remember(mem(1000 + i)).unwrap();
        db.flush().unwrap();
        marks.push(db.snapshots().last().unwrap().txn_id);
    }
    assert!(db.__catalog_shape().1 >= 5);
    db.restore_to(marks[2]).unwrap();
    assert_eq!(db.len(), 2003);
    assert_eq!(db.__catalog_shape().1, 0, "a restore writes a base");
    db.close().unwrap();
    let mut db = Mnemo::open(&path, "pw").unwrap();
    assert_eq!(db.len(), 2003);
    assert_eq!(db.verify().unwrap(), 2003);
}

// --- FMT-9c.3: index deltas ------------------------------------------------

#[test]
fn index_changes_persist_as_deltas_every_commit() {
    // Build an index on a small file (the dev override lowers the 20K
    // threshold; no other test in this binary builds an index).
    std::env::set_var("MNEMO_DEV_ANN_MIN_FACTS", "0");
    let dir = tempdir().unwrap();
    let path = dir.path().join("t.mnemo");
    let mut db = Mnemo::create(&path, "pw", cfg()).unwrap();
    for i in 0..2000 {
        db.remember(mem(i)).unwrap();
    }
    db.flush().unwrap();
    let info = db.build_index().unwrap();
    assert!(info.partitions > 0);
    db.flush().unwrap();
    let (chain, base) = db.__index_shape();
    assert_eq!(chain, 0);
    let fresh: Vec<Ulid> = (0..5)
        .map(|i| {
            let id = db.remember(mem(100_000 + i)).unwrap();
            db.flush().unwrap();
            id
        })
        .collect();
    assert_eq!(db.__index_shape(), (5, base), "one index delta per commit");
    // Simulate a crash: no close, so nothing beyond the commits is written.
    drop(db);
    let mut db = Mnemo::open(&path, "pw").unwrap();
    for (i, id) in fresh.iter().enumerate() {
        let q = mem(100_000 + i).vector;
        let shortlist = db.__ann_shortlist(&q, Some(1_000), Some(4_000)).unwrap();
        assert!(shortlist.contains(id), "delta {i} replayed into the index");
    }
    assert_eq!(db.__index_shape().0, 5);
    assert_eq!(db.verify().unwrap(), 2005);
}