mnemo_engine/store.rs
1//! The Mnemo database — ties the storage engine and the memory model together.
2//!
3//! Layout of a populated file:
4//!
5//! ```text
6//! page 0 : Header (unencrypted)
7//! pages 1..W : write-ahead log region
8//! pages W.. : encrypted record runs + catalog runs + index +
9//! snapshot manifest (append-only)
10//! ```
11//!
12//! Durability is provided by a **write-ahead log** ([`crate::wal`]). A `flush`
13//! is one transaction: record data pages are written copy-on-write to fresh
14//! pages, then the new catalog, ANN index, and header are logged to the WAL
15//! and committed with a single fsync. A checkpoint then folds the WAL into the
16//! home pages. A crash before the commit leaves the previous state intact; a
17//! crash after it is repaired by replaying the WAL on open.
18//!
19//! Because pages are only ever appended, every past transaction's runs survive
20//! on disk. Each flush also appends an entry to a **snapshot manifest**, so
21//! [`Mnemo::restore_to`] can reinstate any past state. Stale pages — and the
22//! history along with them — are reclaimed by [`Mnemo::compact_file`].
23
24use std::fs::{File, OpenOptions};
25use std::io::{Read, Seek, SeekFrom, Write};
26use std::path::{Path, PathBuf};
27
28use fs4::fs_std::FileExt;
29use serde::{Deserialize, Serialize};
30use serde_json::{Map, Value};
31use ulid::Ulid;
32
33use crate::crypto::{self, KdfParams};
34use crate::embed::{self, Embedder};
35use crate::error::{MnemoError, Result};
36use crate::format::{
37 Header, RunPtr, FLAG_ENCRYPTED, PAGE_SIZE, PAYLOAD, VERSION, VERSION_V9, WRAPPED_DEK_LEN,
38};
39use crate::freelist::FreeList;
40use crate::index::{IndexConfig, IndexInfo, IndexOp, IvfPqIndex};
41use crate::keyslot::{KeySlot as KeySlotEntry, KeySlotRun};
42use crate::memory::{
43 self, status as essence_status, CommitBatch, CommitReport, EssenceOp, EssenceOpKind,
44 EssenceScope, EssenceSection, ExportedTurn, Filter, FilterTerm, IngestReport, KeyScope, Memory,
45 MemoryPatch, MemoryType, Metric, Scope, ScoreWeights, ThreadTurn, TurnPolicy, WriteOutcome,
46 CONTINUATION_TIMELINE_ENTRIES, ESSENCE_DEFAULT_RENDER_BYTES, ESSENCE_TRANSITIONS,
47 MNEMO_META_CONTINUES, MNEMO_META_INVALID_AT, MNEMO_META_KEY, MNEMO_META_KIND,
48 MNEMO_META_KIND_ESSENCE, MNEMO_META_KIND_THREAD_LINK, MNEMO_META_KIND_TURN, MNEMO_META_LENGTH,
49 MNEMO_META_PART, MNEMO_META_REFS, MNEMO_META_ROLE, MNEMO_META_SECTION, MNEMO_META_SEQ,
50 MNEMO_META_SHA256, MNEMO_META_STATUS, MNEMO_META_STATUS_HISTORY, MNEMO_META_SUPERSEDED_BY,
51 MNEMO_META_SUPERSEDES, MNEMO_META_THREAD, MNEMO_META_TURN_ID, MNEMO_META_UPDATED_AT,
52 THREAD_TOOL_RESULT_TRUNCATE_BYTES, THREAD_TURN_CHUNK_BYTES,
53};
54use crate::pager::Pager;
55use crate::result_cache::{
56 self, BatchState, CacheBudget, CacheDirectoryEntry, CacheEntry, CacheFlushPolicy, CachePutOpts,
57 CacheStats, CachedValue, CounterState, DirectoryIndex, SemanticCachePutOpts,
58};
59use crate::text;
60use crate::veccol;
61use crate::wal;
62
63/// Default initial size of the write-ahead log region, in pages (64 KiB).
64/// The region grows automatically when a transaction's control plane outgrows
65/// it, so this is a *floor* on the fresh-file footprint, not a cap on
66/// transaction size. Was 64 pages (512 KiB) in v0.1.0 — lowered because that
67/// reservation dominated small-file size (~62% of a 31-memory dogfood file).
68const DEFAULT_WAL_PAGES: u64 = 8;
69
70/// Hard floor on the WAL reservation. A single header-page flush already
71/// needs one full DATA frame plus a COMMIT frame; two pages are the bare
72/// minimum that lets the first transaction commit without immediate growth.
73const MIN_WAL_PAGES: u64 = 2;
74
75/// Default cap on retained snapshot manifest entries. Each `flush` appends
76/// one entry; without a cap, long-running processes accumulate entries
77/// forever, growing per-flush manifest-serialize cost as O(total flushes)
78/// and the manifest run size on disk in lockstep. 256 keeps roughly a
79/// week of hourly snapshots while staying tiny (256 × ~82 bytes ≈ 21 KiB
80/// of manifest). Set `MnemoConfig::max_snapshots = 0` to disable the cap.
81pub const DEFAULT_MAX_SNAPSHOTS: usize = 256;
82
83/// PERF-1: the ANN index is derived data, so `flush` persists it only every
84/// this many commits that changed it (or see
85/// [`INDEX_PERSIST_EVERY_CHANGES`], [`Mnemo::flush_index`], [`Mnemo::close`]).
86/// Between persists, only the catalog is rewritten.
87pub const INDEX_PERSIST_EVERY_COMMITS: u32 = 64;
88
89/// PERF-1: …or once this many vectors were added, changed or removed since
90/// the last persist. Records written after the persisted index are
91/// re-assigned when the index is next loaded, so this bounds that work.
92pub const INDEX_PERSIST_EVERY_CHANGES: usize = 1_000;
93
94/// FMT-9d: default weight ε of the normalised BM25 score in hybrid recall
95/// (`relevance = similarity + ε · bm25 / max bm25`).
96pub const DEFAULT_KEYWORD_WEIGHT: f32 = 0.5;
97
98/// FMT-9b: default [`Mnemo::set_vector_cache_budget`]: the fp16 vector
99/// column stays resident while it fits (100K × 384 dims is about 77 MB) and
100/// is read through the page cache above it. PLANNER-A, 2026-11-07.
101pub const DEFAULT_VECTOR_CACHE_BUDGET: usize = 96 * 1024 * 1024;
102
103/// FMT-9b: below this many facts a v9 file has no ANN index: an exact
104/// fp16 scan of 20K × 384 dims is about a millisecond, and the IVF+PQ
105/// codebook alone would cost more disk than it saves time.
106pub const V9_ANN_MIN_FACTS: usize = 20_000;
107
108/// FMT-9f (PLANNER-A, 2026-11-08): on v9 an unpinned recall reranks at
109/// least this many candidates per requested hit (500 for `top_k` 10), or
110/// the index's own auto depth when that's deeper (above 100K). The v9
111/// index is trained on fp16-rounded vectors, and at 20K–50K the auto
112/// depth (×10–×25) lost up to 0.03 recall@10 against v8; ×50 is within
113/// 0.005 at every measured tier. v8 is unchanged.
114pub const V9_RERANK_PER_HIT: usize = 50;
115
116/// FMT-9c: page runs longer than this are written copy-on-write straight
117/// to their home pages (and fsynced) before the commit, instead of through
118/// the WAL, on v9. The WAL then carries only the header and small runs, so
119/// it stays at its 64 KiB default instead of growing to the largest
120/// catalog or index (26 MB at 100K on v8). Safe for the same reason data
121/// pages are: the pages come from the tail or from quarantined free runs,
122/// so nothing committed references them until the header flips.
123const V9_WAL_RUN_MAX_PAGES: u64 = 2;
124
125/// FMT-9c: snapshot retention on v9: the newest 16 snapshots, plus any
126/// from the last 24 hours, with retained snapshots never pinning more than
127/// a quarter of the file (past that, the oldest go first, whatever their
128/// age). `set_max_snapshots` changes the count.
129pub const V9_DEFAULT_MAX_SNAPSHOTS: usize = 16;
130const V9_SNAPSHOT_KEEP_SECS: i64 = 24 * 3600;
131/// FMT-9c: the pin cap never drops below 128 pages (1 MiB).
132const V9_PIN_FLOOR_PAGES: u64 = 128;
133
134/// FMT-9d: the layout byte at the start of a v9 ANN index run (1: the
135/// whole IVF+PQ index, followed by delta runs). A reader treats an unknown
136/// layout as no index, so a later layout (partition-major, PERF-2) ships
137/// without a format bump.
138const V9_INDEX_LAYOUT: u8 = 1;
139
140/// FMT-9b: how many sparse vector pages one commit folds into its new
141/// pages (the vacuum). Bounded so a commit's cost stays bounded.
142const VACUUM_PAGES_PER_COMMIT: usize = 4;
143
144/// FMT-9c.2: and at most this many rows moved by the vacuum per commit (a
145/// moved row is a changed catalog row, so it lands in the commit's delta).
146/// Pages with more live rows than this stay as they are until deletes thin
147/// them out.
148const VACUUM_MAX_ROWS: usize = 64;
149
150/// FMT-9b: header flag: the file also keeps each record's float vector
151/// (`MnemoConfig::store_float_vectors`).
152const FLAG_FLOAT_VECTORS: u32 = 2;
153
154/// PERF-1: when the in-memory ANN index was last persisted, and whether the
155/// next flush must persist it.
156#[derive(Debug, Default, Clone, Copy)]
157struct IndexPersist {
158 /// Persist on the next flush (build, rebuild, drop, `flush_index`,
159 /// `close`, snapshot restore).
160 forced: bool,
161 /// Commits since the last persist that left the index dirty.
162 commits: u32,
163 /// Vectors added, re-assigned or removed since the last persist.
164 changes: usize,
165}
166
167/// R-004.2: where the thread layers and the onboarding briefing live,
168/// built by the handle's one lazy scan (`ensure_key_caches`, together with
169/// the 12.1 key caches) and then kept current by `put` and `delete`. It
170/// lets `about`, `essence`, `ingest_thread`, `thread_export` and
171/// thread-pinned recall read just their own records instead of decrypting
172/// the whole file each time.
173#[derive(Debug, Default)]
174struct LayerCache {
175 /// Live essence item ids.
176 essence: std::collections::HashSet<u128>,
177 /// Live ids tagged `metadata.area = "onboarding"` (superseded versions
178 /// included; `about` drops those when it reads them).
179 onboarding: std::collections::HashSet<u128>,
180 /// Per `_mnemo.thread`: its records and archive state.
181 threads: std::collections::HashMap<String, ThreadCache>,
182 /// Thread → (the thread it continues, the link record's id).
183 links: std::collections::HashMap<String, (String, u128)>,
184 /// FMT-9e: on v9, thread-turn records the lazy scan didn't decode,
185 /// by their row's `thread_hash`. A thread's bucket is decoded into
186 /// `threads` the first time that thread is used
187 /// ([`Mnemo::warm_thread`]), so a cold handle never decodes (or
188 /// decompresses) the turns of threads it doesn't touch.
189 cold_turns: std::collections::HashMap<u32, Vec<u128>>,
190}
191
192/// One thread's records (turns, essence items, link), the turn ids already
193/// archived (with how many records each has: long turns are chunked), and
194/// the highest `_mnemo.seq` seen.
195#[derive(Debug, Default)]
196struct ThreadCache {
197 records: std::collections::HashSet<u128>,
198 turns: std::collections::HashMap<String, u32>,
199 max_seq: Option<u64>,
200}
201
202fn meta_str<'a>(m: &'a Memory, key: &str) -> Option<&'a str> {
203 m.metadata.get(key).and_then(|v| v.as_str())
204}
205
206fn is_onboarding(m: &Memory) -> bool {
207 meta_str(m, "area").is_some_and(|s| s.eq_ignore_ascii_case("onboarding"))
208}
209
210impl LayerCache {
211 fn add(&mut self, m: &Memory) {
212 let id = m.id.0;
213 let kind = meta_str(m, MNEMO_META_KIND);
214 if kind == Some(MNEMO_META_KIND_ESSENCE) {
215 self.essence.insert(id);
216 }
217 if is_onboarding(m) {
218 self.onboarding.insert(id);
219 }
220 let Some(thread) = meta_str(m, MNEMO_META_THREAD) else {
221 return;
222 };
223 let tc = self.threads.entry(thread.to_string()).or_default();
224 tc.records.insert(id);
225 if kind == Some(MNEMO_META_KIND_TURN) {
226 if let Some(turn_id) = meta_str(m, MNEMO_META_TURN_ID) {
227 *tc.turns.entry(turn_id.to_string()).or_insert(0) += 1;
228 }
229 if let Some(seq) = m.metadata.get(MNEMO_META_SEQ).and_then(|v| v.as_u64()) {
230 tc.max_seq = Some(tc.max_seq.map_or(seq, |s| s.max(seq)));
231 }
232 }
233 if kind == Some(MNEMO_META_KIND_THREAD_LINK) {
234 if let Some(from) = meta_str(m, MNEMO_META_CONTINUES) {
235 self.links
236 .entry(thread.to_string())
237 .or_insert_with(|| (from.to_string(), id));
238 }
239 }
240 }
241
242 /// Undo [`LayerCache::add`] for the stored version of a record that is
243 /// being overwritten or deleted. A thread with no records left is
244 /// dropped, so a forgotten thread's `_mnemo.seq` restarts at 0. Partial
245 /// deletes keep the old maximum (sequence gaps are harmless).
246 fn remove(&mut self, m: &Memory) {
247 let id = m.id.0;
248 self.essence.remove(&id);
249 self.onboarding.remove(&id);
250 let Some(thread) = meta_str(m, MNEMO_META_THREAD) else {
251 return;
252 };
253 if self.links.get(thread).is_some_and(|(_, link)| *link == id) {
254 self.links.remove(thread);
255 }
256 let Some(tc) = self.threads.get_mut(thread) else {
257 return;
258 };
259 tc.records.remove(&id);
260 if meta_str(m, MNEMO_META_KIND) == Some(MNEMO_META_KIND_TURN) {
261 if let Some(turn_id) = meta_str(m, MNEMO_META_TURN_ID) {
262 if let Some(n) = tc.turns.get_mut(turn_id) {
263 *n -= 1;
264 if *n == 0 {
265 tc.turns.remove(turn_id);
266 }
267 }
268 }
269 }
270 if tc.records.is_empty() {
271 self.threads.remove(thread);
272 }
273 }
274}
275
276/// The thread a recall filter pins with `_mnemo.thread` equality, if any.
277fn filter_thread_pin(f: &Filter) -> Option<&str> {
278 f.terms.iter().find_map(|t| match t {
279 FilterTerm::Eq(k, Value::String(s)) if k == MNEMO_META_THREAD => Some(s.as_str()),
280 _ => None,
281 })
282}
283
284/// Whether a recall filter can only match records that aren't in the ANN
285/// (R-004.2): it pins `_mnemo.thread`, or restricts `_mnemo.kind` to turns,
286/// essence items or thread links. Such recalls scan exactly instead.
287fn filter_bypasses_ann(f: &Filter) -> bool {
288 let non_ann = |v: &Value| {
289 matches!(
290 v.as_str(),
291 Some(MNEMO_META_KIND_TURN)
292 | Some(MNEMO_META_KIND_ESSENCE)
293 | Some(MNEMO_META_KIND_THREAD_LINK)
294 )
295 };
296 filter_thread_pin(f).is_some()
297 || f.terms.iter().any(|t| match t {
298 FilterTerm::Eq(k, v) => k == MNEMO_META_KIND && non_ann(v),
299 FilterTerm::In(k, vs) => {
300 k == MNEMO_META_KIND && !vs.is_empty() && vs.iter().all(non_ann)
301 }
302 _ => false,
303 })
304}
305
306/// Chunk size for the torn-header scan's streaming read of the file.
307/// We walk the file 1 MiB at a time looking for 8 KiB-aligned `MWAL`
308/// magic. There is **no** cap on how far a given candidate's frame
309/// chain may extend — a 1M-vector IVF+PQ relocation can push a single
310/// transaction well past 16 MiB — so each candidate is read frame by
311/// frame via [`wal::scan_committed_txn`] rather than slurped into a
312/// bounded buffer. (DUR-1 R-002.1 part 2.)
313const TORN_HEADER_SCAN_CHUNK: usize = 1 << 20; // 1 MiB
314
315/// Walk the file for every 8 KiB-aligned page whose first four bytes are
316/// `MWAL`, treating each as a candidate WAL-region start, and return the
317/// frames of the highest-`txn_id` committed transaction whose header
318/// frame fully validates under `passphrase`.
319///
320/// A candidate is accepted only if:
321/// * `wal::scan_committed_txn` returns a committed transaction (chained
322/// CRC verifies, single `txn_id`, `KIND_COMMIT` closes it);
323/// * that transaction contains a `(0, header_page)` frame;
324/// * the header parses with `wal_start == candidate_page_no` and
325/// `wal_seq == txn_id` (so a frame cannot forge a region it was not
326/// written into);
327/// * deriving the KEK from `passphrase + header.salt`, unwrapping the
328/// wrapped DEK, and validating the header seal all succeed.
329///
330/// Returns `Ok(None)` when the scan finds no valid candidate — the
331/// caller is expected to propagate the original parse error so
332/// non-mnemo files still report `BadMagic`.
333///
334/// DUR-1 (R-002.1 part 2): there is no fixed scan window — the whole
335/// file is scanned. Reading at page granularity (not byte granularity)
336/// is sufficient because every WAL region in the format is page-aligned
337/// and every WAL commit writes its first frame at the region's start.
338fn scan_header_recovery_candidates(
339 file: &mut File,
340 secret: &[u8],
341) -> Result<Option<Vec<wal::Frame>>> {
342 let file_len = file.seek(SeekFrom::End(0))?;
343 if file_len < PAGE_SIZE as u64 * 2 {
344 return Ok(None); // No room for even one WAL page past the header.
345 }
346
347 let mut best: Option<(u64, Vec<wal::Frame>)> = None;
348 let page_sz = PAGE_SIZE as u64;
349 let mut offset = page_sz; // Skip page 0.
350
351 // Stream through the file in 1 MiB chunks, scanning each chunk for
352 // page-aligned `MWAL` candidates. Each candidate's frame chain is
353 // then read incrementally via `wal::scan_committed_txn`, which has
354 // no window cap — a 1M-vector IVF+PQ relocation can span tens of
355 // MiB in one committed transaction.
356 let mut chunk = vec![0u8; TORN_HEADER_SCAN_CHUNK];
357 while offset < file_len {
358 let want = (TORN_HEADER_SCAN_CHUNK as u64).min(file_len - offset) as usize;
359 chunk.resize(want, 0);
360 file.seek(SeekFrom::Start(offset))?;
361 let mut filled = 0;
362 while filled < want {
363 match file.read(&mut chunk[filled..])? {
364 0 => break,
365 n => filled += n,
366 }
367 }
368 if filled == 0 {
369 break;
370 }
371 let mut inner = 0usize;
372 while inner + 4 <= filled {
373 if &chunk[inner..inner + 4] == wal::FRAME_MAGIC_BYTES {
374 let page_no = (offset + inner as u64) / page_sz;
375 if let Some((txn_id, frames)) = try_recover_candidate(file, page_no, secret)? {
376 let keep = match &best {
377 None => true,
378 Some((id, _)) => txn_id > *id,
379 };
380 if keep {
381 best = Some((txn_id, frames));
382 }
383 }
384 }
385 inner += PAGE_SIZE;
386 }
387 offset += filled as u64;
388 }
389
390 Ok(best.map(|(_, f)| f))
391}
392
393/// Try to recover a committed transaction from `candidate_page_no`,
394/// returning `(txn_id, frames)` only if the candidate's
395/// `(0, header_page)` frame fully validates: the frame chain commits
396/// (via [`wal::scan_committed_txn`]'s streaming reader), the header
397/// parses with `wal_start == candidate_page_no` **and**
398/// `wal_seq == txn_id`, the DEK unwraps under `passphrase`, and the
399/// AES-GCM seal validates. DUR-1 (R-002.1 item 2) requires the
400/// `wal_seq == txn_id` check and no fixed scan window — the streaming
401/// reader handles transactions of arbitrary size.
402fn try_recover_candidate(
403 file: &mut File,
404 candidate_page_no: u64,
405 secret: &[u8],
406) -> Result<Option<(u64, Vec<wal::Frame>)>> {
407 let Some((txn_id, frames)) = wal::scan_committed_txn(file, candidate_page_no)? else {
408 return Ok(None);
409 };
410
411 // Must contain a header frame at page 0.
412 let header_bytes = match frames.iter().find(|(p, _)| *p == 0) {
413 Some((_, b)) => b,
414 None => return Ok(None),
415 };
416 if header_bytes.len() != PAGE_SIZE {
417 return Ok(None);
418 }
419 let mut hbuf = [0u8; PAGE_SIZE];
420 hbuf.copy_from_slice(header_bytes);
421 let candidate = match Header::from_page(&hbuf) {
422 Ok(h) => h,
423 Err(_) => return Ok(None),
424 };
425
426 // The frame's claimed region must match where we found it, and the
427 // header frame's `wal_seq` must match the committed transaction's
428 // `txn_id` — this is what pins the frame to the region it was
429 // written into and keeps a stale checkpointed-and-copied frame
430 // from being accepted as a new region's root.
431 if candidate.wal_start != candidate_page_no || candidate.wal_seq != txn_id {
432 return Ok(None);
433 }
434 // DEK unwrap under this candidate's own salt + KDF params.
435 let kdf = KdfParams {
436 m_cost: candidate.m_cost,
437 t_cost: candidate.t_cost,
438 p_cost: candidate.p_cost,
439 };
440 let kek = match crypto::derive_kek(secret, &candidate.salt, kdf) {
441 Ok(k) => k,
442 Err(_) => return Ok(None),
443 };
444 let dek = match crypto::unwrap_dek(&kek, &candidate.dek_nonce, &candidate.wrapped_dek) {
445 Ok(d) => d,
446 Err(_) => return Ok(None),
447 };
448 if candidate.validate_seal(&dek).is_err() {
449 return Ok(None);
450 }
451
452 Ok(Some((txn_id, frames)))
453}
454
455/// Configuration for creating a new database.
456#[derive(Clone, Copy, Debug)]
457pub struct MnemoConfig {
458 /// Embedding dimensionality. Every stored vector must match this.
459 pub dimensions: usize,
460 /// Key-derivation parameters.
461 pub kdf: KdfParams,
462 /// Initial size of the WAL region in 8 KiB pages. Default is
463 /// [`DEFAULT_WAL_PAGES`] (8 pages = 64 KiB), which keeps fresh files
464 /// small; the region auto-grows when a transaction's control plane needs
465 /// more space, so raising this is only a hint for the expected steady-state
466 /// transaction size. Write-heavy databases that routinely flush large
467 /// catalogs or ANN indexes can set this higher (e.g. 64 = 512 KiB) to
468 /// avoid early grow events; tiny embedded uses can leave it at the
469 /// default. Clamped to a minimum of [`MIN_WAL_PAGES`] (2 pages).
470 pub wal_pages_initial: u64,
471 /// Maximum number of snapshot manifest entries to retain. Each `flush`
472 /// appends one entry; once the cap is hit, the *oldest* entry is dropped
473 /// from the in-memory manifest before the new one is added, so the
474 /// retained set is always the most-recent N. Defaults to
475 /// [`DEFAULT_MAX_SNAPSHOTS`] (256). Set to `0` to disable the cap and
476 /// retain every snapshot forever (the pre-v0.3 behavior).
477 ///
478 /// Pages referenced *only* by pruned snapshots stay on disk until the
479 /// next [`Mnemo::compact_file`], which reclaims them. Point-in-time
480 /// recovery via [`Mnemo::restore_to`] into a pruned `txn_id` returns
481 /// [`MnemoError::NotFound`].
482 ///
483 /// Apply to an already-open database with [`Mnemo::set_max_snapshots`].
484 pub max_snapshots: usize,
485 /// The on-disk format a new file is created in. `0` (the default)
486 /// means v9, the current format. `8` creates a v8 file, for tools that
487 /// still need to read it with v0.10 or earlier; it's kept for one
488 /// release (v0.12 makes v8 read-only). **Not a stable API.**
489 #[doc(hidden)]
490 pub format_version: u16,
491 /// FMT-9b (v9 files only; ignored for v8): also keep each memory's
492 /// full-precision vector in its record, so `get()` and `recall()`
493 /// return it exactly. Off by default: a v9 file stores vectors as
494 /// fp16 (about 1e-3 relative error) in the vector column, which recall
495 /// scores from either way. Costs `4 × dimensions` bytes per memory.
496 pub store_float_vectors: bool,
497}
498
499impl Default for MnemoConfig {
500 fn default() -> Self {
501 Self {
502 dimensions: 768,
503 kdf: KdfParams::secure(),
504 wal_pages_initial: DEFAULT_WAL_PAGES,
505 max_snapshots: DEFAULT_MAX_SNAPSHOTS,
506 format_version: 0,
507 store_float_vectors: false,
508 }
509 }
510}
511
512/// FMT-9: resolve [`MnemoConfig::format_version`] for a new file.
513fn resolve_format_version(requested: u16) -> Result<u16> {
514 match requested {
515 // FMT-9f: new files are v9; the `MNEMO_DEV_FORMAT` switch is gone.
516 0 => Ok(crate::format::DEFAULT_NEW_VERSION),
517 VERSION | VERSION_V9 => Ok(requested),
518 v => Err(MnemoError::Invalid(format!(
519 "format_version {v} is not supported for new files (use 8 or 9)"
520 ))),
521 }
522}
523
524/// Catalog entry: maps a memory ID to its page run on disk.
525///
526/// **Schema (v5+):** carries `accessed_at` and `access_count` so `recall`
527/// can update access stats by mutating the catalog entry rather than
528/// rewriting the entire record (Phase 2.1 of the improvement plan). The
529/// values in a memory's serialized record body become a stale snapshot
530/// after the first recall; [`Mnemo::read_memory`] re-populates them from
531/// the catalog so consumers always see the live values.
532#[derive(Serialize, Deserialize, Clone, Debug)]
533struct CatalogEntry {
534 /// ULID as a raw `u128`.
535 id: u128,
536 start_page: u64,
537 page_count: u32,
538 /// Exact serialized byte length of the record.
539 len: u32,
540 deleted: bool,
541 /// Unix seconds of the most recent recall hit (or write, for fresh entries).
542 accessed_at: i64,
543 /// How many times `recall` has surfaced this memory.
544 access_count: u32,
545 /// FMT-9: the generation the record's pages were sealed under (v9 page
546 /// AAD). `0`, and left out of the encoding, on v8 files, so a v8
547 /// catalog is byte-for-byte what v0.10 writes.
548 #[serde(default, skip_serializing_if = "crate::format::is_zero_u64")]
549 gen: u64,
550 /// FMT-9b: the v9 row's tags, scoring inputs and vector location
551 /// (`None`, and left out of the encoding, on v8 files).
552 #[serde(default, skip_serializing_if = "Option::is_none")]
553 v9: Option<RowV9>,
554}
555
556/// FMT-9b: what a v9 catalog row carries beyond the v8 fields, so recall
557/// can filter and score a candidate without decrypting its record, and a
558/// cold handle can select records by kind or thread.
559#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default)]
560struct RowV9 {
561 /// The vector column entry: page (`0` while it's waiting for the next
562 /// flush), slot, and the page's generation.
563 vec_page: u64,
564 vec_slot: u16,
565 vec_gen: u64,
566 /// FMT-9b.2: the record's slot on its packed page (`start_page`), or
567 /// [`REC_SPAN`] when the record owns a run of whole pages.
568 rec_slot: u16,
569 /// `ROW_KIND_*`.
570 kind: u8,
571 /// `ROW_SUPERSEDED | ROW_HAS_TTL | ROW_SHARED`.
572 flags: u8,
573 /// [`memory_type_code`].
574 memory_type: u8,
575 /// FNV-1a of `_mnemo.thread`, of the 12.1 key slot, and of `agent_id`
576 /// (`0` = none). A collision only lets a non-match through to the
577 /// decrypted check.
578 thread_hash: u32,
579 key_hash: u32,
580 agent_hash: u32,
581 created_at: i64,
582 importance: f32,
583 /// `created_at + ttl_secs` when `ROW_HAS_TTL` is set.
584 expires_at: i64,
585 /// FMT-9d: the row's BM25 document id (`0`: not keyword-indexed) and
586 /// its token count. Live rows define which documents are live, and
587 /// N and avgdl, so segments never hold lengths or tombstones.
588 doc: u32,
589 doc_len: u16,
590}
591
592/// FMT-9b.2: `RowV9::rec_slot` of a record too big for a packed page; it
593/// owns `page_count` whole pages from `start_page`, as in v8.
594const REC_SPAN: u16 = u16::MAX;
595/// FMT-9b.2: `kind` byte of a packed record page.
596const PAGE_KIND_RECORD: u8 = 1;
597/// FMT-9b.2: bytes of a packed record page available to slots and data.
598const REC_PAGE_CAP: usize = PAYLOAD - veccol::PAGE_HEADER;
599/// FMT-9b.2: bytes per slot-directory entry (offset u16, length u16).
600const REC_SLOT_BYTES: usize = 4;
601/// FMT-9b.2: thread-archive chunk size on v9. A chunk plus its record's
602/// metadata (~200 B) then fits one packed page (the FMT-9a measurement:
603/// 8 KB chunks spill into a second page almost every time, 300 MB vs
604/// 212 MB for the 50K-turn profile).
605const V9_TURN_CHUNK_BYTES: usize = 7_900;
606
607/// FMT-9e: the zstd frame magic. A v9 record body is either a MessagePack
608/// `Memory` (which always starts with an array marker, `0x9_` or
609/// `0xdc`/`0xdd`) or, for a compressed thread turn, a zstd frame, so the
610/// first four bytes tell them apart without a row flag.
611const ZSTD_MAGIC: [u8; 4] = [0x28, 0xB5, 0x2F, 0xFD];
612/// FMT-9e: zstd level for turn records (the FMT-9a measurement: 2.15× on
613/// this repo's Markdown at level 3).
614const TURN_ZSTD_LEVEL: i32 = 3;
615/// FMT-9e: turn records shorter than this stay uncompressed (a frame's
616/// overhead eats the gain on a one-line turn).
617const TURN_ZSTD_MIN_BYTES: usize = 256;
618/// FMT-9e: the largest decompressed record accepted. A turn record is a
619/// chunk of at most [`V9_TURN_CHUNK_BYTES`] plus its metadata, so this is
620/// a generous bound, not a size limit anyone meets; it stops a damaged
621/// (but authenticated) frame from asking for an unbounded allocation.
622const RECORD_PLAIN_MAX: u64 = 64 << 20;
623/// FMT-9e: by default a thread turn is written plain and compressed once
624/// it's a week old (by its `created_at`), so a live session's recent turns
625/// read at full speed and archived history is small.
626pub const DEFAULT_TURN_COMPRESS_AFTER_SECS: i64 = 7 * 24 * 3600;
627/// FMT-9e: at most this many aged turns are compressed per commit, so a
628/// commit's extra work stays bounded (like the vacuum).
629const COLD_TURNS_PER_COMMIT: usize = 256;
630
631/// FMT-9e: is turn-record compression on? Always, unless the dev switch
632/// `MNEMO_DEV_TURN_ZSTD=0` turns it off for an A/B measurement (read once
633/// per process). Reading never depends on it.
634fn turn_zstd_enabled() -> bool {
635 static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
636 *ON.get_or_init(|| std::env::var("MNEMO_DEV_TURN_ZSTD").as_deref() != Ok("0"))
637}
638
639/// FMT-9e: compress a v9 thread-turn record body, before the page
640/// encryption. Each record is its own zstd frame with no dictionary and no
641/// compression history from any other record, so how densely one record
642/// compresses says nothing about another's content (design note (d)).
643/// Kept only when it saves at least an eighth; otherwise the plain body is
644/// returned.
645fn compress_turn_record(bytes: Vec<u8>) -> Vec<u8> {
646 if bytes.len() < TURN_ZSTD_MIN_BYTES || !turn_zstd_enabled() {
647 return bytes;
648 }
649 let c = ZSTD_COMPRESSOR.with(|cell| {
650 let mut slot = cell.borrow_mut();
651 if slot.is_none() {
652 *slot = zstd::bulk::Compressor::new(TURN_ZSTD_LEVEL).ok();
653 }
654 let out = slot.as_mut().and_then(|z| z.compress(&bytes).ok());
655 if out.is_none() {
656 // Never reuse a context a call failed on.
657 *slot = None;
658 }
659 out
660 });
661 match c {
662 Some(c) if c.len() * 8 <= bytes.len() * 7 => c,
663 _ => bytes,
664 }
665}
666
667// FMT-9e: one zstd context of each kind per thread, made on first use and
668// reused for every record (a fresh context per record made reads about 15%
669// slower). Only the one-shot (`bulk`) calls use them, and each call starts
670// a new frame from a reset session, so no state crosses records; a context
671// a call failed on is dropped, not reused (cf. RUSTSEC-2026-0391, which
672// affected the stream API's `with_context`; zstd 0.13.4 carries its fix).
673// One-shot calls size their tables to the record, so a context stays small.
674thread_local! {
675 static ZSTD_COMPRESSOR: std::cell::RefCell<Option<zstd::bulk::Compressor<'static>>> =
676 const { std::cell::RefCell::new(None) };
677 static ZSTD_DECOMPRESSOR: std::cell::RefCell<Option<zstd::bulk::Decompressor<'static>>> =
678 const { std::cell::RefCell::new(None) };
679}
680
681/// FMT-9e: the plain body of a v9 record: decompressed when it's a zstd
682/// frame, borrowed as is otherwise. The frame must state its content size,
683/// and that size must be at most [`RECORD_PLAIN_MAX`].
684pub(crate) fn plain_record(bytes: &[u8]) -> Result<std::borrow::Cow<'_, [u8]>> {
685 if !bytes.starts_with(&ZSTD_MAGIC) {
686 return Ok(std::borrow::Cow::Borrowed(bytes));
687 }
688 let bad = |why: &str| MnemoError::Invalid(format!("compressed record: {why}"));
689 let size = match zstd::zstd_safe::get_frame_content_size(bytes) {
690 Ok(Some(n)) if n <= RECORD_PLAIN_MAX => n as usize,
691 Ok(Some(_)) => return Err(bad("too large")),
692 _ => return Err(bad("no content size")),
693 };
694 let out = ZSTD_DECOMPRESSOR.with(|cell| {
695 let mut slot = cell.borrow_mut();
696 if slot.is_none() {
697 *slot = Some(zstd::bulk::Decompressor::new().map_err(|e| bad(&e.to_string()))?);
698 }
699 let out = match slot.as_mut() {
700 Some(z) => z.decompress(bytes, size).map_err(|e| bad(&e.to_string())),
701 None => Err(bad("no context")),
702 };
703 if out.is_err() {
704 // Never reuse a context a call failed on.
705 *slot = None;
706 }
707 out
708 })?;
709 if out.len() != size {
710 return Err(bad("length mismatch"));
711 }
712 Ok(std::borrow::Cow::Owned(out))
713}
714
715/// FMT-9c.2: pick the sparse pages (`(page, live rows)`, in page order) one
716/// commit's vacuum folds in: at most [`VACUUM_PAGES_PER_COMMIT`] pages and
717/// [`VACUUM_MAX_ROWS`] rows.
718fn vacuum_pick(sparse: Vec<(u64, usize)>) -> Vec<u64> {
719 let mut rows = 0usize;
720 let mut out = Vec::new();
721 for (p, n) in sparse {
722 if out.len() == VACUUM_PAGES_PER_COMMIT {
723 break;
724 }
725 if rows + n > VACUUM_MAX_ROWS {
726 continue;
727 }
728 rows += n;
729 out.push(p);
730 }
731 out
732}
733
734/// FMT-9b.2: encode a packed record page: the header, the slot directory,
735/// then the records in slot order.
736fn encode_record_page(gen: u64, recs: &[&[u8]]) -> Vec<u8> {
737 let dir = recs.len() * REC_SLOT_BYTES;
738 let data: usize = recs.iter().map(|r| r.len()).sum();
739 let mut out = Vec::with_capacity(veccol::PAGE_HEADER + dir + data);
740 out.extend_from_slice(&[PAGE_KIND_RECORD, 0, 0, 0]);
741 out.extend_from_slice(&(recs.len() as u16).to_le_bytes());
742 out.extend_from_slice(&((dir + data) as u16).to_le_bytes());
743 out.extend_from_slice(&gen.to_le_bytes());
744 let mut off = veccol::PAGE_HEADER + dir;
745 for r in recs {
746 out.extend_from_slice(&(off as u16).to_le_bytes());
747 out.extend_from_slice(&(r.len() as u16).to_le_bytes());
748 off += r.len();
749 }
750 for r in recs {
751 out.extend_from_slice(r);
752 }
753 out
754}
755
756/// FMT-9b.2: the bytes of slot `slot` of a packed record page.
757fn record_slot(page: &[u8], slot: u16) -> Result<Vec<u8>> {
758 let bad = || MnemoError::Invalid("malformed record page".into());
759 if page.len() < veccol::PAGE_HEADER || page[0] != PAGE_KIND_RECORD {
760 return Err(bad());
761 }
762 let n = u16::from_le_bytes([page[4], page[5]]);
763 if slot >= n {
764 return Err(bad());
765 }
766 let d = veccol::PAGE_HEADER + slot as usize * REC_SLOT_BYTES;
767 let dir = page.get(d..d + REC_SLOT_BYTES).ok_or_else(bad)?;
768 let off = u16::from_le_bytes([dir[0], dir[1]]) as usize;
769 let len = u16::from_le_bytes([dir[2], dir[3]]) as usize;
770 page.get(off..off + len).map(|b| b.to_vec()).ok_or_else(bad)
771}
772
773/// FMT-9d: the BM25 manifest run: tokenizer, the next document id, and the
774/// segment runs (oldest first).
775#[derive(Serialize, Deserialize, Clone, Debug)]
776struct Bm25Manifest {
777 tokenizer: u8,
778 next_doc: u32,
779 segments: Vec<(u64, u64, u64, u64)>,
780 /// Documents each segment was written with (live or not since).
781 #[serde(default)]
782 seg_docs: Vec<u32>,
783}
784
785impl Default for Bm25Manifest {
786 fn default() -> Self {
787 Bm25Manifest {
788 tokenizer: text::TOKENIZER_V1,
789 next_doc: 1,
790 segments: Vec::new(),
791 seg_docs: Vec::new(),
792 }
793 }
794}
795
796/// FMT-9d: live documents: `doc -> (row, length)`, their count and mean
797/// length. Rebuilt from the rows after any write.
798#[derive(Debug, Default)]
799struct DocStats {
800 docs: std::collections::HashMap<u32, (usize, u16)>,
801 avgdl: f32,
802 /// FMT-9f: the sum of the live documents' lengths, so `put` and
803 /// `delete` keep `avgdl` current without a rescan of the rows.
804 total: u64,
805}
806
807impl DocStats {
808 /// FMT-9f: drop document `doc` (if live) and keep `avgdl` current.
809 fn remove(&mut self, doc: u32) {
810 if let Some((_, len)) = self.docs.remove(&doc) {
811 self.total = self.total.saturating_sub(len as u64);
812 self.refresh();
813 }
814 }
815
816 /// FMT-9f: add document `doc` at catalog row `row`.
817 fn insert(&mut self, doc: u32, row: usize, len: u16) {
818 if let Some((_, old)) = self.docs.insert(doc, (row, len)) {
819 self.total = self.total.saturating_sub(old as u64);
820 }
821 self.total += len as u64;
822 self.refresh();
823 }
824
825 fn refresh(&mut self) {
826 self.avgdl = if self.docs.is_empty() {
827 1.0
828 } else {
829 self.total as f32 / self.docs.len() as f32
830 };
831 }
832}
833
834/// FMT-9d: the handle's BM25 state.
835#[derive(Debug, Default)]
836struct Bm25State {
837 manifest: Bm25Manifest,
838 /// Per segment, its dict, once loaded (resident; postings stay on disk).
839 dicts: Option<Vec<Vec<text::DictEntry>>>,
840 /// Documents written since the last commit, which writes them as a
841 /// new segment.
842 pending: Vec<(u32, Vec<(String, u8)>)>,
843 stats: Option<DocStats>,
844 /// Documents in the segment the current flush is merging.
845 merged_docs: u32,
846}
847
848/// FMT-9d: a manifest's segment runs and their document counts.
849type Bm25Segments = (Vec<(u64, u64, u64, u64)>, Vec<u32>);
850
851/// FMT-9d: segments past this many are merged into one by the next commit.
852const BM25_MAX_SEGMENTS: usize = 8;
853
854/// FMT-9d: is this row keyword-indexed? Facts and onboarding records;
855/// thread turns, essence items and links stay out (FMT-9a: turn postings
856/// would be 3× all the fact data).
857fn bm25_kind(kind: u8) -> bool {
858 kind == ROW_KIND_FACT || kind == ROW_KIND_ONBOARDING
859}
860
861/// FMT-9d: read `len` bytes at byte `off` of a page run.
862fn read_run_range(pager: &mut Pager, ptr: RunPtr, off: u64, len: u64) -> Result<Vec<u8>> {
863 let mut out = Vec::with_capacity(len as usize);
864 if len == 0 {
865 return Ok(out);
866 }
867 if off + len > ptr.len {
868 return Err(MnemoError::Invalid("BM25 range past its run".into()));
869 }
870 let p = PAYLOAD as u64;
871 let (first, last) = (off / p, (off + len - 1) / p);
872 for i in first..=last {
873 let page = pager.read_page(ptr.start + i, ptr.gen)?;
874 let lo = if i == first { (off % p) as usize } else { 0 };
875 let hi = if i == last {
876 ((off + len - 1) % p) as usize + 1
877 } else {
878 PAYLOAD
879 };
880 out.extend_from_slice(&page[lo..hi]);
881 }
882 Ok(out)
883}
884
885/// FMT-9d: a segment's dict (the `[u32 len][dict]` head of its run).
886fn load_segment_dict(pager: &mut Pager, ptr: RunPtr) -> Result<Vec<text::DictEntry>> {
887 let head = read_run_range(pager, ptr, 0, 4)?;
888 let n = u32::from_le_bytes([head[0], head[1], head[2], head[3]]) as u64;
889 let bytes = read_run_range(pager, ptr, 4, n)?;
890 rmp_serde::from_slice(&bytes).map_err(|e| MnemoError::Serialize(e.to_string()))
891}
892
893/// FMT-9d: a term's postings in one segment (empty when absent).
894fn segment_postings(
895 pager: &mut Pager,
896 ptr: RunPtr,
897 dict: &[text::DictEntry],
898 term: &str,
899) -> Result<Vec<(u32, u8)>> {
900 let Ok(i) = dict.binary_search_by(|e| e.term.as_str().cmp(term)) else {
901 return Ok(Vec::new());
902 };
903 let head = read_run_range(pager, ptr, 0, 4)?;
904 let base = 4 + u32::from_le_bytes([head[0], head[1], head[2], head[3]]) as u64;
905 let e = &dict[i];
906 let bytes = read_run_range(pager, ptr, base + e.off as u64, e.len as u64)?;
907 text::decode_postings(&bytes)
908}
909
910/// FMT-9d: load the BM25 manifest (an empty pointer yields a fresh one).
911fn load_bm25_manifest(pager: &mut Pager, ptr: RunPtr) -> Result<Bm25Manifest> {
912 if ptr.is_empty() {
913 return Ok(Bm25Manifest::default());
914 }
915 let buf = read_run_bytes(pager, ptr.start, ptr.pages, ptr.len, ptr.gen)?;
916 let m: Bm25Manifest =
917 rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
918 if m.tokenizer != text::TOKENIZER_V1 {
919 return Err(MnemoError::Invalid(format!(
920 "BM25 tokenizer {} is newer than this build understands",
921 m.tokenizer
922 )));
923 }
924 Ok(m)
925}
926
927/// FMT-9b.2: the handle's view of packed record pages.
928#[derive(Debug, Default)]
929struct RecPages {
930 /// Rows whose records aren't on a page yet; the next flush packs them.
931 hot: std::collections::HashMap<usize, Vec<u8>>,
932 /// Rows to pack, in order (may repeat; the flush skips repeats).
933 pending: Vec<usize>,
934 /// Packed page → the live rows on it.
935 page_rows: std::collections::HashMap<u64, Vec<usize>>,
936}
937
938impl RecPages {
939 fn from_catalog(catalog: &[CatalogEntry]) -> RecPages {
940 let mut recs = RecPages::default();
941 for (i, e) in catalog.iter().enumerate() {
942 if e.deleted || e.start_page == 0 {
943 continue;
944 }
945 if let Some(r) = e.v9 {
946 if r.rec_slot != REC_SPAN {
947 recs.page_rows.entry(e.start_page).or_default().push(i);
948 }
949 }
950 }
951 recs
952 }
953}
954
955const ROW_SUPERSEDED: u8 = 1;
956const ROW_HAS_TTL: u8 = 2;
957const ROW_SHARED: u8 = 4;
958/// FMT-9e: a thread turn whose record compression is settled: stored as
959/// a zstd frame, or tried and not worth it. Unsettled turns are plain and
960/// get compressed once they're older than the handle's
961/// `turn_compress_after` (see [`Mnemo::set_turn_compress_after`]).
962const ROW_SETTLED: u8 = 8;
963const ROW_KIND_FACT: u8 = 0;
964const ROW_KIND_TURN: u8 = 1;
965const ROW_KIND_ESSENCE: u8 = 2;
966const ROW_KIND_THREAD_LINK: u8 = 3;
967const ROW_KIND_ONBOARDING: u8 = 4;
968
969/// 32-bit FNV-1a over `parts` (separated), never `0` ("none" in a row).
970fn fnv1a32(parts: &[&[u8]]) -> u32 {
971 let mut h: u32 = 0x811c_9dc5;
972 for (i, p) in parts.iter().enumerate() {
973 if i > 0 {
974 h ^= 0xff;
975 h = h.wrapping_mul(0x0100_0193);
976 }
977 for &b in p.iter() {
978 h ^= b as u32;
979 h = h.wrapping_mul(0x0100_0193);
980 }
981 }
982 h.max(1)
983}
984
985fn memory_type_code(t: MemoryType) -> u8 {
986 match t {
987 MemoryType::Episodic => 0,
988 MemoryType::Semantic => 1,
989 MemoryType::Procedural => 2,
990 MemoryType::Working => 3,
991 }
992}
993
994/// FMT-9b: a memory's row tags and scoring inputs (the vector location is
995/// filled in when its column entry is written).
996fn row_tags(m: &Memory) -> RowV9 {
997 let kind = match meta_str(m, MNEMO_META_KIND) {
998 Some(MNEMO_META_KIND_TURN) => ROW_KIND_TURN,
999 Some(MNEMO_META_KIND_ESSENCE) => ROW_KIND_ESSENCE,
1000 Some(MNEMO_META_KIND_THREAD_LINK) => ROW_KIND_THREAD_LINK,
1001 _ if is_onboarding(m) => ROW_KIND_ONBOARDING,
1002 _ => ROW_KIND_FACT,
1003 };
1004 let mut flags = 0u8;
1005 if m.metadata.contains_key(MNEMO_META_SUPERSEDED_BY) {
1006 flags |= ROW_SUPERSEDED;
1007 }
1008 if m.ttl_secs.is_some() {
1009 flags |= ROW_HAS_TTL;
1010 }
1011 if m.scope == Scope::Shared {
1012 flags |= ROW_SHARED;
1013 }
1014 let key_hash = meta_str(m, MNEMO_META_KEY).map_or(0, |k| {
1015 let slot = Mnemo::key_slot_for(m, k);
1016 fnv1a32(&[
1017 slot.scope_owner.as_deref().unwrap_or("").as_bytes(),
1018 slot.key.as_bytes(),
1019 ])
1020 });
1021 RowV9 {
1022 kind,
1023 flags,
1024 memory_type: memory_type_code(m.memory_type),
1025 thread_hash: meta_str(m, MNEMO_META_THREAD).map_or(0, |t| fnv1a32(&[t.as_bytes()])),
1026 key_hash,
1027 agent_hash: fnv1a32(&[m.agent_id.as_bytes()]),
1028 created_at: m.created_at,
1029 importance: m.importance,
1030 expires_at: m.ttl_secs.map_or(0, |t| m.created_at.saturating_add(t)),
1031 ..RowV9::default()
1032 }
1033}
1034
1035/// FMT-9b: the handle's view of the fp16 vector column.
1036#[derive(Debug, Default)]
1037struct VecColumn {
1038 /// Resident fp16 values by catalog row (`rows × dims`), once loaded.
1039 data: Vec<u16>,
1040 norms: Vec<f32>,
1041 /// Which rows of `data` are filled. The column is allocated zeroed for
1042 /// every row (the OS maps those pages lazily) and filled a vector page
1043 /// at a time as recall touches rows, so the first recall reads only the
1044 /// pages its candidates live on.
1045 have: Vec<bool>,
1046 resident: bool,
1047 /// Rows whose entries aren't on a page yet; the next flush writes them.
1048 /// A row here has no page entry (its old one was released).
1049 hot: std::collections::HashMap<usize, (Vec<u16>, f32)>,
1050 /// Rows to write, in order (may repeat; the flush skips repeats).
1051 pending: Vec<usize>,
1052 /// Vector page → the live rows whose entries it holds.
1053 page_rows: std::collections::HashMap<u64, Vec<usize>>,
1054}
1055
1056impl VecColumn {
1057 fn from_catalog(catalog: &[CatalogEntry]) -> VecColumn {
1058 let mut col = VecColumn::default();
1059 for (i, e) in catalog.iter().enumerate() {
1060 if e.deleted {
1061 continue;
1062 }
1063 if let Some(r) = e.v9 {
1064 if r.vec_page != 0 {
1065 col.page_rows.entry(r.vec_page).or_default().push(i);
1066 }
1067 }
1068 }
1069 col
1070 }
1071}
1072
1073/// Frozen v4 catalog entry shape, used only by the v4→v5 migration path in
1074/// [`Mnemo::open`]. Its layout matches the catalog encoding written by
1075/// pre-v5 builds (5 positional fields). Do not change this struct.
1076#[derive(Serialize, Deserialize, Clone, Debug)]
1077struct CatalogEntryV4 {
1078 id: u128,
1079 start_page: u64,
1080 page_count: u32,
1081 len: u32,
1082 deleted: bool,
1083}
1084
1085/// Returned by [`Mnemo::prepare_for_flush`] and consumed by [`Mnemo::flush`].
1086/// Carries the already-serialized control-plane bytes through the prelude
1087/// so we don't re-serialize after the lease has been persisted.
1088struct FlushPrelude {
1089 cat_bytes: Option<Vec<u8>>,
1090 /// FMT-9c.2: `cat_bytes` is a delta run, not a base catalog.
1091 cat_delta: bool,
1092 /// FMT-9c.3: `idx_bytes` is an index delta run, not a whole index.
1093 idx_delta: bool,
1094 /// FMT-9d: this commit's BM25 segment, and the segments it replaces.
1095 bm25_seg: Option<(Vec<u8>, Option<usize>)>,
1096 idx_bytes: Option<Vec<u8>>,
1097 /// Serialized result-cache directory bytes when `dirty_cache` is
1098 /// set (Phase 10.1). `None` if the cache didn't change since the
1099 /// last flush — in that case the header keeps its current
1100 /// `cache_start` / `cache_pages` / `cache_len` and no fresh cache
1101 /// run is written.
1102 cache_bytes: Option<Vec<u8>>,
1103}
1104
1105/// One entry in the append-only snapshot manifest. Because record, catalog,
1106/// and index pages are only ever *appended*, the runs a past flush wrote are
1107/// still on disk; a `Snapshot` is the set of pointers needed to reconstruct
1108/// the database exactly as that flush left it.
1109#[derive(Serialize, Deserialize, Clone, Debug)]
1110struct Snapshot {
1111 txn_id: u64,
1112 created_at: i64,
1113 catalog_start: u64,
1114 catalog_pages: u64,
1115 catalog_len: u64,
1116 index_start: u64,
1117 index_pages: u64,
1118 index_len: u64,
1119 memory_count: u64,
1120 /// FMT-9: generations of the catalog and index runs (v9 only; `None`,
1121 /// and left out of the encoding, on v8 files).
1122 #[serde(default, skip_serializing_if = "Option::is_none")]
1123 gens: Option<SnapshotGens>,
1124}
1125
1126/// FMT-9: the generations a v9 snapshot's runs were sealed under.
1127#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default)]
1128struct SnapshotGens {
1129 catalog: u64,
1130 index: u64,
1131 /// FMT-9c.2: the catalog's delta-chain head at this snapshot.
1132 #[serde(default)]
1133 delta: (u64, u64, u64, u64),
1134 /// FMT-9c.3: the index's delta-chain head at this snapshot.
1135 #[serde(default)]
1136 index_delta: (u64, u64, u64, u64),
1137 /// FMT-9d: the BM25 manifest at this snapshot.
1138 #[serde(default)]
1139 bm25: (u64, u64, u64, u64),
1140}
1141
1142impl Snapshot {
1143 fn catalog_gen(&self) -> u64 {
1144 self.gens.map_or(0, |g| g.catalog)
1145 }
1146 fn index_gen(&self) -> u64 {
1147 self.gens.map_or(0, |g| g.index)
1148 }
1149 fn delta_head(&self) -> RunPtr {
1150 self.gens.map_or(RunPtr::default(), |g| tuple_ptr(g.delta))
1151 }
1152 fn index_delta_head(&self) -> RunPtr {
1153 self.gens
1154 .map_or(RunPtr::default(), |g| tuple_ptr(g.index_delta))
1155 }
1156}
1157
1158/// A restorable point in a database's history — see [`Mnemo::snapshots`].
1159#[derive(Clone, Copy, Debug)]
1160pub struct SnapshotInfo {
1161 /// Id of the transaction that produced this snapshot (monotonic from 1).
1162 pub txn_id: u64,
1163 /// When the snapshot was committed (unix seconds).
1164 pub created_at: i64,
1165 /// Live memory count captured in the snapshot.
1166 pub memory_count: u64,
1167}
1168
1169/// A retrieval request for [`Mnemo::recall`].
1170#[derive(Clone, Debug)]
1171pub struct RecallRequest {
1172 /// Query embedding.
1173 pub query: Vec<f32>,
1174 /// Maximum results to return.
1175 pub top_k: usize,
1176 /// Restrict to these memory types (`None` = all types).
1177 pub memory_types: Option<Vec<MemoryType>>,
1178 /// Restrict to a single agent's view: its own memories plus shared ones.
1179 pub agent_id: Option<String>,
1180 /// Similarity metric.
1181 pub metric: Metric,
1182 /// Multi-signal score weights.
1183 pub weights: ScoreWeights,
1184 /// Index override: partitions to probe (`None` = index default). Ignored
1185 /// when no ANN index is present.
1186 pub n_probe: Option<usize>,
1187 /// Index override: candidates to rerank (`None` = index default). Ignored
1188 /// when no ANN index is present.
1189 pub n_rerank: Option<usize>,
1190 /// Whether to update each result's `accessed_at` and `access_count` in
1191 /// the catalog. Defaults to `true`, matching pre-v5 behavior.
1192 ///
1193 /// Set to `false` for **fully read-only recall** — useful for batch
1194 /// scoring, dry-run scoring, or recalls run by introspection tooling
1195 /// (e.g. `mnemo about` consumers) where you don't want the score's own
1196 /// observation to perturb the database. With `false`, recall does not
1197 /// dirty the catalog and the next `flush` is a no-op.
1198 pub track_access: bool,
1199 /// Phase 12.6 pre-scoring filter: `session_id` + AND-ed metadata
1200 /// conditions applied before the candidate is scored. Default is
1201 /// [`Filter::default`] (no constraints). When the ANN index is on
1202 /// and this filter is selective enough to starve the shortlist
1203 /// below `top_k`, Phase 12.1's widen-then-exact-scan logic kicks
1204 /// in so the result still carries up to `top_k` correct hits.
1205 pub filter: Filter,
1206 /// Phase 12.8 post-scoring budget: stop adding hits once the UTF-8
1207 /// bytes of their `content` fields (not the serialized record
1208 /// body) would exceed this many bytes. `None` (default) means no
1209 /// budget and every hit through `top_k` is returned. If the top-1
1210 /// hit is already over budget it is still returned — callers that
1211 /// need an absolute cap should also pre-size `top_k`.
1212 ///
1213 /// Counts only the UTF-8 `content` bytes so the number is stable
1214 /// across the Rust / Python / CLI / MCP surfaces; the JSON or
1215 /// MessagePack envelope is excluded. Roughly 4 bytes per BPE
1216 /// token for English text — a 4 KB budget comfortably holds ~1K
1217 /// tokens.
1218 ///
1219 /// Observable through [`RecallResponse::truncated`]
1220 /// ([`Mnemo::recall_budgeted`]): `true` iff at least one scored
1221 /// hit was dropped by this budget, `false` otherwise.
1222 pub max_bytes: Option<usize>,
1223 /// FMT-9d (Phase 2, v9 files): **identifiers or exact terms to match**
1224 /// (a version, a PR number, a function name) for hybrid recall. Don't
1225 /// paste the question: the vector covers its meaning, and its common
1226 /// words would match the wrong records. BM25 scores, divided by this
1227 /// query's best, are added to each candidate's similarity with weight
1228 /// [`RecallRequest::keyword_weight`] before the usual scoring. With an
1229 /// empty `query` vector ([`RecallRequest::keywords_only`]), recall is
1230 /// keyword-only. `None` (the default) is plain vector recall.
1231 pub keywords: Option<String>,
1232 /// FMT-9d: weight ε of the normalised BM25 score in hybrid recall
1233 /// (default [`DEFAULT_KEYWORD_WEIGHT`]).
1234 pub keyword_weight: f32,
1235 /// FMT-9d: BM25 term-frequency saturation `k1` (default 1.2).
1236 pub bm25_k1: f32,
1237 /// FMT-9d: BM25 length normalisation `b` (default 0.75).
1238 pub bm25_b: f32,
1239}
1240
1241impl RecallRequest {
1242 /// A request with sensible defaults for a query embedding.
1243 pub fn new(query: Vec<f32>) -> Self {
1244 Self {
1245 query,
1246 top_k: 10,
1247 memory_types: None,
1248 agent_id: None,
1249 metric: Metric::Cosine,
1250 weights: ScoreWeights::default(),
1251 n_probe: None,
1252 n_rerank: None,
1253 track_access: true,
1254 filter: Filter::default(),
1255 max_bytes: None,
1256 keywords: None,
1257 keyword_weight: DEFAULT_KEYWORD_WEIGHT,
1258 bm25_k1: 1.2,
1259 bm25_b: 0.75,
1260 }
1261 }
1262
1263 /// FMT-9d (v9 files): a keyword-only request: BM25 over `keywords`
1264 /// (identifiers or exact terms), no query vector, so it works without
1265 /// an embedder.
1266 pub fn keywords_only(keywords: impl Into<String>) -> Self {
1267 Self::new(Vec::new()).keywords(keywords)
1268 }
1269
1270 /// FMT-9d (v9 files): add identifiers or exact terms to match (hybrid
1271 /// recall). Pass identifiers, not the question. See the `keywords`
1272 /// field.
1273 pub fn keywords(mut self, text: impl Into<String>) -> Self {
1274 self.keywords = Some(text.into());
1275 self
1276 }
1277
1278 /// FMT-9d: the weight ε of the keyword score in hybrid recall.
1279 pub fn keyword_weight(mut self, w: f32) -> Self {
1280 self.keyword_weight = w;
1281 self
1282 }
1283
1284 /// Phase 12.6: attach a pre-scoring filter. See [`Filter`] and
1285 /// [`FilterTerm`] for term shapes.
1286 pub fn filter(mut self, filter: Filter) -> Self {
1287 self.filter = filter;
1288 self
1289 }
1290
1291 /// Phase 12.8: cap the total UTF-8 bytes of returned `content`
1292 /// fields. See [`RecallRequest::max_bytes`].
1293 pub fn max_bytes(mut self, n: usize) -> Self {
1294 self.max_bytes = Some(n);
1295 self
1296 }
1297
1298 /// Build a request by embedding `query_text` with the supplied
1299 /// embedder (Phase 3). Shorthand for `embedder.embed(&[text])` +
1300 /// [`RecallRequest::new`]; same defaults as `new`. Fails if the
1301 /// embedder returns an empty vector list or an embedding failure.
1302 pub fn from_text(embedder: &dyn Embedder, query_text: &str) -> Result<Self> {
1303 let mut vectors = embedder.embed(&[query_text])?;
1304 let v = vectors.pop().ok_or_else(|| {
1305 MnemoError::Invalid("embedder returned no vector for the query text".into())
1306 })?;
1307 Ok(Self::new(v))
1308 }
1309 /// Set the result cap.
1310 pub fn top_k(mut self, k: usize) -> Self {
1311 self.top_k = k;
1312 self
1313 }
1314 /// Restrict to specific memory types.
1315 pub fn types(mut self, t: Vec<MemoryType>) -> Self {
1316 self.memory_types = Some(t);
1317 self
1318 }
1319 /// Restrict to one agent's view.
1320 pub fn agent(mut self, agent_id: impl Into<String>) -> Self {
1321 self.agent_id = Some(agent_id.into());
1322 self
1323 }
1324 /// Set the similarity metric (default: cosine).
1325 pub fn metric(mut self, metric: Metric) -> Self {
1326 self.metric = metric;
1327 self
1328 }
1329 /// Replace the multi-signal score weights.
1330 pub fn weights(mut self, weights: ScoreWeights) -> Self {
1331 self.weights = weights;
1332 self
1333 }
1334 /// Override the number of IVF partitions probed (accuracy/speed dial).
1335 pub fn n_probe(mut self, n: usize) -> Self {
1336 self.n_probe = Some(n);
1337 self
1338 }
1339 /// Override the number of candidates reranked exactly (accuracy dial).
1340 pub fn n_rerank(mut self, n: usize) -> Self {
1341 self.n_rerank = Some(n);
1342 self
1343 }
1344 /// Toggle whether recall updates `accessed_at` / `access_count` on
1345 /// the returned memories' catalog entries (default `true`). Pass
1346 /// `false` for a fully read-only recall — see the field doc.
1347 pub fn track_access(mut self, track: bool) -> Self {
1348 self.track_access = track;
1349 self
1350 }
1351}
1352
1353/// One scored result from [`Mnemo::recall`].
1354#[derive(Clone, Debug)]
1355pub struct RecallResult {
1356 /// The retrieved memory.
1357 pub memory: Memory,
1358 /// Combined multi-signal score.
1359 pub score: f32,
1360 /// The bare similarity component (before other signals).
1361 pub similarity: f32,
1362}
1363
1364/// Phase 12.8 budgeted-recall envelope returned by
1365/// [`Mnemo::recall_budgeted`]. Wraps the `Vec<RecallResult>` the
1366/// legacy [`Mnemo::recall`] returns so callers can tell whether
1367/// [`RecallRequest::max_bytes`] cut anything.
1368#[derive(Clone, Debug)]
1369pub struct RecallResponse {
1370 /// Scored hits, best first, truncated to at most `top_k` and then
1371 /// to fit the `max_bytes` budget if set.
1372 pub hits: Vec<RecallResult>,
1373 /// `true` iff the `max_bytes` budget dropped at least one hit
1374 /// that would otherwise have been returned (i.e. the result is
1375 /// shorter than what `top_k` alone would produce).
1376 pub truncated: bool,
1377 /// Phase 12.1 + 12.6 observability: `true` iff recall fell
1378 /// through to an exact scan (ANN shortlist too narrow after
1379 /// filtering, and the widen pass also came up short, so the
1380 /// index was skipped for one full-corpus pass). Monitoring on
1381 /// this flag is how callers detect selective-filter regressions
1382 /// or supersession chains outgrowing the ANN shortlist.
1383 pub used_exact_scan: bool,
1384}
1385
1386/// Where a file's bytes and an open handle's heap go (P-0 device budget,
1387/// `examples/device_budget.rs`). **Unstable diagnostic**: fields may
1388/// change in any release. File figures come from the header and the
1389/// catalog. RAM figures are buffer capacities, with hash maps estimated
1390/// from their capacity; they cover the engine's main structures, not
1391/// every allocation, and not allocator overhead.
1392#[doc(hidden)]
1393#[derive(Clone, Debug, Default, serde::Serialize)]
1394pub struct BudgetReport {
1395 pub file_bytes: u64,
1396 pub page_size: u64,
1397 /// Page 0.
1398 pub header_bytes: u64,
1399 /// Whole pages holding live records (superseded versions included).
1400 pub record_page_bytes: u64,
1401 /// Serialized bytes of those records; the rest of their pages is
1402 /// slack.
1403 pub record_payload_bytes: u64,
1404 /// Pages still holding soft-deleted records (reclaimed by compact).
1405 pub deleted_record_page_bytes: u64,
1406 pub catalog_run_bytes: u64,
1407 pub catalog_payload_bytes: u64,
1408 pub index_run_bytes: u64,
1409 pub index_payload_bytes: u64,
1410 /// First page of the persisted index run (0 = none).
1411 pub index_start_page: u64,
1412 pub wal_bytes: u64,
1413 pub snapshot_manifest_bytes: u64,
1414 pub cache_dir_run_bytes: u64,
1415 /// Everything else: superseded catalog, index and directory runs from
1416 /// earlier flushes, and freed pages, until the next compact.
1417 pub unreferenced_bytes: u64,
1418 pub live_records: u64,
1419 pub ram_catalog: u64,
1420 pub ram_id_map: u64,
1421 pub ram_ann_centroids: u64,
1422 pub ram_ann_codebook: u64,
1423 pub ram_ann_posting_ids: u64,
1424 pub ram_ann_posting_codes: u64,
1425 pub ram_ann_assignment: u64,
1426 pub ram_cache_dir: u64,
1427 pub ram_key_caches: u64,
1428 /// FMT-9b: the resident fp16 vector column as allocated (rows not yet
1429 /// touched by recall cost no physical memory).
1430 pub ram_vector_column: u64,
1431 /// Sum of the `ram_*` fields.
1432 pub ram_total: u64,
1433}
1434
1435/// Summary statistics for a database.
1436#[derive(Clone, Debug)]
1437pub struct Stats {
1438 /// Live (non-deleted) memory count.
1439 pub memories: usize,
1440 /// Tombstoned entries awaiting compaction.
1441 pub deleted: usize,
1442 /// Embedding dimensionality.
1443 pub dimensions: usize,
1444 /// File size in bytes.
1445 pub file_bytes: u64,
1446 /// Distinct agent IDs present.
1447 pub agents: Vec<String>,
1448 /// Whether pages are encrypted (always true in v1).
1449 pub encrypted: bool,
1450 /// Creation time (unix seconds).
1451 pub created_at: i64,
1452 /// ANN index shape, if an index has been built.
1453 pub index: Option<IndexInfo>,
1454 /// Current size of the write-ahead log region, in 8 KiB pages.
1455 pub wal_pages: u64,
1456 /// Fsyncs issued by the most recent successful [`Mnemo::flush`] on
1457 /// this handle. `0` on a fresh handle before any flush, or when
1458 /// `flush()` short-circuited (nothing dirty). Phase 6 instrumentation
1459 /// for the Phase 11.1 write-latency work — the planner wants this
1460 /// before deciding whether cutting fsync count is a viable
1461 /// durability-preserving fix.
1462 pub fsyncs_last_flush: u32,
1463 /// FMT-9f: the file's on-disk format (`8` or `9`). A v8 file still
1464 /// opens and writes as v8; `mnemo compact` upgrades it to v9.
1465 pub format_version: u16,
1466}
1467
1468/// Result of a [`Mnemo::compact_file`] run.
1469#[derive(Clone, Copy, Debug, Default)]
1470pub struct CompactReport {
1471 /// Live memories before compaction.
1472 pub before: usize,
1473 /// Live memories after compaction (expired ones dropped).
1474 pub after: usize,
1475 /// Live, non-expired cache entries before compaction. FIX-1
1476 /// R-001.1: compaction now preserves the result cache instead of
1477 /// silently dropping it.
1478 pub cache_before: usize,
1479 /// Live, non-expired cache entries after compaction (tombstoned
1480 /// and TTL-expired ones dropped).
1481 pub cache_after: usize,
1482}
1483
1484/// An encrypted, single-file agent memory database.
1485pub struct Mnemo {
1486 pager: Pager,
1487 header: Header,
1488 catalog: Vec<CatalogEntry>,
1489 index: IdIndex,
1490 #[allow(dead_code)]
1491 path: PathBuf,
1492 dimensions: usize,
1493 kdf: KdfParams,
1494 /// Set whenever the catalog changes; drives whether `flush` rewrites it.
1495 dirty_catalog: bool,
1496 /// Optional IVF+PQ approximate-nearest-neighbour index.
1497 ann: Option<IvfPqIndex>,
1498 /// Set whenever the in-memory ANN index differs from the persisted one.
1499 /// PERF-1: `flush` persists it only per [`IndexPersist`]'s policy.
1500 dirty_index: bool,
1501 /// PERF-1: index persistence policy state.
1502 index_persist: IndexPersist,
1503 /// PERF-1 step 3: the header points at a persisted ANN index that
1504 /// hasn't been loaded yet. It loads on first use (`ensure_ann`).
1505 ann_unloaded: bool,
1506 /// Append-only manifest of committed snapshots, oldest first. Capped
1507 /// at [`Mnemo::max_snapshots`] entries on every flush — the oldest
1508 /// entries are pruned first.
1509 manifest: Vec<Snapshot>,
1510 /// Maximum manifest entries to retain across flushes. `0` disables
1511 /// the cap (retain forever). Defaults to [`DEFAULT_MAX_SNAPSHOTS`].
1512 max_snapshots: usize,
1513 /// If `true`, this handle was opened via [`Mnemo::open_read_only`] and
1514 /// every mutating API returns [`MnemoError::ReadOnly`]. The underlying
1515 /// file was opened without write permission and holds an OS shared
1516 /// lock; the OS would reject a write() anyway, but gating in Rust
1517 /// gives callers a clean, typed error.
1518 read_only: bool,
1519 /// Result-cache directory (Phase 10.1). See [`crate::result_cache`].
1520 /// Loaded from `header.cache_start` on open; persisted on `flush`
1521 /// via the same seal_run pipeline as catalog/index/manifest.
1522 cache_dir: Vec<CacheDirectoryEntry>,
1523 /// In-memory index over `cache_dir` for O(1) `(namespace, key_hash)`
1524 /// lookups. Rebuilt on open + kept in sync on `cache_put`/`delete`.
1525 cache_idx: DirectoryIndex,
1526 /// Set whenever `cache_dir` changes; drives whether `flush` rewrites
1527 /// the cache directory run.
1528 dirty_cache: bool,
1529 /// Per-namespace eviction budgets. Missing entries fall back to
1530 /// [`CacheBudget::default`]. Not persisted (per-handle policy).
1531 cache_budgets: std::collections::HashMap<String, CacheBudget>,
1532 /// Flush policy for cache mutations. Memory writes ignore this and
1533 /// always behave as `Strict`.
1534 cache_flush_policy: CacheFlushPolicy,
1535 /// Bookkeeping for `CacheFlushPolicy::Batched` — pending mutation
1536 /// count and first-dirty timestamp.
1537 cache_batch: BatchState,
1538 /// In-memory hit/miss/eviction counters (reset on reopen).
1539 cache_counters: CounterState,
1540 /// Optional handle-scoped embedder (Phase 3, PR 10a). Registered
1541 /// via [`Mnemo::set_embedder`]; consulted by [`Mnemo::remember_text`]
1542 /// and [`RecallRequest::from_text`]. `None` means text-side APIs
1543 /// return [`MnemoError::Invalid`] with a "no embedder" message.
1544 /// Not persisted — the handle is re-registered after every open.
1545 embedder: Option<std::sync::Arc<dyn Embedder>>,
1546 /// Fsyncs issued by the most recent `flush()` call. Snapshotted
1547 /// from the thread-local fsync counter in `lib.rs::fsync_counter`:
1548 /// `flush()` reads it on entry, does its work (each `sync_all` the
1549 /// pager / wal / recovery path makes bumps the counter), and
1550 /// stores the delta here. `0` on a fresh handle before any flush,
1551 /// or on a read-only handle where `flush()` short-circuits.
1552 /// Exposed through [`Stats::fsyncs_last_flush`] for diagnostics —
1553 /// the Phase 6 planner wants this before deciding whether cutting
1554 /// fsync count is a viable write-latency win.
1555 fsyncs_last_flush: u32,
1556 /// FIX-1 R-001.8 part 2: lazy cache of the distinct agent IDs
1557 /// across every live memory, built once on first `stats()` call
1558 /// per handle. `remember()` adds to it in O(1); `delete()` can't
1559 /// cheaply tell whether the removed memory was the last of its
1560 /// agent (the agent lives in the encrypted body, not the catalog
1561 /// — pre-v9 format), so it just invalidates the set and the next
1562 /// `stats()` call rebuilds. Pre-FIX-1 every `stats()` call
1563 /// decrypted every live memory to collect agents, turning an
1564 /// otherwise-cheap diagnostic into an O(N) decrypt.
1565 agent_set: Option<std::collections::BTreeSet<String>>,
1566 /// Phase 12.1 lazy caches, built together on the first keyed op this
1567 /// handle sees (`remember_keyed`, `update` that touches `_mnemo.key`,
1568 /// or any `history(...)` call). Both are maintained incrementally
1569 /// after that. `None` → not built yet; `Some(…)` → authoritative.
1570 ///
1571 /// * `key_index`: `(scope_owner, key) → ULID bytes` of the **live**
1572 /// record for each keyed slot. `scope_owner` is `Some(agent_id)`
1573 /// for a `Private` scope slot and `None` for a `Shared` one, so a
1574 /// private write never collides with a shared one.
1575 /// * `superseded`: the ULID bytes of every record that has been
1576 /// superseded (`_mnemo.superseded_by` set on it). `recall` and
1577 /// `about` skip these before any decrypt.
1578 ///
1579 /// Building both costs one scan over live records (~1s at 100K with
1580 /// hardware AES) and happens at most once per handle. A format bump
1581 /// to v9 moves the key into the catalog row so the scan goes away.
1582 key_index: Option<std::collections::HashMap<KeySlot, u128>>,
1583 superseded: Option<std::collections::HashSet<u128>>,
1584 /// R-004.2: thread layers + onboarding ids, built by the same lazy scan.
1585 layers: Option<LayerCache>,
1586 /// FMT-9e: thread turns are compressed once older than this many
1587 /// seconds (`0`: at write). Per handle, not persisted.
1588 turn_compress_after: i64,
1589 /// FMT-9 (a): the committed free list (v9; always empty on v8). Pages
1590 /// allocated from it since the last commit are already removed.
1591 free: FreeList,
1592 /// FMT-9 (a): runs this transaction stopped referencing (overwritten
1593 /// or deleted records, tombstoned cache entries, a relocated WAL
1594 /// region). The next commit adds them to the free list, freed at its
1595 /// own transaction id.
1596 pending_free: Vec<(u64, u64)>,
1597 /// FMT-9 (a): the free list changed (an allocation took pages from it)
1598 /// and must be rewritten by the next commit.
1599 free_dirty: bool,
1600 /// FMT-9 (b): the key-slot run (v9; empty until Phase 13.2 adds slots).
1601 key_slots: KeySlotRun,
1602 /// FMT-9 (b): `key_slots` changed since the last commit.
1603 dirty_keys: bool,
1604 /// FMT-9: this writable session has burned a transaction id with a
1605 /// durable header-only commit (`ensure_gen_lease`), so the generations
1606 /// its pages carry can't repeat ones a crashed earlier session used.
1607 gen_leased: bool,
1608 /// FMT-9b: the fp16 vector column (v9; unused on v8).
1609 col: VecColumn,
1610 /// FMT-9b.2: packed record pages (v9; unused on v8).
1611 recs: RecPages,
1612 /// FMT-9d: the BM25 keyword index (v9; unused on v8).
1613 bm25: Bm25State,
1614 /// FMT-9d: test hook: write this layout byte instead of the current
1615 /// one (to exercise the unknown-layout path).
1616 index_layout_override: Option<u8>,
1617 /// FMT-9d: during one recall, each keyword match's normalised BM25
1618 /// score, by id.
1619 kw: Option<std::collections::HashMap<u128, f32>>,
1620 /// FMT-9c.2: catalog rows changed since the last commit (v9 writes
1621 /// them as a delta run instead of the whole catalog).
1622 dirty_rows: std::collections::BTreeSet<usize>,
1623 /// FMT-9c.2: the delta runs on top of the base catalog, oldest first,
1624 /// and how many rows they hold in all.
1625 delta_chain: Vec<RunPtr>,
1626 delta_rows: usize,
1627 /// FMT-9c.3: the ANN index's delta runs on top of its base run, oldest
1628 /// first, and how many changes they hold in all.
1629 index_chain: Vec<RunPtr>,
1630 index_chain_ops: usize,
1631 /// FMT-9c.3: the next commit writes the whole index (after a build,
1632 /// rebuild or restore).
1633 index_rebase: bool,
1634 /// FMT-9c.3: this flush's index changes, when it writes them as a
1635 /// delta run.
1636 index_delta_ops: Option<Vec<IndexOp>>,
1637 /// FMT-9c.2: the next commit writes a whole base catalog (after a
1638 /// restore, or when the chain has grown too long).
1639 catalog_rewrite: bool,
1640 /// FMT-9c: the snapshot count was set explicitly (config or
1641 /// `set_max_snapshots`), which replaces v9's default retention policy.
1642 snapshot_cap_explicit: bool,
1643 /// FMT-9c: this flush wrote big runs straight to their home pages,
1644 /// which must be fsynced before the commit.
1645 direct_writes: bool,
1646 /// FMT-9b: keep the column resident while it fits this many bytes.
1647 vector_cache_budget: usize,
1648}
1649
1650/// Composite key for the lazy `key_index` on `Mnemo`. A slot is identified
1651/// by `(scope_owner, key)`:
1652/// * `Scope::Private` writes carry the owning agent → `Some(agent_id)`.
1653/// * `Scope::Shared` writes are per-file → `None`.
1654///
1655/// A private write never supersedes a shared one, and the reverse is also
1656/// true. Scope lives on `Memory.scope`, agent on `Memory.agent_id`.
1657#[derive(Debug, Clone, PartialEq, Eq, Hash)]
1658pub(crate) struct KeySlot {
1659 /// `Some(agent_id)` for `Scope::Private`, `None` for `Scope::Shared`.
1660 pub(crate) scope_owner: Option<String>,
1661 /// The caller-supplied key string, exactly as given (no case-fold).
1662 pub(crate) key: String,
1663}
1664
1665/// Open a file and take an OS advisory lock, returning `Locked { path }`
1666/// if the requested lock cannot be acquired. On drop of the returned
1667/// [`File`], the OS releases the lock. `flock`/`LockFileEx` semantics
1668/// (per file-descriptor on Unix, per-open-file-handle on Windows) mean
1669/// the lock is scoped to this handle: cloning the file (or the returned
1670/// `File`) shares the lock, but a fresh `open()` from any process gets a
1671/// distinct handle and will collide.
1672fn open_with_lock(path: &Path, write: bool, create: bool) -> Result<File> {
1673 let mut opts = OpenOptions::new();
1674 opts.read(true);
1675 if write {
1676 opts.write(true);
1677 }
1678 if create {
1679 opts.create(true).truncate(true);
1680 }
1681 let file = opts.open(path)?;
1682 // On success the returned `Result<()>` is `Ok(())`. On conflict:
1683 // * Unix `flock(2)` → `ErrorKind::WouldBlock` (EWOULDBLOCK / EAGAIN)
1684 // * Windows `LockFileEx` → raw OS code 33 (ERROR_LOCK_VIOLATION)
1685 // or 32 (ERROR_SHARING_VIOLATION), both surfaced by Rust as
1686 // `ErrorKind::Uncategorized` because `std::io::ErrorKind`
1687 // doesn't carry a portable "locked" variant.
1688 // Anything else is a real I/O failure (permissions, invalid handle,
1689 // ...) and gets bubbled up as `MnemoError::Io`.
1690 let result = if write {
1691 FileExt::try_lock_exclusive(&file)
1692 } else {
1693 FileExt::try_lock_shared(&file)
1694 };
1695 match result {
1696 Ok(()) => Ok(file),
1697 Err(e) if is_lock_conflict(&e) => Err(MnemoError::Locked {
1698 path: path.to_path_buf(),
1699 }),
1700 Err(e) => Err(MnemoError::Io(e)),
1701 }
1702}
1703
1704/// Do two paths resolve to the same on-disk file? Used by
1705/// [`Mnemo::export_encrypted`] to refuse overwriting the live
1706/// database. On Unix we compare `(dev, ino)` via `MetadataExt`, which
1707/// catches hard links and symlinks. On Windows we compare canonicalized
1708/// paths, since there is no stable device+inode equivalent in `std`.
1709/// Returns `None` if either side can't be stat'd / canonicalized —
1710/// the caller treats that as "not the same file" so a missing `dest`
1711/// (the normal export case) proceeds.
1712fn paths_refer_to_same_file(a: &Path, b: &Path) -> Option<bool> {
1713 #[cfg(unix)]
1714 {
1715 use std::os::unix::fs::MetadataExt;
1716 let ma = std::fs::metadata(a).ok()?;
1717 let mb = std::fs::metadata(b).ok()?;
1718 Some(ma.dev() == mb.dev() && ma.ino() == mb.ino())
1719 }
1720 #[cfg(not(unix))]
1721 {
1722 let ca = std::fs::canonicalize(a).ok()?;
1723 let cb = std::fs::canonicalize(b).ok()?;
1724 Some(ca == cb)
1725 }
1726}
1727
1728/// A sibling temp-file path next to `dest`, suffixed with
1729/// `.export-tmp`. Lives in `dest`'s own parent directory so the final
1730/// [`std::fs::rename`] is same-filesystem (and therefore atomic on
1731/// every platform mnemo supports).
1732fn temp_sibling(dest: &Path) -> PathBuf {
1733 let mut buf = dest.as_os_str().to_os_string();
1734 buf.push(".export-tmp");
1735 PathBuf::from(buf)
1736}
1737
1738/// Was this I/O error the OS reporting a conflicting file lock, or a
1739/// genuine I/O failure? Platform-conditional because
1740/// [`std::io::ErrorKind`] carries no portable "resource-locked" variant
1741/// as of Rust 1.75 — Windows surfaces `ERROR_LOCK_VIOLATION` /
1742/// `ERROR_SHARING_VIOLATION` as `Uncategorized`.
1743fn is_lock_conflict(e: &std::io::Error) -> bool {
1744 if e.kind() == std::io::ErrorKind::WouldBlock {
1745 return true;
1746 }
1747 #[cfg(windows)]
1748 if let Some(code) = e.raw_os_error() {
1749 // 33 = ERROR_LOCK_VIOLATION, 32 = ERROR_SHARING_VIOLATION —
1750 // both mean "someone else holds an incompatible lock on this
1751 // region of the file". See
1752 // https://learn.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-
1753 if code == 33 || code == 32 {
1754 return true;
1755 }
1756 }
1757 false
1758}
1759
1760/// Read a run of consecutive encrypted pages and concatenate their plaintext.
1761/// `gen` is the generation the run's pointer expects (v9; `0` before).
1762fn read_run_bytes(
1763 pager: &mut Pager,
1764 start: u64,
1765 pages: u64,
1766 len: u64,
1767 gen: u64,
1768) -> Result<Vec<u8>> {
1769 let mut buf = Vec::with_capacity(len as usize);
1770 for i in 0..pages {
1771 buf.extend_from_slice(&pager.read_page(start + i, gen)?);
1772 }
1773 buf.truncate(len as usize);
1774 Ok(buf)
1775}
1776
1777/// FMT-9b: a page run read as a byte stream, one page at a time and
1778/// without caching it, so loading a large run (a 100K catalog is several
1779/// MB, its index about 17 MB) never holds the whole serialized run in
1780/// memory next to the structure being decoded from it.
1781struct RunReader<'a> {
1782 pager: &'a mut Pager,
1783 /// The first page error, kept so the caller reports it as itself
1784 /// (e.g. `PageAuthFailed`) rather than as a decode error.
1785 err: &'a std::cell::RefCell<Option<MnemoError>>,
1786 next_page: u64,
1787 end_page: u64,
1788 gen: u64,
1789 remaining: u64,
1790 buf: Vec<u8>,
1791 pos: usize,
1792}
1793
1794impl<'a> RunReader<'a> {
1795 fn new(
1796 pager: &'a mut Pager,
1797 err: &'a std::cell::RefCell<Option<MnemoError>>,
1798 start: u64,
1799 pages: u64,
1800 len: u64,
1801 gen: u64,
1802 ) -> Self {
1803 RunReader {
1804 pager,
1805 err,
1806 next_page: start,
1807 end_page: start + pages,
1808 gen,
1809 remaining: len,
1810 buf: Vec::new(),
1811 pos: 0,
1812 }
1813 }
1814}
1815
1816impl Read for RunReader<'_> {
1817 /// The decoder reads a byte or a few at a time; serve those straight
1818 /// from the current page.
1819 #[inline]
1820 fn read_exact(&mut self, out: &mut [u8]) -> std::io::Result<()> {
1821 if self.buf.len() - self.pos >= out.len() {
1822 out.copy_from_slice(&self.buf[self.pos..self.pos + out.len()]);
1823 self.pos += out.len();
1824 return Ok(());
1825 }
1826 let mut done = 0;
1827 while done < out.len() {
1828 match self.read(&mut out[done..])? {
1829 0 => return Err(std::io::ErrorKind::UnexpectedEof.into()),
1830 n => done += n,
1831 }
1832 }
1833 Ok(())
1834 }
1835
1836 #[inline]
1837 fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
1838 if self.pos == self.buf.len() {
1839 if self.remaining == 0 || self.next_page >= self.end_page {
1840 return Ok(0);
1841 }
1842 let mut page = match self.pager.read_page_uncached(self.next_page, self.gen) {
1843 Ok(p) => p,
1844 Err(e) => {
1845 let msg = e.to_string();
1846 *self.err.borrow_mut() = Some(e);
1847 return Err(std::io::Error::other(msg));
1848 }
1849 };
1850 self.next_page += 1;
1851 page.truncate((self.remaining as usize).min(page.len()));
1852 self.remaining -= page.len() as u64;
1853 self.buf = page;
1854 self.pos = 0;
1855 }
1856 let n = out.len().min(self.buf.len() - self.pos);
1857 out[..n].copy_from_slice(&self.buf[self.pos..self.pos + n]);
1858 self.pos += n;
1859 Ok(n)
1860 }
1861}
1862
1863/// FMT-9b: deserialize a sequence into a `Vec` sized from the encoded
1864/// length up front (bounded by `cap`, so a hostile length can't force a
1865/// huge allocation). serde's own `Vec` impl pre-sizes only up to 1 MiB and
1866/// then doubles, which for a 100K catalog leaves a quarter of the capacity
1867/// unused and a freed half-size copy behind in the allocator.
1868struct ExactVec<T> {
1869 cap: usize,
1870 _t: std::marker::PhantomData<T>,
1871}
1872
1873impl<'de, T: Deserialize<'de>> serde::de::DeserializeSeed<'de> for ExactVec<T> {
1874 type Value = Vec<T>;
1875 fn deserialize<D: serde::Deserializer<'de>>(
1876 self,
1877 d: D,
1878 ) -> std::result::Result<Vec<T>, D::Error> {
1879 d.deserialize_seq(self)
1880 }
1881}
1882
1883impl<'de, T: Deserialize<'de>> serde::de::Visitor<'de> for ExactVec<T> {
1884 type Value = Vec<T>;
1885 fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
1886 f.write_str("a sequence")
1887 }
1888 fn visit_seq<A: serde::de::SeqAccess<'de>>(
1889 self,
1890 mut seq: A,
1891 ) -> std::result::Result<Vec<T>, A::Error> {
1892 let mut v = Vec::with_capacity(seq.size_hint().unwrap_or(0).min(self.cap));
1893 while let Some(x) = seq.next_element()? {
1894 v.push(x);
1895 }
1896 Ok(v)
1897 }
1898}
1899
1900/// Load a v5+ catalog run (an empty `pages` yields an empty catalog).
1901fn load_catalog(
1902 pager: &mut Pager,
1903 start: u64,
1904 pages: u64,
1905 len: u64,
1906 gen: u64,
1907) -> Result<Vec<CatalogEntry>> {
1908 if pages == 0 {
1909 return Ok(Vec::new());
1910 }
1911 // Sized exactly (every row encodes to at least 8 bytes, which bounds
1912 // the up-front allocation by the run's length). v8 decodes from one
1913 // buffer, as before; v9 streams the run (it's larger, and it's paired
1914 // with a resident vector column, so the transient buffer costs RSS).
1915 let cap = (len / 8) as usize;
1916 if pager.version() < VERSION_V9 {
1917 let buf = read_run_bytes(pager, start, pages, len, gen)?;
1918 let mut de = rmp_serde::Deserializer::from_read_ref(&buf);
1919 return serde::de::DeserializeSeed::deserialize(
1920 ExactVec::<CatalogEntry> {
1921 cap,
1922 _t: std::marker::PhantomData,
1923 },
1924 &mut de,
1925 )
1926 .map_err(|e| MnemoError::Serialize(e.to_string()));
1927 }
1928 let err = std::cell::RefCell::new(None);
1929 let mut de = rmp_serde::Deserializer::new(RunReader::new(pager, &err, start, pages, len, gen));
1930 let res = serde::de::DeserializeSeed::deserialize(
1931 ExactVec::<CatalogEntry> {
1932 cap,
1933 _t: std::marker::PhantomData,
1934 },
1935 &mut de,
1936 );
1937 drop(de);
1938 match (res, err.into_inner()) {
1939 (Ok(v), _) => Ok(v),
1940 (Err(_), Some(page_err)) => Err(page_err),
1941 (Err(e), None) => Err(MnemoError::Serialize(e.to_string())),
1942 }
1943}
1944
1945/// FMT-9c.2: one delta run of a v9 catalog: the rows a commit changed or
1946/// added, by position, and the previous delta in the chain.
1947#[derive(Serialize, Deserialize)]
1948struct CatalogDelta {
1949 prev: (u64, u64, u64, u64),
1950 rows: Vec<(u32, CatalogEntry)>,
1951}
1952
1953fn ptr_tuple(p: RunPtr) -> (u64, u64, u64, u64) {
1954 (p.start, p.pages, p.len, p.gen)
1955}
1956
1957fn tuple_ptr(t: (u64, u64, u64, u64)) -> RunPtr {
1958 RunPtr {
1959 start: t.0,
1960 pages: t.1,
1961 len: t.2,
1962 gen: t.3,
1963 }
1964}
1965
1966/// FMT-9c.2: walk the delta chain from `head` back to the base and replay
1967/// it onto `catalog`, oldest first. Returns the catalog, the chain (oldest
1968/// first) and its total row count. An empty `head` is a no-op (and v8).
1969fn apply_delta_chain(
1970 pager: &mut Pager,
1971 mut catalog: Vec<CatalogEntry>,
1972 head: RunPtr,
1973) -> Result<(Vec<CatalogEntry>, Vec<RunPtr>, usize)> {
1974 let mut deltas: Vec<(RunPtr, Vec<(u32, CatalogEntry)>)> = Vec::new();
1975 let mut at = head;
1976 while !at.is_empty() {
1977 if deltas.len() > 4096 {
1978 return Err(MnemoError::Invalid("catalog delta chain too long".into()));
1979 }
1980 let buf = read_run_bytes(pager, at.start, at.pages, at.len, at.gen)?;
1981 let d: CatalogDelta =
1982 rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
1983 deltas.push((at, d.rows));
1984 at = tuple_ptr(d.prev);
1985 }
1986 deltas.reverse();
1987 let mut rows = 0usize;
1988 let mut chain = Vec::with_capacity(deltas.len());
1989 for (ptr, d) in deltas {
1990 rows += d.len();
1991 chain.push(ptr);
1992 for (i, row) in d {
1993 let i = i as usize;
1994 match i.cmp(&catalog.len()) {
1995 std::cmp::Ordering::Less => catalog[i] = row,
1996 std::cmp::Ordering::Equal => catalog.push(row),
1997 std::cmp::Ordering::Greater => {
1998 return Err(MnemoError::Invalid("catalog delta skips a row".into()))
1999 }
2000 }
2001 }
2002 }
2003 Ok((catalog, chain, rows))
2004}
2005
2006/// FMT-9c.3: one delta run of a v9 ANN index: the changes a commit made,
2007/// and the previous delta in the chain.
2008#[derive(Serialize)]
2009struct IndexDeltaRef<'a> {
2010 prev: (u64, u64, u64, u64),
2011 ops: &'a [IndexOp],
2012}
2013
2014#[derive(Deserialize)]
2015struct IndexDeltaOwned {
2016 prev: (u64, u64, u64, u64),
2017 ops: Vec<IndexOp>,
2018}
2019
2020/// FMT-9c.3: walk the index delta chain from `head` and replay it onto a
2021/// just-loaded base index, oldest first. Returns the chain and its change
2022/// count.
2023fn apply_index_chain(
2024 pager: &mut Pager,
2025 ann: &mut IvfPqIndex,
2026 head: RunPtr,
2027) -> Result<(Vec<RunPtr>, usize)> {
2028 let mut deltas: Vec<(RunPtr, Vec<IndexOp>)> = Vec::new();
2029 let mut at = head;
2030 while !at.is_empty() {
2031 if deltas.len() > 4096 {
2032 return Err(MnemoError::Invalid("index delta chain too long".into()));
2033 }
2034 let buf = read_run_bytes(pager, at.start, at.pages, at.len, at.gen)?;
2035 let d: IndexDeltaOwned =
2036 rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
2037 deltas.push((at, d.ops));
2038 at = tuple_ptr(d.prev);
2039 }
2040 deltas.reverse();
2041 let mut chain = Vec::with_capacity(deltas.len());
2042 let mut n = 0usize;
2043 for (ptr, ops) in deltas {
2044 n += ops.len();
2045 chain.push(ptr);
2046 ann.apply_ops(ops)?;
2047 }
2048 Ok((chain, n))
2049}
2050
2051/// FMT-9f: the runs of an index delta chain whose base didn't load (a
2052/// layout this build doesn't know), oldest first, without replaying them,
2053/// so the next `build_index` or `drop_index` frees them with the base.
2054/// Best effort: a delta this build can't decode ends the walk, and the
2055/// runs before it stay unreferenced until the next compaction.
2056fn index_chain_runs(pager: &mut Pager, head: RunPtr) -> Vec<RunPtr> {
2057 let mut runs = Vec::new();
2058 let mut at = head;
2059 while !at.is_empty() && runs.len() <= 4096 {
2060 let Ok(buf) = read_run_bytes(pager, at.start, at.pages, at.len, at.gen) else {
2061 break;
2062 };
2063 let Ok(d) = rmp_serde::from_slice::<IndexDeltaOwned>(&buf) else {
2064 break;
2065 };
2066 runs.push(at);
2067 at = tuple_ptr(d.prev);
2068 }
2069 runs.reverse();
2070 runs
2071}
2072
2073/// Load a **v4** catalog run using the frozen [`CatalogEntryV4`] shape.
2074/// Used only by the v4→v5 migration path in [`Mnemo::open`].
2075fn load_catalog_v4(
2076 pager: &mut Pager,
2077 start: u64,
2078 pages: u64,
2079 len: u64,
2080) -> Result<Vec<CatalogEntryV4>> {
2081 if pages == 0 {
2082 return Ok(Vec::new());
2083 }
2084 let buf = read_run_bytes(pager, start, pages, len, 0)?;
2085 rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))
2086}
2087
2088/// Walk a v4 catalog and synthesize the v5 shape. For each live entry,
2089/// reads the memory's serialized body and copies its `accessed_at` and
2090/// `access_count` into the catalog row (the v4 record body still carries
2091/// them — they're moving *from* the body *to* the catalog). Deleted
2092/// entries skip the read and zero the fields.
2093fn migrate_v4_catalog(pager: &mut Pager, v4: Vec<CatalogEntryV4>) -> Result<Vec<CatalogEntry>> {
2094 let mut out = Vec::with_capacity(v4.len());
2095 for e in &v4 {
2096 let (accessed_at, access_count) = if e.deleted {
2097 (0i64, 0u32)
2098 } else {
2099 let mut buf = Vec::with_capacity(e.len as usize);
2100 for i in 0..e.page_count as u64 {
2101 buf.extend_from_slice(&pager.read_page(e.start_page + i, 0)?);
2102 }
2103 buf.truncate(e.len as usize);
2104 let mem: Memory = rmp_serde::from_slice(&buf)
2105 .map_err(|err| MnemoError::Serialize(err.to_string()))?;
2106 (mem.accessed_at, mem.access_count)
2107 };
2108 out.push(CatalogEntry {
2109 id: e.id,
2110 start_page: e.start_page,
2111 page_count: e.page_count,
2112 len: e.len,
2113 deleted: e.deleted,
2114 accessed_at,
2115 access_count,
2116 gen: 0,
2117 v9: None,
2118 });
2119 }
2120 Ok(out)
2121}
2122
2123/// Load the snapshot manifest (an empty `pages` yields an empty manifest).
2124fn load_manifest(
2125 pager: &mut Pager,
2126 start: u64,
2127 pages: u64,
2128 len: u64,
2129 gen: u64,
2130) -> Result<Vec<Snapshot>> {
2131 if pages == 0 {
2132 return Ok(Vec::new());
2133 }
2134 let buf = read_run_bytes(pager, start, pages, len, gen)?;
2135 rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))
2136}
2137
2138/// Load the result-cache directory (v8+). An empty `pages` yields an
2139/// empty directory, which is the natural state for v7 files migrated
2140/// to v8 (no cache entries yet) and for freshly created v8 files.
2141fn load_cache_directory(
2142 pager: &mut Pager,
2143 start: u64,
2144 pages: u64,
2145 len: u64,
2146 gen: u64,
2147) -> Result<Vec<CacheDirectoryEntry>> {
2148 if pages == 0 {
2149 return Ok(Vec::new());
2150 }
2151 let buf = read_run_bytes(pager, start, pages, len, gen)?;
2152 rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))
2153}
2154
2155/// Load an ANN index run, validating its dimensionality.
2156fn load_index(
2157 pager: &mut Pager,
2158 start: u64,
2159 pages: u64,
2160 len: u64,
2161 gen: u64,
2162 dims: usize,
2163) -> Result<Option<IvfPqIndex>> {
2164 if pages == 0 {
2165 return Ok(None);
2166 }
2167 // v8 decodes from one buffer, as before. v9 streams the run (a 100K
2168 // index is ~17 MB), trading decode speed for not holding it twice.
2169 if pager.version() < VERSION_V9 {
2170 let buf = read_run_bytes(pager, start, pages, len, gen)?;
2171 let mut idx: IvfPqIndex =
2172 rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
2173 if idx.dims() != dims {
2174 return Err(MnemoError::Invalid(
2175 "ANN index dimensionality does not match the database".into(),
2176 ));
2177 }
2178 idx.rebuild_assignment();
2179 return Ok(Some(idx));
2180 }
2181 // v9 index runs start with a layout byte. An index is derived data, so
2182 // a layout this build doesn't know means "no index" (recall scans
2183 // exactly; `rebuild_index` writes a current one), never an error.
2184 let err = std::cell::RefCell::new(None);
2185 let mut rd = RunReader::new(pager, &err, start, pages, len, gen);
2186 let mut layout = [0u8; 1];
2187 if rd.read_exact(&mut layout).is_err() {
2188 drop(rd);
2189 return match err.into_inner() {
2190 Some(page_err) => Err(page_err),
2191 None => Ok(None),
2192 };
2193 }
2194 if layout[0] != V9_INDEX_LAYOUT {
2195 return Ok(None);
2196 }
2197 let res: std::result::Result<IvfPqIndex, _> = rmp_serde::from_read(rd);
2198 let mut idx = match (res, err.into_inner()) {
2199 (Ok(i), _) => i,
2200 (Err(_), Some(page_err)) => return Err(page_err),
2201 (Err(e), None) => return Err(MnemoError::Serialize(e.to_string())),
2202 };
2203 if idx.dims() != dims {
2204 return Err(MnemoError::Invalid(
2205 "ANN index dimensionality does not match the database".into(),
2206 ));
2207 }
2208 idx.rebuild_assignment();
2209 Ok(Some(idx))
2210}
2211
2212/// FMT-9 (a): load the v9 free list (an empty pointer yields an empty list).
2213fn load_free_list(pager: &mut Pager, ptr: RunPtr) -> Result<FreeList> {
2214 if ptr.is_empty() {
2215 return Ok(FreeList::default());
2216 }
2217 let buf = read_run_bytes(pager, ptr.start, ptr.pages, ptr.len, ptr.gen)?;
2218 let runs: Vec<crate::freelist::FreeRun> =
2219 rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
2220 FreeList::from_runs(runs)
2221}
2222
2223/// FMT-9 (b): load the v9 key-slot run (an empty pointer yields no slots).
2224fn load_key_slots(pager: &mut Pager, ptr: RunPtr) -> Result<KeySlotRun> {
2225 if ptr.is_empty() {
2226 return Ok(KeySlotRun::default());
2227 }
2228 let buf = read_run_bytes(pager, ptr.start, ptr.pages, ptr.len, ptr.gen)?;
2229 let run: KeySlotRun =
2230 rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
2231 run.validate()?;
2232 Ok(run)
2233}
2234
2235/// True if a memory's metadata marks it as the canonical onboarding manifest
2236/// (`metadata.topic == "manifest"`). Used by [`Mnemo::about`] to hoist the
2237/// manifest to the top of the briefing regardless of importance ordering.
2238fn is_manifest(m: &Memory) -> bool {
2239 m.metadata
2240 .get("topic")
2241 .and_then(|v| v.as_str())
2242 .map(|s| s.eq_ignore_ascii_case("manifest"))
2243 .unwrap_or(false)
2244}
2245
2246/// PERF-1 step 4: ULID → catalog row, as an open-addressing table of `u32`
2247/// row numbers: 4 bytes a slot, about 5 bytes an entry, against about 43
2248/// for the `HashMap<u128, usize>` it replaces. A probe compares against the
2249/// catalog's own id at that row, so the ids aren't stored twice. Rows are
2250/// never removed (a delete only tombstones the catalog entry), so there's
2251/// no delete path.
2252#[derive(Debug, Clone, Default)]
2253struct IdIndex {
2254 slots: Vec<u32>,
2255 len: usize,
2256}
2257
2258const NO_ROW: u32 = u32::MAX;
2259
2260impl IdIndex {
2261 fn build(catalog: &[CatalogEntry]) -> IdIndex {
2262 let mut t = IdIndex {
2263 slots: vec![NO_ROW; crate::index::table_capacity_for(catalog.len())],
2264 len: 0,
2265 };
2266 for (row, e) in catalog.iter().enumerate() {
2267 t.insert(e.id, row, catalog);
2268 }
2269 t
2270 }
2271
2272 /// The catalog row holding `id`, if any.
2273 fn get(&self, id: u128, catalog: &[CatalogEntry]) -> Option<usize> {
2274 if self.slots.is_empty() {
2275 return None;
2276 }
2277 let cap = self.slots.len();
2278 let mut i = crate::index::slot_home(id, cap);
2279 loop {
2280 let row = self.slots[i];
2281 if row == NO_ROW {
2282 return None;
2283 }
2284 if catalog[row as usize].id == id {
2285 return Some(row as usize);
2286 }
2287 i = crate::index::next_slot(i, cap);
2288 }
2289 }
2290
2291 /// Map `id` to `row`. `catalog` must hold every row already in the
2292 /// table (the new row may not be pushed yet).
2293 fn insert(&mut self, id: u128, row: usize, catalog: &[CatalogEntry]) {
2294 if (self.len + 1) * 4 > self.slots.len() * 3 {
2295 let cap = (self.slots.len() * 2).max(crate::index::table_capacity_for(self.len + 1));
2296 let old = std::mem::replace(&mut self.slots, vec![NO_ROW; cap]);
2297 self.len = 0;
2298 for r in old.into_iter().filter(|&r| r != NO_ROW) {
2299 self.insert(catalog[r as usize].id, r as usize, catalog);
2300 }
2301 }
2302 let cap = self.slots.len();
2303 let mut i = crate::index::slot_home(id, cap);
2304 loop {
2305 let r = self.slots[i];
2306 if r == NO_ROW {
2307 self.slots[i] = row as u32;
2308 self.len += 1;
2309 return;
2310 }
2311 if catalog[r as usize].id == id {
2312 self.slots[i] = row as u32;
2313 return;
2314 }
2315 i = crate::index::next_slot(i, cap);
2316 }
2317 }
2318
2319 fn heap_bytes(&self) -> usize {
2320 self.slots.capacity() * std::mem::size_of::<u32>()
2321 }
2322}
2323
2324/// Build the ULID → catalog-row lookup.
2325fn build_id_index(catalog: &[CatalogEntry]) -> IdIndex {
2326 IdIndex::build(catalog)
2327}
2328
2329// ---------------------------------------------------------------------------
2330// Phase 12.7a helpers used by `ingest_thread`.
2331// ---------------------------------------------------------------------------
2332
2333/// One incoming turn after chunking or truncation, ready to embed and
2334/// write (Phase 12.7a, split out for 12.7d's batch embed).
2335struct PendingTurn {
2336 turn_id: String,
2337 role: String,
2338 created_at: i64,
2339 parts: Vec<String>,
2340 /// Hex sha256 + original length, set only for a truncated
2341 /// tool-result turn. Applied to the single chunk the truncation
2342 /// yielded.
2343 truncation: Option<(String, u64)>,
2344}
2345
2346/// Output of `Mnemo::plan_ingest`: the turns to write, the first free
2347/// `_mnemo.seq`, and the counts gathered so far (skips, chunking,
2348/// truncation).
2349struct IngestPlan {
2350 pending: Vec<PendingTurn>,
2351 next_seq: u64,
2352 report: IngestReport,
2353}
2354
2355impl IngestPlan {
2356 /// Every text to embed, in write order (turns, then parts).
2357 fn texts(&self) -> Vec<&str> {
2358 self.pending
2359 .iter()
2360 .flat_map(|p| p.parts.iter().map(String::as_str))
2361 .collect()
2362 }
2363
2364 fn text_count(&self) -> usize {
2365 self.pending.iter().map(|p| p.parts.len()).sum()
2366 }
2367}
2368
2369/// PERF-1 / FMT-9: was this catalog row written after the persisted index
2370/// run? v8 allocates pages monotonically, so a record past the run's first
2371/// page is newer. v9 reuses pages, so it compares generations instead.
2372fn written_after_index(e: &CatalogEntry, v9: bool, index_start: u64, index_gen: u64) -> bool {
2373 if v9 {
2374 // `start_page` 0: packed at the next flush, so certainly newer.
2375 e.start_page == 0 || e.gen > index_gen
2376 } else {
2377 e.start_page > index_start
2378 }
2379}
2380
2381/// Whether a record belongs in the ANN index: facts only. Thread-archive
2382/// turns, essence items and thread links are read by kind and scope, never
2383/// by similarity, so they stay out (R-004.2).
2384fn is_ann_kind(m: &Memory) -> bool {
2385 !matches!(
2386 m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str()),
2387 Some(MNEMO_META_KIND_TURN)
2388 | Some(MNEMO_META_KIND_ESSENCE)
2389 | Some(MNEMO_META_KIND_THREAD_LINK)
2390 )
2391}
2392
2393/// Hex-encode a byte slice. Low-dependency alternative to pulling in a
2394/// hex crate for the lone call site in `ingest_thread`'s SHA-256
2395/// recording.
2396fn hex_encode(bytes: &[u8]) -> String {
2397 const HEX: &[u8; 16] = b"0123456789abcdef";
2398 let mut out = String::with_capacity(bytes.len() * 2);
2399 for b in bytes {
2400 out.push(HEX[(b >> 4) as usize] as char);
2401 out.push(HEX[(b & 0x0f) as usize] as char);
2402 }
2403 out
2404}
2405
2406/// Return the longest UTF-8 prefix of `s` whose byte length is `<=
2407/// max_bytes`. Always lands on a `char` boundary.
2408fn take_utf8_prefix(s: &str, max_bytes: usize) -> String {
2409 if s.len() <= max_bytes {
2410 return s.to_string();
2411 }
2412 // Walk forward through char indices; the last index whose byte
2413 // offset fits inside `max_bytes` tells us where to cut.
2414 let mut cut = 0usize;
2415 for (idx, _) in s.char_indices() {
2416 if idx > max_bytes {
2417 break;
2418 }
2419 cut = idx;
2420 }
2421 s[..cut].to_string()
2422}
2423
2424/// Split `s` into chunks, each with `<= max_bytes` of UTF-8 content,
2425/// landing on `char` boundaries. Preserves original ordering; a trailing
2426/// chunk may be shorter than `max_bytes`.
2427/// Phase 12.7b helper: pull the `_mnemo.status` string from an
2428/// essence record. Returns `None` for timeline entries (they have no
2429/// status) and for malformed records.
2430fn status_of(m: &Memory) -> Option<&str> {
2431 m.metadata.get(MNEMO_META_STATUS).and_then(|v| v.as_str())
2432}
2433
2434/// Phase 12.7b helper: render one essence item as a markdown bullet.
2435/// Timeline entries get a date tag; section items get `[open]` /
2436/// `[resolved]` so the reader can scan active work quickly.
2437fn render_essence_item(section: EssenceSection, m: &Memory) -> String {
2438 if section == EssenceSection::Timeline {
2439 let date = format_unix_date(m.created_at);
2440 format!("- [{date}] {}\n", m.content)
2441 } else {
2442 let status = status_of(m).unwrap_or(essence_status::OPEN);
2443 format!("- [{status}] {}\n", m.content)
2444 }
2445}
2446
2447/// Minimal unix-seconds → `YYYY-MM-DD` formatter (no chrono/time
2448/// dependency; mirrors the ISO 8601 parser in `transcript.rs`).
2449fn format_unix_date(secs: i64) -> String {
2450 let days_total = secs.div_euclid(86_400);
2451 let (y, m, d) = civil_from_days(days_total);
2452 format!("{y:04}-{m:02}-{d:02}")
2453}
2454
2455/// Howard Hinnant's `civil_from_days` — inverse of `days_from_civil`,
2456/// adequate for Gregorian dates since 1970. Pure arithmetic, no
2457/// dependencies.
2458fn civil_from_days(z: i64) -> (i64, u32, u32) {
2459 let z = z + 719_468;
2460 let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
2461 let doe = (z - era * 146_097) as u64;
2462 let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
2463 let y = yoe as i64 + era * 400;
2464 let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
2465 let mp = (5 * doy + 2) / 153;
2466 let d = (doy - (153 * mp + 2) / 5 + 1) as u32;
2467 let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32;
2468 let y = if m <= 2 { y + 1 } else { y };
2469 (y, m, d)
2470}
2471
2472fn chunk_utf8(s: &str, max_bytes: usize) -> Vec<String> {
2473 if s.len() <= max_bytes {
2474 return vec![s.to_string()];
2475 }
2476 let mut out = Vec::new();
2477 let mut rest = s;
2478 while rest.len() > max_bytes {
2479 let head = take_utf8_prefix(rest, max_bytes);
2480 let head_bytes = head.len();
2481 out.push(head);
2482 rest = &rest[head_bytes..];
2483 }
2484 if !rest.is_empty() {
2485 out.push(rest.to_string());
2486 }
2487 out
2488}
2489
2490impl Mnemo {
2491 /// Create a brand-new encrypted database at `path`.
2492 pub fn create(
2493 path: impl Into<PathBuf>,
2494 passphrase: &str,
2495 config: MnemoConfig,
2496 ) -> Result<Mnemo> {
2497 Self::create_secret(path, passphrase.as_bytes(), config)
2498 }
2499
2500 /// PERF-1 step 7 (R-004.5): create a database unlocked by a **raw
2501 /// 256-bit key** instead of a passphrase, for keys held by an OS
2502 /// keychain or keystore. The key must be uniformly random (see
2503 /// [`Mnemo::generate_key`]); it gets no stretching. The header records
2504 /// minimal Argon2id parameters ([`KdfParams::raw_key`]), so opening
2505 /// costs microseconds instead of tens of milliseconds. `config.kdf` is
2506 /// ignored. No format change: the key goes through the same KEK
2507 /// derivation a passphrase does.
2508 pub fn create_with_key(
2509 path: impl Into<PathBuf>,
2510 key: &[u8; crypto::KEY_LEN],
2511 config: MnemoConfig,
2512 ) -> Result<Mnemo> {
2513 let config = MnemoConfig {
2514 kdf: KdfParams::raw_key(),
2515 ..config
2516 };
2517 Self::create_secret(path, key, config)
2518 }
2519
2520 /// A fresh random 256-bit key for [`Mnemo::create_with_key`], from the
2521 /// thread-local CSPRNG (seeded from the OS). Store it in a keychain;
2522 /// it can't be recovered from the file.
2523 pub fn generate_key() -> [u8; crypto::KEY_LEN] {
2524 *crypto::random_dek()
2525 }
2526
2527 fn create_secret(
2528 path: impl Into<PathBuf>,
2529 secret: &[u8],
2530 config: MnemoConfig,
2531 ) -> Result<Mnemo> {
2532 let path: PathBuf = path.into();
2533 if config.dimensions == 0 {
2534 return Err(MnemoError::Invalid("dimensions must be > 0".into()));
2535 }
2536 let version = resolve_format_version(config.format_version)?;
2537 if version >= VERSION_V9 && veccol::per_page(config.dimensions) == 0 {
2538 return Err(MnemoError::Invalid(format!(
2539 "a v9 file supports up to {} dimensions for now (one fp16 vector \
2540 must fit a page); vector entries that span pages are not supported yet",
2541 (PAYLOAD - veccol::PAGE_HEADER - 4) / 2
2542 )));
2543 }
2544 let flags = if version >= VERSION_V9 && config.store_float_vectors {
2545 FLAG_ENCRYPTED | FLAG_FLOAT_VECTORS
2546 } else {
2547 FLAG_ENCRYPTED
2548 };
2549 // Exclusive lock covers the write path; second writer sees `Locked`.
2550 let file = open_with_lock(&path, /*write=*/ true, /*create=*/ true)?;
2551
2552 let salt = crypto::random_salt();
2553 let dek = crypto::random_dek();
2554 let kek = crypto::derive_kek(secret, &salt, config.kdf)?;
2555 let dek_nonce = crypto::random_nonce();
2556 let wrapped = crypto::wrap_dek(&kek, &dek_nonce, &dek)?;
2557 if wrapped.len() != WRAPPED_DEK_LEN {
2558 return Err(MnemoError::Crypto("unexpected wrapped DEK length".into()));
2559 }
2560 let mut wrapped_dek = [0u8; WRAPPED_DEK_LEN];
2561 wrapped_dek.copy_from_slice(&wrapped);
2562
2563 let mut header = Header {
2564 version,
2565 page_size: PAGE_SIZE as u32,
2566 flags,
2567 dimensions: config.dimensions as u32,
2568 created_at: memory::now_secs(),
2569 write_counter: 0,
2570 // Page 0 is the header; pages 1..=wal_pages are the WAL;
2571 // record/catalog/index pages start after it. Initial WAL size is
2572 // chosen per-config (default 8 pages = 64 KiB) and clamped to a
2573 // sane floor so the first transaction can always commit.
2574 next_page: 1 + config.wal_pages_initial.max(MIN_WAL_PAGES),
2575 catalog_start: 0,
2576 catalog_pages: 0,
2577 catalog_len: 0,
2578 vector_count: 0,
2579 m_cost: config.kdf.m_cost,
2580 t_cost: config.kdf.t_cost,
2581 p_cost: config.kdf.p_cost,
2582 salt,
2583 dek_nonce,
2584 wrapped_dek,
2585 index_start: 0,
2586 index_pages: 0,
2587 index_len: 0,
2588 wal_start: 1,
2589 wal_pages: config.wal_pages_initial.max(MIN_WAL_PAGES),
2590 wal_seq: 0,
2591 manifest_start: 0,
2592 manifest_pages: 0,
2593 manifest_len: 0,
2594 // Regenerated per write by `apply_seal`; `None` until we seal.
2595 seal_nonce: None,
2596 seal_tag: None,
2597 // v8+ result-cache pointer — zero-initialized on create,
2598 // populated only when `cache_put` runs at least once and
2599 // the next `flush` seals a directory run.
2600 cache_start: 0,
2601 cache_pages: 0,
2602 cache_len: 0,
2603 // FMT-9 v9 fields; all zero on a fresh file (and unused on v8).
2604 catalog_gen: 0,
2605 index_gen: 0,
2606 manifest_gen: 0,
2607 cache_gen: 0,
2608 free: RunPtr::default(),
2609 keys: RunPtr::default(),
2610 bm25: RunPtr::default(),
2611 index_delta: RunPtr::default(),
2612 catalog_delta: RunPtr::default(),
2613 };
2614
2615 let mut pager = Pager::new(file, dek, 0, version);
2616 let mut page = header.to_page();
2617 header.apply_seal(&mut page, pager.dek())?;
2618 pager.write_raw(0, &page)?;
2619 pager.sync()?;
2620
2621 Ok(Mnemo {
2622 pager,
2623 header,
2624 catalog: Vec::new(),
2625 index: IdIndex::default(),
2626 path,
2627 dimensions: config.dimensions,
2628 kdf: config.kdf,
2629 dirty_catalog: false,
2630 ann: None,
2631 dirty_index: false,
2632 index_persist: IndexPersist::default(),
2633 ann_unloaded: false,
2634 manifest: Vec::new(),
2635 // FMT-9c: v9 keeps fewer snapshots by default: with page reuse,
2636 // retention is what pins pages.
2637 max_snapshots: if version >= VERSION_V9 && config.max_snapshots == DEFAULT_MAX_SNAPSHOTS
2638 {
2639 V9_DEFAULT_MAX_SNAPSHOTS
2640 } else {
2641 config.max_snapshots
2642 },
2643 read_only: false,
2644 cache_dir: Vec::new(),
2645 cache_idx: DirectoryIndex::new(),
2646 dirty_cache: false,
2647 cache_budgets: std::collections::HashMap::new(),
2648 cache_flush_policy: CacheFlushPolicy::default(),
2649 cache_batch: BatchState::default(),
2650 cache_counters: CounterState::default(),
2651 embedder: None,
2652 fsyncs_last_flush: 0,
2653 agent_set: None,
2654 key_index: None,
2655 superseded: None,
2656 layers: None,
2657 turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
2658 free: FreeList::default(),
2659 pending_free: Vec::new(),
2660 free_dirty: false,
2661 key_slots: KeySlotRun::default(),
2662 dirty_keys: false,
2663 gen_leased: false,
2664 col: VecColumn::default(),
2665 recs: RecPages::default(),
2666 bm25: Bm25State::default(),
2667 kw: None,
2668 index_layout_override: None,
2669 dirty_rows: std::collections::BTreeSet::new(),
2670 delta_chain: Vec::new(),
2671 delta_rows: 0,
2672 catalog_rewrite: false,
2673 index_chain: Vec::new(),
2674 index_chain_ops: 0,
2675 index_rebase: false,
2676 index_delta_ops: None,
2677 snapshot_cap_explicit: config.max_snapshots != DEFAULT_MAX_SNAPSHOTS,
2678 direct_writes: false,
2679 vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
2680 })
2681 }
2682
2683 /// Open an existing database. A wrong passphrase fails cleanly with
2684 /// [`MnemoError::WrongPassphrase`]. Takes an OS exclusive lock for
2685 /// the lifetime of the returned handle; a second concurrent open
2686 /// (from any process) returns [`MnemoError::Locked`].
2687 ///
2688 /// For a caller that only needs to read (and can tolerate the
2689 /// pending-recovery / pending-migration cases documented on
2690 /// [`Mnemo::open_read_only`]), prefer that entry point — it takes a
2691 /// shared lock and coexists with other readers.
2692 pub fn open(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
2693 Self::open_secret(path, passphrase.as_bytes())
2694 }
2695
2696 /// PERF-1 step 7: open a database created by [`Mnemo::create_with_key`]
2697 /// (or rekeyed with [`Mnemo::rekey_with_key`]) with its raw key.
2698 pub fn open_with_key(path: impl Into<PathBuf>, key: &[u8; crypto::KEY_LEN]) -> Result<Mnemo> {
2699 Self::open_secret(path, key)
2700 }
2701
2702 fn open_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
2703 let path: PathBuf = path.into();
2704 let mut file = open_with_lock(&path, /*write=*/ true, /*create=*/ false)?;
2705
2706 let mut hbuf = [0u8; PAGE_SIZE];
2707 file.seek(SeekFrom::Start(0))?;
2708 file.read_exact(&mut hbuf)?;
2709 let mut header = match Header::from_page(&hbuf) {
2710 Ok(h) => h,
2711 Err(e) => {
2712 // DUR-1 (R-002.1 part 2+3): page 0 did not parse — any
2713 // error is treated as a torn-header candidate, not just
2714 // `HeaderChecksum`. The whole 294-byte header lives in
2715 // the first 512-byte sector, so the realistic failure is
2716 // the sector zeroed or garbled, which returns `BadMagic`
2717 // or `UnsupportedVersion` *before* the CRC gate.
2718 //
2719 // Scan the file for valid committed header-frame WAL
2720 // regions, pick the highest `txn_id`, and replay it. If
2721 // nothing valid is found, return the *original* error so
2722 // non-mnemo files still report `BadMagic`.
2723 let scan = scan_header_recovery_candidates(&mut file, secret)?;
2724 let frames = scan.ok_or(e)?;
2725 for (page_no, bytes) in &frames {
2726 if bytes.len() != PAGE_SIZE {
2727 return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
2728 }
2729 file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
2730 file.write_all(bytes)?;
2731 }
2732 file.sync_all()?;
2733 crate::bump_fsync();
2734 file.seek(SeekFrom::Start(0))?;
2735 file.read_exact(&mut hbuf)?;
2736 Header::from_page(&hbuf)?
2737 }
2738 };
2739
2740 // Crash recovery: replay a committed-but-uncheckpointed transaction.
2741 // Each frame is a finished page image; the header frame, replayed to
2742 // page 0, supersedes the header just read.
2743 if let Some(frames) = wal::recover(
2744 &mut file,
2745 header.wal_start,
2746 header.wal_pages,
2747 header.wal_seq,
2748 )? {
2749 for (page_no, bytes) in &frames {
2750 if bytes.len() != PAGE_SIZE {
2751 return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
2752 }
2753 file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
2754 file.write_all(bytes)?;
2755 }
2756 file.sync_all()?;
2757 crate::bump_fsync();
2758 // Re-read the now-current header from the replayed page 0.
2759 file.seek(SeekFrom::Start(0))?;
2760 file.read_exact(&mut hbuf)?;
2761 header = Header::from_page(&hbuf)?;
2762 }
2763
2764 let kdf = KdfParams {
2765 m_cost: header.m_cost,
2766 t_cost: header.t_cost,
2767 p_cost: header.p_cost,
2768 };
2769 let kek = crypto::derive_kek(secret, &header.salt, kdf)?;
2770 let dek = crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek)?;
2771
2772 // Validate the v7 header seal now that we have the DEK. Returns
2773 // `MnemoError::HeaderTampered` if any of the sealed mutable fields
2774 // were rewritten after the last legitimate flush. No-op for v6 and
2775 // older — pre-v7 files don't carry a seal.
2776 //
2777 // DUR-1 (Planner decision 2026-11-01, Option B): a seal failure is
2778 // **always** reported as tampering, never auto-healed. R-002.1
2779 // part 3's seal-mismatch WAL fallback is withdrawn: the seal
2780 // exists for tamper-evidence; silently loading a stale committed
2781 // header from the WAL would erase that evidence. A real crash
2782 // cannot produce "CRC valid + seal invalid" either, because the
2783 // whole 294-byte header lies inside sector 0 which storage
2784 // writes atomically — a torn sector breaks magic, version or
2785 // CRC first, and *those* are healed by the scan above.
2786 header.validate_seal(&dek)?;
2787
2788 // Build the pager at the file's *current* version so initial reads
2789 // use the matching AAD scheme — `Pager::page_aad` returns empty AAD
2790 // for v4/v5 (no page binding) and `page_no.to_le_bytes()` for v6+.
2791 let on_disk_version = header.version;
2792 let mut pager = Pager::new(file, dek, header.write_counter, on_disk_version);
2793 let dimensions = header.dimensions as usize;
2794
2795 // === v4→v5 catalog migration =====================================
2796 // v6 reads directly; v5 reads directly; v4 replays the catalog into
2797 // the v5 shape by reading each memory's body to populate the new
2798 // `accessed_at` / `access_count` fields. All page reads here still
2799 // use the v4/v5 AAD scheme (none).
2800 let (catalog, migrated_from_v4) = if on_disk_version >= 5 {
2801 (
2802 load_catalog(
2803 &mut pager,
2804 header.catalog_start,
2805 header.catalog_pages,
2806 header.catalog_len,
2807 header.catalog_gen,
2808 )?,
2809 false,
2810 )
2811 } else if on_disk_version == 4 {
2812 let v4 = load_catalog_v4(
2813 &mut pager,
2814 header.catalog_start,
2815 header.catalog_pages,
2816 header.catalog_len,
2817 )?;
2818 (migrate_v4_catalog(&mut pager, v4)?, true)
2819 } else {
2820 return Err(MnemoError::UnsupportedVersion(on_disk_version));
2821 };
2822 // FMT-9c.2: replay the v9 delta chain onto the base catalog.
2823 let (catalog, delta_chain, delta_rows) =
2824 apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
2825 let index = build_id_index(&catalog);
2826 // PERF-1 step 3: the ANN index loads on first use, not at open. A
2827 // v5→v6 page migration is the exception: it must read the index
2828 // now, under the old page AAD, to rewrite it.
2829 let ann = if on_disk_version < 6 {
2830 load_index(
2831 &mut pager,
2832 header.index_start,
2833 header.index_pages,
2834 header.index_len,
2835 header.index_gen,
2836 dimensions,
2837 )?
2838 } else {
2839 None
2840 };
2841 let ann_unloaded = ann.is_none() && header.index_pages > 0;
2842 let mut manifest = load_manifest(
2843 &mut pager,
2844 header.manifest_start,
2845 header.manifest_pages,
2846 header.manifest_len,
2847 header.manifest_gen,
2848 )?;
2849
2850 // === v5→v6 page-encryption migration =============================
2851 // Pre-v6 data pages were encrypted under no AAD; v6 binds the home
2852 // `page_no` as AAD so a page-transplant attack fails authentication
2853 // on read. Re-stage every live record page through `write_page` so
2854 // the next flush re-encrypts it under v6 AAD at the same home slot.
2855 // Tombstoned entries skip — they're reclaimed by `compact_file`.
2856 // Old catalog / ANN / manifest pages stay as orphan ciphertext;
2857 // the next flush writes the new runs at fresh slots.
2858 let migrated_pages_to_v6 = on_disk_version < 6;
2859 if migrated_pages_to_v6 {
2860 for e in &catalog {
2861 if e.deleted {
2862 continue;
2863 }
2864 for i in 0..e.page_count as u64 {
2865 let payload = pager.read_page(e.start_page + i, 0)?;
2866 pager.write_page(e.start_page + i, 0, &payload)?;
2867 }
2868 }
2869 pager.set_version(6);
2870 }
2871
2872 // === v6→v7 header-seal migration =================================
2873 // v7 adds an AES-GCM seal at the tail of the header page that
2874 // authenticates every mutable header field. Pre-v7 files have no
2875 // seal — we just bump the version in memory and mark `dirty_catalog`
2876 // so the next flush writes a sealed v7 header. Pages are untouched
2877 // (page crypto is unchanged from v6).
2878 let migrated_header_to_v7 = on_disk_version < 7;
2879 if migrated_header_to_v7 {
2880 pager.set_version(7);
2881 }
2882
2883 // === v7→v8 cache-directory migration =============================
2884 // v8 adds three u64 header fields (cache_start/pages/len) for the
2885 // result-cache directory (Phase 10.1). Pre-v8 files have those
2886 // three bytes as zero (the on-disk region beyond the v7 seal is
2887 // zero-initialized), which is the legal "empty cache directory"
2888 // state — no data migration required. We bump the version in
2889 // memory and mark dirty_catalog so the next flush writes a
2890 // v8-sealed header with the new AAD shape covering the cache
2891 // pointer. Pages are untouched.
2892 let migrated_header_to_v8 = on_disk_version < 8;
2893 if migrated_header_to_v8 {
2894 pager.set_version(VERSION);
2895 }
2896
2897 let migrated = migrated_from_v4
2898 || migrated_pages_to_v6
2899 || migrated_header_to_v7
2900 || migrated_header_to_v8;
2901 if migrated {
2902 // Stamp the version forward in memory; the next flush persists
2903 // it via the WAL-committed header frame (and the v7 seal).
2904 header.version = VERSION;
2905 // Snapshots written under an older format reference page runs
2906 // encrypted under the old AAD scheme; this build can't decrypt
2907 // them under the new pager. Drop the manifest so the next
2908 // flush records a fresh snapshot of the migrated state.
2909 // Point-in-time recovery into the pre-migration past is
2910 // sacrificed for migration simplicity. Old pages remain on
2911 // disk until `compact_file`. (For v6→v7 alone — which doesn't
2912 // change page crypto — the old snapshots would technically
2913 // still be readable, but dropping them uniformly keeps the
2914 // migration policy simple.)
2915 manifest.clear();
2916 }
2917
2918 // The ANN index pages on disk were sealed under the *old* AAD scheme
2919 // if we migrated pages to v6; force a rewrite at fresh v6-encrypted
2920 // slots on the next flush so a future reopen doesn't try to read
2921 // them under the new AAD and fail. (Catalog and manifest already
2922 // get rewritten via dirty_catalog and the manifest.clear() above.)
2923 let dirty_index = migrated_pages_to_v6 && ann.is_some();
2924
2925 // Load the cache directory (v8+ only). Pre-v8 files have
2926 // cache_pages == 0 by header layout, so this trivially yields
2927 // an empty directory without touching the disk.
2928 let cache_dir = load_cache_directory(
2929 &mut pager,
2930 header.cache_start,
2931 header.cache_pages,
2932 header.cache_len,
2933 header.cache_gen,
2934 )?;
2935 let cache_idx = DirectoryIndex::build(&cache_dir);
2936 // FMT-9 (a)/(b): the free list and key-slot run (empty pointers on
2937 // v8, so these don't touch the disk there).
2938 let free = load_free_list(&mut pager, header.free)?;
2939 let key_slots = load_key_slots(&mut pager, header.keys)?;
2940 let col = VecColumn::from_catalog(&catalog);
2941 let recs = RecPages::from_catalog(&catalog);
2942 let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
2943 let max_snapshots = if header.version >= VERSION_V9 {
2944 V9_DEFAULT_MAX_SNAPSHOTS
2945 } else {
2946 DEFAULT_MAX_SNAPSHOTS
2947 };
2948
2949 Ok(Mnemo {
2950 pager,
2951 header,
2952 catalog,
2953 index,
2954 path,
2955 dimensions,
2956 kdf,
2957 dirty_catalog: migrated,
2958 ann,
2959 dirty_index,
2960 // A v5→v6 page migration leaves the index sealed under the old
2961 // page AAD, so it must be rewritten on the next flush.
2962 index_persist: IndexPersist {
2963 forced: dirty_index,
2964 ..IndexPersist::default()
2965 },
2966 ann_unloaded,
2967 manifest,
2968 // `Mnemo::open` has no [`MnemoConfig`] to read; default the cap
2969 // to [`DEFAULT_MAX_SNAPSHOTS`]. Override with
2970 // [`Mnemo::set_max_snapshots`] right after open if you need
2971 // unlimited retention or a different cap for this handle.
2972 max_snapshots,
2973 read_only: false,
2974 cache_dir,
2975 cache_idx,
2976 dirty_cache: false,
2977 cache_budgets: std::collections::HashMap::new(),
2978 cache_flush_policy: CacheFlushPolicy::default(),
2979 cache_batch: BatchState::default(),
2980 cache_counters: CounterState::default(),
2981 embedder: None,
2982 fsyncs_last_flush: 0,
2983 agent_set: None,
2984 key_index: None,
2985 superseded: None,
2986 layers: None,
2987 turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
2988 free,
2989 pending_free: Vec::new(),
2990 free_dirty: false,
2991 key_slots,
2992 dirty_keys: false,
2993 gen_leased: false,
2994 col,
2995 recs,
2996 bm25: Bm25State {
2997 manifest: bm25_manifest,
2998 ..Bm25State::default()
2999 },
3000 kw: None,
3001 index_layout_override: None,
3002 dirty_rows: std::collections::BTreeSet::new(),
3003 delta_chain,
3004 delta_rows,
3005 catalog_rewrite: false,
3006 index_chain: Vec::new(),
3007 index_chain_ops: 0,
3008 index_rebase: false,
3009 index_delta_ops: None,
3010 snapshot_cap_explicit: false,
3011 direct_writes: false,
3012 vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
3013 })
3014 }
3015
3016 /// Open an existing database in **read-only** mode. Takes an OS shared
3017 /// lock; multiple read-only handles can coexist, but any attempt to
3018 /// open the same file read-write elsewhere returns
3019 /// [`MnemoError::Locked`]. All mutating methods on the returned handle
3020 /// return [`MnemoError::ReadOnly`], including `recall` with
3021 /// `track_access(true)`.
3022 ///
3023 /// Two conditions block a read-only open and return
3024 /// [`MnemoError::NeedsWriteOpen`]:
3025 ///
3026 /// 1. The file has a committed-but-uncheckpointed WAL transaction that
3027 /// would ordinarily be replayed on open. A read-only handle
3028 /// cannot write the recovered pages back.
3029 /// 2. The on-disk format version is older than the current
3030 /// `format::VERSION`, requiring an on-disk migration.
3031 ///
3032 /// In either case, open the file read-write once (which recovers /
3033 /// migrates in place), close the handle cleanly, and reopen
3034 /// read-only. This is documented in the README's Durability section.
3035 pub fn open_read_only(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
3036 Self::open_read_only_secret(path, passphrase.as_bytes())
3037 }
3038
3039 /// PERF-1 step 7: [`Mnemo::open_read_only`] for a raw-key database.
3040 pub fn open_read_only_with_key(
3041 path: impl Into<PathBuf>,
3042 key: &[u8; crypto::KEY_LEN],
3043 ) -> Result<Mnemo> {
3044 Self::open_read_only_secret(path, key)
3045 }
3046
3047 fn open_read_only_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
3048 let path: PathBuf = path.into();
3049 let mut file = open_with_lock(&path, /*write=*/ false, /*create=*/ false)?;
3050
3051 let mut hbuf = [0u8; PAGE_SIZE];
3052 file.seek(SeekFrom::Start(0))?;
3053 file.read_exact(&mut hbuf)?;
3054 // A torn header on a read-only open cannot be healed (that requires
3055 // writing recovered frames back). Surface as `NeedsWriteOpen` so the
3056 // caller knows to reopen read-write once. DUR-1 (R-002.1 part 4):
3057 // any `Header::from_page` failure (`BadMagic`,
3058 // `UnsupportedVersion`, `HeaderChecksum`) qualifies — but only
3059 // when the file-wide scan actually finds a valid committed
3060 // header frame to recover from, so a non-mnemo file still
3061 // reports `BadMagic` cleanly. A `validate_seal` failure is NOT
3062 // routed here (Option B: tampering is never auto-healed).
3063 let header = match Header::from_page(&hbuf) {
3064 Ok(h) => h,
3065 Err(parse_err) => {
3066 let recoverable = scan_header_recovery_candidates(&mut file, secret)?;
3067 return Err(match (parse_err, recoverable) {
3068 (_, Some(_)) => MnemoError::NeedsWriteOpen {
3069 reason: "torn header page needs recovery from WAL",
3070 },
3071 (e, None) => e,
3072 });
3073 }
3074 };
3075
3076 // Refuse to migrate on read-only opens. v6→v7 is header-only
3077 // (no page writes required), but even that bumps `version` on
3078 // disk and re-seals the header — which needs write access. Keep
3079 // the policy uniform across all sub-version bumps.
3080 if header.version < VERSION {
3081 return Err(MnemoError::NeedsWriteOpen {
3082 reason: "pending format-version migration",
3083 });
3084 }
3085
3086 // Refuse if there is a committed-but-uncheckpointed transaction
3087 // waiting for `wal::recover` to fold it into home pages.
3088 if wal::recover(
3089 &mut file,
3090 header.wal_start,
3091 header.wal_pages,
3092 header.wal_seq,
3093 )?
3094 .is_some()
3095 {
3096 return Err(MnemoError::NeedsWriteOpen {
3097 reason: "pending WAL recovery",
3098 });
3099 }
3100
3101 let kdf = KdfParams {
3102 m_cost: header.m_cost,
3103 t_cost: header.t_cost,
3104 p_cost: header.p_cost,
3105 };
3106 let kek = crypto::derive_kek(secret, &header.salt, kdf)?;
3107 let dek = crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek)?;
3108 header.validate_seal(&dek)?;
3109
3110 let mut pager = Pager::new(file, dek, header.write_counter, header.version);
3111 let dimensions = header.dimensions as usize;
3112
3113 let catalog = load_catalog(
3114 &mut pager,
3115 header.catalog_start,
3116 header.catalog_pages,
3117 header.catalog_len,
3118 header.catalog_gen,
3119 )?;
3120 // FMT-9c.2: replay the v9 delta chain onto the base catalog.
3121 let (catalog, delta_chain, delta_rows) =
3122 apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
3123 let index = build_id_index(&catalog);
3124 // PERF-1 step 3: the ANN index loads on first use, not at open.
3125 let ann: Option<IvfPqIndex> = None;
3126 let ann_unloaded = header.index_pages > 0;
3127 let manifest = load_manifest(
3128 &mut pager,
3129 header.manifest_start,
3130 header.manifest_pages,
3131 header.manifest_len,
3132 header.manifest_gen,
3133 )?;
3134 let cache_dir = load_cache_directory(
3135 &mut pager,
3136 header.cache_start,
3137 header.cache_pages,
3138 header.cache_len,
3139 header.cache_gen,
3140 )?;
3141 let cache_idx = DirectoryIndex::build(&cache_dir);
3142 let free = load_free_list(&mut pager, header.free)?;
3143 let key_slots = load_key_slots(&mut pager, header.keys)?;
3144 let col = VecColumn::from_catalog(&catalog);
3145 let recs = RecPages::from_catalog(&catalog);
3146 let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
3147 let max_snapshots = if header.version >= VERSION_V9 {
3148 V9_DEFAULT_MAX_SNAPSHOTS
3149 } else {
3150 DEFAULT_MAX_SNAPSHOTS
3151 };
3152
3153 Ok(Mnemo {
3154 pager,
3155 header,
3156 catalog,
3157 index,
3158 path,
3159 dimensions,
3160 kdf,
3161 // No mutation is possible; leave both dirty flags false so
3162 // `close()` is a genuine no-op even if we later add an
3163 // auto-flush on drop.
3164 dirty_catalog: false,
3165 ann,
3166 dirty_index: false,
3167 index_persist: IndexPersist::default(),
3168 ann_unloaded,
3169 manifest,
3170 max_snapshots,
3171 read_only: true,
3172 cache_dir,
3173 cache_idx,
3174 dirty_cache: false,
3175 cache_budgets: std::collections::HashMap::new(),
3176 cache_flush_policy: CacheFlushPolicy::default(),
3177 cache_batch: BatchState::default(),
3178 cache_counters: CounterState::default(),
3179 embedder: None,
3180 fsyncs_last_flush: 0,
3181 agent_set: None,
3182 key_index: None,
3183 superseded: None,
3184 layers: None,
3185 turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
3186 free,
3187 pending_free: Vec::new(),
3188 free_dirty: false,
3189 key_slots,
3190 dirty_keys: false,
3191 gen_leased: false,
3192 col,
3193 recs,
3194 bm25: Bm25State {
3195 manifest: bm25_manifest,
3196 ..Bm25State::default()
3197 },
3198 kw: None,
3199 index_layout_override: None,
3200 dirty_rows: std::collections::BTreeSet::new(),
3201 delta_chain,
3202 delta_rows,
3203 catalog_rewrite: false,
3204 index_chain: Vec::new(),
3205 index_chain_ops: 0,
3206 index_rebase: false,
3207 index_delta_ops: None,
3208 snapshot_cap_explicit: false,
3209 direct_writes: false,
3210 vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
3211 })
3212 }
3213
3214 /// FMT-9e: compress v9 thread-turn records once they're older than
3215 /// `secs` (by their `created_at`). The default,
3216 /// [`DEFAULT_TURN_COMPRESS_AFTER_SECS`], is 7 days: recent turns stay
3217 /// plain, so a live session reads them at full speed, and each commit
3218 /// compresses up to 256 turns that have aged past the cutoff. `0`
3219 /// compresses every turn as it's written. Compaction always compresses
3220 /// every turn, and so does a vacuum that moves one. Applies to this
3221 /// handle only (not persisted); a no-op on v8 files, which never
3222 /// compress. Reading never depends on it.
3223 pub fn set_turn_compress_after(&mut self, secs: u64) {
3224 self.turn_compress_after = secs.min(i64::MAX as u64) as i64;
3225 }
3226
3227 /// Override the manifest snapshot cap on this open handle. `0` disables
3228 /// the cap; any positive value keeps the most-recent `max` snapshots and
3229 /// drops the rest on the next flush. See [`MnemoConfig::max_snapshots`].
3230 pub fn set_max_snapshots(&mut self, max: usize) {
3231 self.max_snapshots = max;
3232 // FMT-9c: an explicit count is a hard cap on v9 (see
3233 // `apply_retention`).
3234 self.snapshot_cap_explicit = true;
3235 }
3236
3237 /// Embedding dimensionality this database expects.
3238 pub fn dimensions(&self) -> usize {
3239 self.dimensions
3240 }
3241
3242 /// Number of live (non-deleted) memories.
3243 pub fn len(&self) -> usize {
3244 self.catalog.iter().filter(|e| !e.deleted).count()
3245 }
3246
3247 /// True if the database holds no live memories.
3248 pub fn is_empty(&self) -> bool {
3249 self.len() == 0
3250 }
3251
3252 // --- internal page/record helpers -------------------------------------
3253
3254 /// FMT-9: true for a v9 file (page reuse, generations in the AAD).
3255 fn is_v9(&self) -> bool {
3256 self.header.version >= VERSION_V9
3257 }
3258
3259 /// FMT-9: the generation for pages written now: the id of the
3260 /// transaction that will commit them, or a lower bound on it (a WAL
3261 /// relocation inside the next flush can push the commit id higher).
3262 /// Either way it's strictly greater than the generation any earlier
3263 /// write to a reused page carried, which is all rollback protection
3264 /// needs: a page is reused only after the transaction that freed it
3265 /// committed, and that transaction came after the one that wrote it.
3266 /// `0` on v8, whose page AAD has no generation.
3267 fn write_gen(&self) -> u64 {
3268 if self.is_v9() {
3269 self.header.wal_seq + 1
3270 } else {
3271 0
3272 }
3273 }
3274
3275 /// FMT-9 (a): the oldest retained snapshot's transaction id
3276 /// (`u64::MAX` when none is retained). Only pages freed at or before it
3277 /// may be reused.
3278 fn min_snapshot_txn(&self) -> u64 {
3279 self.manifest
3280 .iter()
3281 .map(|s| s.txn_id)
3282 .min()
3283 .unwrap_or(u64::MAX)
3284 }
3285
3286 /// Allocate `n` contiguous pages. v9 takes them from the free list when
3287 /// an eligible run fits (freed by a committed transaction, and not
3288 /// referenced by a retained snapshot); otherwise, and always on v8, the
3289 /// file grows.
3290 fn alloc_pages(&mut self, n: u64) -> u64 {
3291 if self.is_v9() {
3292 let committed = self.header.wal_seq;
3293 let min_snap = self.min_snapshot_txn();
3294 if let Some(p) = self.free.alloc(n, committed, min_snap) {
3295 self.free_dirty = true;
3296 return p;
3297 }
3298 }
3299 let p = self.header.next_page;
3300 self.header.next_page += n;
3301 p
3302 }
3303
3304 /// FMT-9 (a): record that the current transaction stopped referencing
3305 /// `n` pages from `page`. The next commit frees them. No-op on v8,
3306 /// which never reuses a page.
3307 fn release_pages(&mut self, page: u64, n: u64) {
3308 if self.is_v9() && n > 0 && page != 0 {
3309 self.pending_free.push((page, n));
3310 }
3311 }
3312
3313 /// FMT-9d: make sure the live-document stats and the segment dicts are
3314 /// loaded. Stats come from the rows; dicts from each segment's head.
3315 fn ensure_bm25(&mut self) -> Result<()> {
3316 self.ensure_bm25_stats();
3317 self.ensure_bm25_dicts()
3318 }
3319
3320 /// FMT-9d: the live-document stats, from the rows (once per handle, or
3321 /// after a catalog rebase moves rows; `put` and `delete` keep them
3322 /// current, FMT-9f).
3323 fn ensure_bm25_stats(&mut self) {
3324 if self.bm25.stats.is_none() {
3325 let mut docs = std::collections::HashMap::new();
3326 let mut total = 0u64;
3327 for (i, e) in self.catalog.iter().enumerate() {
3328 if e.deleted {
3329 continue;
3330 }
3331 if let Some(r) = e.v9 {
3332 if r.doc != 0 && bm25_kind(r.kind) {
3333 docs.insert(r.doc, (i, r.doc_len));
3334 total += r.doc_len as u64;
3335 }
3336 }
3337 }
3338 let mut stats = DocStats {
3339 docs,
3340 avgdl: 1.0,
3341 total,
3342 };
3343 stats.refresh();
3344 self.bm25.stats = Some(stats);
3345 }
3346 }
3347
3348 /// FMT-9d: the segment dicts, read from each segment's head once; after
3349 /// that a commit adds or replaces only the segments it wrote (FMT-9f).
3350 fn ensure_bm25_dicts(&mut self) -> Result<()> {
3351 if self.bm25.dicts.is_none() {
3352 let mut dicts = Vec::with_capacity(self.bm25.manifest.segments.len());
3353 for &t in &self.bm25.manifest.segments {
3354 dicts.push(load_segment_dict(&mut self.pager, tuple_ptr(t))?);
3355 }
3356 self.bm25.dicts = Some(dicts);
3357 }
3358 Ok(())
3359 }
3360
3361 /// FMT-9d: BM25 scores of the live keyword-indexed rows that match
3362 /// `query`: catalog row → score. Empty on v8 (no keyword index).
3363 fn bm25_scores(
3364 &mut self,
3365 query: &str,
3366 k1: f32,
3367 b: f32,
3368 ) -> Result<std::collections::HashMap<usize, f32>> {
3369 let mut scores: std::collections::HashMap<usize, f32> = std::collections::HashMap::new();
3370 if !self.is_v9() {
3371 return Ok(scores);
3372 }
3373 self.ensure_bm25()?;
3374 let terms: std::collections::BTreeSet<String> = text::tokenize(query).into_iter().collect();
3375 let segs: Vec<RunPtr> = self
3376 .bm25
3377 .manifest
3378 .segments
3379 .iter()
3380 .map(|&t| tuple_ptr(t))
3381 .collect();
3382 let stats = self.bm25.stats.as_ref().expect("loaded above");
3383 let dicts = self.bm25.dicts.as_ref().expect("loaded above");
3384 let n = stats.docs.len();
3385 for term in &terms {
3386 let mut posts: Vec<(u32, u8)> = Vec::new();
3387 for (si, seg) in segs.iter().enumerate() {
3388 posts.extend(segment_postings(&mut self.pager, *seg, &dicts[si], term)?);
3389 }
3390 for (doc, tfs) in &self.bm25.pending {
3391 if let Ok(i) = tfs.binary_search_by(|(t, _)| t.as_str().cmp(term)) {
3392 posts.push((*doc, tfs[i].1));
3393 }
3394 }
3395 posts.retain(|(d, _)| stats.docs.contains_key(d));
3396 if posts.is_empty() {
3397 continue;
3398 }
3399 let idf = text::idf(n, posts.len());
3400 for (d, tf) in posts {
3401 let (row, len) = stats.docs[&d];
3402 *scores.entry(row).or_default() += text::bm25(tf, idf, len, stats.avgdl, k1, b);
3403 }
3404 }
3405 Ok(scores)
3406 }
3407
3408 /// FMT-9d: the next commit's BM25 segment: the pending documents, plus,
3409 /// past [`BM25_MAX_SEGMENTS`], the small segments (all but the first),
3410 /// or all of them once the small ones outgrow a tenth of the first.
3411 /// Dead documents are dropped from merged segments. Returns the segment
3412 /// bytes and, for a merge, the index from which it replaces the
3413 /// segments (`None`: it's appended).
3414 fn bm25_next_segment(&mut self) -> Result<Option<(Vec<u8>, Option<usize>)>> {
3415 if !self.is_v9() || self.bm25.pending.is_empty() {
3416 return Ok(None);
3417 }
3418 let segs: Vec<RunPtr> = self
3419 .bm25
3420 .manifest
3421 .segments
3422 .iter()
3423 .map(|&t| tuple_ptr(t))
3424 .collect();
3425 // Mostly dead (a mass delete): merge everything, so the dead
3426 // postings go and the old segments' pages can be reused or cut off.
3427 // FMT-9f: only the stats; the dicts aren't needed to write a segment.
3428 self.ensure_bm25_stats();
3429 let written: u64 = self.bm25.manifest.seg_docs.iter().map(|&d| d as u64).sum();
3430 let live_docs = self.bm25.stats.as_ref().map_or(0, |s| s.docs.len()) as u64;
3431 let mostly_dead = !segs.is_empty() && written > 64 && live_docs * 2 < written;
3432 if segs.len() < BM25_MAX_SEGMENTS && !mostly_dead {
3433 return Ok(Some((text::build_segment(&self.bm25.pending)?, None)));
3434 }
3435 let small: u64 = segs.iter().skip(1).map(|p| p.len).sum();
3436 let from = if mostly_dead || small * 10 > segs[0].len {
3437 0
3438 } else {
3439 1
3440 };
3441 let live: std::collections::HashSet<u32> = self
3442 .bm25
3443 .stats
3444 .as_ref()
3445 .expect("loaded above")
3446 .docs
3447 .keys()
3448 .copied()
3449 .collect();
3450 let mut by_doc: std::collections::BTreeMap<u32, Vec<(String, u8)>> =
3451 std::collections::BTreeMap::new();
3452 for seg in &segs[from..] {
3453 let bytes = read_run_bytes(&mut self.pager, seg.start, seg.pages, seg.len, seg.gen)?;
3454 let n = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as usize;
3455 let dict: Vec<text::DictEntry> = rmp_serde::from_slice(&bytes[4..4 + n])
3456 .map_err(|e| MnemoError::Serialize(e.to_string()))?;
3457 let post = &bytes[4 + n..];
3458 for e in dict {
3459 let range = e.off as usize..(e.off + e.len) as usize;
3460 let list = text::decode_postings(post.get(range).ok_or_else(|| {
3461 MnemoError::Invalid("BM25 dict entry past its postings".into())
3462 })?)?;
3463 for (d, tf) in list {
3464 if live.contains(&d) {
3465 by_doc.entry(d).or_default().push((e.term.clone(), tf));
3466 }
3467 }
3468 }
3469 }
3470 for (d, tfs) in &self.bm25.pending {
3471 if live.contains(d) {
3472 by_doc.insert(*d, tfs.clone());
3473 }
3474 }
3475 let docs: Vec<(u32, Vec<(String, u8)>)> = by_doc.into_iter().collect();
3476 self.bm25.merged_docs = docs.len() as u32;
3477 Ok(Some((text::build_segment(&docs)?, Some(from))))
3478 }
3479
3480 /// FMT-9c.3: the generation the persisted index is current to: its
3481 /// base run's, or its newest delta run's.
3482 fn index_gen_now(&self) -> u64 {
3483 self.header.index_gen.max(self.header.index_delta.gen)
3484 }
3485
3486 /// FMT-9c.3: should this v9 flush persist the index, and how? Every
3487 /// commit that changed it writes those changes as a delta run (so the
3488 /// index is never stale after a crash); a build, rebuild, drop or
3489 /// restore, or a chain past 64 runs or a quarter of the index, writes
3490 /// the whole index instead. Sets `index_delta_ops` for a delta.
3491 fn plan_index_v9(&mut self) -> bool {
3492 self.index_delta_ops = None;
3493 let Some(ann) = self.ann.as_mut() else {
3494 // Dropped (persisted as "no index"), or not loaded (unchanged).
3495 return self.dirty_index;
3496 };
3497 let ops = ann.take_log();
3498 let n = ann.len();
3499 if self.index_rebase {
3500 return true;
3501 }
3502 if ops.is_empty() {
3503 return false;
3504 }
3505 if self.index_chain.len() >= 64 || (self.index_chain_ops + ops.len()) * 4 > n.max(256) {
3506 return true;
3507 }
3508 self.index_delta_ops = Some(ops);
3509 true
3510 }
3511
3512 /// FMT-9c.2: catalog row `idx` changed (v9 writes it in the next
3513 /// commit's delta run). No-op on v8, which rewrites the whole catalog.
3514 fn touch(&mut self, idx: usize) {
3515 if self.is_v9() {
3516 self.dirty_rows.insert(idx);
3517 }
3518 }
3519
3520 /// FMT-9c.2: write this commit's catalog change as a delta run on the
3521 /// chain, rather than a new base? Not after a restore, not before the
3522 /// first base, and not once the chain has 64 runs or holds more than a
3523 /// quarter of the base's rows (the next base then folds it in).
3524 fn catalog_as_delta(&self) -> bool {
3525 self.is_v9()
3526 && !self.catalog_rewrite
3527 && self.header.catalog_pages > 0
3528 && self.delta_chain.len() < 64
3529 && (self.delta_rows + self.dirty_rows.len()) * 4 <= self.catalog.len()
3530 }
3531
3532 /// FMT-9c.2: before writing a new base, drop deleted rows once they're
3533 /// more than a quarter of the catalog. Rows move, so the id index and
3534 /// the page maps are rebuilt, and the resident column reloads lazily.
3535 /// Runs inside a flush, after packing, when nothing is waiting.
3536 fn compact_tombstones(&mut self) {
3537 let dead = self.catalog.iter().filter(|e| e.deleted).count();
3538 if dead == 0 || dead * 4 <= self.catalog.len() {
3539 return;
3540 }
3541 self.catalog.retain(|e| !e.deleted);
3542 self.catalog.shrink_to_fit();
3543 self.index = build_id_index(&self.catalog);
3544 self.col = VecColumn::from_catalog(&self.catalog);
3545 self.recs = RecPages::from_catalog(&self.catalog);
3546 self.dirty_rows.clear();
3547 self.bm25.stats = None;
3548 }
3549
3550 /// FMT-9b: the file keeps float vectors in its records.
3551 fn float_vectors(&self) -> bool {
3552 self.is_v9() && self.header.flags & FLAG_FLOAT_VECTORS != 0
3553 }
3554
3555 /// FMT-9b: quantize `v` into the column for catalog row `idx` (written
3556 /// by the next flush).
3557 fn stage_vector(&mut self, idx: usize, v: &[f32]) {
3558 let (h, norm) = veccol::quantize(v);
3559 let d = self.dimensions;
3560 if self.col.resident {
3561 let rows = self.catalog.len();
3562 let fits = rows * veccol::entry_len(d) <= self.vector_cache_budget;
3563 if fits {
3564 if self.col.data.len() < rows * d {
3565 self.col.data.resize(rows * d, 0);
3566 self.col.norms.resize(rows, 0.0);
3567 self.col.have.resize(rows, false);
3568 }
3569 self.col.data[idx * d..(idx + 1) * d].copy_from_slice(&h);
3570 self.col.norms[idx] = norm;
3571 self.col.have[idx] = true;
3572 } else {
3573 // Grown past the budget: page the column from now on.
3574 self.col.data = Vec::new();
3575 self.col.norms = Vec::new();
3576 self.col.have = Vec::new();
3577 self.col.resident = false;
3578 }
3579 }
3580 self.col.hot.insert(idx, (h, norm));
3581 self.col.pending.push(idx);
3582 self.touch(idx);
3583 if let Some(r) = self.catalog[idx].v9.as_mut() {
3584 r.vec_page = 0;
3585 r.vec_slot = 0;
3586 r.vec_gen = 0;
3587 }
3588 }
3589
3590 /// FMT-9b: row `idx` no longer needs its column entry (`row` is the row
3591 /// as it was): drop a staged one, or take it off its page and free the
3592 /// page once nothing live is left on it.
3593 fn kill_vector(&mut self, idx: usize, row: &CatalogEntry) {
3594 self.col.hot.remove(&idx);
3595 let Some(r) = row.v9 else { return };
3596 if r.vec_page == 0 {
3597 return;
3598 }
3599 let empty = match self.col.page_rows.get_mut(&r.vec_page) {
3600 Some(rows) => {
3601 rows.retain(|&i| i != idx);
3602 rows.is_empty()
3603 }
3604 None => false,
3605 };
3606 if empty {
3607 self.col.page_rows.remove(&r.vec_page);
3608 self.release_pages(r.vec_page, 1);
3609 }
3610 }
3611
3612 /// FMT-9b.2: row `idx` no longer needs its record (`row` is the row as
3613 /// it was): drop a waiting one, free a span, or take it off its packed
3614 /// page and free the page once nothing live is left on it.
3615 /// FMT-9f: the rerank depth for an ANN query: the caller's pin, else
3616 /// on v9 at least [`V9_RERANK_PER_HIT`] × `top_k`, else the index's
3617 /// own default (`None`).
3618 fn rerank_for(&self, ann: &IvfPqIndex, req: &RecallRequest) -> Option<usize> {
3619 if req.n_rerank.is_some() || !self.is_v9() {
3620 return req.n_rerank;
3621 }
3622 let (_, auto) = ann.effective_defaults();
3623 Some(auto.max(V9_RERANK_PER_HIT.saturating_mul(req.top_k)))
3624 }
3625
3626 /// FMT-9e: is a turn created at `created_at` past this handle's
3627 /// compression cutoff?
3628 fn turn_is_cold(&self, created_at: i64) -> bool {
3629 self.turn_compress_after == 0
3630 || created_at <= memory::now_secs().saturating_sub(self.turn_compress_after)
3631 }
3632
3633 /// FMT-9e: is row `idx` a live, unsettled v9 turn?
3634 fn unsettled_turn(&self, idx: usize) -> bool {
3635 let e = &self.catalog[idx];
3636 !e.deleted
3637 && e.v9
3638 .is_some_and(|r| r.kind == ROW_KIND_TURN && r.flags & ROW_SETTLED == 0)
3639 }
3640
3641 /// FMT-9e: settle row `idx`'s turn record, whose stored bytes are
3642 /// `bytes`: compress them if it pays, and mark the row settled.
3643 /// Returns the bytes to store (unchanged if not worth compressing);
3644 /// the caller writes them. Updates the row's length.
3645 fn settle_turn(&mut self, idx: usize, bytes: Vec<u8>) -> Vec<u8> {
3646 let out = if bytes.starts_with(&ZSTD_MAGIC) {
3647 bytes
3648 } else {
3649 compress_turn_record(bytes)
3650 };
3651 let e = &mut self.catalog[idx];
3652 e.len = out.len() as u32;
3653 if let Some(r) = e.v9.as_mut() {
3654 r.flags |= ROW_SETTLED;
3655 }
3656 self.touch(idx);
3657 self.dirty_catalog = true;
3658 out
3659 }
3660
3661 /// FMT-9e: compress up to [`COLD_TURNS_PER_COMMIT`] plain turns that
3662 /// have aged past the cutoff. Runs at the top of a flush that's
3663 /// already writing, before [`Mnemo::pack_records`], which packs the
3664 /// rewritten records.
3665 fn settle_cold_turns(&mut self) -> Result<()> {
3666 if !self.is_v9() || !self.dirty_catalog {
3667 return Ok(());
3668 }
3669 let picked: Vec<usize> = (0..self.catalog.len())
3670 .filter(|&i| {
3671 self.unsettled_turn(i)
3672 && self.catalog[i]
3673 .v9
3674 .is_some_and(|r| self.turn_is_cold(r.created_at))
3675 })
3676 .take(COLD_TURNS_PER_COMMIT)
3677 .collect();
3678 for idx in picked {
3679 let old = self.catalog[idx].clone();
3680 let bytes = self.read_record(&old)?;
3681 let before = bytes.len();
3682 let out = self.settle_turn(idx, bytes);
3683 if out.len() == before {
3684 continue;
3685 }
3686 self.kill_record(idx, &old);
3687 let gen = self.write_gen();
3688 let e = &mut self.catalog[idx];
3689 if out.len() + REC_SLOT_BYTES <= REC_PAGE_CAP {
3690 e.start_page = 0;
3691 e.page_count = 1;
3692 e.gen = gen;
3693 if let Some(r) = e.v9.as_mut() {
3694 r.rec_slot = 0;
3695 }
3696 self.recs.hot.insert(idx, out);
3697 self.recs.pending.push(idx);
3698 } else {
3699 let (start, pc, gen) = self.write_record(&out)?;
3700 let e = &mut self.catalog[idx];
3701 e.start_page = start;
3702 e.page_count = pc;
3703 e.gen = gen;
3704 if let Some(r) = e.v9.as_mut() {
3705 r.rec_slot = REC_SPAN;
3706 }
3707 }
3708 }
3709 Ok(())
3710 }
3711
3712 fn kill_record(&mut self, idx: usize, row: &CatalogEntry) {
3713 self.recs.hot.remove(&idx);
3714 let Some(r) = row.v9 else { return };
3715 if r.rec_slot == REC_SPAN {
3716 self.release_pages(row.start_page, row.page_count as u64);
3717 return;
3718 }
3719 if row.start_page == 0 {
3720 return;
3721 }
3722 let empty = match self.recs.page_rows.get_mut(&row.start_page) {
3723 Some(rows) => {
3724 rows.retain(|&i| i != idx);
3725 rows.is_empty()
3726 }
3727 None => false,
3728 };
3729 if empty {
3730 self.recs.page_rows.remove(&row.start_page);
3731 self.release_pages(row.start_page, 1);
3732 }
3733 }
3734
3735 /// FMT-9b.2: pack the waiting records, plus the live records of up to
3736 /// [`VACUUM_PAGES_PER_COMMIT`] sparse pages (under 3/4 full), into
3737 /// fresh record pages, filling each before starting the next. Runs at
3738 /// the top of a flush, before the catalog is serialized (it moves rows'
3739 /// record locations), and only when something is waiting.
3740 fn pack_records(&mut self) -> Result<()> {
3741 if !self.is_v9() || self.recs.pending.is_empty() {
3742 return Ok(());
3743 }
3744 let mut sparse: Vec<(u64, usize)> = self
3745 .recs
3746 .page_rows
3747 .iter()
3748 .filter(|(_, rows)| {
3749 let live: usize = rows
3750 .iter()
3751 .map(|&i| self.catalog[i].len as usize + REC_SLOT_BYTES)
3752 .sum();
3753 live * 4 < REC_PAGE_CAP * 3
3754 })
3755 .map(|(&p, rows)| (p, rows.len()))
3756 .collect();
3757 sparse.sort_unstable();
3758 let sparse = vacuum_pick(sparse);
3759 let mut order: Vec<usize> = Vec::new();
3760 let mut seen: std::collections::HashSet<usize> = std::collections::HashSet::new();
3761 for p in sparse {
3762 let rows = self.recs.page_rows.remove(&p).unwrap_or_default();
3763 if let Some(&first) = rows.first() {
3764 let page = self.pager.read_page_uncached(p, self.catalog[first].gen)?;
3765 for idx in rows {
3766 let slot = self.catalog[idx].v9.map_or(0, |r| r.rec_slot);
3767 let mut b = record_slot(&page, slot)?;
3768 // FMT-9e: a vacuum that moves a plain turn compresses it.
3769 if self.unsettled_turn(idx) {
3770 b = self.settle_turn(idx, b);
3771 }
3772 self.recs.hot.insert(idx, b);
3773 if seen.insert(idx) {
3774 order.push(idx);
3775 }
3776 }
3777 }
3778 self.release_pages(p, 1);
3779 }
3780 let pending = std::mem::take(&mut self.recs.pending);
3781 for idx in pending {
3782 if self.recs.hot.contains_key(&idx) && seen.insert(idx) {
3783 order.push(idx);
3784 }
3785 }
3786 let gen = self.write_gen();
3787 let mut i = 0;
3788 while i < order.len() {
3789 let mut used = 0usize;
3790 let mut j = i;
3791 while j < order.len() {
3792 let l = self.recs.hot[&order[j]].len() + REC_SLOT_BYTES;
3793 if j > i && used + l > REC_PAGE_CAP {
3794 break;
3795 }
3796 used += l;
3797 j += 1;
3798 }
3799 let page = self.alloc_pages(1);
3800 let bytes = {
3801 let recs: Vec<&[u8]> = order[i..j]
3802 .iter()
3803 .map(|k| self.recs.hot[k].as_slice())
3804 .collect();
3805 encode_record_page(gen, &recs)
3806 };
3807 self.pager.write_page(page, gen, &bytes)?;
3808 for (slot, &k) in order[i..j].iter().enumerate() {
3809 self.touch(k);
3810 let e = &mut self.catalog[k];
3811 e.start_page = page;
3812 e.page_count = 1;
3813 e.gen = gen;
3814 if let Some(r) = e.v9.as_mut() {
3815 r.rec_slot = slot as u16;
3816 }
3817 }
3818 self.recs.page_rows.insert(page, order[i..j].to_vec());
3819 i = j;
3820 }
3821 for k in order {
3822 self.recs.hot.remove(&k);
3823 }
3824 self.dirty_catalog = true;
3825 Ok(())
3826 }
3827
3828 /// FMT-9b: row `e`'s column entry (fp16 values and norm), from the
3829 /// staged set, the resident column, or its page.
3830 fn column_entry(&mut self, e: &CatalogEntry) -> Result<(Vec<u16>, f32)> {
3831 let d = self.dimensions;
3832 if let Some(idx) = self.index.get(e.id, &self.catalog) {
3833 if let Some((h, n)) = self.col.hot.get(&idx) {
3834 return Ok((h.clone(), *n));
3835 }
3836 if self.col.resident && self.col.have.get(idx).copied().unwrap_or(false) {
3837 return Ok((
3838 self.col.data[idx * d..(idx + 1) * d].to_vec(),
3839 self.col.norms[idx],
3840 ));
3841 }
3842 }
3843 let r =
3844 e.v9.ok_or_else(|| MnemoError::Invalid("v9 row without a vector entry".into()))?;
3845 if r.vec_page == 0 {
3846 return Err(MnemoError::Invalid("vector entry not written yet".into()));
3847 }
3848 let page = self.pager.read_page(r.vec_page, r.vec_gen)?;
3849 veccol::decode_entry(&page, d, r.vec_slot as usize)
3850 }
3851
3852 /// FMT-9b: make the column resident if it fits the budget: allocate it
3853 /// (zeroed, so untouched rows cost no memory) and copy in the staged
3854 /// entries. Rows fill a vector page at a time on first use
3855 /// ([`Mnemo::fill_column_page`]). A no-op once resident, and on v8.
3856 fn ensure_column(&mut self) -> Result<()> {
3857 if !self.is_v9() || self.col.resident {
3858 return Ok(());
3859 }
3860 let d = self.dimensions;
3861 let rows = self.catalog.len();
3862 if rows * veccol::entry_len(d) > self.vector_cache_budget {
3863 return Ok(());
3864 }
3865 self.col.data = vec![0u16; rows * d];
3866 self.col.norms = vec![0f32; rows];
3867 self.col.have = vec![false; rows];
3868 let hot: Vec<(usize, Vec<u16>, f32)> = self
3869 .col
3870 .hot
3871 .iter()
3872 .filter(|(&i, _)| i < rows)
3873 .map(|(&i, (h, n))| (i, h.clone(), *n))
3874 .collect();
3875 for (i, h, n) in hot {
3876 self.col.data[i * d..(i + 1) * d].copy_from_slice(&h);
3877 self.col.norms[i] = n;
3878 self.col.have[i] = true;
3879 }
3880 self.col.resident = true;
3881 Ok(())
3882 }
3883
3884 /// FMT-9b: fill the resident column from row `idx`'s vector page: every
3885 /// live row on that page, in one decrypt that bypasses the page cache.
3886 fn fill_column_page(&mut self, idx: usize) -> Result<()> {
3887 let Some(r) = self.catalog[idx].v9 else {
3888 return Ok(());
3889 };
3890 if r.vec_page == 0 {
3891 return Ok(());
3892 }
3893 let d = self.dimensions;
3894 let bytes = self.pager.read_page_uncached(r.vec_page, r.vec_gen)?;
3895 let rows = self
3896 .col
3897 .page_rows
3898 .get(&r.vec_page)
3899 .cloned()
3900 .unwrap_or_default();
3901 for i in rows.into_iter().chain(std::iter::once(idx)) {
3902 if i >= self.col.have.len() || self.col.have[i] {
3903 continue;
3904 }
3905 let Some(ri) = self.catalog[i].v9 else {
3906 continue;
3907 };
3908 if ri.vec_page != r.vec_page {
3909 continue;
3910 }
3911 let n = veccol::decode_entry_into(
3912 &bytes,
3913 d,
3914 ri.vec_slot as usize,
3915 &mut self.col.data[i * d..(i + 1) * d],
3916 )?;
3917 self.col.norms[i] = n;
3918 self.col.have[i] = true;
3919 }
3920 Ok(())
3921 }
3922
3923 /// FMT-9b: write the staged column entries, plus the live entries of up
3924 /// to [`VACUUM_PAGES_PER_COMMIT`] sparse pages (under 3/4 full), into
3925 /// fresh vector pages. Runs at the top of a flush, before the catalog
3926 /// is serialized, since it moves rows' vector locations. Only when
3927 /// something is staged, so a flush with no new vectors writes none.
3928 fn pack_vectors(&mut self) -> Result<()> {
3929 if !self.is_v9() || self.col.pending.is_empty() {
3930 return Ok(());
3931 }
3932 let d = self.dimensions;
3933 let per = veccol::per_page(d);
3934 let mut sparse: Vec<(u64, usize)> = self
3935 .col
3936 .page_rows
3937 .iter()
3938 .filter(|(_, rows)| rows.len() * 4 < per * 3)
3939 .map(|(&p, rows)| (p, rows.len()))
3940 .collect();
3941 sparse.sort_unstable();
3942 let sparse = vacuum_pick(sparse);
3943 let mut order: Vec<usize> = Vec::new();
3944 let mut seen: std::collections::HashSet<usize> = std::collections::HashSet::new();
3945 for p in sparse {
3946 let rows = self.col.page_rows.remove(&p).unwrap_or_default();
3947 for idx in rows {
3948 let e = self.catalog[idx].clone();
3949 let v = self.column_entry(&e)?;
3950 self.col.hot.insert(idx, v);
3951 if seen.insert(idx) {
3952 order.push(idx);
3953 }
3954 }
3955 self.release_pages(p, 1);
3956 }
3957 let pending = std::mem::take(&mut self.col.pending);
3958 for idx in pending {
3959 if self.col.hot.contains_key(&idx) && seen.insert(idx) {
3960 order.push(idx);
3961 }
3962 }
3963 let gen = self.write_gen();
3964 for chunk in order.chunks(per) {
3965 let page = self.alloc_pages(1);
3966 let bytes = {
3967 let entries: Vec<(&[u16], f32)> = chunk
3968 .iter()
3969 .map(|i| {
3970 let (h, n) = &self.col.hot[i];
3971 (h.as_slice(), *n)
3972 })
3973 .collect();
3974 veccol::encode_page(d, gen, &entries)
3975 };
3976 self.pager.write_page(page, gen, &bytes)?;
3977 for (slot, &i) in chunk.iter().enumerate() {
3978 self.touch(i);
3979 if let Some(r) = self.catalog[i].v9.as_mut() {
3980 r.vec_page = page;
3981 r.vec_slot = slot as u16;
3982 r.vec_gen = gen;
3983 }
3984 }
3985 self.col.page_rows.insert(page, chunk.to_vec());
3986 }
3987 for i in order {
3988 self.col.hot.remove(&i);
3989 }
3990 self.dirty_catalog = true;
3991 Ok(())
3992 }
3993
3994 /// FMT-9: before this session seals its first v9 page, commit a
3995 /// header-only transaction that bumps `wal_seq`.
3996 ///
3997 /// A page's generation is the id of the transaction that writes it,
3998 /// and that id is `wal_seq + 1`, which is only made durable by the
3999 /// commit. If a session crashes before committing, its pages (orphan
4000 /// data pages, or uncommitted WAL frames) carry an id the next session
4001 /// would hand out again, for the same pages (allocation is
4002 /// deterministic). An attacker holding a copy of the crashed file could
4003 /// then swap an orphan back in under a matching generation. Burning
4004 /// one id durably per writable session rules that out: every id this
4005 /// session uses is above every id any earlier session used. Costs one
4006 /// small commit per session, on its first write only.
4007 fn ensure_gen_lease(&mut self) -> Result<()> {
4008 if !self.is_v9() || self.gen_leased || self.read_only {
4009 return Ok(());
4010 }
4011 self.header.wal_seq += 1;
4012 let wal_start = self.header.wal_start;
4013 self.relocate_header_through_wal(wal_start)?;
4014 self.gen_leased = true;
4015 Ok(())
4016 }
4017
4018 /// Write a serialized record (memory or cache entry) to fresh pages.
4019 /// Returns `(start_page, page_count, generation)`.
4020 fn write_record(&mut self, bytes: &[u8]) -> Result<(u64, u32, u64)> {
4021 self.ensure_gen_lease()?;
4022 let pc = bytes.len().div_ceil(PAYLOAD).max(1);
4023 let start = self.alloc_pages(pc as u64);
4024 let gen = self.write_gen();
4025 for i in 0..pc {
4026 let lo = i * PAYLOAD;
4027 let hi = ((i + 1) * PAYLOAD).min(bytes.len());
4028 self.pager
4029 .write_page(start + i as u64, gen, &bytes[lo..hi])?;
4030 }
4031 Ok((start, pc as u32, gen))
4032 }
4033
4034 /// Read a record run of `pages` pages at `start`, sealed under `gen`.
4035 fn read_record_run(&mut self, start: u64, pages: u32, len: u32, gen: u64) -> Result<Vec<u8>> {
4036 let mut buf = Vec::with_capacity(len as usize);
4037 for i in 0..pages as u64 {
4038 buf.extend_from_slice(&self.pager.read_page(start + i, gen)?);
4039 }
4040 buf.truncate(len as usize);
4041 Ok(buf)
4042 }
4043
4044 fn read_record(&mut self, e: &CatalogEntry) -> Result<Vec<u8>> {
4045 // FMT-9b.2: a packed v9 record is one slot of a shared page, or
4046 // still waiting for the next flush.
4047 if let Some(r) = e.v9 {
4048 if r.rec_slot != REC_SPAN {
4049 if e.start_page == 0 {
4050 return self
4051 .index
4052 .get(e.id, &self.catalog)
4053 .and_then(|idx| self.recs.hot.get(&idx).cloned())
4054 .ok_or_else(|| MnemoError::Invalid("record not written yet".into()));
4055 }
4056 let page = self.pager.read_page(e.start_page, e.gen)?;
4057 return record_slot(&page, r.rec_slot);
4058 }
4059 }
4060 let mut buf = Vec::with_capacity(e.len as usize);
4061 for i in 0..e.page_count as u64 {
4062 buf.extend_from_slice(&self.pager.read_page(e.start_page + i, e.gen)?);
4063 }
4064 buf.truncate(e.len as usize);
4065 Ok(buf)
4066 }
4067
4068 /// Decode a stored memory record body. FMT-9e: on v9 it may be a zstd
4069 /// frame (a compressed turn); v8 records are always plain.
4070 fn decode_memory(&self, bytes: &[u8]) -> Result<Memory> {
4071 let plain = if self.is_v9() {
4072 plain_record(bytes)?
4073 } else {
4074 std::borrow::Cow::Borrowed(bytes)
4075 };
4076 rmp_serde::from_slice(&plain).map_err(|err| MnemoError::Serialize(err.to_string()))
4077 }
4078
4079 fn read_memory(&mut self, e: &CatalogEntry) -> Result<Memory> {
4080 let m = self.read_memory_meta(e)?;
4081 self.finish_memory(e, m)
4082 }
4083
4084 /// Complete a record decoded by [`Mnemo::read_memory_meta`]: its
4085 /// vector and the catalog's live access stats.
4086 fn finish_memory(&mut self, e: &CatalogEntry, mut m: Memory) -> Result<Memory> {
4087 // FMT-9b: a v9 record carries no vector (unless the file keeps
4088 // float vectors); it comes from the column, fp16-rounded.
4089 if e.v9.is_some() && !self.float_vectors() {
4090 let (h, _) = self.column_entry(e)?;
4091 m.vector = veccol::dequantize(&h);
4092 }
4093 // The catalog is the source of truth for access stats — the values
4094 // in the record body are a stale snapshot from when the record was
4095 // last written. Overwrite them here so consumers always see live
4096 // values regardless of when the record was last serialized.
4097 m.accessed_at = e.accessed_at;
4098 m.access_count = e.access_count;
4099 Ok(m)
4100 }
4101
4102 /// Serialize and store a memory (insert or overwrite by ID).
4103 fn put(&mut self, mut m: Memory) -> Result<Ulid> {
4104 if m.vector.len() != self.dimensions {
4105 return Err(MnemoError::DimensionMismatch {
4106 expected: self.dimensions,
4107 got: m.vector.len(),
4108 });
4109 }
4110 // Reject all-zero vectors (Phase 3 PR 10a; breaking vs 0.4.x).
4111 // A zero vector scores exactly 0.0 on every cosine query — it
4112 // can't contribute signal, but recency / importance can still
4113 // pull it into top-k as noise. We'd rather fail loudly at
4114 // `remember` than let a mis-embedded entry pollute recall
4115 // quality for the life of the database. Scaffold manifests now
4116 // use a sentinel unit vector so this check doesn't trip them.
4117 if embed::is_zero_vector(&m.vector) {
4118 return Err(MnemoError::Invalid(
4119 "all-zero vector refused: it scores 0.0 on every cosine query and \
4120 pollutes recall; check your embedder returned a real embedding"
4121 .into(),
4122 ));
4123 }
4124 if m.id == Ulid::nil() {
4125 m.id = Ulid::new();
4126 }
4127 let id_u: u128 = m.id.0;
4128 // R-004.2: an overwrite with the handle caches built reads the
4129 // stored version first (one decrypt), so the caches drop exactly
4130 // what it contributed.
4131 if self.layers.is_some() {
4132 if let Some(idx) = self.index.get(id_u, &self.catalog) {
4133 let old_entry = self.catalog[idx].clone();
4134 if !old_entry.deleted {
4135 let old = self.read_memory(&old_entry)?;
4136 self.note_removed(&old)?;
4137 }
4138 }
4139 }
4140 // FMT-9b: a v9 record is stored without its vector (unless the file
4141 // keeps float vectors); the vector goes to the fp16 column.
4142 let v9 = self.is_v9();
4143 let bytes = if v9 && !self.float_vectors() {
4144 let vector = std::mem::take(&mut m.vector);
4145 let b = rmp_serde::to_vec(&m);
4146 m.vector = vector;
4147 b
4148 } else {
4149 rmp_serde::to_vec(&m)
4150 }
4151 .map_err(|e| MnemoError::Serialize(e.to_string()))?;
4152 // FMT-9e: a v9 thread turn is stored zstd-compressed (turns are
4153 // most of a thread-heavy file). The catalog's `len` is the stored
4154 // length, so packing and vacuum move the frame as is.
4155 // FMT-9e (compress when cold): a turn is compressed at write only
4156 // when it's already past the handle's cutoff (an imported
4157 // archive, or `set_turn_compress_after(0)`); otherwise the commit
4158 // that sees it age compresses it.
4159 let is_turn = v9 && meta_str(&m, MNEMO_META_KIND) == Some(MNEMO_META_KIND_TURN);
4160 let settle =
4161 is_turn && (bytes.len() < TURN_ZSTD_MIN_BYTES || self.turn_is_cold(m.created_at));
4162 let bytes = if settle {
4163 compress_turn_record(bytes)
4164 } else {
4165 bytes
4166 };
4167 // FMT-9b.2: a v9 record that fits a packed page waits for the next
4168 // flush, which packs it with others; a bigger one owns a run of
4169 // whole pages, written now as on v8.
4170 let packed = v9 && bytes.len() + REC_SLOT_BYTES <= REC_PAGE_CAP;
4171 // A waiting record has `start_page` 0. Its generation is never 0 on
4172 // v9: the row encoding leaves out a zero `gen`, which would shift
4173 // the `v9` field into its place (a deleted row can stay unpacked).
4174 let (start, pc, gen) = if packed {
4175 self.ensure_gen_lease()?;
4176 (0, 1, self.write_gen())
4177 } else {
4178 self.write_record(&bytes)?
4179 };
4180 let rec_len = bytes.len();
4181 // FMT-9d: a keyword-indexed v9 row gets a new BM25 document.
4182 let mut tags = v9.then(|| row_tags(&m));
4183 if let Some(r) = tags.as_mut() {
4184 if bm25_kind(r.kind) {
4185 let (terms, len) = text::term_freqs(&m.content);
4186 r.doc = self.bm25.manifest.next_doc;
4187 r.doc_len = len;
4188 self.bm25.manifest.next_doc += 1;
4189 self.bm25.pending.push((r.doc, terms));
4190 }
4191 }
4192 // FMT-9f: keep the BM25 stats current (below, once the row is
4193 // placed) instead of rescanning every row at the next commit.
4194 let bm25_new = tags.filter(|r| r.doc != 0 && bm25_kind(r.kind));
4195 // For overwrites preserve the existing access stats; for inserts
4196 // seed them from the memory (which carries them in its body).
4197 let (prev_accessed, prev_count) = match self.index.get(id_u, &self.catalog) {
4198 Some(idx) => (
4199 self.catalog[idx].accessed_at,
4200 self.catalog[idx].access_count,
4201 ),
4202 None => (m.accessed_at, m.access_count),
4203 };
4204 let entry = CatalogEntry {
4205 id: id_u,
4206 start_page: start,
4207 page_count: pc,
4208 len: rec_len as u32,
4209 deleted: false,
4210 accessed_at: prev_accessed,
4211 access_count: prev_count,
4212 gen,
4213 v9: tags.map(|mut r| {
4214 r.rec_slot = if packed { 0 } else { REC_SPAN };
4215 if settle {
4216 r.flags |= ROW_SETTLED;
4217 }
4218 r
4219 }),
4220 };
4221 match self.index.get(id_u, &self.catalog) {
4222 Some(idx) => {
4223 // FMT-9 (a): the overwritten version's pages are freed by
4224 // this transaction's commit. A tombstone's pages were
4225 // already freed when its delete committed.
4226 if !self.catalog[idx].deleted {
4227 let old = self.catalog[idx].clone();
4228 if v9 {
4229 self.kill_vector(idx, &old);
4230 self.kill_record(idx, &old);
4231 } else {
4232 self.release_pages(old.start_page, old.page_count as u64);
4233 }
4234 }
4235 if let (Some(stats), Some(old)) = (self.bm25.stats.as_mut(), self.catalog[idx].v9) {
4236 if !self.catalog[idx].deleted && old.doc != 0 {
4237 stats.remove(old.doc);
4238 }
4239 }
4240 self.catalog[idx] = entry;
4241 self.touch(idx);
4242 }
4243 None => {
4244 self.index.insert(id_u, self.catalog.len(), &self.catalog);
4245 self.touch(self.catalog.len());
4246 self.catalog.push(entry);
4247 }
4248 }
4249 if let (Some(stats), Some(r)) = (self.bm25.stats.as_mut(), bm25_new) {
4250 if let Some(idx) = self.index.get(id_u, &self.catalog) {
4251 stats.insert(r.doc, idx, r.doc_len);
4252 }
4253 }
4254 self.dirty_catalog = true;
4255 if v9 {
4256 if let Some(idx) = self.index.get(id_u, &self.catalog) {
4257 self.stage_vector(idx, &m.vector);
4258 if packed {
4259 self.recs.hot.insert(idx, bytes);
4260 self.recs.pending.push(idx);
4261 }
4262 }
4263 }
4264
4265 // Keep the ANN index complete: assign the (new or changed) vector to
4266 // its nearest partition. Centroids/codebook stay fixed until rebuild.
4267 // R-004.2: only facts go in; turns, essence items and thread links
4268 // are read by kind and scope.
4269 if let Some(ann) = &mut self.ann {
4270 if is_ann_kind(&m) {
4271 ann.add(id_u, &m.vector);
4272 self.dirty_index = true;
4273 self.index_persist.changes += 1;
4274 } else if ann.contains(id_u) {
4275 ann.remove(id_u);
4276 self.dirty_index = true;
4277 self.index_persist.changes += 1;
4278 }
4279 }
4280 self.note_added(&m)?;
4281
4282 // FIX-1 R-001.8 part 2: keep the lazy agent set in sync. If
4283 // it hasn't been built yet (no `stats()` call this session),
4284 // skip — the first `stats()` will build it from scratch. If
4285 // it has, add this memory's agent_id. An overwrite with the
4286 // same id and same agent is a no-op (BTreeSet::insert returns
4287 // false); an overwrite with a *changed* agent_id adds the new
4288 // agent but doesn't drop the old (we can't tell without a
4289 // decrypt whether the old agent still has live memories).
4290 // That's conservative in the right direction — `stats()` may
4291 // over-report by one agent until the next delete invalidates.
4292 if let Some(set) = &mut self.agent_set {
4293 set.insert(m.agent_id.clone());
4294 }
4295 Ok(m.id)
4296 }
4297
4298 // --- public CRUD ------------------------------------------------------
4299
4300 /// Register a text embedder for this handle. Enables
4301 /// [`Mnemo::remember_text`] and the text-side `RecallRequest`
4302 /// convenience path. Does two checks before storing the embedder:
4303 ///
4304 /// 1. **Dimension check.** The embedder's `identity().dimensions`
4305 /// must equal [`Mnemo::dimensions`] (the width this database
4306 /// was created with). Mismatch returns
4307 /// [`MnemoError::DimensionMismatch`].
4308 /// 2. **Manifest identity check.** If the scaffold / onboarding
4309 /// manifest already records an embedder — the canonical shape
4310 /// is `metadata.embedder = { name, dimensions, normalize }`,
4311 /// with the legacy flat `embedder_name` / `embedder_dimensions`
4312 /// / `embedder_normalize` keys accepted as an alias — the new
4313 /// embedder's identity must match exactly. A different name —
4314 /// even at the same dimensions — returns
4315 /// [`MnemoError::Invalid`] with
4316 /// [`EmbedderIdentity::MISMATCH_MSG`] in the message. If the
4317 /// manifest has no embedder metadata yet, this call upserts
4318 /// the nested object into the first onboarding/manifest memory
4319 /// (write + flush so a subsequent reopen sees the identity).
4320 ///
4321 /// **Read-only handles.** `set_embedder` on an `open_read_only`
4322 /// handle performs both checks and adopts the embedder in memory;
4323 /// it never writes. Readers (CLI `--text`, agent-side `recall_text`)
4324 /// can therefore use the text path next to a writer that holds
4325 /// the exclusive lock.
4326 ///
4327 /// On both failure paths the handle's embedder slot is left
4328 /// unchanged. On success the embedder is wrapped in an `Arc<dyn
4329 /// Embedder>` and consulted by every text-side call made through
4330 /// this handle. The slot is per-handle and not persisted — a
4331 /// fresh open must re-register (same identity, so the manifest
4332 /// check round-trips cleanly).
4333 pub fn set_embedder<E: Embedder + 'static>(&mut self, embedder: E) -> Result<()> {
4334 let identity = embedder.identity();
4335 if identity.dimensions != self.dimensions {
4336 return Err(MnemoError::DimensionMismatch {
4337 expected: self.dimensions,
4338 got: identity.dimensions,
4339 });
4340 }
4341
4342 // Find a scaffold/onboarding manifest to inspect (or upsert
4343 // into). `about()` already filters + sorts the onboarding set.
4344 let onboarding = self.about()?;
4345 let mut upsert_target: Option<usize> = None;
4346 for (i, m) in onboarding.iter().enumerate() {
4347 if let Some(saw) = embed::read_identity_from_metadata(&m.metadata) {
4348 if saw != identity {
4349 return Err(embed::identity_mismatch_err(&saw, &identity));
4350 }
4351 // Match — nothing to persist; adopt + done. Works the
4352 // same on read-only and read-write handles.
4353 self.embedder = Some(std::sync::Arc::new(embedder));
4354 return Ok(());
4355 }
4356 if upsert_target.is_none() {
4357 upsert_target = Some(i);
4358 }
4359 }
4360
4361 // No existing identity on the manifest. On a read-write handle
4362 // upsert into the first onboarding memory so later opens see
4363 // it; on a read-only handle skip the write (readers must not
4364 // mutate the file), but still adopt the embedder in memory.
4365 if !self.read_only {
4366 if let Some(idx) = upsert_target {
4367 let mut m = onboarding[idx].clone();
4368 embed::write_identity_into_metadata(&mut m.metadata, &identity);
4369 // Reuse the memory's existing id so this is an in-place
4370 // update of the scaffold manifest, not a new entry.
4371 self.put(m)?;
4372 self.flush()?;
4373 }
4374 }
4375
4376 self.embedder = Some(std::sync::Arc::new(embedder));
4377 Ok(())
4378 }
4379
4380 /// Borrow the currently-registered embedder, if any. Returns
4381 /// `None` when no embedder has been set via
4382 /// [`Mnemo::set_embedder`]. Bindings use this to drive their own
4383 /// text-side convenience methods without duplicating the identity
4384 /// check.
4385 pub fn embedder(&self) -> Option<&dyn Embedder> {
4386 self.embedder.as_deref()
4387 }
4388
4389 /// Shared handle to the registered embedder, cloned so the caller
4390 /// can drop the borrow on this `Mnemo` before invoking `embed()`.
4391 /// FIX-1 / PY-1 (R-001.11): the Python binding needs this so an
4392 /// embedder callback that re-enters `Mnemo` (e.g. a Python
4393 /// embedder that calls `db.dimensions()` or `db.cache_get(...)`)
4394 /// doesn't hit PyO3's "Already borrowed" — we clone the Arc under
4395 /// a short `&self` borrow, release that borrow, then invoke
4396 /// `embed()` with no `&mut Mnemo` held.
4397 pub fn embedder_arc(&self) -> Option<std::sync::Arc<dyn Embedder>> {
4398 self.embedder.clone()
4399 }
4400
4401 /// Embed `content` with the registered embedder and `remember` the
4402 /// result. Returns the new memory's ULID. Convenience path for
4403 /// callers who have a string, want it stored, and don't need any
4404 /// of the per-memory options [`Memory`] exposes (agent, importance,
4405 /// metadata, ttl, scope) — build a [`Memory`] by hand and call
4406 /// [`Mnemo::remember`] for those.
4407 ///
4408 /// Fails with [`MnemoError::Invalid`] if no embedder has been
4409 /// registered on this handle (call [`Mnemo::set_embedder`] first);
4410 /// with [`MnemoError::DimensionMismatch`] if the embedder returned
4411 /// the wrong width for this database.
4412 pub fn remember_text(&mut self, content: &str, memory_type: MemoryType) -> Result<Ulid> {
4413 if self.read_only {
4414 return Err(MnemoError::ReadOnly);
4415 }
4416 let embedder = self.embedder.as_ref().ok_or_else(|| {
4417 MnemoError::Invalid(
4418 "no embedder registered on this handle; call set_embedder first".into(),
4419 )
4420 })?;
4421 let mut vectors = embedder.embed(&[content])?;
4422 let vector = vectors.pop().ok_or_else(|| {
4423 MnemoError::Invalid("embedder returned no vector for the input text".into())
4424 })?;
4425 if vector.len() != self.dimensions {
4426 return Err(MnemoError::DimensionMismatch {
4427 expected: self.dimensions,
4428 got: vector.len(),
4429 });
4430 }
4431 let memory = Memory::new(content.to_string(), memory_type, vector);
4432 self.remember(memory)
4433 }
4434
4435 /// Store a memory. Returns its ULID. If the memory's ID is nil a fresh
4436 /// one is assigned; an existing ID overwrites in place.
4437 pub fn remember(&mut self, memory: Memory) -> Result<Ulid> {
4438 if self.read_only {
4439 return Err(MnemoError::ReadOnly);
4440 }
4441 self.put(memory)
4442 }
4443
4444 /// Fetch a memory by ID.
4445 pub fn get(&mut self, id: &Ulid) -> Result<Memory> {
4446 let idx = self
4447 .index
4448 .get(id.0, &self.catalog)
4449 .ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
4450 let entry = self.catalog[idx].clone();
4451 if entry.deleted {
4452 return Err(MnemoError::NotFound(id.to_string()));
4453 }
4454 self.read_memory(&entry)
4455 }
4456
4457 /// Soft-delete a memory (tombstoned; space reclaimed by `compact`).
4458 pub fn delete(&mut self, id: &Ulid) -> Result<()> {
4459 if self.read_only {
4460 return Err(MnemoError::ReadOnly);
4461 }
4462 let idx = self
4463 .index
4464 .get(id.0, &self.catalog)
4465 .ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
4466 // R-004.2 / 12.1: with the handle caches built, read the record
4467 // being deleted (one decrypt) so the key slot, superseded set and
4468 // thread layers drop exactly what it contributed.
4469 let entry = self.catalog[idx].clone();
4470 if self.layers.is_some() && !entry.deleted {
4471 let old = self.read_memory(&entry)?;
4472 self.note_removed(&old)?;
4473 }
4474 if !self.catalog[idx].deleted {
4475 self.catalog[idx].deleted = true;
4476 self.dirty_catalog = true;
4477 self.touch(idx);
4478 // FMT-9f: drop its document from the stats in place.
4479 if let (Some(stats), Some(r)) = (self.bm25.stats.as_mut(), entry.v9) {
4480 if r.doc != 0 {
4481 stats.remove(r.doc);
4482 }
4483 }
4484 if self.is_v9() {
4485 self.kill_record(idx, &entry);
4486 self.kill_vector(idx, &entry);
4487 } else {
4488 self.release_pages(entry.start_page, entry.page_count as u64);
4489 }
4490 }
4491 if let Some(ann) = &mut self.ann {
4492 if ann.contains(id.0) {
4493 ann.remove(id.0);
4494 self.dirty_index = true;
4495 self.index_persist.changes += 1;
4496 }
4497 }
4498 // FIX-1 R-001.8 part 2: we can't tell whether this delete
4499 // removed the last memory of its agent without a decrypt
4500 // (the catalog entry doesn't carry `agent_id` in format v8),
4501 // so just invalidate the lazy set. The next `stats()` call
4502 // rebuilds from live memories.
4503 self.agent_set = None;
4504 // Phase 12.1: delete of a keyed record removes it from the slot,
4505 // and delete of a superseded record frees a `history` entry. When
4506 // the caches are built, `note_removed` above did that exactly;
4507 // otherwise they're built from scratch on first use anyway.
4508 Ok(())
4509 }
4510
4511 /// Keep the handle caches (12.1 key slots, superseded set, R-004.2
4512 /// layers) current after `put` writes `m`. No-op until they're built.
4513 /// FMT-9e: a cold thread is warmed first, so its layer entry is whole.
4514 fn note_added(&mut self, m: &Memory) -> Result<()> {
4515 if let Some(thread) = meta_str(m, MNEMO_META_THREAD) {
4516 self.warm_thread(thread)?;
4517 }
4518 let id = m.id.0;
4519 if let (Some(keys), Some(sup)) = (self.key_index.as_mut(), self.superseded.as_mut()) {
4520 if m.metadata.contains_key(MNEMO_META_SUPERSEDED_BY) {
4521 sup.insert(id);
4522 } else if let Some(Value::String(k)) = m.metadata.get(MNEMO_META_KEY) {
4523 keys.insert(Self::key_slot_for(m, k), id);
4524 }
4525 }
4526 if let Some(layers) = self.layers.as_mut() {
4527 layers.add(m);
4528 }
4529 Ok(())
4530 }
4531
4532 /// Undo [`Mnemo::note_added`] for the stored version of a record that's
4533 /// about to be overwritten or deleted.
4534 fn note_removed(&mut self, m: &Memory) -> Result<()> {
4535 if let Some(thread) = meta_str(m, MNEMO_META_THREAD) {
4536 self.warm_thread(thread)?;
4537 }
4538 let id = m.id.0;
4539 if let (Some(keys), Some(sup)) = (self.key_index.as_mut(), self.superseded.as_mut()) {
4540 sup.remove(&id);
4541 if let Some(Value::String(k)) = m.metadata.get(MNEMO_META_KEY) {
4542 let slot = Self::key_slot_for(m, k);
4543 if keys.get(&slot) == Some(&id) {
4544 keys.remove(&slot);
4545 }
4546 }
4547 }
4548 if let Some(layers) = self.layers.as_mut() {
4549 layers.remove(m);
4550 }
4551 Ok(())
4552 }
4553
4554 /// FMT-9e: decode `thread`'s cold turn records (see
4555 /// [`LayerCache::cold_turns`]) into the layer cache. Every record in
4556 /// the thread's hash bucket is decoded and filed under its own thread,
4557 /// so a hash collision just warms both threads. A no-op when the
4558 /// caches aren't built or the bucket is already warm.
4559 fn warm_thread(&mut self, thread: &str) -> Result<()> {
4560 self.warm_thread_decoded(thread).map(|_| ())
4561 }
4562
4563 /// [`Mnemo::warm_thread`], returning the records it decoded (without
4564 /// vectors), so a caller about to read the thread decodes each once.
4565 fn warm_thread_decoded(
4566 &mut self,
4567 thread: &str,
4568 ) -> Result<std::collections::HashMap<u128, Memory>> {
4569 let mut decoded = std::collections::HashMap::new();
4570 let h = fnv1a32(&[thread.as_bytes()]);
4571 let Some(ids) = self.layers.as_mut().and_then(|l| l.cold_turns.remove(&h)) else {
4572 return Ok(decoded);
4573 };
4574 for e in self.live_entries(ids) {
4575 let m = self.read_memory_meta(&e)?;
4576 if let Some(layers) = self.layers.as_mut() {
4577 layers.add(&m);
4578 }
4579 decoded.insert(e.id, m);
4580 }
4581 Ok(decoded)
4582 }
4583
4584 /// Decode a record's memory without its vector or live access stats:
4585 /// what the handle caches need (metadata, scope, agent), without the
4586 /// v9 vector-column read [`Mnemo::read_memory`] adds.
4587 fn read_memory_meta(&mut self, e: &CatalogEntry) -> Result<Memory> {
4588 let bytes = self.read_record(e)?;
4589 self.decode_memory(&bytes)
4590 }
4591
4592 /// The live catalog entries for `ids` (ids that are gone are skipped).
4593 fn live_entries(&self, ids: impl IntoIterator<Item = u128>) -> Vec<CatalogEntry> {
4594 ids.into_iter()
4595 .filter_map(|id| {
4596 self.index
4597 .get(id, &self.catalog)
4598 .map(|i| self.catalog[i].clone())
4599 })
4600 .filter(|e| !e.deleted)
4601 .collect()
4602 }
4603
4604 /// Catalog entries of the essence items, from the handle caches.
4605 fn essence_entries(&mut self) -> Result<Vec<CatalogEntry>> {
4606 self.ensure_key_caches()?;
4607 let ids: Vec<u128> = self
4608 .layers
4609 .as_ref()
4610 .map(|l| l.essence.iter().copied().collect())
4611 .unwrap_or_default();
4612 Ok(self.live_entries(ids))
4613 }
4614
4615 /// Catalog entries of every record filed under `thread`, from the
4616 /// handle caches.
4617 fn thread_entries(&mut self, thread: &str) -> Result<Vec<CatalogEntry>> {
4618 Ok(self.thread_entries_decoded(thread)?.0)
4619 }
4620
4621 /// [`Mnemo::thread_entries`], plus the records that warming a cold
4622 /// thread just decoded (see [`Mnemo::warm_thread_decoded`]).
4623 fn thread_entries_decoded(
4624 &mut self,
4625 thread: &str,
4626 ) -> Result<(Vec<CatalogEntry>, std::collections::HashMap<u128, Memory>)> {
4627 self.ensure_key_caches()?;
4628 let decoded = self.warm_thread_decoded(thread)?;
4629 let ids: Vec<u128> = self
4630 .layers
4631 .as_ref()
4632 .and_then(|l| l.threads.get(thread))
4633 .map(|tc| tc.records.iter().copied().collect())
4634 .unwrap_or_default();
4635 Ok((self.live_entries(ids), decoded))
4636 }
4637
4638 // --- Phase 12.1: edit in place + keyed supersession -----------------
4639
4640 /// Scope-aware key-slot identifier for `memory` and a caller key.
4641 ///
4642 /// * [`Scope::Private`] → owned by the memory's `agent_id`.
4643 /// * [`Scope::Shared`] → one slot across the whole file.
4644 fn key_slot_for(memory: &Memory, key: &str) -> crate::store::KeySlot {
4645 let scope_owner = match memory.scope {
4646 Scope::Private => Some(memory.agent_id.clone()),
4647 Scope::Shared => None,
4648 };
4649 crate::store::KeySlot {
4650 scope_owner,
4651 key: key.to_string(),
4652 }
4653 }
4654
4655 /// The handle's one lazy scan: builds the 12.1 `key_index` +
4656 /// `superseded` caches and the R-004.2 layer cache (essence ids,
4657 /// onboarding ids, threads, links) from live records, and prunes
4658 /// non-fact ids from a loaded ANN index (files written before v0.10.0
4659 /// indexed every kind). Decrypts every live record once per handle;
4660 /// `put` and `delete` keep the result current after that.
4661 fn ensure_key_caches(&mut self) -> Result<()> {
4662 if self.key_index.is_some() && self.superseded.is_some() && self.layers.is_some() {
4663 return Ok(());
4664 }
4665 let entries: Vec<CatalogEntry> = self
4666 .catalog
4667 .iter()
4668 .filter(|e| !e.deleted)
4669 .cloned()
4670 .collect();
4671 let mut keys: std::collections::HashMap<crate::store::KeySlot, u128> =
4672 std::collections::HashMap::new();
4673 let mut superseded: std::collections::HashSet<u128> = std::collections::HashSet::new();
4674 let mut layers = LayerCache::default();
4675 let mut non_facts: Vec<u128> = Vec::new();
4676 for e in &entries {
4677 // FMT-9e: a v9 turn row with no key slot and no supersession
4678 // contributes only to its thread's layer entry, so it's left
4679 // undecoded in the thread's cold bucket (most of a
4680 // thread-heavy file). Its row says it isn't a fact.
4681 if let Some(r) = e.v9 {
4682 if r.kind == ROW_KIND_TURN
4683 && r.thread_hash != 0
4684 && r.key_hash == 0
4685 && r.flags & ROW_SUPERSEDED == 0
4686 {
4687 layers
4688 .cold_turns
4689 .entry(r.thread_hash)
4690 .or_default()
4691 .push(e.id);
4692 non_facts.push(e.id);
4693 continue;
4694 }
4695 }
4696 let m = self.read_memory_meta(e)?;
4697 layers.add(&m);
4698 if !is_ann_kind(&m) {
4699 non_facts.push(e.id);
4700 }
4701 if m.metadata.get(MNEMO_META_SUPERSEDED_BY).is_some() {
4702 superseded.insert(e.id);
4703 continue;
4704 }
4705 if let Some(Value::String(k)) = m.metadata.get(MNEMO_META_KEY) {
4706 let slot = Self::key_slot_for(&m, k);
4707 // Catalog order is insertion order; a later live record
4708 // with the same slot would only exist if a prior build
4709 // of this code allowed it. Keep the later id (newer
4710 // ULID), in case of a tie.
4711 keys.entry(slot)
4712 .and_modify(|cur| {
4713 if *cur < e.id {
4714 *cur = e.id;
4715 }
4716 })
4717 .or_insert(e.id);
4718 }
4719 }
4720 self.key_index = Some(keys);
4721 self.superseded = Some(superseded);
4722 self.layers = Some(layers);
4723 if let Some(ann) = &mut self.ann {
4724 let mut pruned = 0usize;
4725 for id in non_facts {
4726 if ann.contains(id) {
4727 ann.remove(id);
4728 pruned += 1;
4729 }
4730 }
4731 if pruned > 0 {
4732 self.dirty_index = true;
4733 self.index_persist.changes += pruned;
4734 }
4735 }
4736 Ok(())
4737 }
4738
4739 /// Stage-identical check for a 12.1 keyed no-op. The incoming write
4740 /// is byte-equal to the live record iff every user-visible field
4741 /// (content, type, importance, vector, user metadata) matches. The
4742 /// `_mnemo.*` reserved keys on the live record are **ignored** for
4743 /// this comparison — a repeat of an identical write re-issued by an
4744 /// extractor must be a NOOP even if the live record already carries
4745 /// supersession bookkeeping from a prior op.
4746 fn is_identical_keyed_write(incoming: &Memory, live: &Memory) -> bool {
4747 if incoming.content != live.content {
4748 return false;
4749 }
4750 if incoming.memory_type != live.memory_type {
4751 return false;
4752 }
4753 if (incoming.importance - live.importance).abs() > f32::EPSILON {
4754 return false;
4755 }
4756 if incoming.vector != live.vector {
4757 return false;
4758 }
4759 // Strip `_mnemo.*` from both sides before comparing user metadata.
4760 let strip = |m: &Map<String, Value>| -> Map<String, Value> {
4761 m.iter()
4762 .filter(|(k, _)| !k.starts_with(crate::memory::MNEMO_META_NS))
4763 .map(|(k, v)| (k.clone(), v.clone()))
4764 .collect()
4765 };
4766 strip(&incoming.metadata) == strip(&live.metadata)
4767 }
4768
4769 /// Store a memory under a keyed upsert slot. Returns the live record's
4770 /// id and a [`WriteOutcome`] saying whether this write created a new
4771 /// slot (`Added`), replaced a prior record (`Superseded`), or matched
4772 /// the existing live record byte-for-byte (`Unchanged`, no file write).
4773 ///
4774 /// The slot is identified by `(scope_owner, key)`:
4775 /// * `Scope::Private` memories share a slot per `(agent_id, key)`.
4776 /// * `Scope::Shared` memories share one slot per `key` across the file.
4777 ///
4778 /// On supersession the prior record is **not** deleted — it remains
4779 /// retrievable through [`Mnemo::history`] — but it is excluded from
4780 /// default [`Mnemo::recall`] and [`Mnemo::about`]. A caller that wants
4781 /// the full lineage of a key walks `history(key)`.
4782 ///
4783 /// This is the atomic 12.1 "supersede-or-noop" write path. Plain
4784 /// [`Mnemo::remember`] without a key is unchanged.
4785 pub fn remember_keyed(
4786 &mut self,
4787 mut memory: Memory,
4788 key: &str,
4789 ) -> Result<(Ulid, WriteOutcome)> {
4790 if self.read_only {
4791 return Err(MnemoError::ReadOnly);
4792 }
4793 if key.is_empty() {
4794 return Err(MnemoError::Invalid(
4795 "remember_keyed: 'key' must not be empty".into(),
4796 ));
4797 }
4798 self.ensure_key_caches()?;
4799 let slot = Self::key_slot_for(&memory, key);
4800 let existing = self.key_index.as_ref().and_then(|k| k.get(&slot).copied());
4801
4802 // Caller may already have set `_mnemo.key` on the memory; rewrite
4803 // it to our canonical value regardless so the record is
4804 // self-identifying.
4805 memory
4806 .metadata
4807 .insert(MNEMO_META_KEY.into(), Value::String(key.to_string()));
4808
4809 // NOOP check: if a live record exists for this slot and its
4810 // user-visible fields match, return the existing id without a
4811 // write.
4812 if let Some(prev_id_u) = existing {
4813 let prev_idx = self
4814 .index
4815 .get(prev_id_u, &self.catalog)
4816 .ok_or_else(|| MnemoError::Invalid("key_index drift".into()))?;
4817 let prev_entry = self.catalog[prev_idx].clone();
4818 let prev_mem = self.read_memory(&prev_entry)?;
4819 if Self::is_identical_keyed_write(&memory, &prev_mem) {
4820 return Ok((Ulid(prev_id_u), WriteOutcome::Unchanged));
4821 }
4822
4823 // SUPERSESSION: insert the new record, then rewrite the old
4824 // with `_mnemo.superseded_by` + `_mnemo.invalid_at`.
4825 let now = memory::now_secs();
4826 memory.id = Ulid::nil(); // force a fresh ULID in `put`
4827 memory.metadata.insert(
4828 MNEMO_META_SUPERSEDES.into(),
4829 Value::String(Ulid(prev_id_u).to_string()),
4830 );
4831 let new_id = self.put(memory)?;
4832 // Mark the prior record.
4833 let mut prev_mut = prev_mem;
4834 prev_mut.metadata.insert(
4835 MNEMO_META_SUPERSEDED_BY.into(),
4836 Value::String(new_id.to_string()),
4837 );
4838 prev_mut
4839 .metadata
4840 .insert(MNEMO_META_INVALID_AT.into(), Value::Number(now.into()));
4841 // keep prev_mut.id so `put` overwrites in place.
4842 self.put(prev_mut)?;
4843 // Update caches incrementally.
4844 if let Some(k) = self.key_index.as_mut() {
4845 k.insert(slot, new_id.0);
4846 }
4847 if let Some(s) = self.superseded.as_mut() {
4848 s.insert(prev_id_u);
4849 }
4850 return Ok((new_id, WriteOutcome::Superseded));
4851 }
4852
4853 // ADDED: no prior record for this slot.
4854 memory.id = Ulid::nil();
4855 let new_id = self.put(memory)?;
4856 if let Some(k) = self.key_index.as_mut() {
4857 k.insert(slot, new_id.0);
4858 }
4859 Ok((new_id, WriteOutcome::Added))
4860 }
4861
4862 /// Text-side keyed write: embed `content` through the registered
4863 /// embedder, then [`Mnemo::remember_keyed`]. Same returns.
4864 pub fn remember_keyed_text(
4865 &mut self,
4866 content: &str,
4867 memory_type: MemoryType,
4868 key: &str,
4869 ) -> Result<(Ulid, WriteOutcome)> {
4870 if self.read_only {
4871 return Err(MnemoError::ReadOnly);
4872 }
4873 let embedder = self.embedder.as_ref().ok_or_else(|| {
4874 MnemoError::Invalid(
4875 "no embedder registered on this handle; call set_embedder first".into(),
4876 )
4877 })?;
4878 let mut vectors = embedder.embed(&[content])?;
4879 let vector = vectors.pop().ok_or_else(|| {
4880 MnemoError::Invalid("embedder returned no vector for the input text".into())
4881 })?;
4882 if vector.len() != self.dimensions {
4883 return Err(MnemoError::DimensionMismatch {
4884 expected: self.dimensions,
4885 got: vector.len(),
4886 });
4887 }
4888 let memory = Memory::new(content.to_string(), memory_type, vector);
4889 self.remember_keyed(memory, key)
4890 }
4891
4892 /// Update an existing memory in place, preserving its ULID,
4893 /// `created_at` and access stats. Returns [`WriteOutcome::Updated`]
4894 /// on a real change, [`WriteOutcome::Unchanged`] when every field in
4895 /// the patch already matches the live record.
4896 ///
4897 /// Reserved `_mnemo.*` metadata keys on the existing record are
4898 /// preserved across a `metadata` replacement; a `metadata` of
4899 /// `Some(map)` replaces every **user-visible** key with `map`'s
4900 /// entries. A `content` change requires a matching `vector`
4901 /// (use [`Mnemo::update_text`] to re-embed) — a bare `content` patch
4902 /// is rejected to stop text and embedding drifting apart silently.
4903 pub fn update(&mut self, id: &Ulid, patch: MemoryPatch) -> Result<WriteOutcome> {
4904 if self.read_only {
4905 return Err(MnemoError::ReadOnly);
4906 }
4907 if patch.content.is_some() && patch.vector.is_none() {
4908 return Err(MnemoError::Invalid(
4909 "update: a content change needs a matching vector (or call update_text \
4910 to re-embed through the registered embedder)"
4911 .into(),
4912 ));
4913 }
4914 let idx = self
4915 .index
4916 .get(id.0, &self.catalog)
4917 .ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
4918 if self.catalog[idx].deleted {
4919 return Err(MnemoError::NotFound(id.to_string()));
4920 }
4921 let entry = self.catalog[idx].clone();
4922 let existing = self.read_memory(&entry)?;
4923
4924 // Apply patch into a candidate, detect no-op after the fact.
4925 let mut cand = existing.clone();
4926 if let Some(c) = &patch.content {
4927 cand.content = c.clone();
4928 }
4929 if let Some(v) = &patch.vector {
4930 if v.len() != self.dimensions {
4931 return Err(MnemoError::DimensionMismatch {
4932 expected: self.dimensions,
4933 got: v.len(),
4934 });
4935 }
4936 cand.vector = v.clone();
4937 }
4938 if let Some(i) = patch.importance {
4939 cand.importance = i.clamp(0.0, 1.0);
4940 }
4941 if let Some(mt) = patch.memory_type {
4942 cand.memory_type = mt;
4943 }
4944 if let Some(new_meta) = patch.metadata {
4945 // Keep reserved `_mnemo.*` keys from the existing record.
4946 let reserved: Vec<(String, Value)> = existing
4947 .metadata
4948 .iter()
4949 .filter(|(k, _)| k.starts_with(crate::memory::MNEMO_META_NS))
4950 .map(|(k, v)| (k.clone(), v.clone()))
4951 .collect();
4952 cand.metadata = new_meta;
4953 for (k, v) in reserved {
4954 cand.metadata.insert(k, v);
4955 }
4956 }
4957 if let Some(ttl) = patch.ttl_secs {
4958 cand.ttl_secs = ttl;
4959 }
4960
4961 // No-op detection: compare all user-visible fields.
4962 if cand.content == existing.content
4963 && cand.vector == existing.vector
4964 && (cand.importance - existing.importance).abs() <= f32::EPSILON
4965 && cand.memory_type == existing.memory_type
4966 && cand.metadata == existing.metadata
4967 && cand.ttl_secs == existing.ttl_secs
4968 {
4969 return Ok(WriteOutcome::Unchanged);
4970 }
4971
4972 // Stamp `_mnemo.updated_at` and preserve id + created_at.
4973 let now = memory::now_secs();
4974 cand.metadata
4975 .insert(MNEMO_META_UPDATED_AT.into(), Value::Number(now.into()));
4976 cand.id = existing.id;
4977 cand.created_at = existing.created_at;
4978 // put() preserves accessed_at/access_count via the index lookup.
4979 self.put(cand)?;
4980 // `put` kept the key slot and the other handle caches current.
4981 Ok(WriteOutcome::Updated)
4982 }
4983
4984 /// In-place update that re-embeds `new_text` through the registered
4985 /// embedder. Shorthand for `update(id, patch.with_content_and_vector)`.
4986 pub fn update_text(
4987 &mut self,
4988 id: &Ulid,
4989 new_text: &str,
4990 mut patch: MemoryPatch,
4991 ) -> Result<WriteOutcome> {
4992 if self.read_only {
4993 return Err(MnemoError::ReadOnly);
4994 }
4995 let embedder = self.embedder.as_ref().ok_or_else(|| {
4996 MnemoError::Invalid(
4997 "no embedder registered on this handle; call set_embedder first".into(),
4998 )
4999 })?;
5000 let mut vectors = embedder.embed(&[new_text])?;
5001 let vector = vectors.pop().ok_or_else(|| {
5002 MnemoError::Invalid("embedder returned no vector for the input text".into())
5003 })?;
5004 if vector.len() != self.dimensions {
5005 return Err(MnemoError::DimensionMismatch {
5006 expected: self.dimensions,
5007 got: vector.len(),
5008 });
5009 }
5010 patch.content = Some(new_text.to_string());
5011 patch.vector = Some(vector);
5012 self.update(id, patch)
5013 }
5014
5015 /// Walk the full chain for a keyed slot: every record this key has
5016 /// ever held, newest first. Entry 0 is the **live** record (unless
5017 /// the slot is empty, in which case the vector is empty); subsequent
5018 /// entries are the superseded history in reverse-supersession order.
5019 ///
5020 /// **Slot resolution (v0.8.1 bugfix, lane A).** The lookup slot
5021 /// matches a keyed `remember` with the same arguments, so the
5022 /// obvious `history(key, None)` walks the private slot of the
5023 /// default agent — the same slot
5024 /// `remember_keyed(Memory::new(...), key)` writes to when the
5025 /// caller omits an explicit `agent_id` / `scope`. Pre-v0.8.1
5026 /// `agent_id = None` incorrectly looked up the shared slot and
5027 /// returned an empty chain for the common case; the old behavior
5028 /// is still reachable via `history_in(key, KeyScope::Shared)`.
5029 ///
5030 /// * `agent_id = Some(a)` → private slot for that agent.
5031 /// * `agent_id = None` → private slot for the default agent (same
5032 /// as [`KeyScope::private_default`]).
5033 ///
5034 /// Use [`Mnemo::history_in`] with [`KeyScope::Shared`] for the
5035 /// shared slot.
5036 pub fn history(&mut self, key: &str, agent_id: Option<&str>) -> Result<Vec<Memory>> {
5037 let scope = match agent_id {
5038 Some(a) => KeyScope::Private(a.to_string()),
5039 None => KeyScope::private_default(),
5040 };
5041 self.history_in(key, scope)
5042 }
5043
5044 /// Walk the full chain for a keyed slot identified by an explicit
5045 /// [`KeyScope`]. Returns the same shape as [`Mnemo::history`] —
5046 /// live record first, then the superseded chain newest-first.
5047 ///
5048 /// Use this when you need the **shared** slot (`KeyScope::Shared`),
5049 /// which `history(key, None)` no longer reaches as of v0.8.1.
5050 pub fn history_in(&mut self, key: &str, scope: KeyScope) -> Result<Vec<Memory>> {
5051 self.ensure_key_caches()?;
5052 let slot = crate::store::KeySlot {
5053 scope_owner: match scope {
5054 KeyScope::Private(a) => Some(a),
5055 KeyScope::Shared => None,
5056 },
5057 key: key.to_string(),
5058 };
5059 let live_id = self.key_index.as_ref().and_then(|k| k.get(&slot).copied());
5060 let mut out = Vec::new();
5061 let mut cursor = live_id;
5062 while let Some(id_u) = cursor {
5063 let idx = self.index.get(id_u, &self.catalog).ok_or_else(|| {
5064 MnemoError::Invalid("history: superseded chain references missing id".into())
5065 })?;
5066 let entry = self.catalog[idx].clone();
5067 if entry.deleted {
5068 break;
5069 }
5070 let m = self.read_memory(&entry)?;
5071 let next = m
5072 .metadata
5073 .get(MNEMO_META_SUPERSEDES)
5074 .and_then(|v| v.as_str())
5075 .and_then(|s| Ulid::from_string(s).ok())
5076 .map(|u| u.0);
5077 out.push(m);
5078 cursor = next;
5079 }
5080 Ok(out)
5081 }
5082
5083 // --- Phase 12.7a: thread archive --------------------------------
5084
5085 /// Ingest a batch of turns into the archive layer of `thread_id`.
5086 ///
5087 /// Each [`ThreadTurn`] becomes one or more `Episodic` records
5088 /// tagged with the usual thread-archive reserved-metadata set
5089 /// (`_mnemo.kind = "turn"`, `_mnemo.thread`, `_mnemo.turn_id`,
5090 /// `_mnemo.seq`, `_mnemo.role`). The ingest path is
5091 /// **idempotent** per `turn_id`: a turn whose `turn_id` already
5092 /// appears on a record in this thread is skipped, so re-running
5093 /// `ingest_thread` with an extended transcript appends only
5094 /// what's new.
5095 ///
5096 /// **Chunking.** A non-tool turn whose UTF-8 content exceeds
5097 /// [`THREAD_TURN_CHUNK_BYTES`] (8 KB) is split into chunks at
5098 /// char boundaries, each stored as its own record with
5099 /// `_mnemo.part = 0, 1, …` under the same `turn_id`.
5100 ///
5101 /// **Truncation.** A `role = "tool"` turn whose content exceeds
5102 /// [`THREAD_TOOL_RESULT_TRUNCATE_BYTES`] (16 KB) is stored
5103 /// truncated — the first 16 KB of UTF-8 goes into the record,
5104 /// and `_mnemo.sha256` + `_mnemo.length` capture the original
5105 /// hash and byte length so the full content can be recovered
5106 /// (or verified) out-of-band.
5107 ///
5108 /// **Embedding.** An embedder must be registered
5109 /// ([`Mnemo::set_embedder`]); the content strings (one per
5110 /// resulting chunk / truncated record) are batched into a single
5111 /// `embed()` call.
5112 ///
5113 /// Call `self.flush()` after a successful ingest to persist.
5114 pub fn ingest_thread(&mut self, thread_id: &str, turns: &[ThreadTurn]) -> Result<IngestReport> {
5115 if self.read_only {
5116 return Err(MnemoError::ReadOnly);
5117 }
5118 let plan = self.plan_ingest(thread_id, turns)?;
5119 if plan.pending.is_empty() {
5120 return Ok(plan.report);
5121 }
5122 // Batched embed: one round trip to the embedder for the whole
5123 // ingest, whatever the turn count.
5124 let vectors = {
5125 let embedder = self.embedder.as_ref().ok_or_else(|| {
5126 MnemoError::Invalid(
5127 "ingest_thread: no embedder registered on this handle; call set_embedder first"
5128 .into(),
5129 )
5130 })?;
5131 embedder.embed(&plan.texts())?
5132 };
5133 self.check_vectors("ingest_thread", &vectors, plan.text_count())?;
5134 self.apply_ingest(thread_id, plan, vectors, None)
5135 }
5136
5137 /// Check a batch of embedder output before anything is written:
5138 /// one vector per input, the file's width, finite and non-zero.
5139 fn check_vectors(&self, ctx: &str, vectors: &[Vec<f32>], expected: usize) -> Result<()> {
5140 if vectors.len() != expected {
5141 return Err(MnemoError::Invalid(format!(
5142 "{ctx}: embedder returned {} vectors for {expected} texts",
5143 vectors.len()
5144 )));
5145 }
5146 for v in vectors {
5147 self.check_vector(ctx, v)?;
5148 }
5149 Ok(())
5150 }
5151
5152 /// The checks `put` applies to one vector, run up front so a batch
5153 /// can't fail half-way through its writes.
5154 fn check_vector(&self, ctx: &str, v: &[f32]) -> Result<()> {
5155 if v.len() != self.dimensions {
5156 return Err(MnemoError::DimensionMismatch {
5157 expected: self.dimensions,
5158 got: v.len(),
5159 });
5160 }
5161 if embed::is_zero_vector(v) {
5162 return Err(MnemoError::Invalid(format!(
5163 "{ctx}: all-zero vector refused"
5164 )));
5165 }
5166 if v.iter().any(|x| !x.is_finite()) {
5167 return Err(MnemoError::Invalid(format!(
5168 "{ctx}: vector contains a non-finite value"
5169 )));
5170 }
5171 Ok(())
5172 }
5173
5174 /// Planning half of [`Mnemo::ingest_thread`]: validate the turns, skip
5175 /// the ones already archived, and chunk or truncate the rest. Reads
5176 /// only; nothing is written.
5177 fn plan_ingest(&mut self, thread_id: &str, turns: &[ThreadTurn]) -> Result<IngestPlan> {
5178 if thread_id.is_empty() {
5179 return Err(MnemoError::Invalid(
5180 "ingest_thread: 'thread_id' must not be empty".into(),
5181 ));
5182 }
5183 // FMT-9b.2: v9 chunks fit one packed page with their metadata.
5184 let chunk_bytes = if self.is_v9() {
5185 V9_TURN_CHUNK_BYTES
5186 } else {
5187 THREAD_TURN_CHUNK_BYTES
5188 };
5189 let mut report = IngestReport::default();
5190 if turns.is_empty() {
5191 return Ok(IngestPlan {
5192 pending: Vec::new(),
5193 next_seq: 0,
5194 report,
5195 });
5196 }
5197
5198 // Build the set of turn_ids already in this thread so the
5199 // idempotence check is O(1) per incoming turn. First-pass
5200 // cost: one decrypt per existing thread record. A future v9
5201 // format bump can move `_mnemo.turn_id` into the catalog row
5202 // and drop this scan.
5203 // R-004.2: the turn ids already archived and the next `_mnemo.seq`
5204 // come from the handle caches: one lazy scan per handle, then O(1)
5205 // per ingest instead of two full decrypt scans each time.
5206 self.ensure_key_caches()?;
5207 self.warm_thread(thread_id)?;
5208 let cached = self.layers.as_ref().and_then(|l| l.threads.get(thread_id));
5209 let existing_turn_ids: std::collections::HashSet<&str> = cached
5210 .map(|tc| tc.turns.keys().map(String::as_str).collect())
5211 .unwrap_or_default();
5212 // A repeat ingest continues the sequence instead of restarting at 0.
5213 let next_seq = cached.and_then(|tc| tc.max_seq).map_or(0, |n| n + 1);
5214
5215 // Determine each incoming turn's final text pieces (post-
5216 // chunking / truncation) before any writes, so the caller can
5217 // embed the whole batch in one `embed()` call.
5218 let mut pending: Vec<PendingTurn> = Vec::new();
5219 let now = memory::now_secs();
5220 for t in turns {
5221 if t.turn_id.is_empty() {
5222 return Err(MnemoError::Invalid(
5223 "ingest_thread: turn_id must not be empty".into(),
5224 ));
5225 }
5226 if existing_turn_ids.contains(t.turn_id.as_str()) {
5227 report.skipped_existing_turns += 1;
5228 continue;
5229 }
5230 let content_bytes = t.content.len();
5231 let is_tool = t.role == "tool";
5232 let (parts, truncation) =
5233 if is_tool && content_bytes > THREAD_TOOL_RESULT_TRUNCATE_BYTES {
5234 report.truncated_turns += 1;
5235 let hash = {
5236 use sha2::{Digest, Sha256};
5237 let h = Sha256::digest(t.content.as_bytes());
5238 hex_encode(&h)
5239 };
5240 let truncated = take_utf8_prefix(&t.content, THREAD_TOOL_RESULT_TRUNCATE_BYTES);
5241 (vec![truncated], Some((hash, content_bytes as u64)))
5242 } else if content_bytes > chunk_bytes {
5243 report.chunked_turns += 1;
5244 (chunk_utf8(&t.content, chunk_bytes), None)
5245 } else {
5246 (vec![t.content.clone()], None)
5247 };
5248 pending.push(PendingTurn {
5249 turn_id: t.turn_id.clone(),
5250 role: t.role.clone(),
5251 created_at: t.timestamp.unwrap_or(now),
5252 parts,
5253 truncation,
5254 });
5255 }
5256 Ok(IngestPlan {
5257 pending,
5258 next_seq,
5259 report,
5260 })
5261 }
5262
5263 /// Write half of [`Mnemo::ingest_thread`]. `vectors` holds one vector
5264 /// per text of `plan.texts()`, already checked. `ttl` stamps every
5265 /// new record (the `commit_deltas` `ExpireAfter` policy).
5266 fn apply_ingest(
5267 &mut self,
5268 thread_id: &str,
5269 plan: IngestPlan,
5270 vectors: Vec<Vec<f32>>,
5271 ttl: Option<i64>,
5272 ) -> Result<IngestReport> {
5273 let IngestPlan {
5274 pending,
5275 mut next_seq,
5276 mut report,
5277 } = plan;
5278 // Order: each pending turn's chunks in part order; each turn
5279 // gets a single `_mnemo.seq` slot, so `_mnemo.part`
5280 // disambiguates chunks of the same turn.
5281 let mut vectors = vectors.into_iter();
5282 for p in &pending {
5283 let this_seq = next_seq;
5284 next_seq += 1;
5285 for (part_idx, part_text) in p.parts.iter().enumerate() {
5286 let vec = vectors.next().ok_or_else(|| {
5287 MnemoError::Invalid("ingest_thread: ran out of vectors".into())
5288 })?;
5289 let mut m =
5290 Memory::new(part_text.clone(), MemoryType::Episodic, vec).with_importance(0.5);
5291 if let Some(t) = ttl {
5292 m = m.with_ttl(t);
5293 }
5294 m.created_at = p.created_at;
5295 m.accessed_at = p.created_at;
5296 m.metadata.insert(
5297 MNEMO_META_KIND.into(),
5298 Value::String(MNEMO_META_KIND_TURN.into()),
5299 );
5300 m.metadata.insert(
5301 MNEMO_META_THREAD.into(),
5302 Value::String(thread_id.to_string()),
5303 );
5304 m.metadata
5305 .insert(MNEMO_META_TURN_ID.into(), Value::String(p.turn_id.clone()));
5306 m.metadata
5307 .insert(MNEMO_META_SEQ.into(), Value::Number(this_seq.into()));
5308 m.metadata
5309 .insert(MNEMO_META_ROLE.into(), Value::String(p.role.clone()));
5310 if p.parts.len() > 1 {
5311 m.metadata.insert(
5312 MNEMO_META_PART.into(),
5313 Value::Number((part_idx as u64).into()),
5314 );
5315 }
5316 if let Some((sha, len)) = &p.truncation {
5317 m.metadata
5318 .insert(MNEMO_META_SHA256.into(), Value::String(sha.clone()));
5319 m.metadata
5320 .insert(MNEMO_META_LENGTH.into(), Value::Number((*len).into()));
5321 }
5322 self.put(m)?;
5323 report.added_records += 1;
5324 }
5325 }
5326 Ok(report)
5327 }
5328
5329 /// Export the archive layer for `thread_id` in `_mnemo.seq`
5330 /// order. Each entry is the Memory as stored; chunks of a
5331 /// multi-part turn come out in `_mnemo.part` order within their
5332 /// `_mnemo.seq`. The record `.content` is the text that was
5333 /// stored (post-chunking / post-truncation); callers that need
5334 /// the pre-truncation original should round-trip via
5335 /// `_mnemo.sha256` + `_mnemo.length`.
5336 pub fn thread_export(&mut self, thread_id: &str) -> Result<Vec<Memory>> {
5337 // R-004.2: the thread's own records, from the handle caches.
5338 // FMT-9e: a cold thread's records were just decoded to warm it;
5339 // reuse them instead of decoding (and decompressing) twice.
5340 let (entries, mut decoded) = self.thread_entries_decoded(thread_id)?;
5341 let mut out: Vec<Memory> = Vec::new();
5342 for e in &entries {
5343 let m = match decoded.remove(&e.id) {
5344 Some(m) => self.finish_memory(e, m)?,
5345 None => self.read_memory(e)?,
5346 };
5347 if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(thread_id) {
5348 continue;
5349 }
5350 if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
5351 != Some(MNEMO_META_KIND_TURN)
5352 {
5353 continue;
5354 }
5355 out.push(m);
5356 }
5357 out.sort_by(|a, b| {
5358 let seq_a = a
5359 .metadata
5360 .get(MNEMO_META_SEQ)
5361 .and_then(|v| v.as_u64())
5362 .unwrap_or(0);
5363 let seq_b = b
5364 .metadata
5365 .get(MNEMO_META_SEQ)
5366 .and_then(|v| v.as_u64())
5367 .unwrap_or(0);
5368 let part_a = a
5369 .metadata
5370 .get(MNEMO_META_PART)
5371 .and_then(|v| v.as_u64())
5372 .unwrap_or(0);
5373 let part_b = b
5374 .metadata
5375 .get(MNEMO_META_PART)
5376 .and_then(|v| v.as_u64())
5377 .unwrap_or(0);
5378 seq_a.cmp(&seq_b).then_with(|| part_a.cmp(&part_b))
5379 });
5380 Ok(out)
5381 }
5382
5383 /// Export a thread's archive in the **ingest shape** (Phase 12.7d):
5384 /// one [`ExportedTurn`] per source turn, in `_mnemo.seq` order, with
5385 /// a chunked turn's parts joined back into one `content`. Passing
5386 /// the result back to [`Mnemo::ingest_thread`] for the same thread
5387 /// skips every turn; for a new thread it re-creates the same records.
5388 /// This is what Python `db.thread_export`, MCP `thread_export` and
5389 /// `mnemo thread export` return; [`Mnemo::thread_export`] still
5390 /// returns the raw records.
5391 pub fn thread_export_turns(&mut self, thread_id: &str) -> Result<Vec<ExportedTurn>> {
5392 let records = self.thread_export(thread_id)?;
5393 let mut out: Vec<ExportedTurn> = Vec::new();
5394 let mut last_seq: Option<u64> = None;
5395 for m in records {
5396 let meta_str = |k: &str| {
5397 m.metadata
5398 .get(k)
5399 .and_then(|v| v.as_str())
5400 .unwrap_or("")
5401 .to_string()
5402 };
5403 let seq = m.metadata.get(MNEMO_META_SEQ).and_then(|v| v.as_u64());
5404 // Records are sorted by (seq, part), so the parts of one turn
5405 // are adjacent and share a seq.
5406 if seq.is_some() && seq == last_seq {
5407 if let Some(prev) = out.last_mut() {
5408 prev.content.push_str(&m.content);
5409 continue;
5410 }
5411 }
5412 last_seq = seq;
5413 out.push(ExportedTurn {
5414 turn_id: meta_str(MNEMO_META_TURN_ID),
5415 role: meta_str(MNEMO_META_ROLE),
5416 timestamp: m.created_at,
5417 sha256: m
5418 .metadata
5419 .get(MNEMO_META_SHA256)
5420 .and_then(|v| v.as_str())
5421 .map(String::from),
5422 original_length: m.metadata.get(MNEMO_META_LENGTH).and_then(|v| v.as_u64()),
5423 content: m.content,
5424 });
5425 }
5426 Ok(out)
5427 }
5428
5429 /// Soft-delete every record in the archive layer of `thread_id`.
5430 /// Returns the number of records marked deleted. Permanent
5431 /// removal still requires a subsequent `compact_file`, same as
5432 /// any `Mnemo::delete`.
5433 ///
5434 /// Does **not** touch `_mnemo.kind = "essence"` records or other
5435 /// non-turn metadata the thread may carry — only `_mnemo.kind =
5436 /// "turn"` records matching the thread id.
5437 pub fn thread_forget(&mut self, thread_id: &str) -> Result<usize> {
5438 if self.read_only {
5439 return Err(MnemoError::ReadOnly);
5440 }
5441 // Collect first, then delete — can't hold catalog borrows
5442 // across `self.delete`.
5443 let entries = self.thread_entries(thread_id)?;
5444 let mut victims: Vec<Ulid> = Vec::new();
5445 for e in &entries {
5446 let m = self.read_memory(e)?;
5447 if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(thread_id) {
5448 continue;
5449 }
5450 if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
5451 != Some(MNEMO_META_KIND_TURN)
5452 {
5453 continue;
5454 }
5455 victims.push(m.id);
5456 }
5457 let n = victims.len();
5458 for id in &victims {
5459 self.delete(id)?;
5460 }
5461 Ok(n)
5462 }
5463
5464 // --- Phase 12.7b: essence (timeline + sections) -----------------
5465
5466 /// Phase 12.7b: add a new essence item to a section (any section
5467 /// except [`EssenceSection::Timeline`]). Content ≤
5468 /// [`memory::ESSENCE_SECTION_MAX_BYTES`] (280 bytes); items over
5469 /// that are rejected. Starting status is `"open"`.
5470 ///
5471 /// Use [`Mnemo::essence_append_timeline`] for timeline entries —
5472 /// they have a different cap (600 bytes), are append-only, and
5473 /// carry no status lifecycle.
5474 pub fn essence_add(
5475 &mut self,
5476 scope: EssenceScope,
5477 section: EssenceSection,
5478 text: &str,
5479 refs: Option<Vec<String>>,
5480 ) -> Result<Ulid> {
5481 if self.read_only {
5482 return Err(MnemoError::ReadOnly);
5483 }
5484 Self::check_essence_add(section, text)?;
5485 self.essence_add_with(scope, section, text, refs, None)
5486 }
5487
5488 /// Validation shared by [`Mnemo::essence_add`] and `commit_deltas`.
5489 fn check_essence_add(section: EssenceSection, text: &str) -> Result<()> {
5490 if section == EssenceSection::Timeline {
5491 return Err(MnemoError::Invalid(
5492 "essence_add: use essence_append_timeline for the timeline section".into(),
5493 ));
5494 }
5495 if text.is_empty() {
5496 return Err(MnemoError::Invalid(
5497 "essence_add: text must not be empty".into(),
5498 ));
5499 }
5500 if text.len() > section.item_max_bytes() {
5501 return Err(MnemoError::Invalid(format!(
5502 "essence_add: '{}' items must be <= {} bytes; got {}",
5503 section.as_str(),
5504 section.item_max_bytes(),
5505 text.len()
5506 )));
5507 }
5508 Ok(())
5509 }
5510
5511 /// Write half of `essence_add`, with an optional precomputed vector.
5512 fn essence_add_with(
5513 &mut self,
5514 scope: EssenceScope,
5515 section: EssenceSection,
5516 text: &str,
5517 refs: Option<Vec<String>>,
5518 vector: Option<Vec<f32>>,
5519 ) -> Result<Ulid> {
5520 let now = memory::now_secs();
5521 let mem = self.build_essence_record(scope, section, text, refs, Some(now), vector)?;
5522 self.put(mem)
5523 }
5524
5525 /// Phase 12.7b: append a new timeline entry (append-only
5526 /// milestone). Content ≤ [`memory::ESSENCE_TIMELINE_MAX_BYTES`]
5527 /// (600 bytes); entries over that are rejected (not truncated —
5528 /// truncating a milestone silently loses the narrative).
5529 pub fn essence_append_timeline(
5530 &mut self,
5531 scope: EssenceScope,
5532 text: &str,
5533 refs: Option<Vec<String>>,
5534 ) -> Result<Ulid> {
5535 if self.read_only {
5536 return Err(MnemoError::ReadOnly);
5537 }
5538 Self::check_essence_append(text)?;
5539 self.essence_append_with(scope, text, refs, None)
5540 }
5541
5542 /// Validation shared by [`Mnemo::essence_append_timeline`] and
5543 /// `commit_deltas`.
5544 fn check_essence_append(text: &str) -> Result<()> {
5545 if text.is_empty() {
5546 return Err(MnemoError::Invalid(
5547 "essence_append_timeline: text must not be empty".into(),
5548 ));
5549 }
5550 if text.len() > EssenceSection::Timeline.item_max_bytes() {
5551 return Err(MnemoError::Invalid(format!(
5552 "essence_append_timeline: entries must be <= {} bytes; got {}",
5553 EssenceSection::Timeline.item_max_bytes(),
5554 text.len()
5555 )));
5556 }
5557 Ok(())
5558 }
5559
5560 /// Write half of `essence_append_timeline`, with an optional
5561 /// precomputed vector.
5562 fn essence_append_with(
5563 &mut self,
5564 scope: EssenceScope,
5565 text: &str,
5566 refs: Option<Vec<String>>,
5567 vector: Option<Vec<f32>>,
5568 ) -> Result<Ulid> {
5569 let mem =
5570 self.build_essence_record(scope, EssenceSection::Timeline, text, refs, None, vector)?;
5571 self.put(mem)
5572 }
5573
5574 /// Phase 12.7b: update an existing essence section item in place.
5575 /// Returns [`WriteOutcome::Updated`] / [`WriteOutcome::Unchanged`].
5576 /// Timeline entries are append-only — calling this on a timeline
5577 /// record is an error.
5578 pub fn essence_update(
5579 &mut self,
5580 id: &Ulid,
5581 text: &str,
5582 refs: Option<Vec<String>>,
5583 ) -> Result<WriteOutcome> {
5584 if self.read_only {
5585 return Err(MnemoError::ReadOnly);
5586 }
5587 let existing = self.load_essence_record(id)?;
5588 let section = self.section_of(&existing)?;
5589 if section == EssenceSection::Timeline {
5590 return Err(MnemoError::Invalid(
5591 "essence_update: timeline entries are append-only; append a new one instead".into(),
5592 ));
5593 }
5594 if text.is_empty() {
5595 return Err(MnemoError::Invalid(
5596 "essence_update: text must not be empty".into(),
5597 ));
5598 }
5599 if text.len() > section.item_max_bytes() {
5600 return Err(MnemoError::Invalid(format!(
5601 "essence_update: '{}' items must be <= {} bytes; got {}",
5602 section.as_str(),
5603 section.item_max_bytes(),
5604 text.len()
5605 )));
5606 }
5607 let mut patch = MemoryPatch {
5608 content: Some(text.to_string()),
5609 ..Default::default()
5610 };
5611 // We don't have a usable embedder-free path for an in-place
5612 // content update, so provide the same vector as the existing
5613 // record (the text-vector drift guard in `update` will accept
5614 // this).
5615 patch.vector = Some(existing.vector.clone());
5616 if let Some(r) = refs {
5617 let mut meta = Map::new();
5618 // Preserve the user-visible metadata (none for essence —
5619 // everything lives under `_mnemo.*`) and let `update`'s
5620 // metadata-replacement path keep the reserved keys.
5621 meta.insert(
5622 MNEMO_META_REFS.into(),
5623 Value::Array(r.into_iter().map(Value::String).collect()),
5624 );
5625 patch.metadata = Some(meta);
5626 }
5627 self.update(id, patch)
5628 }
5629
5630 /// Phase 12.7b: mark a section item as resolved (open → resolved
5631 /// transition). A new history entry lands on
5632 /// `_mnemo.status_history`. Idempotent — resolving a record that
5633 /// is already resolved is a no-op.
5634 pub fn essence_resolve(&mut self, id: &Ulid) -> Result<WriteOutcome> {
5635 self.essence_transition_status(id, essence_status::RESOLVED)
5636 }
5637
5638 /// Phase 12.7b: soft-delete a section item. Same semantics as
5639 /// [`Mnemo::delete`] on any record — space reclaimed by a later
5640 /// compact.
5641 pub fn essence_remove(&mut self, id: &Ulid) -> Result<()> {
5642 if self.read_only {
5643 return Err(MnemoError::ReadOnly);
5644 }
5645 self.delete(id)
5646 }
5647
5648 /// Phase 12.7b: deterministic, budgeted markdown render of the
5649 /// essence layer for a thread (or the project layer when
5650 /// `scope = EssenceScope::Project`). `max_bytes` caps the output;
5651 /// sections are emitted in the fixed [`EssenceSection::render_order`],
5652 /// and each section's own budget is proportional to its position
5653 /// in the order. Oldest timeline entries fold into a
5654 /// `+N earlier milestones` tail when the budget runs out.
5655 ///
5656 /// The render always ends with an archive pointer so the reader
5657 /// knows how to drill in deeper (`mnemo thread export …` or
5658 /// `recall --thread …`).
5659 pub fn essence(&mut self, scope: EssenceScope, max_bytes: usize) -> Result<String> {
5660 let max_bytes = if max_bytes == 0 {
5661 ESSENCE_DEFAULT_RENDER_BYTES
5662 } else {
5663 max_bytes
5664 };
5665 // Collect every live essence record for this scope.
5666 // R-004.2: only the essence items, from the handle caches.
5667 let entries = self.essence_entries()?;
5668 let mut by_section: std::collections::BTreeMap<&'static str, Vec<Memory>> =
5669 std::collections::BTreeMap::new();
5670 for section in EssenceSection::render_order() {
5671 by_section.insert(section.as_str(), Vec::new());
5672 }
5673 for e in &entries {
5674 let m = self.read_memory(e)?;
5675 if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
5676 != Some(MNEMO_META_KIND_ESSENCE)
5677 {
5678 continue;
5679 }
5680 // Scope filter. Project-scope records are shown in every
5681 // thread's render (and in the project render); thread-scope
5682 // records only match when scope == same thread.
5683 let rec_thread = m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str());
5684 let matches_scope = match (&scope, rec_thread) {
5685 (EssenceScope::Project, None) => true,
5686 (EssenceScope::Thread(_), None) => true, // project-scope always shows
5687 (EssenceScope::Thread(t), Some(rt)) => rt == t,
5688 (EssenceScope::Project, Some(_)) => false,
5689 };
5690 if !matches_scope {
5691 continue;
5692 }
5693 let section_str = m
5694 .metadata
5695 .get(MNEMO_META_SECTION)
5696 .and_then(|v| v.as_str())
5697 .unwrap_or("timeline");
5698 by_section
5699 .entry(
5700 EssenceSection::parse(section_str)
5701 .map(|s| s.as_str())
5702 .unwrap_or("timeline"),
5703 )
5704 .or_default()
5705 .push(m);
5706 }
5707
5708 // Sort each section's records:
5709 // - Decisions + Timeline: newest-first (by created_at desc).
5710 // - Everything else: sort open items before resolved, then
5711 // newest-first. Cheap, deterministic.
5712 for (name, items) in by_section.iter_mut() {
5713 let newest_first = matches!(*name, "decisions" | "timeline");
5714 items.sort_by(|a, b| {
5715 let a_resolved = status_of(a) == Some(essence_status::RESOLVED);
5716 let b_resolved = status_of(b) == Some(essence_status::RESOLVED);
5717 match (a_resolved, b_resolved) {
5718 (false, true) => std::cmp::Ordering::Less,
5719 (true, false) => std::cmp::Ordering::Greater,
5720 _ => {
5721 if newest_first {
5722 b.created_at.cmp(&a.created_at)
5723 } else {
5724 a.created_at.cmp(&b.created_at)
5725 }
5726 }
5727 }
5728 });
5729 }
5730
5731 // Per-section budget: give intent / conventions / decisions a
5732 // larger share (they frame the thread), timeline a smaller
5733 // share (it can fold). This is deterministic because it
5734 // depends only on `max_bytes`.
5735 let header_reserve = 256usize; // header + archive pointer
5736 let body_budget = max_bytes.saturating_sub(header_reserve);
5737 let per_section_budget = |section: EssenceSection| -> usize {
5738 match section {
5739 EssenceSection::Intent => body_budget / 8,
5740 EssenceSection::Conventions => body_budget / 6,
5741 EssenceSection::Decisions => body_budget / 4,
5742 EssenceSection::State => body_budget / 8,
5743 EssenceSection::Open => body_budget / 8,
5744 EssenceSection::Next => body_budget / 12,
5745 EssenceSection::Lessons => body_budget / 12,
5746 EssenceSection::Timeline => body_budget / 4,
5747 }
5748 };
5749
5750 let mut out = String::with_capacity(max_bytes);
5751 let scope_label = match &scope {
5752 EssenceScope::Thread(t) => format!("thread `{t}`"),
5753 EssenceScope::Project => "project".to_string(),
5754 };
5755 out.push_str(&format!("# essence — {scope_label}\n\n"));
5756
5757 // Phase 12.7c: continuation. If this is a thread render AND
5758 // the thread has a `_mnemo.continues` link, pull the prior
5759 // thread's open section items, its `next`, and its last-N
5760 // timeline entries in ahead of the new thread's own essence.
5761 if let EssenceScope::Thread(new_id) = &scope {
5762 if let Some(prev_id) = self.predecessor_thread(new_id)? {
5763 let continuation = self
5764 .render_continuation_prelude(&prev_id, max_bytes.saturating_sub(out.len()))?;
5765 if !continuation.is_empty() && out.len() + continuation.len() <= max_bytes {
5766 out.push_str(&continuation);
5767 }
5768 }
5769 }
5770
5771 for section in EssenceSection::render_order() {
5772 let items = match by_section.get(section.as_str()) {
5773 Some(v) if !v.is_empty() => v,
5774 _ => continue,
5775 };
5776 let budget = per_section_budget(*section);
5777 let section_header = format!("## {}\n", section.as_str());
5778 if out.len() + section_header.len() > max_bytes {
5779 break;
5780 }
5781 out.push_str(§ion_header);
5782 let mut used = 0usize;
5783 let mut rendered = 0usize;
5784 for item in items {
5785 let line = render_essence_item(*section, item);
5786 if used + line.len() > budget {
5787 break;
5788 }
5789 if out.len() + line.len() > max_bytes {
5790 break;
5791 }
5792 out.push_str(&line);
5793 used += line.len();
5794 rendered += 1;
5795 }
5796 // Fold the remainder for timeline.
5797 if *section == EssenceSection::Timeline && rendered < items.len() {
5798 let fold = format!("- +{} earlier milestones\n", items.len() - rendered);
5799 if out.len() + fold.len() <= max_bytes {
5800 out.push_str(&fold);
5801 }
5802 }
5803 if out.len() < max_bytes {
5804 out.push('\n');
5805 }
5806 }
5807
5808 // Archive pointer.
5809 let pointer = match &scope {
5810 EssenceScope::Thread(t) => format!(
5811 "> full history: `mnemo thread export <file> --thread {t}` or `recall --thread {t}`\n"
5812 ),
5813 EssenceScope::Project => String::from(
5814 "> full history: `mnemo thread export <file> --thread <id>` or `recall --thread <id>`\n",
5815 ),
5816 };
5817 if out.len() + pointer.len() <= max_bytes {
5818 out.push_str(&pointer);
5819 }
5820 Ok(out)
5821 }
5822
5823 // --- essence internal helpers ----------------------------------
5824
5825 /// Build an essence record. `vector` is a precomputed embedding (the
5826 /// 12.7d `commit_deltas` batch path); `None` embeds `text` with the
5827 /// registered embedder, or falls back to an axis-neutral unit vector
5828 /// when there is none. Essence records are read by `_mnemo.kind` and
5829 /// scope, not by similarity, so the fallback costs no retrieval
5830 /// quality; it only has to pass `put`'s non-zero guard.
5831 fn build_essence_record(
5832 &self,
5833 scope: EssenceScope,
5834 section: EssenceSection,
5835 text: &str,
5836 refs: Option<Vec<String>>,
5837 status_at: Option<i64>,
5838 vector: Option<Vec<f32>>,
5839 ) -> Result<Memory> {
5840 let v = match (vector, self.embedder.as_ref()) {
5841 (Some(v), _) => v,
5842 (None, Some(e)) => e.embed(&[text])?.pop().ok_or_else(|| {
5843 MnemoError::Invalid("essence: embedder returned no vector".into())
5844 })?,
5845 (None, None) => self.sentinel_vector(),
5846 };
5847 let mut m = Memory::new(text.to_string(), MemoryType::Semantic, v).with_importance(0.5);
5848 m.metadata.insert(
5849 MNEMO_META_KIND.into(),
5850 Value::String(MNEMO_META_KIND_ESSENCE.into()),
5851 );
5852 m.metadata.insert(
5853 MNEMO_META_SECTION.into(),
5854 Value::String(section.as_str().into()),
5855 );
5856 if let EssenceScope::Thread(t) = &scope {
5857 m.metadata
5858 .insert(MNEMO_META_THREAD.into(), Value::String(t.clone()));
5859 }
5860 if let Some(r) = refs {
5861 m.metadata.insert(
5862 MNEMO_META_REFS.into(),
5863 Value::Array(r.into_iter().map(Value::String).collect()),
5864 );
5865 }
5866 if let Some(at) = status_at {
5867 m.metadata.insert(
5868 MNEMO_META_STATUS.into(),
5869 Value::String(essence_status::OPEN.into()),
5870 );
5871 m.metadata.insert(
5872 MNEMO_META_STATUS_HISTORY.into(),
5873 Value::Array(vec![serde_json::json!({
5874 "status": essence_status::OPEN,
5875 "at": at,
5876 })]),
5877 );
5878 }
5879 Ok(m)
5880 }
5881
5882 /// Axis-neutral unit vector (every component `1/sqrt(dims)`), used by
5883 /// essence records without an embedder and by thread-link markers.
5884 fn sentinel_vector(&self) -> Vec<f32> {
5885 let dims = self.dimensions.max(1);
5886 vec![1.0 / (dims as f32).sqrt(); dims]
5887 }
5888
5889 fn load_essence_record(&mut self, id: &Ulid) -> Result<Memory> {
5890 let m = self.get(id)?;
5891 if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str()) != Some(MNEMO_META_KIND_ESSENCE)
5892 {
5893 return Err(MnemoError::Invalid(format!(
5894 "essence: record {id} is not an essence record (missing `_mnemo.kind = essence`)"
5895 )));
5896 }
5897 Ok(m)
5898 }
5899
5900 fn section_of(&self, m: &Memory) -> Result<EssenceSection> {
5901 let s = m
5902 .metadata
5903 .get(MNEMO_META_SECTION)
5904 .and_then(|v| v.as_str())
5905 .ok_or_else(|| {
5906 MnemoError::Invalid("essence: record missing `_mnemo.section`".into())
5907 })?;
5908 EssenceSection::parse(s)
5909 .ok_or_else(|| MnemoError::Invalid(format!("essence: unknown section '{s}' on record")))
5910 }
5911
5912 fn essence_transition_status(&mut self, id: &Ulid, to: &str) -> Result<WriteOutcome> {
5913 if self.read_only {
5914 return Err(MnemoError::ReadOnly);
5915 }
5916 let existing = self.load_essence_record(id)?;
5917 let section = self.section_of(&existing)?;
5918 if !section.has_status_lifecycle() {
5919 return Err(MnemoError::Invalid(format!(
5920 "essence_transition: '{}' section has no status lifecycle",
5921 section.as_str()
5922 )));
5923 }
5924 let from = existing
5925 .metadata
5926 .get(MNEMO_META_STATUS)
5927 .and_then(|v| v.as_str())
5928 .unwrap_or(essence_status::OPEN);
5929 if from == to {
5930 return Ok(WriteOutcome::Unchanged);
5931 }
5932 let allowed = ESSENCE_TRANSITIONS
5933 .iter()
5934 .any(|(f, t)| *f == from && *t == to);
5935 if !allowed {
5936 return Err(MnemoError::Invalid(format!(
5937 "essence_transition: '{from}' → '{to}' is not a valid transition for essence"
5938 )));
5939 }
5940 // Mutate metadata in place and `put` back under the same id
5941 // so `_mnemo.updated_at` gets stamped and access stats are
5942 // preserved.
5943 let mut new = existing.clone();
5944 new.metadata
5945 .insert(MNEMO_META_STATUS.into(), Value::String(to.to_string()));
5946 let now = memory::now_secs();
5947 let history_entry = serde_json::json!({ "status": to, "at": now });
5948 let history = new
5949 .metadata
5950 .entry(MNEMO_META_STATUS_HISTORY.to_string())
5951 .or_insert_with(|| Value::Array(Vec::new()));
5952 if let Value::Array(arr) = history {
5953 arr.push(history_entry);
5954 }
5955 new.metadata
5956 .insert(MNEMO_META_UPDATED_AT.into(), Value::Number(now.into()));
5957 new.id = existing.id;
5958 new.created_at = existing.created_at;
5959 self.put(new)?;
5960 Ok(WriteOutcome::Updated)
5961 }
5962
5963 // --- Phase 12.7c: continuation + commit_deltas ------------------
5964
5965 /// Start a new thread that continues from a previous one
5966 /// (Phase 12.7c). Writes a thread-link marker record with
5967 /// `_mnemo.kind = "thread_link"` + `_mnemo.thread = <new_id>` +
5968 /// `_mnemo.continues = <from_id>`. The marker holds no user
5969 /// content; its purpose is to let `essence(Thread(new_id))`
5970 /// discover the continuation chain.
5971 ///
5972 /// The render for the new thread pulls the previous thread's
5973 /// open section items, its `next`, and its last
5974 /// [`CONTINUATION_TIMELINE_ENTRIES`] timeline entries ahead of
5975 /// the new thread's own essence — the "new thread, same
5976 /// experience" path.
5977 pub fn thread_start(&mut self, new_thread_id: &str, from_thread_id: &str) -> Result<Ulid> {
5978 if self.read_only {
5979 return Err(MnemoError::ReadOnly);
5980 }
5981 if new_thread_id.is_empty() || from_thread_id.is_empty() {
5982 return Err(MnemoError::Invalid(
5983 "thread_start: 'new_thread_id' and 'from_thread_id' must not be empty".into(),
5984 ));
5985 }
5986 if new_thread_id == from_thread_id {
5987 return Err(MnemoError::Invalid(
5988 "thread_start: 'new_thread_id' must differ from 'from_thread_id'".into(),
5989 ));
5990 }
5991 // Store a tiny marker record so the continuation chain is
5992 // queryable from a fresh handle. The sentinel vector passes
5993 // `put`'s zero-vector guard; default recall skips link records.
5994 let v = self.sentinel_vector();
5995 let content = format!("thread link: {new_thread_id} continues from {from_thread_id}");
5996 let mut m = Memory::new(content, MemoryType::Semantic, v).with_importance(0.1);
5997 m.metadata.insert(
5998 MNEMO_META_KIND.into(),
5999 Value::String(MNEMO_META_KIND_THREAD_LINK.into()),
6000 );
6001 m.metadata.insert(
6002 MNEMO_META_THREAD.into(),
6003 Value::String(new_thread_id.to_string()),
6004 );
6005 m.metadata.insert(
6006 MNEMO_META_CONTINUES.into(),
6007 Value::String(from_thread_id.to_string()),
6008 );
6009 let id = self.put(m)?;
6010 Ok(id)
6011 }
6012
6013 /// Phase 12.7c helper: render the predecessor thread's open
6014 /// items + `next` + last-N timeline entries as a "continued from"
6015 /// prelude. Used by [`Mnemo::essence`] to carry forward the
6016 /// prior thread's live state into a fresh thread.
6017 fn render_continuation_prelude(
6018 &mut self,
6019 prev_thread: &str,
6020 max_bytes: usize,
6021 ) -> Result<String> {
6022 if max_bytes == 0 {
6023 return Ok(String::new());
6024 }
6025 let entries = self.essence_entries()?;
6026 let mut open_items: Vec<Memory> = Vec::new();
6027 let mut next_items: Vec<Memory> = Vec::new();
6028 let mut timeline_items: Vec<Memory> = Vec::new();
6029 for e in &entries {
6030 let m = self.read_memory(e)?;
6031 if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
6032 != Some(MNEMO_META_KIND_ESSENCE)
6033 {
6034 continue;
6035 }
6036 if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(prev_thread) {
6037 continue;
6038 }
6039 let section = match m
6040 .metadata
6041 .get(MNEMO_META_SECTION)
6042 .and_then(|v| v.as_str())
6043 .and_then(EssenceSection::parse)
6044 {
6045 Some(s) => s,
6046 None => continue,
6047 };
6048 let status = m
6049 .metadata
6050 .get(MNEMO_META_STATUS)
6051 .and_then(|v| v.as_str())
6052 .unwrap_or(essence_status::OPEN);
6053 match section {
6054 EssenceSection::Open if status == essence_status::OPEN => {
6055 open_items.push(m);
6056 }
6057 EssenceSection::Next if status == essence_status::OPEN => {
6058 next_items.push(m);
6059 }
6060 EssenceSection::Timeline => timeline_items.push(m),
6061 _ => {}
6062 }
6063 }
6064 // Newest-first timeline, cap to last N.
6065 timeline_items.sort_by_key(|m| std::cmp::Reverse(m.created_at));
6066 timeline_items.truncate(CONTINUATION_TIMELINE_ENTRIES);
6067 // Oldest-first for stable chronological read in the render.
6068 timeline_items.reverse();
6069
6070 let mut out = String::new();
6071 let header = format!("## continued from `{prev_thread}`\n");
6072 if header.len() > max_bytes {
6073 return Ok(String::new());
6074 }
6075 out.push_str(&header);
6076 if !open_items.is_empty() {
6077 let sub = "### open\n";
6078 if out.len() + sub.len() <= max_bytes {
6079 out.push_str(sub);
6080 }
6081 for m in &open_items {
6082 let line = format!("- {}\n", m.content);
6083 if out.len() + line.len() > max_bytes {
6084 break;
6085 }
6086 out.push_str(&line);
6087 }
6088 }
6089 if !next_items.is_empty() {
6090 let sub = "### next\n";
6091 if out.len() + sub.len() <= max_bytes {
6092 out.push_str(sub);
6093 }
6094 for m in &next_items {
6095 let line = format!("- {}\n", m.content);
6096 if out.len() + line.len() > max_bytes {
6097 break;
6098 }
6099 out.push_str(&line);
6100 }
6101 }
6102 if !timeline_items.is_empty() {
6103 let sub = "### recent milestones\n";
6104 if out.len() + sub.len() <= max_bytes {
6105 out.push_str(sub);
6106 }
6107 for m in &timeline_items {
6108 let line = format!("- {}\n", m.content);
6109 if out.len() + line.len() > max_bytes {
6110 break;
6111 }
6112 out.push_str(&line);
6113 }
6114 }
6115 if out.len() < max_bytes {
6116 out.push('\n');
6117 }
6118 Ok(out)
6119 }
6120
6121 /// Phase 12.7c helper: find the predecessor thread id for
6122 /// `thread_id` by reading its `_mnemo.kind = "thread_link"`
6123 /// marker. Returns `None` if the thread has no predecessor
6124 /// (fresh thread, no `thread_start` call).
6125 fn predecessor_thread(&mut self, thread_id: &str) -> Result<Option<String>> {
6126 self.ensure_key_caches()?;
6127 Ok(self
6128 .layers
6129 .as_ref()
6130 .and_then(|l| l.links.get(thread_id))
6131 .map(|(from, _)| from.clone()))
6132 }
6133
6134 /// Atomic batch write combining 12.1 facts, 12.7b essence ops,
6135 /// and 12.7a archive turns in one flush (Phase 12.7c; batch
6136 /// embedding added in 12.7d).
6137 ///
6138 /// The batch runs in four steps:
6139 ///
6140 /// 1. **Structural checks** on every item: required fields, size
6141 /// caps, empty keys, `thread_id` for turns, and that every
6142 /// `Update` / `Resolve` / `Remove` targets a live essence record
6143 /// the op can apply to.
6144 /// 2. **One `Embedder::embed` call** for every text that needs a
6145 /// vector: facts whose `vector` is empty, essence `Add` /
6146 /// `AppendTimeline` texts (when an embedder is registered;
6147 /// without one, essence keeps its sentinel vector), and the
6148 /// archive turns' chunks.
6149 /// 3. **Vector checks** (width, non-zero, finite) on supplied and
6150 /// embedded vectors alike.
6151 /// 4. **Writes**: facts, then essence ops, then turns per
6152 /// `turn_policy`, then exactly one `flush()`.
6153 ///
6154 /// A failure in steps 1–3 returns an error with nothing staged. A
6155 /// text-only fact, or any turn to archive, with no embedder
6156 /// registered is a step-1 error. `put` has no rollback, so an I/O
6157 /// error during step 4 can leave the batch's earlier writes staged
6158 /// on the handle; steps 1–3 exist so nothing else can fail that
6159 /// late.
6160 pub fn commit_deltas(&mut self, batch: CommitBatch) -> Result<CommitReport> {
6161 if self.read_only {
6162 return Err(MnemoError::ReadOnly);
6163 }
6164 let CommitBatch {
6165 mut facts,
6166 essence_ops,
6167 turns,
6168 thread_id,
6169 turn_policy,
6170 } = batch;
6171 let has_embedder = self.embedder.is_some();
6172 let in_ctx = |ctx: &str, e: MnemoError| MnemoError::Invalid(format!("{ctx}: {e}"));
6173
6174 // --- 1. Structural checks (reads only) ----------------------
6175 for (i, f) in facts.iter().enumerate() {
6176 if let Some(k) = &f.key {
6177 if k.is_empty() {
6178 return Err(MnemoError::Invalid(format!(
6179 "commit_deltas: facts[{i}] key is empty"
6180 )));
6181 }
6182 }
6183 if f.memory.vector.is_empty() {
6184 if f.memory.content.is_empty() {
6185 return Err(MnemoError::Invalid(format!(
6186 "commit_deltas: facts[{i}] has neither a vector nor content to embed"
6187 )));
6188 }
6189 if !has_embedder {
6190 return Err(MnemoError::Invalid(format!(
6191 "commit_deltas: facts[{i}] has no vector and no embedder is registered \
6192 on this handle; pass a vector or call set_embedder first"
6193 )));
6194 }
6195 }
6196 }
6197 let mut removed_in_batch: std::collections::HashSet<u128> =
6198 std::collections::HashSet::new();
6199 for (i, op) in essence_ops.iter().enumerate() {
6200 let ctx = format!("commit_deltas: essence_ops[{i}]");
6201 match op.kind {
6202 EssenceOpKind::Add => {
6203 if op.scope.is_none() {
6204 return Err(MnemoError::Invalid(format!("{ctx} Add missing 'scope'")));
6205 }
6206 let section = op.section.ok_or_else(|| {
6207 MnemoError::Invalid(format!("{ctx} Add missing 'section'"))
6208 })?;
6209 let text = op
6210 .text
6211 .as_deref()
6212 .ok_or_else(|| MnemoError::Invalid(format!("{ctx} Add missing 'text'")))?;
6213 Self::check_essence_add(section, text).map_err(|e| in_ctx(&ctx, e))?;
6214 }
6215 EssenceOpKind::AppendTimeline => {
6216 if op.scope.is_none() {
6217 return Err(MnemoError::Invalid(format!(
6218 "{ctx} AppendTimeline missing 'scope'"
6219 )));
6220 }
6221 let text = op.text.as_deref().ok_or_else(|| {
6222 MnemoError::Invalid(format!("{ctx} AppendTimeline missing 'text'"))
6223 })?;
6224 Self::check_essence_append(text).map_err(|e| in_ctx(&ctx, e))?;
6225 }
6226 EssenceOpKind::Update | EssenceOpKind::Resolve | EssenceOpKind::Remove => {
6227 let id = op
6228 .id
6229 .ok_or_else(|| MnemoError::Invalid(format!("{ctx} needs 'id'")))?;
6230 if removed_in_batch.contains(&id.0) {
6231 return Err(MnemoError::Invalid(format!(
6232 "{ctx}: record {id} is removed earlier in this batch"
6233 )));
6234 }
6235 let existing = self.load_essence_record(&id).map_err(|e| in_ctx(&ctx, e))?;
6236 let section = self.section_of(&existing).map_err(|e| in_ctx(&ctx, e))?;
6237 match op.kind {
6238 EssenceOpKind::Update => {
6239 let text = op.text.as_deref().ok_or_else(|| {
6240 MnemoError::Invalid(format!("{ctx} Update needs 'text'"))
6241 })?;
6242 if section == EssenceSection::Timeline {
6243 return Err(MnemoError::Invalid(format!(
6244 "{ctx}: timeline entries are append-only"
6245 )));
6246 }
6247 if text.is_empty() || text.len() > section.item_max_bytes() {
6248 return Err(MnemoError::Invalid(format!(
6249 "{ctx}: '{}' items must be 1..={} bytes; got {}",
6250 section.as_str(),
6251 section.item_max_bytes(),
6252 text.len()
6253 )));
6254 }
6255 }
6256 EssenceOpKind::Resolve => {
6257 if !section.has_status_lifecycle() {
6258 return Err(MnemoError::Invalid(format!(
6259 "{ctx}: '{}' section has no status lifecycle",
6260 section.as_str()
6261 )));
6262 }
6263 }
6264 _ => {
6265 removed_in_batch.insert(id.0);
6266 }
6267 }
6268 }
6269 }
6270 }
6271 if !turns.is_empty() && thread_id.is_none() {
6272 return Err(MnemoError::Invalid(
6273 "commit_deltas: 'turns' non-empty but 'thread_id' missing".into(),
6274 ));
6275 }
6276 let archive_turns = !turns.is_empty() && !matches!(turn_policy, TurnPolicy::Drop);
6277 if archive_turns && !has_embedder {
6278 return Err(MnemoError::Invalid(
6279 "commit_deltas: archiving 'turns' needs an embedder registered on this handle; \
6280 call set_embedder first or use turn_policy Drop"
6281 .into(),
6282 ));
6283 }
6284 let plan = match (&thread_id, archive_turns) {
6285 (Some(t), true) => Some(self.plan_ingest(t, &turns)?),
6286 _ => None,
6287 };
6288
6289 // --- 2. One batched embed -----------------------------------
6290 let embeds_essence = |op: &EssenceOp| {
6291 has_embedder && matches!(op.kind, EssenceOpKind::Add | EssenceOpKind::AppendTimeline)
6292 };
6293 let (vectors, n_texts) = {
6294 let mut texts: Vec<&str> = Vec::new();
6295 for f in &facts {
6296 if f.memory.vector.is_empty() {
6297 texts.push(f.memory.content.as_str());
6298 }
6299 }
6300 for op in essence_ops.iter().filter(|op| embeds_essence(op)) {
6301 texts.push(op.text.as_deref().expect("checked in step 1"));
6302 }
6303 if let Some(p) = &plan {
6304 texts.extend(p.texts());
6305 }
6306 let vectors = match (texts.is_empty(), self.embedder.as_ref()) {
6307 (true, _) => Vec::new(),
6308 (false, Some(e)) => e.embed(&texts)?,
6309 (false, None) => unreachable!("texts need an embedder; checked in step 1"),
6310 };
6311 (vectors, texts.len())
6312 };
6313
6314 // --- 3. Vector checks ---------------------------------------
6315 self.check_vectors("commit_deltas", &vectors, n_texts)?;
6316 for (i, f) in facts.iter().enumerate() {
6317 let v = &f.memory.vector;
6318 if v.is_empty() {
6319 continue;
6320 }
6321 if v.len() != self.dimensions {
6322 return Err(MnemoError::Invalid(format!(
6323 "commit_deltas: facts[{i}] vector has {} dims; this file is {}",
6324 v.len(),
6325 self.dimensions
6326 )));
6327 }
6328 let ctx = format!("commit_deltas: facts[{i}]");
6329 self.check_vector(&ctx, v)?;
6330 }
6331
6332 // Hand the embedded vectors back out, in the order they were
6333 // requested: facts, then essence ops, then turn chunks.
6334 let mut vectors = vectors.into_iter();
6335 for f in facts.iter_mut() {
6336 if f.memory.vector.is_empty() {
6337 f.memory.vector = vectors.next().expect("one vector per text");
6338 }
6339 }
6340 let essence_vectors: Vec<Option<Vec<f32>>> = essence_ops
6341 .iter()
6342 .map(|op| embeds_essence(op).then(|| vectors.next().expect("one vector per text")))
6343 .collect();
6344 let turn_vectors: Vec<Vec<f32>> = vectors.collect();
6345
6346 // --- 4. Writes ----------------------------------------------
6347 let mut report = CommitReport::default();
6348
6349 for f in facts {
6350 match f.key {
6351 Some(k) => {
6352 let (_id, outcome) = self.remember_keyed(f.memory, &k)?;
6353 match outcome {
6354 WriteOutcome::Added => report.facts_added += 1,
6355 WriteOutcome::Superseded | WriteOutcome::Updated => {
6356 report.facts_superseded += 1
6357 }
6358 WriteOutcome::Unchanged => report.facts_unchanged += 1,
6359 }
6360 }
6361 None => {
6362 self.remember(f.memory)?;
6363 report.facts_added += 1;
6364 }
6365 }
6366 }
6367
6368 for (op, vector) in essence_ops.into_iter().zip(essence_vectors) {
6369 let outcome = match op.kind {
6370 EssenceOpKind::Add => {
6371 self.essence_add_with(
6372 op.scope.expect("checked in step 1"),
6373 op.section.expect("checked in step 1"),
6374 op.text.as_deref().expect("checked in step 1"),
6375 op.refs,
6376 vector,
6377 )?;
6378 WriteOutcome::Added
6379 }
6380 EssenceOpKind::AppendTimeline => {
6381 self.essence_append_with(
6382 op.scope.expect("checked in step 1"),
6383 op.text.as_deref().expect("checked in step 1"),
6384 op.refs,
6385 vector,
6386 )?;
6387 WriteOutcome::Added
6388 }
6389 EssenceOpKind::Update => self.essence_update(
6390 op.id.as_ref().expect("checked in step 1"),
6391 op.text.as_deref().expect("checked in step 1"),
6392 op.refs,
6393 )?,
6394 EssenceOpKind::Resolve => {
6395 self.essence_resolve(op.id.as_ref().expect("checked in step 1"))?
6396 }
6397 EssenceOpKind::Remove => {
6398 self.essence_remove(op.id.as_ref().expect("checked in step 1"))?;
6399 WriteOutcome::Updated
6400 }
6401 };
6402 match outcome {
6403 WriteOutcome::Unchanged => report.essence_ops_unchanged += 1,
6404 _ => report.essence_ops_applied += 1,
6405 }
6406 }
6407
6408 if let (Some(plan), Some(t)) = (plan, thread_id.as_deref()) {
6409 let ttl = match turn_policy {
6410 TurnPolicy::ExpireAfter(secs) => Some(secs),
6411 _ => None,
6412 };
6413 let r = self.apply_ingest(t, plan, turn_vectors, ttl)?;
6414 report.turns_added += r.added_records;
6415 report.turns_skipped += r.skipped_existing_turns;
6416 report.turns_truncated += r.truncated_turns;
6417 report.turns_chunked += r.chunked_turns;
6418 }
6419
6420 self.flush()?;
6421 Ok(report)
6422 }
6423
6424 /// **Not a stable API.** The file's on-disk format version.
6425 #[doc(hidden)]
6426 pub fn __format_version(&self) -> u16 {
6427 self.header.version
6428 }
6429
6430 /// **Not a stable API.** FMT-9 (a): the committed free list as
6431 /// `(freed_at_txn, first_page, page_count)` runs, plus the number of
6432 /// runs pending for the next commit.
6433 #[doc(hidden)]
6434 pub fn __free_list(&self) -> (Vec<(u64, u64, u64)>, usize) {
6435 (
6436 self.free
6437 .to_runs()
6438 .into_iter()
6439 .map(|r| (r.freed_at, r.page, r.count))
6440 .collect(),
6441 self.pending_free.len(),
6442 )
6443 }
6444
6445 /// **Not a stable API.** FMT-9: where a memory's record lives:
6446 /// `(first_page, page_count, generation)`.
6447 #[doc(hidden)]
6448 pub fn __record_location(&self, id: &Ulid) -> Option<(u64, u32, u64)> {
6449 let idx = self.index.get(id.0, &self.catalog)?;
6450 let e = &self.catalog[idx];
6451 (!e.deleted).then_some((e.start_page, e.page_count, e.gen))
6452 }
6453
6454 /// **Not a stable API.** FMT-9e: how many thread-hash buckets of turn
6455 /// records the handle caches hold undecoded (`None` until built).
6456 #[doc(hidden)]
6457 pub fn __cold_thread_buckets(&self) -> Option<usize> {
6458 self.layers.as_ref().map(|l| l.cold_turns.len())
6459 }
6460
6461 /// **Not a stable API.** FMT-9e: a live memory's stored record length
6462 /// and whether the stored body is a zstd frame.
6463 #[doc(hidden)]
6464 pub fn __record_storage(&mut self, id: &Ulid) -> Result<(u32, bool)> {
6465 let idx = self
6466 .index
6467 .get(id.0, &self.catalog)
6468 .ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
6469 let e = self.catalog[idx].clone();
6470 let bytes = self.read_record(&e)?;
6471 Ok((e.len, bytes.starts_with(&ZSTD_MAGIC)))
6472 }
6473
6474 /// **Not a stable API.** FMT-9b: whether the vector column is resident,
6475 /// and how many vector pages hold live entries.
6476 #[doc(hidden)]
6477 pub fn __vector_column(&self) -> (bool, usize) {
6478 (self.col.resident, self.col.page_rows.len())
6479 }
6480
6481 /// **Not a stable API.** FMT-9c.2: `(catalog rows incl. tombstones,
6482 /// delta runs on the chain, first page of the base run)`.
6483 #[doc(hidden)]
6484 pub fn __catalog_shape(&self) -> (usize, usize, u64) {
6485 (
6486 self.catalog.len(),
6487 self.delta_chain.len(),
6488 self.header.catalog_start,
6489 )
6490 }
6491
6492 /// **Not a stable API.** FMT-9c.3: `(index delta runs on the chain,
6493 /// first page of the base index run)`.
6494 #[doc(hidden)]
6495 pub fn __index_shape(&self) -> (usize, u64) {
6496 (self.index_chain.len(), self.header.index_start)
6497 }
6498
6499 /// **Not a stable API.** FMT-9d: BM25 keyword scores for `query`, best
6500 /// first: `(id, score)`, with k1 1.2 and b 0.75.
6501 #[doc(hidden)]
6502 pub fn __bm25(&mut self, query: &str) -> Result<Vec<(Ulid, f32)>> {
6503 let scores = self.bm25_scores(query, 1.2, 0.75)?;
6504 let mut out: Vec<(Ulid, f32)> = scores
6505 .into_iter()
6506 .map(|(row, s)| (Ulid(self.catalog[row].id), s))
6507 .collect();
6508 out.sort_by(|a, b| b.1.total_cmp(&a.1));
6509 Ok(out)
6510 }
6511
6512 /// **Not a stable API.** FMT-9d: write the next whole index with this
6513 /// layout byte (an unknown one exercises the "treat as no index" path).
6514 #[doc(hidden)]
6515 pub fn __set_index_layout_for_testing(&mut self, layout: u8) {
6516 self.index_layout_override = Some(layout);
6517 }
6518
6519 /// **Not a stable API.** FMT-9d: BM25 segments on disk.
6520 #[doc(hidden)]
6521 pub fn __bm25_segments(&self) -> usize {
6522 self.bm25.manifest.segments.len()
6523 }
6524
6525 /// **Not a stable API.** FMT-9: the header's next unallocated page.
6526 #[doc(hidden)]
6527 pub fn __next_page(&self) -> u64 {
6528 self.header.next_page
6529 }
6530
6531 /// **Not a stable API.** FMT-9 (b): add a placeholder key slot (its
6532 /// wrapped DEK is random bytes, so it unlocks nothing) to exercise the
6533 /// key-slot run until Phase 13.2 adds real slots. v9 only.
6534 #[doc(hidden)]
6535 pub fn __add_key_slot_for_testing(&mut self, label: &str) -> Result<()> {
6536 if self.read_only {
6537 return Err(MnemoError::ReadOnly);
6538 }
6539 if !self.is_v9() {
6540 return Err(MnemoError::Invalid("key slots need a v9 file".into()));
6541 }
6542 let mut wrapped = vec![0u8; WRAPPED_DEK_LEN];
6543 wrapped[..crypto::KEY_LEN].copy_from_slice(&crypto::random_dek()[..]);
6544 self.key_slots.slots.push(KeySlotEntry {
6545 label: label.to_string(),
6546 role: 0,
6547 m_cost: KdfParams::raw_key().m_cost,
6548 t_cost: KdfParams::raw_key().t_cost,
6549 p_cost: KdfParams::raw_key().p_cost,
6550 salt: crypto::random_salt(),
6551 dek_nonce: crypto::random_nonce(),
6552 wrapped_dek: wrapped,
6553 });
6554 self.dirty_keys = true;
6555 Ok(())
6556 }
6557
6558 /// **Not a stable API.** FMT-9 (b): the key-slot labels.
6559 #[doc(hidden)]
6560 pub fn __key_slot_labels(&self) -> Vec<String> {
6561 self.key_slots
6562 .slots
6563 .iter()
6564 .map(|s| s.label.clone())
6565 .collect()
6566 }
6567
6568 /// **Not a stable API.** FMT-9 (b): the key-slot run's pointer
6569 /// `(first_page, page_count, generation)`.
6570 #[doc(hidden)]
6571 pub fn __key_slot_run(&self) -> (u64, u64, u64) {
6572 (
6573 self.header.keys.start,
6574 self.header.keys.pages,
6575 self.header.keys.gen,
6576 )
6577 }
6578
6579 /// FMT-9a measurement hook: the ANN shortlist (candidate ids, nearest
6580 /// first by PQ distance) that recall would rerank, at the given or the
6581 /// index's effective `n_probe` / `n_rerank`. Empty without an index.
6582 /// **Unstable diagnostic.**
6583 #[doc(hidden)]
6584 pub fn __ann_shortlist(
6585 &mut self,
6586 query: &[f32],
6587 n_probe: Option<usize>,
6588 n_rerank: Option<usize>,
6589 ) -> Result<Vec<Ulid>> {
6590 self.ensure_ann()?;
6591 Ok(match &self.ann {
6592 Some(ann) => ann
6593 .query(query, n_probe, n_rerank)
6594 .into_iter()
6595 .map(Ulid)
6596 .collect(),
6597 None => Vec::new(),
6598 })
6599 }
6600
6601 /// P-0 device budget breakdown of this file and handle. **Unstable
6602 /// diagnostic**; see [`BudgetReport`]. Reads the header, the catalog
6603 /// and file metadata only; nothing is decrypted.
6604 #[doc(hidden)]
6605 pub fn __budget_report(&self) -> Result<BudgetReport> {
6606 let h = &self.header;
6607 let ps = h.page_size as u64;
6608 let mut r = BudgetReport {
6609 file_bytes: std::fs::metadata(&self.path)?.len(),
6610 page_size: ps,
6611 header_bytes: ps,
6612 catalog_run_bytes: h.catalog_pages * ps,
6613 catalog_payload_bytes: h.catalog_len,
6614 index_run_bytes: h.index_pages * ps,
6615 index_payload_bytes: h.index_len,
6616 index_start_page: h.index_start,
6617 wal_bytes: h.wal_pages * ps,
6618 snapshot_manifest_bytes: h.manifest_pages * ps,
6619 cache_dir_run_bytes: h.cache_pages * ps,
6620 ..Default::default()
6621 };
6622 for e in &self.catalog {
6623 // FMT-9b.2: packed v9 records share pages (counted once below);
6624 // a v9 delete frees its pages, so there are no deleted ones.
6625 let packed = e.v9.is_some_and(|r| r.rec_slot != REC_SPAN);
6626 let pages = if packed { 0 } else { e.page_count as u64 * ps };
6627 if e.deleted {
6628 if e.v9.is_none() {
6629 r.deleted_record_page_bytes += pages;
6630 }
6631 } else {
6632 r.record_page_bytes += pages;
6633 r.record_payload_bytes += e.len as u64;
6634 r.live_records += 1;
6635 }
6636 }
6637 r.record_page_bytes += self.recs.page_rows.len() as u64 * ps;
6638 // FMT-9b: the vector column, reported with the records.
6639 r.record_page_bytes += self.col.page_rows.len() as u64 * ps;
6640 let accounted = r.header_bytes
6641 + r.record_page_bytes
6642 + r.deleted_record_page_bytes
6643 + r.catalog_run_bytes
6644 + r.index_run_bytes
6645 + r.wal_bytes
6646 + r.snapshot_manifest_bytes
6647 + r.cache_dir_run_bytes;
6648 r.unreferenced_bytes = r.file_bytes.saturating_sub(accounted);
6649
6650 r.ram_catalog = (self.catalog.capacity() * std::mem::size_of::<CatalogEntry>()) as u64;
6651 r.ram_id_map = self.index.heap_bytes() as u64;
6652 if let Some(ann) = &self.ann {
6653 let (c, cb, ids, codes, assign) = ann.heap_bytes();
6654 r.ram_ann_centroids = c as u64;
6655 r.ram_ann_codebook = cb as u64;
6656 r.ram_ann_posting_ids = ids as u64;
6657 r.ram_ann_posting_codes = codes as u64;
6658 r.ram_ann_assignment = assign as u64;
6659 }
6660 r.ram_cache_dir =
6661 (self.cache_dir.capacity() * std::mem::size_of::<CacheDirectoryEntry>()) as u64;
6662 let key_index = self.key_index.as_ref().map_or(0, |k| {
6663 crate::index::hash_map_bytes::<KeySlot, u128>(k.capacity())
6664 + k.keys()
6665 .map(|s| s.key.capacity() + s.scope_owner.as_ref().map_or(0, |o| o.capacity()))
6666 .sum::<usize>()
6667 });
6668 let superseded = self.superseded.as_ref().map_or(0, |s| {
6669 crate::index::hash_map_bytes::<u128, ()>(s.capacity())
6670 });
6671 r.ram_key_caches = (key_index + superseded) as u64;
6672 r.ram_vector_column = (self.col.data.capacity() * 2
6673 + self.col.norms.capacity() * 4
6674 + self.col.have.capacity()) as u64;
6675 r.ram_total = r.ram_catalog
6676 + r.ram_id_map
6677 + r.ram_ann_centroids
6678 + r.ram_ann_codebook
6679 + r.ram_ann_posting_ids
6680 + r.ram_ann_posting_codes
6681 + r.ram_ann_assignment
6682 + r.ram_cache_dir
6683 + r.ram_key_caches
6684 + r.ram_vector_column;
6685 Ok(r)
6686 }
6687
6688 /// Return every live memory (used by tooling and compaction).
6689 pub fn memories(&mut self) -> Result<Vec<Memory>> {
6690 // FMT-9b: one pass over the vector pages instead of one per record.
6691 self.ensure_column()?;
6692 let entries: Vec<CatalogEntry> = self
6693 .catalog
6694 .iter()
6695 .filter(|e| !e.deleted)
6696 .cloned()
6697 .collect();
6698 let mut out = Vec::with_capacity(entries.len());
6699 for e in &entries {
6700 out.push(self.read_memory(e)?);
6701 }
6702 Ok(out)
6703 }
6704
6705 /// Return the database's **self-describing onboarding memories** — the
6706 /// ones tagged `metadata.area = "onboarding"`. This is the engine-level
6707 /// surface for the *single-file philosophy*: an agent who receives a
6708 /// `.mnemo` file (and its passphrase) can call this to learn what the
6709 /// file is, which embedder it expects, the recommended agent id, and any
6710 /// other conventions the file's author chose to record — all without
6711 /// needing any external documentation.
6712 ///
6713 /// Ordering: the canonical manifest (tag `metadata.topic = "manifest"`)
6714 /// always comes first; everything else follows in `importance` descending,
6715 /// then `created_at` ascending for deterministic results.
6716 pub fn about(&mut self) -> Result<Vec<Memory>> {
6717 // Phase 12.1: drop records that have been superseded by a newer
6718 // keyed write. They still live in snapshot history and
6719 // `history(key)` — just not here.
6720 // R-004.2: read only the onboarding records, from the handle caches.
6721 self.ensure_key_caches()?;
6722 let ids: Vec<u128> = self
6723 .layers
6724 .as_ref()
6725 .map(|l| l.onboarding.iter().copied().collect())
6726 .unwrap_or_default();
6727 let mut out: Vec<Memory> = Vec::new();
6728 for e in self.live_entries(ids) {
6729 let m = self.read_memory(&e)?;
6730 if !m.metadata.contains_key(MNEMO_META_SUPERSEDED_BY) && is_onboarding(&m) {
6731 out.push(m);
6732 }
6733 }
6734 out.sort_by(|a, b| {
6735 let a_manifest = is_manifest(a);
6736 let b_manifest = is_manifest(b);
6737 // Manifest topic wins first; then importance desc; then created_at asc.
6738 b_manifest
6739 .cmp(&a_manifest)
6740 .then_with(|| b.importance.total_cmp(&a.importance))
6741 .then_with(|| a.created_at.cmp(&b.created_at))
6742 });
6743 Ok(out)
6744 }
6745
6746 /// The project essence for the `about` briefing (Phase 12.7d): the
6747 /// [`EssenceScope::Project`] render at the default budget
6748 /// ([`ESSENCE_DEFAULT_RENDER_BYTES`]), or `None` when the file has no
6749 /// open project-scope item or timeline entry. The CLI and MCP `about`
6750 /// append it after the manifest and onboarding memories; [`Mnemo::about`]
6751 /// itself keeps returning records.
6752 pub fn about_essence(&mut self) -> Result<Option<String>> {
6753 let entries = self.essence_entries()?;
6754 let mut any = false;
6755 for e in &entries {
6756 let m = self.read_memory(e)?;
6757 let is_project_essence = m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
6758 == Some(MNEMO_META_KIND_ESSENCE)
6759 && !m.metadata.contains_key(MNEMO_META_THREAD);
6760 if is_project_essence && status_of(&m) != Some(essence_status::RESOLVED) {
6761 any = true;
6762 break;
6763 }
6764 }
6765 if !any {
6766 return Ok(None);
6767 }
6768 self.essence(EssenceScope::Project, ESSENCE_DEFAULT_RENDER_BYTES)
6769 .map(Some)
6770 }
6771
6772 // --- retrieval --------------------------------------------------------
6773
6774 /// Phase-1 brute-force search: rank live memories by raw similarity only.
6775 /// Read-only — does not update access statistics.
6776 pub fn search(
6777 &mut self,
6778 query: &[f32],
6779 top_k: usize,
6780 metric: Metric,
6781 ) -> Result<Vec<(Memory, f32)>> {
6782 if query.len() != self.dimensions {
6783 return Err(MnemoError::DimensionMismatch {
6784 expected: self.dimensions,
6785 got: query.len(),
6786 });
6787 }
6788 let now = memory::now_secs();
6789 let entries: Vec<CatalogEntry> = self
6790 .catalog
6791 .iter()
6792 .filter(|e| !e.deleted)
6793 .cloned()
6794 .collect();
6795
6796 let mut scored: Vec<(Memory, f32)> = Vec::new();
6797 for e in &entries {
6798 let m = self.read_memory(e)?;
6799 if m.is_expired(now) {
6800 continue;
6801 }
6802 let sim = memory::similarity(metric, query, &m.vector);
6803 scored.push((m, sim));
6804 }
6805 scored.sort_by(|a, b| b.1.total_cmp(&a.1));
6806 scored.truncate(top_k);
6807 Ok(scored)
6808 }
6809
6810 /// Phase-5 multi-signal recall: rank by
6811 /// `α·similarity + β·recency + γ·importance + δ·ln(freq)`, with type and
6812 /// agent filtering. Updates `accessed_at` / `access_count` on the
6813 /// returned memories (persisted on the next `flush`).
6814 ///
6815 /// When an ANN index has been built ([`Mnemo::build_index`]), recall runs
6816 /// the tiered IVF→PQ→rerank pipeline: it scores only the similarity-nearest
6817 /// `n_rerank` candidates rather than every memory. Without an index it
6818 /// scores every live memory exactly.
6819 /// One pass of filter + score for [`Mnemo::recall`]. Factored out of
6820 /// `recall` so the outer path can retry with a wider shortlist when
6821 /// the type/agent/supersession/TTL filters starve the ANN-narrowed
6822 /// result below `top_k` (Phase 12.1 short-results-on-indexed-path
6823 /// fix).
6824 ///
6825 /// * `force_exact = false` → use the ANN index when one exists; fall
6826 /// back to a full scan only if none is built.
6827 /// * `force_exact = true` → skip the ANN index and scan every live
6828 /// non-superseded record. Used by the outer path as the last-resort
6829 /// attempt when widening the shortlist would already touch half or
6830 /// more of the live non-superseded set.
6831 ///
6832 /// Returns an unsorted, un-truncated `Vec<RecallResult>`; the outer
6833 /// `recall` sorts and truncates once.
6834 fn score_once(
6835 &mut self,
6836 req: &RecallRequest,
6837 now: i64,
6838 force_exact: bool,
6839 ) -> Result<Vec<RecallResult>> {
6840 // Candidate set: ANN-narrowed when an index exists and the
6841 // caller hasn't forced an exact scan, else everything live.
6842 // R-004.2: a filter pinning `_mnemo.thread` scans just that
6843 // thread's records (from the handle caches); one that can only
6844 // match non-fact kinds skips the ANN, which doesn't hold them.
6845 // FMT-9d: keyword matches, normalised by the best one.
6846 self.kw = None;
6847 if let Some(text) = &req.keywords {
6848 let scores = self.bm25_scores(text, req.bm25_k1, req.bm25_b)?;
6849 let max = scores.values().copied().fold(0.0f32, f32::max);
6850 if max > 0.0 {
6851 self.kw = Some(
6852 scores
6853 .into_iter()
6854 .map(|(row, s)| (self.catalog[row].id, s / max))
6855 .collect(),
6856 );
6857 } else {
6858 self.kw = Some(std::collections::HashMap::new());
6859 }
6860 }
6861 if let Some(thread) = filter_thread_pin(&req.filter) {
6862 let thread = thread.to_string();
6863 let entries = self.thread_entries(&thread)?;
6864 return self.score_entries(req, now, &entries);
6865 }
6866 // Keyword-only: the candidates are the matches.
6867 if req.query.is_empty() {
6868 let ids: Vec<u128> = self
6869 .kw
6870 .as_ref()
6871 .map_or(Vec::new(), |k| k.keys().copied().collect());
6872 let entries: Vec<CatalogEntry> = ids
6873 .into_iter()
6874 .filter_map(|id| self.index.get(id, &self.catalog))
6875 .map(|i| self.catalog[i].clone())
6876 .filter(|e| !e.deleted)
6877 .collect();
6878 return self.score_entries(req, now, &entries);
6879 }
6880 let force_exact = force_exact || filter_bypasses_ann(&req.filter);
6881 let mut entries: Vec<CatalogEntry> = match (&self.ann, force_exact) {
6882 (Some(ann), false) => {
6883 let n_rerank = self.rerank_for(ann, req);
6884 let ids = ann.query(&req.query, req.n_probe, n_rerank);
6885 ids.iter()
6886 .filter_map(|&id| self.index.get(id, &self.catalog))
6887 .map(|i| self.catalog[i].clone())
6888 .filter(|e| !e.deleted)
6889 .collect()
6890 }
6891 _ => self
6892 .catalog
6893 .iter()
6894 .filter(|e| !e.deleted)
6895 .cloned()
6896 .collect(),
6897 };
6898 // FMT-9d: an ANN shortlist plus the keyword matches it missed.
6899 if let Some(kw) = &self.kw {
6900 if entries.len() < self.catalog.len() {
6901 let have: std::collections::HashSet<u128> = entries.iter().map(|e| e.id).collect();
6902 let extra: Vec<u128> = kw.keys().copied().filter(|id| !have.contains(id)).collect();
6903 for id in extra {
6904 if let Some(i) = self.index.get(id, &self.catalog) {
6905 if !self.catalog[i].deleted {
6906 entries.push(self.catalog[i].clone());
6907 }
6908 }
6909 }
6910 }
6911 }
6912
6913 self.score_entries(req, now, &entries)
6914 }
6915
6916 /// FMT-9d: a candidate's relevance: its vector similarity plus ε times
6917 /// its normalised keyword score (keyword-only requests: just the
6918 /// keyword score). Returns `(relevance, reported similarity)`.
6919 fn relevance(&self, req: &RecallRequest, id: u128, sim: f32) -> (f32, f32) {
6920 let kw = self
6921 .kw
6922 .as_ref()
6923 .and_then(|k| k.get(&id).copied())
6924 .unwrap_or(0.0);
6925 if req.query.is_empty() {
6926 (kw, kw)
6927 } else {
6928 (sim + req.keyword_weight * kw, sim)
6929 }
6930 }
6931
6932 /// Score one candidate set for [`Mnemo::score_once`].
6933 fn score_entries(
6934 &mut self,
6935 req: &RecallRequest,
6936 now: i64,
6937 entries: &[CatalogEntry],
6938 ) -> Result<Vec<RecallResult>> {
6939 // FMT-9b: score v9 candidates from their rows and the fp16 column,
6940 // decrypting only what makes `top_k`. A metadata filter needs the
6941 // record body, so it takes the general path below.
6942 if self.is_v9() && req.filter.is_empty() {
6943 return self.score_rows(req, now, entries);
6944 }
6945 let mut scored: Vec<RecallResult> = Vec::with_capacity(entries.len());
6946 for e in entries {
6947 // Phase 12.1: drop superseded records.
6948 //
6949 // Fast path: if the lazy `superseded` set has already been
6950 // built by a prior keyed op on this handle, check it before
6951 // decrypt — O(1), no AES work. On a fresh read-only handle
6952 // it is typically `None`, so we fall through to the slow
6953 // path (one `_mnemo.superseded_by` lookup per surviving
6954 // record, O(k) over user metadata), and never trigger the
6955 // full-file scan that `ensure_key_caches()` would do.
6956 if self.superseded.as_ref().is_some_and(|s| s.contains(&e.id)) {
6957 continue;
6958 }
6959 let m = self.read_memory(e)?;
6960 if m.metadata.contains_key(MNEMO_META_SUPERSEDED_BY) {
6961 continue;
6962 }
6963 if m.is_expired(now) {
6964 continue;
6965 }
6966 if let Some(types) = &req.memory_types {
6967 if !types.contains(&m.memory_type) {
6968 continue;
6969 }
6970 }
6971 if let Some(agent) = &req.agent_id {
6972 let visible = m.agent_id == *agent || m.scope == Scope::Shared;
6973 if !visible {
6974 continue;
6975 }
6976 }
6977 // Phase 12.6: pre-scoring filter (session_id + metadata
6978 // conditions). Cheap when `req.filter.is_empty()`; the
6979 // filter struct's `matches` is a short-circuit AND.
6980 if !req.filter.is_empty() && !req.filter.matches(&m) {
6981 continue;
6982 }
6983 // Phase 12.7: default-exclude the thread layers from fact
6984 // recall: archive turns (12.7a), essence items (12.7b; read
6985 // by scope through the render, not by similarity) and
6986 // thread-link markers (12.7c; they carry an all-equal
6987 // sentinel vector and would surface as junk hits). A caller
6988 // that wants them opts in by placing any `_mnemo.*` term in
6989 // the filter (equality, `$in`, `$exists`, or a range on
6990 // `_mnemo.thread` / `_mnemo.kind` / `_mnemo.seq`).
6991 if !req.filter.mentions_mnemo_meta()
6992 && matches!(
6993 m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str()),
6994 Some(MNEMO_META_KIND_TURN)
6995 | Some(MNEMO_META_KIND_ESSENCE)
6996 | Some(MNEMO_META_KIND_THREAD_LINK)
6997 )
6998 {
6999 continue;
7000 }
7001 let sim = if req.query.is_empty() {
7002 0.0
7003 } else {
7004 memory::similarity(req.metric, &req.query, &m.vector)
7005 };
7006 let (rel, sim) = self.relevance(req, e.id, sim);
7007 let age_secs = (now - m.created_at) as f32;
7008 let score = req
7009 .weights
7010 .score(rel, age_secs, m.importance, m.access_count);
7011 scored.push(RecallResult {
7012 memory: m,
7013 score,
7014 similarity: sim,
7015 });
7016 }
7017 Ok(scored)
7018 }
7019
7020 /// FMT-9b: [`Mnemo::score_entries`] for a v9 file with no metadata
7021 /// filter. Every check the general path makes on the decrypted record
7022 /// is made on the row instead (supersession, TTL, type, agent
7023 /// visibility, the thread-layer kinds), the similarity comes from the
7024 /// fp16 column, and the score from the row's `created_at` and
7025 /// `importance`. Candidates are then decrypted best-first until `top_k`
7026 /// survive; the agent check is repeated on the record, since the row
7027 /// only has a hash of the agent id.
7028 fn score_rows(
7029 &mut self,
7030 req: &RecallRequest,
7031 now: i64,
7032 entries: &[CatalogEntry],
7033 ) -> Result<Vec<RecallResult>> {
7034 self.ensure_column()?;
7035 let d = self.dimensions;
7036 let q = &req.query;
7037 let q_norm = q.iter().map(|x| x * x).sum::<f32>().sqrt();
7038 let agent_h = req.agent_id.as_deref().map(|a| fnv1a32(&[a.as_bytes()]));
7039 let types: Option<Vec<u8>> = req
7040 .memory_types
7041 .as_ref()
7042 .map(|ts| ts.iter().map(|&t| memory_type_code(t)).collect());
7043 let mut cand: Vec<(f32, f32, usize)> = Vec::with_capacity(entries.len());
7044 for (pos, e) in entries.iter().enumerate() {
7045 let Some(r) = e.v9 else { continue };
7046 if r.flags & ROW_SUPERSEDED != 0 {
7047 continue;
7048 }
7049 if self.superseded.as_ref().is_some_and(|s| s.contains(&e.id)) {
7050 continue;
7051 }
7052 if r.flags & ROW_HAS_TTL != 0 && now >= r.expires_at {
7053 continue;
7054 }
7055 if let Some(ts) = &types {
7056 if !ts.contains(&r.memory_type) {
7057 continue;
7058 }
7059 }
7060 if let Some(ah) = agent_h {
7061 if r.flags & ROW_SHARED == 0 && r.agent_hash != ah {
7062 continue;
7063 }
7064 }
7065 if matches!(
7066 r.kind,
7067 ROW_KIND_TURN | ROW_KIND_ESSENCE | ROW_KIND_THREAD_LINK
7068 ) {
7069 continue;
7070 }
7071 let Some(idx) = self.index.get(e.id, &self.catalog) else {
7072 continue;
7073 };
7074 if !req.query.is_empty()
7075 && self.col.resident
7076 && idx < self.col.have.len()
7077 && !self.col.have[idx]
7078 && !self.col.hot.contains_key(&idx)
7079 {
7080 self.fill_column_page(idx)?;
7081 }
7082 let sim = if req.query.is_empty() {
7083 0.0
7084 } else if let Some((h, n)) = self.col.hot.get(&idx) {
7085 veccol::similarity_f16(req.metric, q, q_norm, h, *n)
7086 } else if self.col.resident && self.col.have.get(idx).copied().unwrap_or(false) {
7087 veccol::similarity_f16(
7088 req.metric,
7089 q,
7090 q_norm,
7091 &self.col.data[idx * d..(idx + 1) * d],
7092 self.col.norms[idx],
7093 )
7094 } else {
7095 let (h, n) = self.column_entry(e)?;
7096 veccol::similarity_f16(req.metric, q, q_norm, &h, n)
7097 };
7098 let (rel, sim) = self.relevance(req, e.id, sim);
7099 let score = req.weights.score(
7100 rel,
7101 (now - r.created_at) as f32,
7102 r.importance,
7103 e.access_count,
7104 );
7105 cand.push((score, sim, pos));
7106 }
7107 cand.sort_by(|a, b| b.0.total_cmp(&a.0));
7108 let mut out: Vec<RecallResult> = Vec::with_capacity(req.top_k.min(cand.len()));
7109 for (score, sim, pos) in cand {
7110 if out.len() >= req.top_k {
7111 break;
7112 }
7113 let m = self.read_memory(&entries[pos])?;
7114 if let Some(agent) = &req.agent_id {
7115 if !(m.agent_id == *agent || m.scope == Scope::Shared) {
7116 continue;
7117 }
7118 }
7119 out.push(RecallResult {
7120 memory: m,
7121 score,
7122 similarity: sim,
7123 });
7124 }
7125 Ok(out)
7126 }
7127
7128 /// Phase-5 multi-signal recall under the v0.7.0 scoring-v2 formula
7129 /// (see [`crate::ScoreWeights`]). Thin wrapper over
7130 /// [`Mnemo::recall_budgeted`] that drops the `truncated` flag;
7131 /// use the budgeted form if you need to know whether
7132 /// [`RecallRequest::max_bytes`] cut anything.
7133 pub fn recall(&mut self, req: &RecallRequest) -> Result<Vec<RecallResult>> {
7134 Ok(self.recall_budgeted(req)?.hits)
7135 }
7136
7137 /// Phase 12.8 budgeted recall: scoring v2 (see [`crate::ScoreWeights`])
7138 /// with an optional UTF-8-content byte cap. Ranks by
7139 /// `sim + max(sim, 0) · (β·recency + γ·(importance − 0.5) + δ·tanh(uses/5))`,
7140 /// with type / agent / supersession / TTL filters and the 12.6
7141 /// `Filter` applied before the scorer. Updates `accessed_at` /
7142 /// `access_count` on the returned memories (persisted on the next
7143 /// `flush`).
7144 ///
7145 /// When an ANN index has been built ([`Mnemo::build_index`]),
7146 /// recall runs the tiered IVF→PQ→rerank pipeline. If the filters
7147 /// (type / agent / supersession / TTL / 12.6) reduce the ANN
7148 /// shortlist below `top_k`, Phase 12.1's short-results-on-indexed-
7149 /// path fix kicks in: the query is retried with `n_probe` and
7150 /// `n_rerank` doubled, and if that would already touch half or
7151 /// more of the live non-superseded records, the ANN pass is
7152 /// skipped for an exact scan. Without an index, recall scores
7153 /// every live non-superseded memory.
7154 ///
7155 /// Phase 12.8: after `top_k` is applied, if
7156 /// [`RecallRequest::max_bytes`] is set, hits are dropped in rank
7157 /// order once the running sum of `content` UTF-8 bytes would
7158 /// exceed the budget. The top-1 hit is always kept even if it
7159 /// alone exceeds the budget. [`RecallResponse::truncated`] is
7160 /// `true` iff at least one hit was dropped by the budget.
7161 pub fn recall_budgeted(&mut self, req: &RecallRequest) -> Result<RecallResponse> {
7162 // FMT-9d: keyword recall needs a v9 file's BM25 index; a
7163 // keyword-only request has no vector.
7164 if req.keywords.is_some() && !self.is_v9() {
7165 return Err(MnemoError::Invalid(
7166 "keyword recall (keywords) needs a v9 file; run `mnemo compact` to upgrade".into(),
7167 ));
7168 }
7169 let keyword_only = req.query.is_empty() && req.keywords.is_some();
7170 if !keyword_only && req.query.len() != self.dimensions {
7171 return Err(MnemoError::DimensionMismatch {
7172 expected: self.dimensions,
7173 got: req.query.len(),
7174 });
7175 }
7176 // PERF-1 step 3: the ANN index loads on the first recall.
7177 self.ensure_ann()?;
7178 // Access-tracking bumps `accessed_at` / `access_count` in the
7179 // catalog, which is a mutation. On a read-only handle the caller
7180 // must pass `track_access(false)` (via CLI `--no-track`).
7181 if self.read_only && req.track_access {
7182 return Err(MnemoError::ReadOnly);
7183 }
7184 // Phase 12.1 **fix** (lane A v0.8.0 REVIEW): do NOT call
7185 // `ensure_key_caches()` here. A fresh read-only handle that
7186 // only ever recalls would otherwise pay a full-file decrypt
7187 // on its first query to build the lazy superseded set. Instead
7188 // `score_once` checks `_mnemo.superseded_by` per-record after
7189 // decrypt (slow path) and consults the set only when it has
7190 // already been built by a prior keyed op on this handle (fast
7191 // path). `live_non_superseded` is not knowable without a
7192 // decrypt pass either, so we approximate by `live_count`; the
7193 // exact-scan gate uses this as a conservative upper bound,
7194 // which errs toward widening (cheaper) rather than skipping
7195 // straight to a full scan.
7196 let live_count = self.catalog.iter().filter(|e| !e.deleted).count();
7197 let live_non_superseded = live_count;
7198 let now = memory::now_secs();
7199
7200 // Scoring v2 (shipped v0.7.0). Single-pass filter + score:
7201 // * `age_secs = now - m.created_at`, so a recall doesn't bump
7202 // its own results' recency on later queries.
7203 // * `ScoreWeights::score` applies the similarity-gated,
7204 // importance-centered, saturating-usage formula documented
7205 // on `ScoreWeights`.
7206 //
7207 // Phase 12.1 + 12.6 short-results-on-indexed-path fix: when the
7208 // ANN shortlist is narrower than the live count, the filters
7209 // (type / agent / supersession / TTL / 12.6) can reduce the
7210 // result below `top_k`. If that happens, re-query the index
7211 // with `n_probe` / `n_rerank` doubled from their *effective*
7212 // base; if that would already cover ≥50% of the live set,
7213 // skip the index and do an exact scan.
7214 //
7215 // Lane A v0.8.0 REVIEW fix: widen from
7216 // `IvfPqIndex::effective_defaults()` when the caller didn't
7217 // pin the knobs. The pre-fix code only doubled when `req.n_*`
7218 // was already `Some(_)`, which skipped the widen step for the
7219 // default (unpinned) recall path and jumped straight to an
7220 // exact scan on moderately selective filters.
7221 let mut scored = self.score_once(req, now, false)?;
7222 let mut used_exact_scan = self.ann.is_none() || filter_bypasses_ann(&req.filter);
7223 if !used_exact_scan && scored.len() < req.top_k && live_non_superseded > req.top_k {
7224 if let Some(ann) = &self.ann {
7225 let (default_probe, default_rerank) = ann.effective_defaults();
7226 let base_probe = req.n_probe.unwrap_or(default_probe);
7227 let base_rerank = self.rerank_for(ann, req).unwrap_or(default_rerank);
7228 let widen_probe = base_probe.saturating_mul(2);
7229 let widen_rerank = base_rerank.saturating_mul(2);
7230 // Attempt 2: widen once, 2× from the effective base.
7231 // Only worth another ANN pass if either knob actually
7232 // grew (saturating_mul on usize::MAX would cap).
7233 if widen_probe > base_probe || widen_rerank > base_rerank {
7234 let widened = RecallRequest {
7235 n_probe: Some(widen_probe),
7236 n_rerank: Some(widen_rerank),
7237 ..req.clone()
7238 };
7239 scored = self.score_once(&widened, now, false)?;
7240 }
7241 // Attempt 3: if still short after widening, pay the
7242 // exact scan — it's the only path that guarantees
7243 // `top_k` correct hits when the filter is more
7244 // selective than `widen_rerank / live_count`. Before
7245 // the fix this was gated on `widened_rerank * 2 >=
7246 // live_non_superseded`, which silently returned fewer
7247 // than `top_k` hits on selective filters that fell in
7248 // the gap. Exact scan is bounded (one decrypt per
7249 // live record); the previous widen is an optimization.
7250 if scored.len() < req.top_k {
7251 scored = self.score_once(req, now, true)?;
7252 used_exact_scan = true;
7253 }
7254 }
7255 }
7256 scored.sort_by(|a, b| b.score.total_cmp(&a.score));
7257 scored.truncate(req.top_k);
7258
7259 // Update access statistics for everything we surfaced — but only
7260 // mutate the catalog entry, not the full record on disk. Pre-v5
7261 // this loop called `self.put(r.memory.clone())`, which serialized
7262 // and rewrote every result (vector included) to fresh pages,
7263 // turning a top-K recall into K full record rewrites. Now the
7264 // catalog is the source of truth for these two fields and the
7265 // record body stays untouched until the next real edit.
7266 if req.track_access {
7267 for r in &mut scored {
7268 if let Some(idx) = self.index.get(r.memory.id.0, &self.catalog) {
7269 self.touch(idx);
7270 let entry = &mut self.catalog[idx];
7271 entry.accessed_at = now;
7272 entry.access_count = entry.access_count.saturating_add(1);
7273 // Propagate the just-updated values onto the returned
7274 // Memory so the caller sees the post-recall state.
7275 r.memory.accessed_at = entry.accessed_at;
7276 r.memory.access_count = entry.access_count;
7277 }
7278 }
7279 // Mark the catalog dirty so the next flush persists these
7280 // bumps. The records themselves are not dirty.
7281 if !scored.is_empty() {
7282 self.dirty_catalog = true;
7283 }
7284 }
7285
7286 // Phase 12.8: apply the `max_bytes` budget *after* tracking has
7287 // seen the full top-k list. Access stats record what the scorer
7288 // chose; the byte budget shouldn't suppress the signal.
7289 let truncated = if let Some(budget) = req.max_bytes {
7290 let orig_len = scored.len();
7291 let mut running = 0usize;
7292 let mut cut_at: Option<usize> = None;
7293 for (i, r) in scored.iter().enumerate() {
7294 let w = r.memory.content.len();
7295 if i == 0 {
7296 // Always keep the top hit, even if it's over budget
7297 // on its own. Avoid "I asked for top_k results and
7298 // got zero" when a single memory exceeds the cap.
7299 running += w;
7300 continue;
7301 }
7302 if running.saturating_add(w) > budget {
7303 cut_at = Some(i);
7304 break;
7305 }
7306 running += w;
7307 }
7308 if let Some(idx) = cut_at {
7309 scored.truncate(idx);
7310 scored.len() < orig_len
7311 } else {
7312 false
7313 }
7314 } else {
7315 false
7316 };
7317
7318 self.kw = None;
7319 Ok(RecallResponse {
7320 hits: scored,
7321 truncated,
7322 used_exact_scan,
7323 })
7324 }
7325
7326 // --- approximate index ------------------------------------------------
7327
7328 /// Build an IVF+PQ approximate-nearest-neighbour index over every live
7329 /// memory, using default tuning. After this, [`Mnemo::recall`] runs the
7330 /// tiered pipeline instead of an exact scan. Persisted on the next
7331 /// `flush`. Returns a snapshot of the index shape.
7332 pub fn build_index(&mut self) -> Result<IndexInfo> {
7333 self.build_index_with(IndexConfig::default())
7334 }
7335
7336 /// Build the ANN index with explicit tuning.
7337 pub fn build_index_with(&mut self, cfg: IndexConfig) -> Result<IndexInfo> {
7338 if self.read_only {
7339 return Err(MnemoError::ReadOnly);
7340 }
7341 // R-004.2: facts only; turns, essence items and thread links stay
7342 // out of the ANN.
7343 let mems: Vec<Memory> = self.memories()?.into_iter().filter(is_ann_kind).collect();
7344 if mems.is_empty() {
7345 return Err(MnemoError::Invalid(
7346 "cannot build an index over an empty database".into(),
7347 ));
7348 }
7349 // FMT-9b: a v9 file below V9_ANN_MIN_FACTS recalls by an exact fp16
7350 // scan, which is faster than the index there and costs no disk.
7351 // Report it as an index with no partitions, and drop a stale one.
7352 // `MNEMO_DEV_ANN_MIN_FACTS` overrides the threshold. Unsupported,
7353 // for tests (and the recall-grid bench) only: it lets the index
7354 // suites exercise a v9 ANN on small files. Not documented.
7355 let min_facts = std::env::var("MNEMO_DEV_ANN_MIN_FACTS")
7356 .ok()
7357 .and_then(|v| v.parse::<usize>().ok())
7358 .unwrap_or(V9_ANN_MIN_FACTS);
7359 if self.is_v9() && mems.len() < min_facts {
7360 if self.ann.is_some() || self.ann_unloaded {
7361 self.ann = None;
7362 self.ann_unloaded = false;
7363 self.dirty_index = true;
7364 self.index_persist.forced = true;
7365 }
7366 return Ok(IndexInfo {
7367 vectors: mems.len(),
7368 partitions: 0,
7369 subspaces: 0,
7370 n_probe: 0,
7371 n_rerank: 0,
7372 });
7373 }
7374 let items: Vec<(u128, &[f32])> =
7375 mems.iter().map(|m| (m.id.0, m.vector.as_slice())).collect();
7376 let mut idx = IvfPqIndex::build(self.dimensions, &items, cfg)?;
7377 let info = idx.info();
7378 if self.is_v9() {
7379 idx.enable_log();
7380 self.index_rebase = true;
7381 }
7382 self.ann = Some(idx);
7383 self.ann_unloaded = false;
7384 self.dirty_index = true;
7385 self.index_persist.forced = true;
7386 Ok(info)
7387 }
7388
7389 /// Rebuild the ANN index from scratch — re-clusters centroids and retrains
7390 /// the PQ codebook, undoing the cluster drift that accumulates as memories
7391 /// are inserted against fixed centroids.
7392 pub fn rebuild_index(&mut self) -> Result<IndexInfo> {
7393 // ANN-1.1: re-derive every parameter (partitions, subspaces,
7394 // n_probe, n_rerank) for the current corpus size by passing
7395 // `IndexConfig::default()`. Previously this freezed the
7396 // stored `(n_probe, n_rerank)` from the current index,
7397 // defeating the point of rebuilding on a grown corpus. If
7398 // you really want to pin specific values, call
7399 // `build_index_with(IndexConfig { … })` directly.
7400 self.build_index_with(IndexConfig::default())
7401 }
7402
7403 /// Drop the ANN index; recall reverts to exact scans. Persisted on flush.
7404 /// No-op on a read-only handle (signature is infallible — callers of
7405 /// `build_index` / `flush` on a read-only handle already get a typed
7406 /// `ReadOnly` error, which is the enforceable boundary).
7407 pub fn drop_index(&mut self) {
7408 if self.read_only {
7409 return;
7410 }
7411 if self.ann.is_some() || self.ann_unloaded {
7412 self.ann = None;
7413 self.ann_unloaded = false;
7414 self.dirty_index = true;
7415 self.index_persist.forced = true;
7416 }
7417 }
7418
7419 /// PERF-1 (R-004.1): bring a just-loaded ANN index up to date with the
7420 /// catalog. Called wherever an index run is loaded: on first use after
7421 /// `open` / `open_read_only` (`ensure_ann`, PERF-1 step 3) and by
7422 /// `restore_to`, with the first page of that run.
7423 ///
7424 /// `flush` persists the index only every so often, so that run can lag
7425 /// the catalog. It does two things:
7426 ///
7427 /// 1. **Prune** every indexed id the catalog no longer has live
7428 /// (missing or deleted).
7429 /// 2. **Re-add** every live record written after the index run. Pages
7430 /// are allocated monotonically in v8, and every write, including an
7431 /// in-place `update` or `essence_update` (they rewrite the record to
7432 /// fresh pages), lands at a page `> index_start`. Those records are
7433 /// read and re-assigned. A record of a non-fact kind (turn, essence,
7434 /// thread link) is removed instead: those never belong in the ANN.
7435 ///
7436 /// The work is bounded by [`INDEX_PERSIST_EVERY_CHANGES`] in the common
7437 /// case, since a writer persists the index at least that often.
7438 ///
7439 /// **v9:** pages are reused, so "a page past the run" no longer means
7440 /// "written after it". A v9 record is newer than the index run when its
7441 /// generation is greater than the run's (`index_gen`).
7442 fn reconcile_ann_with_catalog(&mut self, index_start: u64, index_gen: u64) -> Result<()> {
7443 if self.ann.is_none() {
7444 return Ok(());
7445 }
7446 let mut changed = 0usize;
7447 // 1. Prune ids that are no longer live.
7448 let stale: Vec<u128> = match &self.ann {
7449 Some(ann) => ann
7450 .ids()
7451 .filter(|&id| !matches!(self.index.get(id, &self.catalog), Some(i) if !self.catalog[i].deleted))
7452 .collect(),
7453 None => Vec::new(),
7454 };
7455 if let Some(ann) = &mut self.ann {
7456 for id in &stale {
7457 ann.remove(*id);
7458 changed += 1;
7459 }
7460 }
7461 // 2. Re-add records written after the index run.
7462 let v9 = self.is_v9();
7463 let pending: Vec<CatalogEntry> = self
7464 .catalog
7465 .iter()
7466 .filter(|e| !e.deleted && written_after_index(e, v9, index_start, index_gen))
7467 .cloned()
7468 .collect();
7469 for e in &pending {
7470 let m = self.read_memory(e)?;
7471 if let Some(ann) = &mut self.ann {
7472 if is_ann_kind(&m) {
7473 ann.add(e.id, &m.vector);
7474 } else {
7475 ann.remove(e.id);
7476 }
7477 changed += 1;
7478 }
7479 }
7480 if changed > 0 {
7481 self.dirty_index = true;
7482 self.index_persist.changes += changed;
7483 }
7484 Ok(())
7485 }
7486
7487 /// Whether the database has an ANN index (loaded, or persisted and
7488 /// loading on first use).
7489 pub fn has_index(&self) -> bool {
7490 self.ann.is_some() || self.ann_unloaded
7491 }
7492
7493 /// PERF-1 step 3: load the persisted ANN index on first use, then bring
7494 /// it up to date with the catalog (`reconcile_ann_with_catalog`) and,
7495 /// when the handle caches are already built, prune non-fact ids an
7496 /// older persisted index may hold.
7497 fn ensure_ann(&mut self) -> Result<()> {
7498 if !self.ann_unloaded {
7499 return Ok(());
7500 }
7501 self.ann_unloaded = false;
7502 self.ann = load_index(
7503 &mut self.pager,
7504 self.header.index_start,
7505 self.header.index_pages,
7506 self.header.index_len,
7507 self.header.index_gen,
7508 self.dimensions,
7509 )?;
7510 // FMT-9c.3: replay the v9 index delta chain, then record changes.
7511 if self.is_v9() {
7512 if let Some(ann) = self.ann.as_mut() {
7513 let (chain, n) = apply_index_chain(&mut self.pager, ann, self.header.index_delta)?;
7514 self.index_chain = chain;
7515 self.index_chain_ops = n;
7516 ann.enable_log();
7517 } else if self.header.index_pages > 0 {
7518 // FMT-9f (the 9d leak): an unknown-layout base loads as no
7519 // index; still track its delta runs, so rebuilding or
7520 // dropping the index frees them along with the base.
7521 self.index_chain = index_chain_runs(&mut self.pager, self.header.index_delta);
7522 self.index_chain_ops = 0;
7523 }
7524 }
7525 let (index_start, index_gen) = (self.header.index_start, self.index_gen_now());
7526 self.reconcile_ann_with_catalog(index_start, index_gen)?;
7527 let non_facts: Vec<u128> = match &self.layers {
7528 Some(l) => l
7529 .essence
7530 .iter()
7531 .copied()
7532 .chain(l.threads.values().flat_map(|t| t.records.iter().copied()))
7533 .chain(l.cold_turns.values().flatten().copied())
7534 .collect(),
7535 None => Vec::new(),
7536 };
7537 if let Some(ann) = &mut self.ann {
7538 let mut pruned = 0usize;
7539 for id in non_facts {
7540 if ann.contains(id) {
7541 ann.remove(id);
7542 pruned += 1;
7543 }
7544 }
7545 if pruned > 0 {
7546 self.dirty_index = true;
7547 self.index_persist.changes += pruned;
7548 }
7549 }
7550 Ok(())
7551 }
7552
7553 // --- durability & maintenance ----------------------------------------
7554
7555 /// Seal a serialized buffer into a fresh run of encrypted home pages and
7556 /// append a WAL frame for each. Returns the run's `(start_page, pages)`.
7557 /// The pages are *not* written to their home locations here — they go to
7558 /// the WAL and are folded in later by [`Mnemo::checkpoint`].
7559 ///
7560 /// `gen` is the run's generation (v9; `0` on v8). On v9 the pages come
7561 /// from the free list when an eligible run fits.
7562 fn seal_run(
7563 &mut self,
7564 bytes: &[u8],
7565 gen: u64,
7566 frames: &mut Vec<wal::Frame>,
7567 ) -> Result<(u64, u32)> {
7568 let pc = bytes.len().div_ceil(PAYLOAD).max(1);
7569 let start = self.alloc_pages(pc as u64);
7570 self.seal_run_at(start, pc as u64, bytes, gen, frames)?;
7571 Ok((start, pc as u32))
7572 }
7573
7574 /// Seal `bytes` into the `pages` already-allocated pages at `start`
7575 /// (pages past the data are sealed empty) and append their WAL frames.
7576 fn seal_run_at(
7577 &mut self,
7578 start: u64,
7579 pages: u64,
7580 bytes: &[u8],
7581 gen: u64,
7582 frames: &mut Vec<wal::Frame>,
7583 ) -> Result<()> {
7584 let direct = self.is_v9() && pages > V9_WAL_RUN_MAX_PAGES;
7585 for i in 0..pages as usize {
7586 let lo = (i * PAYLOAD).min(bytes.len());
7587 let hi = ((i + 1) * PAYLOAD).min(bytes.len());
7588 let page_no = start + i as u64;
7589 let sealed = self.pager.seal_page(page_no, gen, &bytes[lo..hi])?;
7590 if direct {
7591 self.pager.write_sealed(page_no, &sealed)?;
7592 } else {
7593 frames.push((page_no, sealed.to_vec()));
7594 }
7595 }
7596 if direct {
7597 self.direct_writes = true;
7598 }
7599 Ok(())
7600 }
7601
7602 /// Grow the WAL region if `frame_pages` worth of frames would not fit.
7603 /// Called at the top of `flush`, where the WAL is always spent (every
7604 /// flush ends with a checkpoint), so relocating it cannot strand a
7605 /// committed transaction.
7606 fn ensure_wal_capacity(&mut self, frame_pages: usize) -> Result<()> {
7607 let need = wal::txn_byte_len(frame_pages, PAGE_SIZE);
7608 if need <= self.header.wal_pages * PAGE_SIZE as u64 {
7609 return Ok(());
7610 }
7611 // Allocate a new, larger region at the file tail (~1.5x headroom).
7612 let req = need.div_ceil(PAGE_SIZE as u64);
7613 let new_pages = (req + req / 2 + 4).max(DEFAULT_WAL_PAGES);
7614 let old_wal_start = self.header.wal_start;
7615 let old_wal_pages = self.header.wal_pages;
7616 // FMT-9 (a): the new region always comes from the file's tail: the
7617 // relocation commits a header-only transaction, so a region taken
7618 // from the free list would still be listed as free in the
7619 // committed free-list run.
7620 let new_start = self.header.next_page;
7621 self.header.next_page += new_pages;
7622 self.header.wal_start = new_start;
7623 self.header.wal_pages = new_pages;
7624 self.header.wal_seq += 1;
7625 // DUR-1 (R-002.1): WAL-log the header change into the *new* region
7626 // first, checkpoint page 0, then invalidate the old region. A crash
7627 // anywhere here either keeps the old state (if page 0 wasn't yet
7628 // overwritten) or recovers to the new state via the torn-header
7629 // scan in `Mnemo::open`, which prefers the highest-txn-id valid
7630 // committed header frame.
7631 self.relocate_header_through_wal(old_wal_start)?;
7632 // FMT-9 (a): once the relocation has committed, the old region is
7633 // unreferenced; the next commit frees it.
7634 self.release_pages(old_wal_start, old_wal_pages);
7635 Ok(())
7636 }
7637
7638 /// Shared helper for every post-`create` page-0 commit: WAL-log the
7639 /// new header into the current WAL region, checkpoint page 0, and
7640 /// invalidate `old_wal_start` (the previous region, or the current
7641 /// region when it hasn't moved). The caller is expected to have
7642 /// mutated `self.header` to its target values *including a bumped
7643 /// `wal_seq`* (one more than the last on-disk `wal_seq`); this
7644 /// routine applies the seal, writes the frame + commit, folds page 0,
7645 /// then zeroes the old region's first frame header.
7646 ///
7647 /// If `old_wal_start == self.header.wal_start`, the invalidation is
7648 /// skipped — the region didn't move, so overwriting its first frame
7649 /// would destroy the just-committed one.
7650 fn relocate_header_through_wal(&mut self, old_wal_start: u64) -> Result<()> {
7651 self.header.write_counter = self.pager.write_counter;
7652 let mut page = self.header.to_page();
7653 self.header.apply_seal(&mut page, self.pager.dek())?;
7654 let txn_id = self.header.wal_seq;
7655 let frames = vec![(0u64, page.to_vec())];
7656 wal::commit(
7657 self.pager.file_mut(),
7658 self.header.wal_start,
7659 self.header.wal_pages,
7660 txn_id,
7661 &frames,
7662 )?;
7663 self.pager.write_raw(0, &page)?;
7664 self.pager.sync()?;
7665 if old_wal_start != self.header.wal_start {
7666 wal::invalidate_region(self.pager.file_mut(), old_wal_start)?;
7667 }
7668 Ok(())
7669 }
7670
7671 /// Fold a committed transaction's WAL frames into their home pages.
7672 fn checkpoint(&mut self, frames: &[wal::Frame]) -> Result<()> {
7673 for (page_no, bytes) in frames {
7674 let mut img = [0u8; PAGE_SIZE];
7675 img.copy_from_slice(bytes);
7676 self.pager.write_sealed(*page_no, &img)?;
7677 }
7678 self.pager.sync()?;
7679 Ok(())
7680 }
7681
7682 /// Persist all pending changes as one **write-ahead-logged transaction**.
7683 ///
7684 /// Sequence:
7685 ///
7686 /// 1. **Prepare.** Serialize the control plane (catalog, ANN index),
7687 /// grow the WAL if needed, and **lease** counter and page slots —
7688 /// bump `header.write_counter` and `header.next_page` by upper
7689 /// bounds on this transaction's consumption and persist that
7690 /// *leased* header to disk before any encrypted page is written.
7691 /// 2. **Data pages.** [`Pager::flush`] writes dirty record pages
7692 /// copy-on-write under fresh nonces and fsyncs them. The orphan-page
7693 /// nonce-reuse window (see [Phase 1.1 of the improvement plan])
7694 /// is closed because the leased header on disk already records a
7695 /// `write_counter` past anything this step can produce.
7696 /// 3. **Control plane.** Seal the new catalog / ANN index / snapshot
7697 /// manifest into fresh home page runs.
7698 /// 4. **Commit.** Log all of step 3's frames plus the final header
7699 /// frame into the WAL and fsync — the durability point.
7700 /// 5. **Checkpoint.** Fold the WAL into the home pages.
7701 ///
7702 /// A crash before step 4 leaves the previous state intact. A crash
7703 /// after step 4 is repaired by [`Mnemo::open`] replaying the WAL.
7704 /// Safe to call repeatedly.
7705 pub fn flush(&mut self) -> Result<()> {
7706 if self.read_only {
7707 return Err(MnemoError::ReadOnly);
7708 }
7709 // Snapshot the thread-local fsync counter. Every `sync_all` the
7710 // pager / wal / checkpoint path makes between here and the end
7711 // of this function bumps the same counter; `delta` lands on
7712 // `self.fsyncs_last_flush` so `Stats::fsyncs_last_flush` can
7713 // report it. The planner wants this to decide whether cutting
7714 // fsync count is a viable write-latency win before we touch
7715 // the write path's durability model.
7716 let fsync_before = crate::fsync_count();
7717 // PERF-1 step 3: while the index is unloaded, records written since
7718 // its run pile up for the next load to re-assign. Once there are
7719 // INDEX_PERSIST_EVERY_CHANGES of them, load it now so this flush
7720 // persists a current index and keeps that work bounded.
7721 if self.ann_unloaded && self.dirty_catalog {
7722 let (index_start, index_gen) = (self.header.index_start, self.index_gen_now());
7723 let v9 = self.is_v9();
7724 let pending = self
7725 .catalog
7726 .iter()
7727 .filter(|e| !e.deleted && written_after_index(e, v9, index_start, index_gen))
7728 .count();
7729 if pending >= INDEX_PERSIST_EVERY_CHANGES {
7730 self.ensure_ann()?;
7731 }
7732 }
7733 // PERF-1: persist the (derived) ANN index only when forced or due.
7734 // Otherwise the header keeps pointing at the previous index run, and
7735 // records written since then are re-assigned when it's next loaded
7736 // (`reconcile_ann_with_catalog`). `ann == None` with `dirty_index` set is a
7737 // dropped index, which is persisted at once.
7738 let persist_index = if self.is_v9() {
7739 self.plan_index_v9()
7740 } else {
7741 self.dirty_index
7742 && (self.index_persist.forced
7743 || self.ann.is_none()
7744 || self.index_persist.commits + 1 >= INDEX_PERSIST_EVERY_COMMITS
7745 || self.index_persist.changes >= INDEX_PERSIST_EVERY_CHANGES)
7746 };
7747 let free_work =
7748 self.is_v9() && (self.free_dirty || !self.pending_free.is_empty() || self.dirty_keys);
7749 if !self.dirty_catalog && !persist_index && !self.dirty_cache && !free_work {
7750 // No control-plane changes pending. Because every `put` that
7751 // dirties a data page also dirties the catalog, and every
7752 // `cache_put` that dirties a data page also dirties
7753 // `dirty_cache`, there can't be dirty data pages either —
7754 // nothing to do.
7755 self.fsyncs_last_flush = 0;
7756 return Ok(());
7757 }
7758
7759 // FIX-1 R-001.8 part 3: snapshot the memory/index-dirty flags
7760 // before step 3b decides whether to push a restore point.
7761 // Only flushes that change the memory catalog or ANN index
7762 // need a snapshot — a cache-only flush (`dirty_cache` set,
7763 // `dirty_catalog` + `dirty_index` both false) must not push,
7764 // because snapshots don't capture cache state and every push
7765 // counts against `max_snapshots` (default 256), so a cache-
7766 // heavy workload would otherwise evict every real memory
7767 // restore point within ~256 cache writes.
7768 let is_memory_or_index_flush = self.dirty_catalog || persist_index;
7769
7770 // FMT-9: a v9 session's first sealed page needs the generation
7771 // lease (normally taken by its first record write already).
7772 self.ensure_gen_lease()?;
7773 // FMT-9e: turns that aged past the cutoff are rewritten compressed
7774 // (bounded per commit), then packed with the other waiting records.
7775 self.settle_cold_turns()?;
7776 // FMT-9b: column entries first; they move rows' vector locations,
7777 // so they must land before the catalog is serialized.
7778 self.pack_records()?;
7779 self.pack_vectors()?;
7780
7781 // 1. Serialize control plane, ensure WAL capacity, lease counter +
7782 // page slots, and persist the leased header *before* any data
7783 // page hits the disk.
7784 let ctx = self.prepare_for_flush(persist_index)?;
7785
7786 // 2. Record (vector) data pages: copy-on-write to fresh pages, fsynced.
7787 // Safe: the lease guarantees that even if we crash here, reopen
7788 // will see a `write_counter` past every nonce this step uses.
7789 self.pager.flush()?;
7790
7791 // 3. Seal the catalog / index control pages into fresh home runs.
7792 // FMT-9: `txn_id` is this transaction's id and the generation
7793 // of every run it seals. A v9 run replaced here is freed by this
7794 // same commit (`superseded`); v8 never reuses pages.
7795 let txn_id = self.header.wal_seq + 1;
7796 let gen = if self.is_v9() { txn_id } else { 0 };
7797 let v9 = self.is_v9();
7798 let mut superseded: Vec<(u64, u64)> = Vec::new();
7799 let mut frames: Vec<wal::Frame> = Vec::new();
7800 // FMT-9c.2: a delta run goes on the chain; a new base replaces the
7801 // old base and the whole chain (all freed by this commit). The
7802 // chain is adopted once the commit succeeds.
7803 let mut new_chain: Option<(Vec<RunPtr>, usize)> = None;
7804 if let Some(bytes) = &ctx.cat_bytes {
7805 if ctx.cat_delta {
7806 let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
7807 let ptr = RunPtr {
7808 start,
7809 pages: pages as u64,
7810 len: bytes.len() as u64,
7811 gen,
7812 };
7813 self.header.catalog_delta = ptr;
7814 let mut chain = self.delta_chain.clone();
7815 chain.push(ptr);
7816 new_chain = Some((chain, self.delta_rows + self.dirty_rows.len()));
7817 } else {
7818 if v9 && self.header.catalog_pages > 0 {
7819 superseded.push((self.header.catalog_start, self.header.catalog_pages));
7820 }
7821 if v9 {
7822 for r in &self.delta_chain {
7823 superseded.push((r.start, r.pages));
7824 }
7825 self.header.catalog_delta = RunPtr::default();
7826 new_chain = Some((Vec::new(), 0));
7827 }
7828 let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
7829 self.header.catalog_start = start;
7830 self.header.catalog_pages = pages as u64;
7831 self.header.catalog_len = bytes.len() as u64;
7832 self.header.catalog_gen = gen;
7833 }
7834 }
7835 // FMT-9d: the BM25 segment, then the manifest that lists it.
7836 let mut new_bm25: Option<Bm25Segments> = None;
7837 if let Some((bytes, replace_from)) = &ctx.bm25_seg {
7838 let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
7839 let mut segs = self.bm25.manifest.segments.clone();
7840 let mut seg_docs = self.bm25.manifest.seg_docs.clone();
7841 seg_docs.resize(segs.len(), 0);
7842 let docs = if let Some(from) = *replace_from {
7843 for t in segs.drain(from..) {
7844 superseded.push((t.0, t.1));
7845 }
7846 seg_docs.truncate(from);
7847 self.bm25.merged_docs
7848 } else {
7849 self.bm25.pending.len() as u32
7850 };
7851 segs.push((start, pages as u64, bytes.len() as u64, gen));
7852 seg_docs.push(docs);
7853 let man = Bm25Manifest {
7854 tokenizer: text::TOKENIZER_V1,
7855 next_doc: self.bm25.manifest.next_doc,
7856 segments: segs.clone(),
7857 seg_docs: seg_docs.clone(),
7858 };
7859 let mbytes =
7860 rmp_serde::to_vec(&man).map_err(|e| MnemoError::Serialize(e.to_string()))?;
7861 if !self.header.bm25.is_empty() {
7862 superseded.push((self.header.bm25.start, self.header.bm25.pages));
7863 }
7864 let (ms, mp) = self.seal_run(&mbytes, gen, &mut frames)?;
7865 self.header.bm25 = RunPtr {
7866 start: ms,
7867 pages: mp as u64,
7868 len: mbytes.len() as u64,
7869 gen,
7870 };
7871 new_bm25 = Some((segs, seg_docs));
7872 }
7873 let mut new_ichain: Option<(Vec<RunPtr>, usize)> = None;
7874 if persist_index && ctx.idx_delta {
7875 if let Some(bytes) = &ctx.idx_bytes {
7876 let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
7877 let ptr = RunPtr {
7878 start,
7879 pages: pages as u64,
7880 len: bytes.len() as u64,
7881 gen,
7882 };
7883 self.header.index_delta = ptr;
7884 let mut chain = self.index_chain.clone();
7885 chain.push(ptr);
7886 let n = self.index_delta_ops.as_ref().map_or(0, |o| o.len());
7887 new_ichain = Some((chain, self.index_chain_ops + n));
7888 }
7889 } else if persist_index {
7890 if v9 && self.header.index_pages > 0 {
7891 superseded.push((self.header.index_start, self.header.index_pages));
7892 }
7893 if v9 {
7894 for r in &self.index_chain {
7895 superseded.push((r.start, r.pages));
7896 }
7897 self.header.index_delta = RunPtr::default();
7898 new_ichain = Some((Vec::new(), 0));
7899 }
7900 match &ctx.idx_bytes {
7901 Some(bytes) => {
7902 let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
7903 self.header.index_start = start;
7904 self.header.index_pages = pages as u64;
7905 self.header.index_len = bytes.len() as u64;
7906 self.header.index_gen = gen;
7907 }
7908 None => {
7909 self.header.index_start = 0;
7910 self.header.index_pages = 0;
7911 self.header.index_len = 0;
7912 self.header.index_gen = 0;
7913 }
7914 }
7915 }
7916 // v8+ result-cache directory (Phase 10.1). Seal the serialized
7917 // directory into a fresh page run and update the pointer in the
7918 // header. The seal AAD already covers cache_start/pages/len for
7919 // v8+ (see `header_seal_aad`), so tampering with the pointer
7920 // trips the header seal on the next open.
7921 if self.dirty_cache {
7922 if v9 && self.header.cache_pages > 0 {
7923 superseded.push((self.header.cache_start, self.header.cache_pages));
7924 }
7925 match &ctx.cache_bytes {
7926 Some(bytes) if !self.cache_dir.is_empty() => {
7927 let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
7928 self.header.cache_start = start;
7929 self.header.cache_pages = pages as u64;
7930 self.header.cache_len = bytes.len() as u64;
7931 self.header.cache_gen = gen;
7932 }
7933 _ => {
7934 // Either the directory serialized to nothing (all
7935 // entries deleted after tombstone-only migration) or
7936 // it's empty — collapse the pointer to zero.
7937 self.header.cache_start = 0;
7938 self.header.cache_pages = 0;
7939 self.header.cache_len = 0;
7940 self.header.cache_gen = 0;
7941 }
7942 }
7943 }
7944
7945 // 3b. Record this transaction as a restorable snapshot. The manifest
7946 // update is staged in a local and adopted only once the commit
7947 // below succeeds, so a failed flush leaves no phantom entry.
7948 //
7949 // The cap (`self.max_snapshots`) is applied to the staged local
7950 // before the new entry lands, so after appending the manifest
7951 // holds at most `max_snapshots` entries — the most-recent N.
7952 // `max_snapshots == 0` disables the cap. On v8, pages
7953 // referenced only by pruned snapshots stay on disk until
7954 // `compact_file`; on v9 they become reusable once no retained
7955 // snapshot is older than the transaction that freed them.
7956 //
7957 // FIX-1 R-001.8 part 3: skip the snapshot push on a
7958 // cache-only flush. Snapshots don't capture the cache
7959 // anyway, and every push counts against `max_snapshots`
7960 // (default 256) + rewrites the manifest run — so a
7961 // cache-heavy workload would otherwise evict every real
7962 // memory restore point within ~256 cache writes. On a
7963 // cache-only flush we reuse the current manifest
7964 // verbatim; the manifest run already lives at
7965 // `self.header.manifest_{start,pages,len}` (set by the
7966 // last memory-touching flush, or at `create` time).
7967 let live = self.len() as u64;
7968 let mut manifest = self.manifest.clone();
7969 if is_memory_or_index_flush {
7970 if self.max_snapshots > 0 && !v9 {
7971 // Drop the oldest entries so there's room for the new one.
7972 while manifest.len() >= self.max_snapshots {
7973 manifest.remove(0);
7974 }
7975 }
7976 manifest.push(Snapshot {
7977 txn_id,
7978 created_at: memory::now_secs(),
7979 catalog_start: self.header.catalog_start,
7980 catalog_pages: self.header.catalog_pages,
7981 catalog_len: self.header.catalog_len,
7982 index_start: self.header.index_start,
7983 index_pages: self.header.index_pages,
7984 index_len: self.header.index_len,
7985 memory_count: live,
7986 gens: v9.then_some(SnapshotGens {
7987 catalog: self.header.catalog_gen,
7988 index: self.header.index_gen,
7989 delta: ptr_tuple(self.header.catalog_delta),
7990 index_delta: ptr_tuple(self.header.index_delta),
7991 bm25: ptr_tuple(self.header.bm25),
7992 }),
7993 });
7994 if v9 {
7995 let extra = self.pending_free.iter().map(|r| r.1).sum::<u64>()
7996 + superseded.iter().map(|r| r.1).sum::<u64>()
7997 + self.header.manifest_pages;
7998 self.apply_retention(&mut manifest, extra);
7999 }
8000 let man_bytes =
8001 rmp_serde::to_vec(&manifest).map_err(|e| MnemoError::Serialize(e.to_string()))?;
8002 if v9 && self.header.manifest_pages > 0 {
8003 superseded.push((self.header.manifest_start, self.header.manifest_pages));
8004 }
8005 let (m_start, m_pages) = self.seal_run(&man_bytes, gen, &mut frames)?;
8006 self.header.manifest_start = m_start;
8007 self.header.manifest_pages = m_pages as u64;
8008 self.header.manifest_len = man_bytes.len() as u64;
8009 self.header.manifest_gen = gen;
8010 }
8011
8012 // 3c. FMT-9 (b): the key-slot run, rewritten copy-on-write when a
8013 // slot changed. The header seal covers its pointer.
8014 if v9 && self.dirty_keys {
8015 if !self.header.keys.is_empty() {
8016 superseded.push((self.header.keys.start, self.header.keys.pages));
8017 }
8018 if self.key_slots.slots.is_empty() {
8019 self.header.keys = RunPtr::default();
8020 } else {
8021 let bytes = rmp_serde::to_vec(&self.key_slots)
8022 .map_err(|e| MnemoError::Serialize(e.to_string()))?;
8023 let (start, pages) = self.seal_run(&bytes, gen, &mut frames)?;
8024 self.header.keys = RunPtr {
8025 start,
8026 pages: pages as u64,
8027 len: bytes.len() as u64,
8028 gen,
8029 };
8030 }
8031 }
8032
8033 // 3d. FMT-9 (a): the free list. Everything this transaction
8034 // stopped referencing (`pending_free`, the runs replaced above,
8035 // and the previous free-list run itself) is freed *at this
8036 // transaction's id*, in this transaction's own commit, so it
8037 // can't be reused until this commit is durable and no retained
8038 // snapshot predates it. The list is staged in a local and
8039 // adopted only once the commit succeeds (a failed flush that
8040 // is retried must not free the same runs twice).
8041 let mut shrink = false;
8042 let staged_free =
8043 if v9 && (self.free_dirty || !self.pending_free.is_empty() || !superseded.is_empty()) {
8044 let mut staged = self.free.clone();
8045 for &(p, c) in self.pending_free.iter().chain(superseded.iter()) {
8046 staged.free(txn_id, p, c)?;
8047 }
8048 if !self.header.free.is_empty() {
8049 staged.free(txn_id, self.header.free.start, self.header.free.pages)?;
8050 }
8051 // FMT-9c: give eligible free runs at the file's tail back
8052 // to the filesystem: the header's `next_page` moves down in
8053 // this commit, and the file is cut after it.
8054 let min_snap = self.min_snapshot_txn();
8055 while let Some(p) =
8056 staged.take_tail(self.header.next_page, self.header.wal_seq, min_snap)
8057 {
8058 self.header.next_page = p;
8059 shrink = true;
8060 }
8061 // Size the run with room for the one extra entry that
8062 // allocating it out of the list can create (a split), then
8063 // allocate it from the list (eligible runs only) or the tail.
8064 let est = rmp_serde::to_vec(&staged.to_runs())
8065 .map_err(|e| MnemoError::Serialize(e.to_string()))?
8066 .len()
8067 + 64;
8068 let pc = est.div_ceil(PAYLOAD).max(1) as u64;
8069 let start = match staged.alloc(pc, self.header.wal_seq, self.min_snapshot_txn()) {
8070 Some(p) => p,
8071 None => {
8072 let p = self.header.next_page;
8073 self.header.next_page += pc;
8074 p
8075 }
8076 };
8077 let bytes = rmp_serde::to_vec(&staged.to_runs())
8078 .map_err(|e| MnemoError::Serialize(e.to_string()))?;
8079 if bytes.len() as u64 > pc * PAYLOAD as u64 {
8080 return Err(MnemoError::Invalid(
8081 "free list outgrew its reserved run".into(),
8082 ));
8083 }
8084 self.seal_run_at(start, pc, &bytes, gen, &mut frames)?;
8085 self.header.free = RunPtr {
8086 start,
8087 pages: pc,
8088 len: bytes.len() as u64,
8089 gen,
8090 };
8091 Some(staged)
8092 } else {
8093 None
8094 };
8095
8096 // 3e. The header is the transaction's final frame; stamp the new id
8097 // and the *actual* (post-flush) write counter, which is <= the
8098 // leased value persisted in `prepare_for_flush`. This is the
8099 // value that ultimately replaces the leased header on checkpoint.
8100 self.header.vector_count = live;
8101 self.header.write_counter = self.pager.write_counter;
8102 self.header.wal_seq = txn_id;
8103 let mut hpage = self.header.to_page();
8104 self.header.apply_seal(&mut hpage, self.pager.dek())?;
8105 frames.push((0, hpage.to_vec()));
8106
8107 // 4. COMMIT — log the transaction and fsync. Nothing at a home page
8108 // other than the leased header has changed yet; this single
8109 // fsync is what makes the rest durable.
8110 // FMT-9c: big runs went straight to their home pages; make them
8111 // durable before the header that references them can be.
8112 if self.direct_writes {
8113 self.pager.sync()?;
8114 self.direct_writes = false;
8115 }
8116 let (wal_start, wal_pages) = (self.header.wal_start, self.header.wal_pages);
8117 wal::commit(self.pager.file_mut(), wal_start, wal_pages, txn_id, &frames)?;
8118
8119 // 5. Checkpoint — fold the WAL into the home pages.
8120 self.checkpoint(&frames)?;
8121 // FMT-9c: the committed header no longer covers the tail; cut it.
8122 // A crash before this leaves a longer file, which is harmless.
8123 if shrink {
8124 let len = self.header.next_page * PAGE_SIZE as u64;
8125 self.pager.file_mut().set_len(len)?;
8126 }
8127
8128 self.manifest = manifest;
8129 if let Some(staged) = staged_free {
8130 self.free = staged;
8131 }
8132 self.pending_free.clear();
8133 self.free_dirty = false;
8134 self.dirty_keys = false;
8135 if let Some((chain, rows)) = new_chain {
8136 self.delta_chain = chain;
8137 self.delta_rows = rows;
8138 }
8139 if let Some((chain, n)) = new_ichain {
8140 self.index_chain = chain;
8141 self.index_chain_ops = n;
8142 }
8143 if let Some((segs, seg_docs)) = new_bm25 {
8144 // FMT-9f: keep the loaded dicts and read only the new segment's
8145 // (a merge replaced the segments from `kept` on), instead of
8146 // dropping them all and reloading every dict on next use.
8147 let kept = self
8148 .bm25
8149 .manifest
8150 .segments
8151 .iter()
8152 .zip(&segs)
8153 .take_while(|(a, b)| a == b)
8154 .count();
8155 if let Some(mut dicts) = self.bm25.dicts.take() {
8156 dicts.truncate(kept);
8157 let mut ok = dicts.len() == kept;
8158 for &t in &segs[kept..] {
8159 match load_segment_dict(&mut self.pager, tuple_ptr(t)) {
8160 Ok(d) if ok => dicts.push(d),
8161 _ => ok = false,
8162 }
8163 }
8164 if ok {
8165 self.bm25.dicts = Some(dicts);
8166 }
8167 }
8168 self.bm25.manifest.segments = segs;
8169 self.bm25.manifest.seg_docs = seg_docs;
8170 self.bm25.pending.clear();
8171 }
8172 if persist_index {
8173 self.index_rebase = false;
8174 self.index_delta_ops = None;
8175 }
8176 if ctx.cat_bytes.is_some() {
8177 self.dirty_rows.clear();
8178 self.catalog_rewrite = false;
8179 }
8180 self.dirty_catalog = false;
8181 if persist_index {
8182 self.dirty_index = false;
8183 self.index_persist = IndexPersist::default();
8184 } else if self.dirty_index {
8185 self.index_persist.commits += 1;
8186 }
8187 self.dirty_cache = false;
8188 // Batched policy: successful flush consumes the pending batch.
8189 self.cache_batch.reset();
8190 // Record how many fsyncs this flush issued, for Stats reporting.
8191 // Thread-local counter guarantees we see only this handle's own
8192 // sync_all calls between the snapshot above and this read.
8193 self.fsyncs_last_flush =
8194 (crate::fsync_count().wrapping_sub(fsync_before)).min(u32::MAX as u64) as u32;
8195 Ok(())
8196 }
8197
8198 /// Pre-flush prelude shared by [`Mnemo::flush`] and the
8199 /// `__crash_partial_flush_for_testing` hook.
8200 ///
8201 /// Serializes the control plane (catalog, ANN index, cache directory)
8202 /// and grows the WAL if needed. That's it: this used to persist a
8203 /// leased header too (SEC-1 obsoleted it; see DUR-1 R-002.1 note in
8204 /// the body for the full rationale), but after DUR-1 the only page-0
8205 /// writes outside `create` go through the WAL via
8206 /// [`Mnemo::relocate_header_through_wal`].
8207 ///
8208 /// The historical pre-stamping used to close the nonce-reuse window
8209 /// flagged in Phase 1.1 of the improvement plan: a crash anywhere
8210 /// between here and the WAL commit left a file whose on-disk header
8211 /// recorded a `write_counter` past every nonce the rest of this
8212 /// transaction could produce, so the next reopen started from a
8213 /// counter that guaranteed uniqueness against any orphan data pages
8214 /// still on disk. SEC-1's per-session random nonce prefix gives the
8215 /// same guarantee without needing a persisted counter, which is why
8216 /// this routine no longer writes page 0.
8217 fn prepare_for_flush(&mut self, persist_index: bool) -> Result<FlushPrelude> {
8218 // Serialize.
8219 // FMT-9c.2: on v9 usually just the changed rows, as a delta run.
8220 let cat_delta = self.dirty_catalog && self.catalog_as_delta();
8221 let cat_bytes = if !self.dirty_catalog {
8222 None
8223 } else if cat_delta {
8224 let rows: Vec<(u32, CatalogEntry)> = self
8225 .dirty_rows
8226 .iter()
8227 .filter(|&&i| i < self.catalog.len())
8228 .map(|&i| (i as u32, self.catalog[i].clone()))
8229 .collect();
8230 let d = CatalogDelta {
8231 prev: ptr_tuple(self.header.catalog_delta),
8232 rows,
8233 };
8234 Some(rmp_serde::to_vec(&d).map_err(|e| MnemoError::Serialize(e.to_string()))?)
8235 } else {
8236 if self.is_v9() {
8237 self.compact_tombstones();
8238 }
8239 Some(
8240 rmp_serde::to_vec(&self.catalog)
8241 .map_err(|e| MnemoError::Serialize(e.to_string()))?,
8242 )
8243 };
8244 // FMT-9c.3: a v9 commit usually writes just its index changes.
8245 let idx_delta = persist_index && self.index_delta_ops.is_some();
8246 let idx_bytes: Option<Vec<u8>> = if !persist_index {
8247 None
8248 } else if let Some(ops) = &self.index_delta_ops {
8249 let d = IndexDeltaRef {
8250 prev: ptr_tuple(self.header.index_delta),
8251 ops,
8252 };
8253 Some(rmp_serde::to_vec(&d).map_err(|e| MnemoError::Serialize(e.to_string()))?)
8254 } else {
8255 match &self.ann {
8256 Some(ann) => {
8257 let mut b =
8258 rmp_serde::to_vec(ann).map_err(|e| MnemoError::Serialize(e.to_string()))?;
8259 // FMT-9: v9 index runs carry a layout byte first.
8260 if self.is_v9() {
8261 b.insert(0, self.index_layout_override.unwrap_or(V9_INDEX_LAYOUT));
8262 }
8263 Some(b)
8264 }
8265 None => None,
8266 }
8267 };
8268 // Serialize the result-cache directory when it's dirty. An
8269 // empty directory serializes to a small header + 0-element
8270 // array; still cheap enough that we don't special-case it.
8271 let cache_bytes: Option<Vec<u8>> = if self.dirty_cache {
8272 Some(
8273 rmp_serde::to_vec(&self.cache_dir)
8274 .map_err(|e| MnemoError::Serialize(e.to_string()))?,
8275 )
8276 } else {
8277 None
8278 };
8279
8280 let cat_pc = cat_bytes
8281 .as_ref()
8282 .map_or(0, |b| b.len().div_ceil(PAYLOAD).max(1));
8283 let idx_pc = idx_bytes
8284 .as_ref()
8285 .map_or(0, |b| b.len().div_ceil(PAYLOAD).max(1));
8286 let cache_pc = cache_bytes
8287 .as_ref()
8288 .map_or(0, |b| b.len().div_ceil(PAYLOAD).max(1));
8289 // Upper bound on the manifest run: one extra entry, <=82 bytes each
8290 // (rmp_serde encodes a Snapshot as a 9-element fixarray of i64/u64,
8291 // worst case 81 bytes plus a 1-byte array header).
8292 // v9 snapshots carry two more u64s (≤ 20 bytes with the array header).
8293 let per_snapshot = if self.is_v9() { 102 } else { 82 };
8294 let man_upper = 9 + (self.manifest.len() + 1) * per_snapshot;
8295 let man_pc_est = man_upper.div_ceil(PAYLOAD).max(1);
8296 // FMT-9: the free-list run (≤ 28 bytes per run: every pending and
8297 // superseded run plus a split and a relocated WAL region) and the
8298 // key-slot run.
8299 let (free_pc_est, keys_pc) = if self.is_v9() {
8300 let runs = self.free.len() + self.pending_free.len() + 12;
8301 let free_pc = (runs * 28 + 64).div_ceil(PAYLOAD).max(1);
8302 let keys_pc = if self.dirty_keys {
8303 rmp_serde::to_vec(&self.key_slots)
8304 .map_err(|e| MnemoError::Serialize(e.to_string()))?
8305 .len()
8306 .div_ceil(PAYLOAD)
8307 .max(1)
8308 } else {
8309 0
8310 };
8311 (free_pc, keys_pc)
8312 } else {
8313 (0, 0)
8314 };
8315
8316 // Grow WAL if needed (DUR-1: relocation is WAL-logged and does
8317 // its own fsyncs). The cache directory rides in the same
8318 // transaction so its page count contributes to the reservation.
8319 // FMT-9d: the BM25 segment (and its manifest, one page).
8320 let bm25_seg = self.bm25_next_segment()?;
8321 let bm25_pc = bm25_seg
8322 .as_ref()
8323 .map_or(0, |(b, _)| b.len().div_ceil(PAYLOAD).max(1));
8324 // FMT-9c: on v9 only runs of up to V9_WAL_RUN_MAX_PAGES go through
8325 // the WAL; longer ones are written in place.
8326 let v9 = self.is_v9();
8327 let in_wal = |pc: usize| {
8328 if v9 && pc as u64 > V9_WAL_RUN_MAX_PAGES {
8329 0
8330 } else {
8331 pc
8332 }
8333 };
8334 self.ensure_wal_capacity(
8335 in_wal(cat_pc)
8336 + in_wal(idx_pc)
8337 + in_wal(cache_pc)
8338 + in_wal(man_pc_est)
8339 + in_wal(free_pc_est)
8340 + in_wal(keys_pc)
8341 + in_wal(bm25_pc)
8342 + usize::from(bm25_pc > 0)
8343 + 1,
8344 )?;
8345
8346 // DUR-1 (R-002.1): the pre-flush "leased header" write that used
8347 // to live here is gone. It existed to close the Phase 1.1 nonce
8348 // reuse window (reopen must see a `write_counter` past any nonce
8349 // this flush could produce before data pages hit disk). SEC-1
8350 // superseded that: nonces now carry a per-session random prefix
8351 // and `write_counter` is no longer a nonce input, so the leased
8352 // write bought us nothing but an extra fsync per flush. Orphan
8353 // data pages past `header.next_page` on a crashed flush are
8354 // still safe — the next session's random prefix makes any later
8355 // write at the same page_no uniquely nonced regardless of
8356 // `write_counter`.
8357
8358 Ok(FlushPrelude {
8359 bm25_seg,
8360 cat_bytes,
8361 cat_delta,
8362 idx_delta,
8363 idx_bytes,
8364 cache_bytes,
8365 })
8366 }
8367
8368 /// Flush and close. Equivalent to `flush()` on a read-write handle; on
8369 /// a read-only handle it's a no-op (no dirty state to persist). The
8370 /// file is released on drop, which also drops the OS advisory lock.
8371 /// **Caution:** the handle keeps the file and its lock until it is
8372 /// dropped, so `drop` it before reopening the same file (or the open
8373 /// fails with [`MnemoError::Locked`]).
8374 pub fn close(&mut self) -> Result<()> {
8375 if self.read_only {
8376 return Ok(());
8377 }
8378 // PERF-1: closing persists a dirty ANN index too, so the next open
8379 // has nothing to re-assign.
8380 self.index_persist.forced = true;
8381 self.flush()
8382 }
8383
8384 /// PERF-1: flush, persisting the ANN index even if the
8385 /// [`INDEX_PERSIST_EVERY_COMMITS`] / [`INDEX_PERSIST_EVERY_CHANGES`]
8386 /// policy wouldn't yet. Plain [`Mnemo::flush`] writes only the catalog
8387 /// between persists; the index is derived, so a crash costs nothing but
8388 /// re-assigning the newer records on the next load.
8389 pub fn flush_index(&mut self) -> Result<()> {
8390 if self.read_only {
8391 return Err(MnemoError::ReadOnly);
8392 }
8393 self.index_persist.forced = true;
8394 self.flush()
8395 }
8396
8397 /// **TEST ONLY — do not use.** Reproduces a crash inside
8398 /// [`Mnemo::flush`] *after* the prelude has leased counter+page slots
8399 /// and persisted the leased header, but *before* the WAL is committed.
8400 /// Runs `prepare_for_flush` + `pager.flush` and returns — leaving
8401 /// data pages and a leased header on disk, with no commit frame.
8402 ///
8403 /// Used by `tests/integration.rs::nonce_unique_after_crashed_data_flush`
8404 /// to verify that this window (Phase 1.1 of the improvement plan) does
8405 /// not enable AES-GCM nonce reuse. Calling this in production code
8406 /// strands data pages; never expose it from a binding.
8407 #[doc(hidden)]
8408 pub fn __crash_partial_flush_for_testing(&mut self) -> Result<()> {
8409 if self.read_only {
8410 return Err(MnemoError::ReadOnly);
8411 }
8412 if !self.dirty_catalog && !self.dirty_index {
8413 return Ok(());
8414 }
8415 // FMT-9b: v9 records and vectors reach their pages at the top of a
8416 // flush; do that part too.
8417 self.ensure_gen_lease()?;
8418 self.pack_records()?;
8419 self.pack_vectors()?;
8420 let _prelude = self.prepare_for_flush(self.dirty_index)?;
8421 self.pager.flush()
8422 }
8423
8424 /// Bound the in-memory page cache to `pages` decrypted pages.
8425 ///
8426 /// The cache holds decrypted page payloads to speed repeated reads. By
8427 /// default it is capped at 1024 pages (~8 MiB; 8192 before v0.10.0).
8428 /// Lower the cap to trade hit rate for a smaller footprint, or raise
8429 /// it for a hotter cache. The
8430 /// cap governs *clean* pages — pages with un-flushed writes are always
8431 /// retained until [`Mnemo::flush`], regardless of the cap.
8432 pub fn set_cache_capacity(&mut self, pages: usize) {
8433 self.pager.set_cache_capacity(pages);
8434 }
8435
8436 /// FMT-9b (v9 files): keep the fp16 vector column in memory while it
8437 /// takes at most `bytes` (default [`DEFAULT_VECTOR_CACHE_BUDGET`],
8438 /// 96 MiB, about 125K vectors at 384 dims). Above it, recall reads
8439 /// vector entries through the page cache instead: less memory, slower
8440 /// recall. Lowering it below the resident size drops the resident copy.
8441 pub fn set_vector_cache_budget(&mut self, bytes: usize) {
8442 self.vector_cache_budget = bytes;
8443 let need = self.catalog.len() * veccol::entry_len(self.dimensions);
8444 if self.col.resident && need > bytes {
8445 self.col.data = Vec::new();
8446 self.col.norms = Vec::new();
8447 self.col.have = Vec::new();
8448 self.col.resident = false;
8449 }
8450 }
8451
8452 /// Page-cache occupancy: `(pages_cached, capacity)`. Renamed from
8453 /// `cache_stats` in v0.4.0: the result-cache API (Phase 10.1) took
8454 /// that name for its per-namespace [`CacheStats`] summary, and the
8455 /// page-cache pair is the older/less-user-facing of the two, so it
8456 /// gets the more-specific name.
8457 pub fn page_cache_stats(&self) -> (usize, usize) {
8458 self.pager.cache_stats()
8459 }
8460
8461 // === Result cache (Phase 10.1 + 10.3) ============================
8462
8463 /// Insert or overwrite a cache entry at `(namespace, key)`.
8464 ///
8465 /// The engine SHA-256-hashes `key` — callers pass the raw input
8466 /// string. Persists to disk on the next `flush()` under the current
8467 /// flush policy (see [`Mnemo::set_cache_flush_policy`]); in
8468 /// [`CacheFlushPolicy::Batched`] the engine may call `flush()`
8469 /// internally when a threshold trips.
8470 ///
8471 /// On overflow of the namespace's [`CacheBudget`] (default 10,000
8472 /// entries / 64 MiB) the LRU entry is tombstoned before the new
8473 /// insert. Tombstones are reclaimed by
8474 /// [`Mnemo::compact_file`], not the runtime flush.
8475 ///
8476 /// Returns [`MnemoError::ReadOnly`] on a read-only handle.
8477 pub fn cache_put(
8478 &mut self,
8479 namespace: &str,
8480 key: &str,
8481 value: &[u8],
8482 opts: CachePutOpts,
8483 ) -> Result<()> {
8484 if self.read_only {
8485 return Err(MnemoError::ReadOnly);
8486 }
8487 let key_hash = result_cache::hash_key(key);
8488 let key_preview = result_cache::key_preview(key);
8489 let now = memory::now_secs();
8490 let size_bytes = value.len() as u64;
8491
8492 let entry = CacheEntry {
8493 namespace: namespace.to_string(),
8494 key_hash,
8495 key_preview: key_preview.clone(),
8496 value: value.to_vec(),
8497 content_type: opts.content_type.clone(),
8498 created_at: now,
8499 accessed_at: now,
8500 access_count: 0,
8501 ttl_secs: opts.ttl_secs,
8502 cost_hint_ms: opts.cost_hint_ms,
8503 size_bytes,
8504 vector: None,
8505 model: None,
8506 };
8507 let bytes = rmp_serde::to_vec(&entry).map_err(|e| MnemoError::Serialize(e.to_string()))?;
8508 let (start_page, page_count, gen) = self.write_record(&bytes)?;
8509
8510 // Tombstone any prior entry for this key. Its pages remain on
8511 // disk until `compact_file`, exactly like memory-record
8512 // overwrites — the tombstone just hides it from lookups.
8513 if let Some(prev_pos) = self.cache_idx.get(namespace, &key_hash) {
8514 self.cache_dir[prev_pos].deleted = true;
8515 self.release_cache_pages(prev_pos);
8516 self.cache_idx.remove(namespace, &key_hash);
8517 }
8518
8519 let dir_entry = CacheDirectoryEntry {
8520 namespace: namespace.to_string(),
8521 key_hash,
8522 start_page,
8523 page_count,
8524 len: bytes.len() as u32,
8525 deleted: false,
8526 accessed_at: now,
8527 access_count: 0,
8528 size_bytes,
8529 ttl_secs: opts.ttl_secs,
8530 created_at: now,
8531 vector: None,
8532 model: None,
8533 gen,
8534 };
8535 let pos = self.cache_dir.len();
8536 self.cache_dir.push(dir_entry);
8537 self.cache_idx.insert(namespace, key_hash, pos);
8538
8539 // Enforce budget — evict LRU tombstones until the namespace
8540 // fits under both caps.
8541 self.evict_to_budget(namespace)?;
8542
8543 self.dirty_cache = true;
8544 self.cache_batch.record_dirty();
8545
8546 // Batched policy: auto-flush if a threshold tripped. Successful
8547 // `flush()` calls `cache_batch.reset()` for us.
8548 if self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
8549 self.flush()?;
8550 }
8551 Ok(())
8552 }
8553
8554 /// Look up `(namespace, key)`. Returns `None` on miss (either the
8555 /// key was never inserted or the entry is TTL-expired). On hit
8556 /// updates catalog-side access stats (mirrors the recall v5-trick);
8557 /// on a read-only handle the stats update is silently skipped.
8558 ///
8559 /// A miss is recorded as a miss in this handle's counters even if
8560 /// the key was TTL-expired — from the caller's perspective the two
8561 /// are indistinguishable.
8562 pub fn cache_get(&mut self, namespace: &str, key: &str) -> Result<Option<CachedValue>> {
8563 // FIX-1 R-001.8 part 4: under `Batched`, honor `max_age` on
8564 // reads too. Pre-FIX-1 the age check only fired on `cache_put`,
8565 // so a workload that wrote a few entries and then switched to
8566 // pure reads could sit with its pending batch past the
8567 // nominal max_age indefinitely. `max_dirty` is already
8568 // maintained per-write; this adds the symmetric time-bound
8569 // check on the read path. Only runs on read-write handles —
8570 // flush() returns ReadOnly for shared-lock readers.
8571 if !self.read_only && self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
8572 self.flush()?;
8573 }
8574 let key_hash = result_cache::hash_key(key);
8575 let pos = match self.cache_idx.get(namespace, &key_hash) {
8576 Some(p) => p,
8577 None => {
8578 self.cache_counters.misses += 1;
8579 return Ok(None);
8580 }
8581 };
8582 let now = memory::now_secs();
8583 // Snapshot the directory entry fields we need before any
8584 // mutation — the borrow checker gets grumpy otherwise.
8585 let (start_page, page_count, len, ttl_secs, expired, created_at, gen) = {
8586 let e = &self.cache_dir[pos];
8587 (
8588 e.start_page,
8589 e.page_count,
8590 e.len,
8591 e.ttl_secs,
8592 e.is_expired(now),
8593 e.created_at,
8594 e.gen,
8595 )
8596 };
8597 if expired {
8598 // Treat expired as a miss for the caller. Don't tombstone
8599 // here — cache_purge does that. Just skip.
8600 self.cache_counters.misses += 1;
8601 return Ok(None);
8602 }
8603 // Decrypt and decode the record body.
8604 let buf = self.read_record_run(start_page, page_count, len, gen)?;
8605 let entry: CacheEntry =
8606 rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
8607
8608 // Bump access stats catalog-only (v5 trick — no full record
8609 // rewrite on a hit). Skipped on read-only handles so a shared-lock
8610 // reader can't dirty the catalog.
8611 if !self.read_only {
8612 self.cache_dir[pos].accessed_at = now;
8613 self.cache_dir[pos].access_count = self.cache_dir[pos].access_count.saturating_add(1);
8614 self.dirty_cache = true;
8615 }
8616 self.cache_counters.hits += 1;
8617 // On a mutating handle: access_count reflects the just-bumped
8618 // value. On a read-only handle: the on-disk directory value is
8619 // unchanged, so returning it verbatim is honest (the caller's
8620 // reads did not, and cannot, alter the persisted counter).
8621 //
8622 // FIX-1 R-002.2 part 1: `accessed_at` must also come from the
8623 // directory on a read-only handle, not from `entry.accessed_at`
8624 // in the record body. The record body is written once at insert
8625 // time and never updated on a hit — every live `accessed_at`
8626 // update lands in the directory (see the `!self.read_only`
8627 // branch above). The pre-FIX-1 code returned
8628 // `entry.accessed_at`, which is really the entry's **creation
8629 // time** by then, so a read-only reader saw "last accessed =
8630 // when it was inserted" even after many successful hits by
8631 // concurrent writers.
8632 let access_count = self.cache_dir[pos].access_count;
8633 let accessed_at = if self.read_only {
8634 self.cache_dir[pos].accessed_at
8635 } else {
8636 now
8637 };
8638 Ok(Some(CachedValue {
8639 value: entry.value,
8640 content_type: entry.content_type,
8641 created_at,
8642 accessed_at,
8643 access_count,
8644 ttl_secs,
8645 }))
8646 }
8647
8648 /// Tombstone the entry at `(namespace, key)`. Silently succeeds if
8649 /// no such entry exists. Space is reclaimed by `compact_file`.
8650 pub fn cache_delete(&mut self, namespace: &str, key: &str) -> Result<()> {
8651 if self.read_only {
8652 return Err(MnemoError::ReadOnly);
8653 }
8654 let key_hash = result_cache::hash_key(key);
8655 if let Some(pos) = self.cache_idx.get(namespace, &key_hash) {
8656 self.cache_dir[pos].deleted = true;
8657 self.release_cache_pages(pos);
8658 self.cache_idx.remove(namespace, &key_hash);
8659 self.dirty_cache = true;
8660 self.cache_batch.record_dirty();
8661 if self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
8662 self.flush()?;
8663 }
8664 }
8665 Ok(())
8666 }
8667
8668 /// Purge cache entries. If `namespace` is `Some`, restrict to that
8669 /// namespace; otherwise purge across the whole cache. If
8670 /// `expired_only` is `true`, only TTL-expired entries are removed;
8671 /// otherwise every live entry in the target scope is tombstoned.
8672 /// Returns the number of entries newly tombstoned.
8673 pub fn cache_purge(&mut self, namespace: Option<&str>, expired_only: bool) -> Result<usize> {
8674 if self.read_only {
8675 return Err(MnemoError::ReadOnly);
8676 }
8677 let now = memory::now_secs();
8678 let mut purged = 0usize;
8679 // Iterate the directory by index; mark matching entries as
8680 // deleted and pull them out of the in-memory index.
8681 for pos in 0..self.cache_dir.len() {
8682 if self.cache_dir[pos].deleted {
8683 continue;
8684 }
8685 if let Some(ns) = namespace {
8686 if self.cache_dir[pos].namespace != ns {
8687 continue;
8688 }
8689 }
8690 if expired_only && !self.cache_dir[pos].is_expired(now) {
8691 continue;
8692 }
8693 let key_hash = self.cache_dir[pos].key_hash;
8694 let ns = self.cache_dir[pos].namespace.clone();
8695 self.cache_dir[pos].deleted = true;
8696 self.release_cache_pages(pos);
8697 self.cache_idx.remove(&ns, &key_hash);
8698 purged += 1;
8699 }
8700 if purged > 0 {
8701 self.dirty_cache = true;
8702 self.cache_batch.record_dirty();
8703 if self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
8704 self.flush()?;
8705 }
8706 }
8707 Ok(purged)
8708 }
8709
8710 /// Statistics for `namespace` (or the whole cache if `None`).
8711 /// `hits`/`misses`/`evictions` are per-handle in-memory counters
8712 /// that reset on reopen; `entries`/`bytes` reflect the durable
8713 /// on-disk state (live entries only, tombstones excluded).
8714 pub fn cache_stats(&self, namespace: Option<&str>) -> CacheStats {
8715 let mut entries = 0usize;
8716 let mut bytes = 0u64;
8717 for e in &self.cache_dir {
8718 if e.deleted {
8719 continue;
8720 }
8721 if let Some(ns) = namespace {
8722 if e.namespace != ns {
8723 continue;
8724 }
8725 }
8726 entries += 1;
8727 bytes += e.size_bytes;
8728 }
8729 let hits = self.cache_counters.hits;
8730 let misses = self.cache_counters.misses;
8731 let total = hits + misses;
8732 let hit_rate = if total > 0 {
8733 hits as f64 / total as f64
8734 } else {
8735 0.0
8736 };
8737 CacheStats {
8738 entries,
8739 bytes,
8740 hits,
8741 misses,
8742 hit_rate,
8743 evictions: self.cache_counters.evictions,
8744 }
8745 }
8746
8747 /// Set (or update) the eviction budget for `namespace`. Applies to
8748 /// subsequent `cache_put` calls; does not retroactively evict
8749 /// entries that already sit under a looser cap.
8750 pub fn set_cache_budget(&mut self, namespace: &str, budget: CacheBudget) {
8751 self.cache_budgets.insert(namespace.to_string(), budget);
8752 }
8753
8754 /// Set the per-handle cache flush policy. Defaults to
8755 /// [`CacheFlushPolicy::Strict`]. Memory writes are never batched;
8756 /// this affects only cache mutations.
8757 pub fn set_cache_flush_policy(&mut self, policy: CacheFlushPolicy) {
8758 self.cache_flush_policy = policy;
8759 // Switching mid-run: if we're moving to Strict and the batch is
8760 // non-empty, the pending count still counts toward dirty_cache
8761 // which the next flush() will consume.
8762 }
8763
8764 /// FMT-9 (a): free a just-tombstoned cache entry's record pages at the
8765 /// next commit (v9; no-op on v8).
8766 fn release_cache_pages(&mut self, pos: usize) {
8767 let (p, c) = (
8768 self.cache_dir[pos].start_page,
8769 self.cache_dir[pos].page_count as u64,
8770 );
8771 self.release_pages(p, c);
8772 }
8773
8774 /// Evict LRU live entries in `namespace` until it fits under both
8775 /// budget caps. Called from `cache_put` after the insert; the new
8776 /// entry is included in the count so we always leave room.
8777 fn evict_to_budget(&mut self, namespace: &str) -> Result<()> {
8778 let budget = self
8779 .cache_budgets
8780 .get(namespace)
8781 .copied()
8782 .unwrap_or_default();
8783 loop {
8784 // Sum current live count + bytes in the namespace.
8785 let mut live_count = 0usize;
8786 let mut live_bytes = 0u64;
8787 for e in &self.cache_dir {
8788 if !e.deleted && e.namespace == namespace {
8789 live_count += 1;
8790 live_bytes += e.size_bytes;
8791 }
8792 }
8793 if live_count <= budget.max_entries && live_bytes <= budget.max_bytes {
8794 return Ok(());
8795 }
8796 // Find the LRU (oldest accessed_at) live entry.
8797 let victim_pos = self
8798 .cache_dir
8799 .iter()
8800 .enumerate()
8801 .filter(|(_, e)| !e.deleted && e.namespace == namespace)
8802 .min_by_key(|(_, e)| e.accessed_at)
8803 .map(|(i, _)| i);
8804 let Some(pos) = victim_pos else {
8805 // Nothing to evict but still over budget — the single
8806 // just-inserted entry itself is bigger than
8807 // max_bytes. Nothing we can safely do here without
8808 // rejecting the put; leave it and return.
8809 return Ok(());
8810 };
8811 let key_hash = self.cache_dir[pos].key_hash;
8812 let ns = self.cache_dir[pos].namespace.clone();
8813 self.cache_dir[pos].deleted = true;
8814 self.release_cache_pages(pos);
8815 self.cache_idx.remove(&ns, &key_hash);
8816 self.cache_counters.evictions += 1;
8817 }
8818 }
8819
8820 // === Semantic cache (Phase 10.2) ==================================
8821
8822 /// Insert or overwrite a **semantic** cache entry — same as
8823 /// [`Mnemo::cache_put`] but with a required embedding vector and
8824 /// model identifier. The vector is stored on the directory entry
8825 /// so [`Mnemo::cache_get_semantic`] can score without decrypting
8826 /// record bodies.
8827 ///
8828 /// Vector dimensionality is validated against the database's
8829 /// configured `dimensions` — a mismatch returns
8830 /// [`MnemoError::DimensionMismatch`], the same signal `remember`
8831 /// gives for the memory catalog.
8832 ///
8833 /// The `model` string is matched **exactly** on lookup — a
8834 /// different model's cache is invisible to a given query. Pick
8835 /// canonical identifiers (`"text-embedding-3-small"`,
8836 /// `"bge-large-en-v1.5"`, ...) and stick to them.
8837 pub fn cache_put_semantic(
8838 &mut self,
8839 namespace: &str,
8840 key: &str,
8841 vector: Vec<f32>,
8842 value: &[u8],
8843 opts: SemanticCachePutOpts,
8844 ) -> Result<()> {
8845 if self.read_only {
8846 return Err(MnemoError::ReadOnly);
8847 }
8848 if vector.len() != self.dimensions {
8849 return Err(MnemoError::DimensionMismatch {
8850 expected: self.dimensions,
8851 got: vector.len(),
8852 });
8853 }
8854 let key_hash = result_cache::hash_key(key);
8855 let key_preview = result_cache::key_preview(key);
8856 let now = memory::now_secs();
8857 let size_bytes = value.len() as u64;
8858
8859 let entry = CacheEntry {
8860 namespace: namespace.to_string(),
8861 key_hash,
8862 key_preview: key_preview.clone(),
8863 value: value.to_vec(),
8864 content_type: opts.content_type.clone(),
8865 created_at: now,
8866 accessed_at: now,
8867 access_count: 0,
8868 ttl_secs: opts.ttl_secs,
8869 cost_hint_ms: opts.cost_hint_ms,
8870 size_bytes,
8871 vector: Some(vector.clone()),
8872 model: Some(opts.model.clone()),
8873 };
8874 let bytes = rmp_serde::to_vec(&entry).map_err(|e| MnemoError::Serialize(e.to_string()))?;
8875 let (start_page, page_count, gen) = self.write_record(&bytes)?;
8876
8877 if let Some(prev_pos) = self.cache_idx.get(namespace, &key_hash) {
8878 self.cache_dir[prev_pos].deleted = true;
8879 self.release_cache_pages(prev_pos);
8880 self.cache_idx.remove(namespace, &key_hash);
8881 }
8882
8883 let dir_entry = CacheDirectoryEntry {
8884 namespace: namespace.to_string(),
8885 key_hash,
8886 start_page,
8887 page_count,
8888 len: bytes.len() as u32,
8889 deleted: false,
8890 accessed_at: now,
8891 access_count: 0,
8892 size_bytes,
8893 ttl_secs: opts.ttl_secs,
8894 created_at: now,
8895 vector: Some(vector),
8896 model: Some(opts.model),
8897 gen,
8898 };
8899 let pos = self.cache_dir.len();
8900 self.cache_dir.push(dir_entry);
8901 self.cache_idx.insert(namespace, key_hash, pos);
8902
8903 self.evict_to_budget(namespace)?;
8904
8905 self.dirty_cache = true;
8906 self.cache_batch.record_dirty();
8907 if self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
8908 self.flush()?;
8909 }
8910 Ok(())
8911 }
8912
8913 /// Semantic recall over the cache: score every live entry in
8914 /// `namespace` whose `model` matches against `query_vector` using
8915 /// cosine similarity; return the top-1 hit and its similarity iff
8916 /// `sim >= threshold`. `None` otherwise.
8917 ///
8918 /// Uses [`crate::result_cache::DEFAULT_SEMANTIC_THRESHOLD`] if you
8919 /// want a sensible default — pass it explicitly (`0.97` today).
8920 ///
8921 /// Exact-key entries (those without a vector) are transparently
8922 /// skipped, as are entries whose model doesn't match — a
8923 /// different-model hit would be a semantic bug, not a win.
8924 /// TTL-expired entries are also skipped.
8925 ///
8926 /// Access-stat updates on hit follow the same policy as
8927 /// [`Mnemo::cache_get`]: bumped on read-write handles, silently
8928 /// skipped on read-only. Hits and misses are recorded in the
8929 /// per-handle counters just like exact-key lookups.
8930 pub fn cache_get_semantic(
8931 &mut self,
8932 namespace: &str,
8933 query_vector: &[f32],
8934 threshold: f32,
8935 model: &str,
8936 ) -> Result<Option<(CachedValue, f32)>> {
8937 if query_vector.len() != self.dimensions {
8938 return Err(MnemoError::DimensionMismatch {
8939 expected: self.dimensions,
8940 got: query_vector.len(),
8941 });
8942 }
8943 // FIX-1 R-001.8 part 4: same Batched max_age check as
8944 // `cache_get`. See the comment there.
8945 if !self.read_only && self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
8946 self.flush()?;
8947 }
8948 let now = memory::now_secs();
8949
8950 // Scan namespace live entries, filter by model, compute
8951 // cosine, keep the best. Exact-key entries have `vector: None`
8952 // and are skipped by the filter.
8953 let mut best: Option<(usize, f32)> = None;
8954 for (i, e) in self.cache_dir.iter().enumerate() {
8955 if e.deleted || e.namespace != namespace || e.is_expired(now) {
8956 continue;
8957 }
8958 let Some(ref v) = e.vector else { continue };
8959 let Some(ref m) = e.model else { continue };
8960 if m != model {
8961 continue;
8962 }
8963 let sim = memory::cosine(query_vector, v);
8964 if best.map(|(_, s)| sim > s).unwrap_or(true) {
8965 best = Some((i, sim));
8966 }
8967 }
8968
8969 let Some((pos, sim)) = best else {
8970 self.cache_counters.misses += 1;
8971 return Ok(None);
8972 };
8973 if sim < threshold {
8974 self.cache_counters.misses += 1;
8975 return Ok(None);
8976 }
8977
8978 // Extract fields before any mutation to keep the borrow checker
8979 // happy.
8980 let (start_page, page_count, len, ttl_secs, created_at, gen) = {
8981 let e = &self.cache_dir[pos];
8982 (
8983 e.start_page,
8984 e.page_count,
8985 e.len,
8986 e.ttl_secs,
8987 e.created_at,
8988 e.gen,
8989 )
8990 };
8991
8992 // Decrypt and decode the record body.
8993 let buf = self.read_record_run(start_page, page_count, len, gen)?;
8994 let entry: CacheEntry =
8995 rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
8996
8997 if !self.read_only {
8998 self.cache_dir[pos].accessed_at = now;
8999 self.cache_dir[pos].access_count = self.cache_dir[pos].access_count.saturating_add(1);
9000 self.dirty_cache = true;
9001 }
9002 self.cache_counters.hits += 1;
9003 let access_count = self.cache_dir[pos].access_count;
9004 // FIX-1 R-002.2 part 1: same read-only `accessed_at` fix as
9005 // `cache_get`. The directory carries the live value; the record
9006 // body only holds the insert-time snapshot.
9007 let accessed_at = if self.read_only {
9008 self.cache_dir[pos].accessed_at
9009 } else {
9010 now
9011 };
9012 Ok(Some((
9013 CachedValue {
9014 value: entry.value,
9015 content_type: entry.content_type,
9016 created_at,
9017 accessed_at,
9018 access_count,
9019 ttl_secs,
9020 },
9021 sim,
9022 )))
9023 }
9024
9025 /// Begin a conversation [`Session`](crate::Session) for `agent_id`.
9026 ///
9027 /// The session borrows the database for its lifetime, records turns as
9028 /// working memory, and consolidates them into episodic memory when closed.
9029 pub fn session(&mut self, agent_id: impl Into<String>) -> crate::session::Session<'_> {
9030 crate::session::Session::new(self, agent_id.into())
9031 }
9032
9033 // --- snapshots & point-in-time recovery ------------------------------
9034
9035 /// Every committed transaction, oldest first — the restore points
9036 /// available to [`Mnemo::restore_to`] and [`Mnemo::restore_to_time`].
9037 ///
9038 /// Each `flush` appends one snapshot. Because the storage engine is
9039 /// append-only, the pages a past flush wrote are still on disk, so any
9040 /// listed snapshot can be reinstated exactly. The history reaches back to
9041 /// the last [`Mnemo::compact_file`], which reclaims space by collapsing
9042 /// it.
9043 pub fn snapshots(&self) -> Vec<SnapshotInfo> {
9044 let mut v: Vec<SnapshotInfo> = self
9045 .manifest
9046 .iter()
9047 .map(|s| SnapshotInfo {
9048 txn_id: s.txn_id,
9049 created_at: s.created_at,
9050 memory_count: s.memory_count,
9051 })
9052 .collect();
9053 v.sort_by_key(|s| s.txn_id);
9054 v
9055 }
9056
9057 /// Load a past snapshot's state and commit it as a new transaction.
9058 fn apply_snapshot(&mut self, snap: &Snapshot) -> Result<()> {
9059 let catalog = load_catalog(
9060 &mut self.pager,
9061 snap.catalog_start,
9062 snap.catalog_pages,
9063 snap.catalog_len,
9064 snap.catalog_gen(),
9065 )?;
9066 let (catalog, _, _) = apply_delta_chain(&mut self.pager, catalog, snap.delta_head())?;
9067 // FMT-9c.2: the restored catalog is written whole.
9068 self.catalog_rewrite = true;
9069 let ann = load_index(
9070 &mut self.pager,
9071 snap.index_start,
9072 snap.index_pages,
9073 snap.index_len,
9074 snap.index_gen(),
9075 self.dimensions,
9076 )?;
9077 let mut ann = ann;
9078 if self.is_v9() {
9079 self.reclaim_for_restore(&catalog);
9080 if let Some(a) = ann.as_mut() {
9081 apply_index_chain(&mut self.pager, a, snap.index_delta_head())?;
9082 a.enable_log();
9083 }
9084 self.index_rebase = true;
9085 // FMT-9d: the snapshot's BM25 segments (quarantined while it's
9086 // retained) describe its documents.
9087 let bptr = snap.gens.map_or(RunPtr::default(), |g| tuple_ptr(g.bm25));
9088 let manifest = load_bm25_manifest(&mut self.pager, bptr)?;
9089 // Its runs come back off the free list; the current ones it
9090 // doesn't share are freed, and the header points at its run.
9091 let runs = |ptr: RunPtr, m: &Bm25Manifest| {
9092 let mut v: Vec<(u64, u64)> = m.segments.iter().map(|t| (t.0, t.1)).collect();
9093 if !ptr.is_empty() {
9094 v.push((ptr.start, ptr.pages));
9095 }
9096 v
9097 };
9098 let then = runs(bptr, &manifest);
9099 let now = runs(self.header.bm25, &self.bm25.manifest);
9100 self.pending_free.retain(|r| !then.contains(r));
9101 for &(p, c) in &then {
9102 self.free.unfree(p, c);
9103 }
9104 for r in now {
9105 if !then.contains(&r) {
9106 self.release_pages(r.0, r.1);
9107 }
9108 }
9109 self.header.bm25 = bptr;
9110 self.free_dirty = true;
9111 self.bm25 = Bm25State {
9112 manifest,
9113 ..Bm25State::default()
9114 };
9115 }
9116 self.index = build_id_index(&catalog);
9117 self.catalog = catalog;
9118 // FMT-9b: the column and the packed pages described the catalog
9119 // being replaced.
9120 self.col = VecColumn::from_catalog(&self.catalog);
9121 self.recs = RecPages::from_catalog(&self.catalog);
9122 self.ann = ann;
9123 self.ann_unloaded = false;
9124 // The handle caches describe the catalog being replaced.
9125 self.key_index = None;
9126 self.superseded = None;
9127 self.layers = None;
9128 self.agent_set = None;
9129 // PERF-1: the snapshot's index may predate its catalog; bring it up
9130 // to date before it's persisted again.
9131 let snap_index_gen = snap.index_gen().max(snap.index_delta_head().gen);
9132 self.reconcile_ann_with_catalog(snap.index_start, snap_index_gen)?;
9133 // Re-commit as a fresh transaction: crash-safe, and itself recorded
9134 // as a new snapshot so a restore can always be undone.
9135 self.dirty_catalog = true;
9136 self.dirty_index = true;
9137 self.index_persist.forced = true;
9138 self.flush()
9139 }
9140
9141 /// FMT-9c: v9 snapshot retention, applied to the staged manifest (the
9142 /// new snapshot already appended): keep the newest `max_snapshots`
9143 /// plus any younger than 24 hours, then drop the oldest while retained
9144 /// snapshots pin more than a quarter of the file's live pages. The
9145 /// newest snapshot always stays.
9146 ///
9147 /// The pin cap is a quarter of the live pages or [`V9_PIN_FLOOR_PAGES`]
9148 /// (1 MiB), whichever is larger, so tiny files keep their history.
9149 /// `extra` is what this commit frees (pinned by every older snapshot).
9150 ///
9151 /// An explicit count (`set_max_snapshots`, or a non-default
9152 /// `MnemoConfig::max_snapshots`) replaces the policy: exactly the
9153 /// newest `n` are kept (`0` keeps all), whatever they pin.
9154 fn apply_retention(&self, manifest: &mut Vec<Snapshot>, extra: u64) {
9155 if self.snapshot_cap_explicit {
9156 if self.max_snapshots > 0 {
9157 while manifest.len() > self.max_snapshots {
9158 manifest.remove(0);
9159 }
9160 }
9161 return;
9162 }
9163 let now = memory::now_secs();
9164 if self.max_snapshots > 0 {
9165 while manifest.len() > self.max_snapshots
9166 && now - manifest[0].created_at > V9_SNAPSHOT_KEEP_SECS
9167 {
9168 manifest.remove(0);
9169 }
9170 }
9171 let live_pages = self.header.next_page.saturating_sub(self.free.pages());
9172 let cap = (live_pages / 4).max(V9_PIN_FLOOR_PAGES);
9173 while manifest.len() > 1 {
9174 let pinned = self.free.pinned_pages(manifest[0].txn_id) + extra;
9175 if pinned <= cap {
9176 break;
9177 }
9178 manifest.remove(0);
9179 }
9180 }
9181
9182 /// FMT-9 (a): a restore makes the snapshot's records live again and the
9183 /// current ones (those the snapshot doesn't have) dead. The snapshot's
9184 /// record runs may be on the free list (freed after the snapshot, and
9185 /// quarantined while it's retained), so they come back off it, and off
9186 /// this transaction's pending frees; the current runs it doesn't share
9187 /// are freed by the restoring commit.
9188 fn reclaim_for_restore(&mut self, restored: &[CatalogEntry]) {
9189 // Record runs, deduplicated (FMT-9b.2: packed records share a page);
9190 // rows still waiting to be packed own no page yet.
9191 let runs = |cat: &[CatalogEntry]| -> std::collections::BTreeMap<u64, u64> {
9192 cat.iter()
9193 .filter(|e| !e.deleted && e.start_page != 0)
9194 .map(|e| (e.start_page, e.page_count as u64))
9195 .collect()
9196 };
9197 let live_then = runs(restored);
9198 let live_now = runs(&self.catalog);
9199 let overlaps = |p: u64, c: u64| {
9200 live_then
9201 .range(..p + c)
9202 .next_back()
9203 .is_some_and(|(&q, &n)| q + n > p)
9204 };
9205 let dying: Vec<(u64, u64)> = live_now
9206 .iter()
9207 .filter(|(p, _)| !live_then.contains_key(p))
9208 .map(|(&p, &c)| (p, c))
9209 .collect();
9210 // FMT-9b: vector pages hold many rows, so they're compared as a
9211 // set: the snapshot's come back, the current ones it lacks go.
9212 let vec_then: std::collections::BTreeSet<u64> = restored
9213 .iter()
9214 .filter(|e| !e.deleted)
9215 .filter_map(|e| e.v9.map(|r| r.vec_page))
9216 .filter(|&p| p != 0)
9217 .collect();
9218 let vec_dying: Vec<u64> = self
9219 .col
9220 .page_rows
9221 .keys()
9222 .copied()
9223 .filter(|p| !vec_then.contains(p))
9224 .collect();
9225 self.pending_free
9226 .retain(|&(p, c)| !(overlaps(p, c) || (c == 1 && vec_then.contains(&p))));
9227 for (&p, &c) in &live_then {
9228 self.free.unfree(p, c);
9229 }
9230 for &p in &vec_then {
9231 self.free.unfree(p, 1);
9232 }
9233 for (p, c) in dying {
9234 self.release_pages(p, c);
9235 }
9236 for p in vec_dying {
9237 self.release_pages(p, 1);
9238 }
9239 self.free_dirty = true;
9240 }
9241
9242 /// Restore the database to the snapshot produced by transaction `txn_id`.
9243 ///
9244 /// The restore is itself a new committed transaction (and a new
9245 /// snapshot), so it is crash-safe and reversible — restoring forward to a
9246 /// later snapshot afterwards works just as well.
9247 pub fn restore_to(&mut self, txn_id: u64) -> Result<SnapshotInfo> {
9248 if self.read_only {
9249 return Err(MnemoError::ReadOnly);
9250 }
9251 let snap = self
9252 .manifest
9253 .iter()
9254 .find(|s| s.txn_id == txn_id)
9255 .cloned()
9256 .ok_or_else(|| {
9257 let oldest = self.manifest.iter().map(|s| s.txn_id).min();
9258 match oldest {
9259 Some(o) if self.is_v9() && txn_id < o => MnemoError::NotFound(format!(
9260 "snapshot for transaction {txn_id}: older than the retention window \
9261 (the newest {} snapshots and any from the last 24 hours, with at \
9262 most a quarter of the file pinned); the oldest kept is {o}",
9263 self.max_snapshots
9264 )),
9265 _ => MnemoError::NotFound(format!("snapshot for transaction {txn_id}")),
9266 }
9267 })?;
9268 self.apply_snapshot(&snap)?;
9269 Ok(SnapshotInfo {
9270 txn_id: snap.txn_id,
9271 created_at: snap.created_at,
9272 memory_count: snap.memory_count,
9273 })
9274 }
9275
9276 /// Restore the database to the latest snapshot committed at or before
9277 /// `unix_secs`. Returns [`MnemoError::NotFound`] if no snapshot is that
9278 /// old. Like [`Mnemo::restore_to`], the restore is a new transaction.
9279 pub fn restore_to_time(&mut self, unix_secs: i64) -> Result<SnapshotInfo> {
9280 if self.read_only {
9281 return Err(MnemoError::ReadOnly);
9282 }
9283 let snap = self
9284 .manifest
9285 .iter()
9286 .filter(|s| s.created_at <= unix_secs)
9287 .max_by_key(|s| s.txn_id)
9288 .cloned()
9289 .ok_or_else(|| {
9290 MnemoError::NotFound(format!("no snapshot at or before time {unix_secs}"))
9291 })?;
9292 self.apply_snapshot(&snap)?;
9293 Ok(SnapshotInfo {
9294 txn_id: snap.txn_id,
9295 created_at: snap.created_at,
9296 memory_count: snap.memory_count,
9297 })
9298 }
9299
9300 /// Return the KDF (Argon2id) parameters this file currently uses to
9301 /// wrap its DEK. Needed by callers that want to [`rekey`] while
9302 /// keeping the file's original protection level (e.g. the Python
9303 /// binding's `rekey(fast=False)`, whose docstring promises exactly
9304 /// that). FIX-1 R-001.5.
9305 pub fn kdf_params(&self) -> KdfParams {
9306 self.kdf
9307 }
9308
9309 /// Flush any pending writes, then stream the (encrypted) file to
9310 /// `dest` atomically: writes to a sibling temp file in `dest`'s
9311 /// parent directory, `sync_all`s it, renames onto `dest`, and
9312 /// fsyncs the parent directory on Unix so a crash never leaves a
9313 /// partial export.
9314 ///
9315 /// Refuses when `dest` resolves to the file this handle has open
9316 /// — that case would otherwise `File::create`-truncate the live
9317 /// database to 0 bytes (`fs::copy` on Unix reads *after* the
9318 /// `O_TRUNC`, so a naive export-to-self destroys the source).
9319 /// Unix: compares `(dev, ino)` of both paths via `MetadataExt`,
9320 /// catching symlinks and hard links. Windows: compares
9321 /// canonicalized paths; a rename onto an open file also fails on
9322 /// Windows because of the mandatory lock the handle holds, so the
9323 /// rename step is a second line of defense.
9324 ///
9325 /// Reads the bytes from the already-open file handle, not by
9326 /// `fs::copy`. On Windows that matters: `LockFileEx` locks are
9327 /// mandatory, so a second handle opened by `fs::copy` would fail
9328 /// while this writer still holds the lock.
9329 ///
9330 /// FIX-1 R-001.5 + R-002.2 part 3.
9331 pub fn export_encrypted(&mut self, dest: impl AsRef<Path>) -> Result<()> {
9332 let dest = dest.as_ref();
9333
9334 // Step 1 — refuse export to the open file itself. We compare
9335 // by identity, not string, so symlinks and hard links to
9336 // `self.path` are caught too. The check runs before any flush
9337 // so a self-targeted export is a pure no-op.
9338 if paths_refer_to_same_file(&self.path, dest).unwrap_or(false) {
9339 return Err(MnemoError::Invalid(format!(
9340 "export_encrypted refuses to overwrite the open database at {}",
9341 dest.display()
9342 )));
9343 }
9344
9345 // Step 2 — flush, so on-disk bytes match the live state.
9346 if !self.read_only {
9347 self.flush()?;
9348 }
9349
9350 // Step 3 — stream from the open handle into a sibling temp.
9351 // Using the open handle (instead of a second open on `self.path`)
9352 // means a Windows mandatory lock can't block the copy.
9353 let parent = dest.parent().ok_or_else(|| {
9354 MnemoError::Invalid(format!(
9355 "export_encrypted: dest {} has no parent directory",
9356 dest.display()
9357 ))
9358 })?;
9359 let tmp = temp_sibling(dest);
9360 // Scope the temp handle so it closes before the rename on
9361 // Windows (which refuses to rename over an open file).
9362 {
9363 let src = self.pager.file_mut();
9364 src.seek(SeekFrom::Start(0))?;
9365 let mut out = OpenOptions::new()
9366 .write(true)
9367 .create(true)
9368 .truncate(true)
9369 .open(&tmp)?;
9370 let mut buf = vec![0u8; 1 << 20]; // 1 MiB
9371 loop {
9372 let n = src.read(&mut buf)?;
9373 if n == 0 {
9374 break;
9375 }
9376 out.write_all(&buf[..n])?;
9377 }
9378 out.sync_all()?;
9379 crate::bump_fsync();
9380 }
9381
9382 // Step 4 — atomic rename onto `dest`.
9383 if let Err(e) = std::fs::rename(&tmp, dest) {
9384 // Best-effort cleanup; propagate the rename error.
9385 let _ = std::fs::remove_file(&tmp);
9386 return Err(MnemoError::Io(e));
9387 }
9388
9389 // Step 5 — fsync the parent directory on Unix so the rename
9390 // itself is durable. On Windows directory syncs aren't
9391 // supported; the rename above is atomic enough on NTFS.
9392 #[cfg(unix)]
9393 {
9394 if let Ok(dir) = File::open(parent) {
9395 let _ = dir.sync_all();
9396 crate::bump_fsync();
9397 }
9398 }
9399 #[cfg(not(unix))]
9400 {
9401 let _ = parent;
9402 }
9403
9404 Ok(())
9405 }
9406
9407 /// Change the passphrase. Cheap: re-derives the KEK and re-wraps the DEK;
9408 /// the encrypted pages are never rewritten.
9409 pub fn rekey(&mut self, new_passphrase: &str, kdf: KdfParams) -> Result<()> {
9410 self.rekey_secret(new_passphrase.as_bytes(), kdf)
9411 }
9412
9413 /// PERF-1 step 7: switch to (or rotate) a raw 256-bit key, recording
9414 /// [`KdfParams::raw_key`] in the header. Like [`Mnemo::rekey`], only the
9415 /// wrapped DEK changes; pages aren't rewritten.
9416 pub fn rekey_with_key(&mut self, new_key: &[u8; crypto::KEY_LEN]) -> Result<()> {
9417 self.rekey_secret(new_key, KdfParams::raw_key())
9418 }
9419
9420 fn rekey_secret(&mut self, secret: &[u8], kdf: KdfParams) -> Result<()> {
9421 if self.read_only {
9422 return Err(MnemoError::ReadOnly);
9423 }
9424 self.flush()?;
9425 let new_salt = crypto::random_salt();
9426 let new_kek = crypto::derive_kek(secret, &new_salt, kdf)?;
9427 let new_nonce = crypto::random_nonce();
9428 let wrapped = crypto::wrap_dek(&new_kek, &new_nonce, self.pager.dek())?;
9429 if wrapped.len() != WRAPPED_DEK_LEN {
9430 return Err(MnemoError::Crypto("unexpected wrapped DEK length".into()));
9431 }
9432 let mut wrapped_dek = [0u8; WRAPPED_DEK_LEN];
9433 wrapped_dek.copy_from_slice(&wrapped);
9434
9435 self.header.salt = new_salt;
9436 self.header.dek_nonce = new_nonce;
9437 self.header.wrapped_dek = wrapped_dek;
9438 self.header.m_cost = kdf.m_cost;
9439 self.header.t_cost = kdf.t_cost;
9440 self.header.p_cost = kdf.p_cost;
9441 self.kdf = kdf;
9442
9443 // DUR-1 (R-002.1): WAL-log the new page 0 before touching it.
9444 // Allocate a fresh WAL region at the tail so that a crash during
9445 // the rekey leaves either the old region (old passphrase works)
9446 // or the new region (new passphrase works) fully committed —
9447 // overwriting the existing region would risk destroying both.
9448 let old_wal_start = self.header.wal_start;
9449 let new_pages = self.header.wal_pages;
9450 let new_start = self.header.next_page;
9451 self.header.next_page += new_pages;
9452 self.header.wal_start = new_start;
9453 self.header.wal_pages = new_pages;
9454 self.header.wal_seq += 1;
9455 self.relocate_header_through_wal(old_wal_start)?;
9456 // FMT-9 (a): the old region is freed by the next commit (see
9457 // `ensure_wal_capacity`).
9458 self.release_pages(old_wal_start, new_pages);
9459 Ok(())
9460 }
9461
9462 /// Decrypt and validate every live record. Returns the count verified
9463 /// across memories **and** cache entries (FIX-1 R-001.1: now that the
9464 /// cache survives compaction, `verify()` has to walk it too — the
9465 /// pre-FIX-1 implementation only checked `self.catalog`, so a
9466 /// corrupted cache record went unnoticed).
9467 pub fn verify(&mut self) -> Result<usize> {
9468 let entries: Vec<CatalogEntry> = self
9469 .catalog
9470 .iter()
9471 .filter(|e| !e.deleted)
9472 .cloned()
9473 .collect();
9474 let mut n = entries.len();
9475 for e in &entries {
9476 let bytes = self.read_record(e)?;
9477 let _m: Memory = self.decode_memory(&bytes)?;
9478 // FMT-9b: and the row's vector column entry.
9479 if e.v9.is_some() {
9480 let (h, _) = self.column_entry(e)?;
9481 if h.len() != self.dimensions {
9482 return Err(MnemoError::Invalid(
9483 "vector entry has the wrong width".into(),
9484 ));
9485 }
9486 }
9487 }
9488
9489 // Also decode every live cache entry. Tombstoned entries are
9490 // skipped — their pages are orphan ciphertext awaiting
9491 // `compact_file`, same as deleted memories. Expired entries
9492 // are *still* on disk until compaction, so we verify them too.
9493 let cache_positions: Vec<(u64, u32, u32, u64)> = self
9494 .cache_dir
9495 .iter()
9496 .filter(|e| !e.deleted)
9497 .map(|e| (e.start_page, e.page_count, e.len, e.gen))
9498 .collect();
9499 for (start_page, page_count, len, gen) in cache_positions {
9500 let buf = self.read_record_run(start_page, page_count, len, gen)?;
9501 let _c: CacheEntry = rmp_serde::from_slice(&buf)
9502 .map_err(|err| MnemoError::Serialize(err.to_string()))?;
9503 n += 1;
9504 }
9505 // FMT-9 (a): no free run may overlap a page the committed state
9506 // references (records, cache entries, control runs, the WAL).
9507 if self.is_v9() {
9508 self.check_free_list_disjoint()?;
9509 }
9510 Ok(n)
9511 }
9512
9513 /// FMT-9 (a): fail if the free list (or a pending free) overlaps any
9514 /// page the current state references.
9515 fn check_free_list_disjoint(&self) -> Result<()> {
9516 let h = &self.header;
9517 let mut live: Vec<(u64, u64, &str)> = vec![
9518 (h.wal_start, h.wal_pages, "WAL region"),
9519 (h.catalog_start, h.catalog_pages, "catalog run"),
9520 (h.index_start, h.index_pages, "index run"),
9521 (h.manifest_start, h.manifest_pages, "manifest run"),
9522 (h.cache_start, h.cache_pages, "cache directory run"),
9523 (h.free.start, h.free.pages, "free-list run"),
9524 (h.keys.start, h.keys.pages, "key-slot run"),
9525 ];
9526 live.extend(
9527 self.catalog
9528 .iter()
9529 .filter(|e| !e.deleted)
9530 .map(|e| (e.start_page, e.page_count as u64, "record")),
9531 );
9532 live.extend(
9533 self.cache_dir
9534 .iter()
9535 .filter(|e| !e.deleted)
9536 .map(|e| (e.start_page, e.page_count as u64, "cache entry")),
9537 );
9538 live.extend(self.col.page_rows.keys().map(|&p| (p, 1, "vector page")));
9539 live.extend(
9540 self.delta_chain
9541 .iter()
9542 .map(|r| (r.start, r.pages, "catalog delta run")),
9543 );
9544 live.extend(
9545 self.index_chain
9546 .iter()
9547 .map(|r| (r.start, r.pages, "index delta run")),
9548 );
9549 live.push((
9550 self.header.bm25.start,
9551 self.header.bm25.pages,
9552 "BM25 manifest",
9553 ));
9554 live.extend(
9555 self.bm25
9556 .manifest
9557 .segments
9558 .iter()
9559 .map(|t| (t.0, t.1, "BM25 segment")),
9560 );
9561 for (p, c, what) in live {
9562 if c == 0 {
9563 continue;
9564 }
9565 if self.free.overlaps(p, c) {
9566 return Err(MnemoError::Invalid(format!(
9567 "free list overlaps a live {what} at page {p} (+{c})"
9568 )));
9569 }
9570 if self
9571 .pending_free
9572 .iter()
9573 .any(|&(q, n)| q < p + c && p < q + n)
9574 {
9575 return Err(MnemoError::Invalid(format!(
9576 "a pending free overlaps a live {what} at page {p} (+{c})"
9577 )));
9578 }
9579 }
9580 Ok(())
9581 }
9582
9583 /// Summary statistics for the open database.
9584 pub fn stats(&mut self) -> Result<Stats> {
9585 let deleted = self.catalog.iter().filter(|e| e.deleted).count();
9586 // FIX-1 R-001.8 part 2: build the agent set lazily on first
9587 // call per handle, then maintain in `put` / invalidate in
9588 // `delete`. Pre-FIX-1 every `stats()` call decrypted every
9589 // live memory (O(N) AES-GCM decrypts just to list a usually-
9590 // tiny set of distinct agent IDs). On a 1M-memory database
9591 // this was in the multi-second range on hardware AES.
9592 if self.agent_set.is_none() {
9593 let built: std::collections::BTreeSet<String> =
9594 self.memories()?.into_iter().map(|m| m.agent_id).collect();
9595 self.agent_set = Some(built);
9596 }
9597 let agents: Vec<String> = self
9598 .agent_set
9599 .as_ref()
9600 .expect("agent_set just filled above")
9601 .iter()
9602 .cloned()
9603 .collect();
9604 let file_bytes = self.header.next_page * PAGE_SIZE as u64;
9605 Ok(Stats {
9606 memories: self.len(),
9607 deleted,
9608 dimensions: self.dimensions,
9609 file_bytes,
9610 agents,
9611 encrypted: self.header.flags & FLAG_ENCRYPTED != 0,
9612 created_at: self.header.created_at,
9613 index: {
9614 self.ensure_ann()?;
9615 self.ann.as_ref().map(|a| a.info())
9616 },
9617 wal_pages: self.header.wal_pages,
9618 fsyncs_last_flush: self.fsyncs_last_flush,
9619 format_version: self.header.version,
9620 })
9621 }
9622
9623 /// Rewrite the file, dropping tombstoned and expired memories and
9624 /// reclaiming stale pages left by updates. Done out-of-place via a temp
9625 /// file and an atomic rename.
9626 ///
9627 /// FIX-1 R-001.1 + R-002.2 part 2: the result cache now survives
9628 /// compaction. Every live, non-expired directory entry is copied
9629 /// into the new file verbatim — `created_at`, `ttl_secs`,
9630 /// `cost_hint_ms`, `key_preview`, `size_bytes`, `content_type`,
9631 /// `vector` and `model` come from the record body; `accessed_at`
9632 /// and `access_count` come from the *directory* (the record body
9633 /// only holds the insert-time snapshot — the directory is the
9634 /// source of truth for the live counts, as of v5's recall trick).
9635 /// Tombstoned and TTL-expired entries are dropped. Pre-FIX-1
9636 /// compaction kept only the memory catalog and silently emptied
9637 /// the cache; the Planner's interim README caveat (c417d2e) called
9638 /// that out — this PR removes the surprise.
9639 pub fn compact_file(path: &str, passphrase: &str) -> Result<CompactReport> {
9640 Self::compact_file_secret(path, passphrase.as_bytes(), 0)
9641 }
9642
9643 /// PERF-1 step 7: [`Mnemo::compact_file`] for a raw-key database.
9644 pub fn compact_file_with_key(path: &str, key: &[u8; crypto::KEY_LEN]) -> Result<CompactReport> {
9645 Self::compact_file_secret(path, key, 0)
9646 }
9647
9648 /// FMT-9f: [`Mnemo::compact_file`] into an explicit format: `9` (what
9649 /// `compact_file` does) or `8`, which keeps a v8 file readable by
9650 /// v0.10 and earlier. The v8 target is kept for one release.
9651 pub fn compact_file_to(
9652 path: &str,
9653 passphrase: &str,
9654 format_version: u16,
9655 ) -> Result<CompactReport> {
9656 if format_version != VERSION && format_version != VERSION_V9 {
9657 return Err(MnemoError::Invalid(format!(
9658 "compact target format {format_version} isn't supported (use 8 or 9)"
9659 )));
9660 }
9661 Self::compact_file_secret(path, passphrase.as_bytes(), format_version)
9662 }
9663
9664 fn compact_file_secret(
9665 path: &str,
9666 secret: &[u8],
9667 format_version: u16,
9668 ) -> Result<CompactReport> {
9669 let mut old = Mnemo::open_secret(path, secret)?;
9670 let dims = old.dimensions;
9671 let kdf = old.kdf;
9672 let tmp = format!("{path}.compact-tmp");
9673
9674 let mut new = Mnemo::create_secret(
9675 &tmp,
9676 secret,
9677 MnemoConfig {
9678 dimensions: dims,
9679 kdf,
9680 // FMT-9f: compaction writes v9 (upgrading a v8 file),
9681 // unless `compact_file_to` asked for v8. A v9 file that
9682 // keeps float vectors keeps them.
9683 format_version,
9684 store_float_vectors: old.float_vectors(),
9685 ..Default::default()
9686 },
9687 )?;
9688 // FMT-9e: compaction rewrites everything, so it compresses every
9689 // thread turn, recent or not.
9690 new.turn_compress_after = 0;
9691 // ANN-1.1: compaction is "re-tune for the current size" by
9692 // default. Pass IndexConfig::default() so partitions,
9693 // subspaces, n_probe and n_rerank are all re-derived from the
9694 // compacted vector count. Previously we froze the pre-compact
9695 // (n_probe, n_rerank), which left the new index on stale
9696 // numbers for the smaller post-compact corpus.
9697 let want_index = old.has_index();
9698 let now = memory::now_secs();
9699 let all = old.memories()?;
9700 let before = all.len();
9701 let mut after = 0;
9702 for m in all {
9703 if m.is_expired(now) {
9704 continue;
9705 }
9706 new.remember(m)?;
9707 after += 1;
9708 }
9709 if want_index && after > 0 {
9710 new.build_index_with(IndexConfig::default())?;
9711 }
9712
9713 // Count + copy cache entries. We walk `old.cache_dir` by index
9714 // so each hop reads the directory's live `accessed_at` /
9715 // `access_count` directly rather than going through
9716 // `cache_get` (which would also treat read-only semantics,
9717 // tombstones, etc. — we want the raw, authoritative values).
9718 let mut cache_before = 0usize;
9719 let mut cache_after = 0usize;
9720 let positions: Vec<usize> = (0..old.cache_dir.len()).collect();
9721 for pos in positions {
9722 let keep = {
9723 let e = &old.cache_dir[pos];
9724 !e.deleted && !e.is_expired(now)
9725 };
9726 if keep {
9727 cache_before += 1;
9728 new.copy_cache_entry_from(&mut old, pos)?;
9729 cache_after += 1;
9730 }
9731 }
9732
9733 new.flush()?;
9734 drop(new);
9735 drop(old);
9736
9737 std::fs::rename(&tmp, path)?;
9738 Ok(CompactReport {
9739 before,
9740 after,
9741 cache_before,
9742 cache_after,
9743 })
9744 }
9745
9746 /// Copy one live cache entry from `old` at directory position
9747 /// `pos` into `self` (a freshly-created compaction target).
9748 /// Decrypts the old record body, substitutes the old *directory*'s
9749 /// `accessed_at` + `access_count` (the record body's copies are
9750 /// stale as of v5's directory-only hit path), re-encodes, writes
9751 /// a fresh record run, and pushes a matching directory entry.
9752 ///
9753 /// Caller is expected to have already filtered out tombstoned /
9754 /// expired entries and to call this on a read-write handle.
9755 pub(crate) fn copy_cache_entry_from(&mut self, old: &mut Mnemo, pos: usize) -> Result<()> {
9756 if self.read_only {
9757 return Err(MnemoError::ReadOnly);
9758 }
9759 // Snapshot the directory fields we need, so the borrow on
9760 // `old.cache_dir` ends before we touch `old.pager`.
9761 let (start_page, page_count, len, dir_accessed_at, dir_access_count, old_gen) = {
9762 let e = &old.cache_dir[pos];
9763 (
9764 e.start_page,
9765 e.page_count,
9766 e.len,
9767 e.accessed_at,
9768 e.access_count,
9769 e.gen,
9770 )
9771 };
9772
9773 // Read + decrypt the old record body.
9774 let buf = old.read_record_run(start_page, page_count, len, old_gen)?;
9775 let mut entry: CacheEntry =
9776 rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
9777
9778 // Overwrite the stale body copies with the directory's live
9779 // values before persisting. Everything else — `created_at`,
9780 // `ttl_secs`, `cost_hint_ms`, `key_preview`, `size_bytes`,
9781 // `content_type`, `value`, `vector`, `model` — stays as the
9782 // body already recorded it.
9783 entry.accessed_at = dir_accessed_at;
9784 entry.access_count = dir_access_count;
9785
9786 let bytes =
9787 rmp_serde::to_vec(&entry).map_err(|err| MnemoError::Serialize(err.to_string()))?;
9788 let (new_start_page, new_page_count, gen) = self.write_record(&bytes)?;
9789
9790 let dir_entry = CacheDirectoryEntry {
9791 namespace: entry.namespace.clone(),
9792 key_hash: entry.key_hash,
9793 start_page: new_start_page,
9794 page_count: new_page_count,
9795 len: bytes.len() as u32,
9796 deleted: false,
9797 accessed_at: dir_accessed_at,
9798 access_count: dir_access_count,
9799 size_bytes: entry.size_bytes,
9800 ttl_secs: entry.ttl_secs,
9801 created_at: entry.created_at,
9802 vector: entry.vector.clone(),
9803 model: entry.model.clone(),
9804 gen,
9805 };
9806 let new_pos = self.cache_dir.len();
9807 self.cache_idx
9808 .insert(&entry.namespace, entry.key_hash, new_pos);
9809 self.cache_dir.push(dir_entry);
9810 self.dirty_cache = true;
9811 Ok(())
9812 }
9813}