Skip to main content

mnemo_engine/
store.rs

1//! The Mnemo database — ties the storage engine and the memory model together.
2//!
3//! Layout of a populated file:
4//!
5//! ```text
6//!   page 0           : Header (unencrypted)
7//!   pages 1..W       : write-ahead log region
8//!   pages W..        : encrypted record runs + catalog runs + index +
9//!                      snapshot manifest (append-only)
10//! ```
11//!
12//! Durability is provided by a **write-ahead log** ([`crate::wal`]). A `flush`
13//! is one transaction: record data pages are written copy-on-write to fresh
14//! pages, then the new catalog, ANN index, and header are logged to the WAL
15//! and committed with a single fsync. A checkpoint then folds the WAL into the
16//! home pages. A crash before the commit leaves the previous state intact; a
17//! crash after it is repaired by replaying the WAL on open.
18//!
19//! Because pages are only ever appended, every past transaction's runs survive
20//! on disk. Each flush also appends an entry to a **snapshot manifest**, so
21//! [`Mnemo::restore_to`] can reinstate any past state. Stale pages — and the
22//! history along with them — are reclaimed by [`Mnemo::compact_file`].
23
24use std::fs::{File, OpenOptions};
25use std::io::{Read, Seek, SeekFrom, Write};
26use std::path::{Path, PathBuf};
27
28use fs4::fs_std::FileExt;
29use serde::{Deserialize, Serialize};
30use serde_json::{Map, Value};
31use ulid::Ulid;
32
33use crate::crypto::{self, KdfParams};
34use crate::embed::{self, Embedder};
35use crate::error::{MnemoError, Result};
36use crate::format::{
37    Header, RunPtr, FLAG_ENCRYPTED, PAGE_SIZE, PAYLOAD, VERSION, VERSION_V9, WRAPPED_DEK_LEN,
38};
39use crate::freelist::FreeList;
40use crate::index::{IndexConfig, IndexInfo, IndexOp, IvfPqIndex};
41use crate::keyslot::{KeySlot as KeySlotEntry, KeySlotRun};
42use crate::memory::{
43    self, status as essence_status, CommitBatch, CommitReport, EssenceOp, EssenceOpKind,
44    EssenceScope, EssenceSection, ExportedTurn, Filter, FilterTerm, IngestReport, KeyScope, Memory,
45    MemoryPatch, MemoryType, Metric, Scope, ScoreWeights, ThreadTurn, TurnPolicy, WriteOutcome,
46    CONTINUATION_TIMELINE_ENTRIES, ESSENCE_DEFAULT_RENDER_BYTES, ESSENCE_TRANSITIONS,
47    MNEMO_META_CONTINUES, MNEMO_META_INVALID_AT, MNEMO_META_KEY, MNEMO_META_KIND,
48    MNEMO_META_KIND_ESSENCE, MNEMO_META_KIND_THREAD_LINK, MNEMO_META_KIND_TURN, MNEMO_META_LENGTH,
49    MNEMO_META_PART, MNEMO_META_REFS, MNEMO_META_ROLE, MNEMO_META_SECTION, MNEMO_META_SEQ,
50    MNEMO_META_SHA256, MNEMO_META_STATUS, MNEMO_META_STATUS_HISTORY, MNEMO_META_SUPERSEDED_BY,
51    MNEMO_META_SUPERSEDES, MNEMO_META_THREAD, MNEMO_META_TURN_ID, MNEMO_META_UPDATED_AT,
52    THREAD_TOOL_RESULT_TRUNCATE_BYTES, THREAD_TURN_CHUNK_BYTES,
53};
54use crate::pager::Pager;
55use crate::result_cache::{
56    self, BatchState, CacheBudget, CacheDirectoryEntry, CacheEntry, CacheFlushPolicy, CachePutOpts,
57    CacheStats, CachedValue, CounterState, DirectoryIndex, SemanticCachePutOpts,
58};
59use crate::text;
60use crate::veccol;
61use crate::wal;
62
63/// Default initial size of the write-ahead log region, in pages (64 KiB).
64/// The region grows automatically when a transaction's control plane outgrows
65/// it, so this is a *floor* on the fresh-file footprint, not a cap on
66/// transaction size. Was 64 pages (512 KiB) in v0.1.0 — lowered because that
67/// reservation dominated small-file size (~62% of a 31-memory dogfood file).
68const DEFAULT_WAL_PAGES: u64 = 8;
69
70/// Hard floor on the WAL reservation. A single header-page flush already
71/// needs one full DATA frame plus a COMMIT frame; two pages are the bare
72/// minimum that lets the first transaction commit without immediate growth.
73const MIN_WAL_PAGES: u64 = 2;
74
75/// Default cap on retained snapshot manifest entries. Each `flush` appends
76/// one entry; without a cap, long-running processes accumulate entries
77/// forever, growing per-flush manifest-serialize cost as O(total flushes)
78/// and the manifest run size on disk in lockstep. 256 keeps roughly a
79/// week of hourly snapshots while staying tiny (256 × ~82 bytes ≈ 21 KiB
80/// of manifest). Set `MnemoConfig::max_snapshots = 0` to disable the cap.
81pub const DEFAULT_MAX_SNAPSHOTS: usize = 256;
82
83/// PERF-1: the ANN index is derived data, so `flush` persists it only every
84/// this many commits that changed it (or see
85/// [`INDEX_PERSIST_EVERY_CHANGES`], [`Mnemo::flush_index`], [`Mnemo::close`]).
86/// Between persists, only the catalog is rewritten.
87pub const INDEX_PERSIST_EVERY_COMMITS: u32 = 64;
88
89/// PERF-1: …or once this many vectors were added, changed or removed since
90/// the last persist. Records written after the persisted index are
91/// re-assigned when the index is next loaded, so this bounds that work.
92pub const INDEX_PERSIST_EVERY_CHANGES: usize = 1_000;
93
94/// FMT-9d: default weight ε of the normalised BM25 score in hybrid recall
95/// (`relevance = similarity + ε · bm25 / max bm25`).
96pub const DEFAULT_KEYWORD_WEIGHT: f32 = 0.5;
97
98/// FMT-9b: default [`Mnemo::set_vector_cache_budget`]: the fp16 vector
99/// column stays resident while it fits (100K × 384 dims is about 77 MB) and
100/// is read through the page cache above it. PLANNER-A, 2026-11-07.
101pub const DEFAULT_VECTOR_CACHE_BUDGET: usize = 96 * 1024 * 1024;
102
103/// FMT-9b: below this many facts a v9 file has no ANN index: an exact
104/// fp16 scan of 20K × 384 dims is about a millisecond, and the IVF+PQ
105/// codebook alone would cost more disk than it saves time.
106pub const V9_ANN_MIN_FACTS: usize = 20_000;
107
108/// FMT-9f (PLANNER-A, 2026-11-08): on v9 an unpinned recall reranks at
109/// least this many candidates per requested hit (500 for `top_k` 10), or
110/// the index's own auto depth when that's deeper (above 100K). The v9
111/// index is trained on fp16-rounded vectors, and at 20K–50K the auto
112/// depth (×10–×25) lost up to 0.03 recall@10 against v8; ×50 is within
113/// 0.005 at every measured tier. v8 is unchanged.
114pub const V9_RERANK_PER_HIT: usize = 50;
115
116/// FMT-9c: page runs longer than this are written copy-on-write straight
117/// to their home pages (and fsynced) before the commit, instead of through
118/// the WAL, on v9. The WAL then carries only the header and small runs, so
119/// it stays at its 64 KiB default instead of growing to the largest
120/// catalog or index (26 MB at 100K on v8). Safe for the same reason data
121/// pages are: the pages come from the tail or from quarantined free runs,
122/// so nothing committed references them until the header flips.
123const V9_WAL_RUN_MAX_PAGES: u64 = 2;
124
125/// FMT-9c: snapshot retention on v9: the newest 16 snapshots, plus any
126/// from the last 24 hours, with retained snapshots never pinning more than
127/// a quarter of the file (past that, the oldest go first, whatever their
128/// age). `set_max_snapshots` changes the count.
129pub const V9_DEFAULT_MAX_SNAPSHOTS: usize = 16;
130const V9_SNAPSHOT_KEEP_SECS: i64 = 24 * 3600;
131/// FMT-9c: the pin cap never drops below 128 pages (1 MiB).
132const V9_PIN_FLOOR_PAGES: u64 = 128;
133
134/// FMT-9d: the layout byte at the start of a v9 ANN index run (1: the
135/// whole IVF+PQ index, followed by delta runs). A reader treats an unknown
136/// layout as no index, so a later layout (partition-major, PERF-2) ships
137/// without a format bump.
138const V9_INDEX_LAYOUT: u8 = 1;
139
140/// FMT-9b: how many sparse vector pages one commit folds into its new
141/// pages (the vacuum). Bounded so a commit's cost stays bounded.
142const VACUUM_PAGES_PER_COMMIT: usize = 4;
143
144/// FMT-9c.2: and at most this many rows moved by the vacuum per commit (a
145/// moved row is a changed catalog row, so it lands in the commit's delta).
146/// Pages with more live rows than this stay as they are until deletes thin
147/// them out.
148const VACUUM_MAX_ROWS: usize = 64;
149
150/// FMT-9b: header flag: the file also keeps each record's float vector
151/// (`MnemoConfig::store_float_vectors`).
152const FLAG_FLOAT_VECTORS: u32 = 2;
153
154/// PERF-1: when the in-memory ANN index was last persisted, and whether the
155/// next flush must persist it.
156#[derive(Debug, Default, Clone, Copy)]
157struct IndexPersist {
158    /// Persist on the next flush (build, rebuild, drop, `flush_index`,
159    /// `close`, snapshot restore).
160    forced: bool,
161    /// Commits since the last persist that left the index dirty.
162    commits: u32,
163    /// Vectors added, re-assigned or removed since the last persist.
164    changes: usize,
165}
166
167/// R-004.2: where the thread layers and the onboarding briefing live,
168/// built by the handle's one lazy scan (`ensure_key_caches`, together with
169/// the 12.1 key caches) and then kept current by `put` and `delete`. It
170/// lets `about`, `essence`, `ingest_thread`, `thread_export` and
171/// thread-pinned recall read just their own records instead of decrypting
172/// the whole file each time.
173#[derive(Debug, Default)]
174struct LayerCache {
175    /// Live essence item ids.
176    essence: std::collections::HashSet<u128>,
177    /// Live ids tagged `metadata.area = "onboarding"` (superseded versions
178    /// included; `about` drops those when it reads them).
179    onboarding: std::collections::HashSet<u128>,
180    /// Per `_mnemo.thread`: its records and archive state.
181    threads: std::collections::HashMap<String, ThreadCache>,
182    /// Thread → (the thread it continues, the link record's id).
183    links: std::collections::HashMap<String, (String, u128)>,
184    /// FMT-9e: on v9, thread-turn records the lazy scan didn't decode,
185    /// by their row's `thread_hash`. A thread's bucket is decoded into
186    /// `threads` the first time that thread is used
187    /// ([`Mnemo::warm_thread`]), so a cold handle never decodes (or
188    /// decompresses) the turns of threads it doesn't touch.
189    cold_turns: std::collections::HashMap<u32, Vec<u128>>,
190}
191
192/// One thread's records (turns, essence items, link), the turn ids already
193/// archived (with how many records each has: long turns are chunked), and
194/// the highest `_mnemo.seq` seen.
195#[derive(Debug, Default)]
196struct ThreadCache {
197    records: std::collections::HashSet<u128>,
198    turns: std::collections::HashMap<String, u32>,
199    max_seq: Option<u64>,
200}
201
202fn meta_str<'a>(m: &'a Memory, key: &str) -> Option<&'a str> {
203    m.metadata.get(key).and_then(|v| v.as_str())
204}
205
206fn is_onboarding(m: &Memory) -> bool {
207    meta_str(m, "area").is_some_and(|s| s.eq_ignore_ascii_case("onboarding"))
208}
209
210impl LayerCache {
211    fn add(&mut self, m: &Memory) {
212        let id = m.id.0;
213        let kind = meta_str(m, MNEMO_META_KIND);
214        if kind == Some(MNEMO_META_KIND_ESSENCE) {
215            self.essence.insert(id);
216        }
217        if is_onboarding(m) {
218            self.onboarding.insert(id);
219        }
220        let Some(thread) = meta_str(m, MNEMO_META_THREAD) else {
221            return;
222        };
223        let tc = self.threads.entry(thread.to_string()).or_default();
224        tc.records.insert(id);
225        if kind == Some(MNEMO_META_KIND_TURN) {
226            if let Some(turn_id) = meta_str(m, MNEMO_META_TURN_ID) {
227                *tc.turns.entry(turn_id.to_string()).or_insert(0) += 1;
228            }
229            if let Some(seq) = m.metadata.get(MNEMO_META_SEQ).and_then(|v| v.as_u64()) {
230                tc.max_seq = Some(tc.max_seq.map_or(seq, |s| s.max(seq)));
231            }
232        }
233        if kind == Some(MNEMO_META_KIND_THREAD_LINK) {
234            if let Some(from) = meta_str(m, MNEMO_META_CONTINUES) {
235                self.links
236                    .entry(thread.to_string())
237                    .or_insert_with(|| (from.to_string(), id));
238            }
239        }
240    }
241
242    /// Undo [`LayerCache::add`] for the stored version of a record that is
243    /// being overwritten or deleted. A thread with no records left is
244    /// dropped, so a forgotten thread's `_mnemo.seq` restarts at 0. Partial
245    /// deletes keep the old maximum (sequence gaps are harmless).
246    fn remove(&mut self, m: &Memory) {
247        let id = m.id.0;
248        self.essence.remove(&id);
249        self.onboarding.remove(&id);
250        let Some(thread) = meta_str(m, MNEMO_META_THREAD) else {
251            return;
252        };
253        if self.links.get(thread).is_some_and(|(_, link)| *link == id) {
254            self.links.remove(thread);
255        }
256        let Some(tc) = self.threads.get_mut(thread) else {
257            return;
258        };
259        tc.records.remove(&id);
260        if meta_str(m, MNEMO_META_KIND) == Some(MNEMO_META_KIND_TURN) {
261            if let Some(turn_id) = meta_str(m, MNEMO_META_TURN_ID) {
262                if let Some(n) = tc.turns.get_mut(turn_id) {
263                    *n -= 1;
264                    if *n == 0 {
265                        tc.turns.remove(turn_id);
266                    }
267                }
268            }
269        }
270        if tc.records.is_empty() {
271            self.threads.remove(thread);
272        }
273    }
274}
275
276/// The thread a recall filter pins with `_mnemo.thread` equality, if any.
277fn filter_thread_pin(f: &Filter) -> Option<&str> {
278    f.terms.iter().find_map(|t| match t {
279        FilterTerm::Eq(k, Value::String(s)) if k == MNEMO_META_THREAD => Some(s.as_str()),
280        _ => None,
281    })
282}
283
284/// Whether a recall filter can only match records that aren't in the ANN
285/// (R-004.2): it pins `_mnemo.thread`, or restricts `_mnemo.kind` to turns,
286/// essence items or thread links. Such recalls scan exactly instead.
287fn filter_bypasses_ann(f: &Filter) -> bool {
288    let non_ann = |v: &Value| {
289        matches!(
290            v.as_str(),
291            Some(MNEMO_META_KIND_TURN)
292                | Some(MNEMO_META_KIND_ESSENCE)
293                | Some(MNEMO_META_KIND_THREAD_LINK)
294        )
295    };
296    filter_thread_pin(f).is_some()
297        || f.terms.iter().any(|t| match t {
298            FilterTerm::Eq(k, v) => k == MNEMO_META_KIND && non_ann(v),
299            FilterTerm::In(k, vs) => {
300                k == MNEMO_META_KIND && !vs.is_empty() && vs.iter().all(non_ann)
301            }
302            _ => false,
303        })
304}
305
306/// Chunk size for the torn-header scan's streaming read of the file.
307/// We walk the file 1 MiB at a time looking for 8 KiB-aligned `MWAL`
308/// magic. There is **no** cap on how far a given candidate's frame
309/// chain may extend — a 1M-vector IVF+PQ relocation can push a single
310/// transaction well past 16 MiB — so each candidate is read frame by
311/// frame via [`wal::scan_committed_txn`] rather than slurped into a
312/// bounded buffer. (DUR-1 R-002.1 part 2.)
313const TORN_HEADER_SCAN_CHUNK: usize = 1 << 20; // 1 MiB
314
315/// Walk the file for every 8 KiB-aligned page whose first four bytes are
316/// `MWAL`, treating each as a candidate WAL-region start, and return the
317/// frames of the highest-`txn_id` committed transaction whose header
318/// frame fully validates under `passphrase`.
319///
320/// A candidate is accepted only if:
321/// * `wal::scan_committed_txn` returns a committed transaction (chained
322///   CRC verifies, single `txn_id`, `KIND_COMMIT` closes it);
323/// * that transaction contains a `(0, header_page)` frame;
324/// * the header parses with `wal_start == candidate_page_no` and
325///   `wal_seq == txn_id` (so a frame cannot forge a region it was not
326///   written into);
327/// * deriving the KEK from `passphrase + header.salt`, unwrapping the
328///   wrapped DEK, and validating the header seal all succeed.
329///
330/// Returns `Ok(None)` when the scan finds no valid candidate — the
331/// caller is expected to propagate the original parse error so
332/// non-mnemo files still report `BadMagic`.
333///
334/// DUR-1 (R-002.1 part 2): there is no fixed scan window — the whole
335/// file is scanned. Reading at page granularity (not byte granularity)
336/// is sufficient because every WAL region in the format is page-aligned
337/// and every WAL commit writes its first frame at the region's start.
338fn scan_header_recovery_candidates(
339    file: &mut File,
340    secret: &[u8],
341) -> Result<Option<Vec<wal::Frame>>> {
342    let file_len = file.seek(SeekFrom::End(0))?;
343    if file_len < PAGE_SIZE as u64 * 2 {
344        return Ok(None); // No room for even one WAL page past the header.
345    }
346
347    let mut best: Option<(u64, Vec<wal::Frame>)> = None;
348    let page_sz = PAGE_SIZE as u64;
349    let mut offset = page_sz; // Skip page 0.
350
351    // Stream through the file in 1 MiB chunks, scanning each chunk for
352    // page-aligned `MWAL` candidates. Each candidate's frame chain is
353    // then read incrementally via `wal::scan_committed_txn`, which has
354    // no window cap — a 1M-vector IVF+PQ relocation can span tens of
355    // MiB in one committed transaction.
356    let mut chunk = vec![0u8; TORN_HEADER_SCAN_CHUNK];
357    while offset < file_len {
358        let want = (TORN_HEADER_SCAN_CHUNK as u64).min(file_len - offset) as usize;
359        chunk.resize(want, 0);
360        file.seek(SeekFrom::Start(offset))?;
361        let mut filled = 0;
362        while filled < want {
363            match file.read(&mut chunk[filled..])? {
364                0 => break,
365                n => filled += n,
366            }
367        }
368        if filled == 0 {
369            break;
370        }
371        let mut inner = 0usize;
372        while inner + 4 <= filled {
373            if &chunk[inner..inner + 4] == wal::FRAME_MAGIC_BYTES {
374                let page_no = (offset + inner as u64) / page_sz;
375                if let Some((txn_id, frames)) = try_recover_candidate(file, page_no, secret)? {
376                    let keep = match &best {
377                        None => true,
378                        Some((id, _)) => txn_id > *id,
379                    };
380                    if keep {
381                        best = Some((txn_id, frames));
382                    }
383                }
384            }
385            inner += PAGE_SIZE;
386        }
387        offset += filled as u64;
388    }
389
390    Ok(best.map(|(_, f)| f))
391}
392
393/// Try to recover a committed transaction from `candidate_page_no`,
394/// returning `(txn_id, frames)` only if the candidate's
395/// `(0, header_page)` frame fully validates: the frame chain commits
396/// (via [`wal::scan_committed_txn`]'s streaming reader), the header
397/// parses with `wal_start == candidate_page_no` **and**
398/// `wal_seq == txn_id`, the DEK unwraps under `passphrase`, and the
399/// AES-GCM seal validates. DUR-1 (R-002.1 item 2) requires the
400/// `wal_seq == txn_id` check and no fixed scan window — the streaming
401/// reader handles transactions of arbitrary size.
402fn try_recover_candidate(
403    file: &mut File,
404    candidate_page_no: u64,
405    secret: &[u8],
406) -> Result<Option<(u64, Vec<wal::Frame>)>> {
407    let Some((txn_id, frames)) = wal::scan_committed_txn(file, candidate_page_no)? else {
408        return Ok(None);
409    };
410
411    // Must contain a header frame at page 0.
412    let header_bytes = match frames.iter().find(|(p, _)| *p == 0) {
413        Some((_, b)) => b,
414        None => return Ok(None),
415    };
416    if header_bytes.len() != PAGE_SIZE {
417        return Ok(None);
418    }
419    let mut hbuf = [0u8; PAGE_SIZE];
420    hbuf.copy_from_slice(header_bytes);
421    let candidate = match Header::from_page(&hbuf) {
422        Ok(h) => h,
423        Err(_) => return Ok(None),
424    };
425
426    // The frame's claimed region must match where we found it, and the
427    // header frame's `wal_seq` must match the committed transaction's
428    // `txn_id` — this is what pins the frame to the region it was
429    // written into and keeps a stale checkpointed-and-copied frame
430    // from being accepted as a new region's root.
431    if candidate.wal_start != candidate_page_no || candidate.wal_seq != txn_id {
432        return Ok(None);
433    }
434    // DEK unwrap under this candidate's own salt + KDF params.
435    let kdf = KdfParams {
436        m_cost: candidate.m_cost,
437        t_cost: candidate.t_cost,
438        p_cost: candidate.p_cost,
439    };
440    let kek = match crypto::derive_kek(secret, &candidate.salt, kdf) {
441        Ok(k) => k,
442        Err(_) => return Ok(None),
443    };
444    let dek = match crypto::unwrap_dek(&kek, &candidate.dek_nonce, &candidate.wrapped_dek) {
445        Ok(d) => d,
446        Err(_) => return Ok(None),
447    };
448    if candidate.validate_seal(&dek).is_err() {
449        return Ok(None);
450    }
451
452    Ok(Some((txn_id, frames)))
453}
454
455/// Configuration for creating a new database.
456#[derive(Clone, Copy, Debug)]
457pub struct MnemoConfig {
458    /// Embedding dimensionality. Every stored vector must match this.
459    pub dimensions: usize,
460    /// Key-derivation parameters.
461    pub kdf: KdfParams,
462    /// Initial size of the WAL region in 8 KiB pages. Default is
463    /// [`DEFAULT_WAL_PAGES`] (8 pages = 64 KiB), which keeps fresh files
464    /// small; the region auto-grows when a transaction's control plane needs
465    /// more space, so raising this is only a hint for the expected steady-state
466    /// transaction size. Write-heavy databases that routinely flush large
467    /// catalogs or ANN indexes can set this higher (e.g. 64 = 512 KiB) to
468    /// avoid early grow events; tiny embedded uses can leave it at the
469    /// default. Clamped to a minimum of [`MIN_WAL_PAGES`] (2 pages).
470    pub wal_pages_initial: u64,
471    /// Maximum number of snapshot manifest entries to retain. Each `flush`
472    /// appends one entry; once the cap is hit, the *oldest* entry is dropped
473    /// from the in-memory manifest before the new one is added, so the
474    /// retained set is always the most-recent N. Defaults to
475    /// [`DEFAULT_MAX_SNAPSHOTS`] (256). Set to `0` to disable the cap and
476    /// retain every snapshot forever (the pre-v0.3 behavior).
477    ///
478    /// Pages referenced *only* by pruned snapshots stay on disk until the
479    /// next [`Mnemo::compact_file`], which reclaims them. Point-in-time
480    /// recovery via [`Mnemo::restore_to`] into a pruned `txn_id` returns
481    /// [`MnemoError::NotFound`].
482    ///
483    /// Apply to an already-open database with [`Mnemo::set_max_snapshots`].
484    pub max_snapshots: usize,
485    /// The on-disk format a new file is created in. `0` (the default)
486    /// means v9, the current format. `8` creates a v8 file, for tools that
487    /// still need to read it with v0.10 or earlier; it's kept for one
488    /// release (v0.12 makes v8 read-only). **Not a stable API.**
489    #[doc(hidden)]
490    pub format_version: u16,
491    /// FMT-9b (v9 files only; ignored for v8): also keep each memory's
492    /// full-precision vector in its record, so `get()` and `recall()`
493    /// return it exactly. Off by default: a v9 file stores vectors as
494    /// fp16 (about 1e-3 relative error) in the vector column, which recall
495    /// scores from either way. Costs `4 × dimensions` bytes per memory.
496    pub store_float_vectors: bool,
497}
498
499impl Default for MnemoConfig {
500    fn default() -> Self {
501        Self {
502            dimensions: 768,
503            kdf: KdfParams::secure(),
504            wal_pages_initial: DEFAULT_WAL_PAGES,
505            max_snapshots: DEFAULT_MAX_SNAPSHOTS,
506            format_version: 0,
507            store_float_vectors: false,
508        }
509    }
510}
511
512/// FMT-9: resolve [`MnemoConfig::format_version`] for a new file.
513fn resolve_format_version(requested: u16) -> Result<u16> {
514    match requested {
515        // FMT-9f: new files are v9; the `MNEMO_DEV_FORMAT` switch is gone.
516        0 => Ok(crate::format::DEFAULT_NEW_VERSION),
517        VERSION | VERSION_V9 => Ok(requested),
518        v => Err(MnemoError::Invalid(format!(
519            "format_version {v} is not supported for new files (use 8 or 9)"
520        ))),
521    }
522}
523
524/// Catalog entry: maps a memory ID to its page run on disk.
525///
526/// **Schema (v5+):** carries `accessed_at` and `access_count` so `recall`
527/// can update access stats by mutating the catalog entry rather than
528/// rewriting the entire record (Phase 2.1 of the improvement plan). The
529/// values in a memory's serialized record body become a stale snapshot
530/// after the first recall; [`Mnemo::read_memory`] re-populates them from
531/// the catalog so consumers always see the live values.
532#[derive(Serialize, Deserialize, Clone, Debug)]
533struct CatalogEntry {
534    /// ULID as a raw `u128`.
535    id: u128,
536    start_page: u64,
537    page_count: u32,
538    /// Exact serialized byte length of the record.
539    len: u32,
540    deleted: bool,
541    /// Unix seconds of the most recent recall hit (or write, for fresh entries).
542    accessed_at: i64,
543    /// How many times `recall` has surfaced this memory.
544    access_count: u32,
545    /// FMT-9: the generation the record's pages were sealed under (v9 page
546    /// AAD). `0`, and left out of the encoding, on v8 files, so a v8
547    /// catalog is byte-for-byte what v0.10 writes.
548    #[serde(default, skip_serializing_if = "crate::format::is_zero_u64")]
549    gen: u64,
550    /// FMT-9b: the v9 row's tags, scoring inputs and vector location
551    /// (`None`, and left out of the encoding, on v8 files).
552    #[serde(default, skip_serializing_if = "Option::is_none")]
553    v9: Option<RowV9>,
554}
555
556/// FMT-9b: what a v9 catalog row carries beyond the v8 fields, so recall
557/// can filter and score a candidate without decrypting its record, and a
558/// cold handle can select records by kind or thread.
559#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default)]
560struct RowV9 {
561    /// The vector column entry: page (`0` while it's waiting for the next
562    /// flush), slot, and the page's generation.
563    vec_page: u64,
564    vec_slot: u16,
565    vec_gen: u64,
566    /// FMT-9b.2: the record's slot on its packed page (`start_page`), or
567    /// [`REC_SPAN`] when the record owns a run of whole pages.
568    rec_slot: u16,
569    /// `ROW_KIND_*`.
570    kind: u8,
571    /// `ROW_SUPERSEDED | ROW_HAS_TTL | ROW_SHARED`.
572    flags: u8,
573    /// [`memory_type_code`].
574    memory_type: u8,
575    /// FNV-1a of `_mnemo.thread`, of the 12.1 key slot, and of `agent_id`
576    /// (`0` = none). A collision only lets a non-match through to the
577    /// decrypted check.
578    thread_hash: u32,
579    key_hash: u32,
580    agent_hash: u32,
581    created_at: i64,
582    importance: f32,
583    /// `created_at + ttl_secs` when `ROW_HAS_TTL` is set.
584    expires_at: i64,
585    /// FMT-9d: the row's BM25 document id (`0`: not keyword-indexed) and
586    /// its token count. Live rows define which documents are live, and
587    /// N and avgdl, so segments never hold lengths or tombstones.
588    doc: u32,
589    doc_len: u16,
590}
591
592/// FMT-9b.2: `RowV9::rec_slot` of a record too big for a packed page; it
593/// owns `page_count` whole pages from `start_page`, as in v8.
594const REC_SPAN: u16 = u16::MAX;
595/// FMT-9b.2: `kind` byte of a packed record page.
596const PAGE_KIND_RECORD: u8 = 1;
597/// FMT-9b.2: bytes of a packed record page available to slots and data.
598const REC_PAGE_CAP: usize = PAYLOAD - veccol::PAGE_HEADER;
599/// FMT-9b.2: bytes per slot-directory entry (offset u16, length u16).
600const REC_SLOT_BYTES: usize = 4;
601/// FMT-9b.2: thread-archive chunk size on v9. A chunk plus its record's
602/// metadata (~200 B) then fits one packed page (the FMT-9a measurement:
603/// 8 KB chunks spill into a second page almost every time, 300 MB vs
604/// 212 MB for the 50K-turn profile).
605const V9_TURN_CHUNK_BYTES: usize = 7_900;
606
607/// FMT-9e: the zstd frame magic. A v9 record body is either a MessagePack
608/// `Memory` (which always starts with an array marker, `0x9_` or
609/// `0xdc`/`0xdd`) or, for a compressed thread turn, a zstd frame, so the
610/// first four bytes tell them apart without a row flag.
611const ZSTD_MAGIC: [u8; 4] = [0x28, 0xB5, 0x2F, 0xFD];
612/// FMT-9e: zstd level for turn records (the FMT-9a measurement: 2.15× on
613/// this repo's Markdown at level 3).
614const TURN_ZSTD_LEVEL: i32 = 3;
615/// FMT-9e: turn records shorter than this stay uncompressed (a frame's
616/// overhead eats the gain on a one-line turn).
617const TURN_ZSTD_MIN_BYTES: usize = 256;
618/// FMT-9e: the largest decompressed record accepted. A turn record is a
619/// chunk of at most [`V9_TURN_CHUNK_BYTES`] plus its metadata, so this is
620/// a generous bound, not a size limit anyone meets; it stops a damaged
621/// (but authenticated) frame from asking for an unbounded allocation.
622const RECORD_PLAIN_MAX: u64 = 64 << 20;
623/// FMT-9e: by default a thread turn is written plain and compressed once
624/// it's a week old (by its `created_at`), so a live session's recent turns
625/// read at full speed and archived history is small.
626pub const DEFAULT_TURN_COMPRESS_AFTER_SECS: i64 = 7 * 24 * 3600;
627/// FMT-9e: at most this many aged turns are compressed per commit, so a
628/// commit's extra work stays bounded (like the vacuum).
629const COLD_TURNS_PER_COMMIT: usize = 256;
630
631/// FMT-9e: is turn-record compression on? Always, unless the dev switch
632/// `MNEMO_DEV_TURN_ZSTD=0` turns it off for an A/B measurement (read once
633/// per process). Reading never depends on it.
634fn turn_zstd_enabled() -> bool {
635    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
636    *ON.get_or_init(|| std::env::var("MNEMO_DEV_TURN_ZSTD").as_deref() != Ok("0"))
637}
638
639/// FMT-9e: compress a v9 thread-turn record body, before the page
640/// encryption. Each record is its own zstd frame with no dictionary and no
641/// compression history from any other record, so how densely one record
642/// compresses says nothing about another's content (design note (d)).
643/// Kept only when it saves at least an eighth; otherwise the plain body is
644/// returned.
645fn compress_turn_record(bytes: Vec<u8>) -> Vec<u8> {
646    if bytes.len() < TURN_ZSTD_MIN_BYTES || !turn_zstd_enabled() {
647        return bytes;
648    }
649    let c = ZSTD_COMPRESSOR.with(|cell| {
650        let mut slot = cell.borrow_mut();
651        if slot.is_none() {
652            *slot = zstd::bulk::Compressor::new(TURN_ZSTD_LEVEL).ok();
653        }
654        let out = slot.as_mut().and_then(|z| z.compress(&bytes).ok());
655        if out.is_none() {
656            // Never reuse a context a call failed on.
657            *slot = None;
658        }
659        out
660    });
661    match c {
662        Some(c) if c.len() * 8 <= bytes.len() * 7 => c,
663        _ => bytes,
664    }
665}
666
667// FMT-9e: one zstd context of each kind per thread, made on first use and
668// reused for every record (a fresh context per record made reads about 15%
669// slower). Only the one-shot (`bulk`) calls use them, and each call starts
670// a new frame from a reset session, so no state crosses records; a context
671// a call failed on is dropped, not reused (cf. RUSTSEC-2026-0391, which
672// affected the stream API's `with_context`; zstd 0.13.4 carries its fix).
673// One-shot calls size their tables to the record, so a context stays small.
674thread_local! {
675    static ZSTD_COMPRESSOR: std::cell::RefCell<Option<zstd::bulk::Compressor<'static>>> =
676        const { std::cell::RefCell::new(None) };
677    static ZSTD_DECOMPRESSOR: std::cell::RefCell<Option<zstd::bulk::Decompressor<'static>>> =
678        const { std::cell::RefCell::new(None) };
679}
680
681/// FMT-9e: the plain body of a v9 record: decompressed when it's a zstd
682/// frame, borrowed as is otherwise. The frame must state its content size,
683/// and that size must be at most [`RECORD_PLAIN_MAX`].
684pub(crate) fn plain_record(bytes: &[u8]) -> Result<std::borrow::Cow<'_, [u8]>> {
685    if !bytes.starts_with(&ZSTD_MAGIC) {
686        return Ok(std::borrow::Cow::Borrowed(bytes));
687    }
688    let bad = |why: &str| MnemoError::Invalid(format!("compressed record: {why}"));
689    let size = match zstd::zstd_safe::get_frame_content_size(bytes) {
690        Ok(Some(n)) if n <= RECORD_PLAIN_MAX => n as usize,
691        Ok(Some(_)) => return Err(bad("too large")),
692        _ => return Err(bad("no content size")),
693    };
694    let out = ZSTD_DECOMPRESSOR.with(|cell| {
695        let mut slot = cell.borrow_mut();
696        if slot.is_none() {
697            *slot = Some(zstd::bulk::Decompressor::new().map_err(|e| bad(&e.to_string()))?);
698        }
699        let out = match slot.as_mut() {
700            Some(z) => z.decompress(bytes, size).map_err(|e| bad(&e.to_string())),
701            None => Err(bad("no context")),
702        };
703        if out.is_err() {
704            // Never reuse a context a call failed on.
705            *slot = None;
706        }
707        out
708    })?;
709    if out.len() != size {
710        return Err(bad("length mismatch"));
711    }
712    Ok(std::borrow::Cow::Owned(out))
713}
714
715/// FMT-9c.2: pick the sparse pages (`(page, live rows)`, in page order) one
716/// commit's vacuum folds in: at most [`VACUUM_PAGES_PER_COMMIT`] pages and
717/// [`VACUUM_MAX_ROWS`] rows.
718fn vacuum_pick(sparse: Vec<(u64, usize)>) -> Vec<u64> {
719    let mut rows = 0usize;
720    let mut out = Vec::new();
721    for (p, n) in sparse {
722        if out.len() == VACUUM_PAGES_PER_COMMIT {
723            break;
724        }
725        if rows + n > VACUUM_MAX_ROWS {
726            continue;
727        }
728        rows += n;
729        out.push(p);
730    }
731    out
732}
733
734/// FMT-9b.2: encode a packed record page: the header, the slot directory,
735/// then the records in slot order.
736fn encode_record_page(gen: u64, recs: &[&[u8]]) -> Vec<u8> {
737    let dir = recs.len() * REC_SLOT_BYTES;
738    let data: usize = recs.iter().map(|r| r.len()).sum();
739    let mut out = Vec::with_capacity(veccol::PAGE_HEADER + dir + data);
740    out.extend_from_slice(&[PAGE_KIND_RECORD, 0, 0, 0]);
741    out.extend_from_slice(&(recs.len() as u16).to_le_bytes());
742    out.extend_from_slice(&((dir + data) as u16).to_le_bytes());
743    out.extend_from_slice(&gen.to_le_bytes());
744    let mut off = veccol::PAGE_HEADER + dir;
745    for r in recs {
746        out.extend_from_slice(&(off as u16).to_le_bytes());
747        out.extend_from_slice(&(r.len() as u16).to_le_bytes());
748        off += r.len();
749    }
750    for r in recs {
751        out.extend_from_slice(r);
752    }
753    out
754}
755
756/// FMT-9b.2: the bytes of slot `slot` of a packed record page.
757fn record_slot(page: &[u8], slot: u16) -> Result<Vec<u8>> {
758    let bad = || MnemoError::Invalid("malformed record page".into());
759    if page.len() < veccol::PAGE_HEADER || page[0] != PAGE_KIND_RECORD {
760        return Err(bad());
761    }
762    let n = u16::from_le_bytes([page[4], page[5]]);
763    if slot >= n {
764        return Err(bad());
765    }
766    let d = veccol::PAGE_HEADER + slot as usize * REC_SLOT_BYTES;
767    let dir = page.get(d..d + REC_SLOT_BYTES).ok_or_else(bad)?;
768    let off = u16::from_le_bytes([dir[0], dir[1]]) as usize;
769    let len = u16::from_le_bytes([dir[2], dir[3]]) as usize;
770    page.get(off..off + len).map(|b| b.to_vec()).ok_or_else(bad)
771}
772
773/// FMT-9d: the BM25 manifest run: tokenizer, the next document id, and the
774/// segment runs (oldest first).
775#[derive(Serialize, Deserialize, Clone, Debug)]
776struct Bm25Manifest {
777    tokenizer: u8,
778    next_doc: u32,
779    segments: Vec<(u64, u64, u64, u64)>,
780    /// Documents each segment was written with (live or not since).
781    #[serde(default)]
782    seg_docs: Vec<u32>,
783}
784
785impl Default for Bm25Manifest {
786    fn default() -> Self {
787        Bm25Manifest {
788            tokenizer: text::TOKENIZER_V1,
789            next_doc: 1,
790            segments: Vec::new(),
791            seg_docs: Vec::new(),
792        }
793    }
794}
795
796/// FMT-9d: live documents: `doc -> (row, length)`, their count and mean
797/// length. Rebuilt from the rows after any write.
798#[derive(Debug, Default)]
799struct DocStats {
800    docs: std::collections::HashMap<u32, (usize, u16)>,
801    avgdl: f32,
802    /// FMT-9f: the sum of the live documents' lengths, so `put` and
803    /// `delete` keep `avgdl` current without a rescan of the rows.
804    total: u64,
805}
806
807impl DocStats {
808    /// FMT-9f: drop document `doc` (if live) and keep `avgdl` current.
809    fn remove(&mut self, doc: u32) {
810        if let Some((_, len)) = self.docs.remove(&doc) {
811            self.total = self.total.saturating_sub(len as u64);
812            self.refresh();
813        }
814    }
815
816    /// FMT-9f: add document `doc` at catalog row `row`.
817    fn insert(&mut self, doc: u32, row: usize, len: u16) {
818        if let Some((_, old)) = self.docs.insert(doc, (row, len)) {
819            self.total = self.total.saturating_sub(old as u64);
820        }
821        self.total += len as u64;
822        self.refresh();
823    }
824
825    fn refresh(&mut self) {
826        self.avgdl = if self.docs.is_empty() {
827            1.0
828        } else {
829            self.total as f32 / self.docs.len() as f32
830        };
831    }
832}
833
834/// FMT-9d: the handle's BM25 state.
835#[derive(Debug, Default)]
836struct Bm25State {
837    manifest: Bm25Manifest,
838    /// Per segment, its dict, once loaded (resident; postings stay on disk).
839    dicts: Option<Vec<Vec<text::DictEntry>>>,
840    /// Documents written since the last commit, which writes them as a
841    /// new segment.
842    pending: Vec<(u32, Vec<(String, u8)>)>,
843    stats: Option<DocStats>,
844    /// Documents in the segment the current flush is merging.
845    merged_docs: u32,
846}
847
848/// FMT-9d: a manifest's segment runs and their document counts.
849type Bm25Segments = (Vec<(u64, u64, u64, u64)>, Vec<u32>);
850
851/// FMT-9d: segments past this many are merged into one by the next commit.
852const BM25_MAX_SEGMENTS: usize = 8;
853
854/// FMT-9d: is this row keyword-indexed? Facts and onboarding records;
855/// thread turns, essence items and links stay out (FMT-9a: turn postings
856/// would be 3× all the fact data).
857fn bm25_kind(kind: u8) -> bool {
858    kind == ROW_KIND_FACT || kind == ROW_KIND_ONBOARDING
859}
860
861/// FMT-9d: read `len` bytes at byte `off` of a page run.
862fn read_run_range(pager: &mut Pager, ptr: RunPtr, off: u64, len: u64) -> Result<Vec<u8>> {
863    let mut out = Vec::with_capacity(len as usize);
864    if len == 0 {
865        return Ok(out);
866    }
867    if off + len > ptr.len {
868        return Err(MnemoError::Invalid("BM25 range past its run".into()));
869    }
870    let p = PAYLOAD as u64;
871    let (first, last) = (off / p, (off + len - 1) / p);
872    for i in first..=last {
873        let page = pager.read_page(ptr.start + i, ptr.gen)?;
874        let lo = if i == first { (off % p) as usize } else { 0 };
875        let hi = if i == last {
876            ((off + len - 1) % p) as usize + 1
877        } else {
878            PAYLOAD
879        };
880        out.extend_from_slice(&page[lo..hi]);
881    }
882    Ok(out)
883}
884
885/// FMT-9d: a segment's dict (the `[u32 len][dict]` head of its run).
886fn load_segment_dict(pager: &mut Pager, ptr: RunPtr) -> Result<Vec<text::DictEntry>> {
887    let head = read_run_range(pager, ptr, 0, 4)?;
888    let n = u32::from_le_bytes([head[0], head[1], head[2], head[3]]) as u64;
889    let bytes = read_run_range(pager, ptr, 4, n)?;
890    rmp_serde::from_slice(&bytes).map_err(|e| MnemoError::Serialize(e.to_string()))
891}
892
893/// FMT-9d: a term's postings in one segment (empty when absent).
894fn segment_postings(
895    pager: &mut Pager,
896    ptr: RunPtr,
897    dict: &[text::DictEntry],
898    term: &str,
899) -> Result<Vec<(u32, u8)>> {
900    let Ok(i) = dict.binary_search_by(|e| e.term.as_str().cmp(term)) else {
901        return Ok(Vec::new());
902    };
903    let head = read_run_range(pager, ptr, 0, 4)?;
904    let base = 4 + u32::from_le_bytes([head[0], head[1], head[2], head[3]]) as u64;
905    let e = &dict[i];
906    let bytes = read_run_range(pager, ptr, base + e.off as u64, e.len as u64)?;
907    text::decode_postings(&bytes)
908}
909
910/// FMT-9d: load the BM25 manifest (an empty pointer yields a fresh one).
911fn load_bm25_manifest(pager: &mut Pager, ptr: RunPtr) -> Result<Bm25Manifest> {
912    if ptr.is_empty() {
913        return Ok(Bm25Manifest::default());
914    }
915    let buf = read_run_bytes(pager, ptr.start, ptr.pages, ptr.len, ptr.gen)?;
916    let m: Bm25Manifest =
917        rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
918    if m.tokenizer != text::TOKENIZER_V1 {
919        return Err(MnemoError::Invalid(format!(
920            "BM25 tokenizer {} is newer than this build understands",
921            m.tokenizer
922        )));
923    }
924    Ok(m)
925}
926
927/// FMT-9b.2: the handle's view of packed record pages.
928#[derive(Debug, Default)]
929struct RecPages {
930    /// Rows whose records aren't on a page yet; the next flush packs them.
931    hot: std::collections::HashMap<usize, Vec<u8>>,
932    /// Rows to pack, in order (may repeat; the flush skips repeats).
933    pending: Vec<usize>,
934    /// Packed page → the live rows on it.
935    page_rows: std::collections::HashMap<u64, Vec<usize>>,
936}
937
938impl RecPages {
939    fn from_catalog(catalog: &[CatalogEntry]) -> RecPages {
940        let mut recs = RecPages::default();
941        for (i, e) in catalog.iter().enumerate() {
942            if e.deleted || e.start_page == 0 {
943                continue;
944            }
945            if let Some(r) = e.v9 {
946                if r.rec_slot != REC_SPAN {
947                    recs.page_rows.entry(e.start_page).or_default().push(i);
948                }
949            }
950        }
951        recs
952    }
953}
954
955const ROW_SUPERSEDED: u8 = 1;
956const ROW_HAS_TTL: u8 = 2;
957const ROW_SHARED: u8 = 4;
958/// FMT-9e: a thread turn whose record compression is settled: stored as
959/// a zstd frame, or tried and not worth it. Unsettled turns are plain and
960/// get compressed once they're older than the handle's
961/// `turn_compress_after` (see [`Mnemo::set_turn_compress_after`]).
962const ROW_SETTLED: u8 = 8;
963const ROW_KIND_FACT: u8 = 0;
964const ROW_KIND_TURN: u8 = 1;
965const ROW_KIND_ESSENCE: u8 = 2;
966const ROW_KIND_THREAD_LINK: u8 = 3;
967const ROW_KIND_ONBOARDING: u8 = 4;
968
969/// 32-bit FNV-1a over `parts` (separated), never `0` ("none" in a row).
970fn fnv1a32(parts: &[&[u8]]) -> u32 {
971    let mut h: u32 = 0x811c_9dc5;
972    for (i, p) in parts.iter().enumerate() {
973        if i > 0 {
974            h ^= 0xff;
975            h = h.wrapping_mul(0x0100_0193);
976        }
977        for &b in p.iter() {
978            h ^= b as u32;
979            h = h.wrapping_mul(0x0100_0193);
980        }
981    }
982    h.max(1)
983}
984
985fn memory_type_code(t: MemoryType) -> u8 {
986    match t {
987        MemoryType::Episodic => 0,
988        MemoryType::Semantic => 1,
989        MemoryType::Procedural => 2,
990        MemoryType::Working => 3,
991    }
992}
993
994/// FMT-9b: a memory's row tags and scoring inputs (the vector location is
995/// filled in when its column entry is written).
996fn row_tags(m: &Memory) -> RowV9 {
997    let kind = match meta_str(m, MNEMO_META_KIND) {
998        Some(MNEMO_META_KIND_TURN) => ROW_KIND_TURN,
999        Some(MNEMO_META_KIND_ESSENCE) => ROW_KIND_ESSENCE,
1000        Some(MNEMO_META_KIND_THREAD_LINK) => ROW_KIND_THREAD_LINK,
1001        _ if is_onboarding(m) => ROW_KIND_ONBOARDING,
1002        _ => ROW_KIND_FACT,
1003    };
1004    let mut flags = 0u8;
1005    if m.metadata.contains_key(MNEMO_META_SUPERSEDED_BY) {
1006        flags |= ROW_SUPERSEDED;
1007    }
1008    if m.ttl_secs.is_some() {
1009        flags |= ROW_HAS_TTL;
1010    }
1011    if m.scope == Scope::Shared {
1012        flags |= ROW_SHARED;
1013    }
1014    let key_hash = meta_str(m, MNEMO_META_KEY).map_or(0, |k| {
1015        let slot = Mnemo::key_slot_for(m, k);
1016        fnv1a32(&[
1017            slot.scope_owner.as_deref().unwrap_or("").as_bytes(),
1018            slot.key.as_bytes(),
1019        ])
1020    });
1021    RowV9 {
1022        kind,
1023        flags,
1024        memory_type: memory_type_code(m.memory_type),
1025        thread_hash: meta_str(m, MNEMO_META_THREAD).map_or(0, |t| fnv1a32(&[t.as_bytes()])),
1026        key_hash,
1027        agent_hash: fnv1a32(&[m.agent_id.as_bytes()]),
1028        created_at: m.created_at,
1029        importance: m.importance,
1030        expires_at: m.ttl_secs.map_or(0, |t| m.created_at.saturating_add(t)),
1031        ..RowV9::default()
1032    }
1033}
1034
1035/// FMT-9b: the handle's view of the fp16 vector column.
1036#[derive(Debug, Default)]
1037struct VecColumn {
1038    /// Resident fp16 values by catalog row (`rows × dims`), once loaded.
1039    data: Vec<u16>,
1040    norms: Vec<f32>,
1041    /// Which rows of `data` are filled. The column is allocated zeroed for
1042    /// every row (the OS maps those pages lazily) and filled a vector page
1043    /// at a time as recall touches rows, so the first recall reads only the
1044    /// pages its candidates live on.
1045    have: Vec<bool>,
1046    resident: bool,
1047    /// Rows whose entries aren't on a page yet; the next flush writes them.
1048    /// A row here has no page entry (its old one was released).
1049    hot: std::collections::HashMap<usize, (Vec<u16>, f32)>,
1050    /// Rows to write, in order (may repeat; the flush skips repeats).
1051    pending: Vec<usize>,
1052    /// Vector page → the live rows whose entries it holds.
1053    page_rows: std::collections::HashMap<u64, Vec<usize>>,
1054}
1055
1056impl VecColumn {
1057    fn from_catalog(catalog: &[CatalogEntry]) -> VecColumn {
1058        let mut col = VecColumn::default();
1059        for (i, e) in catalog.iter().enumerate() {
1060            if e.deleted {
1061                continue;
1062            }
1063            if let Some(r) = e.v9 {
1064                if r.vec_page != 0 {
1065                    col.page_rows.entry(r.vec_page).or_default().push(i);
1066                }
1067            }
1068        }
1069        col
1070    }
1071}
1072
1073/// Frozen v4 catalog entry shape, used only by the v4→v5 migration path in
1074/// [`Mnemo::open`]. Its layout matches the catalog encoding written by
1075/// pre-v5 builds (5 positional fields). Do not change this struct.
1076#[derive(Serialize, Deserialize, Clone, Debug)]
1077struct CatalogEntryV4 {
1078    id: u128,
1079    start_page: u64,
1080    page_count: u32,
1081    len: u32,
1082    deleted: bool,
1083}
1084
1085/// Returned by [`Mnemo::prepare_for_flush`] and consumed by [`Mnemo::flush`].
1086/// Carries the already-serialized control-plane bytes through the prelude
1087/// so we don't re-serialize after the lease has been persisted.
1088struct FlushPrelude {
1089    cat_bytes: Option<Vec<u8>>,
1090    /// FMT-9c.2: `cat_bytes` is a delta run, not a base catalog.
1091    cat_delta: bool,
1092    /// FMT-9c.3: `idx_bytes` is an index delta run, not a whole index.
1093    idx_delta: bool,
1094    /// FMT-9d: this commit's BM25 segment, and the segments it replaces.
1095    bm25_seg: Option<(Vec<u8>, Option<usize>)>,
1096    idx_bytes: Option<Vec<u8>>,
1097    /// Serialized result-cache directory bytes when `dirty_cache` is
1098    /// set (Phase 10.1). `None` if the cache didn't change since the
1099    /// last flush — in that case the header keeps its current
1100    /// `cache_start` / `cache_pages` / `cache_len` and no fresh cache
1101    /// run is written.
1102    cache_bytes: Option<Vec<u8>>,
1103}
1104
1105/// One entry in the append-only snapshot manifest. Because record, catalog,
1106/// and index pages are only ever *appended*, the runs a past flush wrote are
1107/// still on disk; a `Snapshot` is the set of pointers needed to reconstruct
1108/// the database exactly as that flush left it.
1109#[derive(Serialize, Deserialize, Clone, Debug)]
1110struct Snapshot {
1111    txn_id: u64,
1112    created_at: i64,
1113    catalog_start: u64,
1114    catalog_pages: u64,
1115    catalog_len: u64,
1116    index_start: u64,
1117    index_pages: u64,
1118    index_len: u64,
1119    memory_count: u64,
1120    /// FMT-9: generations of the catalog and index runs (v9 only; `None`,
1121    /// and left out of the encoding, on v8 files).
1122    #[serde(default, skip_serializing_if = "Option::is_none")]
1123    gens: Option<SnapshotGens>,
1124}
1125
1126/// FMT-9: the generations a v9 snapshot's runs were sealed under.
1127#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default)]
1128struct SnapshotGens {
1129    catalog: u64,
1130    index: u64,
1131    /// FMT-9c.2: the catalog's delta-chain head at this snapshot.
1132    #[serde(default)]
1133    delta: (u64, u64, u64, u64),
1134    /// FMT-9c.3: the index's delta-chain head at this snapshot.
1135    #[serde(default)]
1136    index_delta: (u64, u64, u64, u64),
1137    /// FMT-9d: the BM25 manifest at this snapshot.
1138    #[serde(default)]
1139    bm25: (u64, u64, u64, u64),
1140}
1141
1142impl Snapshot {
1143    fn catalog_gen(&self) -> u64 {
1144        self.gens.map_or(0, |g| g.catalog)
1145    }
1146    fn index_gen(&self) -> u64 {
1147        self.gens.map_or(0, |g| g.index)
1148    }
1149    fn delta_head(&self) -> RunPtr {
1150        self.gens.map_or(RunPtr::default(), |g| tuple_ptr(g.delta))
1151    }
1152    fn index_delta_head(&self) -> RunPtr {
1153        self.gens
1154            .map_or(RunPtr::default(), |g| tuple_ptr(g.index_delta))
1155    }
1156}
1157
1158/// A restorable point in a database's history — see [`Mnemo::snapshots`].
1159#[derive(Clone, Copy, Debug)]
1160pub struct SnapshotInfo {
1161    /// Id of the transaction that produced this snapshot (monotonic from 1).
1162    pub txn_id: u64,
1163    /// When the snapshot was committed (unix seconds).
1164    pub created_at: i64,
1165    /// Live memory count captured in the snapshot.
1166    pub memory_count: u64,
1167}
1168
1169/// A retrieval request for [`Mnemo::recall`].
1170#[derive(Clone, Debug)]
1171pub struct RecallRequest {
1172    /// Query embedding.
1173    pub query: Vec<f32>,
1174    /// Maximum results to return.
1175    pub top_k: usize,
1176    /// Restrict to these memory types (`None` = all types).
1177    pub memory_types: Option<Vec<MemoryType>>,
1178    /// Restrict to a single agent's view: its own memories plus shared ones.
1179    pub agent_id: Option<String>,
1180    /// Similarity metric.
1181    pub metric: Metric,
1182    /// Multi-signal score weights.
1183    pub weights: ScoreWeights,
1184    /// Index override: partitions to probe (`None` = index default). Ignored
1185    /// when no ANN index is present.
1186    pub n_probe: Option<usize>,
1187    /// Index override: candidates to rerank (`None` = index default). Ignored
1188    /// when no ANN index is present.
1189    pub n_rerank: Option<usize>,
1190    /// Whether to update each result's `accessed_at` and `access_count` in
1191    /// the catalog. Defaults to `true`, matching pre-v5 behavior.
1192    ///
1193    /// Set to `false` for **fully read-only recall** — useful for batch
1194    /// scoring, dry-run scoring, or recalls run by introspection tooling
1195    /// (e.g. `mnemo about` consumers) where you don't want the score's own
1196    /// observation to perturb the database. With `false`, recall does not
1197    /// dirty the catalog and the next `flush` is a no-op.
1198    pub track_access: bool,
1199    /// Phase 12.6 pre-scoring filter: `session_id` + AND-ed metadata
1200    /// conditions applied before the candidate is scored. Default is
1201    /// [`Filter::default`] (no constraints). When the ANN index is on
1202    /// and this filter is selective enough to starve the shortlist
1203    /// below `top_k`, Phase 12.1's widen-then-exact-scan logic kicks
1204    /// in so the result still carries up to `top_k` correct hits.
1205    pub filter: Filter,
1206    /// Phase 12.8 post-scoring budget: stop adding hits once the UTF-8
1207    /// bytes of their `content` fields (not the serialized record
1208    /// body) would exceed this many bytes. `None` (default) means no
1209    /// budget and every hit through `top_k` is returned. If the top-1
1210    /// hit is already over budget it is still returned — callers that
1211    /// need an absolute cap should also pre-size `top_k`.
1212    ///
1213    /// Counts only the UTF-8 `content` bytes so the number is stable
1214    /// across the Rust / Python / CLI / MCP surfaces; the JSON or
1215    /// MessagePack envelope is excluded. Roughly 4 bytes per BPE
1216    /// token for English text — a 4 KB budget comfortably holds ~1K
1217    /// tokens.
1218    ///
1219    /// Observable through [`RecallResponse::truncated`]
1220    /// ([`Mnemo::recall_budgeted`]): `true` iff at least one scored
1221    /// hit was dropped by this budget, `false` otherwise.
1222    pub max_bytes: Option<usize>,
1223    /// FMT-9d (Phase 2, v9 files): **identifiers or exact terms to match**
1224    /// (a version, a PR number, a function name) for hybrid recall. Don't
1225    /// paste the question: the vector covers its meaning, and its common
1226    /// words would match the wrong records. BM25 scores, divided by this
1227    /// query's best, are added to each candidate's similarity with weight
1228    /// [`RecallRequest::keyword_weight`] before the usual scoring. With an
1229    /// empty `query` vector ([`RecallRequest::keywords_only`]), recall is
1230    /// keyword-only. `None` (the default) is plain vector recall.
1231    pub keywords: Option<String>,
1232    /// FMT-9d: weight ε of the normalised BM25 score in hybrid recall
1233    /// (default [`DEFAULT_KEYWORD_WEIGHT`]).
1234    pub keyword_weight: f32,
1235    /// FMT-9d: BM25 term-frequency saturation `k1` (default 1.2).
1236    pub bm25_k1: f32,
1237    /// FMT-9d: BM25 length normalisation `b` (default 0.75).
1238    pub bm25_b: f32,
1239}
1240
1241impl RecallRequest {
1242    /// A request with sensible defaults for a query embedding.
1243    pub fn new(query: Vec<f32>) -> Self {
1244        Self {
1245            query,
1246            top_k: 10,
1247            memory_types: None,
1248            agent_id: None,
1249            metric: Metric::Cosine,
1250            weights: ScoreWeights::default(),
1251            n_probe: None,
1252            n_rerank: None,
1253            track_access: true,
1254            filter: Filter::default(),
1255            max_bytes: None,
1256            keywords: None,
1257            keyword_weight: DEFAULT_KEYWORD_WEIGHT,
1258            bm25_k1: 1.2,
1259            bm25_b: 0.75,
1260        }
1261    }
1262
1263    /// FMT-9d (v9 files): a keyword-only request: BM25 over `keywords`
1264    /// (identifiers or exact terms), no query vector, so it works without
1265    /// an embedder.
1266    pub fn keywords_only(keywords: impl Into<String>) -> Self {
1267        Self::new(Vec::new()).keywords(keywords)
1268    }
1269
1270    /// FMT-9d (v9 files): add identifiers or exact terms to match (hybrid
1271    /// recall). Pass identifiers, not the question. See the `keywords`
1272    /// field.
1273    pub fn keywords(mut self, text: impl Into<String>) -> Self {
1274        self.keywords = Some(text.into());
1275        self
1276    }
1277
1278    /// FMT-9d: the weight ε of the keyword score in hybrid recall.
1279    pub fn keyword_weight(mut self, w: f32) -> Self {
1280        self.keyword_weight = w;
1281        self
1282    }
1283
1284    /// Phase 12.6: attach a pre-scoring filter. See [`Filter`] and
1285    /// [`FilterTerm`] for term shapes.
1286    pub fn filter(mut self, filter: Filter) -> Self {
1287        self.filter = filter;
1288        self
1289    }
1290
1291    /// Phase 12.8: cap the total UTF-8 bytes of returned `content`
1292    /// fields. See [`RecallRequest::max_bytes`].
1293    pub fn max_bytes(mut self, n: usize) -> Self {
1294        self.max_bytes = Some(n);
1295        self
1296    }
1297
1298    /// Build a request by embedding `query_text` with the supplied
1299    /// embedder (Phase 3). Shorthand for `embedder.embed(&[text])` +
1300    /// [`RecallRequest::new`]; same defaults as `new`. Fails if the
1301    /// embedder returns an empty vector list or an embedding failure.
1302    pub fn from_text(embedder: &dyn Embedder, query_text: &str) -> Result<Self> {
1303        let mut vectors = embedder.embed(&[query_text])?;
1304        let v = vectors.pop().ok_or_else(|| {
1305            MnemoError::Invalid("embedder returned no vector for the query text".into())
1306        })?;
1307        Ok(Self::new(v))
1308    }
1309    /// Set the result cap.
1310    pub fn top_k(mut self, k: usize) -> Self {
1311        self.top_k = k;
1312        self
1313    }
1314    /// Restrict to specific memory types.
1315    pub fn types(mut self, t: Vec<MemoryType>) -> Self {
1316        self.memory_types = Some(t);
1317        self
1318    }
1319    /// Restrict to one agent's view.
1320    pub fn agent(mut self, agent_id: impl Into<String>) -> Self {
1321        self.agent_id = Some(agent_id.into());
1322        self
1323    }
1324    /// Set the similarity metric (default: cosine).
1325    pub fn metric(mut self, metric: Metric) -> Self {
1326        self.metric = metric;
1327        self
1328    }
1329    /// Replace the multi-signal score weights.
1330    pub fn weights(mut self, weights: ScoreWeights) -> Self {
1331        self.weights = weights;
1332        self
1333    }
1334    /// Override the number of IVF partitions probed (accuracy/speed dial).
1335    pub fn n_probe(mut self, n: usize) -> Self {
1336        self.n_probe = Some(n);
1337        self
1338    }
1339    /// Override the number of candidates reranked exactly (accuracy dial).
1340    pub fn n_rerank(mut self, n: usize) -> Self {
1341        self.n_rerank = Some(n);
1342        self
1343    }
1344    /// Toggle whether recall updates `accessed_at` / `access_count` on
1345    /// the returned memories' catalog entries (default `true`). Pass
1346    /// `false` for a fully read-only recall — see the field doc.
1347    pub fn track_access(mut self, track: bool) -> Self {
1348        self.track_access = track;
1349        self
1350    }
1351}
1352
1353/// One scored result from [`Mnemo::recall`].
1354#[derive(Clone, Debug)]
1355pub struct RecallResult {
1356    /// The retrieved memory.
1357    pub memory: Memory,
1358    /// Combined multi-signal score.
1359    pub score: f32,
1360    /// The bare similarity component (before other signals).
1361    pub similarity: f32,
1362}
1363
1364/// Phase 12.8 budgeted-recall envelope returned by
1365/// [`Mnemo::recall_budgeted`]. Wraps the `Vec<RecallResult>` the
1366/// legacy [`Mnemo::recall`] returns so callers can tell whether
1367/// [`RecallRequest::max_bytes`] cut anything.
1368#[derive(Clone, Debug)]
1369pub struct RecallResponse {
1370    /// Scored hits, best first, truncated to at most `top_k` and then
1371    /// to fit the `max_bytes` budget if set.
1372    pub hits: Vec<RecallResult>,
1373    /// `true` iff the `max_bytes` budget dropped at least one hit
1374    /// that would otherwise have been returned (i.e. the result is
1375    /// shorter than what `top_k` alone would produce).
1376    pub truncated: bool,
1377    /// Phase 12.1 + 12.6 observability: `true` iff recall fell
1378    /// through to an exact scan (ANN shortlist too narrow after
1379    /// filtering, and the widen pass also came up short, so the
1380    /// index was skipped for one full-corpus pass). Monitoring on
1381    /// this flag is how callers detect selective-filter regressions
1382    /// or supersession chains outgrowing the ANN shortlist.
1383    pub used_exact_scan: bool,
1384}
1385
1386/// Where a file's bytes and an open handle's heap go (P-0 device budget,
1387/// `examples/device_budget.rs`). **Unstable diagnostic**: fields may
1388/// change in any release. File figures come from the header and the
1389/// catalog. RAM figures are buffer capacities, with hash maps estimated
1390/// from their capacity; they cover the engine's main structures, not
1391/// every allocation, and not allocator overhead.
1392#[doc(hidden)]
1393#[derive(Clone, Debug, Default, serde::Serialize)]
1394pub struct BudgetReport {
1395    pub file_bytes: u64,
1396    pub page_size: u64,
1397    /// Page 0.
1398    pub header_bytes: u64,
1399    /// Whole pages holding live records (superseded versions included).
1400    pub record_page_bytes: u64,
1401    /// Serialized bytes of those records; the rest of their pages is
1402    /// slack.
1403    pub record_payload_bytes: u64,
1404    /// Pages still holding soft-deleted records (reclaimed by compact).
1405    pub deleted_record_page_bytes: u64,
1406    pub catalog_run_bytes: u64,
1407    pub catalog_payload_bytes: u64,
1408    pub index_run_bytes: u64,
1409    pub index_payload_bytes: u64,
1410    /// First page of the persisted index run (0 = none).
1411    pub index_start_page: u64,
1412    pub wal_bytes: u64,
1413    pub snapshot_manifest_bytes: u64,
1414    pub cache_dir_run_bytes: u64,
1415    /// Everything else: superseded catalog, index and directory runs from
1416    /// earlier flushes, and freed pages, until the next compact.
1417    pub unreferenced_bytes: u64,
1418    pub live_records: u64,
1419    pub ram_catalog: u64,
1420    pub ram_id_map: u64,
1421    pub ram_ann_centroids: u64,
1422    pub ram_ann_codebook: u64,
1423    pub ram_ann_posting_ids: u64,
1424    pub ram_ann_posting_codes: u64,
1425    pub ram_ann_assignment: u64,
1426    pub ram_cache_dir: u64,
1427    pub ram_key_caches: u64,
1428    /// FMT-9b: the resident fp16 vector column as allocated (rows not yet
1429    /// touched by recall cost no physical memory).
1430    pub ram_vector_column: u64,
1431    /// Sum of the `ram_*` fields.
1432    pub ram_total: u64,
1433}
1434
1435/// Summary statistics for a database.
1436#[derive(Clone, Debug)]
1437pub struct Stats {
1438    /// Live (non-deleted) memory count.
1439    pub memories: usize,
1440    /// Tombstoned entries awaiting compaction.
1441    pub deleted: usize,
1442    /// Embedding dimensionality.
1443    pub dimensions: usize,
1444    /// File size in bytes.
1445    pub file_bytes: u64,
1446    /// Distinct agent IDs present.
1447    pub agents: Vec<String>,
1448    /// Whether pages are encrypted (always true in v1).
1449    pub encrypted: bool,
1450    /// Creation time (unix seconds).
1451    pub created_at: i64,
1452    /// ANN index shape, if an index has been built.
1453    pub index: Option<IndexInfo>,
1454    /// Current size of the write-ahead log region, in 8 KiB pages.
1455    pub wal_pages: u64,
1456    /// Fsyncs issued by the most recent successful [`Mnemo::flush`] on
1457    /// this handle. `0` on a fresh handle before any flush, or when
1458    /// `flush()` short-circuited (nothing dirty). Phase 6 instrumentation
1459    /// for the Phase 11.1 write-latency work — the planner wants this
1460    /// before deciding whether cutting fsync count is a viable
1461    /// durability-preserving fix.
1462    pub fsyncs_last_flush: u32,
1463    /// FMT-9f: the file's on-disk format (`8` or `9`). A v8 file still
1464    /// opens and writes as v8; `mnemo compact` upgrades it to v9.
1465    pub format_version: u16,
1466}
1467
1468/// Result of a [`Mnemo::compact_file`] run.
1469#[derive(Clone, Copy, Debug, Default)]
1470pub struct CompactReport {
1471    /// Live memories before compaction.
1472    pub before: usize,
1473    /// Live memories after compaction (expired ones dropped).
1474    pub after: usize,
1475    /// Live, non-expired cache entries before compaction. FIX-1
1476    /// R-001.1: compaction now preserves the result cache instead of
1477    /// silently dropping it.
1478    pub cache_before: usize,
1479    /// Live, non-expired cache entries after compaction (tombstoned
1480    /// and TTL-expired ones dropped).
1481    pub cache_after: usize,
1482}
1483
1484/// An encrypted, single-file agent memory database.
1485pub struct Mnemo {
1486    pager: Pager,
1487    header: Header,
1488    catalog: Vec<CatalogEntry>,
1489    index: IdIndex,
1490    #[allow(dead_code)]
1491    path: PathBuf,
1492    dimensions: usize,
1493    kdf: KdfParams,
1494    /// Set whenever the catalog changes; drives whether `flush` rewrites it.
1495    dirty_catalog: bool,
1496    /// Optional IVF+PQ approximate-nearest-neighbour index.
1497    ann: Option<IvfPqIndex>,
1498    /// Set whenever the in-memory ANN index differs from the persisted one.
1499    /// PERF-1: `flush` persists it only per [`IndexPersist`]'s policy.
1500    dirty_index: bool,
1501    /// PERF-1: index persistence policy state.
1502    index_persist: IndexPersist,
1503    /// PERF-1 step 3: the header points at a persisted ANN index that
1504    /// hasn't been loaded yet. It loads on first use (`ensure_ann`).
1505    ann_unloaded: bool,
1506    /// Append-only manifest of committed snapshots, oldest first. Capped
1507    /// at [`Mnemo::max_snapshots`] entries on every flush — the oldest
1508    /// entries are pruned first.
1509    manifest: Vec<Snapshot>,
1510    /// Maximum manifest entries to retain across flushes. `0` disables
1511    /// the cap (retain forever). Defaults to [`DEFAULT_MAX_SNAPSHOTS`].
1512    max_snapshots: usize,
1513    /// If `true`, this handle was opened via [`Mnemo::open_read_only`] and
1514    /// every mutating API returns [`MnemoError::ReadOnly`]. The underlying
1515    /// file was opened without write permission and holds an OS shared
1516    /// lock; the OS would reject a write() anyway, but gating in Rust
1517    /// gives callers a clean, typed error.
1518    read_only: bool,
1519    /// Result-cache directory (Phase 10.1). See [`crate::result_cache`].
1520    /// Loaded from `header.cache_start` on open; persisted on `flush`
1521    /// via the same seal_run pipeline as catalog/index/manifest.
1522    cache_dir: Vec<CacheDirectoryEntry>,
1523    /// In-memory index over `cache_dir` for O(1) `(namespace, key_hash)`
1524    /// lookups. Rebuilt on open + kept in sync on `cache_put`/`delete`.
1525    cache_idx: DirectoryIndex,
1526    /// Set whenever `cache_dir` changes; drives whether `flush` rewrites
1527    /// the cache directory run.
1528    dirty_cache: bool,
1529    /// Per-namespace eviction budgets. Missing entries fall back to
1530    /// [`CacheBudget::default`]. Not persisted (per-handle policy).
1531    cache_budgets: std::collections::HashMap<String, CacheBudget>,
1532    /// Flush policy for cache mutations. Memory writes ignore this and
1533    /// always behave as `Strict`.
1534    cache_flush_policy: CacheFlushPolicy,
1535    /// Bookkeeping for `CacheFlushPolicy::Batched` — pending mutation
1536    /// count and first-dirty timestamp.
1537    cache_batch: BatchState,
1538    /// In-memory hit/miss/eviction counters (reset on reopen).
1539    cache_counters: CounterState,
1540    /// Optional handle-scoped embedder (Phase 3, PR 10a). Registered
1541    /// via [`Mnemo::set_embedder`]; consulted by [`Mnemo::remember_text`]
1542    /// and [`RecallRequest::from_text`]. `None` means text-side APIs
1543    /// return [`MnemoError::Invalid`] with a "no embedder" message.
1544    /// Not persisted — the handle is re-registered after every open.
1545    embedder: Option<std::sync::Arc<dyn Embedder>>,
1546    /// Fsyncs issued by the most recent `flush()` call. Snapshotted
1547    /// from the thread-local fsync counter in `lib.rs::fsync_counter`:
1548    /// `flush()` reads it on entry, does its work (each `sync_all` the
1549    /// pager / wal / recovery path makes bumps the counter), and
1550    /// stores the delta here. `0` on a fresh handle before any flush,
1551    /// or on a read-only handle where `flush()` short-circuits.
1552    /// Exposed through [`Stats::fsyncs_last_flush`] for diagnostics —
1553    /// the Phase 6 planner wants this before deciding whether cutting
1554    /// fsync count is a viable write-latency win.
1555    fsyncs_last_flush: u32,
1556    /// FIX-1 R-001.8 part 2: lazy cache of the distinct agent IDs
1557    /// across every live memory, built once on first `stats()` call
1558    /// per handle. `remember()` adds to it in O(1); `delete()` can't
1559    /// cheaply tell whether the removed memory was the last of its
1560    /// agent (the agent lives in the encrypted body, not the catalog
1561    /// — pre-v9 format), so it just invalidates the set and the next
1562    /// `stats()` call rebuilds. Pre-FIX-1 every `stats()` call
1563    /// decrypted every live memory to collect agents, turning an
1564    /// otherwise-cheap diagnostic into an O(N) decrypt.
1565    agent_set: Option<std::collections::BTreeSet<String>>,
1566    /// Phase 12.1 lazy caches, built together on the first keyed op this
1567    /// handle sees (`remember_keyed`, `update` that touches `_mnemo.key`,
1568    /// or any `history(...)` call). Both are maintained incrementally
1569    /// after that. `None` → not built yet; `Some(…)` → authoritative.
1570    ///
1571    /// * `key_index`: `(scope_owner, key) → ULID bytes` of the **live**
1572    ///   record for each keyed slot. `scope_owner` is `Some(agent_id)`
1573    ///   for a `Private` scope slot and `None` for a `Shared` one, so a
1574    ///   private write never collides with a shared one.
1575    /// * `superseded`: the ULID bytes of every record that has been
1576    ///   superseded (`_mnemo.superseded_by` set on it). `recall` and
1577    ///   `about` skip these before any decrypt.
1578    ///
1579    /// Building both costs one scan over live records (~1s at 100K with
1580    /// hardware AES) and happens at most once per handle. A format bump
1581    /// to v9 moves the key into the catalog row so the scan goes away.
1582    key_index: Option<std::collections::HashMap<KeySlot, u128>>,
1583    superseded: Option<std::collections::HashSet<u128>>,
1584    /// R-004.2: thread layers + onboarding ids, built by the same lazy scan.
1585    layers: Option<LayerCache>,
1586    /// FMT-9e: thread turns are compressed once older than this many
1587    /// seconds (`0`: at write). Per handle, not persisted.
1588    turn_compress_after: i64,
1589    /// FMT-9 (a): the committed free list (v9; always empty on v8). Pages
1590    /// allocated from it since the last commit are already removed.
1591    free: FreeList,
1592    /// FMT-9 (a): runs this transaction stopped referencing (overwritten
1593    /// or deleted records, tombstoned cache entries, a relocated WAL
1594    /// region). The next commit adds them to the free list, freed at its
1595    /// own transaction id.
1596    pending_free: Vec<(u64, u64)>,
1597    /// FMT-9 (a): the free list changed (an allocation took pages from it)
1598    /// and must be rewritten by the next commit.
1599    free_dirty: bool,
1600    /// FMT-9 (b): the key-slot run (v9; empty until Phase 13.2 adds slots).
1601    key_slots: KeySlotRun,
1602    /// FMT-9 (b): `key_slots` changed since the last commit.
1603    dirty_keys: bool,
1604    /// FMT-9: this writable session has burned a transaction id with a
1605    /// durable header-only commit (`ensure_gen_lease`), so the generations
1606    /// its pages carry can't repeat ones a crashed earlier session used.
1607    gen_leased: bool,
1608    /// FMT-9b: the fp16 vector column (v9; unused on v8).
1609    col: VecColumn,
1610    /// FMT-9b.2: packed record pages (v9; unused on v8).
1611    recs: RecPages,
1612    /// FMT-9d: the BM25 keyword index (v9; unused on v8).
1613    bm25: Bm25State,
1614    /// FMT-9d: test hook: write this layout byte instead of the current
1615    /// one (to exercise the unknown-layout path).
1616    index_layout_override: Option<u8>,
1617    /// FMT-9d: during one recall, each keyword match's normalised BM25
1618    /// score, by id.
1619    kw: Option<std::collections::HashMap<u128, f32>>,
1620    /// FMT-9c.2: catalog rows changed since the last commit (v9 writes
1621    /// them as a delta run instead of the whole catalog).
1622    dirty_rows: std::collections::BTreeSet<usize>,
1623    /// FMT-9c.2: the delta runs on top of the base catalog, oldest first,
1624    /// and how many rows they hold in all.
1625    delta_chain: Vec<RunPtr>,
1626    delta_rows: usize,
1627    /// FMT-9c.3: the ANN index's delta runs on top of its base run, oldest
1628    /// first, and how many changes they hold in all.
1629    index_chain: Vec<RunPtr>,
1630    index_chain_ops: usize,
1631    /// FMT-9c.3: the next commit writes the whole index (after a build,
1632    /// rebuild or restore).
1633    index_rebase: bool,
1634    /// FMT-9c.3: this flush's index changes, when it writes them as a
1635    /// delta run.
1636    index_delta_ops: Option<Vec<IndexOp>>,
1637    /// FMT-9c.2: the next commit writes a whole base catalog (after a
1638    /// restore, or when the chain has grown too long).
1639    catalog_rewrite: bool,
1640    /// FMT-9c: the snapshot count was set explicitly (config or
1641    /// `set_max_snapshots`), which replaces v9's default retention policy.
1642    snapshot_cap_explicit: bool,
1643    /// FMT-9c: this flush wrote big runs straight to their home pages,
1644    /// which must be fsynced before the commit.
1645    direct_writes: bool,
1646    /// FMT-9b: keep the column resident while it fits this many bytes.
1647    vector_cache_budget: usize,
1648}
1649
1650/// Composite key for the lazy `key_index` on `Mnemo`. A slot is identified
1651/// by `(scope_owner, key)`:
1652/// * `Scope::Private` writes carry the owning agent → `Some(agent_id)`.
1653/// * `Scope::Shared` writes are per-file → `None`.
1654///
1655/// A private write never supersedes a shared one, and the reverse is also
1656/// true. Scope lives on `Memory.scope`, agent on `Memory.agent_id`.
1657#[derive(Debug, Clone, PartialEq, Eq, Hash)]
1658pub(crate) struct KeySlot {
1659    /// `Some(agent_id)` for `Scope::Private`, `None` for `Scope::Shared`.
1660    pub(crate) scope_owner: Option<String>,
1661    /// The caller-supplied key string, exactly as given (no case-fold).
1662    pub(crate) key: String,
1663}
1664
1665/// Open a file and take an OS advisory lock, returning `Locked { path }`
1666/// if the requested lock cannot be acquired. On drop of the returned
1667/// [`File`], the OS releases the lock. `flock`/`LockFileEx` semantics
1668/// (per file-descriptor on Unix, per-open-file-handle on Windows) mean
1669/// the lock is scoped to this handle: cloning the file (or the returned
1670/// `File`) shares the lock, but a fresh `open()` from any process gets a
1671/// distinct handle and will collide.
1672fn open_with_lock(path: &Path, write: bool, create: bool) -> Result<File> {
1673    let mut opts = OpenOptions::new();
1674    opts.read(true);
1675    if write {
1676        opts.write(true);
1677    }
1678    if create {
1679        opts.create(true).truncate(true);
1680    }
1681    let file = opts.open(path)?;
1682    // On success the returned `Result<()>` is `Ok(())`. On conflict:
1683    //   * Unix `flock(2)` → `ErrorKind::WouldBlock` (EWOULDBLOCK / EAGAIN)
1684    //   * Windows `LockFileEx` → raw OS code 33 (ERROR_LOCK_VIOLATION)
1685    //     or 32 (ERROR_SHARING_VIOLATION), both surfaced by Rust as
1686    //     `ErrorKind::Uncategorized` because `std::io::ErrorKind`
1687    //     doesn't carry a portable "locked" variant.
1688    // Anything else is a real I/O failure (permissions, invalid handle,
1689    // ...) and gets bubbled up as `MnemoError::Io`.
1690    let result = if write {
1691        FileExt::try_lock_exclusive(&file)
1692    } else {
1693        FileExt::try_lock_shared(&file)
1694    };
1695    match result {
1696        Ok(()) => Ok(file),
1697        Err(e) if is_lock_conflict(&e) => Err(MnemoError::Locked {
1698            path: path.to_path_buf(),
1699        }),
1700        Err(e) => Err(MnemoError::Io(e)),
1701    }
1702}
1703
1704/// Do two paths resolve to the same on-disk file? Used by
1705/// [`Mnemo::export_encrypted`] to refuse overwriting the live
1706/// database. On Unix we compare `(dev, ino)` via `MetadataExt`, which
1707/// catches hard links and symlinks. On Windows we compare canonicalized
1708/// paths, since there is no stable device+inode equivalent in `std`.
1709/// Returns `None` if either side can't be stat'd / canonicalized —
1710/// the caller treats that as "not the same file" so a missing `dest`
1711/// (the normal export case) proceeds.
1712fn paths_refer_to_same_file(a: &Path, b: &Path) -> Option<bool> {
1713    #[cfg(unix)]
1714    {
1715        use std::os::unix::fs::MetadataExt;
1716        let ma = std::fs::metadata(a).ok()?;
1717        let mb = std::fs::metadata(b).ok()?;
1718        Some(ma.dev() == mb.dev() && ma.ino() == mb.ino())
1719    }
1720    #[cfg(not(unix))]
1721    {
1722        let ca = std::fs::canonicalize(a).ok()?;
1723        let cb = std::fs::canonicalize(b).ok()?;
1724        Some(ca == cb)
1725    }
1726}
1727
1728/// A sibling temp-file path next to `dest`, suffixed with
1729/// `.export-tmp`. Lives in `dest`'s own parent directory so the final
1730/// [`std::fs::rename`] is same-filesystem (and therefore atomic on
1731/// every platform mnemo supports).
1732fn temp_sibling(dest: &Path) -> PathBuf {
1733    let mut buf = dest.as_os_str().to_os_string();
1734    buf.push(".export-tmp");
1735    PathBuf::from(buf)
1736}
1737
1738/// Was this I/O error the OS reporting a conflicting file lock, or a
1739/// genuine I/O failure? Platform-conditional because
1740/// [`std::io::ErrorKind`] carries no portable "resource-locked" variant
1741/// as of Rust 1.75 — Windows surfaces `ERROR_LOCK_VIOLATION` /
1742/// `ERROR_SHARING_VIOLATION` as `Uncategorized`.
1743fn is_lock_conflict(e: &std::io::Error) -> bool {
1744    if e.kind() == std::io::ErrorKind::WouldBlock {
1745        return true;
1746    }
1747    #[cfg(windows)]
1748    if let Some(code) = e.raw_os_error() {
1749        // 33 = ERROR_LOCK_VIOLATION, 32 = ERROR_SHARING_VIOLATION —
1750        // both mean "someone else holds an incompatible lock on this
1751        // region of the file". See
1752        // https://learn.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-
1753        if code == 33 || code == 32 {
1754            return true;
1755        }
1756    }
1757    false
1758}
1759
1760/// Read a run of consecutive encrypted pages and concatenate their plaintext.
1761/// `gen` is the generation the run's pointer expects (v9; `0` before).
1762fn read_run_bytes(
1763    pager: &mut Pager,
1764    start: u64,
1765    pages: u64,
1766    len: u64,
1767    gen: u64,
1768) -> Result<Vec<u8>> {
1769    let mut buf = Vec::with_capacity(len as usize);
1770    for i in 0..pages {
1771        buf.extend_from_slice(&pager.read_page(start + i, gen)?);
1772    }
1773    buf.truncate(len as usize);
1774    Ok(buf)
1775}
1776
1777/// FMT-9b: a page run read as a byte stream, one page at a time and
1778/// without caching it, so loading a large run (a 100K catalog is several
1779/// MB, its index about 17 MB) never holds the whole serialized run in
1780/// memory next to the structure being decoded from it.
1781struct RunReader<'a> {
1782    pager: &'a mut Pager,
1783    /// The first page error, kept so the caller reports it as itself
1784    /// (e.g. `PageAuthFailed`) rather than as a decode error.
1785    err: &'a std::cell::RefCell<Option<MnemoError>>,
1786    next_page: u64,
1787    end_page: u64,
1788    gen: u64,
1789    remaining: u64,
1790    buf: Vec<u8>,
1791    pos: usize,
1792}
1793
1794impl<'a> RunReader<'a> {
1795    fn new(
1796        pager: &'a mut Pager,
1797        err: &'a std::cell::RefCell<Option<MnemoError>>,
1798        start: u64,
1799        pages: u64,
1800        len: u64,
1801        gen: u64,
1802    ) -> Self {
1803        RunReader {
1804            pager,
1805            err,
1806            next_page: start,
1807            end_page: start + pages,
1808            gen,
1809            remaining: len,
1810            buf: Vec::new(),
1811            pos: 0,
1812        }
1813    }
1814}
1815
1816impl Read for RunReader<'_> {
1817    /// The decoder reads a byte or a few at a time; serve those straight
1818    /// from the current page.
1819    #[inline]
1820    fn read_exact(&mut self, out: &mut [u8]) -> std::io::Result<()> {
1821        if self.buf.len() - self.pos >= out.len() {
1822            out.copy_from_slice(&self.buf[self.pos..self.pos + out.len()]);
1823            self.pos += out.len();
1824            return Ok(());
1825        }
1826        let mut done = 0;
1827        while done < out.len() {
1828            match self.read(&mut out[done..])? {
1829                0 => return Err(std::io::ErrorKind::UnexpectedEof.into()),
1830                n => done += n,
1831            }
1832        }
1833        Ok(())
1834    }
1835
1836    #[inline]
1837    fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
1838        if self.pos == self.buf.len() {
1839            if self.remaining == 0 || self.next_page >= self.end_page {
1840                return Ok(0);
1841            }
1842            let mut page = match self.pager.read_page_uncached(self.next_page, self.gen) {
1843                Ok(p) => p,
1844                Err(e) => {
1845                    let msg = e.to_string();
1846                    *self.err.borrow_mut() = Some(e);
1847                    return Err(std::io::Error::other(msg));
1848                }
1849            };
1850            self.next_page += 1;
1851            page.truncate((self.remaining as usize).min(page.len()));
1852            self.remaining -= page.len() as u64;
1853            self.buf = page;
1854            self.pos = 0;
1855        }
1856        let n = out.len().min(self.buf.len() - self.pos);
1857        out[..n].copy_from_slice(&self.buf[self.pos..self.pos + n]);
1858        self.pos += n;
1859        Ok(n)
1860    }
1861}
1862
1863/// FMT-9b: deserialize a sequence into a `Vec` sized from the encoded
1864/// length up front (bounded by `cap`, so a hostile length can't force a
1865/// huge allocation). serde's own `Vec` impl pre-sizes only up to 1 MiB and
1866/// then doubles, which for a 100K catalog leaves a quarter of the capacity
1867/// unused and a freed half-size copy behind in the allocator.
1868struct ExactVec<T> {
1869    cap: usize,
1870    _t: std::marker::PhantomData<T>,
1871}
1872
1873impl<'de, T: Deserialize<'de>> serde::de::DeserializeSeed<'de> for ExactVec<T> {
1874    type Value = Vec<T>;
1875    fn deserialize<D: serde::Deserializer<'de>>(
1876        self,
1877        d: D,
1878    ) -> std::result::Result<Vec<T>, D::Error> {
1879        d.deserialize_seq(self)
1880    }
1881}
1882
1883impl<'de, T: Deserialize<'de>> serde::de::Visitor<'de> for ExactVec<T> {
1884    type Value = Vec<T>;
1885    fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
1886        f.write_str("a sequence")
1887    }
1888    fn visit_seq<A: serde::de::SeqAccess<'de>>(
1889        self,
1890        mut seq: A,
1891    ) -> std::result::Result<Vec<T>, A::Error> {
1892        let mut v = Vec::with_capacity(seq.size_hint().unwrap_or(0).min(self.cap));
1893        while let Some(x) = seq.next_element()? {
1894            v.push(x);
1895        }
1896        Ok(v)
1897    }
1898}
1899
1900/// Load a v5+ catalog run (an empty `pages` yields an empty catalog).
1901fn load_catalog(
1902    pager: &mut Pager,
1903    start: u64,
1904    pages: u64,
1905    len: u64,
1906    gen: u64,
1907) -> Result<Vec<CatalogEntry>> {
1908    if pages == 0 {
1909        return Ok(Vec::new());
1910    }
1911    // Sized exactly (every row encodes to at least 8 bytes, which bounds
1912    // the up-front allocation by the run's length). v8 decodes from one
1913    // buffer, as before; v9 streams the run (it's larger, and it's paired
1914    // with a resident vector column, so the transient buffer costs RSS).
1915    let cap = (len / 8) as usize;
1916    if pager.version() < VERSION_V9 {
1917        let buf = read_run_bytes(pager, start, pages, len, gen)?;
1918        let mut de = rmp_serde::Deserializer::from_read_ref(&buf);
1919        return serde::de::DeserializeSeed::deserialize(
1920            ExactVec::<CatalogEntry> {
1921                cap,
1922                _t: std::marker::PhantomData,
1923            },
1924            &mut de,
1925        )
1926        .map_err(|e| MnemoError::Serialize(e.to_string()));
1927    }
1928    let err = std::cell::RefCell::new(None);
1929    let mut de = rmp_serde::Deserializer::new(RunReader::new(pager, &err, start, pages, len, gen));
1930    let res = serde::de::DeserializeSeed::deserialize(
1931        ExactVec::<CatalogEntry> {
1932            cap,
1933            _t: std::marker::PhantomData,
1934        },
1935        &mut de,
1936    );
1937    drop(de);
1938    match (res, err.into_inner()) {
1939        (Ok(v), _) => Ok(v),
1940        (Err(_), Some(page_err)) => Err(page_err),
1941        (Err(e), None) => Err(MnemoError::Serialize(e.to_string())),
1942    }
1943}
1944
1945/// FMT-9c.2: one delta run of a v9 catalog: the rows a commit changed or
1946/// added, by position, and the previous delta in the chain.
1947#[derive(Serialize, Deserialize)]
1948struct CatalogDelta {
1949    prev: (u64, u64, u64, u64),
1950    rows: Vec<(u32, CatalogEntry)>,
1951}
1952
1953fn ptr_tuple(p: RunPtr) -> (u64, u64, u64, u64) {
1954    (p.start, p.pages, p.len, p.gen)
1955}
1956
1957fn tuple_ptr(t: (u64, u64, u64, u64)) -> RunPtr {
1958    RunPtr {
1959        start: t.0,
1960        pages: t.1,
1961        len: t.2,
1962        gen: t.3,
1963    }
1964}
1965
1966/// FMT-9c.2: walk the delta chain from `head` back to the base and replay
1967/// it onto `catalog`, oldest first. Returns the catalog, the chain (oldest
1968/// first) and its total row count. An empty `head` is a no-op (and v8).
1969fn apply_delta_chain(
1970    pager: &mut Pager,
1971    mut catalog: Vec<CatalogEntry>,
1972    head: RunPtr,
1973) -> Result<(Vec<CatalogEntry>, Vec<RunPtr>, usize)> {
1974    let mut deltas: Vec<(RunPtr, Vec<(u32, CatalogEntry)>)> = Vec::new();
1975    let mut at = head;
1976    while !at.is_empty() {
1977        if deltas.len() > 4096 {
1978            return Err(MnemoError::Invalid("catalog delta chain too long".into()));
1979        }
1980        let buf = read_run_bytes(pager, at.start, at.pages, at.len, at.gen)?;
1981        let d: CatalogDelta =
1982            rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
1983        deltas.push((at, d.rows));
1984        at = tuple_ptr(d.prev);
1985    }
1986    deltas.reverse();
1987    let mut rows = 0usize;
1988    let mut chain = Vec::with_capacity(deltas.len());
1989    for (ptr, d) in deltas {
1990        rows += d.len();
1991        chain.push(ptr);
1992        for (i, row) in d {
1993            let i = i as usize;
1994            match i.cmp(&catalog.len()) {
1995                std::cmp::Ordering::Less => catalog[i] = row,
1996                std::cmp::Ordering::Equal => catalog.push(row),
1997                std::cmp::Ordering::Greater => {
1998                    return Err(MnemoError::Invalid("catalog delta skips a row".into()))
1999                }
2000            }
2001        }
2002    }
2003    Ok((catalog, chain, rows))
2004}
2005
2006/// FMT-9c.3: one delta run of a v9 ANN index: the changes a commit made,
2007/// and the previous delta in the chain.
2008#[derive(Serialize)]
2009struct IndexDeltaRef<'a> {
2010    prev: (u64, u64, u64, u64),
2011    ops: &'a [IndexOp],
2012}
2013
2014#[derive(Deserialize)]
2015struct IndexDeltaOwned {
2016    prev: (u64, u64, u64, u64),
2017    ops: Vec<IndexOp>,
2018}
2019
2020/// FMT-9c.3: walk the index delta chain from `head` and replay it onto a
2021/// just-loaded base index, oldest first. Returns the chain and its change
2022/// count.
2023fn apply_index_chain(
2024    pager: &mut Pager,
2025    ann: &mut IvfPqIndex,
2026    head: RunPtr,
2027) -> Result<(Vec<RunPtr>, usize)> {
2028    let mut deltas: Vec<(RunPtr, Vec<IndexOp>)> = Vec::new();
2029    let mut at = head;
2030    while !at.is_empty() {
2031        if deltas.len() > 4096 {
2032            return Err(MnemoError::Invalid("index delta chain too long".into()));
2033        }
2034        let buf = read_run_bytes(pager, at.start, at.pages, at.len, at.gen)?;
2035        let d: IndexDeltaOwned =
2036            rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
2037        deltas.push((at, d.ops));
2038        at = tuple_ptr(d.prev);
2039    }
2040    deltas.reverse();
2041    let mut chain = Vec::with_capacity(deltas.len());
2042    let mut n = 0usize;
2043    for (ptr, ops) in deltas {
2044        n += ops.len();
2045        chain.push(ptr);
2046        ann.apply_ops(ops)?;
2047    }
2048    Ok((chain, n))
2049}
2050
2051/// FMT-9f: the runs of an index delta chain whose base didn't load (a
2052/// layout this build doesn't know), oldest first, without replaying them,
2053/// so the next `build_index` or `drop_index` frees them with the base.
2054/// Best effort: a delta this build can't decode ends the walk, and the
2055/// runs before it stay unreferenced until the next compaction.
2056fn index_chain_runs(pager: &mut Pager, head: RunPtr) -> Vec<RunPtr> {
2057    let mut runs = Vec::new();
2058    let mut at = head;
2059    while !at.is_empty() && runs.len() <= 4096 {
2060        let Ok(buf) = read_run_bytes(pager, at.start, at.pages, at.len, at.gen) else {
2061            break;
2062        };
2063        let Ok(d) = rmp_serde::from_slice::<IndexDeltaOwned>(&buf) else {
2064            break;
2065        };
2066        runs.push(at);
2067        at = tuple_ptr(d.prev);
2068    }
2069    runs.reverse();
2070    runs
2071}
2072
2073/// Load a **v4** catalog run using the frozen [`CatalogEntryV4`] shape.
2074/// Used only by the v4→v5 migration path in [`Mnemo::open`].
2075fn load_catalog_v4(
2076    pager: &mut Pager,
2077    start: u64,
2078    pages: u64,
2079    len: u64,
2080) -> Result<Vec<CatalogEntryV4>> {
2081    if pages == 0 {
2082        return Ok(Vec::new());
2083    }
2084    let buf = read_run_bytes(pager, start, pages, len, 0)?;
2085    rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))
2086}
2087
2088/// Walk a v4 catalog and synthesize the v5 shape. For each live entry,
2089/// reads the memory's serialized body and copies its `accessed_at` and
2090/// `access_count` into the catalog row (the v4 record body still carries
2091/// them — they're moving *from* the body *to* the catalog). Deleted
2092/// entries skip the read and zero the fields.
2093fn migrate_v4_catalog(pager: &mut Pager, v4: Vec<CatalogEntryV4>) -> Result<Vec<CatalogEntry>> {
2094    let mut out = Vec::with_capacity(v4.len());
2095    for e in &v4 {
2096        let (accessed_at, access_count) = if e.deleted {
2097            (0i64, 0u32)
2098        } else {
2099            let mut buf = Vec::with_capacity(e.len as usize);
2100            for i in 0..e.page_count as u64 {
2101                buf.extend_from_slice(&pager.read_page(e.start_page + i, 0)?);
2102            }
2103            buf.truncate(e.len as usize);
2104            let mem: Memory = rmp_serde::from_slice(&buf)
2105                .map_err(|err| MnemoError::Serialize(err.to_string()))?;
2106            (mem.accessed_at, mem.access_count)
2107        };
2108        out.push(CatalogEntry {
2109            id: e.id,
2110            start_page: e.start_page,
2111            page_count: e.page_count,
2112            len: e.len,
2113            deleted: e.deleted,
2114            accessed_at,
2115            access_count,
2116            gen: 0,
2117            v9: None,
2118        });
2119    }
2120    Ok(out)
2121}
2122
2123/// Load the snapshot manifest (an empty `pages` yields an empty manifest).
2124fn load_manifest(
2125    pager: &mut Pager,
2126    start: u64,
2127    pages: u64,
2128    len: u64,
2129    gen: u64,
2130) -> Result<Vec<Snapshot>> {
2131    if pages == 0 {
2132        return Ok(Vec::new());
2133    }
2134    let buf = read_run_bytes(pager, start, pages, len, gen)?;
2135    rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))
2136}
2137
2138/// Load the result-cache directory (v8+). An empty `pages` yields an
2139/// empty directory, which is the natural state for v7 files migrated
2140/// to v8 (no cache entries yet) and for freshly created v8 files.
2141fn load_cache_directory(
2142    pager: &mut Pager,
2143    start: u64,
2144    pages: u64,
2145    len: u64,
2146    gen: u64,
2147) -> Result<Vec<CacheDirectoryEntry>> {
2148    if pages == 0 {
2149        return Ok(Vec::new());
2150    }
2151    let buf = read_run_bytes(pager, start, pages, len, gen)?;
2152    rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))
2153}
2154
2155/// Load an ANN index run, validating its dimensionality.
2156fn load_index(
2157    pager: &mut Pager,
2158    start: u64,
2159    pages: u64,
2160    len: u64,
2161    gen: u64,
2162    dims: usize,
2163) -> Result<Option<IvfPqIndex>> {
2164    if pages == 0 {
2165        return Ok(None);
2166    }
2167    // v8 decodes from one buffer, as before. v9 streams the run (a 100K
2168    // index is ~17 MB), trading decode speed for not holding it twice.
2169    if pager.version() < VERSION_V9 {
2170        let buf = read_run_bytes(pager, start, pages, len, gen)?;
2171        let mut idx: IvfPqIndex =
2172            rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
2173        if idx.dims() != dims {
2174            return Err(MnemoError::Invalid(
2175                "ANN index dimensionality does not match the database".into(),
2176            ));
2177        }
2178        idx.rebuild_assignment();
2179        return Ok(Some(idx));
2180    }
2181    // v9 index runs start with a layout byte. An index is derived data, so
2182    // a layout this build doesn't know means "no index" (recall scans
2183    // exactly; `rebuild_index` writes a current one), never an error.
2184    let err = std::cell::RefCell::new(None);
2185    let mut rd = RunReader::new(pager, &err, start, pages, len, gen);
2186    let mut layout = [0u8; 1];
2187    if rd.read_exact(&mut layout).is_err() {
2188        drop(rd);
2189        return match err.into_inner() {
2190            Some(page_err) => Err(page_err),
2191            None => Ok(None),
2192        };
2193    }
2194    if layout[0] != V9_INDEX_LAYOUT {
2195        return Ok(None);
2196    }
2197    let res: std::result::Result<IvfPqIndex, _> = rmp_serde::from_read(rd);
2198    let mut idx = match (res, err.into_inner()) {
2199        (Ok(i), _) => i,
2200        (Err(_), Some(page_err)) => return Err(page_err),
2201        (Err(e), None) => return Err(MnemoError::Serialize(e.to_string())),
2202    };
2203    if idx.dims() != dims {
2204        return Err(MnemoError::Invalid(
2205            "ANN index dimensionality does not match the database".into(),
2206        ));
2207    }
2208    idx.rebuild_assignment();
2209    Ok(Some(idx))
2210}
2211
2212/// FMT-9 (a): load the v9 free list (an empty pointer yields an empty list).
2213fn load_free_list(pager: &mut Pager, ptr: RunPtr) -> Result<FreeList> {
2214    if ptr.is_empty() {
2215        return Ok(FreeList::default());
2216    }
2217    let buf = read_run_bytes(pager, ptr.start, ptr.pages, ptr.len, ptr.gen)?;
2218    let runs: Vec<crate::freelist::FreeRun> =
2219        rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
2220    FreeList::from_runs(runs)
2221}
2222
2223/// FMT-9 (b): load the v9 key-slot run (an empty pointer yields no slots).
2224fn load_key_slots(pager: &mut Pager, ptr: RunPtr) -> Result<KeySlotRun> {
2225    if ptr.is_empty() {
2226        return Ok(KeySlotRun::default());
2227    }
2228    let buf = read_run_bytes(pager, ptr.start, ptr.pages, ptr.len, ptr.gen)?;
2229    let run: KeySlotRun =
2230        rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
2231    run.validate()?;
2232    Ok(run)
2233}
2234
2235/// True if a memory's metadata marks it as the canonical onboarding manifest
2236/// (`metadata.topic == "manifest"`). Used by [`Mnemo::about`] to hoist the
2237/// manifest to the top of the briefing regardless of importance ordering.
2238fn is_manifest(m: &Memory) -> bool {
2239    m.metadata
2240        .get("topic")
2241        .and_then(|v| v.as_str())
2242        .map(|s| s.eq_ignore_ascii_case("manifest"))
2243        .unwrap_or(false)
2244}
2245
2246/// PERF-1 step 4: ULID → catalog row, as an open-addressing table of `u32`
2247/// row numbers: 4 bytes a slot, about 5 bytes an entry, against about 43
2248/// for the `HashMap<u128, usize>` it replaces. A probe compares against the
2249/// catalog's own id at that row, so the ids aren't stored twice. Rows are
2250/// never removed (a delete only tombstones the catalog entry), so there's
2251/// no delete path.
2252#[derive(Debug, Clone, Default)]
2253struct IdIndex {
2254    slots: Vec<u32>,
2255    len: usize,
2256}
2257
2258const NO_ROW: u32 = u32::MAX;
2259
2260impl IdIndex {
2261    fn build(catalog: &[CatalogEntry]) -> IdIndex {
2262        let mut t = IdIndex {
2263            slots: vec![NO_ROW; crate::index::table_capacity_for(catalog.len())],
2264            len: 0,
2265        };
2266        for (row, e) in catalog.iter().enumerate() {
2267            t.insert(e.id, row, catalog);
2268        }
2269        t
2270    }
2271
2272    /// The catalog row holding `id`, if any.
2273    fn get(&self, id: u128, catalog: &[CatalogEntry]) -> Option<usize> {
2274        if self.slots.is_empty() {
2275            return None;
2276        }
2277        let cap = self.slots.len();
2278        let mut i = crate::index::slot_home(id, cap);
2279        loop {
2280            let row = self.slots[i];
2281            if row == NO_ROW {
2282                return None;
2283            }
2284            if catalog[row as usize].id == id {
2285                return Some(row as usize);
2286            }
2287            i = crate::index::next_slot(i, cap);
2288        }
2289    }
2290
2291    /// Map `id` to `row`. `catalog` must hold every row already in the
2292    /// table (the new row may not be pushed yet).
2293    fn insert(&mut self, id: u128, row: usize, catalog: &[CatalogEntry]) {
2294        if (self.len + 1) * 4 > self.slots.len() * 3 {
2295            let cap = (self.slots.len() * 2).max(crate::index::table_capacity_for(self.len + 1));
2296            let old = std::mem::replace(&mut self.slots, vec![NO_ROW; cap]);
2297            self.len = 0;
2298            for r in old.into_iter().filter(|&r| r != NO_ROW) {
2299                self.insert(catalog[r as usize].id, r as usize, catalog);
2300            }
2301        }
2302        let cap = self.slots.len();
2303        let mut i = crate::index::slot_home(id, cap);
2304        loop {
2305            let r = self.slots[i];
2306            if r == NO_ROW {
2307                self.slots[i] = row as u32;
2308                self.len += 1;
2309                return;
2310            }
2311            if catalog[r as usize].id == id {
2312                self.slots[i] = row as u32;
2313                return;
2314            }
2315            i = crate::index::next_slot(i, cap);
2316        }
2317    }
2318
2319    fn heap_bytes(&self) -> usize {
2320        self.slots.capacity() * std::mem::size_of::<u32>()
2321    }
2322}
2323
2324/// Build the ULID → catalog-row lookup.
2325fn build_id_index(catalog: &[CatalogEntry]) -> IdIndex {
2326    IdIndex::build(catalog)
2327}
2328
2329// ---------------------------------------------------------------------------
2330// Phase 12.7a helpers used by `ingest_thread`.
2331// ---------------------------------------------------------------------------
2332
2333/// One incoming turn after chunking or truncation, ready to embed and
2334/// write (Phase 12.7a, split out for 12.7d's batch embed).
2335struct PendingTurn {
2336    turn_id: String,
2337    role: String,
2338    created_at: i64,
2339    parts: Vec<String>,
2340    /// Hex sha256 + original length, set only for a truncated
2341    /// tool-result turn. Applied to the single chunk the truncation
2342    /// yielded.
2343    truncation: Option<(String, u64)>,
2344}
2345
2346/// Output of `Mnemo::plan_ingest`: the turns to write, the first free
2347/// `_mnemo.seq`, and the counts gathered so far (skips, chunking,
2348/// truncation).
2349struct IngestPlan {
2350    pending: Vec<PendingTurn>,
2351    next_seq: u64,
2352    report: IngestReport,
2353}
2354
2355impl IngestPlan {
2356    /// Every text to embed, in write order (turns, then parts).
2357    fn texts(&self) -> Vec<&str> {
2358        self.pending
2359            .iter()
2360            .flat_map(|p| p.parts.iter().map(String::as_str))
2361            .collect()
2362    }
2363
2364    fn text_count(&self) -> usize {
2365        self.pending.iter().map(|p| p.parts.len()).sum()
2366    }
2367}
2368
2369/// PERF-1 / FMT-9: was this catalog row written after the persisted index
2370/// run? v8 allocates pages monotonically, so a record past the run's first
2371/// page is newer. v9 reuses pages, so it compares generations instead.
2372fn written_after_index(e: &CatalogEntry, v9: bool, index_start: u64, index_gen: u64) -> bool {
2373    if v9 {
2374        // `start_page` 0: packed at the next flush, so certainly newer.
2375        e.start_page == 0 || e.gen > index_gen
2376    } else {
2377        e.start_page > index_start
2378    }
2379}
2380
2381/// Whether a record belongs in the ANN index: facts only. Thread-archive
2382/// turns, essence items and thread links are read by kind and scope, never
2383/// by similarity, so they stay out (R-004.2).
2384fn is_ann_kind(m: &Memory) -> bool {
2385    !matches!(
2386        m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str()),
2387        Some(MNEMO_META_KIND_TURN)
2388            | Some(MNEMO_META_KIND_ESSENCE)
2389            | Some(MNEMO_META_KIND_THREAD_LINK)
2390    )
2391}
2392
2393/// Hex-encode a byte slice. Low-dependency alternative to pulling in a
2394/// hex crate for the lone call site in `ingest_thread`'s SHA-256
2395/// recording.
2396fn hex_encode(bytes: &[u8]) -> String {
2397    const HEX: &[u8; 16] = b"0123456789abcdef";
2398    let mut out = String::with_capacity(bytes.len() * 2);
2399    for b in bytes {
2400        out.push(HEX[(b >> 4) as usize] as char);
2401        out.push(HEX[(b & 0x0f) as usize] as char);
2402    }
2403    out
2404}
2405
2406/// Return the longest UTF-8 prefix of `s` whose byte length is `<=
2407/// max_bytes`. Always lands on a `char` boundary.
2408fn take_utf8_prefix(s: &str, max_bytes: usize) -> String {
2409    if s.len() <= max_bytes {
2410        return s.to_string();
2411    }
2412    // Walk forward through char indices; the last index whose byte
2413    // offset fits inside `max_bytes` tells us where to cut.
2414    let mut cut = 0usize;
2415    for (idx, _) in s.char_indices() {
2416        if idx > max_bytes {
2417            break;
2418        }
2419        cut = idx;
2420    }
2421    s[..cut].to_string()
2422}
2423
2424/// Split `s` into chunks, each with `<= max_bytes` of UTF-8 content,
2425/// landing on `char` boundaries. Preserves original ordering; a trailing
2426/// chunk may be shorter than `max_bytes`.
2427/// Phase 12.7b helper: pull the `_mnemo.status` string from an
2428/// essence record. Returns `None` for timeline entries (they have no
2429/// status) and for malformed records.
2430fn status_of(m: &Memory) -> Option<&str> {
2431    m.metadata.get(MNEMO_META_STATUS).and_then(|v| v.as_str())
2432}
2433
2434/// Phase 12.7b helper: render one essence item as a markdown bullet.
2435/// Timeline entries get a date tag; section items get `[open]` /
2436/// `[resolved]` so the reader can scan active work quickly.
2437fn render_essence_item(section: EssenceSection, m: &Memory) -> String {
2438    if section == EssenceSection::Timeline {
2439        let date = format_unix_date(m.created_at);
2440        format!("- [{date}] {}\n", m.content)
2441    } else {
2442        let status = status_of(m).unwrap_or(essence_status::OPEN);
2443        format!("- [{status}] {}\n", m.content)
2444    }
2445}
2446
2447/// Minimal unix-seconds → `YYYY-MM-DD` formatter (no chrono/time
2448/// dependency; mirrors the ISO 8601 parser in `transcript.rs`).
2449fn format_unix_date(secs: i64) -> String {
2450    let days_total = secs.div_euclid(86_400);
2451    let (y, m, d) = civil_from_days(days_total);
2452    format!("{y:04}-{m:02}-{d:02}")
2453}
2454
2455/// Howard Hinnant's `civil_from_days` — inverse of `days_from_civil`,
2456/// adequate for Gregorian dates since 1970. Pure arithmetic, no
2457/// dependencies.
2458fn civil_from_days(z: i64) -> (i64, u32, u32) {
2459    let z = z + 719_468;
2460    let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
2461    let doe = (z - era * 146_097) as u64;
2462    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
2463    let y = yoe as i64 + era * 400;
2464    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
2465    let mp = (5 * doy + 2) / 153;
2466    let d = (doy - (153 * mp + 2) / 5 + 1) as u32;
2467    let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32;
2468    let y = if m <= 2 { y + 1 } else { y };
2469    (y, m, d)
2470}
2471
2472fn chunk_utf8(s: &str, max_bytes: usize) -> Vec<String> {
2473    if s.len() <= max_bytes {
2474        return vec![s.to_string()];
2475    }
2476    let mut out = Vec::new();
2477    let mut rest = s;
2478    while rest.len() > max_bytes {
2479        let head = take_utf8_prefix(rest, max_bytes);
2480        let head_bytes = head.len();
2481        out.push(head);
2482        rest = &rest[head_bytes..];
2483    }
2484    if !rest.is_empty() {
2485        out.push(rest.to_string());
2486    }
2487    out
2488}
2489
2490impl Mnemo {
2491    /// Create a brand-new encrypted database at `path`.
2492    pub fn create(
2493        path: impl Into<PathBuf>,
2494        passphrase: &str,
2495        config: MnemoConfig,
2496    ) -> Result<Mnemo> {
2497        Self::create_secret(path, passphrase.as_bytes(), config)
2498    }
2499
2500    /// PERF-1 step 7 (R-004.5): create a database unlocked by a **raw
2501    /// 256-bit key** instead of a passphrase, for keys held by an OS
2502    /// keychain or keystore. The key must be uniformly random (see
2503    /// [`Mnemo::generate_key`]); it gets no stretching. The header records
2504    /// minimal Argon2id parameters ([`KdfParams::raw_key`]), so opening
2505    /// costs microseconds instead of tens of milliseconds. `config.kdf` is
2506    /// ignored. No format change: the key goes through the same KEK
2507    /// derivation a passphrase does.
2508    pub fn create_with_key(
2509        path: impl Into<PathBuf>,
2510        key: &[u8; crypto::KEY_LEN],
2511        config: MnemoConfig,
2512    ) -> Result<Mnemo> {
2513        let config = MnemoConfig {
2514            kdf: KdfParams::raw_key(),
2515            ..config
2516        };
2517        Self::create_secret(path, key, config)
2518    }
2519
2520    /// A fresh random 256-bit key for [`Mnemo::create_with_key`], from the
2521    /// thread-local CSPRNG (seeded from the OS). Store it in a keychain;
2522    /// it can't be recovered from the file.
2523    pub fn generate_key() -> [u8; crypto::KEY_LEN] {
2524        *crypto::random_dek()
2525    }
2526
2527    fn create_secret(
2528        path: impl Into<PathBuf>,
2529        secret: &[u8],
2530        config: MnemoConfig,
2531    ) -> Result<Mnemo> {
2532        let path: PathBuf = path.into();
2533        if config.dimensions == 0 {
2534            return Err(MnemoError::Invalid("dimensions must be > 0".into()));
2535        }
2536        let version = resolve_format_version(config.format_version)?;
2537        if version >= VERSION_V9 && veccol::per_page(config.dimensions) == 0 {
2538            return Err(MnemoError::Invalid(format!(
2539                "a v9 file supports up to {} dimensions for now (one fp16 vector \
2540                 must fit a page); vector entries that span pages are not supported yet",
2541                (PAYLOAD - veccol::PAGE_HEADER - 4) / 2
2542            )));
2543        }
2544        let flags = if version >= VERSION_V9 && config.store_float_vectors {
2545            FLAG_ENCRYPTED | FLAG_FLOAT_VECTORS
2546        } else {
2547            FLAG_ENCRYPTED
2548        };
2549        // Exclusive lock covers the write path; second writer sees `Locked`.
2550        let file = open_with_lock(&path, /*write=*/ true, /*create=*/ true)?;
2551
2552        let salt = crypto::random_salt();
2553        let dek = crypto::random_dek();
2554        let kek = crypto::derive_kek(secret, &salt, config.kdf)?;
2555        let dek_nonce = crypto::random_nonce();
2556        let wrapped = crypto::wrap_dek(&kek, &dek_nonce, &dek)?;
2557        if wrapped.len() != WRAPPED_DEK_LEN {
2558            return Err(MnemoError::Crypto("unexpected wrapped DEK length".into()));
2559        }
2560        let mut wrapped_dek = [0u8; WRAPPED_DEK_LEN];
2561        wrapped_dek.copy_from_slice(&wrapped);
2562
2563        let mut header = Header {
2564            version,
2565            page_size: PAGE_SIZE as u32,
2566            flags,
2567            dimensions: config.dimensions as u32,
2568            created_at: memory::now_secs(),
2569            write_counter: 0,
2570            // Page 0 is the header; pages 1..=wal_pages are the WAL;
2571            // record/catalog/index pages start after it. Initial WAL size is
2572            // chosen per-config (default 8 pages = 64 KiB) and clamped to a
2573            // sane floor so the first transaction can always commit.
2574            next_page: 1 + config.wal_pages_initial.max(MIN_WAL_PAGES),
2575            catalog_start: 0,
2576            catalog_pages: 0,
2577            catalog_len: 0,
2578            vector_count: 0,
2579            m_cost: config.kdf.m_cost,
2580            t_cost: config.kdf.t_cost,
2581            p_cost: config.kdf.p_cost,
2582            salt,
2583            dek_nonce,
2584            wrapped_dek,
2585            index_start: 0,
2586            index_pages: 0,
2587            index_len: 0,
2588            wal_start: 1,
2589            wal_pages: config.wal_pages_initial.max(MIN_WAL_PAGES),
2590            wal_seq: 0,
2591            manifest_start: 0,
2592            manifest_pages: 0,
2593            manifest_len: 0,
2594            // Regenerated per write by `apply_seal`; `None` until we seal.
2595            seal_nonce: None,
2596            seal_tag: None,
2597            // v8+ result-cache pointer — zero-initialized on create,
2598            // populated only when `cache_put` runs at least once and
2599            // the next `flush` seals a directory run.
2600            cache_start: 0,
2601            cache_pages: 0,
2602            cache_len: 0,
2603            // FMT-9 v9 fields; all zero on a fresh file (and unused on v8).
2604            catalog_gen: 0,
2605            index_gen: 0,
2606            manifest_gen: 0,
2607            cache_gen: 0,
2608            free: RunPtr::default(),
2609            keys: RunPtr::default(),
2610            bm25: RunPtr::default(),
2611            index_delta: RunPtr::default(),
2612            catalog_delta: RunPtr::default(),
2613        };
2614
2615        let mut pager = Pager::new(file, dek, 0, version);
2616        let mut page = header.to_page();
2617        header.apply_seal(&mut page, pager.dek())?;
2618        pager.write_raw(0, &page)?;
2619        pager.sync()?;
2620
2621        Ok(Mnemo {
2622            pager,
2623            header,
2624            catalog: Vec::new(),
2625            index: IdIndex::default(),
2626            path,
2627            dimensions: config.dimensions,
2628            kdf: config.kdf,
2629            dirty_catalog: false,
2630            ann: None,
2631            dirty_index: false,
2632            index_persist: IndexPersist::default(),
2633            ann_unloaded: false,
2634            manifest: Vec::new(),
2635            // FMT-9c: v9 keeps fewer snapshots by default: with page reuse,
2636            // retention is what pins pages.
2637            max_snapshots: if version >= VERSION_V9 && config.max_snapshots == DEFAULT_MAX_SNAPSHOTS
2638            {
2639                V9_DEFAULT_MAX_SNAPSHOTS
2640            } else {
2641                config.max_snapshots
2642            },
2643            read_only: false,
2644            cache_dir: Vec::new(),
2645            cache_idx: DirectoryIndex::new(),
2646            dirty_cache: false,
2647            cache_budgets: std::collections::HashMap::new(),
2648            cache_flush_policy: CacheFlushPolicy::default(),
2649            cache_batch: BatchState::default(),
2650            cache_counters: CounterState::default(),
2651            embedder: None,
2652            fsyncs_last_flush: 0,
2653            agent_set: None,
2654            key_index: None,
2655            superseded: None,
2656            layers: None,
2657            turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
2658            free: FreeList::default(),
2659            pending_free: Vec::new(),
2660            free_dirty: false,
2661            key_slots: KeySlotRun::default(),
2662            dirty_keys: false,
2663            gen_leased: false,
2664            col: VecColumn::default(),
2665            recs: RecPages::default(),
2666            bm25: Bm25State::default(),
2667            kw: None,
2668            index_layout_override: None,
2669            dirty_rows: std::collections::BTreeSet::new(),
2670            delta_chain: Vec::new(),
2671            delta_rows: 0,
2672            catalog_rewrite: false,
2673            index_chain: Vec::new(),
2674            index_chain_ops: 0,
2675            index_rebase: false,
2676            index_delta_ops: None,
2677            snapshot_cap_explicit: config.max_snapshots != DEFAULT_MAX_SNAPSHOTS,
2678            direct_writes: false,
2679            vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
2680        })
2681    }
2682
2683    /// Open an existing database. A wrong passphrase fails cleanly with
2684    /// [`MnemoError::WrongPassphrase`]. Takes an OS exclusive lock for
2685    /// the lifetime of the returned handle; a second concurrent open
2686    /// (from any process) returns [`MnemoError::Locked`].
2687    ///
2688    /// For a caller that only needs to read (and can tolerate the
2689    /// pending-recovery / pending-migration cases documented on
2690    /// [`Mnemo::open_read_only`]), prefer that entry point — it takes a
2691    /// shared lock and coexists with other readers.
2692    pub fn open(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
2693        Self::open_secret(path, passphrase.as_bytes())
2694    }
2695
2696    /// PERF-1 step 7: open a database created by [`Mnemo::create_with_key`]
2697    /// (or rekeyed with [`Mnemo::rekey_with_key`]) with its raw key.
2698    pub fn open_with_key(path: impl Into<PathBuf>, key: &[u8; crypto::KEY_LEN]) -> Result<Mnemo> {
2699        Self::open_secret(path, key)
2700    }
2701
2702    fn open_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
2703        let path: PathBuf = path.into();
2704        let mut file = open_with_lock(&path, /*write=*/ true, /*create=*/ false)?;
2705
2706        let mut hbuf = [0u8; PAGE_SIZE];
2707        file.seek(SeekFrom::Start(0))?;
2708        file.read_exact(&mut hbuf)?;
2709        let mut header = match Header::from_page(&hbuf) {
2710            Ok(h) => h,
2711            Err(e) => {
2712                // DUR-1 (R-002.1 part 2+3): page 0 did not parse — any
2713                // error is treated as a torn-header candidate, not just
2714                // `HeaderChecksum`. The whole 294-byte header lives in
2715                // the first 512-byte sector, so the realistic failure is
2716                // the sector zeroed or garbled, which returns `BadMagic`
2717                // or `UnsupportedVersion` *before* the CRC gate.
2718                //
2719                // Scan the file for valid committed header-frame WAL
2720                // regions, pick the highest `txn_id`, and replay it. If
2721                // nothing valid is found, return the *original* error so
2722                // non-mnemo files still report `BadMagic`.
2723                let scan = scan_header_recovery_candidates(&mut file, secret)?;
2724                let frames = scan.ok_or(e)?;
2725                for (page_no, bytes) in &frames {
2726                    if bytes.len() != PAGE_SIZE {
2727                        return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
2728                    }
2729                    file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
2730                    file.write_all(bytes)?;
2731                }
2732                file.sync_all()?;
2733                crate::bump_fsync();
2734                file.seek(SeekFrom::Start(0))?;
2735                file.read_exact(&mut hbuf)?;
2736                Header::from_page(&hbuf)?
2737            }
2738        };
2739
2740        // Crash recovery: replay a committed-but-uncheckpointed transaction.
2741        // Each frame is a finished page image; the header frame, replayed to
2742        // page 0, supersedes the header just read.
2743        if let Some(frames) = wal::recover(
2744            &mut file,
2745            header.wal_start,
2746            header.wal_pages,
2747            header.wal_seq,
2748        )? {
2749            for (page_no, bytes) in &frames {
2750                if bytes.len() != PAGE_SIZE {
2751                    return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
2752                }
2753                file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
2754                file.write_all(bytes)?;
2755            }
2756            file.sync_all()?;
2757            crate::bump_fsync();
2758            // Re-read the now-current header from the replayed page 0.
2759            file.seek(SeekFrom::Start(0))?;
2760            file.read_exact(&mut hbuf)?;
2761            header = Header::from_page(&hbuf)?;
2762        }
2763
2764        let kdf = KdfParams {
2765            m_cost: header.m_cost,
2766            t_cost: header.t_cost,
2767            p_cost: header.p_cost,
2768        };
2769        let kek = crypto::derive_kek(secret, &header.salt, kdf)?;
2770        let dek = crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek)?;
2771
2772        // Validate the v7 header seal now that we have the DEK. Returns
2773        // `MnemoError::HeaderTampered` if any of the sealed mutable fields
2774        // were rewritten after the last legitimate flush. No-op for v6 and
2775        // older — pre-v7 files don't carry a seal.
2776        //
2777        // DUR-1 (Planner decision 2026-11-01, Option B): a seal failure is
2778        // **always** reported as tampering, never auto-healed. R-002.1
2779        // part 3's seal-mismatch WAL fallback is withdrawn: the seal
2780        // exists for tamper-evidence; silently loading a stale committed
2781        // header from the WAL would erase that evidence. A real crash
2782        // cannot produce "CRC valid + seal invalid" either, because the
2783        // whole 294-byte header lies inside sector 0 which storage
2784        // writes atomically — a torn sector breaks magic, version or
2785        // CRC first, and *those* are healed by the scan above.
2786        header.validate_seal(&dek)?;
2787
2788        // Build the pager at the file's *current* version so initial reads
2789        // use the matching AAD scheme — `Pager::page_aad` returns empty AAD
2790        // for v4/v5 (no page binding) and `page_no.to_le_bytes()` for v6+.
2791        let on_disk_version = header.version;
2792        let mut pager = Pager::new(file, dek, header.write_counter, on_disk_version);
2793        let dimensions = header.dimensions as usize;
2794
2795        // === v4→v5 catalog migration =====================================
2796        // v6 reads directly; v5 reads directly; v4 replays the catalog into
2797        // the v5 shape by reading each memory's body to populate the new
2798        // `accessed_at` / `access_count` fields. All page reads here still
2799        // use the v4/v5 AAD scheme (none).
2800        let (catalog, migrated_from_v4) = if on_disk_version >= 5 {
2801            (
2802                load_catalog(
2803                    &mut pager,
2804                    header.catalog_start,
2805                    header.catalog_pages,
2806                    header.catalog_len,
2807                    header.catalog_gen,
2808                )?,
2809                false,
2810            )
2811        } else if on_disk_version == 4 {
2812            let v4 = load_catalog_v4(
2813                &mut pager,
2814                header.catalog_start,
2815                header.catalog_pages,
2816                header.catalog_len,
2817            )?;
2818            (migrate_v4_catalog(&mut pager, v4)?, true)
2819        } else {
2820            return Err(MnemoError::UnsupportedVersion(on_disk_version));
2821        };
2822        // FMT-9c.2: replay the v9 delta chain onto the base catalog.
2823        let (catalog, delta_chain, delta_rows) =
2824            apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
2825        let index = build_id_index(&catalog);
2826        // PERF-1 step 3: the ANN index loads on first use, not at open. A
2827        // v5→v6 page migration is the exception: it must read the index
2828        // now, under the old page AAD, to rewrite it.
2829        let ann = if on_disk_version < 6 {
2830            load_index(
2831                &mut pager,
2832                header.index_start,
2833                header.index_pages,
2834                header.index_len,
2835                header.index_gen,
2836                dimensions,
2837            )?
2838        } else {
2839            None
2840        };
2841        let ann_unloaded = ann.is_none() && header.index_pages > 0;
2842        let mut manifest = load_manifest(
2843            &mut pager,
2844            header.manifest_start,
2845            header.manifest_pages,
2846            header.manifest_len,
2847            header.manifest_gen,
2848        )?;
2849
2850        // === v5→v6 page-encryption migration =============================
2851        // Pre-v6 data pages were encrypted under no AAD; v6 binds the home
2852        // `page_no` as AAD so a page-transplant attack fails authentication
2853        // on read. Re-stage every live record page through `write_page` so
2854        // the next flush re-encrypts it under v6 AAD at the same home slot.
2855        // Tombstoned entries skip — they're reclaimed by `compact_file`.
2856        // Old catalog / ANN / manifest pages stay as orphan ciphertext;
2857        // the next flush writes the new runs at fresh slots.
2858        let migrated_pages_to_v6 = on_disk_version < 6;
2859        if migrated_pages_to_v6 {
2860            for e in &catalog {
2861                if e.deleted {
2862                    continue;
2863                }
2864                for i in 0..e.page_count as u64 {
2865                    let payload = pager.read_page(e.start_page + i, 0)?;
2866                    pager.write_page(e.start_page + i, 0, &payload)?;
2867                }
2868            }
2869            pager.set_version(6);
2870        }
2871
2872        // === v6→v7 header-seal migration =================================
2873        // v7 adds an AES-GCM seal at the tail of the header page that
2874        // authenticates every mutable header field. Pre-v7 files have no
2875        // seal — we just bump the version in memory and mark `dirty_catalog`
2876        // so the next flush writes a sealed v7 header. Pages are untouched
2877        // (page crypto is unchanged from v6).
2878        let migrated_header_to_v7 = on_disk_version < 7;
2879        if migrated_header_to_v7 {
2880            pager.set_version(7);
2881        }
2882
2883        // === v7→v8 cache-directory migration =============================
2884        // v8 adds three u64 header fields (cache_start/pages/len) for the
2885        // result-cache directory (Phase 10.1). Pre-v8 files have those
2886        // three bytes as zero (the on-disk region beyond the v7 seal is
2887        // zero-initialized), which is the legal "empty cache directory"
2888        // state — no data migration required. We bump the version in
2889        // memory and mark dirty_catalog so the next flush writes a
2890        // v8-sealed header with the new AAD shape covering the cache
2891        // pointer. Pages are untouched.
2892        let migrated_header_to_v8 = on_disk_version < 8;
2893        if migrated_header_to_v8 {
2894            pager.set_version(VERSION);
2895        }
2896
2897        let migrated = migrated_from_v4
2898            || migrated_pages_to_v6
2899            || migrated_header_to_v7
2900            || migrated_header_to_v8;
2901        if migrated {
2902            // Stamp the version forward in memory; the next flush persists
2903            // it via the WAL-committed header frame (and the v7 seal).
2904            header.version = VERSION;
2905            // Snapshots written under an older format reference page runs
2906            // encrypted under the old AAD scheme; this build can't decrypt
2907            // them under the new pager. Drop the manifest so the next
2908            // flush records a fresh snapshot of the migrated state.
2909            // Point-in-time recovery into the pre-migration past is
2910            // sacrificed for migration simplicity. Old pages remain on
2911            // disk until `compact_file`. (For v6→v7 alone — which doesn't
2912            // change page crypto — the old snapshots would technically
2913            // still be readable, but dropping them uniformly keeps the
2914            // migration policy simple.)
2915            manifest.clear();
2916        }
2917
2918        // The ANN index pages on disk were sealed under the *old* AAD scheme
2919        // if we migrated pages to v6; force a rewrite at fresh v6-encrypted
2920        // slots on the next flush so a future reopen doesn't try to read
2921        // them under the new AAD and fail. (Catalog and manifest already
2922        // get rewritten via dirty_catalog and the manifest.clear() above.)
2923        let dirty_index = migrated_pages_to_v6 && ann.is_some();
2924
2925        // Load the cache directory (v8+ only). Pre-v8 files have
2926        // cache_pages == 0 by header layout, so this trivially yields
2927        // an empty directory without touching the disk.
2928        let cache_dir = load_cache_directory(
2929            &mut pager,
2930            header.cache_start,
2931            header.cache_pages,
2932            header.cache_len,
2933            header.cache_gen,
2934        )?;
2935        let cache_idx = DirectoryIndex::build(&cache_dir);
2936        // FMT-9 (a)/(b): the free list and key-slot run (empty pointers on
2937        // v8, so these don't touch the disk there).
2938        let free = load_free_list(&mut pager, header.free)?;
2939        let key_slots = load_key_slots(&mut pager, header.keys)?;
2940        let col = VecColumn::from_catalog(&catalog);
2941        let recs = RecPages::from_catalog(&catalog);
2942        let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
2943        let max_snapshots = if header.version >= VERSION_V9 {
2944            V9_DEFAULT_MAX_SNAPSHOTS
2945        } else {
2946            DEFAULT_MAX_SNAPSHOTS
2947        };
2948
2949        Ok(Mnemo {
2950            pager,
2951            header,
2952            catalog,
2953            index,
2954            path,
2955            dimensions,
2956            kdf,
2957            dirty_catalog: migrated,
2958            ann,
2959            dirty_index,
2960            // A v5→v6 page migration leaves the index sealed under the old
2961            // page AAD, so it must be rewritten on the next flush.
2962            index_persist: IndexPersist {
2963                forced: dirty_index,
2964                ..IndexPersist::default()
2965            },
2966            ann_unloaded,
2967            manifest,
2968            // `Mnemo::open` has no [`MnemoConfig`] to read; default the cap
2969            // to [`DEFAULT_MAX_SNAPSHOTS`]. Override with
2970            // [`Mnemo::set_max_snapshots`] right after open if you need
2971            // unlimited retention or a different cap for this handle.
2972            max_snapshots,
2973            read_only: false,
2974            cache_dir,
2975            cache_idx,
2976            dirty_cache: false,
2977            cache_budgets: std::collections::HashMap::new(),
2978            cache_flush_policy: CacheFlushPolicy::default(),
2979            cache_batch: BatchState::default(),
2980            cache_counters: CounterState::default(),
2981            embedder: None,
2982            fsyncs_last_flush: 0,
2983            agent_set: None,
2984            key_index: None,
2985            superseded: None,
2986            layers: None,
2987            turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
2988            free,
2989            pending_free: Vec::new(),
2990            free_dirty: false,
2991            key_slots,
2992            dirty_keys: false,
2993            gen_leased: false,
2994            col,
2995            recs,
2996            bm25: Bm25State {
2997                manifest: bm25_manifest,
2998                ..Bm25State::default()
2999            },
3000            kw: None,
3001            index_layout_override: None,
3002            dirty_rows: std::collections::BTreeSet::new(),
3003            delta_chain,
3004            delta_rows,
3005            catalog_rewrite: false,
3006            index_chain: Vec::new(),
3007            index_chain_ops: 0,
3008            index_rebase: false,
3009            index_delta_ops: None,
3010            snapshot_cap_explicit: false,
3011            direct_writes: false,
3012            vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
3013        })
3014    }
3015
3016    /// Open an existing database in **read-only** mode. Takes an OS shared
3017    /// lock; multiple read-only handles can coexist, but any attempt to
3018    /// open the same file read-write elsewhere returns
3019    /// [`MnemoError::Locked`]. All mutating methods on the returned handle
3020    /// return [`MnemoError::ReadOnly`], including `recall` with
3021    /// `track_access(true)`.
3022    ///
3023    /// Two conditions block a read-only open and return
3024    /// [`MnemoError::NeedsWriteOpen`]:
3025    ///
3026    /// 1. The file has a committed-but-uncheckpointed WAL transaction that
3027    ///    would ordinarily be replayed on open. A read-only handle
3028    ///    cannot write the recovered pages back.
3029    /// 2. The on-disk format version is older than the current
3030    ///    `format::VERSION`, requiring an on-disk migration.
3031    ///
3032    /// In either case, open the file read-write once (which recovers /
3033    /// migrates in place), close the handle cleanly, and reopen
3034    /// read-only. This is documented in the README's Durability section.
3035    pub fn open_read_only(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
3036        Self::open_read_only_secret(path, passphrase.as_bytes())
3037    }
3038
3039    /// PERF-1 step 7: [`Mnemo::open_read_only`] for a raw-key database.
3040    pub fn open_read_only_with_key(
3041        path: impl Into<PathBuf>,
3042        key: &[u8; crypto::KEY_LEN],
3043    ) -> Result<Mnemo> {
3044        Self::open_read_only_secret(path, key)
3045    }
3046
3047    fn open_read_only_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
3048        let path: PathBuf = path.into();
3049        let mut file = open_with_lock(&path, /*write=*/ false, /*create=*/ false)?;
3050
3051        let mut hbuf = [0u8; PAGE_SIZE];
3052        file.seek(SeekFrom::Start(0))?;
3053        file.read_exact(&mut hbuf)?;
3054        // A torn header on a read-only open cannot be healed (that requires
3055        // writing recovered frames back). Surface as `NeedsWriteOpen` so the
3056        // caller knows to reopen read-write once. DUR-1 (R-002.1 part 4):
3057        // any `Header::from_page` failure (`BadMagic`,
3058        // `UnsupportedVersion`, `HeaderChecksum`) qualifies — but only
3059        // when the file-wide scan actually finds a valid committed
3060        // header frame to recover from, so a non-mnemo file still
3061        // reports `BadMagic` cleanly. A `validate_seal` failure is NOT
3062        // routed here (Option B: tampering is never auto-healed).
3063        let header = match Header::from_page(&hbuf) {
3064            Ok(h) => h,
3065            Err(parse_err) => {
3066                let recoverable = scan_header_recovery_candidates(&mut file, secret)?;
3067                return Err(match (parse_err, recoverable) {
3068                    (_, Some(_)) => MnemoError::NeedsWriteOpen {
3069                        reason: "torn header page needs recovery from WAL",
3070                    },
3071                    (e, None) => e,
3072                });
3073            }
3074        };
3075
3076        // Refuse to migrate on read-only opens. v6→v7 is header-only
3077        // (no page writes required), but even that bumps `version` on
3078        // disk and re-seals the header — which needs write access. Keep
3079        // the policy uniform across all sub-version bumps.
3080        if header.version < VERSION {
3081            return Err(MnemoError::NeedsWriteOpen {
3082                reason: "pending format-version migration",
3083            });
3084        }
3085
3086        // Refuse if there is a committed-but-uncheckpointed transaction
3087        // waiting for `wal::recover` to fold it into home pages.
3088        if wal::recover(
3089            &mut file,
3090            header.wal_start,
3091            header.wal_pages,
3092            header.wal_seq,
3093        )?
3094        .is_some()
3095        {
3096            return Err(MnemoError::NeedsWriteOpen {
3097                reason: "pending WAL recovery",
3098            });
3099        }
3100
3101        let kdf = KdfParams {
3102            m_cost: header.m_cost,
3103            t_cost: header.t_cost,
3104            p_cost: header.p_cost,
3105        };
3106        let kek = crypto::derive_kek(secret, &header.salt, kdf)?;
3107        let dek = crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek)?;
3108        header.validate_seal(&dek)?;
3109
3110        let mut pager = Pager::new(file, dek, header.write_counter, header.version);
3111        let dimensions = header.dimensions as usize;
3112
3113        let catalog = load_catalog(
3114            &mut pager,
3115            header.catalog_start,
3116            header.catalog_pages,
3117            header.catalog_len,
3118            header.catalog_gen,
3119        )?;
3120        // FMT-9c.2: replay the v9 delta chain onto the base catalog.
3121        let (catalog, delta_chain, delta_rows) =
3122            apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
3123        let index = build_id_index(&catalog);
3124        // PERF-1 step 3: the ANN index loads on first use, not at open.
3125        let ann: Option<IvfPqIndex> = None;
3126        let ann_unloaded = header.index_pages > 0;
3127        let manifest = load_manifest(
3128            &mut pager,
3129            header.manifest_start,
3130            header.manifest_pages,
3131            header.manifest_len,
3132            header.manifest_gen,
3133        )?;
3134        let cache_dir = load_cache_directory(
3135            &mut pager,
3136            header.cache_start,
3137            header.cache_pages,
3138            header.cache_len,
3139            header.cache_gen,
3140        )?;
3141        let cache_idx = DirectoryIndex::build(&cache_dir);
3142        let free = load_free_list(&mut pager, header.free)?;
3143        let key_slots = load_key_slots(&mut pager, header.keys)?;
3144        let col = VecColumn::from_catalog(&catalog);
3145        let recs = RecPages::from_catalog(&catalog);
3146        let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
3147        let max_snapshots = if header.version >= VERSION_V9 {
3148            V9_DEFAULT_MAX_SNAPSHOTS
3149        } else {
3150            DEFAULT_MAX_SNAPSHOTS
3151        };
3152
3153        Ok(Mnemo {
3154            pager,
3155            header,
3156            catalog,
3157            index,
3158            path,
3159            dimensions,
3160            kdf,
3161            // No mutation is possible; leave both dirty flags false so
3162            // `close()` is a genuine no-op even if we later add an
3163            // auto-flush on drop.
3164            dirty_catalog: false,
3165            ann,
3166            dirty_index: false,
3167            index_persist: IndexPersist::default(),
3168            ann_unloaded,
3169            manifest,
3170            max_snapshots,
3171            read_only: true,
3172            cache_dir,
3173            cache_idx,
3174            dirty_cache: false,
3175            cache_budgets: std::collections::HashMap::new(),
3176            cache_flush_policy: CacheFlushPolicy::default(),
3177            cache_batch: BatchState::default(),
3178            cache_counters: CounterState::default(),
3179            embedder: None,
3180            fsyncs_last_flush: 0,
3181            agent_set: None,
3182            key_index: None,
3183            superseded: None,
3184            layers: None,
3185            turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
3186            free,
3187            pending_free: Vec::new(),
3188            free_dirty: false,
3189            key_slots,
3190            dirty_keys: false,
3191            gen_leased: false,
3192            col,
3193            recs,
3194            bm25: Bm25State {
3195                manifest: bm25_manifest,
3196                ..Bm25State::default()
3197            },
3198            kw: None,
3199            index_layout_override: None,
3200            dirty_rows: std::collections::BTreeSet::new(),
3201            delta_chain,
3202            delta_rows,
3203            catalog_rewrite: false,
3204            index_chain: Vec::new(),
3205            index_chain_ops: 0,
3206            index_rebase: false,
3207            index_delta_ops: None,
3208            snapshot_cap_explicit: false,
3209            direct_writes: false,
3210            vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
3211        })
3212    }
3213
3214    /// FMT-9e: compress v9 thread-turn records once they're older than
3215    /// `secs` (by their `created_at`). The default,
3216    /// [`DEFAULT_TURN_COMPRESS_AFTER_SECS`], is 7 days: recent turns stay
3217    /// plain, so a live session reads them at full speed, and each commit
3218    /// compresses up to 256 turns that have aged past the cutoff. `0`
3219    /// compresses every turn as it's written. Compaction always compresses
3220    /// every turn, and so does a vacuum that moves one. Applies to this
3221    /// handle only (not persisted); a no-op on v8 files, which never
3222    /// compress. Reading never depends on it.
3223    pub fn set_turn_compress_after(&mut self, secs: u64) {
3224        self.turn_compress_after = secs.min(i64::MAX as u64) as i64;
3225    }
3226
3227    /// Override the manifest snapshot cap on this open handle. `0` disables
3228    /// the cap; any positive value keeps the most-recent `max` snapshots and
3229    /// drops the rest on the next flush. See [`MnemoConfig::max_snapshots`].
3230    pub fn set_max_snapshots(&mut self, max: usize) {
3231        self.max_snapshots = max;
3232        // FMT-9c: an explicit count is a hard cap on v9 (see
3233        // `apply_retention`).
3234        self.snapshot_cap_explicit = true;
3235    }
3236
3237    /// Embedding dimensionality this database expects.
3238    pub fn dimensions(&self) -> usize {
3239        self.dimensions
3240    }
3241
3242    /// Number of live (non-deleted) memories.
3243    pub fn len(&self) -> usize {
3244        self.catalog.iter().filter(|e| !e.deleted).count()
3245    }
3246
3247    /// True if the database holds no live memories.
3248    pub fn is_empty(&self) -> bool {
3249        self.len() == 0
3250    }
3251
3252    // --- internal page/record helpers -------------------------------------
3253
3254    /// FMT-9: true for a v9 file (page reuse, generations in the AAD).
3255    fn is_v9(&self) -> bool {
3256        self.header.version >= VERSION_V9
3257    }
3258
3259    /// FMT-9: the generation for pages written now: the id of the
3260    /// transaction that will commit them, or a lower bound on it (a WAL
3261    /// relocation inside the next flush can push the commit id higher).
3262    /// Either way it's strictly greater than the generation any earlier
3263    /// write to a reused page carried, which is all rollback protection
3264    /// needs: a page is reused only after the transaction that freed it
3265    /// committed, and that transaction came after the one that wrote it.
3266    /// `0` on v8, whose page AAD has no generation.
3267    fn write_gen(&self) -> u64 {
3268        if self.is_v9() {
3269            self.header.wal_seq + 1
3270        } else {
3271            0
3272        }
3273    }
3274
3275    /// FMT-9 (a): the oldest retained snapshot's transaction id
3276    /// (`u64::MAX` when none is retained). Only pages freed at or before it
3277    /// may be reused.
3278    fn min_snapshot_txn(&self) -> u64 {
3279        self.manifest
3280            .iter()
3281            .map(|s| s.txn_id)
3282            .min()
3283            .unwrap_or(u64::MAX)
3284    }
3285
3286    /// Allocate `n` contiguous pages. v9 takes them from the free list when
3287    /// an eligible run fits (freed by a committed transaction, and not
3288    /// referenced by a retained snapshot); otherwise, and always on v8, the
3289    /// file grows.
3290    fn alloc_pages(&mut self, n: u64) -> u64 {
3291        if self.is_v9() {
3292            let committed = self.header.wal_seq;
3293            let min_snap = self.min_snapshot_txn();
3294            if let Some(p) = self.free.alloc(n, committed, min_snap) {
3295                self.free_dirty = true;
3296                return p;
3297            }
3298        }
3299        let p = self.header.next_page;
3300        self.header.next_page += n;
3301        p
3302    }
3303
3304    /// FMT-9 (a): record that the current transaction stopped referencing
3305    /// `n` pages from `page`. The next commit frees them. No-op on v8,
3306    /// which never reuses a page.
3307    fn release_pages(&mut self, page: u64, n: u64) {
3308        if self.is_v9() && n > 0 && page != 0 {
3309            self.pending_free.push((page, n));
3310        }
3311    }
3312
3313    /// FMT-9d: make sure the live-document stats and the segment dicts are
3314    /// loaded. Stats come from the rows; dicts from each segment's head.
3315    fn ensure_bm25(&mut self) -> Result<()> {
3316        self.ensure_bm25_stats();
3317        self.ensure_bm25_dicts()
3318    }
3319
3320    /// FMT-9d: the live-document stats, from the rows (once per handle, or
3321    /// after a catalog rebase moves rows; `put` and `delete` keep them
3322    /// current, FMT-9f).
3323    fn ensure_bm25_stats(&mut self) {
3324        if self.bm25.stats.is_none() {
3325            let mut docs = std::collections::HashMap::new();
3326            let mut total = 0u64;
3327            for (i, e) in self.catalog.iter().enumerate() {
3328                if e.deleted {
3329                    continue;
3330                }
3331                if let Some(r) = e.v9 {
3332                    if r.doc != 0 && bm25_kind(r.kind) {
3333                        docs.insert(r.doc, (i, r.doc_len));
3334                        total += r.doc_len as u64;
3335                    }
3336                }
3337            }
3338            let mut stats = DocStats {
3339                docs,
3340                avgdl: 1.0,
3341                total,
3342            };
3343            stats.refresh();
3344            self.bm25.stats = Some(stats);
3345        }
3346    }
3347
3348    /// FMT-9d: the segment dicts, read from each segment's head once; after
3349    /// that a commit adds or replaces only the segments it wrote (FMT-9f).
3350    fn ensure_bm25_dicts(&mut self) -> Result<()> {
3351        if self.bm25.dicts.is_none() {
3352            let mut dicts = Vec::with_capacity(self.bm25.manifest.segments.len());
3353            for &t in &self.bm25.manifest.segments {
3354                dicts.push(load_segment_dict(&mut self.pager, tuple_ptr(t))?);
3355            }
3356            self.bm25.dicts = Some(dicts);
3357        }
3358        Ok(())
3359    }
3360
3361    /// FMT-9d: BM25 scores of the live keyword-indexed rows that match
3362    /// `query`: catalog row → score. Empty on v8 (no keyword index).
3363    fn bm25_scores(
3364        &mut self,
3365        query: &str,
3366        k1: f32,
3367        b: f32,
3368    ) -> Result<std::collections::HashMap<usize, f32>> {
3369        let mut scores: std::collections::HashMap<usize, f32> = std::collections::HashMap::new();
3370        if !self.is_v9() {
3371            return Ok(scores);
3372        }
3373        self.ensure_bm25()?;
3374        let terms: std::collections::BTreeSet<String> = text::tokenize(query).into_iter().collect();
3375        let segs: Vec<RunPtr> = self
3376            .bm25
3377            .manifest
3378            .segments
3379            .iter()
3380            .map(|&t| tuple_ptr(t))
3381            .collect();
3382        let stats = self.bm25.stats.as_ref().expect("loaded above");
3383        let dicts = self.bm25.dicts.as_ref().expect("loaded above");
3384        let n = stats.docs.len();
3385        for term in &terms {
3386            let mut posts: Vec<(u32, u8)> = Vec::new();
3387            for (si, seg) in segs.iter().enumerate() {
3388                posts.extend(segment_postings(&mut self.pager, *seg, &dicts[si], term)?);
3389            }
3390            for (doc, tfs) in &self.bm25.pending {
3391                if let Ok(i) = tfs.binary_search_by(|(t, _)| t.as_str().cmp(term)) {
3392                    posts.push((*doc, tfs[i].1));
3393                }
3394            }
3395            posts.retain(|(d, _)| stats.docs.contains_key(d));
3396            if posts.is_empty() {
3397                continue;
3398            }
3399            let idf = text::idf(n, posts.len());
3400            for (d, tf) in posts {
3401                let (row, len) = stats.docs[&d];
3402                *scores.entry(row).or_default() += text::bm25(tf, idf, len, stats.avgdl, k1, b);
3403            }
3404        }
3405        Ok(scores)
3406    }
3407
3408    /// FMT-9d: the next commit's BM25 segment: the pending documents, plus,
3409    /// past [`BM25_MAX_SEGMENTS`], the small segments (all but the first),
3410    /// or all of them once the small ones outgrow a tenth of the first.
3411    /// Dead documents are dropped from merged segments. Returns the segment
3412    /// bytes and, for a merge, the index from which it replaces the
3413    /// segments (`None`: it's appended).
3414    fn bm25_next_segment(&mut self) -> Result<Option<(Vec<u8>, Option<usize>)>> {
3415        if !self.is_v9() || self.bm25.pending.is_empty() {
3416            return Ok(None);
3417        }
3418        let segs: Vec<RunPtr> = self
3419            .bm25
3420            .manifest
3421            .segments
3422            .iter()
3423            .map(|&t| tuple_ptr(t))
3424            .collect();
3425        // Mostly dead (a mass delete): merge everything, so the dead
3426        // postings go and the old segments' pages can be reused or cut off.
3427        // FMT-9f: only the stats; the dicts aren't needed to write a segment.
3428        self.ensure_bm25_stats();
3429        let written: u64 = self.bm25.manifest.seg_docs.iter().map(|&d| d as u64).sum();
3430        let live_docs = self.bm25.stats.as_ref().map_or(0, |s| s.docs.len()) as u64;
3431        let mostly_dead = !segs.is_empty() && written > 64 && live_docs * 2 < written;
3432        if segs.len() < BM25_MAX_SEGMENTS && !mostly_dead {
3433            return Ok(Some((text::build_segment(&self.bm25.pending)?, None)));
3434        }
3435        let small: u64 = segs.iter().skip(1).map(|p| p.len).sum();
3436        let from = if mostly_dead || small * 10 > segs[0].len {
3437            0
3438        } else {
3439            1
3440        };
3441        let live: std::collections::HashSet<u32> = self
3442            .bm25
3443            .stats
3444            .as_ref()
3445            .expect("loaded above")
3446            .docs
3447            .keys()
3448            .copied()
3449            .collect();
3450        let mut by_doc: std::collections::BTreeMap<u32, Vec<(String, u8)>> =
3451            std::collections::BTreeMap::new();
3452        for seg in &segs[from..] {
3453            let bytes = read_run_bytes(&mut self.pager, seg.start, seg.pages, seg.len, seg.gen)?;
3454            let n = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as usize;
3455            let dict: Vec<text::DictEntry> = rmp_serde::from_slice(&bytes[4..4 + n])
3456                .map_err(|e| MnemoError::Serialize(e.to_string()))?;
3457            let post = &bytes[4 + n..];
3458            for e in dict {
3459                let range = e.off as usize..(e.off + e.len) as usize;
3460                let list = text::decode_postings(post.get(range).ok_or_else(|| {
3461                    MnemoError::Invalid("BM25 dict entry past its postings".into())
3462                })?)?;
3463                for (d, tf) in list {
3464                    if live.contains(&d) {
3465                        by_doc.entry(d).or_default().push((e.term.clone(), tf));
3466                    }
3467                }
3468            }
3469        }
3470        for (d, tfs) in &self.bm25.pending {
3471            if live.contains(d) {
3472                by_doc.insert(*d, tfs.clone());
3473            }
3474        }
3475        let docs: Vec<(u32, Vec<(String, u8)>)> = by_doc.into_iter().collect();
3476        self.bm25.merged_docs = docs.len() as u32;
3477        Ok(Some((text::build_segment(&docs)?, Some(from))))
3478    }
3479
3480    /// FMT-9c.3: the generation the persisted index is current to: its
3481    /// base run's, or its newest delta run's.
3482    fn index_gen_now(&self) -> u64 {
3483        self.header.index_gen.max(self.header.index_delta.gen)
3484    }
3485
3486    /// FMT-9c.3: should this v9 flush persist the index, and how? Every
3487    /// commit that changed it writes those changes as a delta run (so the
3488    /// index is never stale after a crash); a build, rebuild, drop or
3489    /// restore, or a chain past 64 runs or a quarter of the index, writes
3490    /// the whole index instead. Sets `index_delta_ops` for a delta.
3491    fn plan_index_v9(&mut self) -> bool {
3492        self.index_delta_ops = None;
3493        let Some(ann) = self.ann.as_mut() else {
3494            // Dropped (persisted as "no index"), or not loaded (unchanged).
3495            return self.dirty_index;
3496        };
3497        let ops = ann.take_log();
3498        let n = ann.len();
3499        if self.index_rebase {
3500            return true;
3501        }
3502        if ops.is_empty() {
3503            return false;
3504        }
3505        if self.index_chain.len() >= 64 || (self.index_chain_ops + ops.len()) * 4 > n.max(256) {
3506            return true;
3507        }
3508        self.index_delta_ops = Some(ops);
3509        true
3510    }
3511
3512    /// FMT-9c.2: catalog row `idx` changed (v9 writes it in the next
3513    /// commit's delta run). No-op on v8, which rewrites the whole catalog.
3514    fn touch(&mut self, idx: usize) {
3515        if self.is_v9() {
3516            self.dirty_rows.insert(idx);
3517        }
3518    }
3519
3520    /// FMT-9c.2: write this commit's catalog change as a delta run on the
3521    /// chain, rather than a new base? Not after a restore, not before the
3522    /// first base, and not once the chain has 64 runs or holds more than a
3523    /// quarter of the base's rows (the next base then folds it in).
3524    fn catalog_as_delta(&self) -> bool {
3525        self.is_v9()
3526            && !self.catalog_rewrite
3527            && self.header.catalog_pages > 0
3528            && self.delta_chain.len() < 64
3529            && (self.delta_rows + self.dirty_rows.len()) * 4 <= self.catalog.len()
3530    }
3531
3532    /// FMT-9c.2: before writing a new base, drop deleted rows once they're
3533    /// more than a quarter of the catalog. Rows move, so the id index and
3534    /// the page maps are rebuilt, and the resident column reloads lazily.
3535    /// Runs inside a flush, after packing, when nothing is waiting.
3536    fn compact_tombstones(&mut self) {
3537        let dead = self.catalog.iter().filter(|e| e.deleted).count();
3538        if dead == 0 || dead * 4 <= self.catalog.len() {
3539            return;
3540        }
3541        self.catalog.retain(|e| !e.deleted);
3542        self.catalog.shrink_to_fit();
3543        self.index = build_id_index(&self.catalog);
3544        self.col = VecColumn::from_catalog(&self.catalog);
3545        self.recs = RecPages::from_catalog(&self.catalog);
3546        self.dirty_rows.clear();
3547        self.bm25.stats = None;
3548    }
3549
3550    /// FMT-9b: the file keeps float vectors in its records.
3551    fn float_vectors(&self) -> bool {
3552        self.is_v9() && self.header.flags & FLAG_FLOAT_VECTORS != 0
3553    }
3554
3555    /// FMT-9b: quantize `v` into the column for catalog row `idx` (written
3556    /// by the next flush).
3557    fn stage_vector(&mut self, idx: usize, v: &[f32]) {
3558        let (h, norm) = veccol::quantize(v);
3559        let d = self.dimensions;
3560        if self.col.resident {
3561            let rows = self.catalog.len();
3562            let fits = rows * veccol::entry_len(d) <= self.vector_cache_budget;
3563            if fits {
3564                if self.col.data.len() < rows * d {
3565                    self.col.data.resize(rows * d, 0);
3566                    self.col.norms.resize(rows, 0.0);
3567                    self.col.have.resize(rows, false);
3568                }
3569                self.col.data[idx * d..(idx + 1) * d].copy_from_slice(&h);
3570                self.col.norms[idx] = norm;
3571                self.col.have[idx] = true;
3572            } else {
3573                // Grown past the budget: page the column from now on.
3574                self.col.data = Vec::new();
3575                self.col.norms = Vec::new();
3576                self.col.have = Vec::new();
3577                self.col.resident = false;
3578            }
3579        }
3580        self.col.hot.insert(idx, (h, norm));
3581        self.col.pending.push(idx);
3582        self.touch(idx);
3583        if let Some(r) = self.catalog[idx].v9.as_mut() {
3584            r.vec_page = 0;
3585            r.vec_slot = 0;
3586            r.vec_gen = 0;
3587        }
3588    }
3589
3590    /// FMT-9b: row `idx` no longer needs its column entry (`row` is the row
3591    /// as it was): drop a staged one, or take it off its page and free the
3592    /// page once nothing live is left on it.
3593    fn kill_vector(&mut self, idx: usize, row: &CatalogEntry) {
3594        self.col.hot.remove(&idx);
3595        let Some(r) = row.v9 else { return };
3596        if r.vec_page == 0 {
3597            return;
3598        }
3599        let empty = match self.col.page_rows.get_mut(&r.vec_page) {
3600            Some(rows) => {
3601                rows.retain(|&i| i != idx);
3602                rows.is_empty()
3603            }
3604            None => false,
3605        };
3606        if empty {
3607            self.col.page_rows.remove(&r.vec_page);
3608            self.release_pages(r.vec_page, 1);
3609        }
3610    }
3611
3612    /// FMT-9b.2: row `idx` no longer needs its record (`row` is the row as
3613    /// it was): drop a waiting one, free a span, or take it off its packed
3614    /// page and free the page once nothing live is left on it.
3615    /// FMT-9f: the rerank depth for an ANN query: the caller's pin, else
3616    /// on v9 at least [`V9_RERANK_PER_HIT`] × `top_k`, else the index's
3617    /// own default (`None`).
3618    fn rerank_for(&self, ann: &IvfPqIndex, req: &RecallRequest) -> Option<usize> {
3619        if req.n_rerank.is_some() || !self.is_v9() {
3620            return req.n_rerank;
3621        }
3622        let (_, auto) = ann.effective_defaults();
3623        Some(auto.max(V9_RERANK_PER_HIT.saturating_mul(req.top_k)))
3624    }
3625
3626    /// FMT-9e: is a turn created at `created_at` past this handle's
3627    /// compression cutoff?
3628    fn turn_is_cold(&self, created_at: i64) -> bool {
3629        self.turn_compress_after == 0
3630            || created_at <= memory::now_secs().saturating_sub(self.turn_compress_after)
3631    }
3632
3633    /// FMT-9e: is row `idx` a live, unsettled v9 turn?
3634    fn unsettled_turn(&self, idx: usize) -> bool {
3635        let e = &self.catalog[idx];
3636        !e.deleted
3637            && e.v9
3638                .is_some_and(|r| r.kind == ROW_KIND_TURN && r.flags & ROW_SETTLED == 0)
3639    }
3640
3641    /// FMT-9e: settle row `idx`'s turn record, whose stored bytes are
3642    /// `bytes`: compress them if it pays, and mark the row settled.
3643    /// Returns the bytes to store (unchanged if not worth compressing);
3644    /// the caller writes them. Updates the row's length.
3645    fn settle_turn(&mut self, idx: usize, bytes: Vec<u8>) -> Vec<u8> {
3646        let out = if bytes.starts_with(&ZSTD_MAGIC) {
3647            bytes
3648        } else {
3649            compress_turn_record(bytes)
3650        };
3651        let e = &mut self.catalog[idx];
3652        e.len = out.len() as u32;
3653        if let Some(r) = e.v9.as_mut() {
3654            r.flags |= ROW_SETTLED;
3655        }
3656        self.touch(idx);
3657        self.dirty_catalog = true;
3658        out
3659    }
3660
3661    /// FMT-9e: compress up to [`COLD_TURNS_PER_COMMIT`] plain turns that
3662    /// have aged past the cutoff. Runs at the top of a flush that's
3663    /// already writing, before [`Mnemo::pack_records`], which packs the
3664    /// rewritten records.
3665    fn settle_cold_turns(&mut self) -> Result<()> {
3666        if !self.is_v9() || !self.dirty_catalog {
3667            return Ok(());
3668        }
3669        let picked: Vec<usize> = (0..self.catalog.len())
3670            .filter(|&i| {
3671                self.unsettled_turn(i)
3672                    && self.catalog[i]
3673                        .v9
3674                        .is_some_and(|r| self.turn_is_cold(r.created_at))
3675            })
3676            .take(COLD_TURNS_PER_COMMIT)
3677            .collect();
3678        for idx in picked {
3679            let old = self.catalog[idx].clone();
3680            let bytes = self.read_record(&old)?;
3681            let before = bytes.len();
3682            let out = self.settle_turn(idx, bytes);
3683            if out.len() == before {
3684                continue;
3685            }
3686            self.kill_record(idx, &old);
3687            let gen = self.write_gen();
3688            let e = &mut self.catalog[idx];
3689            if out.len() + REC_SLOT_BYTES <= REC_PAGE_CAP {
3690                e.start_page = 0;
3691                e.page_count = 1;
3692                e.gen = gen;
3693                if let Some(r) = e.v9.as_mut() {
3694                    r.rec_slot = 0;
3695                }
3696                self.recs.hot.insert(idx, out);
3697                self.recs.pending.push(idx);
3698            } else {
3699                let (start, pc, gen) = self.write_record(&out)?;
3700                let e = &mut self.catalog[idx];
3701                e.start_page = start;
3702                e.page_count = pc;
3703                e.gen = gen;
3704                if let Some(r) = e.v9.as_mut() {
3705                    r.rec_slot = REC_SPAN;
3706                }
3707            }
3708        }
3709        Ok(())
3710    }
3711
3712    fn kill_record(&mut self, idx: usize, row: &CatalogEntry) {
3713        self.recs.hot.remove(&idx);
3714        let Some(r) = row.v9 else { return };
3715        if r.rec_slot == REC_SPAN {
3716            self.release_pages(row.start_page, row.page_count as u64);
3717            return;
3718        }
3719        if row.start_page == 0 {
3720            return;
3721        }
3722        let empty = match self.recs.page_rows.get_mut(&row.start_page) {
3723            Some(rows) => {
3724                rows.retain(|&i| i != idx);
3725                rows.is_empty()
3726            }
3727            None => false,
3728        };
3729        if empty {
3730            self.recs.page_rows.remove(&row.start_page);
3731            self.release_pages(row.start_page, 1);
3732        }
3733    }
3734
3735    /// FMT-9b.2: pack the waiting records, plus the live records of up to
3736    /// [`VACUUM_PAGES_PER_COMMIT`] sparse pages (under 3/4 full), into
3737    /// fresh record pages, filling each before starting the next. Runs at
3738    /// the top of a flush, before the catalog is serialized (it moves rows'
3739    /// record locations), and only when something is waiting.
3740    fn pack_records(&mut self) -> Result<()> {
3741        if !self.is_v9() || self.recs.pending.is_empty() {
3742            return Ok(());
3743        }
3744        let mut sparse: Vec<(u64, usize)> = self
3745            .recs
3746            .page_rows
3747            .iter()
3748            .filter(|(_, rows)| {
3749                let live: usize = rows
3750                    .iter()
3751                    .map(|&i| self.catalog[i].len as usize + REC_SLOT_BYTES)
3752                    .sum();
3753                live * 4 < REC_PAGE_CAP * 3
3754            })
3755            .map(|(&p, rows)| (p, rows.len()))
3756            .collect();
3757        sparse.sort_unstable();
3758        let sparse = vacuum_pick(sparse);
3759        let mut order: Vec<usize> = Vec::new();
3760        let mut seen: std::collections::HashSet<usize> = std::collections::HashSet::new();
3761        for p in sparse {
3762            let rows = self.recs.page_rows.remove(&p).unwrap_or_default();
3763            if let Some(&first) = rows.first() {
3764                let page = self.pager.read_page_uncached(p, self.catalog[first].gen)?;
3765                for idx in rows {
3766                    let slot = self.catalog[idx].v9.map_or(0, |r| r.rec_slot);
3767                    let mut b = record_slot(&page, slot)?;
3768                    // FMT-9e: a vacuum that moves a plain turn compresses it.
3769                    if self.unsettled_turn(idx) {
3770                        b = self.settle_turn(idx, b);
3771                    }
3772                    self.recs.hot.insert(idx, b);
3773                    if seen.insert(idx) {
3774                        order.push(idx);
3775                    }
3776                }
3777            }
3778            self.release_pages(p, 1);
3779        }
3780        let pending = std::mem::take(&mut self.recs.pending);
3781        for idx in pending {
3782            if self.recs.hot.contains_key(&idx) && seen.insert(idx) {
3783                order.push(idx);
3784            }
3785        }
3786        let gen = self.write_gen();
3787        let mut i = 0;
3788        while i < order.len() {
3789            let mut used = 0usize;
3790            let mut j = i;
3791            while j < order.len() {
3792                let l = self.recs.hot[&order[j]].len() + REC_SLOT_BYTES;
3793                if j > i && used + l > REC_PAGE_CAP {
3794                    break;
3795                }
3796                used += l;
3797                j += 1;
3798            }
3799            let page = self.alloc_pages(1);
3800            let bytes = {
3801                let recs: Vec<&[u8]> = order[i..j]
3802                    .iter()
3803                    .map(|k| self.recs.hot[k].as_slice())
3804                    .collect();
3805                encode_record_page(gen, &recs)
3806            };
3807            self.pager.write_page(page, gen, &bytes)?;
3808            for (slot, &k) in order[i..j].iter().enumerate() {
3809                self.touch(k);
3810                let e = &mut self.catalog[k];
3811                e.start_page = page;
3812                e.page_count = 1;
3813                e.gen = gen;
3814                if let Some(r) = e.v9.as_mut() {
3815                    r.rec_slot = slot as u16;
3816                }
3817            }
3818            self.recs.page_rows.insert(page, order[i..j].to_vec());
3819            i = j;
3820        }
3821        for k in order {
3822            self.recs.hot.remove(&k);
3823        }
3824        self.dirty_catalog = true;
3825        Ok(())
3826    }
3827
3828    /// FMT-9b: row `e`'s column entry (fp16 values and norm), from the
3829    /// staged set, the resident column, or its page.
3830    fn column_entry(&mut self, e: &CatalogEntry) -> Result<(Vec<u16>, f32)> {
3831        let d = self.dimensions;
3832        if let Some(idx) = self.index.get(e.id, &self.catalog) {
3833            if let Some((h, n)) = self.col.hot.get(&idx) {
3834                return Ok((h.clone(), *n));
3835            }
3836            if self.col.resident && self.col.have.get(idx).copied().unwrap_or(false) {
3837                return Ok((
3838                    self.col.data[idx * d..(idx + 1) * d].to_vec(),
3839                    self.col.norms[idx],
3840                ));
3841            }
3842        }
3843        let r =
3844            e.v9.ok_or_else(|| MnemoError::Invalid("v9 row without a vector entry".into()))?;
3845        if r.vec_page == 0 {
3846            return Err(MnemoError::Invalid("vector entry not written yet".into()));
3847        }
3848        let page = self.pager.read_page(r.vec_page, r.vec_gen)?;
3849        veccol::decode_entry(&page, d, r.vec_slot as usize)
3850    }
3851
3852    /// FMT-9b: make the column resident if it fits the budget: allocate it
3853    /// (zeroed, so untouched rows cost no memory) and copy in the staged
3854    /// entries. Rows fill a vector page at a time on first use
3855    /// ([`Mnemo::fill_column_page`]). A no-op once resident, and on v8.
3856    fn ensure_column(&mut self) -> Result<()> {
3857        if !self.is_v9() || self.col.resident {
3858            return Ok(());
3859        }
3860        let d = self.dimensions;
3861        let rows = self.catalog.len();
3862        if rows * veccol::entry_len(d) > self.vector_cache_budget {
3863            return Ok(());
3864        }
3865        self.col.data = vec![0u16; rows * d];
3866        self.col.norms = vec![0f32; rows];
3867        self.col.have = vec![false; rows];
3868        let hot: Vec<(usize, Vec<u16>, f32)> = self
3869            .col
3870            .hot
3871            .iter()
3872            .filter(|(&i, _)| i < rows)
3873            .map(|(&i, (h, n))| (i, h.clone(), *n))
3874            .collect();
3875        for (i, h, n) in hot {
3876            self.col.data[i * d..(i + 1) * d].copy_from_slice(&h);
3877            self.col.norms[i] = n;
3878            self.col.have[i] = true;
3879        }
3880        self.col.resident = true;
3881        Ok(())
3882    }
3883
3884    /// FMT-9b: fill the resident column from row `idx`'s vector page: every
3885    /// live row on that page, in one decrypt that bypasses the page cache.
3886    fn fill_column_page(&mut self, idx: usize) -> Result<()> {
3887        let Some(r) = self.catalog[idx].v9 else {
3888            return Ok(());
3889        };
3890        if r.vec_page == 0 {
3891            return Ok(());
3892        }
3893        let d = self.dimensions;
3894        let bytes = self.pager.read_page_uncached(r.vec_page, r.vec_gen)?;
3895        let rows = self
3896            .col
3897            .page_rows
3898            .get(&r.vec_page)
3899            .cloned()
3900            .unwrap_or_default();
3901        for i in rows.into_iter().chain(std::iter::once(idx)) {
3902            if i >= self.col.have.len() || self.col.have[i] {
3903                continue;
3904            }
3905            let Some(ri) = self.catalog[i].v9 else {
3906                continue;
3907            };
3908            if ri.vec_page != r.vec_page {
3909                continue;
3910            }
3911            let n = veccol::decode_entry_into(
3912                &bytes,
3913                d,
3914                ri.vec_slot as usize,
3915                &mut self.col.data[i * d..(i + 1) * d],
3916            )?;
3917            self.col.norms[i] = n;
3918            self.col.have[i] = true;
3919        }
3920        Ok(())
3921    }
3922
3923    /// FMT-9b: write the staged column entries, plus the live entries of up
3924    /// to [`VACUUM_PAGES_PER_COMMIT`] sparse pages (under 3/4 full), into
3925    /// fresh vector pages. Runs at the top of a flush, before the catalog
3926    /// is serialized, since it moves rows' vector locations. Only when
3927    /// something is staged, so a flush with no new vectors writes none.
3928    fn pack_vectors(&mut self) -> Result<()> {
3929        if !self.is_v9() || self.col.pending.is_empty() {
3930            return Ok(());
3931        }
3932        let d = self.dimensions;
3933        let per = veccol::per_page(d);
3934        let mut sparse: Vec<(u64, usize)> = self
3935            .col
3936            .page_rows
3937            .iter()
3938            .filter(|(_, rows)| rows.len() * 4 < per * 3)
3939            .map(|(&p, rows)| (p, rows.len()))
3940            .collect();
3941        sparse.sort_unstable();
3942        let sparse = vacuum_pick(sparse);
3943        let mut order: Vec<usize> = Vec::new();
3944        let mut seen: std::collections::HashSet<usize> = std::collections::HashSet::new();
3945        for p in sparse {
3946            let rows = self.col.page_rows.remove(&p).unwrap_or_default();
3947            for idx in rows {
3948                let e = self.catalog[idx].clone();
3949                let v = self.column_entry(&e)?;
3950                self.col.hot.insert(idx, v);
3951                if seen.insert(idx) {
3952                    order.push(idx);
3953                }
3954            }
3955            self.release_pages(p, 1);
3956        }
3957        let pending = std::mem::take(&mut self.col.pending);
3958        for idx in pending {
3959            if self.col.hot.contains_key(&idx) && seen.insert(idx) {
3960                order.push(idx);
3961            }
3962        }
3963        let gen = self.write_gen();
3964        for chunk in order.chunks(per) {
3965            let page = self.alloc_pages(1);
3966            let bytes = {
3967                let entries: Vec<(&[u16], f32)> = chunk
3968                    .iter()
3969                    .map(|i| {
3970                        let (h, n) = &self.col.hot[i];
3971                        (h.as_slice(), *n)
3972                    })
3973                    .collect();
3974                veccol::encode_page(d, gen, &entries)
3975            };
3976            self.pager.write_page(page, gen, &bytes)?;
3977            for (slot, &i) in chunk.iter().enumerate() {
3978                self.touch(i);
3979                if let Some(r) = self.catalog[i].v9.as_mut() {
3980                    r.vec_page = page;
3981                    r.vec_slot = slot as u16;
3982                    r.vec_gen = gen;
3983                }
3984            }
3985            self.col.page_rows.insert(page, chunk.to_vec());
3986        }
3987        for i in order {
3988            self.col.hot.remove(&i);
3989        }
3990        self.dirty_catalog = true;
3991        Ok(())
3992    }
3993
3994    /// FMT-9: before this session seals its first v9 page, commit a
3995    /// header-only transaction that bumps `wal_seq`.
3996    ///
3997    /// A page's generation is the id of the transaction that writes it,
3998    /// and that id is `wal_seq + 1`, which is only made durable by the
3999    /// commit. If a session crashes before committing, its pages (orphan
4000    /// data pages, or uncommitted WAL frames) carry an id the next session
4001    /// would hand out again, for the same pages (allocation is
4002    /// deterministic). An attacker holding a copy of the crashed file could
4003    /// then swap an orphan back in under a matching generation. Burning
4004    /// one id durably per writable session rules that out: every id this
4005    /// session uses is above every id any earlier session used. Costs one
4006    /// small commit per session, on its first write only.
4007    fn ensure_gen_lease(&mut self) -> Result<()> {
4008        if !self.is_v9() || self.gen_leased || self.read_only {
4009            return Ok(());
4010        }
4011        self.header.wal_seq += 1;
4012        let wal_start = self.header.wal_start;
4013        self.relocate_header_through_wal(wal_start)?;
4014        self.gen_leased = true;
4015        Ok(())
4016    }
4017
4018    /// Write a serialized record (memory or cache entry) to fresh pages.
4019    /// Returns `(start_page, page_count, generation)`.
4020    fn write_record(&mut self, bytes: &[u8]) -> Result<(u64, u32, u64)> {
4021        self.ensure_gen_lease()?;
4022        let pc = bytes.len().div_ceil(PAYLOAD).max(1);
4023        let start = self.alloc_pages(pc as u64);
4024        let gen = self.write_gen();
4025        for i in 0..pc {
4026            let lo = i * PAYLOAD;
4027            let hi = ((i + 1) * PAYLOAD).min(bytes.len());
4028            self.pager
4029                .write_page(start + i as u64, gen, &bytes[lo..hi])?;
4030        }
4031        Ok((start, pc as u32, gen))
4032    }
4033
4034    /// Read a record run of `pages` pages at `start`, sealed under `gen`.
4035    fn read_record_run(&mut self, start: u64, pages: u32, len: u32, gen: u64) -> Result<Vec<u8>> {
4036        let mut buf = Vec::with_capacity(len as usize);
4037        for i in 0..pages as u64 {
4038            buf.extend_from_slice(&self.pager.read_page(start + i, gen)?);
4039        }
4040        buf.truncate(len as usize);
4041        Ok(buf)
4042    }
4043
4044    fn read_record(&mut self, e: &CatalogEntry) -> Result<Vec<u8>> {
4045        // FMT-9b.2: a packed v9 record is one slot of a shared page, or
4046        // still waiting for the next flush.
4047        if let Some(r) = e.v9 {
4048            if r.rec_slot != REC_SPAN {
4049                if e.start_page == 0 {
4050                    return self
4051                        .index
4052                        .get(e.id, &self.catalog)
4053                        .and_then(|idx| self.recs.hot.get(&idx).cloned())
4054                        .ok_or_else(|| MnemoError::Invalid("record not written yet".into()));
4055                }
4056                let page = self.pager.read_page(e.start_page, e.gen)?;
4057                return record_slot(&page, r.rec_slot);
4058            }
4059        }
4060        let mut buf = Vec::with_capacity(e.len as usize);
4061        for i in 0..e.page_count as u64 {
4062            buf.extend_from_slice(&self.pager.read_page(e.start_page + i, e.gen)?);
4063        }
4064        buf.truncate(e.len as usize);
4065        Ok(buf)
4066    }
4067
4068    /// Decode a stored memory record body. FMT-9e: on v9 it may be a zstd
4069    /// frame (a compressed turn); v8 records are always plain.
4070    fn decode_memory(&self, bytes: &[u8]) -> Result<Memory> {
4071        let plain = if self.is_v9() {
4072            plain_record(bytes)?
4073        } else {
4074            std::borrow::Cow::Borrowed(bytes)
4075        };
4076        rmp_serde::from_slice(&plain).map_err(|err| MnemoError::Serialize(err.to_string()))
4077    }
4078
4079    fn read_memory(&mut self, e: &CatalogEntry) -> Result<Memory> {
4080        let m = self.read_memory_meta(e)?;
4081        self.finish_memory(e, m)
4082    }
4083
4084    /// Complete a record decoded by [`Mnemo::read_memory_meta`]: its
4085    /// vector and the catalog's live access stats.
4086    fn finish_memory(&mut self, e: &CatalogEntry, mut m: Memory) -> Result<Memory> {
4087        // FMT-9b: a v9 record carries no vector (unless the file keeps
4088        // float vectors); it comes from the column, fp16-rounded.
4089        if e.v9.is_some() && !self.float_vectors() {
4090            let (h, _) = self.column_entry(e)?;
4091            m.vector = veccol::dequantize(&h);
4092        }
4093        // The catalog is the source of truth for access stats — the values
4094        // in the record body are a stale snapshot from when the record was
4095        // last written. Overwrite them here so consumers always see live
4096        // values regardless of when the record was last serialized.
4097        m.accessed_at = e.accessed_at;
4098        m.access_count = e.access_count;
4099        Ok(m)
4100    }
4101
4102    /// Serialize and store a memory (insert or overwrite by ID).
4103    fn put(&mut self, mut m: Memory) -> Result<Ulid> {
4104        if m.vector.len() != self.dimensions {
4105            return Err(MnemoError::DimensionMismatch {
4106                expected: self.dimensions,
4107                got: m.vector.len(),
4108            });
4109        }
4110        // Reject all-zero vectors (Phase 3 PR 10a; breaking vs 0.4.x).
4111        // A zero vector scores exactly 0.0 on every cosine query — it
4112        // can't contribute signal, but recency / importance can still
4113        // pull it into top-k as noise. We'd rather fail loudly at
4114        // `remember` than let a mis-embedded entry pollute recall
4115        // quality for the life of the database. Scaffold manifests now
4116        // use a sentinel unit vector so this check doesn't trip them.
4117        if embed::is_zero_vector(&m.vector) {
4118            return Err(MnemoError::Invalid(
4119                "all-zero vector refused: it scores 0.0 on every cosine query and \
4120                 pollutes recall; check your embedder returned a real embedding"
4121                    .into(),
4122            ));
4123        }
4124        if m.id == Ulid::nil() {
4125            m.id = Ulid::new();
4126        }
4127        let id_u: u128 = m.id.0;
4128        // R-004.2: an overwrite with the handle caches built reads the
4129        // stored version first (one decrypt), so the caches drop exactly
4130        // what it contributed.
4131        if self.layers.is_some() {
4132            if let Some(idx) = self.index.get(id_u, &self.catalog) {
4133                let old_entry = self.catalog[idx].clone();
4134                if !old_entry.deleted {
4135                    let old = self.read_memory(&old_entry)?;
4136                    self.note_removed(&old)?;
4137                }
4138            }
4139        }
4140        // FMT-9b: a v9 record is stored without its vector (unless the file
4141        // keeps float vectors); the vector goes to the fp16 column.
4142        let v9 = self.is_v9();
4143        let bytes = if v9 && !self.float_vectors() {
4144            let vector = std::mem::take(&mut m.vector);
4145            let b = rmp_serde::to_vec(&m);
4146            m.vector = vector;
4147            b
4148        } else {
4149            rmp_serde::to_vec(&m)
4150        }
4151        .map_err(|e| MnemoError::Serialize(e.to_string()))?;
4152        // FMT-9e: a v9 thread turn is stored zstd-compressed (turns are
4153        // most of a thread-heavy file). The catalog's `len` is the stored
4154        // length, so packing and vacuum move the frame as is.
4155        // FMT-9e (compress when cold): a turn is compressed at write only
4156        // when it's already past the handle's cutoff (an imported
4157        // archive, or `set_turn_compress_after(0)`); otherwise the commit
4158        // that sees it age compresses it.
4159        let is_turn = v9 && meta_str(&m, MNEMO_META_KIND) == Some(MNEMO_META_KIND_TURN);
4160        let settle =
4161            is_turn && (bytes.len() < TURN_ZSTD_MIN_BYTES || self.turn_is_cold(m.created_at));
4162        let bytes = if settle {
4163            compress_turn_record(bytes)
4164        } else {
4165            bytes
4166        };
4167        // FMT-9b.2: a v9 record that fits a packed page waits for the next
4168        // flush, which packs it with others; a bigger one owns a run of
4169        // whole pages, written now as on v8.
4170        let packed = v9 && bytes.len() + REC_SLOT_BYTES <= REC_PAGE_CAP;
4171        // A waiting record has `start_page` 0. Its generation is never 0 on
4172        // v9: the row encoding leaves out a zero `gen`, which would shift
4173        // the `v9` field into its place (a deleted row can stay unpacked).
4174        let (start, pc, gen) = if packed {
4175            self.ensure_gen_lease()?;
4176            (0, 1, self.write_gen())
4177        } else {
4178            self.write_record(&bytes)?
4179        };
4180        let rec_len = bytes.len();
4181        // FMT-9d: a keyword-indexed v9 row gets a new BM25 document.
4182        let mut tags = v9.then(|| row_tags(&m));
4183        if let Some(r) = tags.as_mut() {
4184            if bm25_kind(r.kind) {
4185                let (terms, len) = text::term_freqs(&m.content);
4186                r.doc = self.bm25.manifest.next_doc;
4187                r.doc_len = len;
4188                self.bm25.manifest.next_doc += 1;
4189                self.bm25.pending.push((r.doc, terms));
4190            }
4191        }
4192        // FMT-9f: keep the BM25 stats current (below, once the row is
4193        // placed) instead of rescanning every row at the next commit.
4194        let bm25_new = tags.filter(|r| r.doc != 0 && bm25_kind(r.kind));
4195        // For overwrites preserve the existing access stats; for inserts
4196        // seed them from the memory (which carries them in its body).
4197        let (prev_accessed, prev_count) = match self.index.get(id_u, &self.catalog) {
4198            Some(idx) => (
4199                self.catalog[idx].accessed_at,
4200                self.catalog[idx].access_count,
4201            ),
4202            None => (m.accessed_at, m.access_count),
4203        };
4204        let entry = CatalogEntry {
4205            id: id_u,
4206            start_page: start,
4207            page_count: pc,
4208            len: rec_len as u32,
4209            deleted: false,
4210            accessed_at: prev_accessed,
4211            access_count: prev_count,
4212            gen,
4213            v9: tags.map(|mut r| {
4214                r.rec_slot = if packed { 0 } else { REC_SPAN };
4215                if settle {
4216                    r.flags |= ROW_SETTLED;
4217                }
4218                r
4219            }),
4220        };
4221        match self.index.get(id_u, &self.catalog) {
4222            Some(idx) => {
4223                // FMT-9 (a): the overwritten version's pages are freed by
4224                // this transaction's commit. A tombstone's pages were
4225                // already freed when its delete committed.
4226                if !self.catalog[idx].deleted {
4227                    let old = self.catalog[idx].clone();
4228                    if v9 {
4229                        self.kill_vector(idx, &old);
4230                        self.kill_record(idx, &old);
4231                    } else {
4232                        self.release_pages(old.start_page, old.page_count as u64);
4233                    }
4234                }
4235                if let (Some(stats), Some(old)) = (self.bm25.stats.as_mut(), self.catalog[idx].v9) {
4236                    if !self.catalog[idx].deleted && old.doc != 0 {
4237                        stats.remove(old.doc);
4238                    }
4239                }
4240                self.catalog[idx] = entry;
4241                self.touch(idx);
4242            }
4243            None => {
4244                self.index.insert(id_u, self.catalog.len(), &self.catalog);
4245                self.touch(self.catalog.len());
4246                self.catalog.push(entry);
4247            }
4248        }
4249        if let (Some(stats), Some(r)) = (self.bm25.stats.as_mut(), bm25_new) {
4250            if let Some(idx) = self.index.get(id_u, &self.catalog) {
4251                stats.insert(r.doc, idx, r.doc_len);
4252            }
4253        }
4254        self.dirty_catalog = true;
4255        if v9 {
4256            if let Some(idx) = self.index.get(id_u, &self.catalog) {
4257                self.stage_vector(idx, &m.vector);
4258                if packed {
4259                    self.recs.hot.insert(idx, bytes);
4260                    self.recs.pending.push(idx);
4261                }
4262            }
4263        }
4264
4265        // Keep the ANN index complete: assign the (new or changed) vector to
4266        // its nearest partition. Centroids/codebook stay fixed until rebuild.
4267        // R-004.2: only facts go in; turns, essence items and thread links
4268        // are read by kind and scope.
4269        if let Some(ann) = &mut self.ann {
4270            if is_ann_kind(&m) {
4271                ann.add(id_u, &m.vector);
4272                self.dirty_index = true;
4273                self.index_persist.changes += 1;
4274            } else if ann.contains(id_u) {
4275                ann.remove(id_u);
4276                self.dirty_index = true;
4277                self.index_persist.changes += 1;
4278            }
4279        }
4280        self.note_added(&m)?;
4281
4282        // FIX-1 R-001.8 part 2: keep the lazy agent set in sync. If
4283        // it hasn't been built yet (no `stats()` call this session),
4284        // skip — the first `stats()` will build it from scratch. If
4285        // it has, add this memory's agent_id. An overwrite with the
4286        // same id and same agent is a no-op (BTreeSet::insert returns
4287        // false); an overwrite with a *changed* agent_id adds the new
4288        // agent but doesn't drop the old (we can't tell without a
4289        // decrypt whether the old agent still has live memories).
4290        // That's conservative in the right direction — `stats()` may
4291        // over-report by one agent until the next delete invalidates.
4292        if let Some(set) = &mut self.agent_set {
4293            set.insert(m.agent_id.clone());
4294        }
4295        Ok(m.id)
4296    }
4297
4298    // --- public CRUD ------------------------------------------------------
4299
4300    /// Register a text embedder for this handle. Enables
4301    /// [`Mnemo::remember_text`] and the text-side `RecallRequest`
4302    /// convenience path. Does two checks before storing the embedder:
4303    ///
4304    /// 1. **Dimension check.** The embedder's `identity().dimensions`
4305    ///    must equal [`Mnemo::dimensions`] (the width this database
4306    ///    was created with). Mismatch returns
4307    ///    [`MnemoError::DimensionMismatch`].
4308    /// 2. **Manifest identity check.** If the scaffold / onboarding
4309    ///    manifest already records an embedder — the canonical shape
4310    ///    is `metadata.embedder = { name, dimensions, normalize }`,
4311    ///    with the legacy flat `embedder_name` / `embedder_dimensions`
4312    ///    / `embedder_normalize` keys accepted as an alias — the new
4313    ///    embedder's identity must match exactly. A different name —
4314    ///    even at the same dimensions — returns
4315    ///    [`MnemoError::Invalid`] with
4316    ///    [`EmbedderIdentity::MISMATCH_MSG`] in the message. If the
4317    ///    manifest has no embedder metadata yet, this call upserts
4318    ///    the nested object into the first onboarding/manifest memory
4319    ///    (write + flush so a subsequent reopen sees the identity).
4320    ///
4321    /// **Read-only handles.** `set_embedder` on an `open_read_only`
4322    /// handle performs both checks and adopts the embedder in memory;
4323    /// it never writes. Readers (CLI `--text`, agent-side `recall_text`)
4324    /// can therefore use the text path next to a writer that holds
4325    /// the exclusive lock.
4326    ///
4327    /// On both failure paths the handle's embedder slot is left
4328    /// unchanged. On success the embedder is wrapped in an `Arc<dyn
4329    /// Embedder>` and consulted by every text-side call made through
4330    /// this handle. The slot is per-handle and not persisted — a
4331    /// fresh open must re-register (same identity, so the manifest
4332    /// check round-trips cleanly).
4333    pub fn set_embedder<E: Embedder + 'static>(&mut self, embedder: E) -> Result<()> {
4334        let identity = embedder.identity();
4335        if identity.dimensions != self.dimensions {
4336            return Err(MnemoError::DimensionMismatch {
4337                expected: self.dimensions,
4338                got: identity.dimensions,
4339            });
4340        }
4341
4342        // Find a scaffold/onboarding manifest to inspect (or upsert
4343        // into). `about()` already filters + sorts the onboarding set.
4344        let onboarding = self.about()?;
4345        let mut upsert_target: Option<usize> = None;
4346        for (i, m) in onboarding.iter().enumerate() {
4347            if let Some(saw) = embed::read_identity_from_metadata(&m.metadata) {
4348                if saw != identity {
4349                    return Err(embed::identity_mismatch_err(&saw, &identity));
4350                }
4351                // Match — nothing to persist; adopt + done. Works the
4352                // same on read-only and read-write handles.
4353                self.embedder = Some(std::sync::Arc::new(embedder));
4354                return Ok(());
4355            }
4356            if upsert_target.is_none() {
4357                upsert_target = Some(i);
4358            }
4359        }
4360
4361        // No existing identity on the manifest. On a read-write handle
4362        // upsert into the first onboarding memory so later opens see
4363        // it; on a read-only handle skip the write (readers must not
4364        // mutate the file), but still adopt the embedder in memory.
4365        if !self.read_only {
4366            if let Some(idx) = upsert_target {
4367                let mut m = onboarding[idx].clone();
4368                embed::write_identity_into_metadata(&mut m.metadata, &identity);
4369                // Reuse the memory's existing id so this is an in-place
4370                // update of the scaffold manifest, not a new entry.
4371                self.put(m)?;
4372                self.flush()?;
4373            }
4374        }
4375
4376        self.embedder = Some(std::sync::Arc::new(embedder));
4377        Ok(())
4378    }
4379
4380    /// Borrow the currently-registered embedder, if any. Returns
4381    /// `None` when no embedder has been set via
4382    /// [`Mnemo::set_embedder`]. Bindings use this to drive their own
4383    /// text-side convenience methods without duplicating the identity
4384    /// check.
4385    pub fn embedder(&self) -> Option<&dyn Embedder> {
4386        self.embedder.as_deref()
4387    }
4388
4389    /// Shared handle to the registered embedder, cloned so the caller
4390    /// can drop the borrow on this `Mnemo` before invoking `embed()`.
4391    /// FIX-1 / PY-1 (R-001.11): the Python binding needs this so an
4392    /// embedder callback that re-enters `Mnemo` (e.g. a Python
4393    /// embedder that calls `db.dimensions()` or `db.cache_get(...)`)
4394    /// doesn't hit PyO3's "Already borrowed" — we clone the Arc under
4395    /// a short `&self` borrow, release that borrow, then invoke
4396    /// `embed()` with no `&mut Mnemo` held.
4397    pub fn embedder_arc(&self) -> Option<std::sync::Arc<dyn Embedder>> {
4398        self.embedder.clone()
4399    }
4400
4401    /// Embed `content` with the registered embedder and `remember` the
4402    /// result. Returns the new memory's ULID. Convenience path for
4403    /// callers who have a string, want it stored, and don't need any
4404    /// of the per-memory options [`Memory`] exposes (agent, importance,
4405    /// metadata, ttl, scope) — build a [`Memory`] by hand and call
4406    /// [`Mnemo::remember`] for those.
4407    ///
4408    /// Fails with [`MnemoError::Invalid`] if no embedder has been
4409    /// registered on this handle (call [`Mnemo::set_embedder`] first);
4410    /// with [`MnemoError::DimensionMismatch`] if the embedder returned
4411    /// the wrong width for this database.
4412    pub fn remember_text(&mut self, content: &str, memory_type: MemoryType) -> Result<Ulid> {
4413        if self.read_only {
4414            return Err(MnemoError::ReadOnly);
4415        }
4416        let embedder = self.embedder.as_ref().ok_or_else(|| {
4417            MnemoError::Invalid(
4418                "no embedder registered on this handle; call set_embedder first".into(),
4419            )
4420        })?;
4421        let mut vectors = embedder.embed(&[content])?;
4422        let vector = vectors.pop().ok_or_else(|| {
4423            MnemoError::Invalid("embedder returned no vector for the input text".into())
4424        })?;
4425        if vector.len() != self.dimensions {
4426            return Err(MnemoError::DimensionMismatch {
4427                expected: self.dimensions,
4428                got: vector.len(),
4429            });
4430        }
4431        let memory = Memory::new(content.to_string(), memory_type, vector);
4432        self.remember(memory)
4433    }
4434
4435    /// Store a memory. Returns its ULID. If the memory's ID is nil a fresh
4436    /// one is assigned; an existing ID overwrites in place.
4437    pub fn remember(&mut self, memory: Memory) -> Result<Ulid> {
4438        if self.read_only {
4439            return Err(MnemoError::ReadOnly);
4440        }
4441        self.put(memory)
4442    }
4443
4444    /// Fetch a memory by ID.
4445    pub fn get(&mut self, id: &Ulid) -> Result<Memory> {
4446        let idx = self
4447            .index
4448            .get(id.0, &self.catalog)
4449            .ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
4450        let entry = self.catalog[idx].clone();
4451        if entry.deleted {
4452            return Err(MnemoError::NotFound(id.to_string()));
4453        }
4454        self.read_memory(&entry)
4455    }
4456
4457    /// Soft-delete a memory (tombstoned; space reclaimed by `compact`).
4458    pub fn delete(&mut self, id: &Ulid) -> Result<()> {
4459        if self.read_only {
4460            return Err(MnemoError::ReadOnly);
4461        }
4462        let idx = self
4463            .index
4464            .get(id.0, &self.catalog)
4465            .ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
4466        // R-004.2 / 12.1: with the handle caches built, read the record
4467        // being deleted (one decrypt) so the key slot, superseded set and
4468        // thread layers drop exactly what it contributed.
4469        let entry = self.catalog[idx].clone();
4470        if self.layers.is_some() && !entry.deleted {
4471            let old = self.read_memory(&entry)?;
4472            self.note_removed(&old)?;
4473        }
4474        if !self.catalog[idx].deleted {
4475            self.catalog[idx].deleted = true;
4476            self.dirty_catalog = true;
4477            self.touch(idx);
4478            // FMT-9f: drop its document from the stats in place.
4479            if let (Some(stats), Some(r)) = (self.bm25.stats.as_mut(), entry.v9) {
4480                if r.doc != 0 {
4481                    stats.remove(r.doc);
4482                }
4483            }
4484            if self.is_v9() {
4485                self.kill_record(idx, &entry);
4486                self.kill_vector(idx, &entry);
4487            } else {
4488                self.release_pages(entry.start_page, entry.page_count as u64);
4489            }
4490        }
4491        if let Some(ann) = &mut self.ann {
4492            if ann.contains(id.0) {
4493                ann.remove(id.0);
4494                self.dirty_index = true;
4495                self.index_persist.changes += 1;
4496            }
4497        }
4498        // FIX-1 R-001.8 part 2: we can't tell whether this delete
4499        // removed the last memory of its agent without a decrypt
4500        // (the catalog entry doesn't carry `agent_id` in format v8),
4501        // so just invalidate the lazy set. The next `stats()` call
4502        // rebuilds from live memories.
4503        self.agent_set = None;
4504        // Phase 12.1: delete of a keyed record removes it from the slot,
4505        // and delete of a superseded record frees a `history` entry. When
4506        // the caches are built, `note_removed` above did that exactly;
4507        // otherwise they're built from scratch on first use anyway.
4508        Ok(())
4509    }
4510
4511    /// Keep the handle caches (12.1 key slots, superseded set, R-004.2
4512    /// layers) current after `put` writes `m`. No-op until they're built.
4513    /// FMT-9e: a cold thread is warmed first, so its layer entry is whole.
4514    fn note_added(&mut self, m: &Memory) -> Result<()> {
4515        if let Some(thread) = meta_str(m, MNEMO_META_THREAD) {
4516            self.warm_thread(thread)?;
4517        }
4518        let id = m.id.0;
4519        if let (Some(keys), Some(sup)) = (self.key_index.as_mut(), self.superseded.as_mut()) {
4520            if m.metadata.contains_key(MNEMO_META_SUPERSEDED_BY) {
4521                sup.insert(id);
4522            } else if let Some(Value::String(k)) = m.metadata.get(MNEMO_META_KEY) {
4523                keys.insert(Self::key_slot_for(m, k), id);
4524            }
4525        }
4526        if let Some(layers) = self.layers.as_mut() {
4527            layers.add(m);
4528        }
4529        Ok(())
4530    }
4531
4532    /// Undo [`Mnemo::note_added`] for the stored version of a record that's
4533    /// about to be overwritten or deleted.
4534    fn note_removed(&mut self, m: &Memory) -> Result<()> {
4535        if let Some(thread) = meta_str(m, MNEMO_META_THREAD) {
4536            self.warm_thread(thread)?;
4537        }
4538        let id = m.id.0;
4539        if let (Some(keys), Some(sup)) = (self.key_index.as_mut(), self.superseded.as_mut()) {
4540            sup.remove(&id);
4541            if let Some(Value::String(k)) = m.metadata.get(MNEMO_META_KEY) {
4542                let slot = Self::key_slot_for(m, k);
4543                if keys.get(&slot) == Some(&id) {
4544                    keys.remove(&slot);
4545                }
4546            }
4547        }
4548        if let Some(layers) = self.layers.as_mut() {
4549            layers.remove(m);
4550        }
4551        Ok(())
4552    }
4553
4554    /// FMT-9e: decode `thread`'s cold turn records (see
4555    /// [`LayerCache::cold_turns`]) into the layer cache. Every record in
4556    /// the thread's hash bucket is decoded and filed under its own thread,
4557    /// so a hash collision just warms both threads. A no-op when the
4558    /// caches aren't built or the bucket is already warm.
4559    fn warm_thread(&mut self, thread: &str) -> Result<()> {
4560        self.warm_thread_decoded(thread).map(|_| ())
4561    }
4562
4563    /// [`Mnemo::warm_thread`], returning the records it decoded (without
4564    /// vectors), so a caller about to read the thread decodes each once.
4565    fn warm_thread_decoded(
4566        &mut self,
4567        thread: &str,
4568    ) -> Result<std::collections::HashMap<u128, Memory>> {
4569        let mut decoded = std::collections::HashMap::new();
4570        let h = fnv1a32(&[thread.as_bytes()]);
4571        let Some(ids) = self.layers.as_mut().and_then(|l| l.cold_turns.remove(&h)) else {
4572            return Ok(decoded);
4573        };
4574        for e in self.live_entries(ids) {
4575            let m = self.read_memory_meta(&e)?;
4576            if let Some(layers) = self.layers.as_mut() {
4577                layers.add(&m);
4578            }
4579            decoded.insert(e.id, m);
4580        }
4581        Ok(decoded)
4582    }
4583
4584    /// Decode a record's memory without its vector or live access stats:
4585    /// what the handle caches need (metadata, scope, agent), without the
4586    /// v9 vector-column read [`Mnemo::read_memory`] adds.
4587    fn read_memory_meta(&mut self, e: &CatalogEntry) -> Result<Memory> {
4588        let bytes = self.read_record(e)?;
4589        self.decode_memory(&bytes)
4590    }
4591
4592    /// The live catalog entries for `ids` (ids that are gone are skipped).
4593    fn live_entries(&self, ids: impl IntoIterator<Item = u128>) -> Vec<CatalogEntry> {
4594        ids.into_iter()
4595            .filter_map(|id| {
4596                self.index
4597                    .get(id, &self.catalog)
4598                    .map(|i| self.catalog[i].clone())
4599            })
4600            .filter(|e| !e.deleted)
4601            .collect()
4602    }
4603
4604    /// Catalog entries of the essence items, from the handle caches.
4605    fn essence_entries(&mut self) -> Result<Vec<CatalogEntry>> {
4606        self.ensure_key_caches()?;
4607        let ids: Vec<u128> = self
4608            .layers
4609            .as_ref()
4610            .map(|l| l.essence.iter().copied().collect())
4611            .unwrap_or_default();
4612        Ok(self.live_entries(ids))
4613    }
4614
4615    /// Catalog entries of every record filed under `thread`, from the
4616    /// handle caches.
4617    fn thread_entries(&mut self, thread: &str) -> Result<Vec<CatalogEntry>> {
4618        Ok(self.thread_entries_decoded(thread)?.0)
4619    }
4620
4621    /// [`Mnemo::thread_entries`], plus the records that warming a cold
4622    /// thread just decoded (see [`Mnemo::warm_thread_decoded`]).
4623    fn thread_entries_decoded(
4624        &mut self,
4625        thread: &str,
4626    ) -> Result<(Vec<CatalogEntry>, std::collections::HashMap<u128, Memory>)> {
4627        self.ensure_key_caches()?;
4628        let decoded = self.warm_thread_decoded(thread)?;
4629        let ids: Vec<u128> = self
4630            .layers
4631            .as_ref()
4632            .and_then(|l| l.threads.get(thread))
4633            .map(|tc| tc.records.iter().copied().collect())
4634            .unwrap_or_default();
4635        Ok((self.live_entries(ids), decoded))
4636    }
4637
4638    // --- Phase 12.1: edit in place + keyed supersession -----------------
4639
4640    /// Scope-aware key-slot identifier for `memory` and a caller key.
4641    ///
4642    /// * [`Scope::Private`] → owned by the memory's `agent_id`.
4643    /// * [`Scope::Shared`]  → one slot across the whole file.
4644    fn key_slot_for(memory: &Memory, key: &str) -> crate::store::KeySlot {
4645        let scope_owner = match memory.scope {
4646            Scope::Private => Some(memory.agent_id.clone()),
4647            Scope::Shared => None,
4648        };
4649        crate::store::KeySlot {
4650            scope_owner,
4651            key: key.to_string(),
4652        }
4653    }
4654
4655    /// The handle's one lazy scan: builds the 12.1 `key_index` +
4656    /// `superseded` caches and the R-004.2 layer cache (essence ids,
4657    /// onboarding ids, threads, links) from live records, and prunes
4658    /// non-fact ids from a loaded ANN index (files written before v0.10.0
4659    /// indexed every kind). Decrypts every live record once per handle;
4660    /// `put` and `delete` keep the result current after that.
4661    fn ensure_key_caches(&mut self) -> Result<()> {
4662        if self.key_index.is_some() && self.superseded.is_some() && self.layers.is_some() {
4663            return Ok(());
4664        }
4665        let entries: Vec<CatalogEntry> = self
4666            .catalog
4667            .iter()
4668            .filter(|e| !e.deleted)
4669            .cloned()
4670            .collect();
4671        let mut keys: std::collections::HashMap<crate::store::KeySlot, u128> =
4672            std::collections::HashMap::new();
4673        let mut superseded: std::collections::HashSet<u128> = std::collections::HashSet::new();
4674        let mut layers = LayerCache::default();
4675        let mut non_facts: Vec<u128> = Vec::new();
4676        for e in &entries {
4677            // FMT-9e: a v9 turn row with no key slot and no supersession
4678            // contributes only to its thread's layer entry, so it's left
4679            // undecoded in the thread's cold bucket (most of a
4680            // thread-heavy file). Its row says it isn't a fact.
4681            if let Some(r) = e.v9 {
4682                if r.kind == ROW_KIND_TURN
4683                    && r.thread_hash != 0
4684                    && r.key_hash == 0
4685                    && r.flags & ROW_SUPERSEDED == 0
4686                {
4687                    layers
4688                        .cold_turns
4689                        .entry(r.thread_hash)
4690                        .or_default()
4691                        .push(e.id);
4692                    non_facts.push(e.id);
4693                    continue;
4694                }
4695            }
4696            let m = self.read_memory_meta(e)?;
4697            layers.add(&m);
4698            if !is_ann_kind(&m) {
4699                non_facts.push(e.id);
4700            }
4701            if m.metadata.get(MNEMO_META_SUPERSEDED_BY).is_some() {
4702                superseded.insert(e.id);
4703                continue;
4704            }
4705            if let Some(Value::String(k)) = m.metadata.get(MNEMO_META_KEY) {
4706                let slot = Self::key_slot_for(&m, k);
4707                // Catalog order is insertion order; a later live record
4708                // with the same slot would only exist if a prior build
4709                // of this code allowed it. Keep the later id (newer
4710                // ULID), in case of a tie.
4711                keys.entry(slot)
4712                    .and_modify(|cur| {
4713                        if *cur < e.id {
4714                            *cur = e.id;
4715                        }
4716                    })
4717                    .or_insert(e.id);
4718            }
4719        }
4720        self.key_index = Some(keys);
4721        self.superseded = Some(superseded);
4722        self.layers = Some(layers);
4723        if let Some(ann) = &mut self.ann {
4724            let mut pruned = 0usize;
4725            for id in non_facts {
4726                if ann.contains(id) {
4727                    ann.remove(id);
4728                    pruned += 1;
4729                }
4730            }
4731            if pruned > 0 {
4732                self.dirty_index = true;
4733                self.index_persist.changes += pruned;
4734            }
4735        }
4736        Ok(())
4737    }
4738
4739    /// Stage-identical check for a 12.1 keyed no-op. The incoming write
4740    /// is byte-equal to the live record iff every user-visible field
4741    /// (content, type, importance, vector, user metadata) matches. The
4742    /// `_mnemo.*` reserved keys on the live record are **ignored** for
4743    /// this comparison — a repeat of an identical write re-issued by an
4744    /// extractor must be a NOOP even if the live record already carries
4745    /// supersession bookkeeping from a prior op.
4746    fn is_identical_keyed_write(incoming: &Memory, live: &Memory) -> bool {
4747        if incoming.content != live.content {
4748            return false;
4749        }
4750        if incoming.memory_type != live.memory_type {
4751            return false;
4752        }
4753        if (incoming.importance - live.importance).abs() > f32::EPSILON {
4754            return false;
4755        }
4756        if incoming.vector != live.vector {
4757            return false;
4758        }
4759        // Strip `_mnemo.*` from both sides before comparing user metadata.
4760        let strip = |m: &Map<String, Value>| -> Map<String, Value> {
4761            m.iter()
4762                .filter(|(k, _)| !k.starts_with(crate::memory::MNEMO_META_NS))
4763                .map(|(k, v)| (k.clone(), v.clone()))
4764                .collect()
4765        };
4766        strip(&incoming.metadata) == strip(&live.metadata)
4767    }
4768
4769    /// Store a memory under a keyed upsert slot. Returns the live record's
4770    /// id and a [`WriteOutcome`] saying whether this write created a new
4771    /// slot (`Added`), replaced a prior record (`Superseded`), or matched
4772    /// the existing live record byte-for-byte (`Unchanged`, no file write).
4773    ///
4774    /// The slot is identified by `(scope_owner, key)`:
4775    /// * `Scope::Private` memories share a slot per `(agent_id, key)`.
4776    /// * `Scope::Shared` memories share one slot per `key` across the file.
4777    ///
4778    /// On supersession the prior record is **not** deleted — it remains
4779    /// retrievable through [`Mnemo::history`] — but it is excluded from
4780    /// default [`Mnemo::recall`] and [`Mnemo::about`]. A caller that wants
4781    /// the full lineage of a key walks `history(key)`.
4782    ///
4783    /// This is the atomic 12.1 "supersede-or-noop" write path. Plain
4784    /// [`Mnemo::remember`] without a key is unchanged.
4785    pub fn remember_keyed(
4786        &mut self,
4787        mut memory: Memory,
4788        key: &str,
4789    ) -> Result<(Ulid, WriteOutcome)> {
4790        if self.read_only {
4791            return Err(MnemoError::ReadOnly);
4792        }
4793        if key.is_empty() {
4794            return Err(MnemoError::Invalid(
4795                "remember_keyed: 'key' must not be empty".into(),
4796            ));
4797        }
4798        self.ensure_key_caches()?;
4799        let slot = Self::key_slot_for(&memory, key);
4800        let existing = self.key_index.as_ref().and_then(|k| k.get(&slot).copied());
4801
4802        // Caller may already have set `_mnemo.key` on the memory; rewrite
4803        // it to our canonical value regardless so the record is
4804        // self-identifying.
4805        memory
4806            .metadata
4807            .insert(MNEMO_META_KEY.into(), Value::String(key.to_string()));
4808
4809        // NOOP check: if a live record exists for this slot and its
4810        // user-visible fields match, return the existing id without a
4811        // write.
4812        if let Some(prev_id_u) = existing {
4813            let prev_idx = self
4814                .index
4815                .get(prev_id_u, &self.catalog)
4816                .ok_or_else(|| MnemoError::Invalid("key_index drift".into()))?;
4817            let prev_entry = self.catalog[prev_idx].clone();
4818            let prev_mem = self.read_memory(&prev_entry)?;
4819            if Self::is_identical_keyed_write(&memory, &prev_mem) {
4820                return Ok((Ulid(prev_id_u), WriteOutcome::Unchanged));
4821            }
4822
4823            // SUPERSESSION: insert the new record, then rewrite the old
4824            // with `_mnemo.superseded_by` + `_mnemo.invalid_at`.
4825            let now = memory::now_secs();
4826            memory.id = Ulid::nil(); // force a fresh ULID in `put`
4827            memory.metadata.insert(
4828                MNEMO_META_SUPERSEDES.into(),
4829                Value::String(Ulid(prev_id_u).to_string()),
4830            );
4831            let new_id = self.put(memory)?;
4832            // Mark the prior record.
4833            let mut prev_mut = prev_mem;
4834            prev_mut.metadata.insert(
4835                MNEMO_META_SUPERSEDED_BY.into(),
4836                Value::String(new_id.to_string()),
4837            );
4838            prev_mut
4839                .metadata
4840                .insert(MNEMO_META_INVALID_AT.into(), Value::Number(now.into()));
4841            // keep prev_mut.id so `put` overwrites in place.
4842            self.put(prev_mut)?;
4843            // Update caches incrementally.
4844            if let Some(k) = self.key_index.as_mut() {
4845                k.insert(slot, new_id.0);
4846            }
4847            if let Some(s) = self.superseded.as_mut() {
4848                s.insert(prev_id_u);
4849            }
4850            return Ok((new_id, WriteOutcome::Superseded));
4851        }
4852
4853        // ADDED: no prior record for this slot.
4854        memory.id = Ulid::nil();
4855        let new_id = self.put(memory)?;
4856        if let Some(k) = self.key_index.as_mut() {
4857            k.insert(slot, new_id.0);
4858        }
4859        Ok((new_id, WriteOutcome::Added))
4860    }
4861
4862    /// Text-side keyed write: embed `content` through the registered
4863    /// embedder, then [`Mnemo::remember_keyed`]. Same returns.
4864    pub fn remember_keyed_text(
4865        &mut self,
4866        content: &str,
4867        memory_type: MemoryType,
4868        key: &str,
4869    ) -> Result<(Ulid, WriteOutcome)> {
4870        if self.read_only {
4871            return Err(MnemoError::ReadOnly);
4872        }
4873        let embedder = self.embedder.as_ref().ok_or_else(|| {
4874            MnemoError::Invalid(
4875                "no embedder registered on this handle; call set_embedder first".into(),
4876            )
4877        })?;
4878        let mut vectors = embedder.embed(&[content])?;
4879        let vector = vectors.pop().ok_or_else(|| {
4880            MnemoError::Invalid("embedder returned no vector for the input text".into())
4881        })?;
4882        if vector.len() != self.dimensions {
4883            return Err(MnemoError::DimensionMismatch {
4884                expected: self.dimensions,
4885                got: vector.len(),
4886            });
4887        }
4888        let memory = Memory::new(content.to_string(), memory_type, vector);
4889        self.remember_keyed(memory, key)
4890    }
4891
4892    /// Update an existing memory in place, preserving its ULID,
4893    /// `created_at` and access stats. Returns [`WriteOutcome::Updated`]
4894    /// on a real change, [`WriteOutcome::Unchanged`] when every field in
4895    /// the patch already matches the live record.
4896    ///
4897    /// Reserved `_mnemo.*` metadata keys on the existing record are
4898    /// preserved across a `metadata` replacement; a `metadata` of
4899    /// `Some(map)` replaces every **user-visible** key with `map`'s
4900    /// entries. A `content` change requires a matching `vector`
4901    /// (use [`Mnemo::update_text`] to re-embed) — a bare `content` patch
4902    /// is rejected to stop text and embedding drifting apart silently.
4903    pub fn update(&mut self, id: &Ulid, patch: MemoryPatch) -> Result<WriteOutcome> {
4904        if self.read_only {
4905            return Err(MnemoError::ReadOnly);
4906        }
4907        if patch.content.is_some() && patch.vector.is_none() {
4908            return Err(MnemoError::Invalid(
4909                "update: a content change needs a matching vector (or call update_text \
4910                 to re-embed through the registered embedder)"
4911                    .into(),
4912            ));
4913        }
4914        let idx = self
4915            .index
4916            .get(id.0, &self.catalog)
4917            .ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
4918        if self.catalog[idx].deleted {
4919            return Err(MnemoError::NotFound(id.to_string()));
4920        }
4921        let entry = self.catalog[idx].clone();
4922        let existing = self.read_memory(&entry)?;
4923
4924        // Apply patch into a candidate, detect no-op after the fact.
4925        let mut cand = existing.clone();
4926        if let Some(c) = &patch.content {
4927            cand.content = c.clone();
4928        }
4929        if let Some(v) = &patch.vector {
4930            if v.len() != self.dimensions {
4931                return Err(MnemoError::DimensionMismatch {
4932                    expected: self.dimensions,
4933                    got: v.len(),
4934                });
4935            }
4936            cand.vector = v.clone();
4937        }
4938        if let Some(i) = patch.importance {
4939            cand.importance = i.clamp(0.0, 1.0);
4940        }
4941        if let Some(mt) = patch.memory_type {
4942            cand.memory_type = mt;
4943        }
4944        if let Some(new_meta) = patch.metadata {
4945            // Keep reserved `_mnemo.*` keys from the existing record.
4946            let reserved: Vec<(String, Value)> = existing
4947                .metadata
4948                .iter()
4949                .filter(|(k, _)| k.starts_with(crate::memory::MNEMO_META_NS))
4950                .map(|(k, v)| (k.clone(), v.clone()))
4951                .collect();
4952            cand.metadata = new_meta;
4953            for (k, v) in reserved {
4954                cand.metadata.insert(k, v);
4955            }
4956        }
4957        if let Some(ttl) = patch.ttl_secs {
4958            cand.ttl_secs = ttl;
4959        }
4960
4961        // No-op detection: compare all user-visible fields.
4962        if cand.content == existing.content
4963            && cand.vector == existing.vector
4964            && (cand.importance - existing.importance).abs() <= f32::EPSILON
4965            && cand.memory_type == existing.memory_type
4966            && cand.metadata == existing.metadata
4967            && cand.ttl_secs == existing.ttl_secs
4968        {
4969            return Ok(WriteOutcome::Unchanged);
4970        }
4971
4972        // Stamp `_mnemo.updated_at` and preserve id + created_at.
4973        let now = memory::now_secs();
4974        cand.metadata
4975            .insert(MNEMO_META_UPDATED_AT.into(), Value::Number(now.into()));
4976        cand.id = existing.id;
4977        cand.created_at = existing.created_at;
4978        // put() preserves accessed_at/access_count via the index lookup.
4979        self.put(cand)?;
4980        // `put` kept the key slot and the other handle caches current.
4981        Ok(WriteOutcome::Updated)
4982    }
4983
4984    /// In-place update that re-embeds `new_text` through the registered
4985    /// embedder. Shorthand for `update(id, patch.with_content_and_vector)`.
4986    pub fn update_text(
4987        &mut self,
4988        id: &Ulid,
4989        new_text: &str,
4990        mut patch: MemoryPatch,
4991    ) -> Result<WriteOutcome> {
4992        if self.read_only {
4993            return Err(MnemoError::ReadOnly);
4994        }
4995        let embedder = self.embedder.as_ref().ok_or_else(|| {
4996            MnemoError::Invalid(
4997                "no embedder registered on this handle; call set_embedder first".into(),
4998            )
4999        })?;
5000        let mut vectors = embedder.embed(&[new_text])?;
5001        let vector = vectors.pop().ok_or_else(|| {
5002            MnemoError::Invalid("embedder returned no vector for the input text".into())
5003        })?;
5004        if vector.len() != self.dimensions {
5005            return Err(MnemoError::DimensionMismatch {
5006                expected: self.dimensions,
5007                got: vector.len(),
5008            });
5009        }
5010        patch.content = Some(new_text.to_string());
5011        patch.vector = Some(vector);
5012        self.update(id, patch)
5013    }
5014
5015    /// Walk the full chain for a keyed slot: every record this key has
5016    /// ever held, newest first. Entry 0 is the **live** record (unless
5017    /// the slot is empty, in which case the vector is empty); subsequent
5018    /// entries are the superseded history in reverse-supersession order.
5019    ///
5020    /// **Slot resolution (v0.8.1 bugfix, lane A).** The lookup slot
5021    /// matches a keyed `remember` with the same arguments, so the
5022    /// obvious `history(key, None)` walks the private slot of the
5023    /// default agent — the same slot
5024    /// `remember_keyed(Memory::new(...), key)` writes to when the
5025    /// caller omits an explicit `agent_id` / `scope`. Pre-v0.8.1
5026    /// `agent_id = None` incorrectly looked up the shared slot and
5027    /// returned an empty chain for the common case; the old behavior
5028    /// is still reachable via `history_in(key, KeyScope::Shared)`.
5029    ///
5030    /// * `agent_id = Some(a)` → private slot for that agent.
5031    /// * `agent_id = None` → private slot for the default agent (same
5032    ///   as [`KeyScope::private_default`]).
5033    ///
5034    /// Use [`Mnemo::history_in`] with [`KeyScope::Shared`] for the
5035    /// shared slot.
5036    pub fn history(&mut self, key: &str, agent_id: Option<&str>) -> Result<Vec<Memory>> {
5037        let scope = match agent_id {
5038            Some(a) => KeyScope::Private(a.to_string()),
5039            None => KeyScope::private_default(),
5040        };
5041        self.history_in(key, scope)
5042    }
5043
5044    /// Walk the full chain for a keyed slot identified by an explicit
5045    /// [`KeyScope`]. Returns the same shape as [`Mnemo::history`] —
5046    /// live record first, then the superseded chain newest-first.
5047    ///
5048    /// Use this when you need the **shared** slot (`KeyScope::Shared`),
5049    /// which `history(key, None)` no longer reaches as of v0.8.1.
5050    pub fn history_in(&mut self, key: &str, scope: KeyScope) -> Result<Vec<Memory>> {
5051        self.ensure_key_caches()?;
5052        let slot = crate::store::KeySlot {
5053            scope_owner: match scope {
5054                KeyScope::Private(a) => Some(a),
5055                KeyScope::Shared => None,
5056            },
5057            key: key.to_string(),
5058        };
5059        let live_id = self.key_index.as_ref().and_then(|k| k.get(&slot).copied());
5060        let mut out = Vec::new();
5061        let mut cursor = live_id;
5062        while let Some(id_u) = cursor {
5063            let idx = self.index.get(id_u, &self.catalog).ok_or_else(|| {
5064                MnemoError::Invalid("history: superseded chain references missing id".into())
5065            })?;
5066            let entry = self.catalog[idx].clone();
5067            if entry.deleted {
5068                break;
5069            }
5070            let m = self.read_memory(&entry)?;
5071            let next = m
5072                .metadata
5073                .get(MNEMO_META_SUPERSEDES)
5074                .and_then(|v| v.as_str())
5075                .and_then(|s| Ulid::from_string(s).ok())
5076                .map(|u| u.0);
5077            out.push(m);
5078            cursor = next;
5079        }
5080        Ok(out)
5081    }
5082
5083    // --- Phase 12.7a: thread archive --------------------------------
5084
5085    /// Ingest a batch of turns into the archive layer of `thread_id`.
5086    ///
5087    /// Each [`ThreadTurn`] becomes one or more `Episodic` records
5088    /// tagged with the usual thread-archive reserved-metadata set
5089    /// (`_mnemo.kind = "turn"`, `_mnemo.thread`, `_mnemo.turn_id`,
5090    /// `_mnemo.seq`, `_mnemo.role`). The ingest path is
5091    /// **idempotent** per `turn_id`: a turn whose `turn_id` already
5092    /// appears on a record in this thread is skipped, so re-running
5093    /// `ingest_thread` with an extended transcript appends only
5094    /// what's new.
5095    ///
5096    /// **Chunking.** A non-tool turn whose UTF-8 content exceeds
5097    /// [`THREAD_TURN_CHUNK_BYTES`] (8 KB) is split into chunks at
5098    /// char boundaries, each stored as its own record with
5099    /// `_mnemo.part = 0, 1, …` under the same `turn_id`.
5100    ///
5101    /// **Truncation.** A `role = "tool"` turn whose content exceeds
5102    /// [`THREAD_TOOL_RESULT_TRUNCATE_BYTES`] (16 KB) is stored
5103    /// truncated — the first 16 KB of UTF-8 goes into the record,
5104    /// and `_mnemo.sha256` + `_mnemo.length` capture the original
5105    /// hash and byte length so the full content can be recovered
5106    /// (or verified) out-of-band.
5107    ///
5108    /// **Embedding.** An embedder must be registered
5109    /// ([`Mnemo::set_embedder`]); the content strings (one per
5110    /// resulting chunk / truncated record) are batched into a single
5111    /// `embed()` call.
5112    ///
5113    /// Call `self.flush()` after a successful ingest to persist.
5114    pub fn ingest_thread(&mut self, thread_id: &str, turns: &[ThreadTurn]) -> Result<IngestReport> {
5115        if self.read_only {
5116            return Err(MnemoError::ReadOnly);
5117        }
5118        let plan = self.plan_ingest(thread_id, turns)?;
5119        if plan.pending.is_empty() {
5120            return Ok(plan.report);
5121        }
5122        // Batched embed: one round trip to the embedder for the whole
5123        // ingest, whatever the turn count.
5124        let vectors = {
5125            let embedder = self.embedder.as_ref().ok_or_else(|| {
5126                MnemoError::Invalid(
5127                    "ingest_thread: no embedder registered on this handle; call set_embedder first"
5128                        .into(),
5129                )
5130            })?;
5131            embedder.embed(&plan.texts())?
5132        };
5133        self.check_vectors("ingest_thread", &vectors, plan.text_count())?;
5134        self.apply_ingest(thread_id, plan, vectors, None)
5135    }
5136
5137    /// Check a batch of embedder output before anything is written:
5138    /// one vector per input, the file's width, finite and non-zero.
5139    fn check_vectors(&self, ctx: &str, vectors: &[Vec<f32>], expected: usize) -> Result<()> {
5140        if vectors.len() != expected {
5141            return Err(MnemoError::Invalid(format!(
5142                "{ctx}: embedder returned {} vectors for {expected} texts",
5143                vectors.len()
5144            )));
5145        }
5146        for v in vectors {
5147            self.check_vector(ctx, v)?;
5148        }
5149        Ok(())
5150    }
5151
5152    /// The checks `put` applies to one vector, run up front so a batch
5153    /// can't fail half-way through its writes.
5154    fn check_vector(&self, ctx: &str, v: &[f32]) -> Result<()> {
5155        if v.len() != self.dimensions {
5156            return Err(MnemoError::DimensionMismatch {
5157                expected: self.dimensions,
5158                got: v.len(),
5159            });
5160        }
5161        if embed::is_zero_vector(v) {
5162            return Err(MnemoError::Invalid(format!(
5163                "{ctx}: all-zero vector refused"
5164            )));
5165        }
5166        if v.iter().any(|x| !x.is_finite()) {
5167            return Err(MnemoError::Invalid(format!(
5168                "{ctx}: vector contains a non-finite value"
5169            )));
5170        }
5171        Ok(())
5172    }
5173
5174    /// Planning half of [`Mnemo::ingest_thread`]: validate the turns, skip
5175    /// the ones already archived, and chunk or truncate the rest. Reads
5176    /// only; nothing is written.
5177    fn plan_ingest(&mut self, thread_id: &str, turns: &[ThreadTurn]) -> Result<IngestPlan> {
5178        if thread_id.is_empty() {
5179            return Err(MnemoError::Invalid(
5180                "ingest_thread: 'thread_id' must not be empty".into(),
5181            ));
5182        }
5183        // FMT-9b.2: v9 chunks fit one packed page with their metadata.
5184        let chunk_bytes = if self.is_v9() {
5185            V9_TURN_CHUNK_BYTES
5186        } else {
5187            THREAD_TURN_CHUNK_BYTES
5188        };
5189        let mut report = IngestReport::default();
5190        if turns.is_empty() {
5191            return Ok(IngestPlan {
5192                pending: Vec::new(),
5193                next_seq: 0,
5194                report,
5195            });
5196        }
5197
5198        // Build the set of turn_ids already in this thread so the
5199        // idempotence check is O(1) per incoming turn. First-pass
5200        // cost: one decrypt per existing thread record. A future v9
5201        // format bump can move `_mnemo.turn_id` into the catalog row
5202        // and drop this scan.
5203        // R-004.2: the turn ids already archived and the next `_mnemo.seq`
5204        // come from the handle caches: one lazy scan per handle, then O(1)
5205        // per ingest instead of two full decrypt scans each time.
5206        self.ensure_key_caches()?;
5207        self.warm_thread(thread_id)?;
5208        let cached = self.layers.as_ref().and_then(|l| l.threads.get(thread_id));
5209        let existing_turn_ids: std::collections::HashSet<&str> = cached
5210            .map(|tc| tc.turns.keys().map(String::as_str).collect())
5211            .unwrap_or_default();
5212        // A repeat ingest continues the sequence instead of restarting at 0.
5213        let next_seq = cached.and_then(|tc| tc.max_seq).map_or(0, |n| n + 1);
5214
5215        // Determine each incoming turn's final text pieces (post-
5216        // chunking / truncation) before any writes, so the caller can
5217        // embed the whole batch in one `embed()` call.
5218        let mut pending: Vec<PendingTurn> = Vec::new();
5219        let now = memory::now_secs();
5220        for t in turns {
5221            if t.turn_id.is_empty() {
5222                return Err(MnemoError::Invalid(
5223                    "ingest_thread: turn_id must not be empty".into(),
5224                ));
5225            }
5226            if existing_turn_ids.contains(t.turn_id.as_str()) {
5227                report.skipped_existing_turns += 1;
5228                continue;
5229            }
5230            let content_bytes = t.content.len();
5231            let is_tool = t.role == "tool";
5232            let (parts, truncation) =
5233                if is_tool && content_bytes > THREAD_TOOL_RESULT_TRUNCATE_BYTES {
5234                    report.truncated_turns += 1;
5235                    let hash = {
5236                        use sha2::{Digest, Sha256};
5237                        let h = Sha256::digest(t.content.as_bytes());
5238                        hex_encode(&h)
5239                    };
5240                    let truncated = take_utf8_prefix(&t.content, THREAD_TOOL_RESULT_TRUNCATE_BYTES);
5241                    (vec![truncated], Some((hash, content_bytes as u64)))
5242                } else if content_bytes > chunk_bytes {
5243                    report.chunked_turns += 1;
5244                    (chunk_utf8(&t.content, chunk_bytes), None)
5245                } else {
5246                    (vec![t.content.clone()], None)
5247                };
5248            pending.push(PendingTurn {
5249                turn_id: t.turn_id.clone(),
5250                role: t.role.clone(),
5251                created_at: t.timestamp.unwrap_or(now),
5252                parts,
5253                truncation,
5254            });
5255        }
5256        Ok(IngestPlan {
5257            pending,
5258            next_seq,
5259            report,
5260        })
5261    }
5262
5263    /// Write half of [`Mnemo::ingest_thread`]. `vectors` holds one vector
5264    /// per text of `plan.texts()`, already checked. `ttl` stamps every
5265    /// new record (the `commit_deltas` `ExpireAfter` policy).
5266    fn apply_ingest(
5267        &mut self,
5268        thread_id: &str,
5269        plan: IngestPlan,
5270        vectors: Vec<Vec<f32>>,
5271        ttl: Option<i64>,
5272    ) -> Result<IngestReport> {
5273        let IngestPlan {
5274            pending,
5275            mut next_seq,
5276            mut report,
5277        } = plan;
5278        // Order: each pending turn's chunks in part order; each turn
5279        // gets a single `_mnemo.seq` slot, so `_mnemo.part`
5280        // disambiguates chunks of the same turn.
5281        let mut vectors = vectors.into_iter();
5282        for p in &pending {
5283            let this_seq = next_seq;
5284            next_seq += 1;
5285            for (part_idx, part_text) in p.parts.iter().enumerate() {
5286                let vec = vectors.next().ok_or_else(|| {
5287                    MnemoError::Invalid("ingest_thread: ran out of vectors".into())
5288                })?;
5289                let mut m =
5290                    Memory::new(part_text.clone(), MemoryType::Episodic, vec).with_importance(0.5);
5291                if let Some(t) = ttl {
5292                    m = m.with_ttl(t);
5293                }
5294                m.created_at = p.created_at;
5295                m.accessed_at = p.created_at;
5296                m.metadata.insert(
5297                    MNEMO_META_KIND.into(),
5298                    Value::String(MNEMO_META_KIND_TURN.into()),
5299                );
5300                m.metadata.insert(
5301                    MNEMO_META_THREAD.into(),
5302                    Value::String(thread_id.to_string()),
5303                );
5304                m.metadata
5305                    .insert(MNEMO_META_TURN_ID.into(), Value::String(p.turn_id.clone()));
5306                m.metadata
5307                    .insert(MNEMO_META_SEQ.into(), Value::Number(this_seq.into()));
5308                m.metadata
5309                    .insert(MNEMO_META_ROLE.into(), Value::String(p.role.clone()));
5310                if p.parts.len() > 1 {
5311                    m.metadata.insert(
5312                        MNEMO_META_PART.into(),
5313                        Value::Number((part_idx as u64).into()),
5314                    );
5315                }
5316                if let Some((sha, len)) = &p.truncation {
5317                    m.metadata
5318                        .insert(MNEMO_META_SHA256.into(), Value::String(sha.clone()));
5319                    m.metadata
5320                        .insert(MNEMO_META_LENGTH.into(), Value::Number((*len).into()));
5321                }
5322                self.put(m)?;
5323                report.added_records += 1;
5324            }
5325        }
5326        Ok(report)
5327    }
5328
5329    /// Export the archive layer for `thread_id` in `_mnemo.seq`
5330    /// order. Each entry is the Memory as stored; chunks of a
5331    /// multi-part turn come out in `_mnemo.part` order within their
5332    /// `_mnemo.seq`. The record `.content` is the text that was
5333    /// stored (post-chunking / post-truncation); callers that need
5334    /// the pre-truncation original should round-trip via
5335    /// `_mnemo.sha256` + `_mnemo.length`.
5336    pub fn thread_export(&mut self, thread_id: &str) -> Result<Vec<Memory>> {
5337        // R-004.2: the thread's own records, from the handle caches.
5338        // FMT-9e: a cold thread's records were just decoded to warm it;
5339        // reuse them instead of decoding (and decompressing) twice.
5340        let (entries, mut decoded) = self.thread_entries_decoded(thread_id)?;
5341        let mut out: Vec<Memory> = Vec::new();
5342        for e in &entries {
5343            let m = match decoded.remove(&e.id) {
5344                Some(m) => self.finish_memory(e, m)?,
5345                None => self.read_memory(e)?,
5346            };
5347            if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(thread_id) {
5348                continue;
5349            }
5350            if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
5351                != Some(MNEMO_META_KIND_TURN)
5352            {
5353                continue;
5354            }
5355            out.push(m);
5356        }
5357        out.sort_by(|a, b| {
5358            let seq_a = a
5359                .metadata
5360                .get(MNEMO_META_SEQ)
5361                .and_then(|v| v.as_u64())
5362                .unwrap_or(0);
5363            let seq_b = b
5364                .metadata
5365                .get(MNEMO_META_SEQ)
5366                .and_then(|v| v.as_u64())
5367                .unwrap_or(0);
5368            let part_a = a
5369                .metadata
5370                .get(MNEMO_META_PART)
5371                .and_then(|v| v.as_u64())
5372                .unwrap_or(0);
5373            let part_b = b
5374                .metadata
5375                .get(MNEMO_META_PART)
5376                .and_then(|v| v.as_u64())
5377                .unwrap_or(0);
5378            seq_a.cmp(&seq_b).then_with(|| part_a.cmp(&part_b))
5379        });
5380        Ok(out)
5381    }
5382
5383    /// Export a thread's archive in the **ingest shape** (Phase 12.7d):
5384    /// one [`ExportedTurn`] per source turn, in `_mnemo.seq` order, with
5385    /// a chunked turn's parts joined back into one `content`. Passing
5386    /// the result back to [`Mnemo::ingest_thread`] for the same thread
5387    /// skips every turn; for a new thread it re-creates the same records.
5388    /// This is what Python `db.thread_export`, MCP `thread_export` and
5389    /// `mnemo thread export` return; [`Mnemo::thread_export`] still
5390    /// returns the raw records.
5391    pub fn thread_export_turns(&mut self, thread_id: &str) -> Result<Vec<ExportedTurn>> {
5392        let records = self.thread_export(thread_id)?;
5393        let mut out: Vec<ExportedTurn> = Vec::new();
5394        let mut last_seq: Option<u64> = None;
5395        for m in records {
5396            let meta_str = |k: &str| {
5397                m.metadata
5398                    .get(k)
5399                    .and_then(|v| v.as_str())
5400                    .unwrap_or("")
5401                    .to_string()
5402            };
5403            let seq = m.metadata.get(MNEMO_META_SEQ).and_then(|v| v.as_u64());
5404            // Records are sorted by (seq, part), so the parts of one turn
5405            // are adjacent and share a seq.
5406            if seq.is_some() && seq == last_seq {
5407                if let Some(prev) = out.last_mut() {
5408                    prev.content.push_str(&m.content);
5409                    continue;
5410                }
5411            }
5412            last_seq = seq;
5413            out.push(ExportedTurn {
5414                turn_id: meta_str(MNEMO_META_TURN_ID),
5415                role: meta_str(MNEMO_META_ROLE),
5416                timestamp: m.created_at,
5417                sha256: m
5418                    .metadata
5419                    .get(MNEMO_META_SHA256)
5420                    .and_then(|v| v.as_str())
5421                    .map(String::from),
5422                original_length: m.metadata.get(MNEMO_META_LENGTH).and_then(|v| v.as_u64()),
5423                content: m.content,
5424            });
5425        }
5426        Ok(out)
5427    }
5428
5429    /// Soft-delete every record in the archive layer of `thread_id`.
5430    /// Returns the number of records marked deleted. Permanent
5431    /// removal still requires a subsequent `compact_file`, same as
5432    /// any `Mnemo::delete`.
5433    ///
5434    /// Does **not** touch `_mnemo.kind = "essence"` records or other
5435    /// non-turn metadata the thread may carry — only `_mnemo.kind =
5436    /// "turn"` records matching the thread id.
5437    pub fn thread_forget(&mut self, thread_id: &str) -> Result<usize> {
5438        if self.read_only {
5439            return Err(MnemoError::ReadOnly);
5440        }
5441        // Collect first, then delete — can't hold catalog borrows
5442        // across `self.delete`.
5443        let entries = self.thread_entries(thread_id)?;
5444        let mut victims: Vec<Ulid> = Vec::new();
5445        for e in &entries {
5446            let m = self.read_memory(e)?;
5447            if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(thread_id) {
5448                continue;
5449            }
5450            if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
5451                != Some(MNEMO_META_KIND_TURN)
5452            {
5453                continue;
5454            }
5455            victims.push(m.id);
5456        }
5457        let n = victims.len();
5458        for id in &victims {
5459            self.delete(id)?;
5460        }
5461        Ok(n)
5462    }
5463
5464    // --- Phase 12.7b: essence (timeline + sections) -----------------
5465
5466    /// Phase 12.7b: add a new essence item to a section (any section
5467    /// except [`EssenceSection::Timeline`]). Content ≤
5468    /// [`memory::ESSENCE_SECTION_MAX_BYTES`] (280 bytes); items over
5469    /// that are rejected. Starting status is `"open"`.
5470    ///
5471    /// Use [`Mnemo::essence_append_timeline`] for timeline entries —
5472    /// they have a different cap (600 bytes), are append-only, and
5473    /// carry no status lifecycle.
5474    pub fn essence_add(
5475        &mut self,
5476        scope: EssenceScope,
5477        section: EssenceSection,
5478        text: &str,
5479        refs: Option<Vec<String>>,
5480    ) -> Result<Ulid> {
5481        if self.read_only {
5482            return Err(MnemoError::ReadOnly);
5483        }
5484        Self::check_essence_add(section, text)?;
5485        self.essence_add_with(scope, section, text, refs, None)
5486    }
5487
5488    /// Validation shared by [`Mnemo::essence_add`] and `commit_deltas`.
5489    fn check_essence_add(section: EssenceSection, text: &str) -> Result<()> {
5490        if section == EssenceSection::Timeline {
5491            return Err(MnemoError::Invalid(
5492                "essence_add: use essence_append_timeline for the timeline section".into(),
5493            ));
5494        }
5495        if text.is_empty() {
5496            return Err(MnemoError::Invalid(
5497                "essence_add: text must not be empty".into(),
5498            ));
5499        }
5500        if text.len() > section.item_max_bytes() {
5501            return Err(MnemoError::Invalid(format!(
5502                "essence_add: '{}' items must be <= {} bytes; got {}",
5503                section.as_str(),
5504                section.item_max_bytes(),
5505                text.len()
5506            )));
5507        }
5508        Ok(())
5509    }
5510
5511    /// Write half of `essence_add`, with an optional precomputed vector.
5512    fn essence_add_with(
5513        &mut self,
5514        scope: EssenceScope,
5515        section: EssenceSection,
5516        text: &str,
5517        refs: Option<Vec<String>>,
5518        vector: Option<Vec<f32>>,
5519    ) -> Result<Ulid> {
5520        let now = memory::now_secs();
5521        let mem = self.build_essence_record(scope, section, text, refs, Some(now), vector)?;
5522        self.put(mem)
5523    }
5524
5525    /// Phase 12.7b: append a new timeline entry (append-only
5526    /// milestone). Content ≤ [`memory::ESSENCE_TIMELINE_MAX_BYTES`]
5527    /// (600 bytes); entries over that are rejected (not truncated —
5528    /// truncating a milestone silently loses the narrative).
5529    pub fn essence_append_timeline(
5530        &mut self,
5531        scope: EssenceScope,
5532        text: &str,
5533        refs: Option<Vec<String>>,
5534    ) -> Result<Ulid> {
5535        if self.read_only {
5536            return Err(MnemoError::ReadOnly);
5537        }
5538        Self::check_essence_append(text)?;
5539        self.essence_append_with(scope, text, refs, None)
5540    }
5541
5542    /// Validation shared by [`Mnemo::essence_append_timeline`] and
5543    /// `commit_deltas`.
5544    fn check_essence_append(text: &str) -> Result<()> {
5545        if text.is_empty() {
5546            return Err(MnemoError::Invalid(
5547                "essence_append_timeline: text must not be empty".into(),
5548            ));
5549        }
5550        if text.len() > EssenceSection::Timeline.item_max_bytes() {
5551            return Err(MnemoError::Invalid(format!(
5552                "essence_append_timeline: entries must be <= {} bytes; got {}",
5553                EssenceSection::Timeline.item_max_bytes(),
5554                text.len()
5555            )));
5556        }
5557        Ok(())
5558    }
5559
5560    /// Write half of `essence_append_timeline`, with an optional
5561    /// precomputed vector.
5562    fn essence_append_with(
5563        &mut self,
5564        scope: EssenceScope,
5565        text: &str,
5566        refs: Option<Vec<String>>,
5567        vector: Option<Vec<f32>>,
5568    ) -> Result<Ulid> {
5569        let mem =
5570            self.build_essence_record(scope, EssenceSection::Timeline, text, refs, None, vector)?;
5571        self.put(mem)
5572    }
5573
5574    /// Phase 12.7b: update an existing essence section item in place.
5575    /// Returns [`WriteOutcome::Updated`] / [`WriteOutcome::Unchanged`].
5576    /// Timeline entries are append-only — calling this on a timeline
5577    /// record is an error.
5578    pub fn essence_update(
5579        &mut self,
5580        id: &Ulid,
5581        text: &str,
5582        refs: Option<Vec<String>>,
5583    ) -> Result<WriteOutcome> {
5584        if self.read_only {
5585            return Err(MnemoError::ReadOnly);
5586        }
5587        let existing = self.load_essence_record(id)?;
5588        let section = self.section_of(&existing)?;
5589        if section == EssenceSection::Timeline {
5590            return Err(MnemoError::Invalid(
5591                "essence_update: timeline entries are append-only; append a new one instead".into(),
5592            ));
5593        }
5594        if text.is_empty() {
5595            return Err(MnemoError::Invalid(
5596                "essence_update: text must not be empty".into(),
5597            ));
5598        }
5599        if text.len() > section.item_max_bytes() {
5600            return Err(MnemoError::Invalid(format!(
5601                "essence_update: '{}' items must be <= {} bytes; got {}",
5602                section.as_str(),
5603                section.item_max_bytes(),
5604                text.len()
5605            )));
5606        }
5607        let mut patch = MemoryPatch {
5608            content: Some(text.to_string()),
5609            ..Default::default()
5610        };
5611        // We don't have a usable embedder-free path for an in-place
5612        // content update, so provide the same vector as the existing
5613        // record (the text-vector drift guard in `update` will accept
5614        // this).
5615        patch.vector = Some(existing.vector.clone());
5616        if let Some(r) = refs {
5617            let mut meta = Map::new();
5618            // Preserve the user-visible metadata (none for essence —
5619            // everything lives under `_mnemo.*`) and let `update`'s
5620            // metadata-replacement path keep the reserved keys.
5621            meta.insert(
5622                MNEMO_META_REFS.into(),
5623                Value::Array(r.into_iter().map(Value::String).collect()),
5624            );
5625            patch.metadata = Some(meta);
5626        }
5627        self.update(id, patch)
5628    }
5629
5630    /// Phase 12.7b: mark a section item as resolved (open → resolved
5631    /// transition). A new history entry lands on
5632    /// `_mnemo.status_history`. Idempotent — resolving a record that
5633    /// is already resolved is a no-op.
5634    pub fn essence_resolve(&mut self, id: &Ulid) -> Result<WriteOutcome> {
5635        self.essence_transition_status(id, essence_status::RESOLVED)
5636    }
5637
5638    /// Phase 12.7b: soft-delete a section item. Same semantics as
5639    /// [`Mnemo::delete`] on any record — space reclaimed by a later
5640    /// compact.
5641    pub fn essence_remove(&mut self, id: &Ulid) -> Result<()> {
5642        if self.read_only {
5643            return Err(MnemoError::ReadOnly);
5644        }
5645        self.delete(id)
5646    }
5647
5648    /// Phase 12.7b: deterministic, budgeted markdown render of the
5649    /// essence layer for a thread (or the project layer when
5650    /// `scope = EssenceScope::Project`). `max_bytes` caps the output;
5651    /// sections are emitted in the fixed [`EssenceSection::render_order`],
5652    /// and each section's own budget is proportional to its position
5653    /// in the order. Oldest timeline entries fold into a
5654    /// `+N earlier milestones` tail when the budget runs out.
5655    ///
5656    /// The render always ends with an archive pointer so the reader
5657    /// knows how to drill in deeper (`mnemo thread export …` or
5658    /// `recall --thread …`).
5659    pub fn essence(&mut self, scope: EssenceScope, max_bytes: usize) -> Result<String> {
5660        let max_bytes = if max_bytes == 0 {
5661            ESSENCE_DEFAULT_RENDER_BYTES
5662        } else {
5663            max_bytes
5664        };
5665        // Collect every live essence record for this scope.
5666        // R-004.2: only the essence items, from the handle caches.
5667        let entries = self.essence_entries()?;
5668        let mut by_section: std::collections::BTreeMap<&'static str, Vec<Memory>> =
5669            std::collections::BTreeMap::new();
5670        for section in EssenceSection::render_order() {
5671            by_section.insert(section.as_str(), Vec::new());
5672        }
5673        for e in &entries {
5674            let m = self.read_memory(e)?;
5675            if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
5676                != Some(MNEMO_META_KIND_ESSENCE)
5677            {
5678                continue;
5679            }
5680            // Scope filter. Project-scope records are shown in every
5681            // thread's render (and in the project render); thread-scope
5682            // records only match when scope == same thread.
5683            let rec_thread = m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str());
5684            let matches_scope = match (&scope, rec_thread) {
5685                (EssenceScope::Project, None) => true,
5686                (EssenceScope::Thread(_), None) => true, // project-scope always shows
5687                (EssenceScope::Thread(t), Some(rt)) => rt == t,
5688                (EssenceScope::Project, Some(_)) => false,
5689            };
5690            if !matches_scope {
5691                continue;
5692            }
5693            let section_str = m
5694                .metadata
5695                .get(MNEMO_META_SECTION)
5696                .and_then(|v| v.as_str())
5697                .unwrap_or("timeline");
5698            by_section
5699                .entry(
5700                    EssenceSection::parse(section_str)
5701                        .map(|s| s.as_str())
5702                        .unwrap_or("timeline"),
5703                )
5704                .or_default()
5705                .push(m);
5706        }
5707
5708        // Sort each section's records:
5709        // - Decisions + Timeline: newest-first (by created_at desc).
5710        // - Everything else: sort open items before resolved, then
5711        //   newest-first. Cheap, deterministic.
5712        for (name, items) in by_section.iter_mut() {
5713            let newest_first = matches!(*name, "decisions" | "timeline");
5714            items.sort_by(|a, b| {
5715                let a_resolved = status_of(a) == Some(essence_status::RESOLVED);
5716                let b_resolved = status_of(b) == Some(essence_status::RESOLVED);
5717                match (a_resolved, b_resolved) {
5718                    (false, true) => std::cmp::Ordering::Less,
5719                    (true, false) => std::cmp::Ordering::Greater,
5720                    _ => {
5721                        if newest_first {
5722                            b.created_at.cmp(&a.created_at)
5723                        } else {
5724                            a.created_at.cmp(&b.created_at)
5725                        }
5726                    }
5727                }
5728            });
5729        }
5730
5731        // Per-section budget: give intent / conventions / decisions a
5732        // larger share (they frame the thread), timeline a smaller
5733        // share (it can fold). This is deterministic because it
5734        // depends only on `max_bytes`.
5735        let header_reserve = 256usize; // header + archive pointer
5736        let body_budget = max_bytes.saturating_sub(header_reserve);
5737        let per_section_budget = |section: EssenceSection| -> usize {
5738            match section {
5739                EssenceSection::Intent => body_budget / 8,
5740                EssenceSection::Conventions => body_budget / 6,
5741                EssenceSection::Decisions => body_budget / 4,
5742                EssenceSection::State => body_budget / 8,
5743                EssenceSection::Open => body_budget / 8,
5744                EssenceSection::Next => body_budget / 12,
5745                EssenceSection::Lessons => body_budget / 12,
5746                EssenceSection::Timeline => body_budget / 4,
5747            }
5748        };
5749
5750        let mut out = String::with_capacity(max_bytes);
5751        let scope_label = match &scope {
5752            EssenceScope::Thread(t) => format!("thread `{t}`"),
5753            EssenceScope::Project => "project".to_string(),
5754        };
5755        out.push_str(&format!("# essence — {scope_label}\n\n"));
5756
5757        // Phase 12.7c: continuation. If this is a thread render AND
5758        // the thread has a `_mnemo.continues` link, pull the prior
5759        // thread's open section items, its `next`, and its last-N
5760        // timeline entries in ahead of the new thread's own essence.
5761        if let EssenceScope::Thread(new_id) = &scope {
5762            if let Some(prev_id) = self.predecessor_thread(new_id)? {
5763                let continuation = self
5764                    .render_continuation_prelude(&prev_id, max_bytes.saturating_sub(out.len()))?;
5765                if !continuation.is_empty() && out.len() + continuation.len() <= max_bytes {
5766                    out.push_str(&continuation);
5767                }
5768            }
5769        }
5770
5771        for section in EssenceSection::render_order() {
5772            let items = match by_section.get(section.as_str()) {
5773                Some(v) if !v.is_empty() => v,
5774                _ => continue,
5775            };
5776            let budget = per_section_budget(*section);
5777            let section_header = format!("## {}\n", section.as_str());
5778            if out.len() + section_header.len() > max_bytes {
5779                break;
5780            }
5781            out.push_str(&section_header);
5782            let mut used = 0usize;
5783            let mut rendered = 0usize;
5784            for item in items {
5785                let line = render_essence_item(*section, item);
5786                if used + line.len() > budget {
5787                    break;
5788                }
5789                if out.len() + line.len() > max_bytes {
5790                    break;
5791                }
5792                out.push_str(&line);
5793                used += line.len();
5794                rendered += 1;
5795            }
5796            // Fold the remainder for timeline.
5797            if *section == EssenceSection::Timeline && rendered < items.len() {
5798                let fold = format!("- +{} earlier milestones\n", items.len() - rendered);
5799                if out.len() + fold.len() <= max_bytes {
5800                    out.push_str(&fold);
5801                }
5802            }
5803            if out.len() < max_bytes {
5804                out.push('\n');
5805            }
5806        }
5807
5808        // Archive pointer.
5809        let pointer = match &scope {
5810            EssenceScope::Thread(t) => format!(
5811                "> full history: `mnemo thread export <file> --thread {t}` or `recall --thread {t}`\n"
5812            ),
5813            EssenceScope::Project => String::from(
5814                "> full history: `mnemo thread export <file> --thread <id>` or `recall --thread <id>`\n",
5815            ),
5816        };
5817        if out.len() + pointer.len() <= max_bytes {
5818            out.push_str(&pointer);
5819        }
5820        Ok(out)
5821    }
5822
5823    // --- essence internal helpers ----------------------------------
5824
5825    /// Build an essence record. `vector` is a precomputed embedding (the
5826    /// 12.7d `commit_deltas` batch path); `None` embeds `text` with the
5827    /// registered embedder, or falls back to an axis-neutral unit vector
5828    /// when there is none. Essence records are read by `_mnemo.kind` and
5829    /// scope, not by similarity, so the fallback costs no retrieval
5830    /// quality; it only has to pass `put`'s non-zero guard.
5831    fn build_essence_record(
5832        &self,
5833        scope: EssenceScope,
5834        section: EssenceSection,
5835        text: &str,
5836        refs: Option<Vec<String>>,
5837        status_at: Option<i64>,
5838        vector: Option<Vec<f32>>,
5839    ) -> Result<Memory> {
5840        let v = match (vector, self.embedder.as_ref()) {
5841            (Some(v), _) => v,
5842            (None, Some(e)) => e.embed(&[text])?.pop().ok_or_else(|| {
5843                MnemoError::Invalid("essence: embedder returned no vector".into())
5844            })?,
5845            (None, None) => self.sentinel_vector(),
5846        };
5847        let mut m = Memory::new(text.to_string(), MemoryType::Semantic, v).with_importance(0.5);
5848        m.metadata.insert(
5849            MNEMO_META_KIND.into(),
5850            Value::String(MNEMO_META_KIND_ESSENCE.into()),
5851        );
5852        m.metadata.insert(
5853            MNEMO_META_SECTION.into(),
5854            Value::String(section.as_str().into()),
5855        );
5856        if let EssenceScope::Thread(t) = &scope {
5857            m.metadata
5858                .insert(MNEMO_META_THREAD.into(), Value::String(t.clone()));
5859        }
5860        if let Some(r) = refs {
5861            m.metadata.insert(
5862                MNEMO_META_REFS.into(),
5863                Value::Array(r.into_iter().map(Value::String).collect()),
5864            );
5865        }
5866        if let Some(at) = status_at {
5867            m.metadata.insert(
5868                MNEMO_META_STATUS.into(),
5869                Value::String(essence_status::OPEN.into()),
5870            );
5871            m.metadata.insert(
5872                MNEMO_META_STATUS_HISTORY.into(),
5873                Value::Array(vec![serde_json::json!({
5874                    "status": essence_status::OPEN,
5875                    "at": at,
5876                })]),
5877            );
5878        }
5879        Ok(m)
5880    }
5881
5882    /// Axis-neutral unit vector (every component `1/sqrt(dims)`), used by
5883    /// essence records without an embedder and by thread-link markers.
5884    fn sentinel_vector(&self) -> Vec<f32> {
5885        let dims = self.dimensions.max(1);
5886        vec![1.0 / (dims as f32).sqrt(); dims]
5887    }
5888
5889    fn load_essence_record(&mut self, id: &Ulid) -> Result<Memory> {
5890        let m = self.get(id)?;
5891        if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str()) != Some(MNEMO_META_KIND_ESSENCE)
5892        {
5893            return Err(MnemoError::Invalid(format!(
5894                "essence: record {id} is not an essence record (missing `_mnemo.kind = essence`)"
5895            )));
5896        }
5897        Ok(m)
5898    }
5899
5900    fn section_of(&self, m: &Memory) -> Result<EssenceSection> {
5901        let s = m
5902            .metadata
5903            .get(MNEMO_META_SECTION)
5904            .and_then(|v| v.as_str())
5905            .ok_or_else(|| {
5906                MnemoError::Invalid("essence: record missing `_mnemo.section`".into())
5907            })?;
5908        EssenceSection::parse(s)
5909            .ok_or_else(|| MnemoError::Invalid(format!("essence: unknown section '{s}' on record")))
5910    }
5911
5912    fn essence_transition_status(&mut self, id: &Ulid, to: &str) -> Result<WriteOutcome> {
5913        if self.read_only {
5914            return Err(MnemoError::ReadOnly);
5915        }
5916        let existing = self.load_essence_record(id)?;
5917        let section = self.section_of(&existing)?;
5918        if !section.has_status_lifecycle() {
5919            return Err(MnemoError::Invalid(format!(
5920                "essence_transition: '{}' section has no status lifecycle",
5921                section.as_str()
5922            )));
5923        }
5924        let from = existing
5925            .metadata
5926            .get(MNEMO_META_STATUS)
5927            .and_then(|v| v.as_str())
5928            .unwrap_or(essence_status::OPEN);
5929        if from == to {
5930            return Ok(WriteOutcome::Unchanged);
5931        }
5932        let allowed = ESSENCE_TRANSITIONS
5933            .iter()
5934            .any(|(f, t)| *f == from && *t == to);
5935        if !allowed {
5936            return Err(MnemoError::Invalid(format!(
5937                "essence_transition: '{from}' → '{to}' is not a valid transition for essence"
5938            )));
5939        }
5940        // Mutate metadata in place and `put` back under the same id
5941        // so `_mnemo.updated_at` gets stamped and access stats are
5942        // preserved.
5943        let mut new = existing.clone();
5944        new.metadata
5945            .insert(MNEMO_META_STATUS.into(), Value::String(to.to_string()));
5946        let now = memory::now_secs();
5947        let history_entry = serde_json::json!({ "status": to, "at": now });
5948        let history = new
5949            .metadata
5950            .entry(MNEMO_META_STATUS_HISTORY.to_string())
5951            .or_insert_with(|| Value::Array(Vec::new()));
5952        if let Value::Array(arr) = history {
5953            arr.push(history_entry);
5954        }
5955        new.metadata
5956            .insert(MNEMO_META_UPDATED_AT.into(), Value::Number(now.into()));
5957        new.id = existing.id;
5958        new.created_at = existing.created_at;
5959        self.put(new)?;
5960        Ok(WriteOutcome::Updated)
5961    }
5962
5963    // --- Phase 12.7c: continuation + commit_deltas ------------------
5964
5965    /// Start a new thread that continues from a previous one
5966    /// (Phase 12.7c). Writes a thread-link marker record with
5967    /// `_mnemo.kind = "thread_link"` + `_mnemo.thread = <new_id>` +
5968    /// `_mnemo.continues = <from_id>`. The marker holds no user
5969    /// content; its purpose is to let `essence(Thread(new_id))`
5970    /// discover the continuation chain.
5971    ///
5972    /// The render for the new thread pulls the previous thread's
5973    /// open section items, its `next`, and its last
5974    /// [`CONTINUATION_TIMELINE_ENTRIES`] timeline entries ahead of
5975    /// the new thread's own essence — the "new thread, same
5976    /// experience" path.
5977    pub fn thread_start(&mut self, new_thread_id: &str, from_thread_id: &str) -> Result<Ulid> {
5978        if self.read_only {
5979            return Err(MnemoError::ReadOnly);
5980        }
5981        if new_thread_id.is_empty() || from_thread_id.is_empty() {
5982            return Err(MnemoError::Invalid(
5983                "thread_start: 'new_thread_id' and 'from_thread_id' must not be empty".into(),
5984            ));
5985        }
5986        if new_thread_id == from_thread_id {
5987            return Err(MnemoError::Invalid(
5988                "thread_start: 'new_thread_id' must differ from 'from_thread_id'".into(),
5989            ));
5990        }
5991        // Store a tiny marker record so the continuation chain is
5992        // queryable from a fresh handle. The sentinel vector passes
5993        // `put`'s zero-vector guard; default recall skips link records.
5994        let v = self.sentinel_vector();
5995        let content = format!("thread link: {new_thread_id} continues from {from_thread_id}");
5996        let mut m = Memory::new(content, MemoryType::Semantic, v).with_importance(0.1);
5997        m.metadata.insert(
5998            MNEMO_META_KIND.into(),
5999            Value::String(MNEMO_META_KIND_THREAD_LINK.into()),
6000        );
6001        m.metadata.insert(
6002            MNEMO_META_THREAD.into(),
6003            Value::String(new_thread_id.to_string()),
6004        );
6005        m.metadata.insert(
6006            MNEMO_META_CONTINUES.into(),
6007            Value::String(from_thread_id.to_string()),
6008        );
6009        let id = self.put(m)?;
6010        Ok(id)
6011    }
6012
6013    /// Phase 12.7c helper: render the predecessor thread's open
6014    /// items + `next` + last-N timeline entries as a "continued from"
6015    /// prelude. Used by [`Mnemo::essence`] to carry forward the
6016    /// prior thread's live state into a fresh thread.
6017    fn render_continuation_prelude(
6018        &mut self,
6019        prev_thread: &str,
6020        max_bytes: usize,
6021    ) -> Result<String> {
6022        if max_bytes == 0 {
6023            return Ok(String::new());
6024        }
6025        let entries = self.essence_entries()?;
6026        let mut open_items: Vec<Memory> = Vec::new();
6027        let mut next_items: Vec<Memory> = Vec::new();
6028        let mut timeline_items: Vec<Memory> = Vec::new();
6029        for e in &entries {
6030            let m = self.read_memory(e)?;
6031            if m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
6032                != Some(MNEMO_META_KIND_ESSENCE)
6033            {
6034                continue;
6035            }
6036            if m.metadata.get(MNEMO_META_THREAD).and_then(|v| v.as_str()) != Some(prev_thread) {
6037                continue;
6038            }
6039            let section = match m
6040                .metadata
6041                .get(MNEMO_META_SECTION)
6042                .and_then(|v| v.as_str())
6043                .and_then(EssenceSection::parse)
6044            {
6045                Some(s) => s,
6046                None => continue,
6047            };
6048            let status = m
6049                .metadata
6050                .get(MNEMO_META_STATUS)
6051                .and_then(|v| v.as_str())
6052                .unwrap_or(essence_status::OPEN);
6053            match section {
6054                EssenceSection::Open if status == essence_status::OPEN => {
6055                    open_items.push(m);
6056                }
6057                EssenceSection::Next if status == essence_status::OPEN => {
6058                    next_items.push(m);
6059                }
6060                EssenceSection::Timeline => timeline_items.push(m),
6061                _ => {}
6062            }
6063        }
6064        // Newest-first timeline, cap to last N.
6065        timeline_items.sort_by_key(|m| std::cmp::Reverse(m.created_at));
6066        timeline_items.truncate(CONTINUATION_TIMELINE_ENTRIES);
6067        // Oldest-first for stable chronological read in the render.
6068        timeline_items.reverse();
6069
6070        let mut out = String::new();
6071        let header = format!("## continued from `{prev_thread}`\n");
6072        if header.len() > max_bytes {
6073            return Ok(String::new());
6074        }
6075        out.push_str(&header);
6076        if !open_items.is_empty() {
6077            let sub = "### open\n";
6078            if out.len() + sub.len() <= max_bytes {
6079                out.push_str(sub);
6080            }
6081            for m in &open_items {
6082                let line = format!("- {}\n", m.content);
6083                if out.len() + line.len() > max_bytes {
6084                    break;
6085                }
6086                out.push_str(&line);
6087            }
6088        }
6089        if !next_items.is_empty() {
6090            let sub = "### next\n";
6091            if out.len() + sub.len() <= max_bytes {
6092                out.push_str(sub);
6093            }
6094            for m in &next_items {
6095                let line = format!("- {}\n", m.content);
6096                if out.len() + line.len() > max_bytes {
6097                    break;
6098                }
6099                out.push_str(&line);
6100            }
6101        }
6102        if !timeline_items.is_empty() {
6103            let sub = "### recent milestones\n";
6104            if out.len() + sub.len() <= max_bytes {
6105                out.push_str(sub);
6106            }
6107            for m in &timeline_items {
6108                let line = format!("- {}\n", m.content);
6109                if out.len() + line.len() > max_bytes {
6110                    break;
6111                }
6112                out.push_str(&line);
6113            }
6114        }
6115        if out.len() < max_bytes {
6116            out.push('\n');
6117        }
6118        Ok(out)
6119    }
6120
6121    /// Phase 12.7c helper: find the predecessor thread id for
6122    /// `thread_id` by reading its `_mnemo.kind = "thread_link"`
6123    /// marker. Returns `None` if the thread has no predecessor
6124    /// (fresh thread, no `thread_start` call).
6125    fn predecessor_thread(&mut self, thread_id: &str) -> Result<Option<String>> {
6126        self.ensure_key_caches()?;
6127        Ok(self
6128            .layers
6129            .as_ref()
6130            .and_then(|l| l.links.get(thread_id))
6131            .map(|(from, _)| from.clone()))
6132    }
6133
6134    /// Atomic batch write combining 12.1 facts, 12.7b essence ops,
6135    /// and 12.7a archive turns in one flush (Phase 12.7c; batch
6136    /// embedding added in 12.7d).
6137    ///
6138    /// The batch runs in four steps:
6139    ///
6140    /// 1. **Structural checks** on every item: required fields, size
6141    ///    caps, empty keys, `thread_id` for turns, and that every
6142    ///    `Update` / `Resolve` / `Remove` targets a live essence record
6143    ///    the op can apply to.
6144    /// 2. **One `Embedder::embed` call** for every text that needs a
6145    ///    vector: facts whose `vector` is empty, essence `Add` /
6146    ///    `AppendTimeline` texts (when an embedder is registered;
6147    ///    without one, essence keeps its sentinel vector), and the
6148    ///    archive turns' chunks.
6149    /// 3. **Vector checks** (width, non-zero, finite) on supplied and
6150    ///    embedded vectors alike.
6151    /// 4. **Writes**: facts, then essence ops, then turns per
6152    ///    `turn_policy`, then exactly one `flush()`.
6153    ///
6154    /// A failure in steps 1–3 returns an error with nothing staged. A
6155    /// text-only fact, or any turn to archive, with no embedder
6156    /// registered is a step-1 error. `put` has no rollback, so an I/O
6157    /// error during step 4 can leave the batch's earlier writes staged
6158    /// on the handle; steps 1–3 exist so nothing else can fail that
6159    /// late.
6160    pub fn commit_deltas(&mut self, batch: CommitBatch) -> Result<CommitReport> {
6161        if self.read_only {
6162            return Err(MnemoError::ReadOnly);
6163        }
6164        let CommitBatch {
6165            mut facts,
6166            essence_ops,
6167            turns,
6168            thread_id,
6169            turn_policy,
6170        } = batch;
6171        let has_embedder = self.embedder.is_some();
6172        let in_ctx = |ctx: &str, e: MnemoError| MnemoError::Invalid(format!("{ctx}: {e}"));
6173
6174        // --- 1. Structural checks (reads only) ----------------------
6175        for (i, f) in facts.iter().enumerate() {
6176            if let Some(k) = &f.key {
6177                if k.is_empty() {
6178                    return Err(MnemoError::Invalid(format!(
6179                        "commit_deltas: facts[{i}] key is empty"
6180                    )));
6181                }
6182            }
6183            if f.memory.vector.is_empty() {
6184                if f.memory.content.is_empty() {
6185                    return Err(MnemoError::Invalid(format!(
6186                        "commit_deltas: facts[{i}] has neither a vector nor content to embed"
6187                    )));
6188                }
6189                if !has_embedder {
6190                    return Err(MnemoError::Invalid(format!(
6191                        "commit_deltas: facts[{i}] has no vector and no embedder is registered \
6192                         on this handle; pass a vector or call set_embedder first"
6193                    )));
6194                }
6195            }
6196        }
6197        let mut removed_in_batch: std::collections::HashSet<u128> =
6198            std::collections::HashSet::new();
6199        for (i, op) in essence_ops.iter().enumerate() {
6200            let ctx = format!("commit_deltas: essence_ops[{i}]");
6201            match op.kind {
6202                EssenceOpKind::Add => {
6203                    if op.scope.is_none() {
6204                        return Err(MnemoError::Invalid(format!("{ctx} Add missing 'scope'")));
6205                    }
6206                    let section = op.section.ok_or_else(|| {
6207                        MnemoError::Invalid(format!("{ctx} Add missing 'section'"))
6208                    })?;
6209                    let text = op
6210                        .text
6211                        .as_deref()
6212                        .ok_or_else(|| MnemoError::Invalid(format!("{ctx} Add missing 'text'")))?;
6213                    Self::check_essence_add(section, text).map_err(|e| in_ctx(&ctx, e))?;
6214                }
6215                EssenceOpKind::AppendTimeline => {
6216                    if op.scope.is_none() {
6217                        return Err(MnemoError::Invalid(format!(
6218                            "{ctx} AppendTimeline missing 'scope'"
6219                        )));
6220                    }
6221                    let text = op.text.as_deref().ok_or_else(|| {
6222                        MnemoError::Invalid(format!("{ctx} AppendTimeline missing 'text'"))
6223                    })?;
6224                    Self::check_essence_append(text).map_err(|e| in_ctx(&ctx, e))?;
6225                }
6226                EssenceOpKind::Update | EssenceOpKind::Resolve | EssenceOpKind::Remove => {
6227                    let id = op
6228                        .id
6229                        .ok_or_else(|| MnemoError::Invalid(format!("{ctx} needs 'id'")))?;
6230                    if removed_in_batch.contains(&id.0) {
6231                        return Err(MnemoError::Invalid(format!(
6232                            "{ctx}: record {id} is removed earlier in this batch"
6233                        )));
6234                    }
6235                    let existing = self.load_essence_record(&id).map_err(|e| in_ctx(&ctx, e))?;
6236                    let section = self.section_of(&existing).map_err(|e| in_ctx(&ctx, e))?;
6237                    match op.kind {
6238                        EssenceOpKind::Update => {
6239                            let text = op.text.as_deref().ok_or_else(|| {
6240                                MnemoError::Invalid(format!("{ctx} Update needs 'text'"))
6241                            })?;
6242                            if section == EssenceSection::Timeline {
6243                                return Err(MnemoError::Invalid(format!(
6244                                    "{ctx}: timeline entries are append-only"
6245                                )));
6246                            }
6247                            if text.is_empty() || text.len() > section.item_max_bytes() {
6248                                return Err(MnemoError::Invalid(format!(
6249                                    "{ctx}: '{}' items must be 1..={} bytes; got {}",
6250                                    section.as_str(),
6251                                    section.item_max_bytes(),
6252                                    text.len()
6253                                )));
6254                            }
6255                        }
6256                        EssenceOpKind::Resolve => {
6257                            if !section.has_status_lifecycle() {
6258                                return Err(MnemoError::Invalid(format!(
6259                                    "{ctx}: '{}' section has no status lifecycle",
6260                                    section.as_str()
6261                                )));
6262                            }
6263                        }
6264                        _ => {
6265                            removed_in_batch.insert(id.0);
6266                        }
6267                    }
6268                }
6269            }
6270        }
6271        if !turns.is_empty() && thread_id.is_none() {
6272            return Err(MnemoError::Invalid(
6273                "commit_deltas: 'turns' non-empty but 'thread_id' missing".into(),
6274            ));
6275        }
6276        let archive_turns = !turns.is_empty() && !matches!(turn_policy, TurnPolicy::Drop);
6277        if archive_turns && !has_embedder {
6278            return Err(MnemoError::Invalid(
6279                "commit_deltas: archiving 'turns' needs an embedder registered on this handle; \
6280                 call set_embedder first or use turn_policy Drop"
6281                    .into(),
6282            ));
6283        }
6284        let plan = match (&thread_id, archive_turns) {
6285            (Some(t), true) => Some(self.plan_ingest(t, &turns)?),
6286            _ => None,
6287        };
6288
6289        // --- 2. One batched embed -----------------------------------
6290        let embeds_essence = |op: &EssenceOp| {
6291            has_embedder && matches!(op.kind, EssenceOpKind::Add | EssenceOpKind::AppendTimeline)
6292        };
6293        let (vectors, n_texts) = {
6294            let mut texts: Vec<&str> = Vec::new();
6295            for f in &facts {
6296                if f.memory.vector.is_empty() {
6297                    texts.push(f.memory.content.as_str());
6298                }
6299            }
6300            for op in essence_ops.iter().filter(|op| embeds_essence(op)) {
6301                texts.push(op.text.as_deref().expect("checked in step 1"));
6302            }
6303            if let Some(p) = &plan {
6304                texts.extend(p.texts());
6305            }
6306            let vectors = match (texts.is_empty(), self.embedder.as_ref()) {
6307                (true, _) => Vec::new(),
6308                (false, Some(e)) => e.embed(&texts)?,
6309                (false, None) => unreachable!("texts need an embedder; checked in step 1"),
6310            };
6311            (vectors, texts.len())
6312        };
6313
6314        // --- 3. Vector checks ---------------------------------------
6315        self.check_vectors("commit_deltas", &vectors, n_texts)?;
6316        for (i, f) in facts.iter().enumerate() {
6317            let v = &f.memory.vector;
6318            if v.is_empty() {
6319                continue;
6320            }
6321            if v.len() != self.dimensions {
6322                return Err(MnemoError::Invalid(format!(
6323                    "commit_deltas: facts[{i}] vector has {} dims; this file is {}",
6324                    v.len(),
6325                    self.dimensions
6326                )));
6327            }
6328            let ctx = format!("commit_deltas: facts[{i}]");
6329            self.check_vector(&ctx, v)?;
6330        }
6331
6332        // Hand the embedded vectors back out, in the order they were
6333        // requested: facts, then essence ops, then turn chunks.
6334        let mut vectors = vectors.into_iter();
6335        for f in facts.iter_mut() {
6336            if f.memory.vector.is_empty() {
6337                f.memory.vector = vectors.next().expect("one vector per text");
6338            }
6339        }
6340        let essence_vectors: Vec<Option<Vec<f32>>> = essence_ops
6341            .iter()
6342            .map(|op| embeds_essence(op).then(|| vectors.next().expect("one vector per text")))
6343            .collect();
6344        let turn_vectors: Vec<Vec<f32>> = vectors.collect();
6345
6346        // --- 4. Writes ----------------------------------------------
6347        let mut report = CommitReport::default();
6348
6349        for f in facts {
6350            match f.key {
6351                Some(k) => {
6352                    let (_id, outcome) = self.remember_keyed(f.memory, &k)?;
6353                    match outcome {
6354                        WriteOutcome::Added => report.facts_added += 1,
6355                        WriteOutcome::Superseded | WriteOutcome::Updated => {
6356                            report.facts_superseded += 1
6357                        }
6358                        WriteOutcome::Unchanged => report.facts_unchanged += 1,
6359                    }
6360                }
6361                None => {
6362                    self.remember(f.memory)?;
6363                    report.facts_added += 1;
6364                }
6365            }
6366        }
6367
6368        for (op, vector) in essence_ops.into_iter().zip(essence_vectors) {
6369            let outcome = match op.kind {
6370                EssenceOpKind::Add => {
6371                    self.essence_add_with(
6372                        op.scope.expect("checked in step 1"),
6373                        op.section.expect("checked in step 1"),
6374                        op.text.as_deref().expect("checked in step 1"),
6375                        op.refs,
6376                        vector,
6377                    )?;
6378                    WriteOutcome::Added
6379                }
6380                EssenceOpKind::AppendTimeline => {
6381                    self.essence_append_with(
6382                        op.scope.expect("checked in step 1"),
6383                        op.text.as_deref().expect("checked in step 1"),
6384                        op.refs,
6385                        vector,
6386                    )?;
6387                    WriteOutcome::Added
6388                }
6389                EssenceOpKind::Update => self.essence_update(
6390                    op.id.as_ref().expect("checked in step 1"),
6391                    op.text.as_deref().expect("checked in step 1"),
6392                    op.refs,
6393                )?,
6394                EssenceOpKind::Resolve => {
6395                    self.essence_resolve(op.id.as_ref().expect("checked in step 1"))?
6396                }
6397                EssenceOpKind::Remove => {
6398                    self.essence_remove(op.id.as_ref().expect("checked in step 1"))?;
6399                    WriteOutcome::Updated
6400                }
6401            };
6402            match outcome {
6403                WriteOutcome::Unchanged => report.essence_ops_unchanged += 1,
6404                _ => report.essence_ops_applied += 1,
6405            }
6406        }
6407
6408        if let (Some(plan), Some(t)) = (plan, thread_id.as_deref()) {
6409            let ttl = match turn_policy {
6410                TurnPolicy::ExpireAfter(secs) => Some(secs),
6411                _ => None,
6412            };
6413            let r = self.apply_ingest(t, plan, turn_vectors, ttl)?;
6414            report.turns_added += r.added_records;
6415            report.turns_skipped += r.skipped_existing_turns;
6416            report.turns_truncated += r.truncated_turns;
6417            report.turns_chunked += r.chunked_turns;
6418        }
6419
6420        self.flush()?;
6421        Ok(report)
6422    }
6423
6424    /// **Not a stable API.** The file's on-disk format version.
6425    #[doc(hidden)]
6426    pub fn __format_version(&self) -> u16 {
6427        self.header.version
6428    }
6429
6430    /// **Not a stable API.** FMT-9 (a): the committed free list as
6431    /// `(freed_at_txn, first_page, page_count)` runs, plus the number of
6432    /// runs pending for the next commit.
6433    #[doc(hidden)]
6434    pub fn __free_list(&self) -> (Vec<(u64, u64, u64)>, usize) {
6435        (
6436            self.free
6437                .to_runs()
6438                .into_iter()
6439                .map(|r| (r.freed_at, r.page, r.count))
6440                .collect(),
6441            self.pending_free.len(),
6442        )
6443    }
6444
6445    /// **Not a stable API.** FMT-9: where a memory's record lives:
6446    /// `(first_page, page_count, generation)`.
6447    #[doc(hidden)]
6448    pub fn __record_location(&self, id: &Ulid) -> Option<(u64, u32, u64)> {
6449        let idx = self.index.get(id.0, &self.catalog)?;
6450        let e = &self.catalog[idx];
6451        (!e.deleted).then_some((e.start_page, e.page_count, e.gen))
6452    }
6453
6454    /// **Not a stable API.** FMT-9e: how many thread-hash buckets of turn
6455    /// records the handle caches hold undecoded (`None` until built).
6456    #[doc(hidden)]
6457    pub fn __cold_thread_buckets(&self) -> Option<usize> {
6458        self.layers.as_ref().map(|l| l.cold_turns.len())
6459    }
6460
6461    /// **Not a stable API.** FMT-9e: a live memory's stored record length
6462    /// and whether the stored body is a zstd frame.
6463    #[doc(hidden)]
6464    pub fn __record_storage(&mut self, id: &Ulid) -> Result<(u32, bool)> {
6465        let idx = self
6466            .index
6467            .get(id.0, &self.catalog)
6468            .ok_or_else(|| MnemoError::NotFound(id.to_string()))?;
6469        let e = self.catalog[idx].clone();
6470        let bytes = self.read_record(&e)?;
6471        Ok((e.len, bytes.starts_with(&ZSTD_MAGIC)))
6472    }
6473
6474    /// **Not a stable API.** FMT-9b: whether the vector column is resident,
6475    /// and how many vector pages hold live entries.
6476    #[doc(hidden)]
6477    pub fn __vector_column(&self) -> (bool, usize) {
6478        (self.col.resident, self.col.page_rows.len())
6479    }
6480
6481    /// **Not a stable API.** FMT-9c.2: `(catalog rows incl. tombstones,
6482    /// delta runs on the chain, first page of the base run)`.
6483    #[doc(hidden)]
6484    pub fn __catalog_shape(&self) -> (usize, usize, u64) {
6485        (
6486            self.catalog.len(),
6487            self.delta_chain.len(),
6488            self.header.catalog_start,
6489        )
6490    }
6491
6492    /// **Not a stable API.** FMT-9c.3: `(index delta runs on the chain,
6493    /// first page of the base index run)`.
6494    #[doc(hidden)]
6495    pub fn __index_shape(&self) -> (usize, u64) {
6496        (self.index_chain.len(), self.header.index_start)
6497    }
6498
6499    /// **Not a stable API.** FMT-9d: BM25 keyword scores for `query`, best
6500    /// first: `(id, score)`, with k1 1.2 and b 0.75.
6501    #[doc(hidden)]
6502    pub fn __bm25(&mut self, query: &str) -> Result<Vec<(Ulid, f32)>> {
6503        let scores = self.bm25_scores(query, 1.2, 0.75)?;
6504        let mut out: Vec<(Ulid, f32)> = scores
6505            .into_iter()
6506            .map(|(row, s)| (Ulid(self.catalog[row].id), s))
6507            .collect();
6508        out.sort_by(|a, b| b.1.total_cmp(&a.1));
6509        Ok(out)
6510    }
6511
6512    /// **Not a stable API.** FMT-9d: write the next whole index with this
6513    /// layout byte (an unknown one exercises the "treat as no index" path).
6514    #[doc(hidden)]
6515    pub fn __set_index_layout_for_testing(&mut self, layout: u8) {
6516        self.index_layout_override = Some(layout);
6517    }
6518
6519    /// **Not a stable API.** FMT-9d: BM25 segments on disk.
6520    #[doc(hidden)]
6521    pub fn __bm25_segments(&self) -> usize {
6522        self.bm25.manifest.segments.len()
6523    }
6524
6525    /// **Not a stable API.** FMT-9: the header's next unallocated page.
6526    #[doc(hidden)]
6527    pub fn __next_page(&self) -> u64 {
6528        self.header.next_page
6529    }
6530
6531    /// **Not a stable API.** FMT-9 (b): add a placeholder key slot (its
6532    /// wrapped DEK is random bytes, so it unlocks nothing) to exercise the
6533    /// key-slot run until Phase 13.2 adds real slots. v9 only.
6534    #[doc(hidden)]
6535    pub fn __add_key_slot_for_testing(&mut self, label: &str) -> Result<()> {
6536        if self.read_only {
6537            return Err(MnemoError::ReadOnly);
6538        }
6539        if !self.is_v9() {
6540            return Err(MnemoError::Invalid("key slots need a v9 file".into()));
6541        }
6542        let mut wrapped = vec![0u8; WRAPPED_DEK_LEN];
6543        wrapped[..crypto::KEY_LEN].copy_from_slice(&crypto::random_dek()[..]);
6544        self.key_slots.slots.push(KeySlotEntry {
6545            label: label.to_string(),
6546            role: 0,
6547            m_cost: KdfParams::raw_key().m_cost,
6548            t_cost: KdfParams::raw_key().t_cost,
6549            p_cost: KdfParams::raw_key().p_cost,
6550            salt: crypto::random_salt(),
6551            dek_nonce: crypto::random_nonce(),
6552            wrapped_dek: wrapped,
6553        });
6554        self.dirty_keys = true;
6555        Ok(())
6556    }
6557
6558    /// **Not a stable API.** FMT-9 (b): the key-slot labels.
6559    #[doc(hidden)]
6560    pub fn __key_slot_labels(&self) -> Vec<String> {
6561        self.key_slots
6562            .slots
6563            .iter()
6564            .map(|s| s.label.clone())
6565            .collect()
6566    }
6567
6568    /// **Not a stable API.** FMT-9 (b): the key-slot run's pointer
6569    /// `(first_page, page_count, generation)`.
6570    #[doc(hidden)]
6571    pub fn __key_slot_run(&self) -> (u64, u64, u64) {
6572        (
6573            self.header.keys.start,
6574            self.header.keys.pages,
6575            self.header.keys.gen,
6576        )
6577    }
6578
6579    /// FMT-9a measurement hook: the ANN shortlist (candidate ids, nearest
6580    /// first by PQ distance) that recall would rerank, at the given or the
6581    /// index's effective `n_probe` / `n_rerank`. Empty without an index.
6582    /// **Unstable diagnostic.**
6583    #[doc(hidden)]
6584    pub fn __ann_shortlist(
6585        &mut self,
6586        query: &[f32],
6587        n_probe: Option<usize>,
6588        n_rerank: Option<usize>,
6589    ) -> Result<Vec<Ulid>> {
6590        self.ensure_ann()?;
6591        Ok(match &self.ann {
6592            Some(ann) => ann
6593                .query(query, n_probe, n_rerank)
6594                .into_iter()
6595                .map(Ulid)
6596                .collect(),
6597            None => Vec::new(),
6598        })
6599    }
6600
6601    /// P-0 device budget breakdown of this file and handle. **Unstable
6602    /// diagnostic**; see [`BudgetReport`]. Reads the header, the catalog
6603    /// and file metadata only; nothing is decrypted.
6604    #[doc(hidden)]
6605    pub fn __budget_report(&self) -> Result<BudgetReport> {
6606        let h = &self.header;
6607        let ps = h.page_size as u64;
6608        let mut r = BudgetReport {
6609            file_bytes: std::fs::metadata(&self.path)?.len(),
6610            page_size: ps,
6611            header_bytes: ps,
6612            catalog_run_bytes: h.catalog_pages * ps,
6613            catalog_payload_bytes: h.catalog_len,
6614            index_run_bytes: h.index_pages * ps,
6615            index_payload_bytes: h.index_len,
6616            index_start_page: h.index_start,
6617            wal_bytes: h.wal_pages * ps,
6618            snapshot_manifest_bytes: h.manifest_pages * ps,
6619            cache_dir_run_bytes: h.cache_pages * ps,
6620            ..Default::default()
6621        };
6622        for e in &self.catalog {
6623            // FMT-9b.2: packed v9 records share pages (counted once below);
6624            // a v9 delete frees its pages, so there are no deleted ones.
6625            let packed = e.v9.is_some_and(|r| r.rec_slot != REC_SPAN);
6626            let pages = if packed { 0 } else { e.page_count as u64 * ps };
6627            if e.deleted {
6628                if e.v9.is_none() {
6629                    r.deleted_record_page_bytes += pages;
6630                }
6631            } else {
6632                r.record_page_bytes += pages;
6633                r.record_payload_bytes += e.len as u64;
6634                r.live_records += 1;
6635            }
6636        }
6637        r.record_page_bytes += self.recs.page_rows.len() as u64 * ps;
6638        // FMT-9b: the vector column, reported with the records.
6639        r.record_page_bytes += self.col.page_rows.len() as u64 * ps;
6640        let accounted = r.header_bytes
6641            + r.record_page_bytes
6642            + r.deleted_record_page_bytes
6643            + r.catalog_run_bytes
6644            + r.index_run_bytes
6645            + r.wal_bytes
6646            + r.snapshot_manifest_bytes
6647            + r.cache_dir_run_bytes;
6648        r.unreferenced_bytes = r.file_bytes.saturating_sub(accounted);
6649
6650        r.ram_catalog = (self.catalog.capacity() * std::mem::size_of::<CatalogEntry>()) as u64;
6651        r.ram_id_map = self.index.heap_bytes() as u64;
6652        if let Some(ann) = &self.ann {
6653            let (c, cb, ids, codes, assign) = ann.heap_bytes();
6654            r.ram_ann_centroids = c as u64;
6655            r.ram_ann_codebook = cb as u64;
6656            r.ram_ann_posting_ids = ids as u64;
6657            r.ram_ann_posting_codes = codes as u64;
6658            r.ram_ann_assignment = assign as u64;
6659        }
6660        r.ram_cache_dir =
6661            (self.cache_dir.capacity() * std::mem::size_of::<CacheDirectoryEntry>()) as u64;
6662        let key_index = self.key_index.as_ref().map_or(0, |k| {
6663            crate::index::hash_map_bytes::<KeySlot, u128>(k.capacity())
6664                + k.keys()
6665                    .map(|s| s.key.capacity() + s.scope_owner.as_ref().map_or(0, |o| o.capacity()))
6666                    .sum::<usize>()
6667        });
6668        let superseded = self.superseded.as_ref().map_or(0, |s| {
6669            crate::index::hash_map_bytes::<u128, ()>(s.capacity())
6670        });
6671        r.ram_key_caches = (key_index + superseded) as u64;
6672        r.ram_vector_column = (self.col.data.capacity() * 2
6673            + self.col.norms.capacity() * 4
6674            + self.col.have.capacity()) as u64;
6675        r.ram_total = r.ram_catalog
6676            + r.ram_id_map
6677            + r.ram_ann_centroids
6678            + r.ram_ann_codebook
6679            + r.ram_ann_posting_ids
6680            + r.ram_ann_posting_codes
6681            + r.ram_ann_assignment
6682            + r.ram_cache_dir
6683            + r.ram_key_caches
6684            + r.ram_vector_column;
6685        Ok(r)
6686    }
6687
6688    /// Return every live memory (used by tooling and compaction).
6689    pub fn memories(&mut self) -> Result<Vec<Memory>> {
6690        // FMT-9b: one pass over the vector pages instead of one per record.
6691        self.ensure_column()?;
6692        let entries: Vec<CatalogEntry> = self
6693            .catalog
6694            .iter()
6695            .filter(|e| !e.deleted)
6696            .cloned()
6697            .collect();
6698        let mut out = Vec::with_capacity(entries.len());
6699        for e in &entries {
6700            out.push(self.read_memory(e)?);
6701        }
6702        Ok(out)
6703    }
6704
6705    /// Return the database's **self-describing onboarding memories** — the
6706    /// ones tagged `metadata.area = "onboarding"`. This is the engine-level
6707    /// surface for the *single-file philosophy*: an agent who receives a
6708    /// `.mnemo` file (and its passphrase) can call this to learn what the
6709    /// file is, which embedder it expects, the recommended agent id, and any
6710    /// other conventions the file's author chose to record — all without
6711    /// needing any external documentation.
6712    ///
6713    /// Ordering: the canonical manifest (tag `metadata.topic = "manifest"`)
6714    /// always comes first; everything else follows in `importance` descending,
6715    /// then `created_at` ascending for deterministic results.
6716    pub fn about(&mut self) -> Result<Vec<Memory>> {
6717        // Phase 12.1: drop records that have been superseded by a newer
6718        // keyed write. They still live in snapshot history and
6719        // `history(key)` — just not here.
6720        // R-004.2: read only the onboarding records, from the handle caches.
6721        self.ensure_key_caches()?;
6722        let ids: Vec<u128> = self
6723            .layers
6724            .as_ref()
6725            .map(|l| l.onboarding.iter().copied().collect())
6726            .unwrap_or_default();
6727        let mut out: Vec<Memory> = Vec::new();
6728        for e in self.live_entries(ids) {
6729            let m = self.read_memory(&e)?;
6730            if !m.metadata.contains_key(MNEMO_META_SUPERSEDED_BY) && is_onboarding(&m) {
6731                out.push(m);
6732            }
6733        }
6734        out.sort_by(|a, b| {
6735            let a_manifest = is_manifest(a);
6736            let b_manifest = is_manifest(b);
6737            // Manifest topic wins first; then importance desc; then created_at asc.
6738            b_manifest
6739                .cmp(&a_manifest)
6740                .then_with(|| b.importance.total_cmp(&a.importance))
6741                .then_with(|| a.created_at.cmp(&b.created_at))
6742        });
6743        Ok(out)
6744    }
6745
6746    /// The project essence for the `about` briefing (Phase 12.7d): the
6747    /// [`EssenceScope::Project`] render at the default budget
6748    /// ([`ESSENCE_DEFAULT_RENDER_BYTES`]), or `None` when the file has no
6749    /// open project-scope item or timeline entry. The CLI and MCP `about`
6750    /// append it after the manifest and onboarding memories; [`Mnemo::about`]
6751    /// itself keeps returning records.
6752    pub fn about_essence(&mut self) -> Result<Option<String>> {
6753        let entries = self.essence_entries()?;
6754        let mut any = false;
6755        for e in &entries {
6756            let m = self.read_memory(e)?;
6757            let is_project_essence = m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str())
6758                == Some(MNEMO_META_KIND_ESSENCE)
6759                && !m.metadata.contains_key(MNEMO_META_THREAD);
6760            if is_project_essence && status_of(&m) != Some(essence_status::RESOLVED) {
6761                any = true;
6762                break;
6763            }
6764        }
6765        if !any {
6766            return Ok(None);
6767        }
6768        self.essence(EssenceScope::Project, ESSENCE_DEFAULT_RENDER_BYTES)
6769            .map(Some)
6770    }
6771
6772    // --- retrieval --------------------------------------------------------
6773
6774    /// Phase-1 brute-force search: rank live memories by raw similarity only.
6775    /// Read-only — does not update access statistics.
6776    pub fn search(
6777        &mut self,
6778        query: &[f32],
6779        top_k: usize,
6780        metric: Metric,
6781    ) -> Result<Vec<(Memory, f32)>> {
6782        if query.len() != self.dimensions {
6783            return Err(MnemoError::DimensionMismatch {
6784                expected: self.dimensions,
6785                got: query.len(),
6786            });
6787        }
6788        let now = memory::now_secs();
6789        let entries: Vec<CatalogEntry> = self
6790            .catalog
6791            .iter()
6792            .filter(|e| !e.deleted)
6793            .cloned()
6794            .collect();
6795
6796        let mut scored: Vec<(Memory, f32)> = Vec::new();
6797        for e in &entries {
6798            let m = self.read_memory(e)?;
6799            if m.is_expired(now) {
6800                continue;
6801            }
6802            let sim = memory::similarity(metric, query, &m.vector);
6803            scored.push((m, sim));
6804        }
6805        scored.sort_by(|a, b| b.1.total_cmp(&a.1));
6806        scored.truncate(top_k);
6807        Ok(scored)
6808    }
6809
6810    /// Phase-5 multi-signal recall: rank by
6811    /// `α·similarity + β·recency + γ·importance + δ·ln(freq)`, with type and
6812    /// agent filtering. Updates `accessed_at` / `access_count` on the
6813    /// returned memories (persisted on the next `flush`).
6814    ///
6815    /// When an ANN index has been built ([`Mnemo::build_index`]), recall runs
6816    /// the tiered IVF→PQ→rerank pipeline: it scores only the similarity-nearest
6817    /// `n_rerank` candidates rather than every memory. Without an index it
6818    /// scores every live memory exactly.
6819    /// One pass of filter + score for [`Mnemo::recall`]. Factored out of
6820    /// `recall` so the outer path can retry with a wider shortlist when
6821    /// the type/agent/supersession/TTL filters starve the ANN-narrowed
6822    /// result below `top_k` (Phase 12.1 short-results-on-indexed-path
6823    /// fix).
6824    ///
6825    /// * `force_exact = false` → use the ANN index when one exists; fall
6826    ///   back to a full scan only if none is built.
6827    /// * `force_exact = true` → skip the ANN index and scan every live
6828    ///   non-superseded record. Used by the outer path as the last-resort
6829    ///   attempt when widening the shortlist would already touch half or
6830    ///   more of the live non-superseded set.
6831    ///
6832    /// Returns an unsorted, un-truncated `Vec<RecallResult>`; the outer
6833    /// `recall` sorts and truncates once.
6834    fn score_once(
6835        &mut self,
6836        req: &RecallRequest,
6837        now: i64,
6838        force_exact: bool,
6839    ) -> Result<Vec<RecallResult>> {
6840        // Candidate set: ANN-narrowed when an index exists and the
6841        // caller hasn't forced an exact scan, else everything live.
6842        // R-004.2: a filter pinning `_mnemo.thread` scans just that
6843        // thread's records (from the handle caches); one that can only
6844        // match non-fact kinds skips the ANN, which doesn't hold them.
6845        // FMT-9d: keyword matches, normalised by the best one.
6846        self.kw = None;
6847        if let Some(text) = &req.keywords {
6848            let scores = self.bm25_scores(text, req.bm25_k1, req.bm25_b)?;
6849            let max = scores.values().copied().fold(0.0f32, f32::max);
6850            if max > 0.0 {
6851                self.kw = Some(
6852                    scores
6853                        .into_iter()
6854                        .map(|(row, s)| (self.catalog[row].id, s / max))
6855                        .collect(),
6856                );
6857            } else {
6858                self.kw = Some(std::collections::HashMap::new());
6859            }
6860        }
6861        if let Some(thread) = filter_thread_pin(&req.filter) {
6862            let thread = thread.to_string();
6863            let entries = self.thread_entries(&thread)?;
6864            return self.score_entries(req, now, &entries);
6865        }
6866        // Keyword-only: the candidates are the matches.
6867        if req.query.is_empty() {
6868            let ids: Vec<u128> = self
6869                .kw
6870                .as_ref()
6871                .map_or(Vec::new(), |k| k.keys().copied().collect());
6872            let entries: Vec<CatalogEntry> = ids
6873                .into_iter()
6874                .filter_map(|id| self.index.get(id, &self.catalog))
6875                .map(|i| self.catalog[i].clone())
6876                .filter(|e| !e.deleted)
6877                .collect();
6878            return self.score_entries(req, now, &entries);
6879        }
6880        let force_exact = force_exact || filter_bypasses_ann(&req.filter);
6881        let mut entries: Vec<CatalogEntry> = match (&self.ann, force_exact) {
6882            (Some(ann), false) => {
6883                let n_rerank = self.rerank_for(ann, req);
6884                let ids = ann.query(&req.query, req.n_probe, n_rerank);
6885                ids.iter()
6886                    .filter_map(|&id| self.index.get(id, &self.catalog))
6887                    .map(|i| self.catalog[i].clone())
6888                    .filter(|e| !e.deleted)
6889                    .collect()
6890            }
6891            _ => self
6892                .catalog
6893                .iter()
6894                .filter(|e| !e.deleted)
6895                .cloned()
6896                .collect(),
6897        };
6898        // FMT-9d: an ANN shortlist plus the keyword matches it missed.
6899        if let Some(kw) = &self.kw {
6900            if entries.len() < self.catalog.len() {
6901                let have: std::collections::HashSet<u128> = entries.iter().map(|e| e.id).collect();
6902                let extra: Vec<u128> = kw.keys().copied().filter(|id| !have.contains(id)).collect();
6903                for id in extra {
6904                    if let Some(i) = self.index.get(id, &self.catalog) {
6905                        if !self.catalog[i].deleted {
6906                            entries.push(self.catalog[i].clone());
6907                        }
6908                    }
6909                }
6910            }
6911        }
6912
6913        self.score_entries(req, now, &entries)
6914    }
6915
6916    /// FMT-9d: a candidate's relevance: its vector similarity plus ε times
6917    /// its normalised keyword score (keyword-only requests: just the
6918    /// keyword score). Returns `(relevance, reported similarity)`.
6919    fn relevance(&self, req: &RecallRequest, id: u128, sim: f32) -> (f32, f32) {
6920        let kw = self
6921            .kw
6922            .as_ref()
6923            .and_then(|k| k.get(&id).copied())
6924            .unwrap_or(0.0);
6925        if req.query.is_empty() {
6926            (kw, kw)
6927        } else {
6928            (sim + req.keyword_weight * kw, sim)
6929        }
6930    }
6931
6932    /// Score one candidate set for [`Mnemo::score_once`].
6933    fn score_entries(
6934        &mut self,
6935        req: &RecallRequest,
6936        now: i64,
6937        entries: &[CatalogEntry],
6938    ) -> Result<Vec<RecallResult>> {
6939        // FMT-9b: score v9 candidates from their rows and the fp16 column,
6940        // decrypting only what makes `top_k`. A metadata filter needs the
6941        // record body, so it takes the general path below.
6942        if self.is_v9() && req.filter.is_empty() {
6943            return self.score_rows(req, now, entries);
6944        }
6945        let mut scored: Vec<RecallResult> = Vec::with_capacity(entries.len());
6946        for e in entries {
6947            // Phase 12.1: drop superseded records.
6948            //
6949            // Fast path: if the lazy `superseded` set has already been
6950            // built by a prior keyed op on this handle, check it before
6951            // decrypt — O(1), no AES work. On a fresh read-only handle
6952            // it is typically `None`, so we fall through to the slow
6953            // path (one `_mnemo.superseded_by` lookup per surviving
6954            // record, O(k) over user metadata), and never trigger the
6955            // full-file scan that `ensure_key_caches()` would do.
6956            if self.superseded.as_ref().is_some_and(|s| s.contains(&e.id)) {
6957                continue;
6958            }
6959            let m = self.read_memory(e)?;
6960            if m.metadata.contains_key(MNEMO_META_SUPERSEDED_BY) {
6961                continue;
6962            }
6963            if m.is_expired(now) {
6964                continue;
6965            }
6966            if let Some(types) = &req.memory_types {
6967                if !types.contains(&m.memory_type) {
6968                    continue;
6969                }
6970            }
6971            if let Some(agent) = &req.agent_id {
6972                let visible = m.agent_id == *agent || m.scope == Scope::Shared;
6973                if !visible {
6974                    continue;
6975                }
6976            }
6977            // Phase 12.6: pre-scoring filter (session_id + metadata
6978            // conditions). Cheap when `req.filter.is_empty()`; the
6979            // filter struct's `matches` is a short-circuit AND.
6980            if !req.filter.is_empty() && !req.filter.matches(&m) {
6981                continue;
6982            }
6983            // Phase 12.7: default-exclude the thread layers from fact
6984            // recall: archive turns (12.7a), essence items (12.7b; read
6985            // by scope through the render, not by similarity) and
6986            // thread-link markers (12.7c; they carry an all-equal
6987            // sentinel vector and would surface as junk hits). A caller
6988            // that wants them opts in by placing any `_mnemo.*` term in
6989            // the filter (equality, `$in`, `$exists`, or a range on
6990            // `_mnemo.thread` / `_mnemo.kind` / `_mnemo.seq`).
6991            if !req.filter.mentions_mnemo_meta()
6992                && matches!(
6993                    m.metadata.get(MNEMO_META_KIND).and_then(|v| v.as_str()),
6994                    Some(MNEMO_META_KIND_TURN)
6995                        | Some(MNEMO_META_KIND_ESSENCE)
6996                        | Some(MNEMO_META_KIND_THREAD_LINK)
6997                )
6998            {
6999                continue;
7000            }
7001            let sim = if req.query.is_empty() {
7002                0.0
7003            } else {
7004                memory::similarity(req.metric, &req.query, &m.vector)
7005            };
7006            let (rel, sim) = self.relevance(req, e.id, sim);
7007            let age_secs = (now - m.created_at) as f32;
7008            let score = req
7009                .weights
7010                .score(rel, age_secs, m.importance, m.access_count);
7011            scored.push(RecallResult {
7012                memory: m,
7013                score,
7014                similarity: sim,
7015            });
7016        }
7017        Ok(scored)
7018    }
7019
7020    /// FMT-9b: [`Mnemo::score_entries`] for a v9 file with no metadata
7021    /// filter. Every check the general path makes on the decrypted record
7022    /// is made on the row instead (supersession, TTL, type, agent
7023    /// visibility, the thread-layer kinds), the similarity comes from the
7024    /// fp16 column, and the score from the row's `created_at` and
7025    /// `importance`. Candidates are then decrypted best-first until `top_k`
7026    /// survive; the agent check is repeated on the record, since the row
7027    /// only has a hash of the agent id.
7028    fn score_rows(
7029        &mut self,
7030        req: &RecallRequest,
7031        now: i64,
7032        entries: &[CatalogEntry],
7033    ) -> Result<Vec<RecallResult>> {
7034        self.ensure_column()?;
7035        let d = self.dimensions;
7036        let q = &req.query;
7037        let q_norm = q.iter().map(|x| x * x).sum::<f32>().sqrt();
7038        let agent_h = req.agent_id.as_deref().map(|a| fnv1a32(&[a.as_bytes()]));
7039        let types: Option<Vec<u8>> = req
7040            .memory_types
7041            .as_ref()
7042            .map(|ts| ts.iter().map(|&t| memory_type_code(t)).collect());
7043        let mut cand: Vec<(f32, f32, usize)> = Vec::with_capacity(entries.len());
7044        for (pos, e) in entries.iter().enumerate() {
7045            let Some(r) = e.v9 else { continue };
7046            if r.flags & ROW_SUPERSEDED != 0 {
7047                continue;
7048            }
7049            if self.superseded.as_ref().is_some_and(|s| s.contains(&e.id)) {
7050                continue;
7051            }
7052            if r.flags & ROW_HAS_TTL != 0 && now >= r.expires_at {
7053                continue;
7054            }
7055            if let Some(ts) = &types {
7056                if !ts.contains(&r.memory_type) {
7057                    continue;
7058                }
7059            }
7060            if let Some(ah) = agent_h {
7061                if r.flags & ROW_SHARED == 0 && r.agent_hash != ah {
7062                    continue;
7063                }
7064            }
7065            if matches!(
7066                r.kind,
7067                ROW_KIND_TURN | ROW_KIND_ESSENCE | ROW_KIND_THREAD_LINK
7068            ) {
7069                continue;
7070            }
7071            let Some(idx) = self.index.get(e.id, &self.catalog) else {
7072                continue;
7073            };
7074            if !req.query.is_empty()
7075                && self.col.resident
7076                && idx < self.col.have.len()
7077                && !self.col.have[idx]
7078                && !self.col.hot.contains_key(&idx)
7079            {
7080                self.fill_column_page(idx)?;
7081            }
7082            let sim = if req.query.is_empty() {
7083                0.0
7084            } else if let Some((h, n)) = self.col.hot.get(&idx) {
7085                veccol::similarity_f16(req.metric, q, q_norm, h, *n)
7086            } else if self.col.resident && self.col.have.get(idx).copied().unwrap_or(false) {
7087                veccol::similarity_f16(
7088                    req.metric,
7089                    q,
7090                    q_norm,
7091                    &self.col.data[idx * d..(idx + 1) * d],
7092                    self.col.norms[idx],
7093                )
7094            } else {
7095                let (h, n) = self.column_entry(e)?;
7096                veccol::similarity_f16(req.metric, q, q_norm, &h, n)
7097            };
7098            let (rel, sim) = self.relevance(req, e.id, sim);
7099            let score = req.weights.score(
7100                rel,
7101                (now - r.created_at) as f32,
7102                r.importance,
7103                e.access_count,
7104            );
7105            cand.push((score, sim, pos));
7106        }
7107        cand.sort_by(|a, b| b.0.total_cmp(&a.0));
7108        let mut out: Vec<RecallResult> = Vec::with_capacity(req.top_k.min(cand.len()));
7109        for (score, sim, pos) in cand {
7110            if out.len() >= req.top_k {
7111                break;
7112            }
7113            let m = self.read_memory(&entries[pos])?;
7114            if let Some(agent) = &req.agent_id {
7115                if !(m.agent_id == *agent || m.scope == Scope::Shared) {
7116                    continue;
7117                }
7118            }
7119            out.push(RecallResult {
7120                memory: m,
7121                score,
7122                similarity: sim,
7123            });
7124        }
7125        Ok(out)
7126    }
7127
7128    /// Phase-5 multi-signal recall under the v0.7.0 scoring-v2 formula
7129    /// (see [`crate::ScoreWeights`]). Thin wrapper over
7130    /// [`Mnemo::recall_budgeted`] that drops the `truncated` flag;
7131    /// use the budgeted form if you need to know whether
7132    /// [`RecallRequest::max_bytes`] cut anything.
7133    pub fn recall(&mut self, req: &RecallRequest) -> Result<Vec<RecallResult>> {
7134        Ok(self.recall_budgeted(req)?.hits)
7135    }
7136
7137    /// Phase 12.8 budgeted recall: scoring v2 (see [`crate::ScoreWeights`])
7138    /// with an optional UTF-8-content byte cap. Ranks by
7139    /// `sim + max(sim, 0) · (β·recency + γ·(importance − 0.5) + δ·tanh(uses/5))`,
7140    /// with type / agent / supersession / TTL filters and the 12.6
7141    /// `Filter` applied before the scorer. Updates `accessed_at` /
7142    /// `access_count` on the returned memories (persisted on the next
7143    /// `flush`).
7144    ///
7145    /// When an ANN index has been built ([`Mnemo::build_index`]),
7146    /// recall runs the tiered IVF→PQ→rerank pipeline. If the filters
7147    /// (type / agent / supersession / TTL / 12.6) reduce the ANN
7148    /// shortlist below `top_k`, Phase 12.1's short-results-on-indexed-
7149    /// path fix kicks in: the query is retried with `n_probe` and
7150    /// `n_rerank` doubled, and if that would already touch half or
7151    /// more of the live non-superseded records, the ANN pass is
7152    /// skipped for an exact scan. Without an index, recall scores
7153    /// every live non-superseded memory.
7154    ///
7155    /// Phase 12.8: after `top_k` is applied, if
7156    /// [`RecallRequest::max_bytes`] is set, hits are dropped in rank
7157    /// order once the running sum of `content` UTF-8 bytes would
7158    /// exceed the budget. The top-1 hit is always kept even if it
7159    /// alone exceeds the budget. [`RecallResponse::truncated`] is
7160    /// `true` iff at least one hit was dropped by the budget.
7161    pub fn recall_budgeted(&mut self, req: &RecallRequest) -> Result<RecallResponse> {
7162        // FMT-9d: keyword recall needs a v9 file's BM25 index; a
7163        // keyword-only request has no vector.
7164        if req.keywords.is_some() && !self.is_v9() {
7165            return Err(MnemoError::Invalid(
7166                "keyword recall (keywords) needs a v9 file; run `mnemo compact` to upgrade".into(),
7167            ));
7168        }
7169        let keyword_only = req.query.is_empty() && req.keywords.is_some();
7170        if !keyword_only && req.query.len() != self.dimensions {
7171            return Err(MnemoError::DimensionMismatch {
7172                expected: self.dimensions,
7173                got: req.query.len(),
7174            });
7175        }
7176        // PERF-1 step 3: the ANN index loads on the first recall.
7177        self.ensure_ann()?;
7178        // Access-tracking bumps `accessed_at` / `access_count` in the
7179        // catalog, which is a mutation. On a read-only handle the caller
7180        // must pass `track_access(false)` (via CLI `--no-track`).
7181        if self.read_only && req.track_access {
7182            return Err(MnemoError::ReadOnly);
7183        }
7184        // Phase 12.1 **fix** (lane A v0.8.0 REVIEW): do NOT call
7185        // `ensure_key_caches()` here. A fresh read-only handle that
7186        // only ever recalls would otherwise pay a full-file decrypt
7187        // on its first query to build the lazy superseded set. Instead
7188        // `score_once` checks `_mnemo.superseded_by` per-record after
7189        // decrypt (slow path) and consults the set only when it has
7190        // already been built by a prior keyed op on this handle (fast
7191        // path). `live_non_superseded` is not knowable without a
7192        // decrypt pass either, so we approximate by `live_count`; the
7193        // exact-scan gate uses this as a conservative upper bound,
7194        // which errs toward widening (cheaper) rather than skipping
7195        // straight to a full scan.
7196        let live_count = self.catalog.iter().filter(|e| !e.deleted).count();
7197        let live_non_superseded = live_count;
7198        let now = memory::now_secs();
7199
7200        // Scoring v2 (shipped v0.7.0). Single-pass filter + score:
7201        //   * `age_secs = now - m.created_at`, so a recall doesn't bump
7202        //     its own results' recency on later queries.
7203        //   * `ScoreWeights::score` applies the similarity-gated,
7204        //     importance-centered, saturating-usage formula documented
7205        //     on `ScoreWeights`.
7206        //
7207        // Phase 12.1 + 12.6 short-results-on-indexed-path fix: when the
7208        // ANN shortlist is narrower than the live count, the filters
7209        // (type / agent / supersession / TTL / 12.6) can reduce the
7210        // result below `top_k`. If that happens, re-query the index
7211        // with `n_probe` / `n_rerank` doubled from their *effective*
7212        // base; if that would already cover ≥50% of the live set,
7213        // skip the index and do an exact scan.
7214        //
7215        // Lane A v0.8.0 REVIEW fix: widen from
7216        // `IvfPqIndex::effective_defaults()` when the caller didn't
7217        // pin the knobs. The pre-fix code only doubled when `req.n_*`
7218        // was already `Some(_)`, which skipped the widen step for the
7219        // default (unpinned) recall path and jumped straight to an
7220        // exact scan on moderately selective filters.
7221        let mut scored = self.score_once(req, now, false)?;
7222        let mut used_exact_scan = self.ann.is_none() || filter_bypasses_ann(&req.filter);
7223        if !used_exact_scan && scored.len() < req.top_k && live_non_superseded > req.top_k {
7224            if let Some(ann) = &self.ann {
7225                let (default_probe, default_rerank) = ann.effective_defaults();
7226                let base_probe = req.n_probe.unwrap_or(default_probe);
7227                let base_rerank = self.rerank_for(ann, req).unwrap_or(default_rerank);
7228                let widen_probe = base_probe.saturating_mul(2);
7229                let widen_rerank = base_rerank.saturating_mul(2);
7230                // Attempt 2: widen once, 2× from the effective base.
7231                // Only worth another ANN pass if either knob actually
7232                // grew (saturating_mul on usize::MAX would cap).
7233                if widen_probe > base_probe || widen_rerank > base_rerank {
7234                    let widened = RecallRequest {
7235                        n_probe: Some(widen_probe),
7236                        n_rerank: Some(widen_rerank),
7237                        ..req.clone()
7238                    };
7239                    scored = self.score_once(&widened, now, false)?;
7240                }
7241                // Attempt 3: if still short after widening, pay the
7242                // exact scan — it's the only path that guarantees
7243                // `top_k` correct hits when the filter is more
7244                // selective than `widen_rerank / live_count`. Before
7245                // the fix this was gated on `widened_rerank * 2 >=
7246                // live_non_superseded`, which silently returned fewer
7247                // than `top_k` hits on selective filters that fell in
7248                // the gap. Exact scan is bounded (one decrypt per
7249                // live record); the previous widen is an optimization.
7250                if scored.len() < req.top_k {
7251                    scored = self.score_once(req, now, true)?;
7252                    used_exact_scan = true;
7253                }
7254            }
7255        }
7256        scored.sort_by(|a, b| b.score.total_cmp(&a.score));
7257        scored.truncate(req.top_k);
7258
7259        // Update access statistics for everything we surfaced — but only
7260        // mutate the catalog entry, not the full record on disk. Pre-v5
7261        // this loop called `self.put(r.memory.clone())`, which serialized
7262        // and rewrote every result (vector included) to fresh pages,
7263        // turning a top-K recall into K full record rewrites. Now the
7264        // catalog is the source of truth for these two fields and the
7265        // record body stays untouched until the next real edit.
7266        if req.track_access {
7267            for r in &mut scored {
7268                if let Some(idx) = self.index.get(r.memory.id.0, &self.catalog) {
7269                    self.touch(idx);
7270                    let entry = &mut self.catalog[idx];
7271                    entry.accessed_at = now;
7272                    entry.access_count = entry.access_count.saturating_add(1);
7273                    // Propagate the just-updated values onto the returned
7274                    // Memory so the caller sees the post-recall state.
7275                    r.memory.accessed_at = entry.accessed_at;
7276                    r.memory.access_count = entry.access_count;
7277                }
7278            }
7279            // Mark the catalog dirty so the next flush persists these
7280            // bumps. The records themselves are not dirty.
7281            if !scored.is_empty() {
7282                self.dirty_catalog = true;
7283            }
7284        }
7285
7286        // Phase 12.8: apply the `max_bytes` budget *after* tracking has
7287        // seen the full top-k list. Access stats record what the scorer
7288        // chose; the byte budget shouldn't suppress the signal.
7289        let truncated = if let Some(budget) = req.max_bytes {
7290            let orig_len = scored.len();
7291            let mut running = 0usize;
7292            let mut cut_at: Option<usize> = None;
7293            for (i, r) in scored.iter().enumerate() {
7294                let w = r.memory.content.len();
7295                if i == 0 {
7296                    // Always keep the top hit, even if it's over budget
7297                    // on its own. Avoid "I asked for top_k results and
7298                    // got zero" when a single memory exceeds the cap.
7299                    running += w;
7300                    continue;
7301                }
7302                if running.saturating_add(w) > budget {
7303                    cut_at = Some(i);
7304                    break;
7305                }
7306                running += w;
7307            }
7308            if let Some(idx) = cut_at {
7309                scored.truncate(idx);
7310                scored.len() < orig_len
7311            } else {
7312                false
7313            }
7314        } else {
7315            false
7316        };
7317
7318        self.kw = None;
7319        Ok(RecallResponse {
7320            hits: scored,
7321            truncated,
7322            used_exact_scan,
7323        })
7324    }
7325
7326    // --- approximate index ------------------------------------------------
7327
7328    /// Build an IVF+PQ approximate-nearest-neighbour index over every live
7329    /// memory, using default tuning. After this, [`Mnemo::recall`] runs the
7330    /// tiered pipeline instead of an exact scan. Persisted on the next
7331    /// `flush`. Returns a snapshot of the index shape.
7332    pub fn build_index(&mut self) -> Result<IndexInfo> {
7333        self.build_index_with(IndexConfig::default())
7334    }
7335
7336    /// Build the ANN index with explicit tuning.
7337    pub fn build_index_with(&mut self, cfg: IndexConfig) -> Result<IndexInfo> {
7338        if self.read_only {
7339            return Err(MnemoError::ReadOnly);
7340        }
7341        // R-004.2: facts only; turns, essence items and thread links stay
7342        // out of the ANN.
7343        let mems: Vec<Memory> = self.memories()?.into_iter().filter(is_ann_kind).collect();
7344        if mems.is_empty() {
7345            return Err(MnemoError::Invalid(
7346                "cannot build an index over an empty database".into(),
7347            ));
7348        }
7349        // FMT-9b: a v9 file below V9_ANN_MIN_FACTS recalls by an exact fp16
7350        // scan, which is faster than the index there and costs no disk.
7351        // Report it as an index with no partitions, and drop a stale one.
7352        // `MNEMO_DEV_ANN_MIN_FACTS` overrides the threshold. Unsupported,
7353        // for tests (and the recall-grid bench) only: it lets the index
7354        // suites exercise a v9 ANN on small files. Not documented.
7355        let min_facts = std::env::var("MNEMO_DEV_ANN_MIN_FACTS")
7356            .ok()
7357            .and_then(|v| v.parse::<usize>().ok())
7358            .unwrap_or(V9_ANN_MIN_FACTS);
7359        if self.is_v9() && mems.len() < min_facts {
7360            if self.ann.is_some() || self.ann_unloaded {
7361                self.ann = None;
7362                self.ann_unloaded = false;
7363                self.dirty_index = true;
7364                self.index_persist.forced = true;
7365            }
7366            return Ok(IndexInfo {
7367                vectors: mems.len(),
7368                partitions: 0,
7369                subspaces: 0,
7370                n_probe: 0,
7371                n_rerank: 0,
7372            });
7373        }
7374        let items: Vec<(u128, &[f32])> =
7375            mems.iter().map(|m| (m.id.0, m.vector.as_slice())).collect();
7376        let mut idx = IvfPqIndex::build(self.dimensions, &items, cfg)?;
7377        let info = idx.info();
7378        if self.is_v9() {
7379            idx.enable_log();
7380            self.index_rebase = true;
7381        }
7382        self.ann = Some(idx);
7383        self.ann_unloaded = false;
7384        self.dirty_index = true;
7385        self.index_persist.forced = true;
7386        Ok(info)
7387    }
7388
7389    /// Rebuild the ANN index from scratch — re-clusters centroids and retrains
7390    /// the PQ codebook, undoing the cluster drift that accumulates as memories
7391    /// are inserted against fixed centroids.
7392    pub fn rebuild_index(&mut self) -> Result<IndexInfo> {
7393        // ANN-1.1: re-derive every parameter (partitions, subspaces,
7394        // n_probe, n_rerank) for the current corpus size by passing
7395        // `IndexConfig::default()`. Previously this freezed the
7396        // stored `(n_probe, n_rerank)` from the current index,
7397        // defeating the point of rebuilding on a grown corpus. If
7398        // you really want to pin specific values, call
7399        // `build_index_with(IndexConfig { … })` directly.
7400        self.build_index_with(IndexConfig::default())
7401    }
7402
7403    /// Drop the ANN index; recall reverts to exact scans. Persisted on flush.
7404    /// No-op on a read-only handle (signature is infallible — callers of
7405    /// `build_index` / `flush` on a read-only handle already get a typed
7406    /// `ReadOnly` error, which is the enforceable boundary).
7407    pub fn drop_index(&mut self) {
7408        if self.read_only {
7409            return;
7410        }
7411        if self.ann.is_some() || self.ann_unloaded {
7412            self.ann = None;
7413            self.ann_unloaded = false;
7414            self.dirty_index = true;
7415            self.index_persist.forced = true;
7416        }
7417    }
7418
7419    /// PERF-1 (R-004.1): bring a just-loaded ANN index up to date with the
7420    /// catalog. Called wherever an index run is loaded: on first use after
7421    /// `open` / `open_read_only` (`ensure_ann`, PERF-1 step 3) and by
7422    /// `restore_to`, with the first page of that run.
7423    ///
7424    /// `flush` persists the index only every so often, so that run can lag
7425    /// the catalog. It does two things:
7426    ///
7427    /// 1. **Prune** every indexed id the catalog no longer has live
7428    ///    (missing or deleted).
7429    /// 2. **Re-add** every live record written after the index run. Pages
7430    ///    are allocated monotonically in v8, and every write, including an
7431    ///    in-place `update` or `essence_update` (they rewrite the record to
7432    ///    fresh pages), lands at a page `> index_start`. Those records are
7433    ///    read and re-assigned. A record of a non-fact kind (turn, essence,
7434    ///    thread link) is removed instead: those never belong in the ANN.
7435    ///
7436    /// The work is bounded by [`INDEX_PERSIST_EVERY_CHANGES`] in the common
7437    /// case, since a writer persists the index at least that often.
7438    ///
7439    /// **v9:** pages are reused, so "a page past the run" no longer means
7440    /// "written after it". A v9 record is newer than the index run when its
7441    /// generation is greater than the run's (`index_gen`).
7442    fn reconcile_ann_with_catalog(&mut self, index_start: u64, index_gen: u64) -> Result<()> {
7443        if self.ann.is_none() {
7444            return Ok(());
7445        }
7446        let mut changed = 0usize;
7447        // 1. Prune ids that are no longer live.
7448        let stale: Vec<u128> = match &self.ann {
7449            Some(ann) => ann
7450                .ids()
7451                .filter(|&id| !matches!(self.index.get(id, &self.catalog), Some(i) if !self.catalog[i].deleted))
7452                .collect(),
7453            None => Vec::new(),
7454        };
7455        if let Some(ann) = &mut self.ann {
7456            for id in &stale {
7457                ann.remove(*id);
7458                changed += 1;
7459            }
7460        }
7461        // 2. Re-add records written after the index run.
7462        let v9 = self.is_v9();
7463        let pending: Vec<CatalogEntry> = self
7464            .catalog
7465            .iter()
7466            .filter(|e| !e.deleted && written_after_index(e, v9, index_start, index_gen))
7467            .cloned()
7468            .collect();
7469        for e in &pending {
7470            let m = self.read_memory(e)?;
7471            if let Some(ann) = &mut self.ann {
7472                if is_ann_kind(&m) {
7473                    ann.add(e.id, &m.vector);
7474                } else {
7475                    ann.remove(e.id);
7476                }
7477                changed += 1;
7478            }
7479        }
7480        if changed > 0 {
7481            self.dirty_index = true;
7482            self.index_persist.changes += changed;
7483        }
7484        Ok(())
7485    }
7486
7487    /// Whether the database has an ANN index (loaded, or persisted and
7488    /// loading on first use).
7489    pub fn has_index(&self) -> bool {
7490        self.ann.is_some() || self.ann_unloaded
7491    }
7492
7493    /// PERF-1 step 3: load the persisted ANN index on first use, then bring
7494    /// it up to date with the catalog (`reconcile_ann_with_catalog`) and,
7495    /// when the handle caches are already built, prune non-fact ids an
7496    /// older persisted index may hold.
7497    fn ensure_ann(&mut self) -> Result<()> {
7498        if !self.ann_unloaded {
7499            return Ok(());
7500        }
7501        self.ann_unloaded = false;
7502        self.ann = load_index(
7503            &mut self.pager,
7504            self.header.index_start,
7505            self.header.index_pages,
7506            self.header.index_len,
7507            self.header.index_gen,
7508            self.dimensions,
7509        )?;
7510        // FMT-9c.3: replay the v9 index delta chain, then record changes.
7511        if self.is_v9() {
7512            if let Some(ann) = self.ann.as_mut() {
7513                let (chain, n) = apply_index_chain(&mut self.pager, ann, self.header.index_delta)?;
7514                self.index_chain = chain;
7515                self.index_chain_ops = n;
7516                ann.enable_log();
7517            } else if self.header.index_pages > 0 {
7518                // FMT-9f (the 9d leak): an unknown-layout base loads as no
7519                // index; still track its delta runs, so rebuilding or
7520                // dropping the index frees them along with the base.
7521                self.index_chain = index_chain_runs(&mut self.pager, self.header.index_delta);
7522                self.index_chain_ops = 0;
7523            }
7524        }
7525        let (index_start, index_gen) = (self.header.index_start, self.index_gen_now());
7526        self.reconcile_ann_with_catalog(index_start, index_gen)?;
7527        let non_facts: Vec<u128> = match &self.layers {
7528            Some(l) => l
7529                .essence
7530                .iter()
7531                .copied()
7532                .chain(l.threads.values().flat_map(|t| t.records.iter().copied()))
7533                .chain(l.cold_turns.values().flatten().copied())
7534                .collect(),
7535            None => Vec::new(),
7536        };
7537        if let Some(ann) = &mut self.ann {
7538            let mut pruned = 0usize;
7539            for id in non_facts {
7540                if ann.contains(id) {
7541                    ann.remove(id);
7542                    pruned += 1;
7543                }
7544            }
7545            if pruned > 0 {
7546                self.dirty_index = true;
7547                self.index_persist.changes += pruned;
7548            }
7549        }
7550        Ok(())
7551    }
7552
7553    // --- durability & maintenance ----------------------------------------
7554
7555    /// Seal a serialized buffer into a fresh run of encrypted home pages and
7556    /// append a WAL frame for each. Returns the run's `(start_page, pages)`.
7557    /// The pages are *not* written to their home locations here — they go to
7558    /// the WAL and are folded in later by [`Mnemo::checkpoint`].
7559    ///
7560    /// `gen` is the run's generation (v9; `0` on v8). On v9 the pages come
7561    /// from the free list when an eligible run fits.
7562    fn seal_run(
7563        &mut self,
7564        bytes: &[u8],
7565        gen: u64,
7566        frames: &mut Vec<wal::Frame>,
7567    ) -> Result<(u64, u32)> {
7568        let pc = bytes.len().div_ceil(PAYLOAD).max(1);
7569        let start = self.alloc_pages(pc as u64);
7570        self.seal_run_at(start, pc as u64, bytes, gen, frames)?;
7571        Ok((start, pc as u32))
7572    }
7573
7574    /// Seal `bytes` into the `pages` already-allocated pages at `start`
7575    /// (pages past the data are sealed empty) and append their WAL frames.
7576    fn seal_run_at(
7577        &mut self,
7578        start: u64,
7579        pages: u64,
7580        bytes: &[u8],
7581        gen: u64,
7582        frames: &mut Vec<wal::Frame>,
7583    ) -> Result<()> {
7584        let direct = self.is_v9() && pages > V9_WAL_RUN_MAX_PAGES;
7585        for i in 0..pages as usize {
7586            let lo = (i * PAYLOAD).min(bytes.len());
7587            let hi = ((i + 1) * PAYLOAD).min(bytes.len());
7588            let page_no = start + i as u64;
7589            let sealed = self.pager.seal_page(page_no, gen, &bytes[lo..hi])?;
7590            if direct {
7591                self.pager.write_sealed(page_no, &sealed)?;
7592            } else {
7593                frames.push((page_no, sealed.to_vec()));
7594            }
7595        }
7596        if direct {
7597            self.direct_writes = true;
7598        }
7599        Ok(())
7600    }
7601
7602    /// Grow the WAL region if `frame_pages` worth of frames would not fit.
7603    /// Called at the top of `flush`, where the WAL is always spent (every
7604    /// flush ends with a checkpoint), so relocating it cannot strand a
7605    /// committed transaction.
7606    fn ensure_wal_capacity(&mut self, frame_pages: usize) -> Result<()> {
7607        let need = wal::txn_byte_len(frame_pages, PAGE_SIZE);
7608        if need <= self.header.wal_pages * PAGE_SIZE as u64 {
7609            return Ok(());
7610        }
7611        // Allocate a new, larger region at the file tail (~1.5x headroom).
7612        let req = need.div_ceil(PAGE_SIZE as u64);
7613        let new_pages = (req + req / 2 + 4).max(DEFAULT_WAL_PAGES);
7614        let old_wal_start = self.header.wal_start;
7615        let old_wal_pages = self.header.wal_pages;
7616        // FMT-9 (a): the new region always comes from the file's tail: the
7617        // relocation commits a header-only transaction, so a region taken
7618        // from the free list would still be listed as free in the
7619        // committed free-list run.
7620        let new_start = self.header.next_page;
7621        self.header.next_page += new_pages;
7622        self.header.wal_start = new_start;
7623        self.header.wal_pages = new_pages;
7624        self.header.wal_seq += 1;
7625        // DUR-1 (R-002.1): WAL-log the header change into the *new* region
7626        // first, checkpoint page 0, then invalidate the old region. A crash
7627        // anywhere here either keeps the old state (if page 0 wasn't yet
7628        // overwritten) or recovers to the new state via the torn-header
7629        // scan in `Mnemo::open`, which prefers the highest-txn-id valid
7630        // committed header frame.
7631        self.relocate_header_through_wal(old_wal_start)?;
7632        // FMT-9 (a): once the relocation has committed, the old region is
7633        // unreferenced; the next commit frees it.
7634        self.release_pages(old_wal_start, old_wal_pages);
7635        Ok(())
7636    }
7637
7638    /// Shared helper for every post-`create` page-0 commit: WAL-log the
7639    /// new header into the current WAL region, checkpoint page 0, and
7640    /// invalidate `old_wal_start` (the previous region, or the current
7641    /// region when it hasn't moved). The caller is expected to have
7642    /// mutated `self.header` to its target values *including a bumped
7643    /// `wal_seq`* (one more than the last on-disk `wal_seq`); this
7644    /// routine applies the seal, writes the frame + commit, folds page 0,
7645    /// then zeroes the old region's first frame header.
7646    ///
7647    /// If `old_wal_start == self.header.wal_start`, the invalidation is
7648    /// skipped — the region didn't move, so overwriting its first frame
7649    /// would destroy the just-committed one.
7650    fn relocate_header_through_wal(&mut self, old_wal_start: u64) -> Result<()> {
7651        self.header.write_counter = self.pager.write_counter;
7652        let mut page = self.header.to_page();
7653        self.header.apply_seal(&mut page, self.pager.dek())?;
7654        let txn_id = self.header.wal_seq;
7655        let frames = vec![(0u64, page.to_vec())];
7656        wal::commit(
7657            self.pager.file_mut(),
7658            self.header.wal_start,
7659            self.header.wal_pages,
7660            txn_id,
7661            &frames,
7662        )?;
7663        self.pager.write_raw(0, &page)?;
7664        self.pager.sync()?;
7665        if old_wal_start != self.header.wal_start {
7666            wal::invalidate_region(self.pager.file_mut(), old_wal_start)?;
7667        }
7668        Ok(())
7669    }
7670
7671    /// Fold a committed transaction's WAL frames into their home pages.
7672    fn checkpoint(&mut self, frames: &[wal::Frame]) -> Result<()> {
7673        for (page_no, bytes) in frames {
7674            let mut img = [0u8; PAGE_SIZE];
7675            img.copy_from_slice(bytes);
7676            self.pager.write_sealed(*page_no, &img)?;
7677        }
7678        self.pager.sync()?;
7679        Ok(())
7680    }
7681
7682    /// Persist all pending changes as one **write-ahead-logged transaction**.
7683    ///
7684    /// Sequence:
7685    ///
7686    /// 1. **Prepare.** Serialize the control plane (catalog, ANN index),
7687    ///    grow the WAL if needed, and **lease** counter and page slots —
7688    ///    bump `header.write_counter` and `header.next_page` by upper
7689    ///    bounds on this transaction's consumption and persist that
7690    ///    *leased* header to disk before any encrypted page is written.
7691    /// 2. **Data pages.** [`Pager::flush`] writes dirty record pages
7692    ///    copy-on-write under fresh nonces and fsyncs them. The orphan-page
7693    ///    nonce-reuse window (see [Phase 1.1 of the improvement plan])
7694    ///    is closed because the leased header on disk already records a
7695    ///    `write_counter` past anything this step can produce.
7696    /// 3. **Control plane.** Seal the new catalog / ANN index / snapshot
7697    ///    manifest into fresh home page runs.
7698    /// 4. **Commit.** Log all of step 3's frames plus the final header
7699    ///    frame into the WAL and fsync — the durability point.
7700    /// 5. **Checkpoint.** Fold the WAL into the home pages.
7701    ///
7702    /// A crash before step 4 leaves the previous state intact. A crash
7703    /// after step 4 is repaired by [`Mnemo::open`] replaying the WAL.
7704    /// Safe to call repeatedly.
7705    pub fn flush(&mut self) -> Result<()> {
7706        if self.read_only {
7707            return Err(MnemoError::ReadOnly);
7708        }
7709        // Snapshot the thread-local fsync counter. Every `sync_all` the
7710        // pager / wal / checkpoint path makes between here and the end
7711        // of this function bumps the same counter; `delta` lands on
7712        // `self.fsyncs_last_flush` so `Stats::fsyncs_last_flush` can
7713        // report it. The planner wants this to decide whether cutting
7714        // fsync count is a viable write-latency win before we touch
7715        // the write path's durability model.
7716        let fsync_before = crate::fsync_count();
7717        // PERF-1 step 3: while the index is unloaded, records written since
7718        // its run pile up for the next load to re-assign. Once there are
7719        // INDEX_PERSIST_EVERY_CHANGES of them, load it now so this flush
7720        // persists a current index and keeps that work bounded.
7721        if self.ann_unloaded && self.dirty_catalog {
7722            let (index_start, index_gen) = (self.header.index_start, self.index_gen_now());
7723            let v9 = self.is_v9();
7724            let pending = self
7725                .catalog
7726                .iter()
7727                .filter(|e| !e.deleted && written_after_index(e, v9, index_start, index_gen))
7728                .count();
7729            if pending >= INDEX_PERSIST_EVERY_CHANGES {
7730                self.ensure_ann()?;
7731            }
7732        }
7733        // PERF-1: persist the (derived) ANN index only when forced or due.
7734        // Otherwise the header keeps pointing at the previous index run, and
7735        // records written since then are re-assigned when it's next loaded
7736        // (`reconcile_ann_with_catalog`). `ann == None` with `dirty_index` set is a
7737        // dropped index, which is persisted at once.
7738        let persist_index = if self.is_v9() {
7739            self.plan_index_v9()
7740        } else {
7741            self.dirty_index
7742                && (self.index_persist.forced
7743                    || self.ann.is_none()
7744                    || self.index_persist.commits + 1 >= INDEX_PERSIST_EVERY_COMMITS
7745                    || self.index_persist.changes >= INDEX_PERSIST_EVERY_CHANGES)
7746        };
7747        let free_work =
7748            self.is_v9() && (self.free_dirty || !self.pending_free.is_empty() || self.dirty_keys);
7749        if !self.dirty_catalog && !persist_index && !self.dirty_cache && !free_work {
7750            // No control-plane changes pending. Because every `put` that
7751            // dirties a data page also dirties the catalog, and every
7752            // `cache_put` that dirties a data page also dirties
7753            // `dirty_cache`, there can't be dirty data pages either —
7754            // nothing to do.
7755            self.fsyncs_last_flush = 0;
7756            return Ok(());
7757        }
7758
7759        // FIX-1 R-001.8 part 3: snapshot the memory/index-dirty flags
7760        // before step 3b decides whether to push a restore point.
7761        // Only flushes that change the memory catalog or ANN index
7762        // need a snapshot — a cache-only flush (`dirty_cache` set,
7763        // `dirty_catalog` + `dirty_index` both false) must not push,
7764        // because snapshots don't capture cache state and every push
7765        // counts against `max_snapshots` (default 256), so a cache-
7766        // heavy workload would otherwise evict every real memory
7767        // restore point within ~256 cache writes.
7768        let is_memory_or_index_flush = self.dirty_catalog || persist_index;
7769
7770        // FMT-9: a v9 session's first sealed page needs the generation
7771        // lease (normally taken by its first record write already).
7772        self.ensure_gen_lease()?;
7773        // FMT-9e: turns that aged past the cutoff are rewritten compressed
7774        // (bounded per commit), then packed with the other waiting records.
7775        self.settle_cold_turns()?;
7776        // FMT-9b: column entries first; they move rows' vector locations,
7777        // so they must land before the catalog is serialized.
7778        self.pack_records()?;
7779        self.pack_vectors()?;
7780
7781        // 1. Serialize control plane, ensure WAL capacity, lease counter +
7782        //    page slots, and persist the leased header *before* any data
7783        //    page hits the disk.
7784        let ctx = self.prepare_for_flush(persist_index)?;
7785
7786        // 2. Record (vector) data pages: copy-on-write to fresh pages, fsynced.
7787        //    Safe: the lease guarantees that even if we crash here, reopen
7788        //    will see a `write_counter` past every nonce this step uses.
7789        self.pager.flush()?;
7790
7791        // 3. Seal the catalog / index control pages into fresh home runs.
7792        //    FMT-9: `txn_id` is this transaction's id and the generation
7793        //    of every run it seals. A v9 run replaced here is freed by this
7794        //    same commit (`superseded`); v8 never reuses pages.
7795        let txn_id = self.header.wal_seq + 1;
7796        let gen = if self.is_v9() { txn_id } else { 0 };
7797        let v9 = self.is_v9();
7798        let mut superseded: Vec<(u64, u64)> = Vec::new();
7799        let mut frames: Vec<wal::Frame> = Vec::new();
7800        // FMT-9c.2: a delta run goes on the chain; a new base replaces the
7801        // old base and the whole chain (all freed by this commit). The
7802        // chain is adopted once the commit succeeds.
7803        let mut new_chain: Option<(Vec<RunPtr>, usize)> = None;
7804        if let Some(bytes) = &ctx.cat_bytes {
7805            if ctx.cat_delta {
7806                let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
7807                let ptr = RunPtr {
7808                    start,
7809                    pages: pages as u64,
7810                    len: bytes.len() as u64,
7811                    gen,
7812                };
7813                self.header.catalog_delta = ptr;
7814                let mut chain = self.delta_chain.clone();
7815                chain.push(ptr);
7816                new_chain = Some((chain, self.delta_rows + self.dirty_rows.len()));
7817            } else {
7818                if v9 && self.header.catalog_pages > 0 {
7819                    superseded.push((self.header.catalog_start, self.header.catalog_pages));
7820                }
7821                if v9 {
7822                    for r in &self.delta_chain {
7823                        superseded.push((r.start, r.pages));
7824                    }
7825                    self.header.catalog_delta = RunPtr::default();
7826                    new_chain = Some((Vec::new(), 0));
7827                }
7828                let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
7829                self.header.catalog_start = start;
7830                self.header.catalog_pages = pages as u64;
7831                self.header.catalog_len = bytes.len() as u64;
7832                self.header.catalog_gen = gen;
7833            }
7834        }
7835        // FMT-9d: the BM25 segment, then the manifest that lists it.
7836        let mut new_bm25: Option<Bm25Segments> = None;
7837        if let Some((bytes, replace_from)) = &ctx.bm25_seg {
7838            let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
7839            let mut segs = self.bm25.manifest.segments.clone();
7840            let mut seg_docs = self.bm25.manifest.seg_docs.clone();
7841            seg_docs.resize(segs.len(), 0);
7842            let docs = if let Some(from) = *replace_from {
7843                for t in segs.drain(from..) {
7844                    superseded.push((t.0, t.1));
7845                }
7846                seg_docs.truncate(from);
7847                self.bm25.merged_docs
7848            } else {
7849                self.bm25.pending.len() as u32
7850            };
7851            segs.push((start, pages as u64, bytes.len() as u64, gen));
7852            seg_docs.push(docs);
7853            let man = Bm25Manifest {
7854                tokenizer: text::TOKENIZER_V1,
7855                next_doc: self.bm25.manifest.next_doc,
7856                segments: segs.clone(),
7857                seg_docs: seg_docs.clone(),
7858            };
7859            let mbytes =
7860                rmp_serde::to_vec(&man).map_err(|e| MnemoError::Serialize(e.to_string()))?;
7861            if !self.header.bm25.is_empty() {
7862                superseded.push((self.header.bm25.start, self.header.bm25.pages));
7863            }
7864            let (ms, mp) = self.seal_run(&mbytes, gen, &mut frames)?;
7865            self.header.bm25 = RunPtr {
7866                start: ms,
7867                pages: mp as u64,
7868                len: mbytes.len() as u64,
7869                gen,
7870            };
7871            new_bm25 = Some((segs, seg_docs));
7872        }
7873        let mut new_ichain: Option<(Vec<RunPtr>, usize)> = None;
7874        if persist_index && ctx.idx_delta {
7875            if let Some(bytes) = &ctx.idx_bytes {
7876                let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
7877                let ptr = RunPtr {
7878                    start,
7879                    pages: pages as u64,
7880                    len: bytes.len() as u64,
7881                    gen,
7882                };
7883                self.header.index_delta = ptr;
7884                let mut chain = self.index_chain.clone();
7885                chain.push(ptr);
7886                let n = self.index_delta_ops.as_ref().map_or(0, |o| o.len());
7887                new_ichain = Some((chain, self.index_chain_ops + n));
7888            }
7889        } else if persist_index {
7890            if v9 && self.header.index_pages > 0 {
7891                superseded.push((self.header.index_start, self.header.index_pages));
7892            }
7893            if v9 {
7894                for r in &self.index_chain {
7895                    superseded.push((r.start, r.pages));
7896                }
7897                self.header.index_delta = RunPtr::default();
7898                new_ichain = Some((Vec::new(), 0));
7899            }
7900            match &ctx.idx_bytes {
7901                Some(bytes) => {
7902                    let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
7903                    self.header.index_start = start;
7904                    self.header.index_pages = pages as u64;
7905                    self.header.index_len = bytes.len() as u64;
7906                    self.header.index_gen = gen;
7907                }
7908                None => {
7909                    self.header.index_start = 0;
7910                    self.header.index_pages = 0;
7911                    self.header.index_len = 0;
7912                    self.header.index_gen = 0;
7913                }
7914            }
7915        }
7916        // v8+ result-cache directory (Phase 10.1). Seal the serialized
7917        // directory into a fresh page run and update the pointer in the
7918        // header. The seal AAD already covers cache_start/pages/len for
7919        // v8+ (see `header_seal_aad`), so tampering with the pointer
7920        // trips the header seal on the next open.
7921        if self.dirty_cache {
7922            if v9 && self.header.cache_pages > 0 {
7923                superseded.push((self.header.cache_start, self.header.cache_pages));
7924            }
7925            match &ctx.cache_bytes {
7926                Some(bytes) if !self.cache_dir.is_empty() => {
7927                    let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
7928                    self.header.cache_start = start;
7929                    self.header.cache_pages = pages as u64;
7930                    self.header.cache_len = bytes.len() as u64;
7931                    self.header.cache_gen = gen;
7932                }
7933                _ => {
7934                    // Either the directory serialized to nothing (all
7935                    // entries deleted after tombstone-only migration) or
7936                    // it's empty — collapse the pointer to zero.
7937                    self.header.cache_start = 0;
7938                    self.header.cache_pages = 0;
7939                    self.header.cache_len = 0;
7940                    self.header.cache_gen = 0;
7941                }
7942            }
7943        }
7944
7945        // 3b. Record this transaction as a restorable snapshot. The manifest
7946        //     update is staged in a local and adopted only once the commit
7947        //     below succeeds, so a failed flush leaves no phantom entry.
7948        //
7949        //     The cap (`self.max_snapshots`) is applied to the staged local
7950        //     before the new entry lands, so after appending the manifest
7951        //     holds at most `max_snapshots` entries — the most-recent N.
7952        //     `max_snapshots == 0` disables the cap. On v8, pages
7953        //     referenced only by pruned snapshots stay on disk until
7954        //     `compact_file`; on v9 they become reusable once no retained
7955        //     snapshot is older than the transaction that freed them.
7956        //
7957        //     FIX-1 R-001.8 part 3: skip the snapshot push on a
7958        //     cache-only flush. Snapshots don't capture the cache
7959        //     anyway, and every push counts against `max_snapshots`
7960        //     (default 256) + rewrites the manifest run — so a
7961        //     cache-heavy workload would otherwise evict every real
7962        //     memory restore point within ~256 cache writes. On a
7963        //     cache-only flush we reuse the current manifest
7964        //     verbatim; the manifest run already lives at
7965        //     `self.header.manifest_{start,pages,len}` (set by the
7966        //     last memory-touching flush, or at `create` time).
7967        let live = self.len() as u64;
7968        let mut manifest = self.manifest.clone();
7969        if is_memory_or_index_flush {
7970            if self.max_snapshots > 0 && !v9 {
7971                // Drop the oldest entries so there's room for the new one.
7972                while manifest.len() >= self.max_snapshots {
7973                    manifest.remove(0);
7974                }
7975            }
7976            manifest.push(Snapshot {
7977                txn_id,
7978                created_at: memory::now_secs(),
7979                catalog_start: self.header.catalog_start,
7980                catalog_pages: self.header.catalog_pages,
7981                catalog_len: self.header.catalog_len,
7982                index_start: self.header.index_start,
7983                index_pages: self.header.index_pages,
7984                index_len: self.header.index_len,
7985                memory_count: live,
7986                gens: v9.then_some(SnapshotGens {
7987                    catalog: self.header.catalog_gen,
7988                    index: self.header.index_gen,
7989                    delta: ptr_tuple(self.header.catalog_delta),
7990                    index_delta: ptr_tuple(self.header.index_delta),
7991                    bm25: ptr_tuple(self.header.bm25),
7992                }),
7993            });
7994            if v9 {
7995                let extra = self.pending_free.iter().map(|r| r.1).sum::<u64>()
7996                    + superseded.iter().map(|r| r.1).sum::<u64>()
7997                    + self.header.manifest_pages;
7998                self.apply_retention(&mut manifest, extra);
7999            }
8000            let man_bytes =
8001                rmp_serde::to_vec(&manifest).map_err(|e| MnemoError::Serialize(e.to_string()))?;
8002            if v9 && self.header.manifest_pages > 0 {
8003                superseded.push((self.header.manifest_start, self.header.manifest_pages));
8004            }
8005            let (m_start, m_pages) = self.seal_run(&man_bytes, gen, &mut frames)?;
8006            self.header.manifest_start = m_start;
8007            self.header.manifest_pages = m_pages as u64;
8008            self.header.manifest_len = man_bytes.len() as u64;
8009            self.header.manifest_gen = gen;
8010        }
8011
8012        // 3c. FMT-9 (b): the key-slot run, rewritten copy-on-write when a
8013        //     slot changed. The header seal covers its pointer.
8014        if v9 && self.dirty_keys {
8015            if !self.header.keys.is_empty() {
8016                superseded.push((self.header.keys.start, self.header.keys.pages));
8017            }
8018            if self.key_slots.slots.is_empty() {
8019                self.header.keys = RunPtr::default();
8020            } else {
8021                let bytes = rmp_serde::to_vec(&self.key_slots)
8022                    .map_err(|e| MnemoError::Serialize(e.to_string()))?;
8023                let (start, pages) = self.seal_run(&bytes, gen, &mut frames)?;
8024                self.header.keys = RunPtr {
8025                    start,
8026                    pages: pages as u64,
8027                    len: bytes.len() as u64,
8028                    gen,
8029                };
8030            }
8031        }
8032
8033        // 3d. FMT-9 (a): the free list. Everything this transaction
8034        //     stopped referencing (`pending_free`, the runs replaced above,
8035        //     and the previous free-list run itself) is freed *at this
8036        //     transaction's id*, in this transaction's own commit, so it
8037        //     can't be reused until this commit is durable and no retained
8038        //     snapshot predates it. The list is staged in a local and
8039        //     adopted only once the commit succeeds (a failed flush that
8040        //     is retried must not free the same runs twice).
8041        let mut shrink = false;
8042        let staged_free =
8043            if v9 && (self.free_dirty || !self.pending_free.is_empty() || !superseded.is_empty()) {
8044                let mut staged = self.free.clone();
8045                for &(p, c) in self.pending_free.iter().chain(superseded.iter()) {
8046                    staged.free(txn_id, p, c)?;
8047                }
8048                if !self.header.free.is_empty() {
8049                    staged.free(txn_id, self.header.free.start, self.header.free.pages)?;
8050                }
8051                // FMT-9c: give eligible free runs at the file's tail back
8052                // to the filesystem: the header's `next_page` moves down in
8053                // this commit, and the file is cut after it.
8054                let min_snap = self.min_snapshot_txn();
8055                while let Some(p) =
8056                    staged.take_tail(self.header.next_page, self.header.wal_seq, min_snap)
8057                {
8058                    self.header.next_page = p;
8059                    shrink = true;
8060                }
8061                // Size the run with room for the one extra entry that
8062                // allocating it out of the list can create (a split), then
8063                // allocate it from the list (eligible runs only) or the tail.
8064                let est = rmp_serde::to_vec(&staged.to_runs())
8065                    .map_err(|e| MnemoError::Serialize(e.to_string()))?
8066                    .len()
8067                    + 64;
8068                let pc = est.div_ceil(PAYLOAD).max(1) as u64;
8069                let start = match staged.alloc(pc, self.header.wal_seq, self.min_snapshot_txn()) {
8070                    Some(p) => p,
8071                    None => {
8072                        let p = self.header.next_page;
8073                        self.header.next_page += pc;
8074                        p
8075                    }
8076                };
8077                let bytes = rmp_serde::to_vec(&staged.to_runs())
8078                    .map_err(|e| MnemoError::Serialize(e.to_string()))?;
8079                if bytes.len() as u64 > pc * PAYLOAD as u64 {
8080                    return Err(MnemoError::Invalid(
8081                        "free list outgrew its reserved run".into(),
8082                    ));
8083                }
8084                self.seal_run_at(start, pc, &bytes, gen, &mut frames)?;
8085                self.header.free = RunPtr {
8086                    start,
8087                    pages: pc,
8088                    len: bytes.len() as u64,
8089                    gen,
8090                };
8091                Some(staged)
8092            } else {
8093                None
8094            };
8095
8096        // 3e. The header is the transaction's final frame; stamp the new id
8097        //     and the *actual* (post-flush) write counter, which is <= the
8098        //     leased value persisted in `prepare_for_flush`. This is the
8099        //     value that ultimately replaces the leased header on checkpoint.
8100        self.header.vector_count = live;
8101        self.header.write_counter = self.pager.write_counter;
8102        self.header.wal_seq = txn_id;
8103        let mut hpage = self.header.to_page();
8104        self.header.apply_seal(&mut hpage, self.pager.dek())?;
8105        frames.push((0, hpage.to_vec()));
8106
8107        // 4. COMMIT — log the transaction and fsync. Nothing at a home page
8108        //    other than the leased header has changed yet; this single
8109        //    fsync is what makes the rest durable.
8110        // FMT-9c: big runs went straight to their home pages; make them
8111        // durable before the header that references them can be.
8112        if self.direct_writes {
8113            self.pager.sync()?;
8114            self.direct_writes = false;
8115        }
8116        let (wal_start, wal_pages) = (self.header.wal_start, self.header.wal_pages);
8117        wal::commit(self.pager.file_mut(), wal_start, wal_pages, txn_id, &frames)?;
8118
8119        // 5. Checkpoint — fold the WAL into the home pages.
8120        self.checkpoint(&frames)?;
8121        // FMT-9c: the committed header no longer covers the tail; cut it.
8122        // A crash before this leaves a longer file, which is harmless.
8123        if shrink {
8124            let len = self.header.next_page * PAGE_SIZE as u64;
8125            self.pager.file_mut().set_len(len)?;
8126        }
8127
8128        self.manifest = manifest;
8129        if let Some(staged) = staged_free {
8130            self.free = staged;
8131        }
8132        self.pending_free.clear();
8133        self.free_dirty = false;
8134        self.dirty_keys = false;
8135        if let Some((chain, rows)) = new_chain {
8136            self.delta_chain = chain;
8137            self.delta_rows = rows;
8138        }
8139        if let Some((chain, n)) = new_ichain {
8140            self.index_chain = chain;
8141            self.index_chain_ops = n;
8142        }
8143        if let Some((segs, seg_docs)) = new_bm25 {
8144            // FMT-9f: keep the loaded dicts and read only the new segment's
8145            // (a merge replaced the segments from `kept` on), instead of
8146            // dropping them all and reloading every dict on next use.
8147            let kept = self
8148                .bm25
8149                .manifest
8150                .segments
8151                .iter()
8152                .zip(&segs)
8153                .take_while(|(a, b)| a == b)
8154                .count();
8155            if let Some(mut dicts) = self.bm25.dicts.take() {
8156                dicts.truncate(kept);
8157                let mut ok = dicts.len() == kept;
8158                for &t in &segs[kept..] {
8159                    match load_segment_dict(&mut self.pager, tuple_ptr(t)) {
8160                        Ok(d) if ok => dicts.push(d),
8161                        _ => ok = false,
8162                    }
8163                }
8164                if ok {
8165                    self.bm25.dicts = Some(dicts);
8166                }
8167            }
8168            self.bm25.manifest.segments = segs;
8169            self.bm25.manifest.seg_docs = seg_docs;
8170            self.bm25.pending.clear();
8171        }
8172        if persist_index {
8173            self.index_rebase = false;
8174            self.index_delta_ops = None;
8175        }
8176        if ctx.cat_bytes.is_some() {
8177            self.dirty_rows.clear();
8178            self.catalog_rewrite = false;
8179        }
8180        self.dirty_catalog = false;
8181        if persist_index {
8182            self.dirty_index = false;
8183            self.index_persist = IndexPersist::default();
8184        } else if self.dirty_index {
8185            self.index_persist.commits += 1;
8186        }
8187        self.dirty_cache = false;
8188        // Batched policy: successful flush consumes the pending batch.
8189        self.cache_batch.reset();
8190        // Record how many fsyncs this flush issued, for Stats reporting.
8191        // Thread-local counter guarantees we see only this handle's own
8192        // sync_all calls between the snapshot above and this read.
8193        self.fsyncs_last_flush =
8194            (crate::fsync_count().wrapping_sub(fsync_before)).min(u32::MAX as u64) as u32;
8195        Ok(())
8196    }
8197
8198    /// Pre-flush prelude shared by [`Mnemo::flush`] and the
8199    /// `__crash_partial_flush_for_testing` hook.
8200    ///
8201    /// Serializes the control plane (catalog, ANN index, cache directory)
8202    /// and grows the WAL if needed. That's it: this used to persist a
8203    /// leased header too (SEC-1 obsoleted it; see DUR-1 R-002.1 note in
8204    /// the body for the full rationale), but after DUR-1 the only page-0
8205    /// writes outside `create` go through the WAL via
8206    /// [`Mnemo::relocate_header_through_wal`].
8207    ///
8208    /// The historical pre-stamping used to close the nonce-reuse window
8209    /// flagged in Phase 1.1 of the improvement plan: a crash anywhere
8210    /// between here and the WAL commit left a file whose on-disk header
8211    /// recorded a `write_counter` past every nonce the rest of this
8212    /// transaction could produce, so the next reopen started from a
8213    /// counter that guaranteed uniqueness against any orphan data pages
8214    /// still on disk. SEC-1's per-session random nonce prefix gives the
8215    /// same guarantee without needing a persisted counter, which is why
8216    /// this routine no longer writes page 0.
8217    fn prepare_for_flush(&mut self, persist_index: bool) -> Result<FlushPrelude> {
8218        // Serialize.
8219        // FMT-9c.2: on v9 usually just the changed rows, as a delta run.
8220        let cat_delta = self.dirty_catalog && self.catalog_as_delta();
8221        let cat_bytes = if !self.dirty_catalog {
8222            None
8223        } else if cat_delta {
8224            let rows: Vec<(u32, CatalogEntry)> = self
8225                .dirty_rows
8226                .iter()
8227                .filter(|&&i| i < self.catalog.len())
8228                .map(|&i| (i as u32, self.catalog[i].clone()))
8229                .collect();
8230            let d = CatalogDelta {
8231                prev: ptr_tuple(self.header.catalog_delta),
8232                rows,
8233            };
8234            Some(rmp_serde::to_vec(&d).map_err(|e| MnemoError::Serialize(e.to_string()))?)
8235        } else {
8236            if self.is_v9() {
8237                self.compact_tombstones();
8238            }
8239            Some(
8240                rmp_serde::to_vec(&self.catalog)
8241                    .map_err(|e| MnemoError::Serialize(e.to_string()))?,
8242            )
8243        };
8244        // FMT-9c.3: a v9 commit usually writes just its index changes.
8245        let idx_delta = persist_index && self.index_delta_ops.is_some();
8246        let idx_bytes: Option<Vec<u8>> = if !persist_index {
8247            None
8248        } else if let Some(ops) = &self.index_delta_ops {
8249            let d = IndexDeltaRef {
8250                prev: ptr_tuple(self.header.index_delta),
8251                ops,
8252            };
8253            Some(rmp_serde::to_vec(&d).map_err(|e| MnemoError::Serialize(e.to_string()))?)
8254        } else {
8255            match &self.ann {
8256                Some(ann) => {
8257                    let mut b =
8258                        rmp_serde::to_vec(ann).map_err(|e| MnemoError::Serialize(e.to_string()))?;
8259                    // FMT-9: v9 index runs carry a layout byte first.
8260                    if self.is_v9() {
8261                        b.insert(0, self.index_layout_override.unwrap_or(V9_INDEX_LAYOUT));
8262                    }
8263                    Some(b)
8264                }
8265                None => None,
8266            }
8267        };
8268        // Serialize the result-cache directory when it's dirty. An
8269        // empty directory serializes to a small header + 0-element
8270        // array; still cheap enough that we don't special-case it.
8271        let cache_bytes: Option<Vec<u8>> = if self.dirty_cache {
8272            Some(
8273                rmp_serde::to_vec(&self.cache_dir)
8274                    .map_err(|e| MnemoError::Serialize(e.to_string()))?,
8275            )
8276        } else {
8277            None
8278        };
8279
8280        let cat_pc = cat_bytes
8281            .as_ref()
8282            .map_or(0, |b| b.len().div_ceil(PAYLOAD).max(1));
8283        let idx_pc = idx_bytes
8284            .as_ref()
8285            .map_or(0, |b| b.len().div_ceil(PAYLOAD).max(1));
8286        let cache_pc = cache_bytes
8287            .as_ref()
8288            .map_or(0, |b| b.len().div_ceil(PAYLOAD).max(1));
8289        // Upper bound on the manifest run: one extra entry, <=82 bytes each
8290        // (rmp_serde encodes a Snapshot as a 9-element fixarray of i64/u64,
8291        // worst case 81 bytes plus a 1-byte array header).
8292        // v9 snapshots carry two more u64s (≤ 20 bytes with the array header).
8293        let per_snapshot = if self.is_v9() { 102 } else { 82 };
8294        let man_upper = 9 + (self.manifest.len() + 1) * per_snapshot;
8295        let man_pc_est = man_upper.div_ceil(PAYLOAD).max(1);
8296        // FMT-9: the free-list run (≤ 28 bytes per run: every pending and
8297        // superseded run plus a split and a relocated WAL region) and the
8298        // key-slot run.
8299        let (free_pc_est, keys_pc) = if self.is_v9() {
8300            let runs = self.free.len() + self.pending_free.len() + 12;
8301            let free_pc = (runs * 28 + 64).div_ceil(PAYLOAD).max(1);
8302            let keys_pc = if self.dirty_keys {
8303                rmp_serde::to_vec(&self.key_slots)
8304                    .map_err(|e| MnemoError::Serialize(e.to_string()))?
8305                    .len()
8306                    .div_ceil(PAYLOAD)
8307                    .max(1)
8308            } else {
8309                0
8310            };
8311            (free_pc, keys_pc)
8312        } else {
8313            (0, 0)
8314        };
8315
8316        // Grow WAL if needed (DUR-1: relocation is WAL-logged and does
8317        // its own fsyncs). The cache directory rides in the same
8318        // transaction so its page count contributes to the reservation.
8319        // FMT-9d: the BM25 segment (and its manifest, one page).
8320        let bm25_seg = self.bm25_next_segment()?;
8321        let bm25_pc = bm25_seg
8322            .as_ref()
8323            .map_or(0, |(b, _)| b.len().div_ceil(PAYLOAD).max(1));
8324        // FMT-9c: on v9 only runs of up to V9_WAL_RUN_MAX_PAGES go through
8325        // the WAL; longer ones are written in place.
8326        let v9 = self.is_v9();
8327        let in_wal = |pc: usize| {
8328            if v9 && pc as u64 > V9_WAL_RUN_MAX_PAGES {
8329                0
8330            } else {
8331                pc
8332            }
8333        };
8334        self.ensure_wal_capacity(
8335            in_wal(cat_pc)
8336                + in_wal(idx_pc)
8337                + in_wal(cache_pc)
8338                + in_wal(man_pc_est)
8339                + in_wal(free_pc_est)
8340                + in_wal(keys_pc)
8341                + in_wal(bm25_pc)
8342                + usize::from(bm25_pc > 0)
8343                + 1,
8344        )?;
8345
8346        // DUR-1 (R-002.1): the pre-flush "leased header" write that used
8347        // to live here is gone. It existed to close the Phase 1.1 nonce
8348        // reuse window (reopen must see a `write_counter` past any nonce
8349        // this flush could produce before data pages hit disk). SEC-1
8350        // superseded that: nonces now carry a per-session random prefix
8351        // and `write_counter` is no longer a nonce input, so the leased
8352        // write bought us nothing but an extra fsync per flush. Orphan
8353        // data pages past `header.next_page` on a crashed flush are
8354        // still safe — the next session's random prefix makes any later
8355        // write at the same page_no uniquely nonced regardless of
8356        // `write_counter`.
8357
8358        Ok(FlushPrelude {
8359            bm25_seg,
8360            cat_bytes,
8361            cat_delta,
8362            idx_delta,
8363            idx_bytes,
8364            cache_bytes,
8365        })
8366    }
8367
8368    /// Flush and close. Equivalent to `flush()` on a read-write handle; on
8369    /// a read-only handle it's a no-op (no dirty state to persist). The
8370    /// file is released on drop, which also drops the OS advisory lock.
8371    /// **Caution:** the handle keeps the file and its lock until it is
8372    /// dropped, so `drop` it before reopening the same file (or the open
8373    /// fails with [`MnemoError::Locked`]).
8374    pub fn close(&mut self) -> Result<()> {
8375        if self.read_only {
8376            return Ok(());
8377        }
8378        // PERF-1: closing persists a dirty ANN index too, so the next open
8379        // has nothing to re-assign.
8380        self.index_persist.forced = true;
8381        self.flush()
8382    }
8383
8384    /// PERF-1: flush, persisting the ANN index even if the
8385    /// [`INDEX_PERSIST_EVERY_COMMITS`] / [`INDEX_PERSIST_EVERY_CHANGES`]
8386    /// policy wouldn't yet. Plain [`Mnemo::flush`] writes only the catalog
8387    /// between persists; the index is derived, so a crash costs nothing but
8388    /// re-assigning the newer records on the next load.
8389    pub fn flush_index(&mut self) -> Result<()> {
8390        if self.read_only {
8391            return Err(MnemoError::ReadOnly);
8392        }
8393        self.index_persist.forced = true;
8394        self.flush()
8395    }
8396
8397    /// **TEST ONLY — do not use.** Reproduces a crash inside
8398    /// [`Mnemo::flush`] *after* the prelude has leased counter+page slots
8399    /// and persisted the leased header, but *before* the WAL is committed.
8400    /// Runs `prepare_for_flush` + `pager.flush` and returns — leaving
8401    /// data pages and a leased header on disk, with no commit frame.
8402    ///
8403    /// Used by `tests/integration.rs::nonce_unique_after_crashed_data_flush`
8404    /// to verify that this window (Phase 1.1 of the improvement plan) does
8405    /// not enable AES-GCM nonce reuse. Calling this in production code
8406    /// strands data pages; never expose it from a binding.
8407    #[doc(hidden)]
8408    pub fn __crash_partial_flush_for_testing(&mut self) -> Result<()> {
8409        if self.read_only {
8410            return Err(MnemoError::ReadOnly);
8411        }
8412        if !self.dirty_catalog && !self.dirty_index {
8413            return Ok(());
8414        }
8415        // FMT-9b: v9 records and vectors reach their pages at the top of a
8416        // flush; do that part too.
8417        self.ensure_gen_lease()?;
8418        self.pack_records()?;
8419        self.pack_vectors()?;
8420        let _prelude = self.prepare_for_flush(self.dirty_index)?;
8421        self.pager.flush()
8422    }
8423
8424    /// Bound the in-memory page cache to `pages` decrypted pages.
8425    ///
8426    /// The cache holds decrypted page payloads to speed repeated reads. By
8427    /// default it is capped at 1024 pages (~8 MiB; 8192 before v0.10.0).
8428    /// Lower the cap to trade hit rate for a smaller footprint, or raise
8429    /// it for a hotter cache. The
8430    /// cap governs *clean* pages — pages with un-flushed writes are always
8431    /// retained until [`Mnemo::flush`], regardless of the cap.
8432    pub fn set_cache_capacity(&mut self, pages: usize) {
8433        self.pager.set_cache_capacity(pages);
8434    }
8435
8436    /// FMT-9b (v9 files): keep the fp16 vector column in memory while it
8437    /// takes at most `bytes` (default [`DEFAULT_VECTOR_CACHE_BUDGET`],
8438    /// 96 MiB, about 125K vectors at 384 dims). Above it, recall reads
8439    /// vector entries through the page cache instead: less memory, slower
8440    /// recall. Lowering it below the resident size drops the resident copy.
8441    pub fn set_vector_cache_budget(&mut self, bytes: usize) {
8442        self.vector_cache_budget = bytes;
8443        let need = self.catalog.len() * veccol::entry_len(self.dimensions);
8444        if self.col.resident && need > bytes {
8445            self.col.data = Vec::new();
8446            self.col.norms = Vec::new();
8447            self.col.have = Vec::new();
8448            self.col.resident = false;
8449        }
8450    }
8451
8452    /// Page-cache occupancy: `(pages_cached, capacity)`. Renamed from
8453    /// `cache_stats` in v0.4.0: the result-cache API (Phase 10.1) took
8454    /// that name for its per-namespace [`CacheStats`] summary, and the
8455    /// page-cache pair is the older/less-user-facing of the two, so it
8456    /// gets the more-specific name.
8457    pub fn page_cache_stats(&self) -> (usize, usize) {
8458        self.pager.cache_stats()
8459    }
8460
8461    // === Result cache (Phase 10.1 + 10.3) ============================
8462
8463    /// Insert or overwrite a cache entry at `(namespace, key)`.
8464    ///
8465    /// The engine SHA-256-hashes `key` — callers pass the raw input
8466    /// string. Persists to disk on the next `flush()` under the current
8467    /// flush policy (see [`Mnemo::set_cache_flush_policy`]); in
8468    /// [`CacheFlushPolicy::Batched`] the engine may call `flush()`
8469    /// internally when a threshold trips.
8470    ///
8471    /// On overflow of the namespace's [`CacheBudget`] (default 10,000
8472    /// entries / 64 MiB) the LRU entry is tombstoned before the new
8473    /// insert. Tombstones are reclaimed by
8474    /// [`Mnemo::compact_file`], not the runtime flush.
8475    ///
8476    /// Returns [`MnemoError::ReadOnly`] on a read-only handle.
8477    pub fn cache_put(
8478        &mut self,
8479        namespace: &str,
8480        key: &str,
8481        value: &[u8],
8482        opts: CachePutOpts,
8483    ) -> Result<()> {
8484        if self.read_only {
8485            return Err(MnemoError::ReadOnly);
8486        }
8487        let key_hash = result_cache::hash_key(key);
8488        let key_preview = result_cache::key_preview(key);
8489        let now = memory::now_secs();
8490        let size_bytes = value.len() as u64;
8491
8492        let entry = CacheEntry {
8493            namespace: namespace.to_string(),
8494            key_hash,
8495            key_preview: key_preview.clone(),
8496            value: value.to_vec(),
8497            content_type: opts.content_type.clone(),
8498            created_at: now,
8499            accessed_at: now,
8500            access_count: 0,
8501            ttl_secs: opts.ttl_secs,
8502            cost_hint_ms: opts.cost_hint_ms,
8503            size_bytes,
8504            vector: None,
8505            model: None,
8506        };
8507        let bytes = rmp_serde::to_vec(&entry).map_err(|e| MnemoError::Serialize(e.to_string()))?;
8508        let (start_page, page_count, gen) = self.write_record(&bytes)?;
8509
8510        // Tombstone any prior entry for this key. Its pages remain on
8511        // disk until `compact_file`, exactly like memory-record
8512        // overwrites — the tombstone just hides it from lookups.
8513        if let Some(prev_pos) = self.cache_idx.get(namespace, &key_hash) {
8514            self.cache_dir[prev_pos].deleted = true;
8515            self.release_cache_pages(prev_pos);
8516            self.cache_idx.remove(namespace, &key_hash);
8517        }
8518
8519        let dir_entry = CacheDirectoryEntry {
8520            namespace: namespace.to_string(),
8521            key_hash,
8522            start_page,
8523            page_count,
8524            len: bytes.len() as u32,
8525            deleted: false,
8526            accessed_at: now,
8527            access_count: 0,
8528            size_bytes,
8529            ttl_secs: opts.ttl_secs,
8530            created_at: now,
8531            vector: None,
8532            model: None,
8533            gen,
8534        };
8535        let pos = self.cache_dir.len();
8536        self.cache_dir.push(dir_entry);
8537        self.cache_idx.insert(namespace, key_hash, pos);
8538
8539        // Enforce budget — evict LRU tombstones until the namespace
8540        // fits under both caps.
8541        self.evict_to_budget(namespace)?;
8542
8543        self.dirty_cache = true;
8544        self.cache_batch.record_dirty();
8545
8546        // Batched policy: auto-flush if a threshold tripped. Successful
8547        // `flush()` calls `cache_batch.reset()` for us.
8548        if self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
8549            self.flush()?;
8550        }
8551        Ok(())
8552    }
8553
8554    /// Look up `(namespace, key)`. Returns `None` on miss (either the
8555    /// key was never inserted or the entry is TTL-expired). On hit
8556    /// updates catalog-side access stats (mirrors the recall v5-trick);
8557    /// on a read-only handle the stats update is silently skipped.
8558    ///
8559    /// A miss is recorded as a miss in this handle's counters even if
8560    /// the key was TTL-expired — from the caller's perspective the two
8561    /// are indistinguishable.
8562    pub fn cache_get(&mut self, namespace: &str, key: &str) -> Result<Option<CachedValue>> {
8563        // FIX-1 R-001.8 part 4: under `Batched`, honor `max_age` on
8564        // reads too. Pre-FIX-1 the age check only fired on `cache_put`,
8565        // so a workload that wrote a few entries and then switched to
8566        // pure reads could sit with its pending batch past the
8567        // nominal max_age indefinitely. `max_dirty` is already
8568        // maintained per-write; this adds the symmetric time-bound
8569        // check on the read path. Only runs on read-write handles —
8570        // flush() returns ReadOnly for shared-lock readers.
8571        if !self.read_only && self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
8572            self.flush()?;
8573        }
8574        let key_hash = result_cache::hash_key(key);
8575        let pos = match self.cache_idx.get(namespace, &key_hash) {
8576            Some(p) => p,
8577            None => {
8578                self.cache_counters.misses += 1;
8579                return Ok(None);
8580            }
8581        };
8582        let now = memory::now_secs();
8583        // Snapshot the directory entry fields we need before any
8584        // mutation — the borrow checker gets grumpy otherwise.
8585        let (start_page, page_count, len, ttl_secs, expired, created_at, gen) = {
8586            let e = &self.cache_dir[pos];
8587            (
8588                e.start_page,
8589                e.page_count,
8590                e.len,
8591                e.ttl_secs,
8592                e.is_expired(now),
8593                e.created_at,
8594                e.gen,
8595            )
8596        };
8597        if expired {
8598            // Treat expired as a miss for the caller. Don't tombstone
8599            // here — cache_purge does that. Just skip.
8600            self.cache_counters.misses += 1;
8601            return Ok(None);
8602        }
8603        // Decrypt and decode the record body.
8604        let buf = self.read_record_run(start_page, page_count, len, gen)?;
8605        let entry: CacheEntry =
8606            rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
8607
8608        // Bump access stats catalog-only (v5 trick — no full record
8609        // rewrite on a hit). Skipped on read-only handles so a shared-lock
8610        // reader can't dirty the catalog.
8611        if !self.read_only {
8612            self.cache_dir[pos].accessed_at = now;
8613            self.cache_dir[pos].access_count = self.cache_dir[pos].access_count.saturating_add(1);
8614            self.dirty_cache = true;
8615        }
8616        self.cache_counters.hits += 1;
8617        // On a mutating handle: access_count reflects the just-bumped
8618        // value. On a read-only handle: the on-disk directory value is
8619        // unchanged, so returning it verbatim is honest (the caller's
8620        // reads did not, and cannot, alter the persisted counter).
8621        //
8622        // FIX-1 R-002.2 part 1: `accessed_at` must also come from the
8623        // directory on a read-only handle, not from `entry.accessed_at`
8624        // in the record body. The record body is written once at insert
8625        // time and never updated on a hit — every live `accessed_at`
8626        // update lands in the directory (see the `!self.read_only`
8627        // branch above). The pre-FIX-1 code returned
8628        // `entry.accessed_at`, which is really the entry's **creation
8629        // time** by then, so a read-only reader saw "last accessed =
8630        // when it was inserted" even after many successful hits by
8631        // concurrent writers.
8632        let access_count = self.cache_dir[pos].access_count;
8633        let accessed_at = if self.read_only {
8634            self.cache_dir[pos].accessed_at
8635        } else {
8636            now
8637        };
8638        Ok(Some(CachedValue {
8639            value: entry.value,
8640            content_type: entry.content_type,
8641            created_at,
8642            accessed_at,
8643            access_count,
8644            ttl_secs,
8645        }))
8646    }
8647
8648    /// Tombstone the entry at `(namespace, key)`. Silently succeeds if
8649    /// no such entry exists. Space is reclaimed by `compact_file`.
8650    pub fn cache_delete(&mut self, namespace: &str, key: &str) -> Result<()> {
8651        if self.read_only {
8652            return Err(MnemoError::ReadOnly);
8653        }
8654        let key_hash = result_cache::hash_key(key);
8655        if let Some(pos) = self.cache_idx.get(namespace, &key_hash) {
8656            self.cache_dir[pos].deleted = true;
8657            self.release_cache_pages(pos);
8658            self.cache_idx.remove(namespace, &key_hash);
8659            self.dirty_cache = true;
8660            self.cache_batch.record_dirty();
8661            if self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
8662                self.flush()?;
8663            }
8664        }
8665        Ok(())
8666    }
8667
8668    /// Purge cache entries. If `namespace` is `Some`, restrict to that
8669    /// namespace; otherwise purge across the whole cache. If
8670    /// `expired_only` is `true`, only TTL-expired entries are removed;
8671    /// otherwise every live entry in the target scope is tombstoned.
8672    /// Returns the number of entries newly tombstoned.
8673    pub fn cache_purge(&mut self, namespace: Option<&str>, expired_only: bool) -> Result<usize> {
8674        if self.read_only {
8675            return Err(MnemoError::ReadOnly);
8676        }
8677        let now = memory::now_secs();
8678        let mut purged = 0usize;
8679        // Iterate the directory by index; mark matching entries as
8680        // deleted and pull them out of the in-memory index.
8681        for pos in 0..self.cache_dir.len() {
8682            if self.cache_dir[pos].deleted {
8683                continue;
8684            }
8685            if let Some(ns) = namespace {
8686                if self.cache_dir[pos].namespace != ns {
8687                    continue;
8688                }
8689            }
8690            if expired_only && !self.cache_dir[pos].is_expired(now) {
8691                continue;
8692            }
8693            let key_hash = self.cache_dir[pos].key_hash;
8694            let ns = self.cache_dir[pos].namespace.clone();
8695            self.cache_dir[pos].deleted = true;
8696            self.release_cache_pages(pos);
8697            self.cache_idx.remove(&ns, &key_hash);
8698            purged += 1;
8699        }
8700        if purged > 0 {
8701            self.dirty_cache = true;
8702            self.cache_batch.record_dirty();
8703            if self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
8704                self.flush()?;
8705            }
8706        }
8707        Ok(purged)
8708    }
8709
8710    /// Statistics for `namespace` (or the whole cache if `None`).
8711    /// `hits`/`misses`/`evictions` are per-handle in-memory counters
8712    /// that reset on reopen; `entries`/`bytes` reflect the durable
8713    /// on-disk state (live entries only, tombstones excluded).
8714    pub fn cache_stats(&self, namespace: Option<&str>) -> CacheStats {
8715        let mut entries = 0usize;
8716        let mut bytes = 0u64;
8717        for e in &self.cache_dir {
8718            if e.deleted {
8719                continue;
8720            }
8721            if let Some(ns) = namespace {
8722                if e.namespace != ns {
8723                    continue;
8724                }
8725            }
8726            entries += 1;
8727            bytes += e.size_bytes;
8728        }
8729        let hits = self.cache_counters.hits;
8730        let misses = self.cache_counters.misses;
8731        let total = hits + misses;
8732        let hit_rate = if total > 0 {
8733            hits as f64 / total as f64
8734        } else {
8735            0.0
8736        };
8737        CacheStats {
8738            entries,
8739            bytes,
8740            hits,
8741            misses,
8742            hit_rate,
8743            evictions: self.cache_counters.evictions,
8744        }
8745    }
8746
8747    /// Set (or update) the eviction budget for `namespace`. Applies to
8748    /// subsequent `cache_put` calls; does not retroactively evict
8749    /// entries that already sit under a looser cap.
8750    pub fn set_cache_budget(&mut self, namespace: &str, budget: CacheBudget) {
8751        self.cache_budgets.insert(namespace.to_string(), budget);
8752    }
8753
8754    /// Set the per-handle cache flush policy. Defaults to
8755    /// [`CacheFlushPolicy::Strict`]. Memory writes are never batched;
8756    /// this affects only cache mutations.
8757    pub fn set_cache_flush_policy(&mut self, policy: CacheFlushPolicy) {
8758        self.cache_flush_policy = policy;
8759        // Switching mid-run: if we're moving to Strict and the batch is
8760        // non-empty, the pending count still counts toward dirty_cache
8761        // which the next flush() will consume.
8762    }
8763
8764    /// FMT-9 (a): free a just-tombstoned cache entry's record pages at the
8765    /// next commit (v9; no-op on v8).
8766    fn release_cache_pages(&mut self, pos: usize) {
8767        let (p, c) = (
8768            self.cache_dir[pos].start_page,
8769            self.cache_dir[pos].page_count as u64,
8770        );
8771        self.release_pages(p, c);
8772    }
8773
8774    /// Evict LRU live entries in `namespace` until it fits under both
8775    /// budget caps. Called from `cache_put` after the insert; the new
8776    /// entry is included in the count so we always leave room.
8777    fn evict_to_budget(&mut self, namespace: &str) -> Result<()> {
8778        let budget = self
8779            .cache_budgets
8780            .get(namespace)
8781            .copied()
8782            .unwrap_or_default();
8783        loop {
8784            // Sum current live count + bytes in the namespace.
8785            let mut live_count = 0usize;
8786            let mut live_bytes = 0u64;
8787            for e in &self.cache_dir {
8788                if !e.deleted && e.namespace == namespace {
8789                    live_count += 1;
8790                    live_bytes += e.size_bytes;
8791                }
8792            }
8793            if live_count <= budget.max_entries && live_bytes <= budget.max_bytes {
8794                return Ok(());
8795            }
8796            // Find the LRU (oldest accessed_at) live entry.
8797            let victim_pos = self
8798                .cache_dir
8799                .iter()
8800                .enumerate()
8801                .filter(|(_, e)| !e.deleted && e.namespace == namespace)
8802                .min_by_key(|(_, e)| e.accessed_at)
8803                .map(|(i, _)| i);
8804            let Some(pos) = victim_pos else {
8805                // Nothing to evict but still over budget — the single
8806                // just-inserted entry itself is bigger than
8807                // max_bytes. Nothing we can safely do here without
8808                // rejecting the put; leave it and return.
8809                return Ok(());
8810            };
8811            let key_hash = self.cache_dir[pos].key_hash;
8812            let ns = self.cache_dir[pos].namespace.clone();
8813            self.cache_dir[pos].deleted = true;
8814            self.release_cache_pages(pos);
8815            self.cache_idx.remove(&ns, &key_hash);
8816            self.cache_counters.evictions += 1;
8817        }
8818    }
8819
8820    // === Semantic cache (Phase 10.2) ==================================
8821
8822    /// Insert or overwrite a **semantic** cache entry — same as
8823    /// [`Mnemo::cache_put`] but with a required embedding vector and
8824    /// model identifier. The vector is stored on the directory entry
8825    /// so [`Mnemo::cache_get_semantic`] can score without decrypting
8826    /// record bodies.
8827    ///
8828    /// Vector dimensionality is validated against the database's
8829    /// configured `dimensions` — a mismatch returns
8830    /// [`MnemoError::DimensionMismatch`], the same signal `remember`
8831    /// gives for the memory catalog.
8832    ///
8833    /// The `model` string is matched **exactly** on lookup — a
8834    /// different model's cache is invisible to a given query. Pick
8835    /// canonical identifiers (`"text-embedding-3-small"`,
8836    /// `"bge-large-en-v1.5"`, ...) and stick to them.
8837    pub fn cache_put_semantic(
8838        &mut self,
8839        namespace: &str,
8840        key: &str,
8841        vector: Vec<f32>,
8842        value: &[u8],
8843        opts: SemanticCachePutOpts,
8844    ) -> Result<()> {
8845        if self.read_only {
8846            return Err(MnemoError::ReadOnly);
8847        }
8848        if vector.len() != self.dimensions {
8849            return Err(MnemoError::DimensionMismatch {
8850                expected: self.dimensions,
8851                got: vector.len(),
8852            });
8853        }
8854        let key_hash = result_cache::hash_key(key);
8855        let key_preview = result_cache::key_preview(key);
8856        let now = memory::now_secs();
8857        let size_bytes = value.len() as u64;
8858
8859        let entry = CacheEntry {
8860            namespace: namespace.to_string(),
8861            key_hash,
8862            key_preview: key_preview.clone(),
8863            value: value.to_vec(),
8864            content_type: opts.content_type.clone(),
8865            created_at: now,
8866            accessed_at: now,
8867            access_count: 0,
8868            ttl_secs: opts.ttl_secs,
8869            cost_hint_ms: opts.cost_hint_ms,
8870            size_bytes,
8871            vector: Some(vector.clone()),
8872            model: Some(opts.model.clone()),
8873        };
8874        let bytes = rmp_serde::to_vec(&entry).map_err(|e| MnemoError::Serialize(e.to_string()))?;
8875        let (start_page, page_count, gen) = self.write_record(&bytes)?;
8876
8877        if let Some(prev_pos) = self.cache_idx.get(namespace, &key_hash) {
8878            self.cache_dir[prev_pos].deleted = true;
8879            self.release_cache_pages(prev_pos);
8880            self.cache_idx.remove(namespace, &key_hash);
8881        }
8882
8883        let dir_entry = CacheDirectoryEntry {
8884            namespace: namespace.to_string(),
8885            key_hash,
8886            start_page,
8887            page_count,
8888            len: bytes.len() as u32,
8889            deleted: false,
8890            accessed_at: now,
8891            access_count: 0,
8892            size_bytes,
8893            ttl_secs: opts.ttl_secs,
8894            created_at: now,
8895            vector: Some(vector),
8896            model: Some(opts.model),
8897            gen,
8898        };
8899        let pos = self.cache_dir.len();
8900        self.cache_dir.push(dir_entry);
8901        self.cache_idx.insert(namespace, key_hash, pos);
8902
8903        self.evict_to_budget(namespace)?;
8904
8905        self.dirty_cache = true;
8906        self.cache_batch.record_dirty();
8907        if self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
8908            self.flush()?;
8909        }
8910        Ok(())
8911    }
8912
8913    /// Semantic recall over the cache: score every live entry in
8914    /// `namespace` whose `model` matches against `query_vector` using
8915    /// cosine similarity; return the top-1 hit and its similarity iff
8916    /// `sim >= threshold`. `None` otherwise.
8917    ///
8918    /// Uses [`crate::result_cache::DEFAULT_SEMANTIC_THRESHOLD`] if you
8919    /// want a sensible default — pass it explicitly (`0.97` today).
8920    ///
8921    /// Exact-key entries (those without a vector) are transparently
8922    /// skipped, as are entries whose model doesn't match — a
8923    /// different-model hit would be a semantic bug, not a win.
8924    /// TTL-expired entries are also skipped.
8925    ///
8926    /// Access-stat updates on hit follow the same policy as
8927    /// [`Mnemo::cache_get`]: bumped on read-write handles, silently
8928    /// skipped on read-only. Hits and misses are recorded in the
8929    /// per-handle counters just like exact-key lookups.
8930    pub fn cache_get_semantic(
8931        &mut self,
8932        namespace: &str,
8933        query_vector: &[f32],
8934        threshold: f32,
8935        model: &str,
8936    ) -> Result<Option<(CachedValue, f32)>> {
8937        if query_vector.len() != self.dimensions {
8938            return Err(MnemoError::DimensionMismatch {
8939                expected: self.dimensions,
8940                got: query_vector.len(),
8941            });
8942        }
8943        // FIX-1 R-001.8 part 4: same Batched max_age check as
8944        // `cache_get`. See the comment there.
8945        if !self.read_only && self.cache_batch.should_auto_flush(&self.cache_flush_policy) {
8946            self.flush()?;
8947        }
8948        let now = memory::now_secs();
8949
8950        // Scan namespace live entries, filter by model, compute
8951        // cosine, keep the best. Exact-key entries have `vector: None`
8952        // and are skipped by the filter.
8953        let mut best: Option<(usize, f32)> = None;
8954        for (i, e) in self.cache_dir.iter().enumerate() {
8955            if e.deleted || e.namespace != namespace || e.is_expired(now) {
8956                continue;
8957            }
8958            let Some(ref v) = e.vector else { continue };
8959            let Some(ref m) = e.model else { continue };
8960            if m != model {
8961                continue;
8962            }
8963            let sim = memory::cosine(query_vector, v);
8964            if best.map(|(_, s)| sim > s).unwrap_or(true) {
8965                best = Some((i, sim));
8966            }
8967        }
8968
8969        let Some((pos, sim)) = best else {
8970            self.cache_counters.misses += 1;
8971            return Ok(None);
8972        };
8973        if sim < threshold {
8974            self.cache_counters.misses += 1;
8975            return Ok(None);
8976        }
8977
8978        // Extract fields before any mutation to keep the borrow checker
8979        // happy.
8980        let (start_page, page_count, len, ttl_secs, created_at, gen) = {
8981            let e = &self.cache_dir[pos];
8982            (
8983                e.start_page,
8984                e.page_count,
8985                e.len,
8986                e.ttl_secs,
8987                e.created_at,
8988                e.gen,
8989            )
8990        };
8991
8992        // Decrypt and decode the record body.
8993        let buf = self.read_record_run(start_page, page_count, len, gen)?;
8994        let entry: CacheEntry =
8995            rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
8996
8997        if !self.read_only {
8998            self.cache_dir[pos].accessed_at = now;
8999            self.cache_dir[pos].access_count = self.cache_dir[pos].access_count.saturating_add(1);
9000            self.dirty_cache = true;
9001        }
9002        self.cache_counters.hits += 1;
9003        let access_count = self.cache_dir[pos].access_count;
9004        // FIX-1 R-002.2 part 1: same read-only `accessed_at` fix as
9005        // `cache_get`. The directory carries the live value; the record
9006        // body only holds the insert-time snapshot.
9007        let accessed_at = if self.read_only {
9008            self.cache_dir[pos].accessed_at
9009        } else {
9010            now
9011        };
9012        Ok(Some((
9013            CachedValue {
9014                value: entry.value,
9015                content_type: entry.content_type,
9016                created_at,
9017                accessed_at,
9018                access_count,
9019                ttl_secs,
9020            },
9021            sim,
9022        )))
9023    }
9024
9025    /// Begin a conversation [`Session`](crate::Session) for `agent_id`.
9026    ///
9027    /// The session borrows the database for its lifetime, records turns as
9028    /// working memory, and consolidates them into episodic memory when closed.
9029    pub fn session(&mut self, agent_id: impl Into<String>) -> crate::session::Session<'_> {
9030        crate::session::Session::new(self, agent_id.into())
9031    }
9032
9033    // --- snapshots & point-in-time recovery ------------------------------
9034
9035    /// Every committed transaction, oldest first — the restore points
9036    /// available to [`Mnemo::restore_to`] and [`Mnemo::restore_to_time`].
9037    ///
9038    /// Each `flush` appends one snapshot. Because the storage engine is
9039    /// append-only, the pages a past flush wrote are still on disk, so any
9040    /// listed snapshot can be reinstated exactly. The history reaches back to
9041    /// the last [`Mnemo::compact_file`], which reclaims space by collapsing
9042    /// it.
9043    pub fn snapshots(&self) -> Vec<SnapshotInfo> {
9044        let mut v: Vec<SnapshotInfo> = self
9045            .manifest
9046            .iter()
9047            .map(|s| SnapshotInfo {
9048                txn_id: s.txn_id,
9049                created_at: s.created_at,
9050                memory_count: s.memory_count,
9051            })
9052            .collect();
9053        v.sort_by_key(|s| s.txn_id);
9054        v
9055    }
9056
9057    /// Load a past snapshot's state and commit it as a new transaction.
9058    fn apply_snapshot(&mut self, snap: &Snapshot) -> Result<()> {
9059        let catalog = load_catalog(
9060            &mut self.pager,
9061            snap.catalog_start,
9062            snap.catalog_pages,
9063            snap.catalog_len,
9064            snap.catalog_gen(),
9065        )?;
9066        let (catalog, _, _) = apply_delta_chain(&mut self.pager, catalog, snap.delta_head())?;
9067        // FMT-9c.2: the restored catalog is written whole.
9068        self.catalog_rewrite = true;
9069        let ann = load_index(
9070            &mut self.pager,
9071            snap.index_start,
9072            snap.index_pages,
9073            snap.index_len,
9074            snap.index_gen(),
9075            self.dimensions,
9076        )?;
9077        let mut ann = ann;
9078        if self.is_v9() {
9079            self.reclaim_for_restore(&catalog);
9080            if let Some(a) = ann.as_mut() {
9081                apply_index_chain(&mut self.pager, a, snap.index_delta_head())?;
9082                a.enable_log();
9083            }
9084            self.index_rebase = true;
9085            // FMT-9d: the snapshot's BM25 segments (quarantined while it's
9086            // retained) describe its documents.
9087            let bptr = snap.gens.map_or(RunPtr::default(), |g| tuple_ptr(g.bm25));
9088            let manifest = load_bm25_manifest(&mut self.pager, bptr)?;
9089            // Its runs come back off the free list; the current ones it
9090            // doesn't share are freed, and the header points at its run.
9091            let runs = |ptr: RunPtr, m: &Bm25Manifest| {
9092                let mut v: Vec<(u64, u64)> = m.segments.iter().map(|t| (t.0, t.1)).collect();
9093                if !ptr.is_empty() {
9094                    v.push((ptr.start, ptr.pages));
9095                }
9096                v
9097            };
9098            let then = runs(bptr, &manifest);
9099            let now = runs(self.header.bm25, &self.bm25.manifest);
9100            self.pending_free.retain(|r| !then.contains(r));
9101            for &(p, c) in &then {
9102                self.free.unfree(p, c);
9103            }
9104            for r in now {
9105                if !then.contains(&r) {
9106                    self.release_pages(r.0, r.1);
9107                }
9108            }
9109            self.header.bm25 = bptr;
9110            self.free_dirty = true;
9111            self.bm25 = Bm25State {
9112                manifest,
9113                ..Bm25State::default()
9114            };
9115        }
9116        self.index = build_id_index(&catalog);
9117        self.catalog = catalog;
9118        // FMT-9b: the column and the packed pages described the catalog
9119        // being replaced.
9120        self.col = VecColumn::from_catalog(&self.catalog);
9121        self.recs = RecPages::from_catalog(&self.catalog);
9122        self.ann = ann;
9123        self.ann_unloaded = false;
9124        // The handle caches describe the catalog being replaced.
9125        self.key_index = None;
9126        self.superseded = None;
9127        self.layers = None;
9128        self.agent_set = None;
9129        // PERF-1: the snapshot's index may predate its catalog; bring it up
9130        // to date before it's persisted again.
9131        let snap_index_gen = snap.index_gen().max(snap.index_delta_head().gen);
9132        self.reconcile_ann_with_catalog(snap.index_start, snap_index_gen)?;
9133        // Re-commit as a fresh transaction: crash-safe, and itself recorded
9134        // as a new snapshot so a restore can always be undone.
9135        self.dirty_catalog = true;
9136        self.dirty_index = true;
9137        self.index_persist.forced = true;
9138        self.flush()
9139    }
9140
9141    /// FMT-9c: v9 snapshot retention, applied to the staged manifest (the
9142    /// new snapshot already appended): keep the newest `max_snapshots`
9143    /// plus any younger than 24 hours, then drop the oldest while retained
9144    /// snapshots pin more than a quarter of the file's live pages. The
9145    /// newest snapshot always stays.
9146    ///
9147    /// The pin cap is a quarter of the live pages or [`V9_PIN_FLOOR_PAGES`]
9148    /// (1 MiB), whichever is larger, so tiny files keep their history.
9149    /// `extra` is what this commit frees (pinned by every older snapshot).
9150    ///
9151    /// An explicit count (`set_max_snapshots`, or a non-default
9152    /// `MnemoConfig::max_snapshots`) replaces the policy: exactly the
9153    /// newest `n` are kept (`0` keeps all), whatever they pin.
9154    fn apply_retention(&self, manifest: &mut Vec<Snapshot>, extra: u64) {
9155        if self.snapshot_cap_explicit {
9156            if self.max_snapshots > 0 {
9157                while manifest.len() > self.max_snapshots {
9158                    manifest.remove(0);
9159                }
9160            }
9161            return;
9162        }
9163        let now = memory::now_secs();
9164        if self.max_snapshots > 0 {
9165            while manifest.len() > self.max_snapshots
9166                && now - manifest[0].created_at > V9_SNAPSHOT_KEEP_SECS
9167            {
9168                manifest.remove(0);
9169            }
9170        }
9171        let live_pages = self.header.next_page.saturating_sub(self.free.pages());
9172        let cap = (live_pages / 4).max(V9_PIN_FLOOR_PAGES);
9173        while manifest.len() > 1 {
9174            let pinned = self.free.pinned_pages(manifest[0].txn_id) + extra;
9175            if pinned <= cap {
9176                break;
9177            }
9178            manifest.remove(0);
9179        }
9180    }
9181
9182    /// FMT-9 (a): a restore makes the snapshot's records live again and the
9183    /// current ones (those the snapshot doesn't have) dead. The snapshot's
9184    /// record runs may be on the free list (freed after the snapshot, and
9185    /// quarantined while it's retained), so they come back off it, and off
9186    /// this transaction's pending frees; the current runs it doesn't share
9187    /// are freed by the restoring commit.
9188    fn reclaim_for_restore(&mut self, restored: &[CatalogEntry]) {
9189        // Record runs, deduplicated (FMT-9b.2: packed records share a page);
9190        // rows still waiting to be packed own no page yet.
9191        let runs = |cat: &[CatalogEntry]| -> std::collections::BTreeMap<u64, u64> {
9192            cat.iter()
9193                .filter(|e| !e.deleted && e.start_page != 0)
9194                .map(|e| (e.start_page, e.page_count as u64))
9195                .collect()
9196        };
9197        let live_then = runs(restored);
9198        let live_now = runs(&self.catalog);
9199        let overlaps = |p: u64, c: u64| {
9200            live_then
9201                .range(..p + c)
9202                .next_back()
9203                .is_some_and(|(&q, &n)| q + n > p)
9204        };
9205        let dying: Vec<(u64, u64)> = live_now
9206            .iter()
9207            .filter(|(p, _)| !live_then.contains_key(p))
9208            .map(|(&p, &c)| (p, c))
9209            .collect();
9210        // FMT-9b: vector pages hold many rows, so they're compared as a
9211        // set: the snapshot's come back, the current ones it lacks go.
9212        let vec_then: std::collections::BTreeSet<u64> = restored
9213            .iter()
9214            .filter(|e| !e.deleted)
9215            .filter_map(|e| e.v9.map(|r| r.vec_page))
9216            .filter(|&p| p != 0)
9217            .collect();
9218        let vec_dying: Vec<u64> = self
9219            .col
9220            .page_rows
9221            .keys()
9222            .copied()
9223            .filter(|p| !vec_then.contains(p))
9224            .collect();
9225        self.pending_free
9226            .retain(|&(p, c)| !(overlaps(p, c) || (c == 1 && vec_then.contains(&p))));
9227        for (&p, &c) in &live_then {
9228            self.free.unfree(p, c);
9229        }
9230        for &p in &vec_then {
9231            self.free.unfree(p, 1);
9232        }
9233        for (p, c) in dying {
9234            self.release_pages(p, c);
9235        }
9236        for p in vec_dying {
9237            self.release_pages(p, 1);
9238        }
9239        self.free_dirty = true;
9240    }
9241
9242    /// Restore the database to the snapshot produced by transaction `txn_id`.
9243    ///
9244    /// The restore is itself a new committed transaction (and a new
9245    /// snapshot), so it is crash-safe and reversible — restoring forward to a
9246    /// later snapshot afterwards works just as well.
9247    pub fn restore_to(&mut self, txn_id: u64) -> Result<SnapshotInfo> {
9248        if self.read_only {
9249            return Err(MnemoError::ReadOnly);
9250        }
9251        let snap = self
9252            .manifest
9253            .iter()
9254            .find(|s| s.txn_id == txn_id)
9255            .cloned()
9256            .ok_or_else(|| {
9257                let oldest = self.manifest.iter().map(|s| s.txn_id).min();
9258                match oldest {
9259                    Some(o) if self.is_v9() && txn_id < o => MnemoError::NotFound(format!(
9260                        "snapshot for transaction {txn_id}: older than the retention window \
9261                         (the newest {} snapshots and any from the last 24 hours, with at \
9262                         most a quarter of the file pinned); the oldest kept is {o}",
9263                        self.max_snapshots
9264                    )),
9265                    _ => MnemoError::NotFound(format!("snapshot for transaction {txn_id}")),
9266                }
9267            })?;
9268        self.apply_snapshot(&snap)?;
9269        Ok(SnapshotInfo {
9270            txn_id: snap.txn_id,
9271            created_at: snap.created_at,
9272            memory_count: snap.memory_count,
9273        })
9274    }
9275
9276    /// Restore the database to the latest snapshot committed at or before
9277    /// `unix_secs`. Returns [`MnemoError::NotFound`] if no snapshot is that
9278    /// old. Like [`Mnemo::restore_to`], the restore is a new transaction.
9279    pub fn restore_to_time(&mut self, unix_secs: i64) -> Result<SnapshotInfo> {
9280        if self.read_only {
9281            return Err(MnemoError::ReadOnly);
9282        }
9283        let snap = self
9284            .manifest
9285            .iter()
9286            .filter(|s| s.created_at <= unix_secs)
9287            .max_by_key(|s| s.txn_id)
9288            .cloned()
9289            .ok_or_else(|| {
9290                MnemoError::NotFound(format!("no snapshot at or before time {unix_secs}"))
9291            })?;
9292        self.apply_snapshot(&snap)?;
9293        Ok(SnapshotInfo {
9294            txn_id: snap.txn_id,
9295            created_at: snap.created_at,
9296            memory_count: snap.memory_count,
9297        })
9298    }
9299
9300    /// Return the KDF (Argon2id) parameters this file currently uses to
9301    /// wrap its DEK. Needed by callers that want to [`rekey`] while
9302    /// keeping the file's original protection level (e.g. the Python
9303    /// binding's `rekey(fast=False)`, whose docstring promises exactly
9304    /// that). FIX-1 R-001.5.
9305    pub fn kdf_params(&self) -> KdfParams {
9306        self.kdf
9307    }
9308
9309    /// Flush any pending writes, then stream the (encrypted) file to
9310    /// `dest` atomically: writes to a sibling temp file in `dest`'s
9311    /// parent directory, `sync_all`s it, renames onto `dest`, and
9312    /// fsyncs the parent directory on Unix so a crash never leaves a
9313    /// partial export.
9314    ///
9315    /// Refuses when `dest` resolves to the file this handle has open
9316    /// — that case would otherwise `File::create`-truncate the live
9317    /// database to 0 bytes (`fs::copy` on Unix reads *after* the
9318    /// `O_TRUNC`, so a naive export-to-self destroys the source).
9319    /// Unix: compares `(dev, ino)` of both paths via `MetadataExt`,
9320    /// catching symlinks and hard links. Windows: compares
9321    /// canonicalized paths; a rename onto an open file also fails on
9322    /// Windows because of the mandatory lock the handle holds, so the
9323    /// rename step is a second line of defense.
9324    ///
9325    /// Reads the bytes from the already-open file handle, not by
9326    /// `fs::copy`. On Windows that matters: `LockFileEx` locks are
9327    /// mandatory, so a second handle opened by `fs::copy` would fail
9328    /// while this writer still holds the lock.
9329    ///
9330    /// FIX-1 R-001.5 + R-002.2 part 3.
9331    pub fn export_encrypted(&mut self, dest: impl AsRef<Path>) -> Result<()> {
9332        let dest = dest.as_ref();
9333
9334        // Step 1 — refuse export to the open file itself. We compare
9335        // by identity, not string, so symlinks and hard links to
9336        // `self.path` are caught too. The check runs before any flush
9337        // so a self-targeted export is a pure no-op.
9338        if paths_refer_to_same_file(&self.path, dest).unwrap_or(false) {
9339            return Err(MnemoError::Invalid(format!(
9340                "export_encrypted refuses to overwrite the open database at {}",
9341                dest.display()
9342            )));
9343        }
9344
9345        // Step 2 — flush, so on-disk bytes match the live state.
9346        if !self.read_only {
9347            self.flush()?;
9348        }
9349
9350        // Step 3 — stream from the open handle into a sibling temp.
9351        // Using the open handle (instead of a second open on `self.path`)
9352        // means a Windows mandatory lock can't block the copy.
9353        let parent = dest.parent().ok_or_else(|| {
9354            MnemoError::Invalid(format!(
9355                "export_encrypted: dest {} has no parent directory",
9356                dest.display()
9357            ))
9358        })?;
9359        let tmp = temp_sibling(dest);
9360        // Scope the temp handle so it closes before the rename on
9361        // Windows (which refuses to rename over an open file).
9362        {
9363            let src = self.pager.file_mut();
9364            src.seek(SeekFrom::Start(0))?;
9365            let mut out = OpenOptions::new()
9366                .write(true)
9367                .create(true)
9368                .truncate(true)
9369                .open(&tmp)?;
9370            let mut buf = vec![0u8; 1 << 20]; // 1 MiB
9371            loop {
9372                let n = src.read(&mut buf)?;
9373                if n == 0 {
9374                    break;
9375                }
9376                out.write_all(&buf[..n])?;
9377            }
9378            out.sync_all()?;
9379            crate::bump_fsync();
9380        }
9381
9382        // Step 4 — atomic rename onto `dest`.
9383        if let Err(e) = std::fs::rename(&tmp, dest) {
9384            // Best-effort cleanup; propagate the rename error.
9385            let _ = std::fs::remove_file(&tmp);
9386            return Err(MnemoError::Io(e));
9387        }
9388
9389        // Step 5 — fsync the parent directory on Unix so the rename
9390        // itself is durable. On Windows directory syncs aren't
9391        // supported; the rename above is atomic enough on NTFS.
9392        #[cfg(unix)]
9393        {
9394            if let Ok(dir) = File::open(parent) {
9395                let _ = dir.sync_all();
9396                crate::bump_fsync();
9397            }
9398        }
9399        #[cfg(not(unix))]
9400        {
9401            let _ = parent;
9402        }
9403
9404        Ok(())
9405    }
9406
9407    /// Change the passphrase. Cheap: re-derives the KEK and re-wraps the DEK;
9408    /// the encrypted pages are never rewritten.
9409    pub fn rekey(&mut self, new_passphrase: &str, kdf: KdfParams) -> Result<()> {
9410        self.rekey_secret(new_passphrase.as_bytes(), kdf)
9411    }
9412
9413    /// PERF-1 step 7: switch to (or rotate) a raw 256-bit key, recording
9414    /// [`KdfParams::raw_key`] in the header. Like [`Mnemo::rekey`], only the
9415    /// wrapped DEK changes; pages aren't rewritten.
9416    pub fn rekey_with_key(&mut self, new_key: &[u8; crypto::KEY_LEN]) -> Result<()> {
9417        self.rekey_secret(new_key, KdfParams::raw_key())
9418    }
9419
9420    fn rekey_secret(&mut self, secret: &[u8], kdf: KdfParams) -> Result<()> {
9421        if self.read_only {
9422            return Err(MnemoError::ReadOnly);
9423        }
9424        self.flush()?;
9425        let new_salt = crypto::random_salt();
9426        let new_kek = crypto::derive_kek(secret, &new_salt, kdf)?;
9427        let new_nonce = crypto::random_nonce();
9428        let wrapped = crypto::wrap_dek(&new_kek, &new_nonce, self.pager.dek())?;
9429        if wrapped.len() != WRAPPED_DEK_LEN {
9430            return Err(MnemoError::Crypto("unexpected wrapped DEK length".into()));
9431        }
9432        let mut wrapped_dek = [0u8; WRAPPED_DEK_LEN];
9433        wrapped_dek.copy_from_slice(&wrapped);
9434
9435        self.header.salt = new_salt;
9436        self.header.dek_nonce = new_nonce;
9437        self.header.wrapped_dek = wrapped_dek;
9438        self.header.m_cost = kdf.m_cost;
9439        self.header.t_cost = kdf.t_cost;
9440        self.header.p_cost = kdf.p_cost;
9441        self.kdf = kdf;
9442
9443        // DUR-1 (R-002.1): WAL-log the new page 0 before touching it.
9444        // Allocate a fresh WAL region at the tail so that a crash during
9445        // the rekey leaves either the old region (old passphrase works)
9446        // or the new region (new passphrase works) fully committed —
9447        // overwriting the existing region would risk destroying both.
9448        let old_wal_start = self.header.wal_start;
9449        let new_pages = self.header.wal_pages;
9450        let new_start = self.header.next_page;
9451        self.header.next_page += new_pages;
9452        self.header.wal_start = new_start;
9453        self.header.wal_pages = new_pages;
9454        self.header.wal_seq += 1;
9455        self.relocate_header_through_wal(old_wal_start)?;
9456        // FMT-9 (a): the old region is freed by the next commit (see
9457        // `ensure_wal_capacity`).
9458        self.release_pages(old_wal_start, new_pages);
9459        Ok(())
9460    }
9461
9462    /// Decrypt and validate every live record. Returns the count verified
9463    /// across memories **and** cache entries (FIX-1 R-001.1: now that the
9464    /// cache survives compaction, `verify()` has to walk it too — the
9465    /// pre-FIX-1 implementation only checked `self.catalog`, so a
9466    /// corrupted cache record went unnoticed).
9467    pub fn verify(&mut self) -> Result<usize> {
9468        let entries: Vec<CatalogEntry> = self
9469            .catalog
9470            .iter()
9471            .filter(|e| !e.deleted)
9472            .cloned()
9473            .collect();
9474        let mut n = entries.len();
9475        for e in &entries {
9476            let bytes = self.read_record(e)?;
9477            let _m: Memory = self.decode_memory(&bytes)?;
9478            // FMT-9b: and the row's vector column entry.
9479            if e.v9.is_some() {
9480                let (h, _) = self.column_entry(e)?;
9481                if h.len() != self.dimensions {
9482                    return Err(MnemoError::Invalid(
9483                        "vector entry has the wrong width".into(),
9484                    ));
9485                }
9486            }
9487        }
9488
9489        // Also decode every live cache entry. Tombstoned entries are
9490        // skipped — their pages are orphan ciphertext awaiting
9491        // `compact_file`, same as deleted memories. Expired entries
9492        // are *still* on disk until compaction, so we verify them too.
9493        let cache_positions: Vec<(u64, u32, u32, u64)> = self
9494            .cache_dir
9495            .iter()
9496            .filter(|e| !e.deleted)
9497            .map(|e| (e.start_page, e.page_count, e.len, e.gen))
9498            .collect();
9499        for (start_page, page_count, len, gen) in cache_positions {
9500            let buf = self.read_record_run(start_page, page_count, len, gen)?;
9501            let _c: CacheEntry = rmp_serde::from_slice(&buf)
9502                .map_err(|err| MnemoError::Serialize(err.to_string()))?;
9503            n += 1;
9504        }
9505        // FMT-9 (a): no free run may overlap a page the committed state
9506        // references (records, cache entries, control runs, the WAL).
9507        if self.is_v9() {
9508            self.check_free_list_disjoint()?;
9509        }
9510        Ok(n)
9511    }
9512
9513    /// FMT-9 (a): fail if the free list (or a pending free) overlaps any
9514    /// page the current state references.
9515    fn check_free_list_disjoint(&self) -> Result<()> {
9516        let h = &self.header;
9517        let mut live: Vec<(u64, u64, &str)> = vec![
9518            (h.wal_start, h.wal_pages, "WAL region"),
9519            (h.catalog_start, h.catalog_pages, "catalog run"),
9520            (h.index_start, h.index_pages, "index run"),
9521            (h.manifest_start, h.manifest_pages, "manifest run"),
9522            (h.cache_start, h.cache_pages, "cache directory run"),
9523            (h.free.start, h.free.pages, "free-list run"),
9524            (h.keys.start, h.keys.pages, "key-slot run"),
9525        ];
9526        live.extend(
9527            self.catalog
9528                .iter()
9529                .filter(|e| !e.deleted)
9530                .map(|e| (e.start_page, e.page_count as u64, "record")),
9531        );
9532        live.extend(
9533            self.cache_dir
9534                .iter()
9535                .filter(|e| !e.deleted)
9536                .map(|e| (e.start_page, e.page_count as u64, "cache entry")),
9537        );
9538        live.extend(self.col.page_rows.keys().map(|&p| (p, 1, "vector page")));
9539        live.extend(
9540            self.delta_chain
9541                .iter()
9542                .map(|r| (r.start, r.pages, "catalog delta run")),
9543        );
9544        live.extend(
9545            self.index_chain
9546                .iter()
9547                .map(|r| (r.start, r.pages, "index delta run")),
9548        );
9549        live.push((
9550            self.header.bm25.start,
9551            self.header.bm25.pages,
9552            "BM25 manifest",
9553        ));
9554        live.extend(
9555            self.bm25
9556                .manifest
9557                .segments
9558                .iter()
9559                .map(|t| (t.0, t.1, "BM25 segment")),
9560        );
9561        for (p, c, what) in live {
9562            if c == 0 {
9563                continue;
9564            }
9565            if self.free.overlaps(p, c) {
9566                return Err(MnemoError::Invalid(format!(
9567                    "free list overlaps a live {what} at page {p} (+{c})"
9568                )));
9569            }
9570            if self
9571                .pending_free
9572                .iter()
9573                .any(|&(q, n)| q < p + c && p < q + n)
9574            {
9575                return Err(MnemoError::Invalid(format!(
9576                    "a pending free overlaps a live {what} at page {p} (+{c})"
9577                )));
9578            }
9579        }
9580        Ok(())
9581    }
9582
9583    /// Summary statistics for the open database.
9584    pub fn stats(&mut self) -> Result<Stats> {
9585        let deleted = self.catalog.iter().filter(|e| e.deleted).count();
9586        // FIX-1 R-001.8 part 2: build the agent set lazily on first
9587        // call per handle, then maintain in `put` / invalidate in
9588        // `delete`. Pre-FIX-1 every `stats()` call decrypted every
9589        // live memory (O(N) AES-GCM decrypts just to list a usually-
9590        // tiny set of distinct agent IDs). On a 1M-memory database
9591        // this was in the multi-second range on hardware AES.
9592        if self.agent_set.is_none() {
9593            let built: std::collections::BTreeSet<String> =
9594                self.memories()?.into_iter().map(|m| m.agent_id).collect();
9595            self.agent_set = Some(built);
9596        }
9597        let agents: Vec<String> = self
9598            .agent_set
9599            .as_ref()
9600            .expect("agent_set just filled above")
9601            .iter()
9602            .cloned()
9603            .collect();
9604        let file_bytes = self.header.next_page * PAGE_SIZE as u64;
9605        Ok(Stats {
9606            memories: self.len(),
9607            deleted,
9608            dimensions: self.dimensions,
9609            file_bytes,
9610            agents,
9611            encrypted: self.header.flags & FLAG_ENCRYPTED != 0,
9612            created_at: self.header.created_at,
9613            index: {
9614                self.ensure_ann()?;
9615                self.ann.as_ref().map(|a| a.info())
9616            },
9617            wal_pages: self.header.wal_pages,
9618            fsyncs_last_flush: self.fsyncs_last_flush,
9619            format_version: self.header.version,
9620        })
9621    }
9622
9623    /// Rewrite the file, dropping tombstoned and expired memories and
9624    /// reclaiming stale pages left by updates. Done out-of-place via a temp
9625    /// file and an atomic rename.
9626    ///
9627    /// FIX-1 R-001.1 + R-002.2 part 2: the result cache now survives
9628    /// compaction. Every live, non-expired directory entry is copied
9629    /// into the new file verbatim — `created_at`, `ttl_secs`,
9630    /// `cost_hint_ms`, `key_preview`, `size_bytes`, `content_type`,
9631    /// `vector` and `model` come from the record body; `accessed_at`
9632    /// and `access_count` come from the *directory* (the record body
9633    /// only holds the insert-time snapshot — the directory is the
9634    /// source of truth for the live counts, as of v5's recall trick).
9635    /// Tombstoned and TTL-expired entries are dropped. Pre-FIX-1
9636    /// compaction kept only the memory catalog and silently emptied
9637    /// the cache; the Planner's interim README caveat (c417d2e) called
9638    /// that out — this PR removes the surprise.
9639    pub fn compact_file(path: &str, passphrase: &str) -> Result<CompactReport> {
9640        Self::compact_file_secret(path, passphrase.as_bytes(), 0)
9641    }
9642
9643    /// PERF-1 step 7: [`Mnemo::compact_file`] for a raw-key database.
9644    pub fn compact_file_with_key(path: &str, key: &[u8; crypto::KEY_LEN]) -> Result<CompactReport> {
9645        Self::compact_file_secret(path, key, 0)
9646    }
9647
9648    /// FMT-9f: [`Mnemo::compact_file`] into an explicit format: `9` (what
9649    /// `compact_file` does) or `8`, which keeps a v8 file readable by
9650    /// v0.10 and earlier. The v8 target is kept for one release.
9651    pub fn compact_file_to(
9652        path: &str,
9653        passphrase: &str,
9654        format_version: u16,
9655    ) -> Result<CompactReport> {
9656        if format_version != VERSION && format_version != VERSION_V9 {
9657            return Err(MnemoError::Invalid(format!(
9658                "compact target format {format_version} isn't supported (use 8 or 9)"
9659            )));
9660        }
9661        Self::compact_file_secret(path, passphrase.as_bytes(), format_version)
9662    }
9663
9664    fn compact_file_secret(
9665        path: &str,
9666        secret: &[u8],
9667        format_version: u16,
9668    ) -> Result<CompactReport> {
9669        let mut old = Mnemo::open_secret(path, secret)?;
9670        let dims = old.dimensions;
9671        let kdf = old.kdf;
9672        let tmp = format!("{path}.compact-tmp");
9673
9674        let mut new = Mnemo::create_secret(
9675            &tmp,
9676            secret,
9677            MnemoConfig {
9678                dimensions: dims,
9679                kdf,
9680                // FMT-9f: compaction writes v9 (upgrading a v8 file),
9681                // unless `compact_file_to` asked for v8. A v9 file that
9682                // keeps float vectors keeps them.
9683                format_version,
9684                store_float_vectors: old.float_vectors(),
9685                ..Default::default()
9686            },
9687        )?;
9688        // FMT-9e: compaction rewrites everything, so it compresses every
9689        // thread turn, recent or not.
9690        new.turn_compress_after = 0;
9691        // ANN-1.1: compaction is "re-tune for the current size" by
9692        // default. Pass IndexConfig::default() so partitions,
9693        // subspaces, n_probe and n_rerank are all re-derived from the
9694        // compacted vector count. Previously we froze the pre-compact
9695        // (n_probe, n_rerank), which left the new index on stale
9696        // numbers for the smaller post-compact corpus.
9697        let want_index = old.has_index();
9698        let now = memory::now_secs();
9699        let all = old.memories()?;
9700        let before = all.len();
9701        let mut after = 0;
9702        for m in all {
9703            if m.is_expired(now) {
9704                continue;
9705            }
9706            new.remember(m)?;
9707            after += 1;
9708        }
9709        if want_index && after > 0 {
9710            new.build_index_with(IndexConfig::default())?;
9711        }
9712
9713        // Count + copy cache entries. We walk `old.cache_dir` by index
9714        // so each hop reads the directory's live `accessed_at` /
9715        // `access_count` directly rather than going through
9716        // `cache_get` (which would also treat read-only semantics,
9717        // tombstones, etc. — we want the raw, authoritative values).
9718        let mut cache_before = 0usize;
9719        let mut cache_after = 0usize;
9720        let positions: Vec<usize> = (0..old.cache_dir.len()).collect();
9721        for pos in positions {
9722            let keep = {
9723                let e = &old.cache_dir[pos];
9724                !e.deleted && !e.is_expired(now)
9725            };
9726            if keep {
9727                cache_before += 1;
9728                new.copy_cache_entry_from(&mut old, pos)?;
9729                cache_after += 1;
9730            }
9731        }
9732
9733        new.flush()?;
9734        drop(new);
9735        drop(old);
9736
9737        std::fs::rename(&tmp, path)?;
9738        Ok(CompactReport {
9739            before,
9740            after,
9741            cache_before,
9742            cache_after,
9743        })
9744    }
9745
9746    /// Copy one live cache entry from `old` at directory position
9747    /// `pos` into `self` (a freshly-created compaction target).
9748    /// Decrypts the old record body, substitutes the old *directory*'s
9749    /// `accessed_at` + `access_count` (the record body's copies are
9750    /// stale as of v5's directory-only hit path), re-encodes, writes
9751    /// a fresh record run, and pushes a matching directory entry.
9752    ///
9753    /// Caller is expected to have already filtered out tombstoned /
9754    /// expired entries and to call this on a read-write handle.
9755    pub(crate) fn copy_cache_entry_from(&mut self, old: &mut Mnemo, pos: usize) -> Result<()> {
9756        if self.read_only {
9757            return Err(MnemoError::ReadOnly);
9758        }
9759        // Snapshot the directory fields we need, so the borrow on
9760        // `old.cache_dir` ends before we touch `old.pager`.
9761        let (start_page, page_count, len, dir_accessed_at, dir_access_count, old_gen) = {
9762            let e = &old.cache_dir[pos];
9763            (
9764                e.start_page,
9765                e.page_count,
9766                e.len,
9767                e.accessed_at,
9768                e.access_count,
9769                e.gen,
9770            )
9771        };
9772
9773        // Read + decrypt the old record body.
9774        let buf = old.read_record_run(start_page, page_count, len, old_gen)?;
9775        let mut entry: CacheEntry =
9776            rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
9777
9778        // Overwrite the stale body copies with the directory's live
9779        // values before persisting. Everything else — `created_at`,
9780        // `ttl_secs`, `cost_hint_ms`, `key_preview`, `size_bytes`,
9781        // `content_type`, `value`, `vector`, `model` — stays as the
9782        // body already recorded it.
9783        entry.accessed_at = dir_accessed_at;
9784        entry.access_count = dir_access_count;
9785
9786        let bytes =
9787            rmp_serde::to_vec(&entry).map_err(|err| MnemoError::Serialize(err.to_string()))?;
9788        let (new_start_page, new_page_count, gen) = self.write_record(&bytes)?;
9789
9790        let dir_entry = CacheDirectoryEntry {
9791            namespace: entry.namespace.clone(),
9792            key_hash: entry.key_hash,
9793            start_page: new_start_page,
9794            page_count: new_page_count,
9795            len: bytes.len() as u32,
9796            deleted: false,
9797            accessed_at: dir_accessed_at,
9798            access_count: dir_access_count,
9799            size_bytes: entry.size_bytes,
9800            ttl_secs: entry.ttl_secs,
9801            created_at: entry.created_at,
9802            vector: entry.vector.clone(),
9803            model: entry.model.clone(),
9804            gen,
9805        };
9806        let new_pos = self.cache_dir.len();
9807        self.cache_idx
9808            .insert(&entry.namespace, entry.key_hash, new_pos);
9809        self.cache_dir.push(dir_entry);
9810        self.dirty_cache = true;
9811        Ok(())
9812    }
9813}