Skip to main content

mnemo_engine/store/
maintenance.rs

1//! Maintenance: export, rekey, verify, stats, and compaction (which also upgrades v8 files to v9).
2
3use super::*;
4
5impl Mnemo {
6    /// Return the KDF (Argon2id) parameters this file currently uses to
7    /// wrap its DEK. Needed by callers that want to [`rekey`] while
8    /// keeping the file's original protection level (e.g. the Python
9    /// binding's `rekey(fast=False)`, whose docstring promises exactly
10    /// that). FIX-1 R-001.5.
11    pub fn kdf_params(&self) -> KdfParams {
12        self.kdf
13    }
14
15    /// Flush any pending writes, then stream the (encrypted) file to
16    /// `dest` atomically: writes to a sibling temp file in `dest`'s
17    /// parent directory, `sync_all`s it, renames onto `dest`, and
18    /// fsyncs the parent directory on Unix so a crash never leaves a
19    /// partial export.
20    ///
21    /// Refuses when `dest` resolves to the file this handle has open
22    /// — that case would otherwise `File::create`-truncate the live
23    /// database to 0 bytes (`fs::copy` on Unix reads *after* the
24    /// `O_TRUNC`, so a naive export-to-self destroys the source).
25    /// Unix: compares `(dev, ino)` of both paths via `MetadataExt`,
26    /// catching symlinks and hard links. Windows: compares
27    /// canonicalized paths; a rename onto an open file also fails on
28    /// Windows because of the mandatory lock the handle holds, so the
29    /// rename step is a second line of defense.
30    ///
31    /// Reads the bytes from the already-open file handle, not by
32    /// `fs::copy`. On Windows that matters: `LockFileEx` locks are
33    /// mandatory, so a second handle opened by `fs::copy` would fail
34    /// while this writer still holds the lock.
35    ///
36    /// FIX-1 R-001.5 + R-002.2 part 3.
37    pub fn export_encrypted(&mut self, dest: impl AsRef<Path>) -> Result<()> {
38        let dest = dest.as_ref();
39
40        // Step 1 — refuse export to the open file itself. We compare
41        // by identity, not string, so symlinks and hard links to
42        // `self.path` are caught too. The check runs before any flush
43        // so a self-targeted export is a pure no-op.
44        if paths_refer_to_same_file(&self.path, dest).unwrap_or(false) {
45            return Err(MnemoError::Invalid(format!(
46                "export_encrypted refuses to overwrite the open database at {}",
47                dest.display()
48            )));
49        }
50
51        // Step 2 — flush, so on-disk bytes match the live state.
52        if !self.read_only {
53            self.flush()?;
54        }
55
56        // Step 3 — stream from the open handle into a sibling temp.
57        // Using the open handle (instead of a second open on `self.path`)
58        // means a Windows mandatory lock can't block the copy.
59        let parent = dest.parent().ok_or_else(|| {
60            MnemoError::Invalid(format!(
61                "export_encrypted: dest {} has no parent directory",
62                dest.display()
63            ))
64        })?;
65        let tmp = temp_sibling(dest);
66        // Scope the temp handle so it closes before the rename on
67        // Windows (which refuses to rename over an open file).
68        {
69            let src = self.pager.file_mut();
70            src.seek(SeekFrom::Start(0))?;
71            let mut out = OpenOptions::new()
72                .write(true)
73                .create(true)
74                .truncate(true)
75                .open(&tmp)?;
76            let mut buf = vec![0u8; 1 << 20]; // 1 MiB
77            loop {
78                let n = src.read(&mut buf)?;
79                if n == 0 {
80                    break;
81                }
82                out.write_all(&buf[..n])?;
83            }
84            out.sync_all()?;
85            crate::bump_fsync();
86        }
87
88        // Step 4 — atomic rename onto `dest`.
89        if let Err(e) = std::fs::rename(&tmp, dest) {
90            // Best-effort cleanup; propagate the rename error.
91            let _ = std::fs::remove_file(&tmp);
92            return Err(MnemoError::Io(e));
93        }
94
95        // Step 5 — fsync the parent directory on Unix so the rename
96        // itself is durable. On Windows directory syncs aren't
97        // supported; the rename above is atomic enough on NTFS.
98        #[cfg(unix)]
99        {
100            if let Ok(dir) = File::open(parent) {
101                let _ = dir.sync_all();
102                crate::bump_fsync();
103            }
104        }
105        #[cfg(not(unix))]
106        {
107            let _ = parent;
108        }
109
110        Ok(())
111    }
112
113    /// Change the passphrase. Cheap: re-derives the KEK and re-wraps the DEK;
114    /// the encrypted pages are never rewritten.
115    ///
116    /// **This changes the secret that opens this file from now on; anyone who
117    /// had the file and the old secret can still read it and later writes.**
118    /// The data key stays the same, so an earlier copy of the file (or of
119    /// just its header) and the old passphrase still unlock it. The old wrap
120    /// is scrubbed from this file (the previous WAL region is zeroed, and any
121    /// stale copy of the old header's key wrap is overwritten); that's
122    /// hygiene, not revocation. **To revoke, rotate:** rekey, then
123    /// [`Mnemo::compact_file`] (`mnemo rekey --rotate`), which rewrites the
124    /// file under a fresh data key. Rotation can't erase old ciphertext the storage hardware still holds: flash keeps rewritten blocks until it reuses them, and that's out of the engine's hands.
125    ///
126    /// **If it returns an error, the file may already be under the new secret,
127    /// or may move to it at the next write** (RB-004.2). Close the handle
128    /// without writing anything more, then try both secrets.
129    pub fn rekey(&mut self, new_passphrase: &str, kdf: KdfParams) -> Result<()> {
130        self.rekey_secret(new_passphrase.as_bytes(), kdf)
131    }
132
133    /// PERF-1 step 7: switch to (or rotate) a raw 256-bit key, recording
134    /// [`KdfParams::raw_key`] in the header. Like [`Mnemo::rekey`], only the
135    /// wrapped DEK changes; pages aren't rewritten, and it doesn't revoke
136    /// the old key (see [`Mnemo::rekey`]).
137    ///
138    /// **If it returns an error, the file may already be under the new secret,
139    /// or may move to it at the next write** (RB-004.2). Close the handle
140    /// without writing anything more, then try both secrets.
141    pub fn rekey_with_key(&mut self, new_key: &[u8; crypto::KEY_LEN]) -> Result<()> {
142        self.rekey_secret(new_key, KdfParams::raw_key())
143    }
144
145    pub(super) fn rekey_secret(&mut self, secret: &[u8], kdf: KdfParams) -> Result<()> {
146        if self.read_only {
147            return Err(MnemoError::ReadOnly);
148        }
149        self.flush()?;
150        let new_salt = crypto::random_salt();
151        let new_kek = crypto::derive_kek(secret, &new_salt, kdf)?;
152        let new_nonce = crypto::random_nonce();
153        let wrapped = crypto::wrap_dek(&new_kek, &new_nonce, self.pager.dek())?;
154        if wrapped.len() != WRAPPED_DEK_LEN {
155            return Err(MnemoError::Crypto("unexpected wrapped DEK length".into()));
156        }
157        let mut wrapped_dek = [0u8; WRAPPED_DEK_LEN];
158        wrapped_dek.copy_from_slice(&wrapped);
159
160        // SEC-2: the old header's key wrap (salt, DEK nonce, wrapped DEK, as
161        // laid out at header bytes 86..162), to scrub once the new header
162        // is committed.
163        let mut old_wrap =
164            Vec::with_capacity(crypto::SALT_LEN + crypto::NONCE_LEN + WRAPPED_DEK_LEN);
165        old_wrap.extend_from_slice(&self.header.salt);
166        old_wrap.extend_from_slice(&self.header.dek_nonce);
167        old_wrap.extend_from_slice(&self.header.wrapped_dek);
168
169        self.header.salt = new_salt;
170        self.header.dek_nonce = new_nonce;
171        self.header.wrapped_dek = wrapped_dek;
172        self.header.m_cost = kdf.m_cost;
173        self.header.t_cost = kdf.t_cost;
174        self.header.p_cost = kdf.p_cost;
175        self.kdf = kdf;
176
177        // DUR-1 (R-002.1): WAL-log the new page 0 before touching it.
178        // Allocate a fresh WAL region at the tail so that a crash during
179        // the rekey leaves either the old region (old passphrase works)
180        // or the new region (new passphrase works) fully committed —
181        // overwriting the existing region would risk destroying both.
182        let old_wal_start = self.header.wal_start;
183        let new_pages = self.header.wal_pages;
184        let new_start = self.header.next_page;
185        self.header.next_page += new_pages;
186        self.header.wal_start = new_start;
187        self.header.wal_pages = new_pages;
188        self.header.wal_seq += 1;
189        self.relocate_header_through_wal(old_wal_start)?;
190        // SEC-2 (RB-001.30): the old region holds page-0 images with the old
191        // wrap; `relocate_header_through_wal` only zeroed its first frame
192        // header. Zero all of it, then overwrite any other stale copy of
193        // the old wrap in the file (an earlier, grown or relocated region).
194        // Hygiene only: a copy of the file taken before the rekey still
195        // opens with the old secret (see `rekey`).
196        wal::scrub_region(self.pager.file_mut(), old_wal_start, new_pages)?;
197        scrub_bytes(self.pager.file_mut(), &old_wrap)?;
198        // FMT-9 (a): the old region is freed by the next commit (see
199        // `ensure_wal_capacity`).
200        self.release_pages(old_wal_start, new_pages);
201        Ok(())
202    }
203
204    /// SEC-2 (RB-001.30): write a copy of this database at `dest` under a
205    /// **fresh data key**, protected by `passphrase` through `kdf`. It goes
206    /// through the compaction path (live memories, the result cache and
207    /// the index, no history), so the copy shares no key material with
208    /// this file: neither this file's passphrase nor its data key opens
209    /// it, and its passphrase opens nothing here. This file is flushed
210    /// first (unless read-only) and is otherwise unchanged; `dest` must not
211    /// exist. The copy is v9.
212    pub fn export_rekeyed(
213        &mut self,
214        dest: impl AsRef<Path>,
215        passphrase: &str,
216        kdf: KdfParams,
217    ) -> Result<CompactReport> {
218        self.export_rekeyed_secret(dest.as_ref(), passphrase.as_bytes(), kdf)
219    }
220
221    /// [`Mnemo::export_rekeyed`] protected by a raw 256-bit key (no KDF).
222    pub fn export_rekeyed_with_key(
223        &mut self,
224        dest: impl AsRef<Path>,
225        key: &[u8; crypto::KEY_LEN],
226    ) -> Result<CompactReport> {
227        self.export_rekeyed_secret(dest.as_ref(), key, KdfParams::raw_key())
228    }
229
230    pub(super) fn export_rekeyed_secret(
231        &mut self,
232        dest: &Path,
233        secret: &[u8],
234        kdf: KdfParams,
235    ) -> Result<CompactReport> {
236        if paths_refer_to_same_file(&self.path, dest).unwrap_or(false) {
237            return Err(MnemoError::Invalid(format!(
238                "export_rekeyed refuses to overwrite the open database at {}",
239                dest.display()
240            )));
241        }
242        if dest.exists() {
243            return Err(MnemoError::Invalid(format!(
244                "{} already exists",
245                dest.display()
246            )));
247        }
248        if !self.read_only {
249            self.flush()?;
250        }
251        let mut name = dest.file_name().unwrap_or_default().to_os_string();
252        name.push(".export-tmp");
253        let tmp = dest.with_file_name(name);
254        let _ = std::fs::remove_file(&tmp);
255        let result = (|| {
256            let mut new = Mnemo::create_secret(
257                &tmp,
258                secret,
259                MnemoConfig {
260                    dimensions: self.dimensions,
261                    kdf,
262                    store_float_vectors: self.float_vectors(),
263                    ..Default::default()
264                },
265            )?;
266            let report = self.copy_live_into(&mut new)?;
267            new.flush()?;
268            drop(new);
269            std::fs::rename(&tmp, dest)?;
270            Ok(report)
271        })();
272        if result.is_err() {
273            let _ = std::fs::remove_file(&tmp);
274        }
275        result
276    }
277
278    /// The compaction copy: every live, unexpired memory and cache entry of
279    /// `self` into `new` (a freshly created file), and the index if `self`
280    /// has one. Counts as [`CompactReport`].
281    pub(super) fn copy_live_into(&mut self, new: &mut Mnemo) -> Result<CompactReport> {
282        // FMT-9e: compaction rewrites everything, so it compresses every
283        // thread turn, recent or not.
284        new.turn_compress_after = 0;
285        // ANN-1.1: compaction is "re-tune for the current size" by
286        // default. Pass IndexConfig::default() so partitions,
287        // subspaces, n_probe and n_rerank are all re-derived from the
288        // compacted vector count. Previously we froze the pre-compact
289        // (n_probe, n_rerank), which left the new index on stale
290        // numbers for the smaller post-compact corpus.
291        let want_index = self.has_index();
292        let now = memory::now_secs();
293        let all = self.memories()?;
294        let before = all.len();
295        let mut after = 0;
296        let mut zero_vector_manifests = 0;
297        let mut zero_vector_memories = 0;
298        for mut m in all {
299            if m.is_expired(now) {
300                continue;
301            }
302            // A file from before the zero-vector rule (Phase 3 PR 10a) can
303            // hold all-zero vectors. A legacy onboarding or manifest memory
304            // gets the sentinel unit vector a current manifest has, so it
305            // behaves like a new one; any other is carried unchanged.
306            if embed::is_zero_vector(&m.vector) {
307                if crud::is_onboarding(&m) || is_manifest(&m) {
308                    m.vector = Memory::scaffold_manifest(self.dimensions).vector;
309                    zero_vector_manifests += 1;
310                } else {
311                    zero_vector_memories += 1;
312                }
313            }
314            // The engine's own copy: `_mnemo.*` keys come through as they
315            // are, and the write-time zero-vector check doesn't apply.
316            new.put_existing(m)?;
317            after += 1;
318        }
319        if want_index && after > 0 {
320            new.build_index_with(IndexConfig::default())?;
321        }
322
323        // Count + copy cache entries. We walk `self.cache_dir` by index
324        // so each hop reads the directory's live `accessed_at` /
325        // `access_count` directly rather than going through
326        // `cache_get` (which would also treat read-only semantics,
327        // tombstones, etc. — we want the raw, authoritative values).
328        let mut cache_before = 0usize;
329        let mut cache_after = 0usize;
330        let positions: Vec<usize> = (0..self.cache_dir.len()).collect();
331        for pos in positions {
332            let (keep, reserved) = {
333                let e = &self.cache_dir[pos];
334                (
335                    !e.deleted && !e.is_expired(now),
336                    embed::is_reserved_namespace(&e.namespace),
337                )
338            };
339            if keep {
340                // EMBED-ID-1: the engine's own entries (the embedder
341                // identity) are copied but not counted.
342                new.copy_cache_entry_from(self, pos)?;
343                if !reserved {
344                    cache_before += 1;
345                    cache_after += 1;
346                }
347            }
348        }
349        Ok(CompactReport {
350            before,
351            after,
352            cache_before,
353            cache_after,
354            zero_vector_manifests,
355            zero_vector_memories,
356        })
357    }
358
359    /// Decrypt and validate every live record. Returns the count verified
360    /// across memories **and** cache entries (FIX-1 R-001.1: now that the
361    /// cache survives compaction, `verify()` has to walk it too — the
362    /// pre-FIX-1 implementation only checked `self.catalog`, so a
363    /// corrupted cache record went unnoticed).
364    pub fn verify(&mut self) -> Result<usize> {
365        Ok(self.verify_report()?.records)
366    }
367
368    /// [`Mnemo::verify`], also counting the memories whose vector is all
369    /// zeros. A file written before the zero-vector rule (Phase 3 PR 10a)
370    /// can hold them; they decode fine but score 0.0 on every vector query.
371    /// `mnemo verify` warns about them, and compaction carries them
372    /// (giving a legacy manifest the sentinel vector).
373    pub fn verify_report(&mut self) -> Result<VerifyReport> {
374        let entries: Vec<CatalogEntry> = self
375            .catalog
376            .iter()
377            .filter(|e| !e.deleted)
378            .cloned()
379            .collect();
380        let mut n = entries.len();
381        let mut zero_vector_memories = 0;
382        for e in &entries {
383            let bytes = self.read_record(e)?;
384            let m: Memory = self.decode_memory(&bytes)?;
385            // FMT-9b: and the row's vector column entry.
386            let zero = if e.v9.is_some() {
387                let (h, _) = self.column_entry(e)?;
388                if h.len() != self.dimensions {
389                    return Err(MnemoError::Invalid(
390                        "vector entry has the wrong width".into(),
391                    ));
392                }
393                // fp16 +0.0 and -0.0.
394                !h.is_empty() && h.iter().all(|b| b & 0x7fff == 0)
395            } else {
396                embed::is_zero_vector(&m.vector)
397            };
398            if zero {
399                zero_vector_memories += 1;
400            }
401        }
402
403        // Also decode every live cache entry. Tombstoned entries are
404        // skipped — their pages are orphan ciphertext awaiting
405        // `compact_file`, same as deleted memories. Expired entries
406        // are *still* on disk until compaction, so we verify them too.
407        let cache_positions: Vec<(u64, u32, u32, u64, bool)> = self
408            .cache_dir
409            .iter()
410            .filter(|e| !e.deleted)
411            .map(|e| {
412                let reserved = embed::is_reserved_namespace(&e.namespace);
413                (e.start_page, e.page_count, e.len, e.gen, reserved)
414            })
415            .collect();
416        for (start_page, page_count, len, gen, reserved) in cache_positions {
417            let buf = self.read_record_run(start_page, page_count, len, gen)?;
418            let _c: CacheEntry = rmp_serde::from_slice(&buf)
419                .map_err(|err| MnemoError::Serialize(err.to_string()))?;
420            // EMBED-ID-1: the engine's own entries are checked, not counted.
421            if !reserved {
422                n += 1;
423            }
424        }
425        // FMT-9 (a): no free run may overlap a page the committed state
426        // references (records, cache entries, control runs, the WAL).
427        if self.is_v9() {
428            self.check_free_list_disjoint()?;
429        }
430        Ok(VerifyReport {
431            records: n,
432            zero_vector_memories,
433        })
434    }
435
436    /// FMT-9 (a): fail if the free list (or a pending free) overlaps any
437    /// page the current state references.
438    pub(super) fn check_free_list_disjoint(&self) -> Result<()> {
439        let h = &self.header;
440        let mut live: Vec<(u64, u64, &str)> = vec![
441            (h.wal_start, h.wal_pages, "WAL region"),
442            (h.catalog_start, h.catalog_pages, "catalog run"),
443            (h.index_start, h.index_pages, "index run"),
444            (h.manifest_start, h.manifest_pages, "manifest run"),
445            (h.cache_start, h.cache_pages, "cache directory run"),
446            (h.free.start, h.free.pages, "free-list run"),
447            (h.keys.start, h.keys.pages, "key-slot run"),
448        ];
449        live.extend(
450            self.catalog
451                .iter()
452                .filter(|e| !e.deleted)
453                .map(|e| (e.start_page, e.page_count as u64, "record")),
454        );
455        live.extend(
456            self.cache_dir
457                .iter()
458                .filter(|e| !e.deleted)
459                .map(|e| (e.start_page, e.page_count as u64, "cache entry")),
460        );
461        live.extend(self.col.page_rows.keys().map(|&p| (p, 1, "vector page")));
462        live.extend(
463            self.delta_chain
464                .iter()
465                .map(|r| (r.start, r.pages, "catalog delta run")),
466        );
467        live.extend(
468            self.index_chain
469                .iter()
470                .map(|r| (r.start, r.pages, "index delta run")),
471        );
472        live.push((
473            self.header.bm25.start,
474            self.header.bm25.pages,
475            "BM25 manifest",
476        ));
477        live.extend(
478            self.bm25
479                .manifest
480                .segments
481                .iter()
482                .map(|t| (t.0, t.1, "BM25 segment")),
483        );
484        for (p, c, what) in live {
485            if c == 0 {
486                continue;
487            }
488            if self.free.overlaps(p, c) {
489                return Err(MnemoError::Invalid(format!(
490                    "free list overlaps a live {what} at page {p} (+{c})"
491                )));
492            }
493            if self
494                .pending_free
495                .iter()
496                .any(|&(q, n)| q < p + c && p < q + n)
497            {
498                return Err(MnemoError::Invalid(format!(
499                    "a pending free overlaps a live {what} at page {p} (+{c})"
500                )));
501            }
502        }
503        Ok(())
504    }
505
506    /// The statistics that need no record decrypts and no index load
507    /// (v0.12.2, RB-002.3): counts, sizes, the format, whether an index
508    /// exists, the recall mode and the bytes the vector column needs. The
509    /// first [`Mnemo::stats`] on a handle decrypts every record (for its
510    /// agent list) and loads a persisted index; this is what a phone app
511    /// polling the recall mode should call.
512    pub fn stats_summary(&self) -> StatsSummary {
513        let deleted = self.catalog.iter().filter(|e| e.deleted).count();
514        let v9 = self.is_v9();
515        let column = if v9 {
516            (self.scored_rows() * crate::veccol::entry_len(self.dimensions)) as u64
517        } else {
518            0
519        };
520        let recall_mode = if self.has_index() {
521            RecallMode::Indexed
522        } else if v9 && column <= self.vector_cache_budget as u64 {
523            RecallMode::ExactResident
524        } else {
525            RecallMode::ExactPaged
526        };
527        let rp = self.restore_points();
528        StatsSummary {
529            memories: self.len(),
530            deleted,
531            dimensions: self.dimensions,
532            file_bytes: self.header.next_page * PAGE_SIZE as u64,
533            format_version: self.header.version,
534            wal_pages: self.header.wal_pages,
535            has_index: self.has_index(),
536            recall_mode,
537            vector_column_bytes: column,
538            snapshots_retained: rp.retained,
539            oldest_restorable_at: rp.oldest_at,
540            snapshot_pinned_bytes: rp.pinned_bytes,
541        }
542    }
543
544    /// Summary statistics for the open database. The first call on a
545    /// handle decrypts every record (to list the agents) and loads a
546    /// persisted index; [`Mnemo::stats_summary`] does neither.
547    pub fn stats(&mut self) -> Result<Stats> {
548        let deleted = self.catalog.iter().filter(|e| e.deleted).count();
549        // FIX-1 R-001.8 part 2: build the agent set lazily on first
550        // call per handle, then maintain in `put` / invalidate in
551        // `delete`. Pre-FIX-1 every `stats()` call decrypted every
552        // live memory (O(N) AES-GCM decrypts just to list a usually-
553        // tiny set of distinct agent IDs). On a 1M-memory database
554        // this was in the multi-second range on hardware AES.
555        if self.agent_set.is_none() {
556            let built: std::collections::BTreeSet<String> =
557                self.memories()?.into_iter().map(|m| m.agent_id).collect();
558            self.agent_set = Some(built);
559        }
560        let agents: Vec<String> = self
561            .agent_set
562            .as_ref()
563            .expect("agent_set just filled above")
564            .iter()
565            .cloned()
566            .collect();
567        let file_bytes = self.header.next_page * PAGE_SIZE as u64;
568        let recall_mode = self.recall_mode()?;
569        let rp = self.restore_points();
570        Ok(Stats {
571            memories: self.len(),
572            deleted,
573            dimensions: self.dimensions,
574            file_bytes,
575            agents,
576            encrypted: self.header.flags & FLAG_ENCRYPTED != 0,
577            created_at: self.header.created_at,
578            index: {
579                self.ensure_ann()?;
580                self.ann.as_ref().map(|a| a.info())
581            },
582            wal_pages: self.header.wal_pages,
583            fsyncs_last_flush: self.fsyncs_last_flush,
584            format_version: self.header.version,
585            recall_mode,
586            snapshots_retained: rp.retained,
587            oldest_restorable_at: rp.oldest_at,
588            snapshot_pinned_bytes: rp.pinned_bytes,
589        })
590    }
591
592    /// Rewrite the file, dropping tombstoned and expired memories and
593    /// reclaiming stale pages left by updates. Done out-of-place via a temp
594    /// file and an atomic rename.
595    ///
596    /// FIX-1 R-001.1 + R-002.2 part 2: the result cache now survives
597    /// compaction. Every live, non-expired directory entry is copied
598    /// into the new file verbatim — `created_at`, `ttl_secs`,
599    /// `cost_hint_ms`, `key_preview`, `size_bytes`, `content_type`,
600    /// `vector` and `model` come from the record body; `accessed_at`
601    /// and `access_count` come from the *directory* (the record body
602    /// only holds the insert-time snapshot — the directory is the
603    /// source of truth for the live counts, as of v5's recall trick).
604    /// Tombstoned and TTL-expired entries are dropped. Pre-FIX-1
605    /// compaction kept only the memory catalog and silently emptied
606    /// the cache; the Planner's interim README caveat (c417d2e) called
607    /// that out — this PR removes the surprise.
608    ///
609    /// The output is always v9, so this is also how a v4–v8 file is
610    /// converted: the only read-write open that accepts one (the v8
611    /// sunset, v0.13). It keeps every live memory.
612    pub fn compact_file(path: &str, passphrase: &str) -> Result<CompactReport> {
613        Self::compact_file_secret(path, passphrase.as_bytes(), 0)
614    }
615
616    /// PERF-1 step 7: [`Mnemo::compact_file`] for a raw-key database.
617    pub fn compact_file_with_key(path: &str, key: &[u8; crypto::KEY_LEN]) -> Result<CompactReport> {
618        Self::compact_file_secret(path, key, 0)
619    }
620
621    pub(super) fn compact_file_secret(
622        path: &str,
623        secret: &[u8],
624        format_version: u16,
625    ) -> Result<CompactReport> {
626        // The one read-write open that takes a v4–v8 file (the v8 sunset).
627        let mut old = Mnemo::open_secret_with(path, secret, true)?;
628        let dims = old.dimensions;
629        let kdf = old.kdf;
630        let tmp = format!("{path}.compact-tmp");
631
632        let mut new = Mnemo::create_secret(
633            &tmp,
634            secret,
635            MnemoConfig {
636                dimensions: dims,
637                kdf,
638                // FMT-9f: compaction writes v9 (upgrading a v8 file),
639                // unless `compact_file_to` asked for v8. A v9 file that
640                // keeps float vectors keeps them.
641                format_version,
642                store_float_vectors: old.float_vectors(),
643                ..Default::default()
644            },
645        )?;
646        let report = old.copy_live_into(&mut new)?;
647
648        new.flush()?;
649        drop(new);
650        drop(old);
651
652        std::fs::rename(&tmp, path)?;
653        Ok(report)
654    }
655
656    /// Copy one live cache entry from `old` at directory position
657    /// `pos` into `self` (a freshly-created compaction target).
658    /// Decrypts the old record body, substitutes the old *directory*'s
659    /// `accessed_at` + `access_count` (the record body's copies are
660    /// stale as of v5's directory-only hit path), re-encodes, writes
661    /// a fresh record run, and pushes a matching directory entry.
662    ///
663    /// Caller is expected to have already filtered out tombstoned /
664    /// expired entries and to call this on a read-write handle.
665    pub(crate) fn copy_cache_entry_from(&mut self, old: &mut Mnemo, pos: usize) -> Result<()> {
666        if self.read_only {
667            return Err(MnemoError::ReadOnly);
668        }
669        // Snapshot the directory fields we need, so the borrow on
670        // `old.cache_dir` ends before we touch `old.pager`.
671        let (start_page, page_count, len, dir_accessed_at, dir_access_count, old_gen) = {
672            let e = &old.cache_dir[pos];
673            (
674                e.start_page,
675                e.page_count,
676                e.len,
677                e.accessed_at,
678                e.access_count,
679                e.gen,
680            )
681        };
682
683        // Read + decrypt the old record body.
684        let buf = old.read_record_run(start_page, page_count, len, old_gen)?;
685        let mut entry: CacheEntry =
686            rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
687
688        // Overwrite the stale body copies with the directory's live
689        // values before persisting. Everything else — `created_at`,
690        // `ttl_secs`, `cost_hint_ms`, `key_preview`, `size_bytes`,
691        // `content_type`, `value`, `vector`, `model` — stays as the
692        // body already recorded it.
693        entry.accessed_at = dir_accessed_at;
694        entry.access_count = dir_access_count;
695
696        let bytes =
697            rmp_serde::to_vec(&entry).map_err(|err| MnemoError::Serialize(err.to_string()))?;
698        let (new_start_page, new_page_count, gen) = self.write_record(&bytes)?;
699
700        let dir_entry = CacheDirectoryEntry {
701            namespace: entry.namespace.clone(),
702            key_hash: entry.key_hash,
703            start_page: new_start_page,
704            page_count: new_page_count,
705            len: bytes.len() as u32,
706            deleted: false,
707            accessed_at: dir_accessed_at,
708            access_count: dir_access_count,
709            size_bytes: entry.size_bytes,
710            ttl_secs: entry.ttl_secs,
711            created_at: entry.created_at,
712            vector: entry.vector.clone(),
713            model: entry.model.clone(),
714            gen,
715        };
716        let new_pos = self.cache_dir.len();
717        self.cache_idx
718            .insert(&entry.namespace, entry.key_hash, new_pos);
719        self.cache_dir.push(dir_entry);
720        self.dirty_cache = true;
721        Ok(())
722    }
723}
724
725/// SEC-2 (RB-001.30): overwrite every occurrence of `needle` in the file
726/// with zeros, then sync. Rekey uses it for stale copies of the old header's
727/// key wrap: they're plaintext (page 0 is sealed, not encrypted) and can sit
728/// in WAL regions that were grown, relocated or freed. Every other page is
729/// AES-GCM ciphertext, where a 76-byte random match doesn't happen.
730pub(super) fn scrub_bytes(file: &mut std::fs::File, needle: &[u8]) -> Result<usize> {
731    if needle.is_empty() {
732        return Ok(0);
733    }
734    let len = file.seek(SeekFrom::End(0))?;
735    let chunk = 1 << 20;
736    let overlap = needle.len() - 1;
737    let mut hits: Vec<u64> = Vec::new();
738    let mut buf = vec![0u8; chunk + overlap];
739    let mut pos: u64 = 0;
740    while pos < len {
741        let want = (chunk as u64 + overlap as u64).min(len - pos) as usize;
742        file.seek(SeekFrom::Start(pos))?;
743        file.read_exact(&mut buf[..want])?;
744        let mut i = 0;
745        while i + needle.len() <= want {
746            if buf[i..i + needle.len()] == *needle {
747                hits.push(pos + i as u64);
748                i += needle.len();
749            } else {
750                i += 1;
751            }
752        }
753        pos += chunk as u64;
754    }
755    hits.dedup();
756    let zeros = vec![0u8; needle.len()];
757    for off in &hits {
758        file.seek(SeekFrom::Start(*off))?;
759        file.write_all(&zeros)?;
760    }
761    if !hits.is_empty() {
762        file.sync_all()?;
763        crate::bump_fsync();
764    }
765    Ok(hits.len())
766}