mnemo_engine/store/maintenance.rs
1//! Maintenance: export, rekey, verify, stats, and compaction (which also upgrades v8 files to v9).
2
3use super::*;
4
5impl Mnemo {
6 /// Return the KDF (Argon2id) parameters this file currently uses to
7 /// wrap its DEK. Needed by callers that want to [`rekey`] while
8 /// keeping the file's original protection level (e.g. the Python
9 /// binding's `rekey(fast=False)`, whose docstring promises exactly
10 /// that). FIX-1 R-001.5.
11 pub fn kdf_params(&self) -> KdfParams {
12 self.kdf
13 }
14
15 /// Flush any pending writes, then stream the (encrypted) file to
16 /// `dest` atomically: writes to a sibling temp file in `dest`'s
17 /// parent directory, `sync_all`s it, renames onto `dest`, and
18 /// fsyncs the parent directory on Unix so a crash never leaves a
19 /// partial export.
20 ///
21 /// Refuses when `dest` resolves to the file this handle has open
22 /// — that case would otherwise `File::create`-truncate the live
23 /// database to 0 bytes (`fs::copy` on Unix reads *after* the
24 /// `O_TRUNC`, so a naive export-to-self destroys the source).
25 /// Unix: compares `(dev, ino)` of both paths via `MetadataExt`,
26 /// catching symlinks and hard links. Windows: compares
27 /// canonicalized paths; a rename onto an open file also fails on
28 /// Windows because of the mandatory lock the handle holds, so the
29 /// rename step is a second line of defense.
30 ///
31 /// Reads the bytes from the already-open file handle, not by
32 /// `fs::copy`. On Windows that matters: `LockFileEx` locks are
33 /// mandatory, so a second handle opened by `fs::copy` would fail
34 /// while this writer still holds the lock.
35 ///
36 /// FIX-1 R-001.5 + R-002.2 part 3.
37 pub fn export_encrypted(&mut self, dest: impl AsRef<Path>) -> Result<()> {
38 let dest = dest.as_ref();
39
40 // Step 1 — refuse export to the open file itself. We compare
41 // by identity, not string, so symlinks and hard links to
42 // `self.path` are caught too. The check runs before any flush
43 // so a self-targeted export is a pure no-op.
44 if paths_refer_to_same_file(&self.path, dest).unwrap_or(false) {
45 return Err(MnemoError::Invalid(format!(
46 "export_encrypted refuses to overwrite the open database at {}",
47 dest.display()
48 )));
49 }
50
51 // Step 2 — flush, so on-disk bytes match the live state.
52 if !self.read_only {
53 self.flush()?;
54 }
55
56 // Step 3 — stream from the open handle into a sibling temp.
57 // Using the open handle (instead of a second open on `self.path`)
58 // means a Windows mandatory lock can't block the copy.
59 let parent = dest.parent().ok_or_else(|| {
60 MnemoError::Invalid(format!(
61 "export_encrypted: dest {} has no parent directory",
62 dest.display()
63 ))
64 })?;
65 let tmp = temp_sibling(dest);
66 // Scope the temp handle so it closes before the rename on
67 // Windows (which refuses to rename over an open file).
68 {
69 let src = self.pager.file_mut();
70 src.seek(SeekFrom::Start(0))?;
71 let mut out = OpenOptions::new()
72 .write(true)
73 .create(true)
74 .truncate(true)
75 .open(&tmp)?;
76 let mut buf = vec![0u8; 1 << 20]; // 1 MiB
77 loop {
78 let n = src.read(&mut buf)?;
79 if n == 0 {
80 break;
81 }
82 out.write_all(&buf[..n])?;
83 }
84 out.sync_all()?;
85 crate::bump_fsync();
86 }
87
88 // Step 4 — atomic rename onto `dest`.
89 if let Err(e) = std::fs::rename(&tmp, dest) {
90 // Best-effort cleanup; propagate the rename error.
91 let _ = std::fs::remove_file(&tmp);
92 return Err(MnemoError::Io(e));
93 }
94
95 // Step 5 — fsync the parent directory on Unix so the rename
96 // itself is durable. On Windows directory syncs aren't
97 // supported; the rename above is atomic enough on NTFS.
98 #[cfg(unix)]
99 {
100 if let Ok(dir) = File::open(parent) {
101 let _ = dir.sync_all();
102 crate::bump_fsync();
103 }
104 }
105 #[cfg(not(unix))]
106 {
107 let _ = parent;
108 }
109
110 Ok(())
111 }
112
113 /// Change the passphrase. Cheap: re-derives the KEK and re-wraps the DEK;
114 /// the encrypted pages are never rewritten.
115 ///
116 /// **This changes the secret that opens this file from now on; anyone who
117 /// had the file and the old secret can still read it and later writes.**
118 /// The data key stays the same, so an earlier copy of the file (or of
119 /// just its header) and the old passphrase still unlock it. The old wrap
120 /// is scrubbed from this file (the previous WAL region is zeroed, and any
121 /// stale copy of the old header's key wrap is overwritten); that's
122 /// hygiene, not revocation. **To revoke, rotate:** rekey, then
123 /// [`Mnemo::compact_file`] (`mnemo rekey --rotate`), which rewrites the
124 /// file under a fresh data key. Rotation can't erase old ciphertext the storage hardware still holds: flash keeps rewritten blocks until it reuses them, and that's out of the engine's hands.
125 ///
126 /// **If it returns an error, the file may already be under the new secret,
127 /// or may move to it at the next write** (RB-004.2). Close the handle
128 /// without writing anything more, then try both secrets.
129 pub fn rekey(&mut self, new_passphrase: &str, kdf: KdfParams) -> Result<()> {
130 self.rekey_secret(new_passphrase.as_bytes(), kdf)
131 }
132
133 /// PERF-1 step 7: switch to (or rotate) a raw 256-bit key, recording
134 /// [`KdfParams::raw_key`] in the header. Like [`Mnemo::rekey`], only the
135 /// wrapped DEK changes; pages aren't rewritten, and it doesn't revoke
136 /// the old key (see [`Mnemo::rekey`]).
137 ///
138 /// **If it returns an error, the file may already be under the new secret,
139 /// or may move to it at the next write** (RB-004.2). Close the handle
140 /// without writing anything more, then try both secrets.
141 pub fn rekey_with_key(&mut self, new_key: &[u8; crypto::KEY_LEN]) -> Result<()> {
142 self.rekey_secret(new_key, KdfParams::raw_key())
143 }
144
145 pub(super) fn rekey_secret(&mut self, secret: &[u8], kdf: KdfParams) -> Result<()> {
146 if self.read_only {
147 return Err(MnemoError::ReadOnly);
148 }
149 self.flush()?;
150 let new_salt = crypto::random_salt();
151 let new_kek = crypto::derive_kek(secret, &new_salt, kdf)?;
152 let new_nonce = crypto::random_nonce();
153 let wrapped = crypto::wrap_dek(&new_kek, &new_nonce, self.pager.dek())?;
154 if wrapped.len() != WRAPPED_DEK_LEN {
155 return Err(MnemoError::Crypto("unexpected wrapped DEK length".into()));
156 }
157 let mut wrapped_dek = [0u8; WRAPPED_DEK_LEN];
158 wrapped_dek.copy_from_slice(&wrapped);
159
160 // SEC-2: the old header's key wrap (salt, DEK nonce, wrapped DEK, as
161 // laid out at header bytes 86..162), to scrub once the new header
162 // is committed.
163 let mut old_wrap =
164 Vec::with_capacity(crypto::SALT_LEN + crypto::NONCE_LEN + WRAPPED_DEK_LEN);
165 old_wrap.extend_from_slice(&self.header.salt);
166 old_wrap.extend_from_slice(&self.header.dek_nonce);
167 old_wrap.extend_from_slice(&self.header.wrapped_dek);
168
169 self.header.salt = new_salt;
170 self.header.dek_nonce = new_nonce;
171 self.header.wrapped_dek = wrapped_dek;
172 self.header.m_cost = kdf.m_cost;
173 self.header.t_cost = kdf.t_cost;
174 self.header.p_cost = kdf.p_cost;
175 self.kdf = kdf;
176
177 // DUR-1 (R-002.1): WAL-log the new page 0 before touching it.
178 // Allocate a fresh WAL region at the tail so that a crash during
179 // the rekey leaves either the old region (old passphrase works)
180 // or the new region (new passphrase works) fully committed —
181 // overwriting the existing region would risk destroying both.
182 let old_wal_start = self.header.wal_start;
183 let new_pages = self.header.wal_pages;
184 let new_start = self.header.next_page;
185 self.header.next_page += new_pages;
186 self.header.wal_start = new_start;
187 self.header.wal_pages = new_pages;
188 self.header.wal_seq += 1;
189 self.relocate_header_through_wal(old_wal_start)?;
190 // SEC-2 (RB-001.30): the old region holds page-0 images with the old
191 // wrap; `relocate_header_through_wal` only zeroed its first frame
192 // header. Zero all of it, then overwrite any other stale copy of
193 // the old wrap in the file (an earlier, grown or relocated region).
194 // Hygiene only: a copy of the file taken before the rekey still
195 // opens with the old secret (see `rekey`).
196 wal::scrub_region(self.pager.file_mut(), old_wal_start, new_pages)?;
197 scrub_bytes(self.pager.file_mut(), &old_wrap)?;
198 // FMT-9 (a): the old region is freed by the next commit (see
199 // `ensure_wal_capacity`).
200 self.release_pages(old_wal_start, new_pages);
201 Ok(())
202 }
203
204 /// SEC-2 (RB-001.30): write a copy of this database at `dest` under a
205 /// **fresh data key**, protected by `passphrase` through `kdf`. It goes
206 /// through the compaction path (live memories, the result cache and
207 /// the index, no history), so the copy shares no key material with
208 /// this file: neither this file's passphrase nor its data key opens
209 /// it, and its passphrase opens nothing here. This file is flushed
210 /// first (unless read-only) and is otherwise unchanged; `dest` must not
211 /// exist. The copy is v9.
212 pub fn export_rekeyed(
213 &mut self,
214 dest: impl AsRef<Path>,
215 passphrase: &str,
216 kdf: KdfParams,
217 ) -> Result<CompactReport> {
218 self.export_rekeyed_secret(dest.as_ref(), passphrase.as_bytes(), kdf)
219 }
220
221 /// [`Mnemo::export_rekeyed`] protected by a raw 256-bit key (no KDF).
222 pub fn export_rekeyed_with_key(
223 &mut self,
224 dest: impl AsRef<Path>,
225 key: &[u8; crypto::KEY_LEN],
226 ) -> Result<CompactReport> {
227 self.export_rekeyed_secret(dest.as_ref(), key, KdfParams::raw_key())
228 }
229
230 pub(super) fn export_rekeyed_secret(
231 &mut self,
232 dest: &Path,
233 secret: &[u8],
234 kdf: KdfParams,
235 ) -> Result<CompactReport> {
236 if paths_refer_to_same_file(&self.path, dest).unwrap_or(false) {
237 return Err(MnemoError::Invalid(format!(
238 "export_rekeyed refuses to overwrite the open database at {}",
239 dest.display()
240 )));
241 }
242 if dest.exists() {
243 return Err(MnemoError::Invalid(format!(
244 "{} already exists",
245 dest.display()
246 )));
247 }
248 if !self.read_only {
249 self.flush()?;
250 }
251 let mut name = dest.file_name().unwrap_or_default().to_os_string();
252 name.push(".export-tmp");
253 let tmp = dest.with_file_name(name);
254 let _ = std::fs::remove_file(&tmp);
255 let result = (|| {
256 let mut new = Mnemo::create_secret(
257 &tmp,
258 secret,
259 MnemoConfig {
260 dimensions: self.dimensions,
261 kdf,
262 store_float_vectors: self.float_vectors(),
263 ..Default::default()
264 },
265 )?;
266 let report = self.copy_live_into(&mut new)?;
267 new.flush()?;
268 drop(new);
269 std::fs::rename(&tmp, dest)?;
270 Ok(report)
271 })();
272 if result.is_err() {
273 let _ = std::fs::remove_file(&tmp);
274 }
275 result
276 }
277
278 /// The compaction copy: every live, unexpired memory and cache entry of
279 /// `self` into `new` (a freshly created file), and the index if `self`
280 /// has one. Counts as [`CompactReport`].
281 pub(super) fn copy_live_into(&mut self, new: &mut Mnemo) -> Result<CompactReport> {
282 // FMT-9e: compaction rewrites everything, so it compresses every
283 // thread turn, recent or not.
284 new.turn_compress_after = 0;
285 // ANN-1.1: compaction is "re-tune for the current size" by
286 // default. Pass IndexConfig::default() so partitions,
287 // subspaces, n_probe and n_rerank are all re-derived from the
288 // compacted vector count. Previously we froze the pre-compact
289 // (n_probe, n_rerank), which left the new index on stale
290 // numbers for the smaller post-compact corpus.
291 let want_index = self.has_index();
292 let now = memory::now_secs();
293 let all = self.memories()?;
294 let before = all.len();
295 let mut after = 0;
296 let mut zero_vector_manifests = 0;
297 let mut zero_vector_memories = 0;
298 for mut m in all {
299 if m.is_expired(now) {
300 continue;
301 }
302 // A file from before the zero-vector rule (Phase 3 PR 10a) can
303 // hold all-zero vectors. A legacy onboarding or manifest memory
304 // gets the sentinel unit vector a current manifest has, so it
305 // behaves like a new one; any other is carried unchanged.
306 if embed::is_zero_vector(&m.vector) {
307 if crud::is_onboarding(&m) || is_manifest(&m) {
308 m.vector = Memory::scaffold_manifest(self.dimensions).vector;
309 zero_vector_manifests += 1;
310 } else {
311 zero_vector_memories += 1;
312 }
313 }
314 // The engine's own copy: `_mnemo.*` keys come through as they
315 // are, and the write-time zero-vector check doesn't apply.
316 new.put_existing(m)?;
317 after += 1;
318 }
319 if want_index && after > 0 {
320 new.build_index_with(IndexConfig::default())?;
321 }
322
323 // Count + copy cache entries. We walk `self.cache_dir` by index
324 // so each hop reads the directory's live `accessed_at` /
325 // `access_count` directly rather than going through
326 // `cache_get` (which would also treat read-only semantics,
327 // tombstones, etc. — we want the raw, authoritative values).
328 let mut cache_before = 0usize;
329 let mut cache_after = 0usize;
330 let positions: Vec<usize> = (0..self.cache_dir.len()).collect();
331 for pos in positions {
332 let (keep, reserved) = {
333 let e = &self.cache_dir[pos];
334 (
335 !e.deleted && !e.is_expired(now),
336 embed::is_reserved_namespace(&e.namespace),
337 )
338 };
339 if keep {
340 // EMBED-ID-1: the engine's own entries (the embedder
341 // identity) are copied but not counted.
342 new.copy_cache_entry_from(self, pos)?;
343 if !reserved {
344 cache_before += 1;
345 cache_after += 1;
346 }
347 }
348 }
349 Ok(CompactReport {
350 before,
351 after,
352 cache_before,
353 cache_after,
354 zero_vector_manifests,
355 zero_vector_memories,
356 })
357 }
358
359 /// Decrypt and validate every live record. Returns the count verified
360 /// across memories **and** cache entries (FIX-1 R-001.1: now that the
361 /// cache survives compaction, `verify()` has to walk it too — the
362 /// pre-FIX-1 implementation only checked `self.catalog`, so a
363 /// corrupted cache record went unnoticed).
364 pub fn verify(&mut self) -> Result<usize> {
365 Ok(self.verify_report()?.records)
366 }
367
368 /// [`Mnemo::verify`], also counting the memories whose vector is all
369 /// zeros. A file written before the zero-vector rule (Phase 3 PR 10a)
370 /// can hold them; they decode fine but score 0.0 on every vector query.
371 /// `mnemo verify` warns about them, and compaction carries them
372 /// (giving a legacy manifest the sentinel vector).
373 pub fn verify_report(&mut self) -> Result<VerifyReport> {
374 let entries: Vec<CatalogEntry> = self
375 .catalog
376 .iter()
377 .filter(|e| !e.deleted)
378 .cloned()
379 .collect();
380 let mut n = entries.len();
381 let mut zero_vector_memories = 0;
382 for e in &entries {
383 let bytes = self.read_record(e)?;
384 let m: Memory = self.decode_memory(&bytes)?;
385 // FMT-9b: and the row's vector column entry.
386 let zero = if e.v9.is_some() {
387 let (h, _) = self.column_entry(e)?;
388 if h.len() != self.dimensions {
389 return Err(MnemoError::Invalid(
390 "vector entry has the wrong width".into(),
391 ));
392 }
393 // fp16 +0.0 and -0.0.
394 !h.is_empty() && h.iter().all(|b| b & 0x7fff == 0)
395 } else {
396 embed::is_zero_vector(&m.vector)
397 };
398 if zero {
399 zero_vector_memories += 1;
400 }
401 }
402
403 // Also decode every live cache entry. Tombstoned entries are
404 // skipped — their pages are orphan ciphertext awaiting
405 // `compact_file`, same as deleted memories. Expired entries
406 // are *still* on disk until compaction, so we verify them too.
407 let cache_positions: Vec<(u64, u32, u32, u64, bool)> = self
408 .cache_dir
409 .iter()
410 .filter(|e| !e.deleted)
411 .map(|e| {
412 let reserved = embed::is_reserved_namespace(&e.namespace);
413 (e.start_page, e.page_count, e.len, e.gen, reserved)
414 })
415 .collect();
416 for (start_page, page_count, len, gen, reserved) in cache_positions {
417 let buf = self.read_record_run(start_page, page_count, len, gen)?;
418 let _c: CacheEntry = rmp_serde::from_slice(&buf)
419 .map_err(|err| MnemoError::Serialize(err.to_string()))?;
420 // EMBED-ID-1: the engine's own entries are checked, not counted.
421 if !reserved {
422 n += 1;
423 }
424 }
425 // FMT-9 (a): no free run may overlap a page the committed state
426 // references (records, cache entries, control runs, the WAL).
427 if self.is_v9() {
428 self.check_free_list_disjoint()?;
429 }
430 Ok(VerifyReport {
431 records: n,
432 zero_vector_memories,
433 })
434 }
435
436 /// FMT-9 (a): fail if the free list (or a pending free) overlaps any
437 /// page the current state references.
438 pub(super) fn check_free_list_disjoint(&self) -> Result<()> {
439 let h = &self.header;
440 let mut live: Vec<(u64, u64, &str)> = vec![
441 (h.wal_start, h.wal_pages, "WAL region"),
442 (h.catalog_start, h.catalog_pages, "catalog run"),
443 (h.index_start, h.index_pages, "index run"),
444 (h.manifest_start, h.manifest_pages, "manifest run"),
445 (h.cache_start, h.cache_pages, "cache directory run"),
446 (h.free.start, h.free.pages, "free-list run"),
447 (h.keys.start, h.keys.pages, "key-slot run"),
448 ];
449 live.extend(
450 self.catalog
451 .iter()
452 .filter(|e| !e.deleted)
453 .map(|e| (e.start_page, e.page_count as u64, "record")),
454 );
455 live.extend(
456 self.cache_dir
457 .iter()
458 .filter(|e| !e.deleted)
459 .map(|e| (e.start_page, e.page_count as u64, "cache entry")),
460 );
461 live.extend(self.col.page_rows.keys().map(|&p| (p, 1, "vector page")));
462 live.extend(
463 self.delta_chain
464 .iter()
465 .map(|r| (r.start, r.pages, "catalog delta run")),
466 );
467 live.extend(
468 self.index_chain
469 .iter()
470 .map(|r| (r.start, r.pages, "index delta run")),
471 );
472 live.push((
473 self.header.bm25.start,
474 self.header.bm25.pages,
475 "BM25 manifest",
476 ));
477 live.extend(
478 self.bm25
479 .manifest
480 .segments
481 .iter()
482 .map(|t| (t.0, t.1, "BM25 segment")),
483 );
484 for (p, c, what) in live {
485 if c == 0 {
486 continue;
487 }
488 if self.free.overlaps(p, c) {
489 return Err(MnemoError::Invalid(format!(
490 "free list overlaps a live {what} at page {p} (+{c})"
491 )));
492 }
493 if self
494 .pending_free
495 .iter()
496 .any(|&(q, n)| q < p + c && p < q + n)
497 {
498 return Err(MnemoError::Invalid(format!(
499 "a pending free overlaps a live {what} at page {p} (+{c})"
500 )));
501 }
502 }
503 Ok(())
504 }
505
506 /// The statistics that need no record decrypts and no index load
507 /// (v0.12.2, RB-002.3): counts, sizes, the format, whether an index
508 /// exists, the recall mode and the bytes the vector column needs. The
509 /// first [`Mnemo::stats`] on a handle decrypts every record (for its
510 /// agent list) and loads a persisted index; this is what a phone app
511 /// polling the recall mode should call.
512 pub fn stats_summary(&self) -> StatsSummary {
513 let deleted = self.catalog.iter().filter(|e| e.deleted).count();
514 let v9 = self.is_v9();
515 let column = if v9 {
516 (self.scored_rows() * crate::veccol::entry_len(self.dimensions)) as u64
517 } else {
518 0
519 };
520 let recall_mode = if self.has_index() {
521 RecallMode::Indexed
522 } else if v9 && column <= self.vector_cache_budget as u64 {
523 RecallMode::ExactResident
524 } else {
525 RecallMode::ExactPaged
526 };
527 let rp = self.restore_points();
528 StatsSummary {
529 memories: self.len(),
530 deleted,
531 dimensions: self.dimensions,
532 file_bytes: self.header.next_page * PAGE_SIZE as u64,
533 format_version: self.header.version,
534 wal_pages: self.header.wal_pages,
535 has_index: self.has_index(),
536 recall_mode,
537 vector_column_bytes: column,
538 snapshots_retained: rp.retained,
539 oldest_restorable_at: rp.oldest_at,
540 snapshot_pinned_bytes: rp.pinned_bytes,
541 }
542 }
543
544 /// Summary statistics for the open database. The first call on a
545 /// handle decrypts every record (to list the agents) and loads a
546 /// persisted index; [`Mnemo::stats_summary`] does neither.
547 pub fn stats(&mut self) -> Result<Stats> {
548 let deleted = self.catalog.iter().filter(|e| e.deleted).count();
549 // FIX-1 R-001.8 part 2: build the agent set lazily on first
550 // call per handle, then maintain in `put` / invalidate in
551 // `delete`. Pre-FIX-1 every `stats()` call decrypted every
552 // live memory (O(N) AES-GCM decrypts just to list a usually-
553 // tiny set of distinct agent IDs). On a 1M-memory database
554 // this was in the multi-second range on hardware AES.
555 if self.agent_set.is_none() {
556 let built: std::collections::BTreeSet<String> =
557 self.memories()?.into_iter().map(|m| m.agent_id).collect();
558 self.agent_set = Some(built);
559 }
560 let agents: Vec<String> = self
561 .agent_set
562 .as_ref()
563 .expect("agent_set just filled above")
564 .iter()
565 .cloned()
566 .collect();
567 let file_bytes = self.header.next_page * PAGE_SIZE as u64;
568 let recall_mode = self.recall_mode()?;
569 let rp = self.restore_points();
570 Ok(Stats {
571 memories: self.len(),
572 deleted,
573 dimensions: self.dimensions,
574 file_bytes,
575 agents,
576 encrypted: self.header.flags & FLAG_ENCRYPTED != 0,
577 created_at: self.header.created_at,
578 index: {
579 self.ensure_ann()?;
580 self.ann.as_ref().map(|a| a.info())
581 },
582 wal_pages: self.header.wal_pages,
583 fsyncs_last_flush: self.fsyncs_last_flush,
584 format_version: self.header.version,
585 recall_mode,
586 snapshots_retained: rp.retained,
587 oldest_restorable_at: rp.oldest_at,
588 snapshot_pinned_bytes: rp.pinned_bytes,
589 })
590 }
591
592 /// Rewrite the file, dropping tombstoned and expired memories and
593 /// reclaiming stale pages left by updates. Done out-of-place via a temp
594 /// file and an atomic rename.
595 ///
596 /// FIX-1 R-001.1 + R-002.2 part 2: the result cache now survives
597 /// compaction. Every live, non-expired directory entry is copied
598 /// into the new file verbatim — `created_at`, `ttl_secs`,
599 /// `cost_hint_ms`, `key_preview`, `size_bytes`, `content_type`,
600 /// `vector` and `model` come from the record body; `accessed_at`
601 /// and `access_count` come from the *directory* (the record body
602 /// only holds the insert-time snapshot — the directory is the
603 /// source of truth for the live counts, as of v5's recall trick).
604 /// Tombstoned and TTL-expired entries are dropped. Pre-FIX-1
605 /// compaction kept only the memory catalog and silently emptied
606 /// the cache; the Planner's interim README caveat (c417d2e) called
607 /// that out — this PR removes the surprise.
608 ///
609 /// The output is always v9, so this is also how a v4–v8 file is
610 /// converted: the only read-write open that accepts one (the v8
611 /// sunset, v0.13). It keeps every live memory.
612 pub fn compact_file(path: &str, passphrase: &str) -> Result<CompactReport> {
613 Self::compact_file_secret(path, passphrase.as_bytes(), 0)
614 }
615
616 /// PERF-1 step 7: [`Mnemo::compact_file`] for a raw-key database.
617 pub fn compact_file_with_key(path: &str, key: &[u8; crypto::KEY_LEN]) -> Result<CompactReport> {
618 Self::compact_file_secret(path, key, 0)
619 }
620
621 pub(super) fn compact_file_secret(
622 path: &str,
623 secret: &[u8],
624 format_version: u16,
625 ) -> Result<CompactReport> {
626 // The one read-write open that takes a v4–v8 file (the v8 sunset).
627 let mut old = Mnemo::open_secret_with(path, secret, true)?;
628 let dims = old.dimensions;
629 let kdf = old.kdf;
630 let tmp = format!("{path}.compact-tmp");
631
632 let mut new = Mnemo::create_secret(
633 &tmp,
634 secret,
635 MnemoConfig {
636 dimensions: dims,
637 kdf,
638 // FMT-9f: compaction writes v9 (upgrading a v8 file),
639 // unless `compact_file_to` asked for v8. A v9 file that
640 // keeps float vectors keeps them.
641 format_version,
642 store_float_vectors: old.float_vectors(),
643 ..Default::default()
644 },
645 )?;
646 let report = old.copy_live_into(&mut new)?;
647
648 new.flush()?;
649 drop(new);
650 drop(old);
651
652 std::fs::rename(&tmp, path)?;
653 Ok(report)
654 }
655
656 /// Copy one live cache entry from `old` at directory position
657 /// `pos` into `self` (a freshly-created compaction target).
658 /// Decrypts the old record body, substitutes the old *directory*'s
659 /// `accessed_at` + `access_count` (the record body's copies are
660 /// stale as of v5's directory-only hit path), re-encodes, writes
661 /// a fresh record run, and pushes a matching directory entry.
662 ///
663 /// Caller is expected to have already filtered out tombstoned /
664 /// expired entries and to call this on a read-write handle.
665 pub(crate) fn copy_cache_entry_from(&mut self, old: &mut Mnemo, pos: usize) -> Result<()> {
666 if self.read_only {
667 return Err(MnemoError::ReadOnly);
668 }
669 // Snapshot the directory fields we need, so the borrow on
670 // `old.cache_dir` ends before we touch `old.pager`.
671 let (start_page, page_count, len, dir_accessed_at, dir_access_count, old_gen) = {
672 let e = &old.cache_dir[pos];
673 (
674 e.start_page,
675 e.page_count,
676 e.len,
677 e.accessed_at,
678 e.access_count,
679 e.gen,
680 )
681 };
682
683 // Read + decrypt the old record body.
684 let buf = old.read_record_run(start_page, page_count, len, old_gen)?;
685 let mut entry: CacheEntry =
686 rmp_serde::from_slice(&buf).map_err(|e| MnemoError::Serialize(e.to_string()))?;
687
688 // Overwrite the stale body copies with the directory's live
689 // values before persisting. Everything else — `created_at`,
690 // `ttl_secs`, `cost_hint_ms`, `key_preview`, `size_bytes`,
691 // `content_type`, `value`, `vector`, `model` — stays as the
692 // body already recorded it.
693 entry.accessed_at = dir_accessed_at;
694 entry.access_count = dir_access_count;
695
696 let bytes =
697 rmp_serde::to_vec(&entry).map_err(|err| MnemoError::Serialize(err.to_string()))?;
698 let (new_start_page, new_page_count, gen) = self.write_record(&bytes)?;
699
700 let dir_entry = CacheDirectoryEntry {
701 namespace: entry.namespace.clone(),
702 key_hash: entry.key_hash,
703 start_page: new_start_page,
704 page_count: new_page_count,
705 len: bytes.len() as u32,
706 deleted: false,
707 accessed_at: dir_accessed_at,
708 access_count: dir_access_count,
709 size_bytes: entry.size_bytes,
710 ttl_secs: entry.ttl_secs,
711 created_at: entry.created_at,
712 vector: entry.vector.clone(),
713 model: entry.model.clone(),
714 gen,
715 };
716 let new_pos = self.cache_dir.len();
717 self.cache_idx
718 .insert(&entry.namespace, entry.key_hash, new_pos);
719 self.cache_dir.push(dir_entry);
720 self.dirty_cache = true;
721 Ok(())
722 }
723}
724
725/// SEC-2 (RB-001.30): overwrite every occurrence of `needle` in the file
726/// with zeros, then sync. Rekey uses it for stale copies of the old header's
727/// key wrap: they're plaintext (page 0 is sealed, not encrypted) and can sit
728/// in WAL regions that were grown, relocated or freed. Every other page is
729/// AES-GCM ciphertext, where a 76-byte random match doesn't happen.
730pub(super) fn scrub_bytes(file: &mut std::fs::File, needle: &[u8]) -> Result<usize> {
731 if needle.is_empty() {
732 return Ok(0);
733 }
734 let len = file.seek(SeekFrom::End(0))?;
735 let chunk = 1 << 20;
736 let overlap = needle.len() - 1;
737 let mut hits: Vec<u64> = Vec::new();
738 let mut buf = vec![0u8; chunk + overlap];
739 let mut pos: u64 = 0;
740 while pos < len {
741 let want = (chunk as u64 + overlap as u64).min(len - pos) as usize;
742 file.seek(SeekFrom::Start(pos))?;
743 file.read_exact(&mut buf[..want])?;
744 let mut i = 0;
745 while i + needle.len() <= want {
746 if buf[i..i + needle.len()] == *needle {
747 hits.push(pos + i as u64);
748 i += needle.len();
749 } else {
750 i += 1;
751 }
752 }
753 pos += chunk as u64;
754 }
755 hits.dedup();
756 let zeros = vec![0u8; needle.len()];
757 for off in &hits {
758 file.seek(SeekFrom::Start(*off))?;
759 file.write_all(&zeros)?;
760 }
761 if !hits.is_empty() {
762 file.sync_all()?;
763 crate::bump_fsync();
764 }
765 Ok(hits.len())
766}