mnemo_engine/store/open.rs
1//! Create, open (read-write, read-only, raw key) and the handle's basic accessors and per-handle settings.
2
3use super::*;
4
5/// Chunk size for the torn-header scan's streaming read of the file.
6/// We walk the file 1 MiB at a time looking for 8 KiB-aligned `MWAL`
7/// magic. There is **no** cap on how far a given candidate's frame
8/// chain may extend — a 1M-vector IVF+PQ relocation can push a single
9/// transaction well past 16 MiB — so each candidate is read frame by
10/// frame via [`wal::scan_committed_txn`] rather than slurped into a
11/// bounded buffer. (DUR-1 R-002.1 part 2.)
12const TORN_HEADER_SCAN_CHUNK: usize = 1 << 20; // 1 MiB
13
14/// The header a set of recovery frames would write to page 0, if they hold
15/// one that parses.
16fn page_zero_header(frames: &[(u64, Vec<u8>)]) -> Option<Header> {
17 frames
18 .iter()
19 .find(|(page_no, _)| *page_no == 0)
20 .and_then(|(_, bytes)| Header::from_page(bytes).ok())
21}
22
23/// Walk the file for every 8 KiB-aligned page whose first four bytes are
24/// `MWAL`, treating each as a candidate WAL-region start, and return the
25/// frames of the highest-`txn_id` committed transaction whose header
26/// frame fully validates under `passphrase`.
27///
28/// A candidate is accepted only if:
29/// * `wal::scan_committed_txn` returns a committed transaction (chained
30/// CRC verifies, single `txn_id`, `KIND_COMMIT` closes it);
31/// * that transaction contains a `(0, header_page)` frame;
32/// * the header parses with `wal_start == candidate_page_no` and
33/// `wal_seq == txn_id` (so a frame cannot forge a region it was not
34/// written into);
35/// * deriving the KEK from `passphrase + header.salt`, unwrapping the
36/// wrapped DEK, and validating the header seal all succeed.
37///
38/// Returns `Ok(None)` when the scan finds no valid candidate — the
39/// caller is expected to propagate the original parse error so
40/// non-mnemo files still report `BadMagic`.
41///
42/// DUR-1 (R-002.1 part 2): there is no fixed scan window — the whole
43/// file is scanned. Reading at page granularity (not byte granularity)
44/// is sufficient because every WAL region in the format is page-aligned
45/// and every WAL commit writes its first frame at the region's start.
46fn scan_header_recovery_candidates(
47 file: &mut File,
48 secret: &[u8],
49) -> Result<Option<Vec<wal::Frame>>> {
50 let file_len = file.seek(SeekFrom::End(0))?;
51 if file_len < PAGE_SIZE as u64 * 2 {
52 return Ok(None); // No room for even one WAL page past the header.
53 }
54
55 let mut best: Option<(u64, Vec<wal::Frame>)> = None;
56 let page_sz = PAGE_SIZE as u64;
57 let mut offset = page_sz; // Skip page 0.
58
59 // Stream through the file in 1 MiB chunks, scanning each chunk for
60 // page-aligned `MWAL` candidates. Each candidate's frame chain is
61 // then read incrementally via `wal::scan_committed_txn`, which has
62 // no window cap — a 1M-vector IVF+PQ relocation can span tens of
63 // MiB in one committed transaction.
64 let mut chunk = vec![0u8; TORN_HEADER_SCAN_CHUNK];
65 while offset < file_len {
66 let want = (TORN_HEADER_SCAN_CHUNK as u64).min(file_len - offset) as usize;
67 chunk.resize(want, 0);
68 file.seek(SeekFrom::Start(offset))?;
69 let mut filled = 0;
70 while filled < want {
71 match file.read(&mut chunk[filled..])? {
72 0 => break,
73 n => filled += n,
74 }
75 }
76 if filled == 0 {
77 break;
78 }
79 let mut inner = 0usize;
80 while inner + 4 <= filled {
81 if &chunk[inner..inner + 4] == wal::FRAME_MAGIC_BYTES {
82 let page_no = (offset + inner as u64) / page_sz;
83 if let Some((txn_id, frames)) = try_recover_candidate(file, page_no, secret)? {
84 let keep = match &best {
85 None => true,
86 Some((id, _)) => txn_id > *id,
87 };
88 if keep {
89 best = Some((txn_id, frames));
90 }
91 }
92 }
93 inner += PAGE_SIZE;
94 }
95 offset += filled as u64;
96 }
97
98 Ok(best.map(|(_, f)| f))
99}
100
101/// Try to recover a committed transaction from `candidate_page_no`,
102/// returning `(txn_id, frames)` only if the candidate's
103/// `(0, header_page)` frame fully validates: the frame chain commits
104/// (via [`wal::scan_committed_txn`]'s streaming reader), the header
105/// parses with `wal_start == candidate_page_no` **and**
106/// `wal_seq == txn_id`, the DEK unwraps under `passphrase`, and the
107/// AES-GCM seal validates. DUR-1 (R-002.1 item 2) requires the
108/// `wal_seq == txn_id` check and no fixed scan window — the streaming
109/// reader handles transactions of arbitrary size.
110fn try_recover_candidate(
111 file: &mut File,
112 candidate_page_no: u64,
113 secret: &[u8],
114) -> Result<Option<(u64, Vec<wal::Frame>)>> {
115 let Some((txn_id, frames)) = wal::scan_committed_txn(file, candidate_page_no)? else {
116 return Ok(None);
117 };
118
119 // Must contain a header frame at page 0.
120 let header_bytes = match frames.iter().find(|(p, _)| *p == 0) {
121 Some((_, b)) => b,
122 None => return Ok(None),
123 };
124 if header_bytes.len() != PAGE_SIZE {
125 return Ok(None);
126 }
127 let mut hbuf = [0u8; PAGE_SIZE];
128 hbuf.copy_from_slice(header_bytes);
129 let candidate = match Header::from_page(&hbuf) {
130 Ok(h) => h,
131 Err(_) => return Ok(None),
132 };
133
134 // The frame's claimed region must match where we found it, and the
135 // header frame's `wal_seq` must match the committed transaction's
136 // `txn_id` — this is what pins the frame to the region it was
137 // written into and keeps a stale checkpointed-and-copied frame
138 // from being accepted as a new region's root.
139 if candidate.wal_start != candidate_page_no || candidate.wal_seq != txn_id {
140 return Ok(None);
141 }
142 // DEK unwrap under this candidate's own salt + KDF params.
143 let kdf = KdfParams {
144 m_cost: candidate.m_cost,
145 t_cost: candidate.t_cost,
146 p_cost: candidate.p_cost,
147 };
148 let kek = match crypto::derive_kek(secret, &candidate.salt, kdf) {
149 Ok(k) => k,
150 Err(_) => return Ok(None),
151 };
152 let dek = match crypto::unwrap_dek(&kek, &candidate.dek_nonce, &candidate.wrapped_dek) {
153 Ok(d) => d,
154 Err(_) => return Ok(None),
155 };
156 if candidate.validate_seal(&dek).is_err() {
157 return Ok(None);
158 }
159
160 Ok(Some((txn_id, frames)))
161}
162
163impl Mnemo {
164 /// Create a brand-new encrypted database at `path`.
165 pub fn create(
166 path: impl Into<PathBuf>,
167 passphrase: &str,
168 config: MnemoConfig,
169 ) -> Result<Mnemo> {
170 Self::create_secret(path, passphrase.as_bytes(), config)
171 }
172
173 /// PERF-1 step 7 (R-004.5): create a database unlocked by a **raw
174 /// 256-bit key** instead of a passphrase, for keys held by an OS
175 /// keychain or keystore. The key must be uniformly random (see
176 /// [`Mnemo::generate_key`]); it gets no stretching. The header records
177 /// minimal Argon2id parameters ([`KdfParams::raw_key`]), so opening
178 /// costs microseconds instead of tens of milliseconds. `config.kdf` is
179 /// ignored. No format change: the key goes through the same KEK
180 /// derivation a passphrase does.
181 pub fn create_with_key(
182 path: impl Into<PathBuf>,
183 key: &[u8; crypto::KEY_LEN],
184 config: MnemoConfig,
185 ) -> Result<Mnemo> {
186 let config = MnemoConfig {
187 kdf: KdfParams::raw_key(),
188 ..config
189 };
190 Self::create_secret(path, key, config)
191 }
192
193 /// A fresh random 256-bit key for [`Mnemo::create_with_key`], from the
194 /// thread-local CSPRNG (seeded from the OS). Store it in a keychain;
195 /// it can't be recovered from the file.
196 pub fn generate_key() -> [u8; crypto::KEY_LEN] {
197 *crypto::random_dek()
198 }
199
200 pub(super) fn create_secret(
201 path: impl Into<PathBuf>,
202 secret: &[u8],
203 config: MnemoConfig,
204 ) -> Result<Mnemo> {
205 let path: PathBuf = path.into();
206 if config.dimensions == 0 {
207 return Err(MnemoError::Invalid("dimensions must be > 0".into()));
208 }
209 let version = resolve_format_version(config.format_version)?;
210 if version >= VERSION_V9 && veccol::per_page(config.dimensions) == 0 {
211 return Err(MnemoError::Invalid(format!(
212 "a v9 file supports up to {} dimensions for now (one fp16 vector \
213 must fit a page); vector entries that span pages are not supported yet",
214 (PAYLOAD - veccol::PAGE_HEADER - 4) / 2
215 )));
216 }
217 let flags = if version >= VERSION_V9 && config.store_float_vectors {
218 FLAG_ENCRYPTED | FLAG_FLOAT_VECTORS
219 } else {
220 FLAG_ENCRYPTED
221 };
222 // Exclusive lock covers the write path; second writer sees `Locked`.
223 let file = open_with_lock(&path, /*write=*/ true, /*create=*/ true)?;
224
225 let salt = crypto::random_salt();
226 let dek = crypto::random_dek();
227 let kek = crypto::derive_kek(secret, &salt, config.kdf)?;
228 let dek_nonce = crypto::random_nonce();
229 let wrapped = crypto::wrap_dek(&kek, &dek_nonce, &dek)?;
230 if wrapped.len() != WRAPPED_DEK_LEN {
231 return Err(MnemoError::Crypto("unexpected wrapped DEK length".into()));
232 }
233 let mut wrapped_dek = [0u8; WRAPPED_DEK_LEN];
234 wrapped_dek.copy_from_slice(&wrapped);
235
236 let mut header = Header {
237 version,
238 page_size: PAGE_SIZE as u32,
239 flags,
240 dimensions: config.dimensions as u32,
241 created_at: memory::now_secs(),
242 write_counter: 0,
243 // Page 0 is the header; pages 1..=wal_pages are the WAL;
244 // record/catalog/index pages start after it. Initial WAL size is
245 // chosen per-config (default 8 pages = 64 KiB) and clamped to a
246 // sane floor so the first transaction can always commit.
247 next_page: 1 + config.wal_pages_initial.max(MIN_WAL_PAGES),
248 catalog_start: 0,
249 catalog_pages: 0,
250 catalog_len: 0,
251 vector_count: 0,
252 m_cost: config.kdf.m_cost,
253 t_cost: config.kdf.t_cost,
254 p_cost: config.kdf.p_cost,
255 salt,
256 dek_nonce,
257 wrapped_dek,
258 index_start: 0,
259 index_pages: 0,
260 index_len: 0,
261 wal_start: 1,
262 wal_pages: config.wal_pages_initial.max(MIN_WAL_PAGES),
263 wal_seq: 0,
264 manifest_start: 0,
265 manifest_pages: 0,
266 manifest_len: 0,
267 // Regenerated per write by `apply_seal`; `None` until we seal.
268 seal_nonce: None,
269 seal_tag: None,
270 // v8+ result-cache pointer — zero-initialized on create,
271 // populated only when `cache_put` runs at least once and
272 // the next `flush` seals a directory run.
273 cache_start: 0,
274 cache_pages: 0,
275 cache_len: 0,
276 // FMT-9 v9 fields; all zero on a fresh file (and unused on v8).
277 catalog_gen: 0,
278 index_gen: 0,
279 manifest_gen: 0,
280 cache_gen: 0,
281 free: RunPtr::default(),
282 keys: RunPtr::default(),
283 bm25: RunPtr::default(),
284 index_delta: RunPtr::default(),
285 catalog_delta: RunPtr::default(),
286 };
287
288 let mut pager = Pager::new(file, dek, 0, version);
289 let mut page = header.to_page();
290 header.apply_seal(&mut page, pager.dek())?;
291 pager.write_raw(0, &page)?;
292 pager.sync()?;
293
294 Ok(Mnemo {
295 pager,
296 header,
297 catalog: Vec::new(),
298 index: IdIndex::default(),
299 path,
300 dimensions: config.dimensions,
301 kdf: config.kdf,
302 dirty_catalog: false,
303 ann: None,
304 dirty_index: false,
305 index_persist: IndexPersist::default(),
306 ann_unloaded: false,
307 manifest: Vec::new(),
308 // FMT-9c: v9 keeps fewer snapshots by default: with page reuse,
309 // retention is what pins pages.
310 max_snapshots: if version >= VERSION_V9 && config.max_snapshots == DEFAULT_MAX_SNAPSHOTS
311 {
312 V9_DEFAULT_MAX_SNAPSHOTS
313 } else {
314 config.max_snapshots
315 },
316 read_only: false,
317 cache_dir: Vec::new(),
318 cache_idx: DirectoryIndex::new(),
319 dirty_cache: false,
320 cache_budgets: std::collections::HashMap::new(),
321 cache_flush_policy: CacheFlushPolicy::default(),
322 cache_batch: BatchState::default(),
323 cache_counters: CounterState::default(),
324 embedder: None,
325 fsyncs_last_flush: 0,
326 agent_set: None,
327 key_index: None,
328 superseded: None,
329 layers: None,
330 turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
331 free: FreeList::default(),
332 pending_free: Vec::new(),
333 free_dirty: false,
334 key_slots: KeySlotRun::default(),
335 dirty_keys: false,
336 gen_leased: false,
337 col: VecColumn::default(),
338 recs: RecPages::default(),
339 bm25: Bm25State::default(),
340 kw: None,
341 index_layout_override: None,
342 dirty_rows: std::collections::BTreeSet::new(),
343 delta_chain: Vec::new(),
344 delta_rows: 0,
345 catalog_rewrite: false,
346 index_chain: Vec::new(),
347 index_chain_ops: 0,
348 index_rebase: false,
349 index_delta_ops: None,
350 snapshot_cap_explicit: config.max_snapshots != DEFAULT_MAX_SNAPSHOTS,
351 direct_writes: false,
352 vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
353 })
354 }
355
356 /// Open an existing database. A wrong passphrase fails cleanly with
357 /// [`MnemoError::WrongPassphrase`]. Takes an OS exclusive lock for
358 /// the lifetime of the returned handle; a second concurrent open
359 /// (from any process) returns [`MnemoError::Locked`].
360 ///
361 /// For a caller that only needs to read (and can tolerate the
362 /// pending-recovery / pending-migration cases documented on
363 /// [`Mnemo::open_read_only`]), prefer that entry point — it takes a
364 /// shared lock and coexists with other readers.
365 pub fn open(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
366 Self::open_secret(path, passphrase.as_bytes())
367 }
368
369 /// PERF-1 step 7: open a database created by [`Mnemo::create_with_key`]
370 /// (or rekeyed with [`Mnemo::rekey_with_key`]) with its raw key.
371 pub fn open_with_key(path: impl Into<PathBuf>, key: &[u8; crypto::KEY_LEN]) -> Result<Mnemo> {
372 Self::open_secret(path, key)
373 }
374
375 pub(super) fn open_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
376 Self::open_secret_with(path, secret, false)
377 }
378
379 /// The read-write open. `allow_old` lets a file older than v9 open (and
380 /// migrate in memory to v8): only compaction uses it, to read a v4–v8
381 /// file and write the v9 copy. Every public read-write open refuses
382 /// those with [`MnemoError::FormatTooOld`] (the v8 sunset, v0.13).
383 pub(super) fn open_secret_with(
384 path: impl Into<PathBuf>,
385 secret: &[u8],
386 allow_old: bool,
387 ) -> Result<Mnemo> {
388 let path: PathBuf = path.into();
389 let mut file = open_with_lock(&path, /*write=*/ true, /*create=*/ false)?;
390
391 let mut hbuf = [0u8; PAGE_SIZE];
392 file.seek(SeekFrom::Start(0))?;
393 file.read_exact(&mut hbuf)?;
394 let mut header = match Header::from_page(&hbuf) {
395 Ok(h) => h,
396 Err(e) => {
397 // DUR-1 (R-002.1 part 2+3): page 0 did not parse — any
398 // error is treated as a torn-header candidate, not just
399 // `HeaderChecksum`. The whole 294-byte header lives in
400 // the first 512-byte sector, so the realistic failure is
401 // the sector zeroed or garbled, which returns `BadMagic`
402 // or `UnsupportedVersion` *before* the CRC gate.
403 //
404 // Scan the file for valid committed header-frame WAL
405 // regions, pick the highest `txn_id`, and replay it. If
406 // nothing valid is found, return the *original* error so
407 // non-mnemo files still report `BadMagic`.
408 let scan = scan_header_recovery_candidates(&mut file, secret)?;
409 let frames = scan.ok_or(e)?;
410 // RB-003.3: the scan has checked the key; refuse an old
411 // format before healing anything.
412 if !allow_old {
413 if let Some(h) = page_zero_header(&frames) {
414 if h.version < VERSION_V9 {
415 return Err(MnemoError::FormatTooOld { version: h.version });
416 }
417 }
418 }
419 for (page_no, bytes) in &frames {
420 if bytes.len() != PAGE_SIZE {
421 return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
422 }
423 file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
424 file.write_all(bytes)?;
425 }
426 file.sync_all()?;
427 crate::bump_fsync();
428 file.seek(SeekFrom::Start(0))?;
429 file.read_exact(&mut hbuf)?;
430 Header::from_page(&hbuf)?
431 }
432 };
433
434 // Crash recovery: a committed-but-uncheckpointed transaction is
435 // replayed below. Each frame is a finished page image; the header
436 // frame, replayed to page 0, supersedes the header just read.
437 let pending = wal::recover(
438 &mut file,
439 header.wal_start,
440 header.wal_pages,
441 header.wal_seq,
442 )?;
443
444 // RB-003.3: settle the key and the format before writing anything,
445 // so a wrong secret or a refused old format leaves the file
446 // byte-identical. Both are checked against the header the file will
447 // have once the WAL is replayed (its page-0 frame, else the current
448 // header), since that header is the one the secret must open. No write
449 // the engine makes leaves a pending page-0 frame whose key wrap differs
450 // from page 0's (a rekey logs its new page 0 into a fresh WAL region,
451 // so a cut-off rekey leaves the old secret or the new one, never a
452 // mix), but checking against the replay's header is the right
453 // defence if one ever does.
454 let target = match pending.as_deref().and_then(page_zero_header) {
455 Some(h) => h,
456 None => header.clone(),
457 };
458 let kdf = KdfParams {
459 m_cost: target.m_cost,
460 t_cost: target.t_cost,
461 p_cost: target.p_cost,
462 };
463 let kek = crypto::derive_kek(secret, &target.salt, kdf)?;
464 let dek = crypto::unwrap_dek(&kek, &target.dek_nonce, &target.wrapped_dek)?;
465 if !allow_old && target.version < VERSION_V9 {
466 return Err(MnemoError::FormatTooOld {
467 version: target.version,
468 });
469 }
470
471 if let Some(frames) = pending {
472 for (page_no, bytes) in &frames {
473 if bytes.len() != PAGE_SIZE {
474 return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
475 }
476 file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
477 file.write_all(bytes)?;
478 }
479 file.sync_all()?;
480 crate::bump_fsync();
481 // Re-read the now-current header from the replayed page 0.
482 file.seek(SeekFrom::Start(0))?;
483 file.read_exact(&mut hbuf)?;
484 header = Header::from_page(&hbuf)?;
485 }
486
487 // Validate the v7 header seal now that we have the DEK. Returns
488 // `MnemoError::HeaderTampered` if any of the sealed mutable fields
489 // were rewritten after the last legitimate flush. No-op for v6 and
490 // older — pre-v7 files don't carry a seal.
491 //
492 // DUR-1 (Planner decision 2026-11-01, Option B): a seal failure is
493 // **always** reported as tampering, never auto-healed. R-002.1
494 // part 3's seal-mismatch WAL fallback is withdrawn: the seal
495 // exists for tamper-evidence; silently loading a stale committed
496 // header from the WAL would erase that evidence. A real crash
497 // cannot produce "CRC valid + seal invalid" either, because the
498 // whole 294-byte header lies inside sector 0 which storage
499 // writes atomically — a torn sector breaks magic, version or
500 // CRC first, and *those* are healed by the scan above.
501 header.validate_seal(&dek)?;
502
503 // Build the pager at the file's *current* version so initial reads
504 // use the matching AAD scheme — `Pager::page_aad` returns empty AAD
505 // for v4/v5 (no page binding) and `page_no.to_le_bytes()` for v6+.
506 let on_disk_version = header.version;
507 // The v8 sunset (v0.13): a read-write handle is v9. Older files open
508 // read-only (v8) or through `compact_file`, which converts them.
509 if !allow_old && on_disk_version < VERSION_V9 {
510 return Err(MnemoError::FormatTooOld {
511 version: on_disk_version,
512 });
513 }
514 let mut pager = Pager::new(file, dek, header.write_counter, on_disk_version);
515 let dimensions = header.dimensions as usize;
516
517 // === v4→v5 catalog migration =====================================
518 // v6 reads directly; v5 reads directly; v4 replays the catalog into
519 // the v5 shape by reading each memory's body to populate the new
520 // `accessed_at` / `access_count` fields. All page reads here still
521 // use the v4/v5 AAD scheme (none).
522 let (catalog, migrated_from_v4) = if on_disk_version >= 5 {
523 (
524 load_catalog(
525 &mut pager,
526 header.catalog_start,
527 header.catalog_pages,
528 header.catalog_len,
529 header.catalog_gen,
530 )?,
531 false,
532 )
533 } else if on_disk_version == 4 {
534 let v4 = load_catalog_v4(
535 &mut pager,
536 header.catalog_start,
537 header.catalog_pages,
538 header.catalog_len,
539 )?;
540 (migrate_v4_catalog(&mut pager, v4)?, true)
541 } else {
542 return Err(MnemoError::UnsupportedVersion(on_disk_version));
543 };
544 // FMT-9c.2: replay the v9 delta chain onto the base catalog.
545 let (catalog, delta_chain, delta_rows) =
546 apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
547 let index = build_id_index(&catalog);
548 // PERF-1 step 3: the ANN index loads on first use, not at open. A
549 // v5→v6 page migration is the exception: it must read the index
550 // now, under the old page AAD, to rewrite it.
551 let ann = if on_disk_version < 6 {
552 load_index(
553 &mut pager,
554 header.index_start,
555 header.index_pages,
556 header.index_len,
557 header.index_gen,
558 dimensions,
559 )?
560 } else {
561 None
562 };
563 let ann_unloaded = ann.is_none() && header.index_pages > 0;
564 let mut manifest = load_manifest(
565 &mut pager,
566 header.manifest_start,
567 header.manifest_pages,
568 header.manifest_len,
569 header.manifest_gen,
570 )?;
571
572 // === v5→v6 page-encryption migration =============================
573 // Pre-v6 data pages were encrypted under no AAD; v6 binds the home
574 // `page_no` as AAD so a page-transplant attack fails authentication
575 // on read. Re-stage every live record page through `write_page` so
576 // the next flush re-encrypts it under v6 AAD at the same home slot.
577 // Tombstoned entries skip — they're reclaimed by `compact_file`.
578 // Old catalog / ANN / manifest pages stay as orphan ciphertext;
579 // the next flush writes the new runs at fresh slots.
580 let migrated_pages_to_v6 = on_disk_version < 6;
581 if migrated_pages_to_v6 {
582 for e in &catalog {
583 if e.deleted {
584 continue;
585 }
586 for i in 0..e.page_count as u64 {
587 let payload = pager.read_page(e.start_page + i, 0)?;
588 pager.write_page(e.start_page + i, 0, &payload)?;
589 }
590 }
591 pager.set_version(6);
592 }
593
594 // === v6→v7 header-seal migration =================================
595 // v7 adds an AES-GCM seal at the tail of the header page that
596 // authenticates every mutable header field. Pre-v7 files have no
597 // seal — we just bump the version in memory and mark `dirty_catalog`
598 // so the next flush writes a sealed v7 header. Pages are untouched
599 // (page crypto is unchanged from v6).
600 let migrated_header_to_v7 = on_disk_version < 7;
601 if migrated_header_to_v7 {
602 pager.set_version(7);
603 }
604
605 // === v7→v8 cache-directory migration =============================
606 // v8 adds three u64 header fields (cache_start/pages/len) for the
607 // result-cache directory (Phase 10.1). Pre-v8 files have those
608 // three bytes as zero (the on-disk region beyond the v7 seal is
609 // zero-initialized), which is the legal "empty cache directory"
610 // state — no data migration required. We bump the version in
611 // memory and mark dirty_catalog so the next flush writes a
612 // v8-sealed header with the new AAD shape covering the cache
613 // pointer. Pages are untouched.
614 let migrated_header_to_v8 = on_disk_version < 8;
615 if migrated_header_to_v8 {
616 pager.set_version(VERSION);
617 }
618
619 let migrated = migrated_from_v4
620 || migrated_pages_to_v6
621 || migrated_header_to_v7
622 || migrated_header_to_v8;
623 if migrated {
624 // Stamp the version forward in memory; the next flush persists
625 // it via the WAL-committed header frame (and the v7 seal).
626 header.version = VERSION;
627 // Snapshots written under an older format reference page runs
628 // encrypted under the old AAD scheme; this build can't decrypt
629 // them under the new pager. Drop the manifest so the next
630 // flush records a fresh snapshot of the migrated state.
631 // Point-in-time recovery into the pre-migration past is
632 // sacrificed for migration simplicity. Old pages remain on
633 // disk until `compact_file`. (For v6→v7 alone — which doesn't
634 // change page crypto — the old snapshots would technically
635 // still be readable, but dropping them uniformly keeps the
636 // migration policy simple.)
637 manifest.clear();
638 }
639
640 // The ANN index pages on disk were sealed under the *old* AAD scheme
641 // if we migrated pages to v6; force a rewrite at fresh v6-encrypted
642 // slots on the next flush so a future reopen doesn't try to read
643 // them under the new AAD and fail. (Catalog and manifest already
644 // get rewritten via dirty_catalog and the manifest.clear() above.)
645 let dirty_index = migrated_pages_to_v6 && ann.is_some();
646
647 // Load the cache directory (v8+ only). Pre-v8 files have
648 // cache_pages == 0 by header layout, so this trivially yields
649 // an empty directory without touching the disk.
650 let cache_dir = load_cache_directory(
651 &mut pager,
652 header.cache_start,
653 header.cache_pages,
654 header.cache_len,
655 header.cache_gen,
656 )?;
657 let cache_idx = DirectoryIndex::build(&cache_dir);
658 // FMT-9 (a)/(b): the free list and key-slot run (empty pointers on
659 // v8, so these don't touch the disk there).
660 let free = load_free_list(&mut pager, header.free)?;
661 let key_slots = load_key_slots(&mut pager, header.keys)?;
662 let col = VecColumn::from_catalog(&catalog);
663 let recs = RecPages::from_catalog(&catalog);
664 let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
665 let max_snapshots = if header.version >= VERSION_V9 {
666 V9_DEFAULT_MAX_SNAPSHOTS
667 } else {
668 DEFAULT_MAX_SNAPSHOTS
669 };
670
671 Ok(Mnemo {
672 pager,
673 header,
674 catalog,
675 index,
676 path,
677 dimensions,
678 kdf,
679 dirty_catalog: migrated,
680 ann,
681 dirty_index,
682 // A v5→v6 page migration leaves the index sealed under the old
683 // page AAD, so it must be rewritten on the next flush.
684 index_persist: IndexPersist {
685 forced: dirty_index,
686 ..IndexPersist::default()
687 },
688 ann_unloaded,
689 manifest,
690 // `Mnemo::open` has no [`MnemoConfig`] to read; default the cap
691 // to [`DEFAULT_MAX_SNAPSHOTS`]. Override with
692 // [`Mnemo::set_max_snapshots`] right after open if you need
693 // unlimited retention or a different cap for this handle.
694 max_snapshots,
695 read_only: false,
696 cache_dir,
697 cache_idx,
698 dirty_cache: false,
699 cache_budgets: std::collections::HashMap::new(),
700 cache_flush_policy: CacheFlushPolicy::default(),
701 cache_batch: BatchState::default(),
702 cache_counters: CounterState::default(),
703 embedder: None,
704 fsyncs_last_flush: 0,
705 agent_set: None,
706 key_index: None,
707 superseded: None,
708 layers: None,
709 turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
710 free,
711 pending_free: Vec::new(),
712 free_dirty: false,
713 key_slots,
714 dirty_keys: false,
715 gen_leased: false,
716 col,
717 recs,
718 bm25: Bm25State {
719 manifest: bm25_manifest,
720 ..Bm25State::default()
721 },
722 kw: None,
723 index_layout_override: None,
724 dirty_rows: std::collections::BTreeSet::new(),
725 delta_chain,
726 delta_rows,
727 catalog_rewrite: false,
728 index_chain: Vec::new(),
729 index_chain_ops: 0,
730 index_rebase: false,
731 index_delta_ops: None,
732 snapshot_cap_explicit: false,
733 direct_writes: false,
734 vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
735 })
736 }
737
738 /// Open an existing database in **read-only** mode. Takes an OS shared
739 /// lock; multiple read-only handles can coexist, but any attempt to
740 /// open the same file read-write elsewhere returns
741 /// [`MnemoError::Locked`]. All mutating methods on the returned handle
742 /// return [`MnemoError::ReadOnly`], including `recall` with
743 /// `track_access(true)`.
744 ///
745 /// Two conditions block a read-only open and return
746 /// [`MnemoError::NeedsWriteOpen`]:
747 ///
748 /// 1. The file has a committed-but-uncheckpointed WAL transaction that
749 /// would ordinarily be replayed on open. A read-only handle
750 /// cannot write the recovered pages back.
751 /// 2. The on-disk format version is older than the current
752 /// `format::VERSION`, requiring an on-disk migration.
753 ///
754 /// In either case, open the file read-write once (which recovers /
755 /// migrates in place), close the handle cleanly, and reopen
756 /// read-only. This is documented in the README's Durability section.
757 pub fn open_read_only(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
758 Self::open_read_only_secret(path, passphrase.as_bytes())
759 }
760
761 /// PERF-1 step 7: [`Mnemo::open_read_only`] for a raw-key database.
762 pub fn open_read_only_with_key(
763 path: impl Into<PathBuf>,
764 key: &[u8; crypto::KEY_LEN],
765 ) -> Result<Mnemo> {
766 Self::open_read_only_secret(path, key)
767 }
768
769 /// Whether `key` opens the raw-key file at `path`, decided from its
770 /// header alone: no catalog load, no lock, microseconds. `Ok(false)` is
771 /// "not this key". Unlike a read-only open, it answers for a file that
772 /// has a WAL transaction to recover (the state a kill between a commit
773 /// and its checkpoint leaves), so a caller trying each of its keys in
774 /// turn isn't stopped by [`MnemoError::NeedsWriteOpen`]. A torn header is
775 /// judged from the WAL frame that would heal it. Other failures (not a
776 /// `.mnemo` file, a tampered header, an I/O error) are errors. Added in
777 /// v0.12.2 (RB-001.23).
778 pub fn key_opens(
779 path: impl AsRef<std::path::Path>,
780 key: &[u8; crypto::KEY_LEN],
781 ) -> Result<bool> {
782 let mut file = File::open(path.as_ref())?;
783 let mut hbuf = [0u8; PAGE_SIZE];
784 file.seek(SeekFrom::Start(0))?;
785 file.read_exact(&mut hbuf)?;
786 let header = match Header::from_page(&hbuf) {
787 Ok(h) => h,
788 // A candidate frame only counts when it validates with `key`.
789 Err(parse_err) => {
790 return match scan_header_recovery_candidates(&mut file, key)? {
791 Some(_) => Ok(true),
792 None => Err(parse_err),
793 }
794 }
795 };
796 let kdf = KdfParams {
797 m_cost: header.m_cost,
798 t_cost: header.t_cost,
799 p_cost: header.p_cost,
800 };
801 let kek = crypto::derive_kek(key, &header.salt, kdf)?;
802 let dek = match crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek) {
803 Ok(dek) => dek,
804 Err(MnemoError::WrongPassphrase) => return Ok(false),
805 Err(e) => return Err(e),
806 };
807 header.validate_seal(&dek)?;
808 Ok(true)
809 }
810
811 pub(super) fn open_read_only_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
812 let path: PathBuf = path.into();
813 let mut file = open_with_lock(&path, /*write=*/ false, /*create=*/ false)?;
814
815 let mut hbuf = [0u8; PAGE_SIZE];
816 file.seek(SeekFrom::Start(0))?;
817 file.read_exact(&mut hbuf)?;
818 // A torn header on a read-only open cannot be healed (that requires
819 // writing recovered frames back). Surface as `NeedsWriteOpen` so the
820 // caller knows to reopen read-write once. DUR-1 (R-002.1 part 4):
821 // any `Header::from_page` failure (`BadMagic`,
822 // `UnsupportedVersion`, `HeaderChecksum`) qualifies — but only
823 // when the file-wide scan actually finds a valid committed
824 // header frame to recover from, so a non-mnemo file still
825 // reports `BadMagic` cleanly. A `validate_seal` failure is NOT
826 // routed here (Option B: tampering is never auto-healed).
827 let header = match Header::from_page(&hbuf) {
828 Ok(h) => h,
829 Err(parse_err) => {
830 let recoverable = scan_header_recovery_candidates(&mut file, secret)?;
831 return Err(match (parse_err, recoverable) {
832 (_, Some(_)) => MnemoError::NeedsWriteOpen {
833 reason: "torn header page needs recovery from WAL",
834 },
835 (e, None) => e,
836 });
837 }
838 };
839
840 // Refuse to migrate on read-only opens. v6→v7 is header-only
841 // (no page writes required), but even that bumps `version` on
842 // disk and re-seals the header — which needs write access. Keep
843 // the policy uniform across all sub-version bumps.
844 // A v4–v7 file needs migrating, which a read-only handle can't do,
845 // and since v0.13 no read-write handle opens it either: compaction
846 // converts it to v9.
847 if header.version < VERSION {
848 return Err(MnemoError::FormatTooOld {
849 version: header.version,
850 });
851 }
852
853 // Refuse if there is a committed-but-uncheckpointed transaction
854 // waiting for `wal::recover` to fold it into home pages.
855 if wal::recover(
856 &mut file,
857 header.wal_start,
858 header.wal_pages,
859 header.wal_seq,
860 )?
861 .is_some()
862 {
863 // RB-003.3: a pre-v9 file can't be opened read-write any more,
864 // so the usual advice would send the user in a circle.
865 // Compaction recovers the WAL and converts the file.
866 return Err(MnemoError::NeedsWriteOpen {
867 reason: if header.version < VERSION_V9 {
868 "pending WAL recovery in a pre-v9 file; run `mnemo compact <file>` \
869 (with `--key-file <key>` for a raw-key file), which recovers it and \
870 converts the file to v9"
871 } else {
872 "pending WAL recovery"
873 },
874 });
875 }
876
877 let kdf = KdfParams {
878 m_cost: header.m_cost,
879 t_cost: header.t_cost,
880 p_cost: header.p_cost,
881 };
882 let kek = crypto::derive_kek(secret, &header.salt, kdf)?;
883 let dek = crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek)?;
884 header.validate_seal(&dek)?;
885
886 let mut pager = Pager::new(file, dek, header.write_counter, header.version);
887 let dimensions = header.dimensions as usize;
888
889 let catalog = load_catalog(
890 &mut pager,
891 header.catalog_start,
892 header.catalog_pages,
893 header.catalog_len,
894 header.catalog_gen,
895 )?;
896 // FMT-9c.2: replay the v9 delta chain onto the base catalog.
897 let (catalog, delta_chain, delta_rows) =
898 apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
899 let index = build_id_index(&catalog);
900 // PERF-1 step 3: the ANN index loads on first use, not at open.
901 let ann: Option<IvfPqIndex> = None;
902 let ann_unloaded = header.index_pages > 0;
903 let manifest = load_manifest(
904 &mut pager,
905 header.manifest_start,
906 header.manifest_pages,
907 header.manifest_len,
908 header.manifest_gen,
909 )?;
910 let cache_dir = load_cache_directory(
911 &mut pager,
912 header.cache_start,
913 header.cache_pages,
914 header.cache_len,
915 header.cache_gen,
916 )?;
917 let cache_idx = DirectoryIndex::build(&cache_dir);
918 let free = load_free_list(&mut pager, header.free)?;
919 let key_slots = load_key_slots(&mut pager, header.keys)?;
920 let col = VecColumn::from_catalog(&catalog);
921 let recs = RecPages::from_catalog(&catalog);
922 let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
923 let max_snapshots = if header.version >= VERSION_V9 {
924 V9_DEFAULT_MAX_SNAPSHOTS
925 } else {
926 DEFAULT_MAX_SNAPSHOTS
927 };
928
929 Ok(Mnemo {
930 pager,
931 header,
932 catalog,
933 index,
934 path,
935 dimensions,
936 kdf,
937 // No mutation is possible; leave both dirty flags false so
938 // `close()` is a genuine no-op even if we later add an
939 // auto-flush on drop.
940 dirty_catalog: false,
941 ann,
942 dirty_index: false,
943 index_persist: IndexPersist::default(),
944 ann_unloaded,
945 manifest,
946 max_snapshots,
947 read_only: true,
948 cache_dir,
949 cache_idx,
950 dirty_cache: false,
951 cache_budgets: std::collections::HashMap::new(),
952 cache_flush_policy: CacheFlushPolicy::default(),
953 cache_batch: BatchState::default(),
954 cache_counters: CounterState::default(),
955 embedder: None,
956 fsyncs_last_flush: 0,
957 agent_set: None,
958 key_index: None,
959 superseded: None,
960 layers: None,
961 turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
962 free,
963 pending_free: Vec::new(),
964 free_dirty: false,
965 key_slots,
966 dirty_keys: false,
967 gen_leased: false,
968 col,
969 recs,
970 bm25: Bm25State {
971 manifest: bm25_manifest,
972 ..Bm25State::default()
973 },
974 kw: None,
975 index_layout_override: None,
976 dirty_rows: std::collections::BTreeSet::new(),
977 delta_chain,
978 delta_rows,
979 catalog_rewrite: false,
980 index_chain: Vec::new(),
981 index_chain_ops: 0,
982 index_rebase: false,
983 index_delta_ops: None,
984 snapshot_cap_explicit: false,
985 direct_writes: false,
986 vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
987 })
988 }
989
990 /// FMT-9e: compress v9 thread-turn records once they're older than
991 /// `secs` (by their `created_at`). The default,
992 /// [`DEFAULT_TURN_COMPRESS_AFTER_SECS`], is 7 days: recent turns stay
993 /// plain, so a live session reads them at full speed, and each commit
994 /// compresses up to 256 turns that have aged past the cutoff. `0`
995 /// compresses every turn as it's written. Compaction always compresses
996 /// every turn, and so does a vacuum that moves one. Applies to this
997 /// handle only (not persisted); a no-op on v8 files, which never
998 /// compress. Reading never depends on it.
999 pub fn set_turn_compress_after(&mut self, secs: u64) {
1000 self.turn_compress_after = secs.min(i64::MAX as u64) as i64;
1001 }
1002
1003 /// Override the manifest snapshot cap on this open handle. `0` disables
1004 /// the cap; any positive value keeps the most-recent `max` snapshots and
1005 /// drops the rest on the next flush. See [`MnemoConfig::max_snapshots`].
1006 pub fn set_max_snapshots(&mut self, max: usize) {
1007 self.max_snapshots = max;
1008 // FMT-9c: an explicit count is a hard cap on v9 (see
1009 // `apply_retention`).
1010 self.snapshot_cap_explicit = true;
1011 }
1012
1013 /// Embedding dimensionality this database expects.
1014 pub fn dimensions(&self) -> usize {
1015 self.dimensions
1016 }
1017
1018 /// Number of live (non-deleted) memories.
1019 pub fn len(&self) -> usize {
1020 self.catalog.iter().filter(|e| !e.deleted).count()
1021 }
1022
1023 /// True if the database holds no live memories.
1024 pub fn is_empty(&self) -> bool {
1025 self.len() == 0
1026 }
1027
1028 // --- internal page/record helpers -------------------------------------
1029}