mnemo_engine/store/open.rs
1//! Create, open (read-write, read-only, raw key) and the handle's basic accessors and per-handle settings.
2
3use super::*;
4
5/// Chunk size for the torn-header scan's streaming read of the file.
6/// We walk the file 1 MiB at a time looking for 8 KiB-aligned `MWAL`
7/// magic. There is **no** cap on how far a given candidate's frame
8/// chain may extend — a 1M-vector IVF+PQ relocation can push a single
9/// transaction well past 16 MiB — so each candidate is read frame by
10/// frame via [`wal::scan_committed_txn`] rather than slurped into a
11/// bounded buffer. (DUR-1 R-002.1 part 2.)
12const TORN_HEADER_SCAN_CHUNK: usize = 1 << 20; // 1 MiB
13
14/// Walk the file for every 8 KiB-aligned page whose first four bytes are
15/// `MWAL`, treating each as a candidate WAL-region start, and return the
16/// frames of the highest-`txn_id` committed transaction whose header
17/// frame fully validates under `passphrase`.
18///
19/// A candidate is accepted only if:
20/// * `wal::scan_committed_txn` returns a committed transaction (chained
21/// CRC verifies, single `txn_id`, `KIND_COMMIT` closes it);
22/// * that transaction contains a `(0, header_page)` frame;
23/// * the header parses with `wal_start == candidate_page_no` and
24/// `wal_seq == txn_id` (so a frame cannot forge a region it was not
25/// written into);
26/// * deriving the KEK from `passphrase + header.salt`, unwrapping the
27/// wrapped DEK, and validating the header seal all succeed.
28///
29/// Returns `Ok(None)` when the scan finds no valid candidate — the
30/// caller is expected to propagate the original parse error so
31/// non-mnemo files still report `BadMagic`.
32///
33/// DUR-1 (R-002.1 part 2): there is no fixed scan window — the whole
34/// file is scanned. Reading at page granularity (not byte granularity)
35/// is sufficient because every WAL region in the format is page-aligned
36/// and every WAL commit writes its first frame at the region's start.
37fn scan_header_recovery_candidates(
38 file: &mut File,
39 secret: &[u8],
40) -> Result<Option<Vec<wal::Frame>>> {
41 let file_len = file.seek(SeekFrom::End(0))?;
42 if file_len < PAGE_SIZE as u64 * 2 {
43 return Ok(None); // No room for even one WAL page past the header.
44 }
45
46 let mut best: Option<(u64, Vec<wal::Frame>)> = None;
47 let page_sz = PAGE_SIZE as u64;
48 let mut offset = page_sz; // Skip page 0.
49
50 // Stream through the file in 1 MiB chunks, scanning each chunk for
51 // page-aligned `MWAL` candidates. Each candidate's frame chain is
52 // then read incrementally via `wal::scan_committed_txn`, which has
53 // no window cap — a 1M-vector IVF+PQ relocation can span tens of
54 // MiB in one committed transaction.
55 let mut chunk = vec![0u8; TORN_HEADER_SCAN_CHUNK];
56 while offset < file_len {
57 let want = (TORN_HEADER_SCAN_CHUNK as u64).min(file_len - offset) as usize;
58 chunk.resize(want, 0);
59 file.seek(SeekFrom::Start(offset))?;
60 let mut filled = 0;
61 while filled < want {
62 match file.read(&mut chunk[filled..])? {
63 0 => break,
64 n => filled += n,
65 }
66 }
67 if filled == 0 {
68 break;
69 }
70 let mut inner = 0usize;
71 while inner + 4 <= filled {
72 if &chunk[inner..inner + 4] == wal::FRAME_MAGIC_BYTES {
73 let page_no = (offset + inner as u64) / page_sz;
74 if let Some((txn_id, frames)) = try_recover_candidate(file, page_no, secret)? {
75 let keep = match &best {
76 None => true,
77 Some((id, _)) => txn_id > *id,
78 };
79 if keep {
80 best = Some((txn_id, frames));
81 }
82 }
83 }
84 inner += PAGE_SIZE;
85 }
86 offset += filled as u64;
87 }
88
89 Ok(best.map(|(_, f)| f))
90}
91
92/// Try to recover a committed transaction from `candidate_page_no`,
93/// returning `(txn_id, frames)` only if the candidate's
94/// `(0, header_page)` frame fully validates: the frame chain commits
95/// (via [`wal::scan_committed_txn`]'s streaming reader), the header
96/// parses with `wal_start == candidate_page_no` **and**
97/// `wal_seq == txn_id`, the DEK unwraps under `passphrase`, and the
98/// AES-GCM seal validates. DUR-1 (R-002.1 item 2) requires the
99/// `wal_seq == txn_id` check and no fixed scan window — the streaming
100/// reader handles transactions of arbitrary size.
101fn try_recover_candidate(
102 file: &mut File,
103 candidate_page_no: u64,
104 secret: &[u8],
105) -> Result<Option<(u64, Vec<wal::Frame>)>> {
106 let Some((txn_id, frames)) = wal::scan_committed_txn(file, candidate_page_no)? else {
107 return Ok(None);
108 };
109
110 // Must contain a header frame at page 0.
111 let header_bytes = match frames.iter().find(|(p, _)| *p == 0) {
112 Some((_, b)) => b,
113 None => return Ok(None),
114 };
115 if header_bytes.len() != PAGE_SIZE {
116 return Ok(None);
117 }
118 let mut hbuf = [0u8; PAGE_SIZE];
119 hbuf.copy_from_slice(header_bytes);
120 let candidate = match Header::from_page(&hbuf) {
121 Ok(h) => h,
122 Err(_) => return Ok(None),
123 };
124
125 // The frame's claimed region must match where we found it, and the
126 // header frame's `wal_seq` must match the committed transaction's
127 // `txn_id` — this is what pins the frame to the region it was
128 // written into and keeps a stale checkpointed-and-copied frame
129 // from being accepted as a new region's root.
130 if candidate.wal_start != candidate_page_no || candidate.wal_seq != txn_id {
131 return Ok(None);
132 }
133 // DEK unwrap under this candidate's own salt + KDF params.
134 let kdf = KdfParams {
135 m_cost: candidate.m_cost,
136 t_cost: candidate.t_cost,
137 p_cost: candidate.p_cost,
138 };
139 let kek = match crypto::derive_kek(secret, &candidate.salt, kdf) {
140 Ok(k) => k,
141 Err(_) => return Ok(None),
142 };
143 let dek = match crypto::unwrap_dek(&kek, &candidate.dek_nonce, &candidate.wrapped_dek) {
144 Ok(d) => d,
145 Err(_) => return Ok(None),
146 };
147 if candidate.validate_seal(&dek).is_err() {
148 return Ok(None);
149 }
150
151 Ok(Some((txn_id, frames)))
152}
153
154impl Mnemo {
155 /// Create a brand-new encrypted database at `path`.
156 pub fn create(
157 path: impl Into<PathBuf>,
158 passphrase: &str,
159 config: MnemoConfig,
160 ) -> Result<Mnemo> {
161 Self::create_secret(path, passphrase.as_bytes(), config)
162 }
163
164 /// PERF-1 step 7 (R-004.5): create a database unlocked by a **raw
165 /// 256-bit key** instead of a passphrase, for keys held by an OS
166 /// keychain or keystore. The key must be uniformly random (see
167 /// [`Mnemo::generate_key`]); it gets no stretching. The header records
168 /// minimal Argon2id parameters ([`KdfParams::raw_key`]), so opening
169 /// costs microseconds instead of tens of milliseconds. `config.kdf` is
170 /// ignored. No format change: the key goes through the same KEK
171 /// derivation a passphrase does.
172 pub fn create_with_key(
173 path: impl Into<PathBuf>,
174 key: &[u8; crypto::KEY_LEN],
175 config: MnemoConfig,
176 ) -> Result<Mnemo> {
177 let config = MnemoConfig {
178 kdf: KdfParams::raw_key(),
179 ..config
180 };
181 Self::create_secret(path, key, config)
182 }
183
184 /// A fresh random 256-bit key for [`Mnemo::create_with_key`], from the
185 /// thread-local CSPRNG (seeded from the OS). Store it in a keychain;
186 /// it can't be recovered from the file.
187 pub fn generate_key() -> [u8; crypto::KEY_LEN] {
188 *crypto::random_dek()
189 }
190
191 pub(super) fn create_secret(
192 path: impl Into<PathBuf>,
193 secret: &[u8],
194 config: MnemoConfig,
195 ) -> Result<Mnemo> {
196 let path: PathBuf = path.into();
197 if config.dimensions == 0 {
198 return Err(MnemoError::Invalid("dimensions must be > 0".into()));
199 }
200 let version = resolve_format_version(config.format_version)?;
201 if version >= VERSION_V9 && veccol::per_page(config.dimensions) == 0 {
202 return Err(MnemoError::Invalid(format!(
203 "a v9 file supports up to {} dimensions for now (one fp16 vector \
204 must fit a page); vector entries that span pages are not supported yet",
205 (PAYLOAD - veccol::PAGE_HEADER - 4) / 2
206 )));
207 }
208 let flags = if version >= VERSION_V9 && config.store_float_vectors {
209 FLAG_ENCRYPTED | FLAG_FLOAT_VECTORS
210 } else {
211 FLAG_ENCRYPTED
212 };
213 // Exclusive lock covers the write path; second writer sees `Locked`.
214 let file = open_with_lock(&path, /*write=*/ true, /*create=*/ true)?;
215
216 let salt = crypto::random_salt();
217 let dek = crypto::random_dek();
218 let kek = crypto::derive_kek(secret, &salt, config.kdf)?;
219 let dek_nonce = crypto::random_nonce();
220 let wrapped = crypto::wrap_dek(&kek, &dek_nonce, &dek)?;
221 if wrapped.len() != WRAPPED_DEK_LEN {
222 return Err(MnemoError::Crypto("unexpected wrapped DEK length".into()));
223 }
224 let mut wrapped_dek = [0u8; WRAPPED_DEK_LEN];
225 wrapped_dek.copy_from_slice(&wrapped);
226
227 let mut header = Header {
228 version,
229 page_size: PAGE_SIZE as u32,
230 flags,
231 dimensions: config.dimensions as u32,
232 created_at: memory::now_secs(),
233 write_counter: 0,
234 // Page 0 is the header; pages 1..=wal_pages are the WAL;
235 // record/catalog/index pages start after it. Initial WAL size is
236 // chosen per-config (default 8 pages = 64 KiB) and clamped to a
237 // sane floor so the first transaction can always commit.
238 next_page: 1 + config.wal_pages_initial.max(MIN_WAL_PAGES),
239 catalog_start: 0,
240 catalog_pages: 0,
241 catalog_len: 0,
242 vector_count: 0,
243 m_cost: config.kdf.m_cost,
244 t_cost: config.kdf.t_cost,
245 p_cost: config.kdf.p_cost,
246 salt,
247 dek_nonce,
248 wrapped_dek,
249 index_start: 0,
250 index_pages: 0,
251 index_len: 0,
252 wal_start: 1,
253 wal_pages: config.wal_pages_initial.max(MIN_WAL_PAGES),
254 wal_seq: 0,
255 manifest_start: 0,
256 manifest_pages: 0,
257 manifest_len: 0,
258 // Regenerated per write by `apply_seal`; `None` until we seal.
259 seal_nonce: None,
260 seal_tag: None,
261 // v8+ result-cache pointer — zero-initialized on create,
262 // populated only when `cache_put` runs at least once and
263 // the next `flush` seals a directory run.
264 cache_start: 0,
265 cache_pages: 0,
266 cache_len: 0,
267 // FMT-9 v9 fields; all zero on a fresh file (and unused on v8).
268 catalog_gen: 0,
269 index_gen: 0,
270 manifest_gen: 0,
271 cache_gen: 0,
272 free: RunPtr::default(),
273 keys: RunPtr::default(),
274 bm25: RunPtr::default(),
275 index_delta: RunPtr::default(),
276 catalog_delta: RunPtr::default(),
277 };
278
279 let mut pager = Pager::new(file, dek, 0, version);
280 let mut page = header.to_page();
281 header.apply_seal(&mut page, pager.dek())?;
282 pager.write_raw(0, &page)?;
283 pager.sync()?;
284
285 Ok(Mnemo {
286 pager,
287 header,
288 catalog: Vec::new(),
289 index: IdIndex::default(),
290 path,
291 dimensions: config.dimensions,
292 kdf: config.kdf,
293 dirty_catalog: false,
294 ann: None,
295 dirty_index: false,
296 index_persist: IndexPersist::default(),
297 ann_unloaded: false,
298 manifest: Vec::new(),
299 // FMT-9c: v9 keeps fewer snapshots by default: with page reuse,
300 // retention is what pins pages.
301 max_snapshots: if version >= VERSION_V9 && config.max_snapshots == DEFAULT_MAX_SNAPSHOTS
302 {
303 V9_DEFAULT_MAX_SNAPSHOTS
304 } else {
305 config.max_snapshots
306 },
307 read_only: false,
308 cache_dir: Vec::new(),
309 cache_idx: DirectoryIndex::new(),
310 dirty_cache: false,
311 cache_budgets: std::collections::HashMap::new(),
312 cache_flush_policy: CacheFlushPolicy::default(),
313 cache_batch: BatchState::default(),
314 cache_counters: CounterState::default(),
315 embedder: None,
316 fsyncs_last_flush: 0,
317 agent_set: None,
318 key_index: None,
319 superseded: None,
320 layers: None,
321 turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
322 free: FreeList::default(),
323 pending_free: Vec::new(),
324 free_dirty: false,
325 key_slots: KeySlotRun::default(),
326 dirty_keys: false,
327 gen_leased: false,
328 col: VecColumn::default(),
329 recs: RecPages::default(),
330 bm25: Bm25State::default(),
331 kw: None,
332 index_layout_override: None,
333 dirty_rows: std::collections::BTreeSet::new(),
334 delta_chain: Vec::new(),
335 delta_rows: 0,
336 catalog_rewrite: false,
337 index_chain: Vec::new(),
338 index_chain_ops: 0,
339 index_rebase: false,
340 index_delta_ops: None,
341 snapshot_cap_explicit: config.max_snapshots != DEFAULT_MAX_SNAPSHOTS,
342 direct_writes: false,
343 vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
344 })
345 }
346
347 /// Open an existing database. A wrong passphrase fails cleanly with
348 /// [`MnemoError::WrongPassphrase`]. Takes an OS exclusive lock for
349 /// the lifetime of the returned handle; a second concurrent open
350 /// (from any process) returns [`MnemoError::Locked`].
351 ///
352 /// For a caller that only needs to read (and can tolerate the
353 /// pending-recovery / pending-migration cases documented on
354 /// [`Mnemo::open_read_only`]), prefer that entry point — it takes a
355 /// shared lock and coexists with other readers.
356 pub fn open(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
357 Self::open_secret(path, passphrase.as_bytes())
358 }
359
360 /// PERF-1 step 7: open a database created by [`Mnemo::create_with_key`]
361 /// (or rekeyed with [`Mnemo::rekey_with_key`]) with its raw key.
362 pub fn open_with_key(path: impl Into<PathBuf>, key: &[u8; crypto::KEY_LEN]) -> Result<Mnemo> {
363 Self::open_secret(path, key)
364 }
365
366 pub(super) fn open_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
367 let path: PathBuf = path.into();
368 let mut file = open_with_lock(&path, /*write=*/ true, /*create=*/ false)?;
369
370 let mut hbuf = [0u8; PAGE_SIZE];
371 file.seek(SeekFrom::Start(0))?;
372 file.read_exact(&mut hbuf)?;
373 let mut header = match Header::from_page(&hbuf) {
374 Ok(h) => h,
375 Err(e) => {
376 // DUR-1 (R-002.1 part 2+3): page 0 did not parse — any
377 // error is treated as a torn-header candidate, not just
378 // `HeaderChecksum`. The whole 294-byte header lives in
379 // the first 512-byte sector, so the realistic failure is
380 // the sector zeroed or garbled, which returns `BadMagic`
381 // or `UnsupportedVersion` *before* the CRC gate.
382 //
383 // Scan the file for valid committed header-frame WAL
384 // regions, pick the highest `txn_id`, and replay it. If
385 // nothing valid is found, return the *original* error so
386 // non-mnemo files still report `BadMagic`.
387 let scan = scan_header_recovery_candidates(&mut file, secret)?;
388 let frames = scan.ok_or(e)?;
389 for (page_no, bytes) in &frames {
390 if bytes.len() != PAGE_SIZE {
391 return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
392 }
393 file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
394 file.write_all(bytes)?;
395 }
396 file.sync_all()?;
397 crate::bump_fsync();
398 file.seek(SeekFrom::Start(0))?;
399 file.read_exact(&mut hbuf)?;
400 Header::from_page(&hbuf)?
401 }
402 };
403
404 // Crash recovery: replay a committed-but-uncheckpointed transaction.
405 // Each frame is a finished page image; the header frame, replayed to
406 // page 0, supersedes the header just read.
407 if let Some(frames) = wal::recover(
408 &mut file,
409 header.wal_start,
410 header.wal_pages,
411 header.wal_seq,
412 )? {
413 for (page_no, bytes) in &frames {
414 if bytes.len() != PAGE_SIZE {
415 return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
416 }
417 file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
418 file.write_all(bytes)?;
419 }
420 file.sync_all()?;
421 crate::bump_fsync();
422 // Re-read the now-current header from the replayed page 0.
423 file.seek(SeekFrom::Start(0))?;
424 file.read_exact(&mut hbuf)?;
425 header = Header::from_page(&hbuf)?;
426 }
427
428 let kdf = KdfParams {
429 m_cost: header.m_cost,
430 t_cost: header.t_cost,
431 p_cost: header.p_cost,
432 };
433 let kek = crypto::derive_kek(secret, &header.salt, kdf)?;
434 let dek = crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek)?;
435
436 // Validate the v7 header seal now that we have the DEK. Returns
437 // `MnemoError::HeaderTampered` if any of the sealed mutable fields
438 // were rewritten after the last legitimate flush. No-op for v6 and
439 // older — pre-v7 files don't carry a seal.
440 //
441 // DUR-1 (Planner decision 2026-11-01, Option B): a seal failure is
442 // **always** reported as tampering, never auto-healed. R-002.1
443 // part 3's seal-mismatch WAL fallback is withdrawn: the seal
444 // exists for tamper-evidence; silently loading a stale committed
445 // header from the WAL would erase that evidence. A real crash
446 // cannot produce "CRC valid + seal invalid" either, because the
447 // whole 294-byte header lies inside sector 0 which storage
448 // writes atomically — a torn sector breaks magic, version or
449 // CRC first, and *those* are healed by the scan above.
450 header.validate_seal(&dek)?;
451
452 // Build the pager at the file's *current* version so initial reads
453 // use the matching AAD scheme — `Pager::page_aad` returns empty AAD
454 // for v4/v5 (no page binding) and `page_no.to_le_bytes()` for v6+.
455 let on_disk_version = header.version;
456 let mut pager = Pager::new(file, dek, header.write_counter, on_disk_version);
457 let dimensions = header.dimensions as usize;
458
459 // === v4→v5 catalog migration =====================================
460 // v6 reads directly; v5 reads directly; v4 replays the catalog into
461 // the v5 shape by reading each memory's body to populate the new
462 // `accessed_at` / `access_count` fields. All page reads here still
463 // use the v4/v5 AAD scheme (none).
464 let (catalog, migrated_from_v4) = if on_disk_version >= 5 {
465 (
466 load_catalog(
467 &mut pager,
468 header.catalog_start,
469 header.catalog_pages,
470 header.catalog_len,
471 header.catalog_gen,
472 )?,
473 false,
474 )
475 } else if on_disk_version == 4 {
476 let v4 = load_catalog_v4(
477 &mut pager,
478 header.catalog_start,
479 header.catalog_pages,
480 header.catalog_len,
481 )?;
482 (migrate_v4_catalog(&mut pager, v4)?, true)
483 } else {
484 return Err(MnemoError::UnsupportedVersion(on_disk_version));
485 };
486 // FMT-9c.2: replay the v9 delta chain onto the base catalog.
487 let (catalog, delta_chain, delta_rows) =
488 apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
489 let index = build_id_index(&catalog);
490 // PERF-1 step 3: the ANN index loads on first use, not at open. A
491 // v5→v6 page migration is the exception: it must read the index
492 // now, under the old page AAD, to rewrite it.
493 let ann = if on_disk_version < 6 {
494 load_index(
495 &mut pager,
496 header.index_start,
497 header.index_pages,
498 header.index_len,
499 header.index_gen,
500 dimensions,
501 )?
502 } else {
503 None
504 };
505 let ann_unloaded = ann.is_none() && header.index_pages > 0;
506 let mut manifest = load_manifest(
507 &mut pager,
508 header.manifest_start,
509 header.manifest_pages,
510 header.manifest_len,
511 header.manifest_gen,
512 )?;
513
514 // === v5→v6 page-encryption migration =============================
515 // Pre-v6 data pages were encrypted under no AAD; v6 binds the home
516 // `page_no` as AAD so a page-transplant attack fails authentication
517 // on read. Re-stage every live record page through `write_page` so
518 // the next flush re-encrypts it under v6 AAD at the same home slot.
519 // Tombstoned entries skip — they're reclaimed by `compact_file`.
520 // Old catalog / ANN / manifest pages stay as orphan ciphertext;
521 // the next flush writes the new runs at fresh slots.
522 let migrated_pages_to_v6 = on_disk_version < 6;
523 if migrated_pages_to_v6 {
524 for e in &catalog {
525 if e.deleted {
526 continue;
527 }
528 for i in 0..e.page_count as u64 {
529 let payload = pager.read_page(e.start_page + i, 0)?;
530 pager.write_page(e.start_page + i, 0, &payload)?;
531 }
532 }
533 pager.set_version(6);
534 }
535
536 // === v6→v7 header-seal migration =================================
537 // v7 adds an AES-GCM seal at the tail of the header page that
538 // authenticates every mutable header field. Pre-v7 files have no
539 // seal — we just bump the version in memory and mark `dirty_catalog`
540 // so the next flush writes a sealed v7 header. Pages are untouched
541 // (page crypto is unchanged from v6).
542 let migrated_header_to_v7 = on_disk_version < 7;
543 if migrated_header_to_v7 {
544 pager.set_version(7);
545 }
546
547 // === v7→v8 cache-directory migration =============================
548 // v8 adds three u64 header fields (cache_start/pages/len) for the
549 // result-cache directory (Phase 10.1). Pre-v8 files have those
550 // three bytes as zero (the on-disk region beyond the v7 seal is
551 // zero-initialized), which is the legal "empty cache directory"
552 // state — no data migration required. We bump the version in
553 // memory and mark dirty_catalog so the next flush writes a
554 // v8-sealed header with the new AAD shape covering the cache
555 // pointer. Pages are untouched.
556 let migrated_header_to_v8 = on_disk_version < 8;
557 if migrated_header_to_v8 {
558 pager.set_version(VERSION);
559 }
560
561 let migrated = migrated_from_v4
562 || migrated_pages_to_v6
563 || migrated_header_to_v7
564 || migrated_header_to_v8;
565 if migrated {
566 // Stamp the version forward in memory; the next flush persists
567 // it via the WAL-committed header frame (and the v7 seal).
568 header.version = VERSION;
569 // Snapshots written under an older format reference page runs
570 // encrypted under the old AAD scheme; this build can't decrypt
571 // them under the new pager. Drop the manifest so the next
572 // flush records a fresh snapshot of the migrated state.
573 // Point-in-time recovery into the pre-migration past is
574 // sacrificed for migration simplicity. Old pages remain on
575 // disk until `compact_file`. (For v6→v7 alone — which doesn't
576 // change page crypto — the old snapshots would technically
577 // still be readable, but dropping them uniformly keeps the
578 // migration policy simple.)
579 manifest.clear();
580 }
581
582 // The ANN index pages on disk were sealed under the *old* AAD scheme
583 // if we migrated pages to v6; force a rewrite at fresh v6-encrypted
584 // slots on the next flush so a future reopen doesn't try to read
585 // them under the new AAD and fail. (Catalog and manifest already
586 // get rewritten via dirty_catalog and the manifest.clear() above.)
587 let dirty_index = migrated_pages_to_v6 && ann.is_some();
588
589 // Load the cache directory (v8+ only). Pre-v8 files have
590 // cache_pages == 0 by header layout, so this trivially yields
591 // an empty directory without touching the disk.
592 let cache_dir = load_cache_directory(
593 &mut pager,
594 header.cache_start,
595 header.cache_pages,
596 header.cache_len,
597 header.cache_gen,
598 )?;
599 let cache_idx = DirectoryIndex::build(&cache_dir);
600 // FMT-9 (a)/(b): the free list and key-slot run (empty pointers on
601 // v8, so these don't touch the disk there).
602 let free = load_free_list(&mut pager, header.free)?;
603 let key_slots = load_key_slots(&mut pager, header.keys)?;
604 let col = VecColumn::from_catalog(&catalog);
605 let recs = RecPages::from_catalog(&catalog);
606 let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
607 let max_snapshots = if header.version >= VERSION_V9 {
608 V9_DEFAULT_MAX_SNAPSHOTS
609 } else {
610 DEFAULT_MAX_SNAPSHOTS
611 };
612
613 Ok(Mnemo {
614 pager,
615 header,
616 catalog,
617 index,
618 path,
619 dimensions,
620 kdf,
621 dirty_catalog: migrated,
622 ann,
623 dirty_index,
624 // A v5→v6 page migration leaves the index sealed under the old
625 // page AAD, so it must be rewritten on the next flush.
626 index_persist: IndexPersist {
627 forced: dirty_index,
628 ..IndexPersist::default()
629 },
630 ann_unloaded,
631 manifest,
632 // `Mnemo::open` has no [`MnemoConfig`] to read; default the cap
633 // to [`DEFAULT_MAX_SNAPSHOTS`]. Override with
634 // [`Mnemo::set_max_snapshots`] right after open if you need
635 // unlimited retention or a different cap for this handle.
636 max_snapshots,
637 read_only: false,
638 cache_dir,
639 cache_idx,
640 dirty_cache: false,
641 cache_budgets: std::collections::HashMap::new(),
642 cache_flush_policy: CacheFlushPolicy::default(),
643 cache_batch: BatchState::default(),
644 cache_counters: CounterState::default(),
645 embedder: None,
646 fsyncs_last_flush: 0,
647 agent_set: None,
648 key_index: None,
649 superseded: None,
650 layers: None,
651 turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
652 free,
653 pending_free: Vec::new(),
654 free_dirty: false,
655 key_slots,
656 dirty_keys: false,
657 gen_leased: false,
658 col,
659 recs,
660 bm25: Bm25State {
661 manifest: bm25_manifest,
662 ..Bm25State::default()
663 },
664 kw: None,
665 index_layout_override: None,
666 dirty_rows: std::collections::BTreeSet::new(),
667 delta_chain,
668 delta_rows,
669 catalog_rewrite: false,
670 index_chain: Vec::new(),
671 index_chain_ops: 0,
672 index_rebase: false,
673 index_delta_ops: None,
674 snapshot_cap_explicit: false,
675 direct_writes: false,
676 vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
677 })
678 }
679
680 /// Open an existing database in **read-only** mode. Takes an OS shared
681 /// lock; multiple read-only handles can coexist, but any attempt to
682 /// open the same file read-write elsewhere returns
683 /// [`MnemoError::Locked`]. All mutating methods on the returned handle
684 /// return [`MnemoError::ReadOnly`], including `recall` with
685 /// `track_access(true)`.
686 ///
687 /// Two conditions block a read-only open and return
688 /// [`MnemoError::NeedsWriteOpen`]:
689 ///
690 /// 1. The file has a committed-but-uncheckpointed WAL transaction that
691 /// would ordinarily be replayed on open. A read-only handle
692 /// cannot write the recovered pages back.
693 /// 2. The on-disk format version is older than the current
694 /// `format::VERSION`, requiring an on-disk migration.
695 ///
696 /// In either case, open the file read-write once (which recovers /
697 /// migrates in place), close the handle cleanly, and reopen
698 /// read-only. This is documented in the README's Durability section.
699 pub fn open_read_only(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
700 Self::open_read_only_secret(path, passphrase.as_bytes())
701 }
702
703 /// PERF-1 step 7: [`Mnemo::open_read_only`] for a raw-key database.
704 pub fn open_read_only_with_key(
705 path: impl Into<PathBuf>,
706 key: &[u8; crypto::KEY_LEN],
707 ) -> Result<Mnemo> {
708 Self::open_read_only_secret(path, key)
709 }
710
711 pub(super) fn open_read_only_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
712 let path: PathBuf = path.into();
713 let mut file = open_with_lock(&path, /*write=*/ false, /*create=*/ false)?;
714
715 let mut hbuf = [0u8; PAGE_SIZE];
716 file.seek(SeekFrom::Start(0))?;
717 file.read_exact(&mut hbuf)?;
718 // A torn header on a read-only open cannot be healed (that requires
719 // writing recovered frames back). Surface as `NeedsWriteOpen` so the
720 // caller knows to reopen read-write once. DUR-1 (R-002.1 part 4):
721 // any `Header::from_page` failure (`BadMagic`,
722 // `UnsupportedVersion`, `HeaderChecksum`) qualifies — but only
723 // when the file-wide scan actually finds a valid committed
724 // header frame to recover from, so a non-mnemo file still
725 // reports `BadMagic` cleanly. A `validate_seal` failure is NOT
726 // routed here (Option B: tampering is never auto-healed).
727 let header = match Header::from_page(&hbuf) {
728 Ok(h) => h,
729 Err(parse_err) => {
730 let recoverable = scan_header_recovery_candidates(&mut file, secret)?;
731 return Err(match (parse_err, recoverable) {
732 (_, Some(_)) => MnemoError::NeedsWriteOpen {
733 reason: "torn header page needs recovery from WAL",
734 },
735 (e, None) => e,
736 });
737 }
738 };
739
740 // Refuse to migrate on read-only opens. v6→v7 is header-only
741 // (no page writes required), but even that bumps `version` on
742 // disk and re-seals the header — which needs write access. Keep
743 // the policy uniform across all sub-version bumps.
744 if header.version < VERSION {
745 return Err(MnemoError::NeedsWriteOpen {
746 reason: "pending format-version migration",
747 });
748 }
749
750 // Refuse if there is a committed-but-uncheckpointed transaction
751 // waiting for `wal::recover` to fold it into home pages.
752 if wal::recover(
753 &mut file,
754 header.wal_start,
755 header.wal_pages,
756 header.wal_seq,
757 )?
758 .is_some()
759 {
760 return Err(MnemoError::NeedsWriteOpen {
761 reason: "pending WAL recovery",
762 });
763 }
764
765 let kdf = KdfParams {
766 m_cost: header.m_cost,
767 t_cost: header.t_cost,
768 p_cost: header.p_cost,
769 };
770 let kek = crypto::derive_kek(secret, &header.salt, kdf)?;
771 let dek = crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek)?;
772 header.validate_seal(&dek)?;
773
774 let mut pager = Pager::new(file, dek, header.write_counter, header.version);
775 let dimensions = header.dimensions as usize;
776
777 let catalog = load_catalog(
778 &mut pager,
779 header.catalog_start,
780 header.catalog_pages,
781 header.catalog_len,
782 header.catalog_gen,
783 )?;
784 // FMT-9c.2: replay the v9 delta chain onto the base catalog.
785 let (catalog, delta_chain, delta_rows) =
786 apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
787 let index = build_id_index(&catalog);
788 // PERF-1 step 3: the ANN index loads on first use, not at open.
789 let ann: Option<IvfPqIndex> = None;
790 let ann_unloaded = header.index_pages > 0;
791 let manifest = load_manifest(
792 &mut pager,
793 header.manifest_start,
794 header.manifest_pages,
795 header.manifest_len,
796 header.manifest_gen,
797 )?;
798 let cache_dir = load_cache_directory(
799 &mut pager,
800 header.cache_start,
801 header.cache_pages,
802 header.cache_len,
803 header.cache_gen,
804 )?;
805 let cache_idx = DirectoryIndex::build(&cache_dir);
806 let free = load_free_list(&mut pager, header.free)?;
807 let key_slots = load_key_slots(&mut pager, header.keys)?;
808 let col = VecColumn::from_catalog(&catalog);
809 let recs = RecPages::from_catalog(&catalog);
810 let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
811 let max_snapshots = if header.version >= VERSION_V9 {
812 V9_DEFAULT_MAX_SNAPSHOTS
813 } else {
814 DEFAULT_MAX_SNAPSHOTS
815 };
816
817 Ok(Mnemo {
818 pager,
819 header,
820 catalog,
821 index,
822 path,
823 dimensions,
824 kdf,
825 // No mutation is possible; leave both dirty flags false so
826 // `close()` is a genuine no-op even if we later add an
827 // auto-flush on drop.
828 dirty_catalog: false,
829 ann,
830 dirty_index: false,
831 index_persist: IndexPersist::default(),
832 ann_unloaded,
833 manifest,
834 max_snapshots,
835 read_only: true,
836 cache_dir,
837 cache_idx,
838 dirty_cache: false,
839 cache_budgets: std::collections::HashMap::new(),
840 cache_flush_policy: CacheFlushPolicy::default(),
841 cache_batch: BatchState::default(),
842 cache_counters: CounterState::default(),
843 embedder: None,
844 fsyncs_last_flush: 0,
845 agent_set: None,
846 key_index: None,
847 superseded: None,
848 layers: None,
849 turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
850 free,
851 pending_free: Vec::new(),
852 free_dirty: false,
853 key_slots,
854 dirty_keys: false,
855 gen_leased: false,
856 col,
857 recs,
858 bm25: Bm25State {
859 manifest: bm25_manifest,
860 ..Bm25State::default()
861 },
862 kw: None,
863 index_layout_override: None,
864 dirty_rows: std::collections::BTreeSet::new(),
865 delta_chain,
866 delta_rows,
867 catalog_rewrite: false,
868 index_chain: Vec::new(),
869 index_chain_ops: 0,
870 index_rebase: false,
871 index_delta_ops: None,
872 snapshot_cap_explicit: false,
873 direct_writes: false,
874 vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
875 })
876 }
877
878 /// FMT-9e: compress v9 thread-turn records once they're older than
879 /// `secs` (by their `created_at`). The default,
880 /// [`DEFAULT_TURN_COMPRESS_AFTER_SECS`], is 7 days: recent turns stay
881 /// plain, so a live session reads them at full speed, and each commit
882 /// compresses up to 256 turns that have aged past the cutoff. `0`
883 /// compresses every turn as it's written. Compaction always compresses
884 /// every turn, and so does a vacuum that moves one. Applies to this
885 /// handle only (not persisted); a no-op on v8 files, which never
886 /// compress. Reading never depends on it.
887 pub fn set_turn_compress_after(&mut self, secs: u64) {
888 self.turn_compress_after = secs.min(i64::MAX as u64) as i64;
889 }
890
891 /// Override the manifest snapshot cap on this open handle. `0` disables
892 /// the cap; any positive value keeps the most-recent `max` snapshots and
893 /// drops the rest on the next flush. See [`MnemoConfig::max_snapshots`].
894 pub fn set_max_snapshots(&mut self, max: usize) {
895 self.max_snapshots = max;
896 // FMT-9c: an explicit count is a hard cap on v9 (see
897 // `apply_retention`).
898 self.snapshot_cap_explicit = true;
899 }
900
901 /// Embedding dimensionality this database expects.
902 pub fn dimensions(&self) -> usize {
903 self.dimensions
904 }
905
906 /// Number of live (non-deleted) memories.
907 pub fn len(&self) -> usize {
908 self.catalog.iter().filter(|e| !e.deleted).count()
909 }
910
911 /// True if the database holds no live memories.
912 pub fn is_empty(&self) -> bool {
913 self.len() == 0
914 }
915
916 // --- internal page/record helpers -------------------------------------
917}