Skip to main content

mnemo_engine/store/
open.rs

1//! Create, open (read-write, read-only, raw key) and the handle's basic accessors and per-handle settings.
2
3use super::*;
4
5/// Chunk size for the torn-header scan's streaming read of the file.
6/// We walk the file 1 MiB at a time looking for 8 KiB-aligned `MWAL`
7/// magic. There is **no** cap on how far a given candidate's frame
8/// chain may extend — a 1M-vector IVF+PQ relocation can push a single
9/// transaction well past 16 MiB — so each candidate is read frame by
10/// frame via [`wal::scan_committed_txn`] rather than slurped into a
11/// bounded buffer. (DUR-1 R-002.1 part 2.)
12const TORN_HEADER_SCAN_CHUNK: usize = 1 << 20; // 1 MiB
13
14/// Walk the file for every 8 KiB-aligned page whose first four bytes are
15/// `MWAL`, treating each as a candidate WAL-region start, and return the
16/// frames of the highest-`txn_id` committed transaction whose header
17/// frame fully validates under `passphrase`.
18///
19/// A candidate is accepted only if:
20/// * `wal::scan_committed_txn` returns a committed transaction (chained
21///   CRC verifies, single `txn_id`, `KIND_COMMIT` closes it);
22/// * that transaction contains a `(0, header_page)` frame;
23/// * the header parses with `wal_start == candidate_page_no` and
24///   `wal_seq == txn_id` (so a frame cannot forge a region it was not
25///   written into);
26/// * deriving the KEK from `passphrase + header.salt`, unwrapping the
27///   wrapped DEK, and validating the header seal all succeed.
28///
29/// Returns `Ok(None)` when the scan finds no valid candidate — the
30/// caller is expected to propagate the original parse error so
31/// non-mnemo files still report `BadMagic`.
32///
33/// DUR-1 (R-002.1 part 2): there is no fixed scan window — the whole
34/// file is scanned. Reading at page granularity (not byte granularity)
35/// is sufficient because every WAL region in the format is page-aligned
36/// and every WAL commit writes its first frame at the region's start.
37fn scan_header_recovery_candidates(
38    file: &mut File,
39    secret: &[u8],
40) -> Result<Option<Vec<wal::Frame>>> {
41    let file_len = file.seek(SeekFrom::End(0))?;
42    if file_len < PAGE_SIZE as u64 * 2 {
43        return Ok(None); // No room for even one WAL page past the header.
44    }
45
46    let mut best: Option<(u64, Vec<wal::Frame>)> = None;
47    let page_sz = PAGE_SIZE as u64;
48    let mut offset = page_sz; // Skip page 0.
49
50    // Stream through the file in 1 MiB chunks, scanning each chunk for
51    // page-aligned `MWAL` candidates. Each candidate's frame chain is
52    // then read incrementally via `wal::scan_committed_txn`, which has
53    // no window cap — a 1M-vector IVF+PQ relocation can span tens of
54    // MiB in one committed transaction.
55    let mut chunk = vec![0u8; TORN_HEADER_SCAN_CHUNK];
56    while offset < file_len {
57        let want = (TORN_HEADER_SCAN_CHUNK as u64).min(file_len - offset) as usize;
58        chunk.resize(want, 0);
59        file.seek(SeekFrom::Start(offset))?;
60        let mut filled = 0;
61        while filled < want {
62            match file.read(&mut chunk[filled..])? {
63                0 => break,
64                n => filled += n,
65            }
66        }
67        if filled == 0 {
68            break;
69        }
70        let mut inner = 0usize;
71        while inner + 4 <= filled {
72            if &chunk[inner..inner + 4] == wal::FRAME_MAGIC_BYTES {
73                let page_no = (offset + inner as u64) / page_sz;
74                if let Some((txn_id, frames)) = try_recover_candidate(file, page_no, secret)? {
75                    let keep = match &best {
76                        None => true,
77                        Some((id, _)) => txn_id > *id,
78                    };
79                    if keep {
80                        best = Some((txn_id, frames));
81                    }
82                }
83            }
84            inner += PAGE_SIZE;
85        }
86        offset += filled as u64;
87    }
88
89    Ok(best.map(|(_, f)| f))
90}
91
92/// Try to recover a committed transaction from `candidate_page_no`,
93/// returning `(txn_id, frames)` only if the candidate's
94/// `(0, header_page)` frame fully validates: the frame chain commits
95/// (via [`wal::scan_committed_txn`]'s streaming reader), the header
96/// parses with `wal_start == candidate_page_no` **and**
97/// `wal_seq == txn_id`, the DEK unwraps under `passphrase`, and the
98/// AES-GCM seal validates. DUR-1 (R-002.1 item 2) requires the
99/// `wal_seq == txn_id` check and no fixed scan window — the streaming
100/// reader handles transactions of arbitrary size.
101fn try_recover_candidate(
102    file: &mut File,
103    candidate_page_no: u64,
104    secret: &[u8],
105) -> Result<Option<(u64, Vec<wal::Frame>)>> {
106    let Some((txn_id, frames)) = wal::scan_committed_txn(file, candidate_page_no)? else {
107        return Ok(None);
108    };
109
110    // Must contain a header frame at page 0.
111    let header_bytes = match frames.iter().find(|(p, _)| *p == 0) {
112        Some((_, b)) => b,
113        None => return Ok(None),
114    };
115    if header_bytes.len() != PAGE_SIZE {
116        return Ok(None);
117    }
118    let mut hbuf = [0u8; PAGE_SIZE];
119    hbuf.copy_from_slice(header_bytes);
120    let candidate = match Header::from_page(&hbuf) {
121        Ok(h) => h,
122        Err(_) => return Ok(None),
123    };
124
125    // The frame's claimed region must match where we found it, and the
126    // header frame's `wal_seq` must match the committed transaction's
127    // `txn_id` — this is what pins the frame to the region it was
128    // written into and keeps a stale checkpointed-and-copied frame
129    // from being accepted as a new region's root.
130    if candidate.wal_start != candidate_page_no || candidate.wal_seq != txn_id {
131        return Ok(None);
132    }
133    // DEK unwrap under this candidate's own salt + KDF params.
134    let kdf = KdfParams {
135        m_cost: candidate.m_cost,
136        t_cost: candidate.t_cost,
137        p_cost: candidate.p_cost,
138    };
139    let kek = match crypto::derive_kek(secret, &candidate.salt, kdf) {
140        Ok(k) => k,
141        Err(_) => return Ok(None),
142    };
143    let dek = match crypto::unwrap_dek(&kek, &candidate.dek_nonce, &candidate.wrapped_dek) {
144        Ok(d) => d,
145        Err(_) => return Ok(None),
146    };
147    if candidate.validate_seal(&dek).is_err() {
148        return Ok(None);
149    }
150
151    Ok(Some((txn_id, frames)))
152}
153
154impl Mnemo {
155    /// Create a brand-new encrypted database at `path`.
156    pub fn create(
157        path: impl Into<PathBuf>,
158        passphrase: &str,
159        config: MnemoConfig,
160    ) -> Result<Mnemo> {
161        Self::create_secret(path, passphrase.as_bytes(), config)
162    }
163
164    /// PERF-1 step 7 (R-004.5): create a database unlocked by a **raw
165    /// 256-bit key** instead of a passphrase, for keys held by an OS
166    /// keychain or keystore. The key must be uniformly random (see
167    /// [`Mnemo::generate_key`]); it gets no stretching. The header records
168    /// minimal Argon2id parameters ([`KdfParams::raw_key`]), so opening
169    /// costs microseconds instead of tens of milliseconds. `config.kdf` is
170    /// ignored. No format change: the key goes through the same KEK
171    /// derivation a passphrase does.
172    pub fn create_with_key(
173        path: impl Into<PathBuf>,
174        key: &[u8; crypto::KEY_LEN],
175        config: MnemoConfig,
176    ) -> Result<Mnemo> {
177        let config = MnemoConfig {
178            kdf: KdfParams::raw_key(),
179            ..config
180        };
181        Self::create_secret(path, key, config)
182    }
183
184    /// A fresh random 256-bit key for [`Mnemo::create_with_key`], from the
185    /// thread-local CSPRNG (seeded from the OS). Store it in a keychain;
186    /// it can't be recovered from the file.
187    pub fn generate_key() -> [u8; crypto::KEY_LEN] {
188        *crypto::random_dek()
189    }
190
191    pub(super) fn create_secret(
192        path: impl Into<PathBuf>,
193        secret: &[u8],
194        config: MnemoConfig,
195    ) -> Result<Mnemo> {
196        let path: PathBuf = path.into();
197        if config.dimensions == 0 {
198            return Err(MnemoError::Invalid("dimensions must be > 0".into()));
199        }
200        let version = resolve_format_version(config.format_version)?;
201        if version >= VERSION_V9 && veccol::per_page(config.dimensions) == 0 {
202            return Err(MnemoError::Invalid(format!(
203                "a v9 file supports up to {} dimensions for now (one fp16 vector \
204                 must fit a page); vector entries that span pages are not supported yet",
205                (PAYLOAD - veccol::PAGE_HEADER - 4) / 2
206            )));
207        }
208        let flags = if version >= VERSION_V9 && config.store_float_vectors {
209            FLAG_ENCRYPTED | FLAG_FLOAT_VECTORS
210        } else {
211            FLAG_ENCRYPTED
212        };
213        // Exclusive lock covers the write path; second writer sees `Locked`.
214        let file = open_with_lock(&path, /*write=*/ true, /*create=*/ true)?;
215
216        let salt = crypto::random_salt();
217        let dek = crypto::random_dek();
218        let kek = crypto::derive_kek(secret, &salt, config.kdf)?;
219        let dek_nonce = crypto::random_nonce();
220        let wrapped = crypto::wrap_dek(&kek, &dek_nonce, &dek)?;
221        if wrapped.len() != WRAPPED_DEK_LEN {
222            return Err(MnemoError::Crypto("unexpected wrapped DEK length".into()));
223        }
224        let mut wrapped_dek = [0u8; WRAPPED_DEK_LEN];
225        wrapped_dek.copy_from_slice(&wrapped);
226
227        let mut header = Header {
228            version,
229            page_size: PAGE_SIZE as u32,
230            flags,
231            dimensions: config.dimensions as u32,
232            created_at: memory::now_secs(),
233            write_counter: 0,
234            // Page 0 is the header; pages 1..=wal_pages are the WAL;
235            // record/catalog/index pages start after it. Initial WAL size is
236            // chosen per-config (default 8 pages = 64 KiB) and clamped to a
237            // sane floor so the first transaction can always commit.
238            next_page: 1 + config.wal_pages_initial.max(MIN_WAL_PAGES),
239            catalog_start: 0,
240            catalog_pages: 0,
241            catalog_len: 0,
242            vector_count: 0,
243            m_cost: config.kdf.m_cost,
244            t_cost: config.kdf.t_cost,
245            p_cost: config.kdf.p_cost,
246            salt,
247            dek_nonce,
248            wrapped_dek,
249            index_start: 0,
250            index_pages: 0,
251            index_len: 0,
252            wal_start: 1,
253            wal_pages: config.wal_pages_initial.max(MIN_WAL_PAGES),
254            wal_seq: 0,
255            manifest_start: 0,
256            manifest_pages: 0,
257            manifest_len: 0,
258            // Regenerated per write by `apply_seal`; `None` until we seal.
259            seal_nonce: None,
260            seal_tag: None,
261            // v8+ result-cache pointer — zero-initialized on create,
262            // populated only when `cache_put` runs at least once and
263            // the next `flush` seals a directory run.
264            cache_start: 0,
265            cache_pages: 0,
266            cache_len: 0,
267            // FMT-9 v9 fields; all zero on a fresh file (and unused on v8).
268            catalog_gen: 0,
269            index_gen: 0,
270            manifest_gen: 0,
271            cache_gen: 0,
272            free: RunPtr::default(),
273            keys: RunPtr::default(),
274            bm25: RunPtr::default(),
275            index_delta: RunPtr::default(),
276            catalog_delta: RunPtr::default(),
277        };
278
279        let mut pager = Pager::new(file, dek, 0, version);
280        let mut page = header.to_page();
281        header.apply_seal(&mut page, pager.dek())?;
282        pager.write_raw(0, &page)?;
283        pager.sync()?;
284
285        Ok(Mnemo {
286            pager,
287            header,
288            catalog: Vec::new(),
289            index: IdIndex::default(),
290            path,
291            dimensions: config.dimensions,
292            kdf: config.kdf,
293            dirty_catalog: false,
294            ann: None,
295            dirty_index: false,
296            index_persist: IndexPersist::default(),
297            ann_unloaded: false,
298            manifest: Vec::new(),
299            // FMT-9c: v9 keeps fewer snapshots by default: with page reuse,
300            // retention is what pins pages.
301            max_snapshots: if version >= VERSION_V9 && config.max_snapshots == DEFAULT_MAX_SNAPSHOTS
302            {
303                V9_DEFAULT_MAX_SNAPSHOTS
304            } else {
305                config.max_snapshots
306            },
307            read_only: false,
308            cache_dir: Vec::new(),
309            cache_idx: DirectoryIndex::new(),
310            dirty_cache: false,
311            cache_budgets: std::collections::HashMap::new(),
312            cache_flush_policy: CacheFlushPolicy::default(),
313            cache_batch: BatchState::default(),
314            cache_counters: CounterState::default(),
315            embedder: None,
316            fsyncs_last_flush: 0,
317            agent_set: None,
318            key_index: None,
319            superseded: None,
320            layers: None,
321            turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
322            free: FreeList::default(),
323            pending_free: Vec::new(),
324            free_dirty: false,
325            key_slots: KeySlotRun::default(),
326            dirty_keys: false,
327            gen_leased: false,
328            col: VecColumn::default(),
329            recs: RecPages::default(),
330            bm25: Bm25State::default(),
331            kw: None,
332            index_layout_override: None,
333            dirty_rows: std::collections::BTreeSet::new(),
334            delta_chain: Vec::new(),
335            delta_rows: 0,
336            catalog_rewrite: false,
337            index_chain: Vec::new(),
338            index_chain_ops: 0,
339            index_rebase: false,
340            index_delta_ops: None,
341            snapshot_cap_explicit: config.max_snapshots != DEFAULT_MAX_SNAPSHOTS,
342            direct_writes: false,
343            vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
344        })
345    }
346
347    /// Open an existing database. A wrong passphrase fails cleanly with
348    /// [`MnemoError::WrongPassphrase`]. Takes an OS exclusive lock for
349    /// the lifetime of the returned handle; a second concurrent open
350    /// (from any process) returns [`MnemoError::Locked`].
351    ///
352    /// For a caller that only needs to read (and can tolerate the
353    /// pending-recovery / pending-migration cases documented on
354    /// [`Mnemo::open_read_only`]), prefer that entry point — it takes a
355    /// shared lock and coexists with other readers.
356    pub fn open(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
357        Self::open_secret(path, passphrase.as_bytes())
358    }
359
360    /// PERF-1 step 7: open a database created by [`Mnemo::create_with_key`]
361    /// (or rekeyed with [`Mnemo::rekey_with_key`]) with its raw key.
362    pub fn open_with_key(path: impl Into<PathBuf>, key: &[u8; crypto::KEY_LEN]) -> Result<Mnemo> {
363        Self::open_secret(path, key)
364    }
365
366    pub(super) fn open_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
367        let path: PathBuf = path.into();
368        let mut file = open_with_lock(&path, /*write=*/ true, /*create=*/ false)?;
369
370        let mut hbuf = [0u8; PAGE_SIZE];
371        file.seek(SeekFrom::Start(0))?;
372        file.read_exact(&mut hbuf)?;
373        let mut header = match Header::from_page(&hbuf) {
374            Ok(h) => h,
375            Err(e) => {
376                // DUR-1 (R-002.1 part 2+3): page 0 did not parse — any
377                // error is treated as a torn-header candidate, not just
378                // `HeaderChecksum`. The whole 294-byte header lives in
379                // the first 512-byte sector, so the realistic failure is
380                // the sector zeroed or garbled, which returns `BadMagic`
381                // or `UnsupportedVersion` *before* the CRC gate.
382                //
383                // Scan the file for valid committed header-frame WAL
384                // regions, pick the highest `txn_id`, and replay it. If
385                // nothing valid is found, return the *original* error so
386                // non-mnemo files still report `BadMagic`.
387                let scan = scan_header_recovery_candidates(&mut file, secret)?;
388                let frames = scan.ok_or(e)?;
389                for (page_no, bytes) in &frames {
390                    if bytes.len() != PAGE_SIZE {
391                        return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
392                    }
393                    file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
394                    file.write_all(bytes)?;
395                }
396                file.sync_all()?;
397                crate::bump_fsync();
398                file.seek(SeekFrom::Start(0))?;
399                file.read_exact(&mut hbuf)?;
400                Header::from_page(&hbuf)?
401            }
402        };
403
404        // Crash recovery: replay a committed-but-uncheckpointed transaction.
405        // Each frame is a finished page image; the header frame, replayed to
406        // page 0, supersedes the header just read.
407        if let Some(frames) = wal::recover(
408            &mut file,
409            header.wal_start,
410            header.wal_pages,
411            header.wal_seq,
412        )? {
413            for (page_no, bytes) in &frames {
414                if bytes.len() != PAGE_SIZE {
415                    return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
416                }
417                file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
418                file.write_all(bytes)?;
419            }
420            file.sync_all()?;
421            crate::bump_fsync();
422            // Re-read the now-current header from the replayed page 0.
423            file.seek(SeekFrom::Start(0))?;
424            file.read_exact(&mut hbuf)?;
425            header = Header::from_page(&hbuf)?;
426        }
427
428        let kdf = KdfParams {
429            m_cost: header.m_cost,
430            t_cost: header.t_cost,
431            p_cost: header.p_cost,
432        };
433        let kek = crypto::derive_kek(secret, &header.salt, kdf)?;
434        let dek = crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek)?;
435
436        // Validate the v7 header seal now that we have the DEK. Returns
437        // `MnemoError::HeaderTampered` if any of the sealed mutable fields
438        // were rewritten after the last legitimate flush. No-op for v6 and
439        // older — pre-v7 files don't carry a seal.
440        //
441        // DUR-1 (Planner decision 2026-11-01, Option B): a seal failure is
442        // **always** reported as tampering, never auto-healed. R-002.1
443        // part 3's seal-mismatch WAL fallback is withdrawn: the seal
444        // exists for tamper-evidence; silently loading a stale committed
445        // header from the WAL would erase that evidence. A real crash
446        // cannot produce "CRC valid + seal invalid" either, because the
447        // whole 294-byte header lies inside sector 0 which storage
448        // writes atomically — a torn sector breaks magic, version or
449        // CRC first, and *those* are healed by the scan above.
450        header.validate_seal(&dek)?;
451
452        // Build the pager at the file's *current* version so initial reads
453        // use the matching AAD scheme — `Pager::page_aad` returns empty AAD
454        // for v4/v5 (no page binding) and `page_no.to_le_bytes()` for v6+.
455        let on_disk_version = header.version;
456        let mut pager = Pager::new(file, dek, header.write_counter, on_disk_version);
457        let dimensions = header.dimensions as usize;
458
459        // === v4→v5 catalog migration =====================================
460        // v6 reads directly; v5 reads directly; v4 replays the catalog into
461        // the v5 shape by reading each memory's body to populate the new
462        // `accessed_at` / `access_count` fields. All page reads here still
463        // use the v4/v5 AAD scheme (none).
464        let (catalog, migrated_from_v4) = if on_disk_version >= 5 {
465            (
466                load_catalog(
467                    &mut pager,
468                    header.catalog_start,
469                    header.catalog_pages,
470                    header.catalog_len,
471                    header.catalog_gen,
472                )?,
473                false,
474            )
475        } else if on_disk_version == 4 {
476            let v4 = load_catalog_v4(
477                &mut pager,
478                header.catalog_start,
479                header.catalog_pages,
480                header.catalog_len,
481            )?;
482            (migrate_v4_catalog(&mut pager, v4)?, true)
483        } else {
484            return Err(MnemoError::UnsupportedVersion(on_disk_version));
485        };
486        // FMT-9c.2: replay the v9 delta chain onto the base catalog.
487        let (catalog, delta_chain, delta_rows) =
488            apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
489        let index = build_id_index(&catalog);
490        // PERF-1 step 3: the ANN index loads on first use, not at open. A
491        // v5→v6 page migration is the exception: it must read the index
492        // now, under the old page AAD, to rewrite it.
493        let ann = if on_disk_version < 6 {
494            load_index(
495                &mut pager,
496                header.index_start,
497                header.index_pages,
498                header.index_len,
499                header.index_gen,
500                dimensions,
501            )?
502        } else {
503            None
504        };
505        let ann_unloaded = ann.is_none() && header.index_pages > 0;
506        let mut manifest = load_manifest(
507            &mut pager,
508            header.manifest_start,
509            header.manifest_pages,
510            header.manifest_len,
511            header.manifest_gen,
512        )?;
513
514        // === v5→v6 page-encryption migration =============================
515        // Pre-v6 data pages were encrypted under no AAD; v6 binds the home
516        // `page_no` as AAD so a page-transplant attack fails authentication
517        // on read. Re-stage every live record page through `write_page` so
518        // the next flush re-encrypts it under v6 AAD at the same home slot.
519        // Tombstoned entries skip — they're reclaimed by `compact_file`.
520        // Old catalog / ANN / manifest pages stay as orphan ciphertext;
521        // the next flush writes the new runs at fresh slots.
522        let migrated_pages_to_v6 = on_disk_version < 6;
523        if migrated_pages_to_v6 {
524            for e in &catalog {
525                if e.deleted {
526                    continue;
527                }
528                for i in 0..e.page_count as u64 {
529                    let payload = pager.read_page(e.start_page + i, 0)?;
530                    pager.write_page(e.start_page + i, 0, &payload)?;
531                }
532            }
533            pager.set_version(6);
534        }
535
536        // === v6→v7 header-seal migration =================================
537        // v7 adds an AES-GCM seal at the tail of the header page that
538        // authenticates every mutable header field. Pre-v7 files have no
539        // seal — we just bump the version in memory and mark `dirty_catalog`
540        // so the next flush writes a sealed v7 header. Pages are untouched
541        // (page crypto is unchanged from v6).
542        let migrated_header_to_v7 = on_disk_version < 7;
543        if migrated_header_to_v7 {
544            pager.set_version(7);
545        }
546
547        // === v7→v8 cache-directory migration =============================
548        // v8 adds three u64 header fields (cache_start/pages/len) for the
549        // result-cache directory (Phase 10.1). Pre-v8 files have those
550        // three bytes as zero (the on-disk region beyond the v7 seal is
551        // zero-initialized), which is the legal "empty cache directory"
552        // state — no data migration required. We bump the version in
553        // memory and mark dirty_catalog so the next flush writes a
554        // v8-sealed header with the new AAD shape covering the cache
555        // pointer. Pages are untouched.
556        let migrated_header_to_v8 = on_disk_version < 8;
557        if migrated_header_to_v8 {
558            pager.set_version(VERSION);
559        }
560
561        let migrated = migrated_from_v4
562            || migrated_pages_to_v6
563            || migrated_header_to_v7
564            || migrated_header_to_v8;
565        if migrated {
566            // Stamp the version forward in memory; the next flush persists
567            // it via the WAL-committed header frame (and the v7 seal).
568            header.version = VERSION;
569            // Snapshots written under an older format reference page runs
570            // encrypted under the old AAD scheme; this build can't decrypt
571            // them under the new pager. Drop the manifest so the next
572            // flush records a fresh snapshot of the migrated state.
573            // Point-in-time recovery into the pre-migration past is
574            // sacrificed for migration simplicity. Old pages remain on
575            // disk until `compact_file`. (For v6→v7 alone — which doesn't
576            // change page crypto — the old snapshots would technically
577            // still be readable, but dropping them uniformly keeps the
578            // migration policy simple.)
579            manifest.clear();
580        }
581
582        // The ANN index pages on disk were sealed under the *old* AAD scheme
583        // if we migrated pages to v6; force a rewrite at fresh v6-encrypted
584        // slots on the next flush so a future reopen doesn't try to read
585        // them under the new AAD and fail. (Catalog and manifest already
586        // get rewritten via dirty_catalog and the manifest.clear() above.)
587        let dirty_index = migrated_pages_to_v6 && ann.is_some();
588
589        // Load the cache directory (v8+ only). Pre-v8 files have
590        // cache_pages == 0 by header layout, so this trivially yields
591        // an empty directory without touching the disk.
592        let cache_dir = load_cache_directory(
593            &mut pager,
594            header.cache_start,
595            header.cache_pages,
596            header.cache_len,
597            header.cache_gen,
598        )?;
599        let cache_idx = DirectoryIndex::build(&cache_dir);
600        // FMT-9 (a)/(b): the free list and key-slot run (empty pointers on
601        // v8, so these don't touch the disk there).
602        let free = load_free_list(&mut pager, header.free)?;
603        let key_slots = load_key_slots(&mut pager, header.keys)?;
604        let col = VecColumn::from_catalog(&catalog);
605        let recs = RecPages::from_catalog(&catalog);
606        let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
607        let max_snapshots = if header.version >= VERSION_V9 {
608            V9_DEFAULT_MAX_SNAPSHOTS
609        } else {
610            DEFAULT_MAX_SNAPSHOTS
611        };
612
613        Ok(Mnemo {
614            pager,
615            header,
616            catalog,
617            index,
618            path,
619            dimensions,
620            kdf,
621            dirty_catalog: migrated,
622            ann,
623            dirty_index,
624            // A v5→v6 page migration leaves the index sealed under the old
625            // page AAD, so it must be rewritten on the next flush.
626            index_persist: IndexPersist {
627                forced: dirty_index,
628                ..IndexPersist::default()
629            },
630            ann_unloaded,
631            manifest,
632            // `Mnemo::open` has no [`MnemoConfig`] to read; default the cap
633            // to [`DEFAULT_MAX_SNAPSHOTS`]. Override with
634            // [`Mnemo::set_max_snapshots`] right after open if you need
635            // unlimited retention or a different cap for this handle.
636            max_snapshots,
637            read_only: false,
638            cache_dir,
639            cache_idx,
640            dirty_cache: false,
641            cache_budgets: std::collections::HashMap::new(),
642            cache_flush_policy: CacheFlushPolicy::default(),
643            cache_batch: BatchState::default(),
644            cache_counters: CounterState::default(),
645            embedder: None,
646            fsyncs_last_flush: 0,
647            agent_set: None,
648            key_index: None,
649            superseded: None,
650            layers: None,
651            turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
652            free,
653            pending_free: Vec::new(),
654            free_dirty: false,
655            key_slots,
656            dirty_keys: false,
657            gen_leased: false,
658            col,
659            recs,
660            bm25: Bm25State {
661                manifest: bm25_manifest,
662                ..Bm25State::default()
663            },
664            kw: None,
665            index_layout_override: None,
666            dirty_rows: std::collections::BTreeSet::new(),
667            delta_chain,
668            delta_rows,
669            catalog_rewrite: false,
670            index_chain: Vec::new(),
671            index_chain_ops: 0,
672            index_rebase: false,
673            index_delta_ops: None,
674            snapshot_cap_explicit: false,
675            direct_writes: false,
676            vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
677        })
678    }
679
680    /// Open an existing database in **read-only** mode. Takes an OS shared
681    /// lock; multiple read-only handles can coexist, but any attempt to
682    /// open the same file read-write elsewhere returns
683    /// [`MnemoError::Locked`]. All mutating methods on the returned handle
684    /// return [`MnemoError::ReadOnly`], including `recall` with
685    /// `track_access(true)`.
686    ///
687    /// Two conditions block a read-only open and return
688    /// [`MnemoError::NeedsWriteOpen`]:
689    ///
690    /// 1. The file has a committed-but-uncheckpointed WAL transaction that
691    ///    would ordinarily be replayed on open. A read-only handle
692    ///    cannot write the recovered pages back.
693    /// 2. The on-disk format version is older than the current
694    ///    `format::VERSION`, requiring an on-disk migration.
695    ///
696    /// In either case, open the file read-write once (which recovers /
697    /// migrates in place), close the handle cleanly, and reopen
698    /// read-only. This is documented in the README's Durability section.
699    pub fn open_read_only(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
700        Self::open_read_only_secret(path, passphrase.as_bytes())
701    }
702
703    /// PERF-1 step 7: [`Mnemo::open_read_only`] for a raw-key database.
704    pub fn open_read_only_with_key(
705        path: impl Into<PathBuf>,
706        key: &[u8; crypto::KEY_LEN],
707    ) -> Result<Mnemo> {
708        Self::open_read_only_secret(path, key)
709    }
710
711    /// Whether `key` opens the raw-key file at `path`, decided from its
712    /// header alone: no catalog load, no lock, microseconds. `Ok(false)` is
713    /// "not this key". Unlike a read-only open, it answers for a file that
714    /// has a WAL transaction to recover (the state a kill between a commit
715    /// and its checkpoint leaves), so a caller trying each of its keys in
716    /// turn isn't stopped by [`MnemoError::NeedsWriteOpen`]. A torn header is
717    /// judged from the WAL frame that would heal it. Other failures (not a
718    /// `.mnemo` file, a tampered header, an I/O error) are errors. Added in
719    /// v0.12.2 (RB-001.23).
720    pub fn key_opens(
721        path: impl AsRef<std::path::Path>,
722        key: &[u8; crypto::KEY_LEN],
723    ) -> Result<bool> {
724        let mut file = File::open(path.as_ref())?;
725        let mut hbuf = [0u8; PAGE_SIZE];
726        file.seek(SeekFrom::Start(0))?;
727        file.read_exact(&mut hbuf)?;
728        let header = match Header::from_page(&hbuf) {
729            Ok(h) => h,
730            // A candidate frame only counts when it validates with `key`.
731            Err(parse_err) => {
732                return match scan_header_recovery_candidates(&mut file, key)? {
733                    Some(_) => Ok(true),
734                    None => Err(parse_err),
735                }
736            }
737        };
738        let kdf = KdfParams {
739            m_cost: header.m_cost,
740            t_cost: header.t_cost,
741            p_cost: header.p_cost,
742        };
743        let kek = crypto::derive_kek(key, &header.salt, kdf)?;
744        let dek = match crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek) {
745            Ok(dek) => dek,
746            Err(MnemoError::WrongPassphrase) => return Ok(false),
747            Err(e) => return Err(e),
748        };
749        header.validate_seal(&dek)?;
750        Ok(true)
751    }
752
753    pub(super) fn open_read_only_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
754        let path: PathBuf = path.into();
755        let mut file = open_with_lock(&path, /*write=*/ false, /*create=*/ false)?;
756
757        let mut hbuf = [0u8; PAGE_SIZE];
758        file.seek(SeekFrom::Start(0))?;
759        file.read_exact(&mut hbuf)?;
760        // A torn header on a read-only open cannot be healed (that requires
761        // writing recovered frames back). Surface as `NeedsWriteOpen` so the
762        // caller knows to reopen read-write once. DUR-1 (R-002.1 part 4):
763        // any `Header::from_page` failure (`BadMagic`,
764        // `UnsupportedVersion`, `HeaderChecksum`) qualifies — but only
765        // when the file-wide scan actually finds a valid committed
766        // header frame to recover from, so a non-mnemo file still
767        // reports `BadMagic` cleanly. A `validate_seal` failure is NOT
768        // routed here (Option B: tampering is never auto-healed).
769        let header = match Header::from_page(&hbuf) {
770            Ok(h) => h,
771            Err(parse_err) => {
772                let recoverable = scan_header_recovery_candidates(&mut file, secret)?;
773                return Err(match (parse_err, recoverable) {
774                    (_, Some(_)) => MnemoError::NeedsWriteOpen {
775                        reason: "torn header page needs recovery from WAL",
776                    },
777                    (e, None) => e,
778                });
779            }
780        };
781
782        // Refuse to migrate on read-only opens. v6→v7 is header-only
783        // (no page writes required), but even that bumps `version` on
784        // disk and re-seals the header — which needs write access. Keep
785        // the policy uniform across all sub-version bumps.
786        if header.version < VERSION {
787            return Err(MnemoError::NeedsWriteOpen {
788                reason: "pending format-version migration",
789            });
790        }
791
792        // Refuse if there is a committed-but-uncheckpointed transaction
793        // waiting for `wal::recover` to fold it into home pages.
794        if wal::recover(
795            &mut file,
796            header.wal_start,
797            header.wal_pages,
798            header.wal_seq,
799        )?
800        .is_some()
801        {
802            return Err(MnemoError::NeedsWriteOpen {
803                reason: "pending WAL recovery",
804            });
805        }
806
807        let kdf = KdfParams {
808            m_cost: header.m_cost,
809            t_cost: header.t_cost,
810            p_cost: header.p_cost,
811        };
812        let kek = crypto::derive_kek(secret, &header.salt, kdf)?;
813        let dek = crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek)?;
814        header.validate_seal(&dek)?;
815
816        let mut pager = Pager::new(file, dek, header.write_counter, header.version);
817        let dimensions = header.dimensions as usize;
818
819        let catalog = load_catalog(
820            &mut pager,
821            header.catalog_start,
822            header.catalog_pages,
823            header.catalog_len,
824            header.catalog_gen,
825        )?;
826        // FMT-9c.2: replay the v9 delta chain onto the base catalog.
827        let (catalog, delta_chain, delta_rows) =
828            apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
829        let index = build_id_index(&catalog);
830        // PERF-1 step 3: the ANN index loads on first use, not at open.
831        let ann: Option<IvfPqIndex> = None;
832        let ann_unloaded = header.index_pages > 0;
833        let manifest = load_manifest(
834            &mut pager,
835            header.manifest_start,
836            header.manifest_pages,
837            header.manifest_len,
838            header.manifest_gen,
839        )?;
840        let cache_dir = load_cache_directory(
841            &mut pager,
842            header.cache_start,
843            header.cache_pages,
844            header.cache_len,
845            header.cache_gen,
846        )?;
847        let cache_idx = DirectoryIndex::build(&cache_dir);
848        let free = load_free_list(&mut pager, header.free)?;
849        let key_slots = load_key_slots(&mut pager, header.keys)?;
850        let col = VecColumn::from_catalog(&catalog);
851        let recs = RecPages::from_catalog(&catalog);
852        let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
853        let max_snapshots = if header.version >= VERSION_V9 {
854            V9_DEFAULT_MAX_SNAPSHOTS
855        } else {
856            DEFAULT_MAX_SNAPSHOTS
857        };
858
859        Ok(Mnemo {
860            pager,
861            header,
862            catalog,
863            index,
864            path,
865            dimensions,
866            kdf,
867            // No mutation is possible; leave both dirty flags false so
868            // `close()` is a genuine no-op even if we later add an
869            // auto-flush on drop.
870            dirty_catalog: false,
871            ann,
872            dirty_index: false,
873            index_persist: IndexPersist::default(),
874            ann_unloaded,
875            manifest,
876            max_snapshots,
877            read_only: true,
878            cache_dir,
879            cache_idx,
880            dirty_cache: false,
881            cache_budgets: std::collections::HashMap::new(),
882            cache_flush_policy: CacheFlushPolicy::default(),
883            cache_batch: BatchState::default(),
884            cache_counters: CounterState::default(),
885            embedder: None,
886            fsyncs_last_flush: 0,
887            agent_set: None,
888            key_index: None,
889            superseded: None,
890            layers: None,
891            turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
892            free,
893            pending_free: Vec::new(),
894            free_dirty: false,
895            key_slots,
896            dirty_keys: false,
897            gen_leased: false,
898            col,
899            recs,
900            bm25: Bm25State {
901                manifest: bm25_manifest,
902                ..Bm25State::default()
903            },
904            kw: None,
905            index_layout_override: None,
906            dirty_rows: std::collections::BTreeSet::new(),
907            delta_chain,
908            delta_rows,
909            catalog_rewrite: false,
910            index_chain: Vec::new(),
911            index_chain_ops: 0,
912            index_rebase: false,
913            index_delta_ops: None,
914            snapshot_cap_explicit: false,
915            direct_writes: false,
916            vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
917        })
918    }
919
920    /// FMT-9e: compress v9 thread-turn records once they're older than
921    /// `secs` (by their `created_at`). The default,
922    /// [`DEFAULT_TURN_COMPRESS_AFTER_SECS`], is 7 days: recent turns stay
923    /// plain, so a live session reads them at full speed, and each commit
924    /// compresses up to 256 turns that have aged past the cutoff. `0`
925    /// compresses every turn as it's written. Compaction always compresses
926    /// every turn, and so does a vacuum that moves one. Applies to this
927    /// handle only (not persisted); a no-op on v8 files, which never
928    /// compress. Reading never depends on it.
929    pub fn set_turn_compress_after(&mut self, secs: u64) {
930        self.turn_compress_after = secs.min(i64::MAX as u64) as i64;
931    }
932
933    /// Override the manifest snapshot cap on this open handle. `0` disables
934    /// the cap; any positive value keeps the most-recent `max` snapshots and
935    /// drops the rest on the next flush. See [`MnemoConfig::max_snapshots`].
936    pub fn set_max_snapshots(&mut self, max: usize) {
937        self.max_snapshots = max;
938        // FMT-9c: an explicit count is a hard cap on v9 (see
939        // `apply_retention`).
940        self.snapshot_cap_explicit = true;
941    }
942
943    /// Embedding dimensionality this database expects.
944    pub fn dimensions(&self) -> usize {
945        self.dimensions
946    }
947
948    /// Number of live (non-deleted) memories.
949    pub fn len(&self) -> usize {
950        self.catalog.iter().filter(|e| !e.deleted).count()
951    }
952
953    /// True if the database holds no live memories.
954    pub fn is_empty(&self) -> bool {
955        self.len() == 0
956    }
957
958    // --- internal page/record helpers -------------------------------------
959}