Skip to main content

mnemo_engine/store/
open.rs

1//! Create, open (read-write, read-only, raw key) and the handle's basic accessors and per-handle settings.
2
3use super::*;
4
5/// Chunk size for the torn-header scan's streaming read of the file.
6/// We walk the file 1 MiB at a time looking for 8 KiB-aligned `MWAL`
7/// magic. There is **no** cap on how far a given candidate's frame
8/// chain may extend — a 1M-vector IVF+PQ relocation can push a single
9/// transaction well past 16 MiB — so each candidate is read frame by
10/// frame via [`wal::scan_committed_txn`] rather than slurped into a
11/// bounded buffer. (DUR-1 R-002.1 part 2.)
12const TORN_HEADER_SCAN_CHUNK: usize = 1 << 20; // 1 MiB
13
14/// The header a set of recovery frames would write to page 0, if they hold
15/// one that parses.
16fn page_zero_header(frames: &[(u64, Vec<u8>)]) -> Option<Header> {
17    frames
18        .iter()
19        .find(|(page_no, _)| *page_no == 0)
20        .and_then(|(_, bytes)| Header::from_page(bytes).ok())
21}
22
23/// Walk the file for every 8 KiB-aligned page whose first four bytes are
24/// `MWAL`, treating each as a candidate WAL-region start, and return the
25/// frames of the highest-`txn_id` committed transaction whose header
26/// frame fully validates under `passphrase`.
27///
28/// A candidate is accepted only if:
29/// * `wal::scan_committed_txn` returns a committed transaction (chained
30///   CRC verifies, single `txn_id`, `KIND_COMMIT` closes it);
31/// * that transaction contains a `(0, header_page)` frame;
32/// * the header parses with `wal_start == candidate_page_no` and
33///   `wal_seq == txn_id` (so a frame cannot forge a region it was not
34///   written into);
35/// * deriving the KEK from `passphrase + header.salt`, unwrapping the
36///   wrapped DEK, and validating the header seal all succeed.
37///
38/// Returns `Ok(None)` when the scan finds no valid candidate — the
39/// caller is expected to propagate the original parse error so
40/// non-mnemo files still report `BadMagic`.
41///
42/// DUR-1 (R-002.1 part 2): there is no fixed scan window — the whole
43/// file is scanned. Reading at page granularity (not byte granularity)
44/// is sufficient because every WAL region in the format is page-aligned
45/// and every WAL commit writes its first frame at the region's start.
46fn scan_header_recovery_candidates(
47    file: &mut File,
48    secret: &[u8],
49) -> Result<Option<Vec<wal::Frame>>> {
50    let file_len = file.seek(SeekFrom::End(0))?;
51    if file_len < PAGE_SIZE as u64 * 2 {
52        return Ok(None); // No room for even one WAL page past the header.
53    }
54
55    let mut best: Option<(u64, Vec<wal::Frame>)> = None;
56    let page_sz = PAGE_SIZE as u64;
57    let mut offset = page_sz; // Skip page 0.
58
59    // Stream through the file in 1 MiB chunks, scanning each chunk for
60    // page-aligned `MWAL` candidates. Each candidate's frame chain is
61    // then read incrementally via `wal::scan_committed_txn`, which has
62    // no window cap — a 1M-vector IVF+PQ relocation can span tens of
63    // MiB in one committed transaction.
64    let mut chunk = vec![0u8; TORN_HEADER_SCAN_CHUNK];
65    while offset < file_len {
66        let want = (TORN_HEADER_SCAN_CHUNK as u64).min(file_len - offset) as usize;
67        chunk.resize(want, 0);
68        file.seek(SeekFrom::Start(offset))?;
69        let mut filled = 0;
70        while filled < want {
71            match file.read(&mut chunk[filled..])? {
72                0 => break,
73                n => filled += n,
74            }
75        }
76        if filled == 0 {
77            break;
78        }
79        let mut inner = 0usize;
80        while inner + 4 <= filled {
81            if &chunk[inner..inner + 4] == wal::FRAME_MAGIC_BYTES {
82                let page_no = (offset + inner as u64) / page_sz;
83                if let Some((txn_id, frames)) = try_recover_candidate(file, page_no, secret)? {
84                    let keep = match &best {
85                        None => true,
86                        Some((id, _)) => txn_id > *id,
87                    };
88                    if keep {
89                        best = Some((txn_id, frames));
90                    }
91                }
92            }
93            inner += PAGE_SIZE;
94        }
95        offset += filled as u64;
96    }
97
98    Ok(best.map(|(_, f)| f))
99}
100
101/// Try to recover a committed transaction from `candidate_page_no`,
102/// returning `(txn_id, frames)` only if the candidate's
103/// `(0, header_page)` frame fully validates: the frame chain commits
104/// (via [`wal::scan_committed_txn`]'s streaming reader), the header
105/// parses with `wal_start == candidate_page_no` **and**
106/// `wal_seq == txn_id`, the DEK unwraps under `passphrase`, and the
107/// AES-GCM seal validates. DUR-1 (R-002.1 item 2) requires the
108/// `wal_seq == txn_id` check and no fixed scan window — the streaming
109/// reader handles transactions of arbitrary size.
110fn try_recover_candidate(
111    file: &mut File,
112    candidate_page_no: u64,
113    secret: &[u8],
114) -> Result<Option<(u64, Vec<wal::Frame>)>> {
115    let Some((txn_id, frames)) = wal::scan_committed_txn(file, candidate_page_no)? else {
116        return Ok(None);
117    };
118
119    // Must contain a header frame at page 0.
120    let header_bytes = match frames.iter().find(|(p, _)| *p == 0) {
121        Some((_, b)) => b,
122        None => return Ok(None),
123    };
124    if header_bytes.len() != PAGE_SIZE {
125        return Ok(None);
126    }
127    let mut hbuf = [0u8; PAGE_SIZE];
128    hbuf.copy_from_slice(header_bytes);
129    let candidate = match Header::from_page(&hbuf) {
130        Ok(h) => h,
131        Err(_) => return Ok(None),
132    };
133
134    // The frame's claimed region must match where we found it, and the
135    // header frame's `wal_seq` must match the committed transaction's
136    // `txn_id` — this is what pins the frame to the region it was
137    // written into and keeps a stale checkpointed-and-copied frame
138    // from being accepted as a new region's root.
139    if candidate.wal_start != candidate_page_no || candidate.wal_seq != txn_id {
140        return Ok(None);
141    }
142    // DEK unwrap under this candidate's own salt + KDF params.
143    let kdf = KdfParams {
144        m_cost: candidate.m_cost,
145        t_cost: candidate.t_cost,
146        p_cost: candidate.p_cost,
147    };
148    let kek = match crypto::derive_kek(secret, &candidate.salt, kdf) {
149        Ok(k) => k,
150        Err(_) => return Ok(None),
151    };
152    let dek = match crypto::unwrap_dek(&kek, &candidate.dek_nonce, &candidate.wrapped_dek) {
153        Ok(d) => d,
154        Err(_) => return Ok(None),
155    };
156    if candidate.validate_seal(&dek).is_err() {
157        return Ok(None);
158    }
159
160    Ok(Some((txn_id, frames)))
161}
162
163impl Mnemo {
164    /// Create a brand-new encrypted database at `path`.
165    pub fn create(
166        path: impl Into<PathBuf>,
167        passphrase: &str,
168        config: MnemoConfig,
169    ) -> Result<Mnemo> {
170        Self::create_secret(path, passphrase.as_bytes(), config)
171    }
172
173    /// PERF-1 step 7 (R-004.5): create a database unlocked by a **raw
174    /// 256-bit key** instead of a passphrase, for keys held by an OS
175    /// keychain or keystore. The key must be uniformly random (see
176    /// [`Mnemo::generate_key`]); it gets no stretching. The header records
177    /// minimal Argon2id parameters ([`KdfParams::raw_key`]), so opening
178    /// costs microseconds instead of tens of milliseconds. `config.kdf` is
179    /// ignored. No format change: the key goes through the same KEK
180    /// derivation a passphrase does.
181    pub fn create_with_key(
182        path: impl Into<PathBuf>,
183        key: &[u8; crypto::KEY_LEN],
184        config: MnemoConfig,
185    ) -> Result<Mnemo> {
186        let config = MnemoConfig {
187            kdf: KdfParams::raw_key(),
188            ..config
189        };
190        Self::create_secret(path, key, config)
191    }
192
193    /// A fresh random 256-bit key for [`Mnemo::create_with_key`], from the
194    /// thread-local CSPRNG (seeded from the OS). Store it in a keychain;
195    /// it can't be recovered from the file.
196    pub fn generate_key() -> [u8; crypto::KEY_LEN] {
197        *crypto::random_dek()
198    }
199
200    pub(super) fn create_secret(
201        path: impl Into<PathBuf>,
202        secret: &[u8],
203        config: MnemoConfig,
204    ) -> Result<Mnemo> {
205        let path: PathBuf = path.into();
206        if config.dimensions == 0 {
207            return Err(MnemoError::Invalid("dimensions must be > 0".into()));
208        }
209        let version = resolve_format_version(config.format_version)?;
210        if version >= VERSION_V9 && veccol::per_page(config.dimensions) == 0 {
211            return Err(MnemoError::Invalid(format!(
212                "a v9 file supports up to {} dimensions for now (one fp16 vector \
213                 must fit a page); vector entries that span pages are not supported yet",
214                (PAYLOAD - veccol::PAGE_HEADER - 4) / 2
215            )));
216        }
217        let flags = if version >= VERSION_V9 && config.store_float_vectors {
218            FLAG_ENCRYPTED | FLAG_FLOAT_VECTORS
219        } else {
220            FLAG_ENCRYPTED
221        };
222        // Exclusive lock covers the write path; second writer sees `Locked`.
223        let file = open_with_lock(&path, /*write=*/ true, /*create=*/ true)?;
224
225        let salt = crypto::random_salt();
226        let dek = crypto::random_dek();
227        let kek = crypto::derive_kek(secret, &salt, config.kdf)?;
228        let dek_nonce = crypto::random_nonce();
229        let wrapped = crypto::wrap_dek(&kek, &dek_nonce, &dek)?;
230        if wrapped.len() != WRAPPED_DEK_LEN {
231            return Err(MnemoError::Crypto("unexpected wrapped DEK length".into()));
232        }
233        let mut wrapped_dek = [0u8; WRAPPED_DEK_LEN];
234        wrapped_dek.copy_from_slice(&wrapped);
235
236        let mut header = Header {
237            version,
238            page_size: PAGE_SIZE as u32,
239            flags,
240            dimensions: config.dimensions as u32,
241            created_at: memory::now_secs(),
242            write_counter: 0,
243            // Page 0 is the header; pages 1..=wal_pages are the WAL;
244            // record/catalog/index pages start after it. Initial WAL size is
245            // chosen per-config (default 8 pages = 64 KiB) and clamped to a
246            // sane floor so the first transaction can always commit.
247            next_page: 1 + config.wal_pages_initial.max(MIN_WAL_PAGES),
248            catalog_start: 0,
249            catalog_pages: 0,
250            catalog_len: 0,
251            vector_count: 0,
252            m_cost: config.kdf.m_cost,
253            t_cost: config.kdf.t_cost,
254            p_cost: config.kdf.p_cost,
255            salt,
256            dek_nonce,
257            wrapped_dek,
258            index_start: 0,
259            index_pages: 0,
260            index_len: 0,
261            wal_start: 1,
262            wal_pages: config.wal_pages_initial.max(MIN_WAL_PAGES),
263            wal_seq: 0,
264            manifest_start: 0,
265            manifest_pages: 0,
266            manifest_len: 0,
267            // Regenerated per write by `apply_seal`; `None` until we seal.
268            seal_nonce: None,
269            seal_tag: None,
270            // v8+ result-cache pointer — zero-initialized on create,
271            // populated only when `cache_put` runs at least once and
272            // the next `flush` seals a directory run.
273            cache_start: 0,
274            cache_pages: 0,
275            cache_len: 0,
276            // FMT-9 v9 fields; all zero on a fresh file (and unused on v8).
277            catalog_gen: 0,
278            index_gen: 0,
279            manifest_gen: 0,
280            cache_gen: 0,
281            free: RunPtr::default(),
282            keys: RunPtr::default(),
283            bm25: RunPtr::default(),
284            index_delta: RunPtr::default(),
285            catalog_delta: RunPtr::default(),
286        };
287
288        let mut pager = Pager::new(file, dek, 0, version);
289        let mut page = header.to_page();
290        header.apply_seal(&mut page, pager.dek())?;
291        pager.write_raw(0, &page)?;
292        pager.sync()?;
293
294        Ok(Mnemo {
295            pager,
296            header,
297            catalog: Vec::new(),
298            index: IdIndex::default(),
299            path,
300            dimensions: config.dimensions,
301            kdf: config.kdf,
302            dirty_catalog: false,
303            ann: None,
304            dirty_index: false,
305            index_persist: IndexPersist::default(),
306            ann_unloaded: false,
307            manifest: Vec::new(),
308            // FMT-9c: v9 keeps fewer snapshots by default: with page reuse,
309            // retention is what pins pages.
310            max_snapshots: if version >= VERSION_V9 && config.max_snapshots == DEFAULT_MAX_SNAPSHOTS
311            {
312                V9_DEFAULT_MAX_SNAPSHOTS
313            } else {
314                config.max_snapshots
315            },
316            read_only: false,
317            cache_dir: Vec::new(),
318            cache_idx: DirectoryIndex::new(),
319            dirty_cache: false,
320            cache_budgets: std::collections::HashMap::new(),
321            cache_flush_policy: CacheFlushPolicy::default(),
322            cache_batch: BatchState::default(),
323            cache_counters: CounterState::default(),
324            embedder: None,
325            fsyncs_last_flush: 0,
326            agent_set: None,
327            key_index: None,
328            superseded: None,
329            layers: None,
330            rekey_cleanup: None,
331            turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
332            free: FreeList::default(),
333            pending_free: Vec::new(),
334            free_dirty: false,
335            key_slots: KeySlotRun::default(),
336            dirty_keys: false,
337            gen_leased: false,
338            col: VecColumn::default(),
339            recs: RecPages::default(),
340            bm25: Bm25State::default(),
341            kw: None,
342            index_layout_override: None,
343            dirty_rows: std::collections::BTreeSet::new(),
344            delta_chain: Vec::new(),
345            delta_rows: 0,
346            catalog_rewrite: false,
347            index_chain: Vec::new(),
348            index_chain_ops: 0,
349            index_rebase: false,
350            index_delta_ops: None,
351            snapshot_cap_explicit: config.max_snapshots != DEFAULT_MAX_SNAPSHOTS,
352            direct_writes: false,
353            vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
354        })
355    }
356
357    /// Open an existing database. A wrong passphrase fails cleanly with
358    /// [`MnemoError::WrongPassphrase`]. Takes an OS exclusive lock for
359    /// the lifetime of the returned handle; a second concurrent open
360    /// (from any process) returns [`MnemoError::Locked`].
361    ///
362    /// For a caller that only needs to read (and can tolerate the
363    /// pending-recovery / pending-migration cases documented on
364    /// [`Mnemo::open_read_only`]), prefer that entry point — it takes a
365    /// shared lock and coexists with other readers.
366    pub fn open(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
367        Self::open_secret(path, passphrase.as_bytes())
368    }
369
370    /// PERF-1 step 7: open a database created by [`Mnemo::create_with_key`]
371    /// (or rekeyed with [`Mnemo::rekey_with_key`]) with its raw key.
372    pub fn open_with_key(path: impl Into<PathBuf>, key: &[u8; crypto::KEY_LEN]) -> Result<Mnemo> {
373        Self::open_secret(path, key)
374    }
375
376    pub(super) fn open_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
377        Self::open_secret_with(path, secret, false)
378    }
379
380    /// The read-write open. `allow_old` lets a file older than v9 open (and
381    /// migrate in memory to v8): only compaction uses it, to read a v4–v8
382    /// file and write the v9 copy. Every public read-write open refuses
383    /// those with [`MnemoError::FormatTooOld`] (the v8 sunset, v0.13).
384    pub(super) fn open_secret_with(
385        path: impl Into<PathBuf>,
386        secret: &[u8],
387        allow_old: bool,
388    ) -> Result<Mnemo> {
389        let path: PathBuf = path.into();
390        let mut file = open_with_lock(&path, /*write=*/ true, /*create=*/ false)?;
391
392        let mut hbuf = [0u8; PAGE_SIZE];
393        file.seek(SeekFrom::Start(0))?;
394        file.read_exact(&mut hbuf)?;
395        let mut header = match Header::from_page(&hbuf) {
396            Ok(h) => h,
397            Err(e) => {
398                // DUR-1 (R-002.1 part 2+3): page 0 did not parse — any
399                // error is treated as a torn-header candidate, not just
400                // `HeaderChecksum`. The whole 294-byte header lives in
401                // the first 512-byte sector, so the realistic failure is
402                // the sector zeroed or garbled, which returns `BadMagic`
403                // or `UnsupportedVersion` *before* the CRC gate.
404                //
405                // Scan the file for valid committed header-frame WAL
406                // regions, pick the highest `txn_id`, and replay it. If
407                // nothing valid is found, return the *original* error so
408                // non-mnemo files still report `BadMagic`.
409                let scan = scan_header_recovery_candidates(&mut file, secret)?;
410                let frames = scan.ok_or(e)?;
411                // RB-003.3: the scan has checked the key; refuse an old
412                // format before healing anything.
413                if !allow_old {
414                    if let Some(h) = page_zero_header(&frames) {
415                        if h.version < VERSION_V9 {
416                            return Err(MnemoError::FormatTooOld { version: h.version });
417                        }
418                    }
419                }
420                for (page_no, bytes) in &frames {
421                    if bytes.len() != PAGE_SIZE {
422                        return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
423                    }
424                    file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
425                    file.write_all(bytes)?;
426                }
427                file.sync_all()?;
428                crate::bump_fsync();
429                file.seek(SeekFrom::Start(0))?;
430                file.read_exact(&mut hbuf)?;
431                Header::from_page(&hbuf)?
432            }
433        };
434
435        // Crash recovery: a committed-but-uncheckpointed transaction is
436        // replayed below. Each frame is a finished page image; the header
437        // frame, replayed to page 0, supersedes the header just read.
438        let pending = wal::recover(
439            &mut file,
440            header.wal_start,
441            header.wal_pages,
442            header.wal_seq,
443        )?;
444
445        // RB-003.3: settle the key and the format before writing anything,
446        // so a wrong secret or a refused old format leaves the file
447        // byte-identical. Both are checked against the header the file will
448        // have once the WAL is replayed (its page-0 frame, else the current
449        // header), since that header is the one the secret must open. No write
450        // the engine makes leaves a pending page-0 frame whose key wrap differs
451        // from page 0's (a rekey logs its new page 0 into a fresh WAL region,
452        // so a cut-off rekey leaves the old secret or the new one, never a
453        // mix), but checking against the replay's header is the right
454        // defence if one ever does.
455        let target = match pending.as_deref().and_then(page_zero_header) {
456            Some(h) => h,
457            None => header.clone(),
458        };
459        let kdf = KdfParams {
460            m_cost: target.m_cost,
461            t_cost: target.t_cost,
462            p_cost: target.p_cost,
463        };
464        let kek = crypto::derive_kek(secret, &target.salt, kdf)?;
465        let dek = crypto::unwrap_dek(&kek, &target.dek_nonce, &target.wrapped_dek)?;
466        if !allow_old && target.version < VERSION_V9 {
467            return Err(MnemoError::FormatTooOld {
468                version: target.version,
469            });
470        }
471
472        if let Some(frames) = pending {
473            for (page_no, bytes) in &frames {
474                if bytes.len() != PAGE_SIZE {
475                    return Err(MnemoError::Invalid("WAL frame is not page-sized".into()));
476                }
477                file.seek(SeekFrom::Start(page_no * PAGE_SIZE as u64))?;
478                file.write_all(bytes)?;
479            }
480            file.sync_all()?;
481            crate::bump_fsync();
482            // Re-read the now-current header from the replayed page 0.
483            file.seek(SeekFrom::Start(0))?;
484            file.read_exact(&mut hbuf)?;
485            header = Header::from_page(&hbuf)?;
486        }
487
488        // Validate the v7 header seal now that we have the DEK. Returns
489        // `MnemoError::HeaderTampered` if any of the sealed mutable fields
490        // were rewritten after the last legitimate flush. No-op for v6 and
491        // older — pre-v7 files don't carry a seal.
492        //
493        // DUR-1 (Planner decision 2026-11-01, Option B): a seal failure is
494        // **always** reported as tampering, never auto-healed. R-002.1
495        // part 3's seal-mismatch WAL fallback is withdrawn: the seal
496        // exists for tamper-evidence; silently loading a stale committed
497        // header from the WAL would erase that evidence. A real crash
498        // cannot produce "CRC valid + seal invalid" either, because the
499        // whole 294-byte header lies inside sector 0 which storage
500        // writes atomically — a torn sector breaks magic, version or
501        // CRC first, and *those* are healed by the scan above.
502        header.validate_seal(&dek)?;
503
504        // Build the pager at the file's *current* version so initial reads
505        // use the matching AAD scheme — `Pager::page_aad` returns empty AAD
506        // for v4/v5 (no page binding) and `page_no.to_le_bytes()` for v6+.
507        let on_disk_version = header.version;
508        // The v8 sunset (v0.13): a read-write handle is v9. Older files open
509        // read-only (v8) or through `compact_file`, which converts them.
510        if !allow_old && on_disk_version < VERSION_V9 {
511            return Err(MnemoError::FormatTooOld {
512                version: on_disk_version,
513            });
514        }
515        let mut pager = Pager::new(file, dek, header.write_counter, on_disk_version);
516        let dimensions = header.dimensions as usize;
517
518        // === v4→v5 catalog migration =====================================
519        // v6 reads directly; v5 reads directly; v4 replays the catalog into
520        // the v5 shape by reading each memory's body to populate the new
521        // `accessed_at` / `access_count` fields. All page reads here still
522        // use the v4/v5 AAD scheme (none).
523        let (catalog, migrated_from_v4) = if on_disk_version >= 5 {
524            (
525                load_catalog(
526                    &mut pager,
527                    header.catalog_start,
528                    header.catalog_pages,
529                    header.catalog_len,
530                    header.catalog_gen,
531                )?,
532                false,
533            )
534        } else if on_disk_version == 4 {
535            let v4 = load_catalog_v4(
536                &mut pager,
537                header.catalog_start,
538                header.catalog_pages,
539                header.catalog_len,
540            )?;
541            (migrate_v4_catalog(&mut pager, v4)?, true)
542        } else {
543            return Err(MnemoError::UnsupportedVersion(on_disk_version));
544        };
545        // FMT-9c.2: replay the v9 delta chain onto the base catalog.
546        let (catalog, delta_chain, delta_rows) =
547            apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
548        let index = build_id_index(&catalog);
549        // PERF-1 step 3: the ANN index loads on first use, not at open. A
550        // v5→v6 page migration is the exception: it must read the index
551        // now, under the old page AAD, to rewrite it.
552        let ann = if on_disk_version < 6 {
553            load_index(
554                &mut pager,
555                header.index_start,
556                header.index_pages,
557                header.index_len,
558                header.index_gen,
559                dimensions,
560            )?
561        } else {
562            None
563        };
564        let ann_unloaded = ann.is_none() && header.index_pages > 0;
565        let mut manifest = load_manifest(
566            &mut pager,
567            header.manifest_start,
568            header.manifest_pages,
569            header.manifest_len,
570            header.manifest_gen,
571        )?;
572
573        // === v5→v6 page-encryption migration =============================
574        // Pre-v6 data pages were encrypted under no AAD; v6 binds the home
575        // `page_no` as AAD so a page-transplant attack fails authentication
576        // on read. Re-stage every live record page through `write_page` so
577        // the next flush re-encrypts it under v6 AAD at the same home slot.
578        // Tombstoned entries skip — they're reclaimed by `compact_file`.
579        // Old catalog / ANN / manifest pages stay as orphan ciphertext;
580        // the next flush writes the new runs at fresh slots.
581        let migrated_pages_to_v6 = on_disk_version < 6;
582        if migrated_pages_to_v6 {
583            for e in &catalog {
584                if e.deleted {
585                    continue;
586                }
587                for i in 0..e.page_count as u64 {
588                    let payload = pager.read_page(e.start_page + i, 0)?;
589                    pager.write_page(e.start_page + i, 0, &payload)?;
590                }
591            }
592            pager.set_version(6);
593        }
594
595        // === v6→v7 header-seal migration =================================
596        // v7 adds an AES-GCM seal at the tail of the header page that
597        // authenticates every mutable header field. Pre-v7 files have no
598        // seal — we just bump the version in memory and mark `dirty_catalog`
599        // so the next flush writes a sealed v7 header. Pages are untouched
600        // (page crypto is unchanged from v6).
601        let migrated_header_to_v7 = on_disk_version < 7;
602        if migrated_header_to_v7 {
603            pager.set_version(7);
604        }
605
606        // === v7→v8 cache-directory migration =============================
607        // v8 adds three u64 header fields (cache_start/pages/len) for the
608        // result-cache directory (Phase 10.1). Pre-v8 files have those
609        // three bytes as zero (the on-disk region beyond the v7 seal is
610        // zero-initialized), which is the legal "empty cache directory"
611        // state — no data migration required. We bump the version in
612        // memory and mark dirty_catalog so the next flush writes a
613        // v8-sealed header with the new AAD shape covering the cache
614        // pointer. Pages are untouched.
615        let migrated_header_to_v8 = on_disk_version < 8;
616        if migrated_header_to_v8 {
617            pager.set_version(VERSION);
618        }
619
620        let migrated = migrated_from_v4
621            || migrated_pages_to_v6
622            || migrated_header_to_v7
623            || migrated_header_to_v8;
624        if migrated {
625            // Stamp the version forward in memory; the next flush persists
626            // it via the WAL-committed header frame (and the v7 seal).
627            header.version = VERSION;
628            // Snapshots written under an older format reference page runs
629            // encrypted under the old AAD scheme; this build can't decrypt
630            // them under the new pager. Drop the manifest so the next
631            // flush records a fresh snapshot of the migrated state.
632            // Point-in-time recovery into the pre-migration past is
633            // sacrificed for migration simplicity. Old pages remain on
634            // disk until `compact_file`. (For v6→v7 alone — which doesn't
635            // change page crypto — the old snapshots would technically
636            // still be readable, but dropping them uniformly keeps the
637            // migration policy simple.)
638            manifest.clear();
639        }
640
641        // The ANN index pages on disk were sealed under the *old* AAD scheme
642        // if we migrated pages to v6; force a rewrite at fresh v6-encrypted
643        // slots on the next flush so a future reopen doesn't try to read
644        // them under the new AAD and fail. (Catalog and manifest already
645        // get rewritten via dirty_catalog and the manifest.clear() above.)
646        let dirty_index = migrated_pages_to_v6 && ann.is_some();
647
648        // Load the cache directory (v8+ only). Pre-v8 files have
649        // cache_pages == 0 by header layout, so this trivially yields
650        // an empty directory without touching the disk.
651        let cache_dir = load_cache_directory(
652            &mut pager,
653            header.cache_start,
654            header.cache_pages,
655            header.cache_len,
656            header.cache_gen,
657        )?;
658        let cache_idx = DirectoryIndex::build(&cache_dir);
659        // FMT-9 (a)/(b): the free list and key-slot run (empty pointers on
660        // v8, so these don't touch the disk there).
661        let free = load_free_list(&mut pager, header.free)?;
662        let key_slots = load_key_slots(&mut pager, header.keys)?;
663        let col = VecColumn::from_catalog(&catalog);
664        let recs = RecPages::from_catalog(&catalog);
665        let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
666        let max_snapshots = if header.version >= VERSION_V9 {
667            V9_DEFAULT_MAX_SNAPSHOTS
668        } else {
669            DEFAULT_MAX_SNAPSHOTS
670        };
671
672        Ok(Mnemo {
673            pager,
674            header,
675            catalog,
676            index,
677            path,
678            dimensions,
679            kdf,
680            dirty_catalog: migrated,
681            ann,
682            dirty_index,
683            // A v5→v6 page migration leaves the index sealed under the old
684            // page AAD, so it must be rewritten on the next flush.
685            index_persist: IndexPersist {
686                forced: dirty_index,
687                ..IndexPersist::default()
688            },
689            ann_unloaded,
690            manifest,
691            // `Mnemo::open` has no [`MnemoConfig`] to read; default the cap
692            // to [`DEFAULT_MAX_SNAPSHOTS`]. Override with
693            // [`Mnemo::set_max_snapshots`] right after open if you need
694            // unlimited retention or a different cap for this handle.
695            max_snapshots,
696            read_only: false,
697            cache_dir,
698            cache_idx,
699            dirty_cache: false,
700            cache_budgets: std::collections::HashMap::new(),
701            cache_flush_policy: CacheFlushPolicy::default(),
702            cache_batch: BatchState::default(),
703            cache_counters: CounterState::default(),
704            embedder: None,
705            fsyncs_last_flush: 0,
706            agent_set: None,
707            key_index: None,
708            superseded: None,
709            layers: None,
710            rekey_cleanup: None,
711            turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
712            free,
713            pending_free: Vec::new(),
714            free_dirty: false,
715            key_slots,
716            dirty_keys: false,
717            gen_leased: false,
718            col,
719            recs,
720            bm25: Bm25State {
721                manifest: bm25_manifest,
722                ..Bm25State::default()
723            },
724            kw: None,
725            index_layout_override: None,
726            dirty_rows: std::collections::BTreeSet::new(),
727            delta_chain,
728            delta_rows,
729            catalog_rewrite: false,
730            index_chain: Vec::new(),
731            index_chain_ops: 0,
732            index_rebase: false,
733            index_delta_ops: None,
734            snapshot_cap_explicit: false,
735            direct_writes: false,
736            vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
737        })
738    }
739
740    /// Open an existing database in **read-only** mode. Takes an OS shared
741    /// lock; multiple read-only handles can coexist, but any attempt to
742    /// open the same file read-write elsewhere returns
743    /// [`MnemoError::Locked`]. All mutating methods on the returned handle
744    /// return [`MnemoError::ReadOnly`], including `recall` with
745    /// `track_access(true)`.
746    ///
747    /// Two conditions block a read-only open and return
748    /// [`MnemoError::NeedsWriteOpen`]:
749    ///
750    /// 1. The file has a committed-but-uncheckpointed WAL transaction that
751    ///    would ordinarily be replayed on open. A read-only handle
752    ///    cannot write the recovered pages back.
753    /// 2. The on-disk format version is older than the current
754    ///    `format::VERSION`, requiring an on-disk migration.
755    ///
756    /// In either case, open the file read-write once (which recovers /
757    /// migrates in place), close the handle cleanly, and reopen
758    /// read-only. This is documented in the README's Durability section.
759    pub fn open_read_only(path: impl Into<PathBuf>, passphrase: &str) -> Result<Mnemo> {
760        Self::open_read_only_secret(path, passphrase.as_bytes())
761    }
762
763    /// PERF-1 step 7: [`Mnemo::open_read_only`] for a raw-key database.
764    pub fn open_read_only_with_key(
765        path: impl Into<PathBuf>,
766        key: &[u8; crypto::KEY_LEN],
767    ) -> Result<Mnemo> {
768        Self::open_read_only_secret(path, key)
769    }
770
771    /// Whether `key` opens the raw-key file at `path`, decided from its
772    /// header alone: no catalog load, no lock, microseconds. `Ok(false)` is
773    /// "not this key". Unlike a read-only open, it answers for a file that
774    /// has a WAL transaction to recover (the state a kill between a commit
775    /// and its checkpoint leaves), so a caller trying each of its keys in
776    /// turn isn't stopped by [`MnemoError::NeedsWriteOpen`]. A torn header is
777    /// judged from the WAL frame that would heal it. Other failures (not a
778    /// `.mnemo` file, a tampered header, an I/O error) are errors. Added in
779    /// v0.12.2 (RB-001.23).
780    pub fn key_opens(
781        path: impl AsRef<std::path::Path>,
782        key: &[u8; crypto::KEY_LEN],
783    ) -> Result<bool> {
784        let mut file = File::open(path.as_ref())?;
785        let mut hbuf = [0u8; PAGE_SIZE];
786        file.seek(SeekFrom::Start(0))?;
787        file.read_exact(&mut hbuf)?;
788        let header = match Header::from_page(&hbuf) {
789            Ok(h) => h,
790            // A candidate frame only counts when it validates with `key`.
791            Err(parse_err) => {
792                return match scan_header_recovery_candidates(&mut file, key)? {
793                    Some(_) => Ok(true),
794                    None => Err(parse_err),
795                }
796            }
797        };
798        let kdf = KdfParams {
799            m_cost: header.m_cost,
800            t_cost: header.t_cost,
801            p_cost: header.p_cost,
802        };
803        let kek = crypto::derive_kek(key, &header.salt, kdf)?;
804        let dek = match crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek) {
805            Ok(dek) => dek,
806            Err(MnemoError::WrongPassphrase) => return Ok(false),
807            Err(e) => return Err(e),
808        };
809        header.validate_seal(&dek)?;
810        Ok(true)
811    }
812
813    pub(super) fn open_read_only_secret(path: impl Into<PathBuf>, secret: &[u8]) -> Result<Mnemo> {
814        let path: PathBuf = path.into();
815        let mut file = open_with_lock(&path, /*write=*/ false, /*create=*/ false)?;
816
817        let mut hbuf = [0u8; PAGE_SIZE];
818        file.seek(SeekFrom::Start(0))?;
819        file.read_exact(&mut hbuf)?;
820        // A torn header on a read-only open cannot be healed (that requires
821        // writing recovered frames back). Surface as `NeedsWriteOpen` so the
822        // caller knows to reopen read-write once. DUR-1 (R-002.1 part 4):
823        // any `Header::from_page` failure (`BadMagic`,
824        // `UnsupportedVersion`, `HeaderChecksum`) qualifies — but only
825        // when the file-wide scan actually finds a valid committed
826        // header frame to recover from, so a non-mnemo file still
827        // reports `BadMagic` cleanly. A `validate_seal` failure is NOT
828        // routed here (Option B: tampering is never auto-healed).
829        let header = match Header::from_page(&hbuf) {
830            Ok(h) => h,
831            Err(parse_err) => {
832                let recoverable = scan_header_recovery_candidates(&mut file, secret)?;
833                return Err(match (parse_err, recoverable) {
834                    // RB-004.5 item 2: as for a pending WAL (RB-003.3), a
835                    // pre-v9 file can't be opened read-write, so the advice
836                    // names the step that does heal it.
837                    (_, Some(frames))
838                        if page_zero_header(&frames).is_some_and(|h| h.version < VERSION_V9) =>
839                    {
840                        MnemoError::NeedsWriteOpen {
841                            reason: "torn header page in a pre-v9 file; run `mnemo compact \
842                                     <file>` (with `--key-file <key>` for a raw-key file), \
843                                     which recovers it and converts the file to v9",
844                        }
845                    }
846                    (_, Some(_)) => MnemoError::NeedsWriteOpen {
847                        reason: "torn header page needs recovery from WAL",
848                    },
849                    (e, None) => e,
850                });
851            }
852        };
853
854        // Refuse to migrate on read-only opens. v6→v7 is header-only
855        // (no page writes required), but even that bumps `version` on
856        // disk and re-seals the header — which needs write access. Keep
857        // the policy uniform across all sub-version bumps.
858        // A v4–v7 file needs migrating, which a read-only handle can't do,
859        // and since v0.13 no read-write handle opens it either: compaction
860        // converts it to v9.
861        if header.version < VERSION {
862            return Err(MnemoError::FormatTooOld {
863                version: header.version,
864            });
865        }
866
867        // Refuse if there is a committed-but-uncheckpointed transaction
868        // waiting for `wal::recover` to fold it into home pages.
869        if wal::recover(
870            &mut file,
871            header.wal_start,
872            header.wal_pages,
873            header.wal_seq,
874        )?
875        .is_some()
876        {
877            // RB-003.3: a pre-v9 file can't be opened read-write any more,
878            // so the usual advice would send the user in a circle.
879            // Compaction recovers the WAL and converts the file.
880            return Err(MnemoError::NeedsWriteOpen {
881                reason: if header.version < VERSION_V9 {
882                    "pending WAL recovery in a pre-v9 file; run `mnemo compact <file>` \
883                     (with `--key-file <key>` for a raw-key file), which recovers it and \
884                     converts the file to v9"
885                } else {
886                    "pending WAL recovery"
887                },
888            });
889        }
890
891        let kdf = KdfParams {
892            m_cost: header.m_cost,
893            t_cost: header.t_cost,
894            p_cost: header.p_cost,
895        };
896        let kek = crypto::derive_kek(secret, &header.salt, kdf)?;
897        let dek = crypto::unwrap_dek(&kek, &header.dek_nonce, &header.wrapped_dek)?;
898        header.validate_seal(&dek)?;
899
900        let mut pager = Pager::new(file, dek, header.write_counter, header.version);
901        let dimensions = header.dimensions as usize;
902
903        let catalog = load_catalog(
904            &mut pager,
905            header.catalog_start,
906            header.catalog_pages,
907            header.catalog_len,
908            header.catalog_gen,
909        )?;
910        // FMT-9c.2: replay the v9 delta chain onto the base catalog.
911        let (catalog, delta_chain, delta_rows) =
912            apply_delta_chain(&mut pager, catalog, header.catalog_delta)?;
913        let index = build_id_index(&catalog);
914        // PERF-1 step 3: the ANN index loads on first use, not at open.
915        let ann: Option<IvfPqIndex> = None;
916        let ann_unloaded = header.index_pages > 0;
917        let manifest = load_manifest(
918            &mut pager,
919            header.manifest_start,
920            header.manifest_pages,
921            header.manifest_len,
922            header.manifest_gen,
923        )?;
924        let cache_dir = load_cache_directory(
925            &mut pager,
926            header.cache_start,
927            header.cache_pages,
928            header.cache_len,
929            header.cache_gen,
930        )?;
931        let cache_idx = DirectoryIndex::build(&cache_dir);
932        let free = load_free_list(&mut pager, header.free)?;
933        let key_slots = load_key_slots(&mut pager, header.keys)?;
934        let col = VecColumn::from_catalog(&catalog);
935        let recs = RecPages::from_catalog(&catalog);
936        let bm25_manifest = load_bm25_manifest(&mut pager, header.bm25)?;
937        let max_snapshots = if header.version >= VERSION_V9 {
938            V9_DEFAULT_MAX_SNAPSHOTS
939        } else {
940            DEFAULT_MAX_SNAPSHOTS
941        };
942
943        Ok(Mnemo {
944            pager,
945            header,
946            catalog,
947            index,
948            path,
949            dimensions,
950            kdf,
951            // No mutation is possible; leave both dirty flags false so
952            // `close()` is a genuine no-op even if we later add an
953            // auto-flush on drop.
954            dirty_catalog: false,
955            ann,
956            dirty_index: false,
957            index_persist: IndexPersist::default(),
958            ann_unloaded,
959            manifest,
960            max_snapshots,
961            read_only: true,
962            cache_dir,
963            cache_idx,
964            dirty_cache: false,
965            cache_budgets: std::collections::HashMap::new(),
966            cache_flush_policy: CacheFlushPolicy::default(),
967            cache_batch: BatchState::default(),
968            cache_counters: CounterState::default(),
969            embedder: None,
970            fsyncs_last_flush: 0,
971            agent_set: None,
972            key_index: None,
973            superseded: None,
974            layers: None,
975            rekey_cleanup: None,
976            turn_compress_after: DEFAULT_TURN_COMPRESS_AFTER_SECS,
977            free,
978            pending_free: Vec::new(),
979            free_dirty: false,
980            key_slots,
981            dirty_keys: false,
982            gen_leased: false,
983            col,
984            recs,
985            bm25: Bm25State {
986                manifest: bm25_manifest,
987                ..Bm25State::default()
988            },
989            kw: None,
990            index_layout_override: None,
991            dirty_rows: std::collections::BTreeSet::new(),
992            delta_chain,
993            delta_rows,
994            catalog_rewrite: false,
995            index_chain: Vec::new(),
996            index_chain_ops: 0,
997            index_rebase: false,
998            index_delta_ops: None,
999            snapshot_cap_explicit: false,
1000            direct_writes: false,
1001            vector_cache_budget: DEFAULT_VECTOR_CACHE_BUDGET,
1002        })
1003    }
1004
1005    /// FMT-9e: compress v9 thread-turn records once they're older than
1006    /// `secs` (by their `created_at`). The default,
1007    /// [`DEFAULT_TURN_COMPRESS_AFTER_SECS`], is 7 days: recent turns stay
1008    /// plain, so a live session reads them at full speed, and each commit
1009    /// compresses up to 256 turns that have aged past the cutoff. `0`
1010    /// compresses every turn as it's written. Compaction always compresses
1011    /// every turn, and so does a vacuum that moves one. Applies to this
1012    /// handle only (not persisted); a no-op on v8 files, which never
1013    /// compress. Reading never depends on it.
1014    pub fn set_turn_compress_after(&mut self, secs: u64) {
1015        self.turn_compress_after = secs.min(i64::MAX as u64) as i64;
1016    }
1017
1018    /// Override the manifest snapshot cap on this open handle. `0` disables
1019    /// the cap; any positive value keeps the most-recent `max` snapshots and
1020    /// drops the rest on the next flush. See [`MnemoConfig::max_snapshots`].
1021    pub fn set_max_snapshots(&mut self, max: usize) {
1022        self.max_snapshots = max;
1023        // FMT-9c: an explicit count is a hard cap on v9 (see
1024        // `apply_retention`).
1025        self.snapshot_cap_explicit = true;
1026    }
1027
1028    /// Embedding dimensionality this database expects.
1029    pub fn dimensions(&self) -> usize {
1030        self.dimensions
1031    }
1032
1033    /// Number of live (non-deleted) memories.
1034    pub fn len(&self) -> usize {
1035        self.catalog.iter().filter(|e| !e.deleted).count()
1036    }
1037
1038    /// True if the database holds no live memories.
1039    pub fn is_empty(&self) -> bool {
1040        self.len() == 0
1041    }
1042
1043    // --- internal page/record helpers -------------------------------------
1044}