Skip to main content

mnemo_engine/store/
flush.rs

1//! Commits and durability: sealed runs, the WAL and checkpoint, flush, close, and the page-cache and vector-budget settings.
2
3use super::*;
4
5impl Mnemo {
6    /// Seal a serialized buffer into a fresh run of encrypted home pages and
7    /// append a WAL frame for each. Returns the run's `(start_page, pages)`.
8    /// The pages are *not* written to their home locations here — they go to
9    /// the WAL and are folded in later by [`Mnemo::checkpoint`].
10    ///
11    /// `gen` is the run's generation (v9; `0` on v8). On v9 the pages come
12    /// from the free list when an eligible run fits.
13    pub(super) fn seal_run(
14        &mut self,
15        bytes: &[u8],
16        gen: u64,
17        frames: &mut Vec<wal::Frame>,
18    ) -> Result<(u64, u32)> {
19        let pc = bytes.len().div_ceil(PAYLOAD).max(1);
20        let start = self.alloc_pages(pc as u64);
21        self.seal_run_at(start, pc as u64, bytes, gen, frames)?;
22        Ok((start, pc as u32))
23    }
24
25    /// Seal `bytes` into the `pages` already-allocated pages at `start`
26    /// (pages past the data are sealed empty) and append their WAL frames.
27    pub(super) fn seal_run_at(
28        &mut self,
29        start: u64,
30        pages: u64,
31        bytes: &[u8],
32        gen: u64,
33        frames: &mut Vec<wal::Frame>,
34    ) -> Result<()> {
35        // RB-007.6: no v9 page is sealed before this window's lease.
36        debug_assert!(
37            !self.is_v9() || self.gen_leased,
38            "a v9 run sealed before the generation lease"
39        );
40        let direct = self.is_v9() && pages > V9_WAL_RUN_MAX_PAGES;
41        for i in 0..pages as usize {
42            let lo = (i * PAYLOAD).min(bytes.len());
43            let hi = ((i + 1) * PAYLOAD).min(bytes.len());
44            let page_no = start + i as u64;
45            let sealed = self.pager.seal_page(page_no, gen, &bytes[lo..hi])?;
46            if direct {
47                self.pager.write_sealed(page_no, &sealed)?;
48            } else {
49                frames.push((page_no, sealed.to_vec()));
50            }
51        }
52        if direct {
53            self.direct_writes = true;
54        }
55        Ok(())
56    }
57
58    /// Grow the WAL region if `frame_pages` worth of frames would not fit.
59    /// Called at the top of `flush`, where the WAL is always spent (every
60    /// flush ends with a checkpoint), so relocating it cannot strand a
61    /// committed transaction.
62    pub(super) fn ensure_wal_capacity(&mut self, frame_pages: usize) -> Result<()> {
63        let need = wal::txn_byte_len(frame_pages, PAGE_SIZE);
64        if need <= self.header.wal_pages * PAGE_SIZE as u64 {
65            return Ok(());
66        }
67        // Allocate a new, larger region at the file tail (~1.5x headroom).
68        let req = need.div_ceil(PAGE_SIZE as u64);
69        let new_pages = (req + req / 2 + 4).max(DEFAULT_WAL_PAGES);
70        let old_wal_start = self.header.wal_start;
71        let old_wal_pages = self.header.wal_pages;
72        // FMT-9 (a): the new region always comes from the file's tail: the
73        // relocation commits a header-only transaction, so a region taken
74        // from the free list would still be listed as free in the
75        // committed free-list run.
76        let new_start = self.header.next_page;
77        self.header.next_page += new_pages;
78        self.header.wal_start = new_start;
79        self.header.wal_pages = new_pages;
80        self.header.wal_seq += 1;
81        // DUR-1 (R-002.1): WAL-log the header change into the *new* region
82        // first, checkpoint page 0, then invalidate the old region. A crash
83        // anywhere here either keeps the old state (if page 0 wasn't yet
84        // overwritten) or recovers to the new state via the torn-header
85        // scan in `Mnemo::open`, which prefers the highest-txn-id valid
86        // committed header frame.
87        self.relocate_header_through_wal(old_wal_start)?;
88        // FMT-9 (a): once the relocation has committed, the old region is
89        // unreferenced; the next commit frees it.
90        self.release_pages(old_wal_start, old_wal_pages);
91        Ok(())
92    }
93
94    /// Shared helper for every post-`create` page-0 commit: WAL-log the
95    /// new header into the current WAL region, checkpoint page 0, and
96    /// invalidate `old_wal_start` (the previous region, or the current
97    /// region when it hasn't moved). The caller is expected to have
98    /// mutated `self.header` to its target values *including a bumped
99    /// `wal_seq`* (one more than the last on-disk `wal_seq`); this
100    /// routine applies the seal, writes the frame + commit, folds page 0,
101    /// then zeroes the old region's first frame header.
102    ///
103    /// If `old_wal_start == self.header.wal_start`, the invalidation is
104    /// skipped — the region didn't move, so overwriting its first frame
105    /// would destroy the just-committed one.
106    pub(super) fn relocate_header_through_wal(&mut self, old_wal_start: u64) -> Result<()> {
107        self.header.write_counter = self.pager.write_counter;
108        let mut page = self.header.to_page();
109        self.header.apply_seal(&mut page, self.pager.dek())?;
110        let txn_id = self.header.wal_seq;
111        let frames = vec![(0u64, page.to_vec())];
112        wal::commit(
113            self.pager.file_mut(),
114            self.header.wal_start,
115            self.header.wal_pages,
116            txn_id,
117            &frames,
118        )?;
119        self.pager.write_raw(0, &page)?;
120        self.pager.sync()?;
121        if old_wal_start != self.header.wal_start {
122            wal::invalidate_region(self.pager.file_mut(), old_wal_start)?;
123        }
124        Ok(())
125    }
126
127    /// Fold a committed transaction's WAL frames into their home pages.
128    pub(super) fn checkpoint(&mut self, frames: &[wal::Frame]) -> Result<()> {
129        for (page_no, bytes) in frames {
130            let mut img = [0u8; PAGE_SIZE];
131            img.copy_from_slice(bytes);
132            self.pager.write_sealed(*page_no, &img)?;
133        }
134        self.pager.sync()?;
135        Ok(())
136    }
137
138    /// Persist all pending changes as one **write-ahead-logged transaction**.
139    ///
140    /// Sequence:
141    ///
142    /// 1. **Prepare.** Serialize the control plane (catalog, ANN index),
143    ///    grow the WAL if needed, and **lease** counter and page slots —
144    ///    bump `header.write_counter` and `header.next_page` by upper
145    ///    bounds on this transaction's consumption and persist that
146    ///    *leased* header to disk before any encrypted page is written.
147    /// 2. **Data pages.** [`Pager::flush`] writes dirty record pages
148    ///    copy-on-write under fresh nonces and fsyncs them. The orphan-page
149    ///    nonce-reuse window (see [Phase 1.1 of the improvement plan])
150    ///    is closed because the leased header on disk already records a
151    ///    `write_counter` past anything this step can produce.
152    /// 3. **Control plane.** Seal the new catalog / ANN index / snapshot
153    ///    manifest into fresh home page runs.
154    /// 4. **Commit.** Log all of step 3's frames plus the final header
155    ///    frame into the WAL and fsync — the durability point.
156    /// 5. **Checkpoint.** Fold the WAL into the home pages.
157    ///
158    /// A crash before step 4 leaves the previous state intact. A crash
159    /// after step 4 is repaired by [`Mnemo::open`] replaying the WAL.
160    /// Safe to call repeatedly.
161    pub fn flush(&mut self) -> Result<()> {
162        if self.read_only {
163            return Err(MnemoError::ReadOnly);
164        }
165        // Snapshot the thread-local fsync counter. Every `sync_all` the
166        // pager / wal / checkpoint path makes between here and the end
167        // of this function bumps the same counter; `delta` lands on
168        // `self.fsyncs_last_flush` so `Stats::fsyncs_last_flush` can
169        // report it. The planner wants this to decide whether cutting
170        // fsync count is a viable write-latency win before we touch
171        // the write path's durability model.
172        let fsync_before = crate::fsync_count();
173        self.commit_point = None;
174        // PERF-1 step 3: while the index is unloaded, records written since
175        // its run pile up for the next load to re-assign. Once there are
176        // INDEX_PERSIST_EVERY_CHANGES of them, load it now so this flush
177        // persists a current index and keeps that work bounded.
178        if self.ann_unloaded && self.dirty_catalog {
179            let (index_start, index_gen) = (self.header.index_start, self.index_gen_now());
180            let v9 = self.is_v9();
181            let pending = self
182                .catalog
183                .iter()
184                .filter(|e| !e.deleted && written_after_index(e, v9, index_start, index_gen))
185                .count();
186            if pending >= INDEX_PERSIST_EVERY_CHANGES {
187                self.ensure_ann()?;
188            }
189        }
190        // RB-015.1: a damaged index run left this handle without an index;
191        // a commit that writes rebuilds it (or drops the run on a file too
192        // small for one).
193        if self.index_repair && (self.dirty_catalog || self.dirty_cache) {
194            self.repair_index()?;
195        }
196        // PERF-1: persist the (derived) ANN index only when forced or due.
197        // Otherwise the header keeps pointing at the previous index run, and
198        // records written since then are re-assigned when it's next loaded
199        // (`reconcile_ann_with_catalog`). `ann == None` with `dirty_index` set is a
200        // dropped index, which is persisted at once.
201        let persist_index = if self.is_v9() {
202            self.plan_index_v9()
203        } else {
204            self.dirty_index
205                && (self.index_persist.forced
206                    || self.ann.is_none()
207                    || self.index_persist.commits + 1 >= INDEX_PERSIST_EVERY_COMMITS
208                    || self.index_persist.changes >= INDEX_PERSIST_EVERY_CHANGES)
209        };
210        let free_work =
211            self.is_v9() && (self.free_dirty || !self.pending_free.is_empty() || self.dirty_keys);
212        if !self.dirty_catalog && !persist_index && !self.dirty_cache && !free_work {
213            // No control-plane changes pending. Because every `put` that
214            // dirties a data page also dirties the catalog, and every
215            // `cache_put` that dirties a data page also dirties
216            // `dirty_cache`, there can't be dirty data pages either —
217            // nothing to do.
218            self.fsyncs_last_flush = 0;
219            return Ok(());
220        }
221        // v0.14.0 step 1: nothing is written to a file that was replaced at
222        // this handle's path.
223        self.ensure_not_replaced()?;
224
225        // FIX-1 R-001.8 part 3: snapshot the memory/index-dirty flags
226        // before step 3b decides whether to push a restore point.
227        // Only flushes that change the memory catalog or ANN index
228        // need a snapshot — a cache-only flush (`dirty_cache` set,
229        // `dirty_catalog` + `dirty_index` both false) must not push,
230        // because snapshots don't capture cache state and every push
231        // counts against `max_snapshots` (default 256), so a cache-
232        // heavy workload would otherwise evict every real memory
233        // restore point within ~256 cache writes.
234        let is_memory_or_index_flush = self.dirty_catalog || persist_index;
235
236        // FMT-9: a v9 session's first sealed page needs the generation
237        // lease (normally taken by its first record write already).
238        self.ensure_gen_lease()?;
239        // FMT-9e: turns that aged past the cutoff are rewritten compressed
240        // (bounded per commit), then packed with the other waiting records.
241        self.settle_cold_turns()?;
242        // FMT-9b: column entries first; they move rows' vector locations,
243        // so they must land before the catalog is serialized.
244        self.pack_records()?;
245        self.pack_vectors()?;
246
247        // 1. Serialize control plane, ensure WAL capacity, lease counter +
248        //    page slots, and persist the leased header *before* any data
249        //    page hits the disk.
250        let ctx = self.prepare_for_flush(persist_index)?;
251
252        // 2. Record (vector) data pages: copy-on-write to fresh pages, fsynced.
253        //    Safe: the lease guarantees that even if we crash here, reopen
254        //    will see a `write_counter` past every nonce this step uses.
255        self.pager.flush()?;
256
257        // 3. Seal the catalog / index control pages into fresh home runs.
258        //    FMT-9: `txn_id` is this transaction's id and the generation
259        //    of every run it seals. A v9 run replaced here is freed by this
260        //    same commit (`superseded`); v8 never reuses pages.
261        let txn_id = self.header.wal_seq + 1;
262        let gen = self.write_gen();
263        debug_assert!(!self.is_v9() || gen == txn_id);
264        let v9 = self.is_v9();
265        let mut superseded: Vec<(u64, u64)> = Vec::new();
266        let mut frames: Vec<wal::Frame> = Vec::new();
267        // FMT-9c.2: a delta run goes on the chain; a new base replaces the
268        // old base and the whole chain (all freed by this commit). The
269        // chain is adopted once the commit succeeds.
270        let mut new_chain: Option<(Vec<RunPtr>, usize)> = None;
271        if let Some(bytes) = &ctx.cat_bytes {
272            if ctx.cat_delta {
273                let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
274                let ptr = RunPtr {
275                    start,
276                    pages: pages as u64,
277                    len: bytes.len() as u64,
278                    gen,
279                };
280                self.header.catalog_delta = ptr;
281                let mut chain = self.delta_chain.clone();
282                chain.push(ptr);
283                new_chain = Some((chain, self.delta_rows + self.dirty_rows.len()));
284            } else {
285                if v9 && self.header.catalog_pages > 0 {
286                    superseded.push((self.header.catalog_start, self.header.catalog_pages));
287                }
288                if v9 {
289                    for r in &self.delta_chain {
290                        superseded.push((r.start, r.pages));
291                    }
292                    self.header.catalog_delta = RunPtr::default();
293                    new_chain = Some((Vec::new(), 0));
294                }
295                let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
296                self.header.catalog_start = start;
297                self.header.catalog_pages = pages as u64;
298                self.header.catalog_len = bytes.len() as u64;
299                self.header.catalog_gen = gen;
300            }
301        }
302        // FMT-9d: the BM25 segment, then the manifest that lists it.
303        let mut new_bm25: Option<Bm25Segments> = None;
304        if let Some((bytes, replace_from)) = &ctx.bm25_seg {
305            let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
306            let mut segs = self.bm25.manifest.segments.clone();
307            let mut seg_docs = self.bm25.manifest.seg_docs.clone();
308            seg_docs.resize(segs.len(), 0);
309            let docs = if let Some(from) = *replace_from {
310                for t in segs.drain(from..) {
311                    superseded.push((t.0, t.1));
312                }
313                seg_docs.truncate(from);
314                self.bm25.merged_docs
315            } else {
316                self.bm25.pending.len() as u32
317            };
318            segs.push((start, pages as u64, bytes.len() as u64, gen));
319            seg_docs.push(docs);
320            let man = Bm25Manifest {
321                tokenizer: text::TOKENIZER_V1,
322                next_doc: self.bm25.manifest.next_doc,
323                segments: segs.clone(),
324                seg_docs: seg_docs.clone(),
325            };
326            let mbytes =
327                rmp_serde::to_vec(&man).map_err(|e| MnemoError::Serialize(e.to_string()))?;
328            if !self.header.bm25.is_empty() {
329                superseded.push((self.header.bm25.start, self.header.bm25.pages));
330            }
331            let (ms, mp) = self.seal_run(&mbytes, gen, &mut frames)?;
332            self.header.bm25 = RunPtr {
333                start: ms,
334                pages: mp as u64,
335                len: mbytes.len() as u64,
336                gen,
337            };
338            new_bm25 = Some((segs, seg_docs));
339        }
340        let mut new_ichain: Option<(Vec<RunPtr>, usize)> = None;
341        if persist_index && ctx.idx_delta {
342            if let Some(bytes) = &ctx.idx_bytes {
343                let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
344                let ptr = RunPtr {
345                    start,
346                    pages: pages as u64,
347                    len: bytes.len() as u64,
348                    gen,
349                };
350                self.header.index_delta = ptr;
351                let mut chain = self.index_chain.clone();
352                chain.push(ptr);
353                let n = self.index_delta_ops.as_ref().map_or(0, |o| o.len());
354                new_ichain = Some((chain, self.index_chain_ops + n));
355            }
356        } else if persist_index {
357            if v9 && self.header.index_pages > 0 {
358                superseded.push((self.header.index_start, self.header.index_pages));
359            }
360            if v9 {
361                for r in &self.index_chain {
362                    superseded.push((r.start, r.pages));
363                }
364                self.header.index_delta = RunPtr::default();
365                new_ichain = Some((Vec::new(), 0));
366            }
367            match &ctx.idx_bytes {
368                Some(bytes) => {
369                    let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
370                    self.header.index_start = start;
371                    self.header.index_pages = pages as u64;
372                    self.header.index_len = bytes.len() as u64;
373                    self.header.index_gen = gen;
374                }
375                None => {
376                    self.header.index_start = 0;
377                    self.header.index_pages = 0;
378                    self.header.index_len = 0;
379                    self.header.index_gen = 0;
380                }
381            }
382        }
383        // v8+ result-cache directory (Phase 10.1). Seal the serialized
384        // directory into a fresh page run and update the pointer in the
385        // header. The seal AAD already covers cache_start/pages/len for
386        // v8+ (see `header_seal_aad`), so tampering with the pointer
387        // trips the header seal on the next open.
388        if self.dirty_cache {
389            if v9 && self.header.cache_pages > 0 {
390                superseded.push((self.header.cache_start, self.header.cache_pages));
391            }
392            match &ctx.cache_bytes {
393                Some(bytes) if !self.cache_dir.is_empty() => {
394                    let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
395                    self.header.cache_start = start;
396                    self.header.cache_pages = pages as u64;
397                    self.header.cache_len = bytes.len() as u64;
398                    self.header.cache_gen = gen;
399                }
400                _ => {
401                    // Either the directory serialized to nothing (all
402                    // entries deleted after tombstone-only migration) or
403                    // it's empty — collapse the pointer to zero.
404                    self.header.cache_start = 0;
405                    self.header.cache_pages = 0;
406                    self.header.cache_len = 0;
407                    self.header.cache_gen = 0;
408                }
409            }
410        }
411
412        // 3b. Record this transaction as a restorable snapshot. The manifest
413        //     update is staged in a local and adopted only once the commit
414        //     below succeeds, so a failed flush leaves no phantom entry.
415        //
416        //     The cap (`self.max_snapshots`) is applied to the staged local
417        //     before the new entry lands, so after appending the manifest
418        //     holds at most `max_snapshots` entries — the most-recent N.
419        //     `max_snapshots == 0` disables the cap. On v8, pages
420        //     referenced only by pruned snapshots stay on disk until
421        //     `compact_file`; on v9 they become reusable once no retained
422        //     snapshot is older than the transaction that freed them.
423        //
424        //     FIX-1 R-001.8 part 3: skip the snapshot push on a
425        //     cache-only flush. Snapshots don't capture the cache
426        //     anyway, and every push counts against `max_snapshots`
427        //     (default 256) + rewrites the manifest run — so a
428        //     cache-heavy workload would otherwise evict every real
429        //     memory restore point within ~256 cache writes. On a
430        //     cache-only flush we reuse the current manifest
431        //     verbatim; the manifest run already lives at
432        //     `self.header.manifest_{start,pages,len}` (set by the
433        //     last memory-touching flush, or at `create` time).
434        let live = self.len() as u64;
435        let mut manifest = self.manifest.clone();
436        if is_memory_or_index_flush {
437            if self.max_snapshots > 0 && !v9 {
438                // Drop the oldest entries so there's room for the new one.
439                while manifest.len() >= self.max_snapshots {
440                    manifest.remove(0);
441                }
442            }
443            manifest.push(Snapshot {
444                txn_id,
445                created_at: memory::now_secs(),
446                catalog_start: self.header.catalog_start,
447                catalog_pages: self.header.catalog_pages,
448                catalog_len: self.header.catalog_len,
449                index_start: self.header.index_start,
450                index_pages: self.header.index_pages,
451                index_len: self.header.index_len,
452                memory_count: live,
453                gens: v9.then_some(SnapshotGens {
454                    catalog: self.header.catalog_gen,
455                    index: self.header.index_gen,
456                    delta: ptr_tuple(self.header.catalog_delta),
457                    index_delta: ptr_tuple(self.header.index_delta),
458                    bm25: ptr_tuple(self.header.bm25),
459                }),
460            });
461            if v9 {
462                let extra = self.pending_free.iter().map(|r| r.1).sum::<u64>()
463                    + superseded.iter().map(|r| r.1).sum::<u64>()
464                    + self.header.manifest_pages;
465                self.apply_retention(&mut manifest, extra);
466            }
467            let man_bytes =
468                rmp_serde::to_vec(&manifest).map_err(|e| MnemoError::Serialize(e.to_string()))?;
469            if v9 && self.header.manifest_pages > 0 {
470                superseded.push((self.header.manifest_start, self.header.manifest_pages));
471            }
472            let (m_start, m_pages) = self.seal_run(&man_bytes, gen, &mut frames)?;
473            self.header.manifest_start = m_start;
474            self.header.manifest_pages = m_pages as u64;
475            self.header.manifest_len = man_bytes.len() as u64;
476            self.header.manifest_gen = gen;
477        }
478
479        // 3c. FMT-9 (b): the key-slot run, rewritten copy-on-write when a
480        //     slot changed. The header seal covers its pointer.
481        if v9 && self.dirty_keys {
482            if !self.header.keys.is_empty() {
483                superseded.push((self.header.keys.start, self.header.keys.pages));
484            }
485            if self.key_slots.slots.is_empty() {
486                self.header.keys = RunPtr::default();
487            } else {
488                let bytes = rmp_serde::to_vec(&self.key_slots)
489                    .map_err(|e| MnemoError::Serialize(e.to_string()))?;
490                let (start, pages) = self.seal_run(&bytes, gen, &mut frames)?;
491                self.header.keys = RunPtr {
492                    start,
493                    pages: pages as u64,
494                    len: bytes.len() as u64,
495                    gen,
496                };
497            }
498        }
499
500        // 3d. FMT-9 (a): the free list. Everything this transaction
501        //     stopped referencing (`pending_free`, the runs replaced above,
502        //     and the previous free-list run itself) is freed *at this
503        //     transaction's id*, in this transaction's own commit, so it
504        //     can't be reused until this commit is durable and no retained
505        //     snapshot predates it. The list is staged in a local and
506        //     adopted only once the commit succeeds (a failed flush that
507        //     is retried must not free the same runs twice).
508        let mut shrink = false;
509        let staged_free =
510            if v9 && (self.free_dirty || !self.pending_free.is_empty() || !superseded.is_empty()) {
511                let mut staged = self.free.clone();
512                for &(p, c) in self.pending_free.iter().chain(superseded.iter()) {
513                    staged.free(txn_id, p, c)?;
514                }
515                if !self.header.free.is_empty() {
516                    staged.free(txn_id, self.header.free.start, self.header.free.pages)?;
517                }
518                // FMT-9c: give eligible free runs at the file's tail back
519                // to the filesystem: the header's `next_page` moves down in
520                // this commit, and the file is cut after it.
521                let min_snap = self.min_snapshot_txn();
522                while let Some(p) =
523                    staged.take_tail(self.header.next_page, self.header.wal_seq, min_snap)
524                {
525                    self.header.next_page = p;
526                    shrink = true;
527                }
528                // Size the run with room for the one extra entry that
529                // allocating it out of the list can create (a split), then
530                // allocate it from the list (eligible runs only) or the tail.
531                let est = rmp_serde::to_vec(&staged.to_runs())
532                    .map_err(|e| MnemoError::Serialize(e.to_string()))?
533                    .len()
534                    + 64;
535                let pc = est.div_ceil(PAYLOAD).max(1) as u64;
536                let start = match staged.alloc(pc, self.header.wal_seq, self.min_snapshot_txn()) {
537                    Some(p) => p,
538                    None => {
539                        let p = self.header.next_page;
540                        self.header.next_page += pc;
541                        p
542                    }
543                };
544                let bytes = rmp_serde::to_vec(&staged.to_runs())
545                    .map_err(|e| MnemoError::Serialize(e.to_string()))?;
546                if bytes.len() as u64 > pc * PAYLOAD as u64 {
547                    return Err(MnemoError::Invalid(
548                        "free list outgrew its reserved run".into(),
549                    ));
550                }
551                self.seal_run_at(start, pc, &bytes, gen, &mut frames)?;
552                self.header.free = RunPtr {
553                    start,
554                    pages: pc,
555                    len: bytes.len() as u64,
556                    gen,
557                };
558                Some(staged)
559            } else {
560                None
561            };
562
563        // 3e. The header is the transaction's final frame; stamp the new id
564        //     and the *actual* (post-flush) write counter, which is <= the
565        //     leased value persisted in `prepare_for_flush`. This is the
566        //     value that ultimately replaces the leased header on checkpoint.
567        self.header.vector_count = live;
568        self.header.write_counter = self.pager.write_counter;
569        self.header.wal_seq = txn_id;
570        let mut hpage = self.header.to_page();
571        self.header.apply_seal(&mut hpage, self.pager.dek())?;
572        frames.push((0, hpage.to_vec()));
573
574        // 4. COMMIT — log the transaction and fsync. Nothing at a home page
575        //    other than the leased header has changed yet; this single
576        //    fsync is what makes the rest durable.
577        // FMT-9c: big runs went straight to their home pages; make them
578        // durable before the header that references them can be.
579        if self.direct_writes {
580            self.pager.sync()?;
581            self.direct_writes = false;
582        }
583        let (wal_start, wal_pages) = (self.header.wal_start, self.header.wal_pages);
584        // 12.4: from here a failure may leave the transaction committed.
585        self.commit_point = Some(txn_id);
586        wal::commit(self.pager.file_mut(), wal_start, wal_pages, txn_id, &frames)?;
587
588        // 5. Checkpoint — fold the WAL into the home pages.
589        self.checkpoint(&frames)?;
590        // FMT-9c: the committed header no longer covers the tail; cut it.
591        // A crash before this leaves a longer file, which is harmless.
592        if shrink {
593            let len = self.header.next_page * PAGE_SIZE as u64;
594            self.pager.file_mut().set_len(len)?;
595        }
596
597        self.manifest = manifest;
598        if let Some(staged) = staged_free {
599            self.free = staged;
600        }
601        self.pending_free.clear();
602        self.free_dirty = false;
603        self.dirty_keys = false;
604        if let Some((chain, rows)) = new_chain {
605            self.delta_chain = chain;
606            self.delta_rows = rows;
607        }
608        if let Some((chain, n)) = new_ichain {
609            self.index_chain = chain;
610            self.index_chain_ops = n;
611        }
612        if let Some((segs, seg_docs)) = new_bm25 {
613            // FMT-9f: keep the loaded dicts and read only the new segment's
614            // (a merge replaced the segments from `kept` on), instead of
615            // dropping them all and reloading every dict on next use.
616            let kept = self
617                .bm25
618                .manifest
619                .segments
620                .iter()
621                .zip(&segs)
622                .take_while(|(a, b)| a == b)
623                .count();
624            if let Some(mut dicts) = self.bm25.dicts.take() {
625                dicts.truncate(kept);
626                let mut ok = dicts.len() == kept;
627                for &t in &segs[kept..] {
628                    match load_segment_dict(&mut self.pager, tuple_ptr(t)) {
629                        Ok(d) if ok => dicts.push(d),
630                        _ => ok = false,
631                    }
632                }
633                if ok {
634                    self.bm25.dicts = Some(dicts);
635                }
636            }
637            self.bm25.manifest.segments = segs;
638            self.bm25.manifest.seg_docs = seg_docs;
639            self.bm25.pending.clear();
640            self.bm25.pending_tokens = None;
641        }
642        if persist_index {
643            self.index_rebase = false;
644            self.index_delta_ops = None;
645        }
646        if ctx.cat_bytes.is_some() {
647            self.dirty_rows.clear();
648            self.catalog_rewrite = false;
649        }
650        self.dirty_catalog = false;
651        if persist_index {
652            self.dirty_index = false;
653            self.index_persist = IndexPersist::default();
654        } else if self.dirty_index {
655            self.index_persist.commits += 1;
656        }
657        self.dirty_cache = false;
658        // Batched policy: successful flush consumes the pending batch.
659        self.cache_batch.reset();
660        // Record how many fsyncs this flush issued, for Stats reporting.
661        // Thread-local counter guarantees we see only this handle's own
662        // sync_all calls between the snapshot above and this read.
663        self.fsyncs_last_flush =
664            (crate::fsync_count().wrapping_sub(fsync_before)).min(u32::MAX as u64) as u32;
665        // 12.4 shared mode (RB-007.5): this commit is in the file.
666        self.window_committed = true;
667        Ok(())
668    }
669
670    /// Pre-flush prelude shared by [`Mnemo::flush`] and the
671    /// `__crash_partial_flush_for_testing` hook.
672    ///
673    /// Serializes the control plane (catalog, ANN index, cache directory)
674    /// and grows the WAL if needed. That's it: this used to persist a
675    /// leased header too (SEC-1 obsoleted it; see DUR-1 R-002.1 note in
676    /// the body for the full rationale), but after DUR-1 the only page-0
677    /// writes outside `create` go through the WAL via
678    /// [`Mnemo::relocate_header_through_wal`].
679    ///
680    /// The historical pre-stamping used to close the nonce-reuse window
681    /// flagged in Phase 1.1 of the improvement plan: a crash anywhere
682    /// between here and the WAL commit left a file whose on-disk header
683    /// recorded a `write_counter` past every nonce the rest of this
684    /// transaction could produce, so the next reopen started from a
685    /// counter that guaranteed uniqueness against any orphan data pages
686    /// still on disk. SEC-1's per-session random nonce prefix gives the
687    /// same guarantee without needing a persisted counter, which is why
688    /// this routine no longer writes page 0.
689    pub(super) fn prepare_for_flush(&mut self, persist_index: bool) -> Result<FlushPrelude> {
690        // Serialize.
691        // FMT-9c.2: on v9 usually just the changed rows, as a delta run.
692        let cat_delta = self.dirty_catalog && self.catalog_as_delta();
693        let cat_bytes = if !self.dirty_catalog {
694            None
695        } else if cat_delta {
696            let rows: Vec<(u32, CatalogEntry)> = self
697                .dirty_rows
698                .iter()
699                .filter(|&&i| i < self.catalog.len())
700                .map(|&i| (i as u32, self.catalog[i].clone()))
701                .collect();
702            let d = CatalogDelta {
703                prev: ptr_tuple(self.header.catalog_delta),
704                rows,
705            };
706            Some(rmp_serde::to_vec(&d).map_err(|e| MnemoError::Serialize(e.to_string()))?)
707        } else {
708            if self.is_v9() {
709                self.compact_tombstones();
710            }
711            Some(
712                rmp_serde::to_vec(&self.catalog)
713                    .map_err(|e| MnemoError::Serialize(e.to_string()))?,
714            )
715        };
716        // FMT-9c.3: a v9 commit usually writes just its index changes.
717        let idx_delta = persist_index && self.index_delta_ops.is_some();
718        let idx_bytes: Option<Vec<u8>> = if !persist_index {
719            None
720        } else if let Some(ops) = &self.index_delta_ops {
721            let d = IndexDeltaRef {
722                prev: ptr_tuple(self.header.index_delta),
723                ops,
724            };
725            Some(rmp_serde::to_vec(&d).map_err(|e| MnemoError::Serialize(e.to_string()))?)
726        } else {
727            match &self.ann {
728                // FMT-9: v9 index runs carry a layout byte first. PERF-2:
729                // layout 2 (flat arrays) unless a test asks for another.
730                Some(ann) if self.is_v9() => {
731                    let layout = self.index_layout_override.unwrap_or(V9_INDEX_LAYOUT);
732                    let mut b = vec![layout];
733                    if layout == V9_INDEX_LAYOUT {
734                        ann.write_layout2(&mut b);
735                    } else {
736                        b.extend(
737                            rmp_serde::to_vec(ann)
738                                .map_err(|e| MnemoError::Serialize(e.to_string()))?,
739                        );
740                    }
741                    Some(b)
742                }
743                Some(ann) => {
744                    Some(rmp_serde::to_vec(ann).map_err(|e| MnemoError::Serialize(e.to_string()))?)
745                }
746                None => None,
747            }
748        };
749        // Serialize the result-cache directory when it's dirty. An
750        // empty directory serializes to a small header + 0-element
751        // array; still cheap enough that we don't special-case it.
752        let cache_bytes: Option<Vec<u8>> = if self.dirty_cache {
753            Some(
754                rmp_serde::to_vec(&self.cache_dir)
755                    .map_err(|e| MnemoError::Serialize(e.to_string()))?,
756            )
757        } else {
758            None
759        };
760
761        let cat_pc = cat_bytes
762            .as_ref()
763            .map_or(0, |b| b.len().div_ceil(PAYLOAD).max(1));
764        let idx_pc = idx_bytes
765            .as_ref()
766            .map_or(0, |b| b.len().div_ceil(PAYLOAD).max(1));
767        let cache_pc = cache_bytes
768            .as_ref()
769            .map_or(0, |b| b.len().div_ceil(PAYLOAD).max(1));
770        // Upper bound on the manifest run: one extra entry, <=82 bytes each
771        // (rmp_serde encodes a Snapshot as a 9-element fixarray of i64/u64,
772        // worst case 81 bytes plus a 1-byte array header).
773        // v9 snapshots carry two more u64s (≤ 20 bytes with the array header).
774        let per_snapshot = if self.is_v9() { 102 } else { 82 };
775        let man_upper = 9 + (self.manifest.len() + 1) * per_snapshot;
776        let man_pc_est = man_upper.div_ceil(PAYLOAD).max(1);
777        // FMT-9: the free-list run (≤ 28 bytes per run: every pending and
778        // superseded run plus a split and a relocated WAL region) and the
779        // key-slot run.
780        let (free_pc_est, keys_pc) = if self.is_v9() {
781            let runs = self.free.len() + self.pending_free.len() + 12;
782            let free_pc = (runs * 28 + 64).div_ceil(PAYLOAD).max(1);
783            let keys_pc = if self.dirty_keys {
784                rmp_serde::to_vec(&self.key_slots)
785                    .map_err(|e| MnemoError::Serialize(e.to_string()))?
786                    .len()
787                    .div_ceil(PAYLOAD)
788                    .max(1)
789            } else {
790                0
791            };
792            (free_pc, keys_pc)
793        } else {
794            (0, 0)
795        };
796
797        // Grow WAL if needed (DUR-1: relocation is WAL-logged and does
798        // its own fsyncs). The cache directory rides in the same
799        // transaction so its page count contributes to the reservation.
800        // FMT-9d: the BM25 segment (and its manifest, one page).
801        let bm25_seg = self.bm25_next_segment()?;
802        let bm25_pc = bm25_seg
803            .as_ref()
804            .map_or(0, |(b, _)| b.len().div_ceil(PAYLOAD).max(1));
805        // FMT-9c: on v9 only runs of up to V9_WAL_RUN_MAX_PAGES go through
806        // the WAL; longer ones are written in place.
807        let v9 = self.is_v9();
808        let in_wal = |pc: usize| {
809            if v9 && pc as u64 > V9_WAL_RUN_MAX_PAGES {
810                0
811            } else {
812                pc
813            }
814        };
815        self.ensure_wal_capacity(
816            in_wal(cat_pc)
817                + in_wal(idx_pc)
818                + in_wal(cache_pc)
819                + in_wal(man_pc_est)
820                + in_wal(free_pc_est)
821                + in_wal(keys_pc)
822                + in_wal(bm25_pc)
823                + usize::from(bm25_pc > 0)
824                + 1,
825        )?;
826
827        // DUR-1 (R-002.1): the pre-flush "leased header" write that used
828        // to live here is gone. It existed to close the Phase 1.1 nonce
829        // reuse window (reopen must see a `write_counter` past any nonce
830        // this flush could produce before data pages hit disk). SEC-1
831        // superseded that: nonces now carry a per-session random prefix
832        // and `write_counter` is no longer a nonce input, so the leased
833        // write bought us nothing but an extra fsync per flush. Orphan
834        // data pages past `header.next_page` on a crashed flush are
835        // still safe — the next session's random prefix makes any later
836        // write at the same page_no uniquely nonced regardless of
837        // `write_counter`.
838
839        Ok(FlushPrelude {
840            bm25_seg,
841            cat_bytes,
842            cat_delta,
843            idx_delta,
844            idx_bytes,
845            cache_bytes,
846        })
847    }
848
849    /// Flush and close: consumes the handle and releases the file and its
850    /// OS advisory lock before returning, so the same file can be reopened
851    /// right away (by this process or another). On a read-write handle it
852    /// flushes first, persisting a dirty ANN index too; on a read-only
853    /// handle there's nothing to persist.
854    ///
855    /// Using the handle after `close` is a compile error. If the flush
856    /// fails, the error is returned and the handle is dropped all the same:
857    /// changes since the last successful [`Mnemo::flush`] are lost, as
858    /// they would be on a plain `drop` (which never flushes).
859    ///
860    /// A struct that owns a handle and closes it from `&mut self` keeps it
861    /// in an `Option` and `take()`s it:
862    ///
863    /// ```no_run
864    /// use mnemo_engine::{Mnemo, Result};
865    ///
866    /// struct App {
867    ///     db: Option<Mnemo>,
868    /// }
869    ///
870    /// impl App {
871    ///     fn shutdown(&mut self) -> Result<()> {
872    ///         match self.db.take() {
873    ///             Some(db) => db.close(),
874    ///             None => Ok(()), // already closed
875    ///         }
876    ///     }
877    /// }
878    /// ```
879    ///
880    /// Changed in v0.12.0: `close` took `&mut self` and left the file and
881    /// lock held until the handle was dropped.
882    ///
883    /// **If the final flush fails, the handle is dropped with it**, and the
884    /// writes that weren't flushed are lost (a full disk on a phone, say).
885    /// [`Mnemo::try_close`] gives the handle back on that path, so the caller
886    /// can free space and retry.
887    pub fn close(mut self) -> Result<()> {
888        if !self.read_only {
889            // PERF-1: closing persists a dirty ANN index too, so the next
890            // open has nothing to re-assign.
891            self.index_persist.forced = true;
892            self.flush()?;
893        }
894        // Dropping the handle closes the file, which releases the lock.
895        drop(self);
896        Ok(())
897    }
898
899    /// [`Mnemo::close`] that keeps the data when the final flush fails: the
900    /// error carries the handle back, with every unflushed write still in
901    /// it, as [`std::io::BufWriter::into_inner`] does. Retry with
902    /// [`Mnemo::flush`] or another `try_close` once the cause is fixed.
903    /// A read-only handle never flushes, so it always closes. Added in
904    /// v0.12.2 (RB-001.22).
905    ///
906    /// ```no_run
907    /// # use mnemo_engine::Mnemo;
908    /// # fn run(db: Mnemo) -> mnemo_engine::Result<()> {
909    /// match db.try_close() {
910    ///     Ok(()) => {}
911    ///     Err(e) => {
912    ///         eprintln!("close failed: {}", e.error());
913    ///         let mut db = e.into_inner(); // still open, still holding the writes
914    ///         db.flush()?;                  // e.g. after freeing space
915    ///         db.close()?;
916    ///     }
917    /// }
918    /// # Ok(()) }
919    /// ```
920    pub fn try_close(mut self) -> std::result::Result<(), CloseError> {
921        if !self.read_only {
922            self.index_persist.forced = true;
923            if let Err(error) = self.flush() {
924                return Err(CloseError {
925                    handle: Box::new(self),
926                    error,
927                });
928            }
929        }
930        drop(self);
931        Ok(())
932    }
933
934    /// PERF-1: flush, persisting the ANN index even if the
935    /// [`INDEX_PERSIST_EVERY_COMMITS`] / [`INDEX_PERSIST_EVERY_CHANGES`]
936    /// policy wouldn't yet. Plain [`Mnemo::flush`] writes only the catalog
937    /// between persists; the index is derived, so a crash costs nothing but
938    /// re-assigning the newer records on the next load.
939    pub fn flush_index(&mut self) -> Result<()> {
940        if self.read_only {
941            return Err(MnemoError::ReadOnly);
942        }
943        self.index_persist.forced = true;
944        self.flush()
945    }
946
947    /// **TEST ONLY — do not use.** Reproduces a crash inside
948    /// [`Mnemo::flush`] *after* the prelude has leased counter+page slots
949    /// and persisted the leased header, but *before* the WAL is committed.
950    /// Runs `prepare_for_flush` + `pager.flush` and returns — leaving
951    /// data pages and a leased header on disk, with no commit frame.
952    ///
953    /// Used by `tests/integration.rs::nonce_unique_after_crashed_data_flush`
954    /// to verify that this window (Phase 1.1 of the improvement plan) does
955    /// not enable AES-GCM nonce reuse. Calling this in production code
956    /// strands data pages; never expose it from a binding.
957    #[doc(hidden)]
958    #[cfg(feature = "dev-hooks")]
959    pub fn __crash_partial_flush_for_testing(&mut self) -> Result<()> {
960        if self.read_only {
961            return Err(MnemoError::ReadOnly);
962        }
963        if !self.dirty_catalog && !self.dirty_index {
964            return Ok(());
965        }
966        // FMT-9b: v9 records and vectors reach their pages at the top of a
967        // flush; do that part too.
968        self.ensure_gen_lease()?;
969        self.pack_records()?;
970        self.pack_vectors()?;
971        let _prelude = self.prepare_for_flush(self.dirty_index)?;
972        self.pager.flush()
973    }
974
975    /// Bound the in-memory page cache to `pages` decrypted pages.
976    ///
977    /// The cache holds decrypted page payloads to speed repeated reads. By
978    /// default it is capped at 1024 pages (~8 MiB; 8192 before v0.10.0).
979    /// Lower the cap to trade hit rate for a smaller footprint, or raise
980    /// it for a hotter cache. The
981    /// cap governs *clean* pages — pages with un-flushed writes are always
982    /// retained until [`Mnemo::flush`], regardless of the cap.
983    pub fn set_cache_capacity(&mut self, pages: usize) {
984        self.pager.set_cache_capacity(pages);
985    }
986
987    /// FMT-9b (v9 files): keep the fp16 vector column in memory while it
988    /// takes at most `bytes` (default [`DEFAULT_VECTOR_CACHE_BUDGET`],
989    /// 96 MiB, about 125K vectors at 384 dims). Above it, recall reads
990    /// vector entries through the page cache instead: less memory, slower
991    /// recall. Lowering it below the resident size drops the resident copy.
992    pub fn set_vector_cache_budget(&mut self, bytes: usize) {
993        self.vector_cache_budget = bytes;
994        let need = self.col.scored_rows * veccol::entry_len(self.dimensions);
995        if self.col.resident && need > bytes {
996            self.col.data = Vec::new();
997            self.col.norms = Vec::new();
998            self.col.have = Vec::new();
999            self.col.resident = false;
1000        }
1001    }
1002
1003    /// Page-cache occupancy: `(pages_cached, capacity)`. Renamed from
1004    /// `cache_stats` in v0.4.0: the result-cache API (Phase 10.1) took
1005    /// that name for its per-namespace [`CacheStats`] summary, and the
1006    /// page-cache pair is the older/less-user-facing of the two, so it
1007    /// gets the more-specific name.
1008    pub fn page_cache_stats(&self) -> (usize, usize) {
1009        self.pager.cache_stats()
1010    }
1011
1012    // === Result cache (Phase 10.1 + 10.3) ============================
1013}