mnemo_engine/store/flush.rs
1//! Commits and durability: sealed runs, the WAL and checkpoint, flush, close, and the page-cache and vector-budget settings.
2
3use super::*;
4
5impl Mnemo {
6 /// Seal a serialized buffer into a fresh run of encrypted home pages and
7 /// append a WAL frame for each. Returns the run's `(start_page, pages)`.
8 /// The pages are *not* written to their home locations here — they go to
9 /// the WAL and are folded in later by [`Mnemo::checkpoint`].
10 ///
11 /// `gen` is the run's generation (v9; `0` on v8). On v9 the pages come
12 /// from the free list when an eligible run fits.
13 pub(super) fn seal_run(
14 &mut self,
15 bytes: &[u8],
16 gen: u64,
17 frames: &mut Vec<wal::Frame>,
18 ) -> Result<(u64, u32)> {
19 let pc = bytes.len().div_ceil(PAYLOAD).max(1);
20 let start = self.alloc_pages(pc as u64);
21 self.seal_run_at(start, pc as u64, bytes, gen, frames)?;
22 Ok((start, pc as u32))
23 }
24
25 /// Seal `bytes` into the `pages` already-allocated pages at `start`
26 /// (pages past the data are sealed empty) and append their WAL frames.
27 pub(super) fn seal_run_at(
28 &mut self,
29 start: u64,
30 pages: u64,
31 bytes: &[u8],
32 gen: u64,
33 frames: &mut Vec<wal::Frame>,
34 ) -> Result<()> {
35 // RB-007.6: no v9 page is sealed before this window's lease.
36 debug_assert!(
37 !self.is_v9() || self.gen_leased,
38 "a v9 run sealed before the generation lease"
39 );
40 let direct = self.is_v9() && pages > V9_WAL_RUN_MAX_PAGES;
41 for i in 0..pages as usize {
42 let lo = (i * PAYLOAD).min(bytes.len());
43 let hi = ((i + 1) * PAYLOAD).min(bytes.len());
44 let page_no = start + i as u64;
45 let sealed = self.pager.seal_page(page_no, gen, &bytes[lo..hi])?;
46 if direct {
47 self.pager.write_sealed(page_no, &sealed)?;
48 } else {
49 frames.push((page_no, sealed.to_vec()));
50 }
51 }
52 if direct {
53 self.direct_writes = true;
54 }
55 Ok(())
56 }
57
58 /// Grow the WAL region if `frame_pages` worth of frames would not fit.
59 /// Called at the top of `flush`, where the WAL is always spent (every
60 /// flush ends with a checkpoint), so relocating it cannot strand a
61 /// committed transaction.
62 pub(super) fn ensure_wal_capacity(&mut self, frame_pages: usize) -> Result<()> {
63 let need = wal::txn_byte_len(frame_pages, PAGE_SIZE);
64 if need <= self.header.wal_pages * PAGE_SIZE as u64 {
65 return Ok(());
66 }
67 // Allocate a new, larger region at the file tail (~1.5x headroom).
68 let req = need.div_ceil(PAGE_SIZE as u64);
69 let new_pages = (req + req / 2 + 4).max(DEFAULT_WAL_PAGES);
70 let old_wal_start = self.header.wal_start;
71 let old_wal_pages = self.header.wal_pages;
72 // FMT-9 (a): the new region always comes from the file's tail: the
73 // relocation commits a header-only transaction, so a region taken
74 // from the free list would still be listed as free in the
75 // committed free-list run.
76 let new_start = self.header.next_page;
77 self.header.next_page += new_pages;
78 self.header.wal_start = new_start;
79 self.header.wal_pages = new_pages;
80 self.header.wal_seq += 1;
81 // DUR-1 (R-002.1): WAL-log the header change into the *new* region
82 // first, checkpoint page 0, then invalidate the old region. A crash
83 // anywhere here either keeps the old state (if page 0 wasn't yet
84 // overwritten) or recovers to the new state via the torn-header
85 // scan in `Mnemo::open`, which prefers the highest-txn-id valid
86 // committed header frame.
87 self.relocate_header_through_wal(old_wal_start)?;
88 // FMT-9 (a): once the relocation has committed, the old region is
89 // unreferenced; the next commit frees it.
90 self.release_pages(old_wal_start, old_wal_pages);
91 Ok(())
92 }
93
94 /// Shared helper for every post-`create` page-0 commit: WAL-log the
95 /// new header into the current WAL region, checkpoint page 0, and
96 /// invalidate `old_wal_start` (the previous region, or the current
97 /// region when it hasn't moved). The caller is expected to have
98 /// mutated `self.header` to its target values *including a bumped
99 /// `wal_seq`* (one more than the last on-disk `wal_seq`); this
100 /// routine applies the seal, writes the frame + commit, folds page 0,
101 /// then zeroes the old region's first frame header.
102 ///
103 /// If `old_wal_start == self.header.wal_start`, the invalidation is
104 /// skipped — the region didn't move, so overwriting its first frame
105 /// would destroy the just-committed one.
106 pub(super) fn relocate_header_through_wal(&mut self, old_wal_start: u64) -> Result<()> {
107 self.header.write_counter = self.pager.write_counter;
108 let mut page = self.header.to_page();
109 self.header.apply_seal(&mut page, self.pager.dek())?;
110 let txn_id = self.header.wal_seq;
111 let frames = vec![(0u64, page.to_vec())];
112 wal::commit(
113 self.pager.file_mut(),
114 self.header.wal_start,
115 self.header.wal_pages,
116 txn_id,
117 &frames,
118 )?;
119 self.pager.write_raw(0, &page)?;
120 self.pager.sync()?;
121 if old_wal_start != self.header.wal_start {
122 wal::invalidate_region(self.pager.file_mut(), old_wal_start)?;
123 }
124 Ok(())
125 }
126
127 /// Fold a committed transaction's WAL frames into their home pages.
128 pub(super) fn checkpoint(&mut self, frames: &[wal::Frame]) -> Result<()> {
129 for (page_no, bytes) in frames {
130 let mut img = [0u8; PAGE_SIZE];
131 img.copy_from_slice(bytes);
132 self.pager.write_sealed(*page_no, &img)?;
133 }
134 self.pager.sync()?;
135 Ok(())
136 }
137
138 /// Persist all pending changes as one **write-ahead-logged transaction**.
139 ///
140 /// Sequence:
141 ///
142 /// 1. **Prepare.** Serialize the control plane (catalog, ANN index),
143 /// grow the WAL if needed, and **lease** counter and page slots —
144 /// bump `header.write_counter` and `header.next_page` by upper
145 /// bounds on this transaction's consumption and persist that
146 /// *leased* header to disk before any encrypted page is written.
147 /// 2. **Data pages.** [`Pager::flush`] writes dirty record pages
148 /// copy-on-write under fresh nonces and fsyncs them. The orphan-page
149 /// nonce-reuse window (see [Phase 1.1 of the improvement plan])
150 /// is closed because the leased header on disk already records a
151 /// `write_counter` past anything this step can produce.
152 /// 3. **Control plane.** Seal the new catalog / ANN index / snapshot
153 /// manifest into fresh home page runs.
154 /// 4. **Commit.** Log all of step 3's frames plus the final header
155 /// frame into the WAL and fsync — the durability point.
156 /// 5. **Checkpoint.** Fold the WAL into the home pages.
157 ///
158 /// A crash before step 4 leaves the previous state intact. A crash
159 /// after step 4 is repaired by [`Mnemo::open`] replaying the WAL.
160 /// Safe to call repeatedly.
161 pub fn flush(&mut self) -> Result<()> {
162 if self.read_only {
163 return Err(MnemoError::ReadOnly);
164 }
165 // Snapshot the thread-local fsync counter. Every `sync_all` the
166 // pager / wal / checkpoint path makes between here and the end
167 // of this function bumps the same counter; `delta` lands on
168 // `self.fsyncs_last_flush` so `Stats::fsyncs_last_flush` can
169 // report it. The planner wants this to decide whether cutting
170 // fsync count is a viable write-latency win before we touch
171 // the write path's durability model.
172 let fsync_before = crate::fsync_count();
173 self.commit_point = None;
174 // PERF-1 step 3: while the index is unloaded, records written since
175 // its run pile up for the next load to re-assign. Once there are
176 // INDEX_PERSIST_EVERY_CHANGES of them, load it now so this flush
177 // persists a current index and keeps that work bounded.
178 if self.ann_unloaded && self.dirty_catalog {
179 let (index_start, index_gen) = (self.header.index_start, self.index_gen_now());
180 let v9 = self.is_v9();
181 let pending = self
182 .catalog
183 .iter()
184 .filter(|e| !e.deleted && written_after_index(e, v9, index_start, index_gen))
185 .count();
186 if pending >= INDEX_PERSIST_EVERY_CHANGES {
187 self.ensure_ann()?;
188 }
189 }
190 // PERF-1: persist the (derived) ANN index only when forced or due.
191 // Otherwise the header keeps pointing at the previous index run, and
192 // records written since then are re-assigned when it's next loaded
193 // (`reconcile_ann_with_catalog`). `ann == None` with `dirty_index` set is a
194 // dropped index, which is persisted at once.
195 let persist_index = if self.is_v9() {
196 self.plan_index_v9()
197 } else {
198 self.dirty_index
199 && (self.index_persist.forced
200 || self.ann.is_none()
201 || self.index_persist.commits + 1 >= INDEX_PERSIST_EVERY_COMMITS
202 || self.index_persist.changes >= INDEX_PERSIST_EVERY_CHANGES)
203 };
204 let free_work =
205 self.is_v9() && (self.free_dirty || !self.pending_free.is_empty() || self.dirty_keys);
206 if !self.dirty_catalog && !persist_index && !self.dirty_cache && !free_work {
207 // No control-plane changes pending. Because every `put` that
208 // dirties a data page also dirties the catalog, and every
209 // `cache_put` that dirties a data page also dirties
210 // `dirty_cache`, there can't be dirty data pages either —
211 // nothing to do.
212 self.fsyncs_last_flush = 0;
213 return Ok(());
214 }
215 // v0.14.0 step 1: nothing is written to a file that was replaced at
216 // this handle's path.
217 self.ensure_not_replaced()?;
218
219 // FIX-1 R-001.8 part 3: snapshot the memory/index-dirty flags
220 // before step 3b decides whether to push a restore point.
221 // Only flushes that change the memory catalog or ANN index
222 // need a snapshot — a cache-only flush (`dirty_cache` set,
223 // `dirty_catalog` + `dirty_index` both false) must not push,
224 // because snapshots don't capture cache state and every push
225 // counts against `max_snapshots` (default 256), so a cache-
226 // heavy workload would otherwise evict every real memory
227 // restore point within ~256 cache writes.
228 let is_memory_or_index_flush = self.dirty_catalog || persist_index;
229
230 // FMT-9: a v9 session's first sealed page needs the generation
231 // lease (normally taken by its first record write already).
232 self.ensure_gen_lease()?;
233 // FMT-9e: turns that aged past the cutoff are rewritten compressed
234 // (bounded per commit), then packed with the other waiting records.
235 self.settle_cold_turns()?;
236 // FMT-9b: column entries first; they move rows' vector locations,
237 // so they must land before the catalog is serialized.
238 self.pack_records()?;
239 self.pack_vectors()?;
240
241 // 1. Serialize control plane, ensure WAL capacity, lease counter +
242 // page slots, and persist the leased header *before* any data
243 // page hits the disk.
244 let ctx = self.prepare_for_flush(persist_index)?;
245
246 // 2. Record (vector) data pages: copy-on-write to fresh pages, fsynced.
247 // Safe: the lease guarantees that even if we crash here, reopen
248 // will see a `write_counter` past every nonce this step uses.
249 self.pager.flush()?;
250
251 // 3. Seal the catalog / index control pages into fresh home runs.
252 // FMT-9: `txn_id` is this transaction's id and the generation
253 // of every run it seals. A v9 run replaced here is freed by this
254 // same commit (`superseded`); v8 never reuses pages.
255 let txn_id = self.header.wal_seq + 1;
256 let gen = self.write_gen();
257 debug_assert!(!self.is_v9() || gen == txn_id);
258 let v9 = self.is_v9();
259 let mut superseded: Vec<(u64, u64)> = Vec::new();
260 let mut frames: Vec<wal::Frame> = Vec::new();
261 // FMT-9c.2: a delta run goes on the chain; a new base replaces the
262 // old base and the whole chain (all freed by this commit). The
263 // chain is adopted once the commit succeeds.
264 let mut new_chain: Option<(Vec<RunPtr>, usize)> = None;
265 if let Some(bytes) = &ctx.cat_bytes {
266 if ctx.cat_delta {
267 let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
268 let ptr = RunPtr {
269 start,
270 pages: pages as u64,
271 len: bytes.len() as u64,
272 gen,
273 };
274 self.header.catalog_delta = ptr;
275 let mut chain = self.delta_chain.clone();
276 chain.push(ptr);
277 new_chain = Some((chain, self.delta_rows + self.dirty_rows.len()));
278 } else {
279 if v9 && self.header.catalog_pages > 0 {
280 superseded.push((self.header.catalog_start, self.header.catalog_pages));
281 }
282 if v9 {
283 for r in &self.delta_chain {
284 superseded.push((r.start, r.pages));
285 }
286 self.header.catalog_delta = RunPtr::default();
287 new_chain = Some((Vec::new(), 0));
288 }
289 let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
290 self.header.catalog_start = start;
291 self.header.catalog_pages = pages as u64;
292 self.header.catalog_len = bytes.len() as u64;
293 self.header.catalog_gen = gen;
294 }
295 }
296 // FMT-9d: the BM25 segment, then the manifest that lists it.
297 let mut new_bm25: Option<Bm25Segments> = None;
298 if let Some((bytes, replace_from)) = &ctx.bm25_seg {
299 let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
300 let mut segs = self.bm25.manifest.segments.clone();
301 let mut seg_docs = self.bm25.manifest.seg_docs.clone();
302 seg_docs.resize(segs.len(), 0);
303 let docs = if let Some(from) = *replace_from {
304 for t in segs.drain(from..) {
305 superseded.push((t.0, t.1));
306 }
307 seg_docs.truncate(from);
308 self.bm25.merged_docs
309 } else {
310 self.bm25.pending.len() as u32
311 };
312 segs.push((start, pages as u64, bytes.len() as u64, gen));
313 seg_docs.push(docs);
314 let man = Bm25Manifest {
315 tokenizer: text::TOKENIZER_V1,
316 next_doc: self.bm25.manifest.next_doc,
317 segments: segs.clone(),
318 seg_docs: seg_docs.clone(),
319 };
320 let mbytes =
321 rmp_serde::to_vec(&man).map_err(|e| MnemoError::Serialize(e.to_string()))?;
322 if !self.header.bm25.is_empty() {
323 superseded.push((self.header.bm25.start, self.header.bm25.pages));
324 }
325 let (ms, mp) = self.seal_run(&mbytes, gen, &mut frames)?;
326 self.header.bm25 = RunPtr {
327 start: ms,
328 pages: mp as u64,
329 len: mbytes.len() as u64,
330 gen,
331 };
332 new_bm25 = Some((segs, seg_docs));
333 }
334 let mut new_ichain: Option<(Vec<RunPtr>, usize)> = None;
335 if persist_index && ctx.idx_delta {
336 if let Some(bytes) = &ctx.idx_bytes {
337 let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
338 let ptr = RunPtr {
339 start,
340 pages: pages as u64,
341 len: bytes.len() as u64,
342 gen,
343 };
344 self.header.index_delta = ptr;
345 let mut chain = self.index_chain.clone();
346 chain.push(ptr);
347 let n = self.index_delta_ops.as_ref().map_or(0, |o| o.len());
348 new_ichain = Some((chain, self.index_chain_ops + n));
349 }
350 } else if persist_index {
351 if v9 && self.header.index_pages > 0 {
352 superseded.push((self.header.index_start, self.header.index_pages));
353 }
354 if v9 {
355 for r in &self.index_chain {
356 superseded.push((r.start, r.pages));
357 }
358 self.header.index_delta = RunPtr::default();
359 new_ichain = Some((Vec::new(), 0));
360 }
361 match &ctx.idx_bytes {
362 Some(bytes) => {
363 let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
364 self.header.index_start = start;
365 self.header.index_pages = pages as u64;
366 self.header.index_len = bytes.len() as u64;
367 self.header.index_gen = gen;
368 }
369 None => {
370 self.header.index_start = 0;
371 self.header.index_pages = 0;
372 self.header.index_len = 0;
373 self.header.index_gen = 0;
374 }
375 }
376 }
377 // v8+ result-cache directory (Phase 10.1). Seal the serialized
378 // directory into a fresh page run and update the pointer in the
379 // header. The seal AAD already covers cache_start/pages/len for
380 // v8+ (see `header_seal_aad`), so tampering with the pointer
381 // trips the header seal on the next open.
382 if self.dirty_cache {
383 if v9 && self.header.cache_pages > 0 {
384 superseded.push((self.header.cache_start, self.header.cache_pages));
385 }
386 match &ctx.cache_bytes {
387 Some(bytes) if !self.cache_dir.is_empty() => {
388 let (start, pages) = self.seal_run(bytes, gen, &mut frames)?;
389 self.header.cache_start = start;
390 self.header.cache_pages = pages as u64;
391 self.header.cache_len = bytes.len() as u64;
392 self.header.cache_gen = gen;
393 }
394 _ => {
395 // Either the directory serialized to nothing (all
396 // entries deleted after tombstone-only migration) or
397 // it's empty — collapse the pointer to zero.
398 self.header.cache_start = 0;
399 self.header.cache_pages = 0;
400 self.header.cache_len = 0;
401 self.header.cache_gen = 0;
402 }
403 }
404 }
405
406 // 3b. Record this transaction as a restorable snapshot. The manifest
407 // update is staged in a local and adopted only once the commit
408 // below succeeds, so a failed flush leaves no phantom entry.
409 //
410 // The cap (`self.max_snapshots`) is applied to the staged local
411 // before the new entry lands, so after appending the manifest
412 // holds at most `max_snapshots` entries — the most-recent N.
413 // `max_snapshots == 0` disables the cap. On v8, pages
414 // referenced only by pruned snapshots stay on disk until
415 // `compact_file`; on v9 they become reusable once no retained
416 // snapshot is older than the transaction that freed them.
417 //
418 // FIX-1 R-001.8 part 3: skip the snapshot push on a
419 // cache-only flush. Snapshots don't capture the cache
420 // anyway, and every push counts against `max_snapshots`
421 // (default 256) + rewrites the manifest run — so a
422 // cache-heavy workload would otherwise evict every real
423 // memory restore point within ~256 cache writes. On a
424 // cache-only flush we reuse the current manifest
425 // verbatim; the manifest run already lives at
426 // `self.header.manifest_{start,pages,len}` (set by the
427 // last memory-touching flush, or at `create` time).
428 let live = self.len() as u64;
429 let mut manifest = self.manifest.clone();
430 if is_memory_or_index_flush {
431 if self.max_snapshots > 0 && !v9 {
432 // Drop the oldest entries so there's room for the new one.
433 while manifest.len() >= self.max_snapshots {
434 manifest.remove(0);
435 }
436 }
437 manifest.push(Snapshot {
438 txn_id,
439 created_at: memory::now_secs(),
440 catalog_start: self.header.catalog_start,
441 catalog_pages: self.header.catalog_pages,
442 catalog_len: self.header.catalog_len,
443 index_start: self.header.index_start,
444 index_pages: self.header.index_pages,
445 index_len: self.header.index_len,
446 memory_count: live,
447 gens: v9.then_some(SnapshotGens {
448 catalog: self.header.catalog_gen,
449 index: self.header.index_gen,
450 delta: ptr_tuple(self.header.catalog_delta),
451 index_delta: ptr_tuple(self.header.index_delta),
452 bm25: ptr_tuple(self.header.bm25),
453 }),
454 });
455 if v9 {
456 let extra = self.pending_free.iter().map(|r| r.1).sum::<u64>()
457 + superseded.iter().map(|r| r.1).sum::<u64>()
458 + self.header.manifest_pages;
459 self.apply_retention(&mut manifest, extra);
460 }
461 let man_bytes =
462 rmp_serde::to_vec(&manifest).map_err(|e| MnemoError::Serialize(e.to_string()))?;
463 if v9 && self.header.manifest_pages > 0 {
464 superseded.push((self.header.manifest_start, self.header.manifest_pages));
465 }
466 let (m_start, m_pages) = self.seal_run(&man_bytes, gen, &mut frames)?;
467 self.header.manifest_start = m_start;
468 self.header.manifest_pages = m_pages as u64;
469 self.header.manifest_len = man_bytes.len() as u64;
470 self.header.manifest_gen = gen;
471 }
472
473 // 3c. FMT-9 (b): the key-slot run, rewritten copy-on-write when a
474 // slot changed. The header seal covers its pointer.
475 if v9 && self.dirty_keys {
476 if !self.header.keys.is_empty() {
477 superseded.push((self.header.keys.start, self.header.keys.pages));
478 }
479 if self.key_slots.slots.is_empty() {
480 self.header.keys = RunPtr::default();
481 } else {
482 let bytes = rmp_serde::to_vec(&self.key_slots)
483 .map_err(|e| MnemoError::Serialize(e.to_string()))?;
484 let (start, pages) = self.seal_run(&bytes, gen, &mut frames)?;
485 self.header.keys = RunPtr {
486 start,
487 pages: pages as u64,
488 len: bytes.len() as u64,
489 gen,
490 };
491 }
492 }
493
494 // 3d. FMT-9 (a): the free list. Everything this transaction
495 // stopped referencing (`pending_free`, the runs replaced above,
496 // and the previous free-list run itself) is freed *at this
497 // transaction's id*, in this transaction's own commit, so it
498 // can't be reused until this commit is durable and no retained
499 // snapshot predates it. The list is staged in a local and
500 // adopted only once the commit succeeds (a failed flush that
501 // is retried must not free the same runs twice).
502 let mut shrink = false;
503 let staged_free =
504 if v9 && (self.free_dirty || !self.pending_free.is_empty() || !superseded.is_empty()) {
505 let mut staged = self.free.clone();
506 for &(p, c) in self.pending_free.iter().chain(superseded.iter()) {
507 staged.free(txn_id, p, c)?;
508 }
509 if !self.header.free.is_empty() {
510 staged.free(txn_id, self.header.free.start, self.header.free.pages)?;
511 }
512 // FMT-9c: give eligible free runs at the file's tail back
513 // to the filesystem: the header's `next_page` moves down in
514 // this commit, and the file is cut after it.
515 let min_snap = self.min_snapshot_txn();
516 while let Some(p) =
517 staged.take_tail(self.header.next_page, self.header.wal_seq, min_snap)
518 {
519 self.header.next_page = p;
520 shrink = true;
521 }
522 // Size the run with room for the one extra entry that
523 // allocating it out of the list can create (a split), then
524 // allocate it from the list (eligible runs only) or the tail.
525 let est = rmp_serde::to_vec(&staged.to_runs())
526 .map_err(|e| MnemoError::Serialize(e.to_string()))?
527 .len()
528 + 64;
529 let pc = est.div_ceil(PAYLOAD).max(1) as u64;
530 let start = match staged.alloc(pc, self.header.wal_seq, self.min_snapshot_txn()) {
531 Some(p) => p,
532 None => {
533 let p = self.header.next_page;
534 self.header.next_page += pc;
535 p
536 }
537 };
538 let bytes = rmp_serde::to_vec(&staged.to_runs())
539 .map_err(|e| MnemoError::Serialize(e.to_string()))?;
540 if bytes.len() as u64 > pc * PAYLOAD as u64 {
541 return Err(MnemoError::Invalid(
542 "free list outgrew its reserved run".into(),
543 ));
544 }
545 self.seal_run_at(start, pc, &bytes, gen, &mut frames)?;
546 self.header.free = RunPtr {
547 start,
548 pages: pc,
549 len: bytes.len() as u64,
550 gen,
551 };
552 Some(staged)
553 } else {
554 None
555 };
556
557 // 3e. The header is the transaction's final frame; stamp the new id
558 // and the *actual* (post-flush) write counter, which is <= the
559 // leased value persisted in `prepare_for_flush`. This is the
560 // value that ultimately replaces the leased header on checkpoint.
561 self.header.vector_count = live;
562 self.header.write_counter = self.pager.write_counter;
563 self.header.wal_seq = txn_id;
564 let mut hpage = self.header.to_page();
565 self.header.apply_seal(&mut hpage, self.pager.dek())?;
566 frames.push((0, hpage.to_vec()));
567
568 // 4. COMMIT — log the transaction and fsync. Nothing at a home page
569 // other than the leased header has changed yet; this single
570 // fsync is what makes the rest durable.
571 // FMT-9c: big runs went straight to their home pages; make them
572 // durable before the header that references them can be.
573 if self.direct_writes {
574 self.pager.sync()?;
575 self.direct_writes = false;
576 }
577 let (wal_start, wal_pages) = (self.header.wal_start, self.header.wal_pages);
578 // 12.4: from here a failure may leave the transaction committed.
579 self.commit_point = Some(txn_id);
580 wal::commit(self.pager.file_mut(), wal_start, wal_pages, txn_id, &frames)?;
581
582 // 5. Checkpoint — fold the WAL into the home pages.
583 self.checkpoint(&frames)?;
584 // FMT-9c: the committed header no longer covers the tail; cut it.
585 // A crash before this leaves a longer file, which is harmless.
586 if shrink {
587 let len = self.header.next_page * PAGE_SIZE as u64;
588 self.pager.file_mut().set_len(len)?;
589 }
590
591 self.manifest = manifest;
592 if let Some(staged) = staged_free {
593 self.free = staged;
594 }
595 self.pending_free.clear();
596 self.free_dirty = false;
597 self.dirty_keys = false;
598 if let Some((chain, rows)) = new_chain {
599 self.delta_chain = chain;
600 self.delta_rows = rows;
601 }
602 if let Some((chain, n)) = new_ichain {
603 self.index_chain = chain;
604 self.index_chain_ops = n;
605 }
606 if let Some((segs, seg_docs)) = new_bm25 {
607 // FMT-9f: keep the loaded dicts and read only the new segment's
608 // (a merge replaced the segments from `kept` on), instead of
609 // dropping them all and reloading every dict on next use.
610 let kept = self
611 .bm25
612 .manifest
613 .segments
614 .iter()
615 .zip(&segs)
616 .take_while(|(a, b)| a == b)
617 .count();
618 if let Some(mut dicts) = self.bm25.dicts.take() {
619 dicts.truncate(kept);
620 let mut ok = dicts.len() == kept;
621 for &t in &segs[kept..] {
622 match load_segment_dict(&mut self.pager, tuple_ptr(t)) {
623 Ok(d) if ok => dicts.push(d),
624 _ => ok = false,
625 }
626 }
627 if ok {
628 self.bm25.dicts = Some(dicts);
629 }
630 }
631 self.bm25.manifest.segments = segs;
632 self.bm25.manifest.seg_docs = seg_docs;
633 self.bm25.pending.clear();
634 self.bm25.pending_tokens = None;
635 }
636 if persist_index {
637 self.index_rebase = false;
638 self.index_delta_ops = None;
639 }
640 if ctx.cat_bytes.is_some() {
641 self.dirty_rows.clear();
642 self.catalog_rewrite = false;
643 }
644 self.dirty_catalog = false;
645 if persist_index {
646 self.dirty_index = false;
647 self.index_persist = IndexPersist::default();
648 } else if self.dirty_index {
649 self.index_persist.commits += 1;
650 }
651 self.dirty_cache = false;
652 // Batched policy: successful flush consumes the pending batch.
653 self.cache_batch.reset();
654 // Record how many fsyncs this flush issued, for Stats reporting.
655 // Thread-local counter guarantees we see only this handle's own
656 // sync_all calls between the snapshot above and this read.
657 self.fsyncs_last_flush =
658 (crate::fsync_count().wrapping_sub(fsync_before)).min(u32::MAX as u64) as u32;
659 // 12.4 shared mode (RB-007.5): this commit is in the file.
660 self.window_committed = true;
661 Ok(())
662 }
663
664 /// Pre-flush prelude shared by [`Mnemo::flush`] and the
665 /// `__crash_partial_flush_for_testing` hook.
666 ///
667 /// Serializes the control plane (catalog, ANN index, cache directory)
668 /// and grows the WAL if needed. That's it: this used to persist a
669 /// leased header too (SEC-1 obsoleted it; see DUR-1 R-002.1 note in
670 /// the body for the full rationale), but after DUR-1 the only page-0
671 /// writes outside `create` go through the WAL via
672 /// [`Mnemo::relocate_header_through_wal`].
673 ///
674 /// The historical pre-stamping used to close the nonce-reuse window
675 /// flagged in Phase 1.1 of the improvement plan: a crash anywhere
676 /// between here and the WAL commit left a file whose on-disk header
677 /// recorded a `write_counter` past every nonce the rest of this
678 /// transaction could produce, so the next reopen started from a
679 /// counter that guaranteed uniqueness against any orphan data pages
680 /// still on disk. SEC-1's per-session random nonce prefix gives the
681 /// same guarantee without needing a persisted counter, which is why
682 /// this routine no longer writes page 0.
683 pub(super) fn prepare_for_flush(&mut self, persist_index: bool) -> Result<FlushPrelude> {
684 // Serialize.
685 // FMT-9c.2: on v9 usually just the changed rows, as a delta run.
686 let cat_delta = self.dirty_catalog && self.catalog_as_delta();
687 let cat_bytes = if !self.dirty_catalog {
688 None
689 } else if cat_delta {
690 let rows: Vec<(u32, CatalogEntry)> = self
691 .dirty_rows
692 .iter()
693 .filter(|&&i| i < self.catalog.len())
694 .map(|&i| (i as u32, self.catalog[i].clone()))
695 .collect();
696 let d = CatalogDelta {
697 prev: ptr_tuple(self.header.catalog_delta),
698 rows,
699 };
700 Some(rmp_serde::to_vec(&d).map_err(|e| MnemoError::Serialize(e.to_string()))?)
701 } else {
702 if self.is_v9() {
703 self.compact_tombstones();
704 }
705 Some(
706 rmp_serde::to_vec(&self.catalog)
707 .map_err(|e| MnemoError::Serialize(e.to_string()))?,
708 )
709 };
710 // FMT-9c.3: a v9 commit usually writes just its index changes.
711 let idx_delta = persist_index && self.index_delta_ops.is_some();
712 let idx_bytes: Option<Vec<u8>> = if !persist_index {
713 None
714 } else if let Some(ops) = &self.index_delta_ops {
715 let d = IndexDeltaRef {
716 prev: ptr_tuple(self.header.index_delta),
717 ops,
718 };
719 Some(rmp_serde::to_vec(&d).map_err(|e| MnemoError::Serialize(e.to_string()))?)
720 } else {
721 match &self.ann {
722 // FMT-9: v9 index runs carry a layout byte first. PERF-2:
723 // layout 2 (flat arrays) unless a test asks for another.
724 Some(ann) if self.is_v9() => {
725 let layout = self.index_layout_override.unwrap_or(V9_INDEX_LAYOUT);
726 let mut b = vec![layout];
727 if layout == V9_INDEX_LAYOUT {
728 ann.write_layout2(&mut b);
729 } else {
730 b.extend(
731 rmp_serde::to_vec(ann)
732 .map_err(|e| MnemoError::Serialize(e.to_string()))?,
733 );
734 }
735 Some(b)
736 }
737 Some(ann) => {
738 Some(rmp_serde::to_vec(ann).map_err(|e| MnemoError::Serialize(e.to_string()))?)
739 }
740 None => None,
741 }
742 };
743 // Serialize the result-cache directory when it's dirty. An
744 // empty directory serializes to a small header + 0-element
745 // array; still cheap enough that we don't special-case it.
746 let cache_bytes: Option<Vec<u8>> = if self.dirty_cache {
747 Some(
748 rmp_serde::to_vec(&self.cache_dir)
749 .map_err(|e| MnemoError::Serialize(e.to_string()))?,
750 )
751 } else {
752 None
753 };
754
755 let cat_pc = cat_bytes
756 .as_ref()
757 .map_or(0, |b| b.len().div_ceil(PAYLOAD).max(1));
758 let idx_pc = idx_bytes
759 .as_ref()
760 .map_or(0, |b| b.len().div_ceil(PAYLOAD).max(1));
761 let cache_pc = cache_bytes
762 .as_ref()
763 .map_or(0, |b| b.len().div_ceil(PAYLOAD).max(1));
764 // Upper bound on the manifest run: one extra entry, <=82 bytes each
765 // (rmp_serde encodes a Snapshot as a 9-element fixarray of i64/u64,
766 // worst case 81 bytes plus a 1-byte array header).
767 // v9 snapshots carry two more u64s (≤ 20 bytes with the array header).
768 let per_snapshot = if self.is_v9() { 102 } else { 82 };
769 let man_upper = 9 + (self.manifest.len() + 1) * per_snapshot;
770 let man_pc_est = man_upper.div_ceil(PAYLOAD).max(1);
771 // FMT-9: the free-list run (≤ 28 bytes per run: every pending and
772 // superseded run plus a split and a relocated WAL region) and the
773 // key-slot run.
774 let (free_pc_est, keys_pc) = if self.is_v9() {
775 let runs = self.free.len() + self.pending_free.len() + 12;
776 let free_pc = (runs * 28 + 64).div_ceil(PAYLOAD).max(1);
777 let keys_pc = if self.dirty_keys {
778 rmp_serde::to_vec(&self.key_slots)
779 .map_err(|e| MnemoError::Serialize(e.to_string()))?
780 .len()
781 .div_ceil(PAYLOAD)
782 .max(1)
783 } else {
784 0
785 };
786 (free_pc, keys_pc)
787 } else {
788 (0, 0)
789 };
790
791 // Grow WAL if needed (DUR-1: relocation is WAL-logged and does
792 // its own fsyncs). The cache directory rides in the same
793 // transaction so its page count contributes to the reservation.
794 // FMT-9d: the BM25 segment (and its manifest, one page).
795 let bm25_seg = self.bm25_next_segment()?;
796 let bm25_pc = bm25_seg
797 .as_ref()
798 .map_or(0, |(b, _)| b.len().div_ceil(PAYLOAD).max(1));
799 // FMT-9c: on v9 only runs of up to V9_WAL_RUN_MAX_PAGES go through
800 // the WAL; longer ones are written in place.
801 let v9 = self.is_v9();
802 let in_wal = |pc: usize| {
803 if v9 && pc as u64 > V9_WAL_RUN_MAX_PAGES {
804 0
805 } else {
806 pc
807 }
808 };
809 self.ensure_wal_capacity(
810 in_wal(cat_pc)
811 + in_wal(idx_pc)
812 + in_wal(cache_pc)
813 + in_wal(man_pc_est)
814 + in_wal(free_pc_est)
815 + in_wal(keys_pc)
816 + in_wal(bm25_pc)
817 + usize::from(bm25_pc > 0)
818 + 1,
819 )?;
820
821 // DUR-1 (R-002.1): the pre-flush "leased header" write that used
822 // to live here is gone. It existed to close the Phase 1.1 nonce
823 // reuse window (reopen must see a `write_counter` past any nonce
824 // this flush could produce before data pages hit disk). SEC-1
825 // superseded that: nonces now carry a per-session random prefix
826 // and `write_counter` is no longer a nonce input, so the leased
827 // write bought us nothing but an extra fsync per flush. Orphan
828 // data pages past `header.next_page` on a crashed flush are
829 // still safe — the next session's random prefix makes any later
830 // write at the same page_no uniquely nonced regardless of
831 // `write_counter`.
832
833 Ok(FlushPrelude {
834 bm25_seg,
835 cat_bytes,
836 cat_delta,
837 idx_delta,
838 idx_bytes,
839 cache_bytes,
840 })
841 }
842
843 /// Flush and close: consumes the handle and releases the file and its
844 /// OS advisory lock before returning, so the same file can be reopened
845 /// right away (by this process or another). On a read-write handle it
846 /// flushes first, persisting a dirty ANN index too; on a read-only
847 /// handle there's nothing to persist.
848 ///
849 /// Using the handle after `close` is a compile error. If the flush
850 /// fails, the error is returned and the handle is dropped all the same:
851 /// changes since the last successful [`Mnemo::flush`] are lost, as
852 /// they would be on a plain `drop` (which never flushes).
853 ///
854 /// A struct that owns a handle and closes it from `&mut self` keeps it
855 /// in an `Option` and `take()`s it:
856 ///
857 /// ```no_run
858 /// use mnemo_engine::{Mnemo, Result};
859 ///
860 /// struct App {
861 /// db: Option<Mnemo>,
862 /// }
863 ///
864 /// impl App {
865 /// fn shutdown(&mut self) -> Result<()> {
866 /// match self.db.take() {
867 /// Some(db) => db.close(),
868 /// None => Ok(()), // already closed
869 /// }
870 /// }
871 /// }
872 /// ```
873 ///
874 /// Changed in v0.12.0: `close` took `&mut self` and left the file and
875 /// lock held until the handle was dropped.
876 ///
877 /// **If the final flush fails, the handle is dropped with it**, and the
878 /// writes that weren't flushed are lost (a full disk on a phone, say).
879 /// [`Mnemo::try_close`] gives the handle back on that path, so the caller
880 /// can free space and retry.
881 pub fn close(mut self) -> Result<()> {
882 if !self.read_only {
883 // PERF-1: closing persists a dirty ANN index too, so the next
884 // open has nothing to re-assign.
885 self.index_persist.forced = true;
886 self.flush()?;
887 }
888 // Dropping the handle closes the file, which releases the lock.
889 drop(self);
890 Ok(())
891 }
892
893 /// [`Mnemo::close`] that keeps the data when the final flush fails: the
894 /// error carries the handle back, with every unflushed write still in
895 /// it, as [`std::io::BufWriter::into_inner`] does. Retry with
896 /// [`Mnemo::flush`] or another `try_close` once the cause is fixed.
897 /// A read-only handle never flushes, so it always closes. Added in
898 /// v0.12.2 (RB-001.22).
899 ///
900 /// ```no_run
901 /// # use mnemo_engine::Mnemo;
902 /// # fn run(db: Mnemo) -> mnemo_engine::Result<()> {
903 /// match db.try_close() {
904 /// Ok(()) => {}
905 /// Err(e) => {
906 /// eprintln!("close failed: {}", e.error());
907 /// let mut db = e.into_inner(); // still open, still holding the writes
908 /// db.flush()?; // e.g. after freeing space
909 /// db.close()?;
910 /// }
911 /// }
912 /// # Ok(()) }
913 /// ```
914 pub fn try_close(mut self) -> std::result::Result<(), CloseError> {
915 if !self.read_only {
916 self.index_persist.forced = true;
917 if let Err(error) = self.flush() {
918 return Err(CloseError {
919 handle: Box::new(self),
920 error,
921 });
922 }
923 }
924 drop(self);
925 Ok(())
926 }
927
928 /// PERF-1: flush, persisting the ANN index even if the
929 /// [`INDEX_PERSIST_EVERY_COMMITS`] / [`INDEX_PERSIST_EVERY_CHANGES`]
930 /// policy wouldn't yet. Plain [`Mnemo::flush`] writes only the catalog
931 /// between persists; the index is derived, so a crash costs nothing but
932 /// re-assigning the newer records on the next load.
933 pub fn flush_index(&mut self) -> Result<()> {
934 if self.read_only {
935 return Err(MnemoError::ReadOnly);
936 }
937 self.index_persist.forced = true;
938 self.flush()
939 }
940
941 /// **TEST ONLY — do not use.** Reproduces a crash inside
942 /// [`Mnemo::flush`] *after* the prelude has leased counter+page slots
943 /// and persisted the leased header, but *before* the WAL is committed.
944 /// Runs `prepare_for_flush` + `pager.flush` and returns — leaving
945 /// data pages and a leased header on disk, with no commit frame.
946 ///
947 /// Used by `tests/integration.rs::nonce_unique_after_crashed_data_flush`
948 /// to verify that this window (Phase 1.1 of the improvement plan) does
949 /// not enable AES-GCM nonce reuse. Calling this in production code
950 /// strands data pages; never expose it from a binding.
951 #[doc(hidden)]
952 #[cfg(feature = "dev-hooks")]
953 pub fn __crash_partial_flush_for_testing(&mut self) -> Result<()> {
954 if self.read_only {
955 return Err(MnemoError::ReadOnly);
956 }
957 if !self.dirty_catalog && !self.dirty_index {
958 return Ok(());
959 }
960 // FMT-9b: v9 records and vectors reach their pages at the top of a
961 // flush; do that part too.
962 self.ensure_gen_lease()?;
963 self.pack_records()?;
964 self.pack_vectors()?;
965 let _prelude = self.prepare_for_flush(self.dirty_index)?;
966 self.pager.flush()
967 }
968
969 /// Bound the in-memory page cache to `pages` decrypted pages.
970 ///
971 /// The cache holds decrypted page payloads to speed repeated reads. By
972 /// default it is capped at 1024 pages (~8 MiB; 8192 before v0.10.0).
973 /// Lower the cap to trade hit rate for a smaller footprint, or raise
974 /// it for a hotter cache. The
975 /// cap governs *clean* pages — pages with un-flushed writes are always
976 /// retained until [`Mnemo::flush`], regardless of the cap.
977 pub fn set_cache_capacity(&mut self, pages: usize) {
978 self.pager.set_cache_capacity(pages);
979 }
980
981 /// FMT-9b (v9 files): keep the fp16 vector column in memory while it
982 /// takes at most `bytes` (default [`DEFAULT_VECTOR_CACHE_BUDGET`],
983 /// 96 MiB, about 125K vectors at 384 dims). Above it, recall reads
984 /// vector entries through the page cache instead: less memory, slower
985 /// recall. Lowering it below the resident size drops the resident copy.
986 pub fn set_vector_cache_budget(&mut self, bytes: usize) {
987 self.vector_cache_budget = bytes;
988 let need = self.col.scored_rows * veccol::entry_len(self.dimensions);
989 if self.col.resident && need > bytes {
990 self.col.data = Vec::new();
991 self.col.norms = Vec::new();
992 self.col.have = Vec::new();
993 self.col.resident = false;
994 }
995 }
996
997 /// Page-cache occupancy: `(pages_cached, capacity)`. Renamed from
998 /// `cache_stats` in v0.4.0: the result-cache API (Phase 10.1) took
999 /// that name for its per-namespace [`CacheStats`] summary, and the
1000 /// page-cache pair is the older/less-user-facing of the two, so it
1001 /// gets the more-specific name.
1002 pub fn page_cache_stats(&self) -> (usize, usize) {
1003 self.pager.cache_stats()
1004 }
1005
1006 // === Result cache (Phase 10.1 + 10.3) ============================
1007}