Skip to main content

mnemo_engine/store/
snapshots.rs

1//! Snapshots and point-in-time recovery (restore, retention), and sessions.
2
3use super::*;
4
5impl Mnemo {
6    /// Begin a conversation [`Session`](crate::Session) for `agent_id`.
7    ///
8    /// The session borrows the database for its lifetime, records turns as
9    /// working memory, and consolidates them into episodic memory when closed.
10    pub fn session(&mut self, agent_id: impl Into<String>) -> crate::session::Session<'_> {
11        crate::session::Session::new(self, agent_id.into())
12    }
13
14    // --- snapshots & point-in-time recovery ------------------------------
15
16    /// Every committed transaction, oldest first — the restore points
17    /// available to [`Mnemo::restore_to`] and [`Mnemo::restore_to_time`].
18    ///
19    /// Each `flush` appends one snapshot. Because the storage engine is
20    /// append-only, the pages a past flush wrote are still on disk, so any
21    /// listed snapshot can be reinstated exactly. The history reaches back to
22    /// the last [`Mnemo::compact_file`], which reclaims space by collapsing
23    /// it.
24    pub fn snapshots(&self) -> Vec<SnapshotInfo> {
25        let mut v: Vec<SnapshotInfo> = self
26            .manifest
27            .iter()
28            .map(|s| SnapshotInfo {
29                txn_id: s.txn_id,
30                created_at: s.created_at,
31                memory_count: s.memory_count,
32            })
33            .collect();
34        v.sort_by_key(|s| s.txn_id);
35        v
36    }
37
38    /// Load a past snapshot's state and commit it as a new transaction.
39    pub(super) fn apply_snapshot(&mut self, snap: &Snapshot) -> Result<()> {
40        let catalog = load_catalog(
41            &mut self.pager,
42            snap.catalog_start,
43            snap.catalog_pages,
44            snap.catalog_len,
45            snap.catalog_gen(),
46        )?;
47        let (catalog, _, _) = apply_delta_chain(&mut self.pager, catalog, snap.delta_head())?;
48        // FMT-9c.2: the restored catalog is written whole.
49        self.catalog_rewrite = true;
50        let ann = load_index(
51            &mut self.pager,
52            snap.index_start,
53            snap.index_pages,
54            snap.index_len,
55            snap.index_gen(),
56            self.dimensions,
57        )?;
58        let mut ann = ann;
59        if self.is_v9() {
60            self.reclaim_for_restore(&catalog);
61            if let Some(a) = ann.as_mut() {
62                apply_index_chain(&mut self.pager, a, snap.index_delta_head())?;
63                a.enable_log();
64            }
65            self.index_rebase = true;
66            // FMT-9d: the snapshot's BM25 segments (quarantined while it's
67            // retained) describe its documents.
68            let bptr = snap.gens.map_or(RunPtr::default(), |g| tuple_ptr(g.bm25));
69            let manifest = load_bm25_manifest(&mut self.pager, bptr)?;
70            // Its runs come back off the free list; the current ones it
71            // doesn't share are freed, and the header points at its run.
72            let runs = |ptr: RunPtr, m: &Bm25Manifest| {
73                let mut v: Vec<(u64, u64)> = m.segments.iter().map(|t| (t.0, t.1)).collect();
74                if !ptr.is_empty() {
75                    v.push((ptr.start, ptr.pages));
76                }
77                v
78            };
79            let then = runs(bptr, &manifest);
80            let now = runs(self.header.bm25, &self.bm25.manifest);
81            self.pending_free.retain(|r| !then.contains(r));
82            for &(p, c) in &then {
83                self.free.unfree(p, c);
84            }
85            for r in now {
86                if !then.contains(&r) {
87                    self.release_pages(r.0, r.1);
88                }
89            }
90            self.header.bm25 = bptr;
91            self.free_dirty = true;
92            self.bm25 = Bm25State {
93                manifest,
94                ..Bm25State::default()
95            };
96        }
97        self.index = build_id_index(&catalog);
98        self.catalog = catalog;
99        // FMT-9b: the column and the packed pages described the catalog
100        // being replaced.
101        self.col = VecColumn::from_catalog(&self.catalog);
102        self.recs = RecPages::from_catalog(&self.catalog);
103        self.ann = ann;
104        self.ann_unloaded = false;
105        // The handle caches describe the catalog being replaced.
106        self.key_index = None;
107        self.superseded = None;
108        self.layers = None;
109        self.agent_set = None;
110        // PERF-1: the snapshot's index may predate its catalog; bring it up
111        // to date before it's persisted again.
112        let snap_index_gen = snap.index_gen().max(snap.index_delta_head().gen);
113        self.reconcile_ann_with_catalog(snap.index_start, snap_index_gen)?;
114        // Re-commit as a fresh transaction: crash-safe, and itself recorded
115        // as a new snapshot so a restore can always be undone.
116        self.dirty_catalog = true;
117        self.dirty_index = true;
118        self.index_persist.forced = true;
119        self.flush()
120    }
121
122    /// FMT-9c: v9 snapshot retention, applied to the staged manifest (the
123    /// new snapshot already appended): keep the newest `max_snapshots`
124    /// plus any younger than 24 hours, then drop the oldest while retained
125    /// snapshots pin more than a quarter of the file's live pages. The
126    /// newest snapshot always stays.
127    ///
128    /// The pin cap is a quarter of the live pages or [`V9_PIN_FLOOR_PAGES`]
129    /// (1 MiB), whichever is larger, so tiny files keep their history.
130    /// `extra` is what this commit frees (pinned by every older snapshot).
131    ///
132    /// An explicit count (`set_max_snapshots`, or a non-default
133    /// `MnemoConfig::max_snapshots`) replaces the policy: exactly the
134    /// newest `n` are kept (`0` keeps all), whatever they pin.
135    pub(super) fn apply_retention(&self, manifest: &mut Vec<Snapshot>, extra: u64) {
136        if self.snapshot_cap_explicit {
137            if self.max_snapshots > 0 {
138                while manifest.len() > self.max_snapshots {
139                    manifest.remove(0);
140                }
141            }
142            return;
143        }
144        let now = memory::now_secs();
145        if self.max_snapshots > 0 {
146            while manifest.len() > self.max_snapshots
147                && now - manifest[0].created_at > V9_SNAPSHOT_KEEP_SECS
148            {
149                manifest.remove(0);
150            }
151        }
152        let live_pages = self.header.next_page.saturating_sub(self.free.pages());
153        let cap = (live_pages / 4).max(V9_PIN_FLOOR_PAGES);
154        while manifest.len() > 1 {
155            let pinned = self.free.pinned_pages(manifest[0].txn_id) + extra;
156            if pinned <= cap {
157                break;
158            }
159            manifest.remove(0);
160        }
161    }
162
163    /// FMT-9 (a): a restore makes the snapshot's records live again and the
164    /// current ones (those the snapshot doesn't have) dead. The snapshot's
165    /// record runs may be on the free list (freed after the snapshot, and
166    /// quarantined while it's retained), so they come back off it, and off
167    /// this transaction's pending frees; the current runs it doesn't share
168    /// are freed by the restoring commit.
169    pub(super) fn reclaim_for_restore(&mut self, restored: &[CatalogEntry]) {
170        // Record runs, deduplicated (FMT-9b.2: packed records share a page);
171        // rows still waiting to be packed own no page yet.
172        let runs = |cat: &[CatalogEntry]| -> std::collections::BTreeMap<u64, u64> {
173            cat.iter()
174                .filter(|e| !e.deleted && e.start_page != 0)
175                .map(|e| (e.start_page, e.page_count as u64))
176                .collect()
177        };
178        let live_then = runs(restored);
179        let live_now = runs(&self.catalog);
180        let overlaps = |p: u64, c: u64| {
181            live_then
182                .range(..p + c)
183                .next_back()
184                .is_some_and(|(&q, &n)| q + n > p)
185        };
186        let dying: Vec<(u64, u64)> = live_now
187            .iter()
188            .filter(|(p, _)| !live_then.contains_key(p))
189            .map(|(&p, &c)| (p, c))
190            .collect();
191        // FMT-9b: vector pages hold many rows, so they're compared as a
192        // set: the snapshot's come back, the current ones it lacks go.
193        let vec_then: std::collections::BTreeSet<u64> = restored
194            .iter()
195            .filter(|e| !e.deleted)
196            .filter_map(|e| e.v9.map(|r| r.vec_page))
197            .filter(|&p| p != 0)
198            .collect();
199        let vec_dying: Vec<u64> = self
200            .col
201            .page_rows
202            .keys()
203            .copied()
204            .filter(|p| !vec_then.contains(p))
205            .collect();
206        self.pending_free
207            .retain(|&(p, c)| !(overlaps(p, c) || (c == 1 && vec_then.contains(&p))));
208        for (&p, &c) in &live_then {
209            self.free.unfree(p, c);
210        }
211        for &p in &vec_then {
212            self.free.unfree(p, 1);
213        }
214        for (p, c) in dying {
215            self.release_pages(p, c);
216        }
217        for p in vec_dying {
218            self.release_pages(p, 1);
219        }
220        self.free_dirty = true;
221    }
222
223    /// Restore the database to the snapshot produced by transaction `txn_id`.
224    ///
225    /// The restore is itself a new committed transaction (and a new
226    /// snapshot), so it is crash-safe and reversible — restoring forward to a
227    /// later snapshot afterwards works just as well.
228    pub fn restore_to(&mut self, txn_id: u64) -> Result<SnapshotInfo> {
229        if self.read_only {
230            return Err(MnemoError::ReadOnly);
231        }
232        let snap = self
233            .manifest
234            .iter()
235            .find(|s| s.txn_id == txn_id)
236            .cloned()
237            .ok_or_else(|| {
238                let oldest = self.manifest.iter().map(|s| s.txn_id).min();
239                match oldest {
240                    Some(o) if self.is_v9() && txn_id < o => MnemoError::NotFound(format!(
241                        "snapshot for transaction {txn_id}: older than the retention window \
242                         (the newest {} snapshots and any from the last 24 hours, with at \
243                         most a quarter of the file pinned); the oldest kept is {o}",
244                        self.max_snapshots
245                    )),
246                    _ => MnemoError::NotFound(format!("snapshot for transaction {txn_id}")),
247                }
248            })?;
249        self.apply_snapshot(&snap)?;
250        Ok(SnapshotInfo {
251            txn_id: snap.txn_id,
252            created_at: snap.created_at,
253            memory_count: snap.memory_count,
254        })
255    }
256
257    /// Restore the database to the latest snapshot committed at or before
258    /// `unix_secs`. Returns [`MnemoError::NotFound`] if no snapshot is that
259    /// old. Like [`Mnemo::restore_to`], the restore is a new transaction.
260    pub fn restore_to_time(&mut self, unix_secs: i64) -> Result<SnapshotInfo> {
261        if self.read_only {
262            return Err(MnemoError::ReadOnly);
263        }
264        let snap = self
265            .manifest
266            .iter()
267            .filter(|s| s.created_at <= unix_secs)
268            .max_by_key(|s| s.txn_id)
269            .cloned()
270            .ok_or_else(|| {
271                MnemoError::NotFound(format!("no snapshot at or before time {unix_secs}"))
272            })?;
273        self.apply_snapshot(&snap)?;
274        Ok(SnapshotInfo {
275            txn_id: snap.txn_id,
276            created_at: snap.created_at,
277            memory_count: snap.memory_count,
278        })
279    }
280}