Skip to main content

mnemo_engine/store/
snapshots.rs

1//! Snapshots and point-in-time recovery (restore, retention), and sessions.
2
3use super::*;
4
5impl Mnemo {
6    /// Begin a conversation [`Session`](crate::Session) for `agent_id`.
7    ///
8    /// The session borrows the database for its lifetime, records turns as
9    /// working memory, and consolidates them into episodic memory when closed.
10    pub fn session(&mut self, agent_id: impl Into<String>) -> crate::session::Session<'_> {
11        crate::session::Session::new(self, agent_id.into())
12    }
13
14    // --- snapshots & point-in-time recovery ------------------------------
15
16    /// Every committed transaction, oldest first — the restore points
17    /// available to [`Mnemo::restore_to`] and [`Mnemo::restore_to_time`].
18    ///
19    /// Each `flush` appends one snapshot. Because the storage engine is
20    /// append-only, the pages a past flush wrote are still on disk, so any
21    /// listed snapshot can be reinstated exactly. The history reaches back to
22    /// the last [`Mnemo::compact_file`], which reclaims space by collapsing
23    /// it.
24    pub fn snapshots(&self) -> Vec<SnapshotInfo> {
25        let mut v: Vec<SnapshotInfo> = self
26            .manifest
27            .iter()
28            .map(|s| SnapshotInfo {
29                txn_id: s.txn_id,
30                created_at: s.created_at,
31                memory_count: s.memory_count,
32            })
33            .collect();
34        v.sort_by_key(|s| s.txn_id);
35        v
36    }
37
38    /// INSP-1 step 4 (RB-001.25): what is restorable right now, from the
39    /// retained snapshots rather than the retention policy: how many there
40    /// are, when the oldest was committed, and the bytes they keep from
41    /// reuse (free pages freed after the oldest one; v9's pin). Reads only
42    /// the manifest and the free list, so it's as cheap as `snapshots()`.
43    pub(super) fn restore_points(&self) -> RestorePoints {
44        let oldest = self.manifest.iter().min_by_key(|s| s.txn_id);
45        RestorePoints {
46            retained: self.manifest.len(),
47            oldest_at: oldest.map(|s| s.created_at),
48            pinned_bytes: match oldest {
49                Some(o) if self.is_v9() => self.free.pinned_pages(o.txn_id) * PAGE_SIZE as u64,
50                _ => 0,
51            },
52        }
53    }
54
55    /// Load a past snapshot's state and commit it as a new transaction.
56    pub(super) fn apply_snapshot(&mut self, snap: &Snapshot) -> Result<()> {
57        let catalog = load_catalog(
58            &mut self.pager,
59            snap.catalog_start,
60            snap.catalog_pages,
61            snap.catalog_len,
62            snap.catalog_gen(),
63        )?;
64        let (catalog, _, _) = apply_delta_chain(&mut self.pager, catalog, snap.delta_head())?;
65        // FMT-9c.2: the restored catalog is written whole.
66        self.catalog_rewrite = true;
67        let ann = load_index(
68            &mut self.pager,
69            snap.index_start,
70            snap.index_pages,
71            snap.index_len,
72            snap.index_gen(),
73            self.dimensions,
74        )?;
75        let mut ann = ann;
76        if self.is_v9() {
77            self.reclaim_for_restore(&catalog);
78            if let Some(a) = ann.as_mut() {
79                apply_index_chain(&mut self.pager, a, snap.index_delta_head())?;
80                a.enable_log();
81            }
82            self.index_rebase = true;
83            // FMT-9d: the snapshot's BM25 segments (quarantined while it's
84            // retained) describe its documents.
85            let bptr = snap.gens.map_or(RunPtr::default(), |g| tuple_ptr(g.bm25));
86            let manifest = load_bm25_manifest(&mut self.pager, bptr)?;
87            // Its runs come back off the free list; the current ones it
88            // doesn't share are freed, and the header points at its run.
89            let runs = |ptr: RunPtr, m: &Bm25Manifest| {
90                let mut v: Vec<(u64, u64)> = m.segments.iter().map(|t| (t.0, t.1)).collect();
91                if !ptr.is_empty() {
92                    v.push((ptr.start, ptr.pages));
93                }
94                v
95            };
96            let then = runs(bptr, &manifest);
97            let now = runs(self.header.bm25, &self.bm25.manifest);
98            self.pending_free.retain(|r| !then.contains(r));
99            for &(p, c) in &then {
100                self.free.unfree(p, c);
101            }
102            for r in now {
103                if !then.contains(&r) {
104                    self.release_pages(r.0, r.1);
105                }
106            }
107            self.header.bm25 = bptr;
108            self.free_dirty = true;
109            self.bm25 = Bm25State {
110                manifest,
111                ..Bm25State::default()
112            };
113        }
114        self.index = build_id_index(&catalog);
115        self.catalog = catalog;
116        // FMT-9b: the column and the packed pages described the catalog
117        // being replaced.
118        self.col = VecColumn::from_catalog(&self.catalog);
119        self.recs = RecPages::from_catalog(&self.catalog);
120        self.ann = ann;
121        self.ann_unloaded = false;
122        // The handle caches describe the catalog being replaced.
123        self.key_index = None;
124        self.superseded = None;
125        self.layers = None;
126        self.agent_set = None;
127        // PERF-1: the snapshot's index may predate its catalog; bring it up
128        // to date before it's persisted again.
129        let snap_index_gen = snap.index_gen().max(snap.index_delta_head().gen);
130        self.reconcile_ann_with_catalog(snap.index_start, snap_index_gen)?;
131        // Re-commit as a fresh transaction: crash-safe, and itself recorded
132        // as a new snapshot so a restore can always be undone.
133        self.dirty_catalog = true;
134        self.dirty_index = true;
135        self.index_persist.forced = true;
136        self.flush()
137    }
138
139    /// FMT-9c: v9 snapshot retention, applied to the staged manifest (the
140    /// new snapshot already appended): keep the newest `max_snapshots`
141    /// plus any younger than 24 hours, then drop the oldest while retained
142    /// snapshots pin more than a quarter of the file's live pages. The
143    /// newest snapshot always stays.
144    ///
145    /// The pin cap is a quarter of the live pages or [`V9_PIN_FLOOR_PAGES`]
146    /// (1 MiB), whichever is larger, so tiny files keep their history.
147    /// `extra` is what this commit frees (pinned by every older snapshot).
148    ///
149    /// An explicit count (`set_max_snapshots`, or a non-default
150    /// `MnemoConfig::max_snapshots`) replaces the policy: exactly the
151    /// newest `n` are kept (`0` keeps all), whatever they pin.
152    pub(super) fn apply_retention(&self, manifest: &mut Vec<Snapshot>, extra: u64) {
153        if self.snapshot_cap_explicit {
154            if self.max_snapshots > 0 {
155                while manifest.len() > self.max_snapshots {
156                    manifest.remove(0);
157                }
158            }
159            return;
160        }
161        let now = memory::now_secs();
162        if self.max_snapshots > 0 {
163            while manifest.len() > self.max_snapshots
164                && now - manifest[0].created_at > V9_SNAPSHOT_KEEP_SECS
165            {
166                manifest.remove(0);
167            }
168        }
169        let live_pages = self.header.next_page.saturating_sub(self.free.pages());
170        let cap = (live_pages / 4).max(V9_PIN_FLOOR_PAGES);
171        while manifest.len() > 1 {
172            let pinned = self.free.pinned_pages(manifest[0].txn_id) + extra;
173            if pinned <= cap {
174                break;
175            }
176            manifest.remove(0);
177        }
178    }
179
180    /// FMT-9 (a): a restore makes the snapshot's records live again and the
181    /// current ones (those the snapshot doesn't have) dead. The snapshot's
182    /// record runs may be on the free list (freed after the snapshot, and
183    /// quarantined while it's retained), so they come back off it, and off
184    /// this transaction's pending frees; the current runs it doesn't share
185    /// are freed by the restoring commit.
186    pub(super) fn reclaim_for_restore(&mut self, restored: &[CatalogEntry]) {
187        // Record runs, deduplicated (FMT-9b.2: packed records share a page);
188        // rows still waiting to be packed own no page yet.
189        let runs = |cat: &[CatalogEntry]| -> std::collections::BTreeMap<u64, u64> {
190            cat.iter()
191                .filter(|e| !e.deleted && e.start_page != 0)
192                .map(|e| (e.start_page, e.page_count as u64))
193                .collect()
194        };
195        let live_then = runs(restored);
196        let live_now = runs(&self.catalog);
197        let overlaps = |p: u64, c: u64| {
198            live_then
199                .range(..p + c)
200                .next_back()
201                .is_some_and(|(&q, &n)| q + n > p)
202        };
203        let dying: Vec<(u64, u64)> = live_now
204            .iter()
205            .filter(|(p, _)| !live_then.contains_key(p))
206            .map(|(&p, &c)| (p, c))
207            .collect();
208        // FMT-9b: vector pages hold many rows, so they're compared as a
209        // set: the snapshot's come back, the current ones it lacks go.
210        let vec_then: std::collections::BTreeSet<u64> = restored
211            .iter()
212            .filter(|e| !e.deleted)
213            .filter_map(|e| e.v9.map(|r| r.vec_page))
214            .filter(|&p| p != 0)
215            .collect();
216        let vec_dying: Vec<u64> = self
217            .col
218            .page_rows
219            .keys()
220            .copied()
221            .filter(|p| !vec_then.contains(p))
222            .collect();
223        self.pending_free
224            .retain(|&(p, c)| !(overlaps(p, c) || (c == 1 && vec_then.contains(&p))));
225        for (&p, &c) in &live_then {
226            self.free.unfree(p, c);
227        }
228        for &p in &vec_then {
229            self.free.unfree(p, 1);
230        }
231        for (p, c) in dying {
232            self.release_pages(p, c);
233        }
234        for p in vec_dying {
235            self.release_pages(p, 1);
236        }
237        self.free_dirty = true;
238    }
239
240    /// Restore the database to the snapshot produced by transaction `txn_id`.
241    ///
242    /// The restore is itself a new committed transaction (and a new
243    /// snapshot), so it is crash-safe and reversible — restoring forward to a
244    /// later snapshot afterwards works just as well.
245    pub fn restore_to(&mut self, txn_id: u64) -> Result<SnapshotInfo> {
246        if self.read_only {
247            return Err(MnemoError::ReadOnly);
248        }
249        let snap = self
250            .manifest
251            .iter()
252            .find(|s| s.txn_id == txn_id)
253            .cloned()
254            .ok_or_else(|| {
255                let oldest = self.manifest.iter().map(|s| s.txn_id).min();
256                match oldest {
257                    Some(o) if self.is_v9() && txn_id < o => MnemoError::NotFound(format!(
258                        "snapshot for transaction {txn_id}: older than the retention window \
259                         (the newest {} snapshots and any from the last 24 hours, with at \
260                         most a quarter of the file pinned); the oldest kept is {o}",
261                        self.max_snapshots
262                    )),
263                    _ => MnemoError::NotFound(format!("snapshot for transaction {txn_id}")),
264                }
265            })?;
266        self.apply_snapshot(&snap)?;
267        Ok(SnapshotInfo {
268            txn_id: snap.txn_id,
269            created_at: snap.created_at,
270            memory_count: snap.memory_count,
271        })
272    }
273
274    /// Restore the database to the latest snapshot committed at or before
275    /// `unix_secs`. Returns [`MnemoError::NotFound`] if no snapshot is that
276    /// old. Like [`Mnemo::restore_to`], the restore is a new transaction.
277    pub fn restore_to_time(&mut self, unix_secs: i64) -> Result<SnapshotInfo> {
278        if self.read_only {
279            return Err(MnemoError::ReadOnly);
280        }
281        let snap = self
282            .manifest
283            .iter()
284            .filter(|s| s.created_at <= unix_secs)
285            .max_by_key(|s| s.txn_id)
286            .cloned()
287            .ok_or_else(|| {
288                MnemoError::NotFound(format!("no snapshot at or before time {unix_secs}"))
289            })?;
290        self.apply_snapshot(&snap)?;
291        Ok(SnapshotInfo {
292            txn_id: snap.txn_id,
293            created_at: snap.created_at,
294            memory_count: snap.memory_count,
295        })
296    }
297}