1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
//! Snapshots and point-in-time recovery (restore, retention), and sessions.
use super::*;
impl Mnemo {
/// Begin a conversation [`Session`](crate::Session) for `agent_id`.
///
/// The session borrows the database for its lifetime, records turns as
/// working memory, and consolidates them into episodic memory when closed.
pub fn session(&mut self, agent_id: impl Into<String>) -> crate::session::Session<'_> {
crate::session::Session::new(self, agent_id.into())
}
// --- snapshots & point-in-time recovery ------------------------------
/// Every committed transaction, oldest first — the restore points
/// available to [`Mnemo::restore_to`] and [`Mnemo::restore_to_time`].
///
/// Each `flush` appends one snapshot. Because the storage engine is
/// append-only, the pages a past flush wrote are still on disk, so any
/// listed snapshot can be reinstated exactly. The history reaches back to
/// the last [`Mnemo::compact_file`], which reclaims space by collapsing
/// it.
pub fn snapshots(&self) -> Vec<SnapshotInfo> {
let mut v: Vec<SnapshotInfo> = self
.manifest
.iter()
.map(|s| SnapshotInfo {
txn_id: s.txn_id,
created_at: s.created_at,
memory_count: s.memory_count,
})
.collect();
v.sort_by_key(|s| s.txn_id);
v
}
/// INSP-1 step 4 (RB-001.25): what is restorable right now, from the
/// retained snapshots rather than the retention policy: how many there
/// are, when the oldest was committed, and the bytes they keep from
/// reuse (free pages freed after the oldest one; v9's pin). Reads only
/// the manifest and the free list, so it's as cheap as `snapshots()`.
pub(super) fn restore_points(&self) -> RestorePoints {
let oldest = self.manifest.iter().min_by_key(|s| s.txn_id);
RestorePoints {
retained: self.manifest.len(),
oldest_at: oldest.map(|s| s.created_at),
pinned_bytes: match oldest {
Some(o) if self.is_v9() => self.free.pinned_pages(o.txn_id) * PAGE_SIZE as u64,
_ => 0,
},
}
}
/// Load a past snapshot's state and commit it as a new transaction.
pub(super) fn apply_snapshot(&mut self, snap: &Snapshot) -> Result<()> {
let catalog = load_catalog(
&mut self.pager,
snap.catalog_start,
snap.catalog_pages,
snap.catalog_len,
snap.catalog_gen(),
)?;
let (catalog, _, _) = apply_delta_chain(&mut self.pager, catalog, snap.delta_head())?;
// FMT-9c.2: the restored catalog is written whole.
self.catalog_rewrite = true;
let ann = load_index(
&mut self.pager,
snap.index_start,
snap.index_pages,
snap.index_len,
snap.index_gen(),
self.dimensions,
)?;
let mut ann = ann;
if self.is_v9() {
self.reclaim_for_restore(&catalog);
if let Some(a) = ann.as_mut() {
apply_index_chain(&mut self.pager, a, snap.index_delta_head())?;
a.enable_log();
}
self.index_rebase = true;
// FMT-9d: the snapshot's BM25 segments (quarantined while it's
// retained) describe its documents.
let bptr = snap.gens.map_or(RunPtr::default(), |g| tuple_ptr(g.bm25));
let manifest = load_bm25_manifest(&mut self.pager, bptr)?;
// Its runs come back off the free list; the current ones it
// doesn't share are freed, and the header points at its run.
let runs = |ptr: RunPtr, m: &Bm25Manifest| {
let mut v: Vec<(u64, u64)> = m.segments.iter().map(|t| (t.0, t.1)).collect();
if !ptr.is_empty() {
v.push((ptr.start, ptr.pages));
}
v
};
let then = runs(bptr, &manifest);
let now = runs(self.header.bm25, &self.bm25.manifest);
self.pending_free.retain(|r| !then.contains(r));
for &(p, c) in &then {
self.free.unfree(p, c);
}
for r in now {
if !then.contains(&r) {
self.release_pages(r.0, r.1);
}
}
self.header.bm25 = bptr;
self.free_dirty = true;
self.bm25 = Bm25State {
manifest,
..Bm25State::default()
};
}
self.index = build_id_index(&catalog);
self.catalog = catalog;
// FMT-9b: the column and the packed pages described the catalog
// being replaced.
self.col = VecColumn::from_catalog(&self.catalog);
self.recs = RecPages::from_catalog(&self.catalog);
self.ann = ann;
self.ann_unloaded = false;
// The handle caches describe the catalog being replaced.
self.key_index = None;
self.superseded = None;
self.layers = None;
self.agent_set = None;
// PERF-1: the snapshot's index may predate its catalog; bring it up
// to date before it's persisted again.
let snap_index_gen = snap.index_gen().max(snap.index_delta_head().gen);
self.reconcile_ann_with_catalog(snap.index_start, snap_index_gen)?;
// Re-commit as a fresh transaction: crash-safe, and itself recorded
// as a new snapshot so a restore can always be undone.
self.dirty_catalog = true;
self.dirty_index = true;
self.index_persist.forced = true;
self.flush()
}
/// FMT-9c: v9 snapshot retention, applied to the staged manifest (the
/// new snapshot already appended): keep the newest `max_snapshots`
/// plus any younger than 24 hours, then drop the oldest while retained
/// snapshots pin more than a quarter of the file's live pages. The
/// newest snapshot always stays.
///
/// The pin cap is a quarter of the live pages or [`V9_PIN_FLOOR_PAGES`]
/// (1 MiB), whichever is larger, so tiny files keep their history.
/// `extra` is what this commit frees (pinned by every older snapshot).
///
/// An explicit count (`set_max_snapshots`, or a non-default
/// `MnemoConfig::max_snapshots`) replaces the policy: exactly the
/// newest `n` are kept (`0` keeps all), whatever they pin.
pub(super) fn apply_retention(&self, manifest: &mut Vec<Snapshot>, extra: u64) {
if self.snapshot_cap_explicit {
if self.max_snapshots > 0 {
while manifest.len() > self.max_snapshots {
manifest.remove(0);
}
}
return;
}
let now = memory::now_secs();
if self.max_snapshots > 0 {
while manifest.len() > self.max_snapshots
&& now - manifest[0].created_at > V9_SNAPSHOT_KEEP_SECS
{
manifest.remove(0);
}
}
let live_pages = self.header.next_page.saturating_sub(self.free.pages());
let cap = (live_pages / 4).max(V9_PIN_FLOOR_PAGES);
while manifest.len() > 1 {
let pinned = self.free.pinned_pages(manifest[0].txn_id) + extra;
if pinned <= cap {
break;
}
manifest.remove(0);
}
}
/// FMT-9 (a): a restore makes the snapshot's records live again and the
/// current ones (those the snapshot doesn't have) dead. The snapshot's
/// record runs may be on the free list (freed after the snapshot, and
/// quarantined while it's retained), so they come back off it, and off
/// this transaction's pending frees; the current runs it doesn't share
/// are freed by the restoring commit.
pub(super) fn reclaim_for_restore(&mut self, restored: &[CatalogEntry]) {
// Record runs, deduplicated (FMT-9b.2: packed records share a page);
// rows still waiting to be packed own no page yet.
let runs = |cat: &[CatalogEntry]| -> std::collections::BTreeMap<u64, u64> {
cat.iter()
.filter(|e| !e.deleted && e.start_page != 0)
.map(|e| (e.start_page, e.page_count as u64))
.collect()
};
let live_then = runs(restored);
let live_now = runs(&self.catalog);
let overlaps = |p: u64, c: u64| {
live_then
.range(..p + c)
.next_back()
.is_some_and(|(&q, &n)| q + n > p)
};
let dying: Vec<(u64, u64)> = live_now
.iter()
.filter(|(p, _)| !live_then.contains_key(p))
.map(|(&p, &c)| (p, c))
.collect();
// FMT-9b: vector pages hold many rows, so they're compared as a
// set: the snapshot's come back, the current ones it lacks go.
let vec_then: std::collections::BTreeSet<u64> = restored
.iter()
.filter(|e| !e.deleted)
.filter_map(|e| e.v9.map(|r| r.vec_page))
.filter(|&p| p != 0)
.collect();
let vec_dying: Vec<u64> = self
.col
.page_rows
.keys()
.copied()
.filter(|p| !vec_then.contains(p))
.collect();
self.pending_free
.retain(|&(p, c)| !(overlaps(p, c) || (c == 1 && vec_then.contains(&p))));
for (&p, &c) in &live_then {
self.free.unfree(p, c);
}
for &p in &vec_then {
self.free.unfree(p, 1);
}
for (p, c) in dying {
self.release_pages(p, c);
}
for p in vec_dying {
self.release_pages(p, 1);
}
self.free_dirty = true;
}
/// Restore the database to the snapshot produced by transaction `txn_id`.
///
/// The restore is itself a new committed transaction (and a new
/// snapshot), so it is crash-safe and reversible — restoring forward to a
/// later snapshot afterwards works just as well.
pub fn restore_to(&mut self, txn_id: u64) -> Result<SnapshotInfo> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let snap = self
.manifest
.iter()
.find(|s| s.txn_id == txn_id)
.cloned()
.ok_or_else(|| {
let oldest = self.manifest.iter().map(|s| s.txn_id).min();
match oldest {
Some(o) if self.is_v9() && txn_id < o => MnemoError::NotFound(format!(
"snapshot for transaction {txn_id}: older than the retention window \
(the newest {} snapshots and any from the last 24 hours, with at \
most a quarter of the file pinned); the oldest kept is {o}",
self.max_snapshots
)),
_ => MnemoError::NotFound(format!("snapshot for transaction {txn_id}")),
}
})?;
self.apply_snapshot(&snap)?;
Ok(SnapshotInfo {
txn_id: snap.txn_id,
created_at: snap.created_at,
memory_count: snap.memory_count,
})
}
/// Restore the database to the latest snapshot committed at or before
/// `unix_secs`. Returns [`MnemoError::NotFound`] if no snapshot is that
/// old. Like [`Mnemo::restore_to`], the restore is a new transaction.
pub fn restore_to_time(&mut self, unix_secs: i64) -> Result<SnapshotInfo> {
if self.read_only {
return Err(MnemoError::ReadOnly);
}
let snap = self
.manifest
.iter()
.filter(|s| s.created_at <= unix_secs)
.max_by_key(|s| s.txn_id)
.cloned()
.ok_or_else(|| {
MnemoError::NotFound(format!("no snapshot at or before time {unix_secs}"))
})?;
self.apply_snapshot(&snap)?;
Ok(SnapshotInfo {
txn_id: snap.txn_id,
created_at: snap.created_at,
memory_count: snap.memory_count,
})
}
}