second_brain_api/
at_rest.rs1use std::sync::{Arc, PoisonError, RwLock};
7use std::time::{Duration, Instant};
8
9use anyhow::Result;
10use second_brain_core::index::raw::RawIndex;
11use second_brain_core::sealed::{SealKeyId, SealKeys};
12use second_brain_core::sqlite_store::{AtRestPurpose, SqliteStore};
13use second_brain_core::transcript_archive::TranscriptArchive;
14
15pub const CHECK_INTERVAL: Duration = Duration::from_secs(60);
16
17pub trait Reseal: Send + Sync {
19 fn reseal_under(&self, keys: SealKeys) -> Result<usize>;
22
23 fn sealed_under(&self) -> Option<SealKeyId>;
25}
26
27impl Reseal for TranscriptArchive {
28 fn reseal_under(&self, keys: SealKeys) -> Result<usize> {
29 self.set_seal_keys(keys);
30 self.reseal()
31 }
32
33 fn sealed_under(&self) -> Option<SealKeyId> {
34 if self.holds_unsealed() {
35 return None;
36 }
37 self.seal_keys().map(|keys| keys.current_id())
38 }
39}
40
41impl Reseal for RwLock<RawIndex> {
42 fn reseal_under(&self, keys: SealKeys) -> Result<usize> {
43 let mut index = self.write().unwrap_or_else(PoisonError::into_inner);
44 index.set_seal_keys(keys)?;
45 index.reseal()
46 }
47
48 fn sealed_under(&self) -> Option<SealKeyId> {
49 self.read()
50 .unwrap_or_else(PoisonError::into_inner)
51 .seal_keys()
52 .map(|keys| keys.current_id())
53 }
54}
55
56pub struct Sealer {
57 pub store: Arc<SqliteStore>,
58 pub purpose: AtRestPurpose,
59 pub target: Arc<dyn Reseal>,
60 sealed_under: Option<SealKeyId>,
61}
62
63impl Sealer {
64 pub fn new(store: Arc<SqliteStore>, purpose: AtRestPurpose, target: Arc<dyn Reseal>) -> Self {
65 Self {
66 store,
67 purpose,
68 target,
69 sealed_under: None,
70 }
71 }
72
73 pub fn spawn(mut self) {
74 std::thread::Builder::new()
75 .name(format!("seal-{:?}", self.purpose).to_lowercase())
76 .spawn(move || {
77 loop {
78 self.check();
79 std::thread::sleep(CHECK_INTERVAL);
80 }
81 })
82 .expect("spawning an at-rest sealing thread");
83 }
84
85 pub fn check(&mut self) -> bool {
88 let keys = match self.store.at_rest_keys(self.purpose) {
89 Ok(Some(keys)) => keys,
90 Ok(None) => return false,
91 Err(e) => {
92 tracing::warn!("reading the {:?} at-rest keys failed: {e:#}", self.purpose);
93 return false;
94 }
95 };
96 let current = keys.keys.current_id();
97 if keys.settled
98 && (self.sealed_under == Some(current) || self.target.sealed_under() == Some(current))
99 {
100 self.sealed_under = Some(current);
101 return false;
102 }
103 let started = Instant::now();
104 let rewritten = match self.target.reseal_under(keys.keys) {
105 Ok(n) => n,
106 Err(e) => {
107 tracing::warn!(
108 "sealing the {:?} files failed and will retry: {e:#}",
109 self.purpose
110 );
111 return false;
112 }
113 };
114 if let Err(e) = self.store.settle_at_rest_keys(self.purpose, current) {
115 tracing::warn!(
116 "recording the sealed {:?} files failed: {e:#}",
117 self.purpose
118 );
119 return false;
120 }
121 self.sealed_under = Some(current);
122 if rewritten > 0 {
123 tracing::info!(
124 "{:?}: {rewritten} files sealed under key {current} in {:.1}s",
125 self.purpose,
126 started.elapsed().as_secs_f64()
127 );
128 }
129 true
130 }
131}
132
133#[cfg(test)]
134mod tests {
135 use std::path::Path;
136
137 use second_brain_core::keys::StoreKey;
138 use second_brain_core::sealed::sealed_key_id;
139 use second_brain_core::sqlite_store::encrypt_migration::RewriteTo;
140 use second_brain_core::transcript_archive::{MANIFEST_FILE, SEALED_MANIFEST_FILE};
141
142 use super::*;
143
144 fn keyed_store(dir: &Path) -> Arc<SqliteStore> {
145 let key = StoreKey::generate().unwrap();
146 Arc::new(SqliteStore::open_with_key(&dir.join("brain.db"), "m".to_string(), &key).unwrap())
147 }
148
149 fn files_under(dir: &Path) -> Vec<std::path::PathBuf> {
150 let mut out = Vec::new();
151 let mut stack = vec![dir.to_path_buf()];
152 while let Some(d) = stack.pop() {
153 for entry in std::fs::read_dir(d).unwrap() {
154 let path = entry.unwrap().path();
155 if path.is_dir() {
156 stack.push(path);
157 } else if !path.to_string_lossy().ends_with(".lock") {
158 out.push(path);
159 }
160 }
161 }
162 out
163 }
164
165 fn assert_sealed_under(dir: &Path, key: SealKeyId) {
166 let files = files_under(dir);
167 assert!(!files.is_empty());
168 for path in files {
169 if path
170 .file_name()
171 .is_some_and(|n| n == MANIFEST_FILE || n == SEALED_MANIFEST_FILE)
172 {
173 let manifest = std::fs::read_to_string(&path).unwrap();
174 assert!(!manifest.contains("session.jsonl"), "{manifest}");
175 continue;
176 }
177 assert_eq!(
178 sealed_key_id(&path).unwrap(),
179 Some(key),
180 "{}",
181 path.display()
182 );
183 }
184 }
185
186 #[test]
187 fn an_archive_written_before_encryption_is_sealed_then_resealed_after_a_rotation() {
188 let data = tempfile::tempdir().unwrap();
189 let store = keyed_store(data.path());
190 let archive_dir = data.path().join("transcripts");
191 let archive = Arc::new(TranscriptArchive::open(&archive_dir).unwrap());
192 let source = data.path().join("session.jsonl");
193 std::fs::write(
194 &source,
195 "{\"text\":\"the zanzibar ferry leaves at noon\"}\n",
196 )
197 .unwrap();
198 archive.archive(&source).unwrap();
199
200 let mut sealer = Sealer::new(
201 store.clone(),
202 AtRestPurpose::TranscriptArchive,
203 archive.clone(),
204 );
205 assert!(sealer.check());
206 let first = store
207 .at_rest_keys(AtRestPurpose::TranscriptArchive)
208 .unwrap()
209 .unwrap();
210 assert!(first.settled);
211 assert_sealed_under(&archive_dir, first.keys.current_id());
212 assert!(!sealer.check());
213 let mut after_restart = Sealer::new(
214 store.clone(),
215 AtRestPurpose::TranscriptArchive,
216 archive.clone(),
217 );
218 assert!(
219 !after_restart.check(),
220 "a settled archive was resealed again"
221 );
222
223 store
224 .rewrite_under_key(RewriteTo::Key(&StoreKey::generate().unwrap()))
225 .unwrap();
226 assert!(sealer.check());
227 let second = store
228 .at_rest_keys(AtRestPurpose::TranscriptArchive)
229 .unwrap()
230 .unwrap();
231 assert_ne!(second.keys.current_id(), first.keys.current_id());
232 assert!(second.keys.retired_ids().is_empty());
233 assert_sealed_under(&archive_dir, second.keys.current_id());
234 assert!(archive.read(&source).unwrap().unwrap().contains("zanzibar"));
235 }
236
237 #[test]
238 fn a_settled_archive_holding_entries_an_older_binary_wrote_is_resealed() {
239 let data = tempfile::tempdir().unwrap();
240 let store = keyed_store(data.path());
241 let archive_dir = data.path().join("transcripts");
242 let keys = || {
243 store
244 .at_rest_keys(AtRestPurpose::TranscriptArchive)
245 .unwrap()
246 .unwrap()
247 };
248 let first = data.path().join("first.jsonl");
249 std::fs::write(&first, "{\"text\":\"the zanzibar ferry\"}\n").unwrap();
250 let archive = Arc::new(TranscriptArchive::open_sealed(&archive_dir, keys().keys).unwrap());
251 archive.archive(&first).unwrap();
252 assert!(
253 Sealer::new(
254 store.clone(),
255 AtRestPurpose::TranscriptArchive,
256 archive.clone()
257 )
258 .check()
259 );
260 assert!(keys().settled);
261 drop(archive);
262
263 let older = data.path().join("older-binary");
264 let later = data.path().join("later.jsonl");
265 std::fs::write(&later, "{\"text\":\"the harbour at dusk\"}\n").unwrap();
266 let plain = TranscriptArchive::open(&older).unwrap();
267 plain.archive(&later).unwrap();
268 let entry = plain.entry(&later).unwrap();
269 drop(plain);
270 std::fs::copy(older.join(&entry.file), archive_dir.join(&entry.file)).unwrap();
271 std::fs::copy(older.join(MANIFEST_FILE), archive_dir.join(MANIFEST_FILE)).unwrap();
272
273 let archive = Arc::new(TranscriptArchive::open_sealed(&archive_dir, keys().keys).unwrap());
274 assert_eq!(archive.len(), 2);
275 let mut after_restart = Sealer::new(
276 store.clone(),
277 AtRestPurpose::TranscriptArchive,
278 archive.clone(),
279 );
280 assert!(after_restart.check());
281 assert_sealed_under(&archive_dir, keys().keys.current_id());
282 assert!(archive.read(&later).unwrap().unwrap().contains("harbour"));
283 }
284
285 #[test]
286 fn a_plaintext_store_seals_nothing() {
287 let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
288 let data = tempfile::tempdir().unwrap();
289 let archive = Arc::new(TranscriptArchive::open(data.path()).unwrap());
290 let mut sealer = Sealer::new(store, AtRestPurpose::TranscriptArchive, archive);
291 assert!(!sealer.check());
292 }
293
294 #[test]
295 fn a_raw_index_is_resealed_under_the_store_keys() {
296 let data = tempfile::tempdir().unwrap();
297 let store = keyed_store(data.path());
298 let raw_dir = data.path().join("brain.raw-index");
299 let index = Arc::new(RwLock::new(
300 RawIndex::open_keyed(
301 &raw_dir,
302 &store
303 .at_rest_keys(AtRestPurpose::RawIndex)
304 .unwrap()
305 .unwrap()
306 .keys,
307 )
308 .unwrap(),
309 ));
310 let mut sealer = Sealer::new(store.clone(), AtRestPurpose::RawIndex, index.clone());
311 assert!(sealer.check());
312 store
313 .rewrite_under_key(RewriteTo::Key(&StoreKey::generate().unwrap()))
314 .unwrap();
315 assert!(sealer.check());
316 let keys = store
317 .at_rest_keys(AtRestPurpose::RawIndex)
318 .unwrap()
319 .unwrap();
320 assert!(keys.settled);
321 assert_sealed_under(&raw_dir, keys.keys.current_id());
322 }
323
324 #[test]
325 fn a_raw_index_opened_before_the_store_was_encrypted_is_reopened_sealed() {
326 let data = tempfile::tempdir().unwrap();
327 let store = keyed_store(data.path());
328 let raw_dir = data.path().join("brain.raw-index");
329 let mut plain = RawIndex::open(&raw_dir).unwrap();
330 let mut embedding = vec![0.0f32; second_brain_core::index::EMBEDDING_DIM];
331 embedding[3] = 1.0;
332 let passage = second_brain_core::index::raw::RawPassage {
333 text: "the zanzibar ferry leaves at noon".to_string(),
334 at: None,
335 project: None,
336 memory: None,
337 };
338 let origin = second_brain_core::index::raw::RawOrigin::Transcript {
339 sha256: "ab".repeat(32),
340 };
341 plain.add_source(&origin, &[(passage, embedding)]).unwrap();
342 plain.commit().unwrap();
343 let index = Arc::new(RwLock::new(plain));
344
345 let mut sealer = Sealer::new(store.clone(), AtRestPurpose::RawIndex, index.clone());
346 assert!(sealer.check());
347 let keys = store
348 .at_rest_keys(AtRestPurpose::RawIndex)
349 .unwrap()
350 .unwrap();
351 assert!(keys.settled);
352 assert_sealed_under(&raw_dir, keys.keys.current_id());
353 let guard = index.read().unwrap();
354 assert_eq!(guard.len(), 1);
355 assert_eq!(guard.bm25_search("zanzibar", 5, None).unwrap().len(), 1);
356 }
357}