1use std::sync::Arc;
7use std::sync::atomic::{AtomicBool, Ordering};
8use std::sync::mpsc::Sender;
9use std::time::Duration;
10
11use anyhow::{Result, anyhow};
12use chrono::Utc;
13use second_brain_core::sqlite_store::encrypt_migration::NotReopened;
14use second_brain_core::sqlite_store::{SqliteStore, VacuumSpace};
15
16use crate::health::{Health, MaintenanceState};
17use crate::maintenance::DrainRequest;
18use crate::write_gate::WriteGate;
19
20pub const SPACE_RETRY: Duration = Duration::from_secs(30 * 60);
21const AWAIT_ENCRYPTION_MICROS: i64 = 3 * 24 * 3600 * 1_000_000;
25const FIRST_RETRY: Duration = Duration::from_secs(30);
26const MAX_RETRY: Duration = Duration::from_secs(3600);
27
28pub struct Compaction {
29 pub store: Arc<SqliteStore>,
30 pub gate: Arc<WriteGate>,
31 pub indexes: Option<Sender<DrainRequest>>,
34 pub health: Arc<Health>,
35 pub space: fn(&SqliteStore) -> Result<VacuumSpace>,
36 pub awaits_encryption: bool,
40 saw_plaintext: AtomicBool,
43}
44
45#[derive(Debug, PartialEq, Eq)]
46enum Step {
47 Done,
48 NoSpace,
49 AwaitingEncryption,
50 IndexesUnavailable,
53}
54
55impl Compaction {
56 pub fn new(
57 store: Arc<SqliteStore>,
58 gate: Arc<WriteGate>,
59 indexes: Option<Sender<DrainRequest>>,
60 health: Arc<Health>,
61 ) -> Self {
62 Self {
63 store,
64 gate,
65 indexes,
66 health,
67 space: SqliteStore::vacuum_space,
68 awaits_encryption: false,
69 saw_plaintext: AtomicBool::new(false),
70 }
71 }
72
73 pub fn run_until_done(&self, mut sleep: impl FnMut(Duration)) {
77 let mut delay = FIRST_RETRY;
78 loop {
79 match self.attempt() {
80 Ok(Step::Done | Step::IndexesUnavailable) => return,
81 Ok(Step::NoSpace | Step::AwaitingEncryption) => sleep(SPACE_RETRY),
82 Err(e) => {
83 tracing::error!(
84 "store compaction failed, retrying in {}s: {e:#}",
85 delay.as_secs()
86 );
87 let now = Utc::now();
88 self.health
89 .maintenance(MaintenanceState::Pending, None, now);
90 self.health.maintenance_failed(&format!("{e:#}"), now);
91 sleep(delay);
92 delay = (delay * 2).min(MAX_RETRY);
93 }
94 }
95 }
96 }
97
98 fn attempt(&self) -> Result<Step> {
99 if self.store.stored_redaction_compacted()? {
100 self.health
101 .maintenance(MaintenanceState::Done, None, Utc::now());
102 return Ok(Step::Done);
103 }
104 if self.awaits_encryption {
105 let encrypted = self.store.is_encrypted();
106 if !encrypted {
107 let now = Utc::now().timestamp_micros();
108 let since = self.store.encryption_awaited_since(now)?;
109 if now - since < AWAIT_ENCRYPTION_MICROS {
110 self.saw_plaintext.store(true, Ordering::SeqCst);
111 return Ok(Step::AwaitingEncryption);
112 }
113 }
114 if encrypted && self.saw_plaintext.load(Ordering::SeqCst) {
115 return self.merge_indexes_after_migration();
116 }
117 }
118 let space = (self.space)(&self.store)?;
119 if !space.fits() {
120 tracing::warn!(
121 "store compaction waits for disk space: needs {} bytes free at {}, has {}",
122 space.needed_bytes,
123 space.path.display(),
124 space.free_bytes
125 );
126 self.health
127 .maintenance(MaintenanceState::WaitingForSpace, Some(space), Utc::now());
128 return Ok(Step::NoSpace);
129 }
130 self.merge_indexes(Some(space))?;
131 let (before, after) = vacuum(&self.store, &self.gate, &self.health)?;
132 let step = self.record()?;
133 if step == Step::Done {
134 tracing::info!("store compacted after redaction: {before} bytes before, {after} after");
135 }
136 Ok(step)
137 }
138
139 fn merge_indexes_after_migration(&self) -> Result<Step> {
140 self.merge_indexes(None)?;
141 let step = self.record()?;
142 if step == Step::Done {
143 tracing::info!("store compacted after redaction by the encryption migration");
144 }
145 Ok(step)
146 }
147
148 fn merge_indexes(&self, space: Option<VacuumSpace>) -> Result<()> {
149 let Some(drain) = &self.indexes else {
150 return Ok(());
151 };
152 self.health
153 .maintenance(MaintenanceState::MergingIndex, space, Utc::now());
154 let (reply, done) = std::sync::mpsc::channel();
155 drain
156 .send(DrainRequest::CompactIndexes(reply))
157 .map_err(|_| anyhow!("the index drain thread is not running"))?;
158 done.recv()
159 .map_err(|_| anyhow!("the index drain thread stopped"))?
160 }
161
162 fn record(&self) -> Result<Step> {
163 if self.indexes.is_none() {
164 self.health.maintenance_failed(
165 "the search indexes are unavailable, so their old segments were not merged; \
166 compaction runs again at the next start",
167 Utc::now(),
168 );
169 return Ok(Step::IndexesUnavailable);
170 }
171 self.store.mark_stored_redaction_compacted()?;
172 self.health
173 .maintenance(MaintenanceState::Done, None, Utc::now());
174 Ok(Step::Done)
175 }
176}
177
178pub fn vacuum(store: &SqliteStore, gate: &WriteGate, health: &Health) -> Result<(u64, u64)> {
181 let _closed = gate.close();
182 let previous = health.report().maintenance.state;
183 health.maintenance(MaintenanceState::Vacuuming, None, Utc::now());
184 let result = store.vacuum_clearing_wal();
185 health.maintenance(previous, None, Utc::now());
186 if let Err(e) = &result
187 && e.downcast_ref::<NotReopened>().is_some()
188 {
189 tracing::error!("{e:#}");
190 std::process::exit(1);
191 }
192 result
193}
194
195#[cfg(test)]
196mod tests {
197 use second_brain_core::keys::{KeyGuard, StoreKey};
198 use second_brain_core::sqlite_store::encrypt_migration::RewriteTo;
199
200 use super::*;
201
202 fn drain_stub() -> Sender<DrainRequest> {
205 let (tx, requests) = std::sync::mpsc::channel();
206 std::thread::spawn(move || {
207 for request in requests {
208 if let DrainRequest::CompactIndexes(reply) = request {
209 let _ = reply.send(Ok(()));
210 }
211 }
212 });
213 tx
214 }
215
216 fn compaction(dir: &std::path::Path) -> Compaction {
217 let store = SqliteStore::open(&dir.join("brain.db"), "m".to_string()).unwrap();
218 Compaction::new(
219 Arc::new(store),
220 Arc::new(WriteGate::default()),
221 Some(drain_stub()),
222 Arc::new(Health::new(Utc::now(), None)),
223 )
224 }
225
226 static SPACE_CHECKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
227
228 fn short_of_space_once(store: &SqliteStore) -> Result<VacuumSpace> {
229 if SPACE_CHECKS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) > 0 {
230 return store.vacuum_space();
231 }
232 Ok(VacuumSpace {
233 path: std::path::PathBuf::from("/data"),
234 needed_bytes: 2_000_000,
235 free_bytes: 1_000_000,
236 })
237 }
238
239 #[test]
240 fn a_compaction_runs_once_and_is_recorded() {
241 let dir = tempfile::tempdir().unwrap();
242 let c = compaction(dir.path());
243 c.run_until_done(|_| panic!("nothing to wait for"));
244 assert!(c.store.stored_redaction_compacted().unwrap());
245 assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
246 assert!(!c.gate.is_closed());
247 }
248
249 #[test]
250 fn a_store_awaiting_encryption_waits_for_the_migration_and_skips_its_own_vacuum() {
251 use std::os::unix::fs::MetadataExt;
252
253 let dir = tempfile::tempdir().unwrap();
254 let db = dir.path().join("brain.db");
255 let mut c = compaction(dir.path());
256 c.awaits_encryption = true;
257 assert_eq!(c.attempt().unwrap(), Step::AwaitingEncryption);
258 assert!(!c.store.stored_redaction_compacted().unwrap());
259
260 let key = StoreKey::from_bytes(KeyGuard::new([4; 32]));
261 c.store.rewrite_under_key(RewriteTo::Key(&key)).unwrap();
262 let migrated = std::fs::metadata(&db).unwrap().ino();
263 c.run_until_done(|_| panic!("the migration has run"));
264 assert!(c.store.stored_redaction_compacted().unwrap());
265 assert_eq!(
266 std::fs::metadata(&db).unwrap().ino(),
267 migrated,
268 "the store was not rewritten a second time"
269 );
270 }
271
272 #[test]
273 fn a_migration_that_never_lands_gives_way_to_the_vacuum_across_restarts() {
274 let dir = tempfile::tempdir().unwrap();
275 let c = compaction(dir.path());
276 let started = Utc::now().timestamp_micros() - AWAIT_ENCRYPTION_MICROS - 1;
277 c.store.encryption_awaited_since(started).unwrap();
278 drop(c);
279
280 let mut restarted = compaction(dir.path());
281 restarted.awaits_encryption = true;
282 restarted.run_until_done(|_| panic!("the wait began over three days ago"));
283 assert!(restarted.store.stored_redaction_compacted().unwrap());
284 assert!(!restarted.store.is_encrypted());
285 }
286
287 #[test]
288 fn a_store_encrypted_before_the_compaction_looked_is_still_vacuumed() {
289 use std::os::unix::fs::MetadataExt;
290
291 let dir = tempfile::tempdir().unwrap();
292 let db = dir.path().join("brain.db");
293 let mut c = compaction(dir.path());
294 c.awaits_encryption = true;
295 let key = StoreKey::from_bytes(KeyGuard::new([5; 32]));
296 c.store.rewrite_under_key(RewriteTo::Key(&key)).unwrap();
297 let migrated = std::fs::metadata(&db).unwrap().ino();
298 c.run_until_done(|_| panic!("nothing to wait for"));
299 assert!(c.store.stored_redaction_compacted().unwrap());
300 assert_ne!(std::fs::metadata(&db).unwrap().ino(), migrated);
301 }
302
303 #[test]
304 fn without_indexes_it_vacuums_but_leaves_the_compaction_unrecorded() {
305 let dir = tempfile::tempdir().unwrap();
306 let mut c = compaction(dir.path());
307 c.indexes = None;
308 c.run_until_done(|_| panic!("retrying cannot bring the indexes back"));
309 assert!(!c.store.stored_redaction_compacted().unwrap());
310 let report = c.health.report();
311 assert!(report.maintenance.last_error.is_some());
312 assert!(!c.gate.is_closed());
313 }
314
315 #[test]
316 fn a_failed_attempt_waits_for_its_retry_as_pending_not_merging() {
317 let dir = tempfile::tempdir().unwrap();
318 let mut c = compaction(dir.path());
319 let (tx, requests) = std::sync::mpsc::channel();
320 std::thread::spawn(move || {
321 for (n, request) in requests.into_iter().enumerate() {
322 if let DrainRequest::CompactIndexes(reply) = request {
323 let outcome = if n == 0 {
324 Err(anyhow!("merge failed"))
325 } else {
326 Ok(())
327 };
328 let _ = reply.send(outcome);
329 }
330 }
331 });
332 c.indexes = Some(tx);
333 let health = c.health.clone();
334 let mut states = Vec::new();
335 c.run_until_done(|_| states.push(health.report().maintenance.state));
336 assert_eq!(states, [MaintenanceState::Pending]);
337 assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
338 }
339
340 #[test]
341 fn without_room_it_reports_and_waits_instead_of_failing() {
342 let dir = tempfile::tempdir().unwrap();
343 let mut c = compaction(dir.path());
344 c.space = short_of_space_once;
345 let health = c.health.clone();
346 let store = c.store.clone();
347 let mut waits = Vec::new();
348 c.run_until_done(|d| {
349 let report = health.report();
350 assert_eq!(report.maintenance.state, MaintenanceState::WaitingForSpace);
351 assert!(
352 report
353 .reason
354 .as_deref()
355 .is_some_and(|r| r.contains("waits for disk space")),
356 "{report:?}"
357 );
358 assert!(!store.stored_redaction_compacted().unwrap());
359 waits.push(d);
360 });
361 assert_eq!(waits, [SPACE_RETRY]);
362 assert!(c.store.stored_redaction_compacted().unwrap());
363 assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
364 assert!(c.health.report().reason.is_none());
365 }
366}