Skip to main content

second_brain_api/
store_compaction.rs

1//! The one-time compaction after the stored-content redaction pass. The pass
2//! rewrites rows, but SQLite keeps their old bytes in free and split pages
3//! and tantivy keeps deleted terms until a segment merges away; a VACUUM and
4//! a forced merge are what take that plaintext off the disk.
5
6use std::sync::Arc;
7use std::sync::atomic::{AtomicBool, Ordering};
8use std::sync::mpsc::Sender;
9use std::time::Duration;
10
11use anyhow::{Result, anyhow};
12use chrono::Utc;
13use second_brain_core::sqlite_store::encrypt_migration::NotReopened;
14use second_brain_core::sqlite_store::{SqliteStore, VacuumSpace};
15
16use crate::health::{Health, MaintenanceState};
17use crate::maintenance::DrainRequest;
18use crate::write_gate::WriteGate;
19
20pub const SPACE_RETRY: Duration = Duration::from_secs(30 * 60);
21/// How long the compaction waits for a pending encryption migration, across
22/// restarts, before it vacuums the plaintext store itself; past a failed
23/// migration's daily retries.
24const AWAIT_ENCRYPTION_MICROS: i64 = 3 * 24 * 3600 * 1_000_000;
25const FIRST_RETRY: Duration = Duration::from_secs(30);
26const MAX_RETRY: Duration = Duration::from_secs(3600);
27
28pub struct Compaction {
29    pub store: Arc<SqliteStore>,
30    pub gate: Arc<WriteGate>,
31    /// The drain thread, which owns the writable indexes; None when the
32    /// indexes never opened.
33    pub indexes: Option<Sender<DrainRequest>>,
34    pub health: Arc<Health>,
35    pub space: fn(&SqliteStore) -> Result<VacuumSpace>,
36    /// The store waits for the one-time encryption migration, whose rewrite
37    /// copies only live pages and so does this compaction's VACUUM; the
38    /// compaction waits for it instead of rewriting the store twice.
39    pub awaits_encryption: bool,
40    /// Set once the store was seen plaintext after the redaction pass, so a
41    /// migration seen afterwards ran on the redacted rows.
42    saw_plaintext: AtomicBool,
43}
44
45#[derive(Debug, PartialEq, Eq)]
46enum Step {
47    Done,
48    NoSpace,
49    AwaitingEncryption,
50    /// Vacuumed, but with no drain thread the lexical index could not be
51    /// merged, so the compaction stays unrecorded and runs at the next start.
52    IndexesUnavailable,
53}
54
55impl Compaction {
56    pub fn new(
57        store: Arc<SqliteStore>,
58        gate: Arc<WriteGate>,
59        indexes: Option<Sender<DrainRequest>>,
60        health: Arc<Health>,
61    ) -> Self {
62        Self {
63            store,
64            gate,
65            indexes,
66            health,
67            space: SqliteStore::vacuum_space,
68            awaits_encryption: false,
69            saw_plaintext: AtomicBool::new(false),
70        }
71    }
72
73    /// Returns once the store records the compaction as done, which may be
74    /// at once; a disk without room is checked again every `SPACE_RETRY`
75    /// and a failure is retried with capped backoff.
76    pub fn run_until_done(&self, mut sleep: impl FnMut(Duration)) {
77        let mut delay = FIRST_RETRY;
78        loop {
79            match self.attempt() {
80                Ok(Step::Done | Step::IndexesUnavailable) => return,
81                Ok(Step::NoSpace | Step::AwaitingEncryption) => sleep(SPACE_RETRY),
82                Err(e) => {
83                    tracing::error!(
84                        "store compaction failed, retrying in {}s: {e:#}",
85                        delay.as_secs()
86                    );
87                    let now = Utc::now();
88                    self.health
89                        .maintenance(MaintenanceState::Pending, None, now);
90                    self.health.maintenance_failed(&format!("{e:#}"), now);
91                    sleep(delay);
92                    delay = (delay * 2).min(MAX_RETRY);
93                }
94            }
95        }
96    }
97
98    fn attempt(&self) -> Result<Step> {
99        if self.store.stored_redaction_compacted()? {
100            self.health
101                .maintenance(MaintenanceState::Done, None, Utc::now());
102            return Ok(Step::Done);
103        }
104        if self.awaits_encryption {
105            let encrypted = self.store.is_encrypted();
106            if !encrypted {
107                let now = Utc::now().timestamp_micros();
108                let since = self.store.encryption_awaited_since(now)?;
109                if now - since < AWAIT_ENCRYPTION_MICROS {
110                    self.saw_plaintext.store(true, Ordering::SeqCst);
111                    return Ok(Step::AwaitingEncryption);
112                }
113            }
114            if encrypted && self.saw_plaintext.load(Ordering::SeqCst) {
115                return self.merge_indexes_after_migration();
116            }
117        }
118        let space = (self.space)(&self.store)?;
119        if !space.fits() {
120            tracing::warn!(
121                "store compaction waits for disk space: needs {} bytes free at {}, has {}",
122                space.needed_bytes,
123                space.path.display(),
124                space.free_bytes
125            );
126            self.health
127                .maintenance(MaintenanceState::WaitingForSpace, Some(space), Utc::now());
128            return Ok(Step::NoSpace);
129        }
130        self.merge_indexes(Some(space))?;
131        let (before, after) = vacuum(&self.store, &self.gate, &self.health)?;
132        let step = self.record()?;
133        if step == Step::Done {
134            tracing::info!("store compacted after redaction: {before} bytes before, {after} after");
135        }
136        Ok(step)
137    }
138
139    fn merge_indexes_after_migration(&self) -> Result<Step> {
140        self.merge_indexes(None)?;
141        let step = self.record()?;
142        if step == Step::Done {
143            tracing::info!("store compacted after redaction by the encryption migration");
144        }
145        Ok(step)
146    }
147
148    fn merge_indexes(&self, space: Option<VacuumSpace>) -> Result<()> {
149        let Some(drain) = &self.indexes else {
150            return Ok(());
151        };
152        self.health
153            .maintenance(MaintenanceState::MergingIndex, space, Utc::now());
154        let (reply, done) = std::sync::mpsc::channel();
155        drain
156            .send(DrainRequest::CompactIndexes(reply))
157            .map_err(|_| anyhow!("the index drain thread is not running"))?;
158        done.recv()
159            .map_err(|_| anyhow!("the index drain thread stopped"))?
160    }
161
162    fn record(&self) -> Result<Step> {
163        if self.indexes.is_none() {
164            self.health.maintenance_failed(
165                "the search indexes are unavailable, so their old segments were not merged; \
166                 compaction runs again at the next start",
167                Utc::now(),
168            );
169            return Ok(Step::IndexesUnavailable);
170        }
171        self.store.mark_stored_redaction_compacted()?;
172        self.health
173            .maintenance(MaintenanceState::Done, None, Utc::now());
174        Ok(Step::Done)
175    }
176}
177
178/// Runs a VACUUM with agent writes held at the gate, reporting it to health
179/// for its length.
180pub fn vacuum(store: &SqliteStore, gate: &WriteGate, health: &Health) -> Result<(u64, u64)> {
181    let _closed = gate.close();
182    let previous = health.report().maintenance.state;
183    health.maintenance(MaintenanceState::Vacuuming, None, Utc::now());
184    let result = store.vacuum_clearing_wal();
185    health.maintenance(previous, None, Utc::now());
186    if let Err(e) = &result
187        && e.downcast_ref::<NotReopened>().is_some()
188    {
189        tracing::error!("{e:#}");
190        std::process::exit(1);
191    }
192    result
193}
194
195#[cfg(test)]
196mod tests {
197    use second_brain_core::keys::{KeyGuard, StoreKey};
198    use second_brain_core::sqlite_store::encrypt_migration::RewriteTo;
199
200    use super::*;
201
202    /// Answers index compaction requests the way the drain thread does when
203    /// the merge succeeds.
204    fn drain_stub() -> Sender<DrainRequest> {
205        let (tx, requests) = std::sync::mpsc::channel();
206        std::thread::spawn(move || {
207            for request in requests {
208                if let DrainRequest::CompactIndexes(reply) = request {
209                    let _ = reply.send(Ok(()));
210                }
211            }
212        });
213        tx
214    }
215
216    fn compaction(dir: &std::path::Path) -> Compaction {
217        let store = SqliteStore::open(&dir.join("brain.db"), "m".to_string()).unwrap();
218        Compaction::new(
219            Arc::new(store),
220            Arc::new(WriteGate::default()),
221            Some(drain_stub()),
222            Arc::new(Health::new(Utc::now(), None)),
223        )
224    }
225
226    static SPACE_CHECKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
227
228    fn short_of_space_once(store: &SqliteStore) -> Result<VacuumSpace> {
229        if SPACE_CHECKS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) > 0 {
230            return store.vacuum_space();
231        }
232        Ok(VacuumSpace {
233            path: std::path::PathBuf::from("/data"),
234            needed_bytes: 2_000_000,
235            free_bytes: 1_000_000,
236        })
237    }
238
239    #[test]
240    fn a_compaction_runs_once_and_is_recorded() {
241        let dir = tempfile::tempdir().unwrap();
242        let c = compaction(dir.path());
243        c.run_until_done(|_| panic!("nothing to wait for"));
244        assert!(c.store.stored_redaction_compacted().unwrap());
245        assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
246        assert!(!c.gate.is_closed());
247    }
248
249    #[test]
250    fn a_store_awaiting_encryption_waits_for_the_migration_and_skips_its_own_vacuum() {
251        use std::os::unix::fs::MetadataExt;
252
253        let dir = tempfile::tempdir().unwrap();
254        let db = dir.path().join("brain.db");
255        let mut c = compaction(dir.path());
256        c.awaits_encryption = true;
257        assert_eq!(c.attempt().unwrap(), Step::AwaitingEncryption);
258        assert!(!c.store.stored_redaction_compacted().unwrap());
259
260        let key = StoreKey::from_bytes(KeyGuard::new([4; 32]));
261        c.store.rewrite_under_key(RewriteTo::Key(&key)).unwrap();
262        let migrated = std::fs::metadata(&db).unwrap().ino();
263        c.run_until_done(|_| panic!("the migration has run"));
264        assert!(c.store.stored_redaction_compacted().unwrap());
265        assert_eq!(
266            std::fs::metadata(&db).unwrap().ino(),
267            migrated,
268            "the store was not rewritten a second time"
269        );
270    }
271
272    #[test]
273    fn a_migration_that_never_lands_gives_way_to_the_vacuum_across_restarts() {
274        let dir = tempfile::tempdir().unwrap();
275        let c = compaction(dir.path());
276        let started = Utc::now().timestamp_micros() - AWAIT_ENCRYPTION_MICROS - 1;
277        c.store.encryption_awaited_since(started).unwrap();
278        drop(c);
279
280        let mut restarted = compaction(dir.path());
281        restarted.awaits_encryption = true;
282        restarted.run_until_done(|_| panic!("the wait began over three days ago"));
283        assert!(restarted.store.stored_redaction_compacted().unwrap());
284        assert!(!restarted.store.is_encrypted());
285    }
286
287    #[test]
288    fn a_store_encrypted_before_the_compaction_looked_is_still_vacuumed() {
289        use std::os::unix::fs::MetadataExt;
290
291        let dir = tempfile::tempdir().unwrap();
292        let db = dir.path().join("brain.db");
293        let mut c = compaction(dir.path());
294        c.awaits_encryption = true;
295        let key = StoreKey::from_bytes(KeyGuard::new([5; 32]));
296        c.store.rewrite_under_key(RewriteTo::Key(&key)).unwrap();
297        let migrated = std::fs::metadata(&db).unwrap().ino();
298        c.run_until_done(|_| panic!("nothing to wait for"));
299        assert!(c.store.stored_redaction_compacted().unwrap());
300        assert_ne!(std::fs::metadata(&db).unwrap().ino(), migrated);
301    }
302
303    #[test]
304    fn without_indexes_it_vacuums_but_leaves_the_compaction_unrecorded() {
305        let dir = tempfile::tempdir().unwrap();
306        let mut c = compaction(dir.path());
307        c.indexes = None;
308        c.run_until_done(|_| panic!("retrying cannot bring the indexes back"));
309        assert!(!c.store.stored_redaction_compacted().unwrap());
310        let report = c.health.report();
311        assert!(report.maintenance.last_error.is_some());
312        assert!(!c.gate.is_closed());
313    }
314
315    #[test]
316    fn a_failed_attempt_waits_for_its_retry_as_pending_not_merging() {
317        let dir = tempfile::tempdir().unwrap();
318        let mut c = compaction(dir.path());
319        let (tx, requests) = std::sync::mpsc::channel();
320        std::thread::spawn(move || {
321            for (n, request) in requests.into_iter().enumerate() {
322                if let DrainRequest::CompactIndexes(reply) = request {
323                    let outcome = if n == 0 {
324                        Err(anyhow!("merge failed"))
325                    } else {
326                        Ok(())
327                    };
328                    let _ = reply.send(outcome);
329                }
330            }
331        });
332        c.indexes = Some(tx);
333        let health = c.health.clone();
334        let mut states = Vec::new();
335        c.run_until_done(|_| states.push(health.report().maintenance.state));
336        assert_eq!(states, [MaintenanceState::Pending]);
337        assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
338    }
339
340    #[test]
341    fn without_room_it_reports_and_waits_instead_of_failing() {
342        let dir = tempfile::tempdir().unwrap();
343        let mut c = compaction(dir.path());
344        c.space = short_of_space_once;
345        let health = c.health.clone();
346        let store = c.store.clone();
347        let mut waits = Vec::new();
348        c.run_until_done(|d| {
349            let report = health.report();
350            assert_eq!(report.maintenance.state, MaintenanceState::WaitingForSpace);
351            assert!(
352                report
353                    .reason
354                    .as_deref()
355                    .is_some_and(|r| r.contains("waits for disk space")),
356                "{report:?}"
357            );
358            assert!(!store.stored_redaction_compacted().unwrap());
359            waits.push(d);
360        });
361        assert_eq!(waits, [SPACE_RETRY]);
362        assert!(c.store.stored_redaction_compacted().unwrap());
363        assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
364        assert!(c.health.report().reason.is_none());
365    }
366}