Skip to main content

second_brain_api/
store_compaction.rs

1//! The one-time compaction after the stored-content redaction pass. The pass
2//! rewrites rows, but SQLite keeps their old bytes in free and split pages
3//! and tantivy keeps deleted terms until a segment merges away; a VACUUM and
4//! a forced merge are what take that plaintext off the disk.
5
6use std::sync::Arc;
7use std::sync::mpsc::Sender;
8use std::time::Duration;
9
10use anyhow::{Result, anyhow};
11use chrono::Utc;
12use second_brain_core::sqlite_store::{SqliteStore, VacuumSpace};
13
14use crate::health::{Health, MaintenanceState};
15use crate::maintenance::DrainRequest;
16use crate::write_gate::WriteGate;
17
18pub const SPACE_RETRY: Duration = Duration::from_secs(30 * 60);
19const FIRST_RETRY: Duration = Duration::from_secs(30);
20const MAX_RETRY: Duration = Duration::from_secs(3600);
21
22pub struct Compaction {
23    pub store: Arc<SqliteStore>,
24    pub gate: Arc<WriteGate>,
25    /// The drain thread, which owns the writable indexes; None when the
26    /// indexes never opened.
27    pub indexes: Option<Sender<DrainRequest>>,
28    pub health: Arc<Health>,
29    pub space: fn(&SqliteStore) -> Result<VacuumSpace>,
30}
31
32#[derive(Debug, PartialEq, Eq)]
33enum Step {
34    Done,
35    NoSpace,
36    /// Vacuumed, but with no drain thread the lexical index could not be
37    /// merged, so the compaction stays unrecorded and runs at the next start.
38    IndexesUnavailable,
39}
40
41impl Compaction {
42    pub fn new(
43        store: Arc<SqliteStore>,
44        gate: Arc<WriteGate>,
45        indexes: Option<Sender<DrainRequest>>,
46        health: Arc<Health>,
47    ) -> Self {
48        Self {
49            store,
50            gate,
51            indexes,
52            health,
53            space: SqliteStore::vacuum_space,
54        }
55    }
56
57    /// Returns once the store records the compaction as done, which may be
58    /// at once; a disk without room is checked again every `SPACE_RETRY`
59    /// and a failure is retried with capped backoff.
60    pub fn run_until_done(&self, mut sleep: impl FnMut(Duration)) {
61        let mut delay = FIRST_RETRY;
62        loop {
63            match self.attempt() {
64                Ok(Step::Done | Step::IndexesUnavailable) => return,
65                Ok(Step::NoSpace) => sleep(SPACE_RETRY),
66                Err(e) => {
67                    tracing::error!(
68                        "store compaction failed, retrying in {}s: {e:#}",
69                        delay.as_secs()
70                    );
71                    let now = Utc::now();
72                    self.health
73                        .maintenance(MaintenanceState::Pending, None, now);
74                    self.health.maintenance_failed(&format!("{e:#}"), now);
75                    sleep(delay);
76                    delay = (delay * 2).min(MAX_RETRY);
77                }
78            }
79        }
80    }
81
82    fn attempt(&self) -> Result<Step> {
83        if self.store.stored_redaction_compacted()? {
84            self.health
85                .maintenance(MaintenanceState::Done, None, Utc::now());
86            return Ok(Step::Done);
87        }
88        let space = (self.space)(&self.store)?;
89        if !space.fits() {
90            tracing::warn!(
91                "store compaction waits for disk space: needs {} bytes free at {}, has {}",
92                space.needed_bytes,
93                space.path.display(),
94                space.free_bytes
95            );
96            self.health
97                .maintenance(MaintenanceState::WaitingForSpace, Some(space), Utc::now());
98            return Ok(Step::NoSpace);
99        }
100        if let Some(drain) = &self.indexes {
101            self.health
102                .maintenance(MaintenanceState::MergingIndex, Some(space), Utc::now());
103            let (reply, done) = std::sync::mpsc::channel();
104            drain
105                .send(DrainRequest::CompactIndexes(reply))
106                .map_err(|_| anyhow!("the index drain thread is not running"))?;
107            done.recv()
108                .map_err(|_| anyhow!("the index drain thread stopped"))??;
109        }
110        let (before, after) = vacuum(&self.store, &self.gate, &self.health)?;
111        if self.indexes.is_none() {
112            self.health.maintenance_failed(
113                "the search indexes are unavailable, so their old segments were not merged; \
114                 compaction runs again at the next start",
115                Utc::now(),
116            );
117            return Ok(Step::IndexesUnavailable);
118        }
119        self.store.mark_stored_redaction_compacted()?;
120        self.health
121            .maintenance(MaintenanceState::Done, None, Utc::now());
122        tracing::info!("store compacted after redaction: {before} bytes before, {after} after");
123        Ok(Step::Done)
124    }
125}
126
127/// Runs a VACUUM with agent writes held at the gate, reporting it to health
128/// for its length.
129pub fn vacuum(store: &SqliteStore, gate: &WriteGate, health: &Health) -> Result<(u64, u64)> {
130    let _closed = gate.close();
131    let previous = health.report().maintenance.state;
132    health.maintenance(MaintenanceState::Vacuuming, None, Utc::now());
133    let result = store.vacuum_clearing_wal();
134    health.maintenance(previous, None, Utc::now());
135    result
136}
137
138#[cfg(test)]
139mod tests {
140    use super::*;
141
142    /// Answers index compaction requests the way the drain thread does when
143    /// the merge succeeds.
144    fn drain_stub() -> Sender<DrainRequest> {
145        let (tx, requests) = std::sync::mpsc::channel();
146        std::thread::spawn(move || {
147            for request in requests {
148                if let DrainRequest::CompactIndexes(reply) = request {
149                    let _ = reply.send(Ok(()));
150                }
151            }
152        });
153        tx
154    }
155
156    fn compaction(dir: &std::path::Path) -> Compaction {
157        let store = SqliteStore::open(&dir.join("brain.db"), "m".to_string()).unwrap();
158        Compaction::new(
159            Arc::new(store),
160            Arc::new(WriteGate::default()),
161            Some(drain_stub()),
162            Arc::new(Health::new(Utc::now(), None)),
163        )
164    }
165
166    static SPACE_CHECKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
167
168    fn short_of_space_once(store: &SqliteStore) -> Result<VacuumSpace> {
169        if SPACE_CHECKS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) > 0 {
170            return store.vacuum_space();
171        }
172        Ok(VacuumSpace {
173            path: std::path::PathBuf::from("/data"),
174            needed_bytes: 2_000_000,
175            free_bytes: 1_000_000,
176        })
177    }
178
179    #[test]
180    fn a_compaction_runs_once_and_is_recorded() {
181        let dir = tempfile::tempdir().unwrap();
182        let c = compaction(dir.path());
183        c.run_until_done(|_| panic!("nothing to wait for"));
184        assert!(c.store.stored_redaction_compacted().unwrap());
185        assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
186        assert!(!c.gate.is_closed());
187    }
188
189    #[test]
190    fn without_indexes_it_vacuums_but_leaves_the_compaction_unrecorded() {
191        let dir = tempfile::tempdir().unwrap();
192        let mut c = compaction(dir.path());
193        c.indexes = None;
194        c.run_until_done(|_| panic!("retrying cannot bring the indexes back"));
195        assert!(!c.store.stored_redaction_compacted().unwrap());
196        let report = c.health.report();
197        assert!(report.maintenance.last_error.is_some());
198        assert!(!c.gate.is_closed());
199    }
200
201    #[test]
202    fn a_failed_attempt_waits_for_its_retry_as_pending_not_merging() {
203        let dir = tempfile::tempdir().unwrap();
204        let mut c = compaction(dir.path());
205        let (tx, requests) = std::sync::mpsc::channel();
206        std::thread::spawn(move || {
207            for (n, request) in requests.into_iter().enumerate() {
208                if let DrainRequest::CompactIndexes(reply) = request {
209                    let outcome = if n == 0 {
210                        Err(anyhow!("merge failed"))
211                    } else {
212                        Ok(())
213                    };
214                    let _ = reply.send(outcome);
215                }
216            }
217        });
218        c.indexes = Some(tx);
219        let health = c.health.clone();
220        let mut states = Vec::new();
221        c.run_until_done(|_| states.push(health.report().maintenance.state));
222        assert_eq!(states, [MaintenanceState::Pending]);
223        assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
224    }
225
226    #[test]
227    fn without_room_it_reports_and_waits_instead_of_failing() {
228        let dir = tempfile::tempdir().unwrap();
229        let mut c = compaction(dir.path());
230        c.space = short_of_space_once;
231        let health = c.health.clone();
232        let store = c.store.clone();
233        let mut waits = Vec::new();
234        c.run_until_done(|d| {
235            let report = health.report();
236            assert_eq!(report.maintenance.state, MaintenanceState::WaitingForSpace);
237            assert!(
238                report
239                    .reason
240                    .as_deref()
241                    .is_some_and(|r| r.contains("waits for disk space")),
242                "{report:?}"
243            );
244            assert!(!store.stored_redaction_compacted().unwrap());
245            waits.push(d);
246        });
247        assert_eq!(waits, [SPACE_RETRY]);
248        assert!(c.store.stored_redaction_compacted().unwrap());
249        assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
250        assert!(c.health.report().reason.is_none());
251    }
252}