Skip to main content

second_brain_api/
store_compaction.rs

1//! The one-time compaction after the stored-content redaction pass. The pass
2//! rewrites rows, but SQLite keeps their old bytes in free and split pages
3//! and tantivy keeps deleted terms until a segment merges away; a VACUUM and
4//! a forced merge are what take that plaintext off the disk.
5
6use std::sync::Arc;
7use std::sync::mpsc::Sender;
8use std::time::Duration;
9
10use anyhow::{Result, anyhow};
11use chrono::Utc;
12use second_brain_core::sqlite_store::{SqliteStore, VacuumSpace};
13
14use crate::health::{Health, MaintenanceState};
15use crate::maintenance::DrainRequest;
16use crate::write_gate::WriteGate;
17
18pub const SPACE_RETRY: Duration = Duration::from_secs(30 * 60);
19const FIRST_RETRY: Duration = Duration::from_secs(30);
20const MAX_RETRY: Duration = Duration::from_secs(3600);
21
22pub struct Compaction {
23    pub store: Arc<SqliteStore>,
24    pub gate: Arc<WriteGate>,
25    /// The drain thread, which owns the writable indexes; None when the
26    /// indexes never opened.
27    pub indexes: Option<Sender<DrainRequest>>,
28    pub health: Arc<Health>,
29    pub space: fn(&SqliteStore) -> Result<VacuumSpace>,
30}
31
32#[derive(Debug, PartialEq, Eq)]
33enum Step {
34    Done,
35    NoSpace,
36    /// Vacuumed, but with no drain thread the lexical index could not be
37    /// merged, so the compaction stays unrecorded and runs at the next start.
38    IndexesUnavailable,
39}
40
41impl Compaction {
42    pub fn new(
43        store: Arc<SqliteStore>,
44        gate: Arc<WriteGate>,
45        indexes: Option<Sender<DrainRequest>>,
46        health: Arc<Health>,
47    ) -> Self {
48        Self {
49            store,
50            gate,
51            indexes,
52            health,
53            space: SqliteStore::vacuum_space,
54        }
55    }
56
57    /// Returns once the store records the compaction as done, which may be
58    /// at once; a disk without room is checked again every `SPACE_RETRY`
59    /// and a failure is retried with capped backoff.
60    pub fn run_until_done(&self, mut sleep: impl FnMut(Duration)) {
61        let mut delay = FIRST_RETRY;
62        loop {
63            match self.attempt() {
64                Ok(Step::Done | Step::IndexesUnavailable) => return,
65                Ok(Step::NoSpace) => sleep(SPACE_RETRY),
66                Err(e) => {
67                    tracing::error!(
68                        "store compaction failed, retrying in {}s: {e:#}",
69                        delay.as_secs()
70                    );
71                    self.health
72                        .maintenance_failed(&format!("{e:#}"), Utc::now());
73                    sleep(delay);
74                    delay = (delay * 2).min(MAX_RETRY);
75                }
76            }
77        }
78    }
79
80    fn attempt(&self) -> Result<Step> {
81        if self.store.stored_redaction_compacted()? {
82            self.health
83                .maintenance(MaintenanceState::Done, None, Utc::now());
84            return Ok(Step::Done);
85        }
86        let space = (self.space)(&self.store)?;
87        if !space.fits() {
88            tracing::warn!(
89                "store compaction waits for disk space: needs {} bytes free at {}, has {}",
90                space.needed_bytes,
91                space.path.display(),
92                space.free_bytes
93            );
94            self.health
95                .maintenance(MaintenanceState::WaitingForSpace, Some(space), Utc::now());
96            return Ok(Step::NoSpace);
97        }
98        if let Some(drain) = &self.indexes {
99            self.health
100                .maintenance(MaintenanceState::MergingIndex, Some(space), Utc::now());
101            let (reply, done) = std::sync::mpsc::channel();
102            drain
103                .send(DrainRequest::CompactIndexes(reply))
104                .map_err(|_| anyhow!("the index drain thread is not running"))?;
105            done.recv()
106                .map_err(|_| anyhow!("the index drain thread stopped"))??;
107        }
108        let (before, after) = vacuum(&self.store, &self.gate, &self.health)?;
109        if self.indexes.is_none() {
110            self.health.maintenance_failed(
111                "the search indexes are unavailable, so their old segments were not merged; \
112                 compaction runs again at the next start",
113                Utc::now(),
114            );
115            return Ok(Step::IndexesUnavailable);
116        }
117        self.store.mark_stored_redaction_compacted()?;
118        self.health
119            .maintenance(MaintenanceState::Done, None, Utc::now());
120        tracing::info!("store compacted after redaction: {before} bytes before, {after} after");
121        Ok(Step::Done)
122    }
123}
124
125/// Runs a VACUUM with agent writes held at the gate, reporting it to health
126/// for its length.
127pub fn vacuum(store: &SqliteStore, gate: &WriteGate, health: &Health) -> Result<(u64, u64)> {
128    let _closed = gate.close();
129    let previous = health.report().maintenance.state;
130    health.maintenance(MaintenanceState::Vacuuming, None, Utc::now());
131    let result = store.vacuum_clearing_wal();
132    health.maintenance(previous, None, Utc::now());
133    result
134}
135
136#[cfg(test)]
137mod tests {
138    use super::*;
139
140    /// Answers index compaction requests the way the drain thread does when
141    /// the merge succeeds.
142    fn drain_stub() -> Sender<DrainRequest> {
143        let (tx, requests) = std::sync::mpsc::channel();
144        std::thread::spawn(move || {
145            for request in requests {
146                if let DrainRequest::CompactIndexes(reply) = request {
147                    let _ = reply.send(Ok(()));
148                }
149            }
150        });
151        tx
152    }
153
154    fn compaction(dir: &std::path::Path) -> Compaction {
155        let store = SqliteStore::open(&dir.join("brain.db"), "m".to_string()).unwrap();
156        Compaction::new(
157            Arc::new(store),
158            Arc::new(WriteGate::default()),
159            Some(drain_stub()),
160            Arc::new(Health::new(Utc::now(), None)),
161        )
162    }
163
164    static SPACE_CHECKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
165
166    fn short_of_space_once(store: &SqliteStore) -> Result<VacuumSpace> {
167        if SPACE_CHECKS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) > 0 {
168            return store.vacuum_space();
169        }
170        Ok(VacuumSpace {
171            path: std::path::PathBuf::from("/data"),
172            needed_bytes: 2_000_000,
173            free_bytes: 1_000_000,
174        })
175    }
176
177    #[test]
178    fn a_compaction_runs_once_and_is_recorded() {
179        let dir = tempfile::tempdir().unwrap();
180        let c = compaction(dir.path());
181        c.run_until_done(|_| panic!("nothing to wait for"));
182        assert!(c.store.stored_redaction_compacted().unwrap());
183        assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
184        assert!(!c.gate.is_closed());
185    }
186
187    #[test]
188    fn without_indexes_it_vacuums_but_leaves_the_compaction_unrecorded() {
189        let dir = tempfile::tempdir().unwrap();
190        let mut c = compaction(dir.path());
191        c.indexes = None;
192        c.run_until_done(|_| panic!("retrying cannot bring the indexes back"));
193        assert!(!c.store.stored_redaction_compacted().unwrap());
194        let report = c.health.report();
195        assert!(report.maintenance.last_error.is_some());
196        assert!(!c.gate.is_closed());
197    }
198
199    #[test]
200    fn without_room_it_reports_and_waits_instead_of_failing() {
201        let dir = tempfile::tempdir().unwrap();
202        let mut c = compaction(dir.path());
203        c.space = short_of_space_once;
204        let health = c.health.clone();
205        let store = c.store.clone();
206        let mut waits = Vec::new();
207        c.run_until_done(|d| {
208            let report = health.report();
209            assert_eq!(report.maintenance.state, MaintenanceState::WaitingForSpace);
210            assert!(
211                report
212                    .reason
213                    .as_deref()
214                    .is_some_and(|r| r.contains("waits for disk space")),
215                "{report:?}"
216            );
217            assert!(!store.stored_redaction_compacted().unwrap());
218            waits.push(d);
219        });
220        assert_eq!(waits, [SPACE_RETRY]);
221        assert!(c.store.stored_redaction_compacted().unwrap());
222        assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
223        assert!(c.health.report().reason.is_none());
224    }
225}