second_brain_api/
store_compaction.rs1use std::sync::Arc;
7use std::sync::mpsc::Sender;
8use std::time::Duration;
9
10use anyhow::{Result, anyhow};
11use chrono::Utc;
12use second_brain_core::sqlite_store::{SqliteStore, VacuumSpace};
13
14use crate::health::{Health, MaintenanceState};
15use crate::maintenance::DrainRequest;
16use crate::write_gate::WriteGate;
17
18pub const SPACE_RETRY: Duration = Duration::from_secs(30 * 60);
19const FIRST_RETRY: Duration = Duration::from_secs(30);
20const MAX_RETRY: Duration = Duration::from_secs(3600);
21
22pub struct Compaction {
23 pub store: Arc<SqliteStore>,
24 pub gate: Arc<WriteGate>,
25 pub indexes: Option<Sender<DrainRequest>>,
28 pub health: Arc<Health>,
29 pub space: fn(&SqliteStore) -> Result<VacuumSpace>,
30}
31
32#[derive(Debug, PartialEq, Eq)]
33enum Step {
34 Done,
35 NoSpace,
36 IndexesUnavailable,
39}
40
41impl Compaction {
42 pub fn new(
43 store: Arc<SqliteStore>,
44 gate: Arc<WriteGate>,
45 indexes: Option<Sender<DrainRequest>>,
46 health: Arc<Health>,
47 ) -> Self {
48 Self {
49 store,
50 gate,
51 indexes,
52 health,
53 space: SqliteStore::vacuum_space,
54 }
55 }
56
57 pub fn run_until_done(&self, mut sleep: impl FnMut(Duration)) {
61 let mut delay = FIRST_RETRY;
62 loop {
63 match self.attempt() {
64 Ok(Step::Done | Step::IndexesUnavailable) => return,
65 Ok(Step::NoSpace) => sleep(SPACE_RETRY),
66 Err(e) => {
67 tracing::error!(
68 "store compaction failed, retrying in {}s: {e:#}",
69 delay.as_secs()
70 );
71 self.health
72 .maintenance_failed(&format!("{e:#}"), Utc::now());
73 sleep(delay);
74 delay = (delay * 2).min(MAX_RETRY);
75 }
76 }
77 }
78 }
79
80 fn attempt(&self) -> Result<Step> {
81 if self.store.stored_redaction_compacted()? {
82 self.health
83 .maintenance(MaintenanceState::Done, None, Utc::now());
84 return Ok(Step::Done);
85 }
86 let space = (self.space)(&self.store)?;
87 if !space.fits() {
88 tracing::warn!(
89 "store compaction waits for disk space: needs {} bytes free at {}, has {}",
90 space.needed_bytes,
91 space.path.display(),
92 space.free_bytes
93 );
94 self.health
95 .maintenance(MaintenanceState::WaitingForSpace, Some(space), Utc::now());
96 return Ok(Step::NoSpace);
97 }
98 if let Some(drain) = &self.indexes {
99 self.health
100 .maintenance(MaintenanceState::MergingIndex, Some(space), Utc::now());
101 let (reply, done) = std::sync::mpsc::channel();
102 drain
103 .send(DrainRequest::CompactIndexes(reply))
104 .map_err(|_| anyhow!("the index drain thread is not running"))?;
105 done.recv()
106 .map_err(|_| anyhow!("the index drain thread stopped"))??;
107 }
108 let (before, after) = vacuum(&self.store, &self.gate, &self.health)?;
109 if self.indexes.is_none() {
110 self.health.maintenance_failed(
111 "the search indexes are unavailable, so their old segments were not merged; \
112 compaction runs again at the next start",
113 Utc::now(),
114 );
115 return Ok(Step::IndexesUnavailable);
116 }
117 self.store.mark_stored_redaction_compacted()?;
118 self.health
119 .maintenance(MaintenanceState::Done, None, Utc::now());
120 tracing::info!("store compacted after redaction: {before} bytes before, {after} after");
121 Ok(Step::Done)
122 }
123}
124
125pub fn vacuum(store: &SqliteStore, gate: &WriteGate, health: &Health) -> Result<(u64, u64)> {
128 let _closed = gate.close();
129 let previous = health.report().maintenance.state;
130 health.maintenance(MaintenanceState::Vacuuming, None, Utc::now());
131 let result = store.vacuum_clearing_wal();
132 health.maintenance(previous, None, Utc::now());
133 result
134}
135
136#[cfg(test)]
137mod tests {
138 use super::*;
139
140 fn drain_stub() -> Sender<DrainRequest> {
143 let (tx, requests) = std::sync::mpsc::channel();
144 std::thread::spawn(move || {
145 for request in requests {
146 if let DrainRequest::CompactIndexes(reply) = request {
147 let _ = reply.send(Ok(()));
148 }
149 }
150 });
151 tx
152 }
153
154 fn compaction(dir: &std::path::Path) -> Compaction {
155 let store = SqliteStore::open(&dir.join("brain.db"), "m".to_string()).unwrap();
156 Compaction::new(
157 Arc::new(store),
158 Arc::new(WriteGate::default()),
159 Some(drain_stub()),
160 Arc::new(Health::new(Utc::now(), None)),
161 )
162 }
163
164 static SPACE_CHECKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
165
166 fn short_of_space_once(store: &SqliteStore) -> Result<VacuumSpace> {
167 if SPACE_CHECKS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) > 0 {
168 return store.vacuum_space();
169 }
170 Ok(VacuumSpace {
171 path: std::path::PathBuf::from("/data"),
172 needed_bytes: 2_000_000,
173 free_bytes: 1_000_000,
174 })
175 }
176
177 #[test]
178 fn a_compaction_runs_once_and_is_recorded() {
179 let dir = tempfile::tempdir().unwrap();
180 let c = compaction(dir.path());
181 c.run_until_done(|_| panic!("nothing to wait for"));
182 assert!(c.store.stored_redaction_compacted().unwrap());
183 assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
184 assert!(!c.gate.is_closed());
185 }
186
187 #[test]
188 fn without_indexes_it_vacuums_but_leaves_the_compaction_unrecorded() {
189 let dir = tempfile::tempdir().unwrap();
190 let mut c = compaction(dir.path());
191 c.indexes = None;
192 c.run_until_done(|_| panic!("retrying cannot bring the indexes back"));
193 assert!(!c.store.stored_redaction_compacted().unwrap());
194 let report = c.health.report();
195 assert!(report.maintenance.last_error.is_some());
196 assert!(!c.gate.is_closed());
197 }
198
199 #[test]
200 fn without_room_it_reports_and_waits_instead_of_failing() {
201 let dir = tempfile::tempdir().unwrap();
202 let mut c = compaction(dir.path());
203 c.space = short_of_space_once;
204 let health = c.health.clone();
205 let store = c.store.clone();
206 let mut waits = Vec::new();
207 c.run_until_done(|d| {
208 let report = health.report();
209 assert_eq!(report.maintenance.state, MaintenanceState::WaitingForSpace);
210 assert!(
211 report
212 .reason
213 .as_deref()
214 .is_some_and(|r| r.contains("waits for disk space")),
215 "{report:?}"
216 );
217 assert!(!store.stored_redaction_compacted().unwrap());
218 waits.push(d);
219 });
220 assert_eq!(waits, [SPACE_RETRY]);
221 assert!(c.store.stored_redaction_compacted().unwrap());
222 assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
223 assert!(c.health.report().reason.is_none());
224 }
225}