second_brain_api/
store_compaction.rs1use std::sync::Arc;
7use std::sync::mpsc::Sender;
8use std::time::Duration;
9
10use anyhow::{Result, anyhow};
11use chrono::Utc;
12use second_brain_core::sqlite_store::{SqliteStore, VacuumSpace};
13
14use crate::health::{Health, MaintenanceState};
15use crate::maintenance::DrainRequest;
16use crate::write_gate::WriteGate;
17
18pub const SPACE_RETRY: Duration = Duration::from_secs(30 * 60);
19const FIRST_RETRY: Duration = Duration::from_secs(30);
20const MAX_RETRY: Duration = Duration::from_secs(3600);
21
22pub struct Compaction {
23 pub store: Arc<SqliteStore>,
24 pub gate: Arc<WriteGate>,
25 pub indexes: Option<Sender<DrainRequest>>,
28 pub health: Arc<Health>,
29 pub space: fn(&SqliteStore) -> Result<VacuumSpace>,
30}
31
32#[derive(Debug, PartialEq, Eq)]
33enum Step {
34 Done,
35 NoSpace,
36 IndexesUnavailable,
39}
40
41impl Compaction {
42 pub fn new(
43 store: Arc<SqliteStore>,
44 gate: Arc<WriteGate>,
45 indexes: Option<Sender<DrainRequest>>,
46 health: Arc<Health>,
47 ) -> Self {
48 Self {
49 store,
50 gate,
51 indexes,
52 health,
53 space: SqliteStore::vacuum_space,
54 }
55 }
56
57 pub fn run_until_done(&self, mut sleep: impl FnMut(Duration)) {
61 let mut delay = FIRST_RETRY;
62 loop {
63 match self.attempt() {
64 Ok(Step::Done | Step::IndexesUnavailable) => return,
65 Ok(Step::NoSpace) => sleep(SPACE_RETRY),
66 Err(e) => {
67 tracing::error!(
68 "store compaction failed, retrying in {}s: {e:#}",
69 delay.as_secs()
70 );
71 let now = Utc::now();
72 self.health
73 .maintenance(MaintenanceState::Pending, None, now);
74 self.health.maintenance_failed(&format!("{e:#}"), now);
75 sleep(delay);
76 delay = (delay * 2).min(MAX_RETRY);
77 }
78 }
79 }
80 }
81
82 fn attempt(&self) -> Result<Step> {
83 if self.store.stored_redaction_compacted()? {
84 self.health
85 .maintenance(MaintenanceState::Done, None, Utc::now());
86 return Ok(Step::Done);
87 }
88 let space = (self.space)(&self.store)?;
89 if !space.fits() {
90 tracing::warn!(
91 "store compaction waits for disk space: needs {} bytes free at {}, has {}",
92 space.needed_bytes,
93 space.path.display(),
94 space.free_bytes
95 );
96 self.health
97 .maintenance(MaintenanceState::WaitingForSpace, Some(space), Utc::now());
98 return Ok(Step::NoSpace);
99 }
100 if let Some(drain) = &self.indexes {
101 self.health
102 .maintenance(MaintenanceState::MergingIndex, Some(space), Utc::now());
103 let (reply, done) = std::sync::mpsc::channel();
104 drain
105 .send(DrainRequest::CompactIndexes(reply))
106 .map_err(|_| anyhow!("the index drain thread is not running"))?;
107 done.recv()
108 .map_err(|_| anyhow!("the index drain thread stopped"))??;
109 }
110 let (before, after) = vacuum(&self.store, &self.gate, &self.health)?;
111 if self.indexes.is_none() {
112 self.health.maintenance_failed(
113 "the search indexes are unavailable, so their old segments were not merged; \
114 compaction runs again at the next start",
115 Utc::now(),
116 );
117 return Ok(Step::IndexesUnavailable);
118 }
119 self.store.mark_stored_redaction_compacted()?;
120 self.health
121 .maintenance(MaintenanceState::Done, None, Utc::now());
122 tracing::info!("store compacted after redaction: {before} bytes before, {after} after");
123 Ok(Step::Done)
124 }
125}
126
127pub fn vacuum(store: &SqliteStore, gate: &WriteGate, health: &Health) -> Result<(u64, u64)> {
130 let _closed = gate.close();
131 let previous = health.report().maintenance.state;
132 health.maintenance(MaintenanceState::Vacuuming, None, Utc::now());
133 let result = store.vacuum_clearing_wal();
134 health.maintenance(previous, None, Utc::now());
135 result
136}
137
138#[cfg(test)]
139mod tests {
140 use super::*;
141
142 fn drain_stub() -> Sender<DrainRequest> {
145 let (tx, requests) = std::sync::mpsc::channel();
146 std::thread::spawn(move || {
147 for request in requests {
148 if let DrainRequest::CompactIndexes(reply) = request {
149 let _ = reply.send(Ok(()));
150 }
151 }
152 });
153 tx
154 }
155
156 fn compaction(dir: &std::path::Path) -> Compaction {
157 let store = SqliteStore::open(&dir.join("brain.db"), "m".to_string()).unwrap();
158 Compaction::new(
159 Arc::new(store),
160 Arc::new(WriteGate::default()),
161 Some(drain_stub()),
162 Arc::new(Health::new(Utc::now(), None)),
163 )
164 }
165
166 static SPACE_CHECKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
167
168 fn short_of_space_once(store: &SqliteStore) -> Result<VacuumSpace> {
169 if SPACE_CHECKS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) > 0 {
170 return store.vacuum_space();
171 }
172 Ok(VacuumSpace {
173 path: std::path::PathBuf::from("/data"),
174 needed_bytes: 2_000_000,
175 free_bytes: 1_000_000,
176 })
177 }
178
179 #[test]
180 fn a_compaction_runs_once_and_is_recorded() {
181 let dir = tempfile::tempdir().unwrap();
182 let c = compaction(dir.path());
183 c.run_until_done(|_| panic!("nothing to wait for"));
184 assert!(c.store.stored_redaction_compacted().unwrap());
185 assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
186 assert!(!c.gate.is_closed());
187 }
188
189 #[test]
190 fn without_indexes_it_vacuums_but_leaves_the_compaction_unrecorded() {
191 let dir = tempfile::tempdir().unwrap();
192 let mut c = compaction(dir.path());
193 c.indexes = None;
194 c.run_until_done(|_| panic!("retrying cannot bring the indexes back"));
195 assert!(!c.store.stored_redaction_compacted().unwrap());
196 let report = c.health.report();
197 assert!(report.maintenance.last_error.is_some());
198 assert!(!c.gate.is_closed());
199 }
200
201 #[test]
202 fn a_failed_attempt_waits_for_its_retry_as_pending_not_merging() {
203 let dir = tempfile::tempdir().unwrap();
204 let mut c = compaction(dir.path());
205 let (tx, requests) = std::sync::mpsc::channel();
206 std::thread::spawn(move || {
207 for (n, request) in requests.into_iter().enumerate() {
208 if let DrainRequest::CompactIndexes(reply) = request {
209 let outcome = if n == 0 {
210 Err(anyhow!("merge failed"))
211 } else {
212 Ok(())
213 };
214 let _ = reply.send(outcome);
215 }
216 }
217 });
218 c.indexes = Some(tx);
219 let health = c.health.clone();
220 let mut states = Vec::new();
221 c.run_until_done(|_| states.push(health.report().maintenance.state));
222 assert_eq!(states, [MaintenanceState::Pending]);
223 assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
224 }
225
226 #[test]
227 fn without_room_it_reports_and_waits_instead_of_failing() {
228 let dir = tempfile::tempdir().unwrap();
229 let mut c = compaction(dir.path());
230 c.space = short_of_space_once;
231 let health = c.health.clone();
232 let store = c.store.clone();
233 let mut waits = Vec::new();
234 c.run_until_done(|d| {
235 let report = health.report();
236 assert_eq!(report.maintenance.state, MaintenanceState::WaitingForSpace);
237 assert!(
238 report
239 .reason
240 .as_deref()
241 .is_some_and(|r| r.contains("waits for disk space")),
242 "{report:?}"
243 );
244 assert!(!store.stored_redaction_compacted().unwrap());
245 waits.push(d);
246 });
247 assert_eq!(waits, [SPACE_RETRY]);
248 assert!(c.store.stored_redaction_compacted().unwrap());
249 assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
250 assert!(c.health.report().reason.is_none());
251 }
252}