second_brain_api/
maintenance.rs1use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, Instant};
9
10use anyhow::Result;
11use chrono::Utc;
12use second_brain_core::config::MaintenanceConfig;
13use second_brain_core::embedding::Embedder;
14use second_brain_core::index::Indexes;
15use second_brain_core::maintenance::{MaintenanceSummary, RepairStats};
16use second_brain_core::sqlite_store::SqliteStore;
17
18use crate::near_duplicates::{NearDupCount, refresh_count};
19
20const DRAIN_INTERVAL: Duration = Duration::from_secs(5);
21
22const CHUNK_IDLE_RESCAN: Duration = Duration::from_secs(600);
25const CHUNK_TICK_BUDGET: Duration = Duration::from_secs(3);
27const CHUNK_MEMORIES_PER_TICK: usize = 32;
29
30struct ChunkThrottle {
32 next_scan: Instant,
33}
34
35impl ChunkThrottle {
36 fn new(now: Instant) -> Self {
37 Self { next_scan: now }
38 }
39
40 fn due(&self, now: Instant) -> bool {
41 now >= self.next_scan
42 }
43
44 fn after_backfill(&mut self, now: Instant, progressed: bool) {
45 self.next_scan = if progressed {
46 now
47 } else {
48 now + CHUNK_IDLE_RESCAN
49 };
50 }
51
52 fn after_drain(&mut self, now: Instant, applied: usize) {
54 if applied > 0 {
55 self.next_scan = now;
56 }
57 }
58}
59
60pub struct RepairRequest {
61 pub dry_run: bool,
62 pub reply: Sender<Result<MaintenanceSummary>>,
63}
64
65pub fn run_repair(
69 store: &SqliteStore,
70 mut indexes: Option<&mut Indexes>,
71 embedder: Option<&Embedder>,
72 dry_run: bool,
73) -> Result<MaintenanceSummary> {
74 let embed = embedder.map(|e| move |text: &str| e.embed(text));
75 let embed_ref = embed
76 .as_ref()
77 .map(|f| f as &dyn Fn(&str) -> Result<Vec<f32>>);
78 let stats = store.repair(indexes.as_deref_mut(), embed_ref, dry_run)?;
79 let report_after = store.integrity_report(indexes.as_deref())?;
80 Ok(MaintenanceSummary {
81 ran_at: Utc::now(),
82 stats,
83 report_after,
84 })
85}
86
87fn log_stats(source: &str, stats: &RepairStats) {
88 if stats.is_clean() {
89 tracing::info!("{source} store repair: nothing to do");
90 } else {
91 tracing::info!("{source} store repair: {}", summarize(stats));
92 }
93}
94
95fn summarize(s: &RepairStats) -> String {
96 let mut parts = Vec::new();
97 let mut push = |name: &str, n: u64| {
98 if n > 0 {
99 parts.push(format!("{name}={n}"));
100 }
101 };
102 push("duplicates_superseded", s.duplicates_superseded);
103 push("markers_inherited", s.distillation_markers_inherited);
104 push("orphaned_markers_dropped", s.orphaned_markers_dropped);
105 push("raws_retired", s.raws_retired);
106 push("relations_pruned", s.relations_pruned);
107 push("vec_keys_deleted", s.vec_keys_deleted);
108 push("sync_log_rows_deleted", s.sync_log_rows_deleted);
109 push("stale_cursors_dropped", s.stale_cursors_dropped);
110 push("embeddings_backfilled", s.embeddings_backfilled);
111 push("embedding_failures", s.embedding_failures);
112 if s.vec_keys_gc_skipped {
113 parts.push("vec_keys_gc=skipped".to_string());
114 }
115 if s.index_rebuilt {
116 parts.push("index=rebuilt".to_string());
117 }
118 if s.dry_run {
119 parts.push("dry_run".to_string());
120 }
121 parts.join(" ")
122}
123
124pub struct DrainLoop {
125 pub store: Arc<SqliteStore>,
126 pub reader: Arc<RwLock<Indexes>>,
127 pub writer: Indexes,
128 pub embedder: Option<Arc<Embedder>>,
129 pub summary: Arc<RwLock<Option<MaintenanceSummary>>>,
130 pub near_duplicates: Arc<RwLock<Option<NearDupCount>>>,
131 pub near_duplicate_threshold: f32,
132 pub config: MaintenanceConfig,
133 pub requests: Receiver<RepairRequest>,
134}
135
136impl DrainLoop {
137 pub fn spawn(self) {
138 std::thread::Builder::new()
139 .name("index-drain".to_string())
140 .spawn(move || self.run())
141 .expect("spawning the index drain thread");
142 }
143
144 fn run(mut self) {
145 let mut last_repair: Option<Instant> = None;
146 let mut chunks = ChunkThrottle::new(Instant::now());
147 loop {
148 match self.requests.recv_timeout(DRAIN_INTERVAL) {
149 Ok(request) => {
150 let result = self.repair("requested", request.dry_run);
151 if result.is_ok() && !request.dry_run {
152 last_repair = Some(Instant::now());
153 }
154 let _ = request.reply.send(result);
156 }
157 Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {
158 let applied = self.drain();
159 chunks.after_drain(Instant::now(), applied.unwrap_or(0));
160 let due = last_repair.is_none_or(|t| {
161 t.elapsed() >= Duration::from_secs(self.config.interval_secs)
162 });
163 if applied == Some(0) && self.config.enabled && due {
164 if self.repair("scheduled", false).is_ok() {
165 last_repair = Some(Instant::now());
166 } else {
167 last_repair = Some(
170 Instant::now()
171 - Duration::from_secs(
172 self.config.interval_secs.saturating_sub(600),
173 ),
174 );
175 }
176 }
177 if applied == Some(0) && chunks.due(Instant::now()) {
178 let progressed = self.backfill_chunks();
179 chunks.after_backfill(Instant::now(), progressed);
180 }
181 }
182 }
183 }
184 }
185
186 fn backfill_chunks(&self) -> bool {
189 let Some(embedder) = self.embedder.as_deref() else {
190 return false;
191 };
192 let embed = |texts: &[&str]| embedder.embed_batch(texts);
193 match self.store.backfill_chunk_embeddings(
194 &embed,
195 CHUNK_MEMORIES_PER_TICK,
196 CHUNK_TICK_BUDGET,
197 ) {
198 Ok(r) => {
199 if r.memories > 0 || r.failed > 0 {
200 tracing::info!(
201 "chunk backfill: memories={} chunks={} failed={} pending={}",
202 r.memories,
203 r.chunks,
204 r.failed,
205 r.pending
206 );
207 }
208 r.memories > 0
209 }
210 Err(e) => {
211 tracing::warn!("chunk backfill failed: {e:#}");
212 false
213 }
214 }
215 }
216
217 fn drain(&mut self) -> Option<usize> {
219 match self.writer.drain(&self.store) {
220 Ok(0) => Some(0),
221 Ok(n) => {
222 tracing::info!("index drain applied {n} outbox rows");
223 self.reload_reader();
224 Some(n)
225 }
226 Err(e) => {
227 tracing::warn!("index drain failed: {e:#}");
228 None
229 }
230 }
231 }
232
233 fn repair(&mut self, source: &str, dry_run: bool) -> Result<MaintenanceSummary> {
234 let started = Instant::now();
235 let result = run_repair(
236 &self.store,
237 Some(&mut self.writer),
238 self.embedder.as_deref(),
239 dry_run,
240 );
241 match &result {
242 Ok(summary) => {
243 log_stats(source, &summary.stats);
244 tracing::info!(
245 "{source} store repair finished in {:.1}s; drift remaining: {:?}",
246 started.elapsed().as_secs_f64(),
247 summary.report_after.drift()
248 );
249 if !dry_run {
250 self.reload_reader();
251 let mut guard = match self.summary.write() {
252 Ok(g) => g,
253 Err(poisoned) => poisoned.into_inner(),
254 };
255 *guard = Some(summary.clone());
256 drop(guard);
257 refresh_count(
258 &self.store,
259 &self.writer,
260 self.near_duplicate_threshold,
261 &self.near_duplicates,
262 );
263 }
264 }
265 Err(e) => tracing::warn!("{source} store repair failed: {e:#}"),
266 }
267 result
268 }
269
270 fn reload_reader(&self) {
271 let mut guard = match self.reader.write() {
272 Ok(g) => g,
273 Err(poisoned) => poisoned.into_inner(),
274 };
275 if let Err(e) = guard.reload_view() {
276 tracing::warn!("index reader reload failed: {e:#}");
277 }
278 }
279}
280
281#[cfg(test)]
282mod tests {
283 use super::*;
284
285 #[test]
286 fn a_fresh_throttle_scans_on_the_first_idle_tick() {
287 let now = Instant::now();
288 assert!(ChunkThrottle::new(now).due(now));
289 }
290
291 #[test]
292 fn a_scan_that_finds_nothing_waits_out_the_idle_interval() {
293 let now = Instant::now();
294 let mut t = ChunkThrottle::new(now);
295 t.after_backfill(now, false);
296 assert!(!t.due(now + CHUNK_IDLE_RESCAN - Duration::from_secs(1)));
297 assert!(t.due(now + CHUNK_IDLE_RESCAN));
298 }
299
300 #[test]
301 fn progress_keeps_the_next_idle_tick_scanning() {
302 let now = Instant::now();
303 let mut t = ChunkThrottle::new(now);
304 t.after_backfill(now, true);
305 assert!(t.due(now));
306 }
307
308 #[test]
309 fn an_applied_drain_resets_the_idle_wait() {
310 let now = Instant::now();
311 let mut t = ChunkThrottle::new(now);
312 t.after_backfill(now, false);
313 t.after_drain(now, 0);
314 assert!(!t.due(now));
315 t.after_drain(now, 3);
316 assert!(t.due(now));
317 }
318}