Skip to main content

second_brain_api/
maintenance.rs

1//! The drain thread: applies the index outbox every few seconds and, when
2//! the outbox is quiet and the interval has passed, repairs the store with
3//! the writable indexes it owns. On-demand repairs arrive over a channel so
4//! the same thread services them; two writers never exist.
5
6use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, Instant};
9
10use anyhow::Result;
11use chrono::Utc;
12use second_brain_core::config::MaintenanceConfig;
13use second_brain_core::embedding::Embedder;
14use second_brain_core::index::Indexes;
15use second_brain_core::maintenance::{MaintenanceSummary, RepairStats};
16use second_brain_core::sqlite_store::SqliteStore;
17
18use crate::near_duplicates::{NearDupCount, refresh_count};
19
20const DRAIN_INTERVAL: Duration = Duration::from_secs(5);
21
22/// Finding chunk work splits every live row, so once a scan comes back
23/// empty-handed the next one waits this long unless a drain applies writes.
24const CHUNK_IDLE_RESCAN: Duration = Duration::from_secs(600);
25/// Per idle tick, so the drain and repair behind it are never held up long.
26const CHUNK_TICK_BUDGET: Duration = Duration::from_secs(3);
27/// Bounds what one tick holds in memory: at most 16 chunks per memory.
28const CHUNK_MEMORIES_PER_TICK: usize = 32;
29
30/// When the drain thread may next look for memories needing chunks.
31struct ChunkThrottle {
32    next_scan: Instant,
33}
34
35impl ChunkThrottle {
36    fn new(now: Instant) -> Self {
37        Self { next_scan: now }
38    }
39
40    fn due(&self, now: Instant) -> bool {
41        now >= self.next_scan
42    }
43
44    fn after_backfill(&mut self, now: Instant, progressed: bool) {
45        self.next_scan = if progressed {
46            now
47        } else {
48            now + CHUNK_IDLE_RESCAN
49        };
50    }
51
52    /// Applied outbox rows may carry content changes that need re-chunking.
53    fn after_drain(&mut self, now: Instant, applied: usize) {
54        if applied > 0 {
55            self.next_scan = now;
56        }
57    }
58}
59
60pub struct RepairRequest {
61    pub dry_run: bool,
62    pub reply: Sender<Result<MaintenanceSummary>>,
63}
64
65/// Runs a repair and re-measures the store afterwards, on whatever index
66/// handle the caller owns (the writer on the drain thread, none when the
67/// indexes never opened).
68pub fn run_repair(
69    store: &SqliteStore,
70    mut indexes: Option<&mut Indexes>,
71    embedder: Option<&Embedder>,
72    dry_run: bool,
73) -> Result<MaintenanceSummary> {
74    let embed = embedder.map(|e| move |text: &str| e.embed(text));
75    let embed_ref = embed
76        .as_ref()
77        .map(|f| f as &dyn Fn(&str) -> Result<Vec<f32>>);
78    let stats = store.repair(indexes.as_deref_mut(), embed_ref, dry_run)?;
79    let report_after = store.integrity_report(indexes.as_deref())?;
80    Ok(MaintenanceSummary {
81        ran_at: Utc::now(),
82        stats,
83        report_after,
84    })
85}
86
87fn log_stats(source: &str, stats: &RepairStats) {
88    if stats.is_clean() {
89        tracing::info!("{source} store repair: nothing to do");
90    } else {
91        tracing::info!("{source} store repair: {}", summarize(stats));
92    }
93}
94
95fn summarize(s: &RepairStats) -> String {
96    let mut parts = Vec::new();
97    let mut push = |name: &str, n: u64| {
98        if n > 0 {
99            parts.push(format!("{name}={n}"));
100        }
101    };
102    push("duplicates_superseded", s.duplicates_superseded);
103    push("markers_inherited", s.distillation_markers_inherited);
104    push("orphaned_markers_dropped", s.orphaned_markers_dropped);
105    push("raws_retired", s.raws_retired);
106    push("relations_pruned", s.relations_pruned);
107    push("vec_keys_deleted", s.vec_keys_deleted);
108    push("sync_log_rows_deleted", s.sync_log_rows_deleted);
109    push("stale_cursors_dropped", s.stale_cursors_dropped);
110    push("embeddings_backfilled", s.embeddings_backfilled);
111    push("embedding_failures", s.embedding_failures);
112    if s.vec_keys_gc_skipped {
113        parts.push("vec_keys_gc=skipped".to_string());
114    }
115    if s.index_rebuilt {
116        parts.push("index=rebuilt".to_string());
117    }
118    if s.dry_run {
119        parts.push("dry_run".to_string());
120    }
121    parts.join(" ")
122}
123
124pub struct DrainLoop {
125    pub store: Arc<SqliteStore>,
126    pub reader: Arc<RwLock<Indexes>>,
127    pub writer: Indexes,
128    pub embedder: Option<Arc<Embedder>>,
129    pub summary: Arc<RwLock<Option<MaintenanceSummary>>>,
130    pub near_duplicates: Arc<RwLock<Option<NearDupCount>>>,
131    pub near_duplicate_threshold: f32,
132    pub config: MaintenanceConfig,
133    pub requests: Receiver<RepairRequest>,
134}
135
136impl DrainLoop {
137    pub fn spawn(self) {
138        std::thread::Builder::new()
139            .name("index-drain".to_string())
140            .spawn(move || self.run())
141            .expect("spawning the index drain thread");
142    }
143
144    fn run(mut self) {
145        let mut last_repair: Option<Instant> = None;
146        let mut chunks = ChunkThrottle::new(Instant::now());
147        loop {
148            match self.requests.recv_timeout(DRAIN_INTERVAL) {
149                Ok(request) => {
150                    let result = self.repair("requested", request.dry_run);
151                    if result.is_ok() && !request.dry_run {
152                        last_repair = Some(Instant::now());
153                    }
154                    // A caller that gave up waiting is not an error here.
155                    let _ = request.reply.send(result);
156                }
157                Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {
158                    let applied = self.drain();
159                    chunks.after_drain(Instant::now(), applied.unwrap_or(0));
160                    let due = last_repair.is_none_or(|t| {
161                        t.elapsed() >= Duration::from_secs(self.config.interval_secs)
162                    });
163                    if applied == Some(0) && self.config.enabled && due {
164                        if self.repair("scheduled", false).is_ok() {
165                            last_repair = Some(Instant::now());
166                        } else {
167                            // Retry sooner than the full interval, but not
168                            // every tick against a store that keeps failing.
169                            last_repair = Some(
170                                Instant::now()
171                                    - Duration::from_secs(
172                                        self.config.interval_secs.saturating_sub(600),
173                                    ),
174                            );
175                        }
176                    }
177                    if applied == Some(0) && chunks.due(Instant::now()) {
178                        let progressed = self.backfill_chunks();
179                        chunks.after_backfill(Instant::now(), progressed);
180                    }
181                }
182            }
183        }
184    }
185
186    /// Embeds pending chunk sets within one tick's budget; true when any set
187    /// was stored, so the next idle tick keeps going.
188    fn backfill_chunks(&self) -> bool {
189        let Some(embedder) = self.embedder.as_deref() else {
190            return false;
191        };
192        let embed = |texts: &[&str]| embedder.embed_batch(texts);
193        match self.store.backfill_chunk_embeddings(
194            &embed,
195            CHUNK_MEMORIES_PER_TICK,
196            CHUNK_TICK_BUDGET,
197        ) {
198            Ok(r) => {
199                if r.memories > 0 || r.failed > 0 {
200                    tracing::info!(
201                        "chunk backfill: memories={} chunks={} failed={} pending={}",
202                        r.memories,
203                        r.chunks,
204                        r.failed,
205                        r.pending
206                    );
207                }
208                r.memories > 0
209            }
210            Err(e) => {
211                tracing::warn!("chunk backfill failed: {e:#}");
212                false
213            }
214        }
215    }
216
217    /// Applies pending outbox rows; None when the drain itself failed.
218    fn drain(&mut self) -> Option<usize> {
219        match self.writer.drain(&self.store) {
220            Ok(0) => Some(0),
221            Ok(n) => {
222                tracing::info!("index drain applied {n} outbox rows");
223                self.reload_reader();
224                Some(n)
225            }
226            Err(e) => {
227                tracing::warn!("index drain failed: {e:#}");
228                None
229            }
230        }
231    }
232
233    fn repair(&mut self, source: &str, dry_run: bool) -> Result<MaintenanceSummary> {
234        let started = Instant::now();
235        let result = run_repair(
236            &self.store,
237            Some(&mut self.writer),
238            self.embedder.as_deref(),
239            dry_run,
240        );
241        match &result {
242            Ok(summary) => {
243                log_stats(source, &summary.stats);
244                tracing::info!(
245                    "{source} store repair finished in {:.1}s; drift remaining: {:?}",
246                    started.elapsed().as_secs_f64(),
247                    summary.report_after.drift()
248                );
249                if !dry_run {
250                    self.reload_reader();
251                    let mut guard = match self.summary.write() {
252                        Ok(g) => g,
253                        Err(poisoned) => poisoned.into_inner(),
254                    };
255                    *guard = Some(summary.clone());
256                    drop(guard);
257                    refresh_count(
258                        &self.store,
259                        &self.writer,
260                        self.near_duplicate_threshold,
261                        &self.near_duplicates,
262                    );
263                }
264            }
265            Err(e) => tracing::warn!("{source} store repair failed: {e:#}"),
266        }
267        result
268    }
269
270    fn reload_reader(&self) {
271        let mut guard = match self.reader.write() {
272            Ok(g) => g,
273            Err(poisoned) => poisoned.into_inner(),
274        };
275        if let Err(e) = guard.reload_view() {
276            tracing::warn!("index reader reload failed: {e:#}");
277        }
278    }
279}
280
281#[cfg(test)]
282mod tests {
283    use super::*;
284
285    #[test]
286    fn a_fresh_throttle_scans_on_the_first_idle_tick() {
287        let now = Instant::now();
288        assert!(ChunkThrottle::new(now).due(now));
289    }
290
291    #[test]
292    fn a_scan_that_finds_nothing_waits_out_the_idle_interval() {
293        let now = Instant::now();
294        let mut t = ChunkThrottle::new(now);
295        t.after_backfill(now, false);
296        assert!(!t.due(now + CHUNK_IDLE_RESCAN - Duration::from_secs(1)));
297        assert!(t.due(now + CHUNK_IDLE_RESCAN));
298    }
299
300    #[test]
301    fn progress_keeps_the_next_idle_tick_scanning() {
302        let now = Instant::now();
303        let mut t = ChunkThrottle::new(now);
304        t.after_backfill(now, true);
305        assert!(t.due(now));
306    }
307
308    #[test]
309    fn an_applied_drain_resets_the_idle_wait() {
310        let now = Instant::now();
311        let mut t = ChunkThrottle::new(now);
312        t.after_backfill(now, false);
313        t.after_drain(now, 0);
314        assert!(!t.due(now));
315        t.after_drain(now, 3);
316        assert!(t.due(now));
317    }
318}