Skip to main content

second_brain_api/
maintenance.rs

1//! The drain thread: applies the index outbox every few seconds and, when
2//! the outbox is quiet and the interval has passed, repairs the store with
3//! the writable indexes it owns. On-demand repairs arrive over a channel so
4//! the same thread services them; two writers never exist.
5
6use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, Instant};
9
10use anyhow::Result;
11use chrono::Utc;
12use second_brain_core::config::MaintenanceConfig;
13use second_brain_core::embedding::Embedder;
14use second_brain_core::index::Indexes;
15use second_brain_core::maintenance::{MaintenanceSummary, RepairStats};
16use second_brain_core::sqlite_store::SqliteStore;
17
18use crate::health::Health;
19use crate::near_duplicates::{NearDupCount, refresh_count};
20
21const DRAIN_INTERVAL: Duration = Duration::from_secs(5);
22
23/// Finding chunk work splits every live row, so once a scan comes back
24/// empty-handed the next one waits this long unless a drain applies writes.
25const CHUNK_IDLE_RESCAN: Duration = Duration::from_secs(600);
26/// Per idle tick, so the drain and repair behind it are never held up long.
27const CHUNK_TICK_BUDGET: Duration = Duration::from_secs(3);
28/// Bounds what one tick holds in memory: at most 16 chunks per memory.
29const CHUNK_MEMORIES_PER_TICK: usize = 32;
30
31/// When the drain thread may next look for memories needing chunks.
32struct ChunkThrottle {
33    next_scan: Instant,
34}
35
36impl ChunkThrottle {
37    fn new(now: Instant) -> Self {
38        Self { next_scan: now }
39    }
40
41    fn due(&self, now: Instant) -> bool {
42        now >= self.next_scan
43    }
44
45    fn after_backfill(&mut self, now: Instant, progressed: bool) {
46        self.next_scan = if progressed {
47            now
48        } else {
49            now + CHUNK_IDLE_RESCAN
50        };
51    }
52
53    /// Applied outbox rows may carry content changes that need re-chunking.
54    fn after_drain(&mut self, now: Instant, applied: usize) {
55        if applied > 0 {
56            self.next_scan = now;
57        }
58    }
59}
60
61pub struct RepairRequest {
62    pub dry_run: bool,
63    pub reply: Sender<Result<MaintenanceSummary>>,
64}
65
66/// Work only the drain thread may do, because it owns the writable indexes.
67pub enum DrainRequest {
68    Repair(RepairRequest),
69    /// Applies the whole outbox, then rewrites the lexical index into fresh
70    /// segments (see `Indexes::compact`).
71    CompactIndexes(Sender<Result<()>>),
72}
73
74/// Runs a repair and re-measures the store afterwards, on whatever index
75/// handle the caller owns (the writer on the drain thread, none when the
76/// indexes never opened).
77pub fn run_repair(
78    store: &SqliteStore,
79    mut indexes: Option<&mut Indexes>,
80    embedder: Option<&Embedder>,
81    dry_run: bool,
82) -> Result<MaintenanceSummary> {
83    let embed = embedder.map(|e| move |text: &str| e.embed(text));
84    let embed_ref = embed
85        .as_ref()
86        .map(|f| f as &dyn Fn(&str) -> Result<Vec<f32>>);
87    let stats = store.repair(indexes.as_deref_mut(), embed_ref, dry_run)?;
88    let report_after = store.integrity_report(indexes.as_deref())?;
89    Ok(MaintenanceSummary {
90        ran_at: Utc::now(),
91        stats,
92        report_after,
93    })
94}
95
96fn log_stats(source: &str, stats: &RepairStats) {
97    if stats.is_clean() {
98        tracing::info!("{source} store repair: nothing to do");
99    } else {
100        tracing::info!("{source} store repair: {}", summarize(stats));
101    }
102}
103
104fn summarize(s: &RepairStats) -> String {
105    let mut parts = Vec::new();
106    let mut push = |name: &str, n: u64| {
107        if n > 0 {
108            parts.push(format!("{name}={n}"));
109        }
110    };
111    push("duplicates_superseded", s.duplicates_superseded);
112    push("markers_inherited", s.distillation_markers_inherited);
113    push("orphaned_markers_dropped", s.orphaned_markers_dropped);
114    push("raws_retired", s.raws_retired);
115    push("relations_pruned", s.relations_pruned);
116    push("vec_keys_deleted", s.vec_keys_deleted);
117    push("sync_log_rows_deleted", s.sync_log_rows_deleted);
118    push("stale_cursors_dropped", s.stale_cursors_dropped);
119    push("embeddings_backfilled", s.embeddings_backfilled);
120    push("embedding_failures", s.embedding_failures);
121    if s.vec_keys_gc_skipped {
122        parts.push("vec_keys_gc=skipped".to_string());
123    }
124    if s.index_rebuilt {
125        parts.push("index=rebuilt".to_string());
126    }
127    if s.dry_run {
128        parts.push("dry_run".to_string());
129    }
130    parts.join(" ")
131}
132
133pub struct DrainLoop {
134    pub store: Arc<SqliteStore>,
135    pub reader: Arc<RwLock<Indexes>>,
136    pub writer: Indexes,
137    pub embedder: Option<Arc<Embedder>>,
138    pub summary: Arc<RwLock<Option<MaintenanceSummary>>>,
139    pub near_duplicates: Arc<RwLock<Option<NearDupCount>>>,
140    pub near_duplicate_threshold: f32,
141    pub config: MaintenanceConfig,
142    pub requests: Receiver<DrainRequest>,
143    pub health: Arc<Health>,
144}
145
146impl DrainLoop {
147    pub fn spawn(self) {
148        std::thread::Builder::new()
149            .name("index-drain".to_string())
150            .spawn(move || self.run())
151            .expect("spawning the index drain thread");
152    }
153
154    fn run(mut self) {
155        let mut last_repair: Option<Instant> = None;
156        let mut chunks = ChunkThrottle::new(Instant::now());
157        loop {
158            match self.requests.recv_timeout(DRAIN_INTERVAL) {
159                Ok(DrainRequest::Repair(request)) => {
160                    let result = self.repair("requested", request.dry_run);
161                    if result.is_ok() && !request.dry_run {
162                        last_repair = Some(Instant::now());
163                    }
164                    // A caller that gave up waiting is not an error here.
165                    let _ = request.reply.send(result);
166                }
167                Ok(DrainRequest::CompactIndexes(reply)) => {
168                    let _ = reply.send(self.compact_indexes());
169                }
170                Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {
171                    let applied = self.drain();
172                    chunks.after_drain(Instant::now(), applied.unwrap_or(0));
173                    let due = last_repair.is_none_or(|t| {
174                        t.elapsed() >= Duration::from_secs(self.config.interval_secs)
175                    });
176                    if applied == Some(0) && self.config.enabled && due {
177                        if self.repair("scheduled", false).is_ok() {
178                            last_repair = Some(Instant::now());
179                        } else {
180                            // Retry sooner than the full interval, but not
181                            // every tick against a store that keeps failing.
182                            last_repair = Some(
183                                Instant::now()
184                                    - Duration::from_secs(
185                                        self.config.interval_secs.saturating_sub(600),
186                                    ),
187                            );
188                        }
189                    }
190                    if applied == Some(0) && chunks.due(Instant::now()) {
191                        let progressed = self.backfill_chunks();
192                        chunks.after_backfill(Instant::now(), progressed);
193                    }
194                }
195            }
196        }
197    }
198
199    /// Embeds pending chunk sets within one tick's budget; true when any set
200    /// was stored, so the next idle tick keeps going.
201    fn backfill_chunks(&self) -> bool {
202        let Some(embedder) = self.embedder.as_deref() else {
203            return false;
204        };
205        let embed = |texts: &[&str]| embedder.embed_batch(texts);
206        match self.store.backfill_chunk_embeddings(
207            &embed,
208            CHUNK_MEMORIES_PER_TICK,
209            CHUNK_TICK_BUDGET,
210        ) {
211            Ok(r) => {
212                if r.memories > 0 || r.failed > 0 {
213                    tracing::info!(
214                        "chunk backfill: memories={} chunks={} failed={} pending={}",
215                        r.memories,
216                        r.chunks,
217                        r.failed,
218                        r.pending
219                    );
220                }
221                r.memories > 0
222            }
223            Err(e) => {
224                tracing::warn!("chunk backfill failed: {e:#}");
225                false
226            }
227        }
228    }
229
230    /// Applies pending outbox rows; None when the drain itself failed.
231    fn drain(&mut self) -> Option<usize> {
232        match self.writer.drain(&self.store) {
233            Ok(0) => {
234                self.health.drained(0, Utc::now());
235                Some(0)
236            }
237            Ok(n) => {
238                tracing::info!("index drain applied {n} outbox rows");
239                self.reload_reader();
240                match self.store.outbox_pending() {
241                    Ok(pending) => self.health.drained(pending, Utc::now()),
242                    Err(e) => tracing::warn!("counting the index outbox failed: {e:#}"),
243                }
244                Some(n)
245            }
246            Err(e) => {
247                tracing::warn!("index drain failed: {e:#}");
248                self.health.drain_failed(&format!("{e:#}"), Utc::now());
249                None
250            }
251        }
252    }
253
254    fn repair(&mut self, source: &str, dry_run: bool) -> Result<MaintenanceSummary> {
255        let started = Instant::now();
256        let result = run_repair(
257            &self.store,
258            Some(&mut self.writer),
259            self.embedder.as_deref(),
260            dry_run,
261        );
262        match &result {
263            Ok(summary) => {
264                self.health.repaired(Utc::now());
265                log_stats(source, &summary.stats);
266                tracing::info!(
267                    "{source} store repair finished in {:.1}s; drift remaining: {:?}",
268                    started.elapsed().as_secs_f64(),
269                    summary.report_after.drift()
270                );
271                if !dry_run {
272                    self.reload_reader();
273                    let mut guard = match self.summary.write() {
274                        Ok(g) => g,
275                        Err(poisoned) => poisoned.into_inner(),
276                    };
277                    *guard = Some(summary.clone());
278                    drop(guard);
279                    refresh_count(
280                        &self.store,
281                        &self.writer,
282                        self.near_duplicate_threshold,
283                        &self.near_duplicates,
284                    );
285                }
286            }
287            Err(e) => {
288                tracing::warn!("{source} store repair failed: {e:#}");
289                self.health.repair_failed(&format!("{e:#}"), Utc::now());
290            }
291        }
292        result
293    }
294
295    fn compact_indexes(&mut self) -> Result<()> {
296        while self.writer.drain(&self.store)? > 0 {}
297        let started = Instant::now();
298        self.writer.compact()?;
299        self.reload_reader();
300        tracing::info!(
301            "lexical index compacted in {:.1}s",
302            started.elapsed().as_secs_f64()
303        );
304        Ok(())
305    }
306
307    fn reload_reader(&self) {
308        let mut guard = match self.reader.write() {
309            Ok(g) => g,
310            Err(poisoned) => poisoned.into_inner(),
311        };
312        if let Err(e) = guard.reload_view() {
313            tracing::warn!("index reader reload failed: {e:#}");
314        }
315    }
316}
317
318#[cfg(test)]
319mod tests {
320    use super::*;
321
322    #[test]
323    fn a_fresh_throttle_scans_on_the_first_idle_tick() {
324        let now = Instant::now();
325        assert!(ChunkThrottle::new(now).due(now));
326    }
327
328    #[test]
329    fn a_scan_that_finds_nothing_waits_out_the_idle_interval() {
330        let now = Instant::now();
331        let mut t = ChunkThrottle::new(now);
332        t.after_backfill(now, false);
333        assert!(!t.due(now + CHUNK_IDLE_RESCAN - Duration::from_secs(1)));
334        assert!(t.due(now + CHUNK_IDLE_RESCAN));
335    }
336
337    #[test]
338    fn progress_keeps_the_next_idle_tick_scanning() {
339        let now = Instant::now();
340        let mut t = ChunkThrottle::new(now);
341        t.after_backfill(now, true);
342        assert!(t.due(now));
343    }
344
345    #[test]
346    fn an_applied_drain_resets_the_idle_wait() {
347        let now = Instant::now();
348        let mut t = ChunkThrottle::new(now);
349        t.after_backfill(now, false);
350        t.after_drain(now, 0);
351        assert!(!t.due(now));
352        t.after_drain(now, 3);
353        assert!(t.due(now));
354    }
355}