Skip to main content

second_brain_api/
maintenance.rs

1//! The drain thread: applies the index outbox every few seconds and, when
2//! the outbox is quiet and the interval has passed, repairs the store with
3//! the writable indexes it owns. On-demand repairs arrive over a channel so
4//! the same thread services them; two writers never exist.
5
6use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, Instant};
9
10use anyhow::Result;
11use chrono::Utc;
12use second_brain_core::config::MaintenanceConfig;
13use second_brain_core::embedding::Embedder;
14use second_brain_core::index::Indexes;
15use second_brain_core::maintenance::{MaintenanceSummary, RepairStats};
16use second_brain_core::sqlite_store::SqliteStore;
17
18const DRAIN_INTERVAL: Duration = Duration::from_secs(5);
19
20pub struct RepairRequest {
21    pub dry_run: bool,
22    pub reply: Sender<Result<MaintenanceSummary>>,
23}
24
25/// Runs a repair and re-measures the store afterwards, on whatever index
26/// handle the caller owns (the writer on the drain thread, none when the
27/// indexes never opened).
28pub fn run_repair(
29    store: &SqliteStore,
30    mut indexes: Option<&mut Indexes>,
31    embedder: Option<&Embedder>,
32    dry_run: bool,
33) -> Result<MaintenanceSummary> {
34    let embed = embedder.map(|e| move |text: &str| e.embed(text));
35    let embed_ref = embed
36        .as_ref()
37        .map(|f| f as &dyn Fn(&str) -> Result<Vec<f32>>);
38    let stats = store.repair(indexes.as_deref_mut(), embed_ref, dry_run)?;
39    let report_after = store.integrity_report(indexes.as_deref())?;
40    Ok(MaintenanceSummary {
41        ran_at: Utc::now(),
42        stats,
43        report_after,
44    })
45}
46
47fn log_stats(source: &str, stats: &RepairStats) {
48    if stats.is_clean() {
49        tracing::info!("{source} store repair: nothing to do");
50    } else {
51        tracing::info!("{source} store repair: {}", summarize(stats));
52    }
53}
54
55fn summarize(s: &RepairStats) -> String {
56    let mut parts = Vec::new();
57    let mut push = |name: &str, n: u64| {
58        if n > 0 {
59            parts.push(format!("{name}={n}"));
60        }
61    };
62    push("duplicates_superseded", s.duplicates_superseded);
63    push("markers_inherited", s.distillation_markers_inherited);
64    push("orphaned_markers_dropped", s.orphaned_markers_dropped);
65    push("raws_retired", s.raws_retired);
66    push("relations_pruned", s.relations_pruned);
67    push("vec_keys_deleted", s.vec_keys_deleted);
68    push("sync_log_rows_deleted", s.sync_log_rows_deleted);
69    push("stale_cursors_dropped", s.stale_cursors_dropped);
70    push("embeddings_backfilled", s.embeddings_backfilled);
71    push("embedding_failures", s.embedding_failures);
72    if s.vec_keys_gc_skipped {
73        parts.push("vec_keys_gc=skipped".to_string());
74    }
75    if s.index_rebuilt {
76        parts.push("index=rebuilt".to_string());
77    }
78    if s.dry_run {
79        parts.push("dry_run".to_string());
80    }
81    parts.join(" ")
82}
83
84pub struct DrainLoop {
85    pub store: Arc<SqliteStore>,
86    pub reader: Arc<RwLock<Indexes>>,
87    pub writer: Indexes,
88    pub embedder: Option<Arc<Embedder>>,
89    pub summary: Arc<RwLock<Option<MaintenanceSummary>>>,
90    pub config: MaintenanceConfig,
91    pub requests: Receiver<RepairRequest>,
92}
93
94impl DrainLoop {
95    pub fn spawn(self) {
96        std::thread::Builder::new()
97            .name("index-drain".to_string())
98            .spawn(move || self.run())
99            .expect("spawning the index drain thread");
100    }
101
102    fn run(mut self) {
103        let mut last_repair: Option<Instant> = None;
104        loop {
105            match self.requests.recv_timeout(DRAIN_INTERVAL) {
106                Ok(request) => {
107                    let result = self.repair("requested", request.dry_run);
108                    if result.is_ok() && !request.dry_run {
109                        last_repair = Some(Instant::now());
110                    }
111                    // A caller that gave up waiting is not an error here.
112                    let _ = request.reply.send(result);
113                }
114                Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {
115                    let applied = self.drain();
116                    let due = last_repair.is_none_or(|t| {
117                        t.elapsed() >= Duration::from_secs(self.config.interval_secs)
118                    });
119                    if applied == Some(0) && self.config.enabled && due {
120                        if self.repair("scheduled", false).is_ok() {
121                            last_repair = Some(Instant::now());
122                        } else {
123                            // Retry sooner than the full interval, but not
124                            // every tick against a store that keeps failing.
125                            last_repair = Some(
126                                Instant::now()
127                                    - Duration::from_secs(
128                                        self.config.interval_secs.saturating_sub(600),
129                                    ),
130                            );
131                        }
132                    }
133                }
134            }
135        }
136    }
137
138    /// Applies pending outbox rows; None when the drain itself failed.
139    fn drain(&mut self) -> Option<usize> {
140        match self.writer.drain(&self.store) {
141            Ok(0) => Some(0),
142            Ok(n) => {
143                tracing::info!("index drain applied {n} outbox rows");
144                self.reload_reader();
145                Some(n)
146            }
147            Err(e) => {
148                tracing::warn!("index drain failed: {e:#}");
149                None
150            }
151        }
152    }
153
154    fn repair(&mut self, source: &str, dry_run: bool) -> Result<MaintenanceSummary> {
155        let started = Instant::now();
156        let result = run_repair(
157            &self.store,
158            Some(&mut self.writer),
159            self.embedder.as_deref(),
160            dry_run,
161        );
162        match &result {
163            Ok(summary) => {
164                log_stats(source, &summary.stats);
165                tracing::info!(
166                    "{source} store repair finished in {:.1}s; drift remaining: {:?}",
167                    started.elapsed().as_secs_f64(),
168                    summary.report_after.drift()
169                );
170                if !dry_run {
171                    self.reload_reader();
172                    let mut guard = match self.summary.write() {
173                        Ok(g) => g,
174                        Err(poisoned) => poisoned.into_inner(),
175                    };
176                    *guard = Some(summary.clone());
177                }
178            }
179            Err(e) => tracing::warn!("{source} store repair failed: {e:#}"),
180        }
181        result
182    }
183
184    fn reload_reader(&self) {
185        let mut guard = match self.reader.write() {
186            Ok(g) => g,
187            Err(poisoned) => poisoned.into_inner(),
188        };
189        if let Err(e) = guard.reload_view() {
190            tracing::warn!("index reader reload failed: {e:#}");
191        }
192    }
193}