1use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
7use std::sync::{Arc, RwLock};
8use std::time::{Duration, Instant};
9
10use anyhow::Result;
11use chrono::Utc;
12use second_brain_core::config::MaintenanceConfig;
13use second_brain_core::embedding::Embedder;
14use second_brain_core::index::Indexes;
15use second_brain_core::maintenance::{MaintenanceSummary, RepairStats};
16use second_brain_core::sqlite_store::SqliteStore;
17
18use crate::health::Health;
19use crate::near_duplicates::{NearDupCount, refresh_count};
20
21const DRAIN_INTERVAL: Duration = Duration::from_secs(5);
22
23const CHUNK_IDLE_RESCAN: Duration = Duration::from_secs(600);
26const CHUNK_TICK_BUDGET: Duration = Duration::from_secs(3);
28const CHUNK_MEMORIES_PER_TICK: usize = 32;
30
31struct ChunkThrottle {
33 next_scan: Instant,
34}
35
36impl ChunkThrottle {
37 fn new(now: Instant) -> Self {
38 Self { next_scan: now }
39 }
40
41 fn due(&self, now: Instant) -> bool {
42 now >= self.next_scan
43 }
44
45 fn after_backfill(&mut self, now: Instant, progressed: bool) {
46 self.next_scan = if progressed {
47 now
48 } else {
49 now + CHUNK_IDLE_RESCAN
50 };
51 }
52
53 fn after_drain(&mut self, now: Instant, applied: usize) {
55 if applied > 0 {
56 self.next_scan = now;
57 }
58 }
59}
60
61pub struct RepairRequest {
62 pub dry_run: bool,
63 pub reply: Sender<Result<MaintenanceSummary>>,
64}
65
66pub enum DrainRequest {
68 Repair(RepairRequest),
69 CompactIndexes(Sender<Result<()>>),
72}
73
74pub fn run_repair(
78 store: &SqliteStore,
79 mut indexes: Option<&mut Indexes>,
80 embedder: Option<&Embedder>,
81 dry_run: bool,
82) -> Result<MaintenanceSummary> {
83 let embed = embedder.map(|e| move |text: &str| e.embed(text));
84 let embed_ref = embed
85 .as_ref()
86 .map(|f| f as &dyn Fn(&str) -> Result<Vec<f32>>);
87 let stats = store.repair(indexes.as_deref_mut(), embed_ref, dry_run)?;
88 let report_after = store.integrity_report(indexes.as_deref())?;
89 Ok(MaintenanceSummary {
90 ran_at: Utc::now(),
91 stats,
92 report_after,
93 })
94}
95
96fn log_stats(source: &str, stats: &RepairStats) {
97 if stats.is_clean() {
98 tracing::info!("{source} store repair: nothing to do");
99 } else {
100 tracing::info!("{source} store repair: {}", summarize(stats));
101 }
102}
103
104fn summarize(s: &RepairStats) -> String {
105 let mut parts = Vec::new();
106 let mut push = |name: &str, n: u64| {
107 if n > 0 {
108 parts.push(format!("{name}={n}"));
109 }
110 };
111 push("duplicates_superseded", s.duplicates_superseded);
112 push("markers_inherited", s.distillation_markers_inherited);
113 push("orphaned_markers_dropped", s.orphaned_markers_dropped);
114 push("raws_retired", s.raws_retired);
115 push("relations_pruned", s.relations_pruned);
116 push("vec_keys_deleted", s.vec_keys_deleted);
117 push("sync_log_rows_deleted", s.sync_log_rows_deleted);
118 push("stale_cursors_dropped", s.stale_cursors_dropped);
119 push("embeddings_backfilled", s.embeddings_backfilled);
120 push("embedding_failures", s.embedding_failures);
121 if s.vec_keys_gc_skipped {
122 parts.push("vec_keys_gc=skipped".to_string());
123 }
124 if s.index_rebuilt {
125 parts.push("index=rebuilt".to_string());
126 }
127 if s.dry_run {
128 parts.push("dry_run".to_string());
129 }
130 parts.join(" ")
131}
132
133pub struct DrainLoop {
134 pub store: Arc<SqliteStore>,
135 pub reader: Arc<RwLock<Indexes>>,
136 pub writer: Indexes,
137 pub embedder: Option<Arc<Embedder>>,
138 pub summary: Arc<RwLock<Option<MaintenanceSummary>>>,
139 pub near_duplicates: Arc<RwLock<Option<NearDupCount>>>,
140 pub near_duplicate_threshold: f32,
141 pub config: MaintenanceConfig,
142 pub requests: Receiver<DrainRequest>,
143 pub health: Arc<Health>,
144}
145
146impl DrainLoop {
147 pub fn spawn(self) {
148 std::thread::Builder::new()
149 .name("index-drain".to_string())
150 .spawn(move || self.run())
151 .expect("spawning the index drain thread");
152 }
153
154 fn run(mut self) {
155 let mut last_repair: Option<Instant> = None;
156 let mut chunks = ChunkThrottle::new(Instant::now());
157 loop {
158 match self.requests.recv_timeout(DRAIN_INTERVAL) {
159 Ok(DrainRequest::Repair(request)) => {
160 let result = self.repair("requested", request.dry_run);
161 if result.is_ok() && !request.dry_run {
162 last_repair = Some(Instant::now());
163 }
164 let _ = request.reply.send(result);
166 }
167 Ok(DrainRequest::CompactIndexes(reply)) => {
168 let _ = reply.send(self.compact_indexes());
169 }
170 Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {
171 let applied = self.drain();
172 chunks.after_drain(Instant::now(), applied.unwrap_or(0));
173 let due = last_repair.is_none_or(|t| {
174 t.elapsed() >= Duration::from_secs(self.config.interval_secs)
175 });
176 if applied == Some(0) && self.config.enabled && due {
177 if self.repair("scheduled", false).is_ok() {
178 last_repair = Some(Instant::now());
179 } else {
180 last_repair = Some(
183 Instant::now()
184 - Duration::from_secs(
185 self.config.interval_secs.saturating_sub(600),
186 ),
187 );
188 }
189 }
190 if applied == Some(0) && chunks.due(Instant::now()) {
191 let progressed = self.backfill_chunks();
192 chunks.after_backfill(Instant::now(), progressed);
193 }
194 }
195 }
196 }
197 }
198
199 fn backfill_chunks(&self) -> bool {
202 let Some(embedder) = self.embedder.as_deref() else {
203 return false;
204 };
205 let embed = |texts: &[&str]| embedder.embed_batch(texts);
206 match self.store.backfill_chunk_embeddings(
207 &embed,
208 CHUNK_MEMORIES_PER_TICK,
209 CHUNK_TICK_BUDGET,
210 ) {
211 Ok(r) => {
212 if r.memories > 0 || r.failed > 0 {
213 tracing::info!(
214 "chunk backfill: memories={} chunks={} failed={} pending={}",
215 r.memories,
216 r.chunks,
217 r.failed,
218 r.pending
219 );
220 }
221 r.memories > 0
222 }
223 Err(e) => {
224 tracing::warn!("chunk backfill failed: {e:#}");
225 false
226 }
227 }
228 }
229
230 fn drain(&mut self) -> Option<usize> {
232 match self.writer.drain(&self.store) {
233 Ok(0) => {
234 self.health.drained(0, Utc::now());
235 Some(0)
236 }
237 Ok(n) => {
238 tracing::info!("index drain applied {n} outbox rows");
239 self.reload_reader();
240 match self.store.outbox_pending() {
241 Ok(pending) => self.health.drained(pending, Utc::now()),
242 Err(e) => tracing::warn!("counting the index outbox failed: {e:#}"),
243 }
244 Some(n)
245 }
246 Err(e) => {
247 tracing::warn!("index drain failed: {e:#}");
248 self.health.drain_failed(&format!("{e:#}"), Utc::now());
249 None
250 }
251 }
252 }
253
254 fn repair(&mut self, source: &str, dry_run: bool) -> Result<MaintenanceSummary> {
255 let started = Instant::now();
256 let result = run_repair(
257 &self.store,
258 Some(&mut self.writer),
259 self.embedder.as_deref(),
260 dry_run,
261 );
262 match &result {
263 Ok(summary) => {
264 self.health.repaired(Utc::now());
265 log_stats(source, &summary.stats);
266 tracing::info!(
267 "{source} store repair finished in {:.1}s; drift remaining: {:?}",
268 started.elapsed().as_secs_f64(),
269 summary.report_after.drift()
270 );
271 if !dry_run {
272 self.reload_reader();
273 let mut guard = match self.summary.write() {
274 Ok(g) => g,
275 Err(poisoned) => poisoned.into_inner(),
276 };
277 *guard = Some(summary.clone());
278 drop(guard);
279 refresh_count(
280 &self.store,
281 &self.writer,
282 self.near_duplicate_threshold,
283 &self.near_duplicates,
284 );
285 }
286 }
287 Err(e) => {
288 tracing::warn!("{source} store repair failed: {e:#}");
289 self.health.repair_failed(&format!("{e:#}"), Utc::now());
290 }
291 }
292 result
293 }
294
295 fn compact_indexes(&mut self) -> Result<()> {
296 while self.writer.drain(&self.store)? > 0 {}
297 let started = Instant::now();
298 self.writer.compact()?;
299 self.reload_reader();
300 tracing::info!(
301 "lexical index compacted in {:.1}s",
302 started.elapsed().as_secs_f64()
303 );
304 Ok(())
305 }
306
307 fn reload_reader(&self) {
308 let mut guard = match self.reader.write() {
309 Ok(g) => g,
310 Err(poisoned) => poisoned.into_inner(),
311 };
312 if let Err(e) = guard.reload_view() {
313 tracing::warn!("index reader reload failed: {e:#}");
314 }
315 }
316}
317
318#[cfg(test)]
319mod tests {
320 use super::*;
321
322 #[test]
323 fn a_fresh_throttle_scans_on_the_first_idle_tick() {
324 let now = Instant::now();
325 assert!(ChunkThrottle::new(now).due(now));
326 }
327
328 #[test]
329 fn a_scan_that_finds_nothing_waits_out_the_idle_interval() {
330 let now = Instant::now();
331 let mut t = ChunkThrottle::new(now);
332 t.after_backfill(now, false);
333 assert!(!t.due(now + CHUNK_IDLE_RESCAN - Duration::from_secs(1)));
334 assert!(t.due(now + CHUNK_IDLE_RESCAN));
335 }
336
337 #[test]
338 fn progress_keeps_the_next_idle_tick_scanning() {
339 let now = Instant::now();
340 let mut t = ChunkThrottle::new(now);
341 t.after_backfill(now, true);
342 assert!(t.due(now));
343 }
344
345 #[test]
346 fn an_applied_drain_resets_the_idle_wait() {
347 let now = Instant::now();
348 let mut t = ChunkThrottle::new(now);
349 t.after_backfill(now, false);
350 t.after_drain(now, 0);
351 assert!(!t.due(now));
352 t.after_drain(now, 3);
353 assert!(t.due(now));
354 }
355}