1use std::path::PathBuf;
7use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
8use std::sync::{Arc, RwLock};
9use std::time::{Duration, Instant};
10
11use anyhow::Result;
12use chrono::Utc;
13use second_brain_core::config::MaintenanceConfig;
14use second_brain_core::embedding::Embedder;
15use second_brain_core::index::{IndexClaim, Indexes};
16use second_brain_core::maintenance::{MaintenanceSummary, RepairStats};
17use second_brain_core::residency::{self, IndexMemory};
18use second_brain_core::sqlite_store::SqliteStore;
19
20use crate::health::Health;
21use crate::near_duplicates::{NearDupCount, refresh_count};
22
23const DRAIN_INTERVAL: Duration = Duration::from_secs(5);
24
25const WRITER_IDLE: Duration = Duration::from_secs(60);
26
27const CHUNK_IDLE_RESCAN: Duration = Duration::from_secs(600);
30const CHUNK_TICK_BUDGET: Duration = Duration::from_secs(3);
32const CHUNK_MEMORIES_PER_TICK: usize = 32;
34
35struct ChunkThrottle {
37 next_scan: Instant,
38}
39
40impl ChunkThrottle {
41 fn new(now: Instant) -> Self {
42 Self { next_scan: now }
43 }
44
45 fn due(&self, now: Instant) -> bool {
46 now >= self.next_scan
47 }
48
49 fn after_backfill(&mut self, now: Instant, progressed: bool) {
50 self.next_scan = if progressed {
51 now
52 } else {
53 now + CHUNK_IDLE_RESCAN
54 };
55 }
56
57 fn after_drain(&mut self, now: Instant, applied: usize) {
59 if applied > 0 {
60 self.next_scan = now;
61 }
62 }
63}
64
65pub struct RepairRequest {
66 pub dry_run: bool,
67 pub reply: Sender<Result<MaintenanceSummary>>,
68}
69
70pub enum DrainRequest {
72 Repair(RepairRequest),
73 CompactIndexes(Sender<Result<()>>),
76}
77
78pub fn run_repair(
82 store: &SqliteStore,
83 mut indexes: Option<&mut Indexes>,
84 embedder: Option<&Embedder>,
85 dry_run: bool,
86) -> Result<MaintenanceSummary> {
87 let embed = embedder.map(|e| move |text: &str| e.embed(text));
88 let embed_ref = embed
89 .as_ref()
90 .map(|f| f as &dyn Fn(&str) -> Result<Vec<f32>>);
91 let stats = store.repair(indexes.as_deref_mut(), embed_ref, dry_run)?;
92 let report_after = store.integrity_report(indexes.as_deref())?;
93 Ok(MaintenanceSummary {
94 ran_at: Utc::now(),
95 stats,
96 report_after,
97 })
98}
99
100fn log_stats(source: &str, stats: &RepairStats) {
101 if stats.is_clean() {
102 tracing::info!("{source} store repair: nothing to do");
103 } else {
104 tracing::info!("{source} store repair: {}", summarize(stats));
105 }
106}
107
108fn summarize(s: &RepairStats) -> String {
109 let mut parts = Vec::new();
110 let mut push = |name: &str, n: u64| {
111 if n > 0 {
112 parts.push(format!("{name}={n}"));
113 }
114 };
115 push("duplicates_superseded", s.duplicates_superseded);
116 push("markers_inherited", s.distillation_markers_inherited);
117 push("orphaned_markers_dropped", s.orphaned_markers_dropped);
118 push("raws_retired", s.raws_retired);
119 push("relations_pruned", s.relations_pruned);
120 push("vec_keys_deleted", s.vec_keys_deleted);
121 push("sync_log_rows_deleted", s.sync_log_rows_deleted);
122 push("stale_cursors_dropped", s.stale_cursors_dropped);
123 push("embeddings_backfilled", s.embeddings_backfilled);
124 push("embedding_failures", s.embedding_failures);
125 if s.vec_keys_gc_skipped {
126 parts.push("vec_keys_gc=skipped".to_string());
127 }
128 if s.index_rebuilt {
129 parts.push("index=rebuilt".to_string());
130 }
131 if s.dry_run {
132 parts.push("dry_run".to_string());
133 }
134 parts.join(" ")
135}
136
137pub struct WriterSlot {
145 dir: PathBuf,
146 open: Option<Indexes>,
147 last_used: Instant,
148 _claim: IndexClaim,
149}
150
151impl WriterSlot {
152 pub fn new(writer: Indexes, claim: IndexClaim) -> Self {
153 crate::heap::record_index_writer(Some(writer.memory()));
154 Self {
155 dir: writer.dir().to_path_buf(),
156 open: Some(writer),
157 last_used: Instant::now(),
158 _claim: claim,
159 }
160 }
161
162 fn open(&mut self) -> Result<&mut Indexes> {
163 if self.open.is_none() {
164 let writer = Indexes::open(&self.dir)?;
165 tracing::info!("index writer reopened at {}", self.dir.display());
166 crate::heap::record_index_writer(Some(writer.memory()));
167 self.open = Some(writer);
168 }
169 Ok(self.open.as_mut().expect("writer opened above"))
170 }
171
172 fn with<R>(&mut self, op: impl FnOnce(&mut Indexes) -> Result<R>) -> Result<R> {
173 let result = op(self.open()?);
174 self.last_used = Instant::now();
175 crate::heap::record_index_writer(self.memory());
176 result
177 }
178
179 fn release_if_idle(&mut self, now: Instant) -> bool {
180 let current = self
181 .open
182 .as_ref()
183 .is_some_and(|w| !w.has_unpersisted_changes());
184 if !current || now.saturating_duration_since(self.last_used) < WRITER_IDLE {
185 return false;
186 }
187 if let Some(writer) = self.open.take()
188 && let Err(e) = writer.wait_for_merges()
189 {
190 tracing::warn!("index writer released without its merges: {e:#}");
191 }
192 crate::heap::record_index_writer(None);
193 residency::release_free_heap();
194 tracing::info!(
195 "index writer released after {}s idle",
196 WRITER_IDLE.as_secs()
197 );
198 true
199 }
200
201 fn memory(&self) -> Option<IndexMemory> {
202 self.open.as_ref().map(Indexes::memory)
203 }
204}
205
206pub struct DrainLoop {
207 pub store: Arc<SqliteStore>,
208 pub reader: Arc<RwLock<Indexes>>,
209 pub writer: WriterSlot,
210 pub embedder: Option<Arc<Embedder>>,
211 pub summary: Arc<RwLock<Option<MaintenanceSummary>>>,
212 pub near_duplicates: Arc<RwLock<Option<NearDupCount>>>,
213 pub near_duplicate_threshold: f32,
214 pub config: MaintenanceConfig,
215 pub requests: Receiver<DrainRequest>,
216 pub health: Arc<Health>,
217}
218
219impl DrainLoop {
220 pub fn spawn(self) {
221 std::thread::Builder::new()
222 .name("index-drain".to_string())
223 .spawn(move || self.run())
224 .expect("spawning the index drain thread");
225 }
226
227 fn run(mut self) {
228 let mut last_repair: Option<Instant> = None;
229 let mut chunks = ChunkThrottle::new(Instant::now());
230 loop {
231 match self.requests.recv_timeout(DRAIN_INTERVAL) {
232 Ok(DrainRequest::Repair(request)) => {
233 let result = self.repair("requested", request.dry_run);
234 if result.is_ok() && !request.dry_run {
235 last_repair = Some(Instant::now());
236 }
237 let _ = request.reply.send(result);
239 }
240 Ok(DrainRequest::CompactIndexes(reply)) => {
241 let _ = reply.send(self.compact_indexes());
242 }
243 Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {
244 let applied = self.drain();
245 chunks.after_drain(Instant::now(), applied.unwrap_or(0));
246 let due = last_repair.is_none_or(|t| {
247 t.elapsed() >= Duration::from_secs(self.config.interval_secs)
248 });
249 if applied == Some(0) && self.config.enabled && due {
250 if self.repair("scheduled", false).is_ok() {
251 last_repair = Some(Instant::now());
252 } else {
253 last_repair = Some(
256 Instant::now()
257 - Duration::from_secs(
258 self.config.interval_secs.saturating_sub(600),
259 ),
260 );
261 }
262 }
263 if applied == Some(0) && chunks.due(Instant::now()) {
264 let progressed = self.backfill_chunks();
265 chunks.after_backfill(Instant::now(), progressed);
266 }
267 }
268 }
269 self.writer.release_if_idle(Instant::now());
270 }
271 }
272
273 fn backfill_chunks(&self) -> bool {
276 let Some(embedder) = self.embedder.as_deref() else {
277 return false;
278 };
279 let embed = |texts: &[&str]| embedder.embed_batch(texts);
280 match self.store.backfill_chunk_embeddings(
281 &embed,
282 CHUNK_MEMORIES_PER_TICK,
283 CHUNK_TICK_BUDGET,
284 ) {
285 Ok(r) => {
286 if r.memories > 0 || r.failed > 0 {
287 tracing::info!(
288 "chunk backfill: memories={} chunks={} failed={} pending={}",
289 r.memories,
290 r.chunks,
291 r.failed,
292 r.pending
293 );
294 }
295 r.memories > 0
296 }
297 Err(e) => {
298 tracing::warn!("chunk backfill failed: {e:#}");
299 false
300 }
301 }
302 }
303
304 fn drain(&mut self) -> Option<usize> {
307 let store = &self.store;
308 let writer = &mut self.writer;
309 let result = store.outbox_pending().and_then(|pending| {
310 if pending == 0 {
311 Ok(0)
312 } else {
313 writer.with(|w| w.drain(store))
314 }
315 });
316 match result {
317 Ok(0) => {
318 self.health.drained(0, Utc::now());
319 Some(0)
320 }
321 Ok(n) => {
322 tracing::info!("index drain applied {n} outbox rows");
323 self.reload_reader();
324 match self.store.outbox_pending() {
325 Ok(pending) => self.health.drained(pending, Utc::now()),
326 Err(e) => tracing::warn!("counting the index outbox failed: {e:#}"),
327 }
328 Some(n)
329 }
330 Err(e) => {
331 tracing::warn!("index drain failed: {e:#}");
332 self.health.drain_failed(&format!("{e:#}"), Utc::now());
333 None
334 }
335 }
336 }
337
338 fn repair(&mut self, source: &str, dry_run: bool) -> Result<MaintenanceSummary> {
339 let started = Instant::now();
340 let store = &self.store;
341 let embedder = self.embedder.as_deref();
342 let repair = |w: &mut Indexes| run_repair(store, Some(w), embedder, dry_run);
343 let result = match self.writer.open().map(|_| ()) {
344 Ok(()) => self.writer.with(repair),
345 Err(e) => {
346 tracing::warn!(
347 "index writer unavailable ({e:#}); repairing the store without the indexes"
348 );
349 match run_repair(store, None, embedder, dry_run) {
350 Ok(summary) => log_stats(source, &summary.stats),
351 Err(store_error) => {
352 tracing::warn!("{source} store repair failed: {store_error:#}")
353 }
354 }
355 Err(e
356 .context("the index writer could not reopen, so the indexes were not repaired"))
357 }
358 };
359 match &result {
360 Ok(summary) => {
361 self.health.repaired(Utc::now());
362 log_stats(source, &summary.stats);
363 tracing::info!(
364 "{source} store repair finished in {:.1}s; drift remaining: {:?}",
365 started.elapsed().as_secs_f64(),
366 summary.report_after.drift()
367 );
368 if !dry_run {
369 self.reload_reader();
370 let mut guard = match self.summary.write() {
371 Ok(g) => g,
372 Err(poisoned) => poisoned.into_inner(),
373 };
374 *guard = Some(summary.clone());
375 drop(guard);
376 let threshold = self.near_duplicate_threshold;
377 let near_duplicates = &self.near_duplicates;
378 let counted = self.writer.with(|w| {
379 refresh_count(store, w, threshold, near_duplicates);
380 Ok(())
381 });
382 if let Err(e) = counted {
383 tracing::warn!("near-duplicate count failed: {e:#}");
384 }
385 if summary.stats.index_rebuilt {
386 residency::release_free_heap();
387 }
388 }
389 }
390 Err(e) => {
391 tracing::warn!("{source} store repair failed: {e:#}");
392 self.health.repair_failed(&format!("{e:#}"), Utc::now());
393 }
394 }
395 result
396 }
397
398 fn compact_indexes(&mut self) -> Result<()> {
399 let store = &self.store;
400 let took = self.writer.with(|w| {
401 while w.drain(store)? > 0 {}
402 let started = Instant::now();
403 w.compact()?;
404 Ok(started.elapsed())
405 })?;
406 self.reload_reader();
407 residency::release_free_heap();
408 tracing::info!("lexical index compacted in {:.1}s", took.as_secs_f64());
409 Ok(())
410 }
411
412 fn reload_reader(&self) {
413 let mut guard = match self.reader.write() {
414 Ok(g) => g,
415 Err(poisoned) => poisoned.into_inner(),
416 };
417 if let Err(e) = guard.reload_view() {
418 tracing::warn!("index reader reload failed: {e:#}");
419 }
420 }
421}
422
423#[cfg(test)]
424mod tests {
425 use std::fs::Permissions;
426 use std::os::unix::fs::PermissionsExt;
427
428 use super::*;
429 use second_brain_core::schema::{Memory, MemoryType};
430 use second_brain_core::store::Store;
431
432 #[test]
433 fn a_fresh_throttle_scans_on_the_first_idle_tick() {
434 let now = Instant::now();
435 assert!(ChunkThrottle::new(now).due(now));
436 }
437
438 #[test]
439 fn a_scan_that_finds_nothing_waits_out_the_idle_interval() {
440 let now = Instant::now();
441 let mut t = ChunkThrottle::new(now);
442 t.after_backfill(now, false);
443 assert!(!t.due(now + CHUNK_IDLE_RESCAN - Duration::from_secs(1)));
444 assert!(t.due(now + CHUNK_IDLE_RESCAN));
445 }
446
447 #[test]
448 fn progress_keeps_the_next_idle_tick_scanning() {
449 let now = Instant::now();
450 let mut t = ChunkThrottle::new(now);
451 t.after_backfill(now, true);
452 assert!(t.due(now));
453 }
454
455 #[test]
456 fn an_applied_drain_resets_the_idle_wait() {
457 let now = Instant::now();
458 let mut t = ChunkThrottle::new(now);
459 t.after_backfill(now, false);
460 t.after_drain(now, 0);
461 assert!(!t.due(now));
462 t.after_drain(now, 3);
463 assert!(t.due(now));
464 }
465
466 fn remember(store: &SqliteStore, content: &str) -> uuid::Uuid {
467 let m = Memory::new(
468 content.to_string(),
469 MemoryType::Semantic,
470 "test".to_string(),
471 String::new(),
472 );
473 store.store_memory(&m).unwrap();
474 m.id
475 }
476
477 fn drain_loop(
478 store: Arc<SqliteStore>,
479 reader: Arc<RwLock<Indexes>>,
480 writer: Indexes,
481 ) -> DrainLoop {
482 let (_tx, requests) = std::sync::mpsc::channel();
483 let claim = IndexClaim::acquire(writer.dir()).unwrap();
484 DrainLoop {
485 store,
486 reader,
487 writer: WriterSlot::new(writer, claim),
488 embedder: None,
489 summary: Arc::new(RwLock::new(None)),
490 near_duplicates: Arc::new(RwLock::new(None)),
491 near_duplicate_threshold: 0.97,
492 config: MaintenanceConfig {
493 enabled: false,
494 interval_secs: 3600,
495 },
496 requests,
497 health: Arc::new(Health::new(Utc::now(), None)),
498 }
499 }
500
501 fn idle_loop(dir: &std::path::Path) -> DrainLoop {
502 let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
503 let mut writer = Indexes::open(dir).unwrap();
504 writer.persist().unwrap();
505 let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir).unwrap()));
506 drain_loop(store, reader, writer)
507 }
508
509 #[test]
510 fn idleness_counts_from_when_an_operation_finishes() {
511 let dir = tempfile::tempdir().unwrap();
512 let mut lp = idle_loop(dir.path());
513 let mut finishing = None;
514 lp.writer
515 .with(|_| {
516 finishing = Some(Instant::now());
517 Ok(())
518 })
519 .unwrap();
520 let finishing = finishing.unwrap();
521 assert!(lp.writer.last_used >= finishing);
522 assert!(
523 !lp.writer
524 .release_if_idle(finishing + WRITER_IDLE - Duration::from_secs(1))
525 );
526 }
527
528 #[test]
529 fn a_writer_holding_changes_its_files_lack_is_never_released() {
530 let dir = tempfile::tempdir().unwrap();
531 let mut lp = idle_loop(dir.path());
532 remember(&lp.store, "the zanzibar ferry leaves at noon");
533 let tantivy = dir.path().join("tantivy");
534 let set_mode =
535 |mode| std::fs::set_permissions(&tantivy, Permissions::from_mode(mode)).unwrap();
536 set_mode(0o500);
537 let drained = lp.drain();
538 set_mode(0o700);
539 assert_eq!(
540 drained, None,
541 "the drain should fail to write the lexical index"
542 );
543 assert!(
544 !lp.writer
545 .release_if_idle(lp.writer.last_used + WRITER_IDLE * 10)
546 );
547 lp.repair("test", true).unwrap();
548 assert!(!lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
549
550 assert_eq!(lp.drain(), Some(1));
551 assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
552 }
553
554 #[test]
555 fn a_failure_that_left_the_files_current_still_releases() {
556 let dir = tempfile::tempdir().unwrap();
557 let mut lp = idle_loop(dir.path());
558 let failed: Result<()> = lp.writer.with(|_| anyhow::bail!("a store step failed"));
559 assert!(failed.is_err());
560 assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
561 }
562
563 #[test]
564 fn a_writer_that_cannot_reopen_fails_the_repair() {
565 let dir = tempfile::tempdir().unwrap();
566 let mut lp = idle_loop(dir.path());
567 assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
568 let blocker = dir.path().join("not-a-dir");
569 std::fs::write(&blocker, b"x").unwrap();
570 lp.writer.dir = blocker.join("indexes");
571
572 let err = lp.repair("test", false).unwrap_err();
573 assert!(
574 format!("{err:#}").contains("indexes were not repaired"),
575 "{err:#}"
576 );
577 assert!(lp.writer.memory().is_none());
578 }
579
580 #[test]
581 fn an_idle_writer_is_released_and_reopened_by_the_next_write() {
582 let dir = tempfile::tempdir().unwrap();
583 let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
584 let mut writer = Indexes::open(dir.path()).unwrap();
585 writer.persist().unwrap();
586 let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir.path()).unwrap()));
587 let mut lp = drain_loop(store.clone(), reader.clone(), writer);
588
589 let first = remember(&store, "the zanzibar ferry leaves at noon");
590 assert_eq!(lp.drain(), Some(1));
591 assert!(lp.writer.memory().is_some());
592
593 let used = lp.writer.last_used;
594 assert!(
595 !lp.writer
596 .release_if_idle(used + WRITER_IDLE - Duration::from_secs(1))
597 );
598 assert!(lp.writer.memory().is_some());
599 assert!(lp.writer.release_if_idle(used + WRITER_IDLE));
600 assert!(lp.writer.memory().is_none());
601
602 assert_eq!(lp.drain(), Some(0));
603 assert!(
604 lp.writer.memory().is_none(),
605 "an empty tick reopened the writer"
606 );
607
608 let second = remember(&store, "the kilimanjaro trail opens in june");
609 assert_eq!(lp.drain(), Some(1));
610 assert!(lp.writer.memory().is_some());
611 assert_eq!(lp.writer.open().unwrap().lexical_len(), 2);
612
613 let guard = reader.read().unwrap();
614 let hits = |q: &str| -> Vec<uuid::Uuid> {
615 guard
616 .bm25_search(q, 5)
617 .unwrap()
618 .into_iter()
619 .map(|(id, _)| id)
620 .collect()
621 };
622 assert_eq!(hits("kilimanjaro"), vec![second]);
623 assert_eq!(hits("zanzibar"), vec![first]);
624 }
625}