Skip to main content

second_brain_api/
maintenance.rs

1//! The drain thread: applies the index outbox every few seconds and, when
2//! the outbox is quiet and the interval has passed, repairs the store with
3//! the writable indexes it owns. On-demand repairs arrive over a channel so
4//! the same thread services them; two writers never exist.
5
6use std::path::PathBuf;
7use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
8use std::sync::{Arc, RwLock};
9use std::time::{Duration, Instant};
10
11use anyhow::Result;
12use chrono::Utc;
13use second_brain_core::config::MaintenanceConfig;
14use second_brain_core::embedding::Embedder;
15use second_brain_core::index::{IndexClaim, Indexes};
16use second_brain_core::maintenance::{MaintenanceSummary, RepairStats};
17use second_brain_core::residency::{self, IndexMemory};
18use second_brain_core::sqlite_store::SqliteStore;
19
20use crate::health::Health;
21use crate::near_duplicates::{NearDupCount, refresh_count};
22
23const DRAIN_INTERVAL: Duration = Duration::from_secs(5);
24
25const WRITER_IDLE: Duration = Duration::from_secs(60);
26
27/// Finding chunk work splits every live row, so once a scan comes back
28/// empty-handed the next one waits this long unless a drain applies writes.
29const CHUNK_IDLE_RESCAN: Duration = Duration::from_secs(600);
30/// Per idle tick, so the drain and repair behind it are never held up long.
31const CHUNK_TICK_BUDGET: Duration = Duration::from_secs(3);
32/// Bounds what one tick holds in memory: at most 16 chunks per memory.
33const CHUNK_MEMORIES_PER_TICK: usize = 32;
34
35/// When the drain thread may next look for memories needing chunks.
36struct ChunkThrottle {
37    next_scan: Instant,
38}
39
40impl ChunkThrottle {
41    fn new(now: Instant) -> Self {
42        Self { next_scan: now }
43    }
44
45    fn due(&self, now: Instant) -> bool {
46        now >= self.next_scan
47    }
48
49    fn after_backfill(&mut self, now: Instant, progressed: bool) {
50        self.next_scan = if progressed {
51            now
52        } else {
53            now + CHUNK_IDLE_RESCAN
54        };
55    }
56
57    /// Applied outbox rows may carry content changes that need re-chunking.
58    fn after_drain(&mut self, now: Instant, applied: usize) {
59        if applied > 0 {
60            self.next_scan = now;
61        }
62    }
63}
64
65pub struct RepairRequest {
66    pub dry_run: bool,
67    pub reply: Sender<Result<MaintenanceSummary>>,
68}
69
70/// Work only the drain thread may do, because it owns the writable indexes.
71pub enum DrainRequest {
72    Repair(RepairRequest),
73    /// Applies the whole outbox, then rewrites the lexical index into fresh
74    /// segments (see `Indexes::compact`).
75    CompactIndexes(Sender<Result<()>>),
76}
77
78/// Runs a repair and re-measures the store afterwards, on whatever index
79/// handle the caller owns (the writer on the drain thread, none when the
80/// indexes never opened or a released writer could not reopen).
81pub fn run_repair(
82    store: &SqliteStore,
83    mut indexes: Option<&mut Indexes>,
84    embedder: Option<&Embedder>,
85    dry_run: bool,
86) -> Result<MaintenanceSummary> {
87    let embed = embedder.map(|e| move |text: &str| e.embed(text));
88    let embed_ref = embed
89        .as_ref()
90        .map(|f| f as &dyn Fn(&str) -> Result<Vec<f32>>);
91    let stats = store.repair(indexes.as_deref_mut(), embed_ref, dry_run)?;
92    let report_after = store.integrity_report(indexes.as_deref())?;
93    Ok(MaintenanceSummary {
94        ran_at: Utc::now(),
95        stats,
96        report_after,
97    })
98}
99
100fn log_stats(source: &str, stats: &RepairStats) {
101    if stats.is_clean() {
102        tracing::info!("{source} store repair: nothing to do");
103    } else {
104        tracing::info!("{source} store repair: {}", summarize(stats));
105    }
106}
107
108fn summarize(s: &RepairStats) -> String {
109    let mut parts = Vec::new();
110    let mut push = |name: &str, n: u64| {
111        if n > 0 {
112            parts.push(format!("{name}={n}"));
113        }
114    };
115    push("duplicates_superseded", s.duplicates_superseded);
116    push("markers_inherited", s.distillation_markers_inherited);
117    push("orphaned_markers_dropped", s.orphaned_markers_dropped);
118    push("raws_retired", s.raws_retired);
119    push("relations_pruned", s.relations_pruned);
120    push("vec_keys_deleted", s.vec_keys_deleted);
121    push("sync_log_rows_deleted", s.sync_log_rows_deleted);
122    push("stale_cursors_dropped", s.stale_cursors_dropped);
123    push("embeddings_backfilled", s.embeddings_backfilled);
124    push("embedding_failures", s.embedding_failures);
125    if s.vec_keys_gc_skipped {
126        parts.push("vec_keys_gc=skipped".to_string());
127    }
128    if s.index_rebuilt {
129        parts.push("index=rebuilt".to_string());
130    }
131    if s.dry_run {
132        parts.push("dry_run".to_string());
133    }
134    parts.join(" ")
135}
136
137/// The drain thread's writable indexes. usearch cannot mutate a viewed index,
138/// so an open writer holds its own anonymous copy of the vectors beside the
139/// reader's mmap; the slot drops it once `WRITER_IDLE` has passed since its
140/// last operation finished, and reopens it from the persisted files when work
141/// arrives. A writer holding changes its files lack is kept, because a drain
142/// commits its store-side deletes before it persists, so only that writer
143/// still holds those removals.
144pub struct WriterSlot {
145    dir: PathBuf,
146    open: Option<Indexes>,
147    last_used: Instant,
148    _claim: IndexClaim,
149}
150
151impl WriterSlot {
152    pub fn new(writer: Indexes, claim: IndexClaim) -> Self {
153        crate::heap::record_index_writer(Some(writer.memory()));
154        Self {
155            dir: writer.dir().to_path_buf(),
156            open: Some(writer),
157            last_used: Instant::now(),
158            _claim: claim,
159        }
160    }
161
162    fn open(&mut self) -> Result<&mut Indexes> {
163        if self.open.is_none() {
164            let writer = Indexes::open(&self.dir)?;
165            tracing::info!("index writer reopened at {}", self.dir.display());
166            crate::heap::record_index_writer(Some(writer.memory()));
167            self.open = Some(writer);
168        }
169        Ok(self.open.as_mut().expect("writer opened above"))
170    }
171
172    fn with<R>(&mut self, op: impl FnOnce(&mut Indexes) -> Result<R>) -> Result<R> {
173        let result = op(self.open()?);
174        self.last_used = Instant::now();
175        crate::heap::record_index_writer(self.memory());
176        result
177    }
178
179    fn release_if_idle(&mut self, now: Instant) -> bool {
180        let current = self
181            .open
182            .as_ref()
183            .is_some_and(|w| !w.has_unpersisted_changes());
184        if !current || now.saturating_duration_since(self.last_used) < WRITER_IDLE {
185            return false;
186        }
187        if let Some(writer) = self.open.take()
188            && let Err(e) = writer.wait_for_merges()
189        {
190            tracing::warn!("index writer released without its merges: {e:#}");
191        }
192        crate::heap::record_index_writer(None);
193        residency::release_free_heap();
194        tracing::info!(
195            "index writer released after {}s idle",
196            WRITER_IDLE.as_secs()
197        );
198        true
199    }
200
201    fn memory(&self) -> Option<IndexMemory> {
202        self.open.as_ref().map(Indexes::memory)
203    }
204}
205
206pub struct DrainLoop {
207    pub store: Arc<SqliteStore>,
208    pub reader: Arc<RwLock<Indexes>>,
209    pub writer: WriterSlot,
210    pub embedder: Option<Arc<Embedder>>,
211    pub summary: Arc<RwLock<Option<MaintenanceSummary>>>,
212    pub near_duplicates: Arc<RwLock<Option<NearDupCount>>>,
213    pub near_duplicate_threshold: f32,
214    pub config: MaintenanceConfig,
215    pub requests: Receiver<DrainRequest>,
216    pub health: Arc<Health>,
217}
218
219impl DrainLoop {
220    pub fn spawn(self) {
221        std::thread::Builder::new()
222            .name("index-drain".to_string())
223            .spawn(move || self.run())
224            .expect("spawning the index drain thread");
225    }
226
227    fn run(mut self) {
228        let mut last_repair: Option<Instant> = None;
229        let mut chunks = ChunkThrottle::new(Instant::now());
230        loop {
231            match self.requests.recv_timeout(DRAIN_INTERVAL) {
232                Ok(DrainRequest::Repair(request)) => {
233                    let result = self.repair("requested", request.dry_run);
234                    if result.is_ok() && !request.dry_run {
235                        last_repair = Some(Instant::now());
236                    }
237                    // A caller that gave up waiting is not an error here.
238                    let _ = request.reply.send(result);
239                }
240                Ok(DrainRequest::CompactIndexes(reply)) => {
241                    let _ = reply.send(self.compact_indexes());
242                }
243                Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {
244                    let applied = self.drain();
245                    chunks.after_drain(Instant::now(), applied.unwrap_or(0));
246                    let due = last_repair.is_none_or(|t| {
247                        t.elapsed() >= Duration::from_secs(self.config.interval_secs)
248                    });
249                    if applied == Some(0) && self.config.enabled && due {
250                        if self.repair("scheduled", false).is_ok() {
251                            last_repair = Some(Instant::now());
252                        } else {
253                            // Retry sooner than the full interval, but not
254                            // every tick against a store that keeps failing.
255                            last_repair = Some(
256                                Instant::now()
257                                    - Duration::from_secs(
258                                        self.config.interval_secs.saturating_sub(600),
259                                    ),
260                            );
261                        }
262                    }
263                    if applied == Some(0) && chunks.due(Instant::now()) {
264                        let progressed = self.backfill_chunks();
265                        chunks.after_backfill(Instant::now(), progressed);
266                    }
267                }
268            }
269            self.writer.release_if_idle(Instant::now());
270        }
271    }
272
273    /// Embeds pending chunk sets within one tick's budget; true when any set
274    /// was stored, so the next idle tick keeps going.
275    fn backfill_chunks(&self) -> bool {
276        let Some(embedder) = self.embedder.as_deref() else {
277            return false;
278        };
279        let embed = |texts: &[&str]| embedder.embed_batch(texts);
280        match self.store.backfill_chunk_embeddings(
281            &embed,
282            CHUNK_MEMORIES_PER_TICK,
283            CHUNK_TICK_BUDGET,
284        ) {
285            Ok(r) => {
286                if r.memories > 0 || r.failed > 0 {
287                    tracing::info!(
288                        "chunk backfill: memories={} chunks={} failed={} pending={}",
289                        r.memories,
290                        r.chunks,
291                        r.failed,
292                        r.pending
293                    );
294                }
295                r.memories > 0
296            }
297            Err(e) => {
298                tracing::warn!("chunk backfill failed: {e:#}");
299                false
300            }
301        }
302    }
303
304    /// Applies pending outbox rows; None when the drain itself failed. An
305    /// empty outbox never touches the writer, so a released one stays closed.
306    fn drain(&mut self) -> Option<usize> {
307        let store = &self.store;
308        let writer = &mut self.writer;
309        let result = store.outbox_pending().and_then(|pending| {
310            if pending == 0 {
311                Ok(0)
312            } else {
313                writer.with(|w| w.drain(store))
314            }
315        });
316        match result {
317            Ok(0) => {
318                self.health.drained(0, Utc::now());
319                Some(0)
320            }
321            Ok(n) => {
322                tracing::info!("index drain applied {n} outbox rows");
323                self.reload_reader();
324                match self.store.outbox_pending() {
325                    Ok(pending) => self.health.drained(pending, Utc::now()),
326                    Err(e) => tracing::warn!("counting the index outbox failed: {e:#}"),
327                }
328                Some(n)
329            }
330            Err(e) => {
331                tracing::warn!("index drain failed: {e:#}");
332                self.health.drain_failed(&format!("{e:#}"), Utc::now());
333                None
334            }
335        }
336    }
337
338    fn repair(&mut self, source: &str, dry_run: bool) -> Result<MaintenanceSummary> {
339        let started = Instant::now();
340        let store = &self.store;
341        let embedder = self.embedder.as_deref();
342        let repair = |w: &mut Indexes| run_repair(store, Some(w), embedder, dry_run);
343        let result = match self.writer.open().map(|_| ()) {
344            Ok(()) => self.writer.with(repair),
345            Err(e) => {
346                tracing::warn!(
347                    "index writer unavailable ({e:#}); repairing the store without the indexes"
348                );
349                match run_repair(store, None, embedder, dry_run) {
350                    Ok(summary) => log_stats(source, &summary.stats),
351                    Err(store_error) => {
352                        tracing::warn!("{source} store repair failed: {store_error:#}")
353                    }
354                }
355                Err(e
356                    .context("the index writer could not reopen, so the indexes were not repaired"))
357            }
358        };
359        match &result {
360            Ok(summary) => {
361                self.health.repaired(Utc::now());
362                log_stats(source, &summary.stats);
363                tracing::info!(
364                    "{source} store repair finished in {:.1}s; drift remaining: {:?}",
365                    started.elapsed().as_secs_f64(),
366                    summary.report_after.drift()
367                );
368                if !dry_run {
369                    self.reload_reader();
370                    let mut guard = match self.summary.write() {
371                        Ok(g) => g,
372                        Err(poisoned) => poisoned.into_inner(),
373                    };
374                    *guard = Some(summary.clone());
375                    drop(guard);
376                    let threshold = self.near_duplicate_threshold;
377                    let near_duplicates = &self.near_duplicates;
378                    let counted = self.writer.with(|w| {
379                        refresh_count(store, w, threshold, near_duplicates);
380                        Ok(())
381                    });
382                    if let Err(e) = counted {
383                        tracing::warn!("near-duplicate count failed: {e:#}");
384                    }
385                    if summary.stats.index_rebuilt {
386                        residency::release_free_heap();
387                    }
388                }
389            }
390            Err(e) => {
391                tracing::warn!("{source} store repair failed: {e:#}");
392                self.health.repair_failed(&format!("{e:#}"), Utc::now());
393            }
394        }
395        result
396    }
397
398    fn compact_indexes(&mut self) -> Result<()> {
399        let store = &self.store;
400        let took = self.writer.with(|w| {
401            while w.drain(store)? > 0 {}
402            let started = Instant::now();
403            w.compact()?;
404            Ok(started.elapsed())
405        })?;
406        self.reload_reader();
407        residency::release_free_heap();
408        tracing::info!("lexical index compacted in {:.1}s", took.as_secs_f64());
409        Ok(())
410    }
411
412    fn reload_reader(&self) {
413        let mut guard = match self.reader.write() {
414            Ok(g) => g,
415            Err(poisoned) => poisoned.into_inner(),
416        };
417        if let Err(e) = guard.reload_view() {
418            tracing::warn!("index reader reload failed: {e:#}");
419        }
420    }
421}
422
423#[cfg(test)]
424mod tests {
425    use std::fs::Permissions;
426    use std::os::unix::fs::PermissionsExt;
427
428    use super::*;
429    use second_brain_core::schema::{Memory, MemoryType};
430    use second_brain_core::store::Store;
431
432    #[test]
433    fn a_fresh_throttle_scans_on_the_first_idle_tick() {
434        let now = Instant::now();
435        assert!(ChunkThrottle::new(now).due(now));
436    }
437
438    #[test]
439    fn a_scan_that_finds_nothing_waits_out_the_idle_interval() {
440        let now = Instant::now();
441        let mut t = ChunkThrottle::new(now);
442        t.after_backfill(now, false);
443        assert!(!t.due(now + CHUNK_IDLE_RESCAN - Duration::from_secs(1)));
444        assert!(t.due(now + CHUNK_IDLE_RESCAN));
445    }
446
447    #[test]
448    fn progress_keeps_the_next_idle_tick_scanning() {
449        let now = Instant::now();
450        let mut t = ChunkThrottle::new(now);
451        t.after_backfill(now, true);
452        assert!(t.due(now));
453    }
454
455    #[test]
456    fn an_applied_drain_resets_the_idle_wait() {
457        let now = Instant::now();
458        let mut t = ChunkThrottle::new(now);
459        t.after_backfill(now, false);
460        t.after_drain(now, 0);
461        assert!(!t.due(now));
462        t.after_drain(now, 3);
463        assert!(t.due(now));
464    }
465
466    fn remember(store: &SqliteStore, content: &str) -> uuid::Uuid {
467        let m = Memory::new(
468            content.to_string(),
469            MemoryType::Semantic,
470            "test".to_string(),
471            String::new(),
472        );
473        store.store_memory(&m).unwrap();
474        m.id
475    }
476
477    fn drain_loop(
478        store: Arc<SqliteStore>,
479        reader: Arc<RwLock<Indexes>>,
480        writer: Indexes,
481    ) -> DrainLoop {
482        let (_tx, requests) = std::sync::mpsc::channel();
483        let claim = IndexClaim::acquire(writer.dir()).unwrap();
484        DrainLoop {
485            store,
486            reader,
487            writer: WriterSlot::new(writer, claim),
488            embedder: None,
489            summary: Arc::new(RwLock::new(None)),
490            near_duplicates: Arc::new(RwLock::new(None)),
491            near_duplicate_threshold: 0.97,
492            config: MaintenanceConfig {
493                enabled: false,
494                interval_secs: 3600,
495            },
496            requests,
497            health: Arc::new(Health::new(Utc::now(), None)),
498        }
499    }
500
501    fn idle_loop(dir: &std::path::Path) -> DrainLoop {
502        let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
503        let mut writer = Indexes::open(dir).unwrap();
504        writer.persist().unwrap();
505        let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir).unwrap()));
506        drain_loop(store, reader, writer)
507    }
508
509    #[test]
510    fn idleness_counts_from_when_an_operation_finishes() {
511        let dir = tempfile::tempdir().unwrap();
512        let mut lp = idle_loop(dir.path());
513        let mut finishing = None;
514        lp.writer
515            .with(|_| {
516                finishing = Some(Instant::now());
517                Ok(())
518            })
519            .unwrap();
520        let finishing = finishing.unwrap();
521        assert!(lp.writer.last_used >= finishing);
522        assert!(
523            !lp.writer
524                .release_if_idle(finishing + WRITER_IDLE - Duration::from_secs(1))
525        );
526    }
527
528    #[test]
529    fn a_writer_holding_changes_its_files_lack_is_never_released() {
530        let dir = tempfile::tempdir().unwrap();
531        let mut lp = idle_loop(dir.path());
532        remember(&lp.store, "the zanzibar ferry leaves at noon");
533        let tantivy = dir.path().join("tantivy");
534        let set_mode =
535            |mode| std::fs::set_permissions(&tantivy, Permissions::from_mode(mode)).unwrap();
536        set_mode(0o500);
537        let drained = lp.drain();
538        set_mode(0o700);
539        assert_eq!(
540            drained, None,
541            "the drain should fail to write the lexical index"
542        );
543        assert!(
544            !lp.writer
545                .release_if_idle(lp.writer.last_used + WRITER_IDLE * 10)
546        );
547        lp.repair("test", true).unwrap();
548        assert!(!lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
549
550        assert_eq!(lp.drain(), Some(1));
551        assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
552    }
553
554    #[test]
555    fn a_failure_that_left_the_files_current_still_releases() {
556        let dir = tempfile::tempdir().unwrap();
557        let mut lp = idle_loop(dir.path());
558        let failed: Result<()> = lp.writer.with(|_| anyhow::bail!("a store step failed"));
559        assert!(failed.is_err());
560        assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
561    }
562
563    #[test]
564    fn a_writer_that_cannot_reopen_fails_the_repair() {
565        let dir = tempfile::tempdir().unwrap();
566        let mut lp = idle_loop(dir.path());
567        assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
568        let blocker = dir.path().join("not-a-dir");
569        std::fs::write(&blocker, b"x").unwrap();
570        lp.writer.dir = blocker.join("indexes");
571
572        let err = lp.repair("test", false).unwrap_err();
573        assert!(
574            format!("{err:#}").contains("indexes were not repaired"),
575            "{err:#}"
576        );
577        assert!(lp.writer.memory().is_none());
578    }
579
580    #[test]
581    fn an_idle_writer_is_released_and_reopened_by_the_next_write() {
582        let dir = tempfile::tempdir().unwrap();
583        let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
584        let mut writer = Indexes::open(dir.path()).unwrap();
585        writer.persist().unwrap();
586        let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir.path()).unwrap()));
587        let mut lp = drain_loop(store.clone(), reader.clone(), writer);
588
589        let first = remember(&store, "the zanzibar ferry leaves at noon");
590        assert_eq!(lp.drain(), Some(1));
591        assert!(lp.writer.memory().is_some());
592
593        let used = lp.writer.last_used;
594        assert!(
595            !lp.writer
596                .release_if_idle(used + WRITER_IDLE - Duration::from_secs(1))
597        );
598        assert!(lp.writer.memory().is_some());
599        assert!(lp.writer.release_if_idle(used + WRITER_IDLE));
600        assert!(lp.writer.memory().is_none());
601
602        assert_eq!(lp.drain(), Some(0));
603        assert!(
604            lp.writer.memory().is_none(),
605            "an empty tick reopened the writer"
606        );
607
608        let second = remember(&store, "the kilimanjaro trail opens in june");
609        assert_eq!(lp.drain(), Some(1));
610        assert!(lp.writer.memory().is_some());
611        assert_eq!(lp.writer.open().unwrap().lexical_len(), 2);
612
613        let guard = reader.read().unwrap();
614        let hits = |q: &str| -> Vec<uuid::Uuid> {
615            guard
616                .bm25_search(q, 5)
617                .unwrap()
618                .into_iter()
619                .map(|(id, _)| id)
620                .collect()
621        };
622        assert_eq!(hits("kilimanjaro"), vec![second]);
623        assert_eq!(hits("zanzibar"), vec![first]);
624    }
625}