Skip to main content

second_brain_api/
watcher.rs

1use std::collections::{HashMap, HashSet};
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4use std::time::{Duration, Instant};
5
6use chrono::{DateTime, Utc};
7use notify::{EventKind, RecursiveMode, Watcher as _};
8
9use second_brain_core::embedding::Embedder;
10use second_brain_core::sqlite_store::SqliteStore;
11use second_brain_core::store::StoreAdmin;
12use second_brain_core::transcript_archive::TranscriptArchive;
13
14use crate::backfill::{LiveIngest, transcript_mtimes, watched_files};
15use crate::bootstrap::Bootstrap;
16use crate::clock_gate::ClockGate;
17use crate::health::Health;
18use crate::ingest::{FileOutcome, capture_transcript, ingest_memory_file, is_memory_file};
19
20const INITIAL_BACKOFF: Duration = Duration::from_secs(30);
21const MAX_BACKOFF: Duration = Duration::from_secs(600);
22const MISSING_DIR_RECHECK: Duration = Duration::from_secs(300);
23const SETTLE: Duration = Duration::from_secs(10);
24/// How often the watch dir is walked for its newest transcript, so capture
25/// lag shows even when change events stop arriving.
26pub const SCAN_INTERVAL: Duration = Duration::from_secs(300);
27
28pub struct Watcher {
29    pub sqlite: Arc<SqliteStore>,
30    pub embedder: Option<Arc<Embedder>>,
31    pub machine_id: String,
32    pub health: Arc<Health>,
33    pub archive: Option<Arc<TranscriptArchive>>,
34    pub live: Arc<LiveIngest>,
35    pub clock: Arc<ClockGate>,
36    pub bootstrap: Arc<Bootstrap>,
37}
38
39impl Watcher {
40    pub fn spawn(self) {
41        std::thread::spawn(move || {
42            let mut backoff = INITIAL_BACKOFF;
43            loop {
44                match self.run() {
45                    // run() only returns Ok when the watch dir is missing; it may be
46                    // created later (fresh machine, first session), so keep checking
47                    Ok(()) => std::thread::sleep(MISSING_DIR_RECHECK),
48                    Err(e) => {
49                        self.health.watcher_failed(&format!("{e:#}"), Utc::now());
50                        tracing::error!(
51                            "watcher failed: {e}; restarting in {}s",
52                            backoff.as_secs()
53                        );
54                        std::thread::sleep(backoff);
55                        backoff = next_backoff(backoff);
56                    }
57                }
58            }
59        });
60    }
61
62    fn run(&self) -> anyhow::Result<()> {
63        let watch_dir = projects_dir();
64        let health = &self.health;
65
66        if !watch_dir.exists() {
67            tracing::info!(
68                "watcher: {} does not exist, re-checking periodically",
69                watch_dir.display()
70            );
71            health.watcher_started(Utc::now());
72            health.transcripts_scanned(None, Utc::now());
73            if let Err(e) = self.bootstrap.ingest_done(&self.sqlite) {
74                tracing::warn!("watcher: cannot record the post-bootstrap ingest pass: {e:#}");
75            }
76            return Ok(());
77        }
78
79        let (tx, rx) = std::sync::mpsc::channel();
80
81        let mut watcher = notify::recommended_watcher(move |res: Result<notify::Event, _>| {
82            if let Ok(event) = res
83                && matches!(event.kind, EventKind::Create(_) | EventKind::Modify(_))
84            {
85                for path in event.paths {
86                    let is_jsonl = path.extension().is_some_and(|e| e == "jsonl");
87                    if is_jsonl || is_memory_file(&path) {
88                        tx.send(path).ok();
89                    }
90                }
91            }
92        })?;
93
94        watcher.watch(&watch_dir, RecursiveMode::Recursive)?;
95        tracing::info!("watcher: monitoring {}", watch_dir.display());
96        health.watcher_started(Utc::now());
97        let mut next_scan = Instant::now();
98
99        let mut pending: HashMap<PathBuf, Instant> = HashMap::new();
100        let mut full_pass = FullPass::default();
101        let store: Arc<dyn StoreAdmin> = self.sqlite.clone();
102
103        loop {
104            if Instant::now() >= next_scan {
105                health.transcripts_scanned(newest_transcript(&watch_dir), Utc::now());
106                next_scan = Instant::now() + SCAN_INTERVAL;
107            }
108
109            while let Ok(path) = rx.try_recv() {
110                pending.insert(path, Instant::now());
111            }
112            full_pass.queue_if_due(&self.bootstrap, &watch_dir, &mut pending, Instant::now());
113
114            let open = self.clock.is_open() && !self.bootstrap.is_pending();
115            let ready = take_settled(&mut pending, Instant::now(), SETTLE, open);
116            let ingesting = self.live.ingesting(ready.len());
117            ingest_ready(
118                ready,
119                &mut pending,
120                &self.clock,
121                &self.bootstrap,
122                health,
123                |path| self.ingest(&store, path),
124            );
125            drop(ingesting);
126            full_pass.finish_if_ingested(&self.bootstrap, &self.sqlite, &pending);
127
128            std::thread::sleep(Duration::from_secs(2));
129        }
130    }
131
132    fn ingest(
133        &self,
134        store: &Arc<dyn StoreAdmin>,
135        path: &Path,
136    ) -> (anyhow::Result<FileOutcome>, Option<DateTime<Utc>>) {
137        let embedder = self.embedder.as_deref();
138        if path.extension().is_some_and(|e| e == "md") {
139            let result = ingest_memory_file(store, embedder, path, &self.machine_id);
140            (result, None)
141        } else {
142            let mtime = modified(path);
143            let archive = self.archive.as_deref();
144            let result = capture_transcript(store, embedder, archive, path, &self.machine_id);
145            (result, mtime)
146        }
147    }
148}
149
150fn next_backoff(prev: Duration) -> Duration {
151    (prev * 2).min(MAX_BACKOFF)
152}
153
154pub fn projects_dir() -> PathBuf {
155    let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
156    PathBuf::from(home).join(".claude").join("projects")
157}
158
159/// The one full pass over the watched files that follows a pairing
160/// bootstrap; it catches files touched while the bootstrap held ingest,
161/// which no change event will report again.
162#[derive(Default)]
163struct FullPass {
164    left: Option<HashSet<PathBuf>>,
165}
166
167impl FullPass {
168    fn queue_if_due(
169        &mut self,
170        bootstrap: &Bootstrap,
171        watch_dir: &Path,
172        pending: &mut HashMap<PathBuf, Instant>,
173        now: Instant,
174    ) {
175        if self.left.is_some() || !bootstrap.ingest_due() {
176            return;
177        }
178        let settled_at = now.checked_sub(SETTLE).unwrap_or(now);
179        let files: HashSet<PathBuf> = watched_files(watch_dir).into_iter().collect();
180        tracing::info!(
181            "watcher: ingesting {} files after the pairing bootstrap",
182            files.len()
183        );
184        for path in &files {
185            pending.insert(path.clone(), settled_at);
186        }
187        self.left = Some(files);
188    }
189
190    fn finish_if_ingested(
191        &mut self,
192        bootstrap: &Bootstrap,
193        sqlite: &SqliteStore,
194        pending: &HashMap<PathBuf, Instant>,
195    ) {
196        let Some(left) = &mut self.left else {
197            return;
198        };
199        left.retain(|path| pending.contains_key(path));
200        if !left.is_empty() {
201            return;
202        }
203        match bootstrap.ingest_done(sqlite) {
204            Ok(()) => self.left = None,
205            Err(e) => {
206                tracing::warn!("watcher: cannot record the post-bootstrap ingest pass: {e:#}")
207            }
208        }
209    }
210}
211
212fn ingest_ready(
213    ready: Vec<PathBuf>,
214    pending: &mut HashMap<PathBuf, Instant>,
215    clock: &ClockGate,
216    bootstrap: &Bootstrap,
217    health: &Health,
218    ingest: impl Fn(&Path) -> (anyhow::Result<FileOutcome>, Option<DateTime<Utc>>),
219) {
220    let mut ready = ready.into_iter();
221    while let Some(path) = ready.next() {
222        let closures = clock.closures();
223        if !clock.is_open() || bootstrap.is_pending() {
224            for path in std::iter::once(path).chain(ready.by_ref()) {
225                pending.insert(path, Instant::now());
226            }
227            break;
228        }
229        let (result, transcript_mtime) = ingest(&path);
230        match result {
231            Err(_) if clock.closures() != closures || !clock.is_open() => {
232                pending.insert(path, Instant::now());
233            }
234            Ok(FileOutcome::Captured) => health.ingested(transcript_mtime, Utc::now()),
235            Ok(FileOutcome::Skipped) => {}
236            Err(e) => health.ingest_failed(&format!("{e:#}"), Utc::now()),
237        }
238    }
239}
240
241/// A file waits while the clock gate holds writes, because its ingest could
242/// only fail, and while a pairing bootstrap holds ingest.
243fn take_settled(
244    pending: &mut HashMap<PathBuf, Instant>,
245    now: Instant,
246    settle: Duration,
247    open: bool,
248) -> Vec<PathBuf> {
249    if !open {
250        return Vec::new();
251    }
252    let ready: Vec<PathBuf> = pending
253        .iter()
254        .filter(|(_, when)| now.duration_since(**when) >= settle)
255        .map(|(p, _)| p.clone())
256        .collect();
257    for path in &ready {
258        pending.remove(path);
259    }
260    ready
261}
262
263fn modified(path: &Path) -> Option<DateTime<Utc>> {
264    std::fs::metadata(path)
265        .and_then(|m| m.modified())
266        .ok()
267        .map(DateTime::<Utc>::from)
268}
269
270fn newest_transcript(dir: &Path) -> Option<DateTime<Utc>> {
271    transcript_mtimes(dir)
272        .into_iter()
273        .filter_map(|(mtime, _)| mtime)
274        .max()
275        .map(DateTime::<Utc>::from)
276}
277
278#[cfg(test)]
279mod tests {
280    use std::sync::atomic::{AtomicBool, AtomicI64, AtomicUsize, Ordering};
281    use std::time::SystemTime;
282
283    use super::*;
284    use crate::clock_gate::TimeSignals;
285
286    #[test]
287    fn backoff_doubles_and_caps() {
288        let mut b = INITIAL_BACKOFF;
289        b = next_backoff(b);
290        assert_eq!(b, Duration::from_secs(60));
291        for _ in 0..10 {
292            b = next_backoff(b);
293        }
294        assert_eq!(b, MAX_BACKOFF);
295    }
296
297    #[test]
298    fn settled_files_stay_pending_while_the_clock_gate_is_closed() {
299        let start = Instant::now();
300        let settle = Duration::from_secs(10);
301        let mut pending = HashMap::from([
302            (PathBuf::from("settled.jsonl"), start),
303            (PathBuf::from("fresh.jsonl"), start + Duration::from_secs(5)),
304        ]);
305        let later = start + settle;
306
307        assert!(take_settled(&mut pending, later, settle, false).is_empty());
308        assert_eq!(pending.len(), 2);
309
310        let ready = take_settled(&mut pending, later, settle, true);
311        assert_eq!(ready, [PathBuf::from("settled.jsonl")]);
312        assert_eq!(
313            pending.keys().collect::<Vec<_>>(),
314            [&PathBuf::from("fresh.jsonl")]
315        );
316    }
317
318    struct SteppingClock {
319        offset: AtomicI64,
320        synced: AtomicBool,
321    }
322
323    impl TimeSignals for SteppingClock {
324        fn now_micros(&self) -> i64 {
325            Utc::now().timestamp_micros() + self.offset.load(Ordering::SeqCst)
326        }
327
328        fn ntp_synced(&self) -> bool {
329            self.synced.load(Ordering::SeqCst)
330        }
331
332        fn rtc_secs(&self) -> Option<i64> {
333            None
334        }
335    }
336
337    #[test]
338    fn a_file_whose_ingest_failed_as_the_gate_closed_is_retried_once_it_reopens() {
339        let dir = tempfile::tempdir().unwrap();
340        let signals = Arc::new(SteppingClock {
341            offset: AtomicI64::new(0),
342            synced: AtomicBool::new(true),
343        });
344        let clock = ClockGate::new(dir.path(), signals.clone(), 0);
345        assert!(clock.is_open());
346        let health = Health::new(Utc::now(), None);
347        let attempts = AtomicUsize::new(0);
348        let ingest = |_: &Path| {
349            if attempts.fetch_add(1, Ordering::SeqCst) > 0 {
350                return (Ok(FileOutcome::Captured), None);
351            }
352            signals.synced.store(false, Ordering::SeqCst);
353            signals.offset.store(-10_000_000, Ordering::SeqCst);
354            assert!(!clock.is_open(), "the backward step closes the gate");
355            (Err(anyhow::anyhow!("wall clock not yet trusted")), None)
356        };
357        let path = PathBuf::from("s.jsonl");
358        let mut pending = HashMap::new();
359        let settle = Duration::from_secs(10);
360
361        ingest_ready(
362            vec![path.clone()],
363            &mut pending,
364            &clock,
365            &Bootstrap::default(),
366            &health,
367            ingest,
368        );
369        assert!(
370            pending.contains_key(&path),
371            "a file the closing gate failed is kept pending"
372        );
373        let later = Instant::now() + settle;
374        assert!(take_settled(&mut pending, later, settle, clock.is_open()).is_empty());
375
376        signals.synced.store(true, Ordering::SeqCst);
377        clock.tick();
378        let ready = take_settled(&mut pending, later, settle, clock.is_open());
379        assert_eq!(ready, [path]);
380        ingest_ready(
381            ready,
382            &mut pending,
383            &clock,
384            &Bootstrap::default(),
385            &health,
386            ingest,
387        );
388        assert!(pending.is_empty());
389        assert_eq!(attempts.load(Ordering::SeqCst), 2);
390    }
391
392    #[test]
393    fn files_after_one_that_closed_the_gate_stay_pending_while_it_stays_closed() {
394        let dir = tempfile::tempdir().unwrap();
395        let signals = Arc::new(SteppingClock {
396            offset: AtomicI64::new(0),
397            synced: AtomicBool::new(true),
398        });
399        let clock = ClockGate::new(dir.path(), signals.clone(), 0);
400        assert!(clock.is_open());
401        let health = Health::new(Utc::now(), None);
402        let attempts = AtomicUsize::new(0);
403        let ingest = |_: &Path| {
404            if attempts.fetch_add(1, Ordering::SeqCst) == 0 {
405                signals.synced.store(false, Ordering::SeqCst);
406                signals.offset.store(-10_000_000, Ordering::SeqCst);
407            }
408            if clock.is_open() {
409                (Ok(FileOutcome::Captured), None)
410            } else {
411                (Err(anyhow::anyhow!("wall clock not yet trusted")), None)
412            }
413        };
414        let (a, b) = (PathBuf::from("a.jsonl"), PathBuf::from("b.jsonl"));
415        let mut pending = HashMap::new();
416        let settle = Duration::from_secs(10);
417
418        ingest_ready(
419            vec![a.clone(), b.clone()],
420            &mut pending,
421            &clock,
422            &Bootstrap::default(),
423            &health,
424            ingest,
425        );
426        assert!(
427            !clock.is_open(),
428            "an unsynced host with no RTC stays closed"
429        );
430        assert!(pending.contains_key(&a) && pending.contains_key(&b));
431        assert!(
432            health.report().last_error.is_none(),
433            "no file reports ingest_failed"
434        );
435
436        signals.synced.store(true, Ordering::SeqCst);
437        clock.tick();
438        let later = Instant::now() + settle;
439        let mut ready = take_settled(&mut pending, later, settle, clock.is_open());
440        ready.sort();
441        assert_eq!(ready, [a, b]);
442        ingest_ready(
443            ready,
444            &mut pending,
445            &clock,
446            &Bootstrap::default(),
447            &health,
448            ingest,
449        );
450        assert!(pending.is_empty());
451        assert_eq!(attempts.load(Ordering::SeqCst), 3);
452        assert!(health.report().last_error.is_none());
453    }
454
455    #[test]
456    fn a_file_after_one_that_closed_the_gate_waits_instead_of_failing_if_it_reopens_mid_ingest() {
457        let dir = tempfile::tempdir().unwrap();
458        let signals = Arc::new(SteppingClock {
459            offset: AtomicI64::new(0),
460            synced: AtomicBool::new(true),
461        });
462        let clock = ClockGate::new(dir.path(), signals.clone(), 0);
463        assert!(clock.is_open());
464        let health = Health::new(Utc::now(), None);
465        let attempts = AtomicUsize::new(0);
466        let ingest = |_: &Path| {
467            if attempts.fetch_add(1, Ordering::SeqCst) == 0 {
468                signals.synced.store(false, Ordering::SeqCst);
469                signals.offset.store(-10_000_000, Ordering::SeqCst);
470                if clock.is_open() {
471                    return (Ok(FileOutcome::Captured), None);
472                }
473            } else {
474                signals.synced.store(true, Ordering::SeqCst);
475                clock.open_by_peer();
476            }
477            (Err(anyhow::anyhow!("wall clock not yet trusted")), None)
478        };
479        let (a, b) = (PathBuf::from("a.jsonl"), PathBuf::from("b.jsonl"));
480        let mut pending = HashMap::new();
481
482        ingest_ready(
483            vec![a.clone(), b.clone()],
484            &mut pending,
485            &clock,
486            &Bootstrap::default(),
487            &health,
488            ingest,
489        );
490
491        assert_eq!(
492            attempts.load(Ordering::SeqCst),
493            1,
494            "b never starts while the gate is closed"
495        );
496        assert!(pending.contains_key(&a) && pending.contains_key(&b));
497        assert!(health.report().last_error.is_none());
498    }
499
500    fn pending_bootstrap() -> (SqliteStore, Bootstrap) {
501        let store = SqliteStore::in_memory("m".to_string()).unwrap();
502        let bootstrap = Bootstrap::default();
503        bootstrap.mark_pending(&store).unwrap();
504        (store, bootstrap)
505    }
506
507    #[test]
508    fn settled_files_stay_queued_while_the_bootstrap_is_pending() {
509        let (_store, bootstrap) = pending_bootstrap();
510        let health = Health::new(Utc::now(), None);
511        let attempts = AtomicUsize::new(0);
512        let (a, b) = (PathBuf::from("a.jsonl"), PathBuf::from("b.jsonl"));
513        let mut pending = HashMap::new();
514
515        ingest_ready(
516            vec![a.clone(), b.clone()],
517            &mut pending,
518            &ClockGate::open_for_tests(),
519            &bootstrap,
520            &health,
521            |_| {
522                attempts.fetch_add(1, Ordering::SeqCst);
523                (Ok(FileOutcome::Captured), None)
524            },
525        );
526
527        assert_eq!(attempts.load(Ordering::SeqCst), 0);
528        assert!(pending.contains_key(&a) && pending.contains_key(&b));
529    }
530
531    #[test]
532    fn files_after_a_pairing_that_marked_the_bootstrap_pending_stay_queued() {
533        let store = SqliteStore::in_memory("m".to_string()).unwrap();
534        let bootstrap = Bootstrap::default();
535        let health = Health::new(Utc::now(), None);
536        let attempts = AtomicUsize::new(0);
537        let (a, b) = (PathBuf::from("a.jsonl"), PathBuf::from("b.jsonl"));
538        let mut pending = HashMap::new();
539
540        ingest_ready(
541            vec![a.clone(), b.clone()],
542            &mut pending,
543            &ClockGate::open_for_tests(),
544            &bootstrap,
545            &health,
546            |_| {
547                attempts.fetch_add(1, Ordering::SeqCst);
548                bootstrap.mark_pending(&store).unwrap();
549                (Ok(FileOutcome::Captured), None)
550            },
551        );
552
553        assert_eq!(attempts.load(Ordering::SeqCst), 1);
554        assert_eq!(pending.len(), 1, "the second file waits: {pending:?}");
555    }
556
557    fn watched_tree() -> (tempfile::TempDir, Vec<PathBuf>) {
558        let dir = tempfile::tempdir().unwrap();
559        let watched = vec![
560            dir.path().join("-home-a-proj").join("s.jsonl"),
561            dir.path()
562                .join("-home-a-proj")
563                .join("s")
564                .join("subagents")
565                .join("agent.jsonl"),
566            dir.path()
567                .join("-home-a-proj")
568                .join("memory")
569                .join("fact.md"),
570        ];
571        let ignored = [
572            dir.path()
573                .join("-home-a-proj")
574                .join("memory")
575                .join("MEMORY.md"),
576            dir.path().join("-home-a-proj").join("notes.md"),
577        ];
578        for path in watched.iter().chain(&ignored) {
579            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
580            std::fs::write(path, "{}\n").unwrap();
581        }
582        (dir, watched)
583    }
584
585    fn sorted(paths: impl IntoIterator<Item = PathBuf>) -> Vec<PathBuf> {
586        let mut paths: Vec<_> = paths.into_iter().collect();
587        paths.sort();
588        paths
589    }
590
591    #[test]
592    fn a_due_ingest_pass_takes_every_watched_file_and_then_clears() {
593        let (dir, watched) = watched_tree();
594        let (store, bootstrap) = pending_bootstrap();
595        let clock = ClockGate::open_for_tests();
596        let health = Health::new(Utc::now(), None);
597        let mut pending = HashMap::new();
598        let mut pass = FullPass::default();
599        let now = Instant::now();
600
601        pass.queue_if_due(&bootstrap, dir.path(), &mut pending, now);
602        assert!(pending.is_empty(), "nothing is queued while still pending");
603
604        assert!(bootstrap.finish(&store).unwrap());
605        pass.queue_if_due(&bootstrap, dir.path(), &mut pending, now);
606        let ready = take_settled(&mut pending, now, SETTLE, true);
607        assert_eq!(sorted(ready.clone()), sorted(watched.clone()));
608
609        let ingested = std::sync::Mutex::new(Vec::new());
610        ingest_ready(ready, &mut pending, &clock, &bootstrap, &health, |path| {
611            ingested.lock().unwrap().push(path.to_path_buf());
612            (Ok(FileOutcome::Skipped), None)
613        });
614        assert_eq!(sorted(ingested.into_inner().unwrap()), sorted(watched));
615        pass.finish_if_ingested(&bootstrap, &store, &pending);
616
617        assert!(!bootstrap.ingest_due());
618        assert_eq!(store.bootstrap_state().unwrap(), None);
619        pass.queue_if_due(&bootstrap, dir.path(), &mut pending, now);
620        assert!(pending.is_empty(), "the pass runs once");
621    }
622
623    #[test]
624    fn a_due_ingest_pass_stays_due_while_one_of_its_files_waits() {
625        let (dir, _) = watched_tree();
626        let (store, bootstrap) = pending_bootstrap();
627        bootstrap.finish(&store).unwrap();
628        let mut pending = HashMap::new();
629        let mut pass = FullPass::default();
630        let now = Instant::now();
631
632        pass.queue_if_due(&bootstrap, dir.path(), &mut pending, now);
633        let mut ready = take_settled(&mut pending, now, SETTLE, true);
634        let waiting = ready.pop().unwrap();
635        pending.insert(waiting.clone(), now);
636        pass.finish_if_ingested(&bootstrap, &store, &pending);
637        assert!(bootstrap.ingest_due());
638        assert_eq!(
639            store.bootstrap_state().unwrap(),
640            Some(second_brain_core::sqlite_store::BootstrapState::IngestDue)
641        );
642
643        pending.remove(&waiting);
644        pass.finish_if_ingested(&bootstrap, &store, &pending);
645        assert!(!bootstrap.ingest_due());
646    }
647
648    #[test]
649    fn the_scan_finds_the_newest_transcript_in_any_project() {
650        let dir = std::env::temp_dir().join(format!("sb-scan-{}", uuid::Uuid::new_v4()));
651        let nested = dir.join("-home-a-proj").join("subagents");
652        std::fs::create_dir_all(&nested).unwrap();
653        assert_eq!(newest_transcript(&dir), None);
654
655        let at = |secs: u64| SystemTime::UNIX_EPOCH + Duration::from_secs(secs);
656        for (file, secs) in [
657            (dir.join("-home-a-proj").join("old.jsonl"), 1_000),
658            (nested.join("agent.jsonl"), 2_000),
659            (dir.join("-home-a-proj").join("notes.md"), 9_000),
660        ] {
661            let f = std::fs::File::create(&file).unwrap();
662            f.set_modified(at(secs)).unwrap();
663        }
664        let newest = newest_transcript(&dir);
665        std::fs::remove_dir_all(&dir).ok();
666        assert_eq!(newest, Some(DateTime::<Utc>::from(at(2_000))));
667    }
668}