Skip to main content

second_brain_api/
watcher.rs

1use std::collections::HashMap;
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4use std::time::{Duration, Instant};
5
6use chrono::{DateTime, Utc};
7use notify::{EventKind, RecursiveMode, Watcher};
8
9use second_brain_core::embedding::Embedder;
10use second_brain_core::store::StoreAdmin;
11use second_brain_core::transcript_archive::TranscriptArchive;
12
13use crate::backfill::{LiveIngest, transcript_mtimes};
14use crate::health::Health;
15use crate::ingest::{FileOutcome, capture_transcript, ingest_memory_file};
16
17const INITIAL_BACKOFF: Duration = Duration::from_secs(30);
18const MAX_BACKOFF: Duration = Duration::from_secs(600);
19const MISSING_DIR_RECHECK: Duration = Duration::from_secs(300);
20/// How often the watch dir is walked for its newest transcript, so capture
21/// lag shows even when change events stop arriving.
22pub const SCAN_INTERVAL: Duration = Duration::from_secs(300);
23
24pub fn spawn(
25    store: Arc<dyn StoreAdmin>,
26    embedder: Option<Arc<Embedder>>,
27    machine_id: String,
28    health: Arc<Health>,
29    archive: Option<Arc<TranscriptArchive>>,
30    live: Arc<LiveIngest>,
31) {
32    std::thread::spawn(move || {
33        let mut backoff = INITIAL_BACKOFF;
34        loop {
35            let outcome = run(
36                store.clone(),
37                embedder.clone(),
38                machine_id.clone(),
39                &health,
40                archive.as_deref(),
41                &live,
42            );
43            match outcome {
44                // run() only returns Ok when the watch dir is missing; it may be
45                // created later (fresh machine, first session), so keep checking
46                Ok(()) => std::thread::sleep(MISSING_DIR_RECHECK),
47                Err(e) => {
48                    health.watcher_failed(&format!("{e:#}"), Utc::now());
49                    tracing::error!("watcher failed: {e}; restarting in {}s", backoff.as_secs());
50                    std::thread::sleep(backoff);
51                    backoff = next_backoff(backoff);
52                }
53            }
54        }
55    });
56}
57
58fn next_backoff(prev: Duration) -> Duration {
59    (prev * 2).min(MAX_BACKOFF)
60}
61
62pub fn projects_dir() -> PathBuf {
63    let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
64    PathBuf::from(home).join(".claude").join("projects")
65}
66
67fn run(
68    store: Arc<dyn StoreAdmin>,
69    embedder: Option<Arc<Embedder>>,
70    machine_id: String,
71    health: &Health,
72    archive: Option<&TranscriptArchive>,
73    live: &LiveIngest,
74) -> anyhow::Result<()> {
75    let watch_dir = projects_dir();
76
77    if !watch_dir.exists() {
78        tracing::info!(
79            "watcher: {} does not exist, re-checking periodically",
80            watch_dir.display()
81        );
82        health.watcher_started(Utc::now());
83        health.transcripts_scanned(None, Utc::now());
84        return Ok(());
85    }
86
87    let (tx, rx) = std::sync::mpsc::channel();
88
89    let mut watcher = notify::recommended_watcher(move |res: Result<notify::Event, _>| {
90        if let Ok(event) = res
91            && matches!(event.kind, EventKind::Create(_) | EventKind::Modify(_))
92        {
93            for path in event.paths {
94                let is_jsonl = path.extension().is_some_and(|e| e == "jsonl");
95                let is_memory_md = path.extension().is_some_and(|e| e == "md")
96                    && path
97                        .parent()
98                        .is_some_and(|p| p.file_name().is_some_and(|n| n == "memory"))
99                    && !path
100                        .file_name()
101                        .is_some_and(|n| n.eq_ignore_ascii_case("MEMORY.md"));
102                if is_jsonl || is_memory_md {
103                    tx.send(path).ok();
104                }
105            }
106        }
107    })?;
108
109    watcher.watch(&watch_dir, RecursiveMode::Recursive)?;
110    tracing::info!("watcher: monitoring {}", watch_dir.display());
111    health.watcher_started(Utc::now());
112    let mut next_scan = Instant::now();
113
114    let mut pending: HashMap<PathBuf, Instant> = HashMap::new();
115    let settle_duration = Duration::from_secs(10);
116
117    loop {
118        if Instant::now() >= next_scan {
119            health.transcripts_scanned(newest_transcript(&watch_dir), Utc::now());
120            next_scan = Instant::now() + SCAN_INTERVAL;
121        }
122
123        while let Ok(path) = rx.try_recv() {
124            pending.insert(path, Instant::now());
125        }
126
127        let now = Instant::now();
128        let ready: Vec<PathBuf> = pending
129            .iter()
130            .filter(|(_, when)| now.duration_since(**when) >= settle_duration)
131            .map(|(p, _)| p.clone())
132            .collect();
133        for path in &ready {
134            pending.remove(path);
135        }
136        let ingesting = live.ingesting(ready.len());
137        for path in ready {
138            let is_memory_md = path.extension().is_some_and(|e| e == "md");
139            let (result, transcript_mtime) = if is_memory_md {
140                let result = ingest_memory_file(&store, embedder.as_deref(), &path, &machine_id);
141                (result, None)
142            } else {
143                let mtime = modified(&path);
144                let result =
145                    capture_transcript(&store, embedder.as_deref(), archive, &path, &machine_id);
146                (result, mtime)
147            };
148            match result {
149                Ok(FileOutcome::Captured) => health.ingested(transcript_mtime, Utc::now()),
150                Ok(FileOutcome::Skipped) => {}
151                Err(e) => health.ingest_failed(&format!("{e:#}"), Utc::now()),
152            }
153        }
154        drop(ingesting);
155
156        std::thread::sleep(Duration::from_secs(2));
157    }
158}
159
160fn modified(path: &Path) -> Option<DateTime<Utc>> {
161    std::fs::metadata(path)
162        .and_then(|m| m.modified())
163        .ok()
164        .map(DateTime::<Utc>::from)
165}
166
167fn newest_transcript(dir: &Path) -> Option<DateTime<Utc>> {
168    transcript_mtimes(dir)
169        .into_iter()
170        .filter_map(|(mtime, _)| mtime)
171        .max()
172        .map(DateTime::<Utc>::from)
173}
174
175#[cfg(test)]
176mod tests {
177    use std::time::SystemTime;
178
179    use super::*;
180
181    #[test]
182    fn backoff_doubles_and_caps() {
183        let mut b = INITIAL_BACKOFF;
184        b = next_backoff(b);
185        assert_eq!(b, Duration::from_secs(60));
186        for _ in 0..10 {
187            b = next_backoff(b);
188        }
189        assert_eq!(b, MAX_BACKOFF);
190    }
191
192    #[test]
193    fn the_scan_finds_the_newest_transcript_in_any_project() {
194        let dir = std::env::temp_dir().join(format!("sb-scan-{}", uuid::Uuid::new_v4()));
195        let nested = dir.join("-home-a-proj").join("subagents");
196        std::fs::create_dir_all(&nested).unwrap();
197        assert_eq!(newest_transcript(&dir), None);
198
199        let at = |secs: u64| SystemTime::UNIX_EPOCH + Duration::from_secs(secs);
200        for (file, secs) in [
201            (dir.join("-home-a-proj").join("old.jsonl"), 1_000),
202            (nested.join("agent.jsonl"), 2_000),
203            (dir.join("-home-a-proj").join("notes.md"), 9_000),
204        ] {
205            let f = std::fs::File::create(&file).unwrap();
206            f.set_modified(at(secs)).unwrap();
207        }
208        let newest = newest_transcript(&dir);
209        std::fs::remove_dir_all(&dir).ok();
210        assert_eq!(newest, Some(DateTime::<Utc>::from(at(2_000))));
211    }
212}