Skip to main content

second_brain_api/
watcher.rs

1use std::collections::HashMap;
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4use std::time::{Duration, Instant, SystemTime};
5
6use chrono::{DateTime, Utc};
7use notify::{EventKind, RecursiveMode, Watcher};
8
9use second_brain_core::embedding::Embedder;
10use second_brain_core::store::StoreAdmin;
11
12use crate::health::Health;
13use crate::ingest::{FileOutcome, ingest_file, ingest_memory_file};
14
15const INITIAL_BACKOFF: Duration = Duration::from_secs(30);
16const MAX_BACKOFF: Duration = Duration::from_secs(600);
17const MISSING_DIR_RECHECK: Duration = Duration::from_secs(300);
18/// How often the watch dir is walked for its newest transcript, so capture
19/// lag shows even when change events stop arriving.
20pub const SCAN_INTERVAL: Duration = Duration::from_secs(300);
21
22pub fn spawn(
23    store: Arc<dyn StoreAdmin>,
24    embedder: Option<Arc<Embedder>>,
25    machine_id: String,
26    health: Arc<Health>,
27) {
28    std::thread::spawn(move || {
29        let mut backoff = INITIAL_BACKOFF;
30        loop {
31            match run(store.clone(), embedder.clone(), machine_id.clone(), &health) {
32                // run() only returns Ok when the watch dir is missing; it may be
33                // created later (fresh machine, first session), so keep checking
34                Ok(()) => std::thread::sleep(MISSING_DIR_RECHECK),
35                Err(e) => {
36                    health.watcher_failed(&format!("{e:#}"), Utc::now());
37                    tracing::error!("watcher failed: {e}; restarting in {}s", backoff.as_secs());
38                    std::thread::sleep(backoff);
39                    backoff = next_backoff(backoff);
40                }
41            }
42        }
43    });
44}
45
46fn next_backoff(prev: Duration) -> Duration {
47    (prev * 2).min(MAX_BACKOFF)
48}
49
50fn run(
51    store: Arc<dyn StoreAdmin>,
52    embedder: Option<Arc<Embedder>>,
53    machine_id: String,
54    health: &Health,
55) -> anyhow::Result<()> {
56    let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
57    let watch_dir = PathBuf::from(&home).join(".claude").join("projects");
58
59    if !watch_dir.exists() {
60        tracing::info!(
61            "watcher: {} does not exist, re-checking periodically",
62            watch_dir.display()
63        );
64        health.watcher_started(Utc::now());
65        health.transcripts_scanned(None, Utc::now());
66        return Ok(());
67    }
68
69    let (tx, rx) = std::sync::mpsc::channel();
70
71    let mut watcher = notify::recommended_watcher(move |res: Result<notify::Event, _>| {
72        if let Ok(event) = res
73            && matches!(event.kind, EventKind::Create(_) | EventKind::Modify(_))
74        {
75            for path in event.paths {
76                let is_jsonl = path.extension().is_some_and(|e| e == "jsonl");
77                let is_memory_md = path.extension().is_some_and(|e| e == "md")
78                    && path
79                        .parent()
80                        .is_some_and(|p| p.file_name().is_some_and(|n| n == "memory"))
81                    && !path
82                        .file_name()
83                        .is_some_and(|n| n.eq_ignore_ascii_case("MEMORY.md"));
84                if is_jsonl || is_memory_md {
85                    tx.send(path).ok();
86                }
87            }
88        }
89    })?;
90
91    watcher.watch(&watch_dir, RecursiveMode::Recursive)?;
92    tracing::info!("watcher: monitoring {}", watch_dir.display());
93    health.watcher_started(Utc::now());
94    let mut next_scan = Instant::now();
95
96    let mut pending: HashMap<PathBuf, Instant> = HashMap::new();
97    let settle_duration = Duration::from_secs(10);
98
99    loop {
100        if Instant::now() >= next_scan {
101            health.transcripts_scanned(newest_transcript(&watch_dir), Utc::now());
102            next_scan = Instant::now() + SCAN_INTERVAL;
103        }
104
105        while let Ok(path) = rx.try_recv() {
106            pending.insert(path, Instant::now());
107        }
108
109        let now = Instant::now();
110        let ready: Vec<PathBuf> = pending
111            .iter()
112            .filter(|(_, when)| now.duration_since(**when) >= settle_duration)
113            .map(|(p, _)| p.clone())
114            .collect();
115
116        for path in ready {
117            pending.remove(&path);
118            let is_memory_md = path.extension().is_some_and(|e| e == "md");
119            let (result, transcript_mtime) = if is_memory_md {
120                let result = ingest_memory_file(&store, embedder.as_deref(), &path, &machine_id);
121                (result, None)
122            } else {
123                let mtime = modified(&path);
124                let result = ingest_file(&store, embedder.as_deref(), &path, &machine_id);
125                (result, mtime)
126            };
127            match result {
128                Ok(FileOutcome::Captured) => health.ingested(transcript_mtime, Utc::now()),
129                Ok(FileOutcome::Skipped) => {}
130                Err(e) => health.ingest_failed(&format!("{e:#}"), Utc::now()),
131            }
132        }
133
134        std::thread::sleep(Duration::from_secs(2));
135    }
136}
137
138fn modified(path: &Path) -> Option<DateTime<Utc>> {
139    std::fs::metadata(path)
140        .and_then(|m| m.modified())
141        .ok()
142        .map(DateTime::<Utc>::from)
143}
144
145fn newest_transcript(dir: &Path) -> Option<DateTime<Utc>> {
146    let mut newest: Option<SystemTime> = None;
147    let mut dirs = vec![dir.to_path_buf()];
148    while let Some(d) = dirs.pop() {
149        let Ok(entries) = std::fs::read_dir(&d) else {
150            continue;
151        };
152        for entry in entries.flatten() {
153            let Ok(kind) = entry.file_type() else {
154                continue;
155            };
156            let path = entry.path();
157            if kind.is_dir() {
158                dirs.push(path);
159            } else if kind.is_file()
160                && path.extension().is_some_and(|e| e == "jsonl")
161                && let Ok(mtime) = entry.metadata().and_then(|m| m.modified())
162            {
163                newest = newest.max(Some(mtime));
164            }
165        }
166    }
167    newest.map(DateTime::<Utc>::from)
168}
169
170#[cfg(test)]
171mod tests {
172    use super::*;
173
174    #[test]
175    fn backoff_doubles_and_caps() {
176        let mut b = INITIAL_BACKOFF;
177        b = next_backoff(b);
178        assert_eq!(b, Duration::from_secs(60));
179        for _ in 0..10 {
180            b = next_backoff(b);
181        }
182        assert_eq!(b, MAX_BACKOFF);
183    }
184
185    #[test]
186    fn the_scan_finds_the_newest_transcript_in_any_project() {
187        let dir = std::env::temp_dir().join(format!("sb-scan-{}", uuid::Uuid::new_v4()));
188        let nested = dir.join("-home-a-proj").join("subagents");
189        std::fs::create_dir_all(&nested).unwrap();
190        assert_eq!(newest_transcript(&dir), None);
191
192        let at = |secs: u64| SystemTime::UNIX_EPOCH + Duration::from_secs(secs);
193        for (file, secs) in [
194            (dir.join("-home-a-proj").join("old.jsonl"), 1_000),
195            (nested.join("agent.jsonl"), 2_000),
196            (dir.join("-home-a-proj").join("notes.md"), 9_000),
197        ] {
198            let f = std::fs::File::create(&file).unwrap();
199            f.set_modified(at(secs)).unwrap();
200        }
201        let newest = newest_transcript(&dir);
202        std::fs::remove_dir_all(&dir).ok();
203        assert_eq!(newest, Some(DateTime::<Utc>::from(at(2_000))));
204    }
205}