second_brain_api/
watcher.rs1use std::collections::HashMap;
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4use std::time::{Duration, Instant, SystemTime};
5
6use chrono::{DateTime, Utc};
7use notify::{EventKind, RecursiveMode, Watcher};
8
9use second_brain_core::embedding::Embedder;
10use second_brain_core::store::StoreAdmin;
11
12use crate::health::Health;
13use crate::ingest::{FileOutcome, ingest_file, ingest_memory_file};
14
15const INITIAL_BACKOFF: Duration = Duration::from_secs(30);
16const MAX_BACKOFF: Duration = Duration::from_secs(600);
17const MISSING_DIR_RECHECK: Duration = Duration::from_secs(300);
18pub const SCAN_INTERVAL: Duration = Duration::from_secs(300);
21
22pub fn spawn(
23 store: Arc<dyn StoreAdmin>,
24 embedder: Option<Arc<Embedder>>,
25 machine_id: String,
26 health: Arc<Health>,
27) {
28 std::thread::spawn(move || {
29 let mut backoff = INITIAL_BACKOFF;
30 loop {
31 match run(store.clone(), embedder.clone(), machine_id.clone(), &health) {
32 Ok(()) => std::thread::sleep(MISSING_DIR_RECHECK),
35 Err(e) => {
36 health.watcher_failed(&format!("{e:#}"), Utc::now());
37 tracing::error!("watcher failed: {e}; restarting in {}s", backoff.as_secs());
38 std::thread::sleep(backoff);
39 backoff = next_backoff(backoff);
40 }
41 }
42 }
43 });
44}
45
46fn next_backoff(prev: Duration) -> Duration {
47 (prev * 2).min(MAX_BACKOFF)
48}
49
50fn run(
51 store: Arc<dyn StoreAdmin>,
52 embedder: Option<Arc<Embedder>>,
53 machine_id: String,
54 health: &Health,
55) -> anyhow::Result<()> {
56 let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
57 let watch_dir = PathBuf::from(&home).join(".claude").join("projects");
58
59 if !watch_dir.exists() {
60 tracing::info!(
61 "watcher: {} does not exist, re-checking periodically",
62 watch_dir.display()
63 );
64 health.watcher_started(Utc::now());
65 health.transcripts_scanned(None, Utc::now());
66 return Ok(());
67 }
68
69 let (tx, rx) = std::sync::mpsc::channel();
70
71 let mut watcher = notify::recommended_watcher(move |res: Result<notify::Event, _>| {
72 if let Ok(event) = res
73 && matches!(event.kind, EventKind::Create(_) | EventKind::Modify(_))
74 {
75 for path in event.paths {
76 let is_jsonl = path.extension().is_some_and(|e| e == "jsonl");
77 let is_memory_md = path.extension().is_some_and(|e| e == "md")
78 && path
79 .parent()
80 .is_some_and(|p| p.file_name().is_some_and(|n| n == "memory"))
81 && !path
82 .file_name()
83 .is_some_and(|n| n.eq_ignore_ascii_case("MEMORY.md"));
84 if is_jsonl || is_memory_md {
85 tx.send(path).ok();
86 }
87 }
88 }
89 })?;
90
91 watcher.watch(&watch_dir, RecursiveMode::Recursive)?;
92 tracing::info!("watcher: monitoring {}", watch_dir.display());
93 health.watcher_started(Utc::now());
94 let mut next_scan = Instant::now();
95
96 let mut pending: HashMap<PathBuf, Instant> = HashMap::new();
97 let settle_duration = Duration::from_secs(10);
98
99 loop {
100 if Instant::now() >= next_scan {
101 health.transcripts_scanned(newest_transcript(&watch_dir), Utc::now());
102 next_scan = Instant::now() + SCAN_INTERVAL;
103 }
104
105 while let Ok(path) = rx.try_recv() {
106 pending.insert(path, Instant::now());
107 }
108
109 let now = Instant::now();
110 let ready: Vec<PathBuf> = pending
111 .iter()
112 .filter(|(_, when)| now.duration_since(**when) >= settle_duration)
113 .map(|(p, _)| p.clone())
114 .collect();
115
116 for path in ready {
117 pending.remove(&path);
118 let is_memory_md = path.extension().is_some_and(|e| e == "md");
119 let (result, transcript_mtime) = if is_memory_md {
120 let result = ingest_memory_file(&store, embedder.as_deref(), &path, &machine_id);
121 (result, None)
122 } else {
123 let mtime = modified(&path);
124 let result = ingest_file(&store, embedder.as_deref(), &path, &machine_id);
125 (result, mtime)
126 };
127 match result {
128 Ok(FileOutcome::Captured) => health.ingested(transcript_mtime, Utc::now()),
129 Ok(FileOutcome::Skipped) => {}
130 Err(e) => health.ingest_failed(&format!("{e:#}"), Utc::now()),
131 }
132 }
133
134 std::thread::sleep(Duration::from_secs(2));
135 }
136}
137
138fn modified(path: &Path) -> Option<DateTime<Utc>> {
139 std::fs::metadata(path)
140 .and_then(|m| m.modified())
141 .ok()
142 .map(DateTime::<Utc>::from)
143}
144
145fn newest_transcript(dir: &Path) -> Option<DateTime<Utc>> {
146 let mut newest: Option<SystemTime> = None;
147 let mut dirs = vec![dir.to_path_buf()];
148 while let Some(d) = dirs.pop() {
149 let Ok(entries) = std::fs::read_dir(&d) else {
150 continue;
151 };
152 for entry in entries.flatten() {
153 let Ok(kind) = entry.file_type() else {
154 continue;
155 };
156 let path = entry.path();
157 if kind.is_dir() {
158 dirs.push(path);
159 } else if kind.is_file()
160 && path.extension().is_some_and(|e| e == "jsonl")
161 && let Ok(mtime) = entry.metadata().and_then(|m| m.modified())
162 {
163 newest = newest.max(Some(mtime));
164 }
165 }
166 }
167 newest.map(DateTime::<Utc>::from)
168}
169
170#[cfg(test)]
171mod tests {
172 use super::*;
173
174 #[test]
175 fn backoff_doubles_and_caps() {
176 let mut b = INITIAL_BACKOFF;
177 b = next_backoff(b);
178 assert_eq!(b, Duration::from_secs(60));
179 for _ in 0..10 {
180 b = next_backoff(b);
181 }
182 assert_eq!(b, MAX_BACKOFF);
183 }
184
185 #[test]
186 fn the_scan_finds_the_newest_transcript_in_any_project() {
187 let dir = std::env::temp_dir().join(format!("sb-scan-{}", uuid::Uuid::new_v4()));
188 let nested = dir.join("-home-a-proj").join("subagents");
189 std::fs::create_dir_all(&nested).unwrap();
190 assert_eq!(newest_transcript(&dir), None);
191
192 let at = |secs: u64| SystemTime::UNIX_EPOCH + Duration::from_secs(secs);
193 for (file, secs) in [
194 (dir.join("-home-a-proj").join("old.jsonl"), 1_000),
195 (nested.join("agent.jsonl"), 2_000),
196 (dir.join("-home-a-proj").join("notes.md"), 9_000),
197 ] {
198 let f = std::fs::File::create(&file).unwrap();
199 f.set_modified(at(secs)).unwrap();
200 }
201 let newest = newest_transcript(&dir);
202 std::fs::remove_dir_all(&dir).ok();
203 assert_eq!(newest, Some(DateTime::<Utc>::from(at(2_000))));
204 }
205}