second_brain_api/
watcher.rs1use std::collections::HashMap;
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4use std::time::{Duration, Instant};
5
6use chrono::{DateTime, Utc};
7use notify::{EventKind, RecursiveMode, Watcher};
8
9use second_brain_core::embedding::Embedder;
10use second_brain_core::store::StoreAdmin;
11use second_brain_core::transcript_archive::TranscriptArchive;
12
13use crate::backfill::{LiveIngest, transcript_mtimes};
14use crate::health::Health;
15use crate::ingest::{FileOutcome, capture_transcript, ingest_memory_file};
16
17const INITIAL_BACKOFF: Duration = Duration::from_secs(30);
18const MAX_BACKOFF: Duration = Duration::from_secs(600);
19const MISSING_DIR_RECHECK: Duration = Duration::from_secs(300);
20pub const SCAN_INTERVAL: Duration = Duration::from_secs(300);
23
24pub fn spawn(
25 store: Arc<dyn StoreAdmin>,
26 embedder: Option<Arc<Embedder>>,
27 machine_id: String,
28 health: Arc<Health>,
29 archive: Option<Arc<TranscriptArchive>>,
30 live: Arc<LiveIngest>,
31) {
32 std::thread::spawn(move || {
33 let mut backoff = INITIAL_BACKOFF;
34 loop {
35 let outcome = run(
36 store.clone(),
37 embedder.clone(),
38 machine_id.clone(),
39 &health,
40 archive.as_deref(),
41 &live,
42 );
43 match outcome {
44 Ok(()) => std::thread::sleep(MISSING_DIR_RECHECK),
47 Err(e) => {
48 health.watcher_failed(&format!("{e:#}"), Utc::now());
49 tracing::error!("watcher failed: {e}; restarting in {}s", backoff.as_secs());
50 std::thread::sleep(backoff);
51 backoff = next_backoff(backoff);
52 }
53 }
54 }
55 });
56}
57
58fn next_backoff(prev: Duration) -> Duration {
59 (prev * 2).min(MAX_BACKOFF)
60}
61
62pub fn projects_dir() -> PathBuf {
63 let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
64 PathBuf::from(home).join(".claude").join("projects")
65}
66
67fn run(
68 store: Arc<dyn StoreAdmin>,
69 embedder: Option<Arc<Embedder>>,
70 machine_id: String,
71 health: &Health,
72 archive: Option<&TranscriptArchive>,
73 live: &LiveIngest,
74) -> anyhow::Result<()> {
75 let watch_dir = projects_dir();
76
77 if !watch_dir.exists() {
78 tracing::info!(
79 "watcher: {} does not exist, re-checking periodically",
80 watch_dir.display()
81 );
82 health.watcher_started(Utc::now());
83 health.transcripts_scanned(None, Utc::now());
84 return Ok(());
85 }
86
87 let (tx, rx) = std::sync::mpsc::channel();
88
89 let mut watcher = notify::recommended_watcher(move |res: Result<notify::Event, _>| {
90 if let Ok(event) = res
91 && matches!(event.kind, EventKind::Create(_) | EventKind::Modify(_))
92 {
93 for path in event.paths {
94 let is_jsonl = path.extension().is_some_and(|e| e == "jsonl");
95 let is_memory_md = path.extension().is_some_and(|e| e == "md")
96 && path
97 .parent()
98 .is_some_and(|p| p.file_name().is_some_and(|n| n == "memory"))
99 && !path
100 .file_name()
101 .is_some_and(|n| n.eq_ignore_ascii_case("MEMORY.md"));
102 if is_jsonl || is_memory_md {
103 tx.send(path).ok();
104 }
105 }
106 }
107 })?;
108
109 watcher.watch(&watch_dir, RecursiveMode::Recursive)?;
110 tracing::info!("watcher: monitoring {}", watch_dir.display());
111 health.watcher_started(Utc::now());
112 let mut next_scan = Instant::now();
113
114 let mut pending: HashMap<PathBuf, Instant> = HashMap::new();
115 let settle_duration = Duration::from_secs(10);
116
117 loop {
118 if Instant::now() >= next_scan {
119 health.transcripts_scanned(newest_transcript(&watch_dir), Utc::now());
120 next_scan = Instant::now() + SCAN_INTERVAL;
121 }
122
123 while let Ok(path) = rx.try_recv() {
124 pending.insert(path, Instant::now());
125 }
126
127 let now = Instant::now();
128 let ready: Vec<PathBuf> = pending
129 .iter()
130 .filter(|(_, when)| now.duration_since(**when) >= settle_duration)
131 .map(|(p, _)| p.clone())
132 .collect();
133 for path in &ready {
134 pending.remove(path);
135 }
136 let ingesting = live.ingesting(ready.len());
137 for path in ready {
138 let is_memory_md = path.extension().is_some_and(|e| e == "md");
139 let (result, transcript_mtime) = if is_memory_md {
140 let result = ingest_memory_file(&store, embedder.as_deref(), &path, &machine_id);
141 (result, None)
142 } else {
143 let mtime = modified(&path);
144 let result =
145 capture_transcript(&store, embedder.as_deref(), archive, &path, &machine_id);
146 (result, mtime)
147 };
148 match result {
149 Ok(FileOutcome::Captured) => health.ingested(transcript_mtime, Utc::now()),
150 Ok(FileOutcome::Skipped) => {}
151 Err(e) => health.ingest_failed(&format!("{e:#}"), Utc::now()),
152 }
153 }
154 drop(ingesting);
155
156 std::thread::sleep(Duration::from_secs(2));
157 }
158}
159
160fn modified(path: &Path) -> Option<DateTime<Utc>> {
161 std::fs::metadata(path)
162 .and_then(|m| m.modified())
163 .ok()
164 .map(DateTime::<Utc>::from)
165}
166
167fn newest_transcript(dir: &Path) -> Option<DateTime<Utc>> {
168 transcript_mtimes(dir)
169 .into_iter()
170 .filter_map(|(mtime, _)| mtime)
171 .max()
172 .map(DateTime::<Utc>::from)
173}
174
175#[cfg(test)]
176mod tests {
177 use std::time::SystemTime;
178
179 use super::*;
180
181 #[test]
182 fn backoff_doubles_and_caps() {
183 let mut b = INITIAL_BACKOFF;
184 b = next_backoff(b);
185 assert_eq!(b, Duration::from_secs(60));
186 for _ in 0..10 {
187 b = next_backoff(b);
188 }
189 assert_eq!(b, MAX_BACKOFF);
190 }
191
192 #[test]
193 fn the_scan_finds_the_newest_transcript_in_any_project() {
194 let dir = std::env::temp_dir().join(format!("sb-scan-{}", uuid::Uuid::new_v4()));
195 let nested = dir.join("-home-a-proj").join("subagents");
196 std::fs::create_dir_all(&nested).unwrap();
197 assert_eq!(newest_transcript(&dir), None);
198
199 let at = |secs: u64| SystemTime::UNIX_EPOCH + Duration::from_secs(secs);
200 for (file, secs) in [
201 (dir.join("-home-a-proj").join("old.jsonl"), 1_000),
202 (nested.join("agent.jsonl"), 2_000),
203 (dir.join("-home-a-proj").join("notes.md"), 9_000),
204 ] {
205 let f = std::fs::File::create(&file).unwrap();
206 f.set_modified(at(secs)).unwrap();
207 }
208 let newest = newest_transcript(&dir);
209 std::fs::remove_dir_all(&dir).ok();
210 assert_eq!(newest, Some(DateTime::<Utc>::from(at(2_000))));
211 }
212}