1use std::collections::{HashMap, HashSet};
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4use std::time::{Duration, Instant};
5
6use chrono::{DateTime, Utc};
7use notify::{EventKind, RecursiveMode, Watcher as _};
8
9use second_brain_core::embedding::Embedder;
10use second_brain_core::sqlite_store::SqliteStore;
11use second_brain_core::store::StoreAdmin;
12use second_brain_core::transcript_archive::TranscriptArchive;
13
14use crate::backfill::{LiveIngest, transcript_mtimes, watched_files};
15use crate::bootstrap::Bootstrap;
16use crate::clock_gate::ClockGate;
17use crate::health::Health;
18use crate::ingest::{FileOutcome, capture_transcript, ingest_memory_file, is_memory_file};
19
20const INITIAL_BACKOFF: Duration = Duration::from_secs(30);
21const MAX_BACKOFF: Duration = Duration::from_secs(600);
22const MISSING_DIR_RECHECK: Duration = Duration::from_secs(300);
23const SETTLE: Duration = Duration::from_secs(10);
24pub const SCAN_INTERVAL: Duration = Duration::from_secs(300);
27
28pub struct Watcher {
29 pub sqlite: Arc<SqliteStore>,
30 pub embedder: Option<Arc<Embedder>>,
31 pub machine_id: String,
32 pub health: Arc<Health>,
33 pub archive: Option<Arc<TranscriptArchive>>,
34 pub live: Arc<LiveIngest>,
35 pub clock: Arc<ClockGate>,
36 pub bootstrap: Arc<Bootstrap>,
37}
38
39impl Watcher {
40 pub fn spawn(self) {
41 std::thread::spawn(move || {
42 let mut backoff = INITIAL_BACKOFF;
43 loop {
44 match self.run() {
45 Ok(()) => std::thread::sleep(MISSING_DIR_RECHECK),
48 Err(e) => {
49 self.health.watcher_failed(&format!("{e:#}"), Utc::now());
50 tracing::error!(
51 "watcher failed: {e}; restarting in {}s",
52 backoff.as_secs()
53 );
54 std::thread::sleep(backoff);
55 backoff = next_backoff(backoff);
56 }
57 }
58 }
59 });
60 }
61
62 fn run(&self) -> anyhow::Result<()> {
63 let watch_dir = projects_dir();
64 let health = &self.health;
65
66 if !watch_dir.exists() {
67 tracing::info!(
68 "watcher: {} does not exist, re-checking periodically",
69 watch_dir.display()
70 );
71 health.watcher_started(Utc::now());
72 health.transcripts_scanned(None, Utc::now());
73 if let Err(e) = self.bootstrap.ingest_done(&self.sqlite) {
74 tracing::warn!("watcher: cannot record the post-bootstrap ingest pass: {e:#}");
75 }
76 return Ok(());
77 }
78
79 let (tx, rx) = std::sync::mpsc::channel();
80
81 let mut watcher = notify::recommended_watcher(move |res: Result<notify::Event, _>| {
82 if let Ok(event) = res
83 && matches!(event.kind, EventKind::Create(_) | EventKind::Modify(_))
84 {
85 for path in event.paths {
86 let is_jsonl = path.extension().is_some_and(|e| e == "jsonl");
87 if is_jsonl || is_memory_file(&path) {
88 tx.send(path).ok();
89 }
90 }
91 }
92 })?;
93
94 watcher.watch(&watch_dir, RecursiveMode::Recursive)?;
95 tracing::info!("watcher: monitoring {}", watch_dir.display());
96 health.watcher_started(Utc::now());
97 let mut next_scan = Instant::now();
98
99 let mut pending: HashMap<PathBuf, Instant> = HashMap::new();
100 let mut full_pass = FullPass::default();
101 let store: Arc<dyn StoreAdmin> = self.sqlite.clone();
102
103 loop {
104 if Instant::now() >= next_scan {
105 health.transcripts_scanned(newest_transcript(&watch_dir), Utc::now());
106 next_scan = Instant::now() + SCAN_INTERVAL;
107 }
108
109 while let Ok(path) = rx.try_recv() {
110 pending.insert(path, Instant::now());
111 }
112 full_pass.queue_if_due(&self.bootstrap, &watch_dir, &mut pending, Instant::now());
113
114 let open = self.clock.is_open() && !self.bootstrap.is_pending();
115 let ready = take_settled(&mut pending, Instant::now(), SETTLE, open);
116 let ingesting = self.live.ingesting(ready.len());
117 ingest_ready(
118 ready,
119 &mut pending,
120 &self.clock,
121 &self.bootstrap,
122 health,
123 |path| self.ingest(&store, path),
124 );
125 drop(ingesting);
126 full_pass.finish_if_ingested(&self.bootstrap, &self.sqlite, &pending);
127
128 std::thread::sleep(Duration::from_secs(2));
129 }
130 }
131
132 fn ingest(
133 &self,
134 store: &Arc<dyn StoreAdmin>,
135 path: &Path,
136 ) -> (anyhow::Result<FileOutcome>, Option<DateTime<Utc>>) {
137 let embedder = self.embedder.as_deref();
138 if path.extension().is_some_and(|e| e == "md") {
139 let result = ingest_memory_file(store, embedder, path, &self.machine_id);
140 (result, None)
141 } else {
142 let mtime = modified(path);
143 let archive = self.archive.as_deref();
144 let result = capture_transcript(store, embedder, archive, path, &self.machine_id);
145 (result, mtime)
146 }
147 }
148}
149
150fn next_backoff(prev: Duration) -> Duration {
151 (prev * 2).min(MAX_BACKOFF)
152}
153
154pub fn projects_dir() -> PathBuf {
155 let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
156 PathBuf::from(home).join(".claude").join("projects")
157}
158
159#[derive(Default)]
163struct FullPass {
164 left: Option<HashSet<PathBuf>>,
165}
166
167impl FullPass {
168 fn queue_if_due(
169 &mut self,
170 bootstrap: &Bootstrap,
171 watch_dir: &Path,
172 pending: &mut HashMap<PathBuf, Instant>,
173 now: Instant,
174 ) {
175 if self.left.is_some() || !bootstrap.ingest_due() {
176 return;
177 }
178 let settled_at = now.checked_sub(SETTLE).unwrap_or(now);
179 let files: HashSet<PathBuf> = watched_files(watch_dir).into_iter().collect();
180 tracing::info!(
181 "watcher: ingesting {} files after the pairing bootstrap",
182 files.len()
183 );
184 for path in &files {
185 pending.insert(path.clone(), settled_at);
186 }
187 self.left = Some(files);
188 }
189
190 fn finish_if_ingested(
191 &mut self,
192 bootstrap: &Bootstrap,
193 sqlite: &SqliteStore,
194 pending: &HashMap<PathBuf, Instant>,
195 ) {
196 let Some(left) = &mut self.left else {
197 return;
198 };
199 left.retain(|path| pending.contains_key(path));
200 if !left.is_empty() {
201 return;
202 }
203 match bootstrap.ingest_done(sqlite) {
204 Ok(()) => self.left = None,
205 Err(e) => {
206 tracing::warn!("watcher: cannot record the post-bootstrap ingest pass: {e:#}")
207 }
208 }
209 }
210}
211
212fn ingest_ready(
213 ready: Vec<PathBuf>,
214 pending: &mut HashMap<PathBuf, Instant>,
215 clock: &ClockGate,
216 bootstrap: &Bootstrap,
217 health: &Health,
218 ingest: impl Fn(&Path) -> (anyhow::Result<FileOutcome>, Option<DateTime<Utc>>),
219) {
220 let mut ready = ready.into_iter();
221 while let Some(path) = ready.next() {
222 let closures = clock.closures();
223 if !clock.is_open() || bootstrap.is_pending() {
224 for path in std::iter::once(path).chain(ready.by_ref()) {
225 pending.insert(path, Instant::now());
226 }
227 break;
228 }
229 let (result, transcript_mtime) = ingest(&path);
230 match result {
231 Err(_) if clock.closures() != closures || !clock.is_open() => {
232 pending.insert(path, Instant::now());
233 }
234 Ok(FileOutcome::Captured) => health.ingested(transcript_mtime, Utc::now()),
235 Ok(FileOutcome::Skipped) => {}
236 Err(e) => health.ingest_failed(&format!("{e:#}"), Utc::now()),
237 }
238 }
239}
240
241fn take_settled(
244 pending: &mut HashMap<PathBuf, Instant>,
245 now: Instant,
246 settle: Duration,
247 open: bool,
248) -> Vec<PathBuf> {
249 if !open {
250 return Vec::new();
251 }
252 let ready: Vec<PathBuf> = pending
253 .iter()
254 .filter(|(_, when)| now.duration_since(**when) >= settle)
255 .map(|(p, _)| p.clone())
256 .collect();
257 for path in &ready {
258 pending.remove(path);
259 }
260 ready
261}
262
263fn modified(path: &Path) -> Option<DateTime<Utc>> {
264 std::fs::metadata(path)
265 .and_then(|m| m.modified())
266 .ok()
267 .map(DateTime::<Utc>::from)
268}
269
270fn newest_transcript(dir: &Path) -> Option<DateTime<Utc>> {
271 transcript_mtimes(dir)
272 .into_iter()
273 .filter_map(|(mtime, _)| mtime)
274 .max()
275 .map(DateTime::<Utc>::from)
276}
277
278#[cfg(test)]
279mod tests {
280 use std::sync::atomic::{AtomicBool, AtomicI64, AtomicUsize, Ordering};
281 use std::time::SystemTime;
282
283 use super::*;
284 use crate::clock_gate::TimeSignals;
285
286 #[test]
287 fn backoff_doubles_and_caps() {
288 let mut b = INITIAL_BACKOFF;
289 b = next_backoff(b);
290 assert_eq!(b, Duration::from_secs(60));
291 for _ in 0..10 {
292 b = next_backoff(b);
293 }
294 assert_eq!(b, MAX_BACKOFF);
295 }
296
297 #[test]
298 fn settled_files_stay_pending_while_the_clock_gate_is_closed() {
299 let start = Instant::now();
300 let settle = Duration::from_secs(10);
301 let mut pending = HashMap::from([
302 (PathBuf::from("settled.jsonl"), start),
303 (PathBuf::from("fresh.jsonl"), start + Duration::from_secs(5)),
304 ]);
305 let later = start + settle;
306
307 assert!(take_settled(&mut pending, later, settle, false).is_empty());
308 assert_eq!(pending.len(), 2);
309
310 let ready = take_settled(&mut pending, later, settle, true);
311 assert_eq!(ready, [PathBuf::from("settled.jsonl")]);
312 assert_eq!(
313 pending.keys().collect::<Vec<_>>(),
314 [&PathBuf::from("fresh.jsonl")]
315 );
316 }
317
318 struct SteppingClock {
319 offset: AtomicI64,
320 synced: AtomicBool,
321 }
322
323 impl TimeSignals for SteppingClock {
324 fn now_micros(&self) -> i64 {
325 Utc::now().timestamp_micros() + self.offset.load(Ordering::SeqCst)
326 }
327
328 fn ntp_synced(&self) -> bool {
329 self.synced.load(Ordering::SeqCst)
330 }
331
332 fn rtc_secs(&self) -> Option<i64> {
333 None
334 }
335 }
336
337 #[test]
338 fn a_file_whose_ingest_failed_as_the_gate_closed_is_retried_once_it_reopens() {
339 let dir = tempfile::tempdir().unwrap();
340 let signals = Arc::new(SteppingClock {
341 offset: AtomicI64::new(0),
342 synced: AtomicBool::new(true),
343 });
344 let clock = ClockGate::new(dir.path(), signals.clone(), 0);
345 assert!(clock.is_open());
346 let health = Health::new(Utc::now(), None);
347 let attempts = AtomicUsize::new(0);
348 let ingest = |_: &Path| {
349 if attempts.fetch_add(1, Ordering::SeqCst) > 0 {
350 return (Ok(FileOutcome::Captured), None);
351 }
352 signals.synced.store(false, Ordering::SeqCst);
353 signals.offset.store(-10_000_000, Ordering::SeqCst);
354 assert!(!clock.is_open(), "the backward step closes the gate");
355 (Err(anyhow::anyhow!("wall clock not yet trusted")), None)
356 };
357 let path = PathBuf::from("s.jsonl");
358 let mut pending = HashMap::new();
359 let settle = Duration::from_secs(10);
360
361 ingest_ready(
362 vec![path.clone()],
363 &mut pending,
364 &clock,
365 &Bootstrap::default(),
366 &health,
367 ingest,
368 );
369 assert!(
370 pending.contains_key(&path),
371 "a file the closing gate failed is kept pending"
372 );
373 let later = Instant::now() + settle;
374 assert!(take_settled(&mut pending, later, settle, clock.is_open()).is_empty());
375
376 signals.synced.store(true, Ordering::SeqCst);
377 clock.tick();
378 let ready = take_settled(&mut pending, later, settle, clock.is_open());
379 assert_eq!(ready, [path]);
380 ingest_ready(
381 ready,
382 &mut pending,
383 &clock,
384 &Bootstrap::default(),
385 &health,
386 ingest,
387 );
388 assert!(pending.is_empty());
389 assert_eq!(attempts.load(Ordering::SeqCst), 2);
390 }
391
392 #[test]
393 fn files_after_one_that_closed_the_gate_stay_pending_while_it_stays_closed() {
394 let dir = tempfile::tempdir().unwrap();
395 let signals = Arc::new(SteppingClock {
396 offset: AtomicI64::new(0),
397 synced: AtomicBool::new(true),
398 });
399 let clock = ClockGate::new(dir.path(), signals.clone(), 0);
400 assert!(clock.is_open());
401 let health = Health::new(Utc::now(), None);
402 let attempts = AtomicUsize::new(0);
403 let ingest = |_: &Path| {
404 if attempts.fetch_add(1, Ordering::SeqCst) == 0 {
405 signals.synced.store(false, Ordering::SeqCst);
406 signals.offset.store(-10_000_000, Ordering::SeqCst);
407 }
408 if clock.is_open() {
409 (Ok(FileOutcome::Captured), None)
410 } else {
411 (Err(anyhow::anyhow!("wall clock not yet trusted")), None)
412 }
413 };
414 let (a, b) = (PathBuf::from("a.jsonl"), PathBuf::from("b.jsonl"));
415 let mut pending = HashMap::new();
416 let settle = Duration::from_secs(10);
417
418 ingest_ready(
419 vec![a.clone(), b.clone()],
420 &mut pending,
421 &clock,
422 &Bootstrap::default(),
423 &health,
424 ingest,
425 );
426 assert!(
427 !clock.is_open(),
428 "an unsynced host with no RTC stays closed"
429 );
430 assert!(pending.contains_key(&a) && pending.contains_key(&b));
431 assert!(
432 health.report().last_error.is_none(),
433 "no file reports ingest_failed"
434 );
435
436 signals.synced.store(true, Ordering::SeqCst);
437 clock.tick();
438 let later = Instant::now() + settle;
439 let mut ready = take_settled(&mut pending, later, settle, clock.is_open());
440 ready.sort();
441 assert_eq!(ready, [a, b]);
442 ingest_ready(
443 ready,
444 &mut pending,
445 &clock,
446 &Bootstrap::default(),
447 &health,
448 ingest,
449 );
450 assert!(pending.is_empty());
451 assert_eq!(attempts.load(Ordering::SeqCst), 3);
452 assert!(health.report().last_error.is_none());
453 }
454
455 #[test]
456 fn a_file_after_one_that_closed_the_gate_waits_instead_of_failing_if_it_reopens_mid_ingest() {
457 let dir = tempfile::tempdir().unwrap();
458 let signals = Arc::new(SteppingClock {
459 offset: AtomicI64::new(0),
460 synced: AtomicBool::new(true),
461 });
462 let clock = ClockGate::new(dir.path(), signals.clone(), 0);
463 assert!(clock.is_open());
464 let health = Health::new(Utc::now(), None);
465 let attempts = AtomicUsize::new(0);
466 let ingest = |_: &Path| {
467 if attempts.fetch_add(1, Ordering::SeqCst) == 0 {
468 signals.synced.store(false, Ordering::SeqCst);
469 signals.offset.store(-10_000_000, Ordering::SeqCst);
470 if clock.is_open() {
471 return (Ok(FileOutcome::Captured), None);
472 }
473 } else {
474 signals.synced.store(true, Ordering::SeqCst);
475 clock.open_by_peer();
476 }
477 (Err(anyhow::anyhow!("wall clock not yet trusted")), None)
478 };
479 let (a, b) = (PathBuf::from("a.jsonl"), PathBuf::from("b.jsonl"));
480 let mut pending = HashMap::new();
481
482 ingest_ready(
483 vec![a.clone(), b.clone()],
484 &mut pending,
485 &clock,
486 &Bootstrap::default(),
487 &health,
488 ingest,
489 );
490
491 assert_eq!(
492 attempts.load(Ordering::SeqCst),
493 1,
494 "b never starts while the gate is closed"
495 );
496 assert!(pending.contains_key(&a) && pending.contains_key(&b));
497 assert!(health.report().last_error.is_none());
498 }
499
500 fn pending_bootstrap() -> (SqliteStore, Bootstrap) {
501 let store = SqliteStore::in_memory("m".to_string()).unwrap();
502 let bootstrap = Bootstrap::default();
503 bootstrap.mark_pending(&store).unwrap();
504 (store, bootstrap)
505 }
506
507 #[test]
508 fn settled_files_stay_queued_while_the_bootstrap_is_pending() {
509 let (_store, bootstrap) = pending_bootstrap();
510 let health = Health::new(Utc::now(), None);
511 let attempts = AtomicUsize::new(0);
512 let (a, b) = (PathBuf::from("a.jsonl"), PathBuf::from("b.jsonl"));
513 let mut pending = HashMap::new();
514
515 ingest_ready(
516 vec![a.clone(), b.clone()],
517 &mut pending,
518 &ClockGate::open_for_tests(),
519 &bootstrap,
520 &health,
521 |_| {
522 attempts.fetch_add(1, Ordering::SeqCst);
523 (Ok(FileOutcome::Captured), None)
524 },
525 );
526
527 assert_eq!(attempts.load(Ordering::SeqCst), 0);
528 assert!(pending.contains_key(&a) && pending.contains_key(&b));
529 }
530
531 #[test]
532 fn files_after_a_pairing_that_marked_the_bootstrap_pending_stay_queued() {
533 let store = SqliteStore::in_memory("m".to_string()).unwrap();
534 let bootstrap = Bootstrap::default();
535 let health = Health::new(Utc::now(), None);
536 let attempts = AtomicUsize::new(0);
537 let (a, b) = (PathBuf::from("a.jsonl"), PathBuf::from("b.jsonl"));
538 let mut pending = HashMap::new();
539
540 ingest_ready(
541 vec![a.clone(), b.clone()],
542 &mut pending,
543 &ClockGate::open_for_tests(),
544 &bootstrap,
545 &health,
546 |_| {
547 attempts.fetch_add(1, Ordering::SeqCst);
548 bootstrap.mark_pending(&store).unwrap();
549 (Ok(FileOutcome::Captured), None)
550 },
551 );
552
553 assert_eq!(attempts.load(Ordering::SeqCst), 1);
554 assert_eq!(pending.len(), 1, "the second file waits: {pending:?}");
555 }
556
557 fn watched_tree() -> (tempfile::TempDir, Vec<PathBuf>) {
558 let dir = tempfile::tempdir().unwrap();
559 let watched = vec![
560 dir.path().join("-home-a-proj").join("s.jsonl"),
561 dir.path()
562 .join("-home-a-proj")
563 .join("s")
564 .join("subagents")
565 .join("agent.jsonl"),
566 dir.path()
567 .join("-home-a-proj")
568 .join("memory")
569 .join("fact.md"),
570 ];
571 let ignored = [
572 dir.path()
573 .join("-home-a-proj")
574 .join("memory")
575 .join("MEMORY.md"),
576 dir.path().join("-home-a-proj").join("notes.md"),
577 ];
578 for path in watched.iter().chain(&ignored) {
579 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
580 std::fs::write(path, "{}\n").unwrap();
581 }
582 (dir, watched)
583 }
584
585 fn sorted(paths: impl IntoIterator<Item = PathBuf>) -> Vec<PathBuf> {
586 let mut paths: Vec<_> = paths.into_iter().collect();
587 paths.sort();
588 paths
589 }
590
591 #[test]
592 fn a_due_ingest_pass_takes_every_watched_file_and_then_clears() {
593 let (dir, watched) = watched_tree();
594 let (store, bootstrap) = pending_bootstrap();
595 let clock = ClockGate::open_for_tests();
596 let health = Health::new(Utc::now(), None);
597 let mut pending = HashMap::new();
598 let mut pass = FullPass::default();
599 let now = Instant::now();
600
601 pass.queue_if_due(&bootstrap, dir.path(), &mut pending, now);
602 assert!(pending.is_empty(), "nothing is queued while still pending");
603
604 assert!(bootstrap.finish(&store).unwrap());
605 pass.queue_if_due(&bootstrap, dir.path(), &mut pending, now);
606 let ready = take_settled(&mut pending, now, SETTLE, true);
607 assert_eq!(sorted(ready.clone()), sorted(watched.clone()));
608
609 let ingested = std::sync::Mutex::new(Vec::new());
610 ingest_ready(ready, &mut pending, &clock, &bootstrap, &health, |path| {
611 ingested.lock().unwrap().push(path.to_path_buf());
612 (Ok(FileOutcome::Skipped), None)
613 });
614 assert_eq!(sorted(ingested.into_inner().unwrap()), sorted(watched));
615 pass.finish_if_ingested(&bootstrap, &store, &pending);
616
617 assert!(!bootstrap.ingest_due());
618 assert_eq!(store.bootstrap_state().unwrap(), None);
619 pass.queue_if_due(&bootstrap, dir.path(), &mut pending, now);
620 assert!(pending.is_empty(), "the pass runs once");
621 }
622
623 #[test]
624 fn a_due_ingest_pass_stays_due_while_one_of_its_files_waits() {
625 let (dir, _) = watched_tree();
626 let (store, bootstrap) = pending_bootstrap();
627 bootstrap.finish(&store).unwrap();
628 let mut pending = HashMap::new();
629 let mut pass = FullPass::default();
630 let now = Instant::now();
631
632 pass.queue_if_due(&bootstrap, dir.path(), &mut pending, now);
633 let mut ready = take_settled(&mut pending, now, SETTLE, true);
634 let waiting = ready.pop().unwrap();
635 pending.insert(waiting.clone(), now);
636 pass.finish_if_ingested(&bootstrap, &store, &pending);
637 assert!(bootstrap.ingest_due());
638 assert_eq!(
639 store.bootstrap_state().unwrap(),
640 Some(second_brain_core::sqlite_store::BootstrapState::IngestDue)
641 );
642
643 pending.remove(&waiting);
644 pass.finish_if_ingested(&bootstrap, &store, &pending);
645 assert!(!bootstrap.ingest_due());
646 }
647
648 #[test]
649 fn the_scan_finds_the_newest_transcript_in_any_project() {
650 let dir = std::env::temp_dir().join(format!("sb-scan-{}", uuid::Uuid::new_v4()));
651 let nested = dir.join("-home-a-proj").join("subagents");
652 std::fs::create_dir_all(&nested).unwrap();
653 assert_eq!(newest_transcript(&dir), None);
654
655 let at = |secs: u64| SystemTime::UNIX_EPOCH + Duration::from_secs(secs);
656 for (file, secs) in [
657 (dir.join("-home-a-proj").join("old.jsonl"), 1_000),
658 (nested.join("agent.jsonl"), 2_000),
659 (dir.join("-home-a-proj").join("notes.md"), 9_000),
660 ] {
661 let f = std::fs::File::create(&file).unwrap();
662 f.set_modified(at(secs)).unwrap();
663 }
664 let newest = newest_transcript(&dir);
665 std::fs::remove_dir_all(&dir).ok();
666 assert_eq!(newest, Some(DateTime::<Utc>::from(at(2_000))));
667 }
668}