use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};
use chrono::{DateTime, Utc};
use notify::{EventKind, RecursiveMode, Watcher as _};
use second_brain_core::embedding::Embedder;
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::StoreAdmin;
use second_brain_core::transcript_archive::TranscriptArchive;
use crate::backfill::{LiveIngest, transcript_mtimes, watched_files};
use crate::bootstrap::Bootstrap;
use crate::clock_gate::ClockGate;
use crate::health::Health;
use crate::ingest::{FileOutcome, capture_transcript, ingest_memory_file, is_memory_file};
const INITIAL_BACKOFF: Duration = Duration::from_secs(30);
const MAX_BACKOFF: Duration = Duration::from_secs(600);
const MISSING_DIR_RECHECK: Duration = Duration::from_secs(300);
const SETTLE: Duration = Duration::from_secs(10);
pub const SCAN_INTERVAL: Duration = Duration::from_secs(300);
pub struct Watcher {
pub sqlite: Arc<SqliteStore>,
pub embedder: Option<Arc<Embedder>>,
pub machine_id: String,
pub health: Arc<Health>,
pub archive: Option<Arc<TranscriptArchive>>,
pub live: Arc<LiveIngest>,
pub clock: Arc<ClockGate>,
pub bootstrap: Arc<Bootstrap>,
}
impl Watcher {
pub fn spawn(self) {
std::thread::spawn(move || {
let mut backoff = INITIAL_BACKOFF;
loop {
match self.run() {
Ok(()) => std::thread::sleep(MISSING_DIR_RECHECK),
Err(e) => {
self.health.watcher_failed(&format!("{e:#}"), Utc::now());
tracing::error!(
"watcher failed: {e}; restarting in {}s",
backoff.as_secs()
);
std::thread::sleep(backoff);
backoff = next_backoff(backoff);
}
}
}
});
}
fn run(&self) -> anyhow::Result<()> {
let watch_dir = projects_dir();
let health = &self.health;
if !watch_dir.exists() {
tracing::info!(
"watcher: {} does not exist, re-checking periodically",
watch_dir.display()
);
health.watcher_started(Utc::now());
health.transcripts_scanned(None, Utc::now());
if let Err(e) = self.bootstrap.ingest_done(&self.sqlite) {
tracing::warn!("watcher: cannot record the post-bootstrap ingest pass: {e:#}");
}
return Ok(());
}
let (tx, rx) = std::sync::mpsc::channel();
let mut watcher = notify::recommended_watcher(move |res: Result<notify::Event, _>| {
if let Ok(event) = res
&& matches!(event.kind, EventKind::Create(_) | EventKind::Modify(_))
{
for path in event.paths {
let is_jsonl = path.extension().is_some_and(|e| e == "jsonl");
if is_jsonl || is_memory_file(&path) {
tx.send(path).ok();
}
}
}
})?;
watcher.watch(&watch_dir, RecursiveMode::Recursive)?;
tracing::info!("watcher: monitoring {}", watch_dir.display());
health.watcher_started(Utc::now());
let mut next_scan = Instant::now();
let mut pending: HashMap<PathBuf, Instant> = HashMap::new();
let mut full_pass = FullPass::default();
let store: Arc<dyn StoreAdmin> = self.sqlite.clone();
loop {
if Instant::now() >= next_scan {
health.transcripts_scanned(newest_transcript(&watch_dir), Utc::now());
next_scan = Instant::now() + SCAN_INTERVAL;
}
while let Ok(path) = rx.try_recv() {
pending.insert(path, Instant::now());
}
full_pass.queue_if_due(&self.bootstrap, &watch_dir, &mut pending, Instant::now());
let open = self.clock.is_open() && !self.bootstrap.is_pending();
let ready = take_settled(&mut pending, Instant::now(), SETTLE, open);
let ingesting = self.live.ingesting(ready.len());
ingest_ready(
ready,
&mut pending,
&self.clock,
&self.bootstrap,
health,
|path| self.ingest(&store, path),
);
drop(ingesting);
full_pass.finish_if_ingested(&self.bootstrap, &self.sqlite, &pending);
std::thread::sleep(Duration::from_secs(2));
}
}
fn ingest(
&self,
store: &Arc<dyn StoreAdmin>,
path: &Path,
) -> (anyhow::Result<FileOutcome>, Option<DateTime<Utc>>) {
let embedder = self.embedder.as_deref();
if path.extension().is_some_and(|e| e == "md") {
let result = ingest_memory_file(store, embedder, path, &self.machine_id);
(result, None)
} else {
let mtime = modified(path);
let archive = self.archive.as_deref();
let result = capture_transcript(store, embedder, archive, path, &self.machine_id);
(result, mtime)
}
}
}
fn next_backoff(prev: Duration) -> Duration {
(prev * 2).min(MAX_BACKOFF)
}
pub fn projects_dir() -> PathBuf {
let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
PathBuf::from(home).join(".claude").join("projects")
}
#[derive(Default)]
struct FullPass {
left: Option<HashSet<PathBuf>>,
}
impl FullPass {
fn queue_if_due(
&mut self,
bootstrap: &Bootstrap,
watch_dir: &Path,
pending: &mut HashMap<PathBuf, Instant>,
now: Instant,
) {
if self.left.is_some() || !bootstrap.ingest_due() {
return;
}
let settled_at = now.checked_sub(SETTLE).unwrap_or(now);
let files: HashSet<PathBuf> = watched_files(watch_dir).into_iter().collect();
tracing::info!(
"watcher: ingesting {} files after the pairing bootstrap",
files.len()
);
for path in &files {
pending.insert(path.clone(), settled_at);
}
self.left = Some(files);
}
fn finish_if_ingested(
&mut self,
bootstrap: &Bootstrap,
sqlite: &SqliteStore,
pending: &HashMap<PathBuf, Instant>,
) {
let Some(left) = &mut self.left else {
return;
};
left.retain(|path| pending.contains_key(path));
if !left.is_empty() {
return;
}
match bootstrap.ingest_done(sqlite) {
Ok(()) => self.left = None,
Err(e) => {
tracing::warn!("watcher: cannot record the post-bootstrap ingest pass: {e:#}")
}
}
}
}
fn ingest_ready(
ready: Vec<PathBuf>,
pending: &mut HashMap<PathBuf, Instant>,
clock: &ClockGate,
bootstrap: &Bootstrap,
health: &Health,
ingest: impl Fn(&Path) -> (anyhow::Result<FileOutcome>, Option<DateTime<Utc>>),
) {
let mut ready = ready.into_iter();
while let Some(path) = ready.next() {
let closures = clock.closures();
if !clock.is_open() || bootstrap.is_pending() {
for path in std::iter::once(path).chain(ready.by_ref()) {
pending.insert(path, Instant::now());
}
break;
}
let (result, transcript_mtime) = ingest(&path);
match result {
Err(_) if clock.closures() != closures || !clock.is_open() => {
pending.insert(path, Instant::now());
}
Ok(FileOutcome::Captured) => health.ingested(transcript_mtime, Utc::now()),
Ok(FileOutcome::Skipped) => {}
Err(e) => health.ingest_failed(&format!("{e:#}"), Utc::now()),
}
}
}
fn take_settled(
pending: &mut HashMap<PathBuf, Instant>,
now: Instant,
settle: Duration,
open: bool,
) -> Vec<PathBuf> {
if !open {
return Vec::new();
}
let ready: Vec<PathBuf> = pending
.iter()
.filter(|(_, when)| now.duration_since(**when) >= settle)
.map(|(p, _)| p.clone())
.collect();
for path in &ready {
pending.remove(path);
}
ready
}
fn modified(path: &Path) -> Option<DateTime<Utc>> {
std::fs::metadata(path)
.and_then(|m| m.modified())
.ok()
.map(DateTime::<Utc>::from)
}
fn newest_transcript(dir: &Path) -> Option<DateTime<Utc>> {
transcript_mtimes(dir)
.into_iter()
.filter_map(|(mtime, _)| mtime)
.max()
.map(DateTime::<Utc>::from)
}
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicUsize, Ordering};
use std::time::SystemTime;
use super::*;
use crate::clock_gate::TimeSignals;
#[test]
fn backoff_doubles_and_caps() {
let mut b = INITIAL_BACKOFF;
b = next_backoff(b);
assert_eq!(b, Duration::from_secs(60));
for _ in 0..10 {
b = next_backoff(b);
}
assert_eq!(b, MAX_BACKOFF);
}
#[test]
fn settled_files_stay_pending_while_the_clock_gate_is_closed() {
let start = Instant::now();
let settle = Duration::from_secs(10);
let mut pending = HashMap::from([
(PathBuf::from("settled.jsonl"), start),
(PathBuf::from("fresh.jsonl"), start + Duration::from_secs(5)),
]);
let later = start + settle;
assert!(take_settled(&mut pending, later, settle, false).is_empty());
assert_eq!(pending.len(), 2);
let ready = take_settled(&mut pending, later, settle, true);
assert_eq!(ready, [PathBuf::from("settled.jsonl")]);
assert_eq!(
pending.keys().collect::<Vec<_>>(),
[&PathBuf::from("fresh.jsonl")]
);
}
struct SteppingClock {
offset: AtomicI64,
synced: AtomicBool,
}
impl TimeSignals for SteppingClock {
fn now_micros(&self) -> i64 {
Utc::now().timestamp_micros() + self.offset.load(Ordering::SeqCst)
}
fn ntp_synced(&self) -> bool {
self.synced.load(Ordering::SeqCst)
}
fn rtc_secs(&self) -> Option<i64> {
None
}
}
#[test]
fn a_file_whose_ingest_failed_as_the_gate_closed_is_retried_once_it_reopens() {
let dir = tempfile::tempdir().unwrap();
let signals = Arc::new(SteppingClock {
offset: AtomicI64::new(0),
synced: AtomicBool::new(true),
});
let clock = ClockGate::new(dir.path(), signals.clone(), 0);
assert!(clock.is_open());
let health = Health::new(Utc::now(), None);
let attempts = AtomicUsize::new(0);
let ingest = |_: &Path| {
if attempts.fetch_add(1, Ordering::SeqCst) > 0 {
return (Ok(FileOutcome::Captured), None);
}
signals.synced.store(false, Ordering::SeqCst);
signals.offset.store(-10_000_000, Ordering::SeqCst);
assert!(!clock.is_open(), "the backward step closes the gate");
(Err(anyhow::anyhow!("wall clock not yet trusted")), None)
};
let path = PathBuf::from("s.jsonl");
let mut pending = HashMap::new();
let settle = Duration::from_secs(10);
ingest_ready(
vec![path.clone()],
&mut pending,
&clock,
&Bootstrap::default(),
&health,
ingest,
);
assert!(
pending.contains_key(&path),
"a file the closing gate failed is kept pending"
);
let later = Instant::now() + settle;
assert!(take_settled(&mut pending, later, settle, clock.is_open()).is_empty());
signals.synced.store(true, Ordering::SeqCst);
clock.tick();
let ready = take_settled(&mut pending, later, settle, clock.is_open());
assert_eq!(ready, [path]);
ingest_ready(
ready,
&mut pending,
&clock,
&Bootstrap::default(),
&health,
ingest,
);
assert!(pending.is_empty());
assert_eq!(attempts.load(Ordering::SeqCst), 2);
}
#[test]
fn files_after_one_that_closed_the_gate_stay_pending_while_it_stays_closed() {
let dir = tempfile::tempdir().unwrap();
let signals = Arc::new(SteppingClock {
offset: AtomicI64::new(0),
synced: AtomicBool::new(true),
});
let clock = ClockGate::new(dir.path(), signals.clone(), 0);
assert!(clock.is_open());
let health = Health::new(Utc::now(), None);
let attempts = AtomicUsize::new(0);
let ingest = |_: &Path| {
if attempts.fetch_add(1, Ordering::SeqCst) == 0 {
signals.synced.store(false, Ordering::SeqCst);
signals.offset.store(-10_000_000, Ordering::SeqCst);
}
if clock.is_open() {
(Ok(FileOutcome::Captured), None)
} else {
(Err(anyhow::anyhow!("wall clock not yet trusted")), None)
}
};
let (a, b) = (PathBuf::from("a.jsonl"), PathBuf::from("b.jsonl"));
let mut pending = HashMap::new();
let settle = Duration::from_secs(10);
ingest_ready(
vec![a.clone(), b.clone()],
&mut pending,
&clock,
&Bootstrap::default(),
&health,
ingest,
);
assert!(
!clock.is_open(),
"an unsynced host with no RTC stays closed"
);
assert!(pending.contains_key(&a) && pending.contains_key(&b));
assert!(
health.report().last_error.is_none(),
"no file reports ingest_failed"
);
signals.synced.store(true, Ordering::SeqCst);
clock.tick();
let later = Instant::now() + settle;
let mut ready = take_settled(&mut pending, later, settle, clock.is_open());
ready.sort();
assert_eq!(ready, [a, b]);
ingest_ready(
ready,
&mut pending,
&clock,
&Bootstrap::default(),
&health,
ingest,
);
assert!(pending.is_empty());
assert_eq!(attempts.load(Ordering::SeqCst), 3);
assert!(health.report().last_error.is_none());
}
#[test]
fn a_file_after_one_that_closed_the_gate_waits_instead_of_failing_if_it_reopens_mid_ingest() {
let dir = tempfile::tempdir().unwrap();
let signals = Arc::new(SteppingClock {
offset: AtomicI64::new(0),
synced: AtomicBool::new(true),
});
let clock = ClockGate::new(dir.path(), signals.clone(), 0);
assert!(clock.is_open());
let health = Health::new(Utc::now(), None);
let attempts = AtomicUsize::new(0);
let ingest = |_: &Path| {
if attempts.fetch_add(1, Ordering::SeqCst) == 0 {
signals.synced.store(false, Ordering::SeqCst);
signals.offset.store(-10_000_000, Ordering::SeqCst);
if clock.is_open() {
return (Ok(FileOutcome::Captured), None);
}
} else {
signals.synced.store(true, Ordering::SeqCst);
clock.open_by_peer();
}
(Err(anyhow::anyhow!("wall clock not yet trusted")), None)
};
let (a, b) = (PathBuf::from("a.jsonl"), PathBuf::from("b.jsonl"));
let mut pending = HashMap::new();
ingest_ready(
vec![a.clone(), b.clone()],
&mut pending,
&clock,
&Bootstrap::default(),
&health,
ingest,
);
assert_eq!(
attempts.load(Ordering::SeqCst),
1,
"b never starts while the gate is closed"
);
assert!(pending.contains_key(&a) && pending.contains_key(&b));
assert!(health.report().last_error.is_none());
}
fn pending_bootstrap() -> (SqliteStore, Bootstrap) {
let store = SqliteStore::in_memory("m".to_string()).unwrap();
let bootstrap = Bootstrap::default();
bootstrap.mark_pending(&store).unwrap();
(store, bootstrap)
}
#[test]
fn settled_files_stay_queued_while_the_bootstrap_is_pending() {
let (_store, bootstrap) = pending_bootstrap();
let health = Health::new(Utc::now(), None);
let attempts = AtomicUsize::new(0);
let (a, b) = (PathBuf::from("a.jsonl"), PathBuf::from("b.jsonl"));
let mut pending = HashMap::new();
ingest_ready(
vec![a.clone(), b.clone()],
&mut pending,
&ClockGate::open_for_tests(),
&bootstrap,
&health,
|_| {
attempts.fetch_add(1, Ordering::SeqCst);
(Ok(FileOutcome::Captured), None)
},
);
assert_eq!(attempts.load(Ordering::SeqCst), 0);
assert!(pending.contains_key(&a) && pending.contains_key(&b));
}
#[test]
fn files_after_a_pairing_that_marked_the_bootstrap_pending_stay_queued() {
let store = SqliteStore::in_memory("m".to_string()).unwrap();
let bootstrap = Bootstrap::default();
let health = Health::new(Utc::now(), None);
let attempts = AtomicUsize::new(0);
let (a, b) = (PathBuf::from("a.jsonl"), PathBuf::from("b.jsonl"));
let mut pending = HashMap::new();
ingest_ready(
vec![a.clone(), b.clone()],
&mut pending,
&ClockGate::open_for_tests(),
&bootstrap,
&health,
|_| {
attempts.fetch_add(1, Ordering::SeqCst);
bootstrap.mark_pending(&store).unwrap();
(Ok(FileOutcome::Captured), None)
},
);
assert_eq!(attempts.load(Ordering::SeqCst), 1);
assert_eq!(pending.len(), 1, "the second file waits: {pending:?}");
}
fn watched_tree() -> (tempfile::TempDir, Vec<PathBuf>) {
let dir = tempfile::tempdir().unwrap();
let watched = vec![
dir.path().join("-home-a-proj").join("s.jsonl"),
dir.path()
.join("-home-a-proj")
.join("s")
.join("subagents")
.join("agent.jsonl"),
dir.path()
.join("-home-a-proj")
.join("memory")
.join("fact.md"),
];
let ignored = [
dir.path()
.join("-home-a-proj")
.join("memory")
.join("MEMORY.md"),
dir.path().join("-home-a-proj").join("notes.md"),
];
for path in watched.iter().chain(&ignored) {
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, "{}\n").unwrap();
}
(dir, watched)
}
fn sorted(paths: impl IntoIterator<Item = PathBuf>) -> Vec<PathBuf> {
let mut paths: Vec<_> = paths.into_iter().collect();
paths.sort();
paths
}
#[test]
fn a_due_ingest_pass_takes_every_watched_file_and_then_clears() {
let (dir, watched) = watched_tree();
let (store, bootstrap) = pending_bootstrap();
let clock = ClockGate::open_for_tests();
let health = Health::new(Utc::now(), None);
let mut pending = HashMap::new();
let mut pass = FullPass::default();
let now = Instant::now();
pass.queue_if_due(&bootstrap, dir.path(), &mut pending, now);
assert!(pending.is_empty(), "nothing is queued while still pending");
assert!(bootstrap.finish(&store).unwrap());
pass.queue_if_due(&bootstrap, dir.path(), &mut pending, now);
let ready = take_settled(&mut pending, now, SETTLE, true);
assert_eq!(sorted(ready.clone()), sorted(watched.clone()));
let ingested = std::sync::Mutex::new(Vec::new());
ingest_ready(ready, &mut pending, &clock, &bootstrap, &health, |path| {
ingested.lock().unwrap().push(path.to_path_buf());
(Ok(FileOutcome::Skipped), None)
});
assert_eq!(sorted(ingested.into_inner().unwrap()), sorted(watched));
pass.finish_if_ingested(&bootstrap, &store, &pending);
assert!(!bootstrap.ingest_due());
assert_eq!(store.bootstrap_state().unwrap(), None);
pass.queue_if_due(&bootstrap, dir.path(), &mut pending, now);
assert!(pending.is_empty(), "the pass runs once");
}
#[test]
fn a_due_ingest_pass_stays_due_while_one_of_its_files_waits() {
let (dir, _) = watched_tree();
let (store, bootstrap) = pending_bootstrap();
bootstrap.finish(&store).unwrap();
let mut pending = HashMap::new();
let mut pass = FullPass::default();
let now = Instant::now();
pass.queue_if_due(&bootstrap, dir.path(), &mut pending, now);
let mut ready = take_settled(&mut pending, now, SETTLE, true);
let waiting = ready.pop().unwrap();
pending.insert(waiting.clone(), now);
pass.finish_if_ingested(&bootstrap, &store, &pending);
assert!(bootstrap.ingest_due());
assert_eq!(
store.bootstrap_state().unwrap(),
Some(second_brain_core::sqlite_store::BootstrapState::IngestDue)
);
pending.remove(&waiting);
pass.finish_if_ingested(&bootstrap, &store, &pending);
assert!(!bootstrap.ingest_due());
}
#[test]
fn the_scan_finds_the_newest_transcript_in_any_project() {
let dir = std::env::temp_dir().join(format!("sb-scan-{}", uuid::Uuid::new_v4()));
let nested = dir.join("-home-a-proj").join("subagents");
std::fs::create_dir_all(&nested).unwrap();
assert_eq!(newest_transcript(&dir), None);
let at = |secs: u64| SystemTime::UNIX_EPOCH + Duration::from_secs(secs);
for (file, secs) in [
(dir.join("-home-a-proj").join("old.jsonl"), 1_000),
(nested.join("agent.jsonl"), 2_000),
(dir.join("-home-a-proj").join("notes.md"), 9_000),
] {
let f = std::fs::File::create(&file).unwrap();
f.set_modified(at(secs)).unwrap();
}
let newest = newest_transcript(&dir);
std::fs::remove_dir_all(&dir).ok();
assert_eq!(newest, Some(DateTime::<Utc>::from(at(2_000))));
}
}