use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime};
use chrono::{DateTime, Utc};
use notify::{EventKind, RecursiveMode, Watcher};
use second_brain_core::embedding::Embedder;
use second_brain_core::store::StoreAdmin;
use crate::health::Health;
use crate::ingest::{FileOutcome, ingest_file, ingest_memory_file};
const INITIAL_BACKOFF: Duration = Duration::from_secs(30);
const MAX_BACKOFF: Duration = Duration::from_secs(600);
const MISSING_DIR_RECHECK: Duration = Duration::from_secs(300);
pub const SCAN_INTERVAL: Duration = Duration::from_secs(300);
pub fn spawn(
store: Arc<dyn StoreAdmin>,
embedder: Option<Arc<Embedder>>,
machine_id: String,
health: Arc<Health>,
) {
std::thread::spawn(move || {
let mut backoff = INITIAL_BACKOFF;
loop {
match run(store.clone(), embedder.clone(), machine_id.clone(), &health) {
Ok(()) => std::thread::sleep(MISSING_DIR_RECHECK),
Err(e) => {
health.watcher_failed(&format!("{e:#}"), Utc::now());
tracing::error!("watcher failed: {e}; restarting in {}s", backoff.as_secs());
std::thread::sleep(backoff);
backoff = next_backoff(backoff);
}
}
}
});
}
fn next_backoff(prev: Duration) -> Duration {
(prev * 2).min(MAX_BACKOFF)
}
fn run(
store: Arc<dyn StoreAdmin>,
embedder: Option<Arc<Embedder>>,
machine_id: String,
health: &Health,
) -> anyhow::Result<()> {
let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
let watch_dir = PathBuf::from(&home).join(".claude").join("projects");
if !watch_dir.exists() {
tracing::info!(
"watcher: {} does not exist, re-checking periodically",
watch_dir.display()
);
health.watcher_started(Utc::now());
health.transcripts_scanned(None, Utc::now());
return Ok(());
}
let (tx, rx) = std::sync::mpsc::channel();
let mut watcher = notify::recommended_watcher(move |res: Result<notify::Event, _>| {
if let Ok(event) = res
&& matches!(event.kind, EventKind::Create(_) | EventKind::Modify(_))
{
for path in event.paths {
let is_jsonl = path.extension().is_some_and(|e| e == "jsonl");
let is_memory_md = path.extension().is_some_and(|e| e == "md")
&& path
.parent()
.is_some_and(|p| p.file_name().is_some_and(|n| n == "memory"))
&& !path
.file_name()
.is_some_and(|n| n.eq_ignore_ascii_case("MEMORY.md"));
if is_jsonl || is_memory_md {
tx.send(path).ok();
}
}
}
})?;
watcher.watch(&watch_dir, RecursiveMode::Recursive)?;
tracing::info!("watcher: monitoring {}", watch_dir.display());
health.watcher_started(Utc::now());
let mut next_scan = Instant::now();
let mut pending: HashMap<PathBuf, Instant> = HashMap::new();
let settle_duration = Duration::from_secs(10);
loop {
if Instant::now() >= next_scan {
health.transcripts_scanned(newest_transcript(&watch_dir), Utc::now());
next_scan = Instant::now() + SCAN_INTERVAL;
}
while let Ok(path) = rx.try_recv() {
pending.insert(path, Instant::now());
}
let now = Instant::now();
let ready: Vec<PathBuf> = pending
.iter()
.filter(|(_, when)| now.duration_since(**when) >= settle_duration)
.map(|(p, _)| p.clone())
.collect();
for path in ready {
pending.remove(&path);
let is_memory_md = path.extension().is_some_and(|e| e == "md");
let (result, transcript_mtime) = if is_memory_md {
let result = ingest_memory_file(&store, embedder.as_deref(), &path, &machine_id);
(result, None)
} else {
let mtime = modified(&path);
let result = ingest_file(&store, embedder.as_deref(), &path, &machine_id);
(result, mtime)
};
match result {
Ok(FileOutcome::Captured) => health.ingested(transcript_mtime, Utc::now()),
Ok(FileOutcome::Skipped) => {}
Err(e) => health.ingest_failed(&format!("{e:#}"), Utc::now()),
}
}
std::thread::sleep(Duration::from_secs(2));
}
}
fn modified(path: &Path) -> Option<DateTime<Utc>> {
std::fs::metadata(path)
.and_then(|m| m.modified())
.ok()
.map(DateTime::<Utc>::from)
}
fn newest_transcript(dir: &Path) -> Option<DateTime<Utc>> {
let mut newest: Option<SystemTime> = None;
let mut dirs = vec![dir.to_path_buf()];
while let Some(d) = dirs.pop() {
let Ok(entries) = std::fs::read_dir(&d) else {
continue;
};
for entry in entries.flatten() {
let Ok(kind) = entry.file_type() else {
continue;
};
let path = entry.path();
if kind.is_dir() {
dirs.push(path);
} else if kind.is_file()
&& path.extension().is_some_and(|e| e == "jsonl")
&& let Ok(mtime) = entry.metadata().and_then(|m| m.modified())
{
newest = newest.max(Some(mtime));
}
}
}
newest.map(DateTime::<Utc>::from)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn backoff_doubles_and_caps() {
let mut b = INITIAL_BACKOFF;
b = next_backoff(b);
assert_eq!(b, Duration::from_secs(60));
for _ in 0..10 {
b = next_backoff(b);
}
assert_eq!(b, MAX_BACKOFF);
}
#[test]
fn the_scan_finds_the_newest_transcript_in_any_project() {
let dir = std::env::temp_dir().join(format!("sb-scan-{}", uuid::Uuid::new_v4()));
let nested = dir.join("-home-a-proj").join("subagents");
std::fs::create_dir_all(&nested).unwrap();
assert_eq!(newest_transcript(&dir), None);
let at = |secs: u64| SystemTime::UNIX_EPOCH + Duration::from_secs(secs);
for (file, secs) in [
(dir.join("-home-a-proj").join("old.jsonl"), 1_000),
(nested.join("agent.jsonl"), 2_000),
(dir.join("-home-a-proj").join("notes.md"), 9_000),
] {
let f = std::fs::File::create(&file).unwrap();
f.set_modified(at(secs)).unwrap();
}
let newest = newest_transcript(&dir);
std::fs::remove_dir_all(&dir).ok();
assert_eq!(newest, Some(DateTime::<Utc>::from(at(2_000))));
}
}