second-brain-api 0.12.0

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};

use chrono::{DateTime, Utc};
use notify::{EventKind, RecursiveMode, Watcher};

use second_brain_core::embedding::Embedder;
use second_brain_core::store::StoreAdmin;
use second_brain_core::transcript_archive::TranscriptArchive;

use crate::backfill::{LiveIngest, transcript_mtimes};
use crate::health::Health;
use crate::ingest::{FileOutcome, capture_transcript, ingest_memory_file};

const INITIAL_BACKOFF: Duration = Duration::from_secs(30);
const MAX_BACKOFF: Duration = Duration::from_secs(600);
const MISSING_DIR_RECHECK: Duration = Duration::from_secs(300);
/// How often the watch dir is walked for its newest transcript, so capture
/// lag shows even when change events stop arriving.
pub const SCAN_INTERVAL: Duration = Duration::from_secs(300);

pub fn spawn(
    store: Arc<dyn StoreAdmin>,
    embedder: Option<Arc<Embedder>>,
    machine_id: String,
    health: Arc<Health>,
    archive: Option<Arc<TranscriptArchive>>,
    live: Arc<LiveIngest>,
) {
    std::thread::spawn(move || {
        let mut backoff = INITIAL_BACKOFF;
        loop {
            let outcome = run(
                store.clone(),
                embedder.clone(),
                machine_id.clone(),
                &health,
                archive.as_deref(),
                &live,
            );
            match outcome {
                // run() only returns Ok when the watch dir is missing; it may be
                // created later (fresh machine, first session), so keep checking
                Ok(()) => std::thread::sleep(MISSING_DIR_RECHECK),
                Err(e) => {
                    health.watcher_failed(&format!("{e:#}"), Utc::now());
                    tracing::error!("watcher failed: {e}; restarting in {}s", backoff.as_secs());
                    std::thread::sleep(backoff);
                    backoff = next_backoff(backoff);
                }
            }
        }
    });
}

fn next_backoff(prev: Duration) -> Duration {
    (prev * 2).min(MAX_BACKOFF)
}

pub fn projects_dir() -> PathBuf {
    let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
    PathBuf::from(home).join(".claude").join("projects")
}

fn run(
    store: Arc<dyn StoreAdmin>,
    embedder: Option<Arc<Embedder>>,
    machine_id: String,
    health: &Health,
    archive: Option<&TranscriptArchive>,
    live: &LiveIngest,
) -> anyhow::Result<()> {
    let watch_dir = projects_dir();

    if !watch_dir.exists() {
        tracing::info!(
            "watcher: {} does not exist, re-checking periodically",
            watch_dir.display()
        );
        health.watcher_started(Utc::now());
        health.transcripts_scanned(None, Utc::now());
        return Ok(());
    }

    let (tx, rx) = std::sync::mpsc::channel();

    let mut watcher = notify::recommended_watcher(move |res: Result<notify::Event, _>| {
        if let Ok(event) = res
            && matches!(event.kind, EventKind::Create(_) | EventKind::Modify(_))
        {
            for path in event.paths {
                let is_jsonl = path.extension().is_some_and(|e| e == "jsonl");
                let is_memory_md = path.extension().is_some_and(|e| e == "md")
                    && path
                        .parent()
                        .is_some_and(|p| p.file_name().is_some_and(|n| n == "memory"))
                    && !path
                        .file_name()
                        .is_some_and(|n| n.eq_ignore_ascii_case("MEMORY.md"));
                if is_jsonl || is_memory_md {
                    tx.send(path).ok();
                }
            }
        }
    })?;

    watcher.watch(&watch_dir, RecursiveMode::Recursive)?;
    tracing::info!("watcher: monitoring {}", watch_dir.display());
    health.watcher_started(Utc::now());
    let mut next_scan = Instant::now();

    let mut pending: HashMap<PathBuf, Instant> = HashMap::new();
    let settle_duration = Duration::from_secs(10);

    loop {
        if Instant::now() >= next_scan {
            health.transcripts_scanned(newest_transcript(&watch_dir), Utc::now());
            next_scan = Instant::now() + SCAN_INTERVAL;
        }

        while let Ok(path) = rx.try_recv() {
            pending.insert(path, Instant::now());
        }

        let now = Instant::now();
        let ready: Vec<PathBuf> = pending
            .iter()
            .filter(|(_, when)| now.duration_since(**when) >= settle_duration)
            .map(|(p, _)| p.clone())
            .collect();
        for path in &ready {
            pending.remove(path);
        }
        let ingesting = live.ingesting(ready.len());
        for path in ready {
            let is_memory_md = path.extension().is_some_and(|e| e == "md");
            let (result, transcript_mtime) = if is_memory_md {
                let result = ingest_memory_file(&store, embedder.as_deref(), &path, &machine_id);
                (result, None)
            } else {
                let mtime = modified(&path);
                let result =
                    capture_transcript(&store, embedder.as_deref(), archive, &path, &machine_id);
                (result, mtime)
            };
            match result {
                Ok(FileOutcome::Captured) => health.ingested(transcript_mtime, Utc::now()),
                Ok(FileOutcome::Skipped) => {}
                Err(e) => health.ingest_failed(&format!("{e:#}"), Utc::now()),
            }
        }
        drop(ingesting);

        std::thread::sleep(Duration::from_secs(2));
    }
}

fn modified(path: &Path) -> Option<DateTime<Utc>> {
    std::fs::metadata(path)
        .and_then(|m| m.modified())
        .ok()
        .map(DateTime::<Utc>::from)
}

fn newest_transcript(dir: &Path) -> Option<DateTime<Utc>> {
    transcript_mtimes(dir)
        .into_iter()
        .filter_map(|(mtime, _)| mtime)
        .max()
        .map(DateTime::<Utc>::from)
}

#[cfg(test)]
mod tests {
    use std::time::SystemTime;

    use super::*;

    #[test]
    fn backoff_doubles_and_caps() {
        let mut b = INITIAL_BACKOFF;
        b = next_backoff(b);
        assert_eq!(b, Duration::from_secs(60));
        for _ in 0..10 {
            b = next_backoff(b);
        }
        assert_eq!(b, MAX_BACKOFF);
    }

    #[test]
    fn the_scan_finds_the_newest_transcript_in_any_project() {
        let dir = std::env::temp_dir().join(format!("sb-scan-{}", uuid::Uuid::new_v4()));
        let nested = dir.join("-home-a-proj").join("subagents");
        std::fs::create_dir_all(&nested).unwrap();
        assert_eq!(newest_transcript(&dir), None);

        let at = |secs: u64| SystemTime::UNIX_EPOCH + Duration::from_secs(secs);
        for (file, secs) in [
            (dir.join("-home-a-proj").join("old.jsonl"), 1_000),
            (nested.join("agent.jsonl"), 2_000),
            (dir.join("-home-a-proj").join("notes.md"), 9_000),
        ] {
            let f = std::fs::File::create(&file).unwrap();
            f.set_modified(at(secs)).unwrap();
        }
        let newest = newest_transcript(&dir);
        std::fs::remove_dir_all(&dir).ok();
        assert_eq!(newest, Some(DateTime::<Utc>::from(at(2_000))));
    }
}