second-brain-api 0.10.0

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};

use notify::{EventKind, RecursiveMode, Watcher};

use second_brain_core::embedding::Embedder;
use second_brain_core::store::StoreAdmin;

use crate::ingest::{ingest_file, ingest_memory_file};

const INITIAL_BACKOFF: Duration = Duration::from_secs(30);
const MAX_BACKOFF: Duration = Duration::from_secs(600);
const MISSING_DIR_RECHECK: Duration = Duration::from_secs(300);

pub fn spawn(store: Arc<dyn StoreAdmin>, embedder: Option<Arc<Embedder>>, machine_id: String) {
    std::thread::spawn(move || {
        let mut backoff = INITIAL_BACKOFF;
        loop {
            match run(store.clone(), embedder.clone(), machine_id.clone()) {
                // run() only returns Ok when the watch dir is missing; it may be
                // created later (fresh machine, first session), so keep checking
                Ok(()) => std::thread::sleep(MISSING_DIR_RECHECK),
                Err(e) => {
                    tracing::error!("watcher failed: {e}; restarting in {}s", backoff.as_secs());
                    std::thread::sleep(backoff);
                    backoff = next_backoff(backoff);
                }
            }
        }
    });
}

fn next_backoff(prev: Duration) -> Duration {
    (prev * 2).min(MAX_BACKOFF)
}

fn run(
    store: Arc<dyn StoreAdmin>,
    embedder: Option<Arc<Embedder>>,
    machine_id: String,
) -> anyhow::Result<()> {
    let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
    let watch_dir = PathBuf::from(&home).join(".claude").join("projects");

    if !watch_dir.exists() {
        tracing::info!(
            "watcher: {} does not exist, re-checking periodically",
            watch_dir.display()
        );
        return Ok(());
    }

    let (tx, rx) = std::sync::mpsc::channel();

    let mut watcher = notify::recommended_watcher(move |res: Result<notify::Event, _>| {
        if let Ok(event) = res
            && matches!(event.kind, EventKind::Create(_) | EventKind::Modify(_))
        {
            for path in event.paths {
                let is_jsonl = path.extension().is_some_and(|e| e == "jsonl");
                let is_memory_md = path.extension().is_some_and(|e| e == "md")
                    && path
                        .parent()
                        .is_some_and(|p| p.file_name().is_some_and(|n| n == "memory"))
                    && !path
                        .file_name()
                        .is_some_and(|n| n.eq_ignore_ascii_case("MEMORY.md"));
                if is_jsonl || is_memory_md {
                    tx.send(path).ok();
                }
            }
        }
    })?;

    watcher.watch(&watch_dir, RecursiveMode::Recursive)?;
    tracing::info!("watcher: monitoring {}", watch_dir.display());

    let mut pending: HashMap<PathBuf, Instant> = HashMap::new();
    let settle_duration = Duration::from_secs(10);

    loop {
        while let Ok(path) = rx.try_recv() {
            pending.insert(path, Instant::now());
        }

        let now = Instant::now();
        let ready: Vec<PathBuf> = pending
            .iter()
            .filter(|(_, when)| now.duration_since(**when) >= settle_duration)
            .map(|(p, _)| p.clone())
            .collect();

        for path in ready {
            pending.remove(&path);
            let is_memory_md = path.extension().is_some_and(|e| e == "md");
            if is_memory_md {
                ingest_memory_file(&store, embedder.as_deref(), &path, &machine_id);
            } else {
                ingest_file(&store, embedder.as_deref(), &path, &machine_id);
            }
        }

        std::thread::sleep(Duration::from_secs(2));
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn backoff_doubles_and_caps() {
        let mut b = INITIAL_BACKOFF;
        b = next_backoff(b);
        assert_eq!(b, Duration::from_secs(60));
        for _ in 0..10 {
            b = next_backoff(b);
        }
        assert_eq!(b, MAX_BACKOFF);
    }
}