second-brain-api 0.12.0

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
use std::collections::BTreeSet;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, SystemTime};

use anyhow::{Context, Result};
use chrono::{DateTime, Utc};
use second_brain_core::sqlite_store::{IngestedSession, SqliteStore};
use second_brain_core::store::StoreAdmin;
use second_brain_sync::peers;

use crate::health::{DistillMode, DistillReport, Health, ProgressSeen, SyncReport};
use crate::note_cover::distill_mode;

pub const INTERVAL: Duration = Duration::from_secs(5 * 60);

const CLAUDE_CODE_SOURCE: &str = "claude-code";

pub struct ProgressCheck {
    pub store: Arc<SqliteStore>,
    /// Where `sync-peers` and the sync success stamp live.
    pub data_dir: PathBuf,
    pub health: Arc<Health>,
    /// False only for a config read cleanly without an `[llm]` section; the
    /// mode then also depends on recent LLM distillation, read each pass.
    pub llm_configured: bool,
}

impl ProgressCheck {
    /// Runs one check before returning, so health holds a full reading
    /// when the caller announces it, then checks every `INTERVAL`.
    pub async fn start(self) -> tokio::task::JoinHandle<()> {
        let check = Arc::new(self);
        check.clone().run_blocking().await;
        tokio::spawn(async move {
            let mut interval = tokio::time::interval(INTERVAL);
            interval.tick().await;
            loop {
                interval.tick().await;
                check.clone().run_blocking().await;
            }
        })
    }

    async fn run_blocking(self: Arc<Self>) {
        if let Err(e) = tokio::task::spawn_blocking(move || self.run_once(Utc::now())).await {
            tracing::error!("sync and distillation check panicked: {e}");
        }
    }

    pub fn run_once(&self, now: DateTime<Utc>) {
        let mut seen = ProgressSeen::default();
        let mut errors = Vec::new();
        match self.sync_seen() {
            Ok(sync) => seen.sync = Some(sync),
            Err(e) => errors.push(format!("{e:#}")),
        }
        let mode = distill_mode(&self.store, self.llm_configured, now).unwrap_or_else(|e| {
            errors.push(format!("reading recent LLM distillation: {e:#}"));
            DistillMode::Llm
        });
        match self.store.pending_distillation() {
            Ok(p) => {
                seen.distill = Some(DistillReport {
                    mode,
                    pending: p.count,
                    oldest_at: p.oldest_created_at,
                    ..DistillReport::default()
                })
            }
            Err(e) => errors.push(format!("reading raws pending distillation: {e:#}")),
        }
        if let Some(since) = self.health.ingest_window_start(now) {
            match self
                .store
                .ingested_sessions_since(CLAUDE_CODE_SOURCE, since)
            {
                Ok(rows) => {
                    seen.ingested_sessions = sessions_started_since(rows, since, |path| {
                        std::fs::metadata(path)?.created()
                    })
                }
                Err(e) => errors.push(format!("reading recently ingested sessions: {e:#}")),
            }
        }
        if !errors.is_empty() {
            let error = errors.join("; ");
            tracing::warn!("sync and distillation check failed: {error}");
            seen.error = Some(error);
        }
        self.health.progress_checked(seen, now);
    }

    fn sync_seen(&self) -> Result<SyncReport> {
        let peers = peers::configured_peers(&self.data_dir)
            .with_context(|| format!("reading {}", peers::PEERS_FILE))?;
        let cursors = self
            .store
            .get_all_sync_states()
            .context("reading sync state")?;
        let last_success_at = peers::last_success(&self.data_dir)?;
        Ok(SyncReport {
            checked_at: None,
            peers: peers.map(|p| p.len()),
            synced_before: !cursors.is_empty(),
            last_success_at,
            last_cursor_at: cursors.iter().map(|c| c.last_sync_at).max(),
        })
    }
}

/// Top-level session keys whose transcript was created at or after `since`,
/// once each and sorted. A re-ingest rewrites `ingested_at` but not the
/// file's creation time, so old sessions cannot inflate the count; a row
/// whose creation time cannot be read (file gone, no birth time) is left out.
/// None when no transcript's filesystem records birth times, since the count
/// would then read zero with nothing to say why.
fn sessions_started_since(
    rows: Vec<IngestedSession>,
    since: DateTime<Utc>,
    created: impl Fn(&Path) -> io::Result<SystemTime>,
) -> Option<Vec<String>> {
    let born: Vec<(String, io::Result<SystemTime>)> = rows
        .into_iter()
        .filter(|row| !row.session_key.contains('/'))
        .map(|row| {
            let at = created(Path::new(&row.file_path));
            (row.session_key, at)
        })
        .collect();
    let no_birth_times = born.iter().all(|(_, at)| at.is_err())
        && born.iter().any(|(_, at)| {
            at.as_ref()
                .is_err_and(|e| e.kind() == io::ErrorKind::Unsupported)
        });
    if no_birth_times {
        return None;
    }
    let kept: BTreeSet<String> = born
        .into_iter()
        .filter(|(_, at)| {
            at.as_ref()
                .is_ok_and(|&at| DateTime::<Utc>::from(at) >= since)
        })
        .map(|(key, _)| key)
        .collect();
    Some(kept.into_iter().collect())
}

#[cfg(test)]
mod tests {
    use super::*;

    fn row(key: &str, path: &str) -> IngestedSession {
        IngestedSession {
            session_key: key.to_string(),
            file_path: path.to_string(),
        }
    }

    #[test]
    fn only_top_level_sessions_whose_transcript_was_created_since_the_bound_count() {
        let since = DateTime::parse_from_rfc3339("2026-11-02T10:00:00Z")
            .unwrap()
            .with_timezone(&Utc);
        let at = |secs: i64| SystemTime::from(since + chrono::Duration::seconds(secs));
        let created = |path: &Path| match path.to_str().unwrap() {
            "/p/sub.jsonl" | "/p/after.jsonl" | "/p/dup-a.jsonl" | "/p/dup-b.jsonl" => Ok(at(5)),
            "/p/at.jsonl" => Ok(at(0)),
            "/p/before.jsonl" => Ok(at(-1)),
            "/p/dup-old.jsonl" => Ok(at(-60)),
            "/p/no-btime.jsonl" => Err(io::Error::from(io::ErrorKind::Unsupported)),
            _ => Err(io::Error::from(io::ErrorKind::NotFound)),
        };
        let cases: [(&str, Vec<IngestedSession>, Option<Vec<&str>>); 10] = [
            (
                "subagent",
                vec![row("s/agent-1", "/p/sub.jsonl")],
                Some(vec![]),
            ),
            (
                "created before",
                vec![row("s", "/p/before.jsonl")],
                Some(vec![]),
            ),
            (
                "created at the bound",
                vec![row("s", "/p/at.jsonl")],
                Some(vec!["s"]),
            ),
            (
                "created after",
                vec![row("s", "/p/after.jsonl")],
                Some(vec!["s"]),
            ),
            (
                "missing file",
                vec![row("s", "/p/gone.jsonl")],
                Some(vec![]),
            ),
            (
                "unreadable btime",
                vec![row("s", "/p/no-btime.jsonl")],
                None,
            ),
            (
                "unreadable btime beside a deleted transcript",
                vec![row("s", "/p/no-btime.jsonl"), row("t", "/p/gone.jsonl")],
                None,
            ),
            (
                "unreadable btime beside a readable one",
                vec![row("s", "/p/no-btime.jsonl"), row("t", "/p/after.jsonl")],
                Some(vec!["t"]),
            ),
            (
                "duplicate keys",
                vec![
                    row("s", "/p/dup-a.jsonl"),
                    row("s", "/p/dup-b.jsonl"),
                    row("s", "/p/dup-old.jsonl"),
                ],
                Some(vec!["s"]),
            ),
            (
                "mixed, sorted",
                vec![
                    row("t", "/p/after.jsonl"),
                    row("s", "/p/at.jsonl"),
                    row("t/agent-2", "/p/sub.jsonl"),
                    row("u", "/p/before.jsonl"),
                ],
                Some(vec!["s", "t"]),
            ),
        ];
        for (name, rows, expected) in cases {
            assert_eq!(
                sessions_started_since(rows, since, created),
                expected.map(|keys| keys.into_iter().map(String::from).collect::<Vec<_>>()),
                "{name}"
            );
        }
    }
}