second-brain-api 0.12.0

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::thread::JoinHandle;
use std::time::{Duration, SystemTime};

use chrono::{DateTime, Utc};

use second_brain_core::embedding::Embedder;
use second_brain_core::residency;
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::StoreAdmin;
use second_brain_core::transcript_archive::TranscriptArchive;

use crate::health::{BackfillReport, BackfillState, Health};
use crate::ingest::{FileOutcome, capture_transcript};

pub const PAUSE_BETWEEN_FILES: Duration = Duration::from_millis(200);
pub const LIVE_POLL: Duration = Duration::from_secs(1);

#[derive(Debug, Default)]
pub struct LiveIngest {
    ingesting: AtomicUsize,
}

impl LiveIngest {
    pub fn ingesting(&self, files: usize) -> Ingesting<'_> {
        self.ingesting.store(files, Ordering::Relaxed);
        Ingesting(self)
    }

    pub fn busy(&self) -> bool {
        self.ingesting.load(Ordering::Relaxed) > 0
    }
}

/// Clears the busy flag on drop, so a watcher that panics mid-ingest cannot
/// leave the backfill waiting forever.
pub struct Ingesting<'a>(&'a LiveIngest);

impl Drop for Ingesting<'_> {
    fn drop(&mut self) {
        self.0.ingesting.store(0, Ordering::Relaxed);
    }
}

pub struct Backfill {
    pub store: Arc<SqliteStore>,
    pub embedder: Option<Arc<Embedder>>,
    pub machine_id: String,
    pub projects_dir: PathBuf,
    pub archive: Option<Arc<TranscriptArchive>>,
    pub live: Arc<LiveIngest>,
    pub health: Arc<Health>,
    pub pause: Duration,
}

#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct BackfillStats {
    pub captured: usize,
    pub skipped: usize,
    pub failed: usize,
}

impl Backfill {
    pub fn spawn(self) -> JoinHandle<()> {
        std::thread::spawn(move || {
            match self.run() {
                Ok(stats) => tracing::info!(
                    "transcript backfill done: {} captured, {} skipped, {} failed",
                    stats.captured,
                    stats.skipped,
                    stats.failed
                ),
                Err(e) => tracing::error!("transcript backfill failed: {e:#}"),
            }
            residency::release_free_heap();
        })
    }

    /// The first complete run walks every transcript; later runs only catch
    /// up on transcripts changed since they were archived, such as sessions
    /// written while the daemon was down.
    pub fn run(&self) -> anyhow::Result<BackfillStats> {
        let catching_up = self.store.transcript_backfill_done()?;
        let files: Vec<PathBuf> = newest_first(&self.projects_dir)
            .into_iter()
            .filter(|(mtime, path)| !catching_up || self.changed_since_archived(path, *mtime))
            .map(|(_, path)| path)
            .collect();
        tracing::info!("transcript backfill: {} transcripts to check", files.len());
        let mut report = BackfillReport {
            state: BackfillState::Running,
            total: files.len(),
            done: 0,
            failed: 0,
            updated_at: Some(Utc::now()),
        };
        self.health.backfill_progress(report.clone());

        let store: Arc<dyn StoreAdmin> = self.store.clone();
        let mut stats = BackfillStats::default();
        for (i, path) in files.iter().enumerate() {
            if i > 0 {
                std::thread::sleep(self.pause);
            }
            while self.live.busy() {
                std::thread::sleep(LIVE_POLL);
            }
            match capture_transcript(
                &store,
                self.embedder.as_deref(),
                self.archive.as_deref(),
                path,
                &self.machine_id,
            ) {
                Ok(FileOutcome::Captured) => stats.captured += 1,
                Ok(FileOutcome::Skipped) => stats.skipped += 1,
                Err(e) => {
                    stats.failed += 1;
                    tracing::warn!("transcript backfill: {}: {e:#}", path.display());
                }
            }
            report.done = i + 1;
            report.failed = stats.failed;
            report.updated_at = Some(Utc::now());
            self.health.backfill_progress(report.clone());
        }

        let marked = if catching_up || stats.failed > 0 {
            Ok(())
        } else {
            self.store.mark_transcript_backfill_done()
        };
        report.state = BackfillState::Done;
        report.updated_at = Some(Utc::now());
        self.health.backfill_progress(report);
        marked?;
        Ok(stats)
    }

    fn changed_since_archived(&self, path: &Path, mtime: Option<SystemTime>) -> bool {
        let Some(archive) = &self.archive else {
            return false;
        };
        match (archive.entry(path), mtime) {
            (None, _) | (Some(_), None) => true,
            (Some(entry), Some(mtime)) => DateTime::<Utc>::from(mtime) > entry.archived_at,
        }
    }
}

pub fn transcripts_newest_first(projects_dir: &Path) -> Vec<PathBuf> {
    newest_first(projects_dir)
        .into_iter()
        .map(|(_, path)| path)
        .collect()
}

fn newest_first(projects_dir: &Path) -> Vec<(Option<SystemTime>, PathBuf)> {
    let mut found = transcript_mtimes(projects_dir);
    found.sort_by(|(a_time, a_path), (b_time, b_path)| {
        b_time.cmp(a_time).then_with(|| a_path.cmp(b_path))
    });
    found
}

pub(crate) fn transcript_mtimes(dir: &Path) -> Vec<(Option<SystemTime>, PathBuf)> {
    let mut found = Vec::new();
    let mut dirs = vec![dir.to_path_buf()];
    while let Some(d) = dirs.pop() {
        let Ok(entries) = std::fs::read_dir(&d) else {
            continue;
        };
        for entry in entries.flatten() {
            let Ok(kind) = entry.file_type() else {
                continue;
            };
            let path = entry.path();
            if kind.is_dir() {
                dirs.push(path);
            } else if kind.is_file() && path.extension().is_some_and(|e| e == "jsonl") {
                let mtime = entry.metadata().and_then(|m| m.modified()).ok();
                found.push((mtime, path));
            }
        }
    }
    found
}