second-brain-api 0.13.5

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
use std::sync::{Arc, Mutex};
use std::time::Duration;

use axum::Json;
use axum::Router;
use axum::extract::State;
use axum::routing::post;
use chrono::Utc;
use second_brain_api::alert;
use second_brain_api::health::{DistillMode, Health, NO_GPU_NOTE, State as HealthState};
use second_brain_api::progress::ProgressCheck;
use second_brain_core::config::AlertConfig;
use second_brain_core::schema::{IngestRecord, Memory, MemoryType, SyncState};
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::{Store, StoreAdmin};
use second_brain_sync::peers;

type Received = Arc<Mutex<Vec<serde_json::Value>>>;

async fn receiver() -> (String, Received) {
    let received: Received = Arc::default();
    let app = Router::new()
        .route(
            "/hook",
            post(
                |State(r): State<Received>, Json(body): Json<serde_json::Value>| async move {
                    r.lock().unwrap().push(body);
                },
            ),
        )
        .with_state(received.clone());
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
    let addr = listener.local_addr().unwrap();
    tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
    (format!("http://{addr}/hook"), received)
}

async fn wait_for(received: &Received, n: usize) {
    for _ in 0..100 {
        if received.lock().unwrap().len() >= n {
            return;
        }
        tokio::time::sleep(Duration::from_millis(50)).await;
    }
}

struct Device {
    _dir: tempfile::TempDir,
    data_dir: std::path::PathBuf,
    store: Arc<SqliteStore>,
}

fn device() -> Device {
    let dir = tempfile::tempdir().unwrap();
    let data_dir = dir.path().join(".second-brain");
    std::fs::create_dir_all(&data_dir).unwrap();
    let store = SqliteStore::open(&data_dir.join("brain.db"), "machine-a".to_string()).unwrap();
    Device {
        _dir: dir,
        data_dir,
        store: Arc::new(store),
    }
}

fn check(device: &Device, health: Arc<Health>) -> ProgressCheck {
    check_in(device, health, true)
}

fn check_in(device: &Device, health: Arc<Health>, llm_configured: bool) -> ProgressCheck {
    ProgressCheck {
        store: device.store.clone(),
        data_dir: device.data_dir.clone(),
        health,
        llm_configured,
    }
}

fn synced_with_a_peer(device: &Device) {
    device
        .store
        .set_sync_state(&SyncState {
            peer_id: "6f1c1d1e-0000-4000-8000-000000000001".to_string(),
            last_seq: 42,
            last_sync_at: Utc::now() - chrono::Duration::days(2),
        })
        .unwrap();
}

async fn announced_on_startup(device: &Device) -> Vec<serde_json::Value> {
    let (url, received) = receiver().await;
    let config = AlertConfig {
        webhook_url: Some(url),
        ..AlertConfig::default()
    };
    let tx = alert::start(&config, "laptop".to_string()).expect("a webhook is set");
    let health = Arc::new(Health::new(Utc::now(), Some(tx)));
    let task = check(device, health.clone()).start().await;
    health.announce(Utc::now());
    wait_for(&received, 1).await;
    tokio::time::sleep(Duration::from_millis(300)).await;
    task.abort();
    received.lock().unwrap().clone()
}

#[tokio::test]
async fn startup_sends_the_ok_state_of_a_solo_device_once() {
    let device = device();
    let bodies = announced_on_startup(&device).await;
    assert_eq!(bodies.len(), 1, "{bodies:?}");
    assert_eq!(bodies[0]["state"], "ok");
    assert!(bodies[0]["reason"].is_null());
    assert_eq!(bodies[0]["machine"], "laptop");
}

#[tokio::test]
async fn startup_sends_a_degraded_state_for_a_device_that_lost_its_peers() {
    let device = device();
    synced_with_a_peer(&device);
    std::fs::write(device.data_dir.join(peers::PEERS_FILE), "").unwrap();
    let bodies = announced_on_startup(&device).await;
    assert_eq!(bodies.len(), 1, "{bodies:?}");
    assert_eq!(bodies[0]["state"], "degraded");
    let reason = bodies[0]["reason"].as_str().unwrap();
    assert!(reason.contains("no paired device"), "{reason}");
}

#[test]
fn a_recorded_sync_success_clears_an_overdue_sync() {
    let device = device();
    synced_with_a_peer(&device);
    std::fs::write(device.data_dir.join(peers::PEERS_FILE), "user@peer\n").unwrap();
    let now = Utc::now();
    let health = Arc::new(Health::new(now, None));
    let check = check(&device, health.clone());

    peers::record_success(&device.data_dir, now - chrono::Duration::hours(30)).unwrap();
    check.run_once(now);
    let report = health.report();
    assert_eq!(report.state, HealthState::Degraded);
    assert!(
        report
            .reason
            .as_deref()
            .unwrap()
            .contains("no sync has succeeded in 30h"),
        "{report:?}"
    );
    assert_eq!(report.sync.peers, Some(1));
    assert!(report.sync.synced_before);

    peers::record_success(&device.data_dir, now).unwrap();
    check.run_once(now);
    let report = health.report();
    assert_eq!(report.state, HealthState::Ok, "{:?}", report.reason);
    assert_eq!(report.sync.last_success_at, Some(now));
}

fn old_raw(device: &Device) {
    let mut raw = Memory::new(
        "a transcript chunk".to_string(),
        MemoryType::Episodic,
        "claude-code".to_string(),
        String::new(),
    );
    raw.created_at = Utc::now() - chrono::Duration::hours(30);
    device.store.store_memory(&raw).unwrap();
}

#[test]
fn without_an_llm_endpoint_waiting_raws_are_counted_under_the_no_gpu_mode() {
    let device = device();
    old_raw(&device);
    let now = Utc::now();
    let health = Arc::new(Health::new(now, None));
    check_in(&device, health.clone(), false).run_once(now);

    let report = health.report();
    assert_eq!(report.state, HealthState::Ok, "{:?}", report.reason);
    assert_eq!(report.distill.pending, 1);
    assert_eq!(report.distill.mode, DistillMode::NoGpu);
    assert_eq!(report.distill.note, Some(NO_GPU_NOTE));
}

#[test]
fn without_an_llm_endpoint_recent_llm_distillation_keeps_the_llm_mode_and_degrades() {
    let device = device();
    old_raw(&device);
    let distilled = Memory::new(
        "a chunk a GPU peer distilled".to_string(),
        MemoryType::Episodic,
        "claude-code".to_string(),
        "/tmp/s2.jsonl".to_string(),
    );
    device.store.store_memory(&distilled).unwrap();
    let summary = Memory::new(
        "the distillate".to_string(),
        MemoryType::Semantic,
        "consolidation".to_string(),
        "batch-1".to_string(),
    );
    device.store.store_memory(&summary).unwrap();
    device
        .store
        .mark_consolidated(distilled.id, summary.id, "some-model")
        .unwrap();
    let now = Utc::now();
    let health = Arc::new(Health::new(now, None));
    check_in(&device, health.clone(), false).run_once(now);

    let report = health.report();
    assert_eq!(report.distill.mode, DistillMode::Llm);
    assert_eq!(report.distill.note, None);
    assert_eq!(report.state, HealthState::Degraded, "{report:?}");
}

#[test]
fn raws_waiting_to_be_distilled_are_counted_from_the_store() {
    let device = device();
    old_raw(&device);
    let now = Utc::now();
    let health = Arc::new(Health::new(now, None));
    check(&device, health.clone()).run_once(now);

    let report = health.report();
    assert_eq!(report.state, HealthState::Degraded);
    assert_eq!(report.distill.pending, 1);
    assert_eq!(report.distill.mode, DistillMode::Llm);
    assert!(
        report
            .reason
            .as_deref()
            .unwrap()
            .contains("1 raw memory has waited 30h"),
        "{report:?}"
    );
}

#[test]
fn an_unreadable_peers_list_degrades_once_it_persists() {
    let device = device();
    std::fs::create_dir(device.data_dir.join(peers::PEERS_FILE)).unwrap();
    let now = Utc::now();
    let health = Arc::new(Health::new(now, None));
    let check = check(&device, health.clone());
    check.run_once(now);
    assert_eq!(health.report().state, HealthState::Ok);
    check.run_once(now + chrono::Duration::minutes(10));
    let report = health.report();
    assert_eq!(report.state, HealthState::Degraded);
    assert!(
        report
            .reason
            .as_deref()
            .unwrap()
            .contains("cannot check sync and distillation"),
        "{report:?}"
    );
}

#[test]
fn an_unreadable_peers_list_is_named_while_the_rest_of_sync_is_still_reported() {
    let device = device();
    synced_with_a_peer(&device);
    let peers_file = device.data_dir.join(peers::PEERS_FILE);
    std::fs::write(&peers_file, "u@pi\n").unwrap();
    let now = Utc::now();
    let health = Arc::new(Health::new(now, None));
    let check = check(&device, health.clone());
    check.run_once(now);
    assert_eq!(health.report().sync.peers, Some(1));

    std::fs::remove_file(&peers_file).unwrap();
    std::fs::create_dir(&peers_file).unwrap();
    check.run_once(now + chrono::Duration::minutes(5));
    let sync = health.report().sync;
    assert!(sync.synced_before, "{sync:?}");
    assert!(sync.last_cursor_at.is_some(), "{sync:?}");
    assert_eq!(
        sync.peers,
        Some(1),
        "the last count stands while the file is unreadable"
    );

    check.run_once(now + chrono::Duration::minutes(15));
    let reason = health.report().reason.unwrap_or_default();
    assert!(reason.contains(peers::PEERS_FILE), "{reason}");
}

fn ingested(device: &Device, path: &str, session_key: &str, source: &str) {
    device
        .store
        .record_ingested(&IngestRecord {
            file_path: path,
            content_hash: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
            session_key: Some(session_key),
            legacy_ids: false,
            memory_count: 1,
            source,
        })
        .unwrap();
}

fn transcript(dir: &tempfile::TempDir, name: &str) -> String {
    let path = dir.path().join(name);
    std::fs::write(&path, "{}\n").unwrap();
    path.to_str().unwrap().to_string()
}

fn counted_if_birth_time_is_readable(path: &str) -> u64 {
    match std::fs::metadata(path).unwrap().created() {
        Ok(_) => 1,
        Err(_) => 0,
    }
}

#[test]
fn a_progress_pass_counts_claude_code_sessions_ingested_since_the_daemon_started() {
    let device = device();
    let transcripts = tempfile::tempdir().unwrap();
    // File times come from the kernel's coarse clock, which can trail Utc::now() by a tick.
    let health = Arc::new(Health::new(Utc::now() - chrono::Duration::seconds(1), None));
    let check = check(&device, health.clone());
    check.run_once(Utc::now());
    assert_eq!(health.report().value.recall.sessions_ingested, Some(0));

    let s1 = transcript(&transcripts, "s1.jsonl");
    let s1_again = transcript(&transcripts, "s1-again.jsonl");
    let sub = transcript(&transcripts, "s1-agent.jsonl");
    let c1 = transcript(&transcripts, "c1.jsonl");
    ingested(&device, &s1, "session-1", "claude-code");
    ingested(&device, &s1_again, "session-1", "claude-code");
    ingested(&device, &sub, "session-1/agent-1", "claude-code");
    ingested(&device, &c1, "cursor-1", "cursor");
    ingested(&device, "/p/gone.jsonl", "session-gone", "claude-code");
    check.run_once(Utc::now());

    let report = health.report();
    assert_eq!(
        report.value.recall.sessions_ingested,
        Some(counted_if_birth_time_is_readable(&s1))
    );
    assert_eq!(report.state, HealthState::Ok, "{:?}", report.reason);
}

#[test]
fn a_session_whose_transcript_predates_the_daemon_is_not_counted_when_reingested() {
    let device = device();
    let transcripts = tempfile::tempdir().unwrap();
    let old = transcript(&transcripts, "old.jsonl");
    std::thread::sleep(Duration::from_millis(1100));
    let health = Arc::new(Health::new(
        Utc::now() - chrono::Duration::milliseconds(100),
        None,
    ));
    let check = check(&device, health.clone());
    let new = transcript(&transcripts, "new.jsonl");
    ingested(&device, &old, "session-old", "claude-code");
    ingested(&device, &new, "session-new", "claude-code");
    check.run_once(Utc::now());

    assert_eq!(
        health.report().value.recall.sessions_ingested,
        Some(counted_if_birth_time_is_readable(&new))
    );
}