second-brain-api 0.13.5

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
mod common;

use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, Instant};

use axum::http::StatusCode;
use chrono::Utc;
use serde_json::json;
use tower::ServiceExt;

use second_brain_api::AppState;
use second_brain_api::backfill::{Backfill, LiveIngest};
use second_brain_api::bootstrap::Bootstrap;
use second_brain_api::clock_gate::ClockGate;
use second_brain_api::health::Health;
use second_brain_api::routes;
use second_brain_api::sources::Sources;
use second_brain_core::sqlite_store::{BootstrapState, SqliteStore};
use second_brain_core::store::Store;

use common::{json_body, post_json, state};

const COUNTERS: [&str; 7] = [
    "new_memories",
    "new_entities",
    "new_conversations",
    "skipped_dupes",
    "unchanged",
    "errors",
    "files_processed",
];

fn pending_state() -> Arc<AppState> {
    let state = state();
    state.bootstrap.mark_pending(&state.sqlite).unwrap();
    state
}

async fn post(
    state: &Arc<AppState>,
    uri: &str,
    body: serde_json::Value,
) -> axum::response::Response {
    routes::router()
        .with_state(state.clone())
        .oneshot(post_json(uri, body))
        .await
        .unwrap()
}

#[test]
fn a_new_store_loads_with_no_bootstrap_pending() {
    let store = SqliteStore::in_memory("m".to_string()).unwrap();
    let bootstrap = Bootstrap::load(&store).unwrap();
    assert!(!bootstrap.is_pending());
    assert!(!bootstrap.ingest_due());
}

#[test]
fn a_pending_bootstrap_is_stored_and_loaded_after_a_restart() {
    let store = SqliteStore::in_memory("m".to_string()).unwrap();
    Bootstrap::default().mark_pending(&store).unwrap();
    assert_eq!(
        store.bootstrap_state().unwrap(),
        Some(BootstrapState::Pending)
    );
    let reloaded = Bootstrap::load(&store).unwrap();
    assert!(reloaded.is_pending());
    assert!(!reloaded.ingest_due());
}

#[test]
fn finishing_a_pending_bootstrap_makes_the_full_ingest_due_until_it_is_done() {
    let store = SqliteStore::in_memory("m".to_string()).unwrap();
    let bootstrap = Bootstrap::default();
    bootstrap.mark_pending(&store).unwrap();

    assert!(bootstrap.finish(&store).unwrap());
    assert!(!bootstrap.is_pending());
    assert!(bootstrap.ingest_due());
    assert!(
        Bootstrap::load(&store).unwrap().ingest_due(),
        "a restart before the pass reruns it"
    );

    bootstrap.ingest_done(&store).unwrap();
    assert!(!bootstrap.ingest_due());
    assert_eq!(store.bootstrap_state().unwrap(), None);
}

#[test]
fn finishing_without_a_pending_bootstrap_changes_nothing() {
    let store = SqliteStore::in_memory("m".to_string()).unwrap();
    let bootstrap = Bootstrap::default();
    assert!(!bootstrap.finish(&store).unwrap());
    assert_eq!(store.bootstrap_state().unwrap(), None);
    assert!(!bootstrap.ingest_due());
}

#[test]
fn a_finished_ingest_pass_does_not_clear_a_bootstrap_pending_again() {
    let store = SqliteStore::in_memory("m".to_string()).unwrap();
    let bootstrap = Bootstrap::default();
    bootstrap.mark_pending(&store).unwrap();
    bootstrap.ingest_done(&store).unwrap();
    assert!(bootstrap.is_pending());
    assert_eq!(
        store.bootstrap_state().unwrap(),
        Some(BootstrapState::Pending)
    );
}

#[tokio::test]
async fn ingest_answers_deferred_with_zero_counts_while_the_bootstrap_is_pending() {
    let state = pending_state();
    let response = post(&state, "/v1/ingest", json!({})).await;
    assert_eq!(response.status(), StatusCode::OK);
    let body = json_body(response).await;
    assert_eq!(body["deferred"], json!(true), "{body}");
    for counter in COUNTERS {
        assert_eq!(body[counter], json!(0), "{counter} in {body}");
    }
}

#[tokio::test]
async fn reingest_answers_deferred_while_the_bootstrap_is_pending() {
    let state = pending_state();
    let response = post(&state, "/v1/reingest", json!({})).await;
    assert_eq!(response.status(), StatusCode::OK);
    let body = json_body(response).await;
    assert_eq!(body["deferred"], json!(true), "{body}");
    assert_eq!(body["cleared_log"], json!(0), "{body}");
    for counter in COUNTERS {
        assert_eq!(body[counter], json!(0), "{counter} in {body}");
    }
}

#[tokio::test]
async fn an_unknown_source_is_still_a_400_while_the_bootstrap_is_pending() {
    let state = pending_state();
    let response = post(&state, "/v1/ingest", json!({"source": "bogus"})).await;
    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
}

#[tokio::test]
async fn remember_still_stores_while_the_bootstrap_is_pending() {
    let state = pending_state();
    let response = post(
        &state,
        "/v1/remember",
        json!({"content": "The bootstrap gate leaves remember open because its ids are random."}),
    )
    .await;
    assert_eq!(response.status(), StatusCode::OK);
    assert_eq!(state.sqlite.memory_count().unwrap(), 1);
}

#[test]
fn sources_skip_their_pass_while_the_bootstrap_is_pending() {
    let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
    let sources = Sources {
        store: store.clone(),
        embedder: None,
        machine_id: "m".to_string(),
        pull_requests: None,
    };
    let bootstrap = Bootstrap::default();
    assert!(sources.run_unless_pending(&bootstrap).is_some());
    bootstrap.mark_pending(&store).unwrap();
    assert!(sources.run_unless_pending(&bootstrap).is_none());
}

fn transcript(session: &str) -> String {
    [
        json!({"type": "summary", "summary": "s"}),
        json!({"type": "user", "sessionId": session, "message": {"content": "question"}}),
        json!({"type": "assistant", "sessionId": session, "message": {"content": [{"type": "text",
            "text": "The watcher waits ten seconds for a transcript to settle before it ingests it, because Claude Code writes each turn in several appends."}]}}),
    ]
    .map(|line| line.to_string())
    .join("\n")
        + "\n"
}

fn write_transcript(projects: &Path) {
    let path = projects.join("-home-a-proj").join("s.jsonl");
    std::fs::create_dir_all(path.parent().unwrap()).unwrap();
    std::fs::write(path, transcript("sess-1")).unwrap();
}

#[test]
fn the_backfill_waits_while_the_bootstrap_is_pending() {
    let dir = tempfile::tempdir().unwrap();
    let projects = dir.path().join("projects");
    write_transcript(&projects);
    let store = Arc::new(SqliteStore::open(&dir.path().join("brain.db"), "m".to_string()).unwrap());
    let bootstrap = Arc::new(Bootstrap::default());
    bootstrap.mark_pending(&store).unwrap();
    let backfill = Backfill {
        store: store.clone(),
        embedder: None,
        machine_id: "m".to_string(),
        projects_dir: projects,
        archive: None,
        live: Arc::new(LiveIngest::default()),
        health: Arc::new(Health::new(Utc::now(), None)),
        pause: Duration::ZERO,
        clock: ClockGate::open_for_tests(),
    };

    let handle = backfill.spawn_gated(bootstrap.clone());
    std::thread::sleep(Duration::from_millis(1500));
    assert!(!handle.is_finished());
    assert_eq!(store.memory_count().unwrap(), 0);

    assert!(bootstrap.finish(&store).unwrap());
    let deadline = Instant::now() + Duration::from_secs(60);
    while !handle.is_finished() {
        assert!(
            Instant::now() < deadline,
            "the backfill never resumed after the bootstrap"
        );
        std::thread::sleep(Duration::from_millis(50));
    }
    handle.join().unwrap();
    assert!(store.memory_count().unwrap() > 0);
}

#[test]
fn a_join_marker_holds_an_empty_store_pending_and_is_read_once() {
    let dir = tempfile::tempdir().unwrap();
    let marker = dir
        .path()
        .join(second_brain_sync::pairing::JOIN_PENDING_FILE);
    std::fs::write(&marker, "").unwrap();
    let store = SqliteStore::in_memory("m".to_string()).unwrap();
    let bootstrap = Bootstrap::start(&store, dir.path());
    assert!(bootstrap.is_pending());
    assert!(!marker.exists());
    assert_eq!(
        store.bootstrap_state().unwrap(),
        Some(BootstrapState::Pending)
    );
}

#[tokio::test(flavor = "multi_thread")]
async fn a_join_marker_on_a_store_with_rows_is_dropped_without_waiting() {
    let state = state();
    let response = post(
        &state,
        "/v1/remember",
        json!({"content": "A note this store held before setup ran again."}),
    )
    .await;
    assert_eq!(response.status(), StatusCode::OK);
    let dir = tempfile::tempdir().unwrap();
    let marker = dir
        .path()
        .join(second_brain_sync::pairing::JOIN_PENDING_FILE);
    std::fs::write(&marker, "").unwrap();
    let bootstrap = Bootstrap::start(&state.sqlite, dir.path());
    assert!(!bootstrap.is_pending());
    assert!(!marker.exists());
}

#[test]
fn an_unreadable_bootstrap_state_holds_ingest_back() {
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("brain.db");
    drop(SqliteStore::open(&path, "m".to_string()).unwrap());
    rusqlite::Connection::open(&path)
        .unwrap()
        .execute(
            "INSERT OR REPLACE INTO index_meta (key, value) VALUES ('bootstrap', 'garbled')",
            [],
        )
        .unwrap();
    let store = SqliteStore::open(&path, "m".to_string()).unwrap();
    assert!(Bootstrap::start(&store, dir.path()).is_pending());
}

#[test]
fn cancelling_ends_only_a_pending_bootstrap() {
    let store = SqliteStore::in_memory("m".to_string()).unwrap();
    let bootstrap = Bootstrap::default();
    bootstrap.mark_pending(&store).unwrap();
    assert!(bootstrap.cancel(&store).unwrap());
    assert!(!bootstrap.is_pending());
    assert_eq!(store.bootstrap_state().unwrap(), None);

    bootstrap.mark_pending(&store).unwrap();
    bootstrap.finish(&store).unwrap();
    assert!(!bootstrap.cancel(&store).unwrap());
    assert!(bootstrap.ingest_due());
}