mod common;
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, Instant};
use axum::http::StatusCode;
use chrono::Utc;
use serde_json::json;
use tower::ServiceExt;
use second_brain_api::AppState;
use second_brain_api::backfill::{Backfill, LiveIngest};
use second_brain_api::bootstrap::Bootstrap;
use second_brain_api::clock_gate::ClockGate;
use second_brain_api::health::Health;
use second_brain_api::routes;
use second_brain_api::sources::Sources;
use second_brain_core::sqlite_store::{BootstrapState, SqliteStore};
use second_brain_core::store::Store;
use common::{json_body, post_json, state};
const COUNTERS: [&str; 7] = [
"new_memories",
"new_entities",
"new_conversations",
"skipped_dupes",
"unchanged",
"errors",
"files_processed",
];
fn pending_state() -> Arc<AppState> {
let state = state();
state.bootstrap.mark_pending(&state.sqlite).unwrap();
state
}
async fn post(
state: &Arc<AppState>,
uri: &str,
body: serde_json::Value,
) -> axum::response::Response {
routes::router()
.with_state(state.clone())
.oneshot(post_json(uri, body))
.await
.unwrap()
}
#[test]
fn a_new_store_loads_with_no_bootstrap_pending() {
let store = SqliteStore::in_memory("m".to_string()).unwrap();
let bootstrap = Bootstrap::load(&store).unwrap();
assert!(!bootstrap.is_pending());
assert!(!bootstrap.ingest_due());
}
#[test]
fn a_pending_bootstrap_is_stored_and_loaded_after_a_restart() {
let store = SqliteStore::in_memory("m".to_string()).unwrap();
Bootstrap::default().mark_pending(&store).unwrap();
assert_eq!(
store.bootstrap_state().unwrap(),
Some(BootstrapState::Pending)
);
let reloaded = Bootstrap::load(&store).unwrap();
assert!(reloaded.is_pending());
assert!(!reloaded.ingest_due());
}
#[test]
fn finishing_a_pending_bootstrap_makes_the_full_ingest_due_until_it_is_done() {
let store = SqliteStore::in_memory("m".to_string()).unwrap();
let bootstrap = Bootstrap::default();
bootstrap.mark_pending(&store).unwrap();
assert!(bootstrap.finish(&store).unwrap());
assert!(!bootstrap.is_pending());
assert!(bootstrap.ingest_due());
assert!(
Bootstrap::load(&store).unwrap().ingest_due(),
"a restart before the pass reruns it"
);
bootstrap.ingest_done(&store).unwrap();
assert!(!bootstrap.ingest_due());
assert_eq!(store.bootstrap_state().unwrap(), None);
}
#[test]
fn finishing_without_a_pending_bootstrap_changes_nothing() {
let store = SqliteStore::in_memory("m".to_string()).unwrap();
let bootstrap = Bootstrap::default();
assert!(!bootstrap.finish(&store).unwrap());
assert_eq!(store.bootstrap_state().unwrap(), None);
assert!(!bootstrap.ingest_due());
}
#[test]
fn a_finished_ingest_pass_does_not_clear_a_bootstrap_pending_again() {
let store = SqliteStore::in_memory("m".to_string()).unwrap();
let bootstrap = Bootstrap::default();
bootstrap.mark_pending(&store).unwrap();
bootstrap.ingest_done(&store).unwrap();
assert!(bootstrap.is_pending());
assert_eq!(
store.bootstrap_state().unwrap(),
Some(BootstrapState::Pending)
);
}
#[tokio::test]
async fn ingest_answers_deferred_with_zero_counts_while_the_bootstrap_is_pending() {
let state = pending_state();
let response = post(&state, "/v1/ingest", json!({})).await;
assert_eq!(response.status(), StatusCode::OK);
let body = json_body(response).await;
assert_eq!(body["deferred"], json!(true), "{body}");
for counter in COUNTERS {
assert_eq!(body[counter], json!(0), "{counter} in {body}");
}
}
#[tokio::test]
async fn reingest_answers_deferred_while_the_bootstrap_is_pending() {
let state = pending_state();
let response = post(&state, "/v1/reingest", json!({})).await;
assert_eq!(response.status(), StatusCode::OK);
let body = json_body(response).await;
assert_eq!(body["deferred"], json!(true), "{body}");
assert_eq!(body["cleared_log"], json!(0), "{body}");
for counter in COUNTERS {
assert_eq!(body[counter], json!(0), "{counter} in {body}");
}
}
#[tokio::test]
async fn an_unknown_source_is_still_a_400_while_the_bootstrap_is_pending() {
let state = pending_state();
let response = post(&state, "/v1/ingest", json!({"source": "bogus"})).await;
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn remember_still_stores_while_the_bootstrap_is_pending() {
let state = pending_state();
let response = post(
&state,
"/v1/remember",
json!({"content": "The bootstrap gate leaves remember open because its ids are random."}),
)
.await;
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(state.sqlite.memory_count().unwrap(), 1);
}
#[test]
fn sources_skip_their_pass_while_the_bootstrap_is_pending() {
let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
let sources = Sources {
store: store.clone(),
embedder: None,
machine_id: "m".to_string(),
pull_requests: None,
};
let bootstrap = Bootstrap::default();
assert!(sources.run_unless_pending(&bootstrap).is_some());
bootstrap.mark_pending(&store).unwrap();
assert!(sources.run_unless_pending(&bootstrap).is_none());
}
fn transcript(session: &str) -> String {
[
json!({"type": "summary", "summary": "s"}),
json!({"type": "user", "sessionId": session, "message": {"content": "question"}}),
json!({"type": "assistant", "sessionId": session, "message": {"content": [{"type": "text",
"text": "The watcher waits ten seconds for a transcript to settle before it ingests it, because Claude Code writes each turn in several appends."}]}}),
]
.map(|line| line.to_string())
.join("\n")
+ "\n"
}
fn write_transcript(projects: &Path) {
let path = projects.join("-home-a-proj").join("s.jsonl");
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, transcript("sess-1")).unwrap();
}
#[test]
fn the_backfill_waits_while_the_bootstrap_is_pending() {
let dir = tempfile::tempdir().unwrap();
let projects = dir.path().join("projects");
write_transcript(&projects);
let store = Arc::new(SqliteStore::open(&dir.path().join("brain.db"), "m".to_string()).unwrap());
let bootstrap = Arc::new(Bootstrap::default());
bootstrap.mark_pending(&store).unwrap();
let backfill = Backfill {
store: store.clone(),
embedder: None,
machine_id: "m".to_string(),
projects_dir: projects,
archive: None,
live: Arc::new(LiveIngest::default()),
health: Arc::new(Health::new(Utc::now(), None)),
pause: Duration::ZERO,
clock: ClockGate::open_for_tests(),
};
let handle = backfill.spawn_gated(bootstrap.clone());
std::thread::sleep(Duration::from_millis(1500));
assert!(!handle.is_finished());
assert_eq!(store.memory_count().unwrap(), 0);
assert!(bootstrap.finish(&store).unwrap());
let deadline = Instant::now() + Duration::from_secs(60);
while !handle.is_finished() {
assert!(
Instant::now() < deadline,
"the backfill never resumed after the bootstrap"
);
std::thread::sleep(Duration::from_millis(50));
}
handle.join().unwrap();
assert!(store.memory_count().unwrap() > 0);
}
#[test]
fn a_join_marker_holds_an_empty_store_pending_and_is_read_once() {
let dir = tempfile::tempdir().unwrap();
let marker = dir
.path()
.join(second_brain_sync::pairing::JOIN_PENDING_FILE);
std::fs::write(&marker, "").unwrap();
let store = SqliteStore::in_memory("m".to_string()).unwrap();
let bootstrap = Bootstrap::start(&store, dir.path());
assert!(bootstrap.is_pending());
assert!(!marker.exists());
assert_eq!(
store.bootstrap_state().unwrap(),
Some(BootstrapState::Pending)
);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_join_marker_on_a_store_with_rows_is_dropped_without_waiting() {
let state = state();
let response = post(
&state,
"/v1/remember",
json!({"content": "A note this store held before setup ran again."}),
)
.await;
assert_eq!(response.status(), StatusCode::OK);
let dir = tempfile::tempdir().unwrap();
let marker = dir
.path()
.join(second_brain_sync::pairing::JOIN_PENDING_FILE);
std::fs::write(&marker, "").unwrap();
let bootstrap = Bootstrap::start(&state.sqlite, dir.path());
assert!(!bootstrap.is_pending());
assert!(!marker.exists());
}
#[test]
fn an_unreadable_bootstrap_state_holds_ingest_back() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("brain.db");
drop(SqliteStore::open(&path, "m".to_string()).unwrap());
rusqlite::Connection::open(&path)
.unwrap()
.execute(
"INSERT OR REPLACE INTO index_meta (key, value) VALUES ('bootstrap', 'garbled')",
[],
)
.unwrap();
let store = SqliteStore::open(&path, "m".to_string()).unwrap();
assert!(Bootstrap::start(&store, dir.path()).is_pending());
}
#[test]
fn cancelling_ends_only_a_pending_bootstrap() {
let store = SqliteStore::in_memory("m".to_string()).unwrap();
let bootstrap = Bootstrap::default();
bootstrap.mark_pending(&store).unwrap();
assert!(bootstrap.cancel(&store).unwrap());
assert!(!bootstrap.is_pending());
assert_eq!(store.bootstrap_state().unwrap(), None);
bootstrap.mark_pending(&store).unwrap();
bootstrap.finish(&store).unwrap();
assert!(!bootstrap.cancel(&store).unwrap());
assert!(bootstrap.ingest_due());
}