second-brain-api 0.13.2

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
mod common;

use std::sync::{Arc, RwLock};

use axum::http::StatusCode;
use tower::ServiceExt;
use uuid::Uuid;

use common::{json_body, post_json};
use second_brain_api::health::Health;
use second_brain_api::ingest::ingest_file;
use second_brain_api::{AppState, routes};
use second_brain_core::embedding::Embedder;
use second_brain_core::index::Indexes;
use second_brain_core::machine::MachineIdentity;
use second_brain_core::retrieval::RetrievalTunables;
use second_brain_core::schema::{Memory, MemoryType, Scope};
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::{Store, StoreAdmin};

struct Fixture {
    app: axum::Router,
    store: Arc<SqliteStore>,
    writer: Indexes,
    reader: Arc<RwLock<Indexes>>,
    _dir: tempfile::TempDir,
}

impl Fixture {
    fn new(embedder: Option<Arc<Embedder>>) -> Self {
        let machine = MachineIdentity {
            id: "redact-machine".to_string(),
            name: "redact".to_string(),
        };
        let store = Arc::new(SqliteStore::in_memory(machine.id.clone()).unwrap());
        store.register_machine(&machine).unwrap();
        let dir = tempfile::tempdir().unwrap();
        let mut writer = Indexes::open(dir.path()).unwrap();
        writer.persist().unwrap();
        let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir.path()).unwrap()));
        let state = Arc::new(AppState {
            data_dir: std::env::temp_dir().join("second-brain-test-no-device-key"),
            pairing: Default::default(),
            bootstrap: Default::default(),
            device_access: Default::default(),
            gate: Default::default(),
            clock: second_brain_api::clock_gate::ClockGate::open_for_tests(),
            store: store.clone(),
            sqlite: store.clone(),
            indexes: Some(reader.clone()),
            raw_index: None,
            raw_coverage: None,
            archive: None,
            reranker: None,
            tunables: RetrievalTunables::default(),
            embedder,
            machine,
            maintenance: Arc::new(RwLock::new(None)),
            repair_tx: None,
            near_duplicate_threshold: 0.97,
            near_duplicates: Arc::new(RwLock::new(None)),
            health: Arc::new(Health::new(chrono::Utc::now(), None)),
        });
        Fixture {
            app: routes::router().with_state(state),
            store,
            writer,
            reader,
            _dir: dir,
        }
    }

    fn drain(&mut self) {
        self.writer.drain(&self.store).unwrap();
        self.reader.write().unwrap().reload_view().unwrap();
    }

    async fn post(&self, uri: &str, body: serde_json::Value) -> serde_json::Value {
        let response = self
            .app
            .clone()
            .oneshot(post_json(uri, body))
            .await
            .unwrap();
        let status = response.status();
        let body = json_body(response).await;
        assert_eq!(status, StatusCode::OK, "{uri}: {body}");
        body
    }
}

fn fake_github_token(body: &str) -> String {
    ["gh", "p_"].concat() + &body.repeat(6)
}

/// Every place a stored secret could surface: memory content and context,
/// and the sync log peers replicate from, which is the only read of a
/// relation's context.
fn assert_stored_nowhere(store: &SqliteStore, secret: &str) {
    for id in store.all_memory_ids().unwrap() {
        let memory = store.get_memory(id).unwrap().unwrap();
        assert!(!memory.content.contains(secret), "{}", memory.content);
        assert!(!memory.context.unwrap_or_default().contains(secret));
    }
    let log = format!("{:?}", store.sync_log_since(0).unwrap());
    assert!(!log.contains(secret), "secret in sync_log");
}

fn id_of(body: &serde_json::Value, key: &str) -> Uuid {
    Uuid::parse_str(body[key].as_str().unwrap()).unwrap()
}

#[tokio::test]
async fn remember_stores_and_indexes_no_secret() {
    let mut f = Fixture::new(None);
    let token = fake_github_token("Qq7xZ2");
    let password = ["S3cret", "Pw9Kx2"].concat();

    let body = f
        .post(
            "/v1/remember",
            serde_json::json!({
                "content": format!(
                    "the deploy bot pushes with {token} and reads postgres://app:{password}@db.internal/app"
                ),
            }),
        )
        .await;
    f.drain();

    let stored = f.store.get_memory(id_of(&body, "id")).unwrap().unwrap();
    assert_eq!(
        stored.content,
        "the deploy bot pushes with [REDACTED:github_token] and reads \
         postgres://[REDACTED:url_credentials]@db.internal/app"
    );
    assert_stored_nowhere(&f.store, &token);
    assert_stored_nowhere(&f.store, &password);
    let index = f.reader.read().unwrap();
    assert!(!index.bm25_search("deploy bot", 5).unwrap().is_empty());
    assert!(index.bm25_search(&token, 5).unwrap().is_empty());
    assert!(index.bm25_search(&password, 5).unwrap().is_empty());
}

#[tokio::test]
async fn remember_dedupes_two_notes_that_differ_only_in_their_secret() {
    let f = Fixture::new(None);
    let note = |token: String| serde_json::json!({"content": format!("ci pushes with {token}")});

    let first = f
        .post("/v1/remember", note(fake_github_token("Aa1bB2")))
        .await;
    let second = f
        .post("/v1/remember", note(fake_github_token("Cc3dD4")))
        .await;

    assert_eq!(second["duplicate"], true);
    assert_eq!(second["id"], first["id"]);
}

#[tokio::test]
async fn consolidate_store_redacts_the_distilled_content() {
    let f = Fixture::new(None);
    let token = fake_github_token("Ee5fF6");

    let body = f
        .post(
            "/v1/consolidate/store",
            serde_json::json!({
                "raw_ids": [],
                "source_id": "session-1",
                "distilled_content": format!("the release job pushes with {token}"),
                "model": "test-model",
            }),
        )
        .await;

    let stored = f
        .store
        .get_memory(id_of(&body, "distilled_id"))
        .unwrap()
        .unwrap();
    assert_eq!(
        stored.content,
        "the release job pushes with [REDACTED:github_token]"
    );
    assert_stored_nowhere(&f.store, &token);
}

#[tokio::test]
async fn merge_redacts_the_merged_content() {
    let embedder = Arc::new(Embedder::new().unwrap());
    let f = Fixture::new(Some(embedder));
    let distillate = |content: &str| {
        let mut m = Memory::new(
            content.to_string(),
            MemoryType::Semantic,
            "consolidation".to_string(),
            String::new(),
        )
        .with_project_path(Some("-repo".to_string()));
        m.embedding = vec![0.0; 384];
        m.embedding[0] = 1.0;
        m
    };
    let a = distillate("the release job pushes with a token");
    let b = distillate("release pushes use a token");
    f.store.store_memory(&a).unwrap();
    f.store.store_memory(&b).unwrap();
    let token = fake_github_token("Gg7hH8");

    let body = f
        .post(
            "/v1/consolidate/merge",
            serde_json::json!({
                "member_ids": [a.id.to_string(), b.id.to_string()],
                "content": format!("the release job pushes with {token}"),
                "model": "test-model",
            }),
        )
        .await;

    let merged = f
        .store
        .get_memory(id_of(&body, "merged_id"))
        .unwrap()
        .unwrap();
    assert_eq!(
        merged.content,
        "the release job pushes with [REDACTED:github_token]"
    );
    assert_stored_nowhere(&f.store, &token);
}

#[tokio::test]
async fn associate_redacts_the_relation_context() {
    let f = Fixture::new(None);
    let token = fake_github_token("Jj9kK0");
    let from = f
        .post("/v1/remember", serde_json::json!({"content": "first note"}))
        .await;
    let to = f
        .post(
            "/v1/remember",
            serde_json::json!({"content": "second note"}),
        )
        .await;

    let response = f
        .app
        .clone()
        .oneshot(post_json(
            "/v1/associate",
            serde_json::json!({
                "from_id": from["id"],
                "to_id": to["id"],
                "relation": "relates_to",
                "context": format!("both use {token}"),
            }),
        ))
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::CREATED);
    assert_stored_nowhere(&f.store, &token);
    let log = format!("{:?}", f.store.sync_log_since(0).unwrap());
    assert!(log.contains("both use [REDACTED:github_token]"));
}

#[test]
fn an_ingested_transcript_stores_no_secret_and_an_unchanged_file_is_skipped() {
    let token = fake_github_token("Mm1nN2");
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("session.jsonl");
    let lines = [
        serde_json::json!({
            "type": "user",
            "timestamp": "2026-07-23T08:59:00Z",
            "message": {"content": "why did the deploy fail"},
        }),
        serde_json::json!({
            "type": "assistant",
            "timestamp": "2026-07-23T09:05:00Z",
            "message": {"content": [{"type": "text", "text": format!(
                "The deploy failed because the old token was revoked, so we rotated it and the \
                 runner now pushes with {token} from the vault."
            )}]},
        }),
    ];
    std::fs::write(
        &path,
        lines
            .iter()
            .map(|l| l.to_string())
            .collect::<Vec<_>>()
            .join("\n"),
    )
    .unwrap();
    let sqlite = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
    let store: Arc<dyn StoreAdmin> = sqlite.clone();

    ingest_file(&store, None, &path, "m").unwrap();

    assert!(sqlite.memory_count().unwrap() > 0);
    assert_stored_nowhere(&sqlite, &token);
    let redacted = sqlite
        .all_memory_ids()
        .unwrap()
        .into_iter()
        .filter_map(|id| sqlite.get_memory(id).unwrap())
        .any(|m| m.content.contains("[REDACTED:github_token]"));
    assert!(redacted);
    let index_dir = tempfile::tempdir().unwrap();
    let mut index = Indexes::open(index_dir.path()).unwrap();
    index.drain(&sqlite).unwrap();
    assert!(!index.bm25_search("deploy", 5).unwrap().is_empty());
    assert!(index.bm25_search(&token, 5).unwrap().is_empty());
    let seq = sqlite.max_sync_seq().unwrap();

    ingest_file(&store, None, &path, "m").unwrap();

    assert_eq!(sqlite.max_sync_seq().unwrap(), seq);
    assert_eq!(sqlite.ingested_file_count().unwrap(), 1);
}

#[tokio::test]
async fn remember_redacts_related_entity_names() {
    let f = Fixture::new(None);
    let password = ["S3cret", "Pw9Kx2"].concat();
    let related = format!("https://bot:{password}@git.example.com/repo");

    f.post(
        "/v1/remember",
        serde_json::json!({"content": "the mirror job clones nightly", "related_to": [related]}),
    )
    .await;

    assert!(f.store.find_entity_by_name(&related).unwrap().is_none());
    assert!(
        f.store
            .find_entity_by_name("https://[REDACTED:url_credentials]@git.example.com/repo")
            .unwrap()
            .is_some()
    );
    assert_stored_nowhere(&f.store, &password);
}

#[tokio::test]
async fn forget_redacts_the_reason() {
    let f = Fixture::new(None);
    let token = fake_github_token("Pp3qQ4");
    let body = f
        .post("/v1/remember", serde_json::json!({"content": "stale note"}))
        .await;
    let id = id_of(&body, "id");

    let response = f
        .app
        .clone()
        .oneshot(common::delete(&format!(
            "/v1/memory/{id}?reason=leaked%20{token}"
        )))
        .await
        .unwrap();

    assert_eq!(response.status(), StatusCode::OK);
    let retired = f.store.get_memory(id).unwrap().unwrap();
    assert_eq!(
        retired.retired_reason.as_deref(),
        Some("leaked [REDACTED:github_token]")
    );
    assert_stored_nowhere(&f.store, &token);
}

#[tokio::test]
async fn a_shared_memory_is_redacted_like_a_private_one() {
    let mut f = Fixture::new(None);
    let token = fake_github_token("Ss5hR8");

    let body = f
        .post(
            "/v1/remember",
            serde_json::json!({
                "content": format!("the team bot pushes with {token}"),
                "scope": "shared",
            }),
        )
        .await;
    f.drain();

    let stored = f.store.get_memory(id_of(&body, "id")).unwrap().unwrap();
    assert_eq!(stored.scope, Scope::Shared);
    assert_eq!(
        stored.content,
        "the team bot pushes with [REDACTED:github_token]"
    );
    assert_stored_nowhere(&f.store, &token);
}