second-brain-api 0.12.0

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
mod common;

use std::io::{BufRead, BufReader, Read, Write};
use std::path::Path;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::thread::{self, JoinHandle};
use std::time::{Duration, Instant};

use anyhow::Result;
use axum::http::StatusCode;
use chrono::{SecondsFormat, Utc};
use serde_json::Value;
use tower::ServiceExt;

use common::{app_with_sqlite, get, json_body, post_json};
use second_brain_api::sync_loop::{Connect, SyncLoop};
use second_brain_api::write_gate::WriteGate;
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_sync::crypto::{XChaChaEncryptor, open_line, seal_line};
use second_brain_sync::export::export_changes;
use second_brain_sync::import::import_changes;
use second_brain_sync::peers::{self, Peer};
use second_brain_sync::protocol::SyncHeader;
use second_brain_sync::session::{Exit, Remote, Transport, read_checked_header};

struct InProcessPeer {
    host: String,
    store: Arc<SqliteStore>,
    encryptor: Arc<XChaChaEncryptor>,
}

impl Transport for InProcessPeer {
    fn host(&self) -> &str {
        &self.host
    }

    fn whoami(&self) -> Result<String> {
        Ok(self.store.machine_id().to_string())
    }

    fn export(&self, after_seq: u64) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
        let (output, mut writer) = std::io::pipe()?;
        let store = self.store.clone();
        let encryptor = self.encryptor.clone();
        let worker = thread::spawn(move || -> Result<()> {
            let mut records = Vec::new();
            export_changes(&*store, after_seq, &mut records)?;
            writeln!(writer, "{}", SyncHeader::local().to_line())?;
            for record in records.split(|b| *b == b'\n').filter(|r| !r.is_empty()) {
                writeln!(writer, "{}", seal_line(&*encryptor, record)?)?;
            }
            Ok(())
        });
        Ok((Box::new(output), Box::new(PipedRemote(Some(worker)))))
    }

    fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Remote>)> {
        let (reader, input) = std::io::pipe()?;
        let store = self.store.clone();
        let encryptor = self.encryptor.clone();
        let worker = thread::spawn(move || -> Result<()> {
            let mut reader = BufReader::new(reader);
            read_checked_header(&mut reader, "the pushing daemon")?;
            let mut records = Vec::new();
            for line in reader.lines() {
                let line = line?;
                if line.trim().is_empty() {
                    continue;
                }
                records.extend(open_line(&*encryptor, &line)?);
                records.push(b'\n');
            }
            import_changes(&*store, records.as_slice())?;
            Ok(())
        });
        Ok((Box::new(input), Box::new(PipedRemote(Some(worker)))))
    }
}

struct PipedRemote(Option<JoinHandle<Result<()>>>);

impl Remote for PipedRemote {
    fn wait(&mut self) -> Result<Exit> {
        let Some(worker) = self.0.take() else {
            anyhow::bail!("remote already waited on");
        };
        Ok(match worker.join() {
            Ok(Ok(())) => Exit {
                success: true,
                status: "exit status: 0".to_string(),
            },
            Ok(Err(e)) => Exit {
                success: false,
                status: format!("{e:#}"),
            },
            Err(_) => Exit {
                success: false,
                status: "panicked".to_string(),
            },
        })
    }

    fn stop(&mut self) -> Result<()> {
        self.wait().map(|_| ())
    }
}

struct DownPeer {
    host: String,
    calls: Arc<AtomicUsize>,
}

impl Transport for DownPeer {
    fn host(&self) -> &str {
        &self.host
    }

    fn whoami(&self) -> Result<String> {
        self.calls.fetch_add(1, Ordering::SeqCst);
        anyhow::bail!("ssh: connect to host down: Connection refused")
    }

    fn export(&self, _after_seq: u64) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
        anyhow::bail!("a down peer exports nothing")
    }

    fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Remote>)> {
        anyhow::bail!("a down peer imports nothing")
    }
}

struct HungPeer {
    host: String,
}

impl Transport for HungPeer {
    fn host(&self) -> &str {
        &self.host
    }

    fn whoami(&self) -> Result<String> {
        thread::sleep(Duration::from_secs(60));
        anyhow::bail!("ssh to a hung peer timed out")
    }

    fn export(&self, _after_seq: u64) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
        anyhow::bail!("a hung peer exports nothing")
    }

    fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Remote>)> {
        anyhow::bail!("a hung peer imports nothing")
    }
}

async fn wait_until(within: Duration, mut done: impl FnMut() -> bool) -> bool {
    let deadline = Instant::now() + within;
    while Instant::now() < deadline {
        if done() {
            return true;
        }
        tokio::time::sleep(Duration::from_millis(200)).await;
    }
    done()
}

fn synced(data_dir: &Path) -> bool {
    peers::last_success(data_dir).unwrap().is_some()
}

async fn recalls(app: &axum::Router, token: &str) -> bool {
    let response = app
        .clone()
        .oneshot(post_json("/v1/recall", serde_json::json!({"query": token})))
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::OK);
    json_body(response).await["memories"]
        .as_array()
        .unwrap()
        .iter()
        .any(|m| m["content"].as_str().is_some_and(|c| c.contains(token)))
}

struct Daemons {
    app_a: axum::Router,
    app_b: axum::Router,
    data_dir: tempfile::TempDir,
    down_calls: Arc<AtomicUsize>,
}

fn spawn_daemons(peer_lines: &str) -> Daemons {
    let (app_a, sqlite_a) = app_with_sqlite(SqliteStore::in_memory("machine-a".into()).unwrap());
    let (app_b, sqlite_b) = app_with_sqlite(SqliteStore::in_memory("machine-b".into()).unwrap());
    let data_dir = tempfile::tempdir().unwrap();
    std::fs::write(data_dir.path().join(peers::PEERS_FILE), peer_lines).unwrap();
    let encryptor = Arc::new(XChaChaEncryptor::new([7u8; 32]));
    std::fs::write(data_dir.path().join("sync.key"), encryptor.key_base64()).unwrap();
    let down_calls = Arc::new(AtomicUsize::new(0));

    let connect: Connect = {
        let encryptor = encryptor.clone();
        let down_calls = down_calls.clone();
        Arc::new(move |peer: &Peer| -> Result<Box<dyn Transport + Send>> {
            let host = peer.host.clone();
            Ok(match peer.host.as_str() {
                "user@b" => Box::new(InProcessPeer {
                    host,
                    store: sqlite_b.clone(),
                    encryptor: encryptor.clone(),
                }),
                "user@down" => Box::new(DownPeer {
                    host,
                    calls: down_calls.clone(),
                }),
                "user@hung" => Box::new(HungPeer { host }),
                other => anyhow::bail!("no such peer {other}"),
            })
        })
    };
    SyncLoop {
        data_dir: data_dir.path().to_path_buf(),
        sqlite: sqlite_a,
        gate: Arc::new(WriteGate::default()),
        connect,
        health: None,
    }
    .spawn();
    Daemons {
        app_a,
        app_b,
        data_dir,
        down_calls,
    }
}

#[tokio::test]
async fn a_memory_remembered_on_one_daemon_is_recallable_on_its_peer_within_ten_seconds() {
    let Daemons {
        app_a,
        app_b,
        data_dir,
        down_calls,
    } = spawn_daemons("user@b\nuser@down\nuser@hung\n");

    assert!(
        wait_until(Duration::from_secs(10), || synced(data_dir.path())).await,
        "the start-up full sync with b never succeeded while the hung peer blocked"
    );
    assert!(
        wait_until(Duration::from_secs(10), || down_calls
            .load(Ordering::SeqCst)
            > 0)
        .await,
        "the down peer was never tried at start-up"
    );

    let token = format!("synctoken{}", uuid::Uuid::new_v4().simple());
    let response = app_a
        .clone()
        .oneshot(post_json(
            "/v1/remember",
            serde_json::json!({"content": format!("the sync loop carried {token} across")}),
        ))
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::OK);
    let remembered = Instant::now();

    let mut found = false;
    while remembered.elapsed() < Duration::from_secs(10) {
        if recalls(&app_b, &token).await {
            found = true;
            break;
        }
        tokio::time::sleep(Duration::from_millis(200)).await;
    }
    assert!(
        found,
        "b could not recall the memory 10 s after a remembered it"
    );

    tokio::time::sleep(Duration::from_secs(2)).await;
    assert_eq!(
        down_calls.load(Ordering::SeqCst),
        1,
        "the write-triggered push retried the down peer inside its backoff"
    );
    assert!(synced(data_dir.path()));
}

async fn sibling_note(app: &axum::Router, project: &str, since: &str, id: &str) -> Option<Value> {
    let response = app
        .clone()
        .oneshot(get(&format!("/v1/notes?project={project}&since={since}")))
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::OK);
    json_body(response).await["notes"]
        .as_array()
        .unwrap()
        .iter()
        .find(|n| n["id"] == id)
        .cloned()
}

#[tokio::test]
async fn a_note_written_on_one_daemon_is_in_its_peers_since_lookup_within_ten_seconds() {
    let daemons = spawn_daemons("user@b\n");
    assert!(
        wait_until(Duration::from_secs(10), || synced(daemons.data_dir.path())).await,
        "the start-up full sync with b never succeeded"
    );

    let response = daemons
        .app_a
        .clone()
        .oneshot(post_json(
            "/v1/notes",
            serde_json::json!({
                "session_id": "session-a",
                "project": "-p-sib",
                "kind": "fact",
                "decision": "the sibling port is 7311",
            }),
        ))
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::OK);
    let id = json_body(response).await["id"]
        .as_str()
        .unwrap()
        .to_string();
    let written = Instant::now();
    let since =
        (Utc::now() - chrono::Duration::hours(2)).to_rfc3339_opts(SecondsFormat::Secs, true);

    let mut found = None;
    while written.elapsed() < Duration::from_secs(10) {
        found = sibling_note(&daemons.app_b, "-p-sib", &since, &id).await;
        if found.is_some() {
            break;
        }
        tokio::time::sleep(Duration::from_millis(200)).await;
    }
    let found = found.expect("b's since lookup lacked a's note 10 s after the write");
    assert_eq!(found["session_id"], "session-a");
    assert_eq!(found["kind"], "fact");
    assert_eq!(found["decision"], "the sibling port is 7311");
}