second-brain-api 0.13.5

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
use std::fs::{self, File};
use std::io::BufWriter;
use std::path::PathBuf;
use std::time::Instant;

use anyhow::{Context, Result};
use clap::Parser;
use second_brain_core::keys::{Epoch, IdentityKey};
use second_brain_core::sqlite_store::{SnapshotOptions, SqliteStore};
use second_brain_sync::protocol::SYNC_PROTOCOL_VERSION;
use second_brain_sync::snapshot::{
    SNAPSHOT_FORMAT, SnapshotHeader, SnapshotKey, SnapshotWriter, spool_rows, verify,
};

#[derive(Parser)]
#[command(
    name = "sb-snapshot-probe",
    about = "Exports a store as a snapshot, then merges it into an empty store, reporting each \
             phase's peak RSS"
)]
struct Args {
    #[arg(long, help = "A copy of the store to export, never a live store")]
    store: PathBuf,
    #[arg(long, help = "Empty directory for the spool and the merged store")]
    work: PathBuf,
}

fn reset_peak_rss() -> Result<()> {
    fs::write("/proc/self/clear_refs", "5").context("resetting VmHWM")
}

fn peak_rss_mib() -> Result<u64> {
    let status = fs::read_to_string("/proc/self/status")?;
    let kib: u64 = status
        .lines()
        .find_map(|l| l.strip_prefix("VmHWM:"))
        .context("no VmHWM in /proc/self/status")?
        .trim()
        .trim_end_matches("kB")
        .trim()
        .parse()?;
    Ok(kib / 1024)
}

fn main() -> Result<()> {
    let args = Args::parse();
    fs::create_dir_all(&args.work)?;
    let identity = IdentityKey::generate()?;
    let epoch = Epoch {
        number: 1,
        key_id: identity.key_id(),
    };
    let key_for = |h: &SnapshotHeader| SnapshotKey::derive(&identity, &h.epoch);
    let spool = args.work.join("snapshot.spool");

    let source = SqliteStore::open(&args.store, "snapshot-probe-exporter".to_string())?;
    reset_peak_rss()?;
    let started = Instant::now();
    let header = SnapshotHeader {
        format: SNAPSHOT_FORMAT,
        snapshot_id: "probe".to_string(),
        exporter: source.machine_id().to_string(),
        epoch,
        sync_protocol: SYNC_PROTOCOL_VERSION,
        schema_version: source.schema_version()?,
        created_at: 0,
    };
    let mut writer = SnapshotWriter::new(
        BufWriter::with_capacity(1024 * 1024, File::create(&spool)?),
        SnapshotKey::derive(&identity, &epoch)?,
        header,
    )?;
    let state = source.export_snapshot(SnapshotOptions { with_chunks: true }, &mut |row| {
        writer.row(&row)
    })?;
    writer.finish(state, None)?;
    let export_secs = started.elapsed().as_secs_f64();
    let export_peak = peak_rss_mib()?;
    drop(source);

    reset_peak_rss()?;
    let started = Instant::now();
    let (header, manifest) = verify(&spool, &key_for)?;
    let target = SqliteStore::open(
        &args.work.join("merged.db"),
        "snapshot-probe-receiver".to_string(),
    )?;
    let mut merge =
        target.begin_snapshot_merge(&header.snapshot_id, &header.exporter, manifest.state.seq)?;
    spool_rows(&spool, &key_for, &mut |row| merge.push(row))?;
    let outcome = merge.finish(&manifest.state.cursors)?;
    let merge_secs = started.elapsed().as_secs_f64();
    let merge_peak = peak_rss_mib()?;

    println!(
        "{}",
        serde_json::json!({
            "spool_bytes": fs::metadata(&spool)?.len(),
            "rows": outcome.rows,
            "changed": outcome.changed,
            "settled": outcome.settled,
            "counts": manifest.state.counts,
            "export_secs": export_secs,
            "export_peak_mib": export_peak,
            "merge_secs": merge_secs,
            "merge_peak_mib": merge_peak,
        })
    );
    Ok(())
}