second-brain-api 0.13.5

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
use std::collections::HashSet;
use std::io::Write;
use std::path::PathBuf;

use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use uuid::Uuid;

use second_brain_api::eval::caching_store::CachingStore;
use second_brain_api::eval::{self, GateSweepReport, QueryRecord};
use second_brain_core::embedding::Embedder;
use second_brain_core::index::{IndexClaim, Indexes};
use second_brain_core::schema::Tier;
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::StoreAdmin;

const EVAL_MACHINE_ID: &str = "sb-recall-eval";
const DEFAULT_LIMIT: usize = 10;

#[derive(Parser)]
#[command(
    name = "sb-recall-eval",
    about = "Recall A/B eval harness for second-brain"
)]
struct Cli {
    #[command(subcommand)]
    command: Command,
}

#[derive(Subcommand)]
enum Command {
    ExtractCorpus {
        #[arg(long)]
        snapshot: PathBuf,
        #[arg(long)]
        out: PathBuf,
    },
    RunAb {
        #[arg(long)]
        snapshot: PathBuf,
        #[arg(long)]
        eval: PathBuf,
        #[arg(long)]
        out: PathBuf,
        #[arg(long, default_value_t = DEFAULT_LIMIT)]
        limit: usize,
        #[arg(long, default_value_t = 0, hide = true)]
        max_queries: usize,
        #[arg(long, default_value = "legacy", value_parser = ["legacy", "hybrid", "hybrid-rerank"])]
        engine: String,
        #[arg(long, default_value = "bge-reranker-v2-m3")]
        rerank_model: String,
        #[arg(long, default_value_t = 100)]
        fuse_top: usize,
    },
    /// Calibrates the exhausted-scope cosine floor on the gold set.
    FloorSweep {
        #[arg(long)]
        snapshot: PathBuf,
        #[arg(long)]
        eval: PathBuf,
        #[arg(long)]
        out: PathBuf,
        /// Only projects holding at least this many live memories serve as
        /// foreign scopes, so a near-empty scope cannot pass as "empty".
        #[arg(long, default_value_t = 20)]
        min_project_memories: u64,
    },
    GateSweep {
        #[arg(long, conflicts_with = "ab")]
        records: Option<PathBuf>,
        #[arg(long, conflicts_with = "records")]
        ab: Option<PathBuf>,
        #[arg(long)]
        out: PathBuf,
    },
}

fn main() -> Result<()> {
    let cli = Cli::parse();
    match cli.command {
        Command::ExtractCorpus { snapshot, out } => extract_corpus(&snapshot, &out),
        Command::RunAb {
            snapshot,
            eval,
            out,
            limit,
            max_queries,
            engine,
            rerank_model,
            fuse_top,
        } => run_ab(
            &snapshot,
            &eval,
            &out,
            limit,
            max_queries,
            &engine,
            &rerank_model,
            fuse_top,
        ),
        Command::GateSweep { records, ab, out } => gate_sweep(records, ab, &out),
        Command::FloorSweep {
            snapshot,
            eval,
            out,
            min_project_memories,
        } => floor_sweep(&snapshot, &eval, &out, min_project_memories),
    }
}

fn floor_sweep(
    snapshot: &std::path::Path,
    eval_path: &std::path::Path,
    out: &std::path::Path,
    min_project_memories: u64,
) -> Result<()> {
    let queries = eval::load_eval_set(eval_path)?;
    let store = open_store(snapshot)?;
    let projects: Vec<String> = store
        .distinct_project_paths()?
        .into_iter()
        .filter(|(_, n)| *n >= min_project_memories)
        .map(|(p, _)| p)
        .collect();
    let embedder = Embedder::new().context("loading embedding model")?;
    eprintln!("embedding {} queries", queries.len());
    let embedded = eval::embed_all_queries(&embedder, &queries)?;
    let pairs = eval::scoped_pairs(&store, &embedded, &projects)?;
    anyhow::ensure!(
        !pairs.is_empty(),
        "no gold memory of {} is live in {}; the eval set was built on another corpus",
        eval_path.display(),
        snapshot.display()
    );
    let report = eval::exhausted_floor_sweep(&pairs);
    println!(
        "{} of {} queries scored over {} foreign projects; chosen floor {:.2}",
        pairs.len(),
        queries.len(),
        projects.len(),
        report.chosen_threshold
    );
    let body = serde_json::json!({"report": report, "pairs": pairs});
    std::fs::write(out, serde_json::to_string_pretty(&body)?)
        .with_context(|| format!("writing {}", out.display()))?;
    Ok(())
}

/// Held for the whole run, so a live daemon's directory is refused even while
/// the daemon has released its writer, and no daemon starts on it mid-run.
fn claim_snapshot_indexes(dir: &std::path::Path) -> Result<IndexClaim> {
    IndexClaim::acquire(dir).with_context(|| {
        format!(
            "refusing {}: a live daemon may own it; pass a copy of the store as --snapshot",
            dir.display()
        )
    })
}

fn open_store(snapshot: &std::path::Path) -> Result<SqliteStore> {
    SqliteStore::open(snapshot, EVAL_MACHINE_ID.to_string())
        .with_context(|| format!("opening snapshot at {}", snapshot.display()))
}

fn extract_corpus(snapshot: &std::path::Path, out: &std::path::Path) -> Result<()> {
    let store = open_store(snapshot)?;
    let count = eval::extract_corpus(&store, out)?;
    println!("wrote {count} corpus entries to {}", out.display());
    Ok(())
}

#[allow(clippy::too_many_arguments)]
fn run_ab(
    snapshot: &std::path::Path,
    eval_path: &std::path::Path,
    out: &std::path::Path,
    limit: usize,
    max_queries: usize,
    engine: &str,
    rerank_model: &str,
    fuse_top: usize,
) -> Result<()> {
    let mut queries = eval::load_eval_set(eval_path)?;
    if max_queries > 0 && queries.len() > max_queries {
        queries.truncate(max_queries);
    }
    let _claim = if engine == "legacy" {
        None
    } else {
        Some(claim_snapshot_indexes(&snapshot.with_extension("indexes"))?)
    };
    let store = open_store(snapshot)?;

    // Hybrid engines search real indexes built beside the snapshot; a rebuild
    // mutates the snapshot's outbox watermark, which is why RunAb should get a
    // copy of the live db, never brain.db itself.
    let indexes_store;
    let indexes;
    let reranker;
    let hybrid_ctx = if engine == "legacy" {
        None
    } else {
        indexes_store = open_store(snapshot)?;
        let dir = snapshot.with_extension("indexes");
        let mut idx = Indexes::open(&dir)?;
        if idx.consistent_with(&indexes_store)? {
            eprintln!("indexes at {} are consistent, reusing", dir.display());
        } else {
            eprintln!("rebuilding indexes at {}", dir.display());
            let n = idx.rebuild(&indexes_store)?;
            eprintln!("indexed {n} memories");
        }
        indexes = idx;
        reranker = if engine == "hybrid-rerank" {
            eprintln!("loading rerank model {rerank_model}");
            Some(
                second_brain_core::retrieval::Reranker::new(rerank_model)
                    .context("loading rerank model")?,
            )
        } else {
            None
        };
        Some(eval::HybridContext {
            sqlite: &indexes_store,
            indexes: &indexes,
            reranker: reranker.as_ref(),
            tunables: second_brain_core::retrieval::RetrievalTunables {
                fuse_top,
                ..Default::default()
            },
        })
    };

    let embedder = Embedder::new().context("loading embedding model")?;

    eprintln!("embedding {} queries", queries.len());
    let embedded = eval::embed_all_queries(&embedder, &queries)?;

    // One cache spans both arms. Prewarming pulls every Memory edge in two bulk
    // scans up front, replacing the ~corpus x 7 serialized point reads that
    // recall would otherwise issue; get_relations is query-independent so this is
    // result-equivalent.
    let caching_store = CachingStore::new(&store);
    eprintln!("prewarming relation cache");
    caching_store
        .prewarm()
        .context("prewarming relation cache")?;

    // One bulk read replaces two full-table scans per query: without this the
    // arms serialize on the store's connection mutex re-parsing 68k embedding
    // blobs per vector_search and the eval runs for hours.
    eprintln!("prewarming vector corpus");
    let corpus: Vec<second_brain_core::schema::Memory> = store
        .all_memories_with_embeddings()?
        .into_iter()
        .filter(|m| !m.embedding.is_empty() && m.tier == Tier::Hot && m.superseded_by.is_none())
        .collect();
    eprintln!("vector corpus: {} memories", corpus.len());
    caching_store.prewarm_vectors(corpus);

    eprintln!("running bare arm ({engine})");
    let bare =
        eval::run_arm_parallel(&caching_store, &embedded, false, limit, hybrid_ctx.as_ref())?;
    eprintln!("running prefixed arm ({engine})");
    let prefixed =
        eval::run_arm_parallel(&caching_store, &embedded, true, limit, hybrid_ctx.as_ref())?;

    let relevant_sets: std::collections::HashMap<String, HashSet<Uuid>> = queries
        .iter()
        .map(|q| {
            (
                q.query_id.clone(),
                q.relevant_memory_ids.iter().copied().collect(),
            )
        })
        .collect();

    let report = eval::aggregate(&bare, &prefixed, &relevant_sets);

    write_records(&out.with_extension("bare.jsonl"), &bare)?;
    write_records(&out.with_extension("prefixed.jsonl"), &prefixed)?;
    write_json(out, &report)?;

    println!("aggregate report written to {}", out.display());
    Ok(())
}

fn gate_sweep(records: Option<PathBuf>, ab: Option<PathBuf>, out: &std::path::Path) -> Result<()> {
    let prefixed = match (records, ab) {
        (Some(path), _) => read_records(&path)?,
        (None, Some(path)) => read_records(&path.with_extension("prefixed.jsonl"))?,
        (None, None) => {
            anyhow::bail!("gate-sweep requires either --records or --ab");
        }
    };
    let report: GateSweepReport = eval::gate_sweep(&prefixed);
    write_json(out, &report)?;
    println!("gate sweep report written to {}", out.display());
    Ok(())
}

fn read_records(path: &std::path::Path) -> Result<Vec<QueryRecord>> {
    let text = std::fs::read_to_string(path)
        .with_context(|| format!("reading records from {}", path.display()))?;
    parse_records(&text)
}

fn parse_records(text: &str) -> Result<Vec<QueryRecord>> {
    let mut out = Vec::new();
    for line in text.lines() {
        let trimmed = line.trim();
        if trimmed.is_empty() {
            continue;
        }
        out.push(serde_json::from_str(trimmed)?);
    }
    Ok(out)
}

fn write_records(path: &std::path::Path, records: &[QueryRecord]) -> Result<()> {
    let mut file = std::fs::File::create(path)?;
    for r in records {
        writeln!(file, "{}", serde_json::to_string(r)?)?;
    }
    Ok(())
}

fn write_json<T: serde::Serialize>(path: &std::path::Path, value: &T) -> Result<()> {
    let json = serde_json::to_string_pretty(value)?;
    std::fs::write(path, json)?;
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn a_claimed_snapshot_index_dir_is_refused() {
        let dir = tempfile::tempdir().unwrap();
        let held = IndexClaim::acquire(dir.path()).unwrap();
        let err = claim_snapshot_indexes(dir.path()).err().unwrap();
        assert!(format!("{err:#}").contains("refusing"), "{err:#}");
        drop(held);
        claim_snapshot_indexes(dir.path()).unwrap();
    }
}