use std::collections::HashSet;
use std::io::Write;
use std::path::PathBuf;
use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use uuid::Uuid;
use second_brain_api::eval::caching_store::CachingStore;
use second_brain_api::eval::{self, GateSweepReport, QueryRecord};
use second_brain_core::embedding::Embedder;
use second_brain_core::index::{IndexClaim, Indexes};
use second_brain_core::schema::Tier;
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::StoreAdmin;
const EVAL_MACHINE_ID: &str = "sb-recall-eval";
const DEFAULT_LIMIT: usize = 10;
#[derive(Parser)]
#[command(
name = "sb-recall-eval",
about = "Recall A/B eval harness for second-brain"
)]
struct Cli {
#[command(subcommand)]
command: Command,
}
#[derive(Subcommand)]
enum Command {
ExtractCorpus {
#[arg(long)]
snapshot: PathBuf,
#[arg(long)]
out: PathBuf,
},
RunAb {
#[arg(long)]
snapshot: PathBuf,
#[arg(long)]
eval: PathBuf,
#[arg(long)]
out: PathBuf,
#[arg(long, default_value_t = DEFAULT_LIMIT)]
limit: usize,
#[arg(long, default_value_t = 0, hide = true)]
max_queries: usize,
#[arg(long, default_value = "legacy", value_parser = ["legacy", "hybrid", "hybrid-rerank"])]
engine: String,
#[arg(long, default_value = "bge-reranker-v2-m3")]
rerank_model: String,
#[arg(long, default_value_t = 100)]
fuse_top: usize,
},
FloorSweep {
#[arg(long)]
snapshot: PathBuf,
#[arg(long)]
eval: PathBuf,
#[arg(long)]
out: PathBuf,
#[arg(long, default_value_t = 20)]
min_project_memories: u64,
},
GateSweep {
#[arg(long, conflicts_with = "ab")]
records: Option<PathBuf>,
#[arg(long, conflicts_with = "records")]
ab: Option<PathBuf>,
#[arg(long)]
out: PathBuf,
},
}
fn main() -> Result<()> {
let cli = Cli::parse();
match cli.command {
Command::ExtractCorpus { snapshot, out } => extract_corpus(&snapshot, &out),
Command::RunAb {
snapshot,
eval,
out,
limit,
max_queries,
engine,
rerank_model,
fuse_top,
} => run_ab(
&snapshot,
&eval,
&out,
limit,
max_queries,
&engine,
&rerank_model,
fuse_top,
),
Command::GateSweep { records, ab, out } => gate_sweep(records, ab, &out),
Command::FloorSweep {
snapshot,
eval,
out,
min_project_memories,
} => floor_sweep(&snapshot, &eval, &out, min_project_memories),
}
}
fn floor_sweep(
snapshot: &std::path::Path,
eval_path: &std::path::Path,
out: &std::path::Path,
min_project_memories: u64,
) -> Result<()> {
let queries = eval::load_eval_set(eval_path)?;
let store = open_store(snapshot)?;
let projects: Vec<String> = store
.distinct_project_paths()?
.into_iter()
.filter(|(_, n)| *n >= min_project_memories)
.map(|(p, _)| p)
.collect();
let embedder = Embedder::new().context("loading embedding model")?;
eprintln!("embedding {} queries", queries.len());
let embedded = eval::embed_all_queries(&embedder, &queries)?;
let pairs = eval::scoped_pairs(&store, &embedded, &projects)?;
anyhow::ensure!(
!pairs.is_empty(),
"no gold memory of {} is live in {}; the eval set was built on another corpus",
eval_path.display(),
snapshot.display()
);
let report = eval::exhausted_floor_sweep(&pairs);
println!(
"{} of {} queries scored over {} foreign projects; chosen floor {:.2}",
pairs.len(),
queries.len(),
projects.len(),
report.chosen_threshold
);
let body = serde_json::json!({"report": report, "pairs": pairs});
std::fs::write(out, serde_json::to_string_pretty(&body)?)
.with_context(|| format!("writing {}", out.display()))?;
Ok(())
}
fn claim_snapshot_indexes(dir: &std::path::Path) -> Result<IndexClaim> {
IndexClaim::acquire(dir).with_context(|| {
format!(
"refusing {}: a live daemon may own it; pass a copy of the store as --snapshot",
dir.display()
)
})
}
fn open_store(snapshot: &std::path::Path) -> Result<SqliteStore> {
SqliteStore::open(snapshot, EVAL_MACHINE_ID.to_string())
.with_context(|| format!("opening snapshot at {}", snapshot.display()))
}
fn extract_corpus(snapshot: &std::path::Path, out: &std::path::Path) -> Result<()> {
let store = open_store(snapshot)?;
let count = eval::extract_corpus(&store, out)?;
println!("wrote {count} corpus entries to {}", out.display());
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn run_ab(
snapshot: &std::path::Path,
eval_path: &std::path::Path,
out: &std::path::Path,
limit: usize,
max_queries: usize,
engine: &str,
rerank_model: &str,
fuse_top: usize,
) -> Result<()> {
let mut queries = eval::load_eval_set(eval_path)?;
if max_queries > 0 && queries.len() > max_queries {
queries.truncate(max_queries);
}
let _claim = if engine == "legacy" {
None
} else {
Some(claim_snapshot_indexes(&snapshot.with_extension("indexes"))?)
};
let store = open_store(snapshot)?;
let indexes_store;
let indexes;
let reranker;
let hybrid_ctx = if engine == "legacy" {
None
} else {
indexes_store = open_store(snapshot)?;
let dir = snapshot.with_extension("indexes");
let mut idx = Indexes::open(&dir)?;
if idx.consistent_with(&indexes_store)? {
eprintln!("indexes at {} are consistent, reusing", dir.display());
} else {
eprintln!("rebuilding indexes at {}", dir.display());
let n = idx.rebuild(&indexes_store)?;
eprintln!("indexed {n} memories");
}
indexes = idx;
reranker = if engine == "hybrid-rerank" {
eprintln!("loading rerank model {rerank_model}");
Some(
second_brain_core::retrieval::Reranker::new(rerank_model)
.context("loading rerank model")?,
)
} else {
None
};
Some(eval::HybridContext {
sqlite: &indexes_store,
indexes: &indexes,
reranker: reranker.as_ref(),
tunables: second_brain_core::retrieval::RetrievalTunables {
fuse_top,
..Default::default()
},
})
};
let embedder = Embedder::new().context("loading embedding model")?;
eprintln!("embedding {} queries", queries.len());
let embedded = eval::embed_all_queries(&embedder, &queries)?;
let caching_store = CachingStore::new(&store);
eprintln!("prewarming relation cache");
caching_store
.prewarm()
.context("prewarming relation cache")?;
eprintln!("prewarming vector corpus");
let corpus: Vec<second_brain_core::schema::Memory> = store
.all_memories_with_embeddings()?
.into_iter()
.filter(|m| !m.embedding.is_empty() && m.tier == Tier::Hot && m.superseded_by.is_none())
.collect();
eprintln!("vector corpus: {} memories", corpus.len());
caching_store.prewarm_vectors(corpus);
eprintln!("running bare arm ({engine})");
let bare =
eval::run_arm_parallel(&caching_store, &embedded, false, limit, hybrid_ctx.as_ref())?;
eprintln!("running prefixed arm ({engine})");
let prefixed =
eval::run_arm_parallel(&caching_store, &embedded, true, limit, hybrid_ctx.as_ref())?;
let relevant_sets: std::collections::HashMap<String, HashSet<Uuid>> = queries
.iter()
.map(|q| {
(
q.query_id.clone(),
q.relevant_memory_ids.iter().copied().collect(),
)
})
.collect();
let report = eval::aggregate(&bare, &prefixed, &relevant_sets);
write_records(&out.with_extension("bare.jsonl"), &bare)?;
write_records(&out.with_extension("prefixed.jsonl"), &prefixed)?;
write_json(out, &report)?;
println!("aggregate report written to {}", out.display());
Ok(())
}
fn gate_sweep(records: Option<PathBuf>, ab: Option<PathBuf>, out: &std::path::Path) -> Result<()> {
let prefixed = match (records, ab) {
(Some(path), _) => read_records(&path)?,
(None, Some(path)) => read_records(&path.with_extension("prefixed.jsonl"))?,
(None, None) => {
anyhow::bail!("gate-sweep requires either --records or --ab");
}
};
let report: GateSweepReport = eval::gate_sweep(&prefixed);
write_json(out, &report)?;
println!("gate sweep report written to {}", out.display());
Ok(())
}
fn read_records(path: &std::path::Path) -> Result<Vec<QueryRecord>> {
let text = std::fs::read_to_string(path)
.with_context(|| format!("reading records from {}", path.display()))?;
parse_records(&text)
}
fn parse_records(text: &str) -> Result<Vec<QueryRecord>> {
let mut out = Vec::new();
for line in text.lines() {
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
out.push(serde_json::from_str(trimmed)?);
}
Ok(out)
}
fn write_records(path: &std::path::Path, records: &[QueryRecord]) -> Result<()> {
let mut file = std::fs::File::create(path)?;
for r in records {
writeln!(file, "{}", serde_json::to_string(r)?)?;
}
Ok(())
}
fn write_json<T: serde::Serialize>(path: &std::path::Path, value: &T) -> Result<()> {
let json = serde_json::to_string_pretty(value)?;
std::fs::write(path, json)?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_claimed_snapshot_index_dir_is_refused() {
let dir = tempfile::tempdir().unwrap();
let held = IndexClaim::acquire(dir.path()).unwrap();
let err = claim_snapshot_indexes(dir.path()).err().unwrap();
assert!(format!("{err:#}").contains("refusing"), "{err:#}");
drop(held);
claim_snapshot_indexes(dir.path()).unwrap();
}
}