use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::thread;
use std::time::{Duration, Instant};
use anyhow::{Context, Result, bail};
use clap::Parser;
use fs4::fs_std::FileExt;
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
use serde::Serialize;
use second_brain_core::embedding::{Embedder, query_prompt};
use second_brain_core::index::{CLAIM_FILE, EMBEDDING_DIM, Indexes};
use second_brain_core::query::{QueryFilters, QueryRequest};
use second_brain_core::retrieval::{RerankScorer, Reranker, RetrievalTunables, Retriever};
use second_brain_core::sqlite_store::SqliteStore;
const DEFAULT_LIMIT: usize = 10;
#[derive(Parser)]
#[command(
name = "sb-recall-bench",
about = "In-process throughput/latency harness for second-brain hybrid recall (Retriever::rank, not the HTTP layer)"
)]
struct Cli {
#[arg(long, help = "Path to the SqliteStore brain.db to bench against")]
store: PathBuf,
#[arg(
long,
help = "Path to the Indexes directory (vectors.usearch + tantivy/) beside the store"
)]
indexes: PathBuf,
#[arg(
long,
help = "A positive integer (synthesize that many queries from the store's own memories, no model download needed) or a path to a recall_eval-style JSONL query file (requires the bge-small embedding model)"
)]
queries: String,
#[arg(
long,
default_value_t = 1,
help = "Number of concurrent worker threads issuing recall calls"
)]
concurrency: usize,
#[arg(
long,
help = "Run the measured phase for this many seconds (mutually exclusive with --iterations)"
)]
duration: Option<u64>,
#[arg(
long,
help = "Run the measured phase for exactly this many total recall calls (mutually exclusive with --duration)"
)]
iterations: Option<u64>,
#[arg(long, help = "Where to write the JSON report")]
out: PathBuf,
#[arg(
long,
default_value = "off",
help = "Cross-encoder rerank model to load (bge-reranker-v2-m3, jina-reranker-v1-turbo), or 'off' to skip reranking"
)]
rerank: String,
#[arg(
long,
default_value_t = DEFAULT_LIMIT,
help = "recall limit passed to Retriever::rank"
)]
limit: usize,
#[arg(
long,
help = "RRF fuse_top override; defaults to the engine's own default"
)]
fuse_top: Option<usize>,
#[arg(
long,
help = "Discarded warmup calls before measurement begins; defaults to concurrency*5 (min 20)"
)]
warmup: Option<usize>,
#[arg(
long,
default_value_t = 42,
help = "Seed for deterministic sampling when --queries is a synthetic count"
)]
seed: u64,
}
#[derive(Clone, Copy)]
enum Workload {
Duration(Duration),
Iterations(u64),
}
#[derive(Serialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
enum QuerySourceConfig {
File { path: String },
Synthetic { count: usize, seed: u64 },
}
#[derive(Serialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
enum WorkloadConfig {
Duration { seconds: u64 },
Iterations { count: u64 },
}
#[derive(Serialize)]
struct BenchConfig {
store: String,
indexes: String,
queries: QuerySourceConfig,
query_count: usize,
concurrency: usize,
workload: WorkloadConfig,
limit: usize,
rerank: String,
fuse_top: usize,
warmup_discarded: usize,
}
#[derive(Serialize, Default)]
struct LatencyStats {
p50_ms: f64,
p95_ms: f64,
p99_ms: f64,
max_ms: f64,
mean_ms: f64,
}
#[derive(Serialize)]
struct ErrorsReport {
count: u64,
kinds: BTreeMap<String, u64>,
}
#[derive(Serialize)]
struct BenchReport {
config: BenchConfig,
total_queries: u64,
elapsed_secs: f64,
achieved_qps: f64,
latency_ms: LatencyStats,
errors: ErrorsReport,
}
fn main() -> Result<()> {
let cli = Cli::parse();
if cli.concurrency == 0 {
bail!("--concurrency must be at least 1, got 0");
}
if cli.limit == 0 {
bail!("--limit must be at least 1, got 0");
}
match (cli.duration, cli.iterations) {
(None, None) => bail!("exactly one of --duration or --iterations is required"),
(Some(_), Some(_)) => {
bail!("--duration and --iterations are mutually exclusive; pass exactly one")
}
_ => {}
}
let workload = match (cli.duration, cli.iterations) {
(Some(secs), None) => {
if secs == 0 {
bail!("--duration must be at least 1 second, got 0");
}
Workload::Duration(Duration::from_secs(secs))
}
(None, Some(n)) => {
if n == 0 {
bail!("--iterations must be at least 1, got 0");
}
Workload::Iterations(n)
}
_ => unreachable!("validated above"),
};
check_no_live_writer(&cli.indexes)?;
let store = SqliteStore::open(&cli.store, "sb-recall-bench".to_string())
.with_context(|| format!("opening store at {}", cli.store.display()))?;
let indexes = Indexes::open_readonly(&cli.indexes)
.with_context(|| format!("opening indexes at {}", cli.indexes.display()))?;
match indexes.consistent_with(&store) {
Ok(true) => eprintln!(
"indexes at {} are consistent with the store",
cli.indexes.display()
),
Ok(false) => eprintln!(
"warning: indexes at {} look stale relative to the store; recall_bench never \
rebuilds indexes, so results reflect them as-is",
cli.indexes.display()
),
Err(e) => eprintln!("warning: could not verify index consistency: {e}"),
}
let reranker = match cli.rerank.as_str() {
"off" => None,
model => {
eprintln!("loading rerank model {model}");
Some(Reranker::new(model).context("loading rerank model")?)
}
};
let fuse_top = cli
.fuse_top
.unwrap_or(RetrievalTunables::default().fuse_top);
let tunables = RetrievalTunables {
fuse_top,
..Default::default()
};
let retriever = Retriever { tunables };
let (queries, query_source) = prepare_queries(&store, &cli)?;
if queries.is_empty() {
bail!("no queries to run; check --queries");
}
println!(
"prepared {} queries ({})",
queries.len(),
match &query_source {
QuerySourceConfig::File { path } => format!("from file {path}, bge-small embeddings"),
QuerySourceConfig::Synthetic { count, seed } => format!(
"synthetic, {count} sampled from the store at seed {seed}, no embedding model"
),
}
);
let warmup_n = cli.warmup.unwrap_or_else(|| (cli.concurrency * 5).max(20));
let warmup_start = Instant::now();
let warmup_outcome = run_workload(
&store,
&indexes,
&retriever,
reranker.as_ref(),
&queries,
cli.concurrency,
Workload::Iterations(warmup_n as u64),
);
println!(
"discarded {} warmup queries in {:.2}s",
warmup_outcome.len(),
warmup_start.elapsed().as_secs_f64()
);
let measure_start = Instant::now();
let outcome = run_workload(
&store,
&indexes,
&retriever,
reranker.as_ref(),
&queries,
cli.concurrency,
workload,
);
let elapsed = measure_start.elapsed();
let report = build_report(
BenchConfig {
store: cli.store.display().to_string(),
indexes: cli.indexes.display().to_string(),
queries: query_source,
query_count: queries.len(),
concurrency: cli.concurrency,
workload: match workload {
Workload::Duration(d) => WorkloadConfig::Duration {
seconds: d.as_secs(),
},
Workload::Iterations(n) => WorkloadConfig::Iterations { count: n },
},
limit: cli.limit,
rerank: cli.rerank.clone(),
fuse_top,
warmup_discarded: warmup_outcome.len() as usize,
},
&outcome,
elapsed,
);
let json = serde_json::to_string_pretty(&report)?;
std::fs::write(&cli.out, &json)
.with_context(|| format!("writing report to {}", cli.out.display()))?;
print_summary(&report);
println!("report written to {}", cli.out.display());
Ok(())
}
fn check_no_live_writer(indexes_dir: &Path) -> Result<()> {
check_lock(&indexes_dir.join(CLAIM_FILE))?;
check_lock(&indexes_dir.join("tantivy").join(".tantivy-writer.lock"))
}
fn check_lock(lock_path: &Path) -> Result<()> {
if !lock_path.exists() {
return Ok(());
}
let file = std::fs::OpenOptions::new()
.write(true)
.open(lock_path)
.with_context(|| format!("opening {} to probe for a live writer", lock_path.display()))?;
let acquired = file
.try_lock_exclusive()
.with_context(|| format!("probing lock state of {}", lock_path.display()))?;
if acquired {
let _ = file.unlock();
return Ok(());
}
match find_lock_holder_pid(lock_path) {
Some(pid) => bail!(
"refusing to start: {} is held by another process (pid {pid}), most likely a live \
daemon. Point --indexes at a private copy, never at a directory an active daemon owns.",
lock_path.display()
),
None => bail!(
"refusing to start: {} is held by another process (pid undetermined), most likely a \
live daemon. Point --indexes at a private copy, never at a directory an active \
daemon owns.",
lock_path.display()
),
}
}
fn find_lock_holder_pid(lock_path: &Path) -> Option<u32> {
let canonical = lock_path.canonicalize().ok()?;
let locks = std::fs::read_to_string("/proc/locks").ok()?;
for line in locks.lines() {
let fields: Vec<&str> = line.split_whitespace().collect();
if fields.len() < 5 || fields[1] != "FLOCK" {
continue;
}
let Ok(pid) = fields[4].parse::<u32>() else {
continue;
};
let Ok(entries) = std::fs::read_dir(format!("/proc/{pid}/fd")) else {
continue;
};
for entry in entries.flatten() {
if let Ok(target) = std::fs::read_link(entry.path())
&& target == canonical
{
return Some(pid);
}
}
}
None
}
fn prepare_queries(
store: &SqliteStore,
cli: &Cli,
) -> Result<(Vec<QueryRequest>, QuerySourceConfig)> {
match cli.queries.parse::<usize>() {
Ok(count) => {
if count == 0 {
bail!("--queries count must be at least 1, got 0");
}
let synthetic = synthesize_queries(store, count, cli.seed, cli.limit)?;
Ok((
synthetic,
QuerySourceConfig::Synthetic {
count,
seed: cli.seed,
},
))
}
Err(_) => {
let path = PathBuf::from(&cli.queries);
let loaded = load_file_queries(&path, cli.limit)?;
Ok((
loaded,
QuerySourceConfig::File {
path: cli.queries.clone(),
},
))
}
}
}
fn synthesize_queries(
store: &SqliteStore,
count: usize,
seed: u64,
limit: usize,
) -> Result<Vec<QueryRequest>> {
let indexable = store.all_indexable()?;
let usable: Vec<(uuid::Uuid, Vec<f32>, String)> = indexable
.into_iter()
.filter(|(_, embedding, _)| embedding.len() == EMBEDDING_DIM)
.collect();
if usable.is_empty() {
bail!(
"the store has no indexable memories with {EMBEDDING_DIM}-dim embeddings; cannot \
synthesize queries from it"
);
}
let mut rng = StdRng::seed_from_u64(seed);
let mut queries = Vec::with_capacity(count);
for _ in 0..count {
let idx = rng.random_range(0..usable.len());
let (_, embedding, body) = &usable[idx];
let text = query_text_from_body(body);
queries.push(build_request(text, embedding.clone(), limit));
}
Ok(queries)
}
fn query_text_from_body(body: &str) -> String {
let words: Vec<&str> = body.split_whitespace().take(10).collect();
if words.is_empty() {
body.chars().take(40).collect()
} else {
words.join(" ")
}
}
fn load_file_queries(path: &Path, limit: usize) -> Result<Vec<QueryRequest>> {
let eval_queries = second_brain_api::eval::load_eval_set(path)
.with_context(|| format!("loading queries from {}", path.display()))?;
if eval_queries.is_empty() {
bail!("query file {} contains no queries", path.display());
}
eprintln!(
"loading embedding model to embed {} file queries",
eval_queries.len()
);
let embedder = Embedder::new().context("loading embedding model")?;
let prefixed: Vec<String> = eval_queries
.iter()
.map(|q| query_prompt(&q.query))
.collect();
let refs: Vec<&str> = prefixed.iter().map(|s| s.as_str()).collect();
let embeddings = embedder
.embed_batch(&refs)
.context("embedding file queries")?;
Ok(eval_queries
.into_iter()
.zip(embeddings)
.map(|(q, embedding)| build_request(q.query, embedding, limit))
.collect())
}
fn build_request(text: String, embedding: Vec<f32>, limit: usize) -> QueryRequest {
let entity_names = second_brain_api::link_query_entities(&text);
QueryRequest {
text,
embedding,
limit,
filters: QueryFilters {
entity_names,
..QueryFilters::default()
},
}
}
struct WorkloadOutcome {
latencies_us_ok: Vec<u64>,
error_kinds: BTreeMap<String, u64>,
}
impl WorkloadOutcome {
fn len(&self) -> u64 {
self.latencies_us_ok.len() as u64 + self.error_kinds.values().sum::<u64>()
}
}
#[allow(clippy::too_many_arguments)]
fn run_workload(
store: &SqliteStore,
indexes: &Indexes,
retriever: &Retriever,
reranker: Option<&Reranker>,
queries: &[QueryRequest],
concurrency: usize,
workload: Workload,
) -> WorkloadOutcome {
let issued = AtomicU64::new(0);
let stop = AtomicBool::new(false);
let deadline = match workload {
Workload::Duration(d) => Some(Instant::now() + d),
Workload::Iterations(_) => None,
};
let target = match workload {
Workload::Iterations(n) => Some(n),
Workload::Duration(_) => None,
};
let collected: Mutex<Vec<(u64, Option<String>)>> = Mutex::new(Vec::new());
thread::scope(|scope| {
for _ in 0..concurrency {
scope.spawn(|| {
let mut local: Vec<(u64, Option<String>)> = Vec::new();
loop {
if stop.load(Ordering::Relaxed) {
break;
}
let i = issued.fetch_add(1, Ordering::Relaxed);
if let Some(t) = target
&& i >= t
{
stop.store(true, Ordering::Relaxed);
break;
}
let request = &queries[(i as usize) % queries.len()];
let started = Instant::now();
let outcome: Result<usize> = (|| {
let pool = retriever.candidates(indexes, store, request)?;
let ranked = retriever.rank(
store,
request,
&pool,
reranker.map(|r| r as &dyn RerankScorer),
)?;
Ok(ranked.len())
})();
let elapsed_us = started.elapsed().as_micros() as u64;
match outcome {
Ok(_) => local.push((elapsed_us, None)),
Err(e) => local.push((elapsed_us, Some(short_kind(&e)))),
}
if let Some(dl) = deadline
&& Instant::now() >= dl
{
stop.store(true, Ordering::Relaxed);
break;
}
}
collected.lock().unwrap().extend(local);
});
}
});
let all = collected.into_inner().unwrap();
let mut latencies_us_ok = Vec::with_capacity(all.len());
let mut error_kinds: BTreeMap<String, u64> = BTreeMap::new();
for (us, kind) in all {
match kind {
None => latencies_us_ok.push(us),
Some(k) => *error_kinds.entry(k).or_insert(0) += 1,
}
}
WorkloadOutcome {
latencies_us_ok,
error_kinds,
}
}
fn short_kind(e: &anyhow::Error) -> String {
let s = e.to_string();
let line = s.lines().next().unwrap_or("unknown error");
if line.len() <= 140 {
line.to_string()
} else {
let mut end = 140;
while end > 0 && !line.is_char_boundary(end) {
end -= 1;
}
format!("{}...", &line[..end])
}
}
fn percentile(sorted_asc: &[f64], p: f64) -> f64 {
if sorted_asc.is_empty() {
return 0.0;
}
if sorted_asc.len() == 1 {
return sorted_asc[0];
}
let rank = (p / 100.0) * (sorted_asc.len() as f64 - 1.0);
let lower = rank.floor() as usize;
let upper = rank.ceil() as usize;
if lower == upper {
sorted_asc[lower]
} else {
let frac = rank - lower as f64;
sorted_asc[lower] + (sorted_asc[upper] - sorted_asc[lower]) * frac
}
}
fn build_report(config: BenchConfig, outcome: &WorkloadOutcome, elapsed: Duration) -> BenchReport {
let mut latencies_ms: Vec<f64> = outcome
.latencies_us_ok
.iter()
.map(|us| *us as f64 / 1000.0)
.collect();
latencies_ms.sort_by(|a, b| a.partial_cmp(b).unwrap());
let latency_ms = if latencies_ms.is_empty() {
LatencyStats::default()
} else {
LatencyStats {
p50_ms: percentile(&latencies_ms, 50.0),
p95_ms: percentile(&latencies_ms, 95.0),
p99_ms: percentile(&latencies_ms, 99.0),
max_ms: *latencies_ms.last().unwrap(),
mean_ms: latencies_ms.iter().sum::<f64>() / latencies_ms.len() as f64,
}
};
let total_queries = outcome.len();
let elapsed_secs = elapsed.as_secs_f64();
let achieved_qps = if elapsed_secs > 0.0 {
total_queries as f64 / elapsed_secs
} else {
0.0
};
let error_count: u64 = outcome.error_kinds.values().sum();
BenchReport {
config,
total_queries,
elapsed_secs,
achieved_qps,
latency_ms,
errors: ErrorsReport {
count: error_count,
kinds: outcome.error_kinds.clone(),
},
}
}
fn print_summary(report: &BenchReport) {
println!(
"{} queries in {:.2}s ({:.1} qps, concurrency {})",
report.total_queries, report.elapsed_secs, report.achieved_qps, report.config.concurrency
);
println!(
"latency ms: p50={:.2} p95={:.2} p99={:.2} max={:.2} mean={:.2}",
report.latency_ms.p50_ms,
report.latency_ms.p95_ms,
report.latency_ms.p99_ms,
report.latency_ms.max_ms,
report.latency_ms.mean_ms,
);
if report.errors.count > 0 {
println!("errors: {} total", report.errors.count);
for (kind, count) in &report.errors.kinds {
println!(" {count}x {kind}");
}
} else {
println!("errors: none");
}
}
#[cfg(test)]
mod tests {
use super::*;
use second_brain_core::index::IndexClaim;
#[test]
fn a_daemon_claim_refuses_the_bench_while_held() {
let dir = tempfile::tempdir().unwrap();
let claim = IndexClaim::acquire(dir.path()).unwrap();
let err = check_no_live_writer(dir.path()).err().unwrap();
assert!(format!("{err:#}").contains("refusing to start"), "{err:#}");
drop(claim);
check_no_live_writer(dir.path()).unwrap();
}
}