use std::collections::{BTreeMap, HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::Instant;
use anyhow::{Context, Result, anyhow, bail};
use serde::{Deserialize, Serialize};
use usearch::{Index, IndexOptions, MetricKind, ScalarKind};
use uuid::Uuid;
use second_brain_core::embedding::{Embedder, query_prompt};
use second_brain_core::index::EMBEDDING_DIM;
use second_brain_core::sqlite_store::SqliteStore;
const DEFAULT_SEARCH_K: usize = 50;
const DEFAULT_RECALL_KS: &str = "10,30,50";
const DEFAULT_KINDS: &str = "f32,f16,i8";
const EVAL_MACHINE_ID: &str = "sb-quant-sweep";
fn print_help() {
println!(
r#"quant_sweep - measure usearch quantization tradeoffs on the real snapshot
USAGE:
quant_sweep <MODE> [OPTIONS]
MODES:
embed-queries Embed every query in an eval set once and cache the vectors to disk.
build-index Rebuild a usearch vector index from a snapshot at one scalar kind.
measure Open a pre-built index read-only and measure RSS, latency, and recall.
sweep Run embed-queries once, then build-index+measure for each kind, and
write the aggregate comparison report.
embed-queries OPTIONS:
--eval <PATH> Eval set JSONL (default: evals/eval_set.jsonl)
--out <PATH> Where to write the embedded-queries JSON (required)
build-index OPTIONS:
--kind <f32|f16|i8> Scalar kind to quantize vectors to (required)
--snapshot <PATH> SqliteStore snapshot to read memories from (default: evals/snapshot.db)
--index-out <PATH> Where to write the .usearch file (required)
--json <PATH> Where to write this run's report as JSON (required)
measure OPTIONS:
--kind <f32|f16|i8> Scalar kind the index at --index was built with (required)
--index <PATH> Pre-built .usearch file to open read-only (required)
--snapshot <PATH> Snapshot to resolve usearch keys back to memory ids (default: evals/snapshot.db)
--embedded-queries <PATH> Output of embed-queries (required)
--search-k <N> Neighbours requested per search (default: {DEFAULT_SEARCH_K}, raised to cover --recall-ks)
--recall-ks <CSV> K values to score dense recall at (default: {DEFAULT_RECALL_KS})
--json <PATH> Where to write this run's report as JSON (required)
sweep OPTIONS:
--snapshot <PATH> (default: evals/snapshot.db)
--eval <PATH> (default: evals/eval_set.jsonl)
--kinds <CSV> Scalar kinds to compare; must include f32 (default: {DEFAULT_KINDS})
--recall-ks <CSV> (default: {DEFAULT_RECALL_KS})
--search-k <N> (default: {DEFAULT_SEARCH_K})
--work-dir <PATH> Scratch dir for built indexes and per-arm reports
(default: a fixed tempdir, never inside the repo)
--out <PATH> Final report path (default: evals/quantization-sweep.json)
--force-reembed Re-embed queries even if a cached embedded-queries.json exists
-h, --help Print this help and exit
"#
);
}
fn default_snapshot() -> PathBuf {
PathBuf::from("evals/snapshot.db")
}
fn default_eval() -> PathBuf {
PathBuf::from("evals/eval_set.jsonl")
}
fn default_work_dir() -> PathBuf {
std::env::temp_dir().join("second-brain-quant-sweep")
}
fn default_out() -> PathBuf {
PathBuf::from("evals/quantization-sweep.json")
}
fn kind_label(kind: ScalarKind) -> &'static str {
match kind {
ScalarKind::F32 => "f32",
ScalarKind::F16 => "f16",
ScalarKind::I8 => "i8",
_ => "unknown",
}
}
fn parse_kind(raw: &str) -> Result<ScalarKind, String> {
match raw.to_ascii_lowercase().as_str() {
"f32" => Ok(ScalarKind::F32),
"f16" => Ok(ScalarKind::F16),
"i8" => Ok(ScalarKind::I8),
other => Err(format!(
"unknown scalar kind '{other}'; expected one of: f32, f16, i8"
)),
}
}
fn parse_kinds_csv(raw: &str) -> Result<Vec<ScalarKind>, String> {
raw.split(',')
.map(str::trim)
.filter(|s| !s.is_empty())
.map(parse_kind)
.collect()
}
fn parse_usize_csv(raw: &str) -> Result<Vec<usize>, String> {
let mut ks: Vec<usize> = raw
.split(',')
.map(str::trim)
.filter(|s| !s.is_empty())
.map(|s| s.parse::<usize>().map_err(|e| format!("'{s}': {e}")))
.collect::<Result<Vec<usize>, String>>()?;
if ks.is_empty() {
return Err("expected at least one value in a comma-separated list".to_string());
}
ks.sort_unstable();
ks.dedup();
Ok(ks)
}
fn collect_flags(raw: &[String]) -> Result<HashMap<String, String>, String> {
let mut flags = HashMap::new();
let mut i = 0;
while i < raw.len() {
let key = raw[i]
.strip_prefix("--")
.ok_or_else(|| format!("expected a --flag, got '{}'", raw[i]))?
.to_string();
let value = raw
.get(i + 1)
.ok_or_else(|| format!("--{key} requires a value"))?
.clone();
flags.insert(key, value);
i += 2;
}
Ok(flags)
}
fn take_required(flags: &HashMap<String, String>, key: &str) -> Result<String, String> {
flags
.get(key)
.cloned()
.ok_or_else(|| format!("--{key} is required"))
}
fn take_path(
flags: &HashMap<String, String>,
key: &str,
default: impl FnOnce() -> PathBuf,
) -> PathBuf {
flags.get(key).map(PathBuf::from).unwrap_or_else(default)
}
fn take_usize(flags: &HashMap<String, String>, key: &str, default: usize) -> Result<usize, String> {
match flags.get(key) {
Some(v) => v.parse::<usize>().map_err(|e| format!("--{key}: {e}")),
None => Ok(default),
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
struct RssSnapshot {
vm_rss_kb: u64,
}
fn parse_kb_field(field: &str) -> Result<u64> {
field
.trim()
.trim_end_matches("kB")
.trim()
.parse::<u64>()
.with_context(|| format!("parsing /proc/self/status field: {field:?}"))
}
fn read_rss() -> Result<RssSnapshot> {
let status =
std::fs::read_to_string("/proc/self/status").context("reading /proc/self/status")?;
for line in status.lines() {
if let Some(rest) = line.strip_prefix("VmRSS:") {
return Ok(RssSnapshot {
vm_rss_kb: parse_kb_field(rest)?,
});
}
}
Err(anyhow!("VmRSS not found in /proc/self/status"))
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct LatencyStats {
p50_us: f64,
p95_us: f64,
max_us: f64,
mean_us: f64,
count: usize,
}
fn latency_stats(mut samples: Vec<f64>) -> LatencyStats {
if samples.is_empty() {
return LatencyStats {
p50_us: 0.0,
p95_us: 0.0,
max_us: 0.0,
mean_us: 0.0,
count: 0,
};
}
samples.sort_by(|a, b| a.partial_cmp(b).expect("latency samples are finite"));
let n = samples.len();
let percentile = |q: f64| -> f64 {
let idx = ((q * (n as f64 - 1.0)).round() as usize).min(n - 1);
samples[idx]
};
let mean = samples.iter().sum::<f64>() / n as f64;
LatencyStats {
p50_us: percentile(0.50),
p95_us: percentile(0.95),
max_us: *samples.last().expect("non-empty"),
mean_us: mean,
count: n,
}
}
fn new_arm_index(kind: ScalarKind) -> Result<Index> {
let options = IndexOptions {
dimensions: EMBEDDING_DIM,
metric: MetricKind::Cos,
quantization: kind,
..IndexOptions::default()
};
Index::new(&options).map_err(|e| anyhow!("creating usearch index: {e}"))
}
fn open_snapshot(path: &Path) -> Result<SqliteStore> {
SqliteStore::open(path, EVAL_MACHINE_ID.to_string())
.with_context(|| format!("opening snapshot at {}", path.display()))
}
fn path_str(path: &Path) -> Result<&str> {
path.to_str()
.with_context(|| format!("{} is not valid utf-8", path.display()))
}
fn write_json<T: Serialize>(path: &Path, value: &T) -> Result<()> {
if let Some(parent) = path.parent()
&& !parent.as_os_str().is_empty()
{
std::fs::create_dir_all(parent)
.with_context(|| format!("creating {}", parent.display()))?;
}
let json = serde_json::to_string_pretty(value).context("serializing report")?;
std::fs::write(path, json).with_context(|| format!("writing {}", path.display()))
}
fn read_json<T: for<'de> Deserialize<'de>>(path: &Path) -> Result<T> {
let text =
std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?;
serde_json::from_str(&text).with_context(|| format!("parsing {}", path.display()))
}
#[derive(Debug, Clone, Deserialize)]
struct EvalQuery {
query_id: String,
query: String,
#[serde(default)]
relevant_memory_ids: Vec<Uuid>,
}
fn load_eval_queries(path: &Path) -> Result<Vec<EvalQuery>> {
let text = std::fs::read_to_string(path)
.with_context(|| format!("reading eval set {}", path.display()))?;
let mut out = Vec::new();
for (lineno, line) in text.lines().enumerate() {
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let q: EvalQuery = serde_json::from_str(trimmed).with_context(|| {
format!("{}:{}: invalid eval query JSON", path.display(), lineno + 1)
})?;
out.push(q);
}
Ok(out)
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct EmbeddedQuery {
query_id: String,
relevant_memory_ids: Vec<Uuid>,
embedding: Vec<f32>,
}
fn load_embedded_queries(path: &Path) -> Result<Vec<EmbeddedQuery>> {
read_json(path)
}
struct EmbedQueriesArgs {
eval: PathBuf,
out: PathBuf,
}
fn parse_embed_queries_args(raw: &[String]) -> Result<EmbedQueriesArgs, String> {
let flags = collect_flags(raw)?;
Ok(EmbedQueriesArgs {
eval: take_path(&flags, "eval", default_eval),
out: PathBuf::from(take_required(&flags, "out")?),
})
}
fn cmd_embed_queries(args: &EmbedQueriesArgs) -> Result<()> {
let queries = load_eval_queries(&args.eval)?;
eprintln!(
"loaded {} eval queries from {}",
queries.len(),
args.eval.display()
);
eprintln!("loading embedding model");
let embedder = Embedder::new().context("loading embedding model")?;
eprintln!("embedding {} queries", queries.len());
let prefixed: Vec<String> = queries.iter().map(|q| query_prompt(&q.query)).collect();
let refs: Vec<&str> = prefixed.iter().map(String::as_str).collect();
let vectors = embedder
.embed_batch(&refs)
.context("batch-embedding eval queries")?;
let embedded: Vec<EmbeddedQuery> = queries
.into_iter()
.zip(vectors)
.map(|(q, embedding)| EmbeddedQuery {
query_id: q.query_id,
relevant_memory_ids: q.relevant_memory_ids,
embedding,
})
.collect();
write_json(&args.out, &embedded)?;
println!(
"embedded {} queries -> {}",
embedded.len(),
args.out.display()
);
Ok(())
}
struct BuildIndexArgs {
kind: ScalarKind,
snapshot: PathBuf,
index_out: PathBuf,
json: PathBuf,
}
fn parse_build_index_args(raw: &[String]) -> Result<BuildIndexArgs, String> {
let flags = collect_flags(raw)?;
Ok(BuildIndexArgs {
kind: parse_kind(&take_required(&flags, "kind")?)?,
snapshot: take_path(&flags, "snapshot", default_snapshot),
index_out: PathBuf::from(take_required(&flags, "index-out")?),
json: PathBuf::from(take_required(&flags, "json")?),
})
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct BuildReport {
kind: String,
candidate_rows: usize,
indexed_vectors: usize,
index_file_bytes: u64,
build_wall_clock_ms: u64,
}
fn cmd_build_index(args: &BuildIndexArgs) -> Result<()> {
let store = open_snapshot(&args.snapshot)?;
let started = Instant::now();
let index = new_arm_index(args.kind)?;
let rows = store
.all_indexable()
.context("reading indexable memories from snapshot")?;
index
.reserve(rows.len().max(1))
.map_err(|e| anyhow!("reserving usearch capacity for {} vectors: {e}", rows.len()))?;
let mut indexed = 0usize;
for (memory_id, embedding, _body) in &rows {
if embedding.len() != EMBEDDING_DIM {
continue;
}
let key = store
.vec_key(*memory_id)
.with_context(|| format!("allocating vec_key for {memory_id}"))?;
index
.add(key, embedding.as_slice())
.map_err(|e| anyhow!("adding {memory_id} (key {key}): {e}"))?;
indexed += 1;
}
let index_out_str = path_str(&args.index_out)?;
if let Some(parent) = args.index_out.parent()
&& !parent.as_os_str().is_empty()
{
std::fs::create_dir_all(parent)
.with_context(|| format!("creating {}", parent.display()))?;
}
index
.save(index_out_str)
.map_err(|e| anyhow!("saving index to {index_out_str}: {e}"))?;
let build_wall_clock_ms = started.elapsed().as_millis() as u64;
let index_file_bytes = std::fs::metadata(&args.index_out)
.with_context(|| format!("statting {}", args.index_out.display()))?
.len();
let report = BuildReport {
kind: kind_label(args.kind).to_string(),
candidate_rows: rows.len(),
indexed_vectors: indexed,
index_file_bytes,
build_wall_clock_ms,
};
println!(
"built {} index: {} vectors ({} candidates) -> {} ({} bytes) in {}ms",
report.kind,
report.indexed_vectors,
report.candidate_rows,
args.index_out.display(),
report.index_file_bytes,
report.build_wall_clock_ms
);
write_json(&args.json, &report)?;
Ok(())
}
struct MeasureArgs {
kind: ScalarKind,
index: PathBuf,
snapshot: PathBuf,
embedded_queries: PathBuf,
search_k: usize,
recall_ks: Vec<usize>,
json: PathBuf,
}
fn parse_measure_args(raw: &[String]) -> Result<MeasureArgs, String> {
let flags = collect_flags(raw)?;
let recall_ks = match flags.get("recall-ks") {
Some(v) => parse_usize_csv(v)?,
None => parse_usize_csv(DEFAULT_RECALL_KS)?,
};
let search_k = take_usize(&flags, "search-k", DEFAULT_SEARCH_K)?;
let max_recall_k = *recall_ks.iter().max().expect("recall_ks is non-empty");
Ok(MeasureArgs {
kind: parse_kind(&take_required(&flags, "kind")?)?,
index: PathBuf::from(take_required(&flags, "index")?),
snapshot: take_path(&flags, "snapshot", default_snapshot),
embedded_queries: PathBuf::from(take_required(&flags, "embedded-queries")?),
search_k: search_k.max(max_recall_k),
recall_ks,
json: PathBuf::from(take_required(&flags, "json")?),
})
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct MeasureReport {
kind: String,
index_file_bytes: u64,
queries_total: usize,
queries_scored: usize,
unresolved_keys: usize,
search_k: usize,
recall_ks: Vec<usize>,
recall: BTreeMap<String, f64>,
rss_before_open_kb: u64,
rss_after_open_kb: u64,
rss_after_queries_kb: u64,
delta_open_vm_rss_kb: i64,
delta_queries_vm_rss_kb: i64,
usearch_memory_usage_after_open_bytes: usize,
usearch_memory_usage_after_queries_bytes: usize,
search_latency: LatencyStats,
pid: u32,
}
fn cmd_measure(args: &MeasureArgs) -> Result<()> {
let rss_before_open = read_rss()?;
let store = open_snapshot(&args.snapshot)?;
let index_path_str = path_str(&args.index)?;
let on_disk = Index::metadata(index_path_str)
.map_err(|e| anyhow!("reading usearch header at {}: {e}", args.index.display()))?;
if on_disk.quantization != args.kind {
bail!(
"index at {} was built as {:?} but --kind says {:?}",
args.index.display(),
on_disk.quantization,
args.kind
);
}
let index = new_arm_index(args.kind)?;
index
.view(index_path_str)
.map_err(|e| anyhow!("viewing {index_path_str}: {e}"))?;
let rss_after_open = read_rss()?;
let usearch_bytes_after_open = index.memory_usage();
let embedded = load_embedded_queries(&args.embedded_queries)?;
let queries_total = embedded.len();
let mut key_cache: HashMap<u64, Option<Uuid>> = HashMap::new();
let mut latencies_us = Vec::with_capacity(queries_total);
let mut hits: HashMap<usize, usize> = args.recall_ks.iter().map(|k| (*k, 0)).collect();
let mut scored_queries = 0usize;
let mut unresolved_keys = 0usize;
for q in &embedded {
if q.relevant_memory_ids.is_empty() {
continue;
}
let relevant: HashSet<Uuid> = q.relevant_memory_ids.iter().copied().collect();
let started = Instant::now();
let matches = index
.search(&q.embedding, args.search_k)
.map_err(|e| anyhow!("search for query {}: {e}", q.query_id))?;
latencies_us.push(started.elapsed().as_secs_f64() * 1_000_000.0);
let mut ranked_ids = Vec::with_capacity(matches.keys.len());
for key in &matches.keys {
let resolved = match key_cache.get(key) {
Some(cached) => *cached,
None => {
let looked_up = store
.memory_id_for_vec_key(*key)
.with_context(|| format!("resolving vec_key {key}"))?;
key_cache.insert(*key, looked_up);
looked_up
}
};
match resolved {
Some(id) => ranked_ids.push(id),
None => unresolved_keys += 1,
}
}
for k in &args.recall_ks {
if ranked_ids.iter().take(*k).any(|id| relevant.contains(id)) {
*hits.get_mut(k).expect("recall_ks keys pre-populated") += 1;
}
}
scored_queries += 1;
}
let rss_after_queries = read_rss()?;
let usearch_bytes_after_queries = index.memory_usage();
let recall: BTreeMap<String, f64> = args
.recall_ks
.iter()
.map(|k| {
let r = if scored_queries == 0 {
0.0
} else {
hits[k] as f64 / scored_queries as f64
};
(k.to_string(), r)
})
.collect();
let report = MeasureReport {
kind: kind_label(args.kind).to_string(),
index_file_bytes: std::fs::metadata(&args.index)
.with_context(|| format!("statting {}", args.index.display()))?
.len(),
queries_total,
queries_scored: scored_queries,
unresolved_keys,
search_k: args.search_k,
recall_ks: args.recall_ks.clone(),
recall,
rss_before_open_kb: rss_before_open.vm_rss_kb,
rss_after_open_kb: rss_after_open.vm_rss_kb,
rss_after_queries_kb: rss_after_queries.vm_rss_kb,
delta_open_vm_rss_kb: rss_after_open.vm_rss_kb as i64 - rss_before_open.vm_rss_kb as i64,
delta_queries_vm_rss_kb: rss_after_queries.vm_rss_kb as i64
- rss_after_open.vm_rss_kb as i64,
usearch_memory_usage_after_open_bytes: usearch_bytes_after_open,
usearch_memory_usage_after_queries_bytes: usearch_bytes_after_queries,
search_latency: latency_stats(latencies_us),
pid: std::process::id(),
};
println!(
"measured {}: rss_after_queries={}kB p50={:.1}us p95={:.1}us recall={:?} unresolved_keys={}",
report.kind,
report.rss_after_queries_kb,
report.search_latency.p50_us,
report.search_latency.p95_us,
report.recall,
report.unresolved_keys
);
write_json(&args.json, &report)?;
Ok(())
}
struct SweepArgs {
snapshot: PathBuf,
eval: PathBuf,
kinds: Vec<ScalarKind>,
recall_ks: Vec<usize>,
search_k: usize,
work_dir: PathBuf,
out: PathBuf,
force_reembed: bool,
}
fn parse_sweep_args(raw: &[String]) -> Result<SweepArgs, String> {
let flags = collect_flags(raw)?;
let recall_ks = match flags.get("recall-ks") {
Some(v) => parse_usize_csv(v)?,
None => parse_usize_csv(DEFAULT_RECALL_KS)?,
};
let search_k = take_usize(&flags, "search-k", DEFAULT_SEARCH_K)?;
let max_recall_k = *recall_ks.iter().max().expect("recall_ks is non-empty");
let kinds = match flags.get("kinds") {
Some(v) => parse_kinds_csv(v)?,
None => parse_kinds_csv(DEFAULT_KINDS)?,
};
if !kinds.contains(&ScalarKind::F32) {
return Err("--kinds must include f32: it is the recall and size baseline every other kind is compared against".to_string());
}
Ok(SweepArgs {
snapshot: take_path(&flags, "snapshot", default_snapshot),
eval: take_path(&flags, "eval", default_eval),
kinds,
recall_ks,
search_k: search_k.max(max_recall_k),
work_dir: take_path(&flags, "work-dir", default_work_dir),
out: take_path(&flags, "out", default_out),
force_reembed: flags.contains_key("force-reembed"),
})
}
#[derive(Debug, Clone, Serialize)]
struct ArmReport {
kind: String,
build: BuildReport,
measure: MeasureReport,
recall_delta_vs_f32: BTreeMap<String, f64>,
}
#[derive(Debug, Clone, Serialize)]
struct SweepReport {
generated_at: String,
snapshot: String,
eval_set: String,
embedding_dim: usize,
queries_total: usize,
search_k: usize,
recall_ks: Vec<usize>,
arms: Vec<ArmReport>,
recommendation: String,
}
fn run_child(exe: &Path, args: &[String]) -> Result<()> {
let status = Command::new(exe)
.args(args)
.status()
.with_context(|| format!("spawning {} {:?}", exe.display(), args))?;
if !status.success() {
bail!(
"child process {} {:?} exited with {status}",
exe.display(),
args
);
}
Ok(())
}
fn recommend(arms: &[ArmReport], recall_ks: &[usize], tolerance: f64) -> String {
let holds_within_tolerance = |arm: &ArmReport| {
recall_ks.iter().all(|k| {
arm.recall_delta_vs_f32
.get(&k.to_string())
.copied()
.unwrap_or(f64::MIN)
>= -tolerance
})
};
for preferred in ["i8", "f16"] {
if let Some(arm) = arms.iter().find(|a| a.kind == preferred)
&& holds_within_tolerance(arm)
{
return format!(
"adopt {preferred}: dense recall stays within {:.1}pp of f32 at every measured K",
tolerance * 100.0
);
}
}
"keep f32: no smaller kind held recall within tolerance at every measured K".to_string()
}
fn cmd_sweep(args: &SweepArgs) -> Result<()> {
std::fs::create_dir_all(&args.work_dir)
.with_context(|| format!("creating work dir {}", args.work_dir.display()))?;
let embedded_queries_path = args.work_dir.join("embedded-queries.json");
if args.force_reembed || !embedded_queries_path.exists() {
cmd_embed_queries(&EmbedQueriesArgs {
eval: args.eval.clone(),
out: embedded_queries_path.clone(),
})?;
} else {
eprintln!(
"reusing cached embedded queries at {}",
embedded_queries_path.display()
);
}
let exe = std::env::current_exe().context("resolving own executable path")?;
let recall_ks_csv = args
.recall_ks
.iter()
.map(usize::to_string)
.collect::<Vec<_>>()
.join(",");
let mut arms = Vec::new();
for kind in &args.kinds {
let label = kind_label(*kind);
eprintln!("=== arm: {label} ===");
let index_out = args.work_dir.join(format!("vectors-{label}.usearch"));
let build_json = args.work_dir.join(format!("build-{label}.json"));
let measure_json = args.work_dir.join(format!("measure-{label}.json"));
run_child(
&exe,
&[
"build-index".to_string(),
"--kind".to_string(),
label.to_string(),
"--snapshot".to_string(),
args.snapshot.display().to_string(),
"--index-out".to_string(),
index_out.display().to_string(),
"--json".to_string(),
build_json.display().to_string(),
],
)?;
run_child(
&exe,
&[
"measure".to_string(),
"--kind".to_string(),
label.to_string(),
"--index".to_string(),
index_out.display().to_string(),
"--snapshot".to_string(),
args.snapshot.display().to_string(),
"--embedded-queries".to_string(),
embedded_queries_path.display().to_string(),
"--search-k".to_string(),
args.search_k.to_string(),
"--recall-ks".to_string(),
recall_ks_csv.clone(),
"--json".to_string(),
measure_json.display().to_string(),
],
)?;
let build: BuildReport = read_json(&build_json)?;
let measure: MeasureReport = read_json(&measure_json)?;
arms.push((label, build, measure));
}
let baseline_recall = arms
.iter()
.find(|(label, _, _)| *label == "f32")
.map(|(_, _, m)| m.recall.clone())
.ok_or_else(|| anyhow!("f32 arm did not produce a report; cannot compute deltas"))?;
let queries_total = arms.first().map(|(_, _, m)| m.queries_total).unwrap_or(0);
let mut arm_reports: Vec<ArmReport> = arms
.into_iter()
.map(|(label, build, measure)| {
let recall_delta_vs_f32 = measure
.recall
.iter()
.map(|(k, v)| {
let base = baseline_recall.get(k).copied().unwrap_or(0.0);
(k.clone(), v - base)
})
.collect();
ArmReport {
kind: label.to_string(),
build,
measure,
recall_delta_vs_f32,
}
})
.collect();
arm_reports.sort_by_key(|a| a.build.index_file_bytes);
let recommendation = recommend(&arm_reports, &args.recall_ks, 0.01);
let report = SweepReport {
generated_at: chrono::Utc::now().to_rfc3339(),
snapshot: args.snapshot.display().to_string(),
eval_set: args.eval.display().to_string(),
embedding_dim: EMBEDDING_DIM,
queries_total,
search_k: args.search_k,
recall_ks: args.recall_ks.clone(),
arms: arm_reports,
recommendation,
};
write_json(&args.out, &report)?;
print_table(&report);
println!("\nwrote {}", args.out.display());
Ok(())
}
fn print_table(report: &SweepReport) {
println!();
print!(
"{:<6} {:>13} {:>10} {:>10} {:>9} {:>9}",
"kind", "file_bytes", "build_ms", "rss_kb", "p50_us", "p95_us"
);
for k in &report.recall_ks {
print!(" {:>9}", format!("r@{k}"));
}
for k in &report.recall_ks {
print!(" {:>9}", format!("d@{k}"));
}
println!();
for arm in &report.arms {
print!(
"{:<6} {:>13} {:>10} {:>10} {:>9.1} {:>9.1}",
arm.kind,
arm.build.index_file_bytes,
arm.build.build_wall_clock_ms,
arm.measure.rss_after_queries_kb,
arm.measure.search_latency.p50_us,
arm.measure.search_latency.p95_us
);
for k in &report.recall_ks {
let r = arm
.measure
.recall
.get(&k.to_string())
.copied()
.unwrap_or(0.0);
print!(" {r:>9.4}");
}
for k in &report.recall_ks {
let d = arm
.recall_delta_vs_f32
.get(&k.to_string())
.copied()
.unwrap_or(0.0);
print!(" {d:>+9.4}");
}
println!();
}
println!("\nrecommendation: {}", report.recommendation);
}
fn main() -> Result<()> {
let raw: Vec<String> = std::env::args().skip(1).collect();
if raw.iter().any(|a| a == "--help" || a == "-h") {
print_help();
return Ok(());
}
if raw.is_empty() {
eprintln!("error: missing MODE argument\n");
print_help();
std::process::exit(2);
}
let mode = raw[0].as_str();
let rest = &raw[1..];
let result = match mode {
"embed-queries" => parse_embed_queries_args(rest)
.map_err(anyhow::Error::msg)
.and_then(|a| cmd_embed_queries(&a)),
"build-index" => parse_build_index_args(rest)
.map_err(anyhow::Error::msg)
.and_then(|a| cmd_build_index(&a)),
"measure" => parse_measure_args(rest)
.map_err(anyhow::Error::msg)
.and_then(|a| cmd_measure(&a)),
"sweep" => parse_sweep_args(rest)
.map_err(anyhow::Error::msg)
.and_then(|a| cmd_sweep(&a)),
other => {
eprintln!(
"error: unknown mode '{other}'; expected embed-queries, build-index, measure, or sweep\n"
);
print_help();
std::process::exit(2);
}
};
if let Err(e) = result {
eprintln!("error: {e:#}");
std::process::exit(1);
}
Ok(())
}