use std::path::{Path, PathBuf};
use crate::core::eval_ab::artifact::{self, SignedAbReportV1};
use crate::core::eval_ab::footprint::{
Footprint, FootprintConfig, FootprintReport, run_footprint_ab,
};
use crate::core::eval_ab::model::{ModelRunner, OpenAiRunner, RecordedRunner, RecordingRunner};
use crate::core::eval_ab::report::ReportConfig;
use crate::core::eval_ab::routing_eval::{RoutingEvalConfig, run_routing_eval};
use crate::core::eval_ab::suite::EvalSuite;
use crate::core::eval_ab::testbench::lockfile::TestbenchLock;
use crate::core::eval_ab::testbench::{self, TestbenchConfig, TestbenchReport, findings};
use crate::core::eval_ab::{AbRunConfig, run_ab};
pub fn cmd_eval(args: &[String]) {
if args.iter().any(|a| a == "--delta") {
let rest: Vec<String> = args.iter().filter(|a| *a != "--delta").cloned().collect();
return cmd_footprint(&rest);
}
match args.first().map(String::as_str) {
Some("ab") => cmd_ab(&args[1..]),
Some("footprint" | "delta") => cmd_footprint(&args[1..]),
Some("routing") => cmd_routing(&args[1..]),
Some("testbench") => cmd_testbench(&args[1..]),
Some("verify") => cmd_verify(&args[1..]),
Some("init") => cmd_init(&args[1..]),
Some("-h" | "--help") | None => print_help(),
Some(other) => {
eprintln!("eval: unknown subcommand '{other}'\n");
print_help();
std::process::exit(2);
}
}
}
fn print_help() {
println!(
"lean-ctx eval — deterministic with/without output-quality proof\n\n\
USAGE:\n\
lean-ctx eval init <dir> Scaffold a runnable starter suite\n\
lean-ctx eval ab --suite <file> [opts] Run the A/B quality comparison\n\
lean-ctx eval footprint --suite <f> [o] Ablate lean-ctx's OWN injected context (#959)\n\
lean-ctx eval routing --suite <f> [o] Router off-vs-on rate-card savings proof\n\
lean-ctx eval testbench --lock <f> [o] Off-vs-on across pinned real repos (#611)\n\
lean-ctx eval verify <artifact.json> Verify signature + determinism digest\n\n\
ab OPTIONS:\n\
--suite <file> NDJSON suite (required)\n\
--budget <n> Token budget per condition (default 4000)\n\
--margin <f> Non-inferiority margin for the gate (default 0.0)\n\
--out <file> Artifact path (default: data dir)\n\
--replay <file> Replay a recording instead of calling a live model (deterministic CI)\n\
--record <file> Call the live model and save responses to a recording\n\
--gate Exit non-zero if the verdict is a regression\n\n\
footprint OPTIONS (also: `eval --delta`):\n\
--suite <file> Footprint-sensitive NDJSON suite (required)\n\
--margin <f> Non-inferiority margin for the per-element gate (default 0.0)\n\
--floor <n> Min marginal tokens before flagging an element to prune (default 50)\n\
--replay <file> Replay a recording (deterministic); --record to capture live\n\
--json Emit the full JSON report instead of the side-by-side table\n\
--gate Exit non-zero if any injected element is actively harmful\n\n\
routing OPTIONS:\n\
--suite <file> NDJSON suite of real task prompts (required)\n\
--requested <m> Model the off-arm assumes (default: [proxy.baseline].reference_model)\n\
--json Emit the full JSON report instead of the table\n\
--gate Exit non-zero if routing downgraded premium work\n\
Rules come from [proxy.routing] in config.toml — the deployment's live rule set.\n\n\
testbench OPTIONS:\n\
--lock <file> Pinned-repo lockfile (default eval/testbench/testbench.lock.json)\n\
--out <dir> Output dir for FINDINGS.md + regressions.json (default testbench-out)\n\
--cache <dir> Clone cache for remote repos (default <out>/cache)\n\
--budget <n> Token budget per condition (default 4000)\n\
--margin <f> Non-inferiority margin for the per-repo gate (default 0.0)\n\
--replay <file> Replay a recording (deterministic CI); --record to capture live\n\
--gate Exit non-zero if any repo regressed\n\n\
LIVE MODEL (when not replaying) is read from the environment:\n\
LEAN_CTX_EVAL_MODEL_URL OpenAI-compatible base URL (e.g. https://api.openai.com/v1)\n\
LEAN_CTX_EVAL_MODEL Model id (e.g. gpt-4o-mini)\n\
LEAN_CTX_EVAL_MODEL_KEY API key (optional for local servers)\n\
LEAN_CTX_EVAL_SEED Decoding seed (default 7)"
);
}
fn flag_value<'a>(args: &'a [String], flag: &str) -> Option<&'a str> {
args.iter()
.position(|a| a == flag)
.and_then(|i| args.get(i + 1))
.map(String::as_str)
}
fn has_flag(args: &[String], flag: &str) -> bool {
args.iter().any(|a| a == flag)
}
fn cmd_ab(args: &[String]) {
let Some(suite_path) = flag_value(args, "--suite") else {
eprintln!("eval ab: --suite <file> is required");
std::process::exit(2);
};
let suite_path = PathBuf::from(suite_path);
let suite = match EvalSuite::load(&suite_path) {
Ok(s) => s,
Err(e) => {
eprintln!("eval ab: {e:#}");
std::process::exit(1);
}
};
let suite_name = suite_path
.file_name()
.map_or_else(|| "suite".to_string(), |s| s.to_string_lossy().into_owned());
let mut cfg = AbRunConfig::default();
if let Some(b) = flag_value(args, "--budget").and_then(|v| v.parse().ok()) {
cfg.budget_tokens = b;
}
cfg.report = ReportConfig {
noninferiority_margin: flag_value(args, "--margin")
.and_then(|v| v.parse().ok())
.unwrap_or(0.0),
..ReportConfig::default()
};
let report = if let Some(replay) = flag_value(args, "--replay") {
let runner = match RecordedRunner::from_file(Path::new(replay)) {
Ok(r) => r,
Err(e) => {
eprintln!("eval ab: {e:#}");
std::process::exit(1);
}
};
run_or_exit(&suite, &suite_name, &runner, &cfg)
} else {
let live = match OpenAiRunner::from_env() {
Ok(r) => r,
Err(e) => {
eprintln!(
"eval ab: no live model configured: {e:#}\n(use --replay <file> for an offline run)"
);
std::process::exit(1);
}
};
if let Some(record_path) = flag_value(args, "--record") {
let recorder = RecordingRunner::new(live);
let report = run_or_exit(&suite, &suite_name, &recorder, &cfg);
if let Err(e) = recorder.into_recording().save(Path::new(record_path)) {
eprintln!("eval ab: failed to save recording: {e:#}");
std::process::exit(1);
}
println!("Recording saved → {record_path}");
report
} else {
run_or_exit(&suite, &suite_name, &live, &cfg)
}
};
let agent_id = crate::core::agent_identity::current_agent_id().to_string();
let mut signed = SignedAbReportV1::from_report(report, &agent_id);
if let Err(e) = signed.sign(&agent_id) {
eprintln!("eval ab: signing failed: {e}");
std::process::exit(1);
}
let out = match flag_value(args, "--out") {
Some(p) => PathBuf::from(p),
None => match artifact::default_artifact_path() {
Ok(p) => p,
Err(e) => {
eprintln!("eval ab: {e}");
std::process::exit(1);
}
},
};
if let Err(e) = artifact::write_artifact(&signed, &out) {
eprintln!("eval ab: {e}");
std::process::exit(1);
}
println!("{}", signed.report.render());
println!("determinism digest: {}", signed.determinism_digest);
println!("artifact: {}", out.display());
if has_flag(args, "--gate") && !signed.verdict.gate_passes() {
eprintln!("\nquality gate FAILED: {}", signed.verdict.label());
std::process::exit(1);
}
}
fn run_or_exit(
suite: &EvalSuite,
suite_name: &str,
runner: &dyn crate::core::eval_ab::model::ModelRunner,
cfg: &AbRunConfig,
) -> crate::core::eval_ab::report::AbReport {
match run_ab(suite, suite_name, runner, cfg) {
Ok(r) => r,
Err(e) => {
eprintln!("eval ab: run failed: {e:#}");
std::process::exit(1);
}
}
}
fn cmd_routing(args: &[String]) {
let Some(suite_path) = flag_value(args, "--suite") else {
eprintln!("eval routing: --suite <file> is required");
std::process::exit(2);
};
let suite_path = PathBuf::from(suite_path);
let suite = match EvalSuite::load(&suite_path) {
Ok(s) => s,
Err(e) => {
eprintln!("eval routing: {e:#}");
std::process::exit(1);
}
};
let suite_name = suite_path
.file_name()
.map_or_else(|| "suite".to_string(), |s| s.to_string_lossy().into_owned());
let config = crate::core::config::Config::load();
let requested_model = flag_value(args, "--requested")
.map(str::to_string)
.or_else(|| config.proxy.baseline.reference_model.clone());
let Some(requested_model) = requested_model else {
eprintln!(
"eval routing: no requested model — pass --requested <model> or set \
[proxy.baseline].reference_model in config.toml"
);
std::process::exit(2);
};
let cfg = RoutingEvalConfig {
requested_model,
rules: config.proxy.routing.clone(),
};
let pricing = crate::core::gain::model_pricing::ModelPricing::load();
let report = match run_routing_eval(&suite, &suite_name, &pricing, &cfg) {
Ok(r) => r,
Err(e) => {
eprintln!("eval routing: {e:#}");
std::process::exit(1);
}
};
if has_flag(args, "--json") {
println!("{}", report.to_json());
} else {
println!("{}", report.render());
}
println!("determinism digest: {}", report.determinism_digest());
if has_flag(args, "--gate") && !report.gate_passes() {
eprintln!(
"\nrouting gate FAILED: {} premium task(s) downgraded",
report.premium_downgrades
);
std::process::exit(1);
}
}
fn cmd_footprint(args: &[String]) {
let Some(suite_path) = flag_value(args, "--suite") else {
eprintln!("eval footprint: --suite <file> is required");
std::process::exit(2);
};
let suite_path = PathBuf::from(suite_path);
let suite = match EvalSuite::load(&suite_path) {
Ok(s) => s,
Err(e) => {
eprintln!("eval footprint: {e:#}");
std::process::exit(1);
}
};
let suite_name = suite_path.file_name().map_or_else(
|| "footprint".to_string(),
|s| s.to_string_lossy().into_owned(),
);
let margin = flag_value(args, "--margin")
.and_then(|v| v.parse().ok())
.unwrap_or(0.0);
let token_floor = flag_value(args, "--floor")
.and_then(|v| v.parse().ok())
.unwrap_or_else(|| FootprintConfig::default().token_floor);
let cfg = FootprintConfig {
report: ReportConfig {
noninferiority_margin: margin,
..ReportConfig::default()
},
token_floor,
};
let project_root = std::env::current_dir()
.map_or_else(|_| ".".to_string(), |p| p.to_string_lossy().into_owned());
let footprint = Footprint::live(&project_root);
let mut report = if let Some(replay) = flag_value(args, "--replay") {
let runner = match RecordedRunner::from_file(Path::new(replay)) {
Ok(r) => r,
Err(e) => {
eprintln!("eval footprint: {e:#}");
std::process::exit(1);
}
};
run_footprint_or_exit(&suite, &suite_name, &footprint, &runner, &cfg)
} else {
let live = match OpenAiRunner::from_env() {
Ok(r) => r,
Err(e) => {
eprintln!(
"eval footprint: no live model configured: {e:#}\n(use --replay <file> for an offline run)"
);
std::process::exit(1);
}
};
if let Some(record_path) = flag_value(args, "--record") {
let recorder = RecordingRunner::new(live);
let report = run_footprint_or_exit(&suite, &suite_name, &footprint, &recorder, &cfg);
if let Err(e) = recorder.into_recording().save(Path::new(record_path)) {
eprintln!("eval footprint: failed to save recording: {e:#}");
std::process::exit(1);
}
println!("Recording saved → {record_path}");
report
} else {
run_footprint_or_exit(&suite, &suite_name, &footprint, &live, &cfg)
}
};
let agent_id = crate::core::agent_identity::current_agent_id().to_string();
if let Err(e) = report.sign(&agent_id) {
eprintln!("eval footprint: signing failed: {e}");
std::process::exit(1);
}
let out = match flag_value(args, "--out") {
Some(p) => PathBuf::from(p),
None => match default_footprint_path() {
Ok(p) => p,
Err(e) => {
eprintln!("eval footprint: {e}");
std::process::exit(1);
}
},
};
if let Some(parent) = out.parent() {
let _ = std::fs::create_dir_all(parent);
}
if let Err(e) = std::fs::write(&out, report.to_json()) {
eprintln!("eval footprint: write {}: {e}", out.display());
std::process::exit(1);
}
if has_flag(args, "--json") {
println!("{}", report.to_json());
} else {
println!("{}", report.render());
println!("artifact: {}", out.display());
}
if has_flag(args, "--gate") && !report.gate_passes() {
eprintln!("\nfootprint gate FAILED: a harmful injected element is present");
std::process::exit(1);
}
}
fn run_footprint_or_exit(
suite: &EvalSuite,
suite_name: &str,
footprint: &Footprint,
runner: &dyn ModelRunner,
cfg: &FootprintConfig,
) -> FootprintReport {
match run_footprint_ab(suite, suite_name, footprint, runner, cfg) {
Ok(r) => r,
Err(e) => {
eprintln!("eval footprint: run failed: {e:#}");
std::process::exit(1);
}
}
}
fn default_footprint_path() -> Result<PathBuf, String> {
let dir = crate::core::data_dir::lean_ctx_data_dir()?.join("eval");
std::fs::create_dir_all(&dir).map_err(|e| format!("mkdir eval: {e}"))?;
let stamp = chrono::Utc::now().format("%Y%m%dT%H%M%SZ");
Ok(dir.join(format!("footprint-report-v1_{stamp}.json")))
}
fn cmd_testbench(args: &[String]) {
let lock_path = flag_value(args, "--lock").map_or_else(
|| PathBuf::from("eval/testbench/testbench.lock.json"),
PathBuf::from,
);
let lock = match TestbenchLock::load(&lock_path) {
Ok(l) => l,
Err(e) => {
eprintln!("eval testbench: {e:#}\n(use --lock <file> to point at a lockfile)");
std::process::exit(1);
}
};
let out_dir =
flag_value(args, "--out").map_or_else(|| PathBuf::from("testbench-out"), PathBuf::from);
let cache_dir =
flag_value(args, "--cache").map_or_else(|| out_dir.join("cache"), PathBuf::from);
let mut cfg = TestbenchConfig::default();
if let Some(b) = flag_value(args, "--budget").and_then(|v| v.parse().ok()) {
cfg.run.budget_tokens = b;
}
cfg.run.report = ReportConfig {
noninferiority_margin: flag_value(args, "--margin")
.and_then(|v| v.parse().ok())
.unwrap_or(0.0),
..ReportConfig::default()
};
let report = if let Some(replay) = flag_value(args, "--replay") {
let runner = match RecordedRunner::from_file(Path::new(replay)) {
Ok(r) => r,
Err(e) => {
eprintln!("eval testbench: {e:#}");
std::process::exit(1);
}
};
run_testbench_or_exit(&lock, &cache_dir, &runner, &cfg)
} else {
let live = match OpenAiRunner::from_env() {
Ok(r) => r,
Err(e) => {
eprintln!(
"eval testbench: no live model configured: {e:#}\n(use --replay <file> for an offline run)"
);
std::process::exit(1);
}
};
if let Some(record_path) = flag_value(args, "--record") {
let recorder = RecordingRunner::new(live);
let report = run_testbench_or_exit(&lock, &cache_dir, &recorder, &cfg);
if let Err(e) = recorder.into_recording().save(Path::new(record_path)) {
eprintln!("eval testbench: failed to save recording: {e:#}");
std::process::exit(1);
}
println!("Recording saved → {record_path}");
report
} else {
run_testbench_or_exit(&lock, &cache_dir, &live, &cfg)
}
};
let (findings_path, regressions_path) = match findings::write(&report, &out_dir) {
Ok(paths) => paths,
Err(e) => {
eprintln!("eval testbench: {e:#}");
std::process::exit(1);
}
};
print!("{}", findings::render_findings(&report));
println!("\nFINDINGS: {}", findings_path.display());
println!("regressions: {}", regressions_path.display());
if has_flag(args, "--gate") && !report.gate_passes() {
eprintln!("\ntestbench gate FAILED: {}", report.verdict.label());
std::process::exit(1);
}
}
fn run_testbench_or_exit(
lock: &TestbenchLock,
cache_dir: &Path,
runner: &dyn ModelRunner,
cfg: &TestbenchConfig,
) -> TestbenchReport {
match testbench::run_testbench(lock, cache_dir, runner, cfg) {
Ok(r) => r,
Err(e) => {
eprintln!("eval testbench: run failed: {e:#}");
std::process::exit(1);
}
}
}
fn cmd_verify(args: &[String]) {
let Some(path) = args.first() else {
eprintln!("eval verify: <artifact.json> is required");
std::process::exit(2);
};
let artifact = match artifact::load_artifact(Path::new(path)) {
Ok(a) => a,
Err(e) => {
eprintln!("eval verify: {e}");
std::process::exit(1);
}
};
let result = artifact.verify();
println!("Artifact: {path}");
println!("Verdict: {}", artifact.verdict.label());
println!("Determinism digest: {}", artifact.determinism_digest);
println!(
"Digest matches: {}",
if result.digest_matches { "yes" } else { "NO" }
);
println!(
"Signature valid: {}",
if result.signature_valid { "yes" } else { "NO" }
);
if let Some(pk) = &result.signer_public_key {
println!("Signer public key: {pk}");
}
if let Some(err) = &result.error {
println!("Error: {err}");
}
if result.ok() {
println!("\nOK — artifact is authentic and internally consistent.");
} else {
eprintln!("\nFAILED — artifact could not be verified.");
std::process::exit(1);
}
}
fn cmd_init(args: &[String]) {
let dir = PathBuf::from(args.first().map_or("eval-suite", |s| s.as_str()));
match write_starter_suite(&dir) {
Ok(suite) => {
println!("Starter suite written to {}", dir.display());
println!("Suite file: {}", suite.display());
println!("\nNext:");
println!(" # 1) record real model answers once (needs a live model in env)");
println!(
" lean-ctx eval ab --suite {} --record {}/recording.json",
suite.display(),
dir.display()
);
println!(" # 2) replay deterministically anywhere (CI)");
println!(
" lean-ctx eval ab --suite {} --replay {}/recording.json --gate",
suite.display(),
dir.display()
);
}
Err(e) => {
eprintln!("eval init: {e:#}");
std::process::exit(1);
}
}
}
fn write_starter_suite(dir: &Path) -> anyhow::Result<PathBuf> {
use anyhow::Context;
let corpus = dir.join("corpus");
let code = dir.join("code");
std::fs::create_dir_all(&corpus).context("creating corpus dir")?;
std::fs::create_dir_all(&code).context("creating code dir")?;
std::fs::write(
corpus.join("architecture.md"),
"# Consolidation pipeline\n\n\
Provider data flows through one consolidation pipeline. Artifacts are persisted to four \
stores: the BM25 index, the Graph index, ProjectKnowledge, and the Session cache. This is \
what lets semantic search, knowledge recall, and cross-source hints share one source of truth.\n",
)
.context("writing corpus/architecture.md")?;
std::fs::write(
corpus.join("overview.md"),
"# Overview\n\nlean-ctx is a context runtime for AI agents. This file is general \
background and intentionally does not list the consolidation stores.\n",
)
.context("writing corpus/overview.md")?;
std::fs::write(
code.join("test.sh"),
"#!/bin/sh\n. ./solution.sh\n[ \"$(add 2 3)\" = \"5\" ] || exit 1\n[ \"$(add 10 20)\" = \"30\" ] || exit 1\n",
)
.context("writing code/test.sh")?;
std::fs::write(
code.join("solution.sh"),
"# TODO: implement add() so that `add a b` prints a+b\nadd() { echo 0; }\n",
)
.context("writing code/solution.sh")?;
let suite = dir.join("suite.ndjson");
let lines = [
r#"{"id":"qa-consolidation-stores","domain":"qa","prompt":"Which four stores does the consolidation pipeline persist artifacts to?","workspace":"corpus","answers":["bm25 index, graph index, projectknowledge, session cache","bm25, graph, knowledge, session"]}"#,
r#"{"id":"code-add","domain":"code","prompt":"Implement the POSIX shell function add in solution.sh so that `add a b` prints the sum a+b. Output only the file contents.","workspace":"code","target_file":"solution.sh","test_cmd":"sh test.sh"}"#,
];
std::fs::write(
&suite,
format!("# lean-ctx eval starter suite\n{}\n", lines.join("\n")),
)
.context("writing suite.ndjson")?;
Ok(suite)
}
#[cfg(test)]
mod recording_guard_tests {
use super::*;
#[test]
fn committed_recording_replays_and_passes_gate() {
let dir = tempfile::tempdir().unwrap();
let suite_path = write_starter_suite(dir.path()).expect("scaffold starter suite");
let suite = EvalSuite::load(&suite_path).expect("load starter suite");
let rec_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("eval/recording.json");
assert!(
rec_path.exists(),
"committed recording missing at {} — CI quality-gate would silently skip",
rec_path.display()
);
let runner = RecordedRunner::from_file(&rec_path).expect("load committed recording");
let report = run_ab(&suite, "suite.ndjson", &runner, &AbRunConfig::default())
.expect("committed recording must cover every replay key");
assert!(
report.verdict.gate_passes(),
"committed recording must not encode a regression, got: {}",
report.verdict.label()
);
}
}