lean_ctx/cli/
eval_cmd.rs

1//! `lean-ctx eval` — deterministic with/without output-quality eval CLI (#238).
2//!
3//! Subcommands:
4//! * `eval init <dir>`     — scaffold a runnable starter suite (one QA + one code task).
5//! * `eval ab --suite P`   — run the A/B comparison and write a signed, reproducible artifact.
6//! * `eval verify <file>`  — verify an artifact's signature + determinism digest offline.
7
8use std::path::{Path, PathBuf};
9
10use crate::core::eval_ab::artifact::{self, SignedAbReportV1};
11use crate::core::eval_ab::model::{OpenAiRunner, RecordedRunner, RecordingRunner};
12use crate::core::eval_ab::report::ReportConfig;
13use crate::core::eval_ab::suite::EvalSuite;
14use crate::core::eval_ab::{run_ab, AbRunConfig};
15
16/// Entry point dispatched from `cli::dispatch`.
17pub fn cmd_eval(args: &[String]) {
18    match args.first().map(String::as_str) {
19        Some("ab") => cmd_ab(&args[1..]),
20        Some("verify") => cmd_verify(&args[1..]),
21        Some("init") => cmd_init(&args[1..]),
22        Some("-h" | "--help") | None => print_help(),
23        Some(other) => {
24            eprintln!("eval: unknown subcommand '{other}'\n");
25            print_help();
26            std::process::exit(2);
27        }
28    }
29}
30
31fn print_help() {
32    println!(
33        "lean-ctx eval — deterministic with/without output-quality proof\n\n\
34USAGE:\n\
35  lean-ctx eval init <dir>                 Scaffold a runnable starter suite\n\
36  lean-ctx eval ab --suite <file> [opts]   Run the A/B quality comparison\n\
37  lean-ctx eval verify <artifact.json>     Verify signature + determinism digest\n\n\
38ab OPTIONS:\n\
39  --suite <file>     NDJSON suite (required)\n\
40  --budget <n>       Token budget per condition (default 4000)\n\
41  --margin <f>       Non-inferiority margin for the gate (default 0.0)\n\
42  --out <file>       Artifact path (default: data dir)\n\
43  --replay <file>    Replay a recording instead of calling a live model (deterministic CI)\n\
44  --record <file>    Call the live model and save responses to a recording\n\
45  --gate             Exit non-zero if the verdict is a regression\n\n\
46LIVE MODEL (when not replaying) is read from the environment:\n\
47  LEAN_CTX_EVAL_MODEL_URL   OpenAI-compatible base URL (e.g. https://api.openai.com/v1)\n\
48  LEAN_CTX_EVAL_MODEL       Model id (e.g. gpt-4o-mini)\n\
49  LEAN_CTX_EVAL_MODEL_KEY   API key (optional for local servers)\n\
50  LEAN_CTX_EVAL_SEED        Decoding seed (default 7)"
51    );
52}
53
54/// Returns the value following `flag` in `args`, if present.
55fn flag_value<'a>(args: &'a [String], flag: &str) -> Option<&'a str> {
56    args.iter()
57        .position(|a| a == flag)
58        .and_then(|i| args.get(i + 1))
59        .map(String::as_str)
60}
61
62fn has_flag(args: &[String], flag: &str) -> bool {
63    args.iter().any(|a| a == flag)
64}
65
66fn cmd_ab(args: &[String]) {
67    let Some(suite_path) = flag_value(args, "--suite") else {
68        eprintln!("eval ab: --suite <file> is required");
69        std::process::exit(2);
70    };
71    let suite_path = PathBuf::from(suite_path);
72    let suite = match EvalSuite::load(&suite_path) {
73        Ok(s) => s,
74        Err(e) => {
75            eprintln!("eval ab: {e:#}");
76            std::process::exit(1);
77        }
78    };
79    let suite_name = suite_path
80        .file_name()
81        .map_or_else(|| "suite".to_string(), |s| s.to_string_lossy().into_owned());
82
83    let mut cfg = AbRunConfig::default();
84    if let Some(b) = flag_value(args, "--budget").and_then(|v| v.parse().ok()) {
85        cfg.budget_tokens = b;
86    }
87    cfg.report = ReportConfig {
88        noninferiority_margin: flag_value(args, "--margin")
89            .and_then(|v| v.parse().ok())
90            .unwrap_or(0.0),
91        ..ReportConfig::default()
92    };
93
94    // Runner selection: replay (deterministic) > live + record > live.
95    let report = if let Some(replay) = flag_value(args, "--replay") {
96        let runner = match RecordedRunner::from_file(Path::new(replay)) {
97            Ok(r) => r,
98            Err(e) => {
99                eprintln!("eval ab: {e:#}");
100                std::process::exit(1);
101            }
102        };
103        run_or_exit(&suite, &suite_name, &runner, &cfg)
104    } else {
105        let live = match OpenAiRunner::from_env() {
106            Ok(r) => r,
107            Err(e) => {
108                eprintln!("eval ab: no live model configured: {e:#}\n(use --replay <file> for an offline run)");
109                std::process::exit(1);
110            }
111        };
112        if let Some(record_path) = flag_value(args, "--record") {
113            let recorder = RecordingRunner::new(live);
114            let report = run_or_exit(&suite, &suite_name, &recorder, &cfg);
115            if let Err(e) = recorder.into_recording().save(Path::new(record_path)) {
116                eprintln!("eval ab: failed to save recording: {e:#}");
117                std::process::exit(1);
118            }
119            println!("Recording saved → {record_path}");
120            report
121        } else {
122            run_or_exit(&suite, &suite_name, &live, &cfg)
123        }
124    };
125
126    // Sign + persist the artifact.
127    let agent_id = crate::core::agent_identity::current_agent_id().to_string();
128    let mut signed = SignedAbReportV1::from_report(report, &agent_id);
129    if let Err(e) = signed.sign(&agent_id) {
130        eprintln!("eval ab: signing failed: {e}");
131        std::process::exit(1);
132    }
133    let out = match flag_value(args, "--out") {
134        Some(p) => PathBuf::from(p),
135        None => match artifact::default_artifact_path() {
136            Ok(p) => p,
137            Err(e) => {
138                eprintln!("eval ab: {e}");
139                std::process::exit(1);
140            }
141        },
142    };
143    if let Err(e) = artifact::write_artifact(&signed, &out) {
144        eprintln!("eval ab: {e}");
145        std::process::exit(1);
146    }
147
148    println!("{}", signed.report.render());
149    println!("determinism digest: {}", signed.determinism_digest);
150    println!("artifact:           {}", out.display());
151
152    if has_flag(args, "--gate") && !signed.verdict.gate_passes() {
153        eprintln!("\nquality gate FAILED: {}", signed.verdict.label());
154        std::process::exit(1);
155    }
156}
157
158fn run_or_exit(
159    suite: &EvalSuite,
160    suite_name: &str,
161    runner: &dyn crate::core::eval_ab::model::ModelRunner,
162    cfg: &AbRunConfig,
163) -> crate::core::eval_ab::report::AbReport {
164    match run_ab(suite, suite_name, runner, cfg) {
165        Ok(r) => r,
166        Err(e) => {
167            eprintln!("eval ab: run failed: {e:#}");
168            std::process::exit(1);
169        }
170    }
171}
172
173fn cmd_verify(args: &[String]) {
174    let Some(path) = args.first() else {
175        eprintln!("eval verify: <artifact.json> is required");
176        std::process::exit(2);
177    };
178    let artifact = match artifact::load_artifact(Path::new(path)) {
179        Ok(a) => a,
180        Err(e) => {
181            eprintln!("eval verify: {e}");
182            std::process::exit(1);
183        }
184    };
185    let result = artifact.verify();
186    println!("Artifact:           {path}");
187    println!("Verdict:            {}", artifact.verdict.label());
188    println!("Determinism digest: {}", artifact.determinism_digest);
189    println!(
190        "Digest matches:     {}",
191        if result.digest_matches { "yes" } else { "NO" }
192    );
193    println!(
194        "Signature valid:    {}",
195        if result.signature_valid { "yes" } else { "NO" }
196    );
197    if let Some(pk) = &result.signer_public_key {
198        println!("Signer public key:  {pk}");
199    }
200    if let Some(err) = &result.error {
201        println!("Error:              {err}");
202    }
203    if result.ok() {
204        println!("\nOK — artifact is authentic and internally consistent.");
205    } else {
206        eprintln!("\nFAILED — artifact could not be verified.");
207        std::process::exit(1);
208    }
209}
210
211fn cmd_init(args: &[String]) {
212    let dir = PathBuf::from(args.first().map_or("eval-suite", |s| s.as_str()));
213    match write_starter_suite(&dir) {
214        Ok(suite) => {
215            println!("Starter suite written to {}", dir.display());
216            println!("Suite file: {}", suite.display());
217            println!("\nNext:");
218            println!("  # 1) record real model answers once (needs a live model in env)");
219            println!(
220                "  lean-ctx eval ab --suite {} --record {}/recording.json",
221                suite.display(),
222                dir.display()
223            );
224            println!("  # 2) replay deterministically anywhere (CI)");
225            println!(
226                "  lean-ctx eval ab --suite {} --replay {}/recording.json --gate",
227                suite.display(),
228                dir.display()
229            );
230        }
231        Err(e) => {
232            eprintln!("eval init: {e:#}");
233            std::process::exit(1);
234        }
235    }
236}
237
238/// Materializes a small, runnable starter suite: one RAG/QA task whose answer lives in the
239/// corpus, and one POSIX-shell code task with a failing stub + unit test.
240fn write_starter_suite(dir: &Path) -> anyhow::Result<PathBuf> {
241    use anyhow::Context;
242    let corpus = dir.join("corpus");
243    let code = dir.join("code");
244    std::fs::create_dir_all(&corpus).context("creating corpus dir")?;
245    std::fs::create_dir_all(&code).context("creating code dir")?;
246
247    std::fs::write(
248        corpus.join("architecture.md"),
249        "# Consolidation pipeline\n\n\
250Provider data flows through one consolidation pipeline. Artifacts are persisted to four \
251stores: the BM25 index, the Graph index, ProjectKnowledge, and the Session cache. This is \
252what lets semantic search, knowledge recall, and cross-source hints share one source of truth.\n",
253    )
254    .context("writing corpus/architecture.md")?;
255    std::fs::write(
256        corpus.join("overview.md"),
257        "# Overview\n\nlean-ctx is a context runtime for AI agents. This file is general \
258background and intentionally does not list the consolidation stores.\n",
259    )
260    .context("writing corpus/overview.md")?;
261
262    std::fs::write(
263        code.join("test.sh"),
264        "#!/bin/sh\n. ./solution.sh\n[ \"$(add 2 3)\" = \"5\" ] || exit 1\n[ \"$(add 10 20)\" = \"30\" ] || exit 1\n",
265    )
266    .context("writing code/test.sh")?;
267    std::fs::write(
268        code.join("solution.sh"),
269        "# TODO: implement add() so that `add a b` prints a+b\nadd() { echo 0; }\n",
270    )
271    .context("writing code/solution.sh")?;
272
273    let suite = dir.join("suite.ndjson");
274    let lines = [
275        r#"{"id":"qa-consolidation-stores","domain":"qa","prompt":"Which four stores does the consolidation pipeline persist artifacts to?","workspace":"corpus","answers":["bm25 index, graph index, projectknowledge, session cache","bm25, graph, knowledge, session"]}"#,
276        r#"{"id":"code-add","domain":"code","prompt":"Implement the POSIX shell function add in solution.sh so that `add a b` prints the sum a+b. Output only the file contents.","workspace":"code","target_file":"solution.sh","test_cmd":"sh test.sh"}"#,
277    ];
278    std::fs::write(
279        &suite,
280        format!("# lean-ctx eval starter suite\n{}\n", lines.join("\n")),
281    )
282    .context("writing suite.ndjson")?;
283    Ok(suite)
284}
lean_ctx/cli/eval_cmd.rs

lean_ctx/cli/
eval_cmd.rs
