harn_cli/cli/eval.rs
1//! Clap definitions for `harn eval` and its subcommands.
2//!
3//! The bare form `harn eval <path>` evaluates a run record, run directory,
4//! eval manifest, or `.harn` pipeline (legacy entrypoint, dispatched through
5//! `eval_run_record`). The `harn eval prompt <file> --fleet <models>`
6//! subcommand renders (and optionally runs / judges) a single
7//! `.harn.prompt` template against a fleet of models so authors can compare
8//! the wire envelope each capability profile materializes.
9
10use std::path::PathBuf;
11
12use clap::{Args, Subcommand, ValueEnum};
13
14#[derive(Debug, Args)]
15#[command(args_conflicts_with_subcommands = true)]
16pub struct EvalArgs {
17 /// Run record path, run directory, eval manifest path, or `.harn` pipeline.
18 /// Required unless a subcommand (e.g. `prompt`) is used.
19 pub path: Option<String>,
20 /// Optional baseline run record for diffing.
21 #[arg(long)]
22 pub compare: Option<String>,
23 /// Run a pipeline twice and compare the baseline against this structural experiment.
24 #[arg(long = "structural-experiment")]
25 pub structural_experiment: Option<String>,
26 /// Replay LLM responses from a JSONL fixture file when `path` is a `.harn` pipeline.
27 #[arg(
28 long = "llm-mock",
29 value_name = "PATH",
30 conflicts_with = "llm_mock_record"
31 )]
32 pub llm_mock: Option<String>,
33 /// Record executed LLM responses into a JSONL fixture file when `path` is a `.harn` pipeline.
34 #[arg(
35 long = "llm-mock-record",
36 value_name = "PATH",
37 conflicts_with = "llm_mock"
38 )]
39 pub llm_mock_record: Option<String>,
40 /// Positional arguments forwarded to `harn run <pipeline.harn> -- ...` when
41 /// `path` is a pipeline file and `--structural-experiment` is set.
42 #[arg(last = true)]
43 pub argv: Vec<String>,
44 #[command(subcommand)]
45 pub command: Option<EvalCommand>,
46}
47
48#[derive(Debug, Subcommand)]
49pub enum EvalCommand {
50 /// Benchmark coding-agent fixtures across providers and tool formats.
51 CodingAgent(EvalCodingAgentArgs),
52 /// Run deterministic context-engineering modes over task fixtures.
53 Context(EvalContextArgs),
54 /// Render and optionally run a `.harn.prompt` across a fleet of models.
55 Prompt(EvalPromptArgs),
56 /// Run tool-call accuracy, latency, and cost evals over a dataset.
57 ToolCalls(EvalToolCallsArgs),
58}
59
60#[derive(Debug, Args)]
61pub struct EvalContextArgs {
62 /// Context eval manifest JSON or TOML.
63 pub manifest: PathBuf,
64 /// Output directory for summary.json, per_run.jsonl, and summary.md.
65 #[arg(long)]
66 pub output: Option<PathBuf>,
67 /// Print the aggregate summary JSON to stdout.
68 #[arg(long)]
69 pub json: bool,
70}
71
72#[derive(Debug, Args)]
73pub struct EvalCodingAgentArgs {
74 /// Fixture ids to run (comma-separated, repeatable). Use `all` for the full suite.
75 #[arg(long = "fixture", value_delimiter = ',', default_value = "all")]
76 pub fixtures: Vec<String>,
77 /// Model selectors to run (comma-separated, repeatable). Each entry may be
78 /// an alias, `provider:model`, or `provider=...,model=...`.
79 #[arg(long = "model", value_delimiter = ',', default_value = "mock:mock")]
80 pub models: Vec<String>,
81 /// Tool-call rendering modes to compare.
82 #[arg(
83 long = "tool-format",
84 value_delimiter = ',',
85 default_value = "native,text"
86 )]
87 pub tool_formats: Vec<String>,
88 /// Output directory for summary.json, per_run.jsonl, transcripts, and markdown reports.
89 #[arg(long)]
90 pub output: Option<PathBuf>,
91 /// Optional .env file(s) to load for provider credentials. Values are never written to artifacts.
92 #[arg(long = "env-file")]
93 pub env_files: Vec<PathBuf>,
94 /// Append reachable local Ollama/llama.cpp/MLX/vLLM models to the selected matrix.
95 #[arg(long = "include-local")]
96 pub include_local: bool,
97 /// Restrict local discovery to one provider id. Repeatable.
98 #[arg(long = "local-provider")]
99 pub local_providers: Vec<String>,
100 /// Maximum discovered local models to append.
101 #[arg(long = "max-local-models", default_value_t = 2)]
102 pub max_local_models: usize,
103 /// Leave newly-loaded Ollama models running after each local benchmark run.
104 #[arg(long = "keep-local-after-run")]
105 pub keep_local_after_run: bool,
106 /// Stop after N matrix entries, useful for cost-capped smoke runs.
107 #[arg(long = "max-runs")]
108 pub max_runs: Option<usize>,
109 /// Maximum repair-agent loop iterations per run.
110 #[arg(long = "max-iterations", default_value_t = 8)]
111 pub max_iterations: usize,
112 /// Python executable used by the fixture and verification command.
113 #[arg(long, default_value = "python3")]
114 pub python: String,
115 /// Treat missing credentials as an error instead of skipping the run.
116 #[arg(long = "fail-on-unauthorized")]
117 pub fail_on_unauthorized: bool,
118 /// Print the aggregate summary JSON to stdout.
119 #[arg(long)]
120 pub json: bool,
121 /// Optional step_judge config applied to every run in this invocation.
122 /// Accepts a preset name (`symmetric-cheap`, `asymmetric`,
123 /// `symmetric-strong`) which expands to a known {model, provider}
124 /// pair, or `custom:<json>` for a literal JSON object passed verbatim
125 /// to `agent_loop({step_judge: ...})`. Omit (or pass `none`) to disable.
126 /// For matrix sweeps across presets, the step-judge experiment driver
127 /// at experiments/step-judge/run.sh invokes the eval runner once per
128 /// preset and aggregates.
129 #[arg(long = "step-judge")]
130 pub step_judge: Option<String>,
131 /// Override the on_veto remediation shape for the step-judge config
132 /// (`replace` or `retain`). Default is `replace`.
133 #[arg(long = "step-judge-on-veto")]
134 pub step_judge_on_veto: Option<String>,
135 /// Use the adversarial rubric variant.
136 #[arg(long = "step-judge-adversarial")]
137 pub step_judge_adversarial: bool,
138 /// Free-form label persisted in summary.json for grouping repeat runs
139 /// (e.g. "replicate-1", "probe-judge-arch-gpt"). Defaults to empty.
140 #[arg(long = "run-label", default_value = "")]
141 pub run_label: String,
142 /// Path to a previous coding-agent `summary.json` (or its parent dir).
143 /// When present, the new summary embeds a `baseline_comparison` block
144 /// listing per-fixture regressions (baseline passed but this cell
145 /// failed) and recoveries (baseline failed but this cell passed),
146 /// plus aggregate counts and a net lift in percentage points.
147 /// Useful for cross-cell A/Bs (provider sweep, prompt change, step
148 /// judge on/off) where net pass-rate hides destructive interactions
149 /// like the cli-help-flag regression the step-judge experiment
150 /// surfaced (harn#2318).
151 #[arg(long = "baseline-comparison-against")]
152 pub baseline_comparison_against: Option<PathBuf>,
153}
154
155#[derive(Debug, Args)]
156pub struct EvalToolCallsArgs {
157 #[command(subcommand)]
158 pub command: Option<EvalToolCallsCommand>,
159 /// Dataset directory or JSON file. Directories prefer a `cases/` child.
160 #[arg(long, default_value = "conformance/tool-call-eval")]
161 pub dataset: PathBuf,
162 /// Planner model selector: alias, `provider:model`, or `provider=...,model=...`.
163 #[arg(long)]
164 pub planner: Option<String>,
165 /// Optional binder model selector. When set, a second model canonicalizes
166 /// the planner's response into a call/refusal decision before scoring.
167 #[arg(long)]
168 pub binder: Option<String>,
169 /// Judge model used only for predicate cases.
170 #[arg(long = "judge-model", default_value = "claude-opus-4-7")]
171 pub judge_model: String,
172 /// Output directory for `summary.json` and `per_case.jsonl`.
173 #[arg(long)]
174 pub output: Option<PathBuf>,
175 /// Override tool rendering for the planner (`native` or `text`).
176 #[arg(long = "tool-format")]
177 pub tool_format: Option<String>,
178 /// Maximum planner response tokens.
179 #[arg(long = "max-tokens", default_value_t = 512)]
180 pub max_tokens: i64,
181 /// Maximum binder response tokens. Default is sized to leave room for
182 /// reasoning-emitting models (e.g. GPT-OSS-120B emits ~200 tokens of
183 /// chain-of-thought before the JSON payload); non-reasoning binders
184 /// will under-fill this budget at no extra cost.
185 #[arg(long = "binder-max-tokens", default_value_t = 1024)]
186 pub binder_max_tokens: i64,
187 /// Run only cases whose id or tag contains this string.
188 #[arg(long)]
189 pub filter: Option<String>,
190 /// Stop after N selected cases, useful for smoke runs.
191 #[arg(long = "max-cases")]
192 pub max_cases: Option<usize>,
193 /// Treat missing credentials as an immediate preflight error.
194 #[arg(long = "fail-on-unauthorized")]
195 pub fail_on_unauthorized: bool,
196}
197
198#[derive(Debug, Subcommand)]
199pub enum EvalToolCallsCommand {
200 /// Compare a current summary against a pinned baseline.
201 RegressionCheck(EvalToolCallsRegressionArgs),
202}
203
204#[derive(Debug, Args)]
205pub struct EvalToolCallsRegressionArgs {
206 /// Current run summary. Defaults to `.harn-runs/tool-call-eval/latest/summary.json`.
207 #[arg(long)]
208 pub current: Option<PathBuf>,
209 /// Optional planner label for diagnostics.
210 #[arg(long)]
211 pub planner: Option<String>,
212 /// Baseline summary JSON to compare against.
213 #[arg(long)]
214 pub against: PathBuf,
215 /// Maximum allowed pass-rate drop in percentage points.
216 #[arg(long = "max-drop-pp", default_value_t = 2.0)]
217 pub max_drop_pp: f64,
218}
219
220#[derive(Debug, Args)]
221pub struct EvalPromptArgs {
222 /// Path to a `.harn.prompt` (or `.prompt`) template.
223 pub file: PathBuf,
224 /// Fleet of model selectors (comma-separated, repeatable).
225 /// Each entry is either a model alias (`claude-opus-4-7`) or a
226 /// `provider:model` selector (`ollama:qwen3.5`). Mutually exclusive
227 /// with `--fleet-name`.
228 #[arg(
229 long,
230 value_delimiter = ',',
231 required_unless_present = "fleet_name",
232 conflicts_with = "fleet_name"
233 )]
234 pub fleet: Vec<String>,
235 /// Named fleet from `harn.toml` `[eval.fleets.<name>]`.
236 #[arg(long = "fleet-name")]
237 pub fleet_name: Option<String>,
238 /// JSON file with bindings injected into the template scope.
239 #[arg(long)]
240 pub bindings: Option<PathBuf>,
241 /// Prompt context-quality fixture(s) that score artifact selection,
242 /// stale/noisy rejection, budget adherence, and logical-section shape.
243 #[arg(long = "context-fixture")]
244 pub context_fixture: Vec<PathBuf>,
245 /// Evaluation mode.
246 #[arg(long, value_enum, default_value_t = EvalPromptMode::Render)]
247 pub mode: EvalPromptMode,
248 /// Output format.
249 #[arg(long, value_enum, default_value_t = EvalPromptOutput::Terminal)]
250 pub output: EvalPromptOutput,
251 /// Output destination for HTML / JSON (defaults to stdout).
252 #[arg(long = "out-file", short = 'o')]
253 pub out_file: Option<PathBuf>,
254 /// Maximum concurrent model invocations in run/judge modes.
255 #[arg(long, default_value_t = 4)]
256 pub max_concurrent: usize,
257 /// Optional judge prompt template. When unset, a built-in equivalence
258 /// judge is used.
259 #[arg(long = "judge-template")]
260 pub judge_template: Option<PathBuf>,
261 /// Model used for `--mode judge` evaluation.
262 #[arg(long = "judge-model", default_value = "claude-opus-4-7")]
263 pub judge_model: String,
264 /// Maximum tokens for `--mode run` / `--mode judge` calls.
265 #[arg(long = "max-tokens", default_value_t = 1024)]
266 pub max_tokens: i64,
267 /// Treat unauthenticated providers as errors rather than skipping them.
268 #[arg(long = "fail-on-unauthorized")]
269 pub fail_on_unauthorized: bool,
270}
271
272#[derive(Debug, Clone, Copy, ValueEnum)]
273pub enum EvalPromptMode {
274 /// Render the template against each model's capability profile.
275 Render,
276 /// Render + execute against each model and collect outputs.
277 Run,
278 /// Render + run + LLM-as-judge equivalence scoring.
279 Judge,
280}
281
282#[derive(Debug, Clone, Copy, ValueEnum)]
283pub enum EvalPromptOutput {
284 Terminal,
285 Json,
286 Html,
287}