harn_cli/cli/
eval.rs

1//! Clap definitions for `harn eval` and its subcommands.
2//!
3//! The bare form `harn eval <path>` evaluates a run record, run directory,
4//! eval manifest, or `.harn` pipeline (legacy entrypoint, dispatched through
5//! `eval_run_record`). The `harn eval prompt <file> --fleet <models>`
6//! subcommand renders (and optionally runs / judges) a single
7//! `.harn.prompt` template against a fleet of models so authors can compare
8//! the wire envelope each capability profile materializes.
9
10use std::path::PathBuf;
11
12use clap::{Args, Subcommand, ValueEnum};
13
14const MAX_CODING_AGENT_REPLICATES: usize = 16;
15
16#[derive(Debug, Args)]
17#[command(args_conflicts_with_subcommands = true)]
18pub struct EvalArgs {
19    /// Run record path, run directory, eval manifest path, or `.harn` pipeline.
20    /// Required unless a subcommand (e.g. `prompt`) is used.
21    pub path: Option<String>,
22    /// Optional baseline run record for diffing.
23    #[arg(long)]
24    pub compare: Option<String>,
25    /// Run a pipeline twice and compare the baseline against this structural experiment.
26    #[arg(long = "structural-experiment")]
27    pub structural_experiment: Option<String>,
28    /// Replay LLM responses from a JSONL fixture file when `path` is a `.harn` pipeline.
29    #[arg(
30        long = "llm-mock",
31        value_name = "PATH",
32        conflicts_with = "llm_mock_record"
33    )]
34    pub llm_mock: Option<String>,
35    /// Record executed LLM responses into a JSONL fixture file when `path` is a `.harn` pipeline.
36    #[arg(
37        long = "llm-mock-record",
38        value_name = "PATH",
39        conflicts_with = "llm_mock"
40    )]
41    pub llm_mock_record: Option<String>,
42    /// Positional arguments forwarded to `harn run <pipeline.harn> -- ...` when
43    /// `path` is a pipeline file and `--structural-experiment` is set.
44    #[arg(last = true)]
45    pub argv: Vec<String>,
46    #[command(subcommand)]
47    pub command: Option<EvalCommand>,
48}
49
50#[derive(Debug, Subcommand)]
51pub enum EvalCommand {
52    /// Benchmark coding-agent fixtures across providers and tool formats.
53    CodingAgent(EvalCodingAgentArgs),
54    /// Run deterministic context-engineering modes over task fixtures.
55    Context(EvalContextArgs),
56    /// Render and optionally run a `.harn.prompt` across a fleet of models.
57    Prompt(EvalPromptArgs),
58    /// Gate skill/guidance variants on contamination-safe held-out eval results.
59    #[command(name = "skill-gate", visible_alias = "skill_gate")]
60    SkillGate(EvalSkillGateArgs),
61    /// Measure pre-turn scope-triage savings and false-positive rates.
62    #[command(name = "scope_triage", visible_alias = "scope-triage")]
63    ScopeTriage(EvalScopeTriageArgs),
64    /// Run tool-call accuracy, latency, and cost evals over a dataset.
65    ToolCalls(EvalToolCallsArgs),
66}
67
68#[derive(Debug, Args)]
69pub struct EvalContextArgs {
70    /// Context eval manifest JSON or TOML.
71    pub manifest: PathBuf,
72    /// Output directory for summary.json, per_run.jsonl, and summary.md.
73    #[arg(long)]
74    pub output: Option<PathBuf>,
75    /// Print the aggregate summary JSON to stdout.
76    #[arg(long)]
77    pub json: bool,
78}
79
80#[derive(Debug, Args)]
81pub struct EvalSkillGateArgs {
82    /// Skill gate manifest JSON or TOML.
83    pub manifest: PathBuf,
84    /// Output directory for summary.json, per_case.jsonl, receipt.json, and summary.md.
85    #[arg(long)]
86    pub output: Option<PathBuf>,
87    /// Print the aggregate report JSON to stdout.
88    #[arg(long)]
89    pub json: bool,
90}
91
92#[derive(Debug, Args)]
93pub struct EvalScopeTriageArgs {
94    /// Scope-triage dataset JSON.
95    #[arg(long, default_value = "evals/scope_triage/dataset.json")]
96    pub dataset: PathBuf,
97    /// Output directory for summary.json, per_case.jsonl, and summary.md.
98    #[arg(long)]
99    pub output: Option<PathBuf>,
100    /// Print the aggregate summary JSON to stdout.
101    #[arg(long)]
102    pub json: bool,
103    /// Run the live default classifier model instead of the deterministic reference classifier.
104    #[arg(long)]
105    pub live: bool,
106    /// Live classifier model selector.
107    #[arg(long, default_value = "ollama:qwen3:1.7b")]
108    pub model: String,
109    /// Confidence threshold below which labels become escalate.
110    #[arg(long = "confidence-threshold", default_value_t = 0.65)]
111    pub confidence_threshold: f64,
112    /// Stop after N cases, useful for smoke runs.
113    #[arg(long = "max-cases")]
114    pub max_cases: Option<usize>,
115}
116
117#[derive(Debug, Args)]
118pub struct EvalCodingAgentArgs {
119    /// Fixture ids to run (comma-separated, repeatable). Use `all` for the full suite.
120    #[arg(long = "fixture", value_delimiter = ',', default_value = "all")]
121    pub fixtures: Vec<String>,
122    /// Model selectors to run (comma-separated, repeatable). Each entry may be
123    /// an alias, `provider:model`, or `provider=...,model=...`.
124    #[arg(long = "model", value_delimiter = ',', default_value = "mock:mock")]
125    pub models: Vec<String>,
126    /// Tool-call rendering modes to compare.
127    #[arg(
128        long = "tool-format",
129        value_delimiter = ',',
130        default_value = "native,text"
131    )]
132    pub tool_formats: Vec<String>,
133    /// Output directory for summary.json, per_run.jsonl, transcripts, and markdown reports.
134    #[arg(long)]
135    pub output: Option<PathBuf>,
136    /// Optional .env file(s) to load for provider credentials. Values are never written to artifacts.
137    #[arg(long = "env-file")]
138    pub env_files: Vec<PathBuf>,
139    /// Append reachable local Ollama/llama.cpp/MLX/vLLM models to the selected matrix.
140    #[arg(long = "include-local")]
141    pub include_local: bool,
142    /// Restrict local discovery to one provider id. Repeatable.
143    #[arg(long = "local-provider")]
144    pub local_providers: Vec<String>,
145    /// Maximum discovered local models to append.
146    #[arg(long = "max-local-models", default_value_t = 2)]
147    pub max_local_models: usize,
148    /// Leave newly-loaded Ollama models running after each local benchmark run.
149    #[arg(long = "keep-local-after-run")]
150    pub keep_local_after_run: bool,
151    /// Stop after N matrix entries, useful for cost-capped smoke runs.
152    #[arg(long = "max-runs")]
153    pub max_runs: Option<usize>,
154    /// Number of independent trials for each fixture/model/tool-format cell.
155    #[arg(long = "replicates", default_value_t = 1, value_parser = parse_replicates)]
156    pub replicates: usize,
157    /// Maximum repair-agent loop iterations per run.
158    #[arg(long = "max-iterations", default_value_t = 8)]
159    pub max_iterations: usize,
160    /// Python executable used by the fixture and verification command.
161    #[arg(long, default_value = "python3")]
162    pub python: String,
163    /// Treat missing credentials as an error instead of skipping the run.
164    #[arg(long = "fail-on-unauthorized")]
165    pub fail_on_unauthorized: bool,
166    /// Print the aggregate summary JSON to stdout.
167    #[arg(long)]
168    pub json: bool,
169    /// Optional step_judge config applied to every run in this invocation.
170    /// Accepts a preset name (`symmetric-cheap`, `asymmetric`,
171    /// `symmetric-strong`) which expands to a known {model, provider}
172    /// pair, or `custom:<json>` for a literal JSON object passed verbatim
173    /// to `agent_loop({step_judge: ...})`. Omit (or pass `none` / `off`) to disable.
174    /// For matrix sweeps across presets, the step-judge experiment driver
175    /// at experiments/step-judge/run.sh invokes the eval runner once per
176    /// preset and aggregates.
177    #[arg(long = "step-judge")]
178    pub step_judge: Option<String>,
179    /// Override the on_veto remediation shape for the step-judge config
180    /// (`replace` or `retain`). Default is `replace`.
181    #[arg(long = "step-judge-on-veto")]
182    pub step_judge_on_veto: Option<String>,
183    /// Use the adversarial rubric variant.
184    #[arg(long = "step-judge-adversarial")]
185    pub step_judge_adversarial: bool,
186    /// Free-form reason attached when forcing a tool format against catalog guidance.
187    #[arg(long = "override-reason")]
188    pub override_reason: Option<String>,
189    /// Structural-validator config applied to every run in this invocation.
190    /// Omit to use the suite default (currently the 4-rule validator).
191    /// Accepts `on` / `default`, `off` / `none`, or `custom:<json>` for a
192    /// literal JSON object passed to `with_structural_validator(...)`.
193    #[arg(long = "structural-validator")]
194    pub structural_validator: Option<String>,
195    /// Free-form label persisted in summary.json for grouping repeat runs
196    /// (e.g. "replicate-1", "probe-judge-arch-gpt"). Defaults to empty.
197    #[arg(long = "run-label", default_value = "")]
198    pub run_label: String,
199    /// Path to a previous coding-agent `summary.json` (or its parent dir).
200    /// When present, the new summary embeds a `baseline_comparison` block
201    /// listing per-fixture regressions (baseline passed but this cell
202    /// failed) and recoveries (baseline failed but this cell passed),
203    /// plus aggregate counts and a net lift in percentage points.
204    /// Useful for cross-cell A/Bs (provider sweep, prompt change, step
205    /// judge on/off) where net pass-rate hides destructive interactions
206    /// like the cli-help-flag regression the step-judge experiment
207    /// surfaced (harn#2318).
208    #[arg(long = "baseline-comparison-against")]
209    pub baseline_comparison_against: Option<PathBuf>,
210}
211
212fn parse_replicates(raw: &str) -> Result<usize, String> {
213    let value = raw
214        .parse::<usize>()
215        .map_err(|_| "replicates must be a positive integer".to_string())?;
216    (1..=MAX_CODING_AGENT_REPLICATES)
217        .contains(&value)
218        .then_some(value)
219        .ok_or_else(|| format!("replicates must be between 1 and {MAX_CODING_AGENT_REPLICATES}"))
220}
221
222#[derive(Debug, Args)]
223pub struct EvalToolCallsArgs {
224    #[command(subcommand)]
225    pub command: Option<EvalToolCallsCommand>,
226    /// Dataset directory or JSON file. Directories prefer a `cases/` child.
227    #[arg(long, default_value = "conformance/tool-call-eval")]
228    pub dataset: PathBuf,
229    /// Planner model selector: alias, `provider:model`, or `provider=...,model=...`.
230    #[arg(long)]
231    pub planner: Option<String>,
232    /// Optional binder model selector. When set, a second model canonicalizes
233    /// the planner's response into a call/refusal decision before scoring.
234    #[arg(long)]
235    pub binder: Option<String>,
236    /// Judge model used only for predicate cases.
237    #[arg(long = "judge-model", default_value = "claude-opus-4-7")]
238    pub judge_model: String,
239    /// Output directory for `summary.json` and `per_case.jsonl`.
240    #[arg(long)]
241    pub output: Option<PathBuf>,
242    /// Override tool rendering for the planner (`native` or `text`).
243    #[arg(long = "tool-format")]
244    pub tool_format: Option<String>,
245    /// Maximum planner response tokens.
246    #[arg(long = "max-tokens", default_value_t = 512)]
247    pub max_tokens: i64,
248    /// Maximum binder response tokens. Default is sized to leave room for
249    /// reasoning-emitting models (e.g. GPT-OSS-120B emits ~200 tokens of
250    /// chain-of-thought before the JSON payload); non-reasoning binders
251    /// will under-fill this budget at no extra cost.
252    #[arg(long = "binder-max-tokens", default_value_t = 1024)]
253    pub binder_max_tokens: i64,
254    /// Run only cases whose id or tag contains this string.
255    #[arg(long)]
256    pub filter: Option<String>,
257    /// Stop after N selected cases, useful for smoke runs.
258    #[arg(long = "max-cases")]
259    pub max_cases: Option<usize>,
260    /// Treat missing credentials as an immediate preflight error.
261    #[arg(long = "fail-on-unauthorized")]
262    pub fail_on_unauthorized: bool,
263}
264
265#[derive(Debug, Subcommand)]
266pub enum EvalToolCallsCommand {
267    /// Compare a current summary against a pinned baseline.
268    RegressionCheck(EvalToolCallsRegressionArgs),
269}
270
271#[derive(Debug, Args)]
272pub struct EvalToolCallsRegressionArgs {
273    /// Current run summary. Defaults to `.harn-runs/tool-call-eval/latest/summary.json`.
274    #[arg(long)]
275    pub current: Option<PathBuf>,
276    /// Optional planner label for diagnostics.
277    #[arg(long)]
278    pub planner: Option<String>,
279    /// Baseline summary JSON to compare against.
280    #[arg(long)]
281    pub against: PathBuf,
282    /// Maximum allowed pass-rate drop in percentage points.
283    #[arg(long = "max-drop-pp", default_value_t = 2.0)]
284    pub max_drop_pp: f64,
285}
286
287#[derive(Debug, Args)]
288pub struct EvalPromptArgs {
289    /// Path to a `.harn.prompt` (or `.prompt`) template.
290    pub file: PathBuf,
291    /// Fleet of model selectors (comma-separated, repeatable).
292    /// Each entry is either a model alias (`claude-opus-4-7`) or a
293    /// `provider:model` selector (`ollama:qwen3.5`). Mutually exclusive
294    /// with `--fleet-name`.
295    #[arg(
296        long,
297        value_delimiter = ',',
298        required_unless_present = "fleet_name",
299        conflicts_with = "fleet_name"
300    )]
301    pub fleet: Vec<String>,
302    /// Named fleet from `harn.toml` `[eval.fleets.<name>]`.
303    #[arg(long = "fleet-name")]
304    pub fleet_name: Option<String>,
305    /// JSON file with bindings injected into the template scope.
306    #[arg(long)]
307    pub bindings: Option<PathBuf>,
308    /// Prompt context-quality fixture(s) that score artifact selection,
309    /// stale/noisy rejection, budget adherence, and logical-section shape.
310    #[arg(long = "context-fixture")]
311    pub context_fixture: Vec<PathBuf>,
312    /// Evaluation mode.
313    #[arg(long, value_enum, default_value_t = EvalPromptMode::Render)]
314    pub mode: EvalPromptMode,
315    /// Output format.
316    #[arg(long, value_enum, default_value_t = EvalPromptOutput::Terminal)]
317    pub output: EvalPromptOutput,
318    /// Output destination for HTML / JSON (defaults to stdout).
319    #[arg(long = "out-file", short = 'o')]
320    pub out_file: Option<PathBuf>,
321    /// Maximum concurrent model invocations in run/judge modes.
322    #[arg(long, default_value_t = 4)]
323    pub max_concurrent: usize,
324    /// Optional judge prompt template. When unset, a built-in equivalence
325    /// judge is used.
326    #[arg(long = "judge-template")]
327    pub judge_template: Option<PathBuf>,
328    /// Model used for `--mode judge` evaluation.
329    #[arg(long = "judge-model", default_value = "claude-opus-4-7")]
330    pub judge_model: String,
331    /// Maximum tokens for `--mode run` / `--mode judge` calls.
332    #[arg(long = "max-tokens", default_value_t = 1024)]
333    pub max_tokens: i64,
334    /// Treat unauthenticated providers as errors rather than skipping them.
335    #[arg(long = "fail-on-unauthorized")]
336    pub fail_on_unauthorized: bool,
337}
338
339#[derive(Debug, Clone, Copy, ValueEnum)]
340pub enum EvalPromptMode {
341    /// Render the template against each model's capability profile.
342    Render,
343    /// Render + execute against each model and collect outputs.
344    Run,
345    /// Render + run + LLM-as-judge equivalence scoring.
346    Judge,
347}
348
349#[derive(Debug, Clone, Copy, ValueEnum)]
350pub enum EvalPromptOutput {
351    Terminal,
352    Json,
353    Html,
354}
harn_cli/cli/eval.rs

harn_cli/cli/
eval.rs
