harn_cli/commands/
eval_skill_gate.rs

1//! `harn eval skill-gate` - contamination-safe gate reports for skill/guidance candidates.
2
3use std::fs;
4use std::io::Write as _;
5use std::path::{Path, PathBuf};
6
7use harn_vm::orchestration::{
8    evaluate_skill_gate_manifest, load_skill_gate_manifest, SkillGateCaseReport, SkillGateReport,
9    SkillGateVariantReport,
10};
11
12use crate::cli::EvalSkillGateArgs;
13use crate::format::escape_md;
14
15pub async fn run(args: EvalSkillGateArgs) -> i32 {
16    let manifest = match load_skill_gate_manifest(&args.manifest) {
17        Ok(manifest) => manifest,
18        Err(error) => {
19            eprintln!("error: {error}");
20            return 1;
21        }
22    };
23    let report = match evaluate_skill_gate_manifest(&manifest) {
24        Ok(report) => report,
25        Err(error) => {
26            eprintln!("error: {error}");
27            return 1;
28        }
29    };
30    let output_dir = args.output.unwrap_or_else(|| default_output_dir(&report));
31    if let Err(error) = fs::create_dir_all(&output_dir) {
32        eprintln!("error: failed to create {}: {error}", output_dir.display());
33        return 1;
34    }
35    if let Err(error) = write_outputs(&output_dir, &report) {
36        eprintln!("error: failed to write skill gate outputs: {error}");
37        return 1;
38    }
39    eprintln!(
40        "wrote {}, {}, {}, and {}",
41        output_dir.join("summary.json").display(),
42        output_dir.join("per_case.jsonl").display(),
43        output_dir.join("receipt.json").display(),
44        output_dir.join("summary.md").display()
45    );
46    if args.json {
47        match serde_json::to_string_pretty(&report) {
48            Ok(payload) => println!("{payload}"),
49            Err(error) => {
50                eprintln!("error: failed to serialize skill gate report: {error}");
51                return 1;
52            }
53        }
54    } else {
55        println!(
56            "skill gate: {} selected={} variants={} included={} excluded={} tamper={}",
57            if report.pass { "PASS" } else { "FAIL" },
58            report.selected_variant_id.as_deref().unwrap_or("none"),
59            report.variants.len(),
60            report.included_task_count,
61            report.excluded_task_count,
62            if report.tamper.pass { "pass" } else { "fail" }
63        );
64    }
65    i32::from(!report.pass)
66}
67
68fn default_output_dir(report: &SkillGateReport) -> PathBuf {
69    Path::new(".harn-runs")
70        .join("skill-gate")
71        .join(&report.manifest_id)
72}
73
74fn write_outputs(output_dir: &Path, report: &SkillGateReport) -> Result<(), String> {
75    write_json(output_dir.join("summary.json"), report)?;
76    write_per_case(output_dir.join("per_case.jsonl"), report)?;
77    write_json(output_dir.join("receipt.json"), &report.receipt)?;
78    fs::write(output_dir.join("summary.md"), render_markdown(report))
79        .map_err(|error| error.to_string())
80}
81
82fn write_json<T: serde::Serialize>(path: PathBuf, value: &T) -> Result<(), String> {
83    let payload = serde_json::to_string_pretty(value).map_err(|error| error.to_string())?;
84    fs::write(path, payload).map_err(|error| error.to_string())
85}
86
87fn write_per_case(path: PathBuf, report: &SkillGateReport) -> Result<(), String> {
88    let mut file = fs::File::create(path).map_err(|error| error.to_string())?;
89    for variant in &report.variants {
90        for case in &variant.cases {
91            let line = serde_json::to_string(&PerCaseLine {
92                variant_id: &variant.id,
93                accepted: variant.accepted,
94                case,
95            })
96            .map_err(|error| error.to_string())?;
97            file.write_all(line.as_bytes())
98                .map_err(|error| error.to_string())?;
99            file.write_all(b"\n").map_err(|error| error.to_string())?;
100        }
101    }
102    Ok(())
103}
104
105#[derive(serde::Serialize)]
106struct PerCaseLine<'a> {
107    variant_id: &'a str,
108    accepted: bool,
109    #[serde(flatten)]
110    case: &'a SkillGateCaseReport,
111}
112
113fn render_markdown(report: &SkillGateReport) -> String {
114    let mut out = String::new();
115    out.push_str(&format!("# Skill Gate: {}\n\n", report.manifest_id));
116    out.push_str(&format!(
117        "- status: {}\n- target model: `{}`\n- selected variant: `{}`\n- included tasks: {}\n- excluded tasks: {}\n- tamper: {}\n- pareto frontier: {}\n\n",
118        if report.pass { "PASS" } else { "FAIL" },
119        escape_md(&report.target_model.id),
120        escape_md(report.selected_variant_id.as_deref().unwrap_or("none")),
121        report.included_task_count,
122        report.excluded_task_count,
123        if report.tamper.pass { "pass" } else { "fail" },
124        if report.pareto_frontier.is_empty() {
125            "none".to_string()
126        } else {
127            report.pareto_frontier.join(", ")
128        }
129    ));
130    out.push_str(
131        "| variant | decision | lift | gap recovery | regressions | context delta | failures |\n",
132    );
133    out.push_str("|---|---|---:|---:|---:|---:|---|\n");
134    for variant in &report.variants {
135        out.push_str(&variant_row(variant));
136    }
137    if !report.task_safety.is_empty() {
138        out.push_str("\n## Held-out Filter\n\n");
139        out.push_str("| task | cluster | included | reason |\n");
140        out.push_str("|---|---|---:|---|\n");
141        for task in &report.task_safety {
142            out.push_str(&format!(
143                "| {} | {} | {} | {} |\n",
144                escape_md(&task.task_id),
145                escape_md(&task.cluster),
146                if task.included { "yes" } else { "no" },
147                escape_md(task.exclusion_reason.as_deref().unwrap_or(""))
148            ));
149        }
150    }
151    if !report.tamper.checks.is_empty() {
152        out.push_str("\n## Immutable Grader Checks\n\n");
153        out.push_str("| path | status | actual sha256 |\n");
154        out.push_str("|---|---|---|\n");
155        for check in &report.tamper.checks {
156            out.push_str(&format!(
157                "| {} | {} | `{}` |\n",
158                escape_md(&check.path),
159                escape_md(&check.status),
160                check.actual_sha256.as_deref().unwrap_or("")
161            ));
162        }
163    }
164    out
165}
166
167fn variant_row(variant: &SkillGateVariantReport) -> String {
168    format!(
169        "| {} | {} | {:.4} | {:.4} | {}/{} | {} | {} |\n",
170        escape_md(&variant.id),
171        if variant.accepted {
172            "accepted"
173        } else {
174            "rejected"
175        },
176        variant.metrics.mean_score_lift,
177        variant.metrics.mean_gap_recovery,
178        variant.metrics.regression_count,
179        variant.metrics.regression_denominator,
180        variant.context.delta_tokens,
181        escape_md(&variant.failures.join("; "))
182    )
183}
harn_cli/commands/eval_skill_gate.rs

harn_cli/commands/
eval_skill_gate.rs
