harn_cli/commands/
eval_context.rs

1//! `harn eval context` — deterministic context-engineering mode runner.
2//!
3//! ## .harn dispatch
4//!
5//! The **evaluation pipeline** (manifest load, `evaluate_context_eval_manifest`
6//! invocation, per-run scoring) stays in Rust — it reaches into
7//! `harn_vm::orchestration::context_eval` internals (mode runs,
8//! projection policies, scoring) that aren't reachable from script-land
9//! today without G4 (#2297) exposing the orchestration surface.
10//!
11//! The **rendering layer** (the markdown body of `summary.md`, the
12//! one-line human summary, the `--json` pretty form) is delegated to
13//! `crates/harn-stdlib/src/stdlib/cli/eval/context.harn`. The Rust shim
14//! pre-serialises the `ContextEvalReport` to JSON, forwards it via
15//! [`CONTEXT_REPORT_ENV`], dispatches three times (markdown for
16//! `summary.md`, summary for stdout, optional JSON for stdout when
17//! `--json` is set), and writes the captured payloads to disk / real
18//! stdout. The artifacts that need byte-identical serde output
19//! (`summary.json`, `per_run.jsonl`) stay on the Rust side because
20//! Harn's `json_stringify_pretty` sorts dict keys alphabetically and
21//! the on-disk format is consumed by the regression-check / hosted
22//! ingestion paths that depend on the serde struct-field order.
23//!
24use std::fs;
25use std::io::Write as _;
26use std::path::{Path, PathBuf};
27
28use harn_vm::orchestration::{
29    context_eval_default_output_dir, evaluate_context_eval_manifest, load_context_eval_manifest,
30    ContextEvalReport, ContextEvalRunReport,
31};
32
33use crate::cli::EvalContextArgs;
34use crate::dispatch;
35use crate::env_guard::ScopedEnvVar;
36
37/// Env var the embedded `cli/eval/context` script reads to pick up the
38/// pre-serialised `ContextEvalReport`. The Rust shim does all the
39/// pipeline work and hands the script the assembled report so it only
40/// has to format it.
41const CONTEXT_REPORT_ENV: &str = "HARN_EVAL_CONTEXT_REPORT_JSON";
42
43/// Env var the script reads to select between the three rendering
44/// modes (`"markdown"` for `summary.md`, `"summary"` for the one-line
45/// stdout summary, `"json"` for the `--json` pretty form).
46const CONTEXT_OUTPUT_MODE_ENV: &str = "HARN_EVAL_CONTEXT_OUTPUT_MODE";
47
48/// Serialises the dispatch-render path so concurrent in-process callers
49/// (the existing `eval_context_cli` integration tests, plus any future
50/// fanout caller) don't race on the process-global env vars the Rust
51/// shim sets to hand the report off to the .harn script. The CLI binary
52/// itself is single-call, so this mutex is uncontended in production;
53/// in tests it serialises the dispatch window only — aggregation still
54/// runs freely.
55///
56/// Mirrors the pattern W5's `eval_prompt.rs` uses (see harn#2305) so
57/// the cross-script env-var hand-off stays consistent across the eval
58/// cluster.
59static DISPATCH_RENDER_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
60
61pub async fn run(args: EvalContextArgs) -> i32 {
62    let report = match aggregate_report(&args) {
63        Ok(report) => report,
64        Err(code) => return code,
65    };
66
67    let output_dir = args.output.unwrap_or_else(context_eval_default_output_dir);
68    if let Err(error) = fs::create_dir_all(&output_dir) {
69        eprintln!("error: failed to create {}: {error}", output_dir.display());
70        return 1;
71    }
72
73    // The JSON artifacts (summary.json, per_run.jsonl) always stay on the
74    // serde-driven Rust path — see module docstring for the byte-format
75    // rationale. They write before any rendering so a render failure
76    // doesn't leave a partially-written report directory.
77    if let Err(error) = write_json_artifacts(&output_dir, &report) {
78        eprintln!("error: failed to write context eval outputs: {error}");
79        return 1;
80    }
81
82    match write_markdown_dispatch(&output_dir, &report).await {
83        Ok(()) => {}
84        Err(code) => return code,
85    }
86    announce_output_paths(&output_dir);
87    if args.json {
88        if let Err(code) = print_json_dispatch(&report).await {
89            return code;
90        }
91    } else if let Err(code) = print_summary_dispatch(&report).await {
92        return code;
93    }
94    post_render_exit_code(&report)
95}
96
97/// Build the aggregated [`ContextEvalReport`] without any rendering.
98fn aggregate_report(args: &EvalContextArgs) -> Result<ContextEvalReport, i32> {
99    let manifest = match load_context_eval_manifest(&args.manifest) {
100        Ok(manifest) => manifest,
101        Err(error) => {
102            eprintln!("error: {error}");
103            return Err(1);
104        }
105    };
106    let report = match evaluate_context_eval_manifest(&manifest) {
107        Ok(report) => report,
108        Err(error) => {
109            eprintln!("error: {error}");
110            return Err(1);
111        }
112    };
113    Ok(report)
114}
115
116fn post_render_exit_code(report: &ContextEvalReport) -> i32 {
117    i32::from(!report.pass)
118}
119
120fn announce_output_paths(output_dir: &Path) {
121    eprintln!(
122        "wrote {}, {}, and {}",
123        output_dir.join("summary.json").display(),
124        output_dir.join("per_run.jsonl").display(),
125        output_dir.join("summary.md").display()
126    );
127}
128
129fn write_json_artifacts(output_dir: &Path, report: &ContextEvalReport) -> Result<(), String> {
130    write_json(output_dir.join("summary.json"), report)?;
131    write_jsonl(output_dir.join("per_run.jsonl"), &report.runs)
132}
133
134fn write_json(path: PathBuf, report: &ContextEvalReport) -> Result<(), String> {
135    let payload = serde_json::to_string_pretty(report).map_err(|error| error.to_string())?;
136    fs::write(path, payload).map_err(|error| error.to_string())
137}
138
139fn write_jsonl(path: PathBuf, runs: &[ContextEvalRunReport]) -> Result<(), String> {
140    let mut file = fs::File::create(path).map_err(|error| error.to_string())?;
141    for run in runs {
142        let line = serde_json::to_string(run).map_err(|error| error.to_string())?;
143        file.write_all(line.as_bytes())
144            .map_err(|error| error.to_string())?;
145        file.write_all(b"\n").map_err(|error| error.to_string())?;
146    }
147    Ok(())
148}
149
150// ─── Dispatch (.harn) render path ────────────────────────────────────────
151
152async fn write_markdown_dispatch(output_dir: &Path, report: &ContextEvalReport) -> Result<(), i32> {
153    let payload = render_via_dispatch(report, "markdown").await?;
154    if let Err(error) = fs::write(output_dir.join("summary.md"), payload) {
155        eprintln!("error: failed to write context eval markdown: {error}");
156        return Err(1);
157    }
158    Ok(())
159}
160
161async fn print_summary_dispatch(report: &ContextEvalReport) -> Result<(), i32> {
162    let payload = render_via_dispatch(report, "summary").await?;
163    print!("{payload}");
164    if !payload.ends_with('\n') {
165        println!();
166    }
167    Ok(())
168}
169
170async fn print_json_dispatch(report: &ContextEvalReport) -> Result<(), i32> {
171    let payload = render_via_dispatch(report, "json").await?;
172    print!("{payload}");
173    if !payload.ends_with('\n') {
174        println!();
175    }
176    Ok(())
177}
178
179/// Dispatch to the embedded `cli/eval/context.harn` script for one of
180/// the three rendering modes (markdown / summary / json). Returns the
181/// captured stdout on success, or a propagated exit code on failure.
182///
183/// **Concurrency.** Held under [`DISPATCH_RENDER_LOCK`] so concurrent
184/// in-process callers don't race on the global env vars the Rust shim
185/// sets to hand the report to the script. See the lock's docstring for
186/// the trade-off rationale.
187async fn render_via_dispatch(report: &ContextEvalReport, mode: &str) -> Result<String, i32> {
188    let report_json = match serde_json::to_string(report) {
189        Ok(json) => json,
190        Err(error) => {
191            eprintln!("error: failed to serialise ContextEvalReport for dispatch: {error}");
192            return Err(1);
193        }
194    };
195
196    let _guard = DISPATCH_RENDER_LOCK.lock().await;
197    let _report = ScopedEnvVar::set(CONTEXT_REPORT_ENV, &report_json);
198    let _mode = ScopedEnvVar::set(CONTEXT_OUTPUT_MODE_ENV, mode);
199
200    let outcome = dispatch::run_embedded_script("eval/context", Vec::new(), false).await;
201    if !outcome.stderr.is_empty() {
202        use std::io::Write as _;
203        let _ = std::io::stderr().write_all(outcome.stderr.as_bytes());
204    }
205    if outcome.exit_code != 0 {
206        return Err(outcome.exit_code);
207    }
208    Ok(outcome.stdout)
209}
harn_cli/commands/eval_context.rs

harn_cli/commands/
eval_context.rs
