aether_cli/eval/
mod.rs

1mod render;
2
3use crate::output::OutputFormat;
4use aether_evals::{
5    EvalFileError, EvalFilesReport, EvalSpec, EvalSpecLoadOptions, EvalStreamEvent, ResolvedEvalSpec,
6    WorkspaceRetention, run_eval_spec_streaming, run_eval_specs,
7};
8use render::render;
9use std::env::current_dir;
10use std::fs::read_to_string;
11use std::io::{Read, Write, stdin, stdout};
12use std::num::NonZeroUsize;
13use std::path::PathBuf;
14use std::process::ExitCode;
15
16const DEFAULT_MAX_CONCURRENCY: NonZeroUsize = NonZeroUsize::new(5).unwrap();
17
18#[derive(clap::Args)]
19pub struct EvalArgs {
20    /// Eval JSON files or directories to discover eval files under. Defaults to ./evals.
21    pub paths: Vec<PathBuf>,
22
23    /// Run only the eval with this name.
24    #[arg(long)]
25    pub name: Option<String>,
26
27    /// Maximum number of eval files to run concurrently.
28    #[arg(long, default_value_t = DEFAULT_MAX_CONCURRENCY)]
29    pub max_concurrency: NonZeroUsize,
30
31    /// Output format
32    #[arg(long, default_value = "text")]
33    pub output: OutputFormat,
34
35    /// Run a single eval from a JSON file, or from stdin with `-`. Emits newline-delimited
36    /// `EvalStreamEvent` JSON on stdout: an `agent_message` event per agent message as it streams,
37    /// terminated by one `outcome` event.
38    #[arg(long, value_name = "PATH_OR_DASH")]
39    pub spec_file: Option<String>,
40
41    /// Retain the eval workspace on disk and include `retainedWorkspace` in JSON output. Used with
42    /// `--spec-file`.
43    #[arg(long)]
44    pub retain_workspace: bool,
45
46    /// Base directory for resolving relative paths in a `--spec-file` spec. Defaults to the current
47    /// working directory.
48    #[arg(long)]
49    pub base_dir: Option<PathBuf>,
50}
51
52pub async fn run(args: EvalArgs) -> Result<ExitCode, EvalFileError> {
53    if let Some(source) = args.spec_file {
54        let base_dir = args.base_dir.or_else(|| current_dir().ok()).unwrap_or_else(|| PathBuf::from("."));
55        let json = read_eval(&source)?;
56        let retention = if args.retain_workspace { WorkspaceRetention::Retain } else { WorkspaceRetention::Discard };
57        let spec: EvalSpec = serde_json::from_str(&json).map_err(|source| EvalFileError::ParseSpec { source })?;
58        let case = ResolvedEvalSpec::resolve(spec, base_dir)?;
59        let outcome = run_eval_spec_streaming(case, retention, |message| {
60            print_json_event(&EvalStreamEvent::AgentMessage { message: message.clone() });
61        })
62        .await?;
63        print_json_event(&EvalStreamEvent::Outcome { outcome });
64        return Ok(ExitCode::SUCCESS);
65    }
66
67    let cases = ResolvedEvalSpec::load(EvalSpecLoadOptions { paths: args.paths, filter: args.name })?;
68    let evals = run_eval_specs(cases, WorkspaceRetention::Discard, args.max_concurrency).await?;
69    let report = EvalFilesReport { evals };
70
71    println!("{}", render(&report, args.output));
72    Ok(if report.passed() { ExitCode::SUCCESS } else { ExitCode::FAILURE })
73}
74
75fn print_json_event(event: &EvalStreamEvent) {
76    println!("{}", serde_json::to_string(event).expect("EvalStreamEvent serializes to JSON"));
77    let _ = stdout().flush();
78}
79
80fn read_eval(source: &str) -> Result<String, EvalFileError> {
81    if source == "-" {
82        let mut json = String::new();
83        stdin()
84            .read_to_string(&mut json)
85            .map_err(|source| EvalFileError::ReadEvalFile { path: PathBuf::from("-"), source })?;
86        return Ok(json);
87    }
88
89    let path = PathBuf::from(source);
90    read_to_string(&path).map_err(|source| EvalFileError::ReadEvalFile { path, source })
91}
aether_cli/eval/mod.rs

aether_cli/eval/
mod.rs
