aether_cli/eval/
mod.rs

1mod render;
2
3use crate::output::OutputFormat;
4use aether_evals::{
5    EvalFileError, EvalFilesReport, EvalSpec, EvalSpecLoadOptions, ResolvedEvalSpec, WorkspaceRetention, run_eval_specs,
6};
7use render::render;
8use std::env::current_dir;
9use std::fs::read_to_string;
10use std::io::{Read, stdin};
11use std::num::NonZeroUsize;
12use std::path::PathBuf;
13use std::process::ExitCode;
14
15const DEFAULT_MAX_CONCURRENCY: NonZeroUsize = NonZeroUsize::new(5).unwrap();
16
17#[derive(clap::Args)]
18pub struct EvalArgs {
19    /// Eval JSON files or directories to discover eval files under. Defaults to ./evals.
20    pub paths: Vec<PathBuf>,
21
22    /// Run only the eval with this name.
23    #[arg(long)]
24    pub name: Option<String>,
25
26    /// Maximum number of eval files to run concurrently.
27    #[arg(long, default_value_t = DEFAULT_MAX_CONCURRENCY)]
28    pub max_concurrency: NonZeroUsize,
29
30    /// Output format
31    #[arg(long, default_value = "text")]
32    pub output: OutputFormat,
33
34    /// Run a single eval from a JSON file, or from stdin with `-`. The outcome is printed to stdout as JSON.
35    #[arg(long, value_name = "PATH_OR_DASH")]
36    pub spec_file: Option<String>,
37
38    /// Retain the eval workspace on disk and include `retainedWorkspace` in JSON output. Used with
39    /// `--spec-file`.
40    #[arg(long)]
41    pub retain_workspace: bool,
42
43    /// Base directory for resolving relative paths in a `--spec-file` spec. Defaults to the current
44    /// working directory.
45    #[arg(long)]
46    pub base_dir: Option<PathBuf>,
47}
48
49pub async fn run(args: EvalArgs) -> Result<ExitCode, EvalFileError> {
50    if let Some(source) = args.spec_file {
51        let base_dir = args.base_dir.or_else(|| current_dir().ok()).unwrap_or_else(|| PathBuf::from("."));
52        let json = read_eval(&source)?;
53        let retention = if args.retain_workspace { WorkspaceRetention::Retain } else { WorkspaceRetention::Discard };
54        let spec: EvalSpec = serde_json::from_str(&json).map_err(|source| EvalFileError::ParseSpec { source })?;
55        let case = ResolvedEvalSpec::resolve(spec, base_dir)?;
56        let mut evals = run_eval_specs([case], retention, NonZeroUsize::MIN).await?;
57        let outcome = evals.pop().expect("one case in, one outcome out");
58        println!("{}", serde_json::to_string(&outcome).expect("EvalOutcome serializes to JSON"));
59        return Ok(ExitCode::SUCCESS);
60    }
61
62    let cases = ResolvedEvalSpec::load(EvalSpecLoadOptions { paths: args.paths, filter: args.name })?;
63    let evals = run_eval_specs(cases, WorkspaceRetention::Discard, args.max_concurrency).await?;
64    let report = EvalFilesReport { evals };
65
66    println!("{}", render(&report, args.output));
67    Ok(if report.passed() { ExitCode::SUCCESS } else { ExitCode::FAILURE })
68}
69
70fn read_eval(source: &str) -> Result<String, EvalFileError> {
71    if source == "-" {
72        let mut json = String::new();
73        stdin()
74            .read_to_string(&mut json)
75            .map_err(|source| EvalFileError::ReadEvalFile { path: PathBuf::from("-"), source })?;
76        return Ok(json);
77    }
78
79    let path = PathBuf::from(source);
80    read_to_string(&path).map_err(|source| EvalFileError::ReadEvalFile { path, source })
81}
aether_cli/eval/mod.rs

aether_cli/eval/
mod.rs
