aether_cli/eval/
mod.rs

1mod render;
2
3use crate::output::OutputFormat;
4use aether_evals::{EvalFileError, EvalRunOptions, run_eval_files};
5use render::render;
6use std::num::NonZeroUsize;
7use std::path::PathBuf;
8use std::process::ExitCode;
9
10const DEFAULT_MAX_CONCURRENCY: NonZeroUsize = NonZeroUsize::new(5).unwrap();
11
12#[derive(clap::Args)]
13pub struct EvalArgs {
14    /// Eval JSON files or directories to discover eval files under. Defaults to ./evals.
15    pub paths: Vec<PathBuf>,
16
17    /// Run only the eval with this name.
18    #[arg(long)]
19    pub name: Option<String>,
20
21    /// Maximum number of eval files to run concurrently.
22    #[arg(long, default_value_t = DEFAULT_MAX_CONCURRENCY)]
23    pub max_concurrency: NonZeroUsize,
24
25    /// Output format
26    #[arg(long, default_value = "text")]
27    pub output: OutputFormat,
28}
29
30pub async fn run(args: EvalArgs) -> Result<ExitCode, EvalFileError> {
31    let report =
32        run_eval_files(EvalRunOptions { paths: args.paths, filter: args.name, max_concurrency: args.max_concurrency })
33            .await?;
34
35    println!("{}", render(&report, args.output));
36    Ok(if report.passed() { ExitCode::SUCCESS } else { ExitCode::FAILURE })
37}
aether_cli/eval/mod.rs

aether_cli/eval/
mod.rs
