eval-magic 0.5.0

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

//! `run` — build the iteration workspace and dispatch plan (the default action).

use crate::cli::args::RunArgs;
use crate::cli::run;
use crate::cli::{parse_id_list, run_context_with_bootstrap};

/// Build the iteration workspace and dispatch plan (the default action).
pub(crate) fn run_run(args: RunArgs) -> anyhow::Result<()> {
    let ctx = run_context_with_bootstrap(&args.common, args.bootstrap.clone())?;
    let only = parse_id_list(args.common.only.as_deref());
    let skip = parse_id_list(args.common.skip.as_deref());
    run::orchestrate::command_run(
        &ctx,
        &run::orchestrate::RunOptions {
            mode: args.common.mode.as_deref(),
            baseline: args.baseline.as_deref(),
            only: only.as_deref(),
            skip: skip.as_deref(),
            iteration: args.common.iteration,
            dry_run: args.dry_run,
            no_stage: args.no_stage,
            guard: match (args.guard, args.no_guard) {
                (true, _) => Some(true),
                (_, true) => Some(false),
                _ => None,
            },
            stage_name: args.stage_name.as_deref(),
            plan_mode: args.plan_mode,
            runs: args.runs,
            agent_model: args.agent_model.as_deref(),
            judge_model: args.judge_model.as_deref(),
            label: args.label.as_deref(),
        },
    )?;
    Ok(())
}