eval-magic 0.5.0

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

//! Phase 1 — validate the request and resolve the iteration coordinates +
//! per-condition skill paths, before any directory is created.

use std::fs;

use serde_json::Value;

use crate::cli::command_target_args;
use crate::core::{Mode, RunContext};
use crate::validation::validate_evals_config;

use super::super::RunError;
use super::super::dispatch::select_evals;
use super::super::fixtures::fixture_pairs;
use super::super::grouping::{GroupInput, compute_groups};
use super::super::util::{condition_names_for, make_run_nonce, next_iteration};
use super::{Resolved, RunOptions};

pub(super) fn resolve_request(ctx: &RunContext, opts: &RunOptions) -> Result<Resolved, RunError> {
    let mode = match opts.mode {
        Some("new-skill") => Mode::NewSkill,
        Some("revision") => Mode::Revision,
        Some(other) => return Err(RunError::msg(format!("unknown --mode: {other}"))),
        None => Mode::NewSkill,
    };
    let baseline =
        (mode == Mode::Revision).then(|| opts.baseline.unwrap_or("baseline").to_string());
    if opts.runs == 0 {
        return Err(RunError::msg("--runs must be at least 1"));
    }

    let skill_md_path = ctx.skill_subdir.join("SKILL.md");
    if !skill_md_path.exists() {
        return Err(RunError::msg(format!(
            "skill not found: {}",
            skill_md_path.display()
        )));
    }
    let skill_md = skill_md_path.to_string_lossy().into_owned();

    let evals_path = ctx.skill_subdir.join("evals").join("evals.json");
    if !evals_path.exists() {
        return Err(RunError::msg(format!(
            "evals.json not found: {}",
            evals_path.display()
        )));
    }
    let value: Value = serde_json::from_str(&fs::read_to_string(&evals_path)?)?;
    let config = validate_evals_config(&value, &evals_path.to_string_lossy())?;
    if config.skill_name != ctx.skill_name {
        eprintln!(
            "warning: evals.json skill_name ({}) does not match the skill folder ({}). Proceeding with {}.",
            config.skill_name, ctx.skill_name, ctx.skill_name
        );
    }

    let selected_evals = select_evals(&config.evals, opts.only, opts.skip)?;
    let total_evals = config.evals.len();

    // Compute isolation groups up front (fixture-conflict + explicit hint), before
    // any env is staged: staging and dispatch both consume this plan. `fixture_pairs`
    // also fails fast here if a declared fixture is missing.
    let fixture_pairs_by_eval = selected_evals
        .iter()
        .map(|ev| fixture_pairs(ev, &ctx.skill_subdir))
        .collect::<Result<Vec<_>, _>>()?;
    let group_inputs: Vec<GroupInput> = selected_evals
        .iter()
        .zip(&fixture_pairs_by_eval)
        .map(|(ev, fixtures)| GroupInput {
            eval_id: &ev.id,
            isolation: ev.isolation,
            fixtures,
        })
        .collect();
    let groups = compute_groups(&group_inputs);

    let workspace_skill_dir = ctx.workspace_root.join(&ctx.skill_name);
    let iteration = next_iteration(&workspace_skill_dir, opts.iteration);
    let iteration_dir = workspace_skill_dir.join(format!("iteration-{iteration}"));
    let run_nonce = make_run_nonce();
    let run_tag = format!("i{iteration}-{run_nonce}");

    if iteration_dir.exists() && opts.iteration.is_none() {
        return Err(RunError::msg(format!(
            "iteration-{iteration} already exists; pass --iteration to overwrite explicitly"
        )));
    }

    let (cond_a, cond_b) = condition_names_for(mode);
    let (skill_path_a, skill_path_b): (Option<String>, Option<String>) = match mode {
        Mode::NewSkill => (Some(skill_md.clone()), None),
        Mode::Revision => {
            let baseline = baseline.as_deref().expect("revision baseline set above");
            let baseline_skill = workspace_skill_dir
                .join("snapshots")
                .join(baseline)
                .join("SKILL.md");
            if !baseline_skill.exists() {
                let target_args = command_target_args(ctx);
                return Err(RunError::msg(format!(
                    "baseline snapshot not found: {}\n  Run: eval-magic snapshot{target_args} --label {} (before editing)",
                    baseline_skill.display(),
                    baseline
                )));
            }
            (
                Some(baseline_skill.to_string_lossy().into_owned()),
                Some(skill_md.clone()),
            )
        }
    };

    Ok(Resolved {
        mode,
        baseline,
        skill_md_path,
        iteration,
        iteration_dir,
        run_nonce,
        run_tag,
        cond_a,
        cond_b,
        skill_path_a,
        skill_path_b,
        selected_evals,
        total_evals,
        groups,
    })
}