eval-magic 0.3.4

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

//! Phase 1 — validate the request and resolve the iteration coordinates +
//! per-condition skill paths, before any directory is created.

use std::fs;

use serde_json::Value;

use crate::cli::command_target_args;
use crate::core::{Mode, RunContext};
use crate::validation::validate_evals_config;

use super::super::RunError;
use super::super::dispatch::select_evals;
use super::super::util::{
    condition_names_for, make_run_nonce, next_iteration, validate_harness_run_options,
};
use super::{Resolved, RunOptions};

pub(super) fn resolve_request(ctx: &RunContext, opts: &RunOptions) -> Result<Resolved, RunError> {
    let mode = match opts.mode {
        Some("new-skill") => Mode::NewSkill,
        Some("revision") => Mode::Revision,
        Some(other) => return Err(RunError::msg(format!("unknown --mode: {other}"))),
        None => Mode::NewSkill,
    };
    let baseline =
        (mode == Mode::Revision).then(|| opts.baseline.unwrap_or("baseline").to_string());
    if opts.runs == 0 {
        return Err(RunError::msg("--runs must be at least 1"));
    }
    validate_harness_run_options(opts, ctx)?;

    let skill_md_path = ctx.skill_subdir.join("SKILL.md");
    if !skill_md_path.exists() {
        return Err(RunError::msg(format!(
            "skill not found: {}",
            skill_md_path.display()
        )));
    }
    let skill_md = skill_md_path.to_string_lossy().into_owned();

    let evals_path = ctx.skill_subdir.join("evals").join("evals.json");
    if !evals_path.exists() {
        return Err(RunError::msg(format!(
            "evals.json not found: {}",
            evals_path.display()
        )));
    }
    let value: Value = serde_json::from_str(&fs::read_to_string(&evals_path)?)?;
    let config = validate_evals_config(&value, &evals_path.to_string_lossy())?;
    if config.skill_name != ctx.skill_name {
        eprintln!(
            "warning: evals.json skill_name ({}) does not match the skill folder ({}). Proceeding with {}.",
            config.skill_name, ctx.skill_name, ctx.skill_name
        );
    }

    let selected_evals = select_evals(&config.evals, opts.only, opts.skip)?;
    let total_evals = config.evals.len();

    let workspace_skill_dir = ctx.workspace_root.join(&ctx.skill_name);
    let iteration = next_iteration(&workspace_skill_dir, opts.iteration);
    let iteration_dir = workspace_skill_dir.join(format!("iteration-{iteration}"));
    let run_nonce = make_run_nonce();
    let run_tag = format!("i{iteration}-{run_nonce}");

    if iteration_dir.exists() && opts.iteration.is_none() {
        return Err(RunError::msg(format!(
            "iteration-{iteration} already exists; pass --iteration to overwrite explicitly"
        )));
    }

    let (cond_a, cond_b) = condition_names_for(mode);
    let (skill_path_a, skill_path_b): (Option<String>, Option<String>) = match mode {
        Mode::NewSkill => (Some(skill_md.clone()), None),
        Mode::Revision => {
            let baseline = baseline.as_deref().expect("revision baseline set above");
            let baseline_skill = workspace_skill_dir
                .join("snapshots")
                .join(baseline)
                .join("SKILL.md");
            if !baseline_skill.exists() {
                let target_args = command_target_args(ctx);
                return Err(RunError::msg(format!(
                    "baseline snapshot not found: {}\n  Run: eval-magic snapshot{target_args} --label {} (before editing)",
                    baseline_skill.display(),
                    baseline
                )));
            }
            (
                Some(baseline_skill.to_string_lossy().into_owned()),
                Some(skill_md.clone()),
            )
        }
    };

    Ok(Resolved {
        mode,
        baseline,
        skill_md_path,
        iteration,
        iteration_dir,
        run_nonce,
        run_tag,
        cond_a,
        cond_b,
        skill_path_a,
        skill_path_b,
        selected_evals,
        total_evals,
    })
}