eval-magic 0.6.0

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84

//! CLI-level command-check finalize behavior.

use crate::helpers::{canonical_root, skill_eval};
use serde_json::json;
use std::fs;

#[test]
fn finalize_folds_command_check_result_into_normal_pass_rate() {
    let (_tmp, root) = canonical_root();
    let skill_dir = root.join("skill-dir");
    let skill_sub = skill_dir.join("mr-review");
    fs::create_dir_all(skill_sub.join("evals")).unwrap();
    fs::write(
        skill_sub.join("SKILL.md"),
        "---\nname: mr-review\ndescription: review MRs\n---\n\nbody\n",
    )
    .unwrap();
    fs::write(
        skill_sub.join("evals/evals.json"),
        serde_json::to_string_pretty(&json!({
            "skill_name": "mr-review",
            "evals": [{
                "id": "pos-eval",
                "prompt": "Fix it.",
                "expected_output": "tests pass",
                "assertions": [{
                    "id": "held-out-tests",
                    "type": "command_check",
                    "command": "true"
                }]
            }]
        }))
        .unwrap(),
    )
    .unwrap();

    let cwd = root.join("work");
    let iteration_dir = cwd
        .join(".eval-magic")
        .join("mr-review")
        .join("iteration-1");
    let cond_dir = iteration_dir.join("eval-pos-eval").join("without_skill");
    fs::create_dir_all(cond_dir.join("command-checks")).unwrap();
    fs::write(
        iteration_dir.join("conditions.json"),
        serde_json::to_string(&json!({
            "mode": "new-skill",
            "conditions": [{"name": "without_skill", "skill_path": null}],
            "timestamp": "2026-06-08T00:00:00.000Z",
            "harness": "claude-code"
        }))
        .unwrap(),
    )
    .unwrap();
    fs::write(
        cond_dir.join("command-checks/held-out-tests.json"),
        serde_json::to_string(&json!({
            "id": "held-out-tests",
            "passed": true,
            "evidence": "exit code matched 0",
            "expected_exit_code": 0,
            "actual_exit_code": 0,
            "stdout": "",
            "stderr": ""
        }))
        .unwrap(),
    )
    .unwrap();

    skill_eval()
        .current_dir(&cwd)
        .args(["grade", "--skill-dir"])
        .arg(&skill_dir)
        .args(["--skill", "mr-review", "--iteration", "1", "--finalize"])
        .assert()
        .success();

    let grading: serde_json::Value =
        serde_json::from_str(&fs::read_to_string(cond_dir.join("grading.json")).unwrap()).unwrap();
    assert_eq!(grading["summary"]["pass_rate"], json!(1.0));
    assert_eq!(grading["assertion_results"][0]["id"], "held-out-tests");
    assert_eq!(grading["assertion_results"][0]["grader"], "command_check");
    assert_eq!(grading["assertion_results"][0]["confidence"], json!(1.0));
}