eval-magic 0.3.4

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

//! Help output, `validate`, and parser-level dispatch (unknown subcommands).

use crate::helpers::skill_eval;
use predicates::str::contains;
use std::fs;
use tempfile::TempDir;

/// A minimal valid `evals.json` body.
const VALID_EVALS: &str = r#"{ "skill_name": "demo", "evals": [
    { "id": "e1", "prompt": "p", "expected_output": "o" } ] }"#;

/// Build `<root>/<skill>/evals/evals.json` with the given contents.
fn write_evals(root: &std::path::Path, skill: &str, contents: &str) {
    let dir = root.join(skill).join("evals");
    fs::create_dir_all(&dir).unwrap();
    fs::write(dir.join("evals.json"), contents).unwrap();
}

/// `--help` succeeds and lists the subcommands.
#[test]
fn help_lists_subcommands() {
    skill_eval()
        .arg("--help")
        .assert()
        .success()
        .stdout(contains("init"))
        .stdout(contains("record-runs"))
        .stdout(contains("grade"))
        .stdout(contains("validate"))
        .stdout(contains("aggregate"));
}

/// The binary name in help output is the published command name.
#[test]
fn help_uses_published_binary_name() {
    skill_eval()
        .arg("--help")
        .assert()
        .success()
        .stdout(contains("eval-magic"));
}

/// `ingest` reaches its own context validation when invoked bare.
#[test]
fn ingest_is_wired_and_validates_context() {
    skill_eval()
        .arg("ingest")
        .assert()
        .failure()
        .stderr(contains("--skill-dir"));
}

/// `validate` over a dir of valid evals succeeds and prints a ✓ per file.
#[test]
fn validate_succeeds_on_valid_evals() {
    let tmp = TempDir::new().unwrap();
    write_evals(tmp.path(), "good", VALID_EVALS);

    skill_eval()
        .arg("validate")
        .arg("--skill-dir")
        .arg(tmp.path())
        .assert()
        .success()
        .stdout(contains("✓ good/evals/evals.json"))
        .stdout(contains("Validated 1 evals.json file(s); 0 failed."));
}

#[test]
fn validate_defaults_to_current_skill_dir() {
    let tmp = TempDir::new().unwrap();
    write_evals(tmp.path(), "good", VALID_EVALS);
    fs::write(
        tmp.path().join("good").join("SKILL.md"),
        "---\nname: good\n---\nbody\n",
    )
    .unwrap();

    skill_eval()
        .current_dir(tmp.path().join("good"))
        .arg("validate")
        .assert()
        .success()
        .stdout(contains("✓ evals/evals.json"))
        .stdout(contains("Validated 1 evals.json file(s); 0 failed."));
}

#[test]
fn validate_accepts_a_skill_path() {
    let tmp = TempDir::new().unwrap();
    write_evals(tmp.path(), "good", VALID_EVALS);
    fs::write(
        tmp.path().join("good").join("SKILL.md"),
        "---\nname: good\n---\nbody\n",
    )
    .unwrap();

    skill_eval()
        .arg("validate")
        .arg("--skill")
        .arg(tmp.path().join("good"))
        .assert()
        .success()
        .stdout(contains("✓ evals/evals.json"));
}

/// `validate` exits non-zero and prints a ✗ when a file fails validation.
#[test]
fn validate_fails_on_invalid_evals() {
    let tmp = TempDir::new().unwrap();
    write_evals(tmp.path(), "bad", r#"{ "skill_name": "x", "evals": [] }"#);

    skill_eval()
        .arg("validate")
        .arg("--skill-dir")
        .arg(tmp.path())
        .assert()
        .failure()
        .stderr(contains("✗"));
}

/// `validate` without a detectable skill fails with our message.
#[test]
fn validate_requires_a_skill_context() {
    skill_eval()
        .arg("validate")
        .assert()
        .failure()
        .stderr(contains("missing skill"));
}

/// An unknown subcommand is rejected by the parser (clap), not silently
/// accepted.
#[test]
fn unknown_subcommand_is_rejected() {
    skill_eval().arg("does-not-exist").assert().failure();
}