eval-magic 0.6.1

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

//! Codex guard arguments across eval-agent and judge dispatch.

use crate::helpers::*;
use std::fs;
use std::path::Path;

#[test]
fn codex_guarded_eval_keeps_hook_bypass_while_judges_omit_it() {
    let tmp = tempfile::TempDir::new().unwrap();
    let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS);
    skill_eval()
        .current_dir(&cwd)
        .args(["run", "--skill-dir"])
        .arg(&skill_dir)
        .args(["--skill", "mr-review", "--harness", "codex"])
        .assert()
        .success();

    let runbook = read_str(&iteration_dir(&cwd).join("RUNBOOK.md"));
    let (eval_section, judge_and_after) = runbook
        .split_once("## 2. Dispatch the judge agents, then finalize")
        .expect("runbook has separate eval and judge sections");
    assert!(
        eval_section.contains("--dangerously-bypass-hook-trust"),
        "guarded eval agents must trust their staged hook: {eval_section}"
    );
    let judge_section = judge_and_after
        .split_once("## 3. Read the result")
        .map(|(section, _)| section)
        .unwrap_or(judge_and_after);
    assert!(
        !judge_section.contains("--dangerously-bypass-hook-trust"),
        "judges run outside guarded task envs: {judge_section}"
    );

    let dispatch = read_json(&iteration_dir(&cwd).join("dispatch.json"));
    for task in dispatch["tasks"].as_array().unwrap() {
        let outputs = Path::new(task["outputs_dir"].as_str().unwrap());
        fs::create_dir_all(outputs).unwrap();
        fs::write(outputs.join("final-message.md"), "Reviewed.\n").unwrap();
    }

    let ingest = skill_eval()
        .current_dir(&cwd)
        .args(["ingest", "--skill-dir"])
        .arg(&skill_dir)
        .args([
            "--skill",
            "mr-review",
            "--harness",
            "codex",
            "--iteration",
            "1",
        ])
        .assert()
        .success();
    let stdout = String::from_utf8(ingest.get_output().stdout.clone()).unwrap();
    assert!(stdout.contains("Dispatch each judge task"), "{stdout}");
    assert!(
        stdout.contains("codex --ask-for-approval never exec"),
        "{stdout}"
    );
    assert!(
        !stdout.contains("--dangerously-bypass-hook-trust"),
        "ingest judge recipe must not inherit task guard flags: {stdout}"
    );
}