eval-magic 0.4.0

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151

//! The hidden `guard` PreToolUse hook entry point and `teardown-guard`.

use crate::helpers::skill_eval;
use predicates::prelude::*;
use predicates::str::contains;
use std::fs;
use tempfile::TempDir;

/// The internal `guard` hook entry point is hidden from `--help` (its unique
/// description never appears) yet remains callable.
#[test]
fn guard_subcommand_is_hidden_but_callable() {
    skill_eval()
        .arg("--help")
        .assert()
        .success()
        .stdout(contains("PreToolUse hook entry point").not());

    skill_eval().arg("guard").arg("--help").assert().success();
}

/// Write an armed guard marker scoping writes to `<allowed>`, and return its path.
fn write_armed_marker(root: &std::path::Path, allowed: &std::path::Path) -> std::path::PathBuf {
    let skills = root.join(".claude").join("skills");
    fs::create_dir_all(&skills).unwrap();
    let marker = skills.join(".slow-powers-eval-guard.json");
    fs::write(
        &marker,
        format!(
            r#"{{ "active": true, "allowedRoots": ["{}"], "expiresAt": "2999-01-01T00:00:00.000Z" }}"#,
            allowed.display()
        ),
    )
    .unwrap();
    marker
}

fn write_codex_armed_marker(
    root: &std::path::Path,
    allowed: &std::path::Path,
) -> std::path::PathBuf {
    let skills = root.join(".agents").join("skills");
    fs::create_dir_all(&skills).unwrap();
    let marker = skills.join(".slow-powers-eval-guard.json");
    fs::write(
        &marker,
        format!(
            r#"{{ "active": true, "allowedRoots": ["{}"], "expiresAt": "2999-01-01T00:00:00.000Z" }}"#,
            allowed.display()
        ),
    )
    .unwrap();
    marker
}

/// `guard` denies a Write outside the sandbox: it prints a PreToolUse deny verdict
/// on stdout and still exits 0 (the hook must never fail the session).
#[test]
fn guard_denies_out_of_bounds_write() {
    let tmp = TempDir::new().unwrap();
    let marker = write_armed_marker(tmp.path(), &tmp.path().join(".eval-magic"));

    skill_eval()
        .arg("guard")
        .arg(&marker)
        .write_stdin(r#"{ "tool_name": "Write", "tool_input": { "file_path": "/etc/passwd" } }"#)
        .assert()
        .success()
        .stdout(contains(r#""permissionDecision":"deny""#));
}

/// `guard` allows an in-bounds write: empty stdout, exit 0.
#[test]
fn guard_allows_in_bounds_write() {
    let tmp = TempDir::new().unwrap();
    let workspace = tmp.path().join(".eval-magic");
    let marker = write_armed_marker(tmp.path(), &workspace);

    skill_eval()
        .arg("guard")
        .arg(&marker)
        .write_stdin(format!(
            r#"{{ "tool_name": "Write", "tool_input": {{ "file_path": "{}/out.md" }} }}"#,
            workspace.display()
        ))
        .assert()
        .success()
        .stdout("");
}

#[test]
fn guard_codex_subcommand_blocks_with_codex_verdict_shape() {
    let tmp = TempDir::new().unwrap();
    let marker = write_codex_armed_marker(tmp.path(), &tmp.path().join(".eval-magic"));

    skill_eval()
        .arg("guard-codex")
        .arg(&marker)
        .write_stdin(
            r#"{ "tool_name": "Bash", "tool_input": { "command": "npm install left-pad" } }"#,
        )
        .assert()
        .success()
        .stdout(contains(r#""decision":"block""#))
        .stdout(contains("blocked Bash"));
}

/// `guard` fails open when the marker is absent: empty stdout, exit 0.
#[test]
fn guard_fails_open_without_marker() {
    let tmp = TempDir::new().unwrap();
    skill_eval()
        .arg("guard")
        .arg(tmp.path().join("nope.json"))
        .write_stdin(r#"{ "tool_name": "Write", "tool_input": { "file_path": "/etc/passwd" } }"#)
        .assert()
        .success()
        .stdout("");
}

/// `teardown-guard` reports when no guard is installed (cwd has no marker).
#[test]
fn teardown_guard_reports_nothing_to_remove() {
    let tmp = TempDir::new().unwrap();
    skill_eval()
        .arg("teardown-guard")
        .current_dir(tmp.path())
        .assert()
        .success()
        .stdout(contains("nothing to remove"));
}

/// `teardown-guard` sweeps a stray marker in the cwd and reports removal.
#[test]
fn teardown_guard_removes_installed_guard() {
    let tmp = TempDir::new().unwrap();
    write_armed_marker(tmp.path(), &tmp.path().join(".eval-magic"));

    skill_eval()
        .arg("teardown-guard")
        .current_dir(tmp.path())
        .assert()
        .success()
        .stdout(contains("Write guard removed"));

    assert!(
        !tmp.path()
            .join(".claude/skills/.slow-powers-eval-guard.json")
            .exists()
    );
}