eval-magic 0.5.0

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

//! Shared fixtures and helpers for the `run` integration tests.

use std::fs;
use std::path::{Path, PathBuf};

use assert_cmd::Command;
use serde_json::Value;

pub const STAGED_MANIFEST: &str = ".slow-powers-eval-manifest.json";
pub const DEFAULT_EVALS: &str = r#"{ "skill_name": "mr-review", "evals": [ { "id": "e1", "prompt": "review this MR", "expected_output": "a review" } ] }"#;

pub fn skill_eval() -> Command {
    let mut cmd = Command::cargo_bin("eval-magic").expect("binary `eval-magic` should build");
    // Disable user-global descriptor discovery so a developer's
    // ~/.config/eval-magic/harnesses never leaks into the tests.
    cmd.env("EVAL_MAGIC_CONFIG_DIR", "");
    cmd
}

/// Build `<root>/skill-dir/mr-review/{SKILL.md,evals/evals.json}` and a `work`
/// cwd; returns `(skill_dir, cwd)`.
pub fn setup(root: &Path, evals_json: &str) -> (PathBuf, PathBuf) {
    let skill_dir = root.join("skill-dir");
    let skill_sub = skill_dir.join("mr-review");
    fs::create_dir_all(skill_sub.join("evals")).unwrap();
    fs::write(
        skill_sub.join("SKILL.md"),
        "---\nname: mr-review\ndescription: review merge requests\n---\n\nbody\n",
    )
    .unwrap();
    fs::write(skill_sub.join("evals").join("evals.json"), evals_json).unwrap();
    let cwd = root.join("work");
    fs::create_dir_all(&cwd).unwrap();
    (skill_dir, cwd)
}

pub fn iteration_dir(cwd: &Path) -> PathBuf {
    cwd.join(".eval-magic")
        .join("mr-review")
        .join("iteration-1")
}

/// A per-`(group, condition)` env dir — the cwd each `claude -p`/`codex exec`
/// subprocess runs from: `iteration-N/env-<group>-<condition>/`. Each holds only
/// that condition's skill (or none, for the control arm) and its group's fixtures.
/// Staging, fixtures, and the guard marker all land under here, below
/// `iteration_dir`; `RUNBOOK.md` lives above it in `iteration_dir`.
pub fn cli_env_dir(cwd: &Path, group: &str, condition: &str) -> PathBuf {
    iteration_dir(cwd).join(format!("env-{group}-{condition}"))
}

/// Staged skill names under the default single-group `with_skill` env's harness
/// skills dir (`env-g1-with_skill/.claude/skills`), excluding the staging
/// manifest, sorted.
pub fn env_staged_entries(cwd: &Path) -> Vec<String> {
    staged_entries(&cli_env_dir(cwd, "g1", "with_skill").join(".claude/skills"))
}

pub fn read_json(path: &Path) -> Value {
    serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap()
}

pub fn read_str(path: &Path) -> String {
    fs::read_to_string(path).unwrap()
}

/// Names directly under `.claude/skills` (or `.agents/skills`), excluding
/// dotfiles (the staging manifest and the guard marker), sorted.
pub fn staged_entries(skills_dir: &Path) -> Vec<String> {
    let mut names: Vec<String> = fs::read_dir(skills_dir)
        .unwrap()
        .map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
        .filter(|n| !n.starts_with('.'))
        .collect();
    names.sort();
    names
}