eval-magic 0.3.2

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

//! Codex CLI command rendering for `DispatchMechanism::Cli` guidance.

fn shell_quote_arg(value: &str) -> String {
    if value.bytes().all(|b| {
        b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.' | b'/' | b':' | b'@' | b'+')
    }) {
        return value.to_string();
    }
    format!("'{}'", value.replace('\'', "'\"'\"'"))
}

fn render_cli_model_arg(flag: Option<&str>, model: Option<&str>) -> String {
    let Some(model) = model.filter(|m| !m.trim().is_empty()) else {
        return String::new();
    };
    let Some(flag) = flag else {
        return String::new();
    };
    format!(" {flag} {}", shell_quote_arg(model))
}

/// Copy/pasteable Codex dispatch command template. Stdin is detached so a
/// surrounding `xargs`/pipe cannot be treated as extra prompt context.
pub(crate) fn codex_exec_command_template(
    model_flag: Option<&str>,
    guard: bool,
    agent_model: Option<&str>,
) -> String {
    let hook_trust = if guard {
        " --dangerously-bypass-hook-trust"
    } else {
        ""
    };
    let model_arg = render_cli_model_arg(model_flag, agent_model);
    [
        format!(
            "codex exec --cd <eval-root> --sandbox workspace-write --ask-for-approval never{hook_trust}{model_arg} --json \\"
        ),
        "  --output-last-message <outputs_dir>/final-message.md \\".to_string(),
        "  \"Read the file at <dispatch_prompt_path> and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to <outputs_dir>/final-message.md.\" \\".to_string(),
        "  </dev/null \\".to_string(),
        "  > <outputs_dir>/codex-events.jsonl \\".to_string(),
        "  2> <outputs_dir>/codex-stderr.log".to_string(),
    ]
    .join("\n")
}

pub(crate) fn codex_parallel_dispatch_recipe(
    model_flag: Option<&str>,
    guard: bool,
    agent_model: Option<&str>,
) -> String {
    let hook_trust = if guard {
        " --dangerously-bypass-hook-trust"
    } else {
        ""
    };
    let model_arg = render_cli_model_arg(model_flag, agent_model);
    [
        "JOBS=${JOBS:-4}".to_string(),
        "jq -j '.tasks[] | [.dispatch_prompt_path, .outputs_dir] | @tsv + \"\\u0000\"' dispatch.json | \\".to_string(),
        "  xargs -0 -P \"$JOBS\" -I{} sh -c '".to_string(),
        "    prompt_path=\"$(printf \"%s\" \"$1\" | cut -f1)\"".to_string(),
        "    outputs_dir=\"$(printf \"%s\" \"$1\" | cut -f2)\"".to_string(),
        "    mkdir -p \"$outputs_dir\"".to_string(),
        format!(
            "    codex exec --cd <eval-root> --sandbox workspace-write --ask-for-approval never{hook_trust}{model_arg} --json \\"
        ),
        "      --output-last-message \"$outputs_dir/final-message.md\" \\".to_string(),
        "      \"Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to $outputs_dir/final-message.md.\" \\".to_string(),
        "      </dev/null \\".to_string(),
        "      > \"$outputs_dir/codex-events.jsonl\" \\".to_string(),
        "      2> \"$outputs_dir/codex-stderr.log\"".to_string(),
        "  ' sh {}".to_string(),
    ]
    .join("\n")
}

pub(crate) fn codex_judge_dispatch_recipe(model_flag: Option<&str>, guard: bool) -> String {
    let hook_trust = if guard {
        " --dangerously-bypass-hook-trust"
    } else {
        ""
    };
    let model_flag = model_flag.unwrap_or("-m");
    [
        "Dispatch each judge task from judge-tasks.json with:".to_string(),
        String::new(),
        "```bash".to_string(),
        "JOBS=${JOBS:-4}".to_string(),
        "jq -j '.tasks[] | [.dispatch_prompt_path, .response_path, (.model // \"\")] | @tsv + \"\\u0000\"' judge-tasks.json | \\".to_string(),
        "  xargs -0 -P \"$JOBS\" -I{} sh -c '".to_string(),
        "    prompt_path=\"$(printf \"%s\" \"$1\" | cut -f1)\"".to_string(),
        "    response_path=\"$(printf \"%s\" \"$1\" | cut -f2)\"".to_string(),
        "    model=\"$(printf \"%s\" \"$1\" | cut -f3)\"".to_string(),
        "    response_base=\"${response_path%.json}\"".to_string(),
        "    mkdir -p \"$(dirname \"$response_path\")\"".to_string(),
        "    if [ -n \"$model\" ]; then".to_string(),
        format!(
            "      codex exec --cd <eval-root> --sandbox workspace-write --ask-for-approval never{hook_trust} {model_flag} \"$model\" --json \\"
        ),
        "        \"Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers.\" \\".to_string(),
        "        </dev/null \\".to_string(),
        "        > \"$response_base.codex-events.jsonl\" \\".to_string(),
        "        2> \"$response_base.codex-stderr.log\"".to_string(),
        "    else".to_string(),
        format!(
            "      codex exec --cd <eval-root> --sandbox workspace-write --ask-for-approval never{hook_trust} --json \\"
        ),
        "        \"Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers.\" \\".to_string(),
        "        </dev/null \\".to_string(),
        "        > \"$response_base.codex-events.jsonl\" \\".to_string(),
        "        2> \"$response_base.codex-stderr.log\"".to_string(),
        "    fi".to_string(),
        "  ' sh {}".to_string(),
        "```".to_string(),
    ]
    .join("\n")
}