eval-magic 0.5.0

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

//! Claude Code harness support — the default harness.
//!
//! The declarative half of this harness lives in `harnesses/claude-code.toml`
//! (including the write-guard data rendered by the generic engine in
//! [`crate::adapters::guard`]); this module tree keeps only the code-backed
//! capabilities the descriptor references: `claude -p` stream-json transcript
//! parsing ([`stream_json`] + [`transcript`]) and plugin-shadow detection
//! ([`plugin_shadow`]).

pub mod plugin_shadow;
pub mod stream_json;
pub mod transcript;

#[cfg(test)]
mod tests {
    use crate::adapters::adapter_for;
    use crate::core::Harness;

    #[test]
    fn claude_adapter_advertises_cli_events_file_and_model_flag() {
        let a = adapter_for(Harness::resolve("claude-code").unwrap());
        assert_eq!(
            a.cli_events_filename().as_deref(),
            Some("claude-events.jsonl")
        );
        assert_eq!(a.cli_model_flag().as_deref(), Some("--model"));
    }

    #[test]
    fn claude_parse_cli_events_full_reads_stream_json_result_event() {
        use serde_json::json;
        let dir = tempfile::TempDir::new().unwrap();
        let path = dir.path().join("claude-events.jsonl");
        // No per-line timestamps; the result event is the only source of duration.
        let lines = [
            json!({"type": "assistant", "message": {"id": "msg_1", "role": "assistant", "content": [
                {"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}}
            ]}}),
            json!({"type": "result", "subtype": "success", "is_error": false, "result": "Done", "duration_ms": 5637, "usage": {"input_tokens": 1, "output_tokens": 2, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}),
        ];
        let body = lines
            .iter()
            .map(|l| l.to_string())
            .collect::<Vec<_>>()
            .join("\n");
        std::fs::write(&path, format!("{body}\n")).unwrap();

        let a = adapter_for(Harness::resolve("claude-code").unwrap());
        let summary = a.parse_cli_events_full(&path).unwrap();
        assert_eq!(summary.final_text, Some("Done".into()));
        assert_eq!(summary.duration_ms, Some(5637));
        assert_eq!(summary.tool_invocations.len(), 1);
        assert_eq!(summary.tool_invocations[0].name, "Bash");
    }
}