eval_magic/pipeline/grade/
transcript_check.rs

1//! Transcript-check grading.
2//!
3//! A
4//! `transcript_check` assertion of kind `tool_invocation_matches` passes when its
5//! `pattern` regex matches the `"<name> <json-args>"` rendering of any tool
6//! invocation in the run.
7
8use regex::Regex;
9
10use crate::core::{AssertionResult, AssertionTranscriptCheck, Grader, ToolInvocation};
11
12/// Render an invocation as `"<name> <compact-json-args>"` (args omitted when
13/// absent) — the text the check's `pattern` regex runs against.
14fn describe_invocation(inv: &ToolInvocation) -> String {
15    match &inv.args {
16        Some(args) => format!(
17            "{} {}",
18            inv.name,
19            serde_json::to_string(args).unwrap_or_default()
20        ),
21        None => inv.name.clone(),
22    }
23}
24
25/// A failed transcript-check result with full confidence.
26fn fail(id: &str, evidence: String) -> AssertionResult {
27    AssertionResult {
28        id: id.to_string(),
29        passed: false,
30        evidence,
31        confidence: Some(1.0),
32        grader: Some(Grader::TranscriptCheck),
33    }
34}
35
36/// Grade a `transcript_check` assertion against a run's tool invocations,
37/// covering the empty-invocations, unsupported-kind, missing-pattern,
38/// invalid-regex, match, and no-match branches.
39pub fn grade_transcript_check(
40    assertion: &AssertionTranscriptCheck,
41    invocations: &[ToolInvocation],
42) -> AssertionResult {
43    if invocations.is_empty() {
44        return fail(
45            &assertion.id,
46            "tool_invocations is empty — run record was not filled by a transcript adapter. \
47             Run `eval-magic fill-transcripts` for Claude Code, or `eval-magic fill-transcripts \
48             --harness codex` when outputs/codex-events.jsonl is present; otherwise rely on \
49             `llm_judge` assertions for harnesses without an adapter."
50                .to_string(),
51        );
52    }
53
54    if assertion.check != "tool_invocation_matches" {
55        return fail(
56            &assertion.id,
57            format!("unsupported transcript_check kind: '{}'", assertion.check),
58        );
59    }
60
61    let Some(pattern) = assertion.pattern.as_deref() else {
62        return fail(
63            &assertion.id,
64            "transcript_check 'tool_invocation_matches' requires a `pattern` field".to_string(),
65        );
66    };
67
68    let re = match Regex::new(pattern) {
69        Ok(re) => re,
70        Err(err) => {
71            return fail(
72                &assertion.id,
73                format!("invalid regex in pattern '{pattern}': {err}"),
74            );
75        }
76    };
77
78    for inv in invocations {
79        let target = describe_invocation(inv);
80        if re.is_match(&target) {
81            let snippet: String = target.chars().take(200).collect();
82            return AssertionResult {
83                id: assertion.id.clone(),
84                passed: true,
85                evidence: format!("matched ordinal {}: {snippet}", inv.ordinal),
86                confidence: Some(1.0),
87                grader: Some(Grader::TranscriptCheck),
88            };
89        }
90    }
91
92    fail(
93        &assertion.id,
94        format!(
95            "no tool invocation matched /{pattern}/ across {} invocation(s)",
96            invocations.len()
97        ),
98    )
99}
100
101#[cfg(test)]
102mod tests {
103    use super::*;
104    use serde_json::json;
105
106    fn check(pattern: Option<&str>) -> AssertionTranscriptCheck {
107        AssertionTranscriptCheck {
108            id: "t1".to_string(),
109            check: "tool_invocation_matches".to_string(),
110            pattern: pattern.map(str::to_string),
111            must_precede: None,
112        }
113    }
114
115    fn inv(name: &str, args: serde_json::Value, ordinal: u32) -> ToolInvocation {
116        ToolInvocation {
117            name: name.to_string(),
118            args: Some(args),
119            result: None,
120            ordinal,
121        }
122    }
123
124    #[test]
125    fn empty_invocations_fail_with_guidance() {
126        let r = grade_transcript_check(&check(Some("Bash")), &[]);
127        assert!(!r.passed);
128        assert!(r.evidence.contains("tool_invocations is empty"));
129        assert_eq!(r.grader, Some(Grader::TranscriptCheck));
130    }
131
132    #[test]
133    fn missing_pattern_fails() {
134        let r = grade_transcript_check(&check(None), &[inv("Bash", json!({"command": "ls"}), 0)]);
135        assert!(!r.passed);
136        assert!(r.evidence.contains("requires a `pattern`"));
137    }
138
139    #[test]
140    fn unsupported_kind_fails() {
141        let mut c = check(Some("x"));
142        c.check = "something_else".to_string();
143        let r = grade_transcript_check(&c, &[inv("Bash", json!({}), 0)]);
144        assert!(!r.passed);
145        assert!(r.evidence.contains("unsupported transcript_check kind"));
146    }
147
148    #[test]
149    fn matching_pattern_passes_with_ordinal() {
150        let invs = [
151            inv("Read", json!({"file_path": "/x"}), 0),
152            inv("Bash", json!({"command": "bun test"}), 1),
153        ];
154        let r = grade_transcript_check(&check(Some("bun test")), &invs);
155        assert!(r.passed);
156        assert!(r.evidence.contains("matched ordinal 1"));
157    }
158
159    #[test]
160    fn no_match_fails_with_count() {
161        let invs = [inv("Read", json!({"file_path": "/x"}), 0)];
162        let r = grade_transcript_check(&check(Some("npm install")), &invs);
163        assert!(!r.passed);
164        assert!(r.evidence.contains("across 1 invocation(s)"));
165    }
166
167    #[test]
168    fn invalid_regex_fails() {
169        let invs = [inv("Bash", json!({"command": "ls"}), 0)];
170        let r = grade_transcript_check(&check(Some("(unclosed")), &invs);
171        assert!(!r.passed);
172        assert!(r.evidence.contains("invalid regex"));
173    }
174}
eval_magic/pipeline/grade/transcript_check.rs

eval_magic/pipeline/grade/
transcript_check.rs
