aether_evals/
assertions.rs

1use crate::Transcript;
2use std::fmt::Write as _;
3
4#[track_caller]
5pub fn assert_tool_called(transcript: &Transcript, name: &str) {
6    assert!(transcript.tool_called(name), "expected tool `{name}` to be called");
7}
8
9#[track_caller]
10pub fn assert_tool_call_count(transcript: &Transcript, name: &str, expected: usize) {
11    assert_eq!(transcript.tool_call_count(name), expected, "unexpected call count for tool `{name}`");
12}
13
14#[track_caller]
15pub fn assert_tool_call_with_args(transcript: &Transcript, name: &str, expected: &serde_json::Value) {
16    let mut matched = false;
17    let mut parse_failures = String::new();
18
19    for call in transcript.tool_calls(name) {
20        match call.arguments_json() {
21            Ok(actual) if actual == *expected => {
22                matched = true;
23                break;
24            }
25            Ok(_) => {}
26            Err(error) => {
27                let _ = writeln!(parse_failures, "  arguments={} parse_error={error}", call.arguments);
28            }
29        }
30    }
31
32    if matched {
33        return;
34    }
35
36    let mut message = format!("expected tool `{name}` to be called with args `{expected}`");
37    if !parse_failures.is_empty() {
38        message.push_str("\nNon-JSON arguments seen for this tool (skipped from match):\n");
39        message.push_str(&parse_failures);
40    }
41    panic!("{message}");
42}
43
44#[cfg(test)]
45mod tests {
46    use super::*;
47    use aether_core::events::AgentMessage;
48    use llm::ToolCallResult;
49
50    #[test]
51    fn assert_tool_call_with_args_accepts_matching_json_arguments() {
52        let trace = transcript_with_messages(vec![tool_result("bash", r#"{"command":"pwd"}"#)]);
53
54        assert_tool_call_with_args(&trace, "bash", &serde_json::json!({ "command": "pwd" }));
55    }
56
57    #[test]
58    #[should_panic(expected = "Non-JSON arguments seen for this tool")]
59    fn assert_tool_call_with_args_surfaces_parse_failures() {
60        let trace = transcript_with_messages(vec![tool_result("bash", "not json")]);
61
62        assert_tool_call_with_args(&trace, "bash", &serde_json::json!({ "command": "pwd" }));
63    }
64
65    #[test]
66    #[should_panic(expected = "expected tool `missing` to be called")]
67    fn assert_tool_called_panics_when_tool_was_not_called() {
68        let trace = transcript_with_messages(vec![]);
69
70        assert_tool_called(&trace, "missing");
71    }
72
73    fn transcript_with_messages(messages: Vec<AgentMessage>) -> Transcript {
74        Transcript::new(messages)
75    }
76
77    fn tool_result(name: &str, arguments: &str) -> AgentMessage {
78        AgentMessage::ToolResult {
79            result: ToolCallResult {
80                id: name.to_string(),
81                name: name.to_string(),
82                arguments: arguments.to_string(),
83                result: "ok".to_string(),
84            },
85            result_meta: None,
86            model_name: "test".to_string(),
87        }
88    }
89}
aether_evals/assertions.rs

aether_evals/
assertions.rs
