aether_evals/
assertions.rs

1use crate::TaskRun;
2use std::fmt::Write as _;
3
4#[track_caller]
5pub fn assert_tool_called(run: &TaskRun, name: &str) {
6    assert!(run.transcript().tool_called(name), "expected tool `{name}` to be called\n\n{}", run.failure_context());
7}
8
9#[track_caller]
10pub fn assert_tool_call_count(run: &TaskRun, name: &str, expected: usize) {
11    assert_eq!(
12        run.transcript().tool_call_count(name),
13        expected,
14        "unexpected call count for tool `{name}`\n\n{}",
15        run.failure_context()
16    );
17}
18
19#[track_caller]
20pub fn assert_tool_call_with_args(run: &TaskRun, name: &str, expected: &serde_json::Value) {
21    let mut matched = false;
22    let mut parse_failures = String::new();
23
24    for call in run.transcript().tool_calls(name) {
25        match call.arguments_json() {
26            Ok(actual) if actual == *expected => {
27                matched = true;
28                break;
29            }
30            Ok(_) => {}
31            Err(error) => {
32                let _ = writeln!(parse_failures, "  arguments={} parse_error={error}", call.arguments);
33            }
34        }
35    }
36
37    if matched {
38        return;
39    }
40
41    let mut message = format!("expected tool `{name}` to be called with args `{expected}`");
42    if !parse_failures.is_empty() {
43        message.push_str("\nNon-JSON arguments seen for this tool (skipped from match):\n");
44        message.push_str(&parse_failures);
45    }
46    panic!("{message}\n\n{}", run.failure_context());
47}
48
49#[cfg(test)]
50mod tests {
51    use super::*;
52    use crate::{TaskRun, Transcript, Workspace};
53    use aether_core::events::AgentMessage;
54    use llm::ToolCallResult;
55
56    #[test]
57    fn assert_tool_call_with_args_accepts_matching_json_arguments() {
58        let run = run_with_messages(vec![tool_result("bash", r#"{"command":"pwd"}"#)]);
59
60        assert_tool_call_with_args(&run, "bash", &serde_json::json!({ "command": "pwd" }));
61    }
62
63    #[test]
64    #[should_panic(expected = "Non-JSON arguments seen for this tool")]
65    fn assert_tool_call_with_args_surfaces_parse_failures() {
66        let run = run_with_messages(vec![tool_result("bash", "not json")]);
67
68        assert_tool_call_with_args(&run, "bash", &serde_json::json!({ "command": "pwd" }));
69    }
70
71    #[test]
72    #[should_panic(expected = "expected tool `missing` to be called")]
73    fn assert_tool_called_panics_when_tool_was_not_called() {
74        let run = run_with_messages(vec![]);
75
76        assert_tool_called(&run, "missing");
77    }
78
79    fn run_with_messages(messages: Vec<AgentMessage>) -> TaskRun {
80        TaskRun::new("prompt".to_string(), Workspace::empty().unwrap(), Transcript::new(messages))
81    }
82
83    fn tool_result(name: &str, arguments: &str) -> AgentMessage {
84        AgentMessage::ToolResult {
85            result: ToolCallResult {
86                id: name.to_string(),
87                name: name.to_string(),
88                arguments: arguments.to_string(),
89                result: "ok".to_string(),
90            },
91            result_meta: None,
92            model_name: "test".to_string(),
93        }
94    }
95}
aether_evals/assertions.rs

aether_evals/
assertions.rs
