aether_evals/evals/
transcript.rs

1use crate::agents::{TRANSCRIPT_PAYLOAD_CHARS, get_transcript_line};
2use aether_core::events::AgentMessage;
3use std::fmt::Write as _;
4
5pub struct Transcript {
6    messages: Vec<AgentMessage>,
7}
8
9pub struct ToolCall<'a> {
10    pub name: &'a str,
11    pub arguments: &'a str,
12}
13
14impl Transcript {
15    pub(crate) fn new(messages: Vec<AgentMessage>) -> Self {
16        Self { messages }
17    }
18
19    pub fn messages(&self) -> &[AgentMessage] {
20        &self.messages
21    }
22
23    pub fn all_tool_calls(&self) -> impl Iterator<Item = ToolCall<'_>> + '_ {
24        self.messages.iter().filter_map(|message| match message {
25            AgentMessage::ToolResult { result, .. } => {
26                Some(ToolCall { name: &result.name, arguments: &result.arguments })
27            }
28            AgentMessage::ToolError { error, .. } => {
29                Some(ToolCall { name: &error.name, arguments: error.arguments.as_deref().unwrap_or("") })
30            }
31            _ => None,
32        })
33    }
34
35    pub fn tool_calls<'a>(&'a self, name: &'a str) -> impl Iterator<Item = ToolCall<'a>> + 'a {
36        self.all_tool_calls().filter(move |call| call.name == name)
37    }
38
39    pub fn tool_called(&self, name: &str) -> bool {
40        self.tool_calls(name).next().is_some()
41    }
42
43    pub fn tool_call_count(&self, name: &str) -> usize {
44        self.tool_calls(name).count()
45    }
46}
47
48impl ToolCall<'_> {
49    pub fn arguments_json(&self) -> Result<serde_json::Value, serde_json::Error> {
50        serde_json::from_str(self.arguments)
51    }
52}
53
54pub(crate) fn format_transcript(messages: &[AgentMessage]) -> String {
55    let mut transcript = String::new();
56    for message in messages {
57        if let Some(line) = get_transcript_line(message, TRANSCRIPT_PAYLOAD_CHARS) {
58            let _ = writeln!(transcript, "{line}");
59        }
60    }
61    transcript
62}
63
64#[cfg(test)]
65mod tests {
66    use super::*;
67    use llm::{ToolCallRequest, ToolCallResult};
68
69    #[test]
70    fn tool_call_count_counts_matching_tool_calls() {
71        let transcript = transcript_with_messages(vec![tool_call("bash"), tool_call("read"), tool_result("bash")]);
72
73        assert!(transcript.tool_called("bash"));
74        assert!(!transcript.tool_called("read"));
75        assert!(!transcript.tool_called("write"));
76        assert_eq!(transcript.tool_call_count("bash"), 1);
77        assert_eq!(transcript.tool_call_count("read"), 0);
78    }
79
80    #[test]
81    fn tool_call_arguments_json_parses_arguments() {
82        let call = ToolCall { name: "bash", arguments: r#"{"command":"pwd"}"# };
83
84        assert_eq!(call.arguments_json().unwrap(), serde_json::json!({ "command": "pwd" }));
85    }
86
87    #[test]
88    fn tool_call_arguments_json_returns_error_for_invalid_json() {
89        let call = ToolCall { name: "bash", arguments: "not json" };
90
91        assert!(call.arguments_json().is_err());
92    }
93
94    pub(crate) fn transcript_with_messages(messages: Vec<AgentMessage>) -> Transcript {
95        Transcript::new(messages)
96    }
97
98    fn tool_call(name: &str) -> AgentMessage {
99        AgentMessage::ToolCall {
100            request: ToolCallRequest { id: name.to_string(), name: name.to_string(), arguments: "{}".to_string() },
101            model_name: "test".to_string(),
102        }
103    }
104
105    fn tool_result(name: &str) -> AgentMessage {
106        AgentMessage::ToolResult {
107            result: ToolCallResult {
108                id: name.to_string(),
109                name: name.to_string(),
110                arguments: "{}".to_string(),
111                result: "ok".to_string(),
112            },
113            result_meta: None,
114            model_name: "test".to_string(),
115        }
116    }
117}
aether_evals/evals/transcript.rs

aether_evals/evals/
transcript.rs
