1use crate::Workspace;
2use crate::agents::{TRANSCRIPT_PAYLOAD_CHARS, get_transcript_line};
3use aether_core::events::AgentMessage;
4use serde::{Deserialize, Serialize};
5use std::fmt::Write as _;
6use std::path::{Path, PathBuf};
7
8pub struct EvalReport {
9 prompt: String,
10 workspace: Workspace,
11 messages: Vec<AgentMessage>,
12 agent_diff: Option<GitDiff>,
13 reference_diff: Option<GitDiff>,
14}
15
16#[derive(Debug, Clone, Serialize, Deserialize)]
17pub struct GitDiff {
18 pub diff: String,
19 pub stats: DiffStats,
20}
21
22#[derive(Debug, Clone, Serialize, Deserialize)]
23pub struct DiffStats {
24 pub files_changed: usize,
25 pub lines_added: usize,
26 pub lines_removed: usize,
27}
28
29pub struct ToolCall<'a> {
30 pub name: &'a str,
31 pub arguments: &'a str,
32}
33
34impl EvalReport {
35 pub(crate) fn new(
36 prompt: String,
37 workspace: Workspace,
38 messages: Vec<AgentMessage>,
39 agent_diff: Option<GitDiff>,
40 reference_diff: Option<GitDiff>,
41 ) -> Self {
42 Self { prompt, workspace, messages, agent_diff, reference_diff }
43 }
44
45 pub fn prompt(&self) -> &str {
46 &self.prompt
47 }
48
49 pub fn workspace(&self) -> &Workspace {
50 &self.workspace
51 }
52
53 pub fn path(&self, relative_path: impl AsRef<Path>) -> PathBuf {
54 self.workspace.path().join(relative_path)
55 }
56
57 pub fn messages(&self) -> &[AgentMessage] {
58 &self.messages
59 }
60
61 pub fn agent_diff(&self) -> Option<&GitDiff> {
62 self.agent_diff.as_ref()
63 }
64
65 pub fn reference_diff(&self) -> Option<&GitDiff> {
66 self.reference_diff.as_ref()
67 }
68
69 pub fn tool_calls<'a>(&'a self, name: &'a str) -> impl Iterator<Item = ToolCall<'a>> + 'a {
70 self.messages.iter().filter_map(move |message| match message {
71 AgentMessage::ToolResult { result, .. } if result.name == name => {
72 Some(ToolCall { name: &result.name, arguments: &result.arguments })
73 }
74 AgentMessage::ToolError { error, .. } if error.name == name => {
75 Some(ToolCall { name: &error.name, arguments: error.arguments.as_deref().unwrap_or("") })
76 }
77 _ => None,
78 })
79 }
80
81 pub fn tool_called(&self, name: &str) -> bool {
82 self.tool_calls(name).next().is_some()
83 }
84
85 pub fn tool_call_count(&self, name: &str) -> usize {
86 self.tool_calls(name).count()
87 }
88
89 pub fn failure_context(&self) -> String {
90 let mut summary = String::new();
91 let _ = writeln!(summary, "Eval failure context");
92 let _ = writeln!(summary, "Workspace: {}", self.workspace.path().display());
93 summary.push_str("Prompt:\n");
94 push_indented(&mut summary, &self.prompt, 2);
95 summary.push('\n');
96
97 if let Some(diff) = &self.agent_diff {
98 summary.push_str("Agent diff summary:\n");
99 push_diff_stats(&mut summary, diff);
100 summary.push('\n');
101 }
102
103 if let Some(diff) = &self.reference_diff {
104 summary.push_str("Reference diff summary:\n");
105 push_diff_stats(&mut summary, diff);
106 summary.push('\n');
107 }
108
109 summary.push_str("Agent messages:\n");
110 if self.messages.is_empty() {
111 summary.push_str(" none\n");
112 } else {
113 push_indented(&mut summary, &format_transcript(&self.messages), 2);
114 }
115
116 summary
117 }
118}
119
120impl ToolCall<'_> {
121 pub fn arguments_json(&self) -> Result<serde_json::Value, serde_json::Error> {
122 serde_json::from_str(self.arguments)
123 }
124}
125
126impl DiffStats {
127 pub fn from_diff(diff: &str) -> Self {
128 let mut lines_added = 0;
129 let mut lines_removed = 0;
130 let mut files_changed = 0;
131
132 for line in diff.lines() {
133 if line.starts_with("diff --git") {
134 files_changed += 1;
135 } else if line.starts_with('+') && !line.starts_with("+++") {
136 lines_added += 1;
137 } else if line.starts_with('-') && !line.starts_with("---") {
138 lines_removed += 1;
139 }
140 }
141
142 Self { files_changed, lines_added, lines_removed }
143 }
144}
145
146pub(crate) fn format_transcript(messages: &[AgentMessage]) -> String {
147 let mut transcript = String::new();
148 for message in messages {
149 if let Some(line) = get_transcript_line(message, TRANSCRIPT_PAYLOAD_CHARS) {
150 let _ = writeln!(transcript, "{line}");
151 }
152 }
153 transcript
154}
155
156fn push_indented(output: &mut String, value: &str, spaces: usize) {
157 let indentation = " ".repeat(spaces);
158 for line in value.lines() {
159 output.push_str(&indentation);
160 output.push_str(line);
161 output.push('\n');
162 }
163}
164
165fn push_diff_stats(output: &mut String, diff: &GitDiff) {
166 let _ = writeln!(output, " Files changed: {}", diff.stats.files_changed);
167 let _ = writeln!(output, " Lines added: {}", diff.stats.lines_added);
168 let _ = writeln!(output, " Lines removed: {}", diff.stats.lines_removed);
169}
170
171#[cfg(test)]
172mod tests {
173 use super::*;
174 use llm::{ToolCallRequest, ToolCallResult};
175
176 #[test]
177 fn diff_stats_from_diff_counts_files_and_changed_lines() {
178 let diff = "diff --git a/a.txt b/a.txt\n--- a/a.txt\n+++ b/a.txt\n@@\n-old\n+new\ndiff --git a/b.txt b/b.txt\n+++ b/b.txt\n+added\n";
179
180 let stats = DiffStats::from_diff(diff);
181
182 assert_eq!(stats.files_changed, 2);
183 assert_eq!(stats.lines_added, 2);
184 assert_eq!(stats.lines_removed, 1);
185 }
186
187 #[test]
188 fn failure_context_includes_prompt_workspace_messages_and_diff_summary() {
189 let report = EvalReport::new(
190 "do the thing".to_string(),
191 Workspace::empty().unwrap(),
192 vec![AgentMessage::text("msg_1", "done", true, "test"), AgentMessage::Done],
193 Some(GitDiff {
194 diff: "diff --git a/a.txt b/a.txt\n+new\n".to_string(),
195 stats: DiffStats { files_changed: 1, lines_added: 1, lines_removed: 0 },
196 }),
197 None,
198 );
199
200 let context = report.failure_context();
201
202 assert!(context.contains("Eval failure context"));
203 assert!(context.contains("Workspace:"));
204 assert!(context.contains("do the thing"));
205 assert!(context.contains("Agent diff summary:"));
206 assert!(context.contains("Files changed: 1"));
207 assert!(context.contains("[agent] done"));
208 }
209
210 #[test]
211 fn tool_call_count_counts_matching_tool_calls() {
212 let report = report_with_messages(vec![tool_call("bash"), tool_call("read"), tool_result("bash")]);
213
214 assert!(report.tool_called("bash"));
215 assert!(!report.tool_called("read"));
216 assert!(!report.tool_called("write"));
217 assert_eq!(report.tool_call_count("bash"), 1);
218 assert_eq!(report.tool_call_count("read"), 0);
219 }
220
221 #[test]
222 fn tool_call_arguments_json_parses_arguments() {
223 let call = ToolCall { name: "bash", arguments: r#"{"command":"pwd"}"# };
224
225 assert_eq!(call.arguments_json().unwrap(), serde_json::json!({ "command": "pwd" }));
226 }
227
228 #[test]
229 fn tool_call_arguments_json_returns_error_for_invalid_json() {
230 let call = ToolCall { name: "bash", arguments: "not json" };
231
232 assert!(call.arguments_json().is_err());
233 }
234
235 pub(crate) fn report_with_messages(messages: Vec<AgentMessage>) -> EvalReport {
236 EvalReport::new("prompt".to_string(), Workspace::empty().unwrap(), messages, None, None)
237 }
238
239 fn tool_call(name: &str) -> AgentMessage {
240 AgentMessage::ToolCall {
241 request: ToolCallRequest { id: name.to_string(), name: name.to_string(), arguments: "{}".to_string() },
242 model_name: "test".to_string(),
243 }
244 }
245
246 fn tool_result(name: &str) -> AgentMessage {
247 AgentMessage::ToolResult {
248 result: ToolCallResult {
249 id: name.to_string(),
250 name: name.to_string(),
251 arguments: "{}".to_string(),
252 result: "ok".to_string(),
253 },
254 result_meta: None,
255 model_name: "test".to_string(),
256 }
257 }
258}