aether_evals/spec/
report.rs

1use super::runner::WorkspaceRetention;
2use crate::evals::{RetainedWorkspaceInfo, TaskRun};
3use aether_core::events::AgentMessage;
4use schemars::{JsonSchema, Schema, schema_for};
5use serde::Serialize;
6use serde_json::Value;
7
8#[derive(Debug, Clone, Serialize, JsonSchema)]
9#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
10pub enum EvalStreamEvent {
11    AgentMessage { message: AgentMessage },
12    Outcome { outcome: EvalOutcome },
13}
14
15impl EvalStreamEvent {
16    pub fn schema() -> Schema {
17        schema_for!(Self)
18    }
19}
20
21/// Outcome of running a collection of eval files.
22#[derive(Debug, Clone, Serialize)]
23#[serde(rename_all = "camelCase")]
24pub struct EvalFilesReport {
25    pub evals: Vec<EvalOutcome>,
26}
27
28#[derive(Debug, Clone, Serialize, JsonSchema)]
29#[serde(rename_all = "camelCase", deny_unknown_fields)]
30pub struct EvalOutcome {
31    pub name: String,
32    pub passed: bool,
33    pub failures: Vec<String>,
34    pub tool_calls: Vec<EvalToolCall>,
35    #[serde(skip_serializing_if = "Option::is_none")]
36    pub judge: Option<JudgeSummary>,
37    #[serde(skip_serializing_if = "Option::is_none")]
38    pub failure_context: Option<String>,
39    /// Retained workspace root and effective cwd, when the eval was run with retention. The caller owns cleanup of `root_path`.
40    #[serde(skip_serializing_if = "Option::is_none")]
41    pub retained_workspace: Option<RetainedWorkspaceInfo>,
42}
43
44#[derive(Debug, Clone, Serialize, JsonSchema)]
45#[serde(rename_all = "camelCase", deny_unknown_fields)]
46pub struct EvalToolCall {
47    pub name: String,
48    pub arguments: Option<Value>,
49    pub raw_arguments: String,
50}
51
52impl EvalOutcome {
53    /// JSON schema for a single eval's outcome (the output returned to out-of-process callers).
54    pub fn schema() -> Schema {
55        schema_for!(Self)
56    }
57
58    pub(crate) fn setup_failed(name: impl Into<String>, error: impl std::fmt::Display) -> Self {
59        Self::failed(name, vec![format!("workspace setup failed: {error}")])
60    }
61
62    pub(crate) fn failed(name: impl Into<String>, failures: Vec<String>) -> Self {
63        Self {
64            name: name.into(),
65            passed: false,
66            failures,
67            tool_calls: Vec::new(),
68            judge: None,
69            failure_context: None,
70            retained_workspace: None,
71        }
72    }
73
74    pub(crate) fn from_task_run(
75        name: impl Into<String>,
76        run: TaskRun,
77        failures: Vec<String>,
78        judge: Option<JudgeSummary>,
79        retention: WorkspaceRetention,
80    ) -> Self {
81        let passed = failures.is_empty();
82        let failure_context = (!passed).then(|| run.failure_context());
83        let tool_calls = EvalToolCall::from_task_run(&run);
84        let retained_workspace = match retention {
85            WorkspaceRetention::Retain => Some(run.into_workspace().persist()),
86            WorkspaceRetention::Discard => None,
87        };
88
89        Self { name: name.into(), passed, failures, tool_calls, judge, failure_context, retained_workspace }
90    }
91}
92
93impl EvalToolCall {
94    fn from_task_run(run: &TaskRun) -> Vec<Self> {
95        run.transcript()
96            .all_tool_calls()
97            .map(|call| Self {
98                name: call.name.to_string(),
99                arguments: call.arguments_json().ok(),
100                raw_arguments: call.arguments.to_string(),
101            })
102            .collect()
103    }
104}
105
106#[derive(Debug, Clone, Serialize, JsonSchema)]
107#[serde(rename_all = "camelCase", deny_unknown_fields)]
108pub struct JudgeSummary {
109    pub passed: bool,
110    pub score: f64,
111    pub reason: String,
112    pub criteria: Vec<JudgeCriterionSummary>,
113}
114
115#[derive(Debug, Clone, Serialize, JsonSchema)]
116#[serde(rename_all = "camelCase", deny_unknown_fields)]
117pub struct JudgeCriterionSummary {
118    pub id: String,
119    pub description: String,
120    pub blocking: bool,
121    pub weight: f64,
122    pub threshold: f64,
123    pub score: f64,
124    pub passed: bool,
125    pub reason: String,
126}
127
128impl EvalFilesReport {
129    pub fn passed(&self) -> bool {
130        self.evals.iter().all(|eval| eval.passed)
131    }
132
133    pub fn passed_count(&self) -> usize {
134        self.evals.iter().filter(|eval| eval.passed).count()
135    }
136
137    pub fn failed_count(&self) -> usize {
138        self.evals.iter().filter(|eval| !eval.passed).count()
139    }
140}
141
142impl JudgeSummary {
143    /// Failure messages for blocking criteria that scored below their threshold.
144    pub fn blocking_failures(&self) -> impl Iterator<Item = String> + '_ {
145        self.criteria
146            .iter()
147            .filter(|criterion| criterion.blocking && !criterion.passed)
148            .map(|criterion| format!("judge criterion `{}`: {}", criterion.id, criterion.reason))
149    }
150}
151
152#[cfg(test)]
153mod tests {
154    use super::*;
155    use aether_core::events::AgentMessage;
156
157    #[test]
158    fn blocking_failures_report_only_blocking_criteria_below_threshold() {
159        let criterion = |id: &str, blocking, score: f64| JudgeCriterionSummary {
160            id: id.to_string(),
161            description: "desc".to_string(),
162            blocking,
163            weight: 1.0,
164            threshold: 0.8,
165            score,
166            passed: score >= 0.8,
167            reason: format!("{id} reason"),
168        };
169        let summary = JudgeSummary {
170            passed: false,
171            score: 0.0,
172            reason: "r".to_string(),
173            criteria: vec![
174                criterion("met", true, 0.9),
175                criterion("failed", true, 0.5),
176                criterion("advisory", false, 0.0),
177            ],
178        };
179
180        let failures: Vec<String> = summary.blocking_failures().collect();
181
182        assert_eq!(failures, vec!["judge criterion `failed`: failed reason".to_string()]);
183    }
184
185    #[test]
186    fn agent_message_event_serializes_with_snake_case_tag_and_nested_message() {
187        let event = EvalStreamEvent::AgentMessage {
188            message: AgentMessage::Text {
189                message_id: "msg_1".to_string(),
190                chunk: "hi".to_string(),
191                is_complete: false,
192                model_name: "fake".to_string(),
193            },
194        };
195        let json = serde_json::to_value(&event).unwrap();
196        assert_eq!(json["type"], "agent_message");
197        assert_eq!(json["message"]["type"], "text");
198        assert_eq!(json["message"]["message_id"], "msg_1");
199        assert_eq!(json["message"]["is_complete"], false);
200    }
201
202    #[test]
203    fn outcome_event_serializes_with_nested_outcome_fields() {
204        let outcome = EvalOutcome {
205            name: "edit-notes".to_string(),
206            passed: true,
207            failures: Vec::new(),
208            tool_calls: Vec::new(),
209            judge: None,
210            failure_context: None,
211            retained_workspace: None,
212        };
213        let event = EvalStreamEvent::Outcome { outcome };
214        let json: serde_json::Value = serde_json::to_value(&event).unwrap();
215        assert_eq!(json["type"], "outcome");
216        assert_eq!(json["outcome"]["name"], "edit-notes");
217        assert_eq!(json["outcome"]["passed"], true);
218    }
219}
aether_evals/spec/report.rs

aether_evals/spec/
report.rs
