aether_evals/spec/
report.rs

1use super::runner::WorkspaceRetention;
2use crate::evals::{RetainedWorkspaceInfo, TaskRun};
3use schemars::{JsonSchema, Schema, schema_for};
4use serde::Serialize;
5use serde_json::Value;
6
7/// Outcome of running a collection of eval files.
8#[derive(Debug, Clone, Serialize)]
9#[serde(rename_all = "camelCase")]
10pub struct EvalFilesReport {
11    pub evals: Vec<EvalOutcome>,
12}
13
14#[derive(Debug, Clone, Serialize, JsonSchema)]
15#[serde(rename_all = "camelCase", deny_unknown_fields)]
16pub struct EvalOutcome {
17    pub name: String,
18    pub passed: bool,
19    pub failures: Vec<String>,
20    pub tool_calls: Vec<EvalToolCall>,
21    #[serde(skip_serializing_if = "Option::is_none")]
22    pub judge: Option<JudgeSummary>,
23    #[serde(skip_serializing_if = "Option::is_none")]
24    pub failure_context: Option<String>,
25    /// Retained workspace root and effective cwd, when the eval was run with retention. The caller owns cleanup of `root_path`.
26    #[serde(skip_serializing_if = "Option::is_none")]
27    pub retained_workspace: Option<RetainedWorkspaceInfo>,
28}
29
30#[derive(Debug, Clone, Serialize, JsonSchema)]
31#[serde(rename_all = "camelCase", deny_unknown_fields)]
32pub struct EvalToolCall {
33    pub name: String,
34    pub arguments: Option<Value>,
35    pub raw_arguments: String,
36}
37
38impl EvalOutcome {
39    /// JSON schema for a single eval's outcome (the output returned to out-of-process callers).
40    pub fn schema() -> Schema {
41        schema_for!(Self)
42    }
43
44    pub(crate) fn setup_failed(name: impl Into<String>, error: impl std::fmt::Display) -> Self {
45        Self::failed(name, vec![format!("workspace setup failed: {error}")])
46    }
47
48    pub(crate) fn failed(name: impl Into<String>, failures: Vec<String>) -> Self {
49        Self {
50            name: name.into(),
51            passed: false,
52            failures,
53            tool_calls: Vec::new(),
54            judge: None,
55            failure_context: None,
56            retained_workspace: None,
57        }
58    }
59
60    pub(crate) fn from_task_run(
61        name: impl Into<String>,
62        run: TaskRun,
63        failures: Vec<String>,
64        judge: Option<JudgeSummary>,
65        retention: WorkspaceRetention,
66    ) -> Self {
67        let passed = failures.is_empty();
68        let failure_context = (!passed).then(|| run.failure_context());
69        let tool_calls = EvalToolCall::from_task_run(&run);
70        let retained_workspace = match retention {
71            WorkspaceRetention::Retain => Some(run.into_workspace().persist()),
72            WorkspaceRetention::Discard => None,
73        };
74
75        Self { name: name.into(), passed, failures, tool_calls, judge, failure_context, retained_workspace }
76    }
77}
78
79impl EvalToolCall {
80    fn from_task_run(run: &TaskRun) -> Vec<Self> {
81        run.transcript()
82            .all_tool_calls()
83            .map(|call| Self {
84                name: call.name.to_string(),
85                arguments: call.arguments_json().ok(),
86                raw_arguments: call.arguments.to_string(),
87            })
88            .collect()
89    }
90}
91
92#[derive(Debug, Clone, Serialize, JsonSchema)]
93#[serde(rename_all = "camelCase", deny_unknown_fields)]
94pub struct JudgeSummary {
95    pub passed: bool,
96    pub score: f64,
97    pub reason: String,
98    pub criteria: Vec<JudgeCriterionSummary>,
99}
100
101#[derive(Debug, Clone, Serialize, JsonSchema)]
102#[serde(rename_all = "camelCase", deny_unknown_fields)]
103pub struct JudgeCriterionSummary {
104    pub id: String,
105    pub description: String,
106    pub blocking: bool,
107    pub weight: f64,
108    pub threshold: f64,
109    pub score: f64,
110    pub reason: String,
111}
112
113impl EvalFilesReport {
114    pub fn passed(&self) -> bool {
115        self.evals.iter().all(|eval| eval.passed)
116    }
117
118    pub fn passed_count(&self) -> usize {
119        self.evals.iter().filter(|eval| eval.passed).count()
120    }
121
122    pub fn failed_count(&self) -> usize {
123        self.evals.iter().filter(|eval| !eval.passed).count()
124    }
125}
126
127impl JudgeSummary {
128    /// Failure messages for blocking criteria that scored below their threshold.
129    pub fn blocking_failures(&self) -> impl Iterator<Item = String> + '_ {
130        self.criteria
131            .iter()
132            .filter(|criterion| criterion.blocking && !criterion.passed())
133            .map(|criterion| format!("judge criterion `{}`: {}", criterion.id, criterion.reason))
134    }
135}
136
137impl JudgeCriterionSummary {
138    pub fn passed(&self) -> bool {
139        self.score >= self.threshold
140    }
141}
142
143#[cfg(test)]
144mod tests {
145    use super::*;
146
147    #[test]
148    fn blocking_failures_report_only_blocking_criteria_below_threshold() {
149        let criterion = |id: &str, blocking, score| JudgeCriterionSummary {
150            id: id.to_string(),
151            description: "desc".to_string(),
152            blocking,
153            weight: 1.0,
154            threshold: 0.8,
155            score,
156            reason: format!("{id} reason"),
157        };
158        let summary = JudgeSummary {
159            passed: false,
160            score: 0.0,
161            reason: "r".to_string(),
162            criteria: vec![
163                criterion("met", true, 0.9),
164                criterion("failed", true, 0.5),
165                criterion("advisory", false, 0.0),
166            ],
167        };
168
169        let failures: Vec<String> = summary.blocking_failures().collect();
170
171        assert_eq!(failures, vec!["judge criterion `failed`: failed reason".to_string()]);
172    }
173}
aether_evals/spec/report.rs

aether_evals/spec/
report.rs
