aether_evals/spec/
report.rs

1use serde::Serialize;
2
3/// Outcome of running a collection of eval files.
4#[derive(Debug, Clone, Serialize)]
5pub struct EvalFilesReport {
6    pub evals: Vec<EvalOutcome>,
7}
8
9#[derive(Debug, Clone, Serialize)]
10pub struct EvalOutcome {
11    pub name: String,
12    pub passed: bool,
13    pub failures: Vec<String>,
14    #[serde(skip_serializing_if = "Option::is_none")]
15    pub judge: Option<JudgeSummary>,
16    #[serde(skip_serializing_if = "Option::is_none")]
17    pub failure_context: Option<String>,
18}
19
20#[derive(Debug, Clone, Serialize)]
21pub struct JudgeSummary {
22    pub passed: bool,
23    pub score: f64,
24    pub reason: String,
25    pub criteria: Vec<JudgeCriterionSummary>,
26}
27
28#[derive(Debug, Clone, Serialize)]
29pub struct JudgeCriterionSummary {
30    pub id: String,
31    pub description: String,
32    pub blocking: bool,
33    pub weight: f64,
34    pub threshold: f64,
35    pub score: f64,
36    pub reason: String,
37}
38
39impl EvalFilesReport {
40    pub fn passed(&self) -> bool {
41        self.evals.iter().all(|eval| eval.passed)
42    }
43
44    pub fn passed_count(&self) -> usize {
45        self.evals.iter().filter(|eval| eval.passed).count()
46    }
47
48    pub fn failed_count(&self) -> usize {
49        self.evals.iter().filter(|eval| !eval.passed).count()
50    }
51}
52
53impl JudgeSummary {
54    /// Failure messages for blocking criteria that scored below their threshold.
55    pub fn blocking_failures(&self) -> impl Iterator<Item = String> + '_ {
56        self.criteria
57            .iter()
58            .filter(|criterion| criterion.blocking && !criterion.passed())
59            .map(|criterion| format!("judge criterion `{}`: {}", criterion.id, criterion.reason))
60    }
61}
62
63impl JudgeCriterionSummary {
64    pub fn passed(&self) -> bool {
65        self.score >= self.threshold
66    }
67}
68
69#[cfg(test)]
70mod tests {
71    use super::*;
72
73    #[test]
74    fn blocking_failures_report_only_blocking_criteria_below_threshold() {
75        let criterion = |id: &str, blocking, score| JudgeCriterionSummary {
76            id: id.to_string(),
77            description: "desc".to_string(),
78            blocking,
79            weight: 1.0,
80            threshold: 0.8,
81            score,
82            reason: format!("{id} reason"),
83        };
84        let summary = JudgeSummary {
85            passed: false,
86            score: 0.0,
87            reason: "r".to_string(),
88            criteria: vec![
89                criterion("met", true, 0.9),
90                criterion("failed", true, 0.5),
91                criterion("advisory", false, 0.0),
92            ],
93        };
94
95        let failures: Vec<String> = summary.blocking_failures().collect();
96
97        assert_eq!(failures, vec!["judge criterion `failed`: failed reason".to_string()]);
98    }
99}
aether_evals/spec/report.rs

aether_evals/spec/
report.rs
