eval_magic/pipeline/grade/
finalize.rs

1//! Grading finalize.
2//!
3//! For each
4//! `(eval, condition)` it grades `transcript_check` assertions directly, folds in
5//! the `llm_judge` responses written by the orchestrator (missing → FAIL),
6//! assembles the skill-invocation meta result, and writes a schema-valid
7//! `grading.json` with pass/fail summaries.
8
9use std::fs;
10
11use serde::Deserialize;
12
13use crate::core::{
14    Assertion, AssertionResult, Grader, GradingResult, GradingSummary, MetaSummary, RunRecord,
15    SKILL_INVOKED_META_ID, ToolInvocation,
16};
17use crate::pipeline::error::PipelineError;
18use crate::pipeline::io::write_json;
19use crate::pipeline::slots::run_slots;
20use crate::validation::{SchemaName, validate_against_schema};
21
22use super::GradeContext;
23use super::transcript_check::grade_transcript_check;
24
25/// What finalize graded, for the CLI summary.
26#[derive(Debug, Default, Clone, Copy)]
27pub struct FinalizeSummary {
28    pub total_graded: usize,
29    pub total_meta_graded: usize,
30    pub total_unverifiable: usize,
31    pub meta_failures: usize,
32}
33
34/// A judge's verdict file. All fields tolerate absence (a sloppy judge response
35/// degrades to FAIL/0 rather than erroring the stage).
36#[derive(Debug, Deserialize)]
37struct JudgeResponse {
38    #[serde(default)]
39    passed: bool,
40    #[serde(default)]
41    evidence: Option<String>,
42    #[serde(default)]
43    confidence: Option<f64>,
44    #[serde(default)]
45    grader: Option<Grader>,
46}
47
48/// Fold judge responses + transcript checks into a `grading.json` per
49/// `(eval, condition)`. See the module docs for the per-assertion behavior.
50pub fn finalize(ctx: &GradeContext) -> Result<FinalizeSummary, PipelineError> {
51    let conds: Vec<(String, Option<String>)> = ctx
52        .conditions
53        .conditions
54        .iter()
55        .map(|c| (c.name.clone(), c.skill_path.clone()))
56        .collect();
57
58    let mut summary = FinalizeSummary::default();
59
60    for ev in &ctx.evals.evals {
61        let assertions = ev.assertions.as_deref().unwrap_or(&[]);
62        let has_assertions = !assertions.is_empty();
63
64        for (cond, cond_skill_path) in &conds {
65            let cond_dir = ctx.iteration_dir.join(format!("eval-{}", ev.id)).join(cond);
66            if !cond_dir.exists() {
67                continue;
68            }
69            for slot in run_slots(&cond_dir) {
70                let judge_responses_dir = slot.dir.join("judge-responses");
71                let grading_path = slot.dir.join("grading.json");
72
73                let run_record_path = slot.dir.join("run.json");
74                let run_record: Option<RunRecord> = if run_record_path.exists() {
75                    Some(validate_against_schema(
76                        SchemaName::RunRecord,
77                        &serde_json::from_str(&fs::read_to_string(&run_record_path)?)?,
78                        &run_record_path.to_string_lossy(),
79                    )?)
80                } else {
81                    None
82                };
83
84                let mut assertion_results: Vec<AssertionResult> = Vec::new();
85                if has_assertions {
86                    for assertion in assertions {
87                        match assertion {
88                            Assertion::TranscriptCheck(tc) => {
89                                let invocations: &[ToolInvocation] = run_record
90                                    .as_ref()
91                                    .map(|r| r.tool_invocations.as_slice())
92                                    .unwrap_or(&[]);
93                                assertion_results.push(grade_transcript_check(tc, invocations));
94                                if invocations.is_empty() {
95                                    summary.total_unverifiable += 1;
96                                } else {
97                                    summary.total_graded += 1;
98                                }
99                            }
100                            Assertion::LlmJudge(j) => {
101                                let response_path =
102                                    judge_responses_dir.join(format!("{}.json", j.id));
103                                if !response_path.exists() {
104                                    eprintln!(
105                                        "warn: missing judge response: {} (assertion will be FAIL)",
106                                        response_path.display()
107                                    );
108                                    assertion_results.push(AssertionResult {
109                                        id: j.id.clone(),
110                                        passed: false,
111                                        evidence: format!(
112                                            "judge response missing at {}",
113                                            response_path.display()
114                                        ),
115                                        confidence: Some(0.0),
116                                        grader: Some(Grader::LlmJudge),
117                                    });
118                                    continue;
119                                }
120                                let response: JudgeResponse =
121                                    serde_json::from_str(&fs::read_to_string(&response_path)?)?;
122                                assertion_results.push(AssertionResult {
123                                    id: j.id.clone(),
124                                    passed: response.passed,
125                                    evidence: response.evidence.unwrap_or_default(),
126                                    confidence: Some(response.confidence.unwrap_or(0.0)),
127                                    grader: Some(Grader::LlmJudge),
128                                });
129                                summary.total_graded += 1;
130                            }
131                        }
132                    }
133                }
134
135                // Mirror the emit gate: negative evals carry no meta-check.
136                let mut meta_results: Vec<AssertionResult> = Vec::new();
137                if cond_skill_path.is_some() && ev.skill_should_trigger != Some(false) {
138                    let response_path =
139                        judge_responses_dir.join(format!("{SKILL_INVOKED_META_ID}.json"));
140                    if response_path.exists() {
141                        let response: JudgeResponse =
142                            serde_json::from_str(&fs::read_to_string(&response_path)?)?;
143                        let passed = response.passed;
144                        meta_results.push(AssertionResult {
145                            id: SKILL_INVOKED_META_ID.to_string(),
146                            passed,
147                            evidence: response.evidence.unwrap_or_default(),
148                            confidence: Some(response.confidence.unwrap_or(0.0)),
149                            grader: Some(response.grader.unwrap_or(Grader::LlmJudge)),
150                        });
151                        summary.total_meta_graded += 1;
152                        if !passed {
153                            summary.meta_failures += 1;
154                        }
155                    } else {
156                        eprintln!(
157                            "warn: missing skill-invocation meta response: {}",
158                            response_path.display()
159                        );
160                        meta_results.push(AssertionResult {
161                            id: SKILL_INVOKED_META_ID.to_string(),
162                            passed: false,
163                            evidence: format!(
164                                "meta judge response missing at {}",
165                                response_path.display()
166                            ),
167                            confidence: Some(0.0),
168                            grader: Some(Grader::LlmJudge),
169                        });
170                    }
171                }
172
173                let passed = assertion_results.iter().filter(|r| r.passed).count() as u32;
174                let total = assertion_results.len() as u32;
175                let meta_len = meta_results.len() as u32;
176                let meta_passed = meta_results.iter().filter(|r| r.passed).count() as u32;
177                let has_meta = !meta_results.is_empty();
178                let skill_invoked = has_meta.then(|| meta_results.iter().all(|r| r.passed));
179
180                let grading = GradingResult {
181                    assertion_results,
182                    meta_results: has_meta.then_some(meta_results),
183                    summary: GradingSummary {
184                        passed,
185                        failed: total - passed,
186                        total,
187                        pass_rate: if total == 0 {
188                            0.0
189                        } else {
190                            f64::from(passed) / f64::from(total)
191                        },
192                    },
193                    meta_summary: has_meta.then_some(MetaSummary {
194                        passed: meta_passed,
195                        failed: meta_len - meta_passed,
196                        total: meta_len,
197                        skill_invoked,
198                    }),
199                };
200
201                validate_against_schema::<serde_json::Value>(
202                    SchemaName::Grading,
203                    &serde_json::to_value(&grading)?,
204                    &grading_path.to_string_lossy(),
205                )?;
206                write_json(&grading_path, &grading)?;
207            }
208        }
209    }
210
211    Ok(summary)
212}
eval_magic/pipeline/grade/finalize.rs

eval_magic/pipeline/grade/
finalize.rs
