eval_magic/pipeline/grade/
finalize.rs

1//! Grading finalize.
2//!
3//! For each
4//! `(eval, condition)` it grades `transcript_check` assertions directly, folds in
5//! persisted `command_check` results, deterministic `diff_scope` thresholds,
6//! and the `llm_judge` responses written by the orchestrator (missing → FAIL),
7//! assembles the skill-invocation meta result, and writes a schema-valid
8//! `grading.json` with pass/fail summaries.
9
10use std::fs;
11
12use serde::Deserialize;
13
14use crate::adapters::adapter_for;
15use crate::core::{
16    Assertion, AssertionResult, Grader, GradingResult, GradingSummary, MetaSummary, RunRecord,
17    SKILL_INVOKED_META_ID, ToolInvocation,
18};
19use crate::pipeline::DiffScopeMetrics;
20use crate::pipeline::error::PipelineError;
21use crate::pipeline::io::write_json;
22use crate::pipeline::slots::run_slots;
23use crate::validation::{SchemaName, validate_against_schema};
24
25use super::GradeContext;
26use super::command_check::CommandCheckResult;
27use super::diff_scope::grade_diff_scope;
28use super::transcript_check::grade_transcript_check_with_context;
29
30/// What finalize graded, for the CLI summary.
31#[derive(Debug, Default, Clone, Copy)]
32pub struct FinalizeSummary {
33    pub total_graded: usize,
34    pub total_meta_graded: usize,
35    pub total_unverifiable: usize,
36    pub meta_failures: usize,
37}
38
39/// A judge's verdict file. All fields tolerate absence (a sloppy judge response
40/// degrades to FAIL/0 rather than erroring the stage).
41#[derive(Debug, Deserialize)]
42struct JudgeResponse {
43    #[serde(default)]
44    passed: bool,
45    #[serde(default)]
46    evidence: Option<String>,
47    #[serde(default)]
48    confidence: Option<f64>,
49    #[serde(default)]
50    grader: Option<Grader>,
51}
52
53/// Fold runner checks and judge responses into a `grading.json` per
54/// `(eval, condition)`. See the module docs for the per-assertion behavior.
55pub fn finalize(ctx: &GradeContext) -> Result<FinalizeSummary, PipelineError> {
56    let conds: Vec<(String, Option<String>)> = ctx
57        .conditions
58        .conditions
59        .iter()
60        .map(|c| (c.name.clone(), c.skill_path.clone()))
61        .collect();
62
63    let mut summary = FinalizeSummary::default();
64    let transcript_vocabulary = ctx
65        .conditions
66        .harness
67        .map(|harness| adapter_for(harness).tool_vocabulary())
68        .unwrap_or_default();
69
70    for ev in &ctx.evals.evals {
71        let assertions = ev.assertions.as_deref().unwrap_or(&[]);
72        let has_assertions = !assertions.is_empty();
73
74        for (cond, cond_skill_path) in &conds {
75            let cond_dir = ctx.iteration_dir.join(format!("eval-{}", ev.id)).join(cond);
76            if !cond_dir.exists() {
77                continue;
78            }
79            for slot in run_slots(&cond_dir) {
80                let judge_responses_dir = slot.dir.join("judge-responses");
81                let grading_path = slot.dir.join("grading.json");
82
83                let run_record_path = slot.dir.join("run.json");
84                let run_record: Option<RunRecord> = if run_record_path.exists() {
85                    Some(validate_against_schema(
86                        SchemaName::RunRecord,
87                        &serde_json::from_str(&fs::read_to_string(&run_record_path)?)?,
88                        &run_record_path.to_string_lossy(),
89                    )?)
90                } else {
91                    None
92                };
93
94                let mut assertion_results: Vec<AssertionResult> = Vec::new();
95                if has_assertions {
96                    for assertion in assertions {
97                        match assertion {
98                            Assertion::TranscriptCheck(tc) => {
99                                let invocations: &[ToolInvocation] = run_record
100                                    .as_ref()
101                                    .map(|r| r.tool_invocations.as_slice())
102                                    .unwrap_or(&[]);
103                                let conversation = run_record
104                                    .as_ref()
105                                    .and_then(|run| run.conversation.as_ref());
106                                assertion_results.push(grade_transcript_check_with_context(
107                                    tc,
108                                    invocations,
109                                    conversation,
110                                    &transcript_vocabulary,
111                                ));
112                                let unverifiable = match tc.check.as_str() {
113                                    "assistant_message_matches" => conversation.is_none(),
114                                    _ => invocations.is_empty(),
115                                };
116                                if unverifiable {
117                                    summary.total_unverifiable += 1;
118                                } else {
119                                    summary.total_graded += 1;
120                                }
121                            }
122                            Assertion::LlmJudge(j) => {
123                                let response_path =
124                                    judge_responses_dir.join(format!("{}.json", j.id));
125                                if !response_path.exists() {
126                                    eprintln!(
127                                        "warn: missing judge response: {} (assertion will be FAIL)",
128                                        response_path.display()
129                                    );
130                                    assertion_results.push(AssertionResult {
131                                        id: j.id.clone(),
132                                        passed: false,
133                                        evidence: format!(
134                                            "judge response missing at {}",
135                                            response_path.display()
136                                        ),
137                                        confidence: Some(0.0),
138                                        grader: Some(Grader::LlmJudge),
139                                    });
140                                    continue;
141                                }
142                                let response: JudgeResponse =
143                                    serde_json::from_str(&fs::read_to_string(&response_path)?)?;
144                                assertion_results.push(AssertionResult {
145                                    id: j.id.clone(),
146                                    passed: response.passed,
147                                    evidence: response.evidence.unwrap_or_default(),
148                                    confidence: Some(response.confidence.unwrap_or(0.0)),
149                                    grader: Some(Grader::LlmJudge),
150                                });
151                                summary.total_graded += 1;
152                            }
153                            Assertion::CommandCheck(check) => {
154                                let result_path = slot
155                                    .dir
156                                    .join("command-checks")
157                                    .join(format!("{}.json", check.id));
158                                if !result_path.exists() {
159                                    return Err(PipelineError::Message(format!(
160                                        "missing command_check result: {}. Run ingest (or grade without --finalize) before finalize.",
161                                        result_path.display()
162                                    )));
163                                }
164                                let result: CommandCheckResult = validate_against_schema(
165                                    SchemaName::CommandCheck,
166                                    &serde_json::from_str(&fs::read_to_string(&result_path)?)?,
167                                    &result_path.to_string_lossy(),
168                                )?;
169                                assertion_results.push(AssertionResult {
170                                    id: check.id.clone(),
171                                    passed: result.passed,
172                                    evidence: result.evidence,
173                                    confidence: Some(1.0),
174                                    grader: Some(Grader::CommandCheck),
175                                });
176                                summary.total_graded += 1;
177                            }
178                            Assertion::DiffScope(check) => {
179                                let result_path = slot.dir.join("diff-scope.json");
180                                if !result_path.exists() {
181                                    return Err(PipelineError::Message(format!(
182                                        "missing diff_scope result: {}. Run ingest before finalize; if this iteration predates diff-scope baselines, rebuild it first.",
183                                        result_path.display()
184                                    )));
185                                }
186                                let metrics: DiffScopeMetrics = validate_against_schema(
187                                    SchemaName::DiffScope,
188                                    &serde_json::from_str(&fs::read_to_string(&result_path)?)?,
189                                    &result_path.to_string_lossy(),
190                                )?;
191                                assertion_results.push(grade_diff_scope(check, metrics));
192                                summary.total_graded += 1;
193                            }
194                        }
195                    }
196                }
197
198                // Mirror the emit gate: negative evals carry no meta-check.
199                let mut meta_results: Vec<AssertionResult> = Vec::new();
200                if cond_skill_path.is_some() && ev.skill_should_trigger != Some(false) {
201                    let response_path =
202                        judge_responses_dir.join(format!("{SKILL_INVOKED_META_ID}.json"));
203                    if response_path.exists() {
204                        let response: JudgeResponse =
205                            serde_json::from_str(&fs::read_to_string(&response_path)?)?;
206                        let passed = response.passed;
207                        meta_results.push(AssertionResult {
208                            id: SKILL_INVOKED_META_ID.to_string(),
209                            passed,
210                            evidence: response.evidence.unwrap_or_default(),
211                            confidence: Some(response.confidence.unwrap_or(0.0)),
212                            grader: Some(response.grader.unwrap_or(Grader::LlmJudge)),
213                        });
214                        summary.total_meta_graded += 1;
215                        if !passed {
216                            summary.meta_failures += 1;
217                        }
218                    } else {
219                        eprintln!(
220                            "warn: missing skill-invocation meta response: {}",
221                            response_path.display()
222                        );
223                        meta_results.push(AssertionResult {
224                            id: SKILL_INVOKED_META_ID.to_string(),
225                            passed: false,
226                            evidence: format!(
227                                "meta judge response missing at {}",
228                                response_path.display()
229                            ),
230                            confidence: Some(0.0),
231                            grader: Some(Grader::LlmJudge),
232                        });
233                    }
234                }
235
236                let passed = assertion_results.iter().filter(|r| r.passed).count() as u32;
237                let total = assertion_results.len() as u32;
238                let meta_len = meta_results.len() as u32;
239                let meta_passed = meta_results.iter().filter(|r| r.passed).count() as u32;
240                let has_meta = !meta_results.is_empty();
241                let skill_invoked = has_meta.then(|| meta_results.iter().all(|r| r.passed));
242
243                let grading = GradingResult {
244                    assertion_results,
245                    meta_results: has_meta.then_some(meta_results),
246                    summary: GradingSummary {
247                        passed,
248                        failed: total - passed,
249                        total,
250                        pass_rate: if total == 0 {
251                            0.0
252                        } else {
253                            f64::from(passed) / f64::from(total)
254                        },
255                    },
256                    meta_summary: has_meta.then_some(MetaSummary {
257                        passed: meta_passed,
258                        failed: meta_len - meta_passed,
259                        total: meta_len,
260                        skill_invoked,
261                    }),
262                };
263
264                validate_against_schema::<serde_json::Value>(
265                    SchemaName::Grading,
266                    &serde_json::to_value(&grading)?,
267                    &grading_path.to_string_lossy(),
268                )?;
269                write_json(&grading_path, &grading)?;
270            }
271        }
272    }
273
274    Ok(summary)
275}
eval_magic/pipeline/grade/finalize.rs

eval_magic/pipeline/grade/
finalize.rs
