use std::fs;
use serde::Deserialize;
use crate::adapters::adapter_for;
use crate::core::{
Assertion, AssertionResult, Grader, GradingResult, GradingSummary, MetaSummary, RunRecord,
SKILL_INVOKED_META_ID, ToolInvocation,
};
use crate::pipeline::DiffScopeMetrics;
use crate::pipeline::error::PipelineError;
use crate::pipeline::io::write_json;
use crate::pipeline::slots::run_slots;
use crate::validation::{SchemaName, validate_against_schema};
use super::GradeContext;
use super::command_check::CommandCheckResult;
use super::diff_scope::grade_diff_scope;
use super::transcript_check::grade_transcript_check_with_context;
#[derive(Debug, Default, Clone, Copy)]
pub struct FinalizeSummary {
pub total_graded: usize,
pub total_meta_graded: usize,
pub total_unverifiable: usize,
pub meta_failures: usize,
}
#[derive(Debug, Deserialize)]
struct JudgeResponse {
#[serde(default)]
passed: bool,
#[serde(default)]
evidence: Option<String>,
#[serde(default)]
confidence: Option<f64>,
#[serde(default)]
grader: Option<Grader>,
}
pub fn finalize(ctx: &GradeContext) -> Result<FinalizeSummary, PipelineError> {
let conds: Vec<(String, Option<String>)> = ctx
.conditions
.conditions
.iter()
.map(|c| (c.name.clone(), c.skill_path.clone()))
.collect();
let mut summary = FinalizeSummary::default();
let transcript_vocabulary = ctx
.conditions
.harness
.map(|harness| adapter_for(harness).tool_vocabulary())
.unwrap_or_default();
for ev in &ctx.evals.evals {
let assertions = ev.assertions.as_deref().unwrap_or(&[]);
let has_assertions = !assertions.is_empty();
for (cond, cond_skill_path) in &conds {
let cond_dir = ctx.iteration_dir.join(format!("eval-{}", ev.id)).join(cond);
if !cond_dir.exists() {
continue;
}
for slot in run_slots(&cond_dir) {
let judge_responses_dir = slot.dir.join("judge-responses");
let grading_path = slot.dir.join("grading.json");
let run_record_path = slot.dir.join("run.json");
let run_record: Option<RunRecord> = if run_record_path.exists() {
Some(validate_against_schema(
SchemaName::RunRecord,
&serde_json::from_str(&fs::read_to_string(&run_record_path)?)?,
&run_record_path.to_string_lossy(),
)?)
} else {
None
};
let mut assertion_results: Vec<AssertionResult> = Vec::new();
if has_assertions {
for assertion in assertions {
match assertion {
Assertion::TranscriptCheck(tc) => {
let invocations: &[ToolInvocation] = run_record
.as_ref()
.map(|r| r.tool_invocations.as_slice())
.unwrap_or(&[]);
let conversation = run_record
.as_ref()
.and_then(|run| run.conversation.as_ref());
assertion_results.push(grade_transcript_check_with_context(
tc,
invocations,
conversation,
&transcript_vocabulary,
));
let unverifiable = match tc.check.as_str() {
"assistant_message_matches" => conversation.is_none(),
_ => invocations.is_empty(),
};
if unverifiable {
summary.total_unverifiable += 1;
} else {
summary.total_graded += 1;
}
}
Assertion::LlmJudge(j) => {
let response_path =
judge_responses_dir.join(format!("{}.json", j.id));
if !response_path.exists() {
eprintln!(
"warn: missing judge response: {} (assertion will be FAIL)",
response_path.display()
);
assertion_results.push(AssertionResult {
id: j.id.clone(),
passed: false,
evidence: format!(
"judge response missing at {}",
response_path.display()
),
confidence: Some(0.0),
grader: Some(Grader::LlmJudge),
});
continue;
}
let response: JudgeResponse =
serde_json::from_str(&fs::read_to_string(&response_path)?)?;
assertion_results.push(AssertionResult {
id: j.id.clone(),
passed: response.passed,
evidence: response.evidence.unwrap_or_default(),
confidence: Some(response.confidence.unwrap_or(0.0)),
grader: Some(Grader::LlmJudge),
});
summary.total_graded += 1;
}
Assertion::CommandCheck(check) => {
let result_path = slot
.dir
.join("command-checks")
.join(format!("{}.json", check.id));
if !result_path.exists() {
return Err(PipelineError::Message(format!(
"missing command_check result: {}. Run ingest (or grade without --finalize) before finalize.",
result_path.display()
)));
}
let result: CommandCheckResult = validate_against_schema(
SchemaName::CommandCheck,
&serde_json::from_str(&fs::read_to_string(&result_path)?)?,
&result_path.to_string_lossy(),
)?;
assertion_results.push(AssertionResult {
id: check.id.clone(),
passed: result.passed,
evidence: result.evidence,
confidence: Some(1.0),
grader: Some(Grader::CommandCheck),
});
summary.total_graded += 1;
}
Assertion::DiffScope(check) => {
let result_path = slot.dir.join("diff-scope.json");
if !result_path.exists() {
return Err(PipelineError::Message(format!(
"missing diff_scope result: {}. Run ingest before finalize; if this iteration predates diff-scope baselines, rebuild it first.",
result_path.display()
)));
}
let metrics: DiffScopeMetrics = validate_against_schema(
SchemaName::DiffScope,
&serde_json::from_str(&fs::read_to_string(&result_path)?)?,
&result_path.to_string_lossy(),
)?;
assertion_results.push(grade_diff_scope(check, metrics));
summary.total_graded += 1;
}
}
}
}
let mut meta_results: Vec<AssertionResult> = Vec::new();
if cond_skill_path.is_some() && ev.skill_should_trigger != Some(false) {
let response_path =
judge_responses_dir.join(format!("{SKILL_INVOKED_META_ID}.json"));
if response_path.exists() {
let response: JudgeResponse =
serde_json::from_str(&fs::read_to_string(&response_path)?)?;
let passed = response.passed;
meta_results.push(AssertionResult {
id: SKILL_INVOKED_META_ID.to_string(),
passed,
evidence: response.evidence.unwrap_or_default(),
confidence: Some(response.confidence.unwrap_or(0.0)),
grader: Some(response.grader.unwrap_or(Grader::LlmJudge)),
});
summary.total_meta_graded += 1;
if !passed {
summary.meta_failures += 1;
}
} else {
eprintln!(
"warn: missing skill-invocation meta response: {}",
response_path.display()
);
meta_results.push(AssertionResult {
id: SKILL_INVOKED_META_ID.to_string(),
passed: false,
evidence: format!(
"meta judge response missing at {}",
response_path.display()
),
confidence: Some(0.0),
grader: Some(Grader::LlmJudge),
});
}
}
let passed = assertion_results.iter().filter(|r| r.passed).count() as u32;
let total = assertion_results.len() as u32;
let meta_len = meta_results.len() as u32;
let meta_passed = meta_results.iter().filter(|r| r.passed).count() as u32;
let has_meta = !meta_results.is_empty();
let skill_invoked = has_meta.then(|| meta_results.iter().all(|r| r.passed));
let grading = GradingResult {
assertion_results,
meta_results: has_meta.then_some(meta_results),
summary: GradingSummary {
passed,
failed: total - passed,
total,
pass_rate: if total == 0 {
0.0
} else {
f64::from(passed) / f64::from(total)
},
},
meta_summary: has_meta.then_some(MetaSummary {
passed: meta_passed,
failed: meta_len - meta_passed,
total: meta_len,
skill_invoked,
}),
};
validate_against_schema::<serde_json::Value>(
SchemaName::Grading,
&serde_json::to_value(&grading)?,
&grading_path.to_string_lossy(),
)?;
write_json(&grading_path, &grading)?;
}
}
}
Ok(summary)
}