rustev_eval/
judge.rs

1//! Judge reproduction (spec 041, 3.8.1 to 3.8.3).
2//!
3//! A model judge is an ordinary plan with a `rubric` step. The host runs it
4//! once per view entry, with capture, and keeps the bundles; this module
5//! reproduces each bundle offline with spec 004's reproduction and writes
6//! the judge's `rustev.scores/1`. Missing or unusable evidence leaves the
7//! case unscored with its reason (I-2). A bundle of another plan, or a
8//! judgment whose derivation contradicts the descriptor, refuses the whole
9//! call: either shows that the judge is not the one the configuration
10//! declares, which taints every score in the document.
11
12use std::collections::BTreeMap;
13use std::fmt;
14
15use rustev_contract::Document;
16use rustev_contract::ids::PlanId;
17use rustev_contract::judgment::{Derivation, OutValue, Outcome};
18use serde_json::Value;
19
20use crate::bundle::BundleInput;
21use crate::replay::{ReplayConfig, ReplayOutcome, reproduce};
22use crate::resolve::Resolver;
23use crate::scores::{SCORES, ScoreEntry, ScoreOutcome, ScoreViews, Scores, reserved};
24use crate::scoring::{ScorerDescriptor, ScorerKind, ScorerOutput};
25
26/// Why a judge reproduction call was refused, by refusal code (3.8.3).
27#[derive(Debug, Clone, PartialEq, Eq)]
28pub enum JudgeError {
29    /// The descriptor is not an ordinal model scorer with a judge, or names
30    /// another plan than the one given.
31    NotAJudge(String),
32    /// A bundle of another plan than the judge's (`judge-plan-mismatch`).
33    PlanMismatch {
34        case: String,
35    },
36    /// An `exact-derived` judgment under a model descriptor
37    /// (`judge-derivation-mismatch`).
38    DerivationMismatch {
39        case: String,
40    },
41    NotCanonical,
42}
43
44impl JudgeError {
45    pub fn code(&self) -> &'static str {
46        match self {
47            JudgeError::NotAJudge(_) => "not-a-judge",
48            JudgeError::PlanMismatch { .. } => "judge-plan-mismatch",
49            JudgeError::DerivationMismatch { .. } => "judge-derivation-mismatch",
50            JudgeError::NotCanonical => "not-canonical",
51        }
52    }
53}
54
55impl fmt::Display for JudgeError {
56    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
57        write!(f, "{}: {self:?}", self.code())
58    }
59}
60
61impl std::error::Error for JudgeError {}
62
63/// Reproduces each view entry's judge bundle (`bundles`, by case id) and
64/// writes the judge's scores (3.8.3). `plan` is the judge plan's identity,
65/// which the descriptor must name.
66pub fn judge_scores(
67    views: &ScoreViews,
68    descriptor: &ScorerDescriptor,
69    plan: &PlanId,
70    bundles: &BTreeMap<String, BundleInput>,
71    resolver: &dyn Resolver,
72    replay: &ReplayConfig,
73) -> Result<Scores, JudgeError> {
74    let (Some(judge), ScorerKind::Model, ScorerOutput::Ordinal { levels }) =
75        (&descriptor.judge, descriptor.kind, &descriptor.output)
76    else {
77        return Err(JudgeError::NotAJudge(
78            "a judge is an ordinal model scorer with a judge reference".into(),
79        ));
80    };
81    if &judge.plan != plan {
82        return Err(JudgeError::NotAJudge(
83            "the descriptor names another judge plan".into(),
84        ));
85    }
86    let mut entries = Vec::with_capacity(views.cases.len());
87    for e in &views.cases {
88        let unscored = |reason: &str| ScoreEntry {
89            case: e.case.clone(),
90            subject: e.subject.clone(),
91            outcome: ScoreOutcome::Unscored(reason.into()),
92            judge_run: None,
93        };
94        let bundle = match bundles.get(&e.case) {
95            None => {
96                entries.push(unscored(reserved::JUDGE_BUNDLE_MISSING));
97                continue;
98            }
99            Some(BundleInput::Failed(_)) => {
100                entries.push(unscored(reserved::JUDGE_NOT_REPRODUCED));
101                continue;
102            }
103            Some(BundleInput::Loaded(b)) => b,
104        };
105        if &bundle.plan_id != plan {
106            return Err(JudgeError::PlanMismatch {
107                case: e.case.clone(),
108            });
109        }
110        let r = match reproduce(bundle, resolver, replay) {
111            ReplayOutcome::Reproduced(r) => r,
112            ReplayOutcome::Diverged { .. } | ReplayOutcome::Incomparable(_) => {
113                entries.push(unscored(reserved::JUDGE_NOT_REPRODUCED));
114                continue;
115            }
116        };
117        let subject = Value::String(e.subject.to_string());
118        let supplied = r
119            .snapshot()
120            .entries
121            .iter()
122            .find(|x| x.field == judge.subject_field)
123            .map(|x| &x.value);
124        if supplied != Some(&subject) {
125            entries.push(unscored(reserved::SUBJECT_MISMATCH));
126            continue;
127        }
128        let j = r.judgment();
129        if j.derivation == Derivation::ExactDerived {
130            return Err(JudgeError::DerivationMismatch {
131                case: e.case.clone(),
132            });
133        }
134        let judge_run = Some(
135            r.run()
136                .record_digest()
137                .map_err(|_| JudgeError::NotCanonical)?,
138        );
139        let outcome = match &j.outcome {
140            Outcome::Propose { params, .. } => match params.get(&judge.level_param) {
141                Some(OutValue::Enum(level)) => match levels.iter().position(|l| l == level) {
142                    Some(i) => ScoreOutcome::Score(i as f64),
143                    None => ScoreOutcome::Unscored(reserved::INVALID_SCORE.into()),
144                },
145                _ => ScoreOutcome::Unscored(reserved::INVALID_SCORE.into()),
146            },
147            // An escalation or any unresolved judgment.
148            _ => ScoreOutcome::Unscored(reserved::JUDGE_UNRESOLVED.into()),
149        };
150        entries.push(ScoreEntry {
151            case: e.case.clone(),
152            subject: e.subject.clone(),
153            outcome,
154            judge_run,
155        });
156    }
157    Ok(Scores {
158        schema: SCORES.into(),
159        scorer: descriptor.clone(),
160        views: views
161            .record_digest()
162            .map_err(|_| JudgeError::NotCanonical)?,
163        entries,
164    })
165}
rustev_eval/judge.rs

rustev_eval/
judge.rs
