rustev_eval/
scores.rs

1//! Scorers, `rustev.score-views/1` and `rustev.scores/1` (spec 041, 3.6
2//! and 3.7).
3//!
4//! Scores enter evaluation only as a scores document bound to the views of
5//! the same evaluation (R-1): eval computes the views, a host's scorer
6//! scores each entry outside eval, and evaluation refuses a document over
7//! other views. Eval stays synchronous and I/O-free; a scorer is a Rust
8//! value the host composes in code (D-06).
9
10use std::collections::BTreeMap;
11
12use rustev_contract::Document;
13use rustev_contract::canonical::{CanonicalError, record_canonical_bytes, tagged_digest};
14use rustev_contract::eval_report::Split;
15use rustev_contract::ids::{ContentDigest, DatasetId, EvaluatorConfigId, PlanId};
16use rustev_contract::judgment::{Derivation, Notice, OutValue};
17use rustev_contract::limits::REPLAY_V1;
18use serde::{Deserialize, Serialize};
19
20use crate::dataset::Label;
21use crate::gold::Gold;
22use crate::scoring::{ScorerDescriptor, ScorerOutput};
23
24pub const SCORE_VIEWS: &str = "rustev.score-views/1";
25pub const SCORES: &str = "rustev.scores/1";
26/// The tag of a view entry's subject digest (3.7.1).
27pub const PROPOSAL_VIEW: &str = "rustev.proposal-view/1";
28
29/// Unscored reasons eval reserves (3.6.1, 3.7, 3.8, 3.9).
30pub mod reserved {
31    pub const INVALID_SCORE: &str = "invalid_score";
32    pub const JUDGE_BUNDLE_MISSING: &str = "judge_bundle_missing";
33    pub const JUDGE_NOT_REPRODUCED: &str = "judge_not_reproduced";
34    pub const SUBJECT_MISMATCH: &str = "subject_mismatch";
35    pub const JUDGE_UNRESOLVED: &str = "judge_unresolved";
36    pub const SCORER_FAILED: &str = "scorer_failed";
37    pub const SCORER_OVERLOADED: &str = "scorer_overloaded";
38    pub const SCORER_DEADLINE: &str = "scorer_deadline";
39    pub const ALL: [&str; 8] = [
40        INVALID_SCORE,
41        JUDGE_BUNDLE_MISSING,
42        JUDGE_NOT_REPRODUCED,
43        SUBJECT_MISMATCH,
44        JUDGE_UNRESOLVED,
45        SCORER_FAILED,
46        SCORER_OVERLOADED,
47        SCORER_DEADLINE,
48    ];
49}
50
51/// The views a scorer scores: one entry per comparable proposal case of
52/// one evaluation, in manifest order (3.7.1).
53#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
54#[serde(deny_unknown_fields)]
55pub struct ScoreViews {
56    pub schema: String,
57    pub dataset: DatasetId,
58    pub split: Split,
59    pub config: EvaluatorConfigId,
60    pub plan: PlanId,
61    pub cases: Vec<ViewEntry>,
62}
63
64crate::doc::eval_document!(ScoreViews, SCORE_VIEWS, REPLAY_V1);
65
66#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
67#[serde(deny_unknown_fields)]
68pub struct ViewEntry {
69    pub case: String,
70    pub case_view: CaseView,
71    pub proposal_view: ProposalView,
72    /// `tagged_digest("rustev.proposal-view/1", proposal view)`.
73    pub subject: ContentDigest,
74}
75
76/// What a scorer knows of a case. `gold` is null for a scorer that does
77/// not declare `reads_gold` (3.7.4).
78#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
79#[serde(deny_unknown_fields)]
80pub struct CaseView {
81    pub id: String,
82    pub label: Label,
83    pub gold: Option<Gold>,
84    pub subgroups: Vec<String>,
85}
86
87/// What a scorer knows of a proposal: the judgment's action, parameters,
88/// derivation and notices.
89#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
90#[serde(deny_unknown_fields)]
91pub struct ProposalView {
92    pub action: String,
93    pub params: BTreeMap<String, OutValue>,
94    pub derivation: Derivation,
95    pub notices: Vec<Notice>,
96}
97
98impl ProposalView {
99    /// The subject a score binds (3.7.1), over the record canonical bytes,
100    /// since parameters may hold binary64 values.
101    pub fn subject(&self) -> Result<ContentDigest, CanonicalError> {
102        let digest = tagged_digest(PROPOSAL_VIEW, &record_canonical_bytes(self)?);
103        Ok(ContentDigest::parse(&digest).expect("a tagged digest is a content digest"))
104    }
105}
106
107/// One scorer's scores over one views document (3.7.2).
108#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
109#[serde(deny_unknown_fields)]
110pub struct Scores {
111    pub schema: String,
112    pub scorer: ScorerDescriptor,
113    /// The views document's record digest.
114    pub views: ContentDigest,
115    pub entries: Vec<ScoreEntry>,
116}
117
118crate::doc::eval_document!(Scores, SCORES, REPLAY_V1);
119
120#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
121#[serde(deny_unknown_fields)]
122pub struct ScoreEntry {
123    pub case: String,
124    pub subject: ContentDigest,
125    pub outcome: ScoreOutcome,
126    /// The judge's run record digest for a model scorer; null otherwise.
127    pub judge_run: Option<ContentDigest>,
128}
129
130/// A score, or why there is none (3.6.1).
131#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
132#[serde(rename_all = "snake_case")]
133pub enum ScoreOutcome {
134    Score(f64),
135    Unscored(String),
136}
137
138/// A scorer the host composes in code (3.6, D-06).
139pub trait Scorer: Send + Sync {
140    fn descriptor(&self) -> &ScorerDescriptor;
141    fn score(&self, case: &CaseView, proposal: &ProposalView) -> ScoreOutcome;
142}
143
144/// 1 to 64 bytes of `[a-z0-9_]` (3.6.1).
145pub fn reason_ok(r: &str) -> bool {
146    (1..=64).contains(&r.len())
147        && r.bytes()
148            .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_')
149}
150
151/// Whether a score lies inside the scorer's output (3.7.5): a level index
152/// of an ordinal scale, or a finite value in a bounded range. Never
153/// clamped.
154pub fn score_ok(output: &ScorerOutput, value: f64) -> bool {
155    match output {
156        ScorerOutput::Ordinal { levels } => {
157            value.is_finite() && value.fract() == 0.0 && value >= 0.0 && value < levels.len() as f64
158        }
159        ScorerOutput::Bounded { min, max } => {
160            value.is_finite() && value >= min.to_f64_nearest() && value <= max.to_f64_nearest()
161        }
162    }
163}
164
165/// Calls the scorer once per entry, in order, synchronously (3.7.3). A
166/// scorer that does not declare `reads_gold` is handed the case view with
167/// `gold` null (3.7.4). A score outside the output, or an unscored reason
168/// that is malformed or reserved for eval, is recorded as unscored
169/// `invalid_score`, never clamped or passed through.
170pub fn score_views(scorer: &dyn Scorer, views: &ScoreViews) -> Result<Scores, CanonicalError> {
171    let d = scorer.descriptor();
172    let entries = views
173        .cases
174        .iter()
175        .map(|e| {
176            let withheld;
177            let case = if d.reads_gold {
178                &e.case_view
179            } else {
180                withheld = CaseView {
181                    gold: None,
182                    ..e.case_view.clone()
183                };
184                &withheld
185            };
186            let outcome = match scorer.score(case, &e.proposal_view) {
187                ScoreOutcome::Score(v) if score_ok(&d.output, v) => ScoreOutcome::Score(v),
188                ScoreOutcome::Unscored(r)
189                    if reason_ok(&r) && !reserved::ALL.contains(&r.as_str()) =>
190                {
191                    ScoreOutcome::Unscored(r)
192                }
193                _ => ScoreOutcome::Unscored(reserved::INVALID_SCORE.into()),
194            };
195            ScoreEntry {
196                case: e.case.clone(),
197                subject: e.subject.clone(),
198                outcome,
199                judge_run: None,
200            }
201        })
202        .collect();
203    Ok(Scores {
204        schema: SCORES.into(),
205        scorer: d.clone(),
206        views: views.record_digest()?,
207        entries,
208    })
209}
rustev_eval/scores.rs

rustev_eval/
scores.rs
