rustev_eval/
scoring.rs

1//! Scorer, rubric, human scale, agreement and origin declarations of
2//! `rustev.evaluator-config/3` (spec 041, 3.5 to 3.8).
3//!
4//! These are declared data: a scorer's descriptor names its identity,
5//! output and rules, so it enters the `EvaluatorConfigId`. A model scorer
6//! is a judge whose plan and backend identities are bound here too. The
7//! checks refuse every declaration that would mix a model metric with a
8//! deterministic one, read an ordinal scale as the wrong kind, or report an
9//! origin family before its evidence is read.
10
11use std::collections::{BTreeMap, BTreeSet};
12
13use rustev_contract::decimal::Decimal;
14use rustev_contract::ids::{ArtifactId, PlanId};
15use serde::{Deserialize, Serialize};
16
17use crate::config::{AdapterRules, ConfigError};
18use crate::gold::MAX_NAME;
19use crate::measures::declared_name;
20
21/// The most scorers one configuration declares (3.6.2).
22pub const MAX_SCORERS: usize = 16;
23/// An ordinal scale holds 2 to 16 levels (3.6.2).
24pub const MAX_LEVELS: usize = 16;
25/// The reserved prefix of every model-derived metric (3.8.5).
26pub const MODEL_PREFIX: &str = "model/";
27/// An agreement side naming a human scale (3.8.4).
28pub const HUMAN_SIDE: &str = "human:";
29
30/// Refusal codes of spec 041 under `invalid_input` (3.10.5).
31pub mod code {
32    pub const SCALE_MISMATCH: &str = "scale-mismatch";
33    pub const JUDGE_OUTPUT_KIND: &str = "judge-output-kind";
34    pub const MODEL_PREFIX: &str = "model-prefix";
35    pub const METRIC_UNAVAILABLE: &str = "metric-unavailable";
36}
37
38/// A scorer's descriptor (3.6.2).
39#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
40#[serde(deny_unknown_fields)]
41pub struct ScorerDescriptor {
42    pub id: String,
43    pub version: String,
44    pub kind: ScorerKind,
45    pub output: ScorerOutput,
46    /// The digest of a canonical document that fully determines the
47    /// scoring, or `opaque`.
48    pub rules: AdapterRules,
49    /// Whether the scorer is handed the case's gold (3.7.4).
50    pub reads_gold: bool,
51    /// A model scorer's judge (3.8.1); absent for a deterministic one.
52    #[serde(default, skip_serializing_if = "Option::is_none")]
53    pub judge: Option<JudgeRef>,
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
57#[serde(rename_all = "snake_case")]
58pub enum ScorerKind {
59    Deterministic,
60    Model,
61}
62
63/// What a score is (3.6.2).
64#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
65#[serde(rename_all = "snake_case", deny_unknown_fields)]
66pub enum ScorerOutput {
67    /// Level names, lowest first; the value is the level index.
68    Ordinal { levels: Vec<String> },
69    /// A finite binary64 in `[min, max]`.
70    Bounded { min: Decimal, max: Decimal },
71}
72
73/// A judge plan and the backend identities serving it (3.8.1).
74#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
75#[serde(deny_unknown_fields)]
76pub struct JudgeRef {
77    pub plan: PlanId,
78    pub artifacts: Vec<ArtifactId>,
79    /// The proposal parameter whose enum option is the level.
80    pub level_param: String,
81    /// The input field the view's subject is supplied as.
82    pub subject_field: String,
83}
84
85/// A rubric metric (3.6.3), reported as `rubric/<name>` or, for a model
86/// scorer, `model/rubric/<name>`.
87#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
88#[serde(deny_unknown_fields)]
89pub struct RubricMetric {
90    pub name: String,
91    pub scorer: String,
92    pub aggregate: RubricAggregate,
93}
94
95#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
96#[serde(rename_all = "snake_case")]
97pub enum RubricAggregate {
98    /// Ordinal: the mean level index.
99    MeanLevelIndex,
100    /// Ordinal: scored cases at or above the level over scored cases.
101    ShareAtLeast(String),
102    /// Bounded: the mean score.
103    Mean,
104}
105
106/// A human rating scale (3.6.4).
107#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
108#[serde(deny_unknown_fields)]
109pub struct HumanScale {
110    pub name: String,
111    pub levels: Vec<String>,
112}
113
114/// An agreement between two ordinal sides (3.8.4): each a declared
115/// scorer id or `human:<scale>`.
116#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
117#[serde(deny_unknown_fields)]
118pub struct AgreementDecl {
119    pub name: String,
120    pub a: String,
121    pub b: String,
122}
123
124/// An origin and evidence-quality family (3.5.1).
125#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
126#[serde(rename_all = "snake_case")]
127pub enum OriginFamily {
128    TextBackstop,
129    Extraction,
130    Screening,
131    Corroboration,
132    AcquisitionTier,
133}
134
135impl OriginFamily {
136    pub fn name(self) -> &'static str {
137        match self {
138            OriginFamily::TextBackstop => "text_backstop",
139            OriginFamily::Extraction => "extraction",
140            OriginFamily::Screening => "screening",
141            OriginFamily::Corroboration => "corroboration",
142            OriginFamily::AcquisitionTier => "acquisition_tier",
143        }
144    }
145
146    /// Whether this build reads the family's evidence (3.5.1): every
147    /// family's owning spec has its evidence on `main`.
148    pub fn available(self) -> bool {
149        true
150    }
151}
152
153/// The version 3 members of an evaluator configuration (3.1.3).
154pub struct Declarations<'a> {
155    pub scorers: &'a [ScorerDescriptor],
156    pub rubric_metrics: &'a [RubricMetric],
157    pub human_scales: &'a [HumanScale],
158    pub agreements: &'a [AgreementDecl],
159    pub origin: &'a [OriginFamily],
160}
161
162fn refuse<T>(code: &str, detail: impl std::fmt::Display) -> Result<T, ConfigError> {
163    Err(ConfigError(format!("{code}: {detail}")))
164}
165
166fn bad<T>(detail: &str) -> Result<T, ConfigError> {
167    Err(ConfigError(detail.into()))
168}
169
170/// A level or scale name: 1 to 256 bytes, as gold names are (3.3.2).
171/// Neither enters a metric name, so no character set is imposed.
172fn label_ok(s: &str) -> bool {
173    (1..=MAX_NAME).contains(&s.len())
174}
175
176fn levels_ok(levels: &[String]) -> bool {
177    let set: BTreeSet<&str> = levels.iter().map(String::as_str).collect();
178    (2..=MAX_LEVELS).contains(&levels.len())
179        && set.len() == levels.len()
180        && levels.iter().all(|l| label_ok(l))
181}
182
183/// One side of an agreement, resolved: its levels and whether it is a
184/// model scorer.
185struct Side<'a> {
186    levels: &'a [String],
187    model: bool,
188    opaque: bool,
189}
190
191impl Declarations<'_> {
192    fn scorer(&self, id: &str) -> Option<&ScorerDescriptor> {
193        self.scorers.iter().find(|s| s.id == id)
194    }
195
196    fn side(&self, side: &str) -> Result<Side<'_>, ConfigError> {
197        if let Some(scale) = side.strip_prefix(HUMAN_SIDE) {
198            let Some(h) = self.human_scales.iter().find(|h| h.name == scale) else {
199                return bad("an agreement names a human scale the configuration does not declare");
200            };
201            return Ok(Side {
202                levels: &h.levels,
203                model: false,
204                opaque: false,
205            });
206        }
207        let Some(s) = self.scorer(side) else {
208            return bad("an agreement names a scorer the configuration does not declare");
209        };
210        match &s.output {
211            ScorerOutput::Ordinal { levels } => Ok(Side {
212                levels,
213                model: s.kind == ScorerKind::Model,
214                opaque: s.rules == AdapterRules::Opaque,
215            }),
216            ScorerOutput::Bounded { .. } => refuse(
217                code::SCALE_MISMATCH,
218                format!("agreement side {side} is not ordinal"),
219            ),
220        }
221    }
222
223    /// Every metric name these declarations report, with whether it is
224    /// model-derived and whether an opaque scorer produces it (3.6.3,
225    /// 3.8.4, 3.12).
226    pub fn metric_names(&self) -> BTreeMap<String, (bool, bool)> {
227        let mut out = BTreeMap::new();
228        for r in self.rubric_metrics {
229            if let Some(s) = self.scorer(&r.scorer) {
230                let model = s.kind == ScorerKind::Model;
231                let prefix = if model { MODEL_PREFIX } else { "" };
232                out.insert(
233                    format!("{prefix}rubric/{}", r.name),
234                    (model, s.rules == AdapterRules::Opaque),
235                );
236            }
237        }
238        for a in self.agreements {
239            if let (Ok(x), Ok(y)) = (self.side(&a.a), self.side(&a.b)) {
240                let model = x.model || y.model;
241                let prefix = if model { MODEL_PREFIX } else { "" };
242                for stat in ["exact", "adjacent", "mean_abs_diff", "qwk"] {
243                    out.insert(
244                        format!("{prefix}agreement/{}/{stat}", a.name),
245                        (model, x.opaque || y.opaque),
246                    );
247                }
248            }
249        }
250        out
251    }
252
253    /// The rules of 3.5 to 3.8.
254    pub fn check(&self) -> Result<(), ConfigError> {
255        if self.scorers.len() > MAX_SCORERS {
256            return bad("at most 16 scorers");
257        }
258        let mut ids = BTreeSet::new();
259        for s in self.scorers {
260            if !declared_name(&s.id) || !ids.insert(s.id.as_str()) {
261                return bad("scorer ids are unique, 1 to 64 bytes of [A-Za-z0-9._-]");
262            }
263            if !(1..=64).contains(&s.version.len()) {
264                return bad("a scorer version is 1 to 64 bytes");
265            }
266            match &s.output {
267                ScorerOutput::Ordinal { levels } if !levels_ok(levels) => {
268                    return bad("an ordinal scorer declares 2 to 16 unique level names");
269                }
270                ScorerOutput::Bounded { min, max } if min >= max => {
271                    return bad("a bounded scorer's min is below its max");
272                }
273                _ => {}
274            }
275            match (s.kind, &s.judge, &s.output) {
276                (ScorerKind::Model, Some(_), ScorerOutput::Bounded { .. }) => {
277                    return refuse(code::JUDGE_OUTPUT_KIND, &s.id);
278                }
279                (ScorerKind::Model, None, _) => {
280                    return bad("a model scorer names its judge");
281                }
282                (ScorerKind::Deterministic, Some(_), _) => {
283                    return bad("only a model scorer names a judge");
284                }
285                _ => {}
286            }
287            if let Some(j) = &s.judge
288                && (!(1..=256).contains(&j.level_param.len())
289                    || !(1..=256).contains(&j.subject_field.len()))
290            {
291                return bad("a judge's level_param and subject_field are 1 to 256 bytes");
292            }
293        }
294        let mut names = BTreeSet::new();
295        for r in self.rubric_metrics {
296            if !declared_name(&r.name) || !names.insert(r.name.as_str()) {
297                return bad("rubric metric names are unique, 1 to 64 bytes of [A-Za-z0-9._-]");
298            }
299            let Some(s) = self.scorer(&r.scorer) else {
300                return bad("a rubric metric names a scorer the configuration does not declare");
301            };
302            match (&r.aggregate, &s.output) {
303                (RubricAggregate::MeanLevelIndex, ScorerOutput::Ordinal { .. })
304                | (RubricAggregate::Mean, ScorerOutput::Bounded { .. }) => {}
305                (RubricAggregate::ShareAtLeast(l), ScorerOutput::Ordinal { levels })
306                    if levels.contains(l) => {}
307                _ => return refuse(code::SCALE_MISMATCH, &r.name),
308            }
309        }
310        let mut scales = BTreeSet::new();
311        for h in self.human_scales {
312            if !label_ok(&h.name) || !scales.insert(h.name.as_str()) {
313                return bad("human scale names are unique, 1 to 256 bytes");
314            }
315            if !levels_ok(&h.levels) {
316                return bad("a human scale declares 2 to 16 unique level names");
317            }
318        }
319        let mut agreements = BTreeSet::new();
320        for a in self.agreements {
321            if !declared_name(&a.name) || !agreements.insert(a.name.as_str()) {
322                return bad("agreement names are unique, 1 to 64 bytes of [A-Za-z0-9._-]");
323            }
324            let (x, y) = (self.side(&a.a)?, self.side(&a.b)?);
325            if x.levels != y.levels {
326                return refuse(code::SCALE_MISMATCH, &a.name);
327            }
328        }
329        let mut families = BTreeSet::new();
330        for f in self.origin {
331            if !families.insert(f) {
332                return bad("origin families are unique");
333            }
334            if !f.available() {
335                return refuse(code::METRIC_UNAVAILABLE, f.name());
336            }
337        }
338        Ok(())
339    }
340}
341
342/// The model-prefix rule for a metric a gate or formula names (3.8.5): a
343/// rubric or agreement metric of a model scorer is named under `model/`,
344/// and a deterministic one never is.
345pub fn check_prefix(
346    metric: &str,
347    names: &BTreeMap<String, (bool, bool)>,
348) -> Result<(), ConfigError> {
349    let bare = metric.strip_prefix(MODEL_PREFIX).unwrap_or(metric);
350    let base = bare.split('/').next().unwrap_or_default();
351    if base != "rubric" && base != "agreement" {
352        return Ok(());
353    }
354    let prefixed = format!("{MODEL_PREFIX}{bare}");
355    let model = names.get(bare).is_none() && names.get(&prefixed).is_some_and(|(m, _)| *m);
356    let named_model = metric.starts_with(MODEL_PREFIX);
357    let known = names.contains_key(bare) || names.contains_key(&prefixed);
358    if known && model != named_model {
359        return refuse(code::MODEL_PREFIX, metric);
360    }
361    Ok(())
362}
rustev_eval/scoring.rs

rustev_eval/
scoring.rs
