rustev_eval/
config.rs

1//! Evaluator configurations, `rustev.evaluator-config/1` and `/2` (spec
2//! 004, 3.5.3; spec 014).
3//!
4//! The configuration is the definition a report's numbers mean anything
5//! under: the task adapter and its version, how labels are read, which
6//! proposal payloads define agreement, which step's probabilities are
7//! scored, subgroup and reliability-bin boundaries, latency quantiles, cost
8//! unit comparability, exclusions, gate thresholds and the temperature grid.
9//! Its canonical digest is the report's `EvaluatorConfigId`. Fractional
10//! values are Decimal strings, so the identity uses the existing form.
11//!
12//! A gate states only metric, direction, tolerance and minimum coverage: the
13//! baseline and candidate cohorts it compares (dataset, split, config) are
14//! the ones their reports name, since a configuration cannot name its own
15//! identity.
16//!
17//! Version 2 binds the task adapter's rules: its adapter reference carries
18//! `rules`, a digest of the document that determines the adapter's label
19//! check and correctness, or `opaque`. Version 1 has no `rules`, keeps its
20//! bytes and identity, and counts as unbound for every gate (spec 014).
21//!
22//! Version 3 (spec 041, 3.1.3) adds the digest of the adapter's measure
23//! declarations to the adapter reference, and declares scorers, rubric
24//! metrics, human scales, agreements and origin families, so every one of
25//! them enters the identity. Versions 1 and 2 produce no new metric.
26
27use std::collections::BTreeSet;
28use std::fmt;
29
30use rustev_contract::bounded::ParseLimits;
31use rustev_contract::canonical::tagged_digest;
32use rustev_contract::decimal::Decimal;
33use rustev_contract::ids::{ContentDigest, EvaluatorConfigId};
34use rustev_contract::limits::REPLAY_V1;
35use rustev_contract::{Document, DocumentError, Identified};
36use serde::{Deserialize, Serialize};
37
38use crate::dataset::AdapterRef;
39use crate::scoring::{
40    AgreementDecl, Declarations, HumanScale, OriginFamily, RubricMetric, ScorerDescriptor,
41    check_prefix,
42};
43pub const EVALUATOR_CONFIG: &str = "rustev.evaluator-config/1";
44/// Version 2: the adapter reference binds the adapter's rules (spec 014).
45pub const EVALUATOR_CONFIG_V2: &str = "rustev.evaluator-config/2";
46/// Version 3: measures, scorers, rubric metrics, agreements and origin
47/// families (spec 041).
48pub const EVALUATOR_CONFIG_V3: &str = "rustev.evaluator-config/3";
49/// The most temperatures one grid may hold (spec 004, 3.6.2).
50pub const MAX_GRID: usize = 4096;
51
52#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
53#[serde(deny_unknown_fields)]
54pub struct EvaluatorConfig {
55    pub schema: String,
56    pub adapter: ConfigAdapter,
57    /// What a label means, in words.
58    pub label_interpretation: String,
59    pub agreement: Agreement,
60    /// The step whose probability vector log loss, Brier and reliability
61    /// score; none when the task has no probability to score.
62    pub probability_step: Option<String>,
63    /// Reliability bin boundaries: strictly increasing from 0 to 1.
64    pub reliability_bins: Vec<Decimal>,
65    /// Declared subgroups, reported besides the whole cohort.
66    pub subgroups: Vec<String>,
67    /// Latency quantiles in (0, 1], strictly increasing (nearest rank).
68    pub latency_quantiles: Vec<Decimal>,
69    /// Whether every backend's cost units may be added together.
70    pub cost_units_comparable: bool,
71    /// Every metric this configuration reports, with its formula in words.
72    pub metrics: Vec<MetricFormula>,
73    pub gates: Vec<Gate>,
74    /// Candidate temperatures for fitting: strictly increasing, in
75    /// (0, 1000], at most 4096; empty when nothing is fitted.
76    pub temperature_grid: Vec<Decimal>,
77    /// Version 3 only, as are the four members below (spec 041, 3.6 to
78    /// 3.8 and 3.5).
79    #[serde(default, skip_serializing_if = "Option::is_none")]
80    pub scorers: Option<Vec<ScorerDescriptor>>,
81    #[serde(default, skip_serializing_if = "Option::is_none")]
82    pub rubric_metrics: Option<Vec<RubricMetric>>,
83    #[serde(default, skip_serializing_if = "Option::is_none")]
84    pub human_scales: Option<Vec<HumanScale>>,
85    #[serde(default, skip_serializing_if = "Option::is_none")]
86    pub agreements: Option<Vec<AgreementDecl>>,
87    #[serde(default, skip_serializing_if = "Option::is_none")]
88    pub origin: Option<Vec<OriginFamily>>,
89}
90
91/// Either version parses; `SCHEMA` names version 1 only because the trait
92/// holds one constant. The identity is tagged with the document's own
93/// schema, so a version 1 identity is what it always was.
94impl Document for EvaluatorConfig {
95    const SCHEMA: &'static str = EVALUATOR_CONFIG;
96    const LIMITS: ParseLimits = REPLAY_V1;
97    fn schema(&self) -> &str {
98        &self.schema
99    }
100
101    fn parse(bytes: &[u8]) -> Result<Self, DocumentError> {
102        let doc: Self = rustev_contract::bounded::parse_bounded(bytes, &Self::LIMITS)
103            .map_err(DocumentError::Parse)?;
104        if ![EVALUATOR_CONFIG, EVALUATOR_CONFIG_V2, EVALUATOR_CONFIG_V3]
105            .contains(&doc.schema.as_str())
106        {
107            return Err(DocumentError::Schema {
108                expected: EVALUATOR_CONFIG_V3,
109                found: doc.schema.chars().take(80).collect(),
110            });
111        }
112        Ok(doc)
113    }
114
115    fn record_digest(&self) -> Result<ContentDigest, rustev_contract::canonical::CanonicalError> {
116        let raw = self.record_canonical()?;
117        Ok(ContentDigest::parse(&tagged_digest(&self.schema, &raw))
118            .expect("a tagged digest is a content digest"))
119    }
120}
121
122impl Identified for EvaluatorConfig {
123    type Id = EvaluatorConfigId;
124    fn wrap(digest: String) -> EvaluatorConfigId {
125        EvaluatorConfigId::parse(&digest).expect("a tagged digest is an identity")
126    }
127
128    fn id(&self) -> Result<EvaluatorConfigId, rustev_contract::canonical::CanonicalError> {
129        Ok(Self::wrap(tagged_digest(&self.schema, &self.canonical()?)))
130    }
131}
132
133/// The configuration's task adapter: name and version, and in version 2
134/// its rules.
135#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
136#[serde(deny_unknown_fields)]
137pub struct ConfigAdapter {
138    pub name: String,
139    pub version: String,
140    /// Absent in version 1, required in versions 2 and 3.
141    #[serde(default, skip_serializing_if = "Option::is_none")]
142    pub rules: Option<AdapterRules>,
143    /// The digest of the adapter's measure declarations; version 3 only
144    /// (spec 041, 3.4.1).
145    #[serde(default, skip_serializing_if = "Option::is_none")]
146    pub measures: Option<ContentDigest>,
147}
148
149impl ConfigAdapter {
150    /// A version 1 reference: no rules.
151    pub fn v1(r: AdapterRef) -> ConfigAdapter {
152        ConfigAdapter {
153            name: r.name,
154            version: r.version,
155            rules: None,
156            measures: None,
157        }
158    }
159
160    /// A version 2 reference.
161    pub fn v2(r: AdapterRef, rules: AdapterRules) -> ConfigAdapter {
162        ConfigAdapter {
163            name: r.name,
164            version: r.version,
165            rules: Some(rules),
166            measures: None,
167        }
168    }
169
170    /// A version 3 reference: rules and the measures digest.
171    pub fn v3(r: AdapterRef, rules: AdapterRules, measures: ContentDigest) -> ConfigAdapter {
172        ConfigAdapter {
173            name: r.name,
174            version: r.version,
175            rules: Some(rules),
176            measures: Some(measures),
177        }
178    }
179
180    pub fn reference(&self) -> AdapterRef {
181        AdapterRef {
182            name: self.name.clone(),
183            version: self.version.clone(),
184        }
185    }
186}
187
188/// What determines a task adapter's label check and correctness (spec
189/// 014, 3.1): the content digest of a canonical document that fully
190/// determines them, or `opaque` when no document does.
191#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
192#[serde(rename_all = "snake_case")]
193pub enum AdapterRules {
194    Bound(ContentDigest),
195    Opaque,
196}
197
198/// Which proposal payloads define an outcome: the action, and these
199/// parameters. Whole-judgment equality is never the agreement metric.
200#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
201#[serde(deny_unknown_fields)]
202pub struct Agreement {
203    pub params: Vec<String>,
204}
205
206#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
207#[serde(deny_unknown_fields)]
208pub struct MetricFormula {
209    pub name: String,
210    pub formula: String,
211}
212
213#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
214#[serde(rename_all = "snake_case")]
215pub enum Direction {
216    /// Lower is better: the candidate may exceed the baseline by at most the
217    /// tolerance.
218    LowerIsBetter,
219    /// Higher is better: the candidate may fall short by at most the
220    /// tolerance.
221    HigherIsBetter,
222}
223
224#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
225#[serde(deny_unknown_fields)]
226pub struct Gate {
227    pub name: String,
228    pub metric: String,
229    pub direction: Direction,
230    /// Absolute, non-negative.
231    pub tolerance: Decimal,
232    /// In [0, 1]: comparable cases / all cases, for both cohorts.
233    pub min_comparable_coverage: Decimal,
234    /// In [0, 1]: labeled proposals / proposals, for both cohorts.
235    pub min_labeled_coverage: Decimal,
236}
237
238/// Why a configuration was refused.
239#[derive(Debug, Clone, PartialEq, Eq)]
240pub struct ConfigError(pub String);
241
242impl fmt::Display for ConfigError {
243    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
244        f.write_str(&self.0)
245    }
246}
247
248impl std::error::Error for ConfigError {}
249
250fn increasing(v: &[Decimal]) -> bool {
251    v.windows(2).all(|w| w[0] < w[1])
252}
253
254impl EvaluatorConfig {
255    /// The rules digest a gate may compare: bound in version 2, otherwise
256    /// none (version 1 rules are unbound, spec 014 3.2.3).
257    pub fn bound_rules(&self) -> Option<&ContentDigest> {
258        match (&self.adapter.rules, self.schema.as_str()) {
259            (Some(AdapterRules::Bound(d)), EVALUATOR_CONFIG_V2 | EVALUATOR_CONFIG_V3) => Some(d),
260            _ => None,
261        }
262    }
263
264    /// The version 3 declarations; none in versions 1 and 2.
265    pub fn declarations(&self) -> Option<Declarations<'_>> {
266        if self.schema != EVALUATOR_CONFIG_V3 {
267            return None;
268        }
269        Some(Declarations {
270            scorers: self.scorers.as_deref()?,
271            rubric_metrics: self.rubric_metrics.as_deref()?,
272            human_scales: self.human_scales.as_deref()?,
273            agreements: self.agreements.as_deref()?,
274            origin: self.origin.as_deref()?,
275        })
276    }
277
278    pub fn check(&self) -> Result<(), ConfigError> {
279        let bad = |m: &str| Err(ConfigError(m.into()));
280        let v3_members = [
281            self.scorers.is_some(),
282            self.rubric_metrics.is_some(),
283            self.human_scales.is_some(),
284            self.agreements.is_some(),
285            self.origin.is_some(),
286        ];
287        let measures = self.adapter.measures.is_some();
288        match (self.schema.as_str(), &self.adapter.rules) {
289            (EVALUATOR_CONFIG, None) | (EVALUATOR_CONFIG_V2 | EVALUATOR_CONFIG_V3, Some(_)) => {}
290            (EVALUATOR_CONFIG, Some(_)) => return bad("version 1 names no adapter rules"),
291            (EVALUATOR_CONFIG_V2, None) => return bad("version 2 names the adapter rules"),
292            (EVALUATOR_CONFIG_V3, None) => return bad("version 3 names the adapter rules"),
293            _ => return bad("schema"),
294        }
295        if self.schema == EVALUATOR_CONFIG_V3 {
296            if !measures || v3_members.contains(&false) {
297                return bad(
298                    "version 3 names the adapter measures, scorers, rubric metrics, human scales, agreements and origin",
299                );
300            }
301        } else if measures || v3_members.contains(&true) {
302            return bad(
303                "only version 3 names measures, scorers, rubric metrics, agreements or origin",
304            );
305        }
306        let one = Decimal::from_i64(1);
307        let b = &self.reliability_bins;
308        if b.len() < 2 || b[0] != Decimal::ZERO || b[b.len() - 1] != one || !increasing(b) {
309            return bad("reliability bins run strictly upward from 0 to 1");
310        }
311        let q = &self.latency_quantiles;
312        if q.iter().any(|x| *x <= Decimal::ZERO || *x > one) || !increasing(q) {
313            return bad("latency quantiles are strictly increasing in (0, 1]");
314        }
315        let names: BTreeSet<&str> = self.metrics.iter().map(|m| m.name.as_str()).collect();
316        if names.len() != self.metrics.len() {
317            return bad("metric names are unique");
318        }
319        let groups: BTreeSet<&str> = self.subgroups.iter().map(String::as_str).collect();
320        if groups.len() != self.subgroups.len() || groups.contains("all") {
321            return bad("subgroups are unique and none is named `all`");
322        }
323        let mut gate_names = BTreeSet::new();
324        for g in &self.gates {
325            if !gate_names.insert(&g.name) {
326                return bad("gate names are unique");
327            }
328            // A series is named `<base>/<suffix>`; its base must be declared.
329            let base = g.metric.split('/').next().unwrap_or_default();
330            if !names.contains(base) {
331                return bad("a gate names a metric the configuration does not declare");
332            }
333            if g.tolerance.is_negative() {
334                return bad("a gate tolerance is non-negative");
335            }
336            for c in [g.min_comparable_coverage, g.min_labeled_coverage] {
337                if c.is_negative() || c > one {
338                    return bad("a minimum coverage is in [0, 1]");
339                }
340            }
341        }
342        if let Some(d) = self.declarations() {
343            d.check()?;
344            let reported = d.metric_names();
345            // Spec 004's declared-metric rule, for every name the
346            // declarations report.
347            for name in reported.keys() {
348                let base = name.split('/').next().unwrap_or_default();
349                if !names.contains(base) {
350                    return bad(
351                        "a reported metric names a base the configuration does not declare",
352                    );
353                }
354            }
355            for g in &self.gates {
356                check_prefix(&g.metric, &reported)?;
357            }
358        }
359        let t = &self.temperature_grid;
360        let max = Decimal::from_i64(1000);
361        if t.len() > MAX_GRID || !increasing(t) || t.iter().any(|x| *x <= Decimal::ZERO || *x > max)
362        {
363            return bad("the temperature grid is strictly increasing in (0, 1000], at most 4096");
364        }
365        Ok(())
366    }
367}
rustev_eval/config.rs

rustev_eval/
config.rs
