rustev_eval/
config.rs

1//! Evaluator configurations, `rustev.evaluator-config/1` and `/2` (spec
2//! 004, 3.5.3; spec 014).
3//!
4//! The configuration is the definition a report's numbers mean anything
5//! under: the task adapter and its version, how labels are read, which
6//! proposal payloads define agreement, which step's probabilities are
7//! scored, subgroup and reliability-bin boundaries, latency quantiles, cost
8//! unit comparability, exclusions, gate thresholds and the temperature grid.
9//! Its canonical digest is the report's `EvaluatorConfigId`. Fractional
10//! values are Decimal strings, so the identity uses the existing form.
11//!
12//! A gate states only metric, direction, tolerance and minimum coverage: the
13//! baseline and candidate cohorts it compares (dataset, split, config) are
14//! the ones their reports name, since a configuration cannot name its own
15//! identity.
16//!
17//! Version 2 binds the task adapter's rules: its adapter reference carries
18//! `rules`, a digest of the document that determines the adapter's label
19//! check and correctness, or `opaque`. Version 1 has no `rules`, keeps its
20//! bytes and identity, and counts as unbound for every gate (spec 014).
21
22use std::collections::BTreeSet;
23use std::fmt;
24
25use rustev_contract::bounded::ParseLimits;
26use rustev_contract::canonical::tagged_digest;
27use rustev_contract::decimal::Decimal;
28use rustev_contract::ids::{ContentDigest, EvaluatorConfigId};
29use rustev_contract::limits::REPLAY_V1;
30use rustev_contract::{Document, DocumentError, Identified};
31use serde::{Deserialize, Serialize};
32
33use crate::dataset::AdapterRef;
34pub const EVALUATOR_CONFIG: &str = "rustev.evaluator-config/1";
35/// Version 2: the adapter reference binds the adapter's rules (spec 014).
36pub const EVALUATOR_CONFIG_V2: &str = "rustev.evaluator-config/2";
37/// The most temperatures one grid may hold (spec 004, 3.6.2).
38pub const MAX_GRID: usize = 4096;
39
40#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
41#[serde(deny_unknown_fields)]
42pub struct EvaluatorConfig {
43    pub schema: String,
44    pub adapter: ConfigAdapter,
45    /// What a label means, in words.
46    pub label_interpretation: String,
47    pub agreement: Agreement,
48    /// The step whose probability vector log loss, Brier and reliability
49    /// score; none when the task has no probability to score.
50    pub probability_step: Option<String>,
51    /// Reliability bin boundaries: strictly increasing from 0 to 1.
52    pub reliability_bins: Vec<Decimal>,
53    /// Declared subgroups, reported besides the whole cohort.
54    pub subgroups: Vec<String>,
55    /// Latency quantiles in (0, 1], strictly increasing (nearest rank).
56    pub latency_quantiles: Vec<Decimal>,
57    /// Whether every backend's cost units may be added together.
58    pub cost_units_comparable: bool,
59    /// Every metric this configuration reports, with its formula in words.
60    pub metrics: Vec<MetricFormula>,
61    pub gates: Vec<Gate>,
62    /// Candidate temperatures for fitting: strictly increasing, in
63    /// (0, 1000], at most 4096; empty when nothing is fitted.
64    pub temperature_grid: Vec<Decimal>,
65}
66
67/// Either version parses; `SCHEMA` names version 1 only because the trait
68/// holds one constant. The identity is tagged with the document's own
69/// schema, so a version 1 identity is what it always was.
70impl Document for EvaluatorConfig {
71    const SCHEMA: &'static str = EVALUATOR_CONFIG;
72    const LIMITS: ParseLimits = REPLAY_V1;
73    fn schema(&self) -> &str {
74        &self.schema
75    }
76
77    fn parse(bytes: &[u8]) -> Result<Self, DocumentError> {
78        let doc: Self = rustev_contract::bounded::parse_bounded(bytes, &Self::LIMITS)
79            .map_err(DocumentError::Parse)?;
80        if doc.schema != EVALUATOR_CONFIG && doc.schema != EVALUATOR_CONFIG_V2 {
81            return Err(DocumentError::Schema {
82                expected: EVALUATOR_CONFIG_V2,
83                found: doc.schema.chars().take(80).collect(),
84            });
85        }
86        Ok(doc)
87    }
88
89    fn record_digest(&self) -> Result<ContentDigest, rustev_contract::canonical::CanonicalError> {
90        let raw = self.record_canonical()?;
91        Ok(ContentDigest::parse(&tagged_digest(&self.schema, &raw))
92            .expect("a tagged digest is a content digest"))
93    }
94}
95
96impl Identified for EvaluatorConfig {
97    type Id = EvaluatorConfigId;
98    fn wrap(digest: String) -> EvaluatorConfigId {
99        EvaluatorConfigId::parse(&digest).expect("a tagged digest is an identity")
100    }
101
102    fn id(&self) -> Result<EvaluatorConfigId, rustev_contract::canonical::CanonicalError> {
103        Ok(Self::wrap(tagged_digest(&self.schema, &self.canonical()?)))
104    }
105}
106
107/// The configuration's task adapter: name and version, and in version 2
108/// its rules.
109#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
110#[serde(deny_unknown_fields)]
111pub struct ConfigAdapter {
112    pub name: String,
113    pub version: String,
114    /// Absent in version 1, required in version 2.
115    #[serde(default, skip_serializing_if = "Option::is_none")]
116    pub rules: Option<AdapterRules>,
117}
118
119impl ConfigAdapter {
120    /// A version 1 reference: no rules.
121    pub fn v1(r: AdapterRef) -> ConfigAdapter {
122        ConfigAdapter {
123            name: r.name,
124            version: r.version,
125            rules: None,
126        }
127    }
128
129    /// A version 2 reference.
130    pub fn v2(r: AdapterRef, rules: AdapterRules) -> ConfigAdapter {
131        ConfigAdapter {
132            name: r.name,
133            version: r.version,
134            rules: Some(rules),
135        }
136    }
137
138    pub fn reference(&self) -> AdapterRef {
139        AdapterRef {
140            name: self.name.clone(),
141            version: self.version.clone(),
142        }
143    }
144}
145
146/// What determines a task adapter's label check and correctness (spec
147/// 014, 3.1): the content digest of a canonical document that fully
148/// determines them, or `opaque` when no document does.
149#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
150#[serde(rename_all = "snake_case")]
151pub enum AdapterRules {
152    Bound(ContentDigest),
153    Opaque,
154}
155
156/// Which proposal payloads define an outcome: the action, and these
157/// parameters. Whole-judgment equality is never the agreement metric.
158#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
159#[serde(deny_unknown_fields)]
160pub struct Agreement {
161    pub params: Vec<String>,
162}
163
164#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
165#[serde(deny_unknown_fields)]
166pub struct MetricFormula {
167    pub name: String,
168    pub formula: String,
169}
170
171#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
172#[serde(rename_all = "snake_case")]
173pub enum Direction {
174    /// Lower is better: the candidate may exceed the baseline by at most the
175    /// tolerance.
176    LowerIsBetter,
177    /// Higher is better: the candidate may fall short by at most the
178    /// tolerance.
179    HigherIsBetter,
180}
181
182#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
183#[serde(deny_unknown_fields)]
184pub struct Gate {
185    pub name: String,
186    pub metric: String,
187    pub direction: Direction,
188    /// Absolute, non-negative.
189    pub tolerance: Decimal,
190    /// In [0, 1]: comparable cases / all cases, for both cohorts.
191    pub min_comparable_coverage: Decimal,
192    /// In [0, 1]: labeled proposals / proposals, for both cohorts.
193    pub min_labeled_coverage: Decimal,
194}
195
196/// Why a configuration was refused.
197#[derive(Debug, Clone, PartialEq, Eq)]
198pub struct ConfigError(pub String);
199
200impl fmt::Display for ConfigError {
201    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
202        f.write_str(&self.0)
203    }
204}
205
206impl std::error::Error for ConfigError {}
207
208fn increasing(v: &[Decimal]) -> bool {
209    v.windows(2).all(|w| w[0] < w[1])
210}
211
212impl EvaluatorConfig {
213    /// The rules digest a gate may compare: bound in version 2, otherwise
214    /// none (version 1 rules are unbound, spec 014 3.2.3).
215    pub fn bound_rules(&self) -> Option<&ContentDigest> {
216        match (&self.adapter.rules, self.schema.as_str()) {
217            (Some(AdapterRules::Bound(d)), EVALUATOR_CONFIG_V2) => Some(d),
218            _ => None,
219        }
220    }
221
222    pub fn check(&self) -> Result<(), ConfigError> {
223        let bad = |m: &str| Err(ConfigError(m.into()));
224        match (self.schema.as_str(), &self.adapter.rules) {
225            (EVALUATOR_CONFIG, None) | (EVALUATOR_CONFIG_V2, Some(_)) => {}
226            (EVALUATOR_CONFIG, Some(_)) => return bad("version 1 names no adapter rules"),
227            (EVALUATOR_CONFIG_V2, None) => return bad("version 2 names the adapter rules"),
228            _ => return bad("schema"),
229        }
230        let one = Decimal::from_i64(1);
231        let b = &self.reliability_bins;
232        if b.len() < 2 || b[0] != Decimal::ZERO || b[b.len() - 1] != one || !increasing(b) {
233            return bad("reliability bins run strictly upward from 0 to 1");
234        }
235        let q = &self.latency_quantiles;
236        if q.iter().any(|x| *x <= Decimal::ZERO || *x > one) || !increasing(q) {
237            return bad("latency quantiles are strictly increasing in (0, 1]");
238        }
239        let names: BTreeSet<&str> = self.metrics.iter().map(|m| m.name.as_str()).collect();
240        if names.len() != self.metrics.len() {
241            return bad("metric names are unique");
242        }
243        let groups: BTreeSet<&str> = self.subgroups.iter().map(String::as_str).collect();
244        if groups.len() != self.subgroups.len() || groups.contains("all") {
245            return bad("subgroups are unique and none is named `all`");
246        }
247        let mut gate_names = BTreeSet::new();
248        for g in &self.gates {
249            if !gate_names.insert(&g.name) {
250                return bad("gate names are unique");
251            }
252            // A series is named `<base>/<suffix>`; its base must be declared.
253            let base = g.metric.split('/').next().unwrap_or_default();
254            if !names.contains(base) {
255                return bad("a gate names a metric the configuration does not declare");
256            }
257            if g.tolerance.is_negative() {
258                return bad("a gate tolerance is non-negative");
259            }
260            for c in [g.min_comparable_coverage, g.min_labeled_coverage] {
261                if c.is_negative() || c > one {
262                    return bad("a minimum coverage is in [0, 1]");
263                }
264            }
265        }
266        let t = &self.temperature_grid;
267        let max = Decimal::from_i64(1000);
268        if t.len() > MAX_GRID || !increasing(t) || t.iter().any(|x| *x <= Decimal::ZERO || *x > max)
269        {
270            return bad("the temperature grid is strictly increasing in (0, 1000], at most 4096");
271        }
272        Ok(())
273    }
274}
rustev_eval/config.rs

rustev_eval/
config.rs
