rustev_eval/
baseline.rs

1//! Named baselines, `rustev.baseline/1`, and the reproducibility check
2//! (spec 029, 3.8 and 3.9).
3//!
4//! A baseline is a measurement record of one repeat set: it binds the
5//! repeat report and every run document by digest, the dataset, split and
6//! role, the evaluator and repeat configurations, the plan, the source,
7//! the run seeds, the held-out record, and every cohort's `pooled`,
8//! interval and band. It is not a quality claim and grants nothing. The
9//! reproducibility check compares a new repeat set's stability metrics
10//! with a baseline of the same configuration, within the baseline's bands.
11
12use std::fmt;
13
14use rustev_contract::eval_report::{DatasetProvenance, Split};
15use rustev_contract::ids::{ContentDigest, DatasetId, EvaluatorConfigId, PlanId};
16use rustev_contract::limits::REPLAY_V1;
17use rustev_contract::{Document, Identified};
18use serde::{Deserialize, Serialize};
19
20use crate::config::AdapterRules;
21use crate::repeat::{
22    BandDoc, CohortStatus, FigureDoc, HeldOut, IntervalDoc, Observation, RepeatInputs,
23    RepeatReport, RunRef, Source, SplitRole, repeat,
24};
25use crate::repeat_config::RepeatConfig;
26
27pub const BASELINE: &str = "rustev.baseline/1";
28/// The longest baseline name.
29pub const MAX_NAME: usize = 64;
30
31#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
32#[serde(deny_unknown_fields)]
33pub struct Baseline {
34    pub schema: String,
35    pub name: String,
36    /// The record digest of the repeat report.
37    pub repeat_report: ContentDigest,
38    pub runs: Vec<RunRef>,
39    pub dataset: DatasetId,
40    pub split: Split,
41    pub split_role: SplitRole,
42    /// A synthetic-provenance baseline stays labeled as such (3.8.2).
43    pub provenance: DatasetProvenance,
44    pub evaluator_config: EvaluatorConfigId,
45    /// The evaluator configuration's schema (`/1` or `/2`).
46    pub evaluator_config_version: String,
47    pub adapter_rules: Option<AdapterRules>,
48    pub repeat_config: ContentDigest,
49    pub plan: PlanId,
50    pub source: Source,
51    pub n: u64,
52    /// Supplied by the host, never read from a clock.
53    pub recorded_at_ms: i64,
54    pub held_out: Option<HeldOut>,
55    pub cohorts: Vec<BaselineCohort>,
56}
57
58crate::doc::eval_document!(Baseline, BASELINE, REPLAY_V1);
59
60#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
61#[serde(deny_unknown_fields)]
62pub struct BaselineCohort {
63    pub name: String,
64    pub status: CohortStatus,
65    pub metrics: Vec<BaselineMetric>,
66}
67
68#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
69#[serde(deny_unknown_fields)]
70pub struct BaselineMetric {
71    pub name: String,
72    pub observation: Option<Observation>,
73    pub pooled: FigureDoc,
74    pub interval: IntervalDoc,
75    pub band: BandDoc,
76}
77
78/// Why no baseline was recorded (`invalid_input`).
79#[derive(Debug, Clone, PartialEq, Eq)]
80pub enum BaselineError {
81    /// 1 to 64 bytes of `[A-Za-z0-9._-]`, starting alphanumeric.
82    Name,
83    /// The supplied documents do not reproduce the repeat report, or a
84    /// digest does not bind.
85    Binding(String),
86}
87
88impl fmt::Display for BaselineError {
89    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
90        match self {
91            BaselineError::Name => write!(
92                f,
93                "a baseline name is 1 to 64 bytes of [A-Za-z0-9._-], starting alphanumeric"
94            ),
95            BaselineError::Binding(m) => write!(f, "binding: {m}"),
96        }
97    }
98}
99
100impl std::error::Error for BaselineError {}
101
102pub fn valid_name(name: &str) -> bool {
103    !name.is_empty()
104        && name.len() <= MAX_NAME
105        && name.as_bytes()[0].is_ascii_alphanumeric()
106        && name
107            .bytes()
108            .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-'))
109}
110
111/// Record a baseline of `report` (3.8). Every binding is verified against
112/// the documents supplied: the repeat set is assembled again from
113/// `inputs` and must equal the report, apart from a held-out count, which
114/// only the host's ledger can state and which is copied as recorded.
115pub fn record(
116    name: &str,
117    inputs: &RepeatInputs<'_>,
118    report: &RepeatReport,
119    recorded_at_ms: i64,
120) -> Result<Baseline, BaselineError> {
121    if !valid_name(name) {
122        return Err(BaselineError::Name);
123    }
124    let again = repeat(inputs).map_err(|e| BaselineError::Binding(e.to_string()))?;
125    let mut masked = report.clone();
126    masked.held_out = again.held_out.clone();
127    if masked != again {
128        return Err(BaselineError::Binding(
129            "the supplied runs and configurations do not reproduce the repeat report".into(),
130        ));
131    }
132    // A count includes its own entry, and no count exceeds the ledger's.
133    if let Some(HeldOut::Counted(r)) = &report.held_out
134        && (r.count == 0
135            || r.plan_count == 0
136            || r.count > r.dataset_count
137            || r.plan_count > r.dataset_count)
138    {
139        return Err(BaselineError::Binding(
140            "held-out count or plan count is zero or exceeds the dataset count".into(),
141        ));
142    }
143    let first = inputs
144        .runs
145        .first()
146        .ok_or_else(|| BaselineError::Binding("no run".into()))?;
147    Ok(Baseline {
148        schema: BASELINE.into(),
149        name: name.into(),
150        repeat_report: report
151            .record_digest()
152            .map_err(|_| BaselineError::Binding("the repeat report has no record digest".into()))?,
153        runs: report.runs.clone(),
154        dataset: report.dataset.clone(),
155        split: report.split,
156        split_role: report.split_role,
157        provenance: report.provenance.clone(),
158        evaluator_config: report.evaluator_config.clone(),
159        evaluator_config_version: first.evaluated.config.schema.clone(),
160        adapter_rules: report.adapter_rules.clone(),
161        repeat_config: report.repeat_config.clone(),
162        plan: report.plan.clone(),
163        source: report.source,
164        n: report.runs.len() as u64,
165        recorded_at_ms,
166        held_out: report.held_out.clone(),
167        cohorts: report
168            .cohorts
169            .iter()
170            .map(|c| BaselineCohort {
171                name: c.name.clone(),
172                status: c.status,
173                metrics: c
174                    .metrics
175                    .iter()
176                    .map(|m| BaselineMetric {
177                        name: m.name.clone(),
178                        observation: m.observation,
179                        pooled: m.pooled.clone(),
180                        interval: m.interval.clone(),
181                        band: m.band.clone(),
182                    })
183                    .collect(),
184            })
185            .collect(),
186    })
187}
188
189/// One stability metric beyond the baseline's band.
190#[derive(Debug, Clone, PartialEq)]
191pub struct Exceedance {
192    pub cohort: String,
193    pub metric: String,
194    pub delta: f64,
195    pub band: f64,
196}
197
198/// The reproducibility verdict (3.9).
199#[derive(Debug, Clone, PartialEq)]
200pub enum Reproducibility {
201    Reproducible,
202    /// A stop-and-report condition (exit 13): every exceedance, whatever
203    /// else is unknown.
204    NotReproducible {
205        exceedances: Vec<Exceedance>,
206    },
207    /// No exceedance, and something needed is unknown.
208    Unknown {
209        reasons: Vec<String>,
210    },
211}
212
213/// Compare a repeat set with a baseline of the same configuration: for
214/// every stability metric in every baseline cohort, reproducible when
215/// `|pooled_run - pooled_baseline| <= band_baseline`.
216pub fn reproducibility(
217    baseline: &Baseline,
218    run: &RepeatReport,
219    config: &RepeatConfig,
220) -> Reproducibility {
221    let mut reasons = vec![];
222    if config.id().ok().as_ref() != Some(&baseline.repeat_config) {
223        reasons.push("the repeat configuration is not the baseline's".to_string());
224    }
225    if run.dataset != baseline.dataset
226        || run.split != baseline.split
227        || run.evaluator_config != baseline.evaluator_config
228        || run.repeat_config != baseline.repeat_config
229        || run.plan != baseline.plan
230    {
231        reasons.push("the repeat set's dataset, split, evaluator configuration, repeat configuration or plan is not the baseline's".into());
232    }
233    let mut exceedances = vec![];
234    for bc in &baseline.cohorts {
235        let rc = run.cohorts.iter().find(|c| c.name == bc.name);
236        for metric in &config.stability_metrics {
237            let base = bc.metrics.iter().find(|m| &m.name == metric);
238            let now = rc.and_then(|c| c.metrics.iter().find(|m| &m.name == metric));
239            let at = format!("{} in {}", metric, bc.name);
240            let (Some(base), Some(now)) = (base, now) else {
241                reasons.push(format!("{at}: not reported on both sides"));
242                continue;
243            };
244            match (&base.pooled, &now.pooled, &base.band) {
245                (FigureDoc::Value(b), FigureDoc::Value(r), BandDoc::Value { value: band, .. }) => {
246                    let delta = r - b;
247                    if delta.abs() > *band {
248                        exceedances.push(Exceedance {
249                            cohort: bc.name.clone(),
250                            metric: metric.clone(),
251                            delta,
252                            band: *band,
253                        });
254                    }
255                }
256                _ => reasons.push(format!("{at}: a value or the baseline band is unknown")),
257            }
258        }
259    }
260    if !exceedances.is_empty() {
261        Reproducibility::NotReproducible { exceedances }
262    } else if !reasons.is_empty() {
263        Reproducibility::Unknown { reasons }
264    } else {
265        Reproducibility::Reproducible
266    }
267}
rustev_eval/baseline.rs

rustev_eval/
baseline.rs
