rustev_eval/
eval_cases.rs

1//! Per-case measures, `rustev.eval-cases/1` (spec 029, 3.2).
2//!
3//! One document per evaluation run: it binds the run's report by record
4//! digest and lists, per case in manifest order, the case id, the run
5//! record it was measured from, its outcome key, its top-k list when
6//! declared, and its contribution to every case-decomposable metric. The
7//! measures reproduce the report's figures bit for bit or the run is
8//! refused (`reconcile`), so a repeat set never aggregates numbers its own
9//! reports do not state.
10
11use std::collections::BTreeMap;
12
13use rustev_contract::Document;
14use rustev_contract::decimal::Decimal;
15use rustev_contract::eval_report::MetricValue;
16use rustev_contract::ids::ContentDigest;
17use rustev_contract::limits::REPLAY_V1;
18use serde::{Deserialize, Serialize};
19
20use crate::repeat_config::TopK;
21use crate::repeat_stats::{Contribution, Figure, Form, MetricCases, run_stats};
22use crate::report::Evaluated;
23
24pub const EVAL_CASES: &str = "rustev.eval-cases/1";
25/// The two metrics repeat reports derive (3.2.5); no run report states
26/// them, so reconciliation has nothing to compare them with.
27pub const COST_PER_1000_OBSERVED: &str = "cost_per_1000/observed";
28pub const COST_PER_1000_ESTIMATED: &str = "cost_per_1000/estimated";
29
30/// Whether `metric` is one of the derived repeat-report metrics.
31pub fn derived(metric: &str) -> bool {
32    metric == COST_PER_1000_OBSERVED || metric == COST_PER_1000_ESTIMATED
33}
34
35#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
36#[serde(deny_unknown_fields)]
37pub struct EvalCases {
38    pub schema: String,
39    /// The record digest of the run's `rustev.eval-report/1`.
40    pub report: ContentDigest,
41    /// The ranking parameter whose first `k` candidates each case lists;
42    /// none when the repeat configuration declares no top-k.
43    pub top_k: Option<TopK>,
44    pub cases: Vec<CaseMeasures>,
45}
46
47crate::doc::eval_document!(EvalCases, EVAL_CASES, REPLAY_V1);
48
49#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
50#[serde(deny_unknown_fields)]
51pub struct CaseMeasures {
52    pub case: String,
53    /// The record digest of the run record the case was measured from;
54    /// none for a case with no comparable run.
55    pub run: Option<ContentDigest>,
56    /// The outcome key of the case's `rustev.eval-detail/1` entry.
57    pub outcome: String,
58    /// The first `k` candidates of the declared ranking parameter, or none
59    /// when the case has no proposal carrying that parameter as a ranking
60    /// or shortlist (always none when no top-k is declared).
61    pub top_k: Option<Vec<String>>,
62    /// Per metric, the case's contribution; a metric whose denominator the
63    /// case is outside has no entry.
64    pub measures: BTreeMap<String, CaseMeasure>,
65}
66
67/// One case's contribution to one metric (3.2.2).
68#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
69#[serde(rename_all = "snake_case", deny_unknown_fields)]
70pub enum CaseMeasure {
71    Count {
72        num: u64,
73        den: u64,
74    },
75    Mean {
76        num: f64,
77        den: u64,
78    },
79    Sample {
80        value: f64,
81    },
82    /// Inside the denominator with no finite contribution, and why.
83    Unknown {
84        reason: String,
85    },
86}
87
88impl CaseMeasure {
89    pub fn contribution(&self) -> Contribution {
90        match self {
91            CaseMeasure::Count { num, den } => Contribution::Count {
92                num: *num,
93                den: *den,
94            },
95            CaseMeasure::Mean { num, den } => Contribution::Mean {
96                num: *num,
97                den: *den,
98            },
99            CaseMeasure::Sample { value } => Contribution::Sample(*value),
100            CaseMeasure::Unknown { .. } => Contribution::Unknown,
101        }
102    }
103}
104
105/// How a spec 004 metric decomposes into per-case measures (3.2.3), by its
106/// reported name; none for a total (cost units, unknown attempts,
107/// liability), for `brier` (see below), or a metric no spec gives a
108/// per-case measure.
109pub fn form_of(metric: &str) -> Option<Form> {
110    match metric {
111        "coverage"
112        | "acceptance_coverage"
113        | "labeled_proposal_coverage"
114        | "error_among_accepted"
115        | "candidate_outcome_change" => return Some(Form::CountRatio),
116        "mean_log_loss" | COST_PER_1000_OBSERVED | COST_PER_1000_ESTIMATED => {
117            return Some(Form::Mean);
118        }
119        // Spec 004 adds every class term of every case to one running sum,
120        // so per-case sums cannot reproduce its bits (3.2.6). Until the
121        // owner decides how 3.2.3 and 3.2.6 meet for it, brier has no
122        // per-case measure: its interval is unknown, never approximated.
123        "brier" => return None,
124        _ => {}
125    }
126    let parts: Vec<&str> = metric.split('/').collect();
127    match parts.as_slice() {
128        ["reliability", _, _, "confidence"] => Some(Form::Mean),
129        ["reliability", _, _, "accuracy"] => Some(Form::CountRatio),
130        ["latency_ms" | "queue_ms", q] => {
131            let q = Decimal::parse(q.strip_prefix('p')?).ok()?;
132            Some(Form::Sample { quantile: q })
133        }
134        _ => None,
135    }
136}
137
138/// One metric's contributions over a run set's case documents:
139/// `cases[case][run]`. Every document must list the same case ids in the
140/// same order (3.1.3); otherwise the set is refused (`mixed-runs`), never
141/// truncated or padded.
142pub fn metric_cases(metric: &str, form: Form, runs: &[&EvalCases]) -> Result<MetricCases, String> {
143    let first = runs.first().map_or(&[][..], |r| r.cases.as_slice());
144    for r in runs {
145        if r.cases.len() != first.len() || r.cases.iter().zip(first).any(|(a, b)| a.case != b.case)
146        {
147            return Err("mixed-runs: the runs do not list the same cases in the same order".into());
148        }
149    }
150    let cases = (0..first.len())
151        .map(|c| {
152            runs.iter()
153                .map(|r| {
154                    r.cases[c]
155                        .measures
156                        .get(metric)
157                        .map(CaseMeasure::contribution)
158                })
159                .collect()
160        })
161        .collect();
162    Ok(MetricCases {
163        name: metric.into(),
164        form,
165        cases,
166    })
167}
168
169/// Whether one run's case measures reproduce its report (3.2.6): the
170/// document binds the report's record digest; its cases are the detail's,
171/// in order, with the same outcome keys; every case-decomposable metric
172/// computed from the measures over the whole cohort equals the report's
173/// value bit for bit (or both are unknown); and the count-ratio sums equal
174/// the detail's counts.
175pub fn reconcile(run: &Evaluated, cases: &EvalCases) -> Result<(), String> {
176    let digest = run
177        .report
178        .record_digest()
179        .map_err(|_| "the report has no record digest".to_string())?;
180    if cases.schema != EVAL_CASES || cases.report != digest || run.detail.report != digest {
181        return Err("the case measures are not bound to this report".into());
182    }
183    if cases.cases.len() != run.detail.cases.len()
184        || cases
185            .cases
186            .iter()
187            .zip(&run.detail.cases)
188            .any(|(m, d)| m.case != d.case || m.outcome != d.outcome)
189    {
190        return Err("the case measures do not list the detail's cases and outcomes".into());
191    }
192    for c in &cases.cases {
193        if let Some(name) = c.measures.keys().find(|m| form_of(m).is_none()) {
194            return Err(format!(
195                "case {} has a measure for `{name}`, which has none",
196                c.case
197            ));
198        }
199    }
200    for m in &run.report.metrics {
201        let Some(form) = form_of(&m.name) else {
202            continue;
203        };
204        let metric = metric_cases(&m.name, form, &[cases])?;
205        let stats = run_stats(&metric, 1, false).map_err(|e| format!("`{}`: {e}", m.name))?;
206        let same = match (&stats.runs[0], &m.value) {
207            (Figure::Known(a), MetricValue::Measured { value, .. }) => {
208                a.to_bits() == value.to_bits()
209            }
210            (Figure::Unknown { .. }, MetricValue::Unknown { .. }) => true,
211            _ => false,
212        };
213        if !same {
214            return Err(format!(
215                "`{}` computed from the case measures is not the report's value",
216                m.name
217            ));
218        }
219        if let Some(count) = run.detail.count(&m.name) {
220            let (mut num, mut den) = (0u128, 0u128);
221            for per_run in &metric.cases {
222                if let Some(Contribution::Count { num: n, den: d }) = per_run[0] {
223                    num += u128::from(n);
224                    den += u128::from(d);
225                }
226            }
227            if (num, den) != (u128::from(count.numerator), u128::from(count.denominator)) {
228                return Err(format!("`{}` sums are not the detail's counts", m.name));
229            }
230        }
231    }
232    // A measure for a metric the report does not state is not a measure of
233    // this run.
234    for c in &cases.cases {
235        if let Some(name) = c
236            .measures
237            .keys()
238            .find(|n| !derived(n) && !run.report.metrics.iter().any(|m| &m.name == *n))
239        {
240            return Err(format!(
241                "case {} has a measure for unreported `{name}`",
242                c.case
243            ));
244        }
245    }
246    Ok(())
247}
rustev_eval/eval_cases.rs

rustev_eval/
eval_cases.rs
