rustev_eval/
eval_cases.rs

1//! Per-case measures, `rustev.eval-cases/1` (spec 029, 3.2).
2//!
3//! One document per evaluation run: it binds the run's report by record
4//! digest and lists, per case in manifest order, the case id, the run
5//! record it was measured from, its outcome key, its top-k list when
6//! declared, and its contribution to every case-decomposable metric. The
7//! measures reproduce the report's figures bit for bit or the run is
8//! refused (`reconcile`), so a repeat set never aggregates numbers its own
9//! reports do not state.
10
11use std::collections::BTreeMap;
12
13use rustev_contract::Document;
14use rustev_contract::decimal::Decimal;
15use rustev_contract::eval_report::MetricValue;
16use rustev_contract::ids::ContentDigest;
17use rustev_contract::limits::REPLAY_V1;
18use serde::{Deserialize, Serialize};
19
20use crate::repeat_config::TopK;
21use crate::repeat_stats::{Contribution, Figure, Form, MetricCases, run_stats};
22use crate::report::Evaluated;
23
24pub const EVAL_CASES: &str = "rustev.eval-cases/1";
25/// The two metrics repeat reports derive (3.2.5); no run report states
26/// them, so reconciliation has nothing to compare them with.
27pub const COST_PER_1000_OBSERVED: &str = "cost_per_1000/observed";
28pub const COST_PER_1000_ESTIMATED: &str = "cost_per_1000/estimated";
29
30/// Whether `metric` is one of the derived repeat-report metrics.
31pub fn derived(metric: &str) -> bool {
32    metric == COST_PER_1000_OBSERVED || metric == COST_PER_1000_ESTIMATED
33}
34
35#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
36#[serde(deny_unknown_fields)]
37pub struct EvalCases {
38    pub schema: String,
39    /// The record digest of the run's `rustev.eval-report/1`.
40    pub report: ContentDigest,
41    /// The ranking parameter whose first `k` candidates each case lists;
42    /// none when the repeat configuration declares no top-k.
43    pub top_k: Option<TopK>,
44    pub cases: Vec<CaseMeasures>,
45}
46
47crate::doc::eval_document!(EvalCases, EVAL_CASES, REPLAY_V1);
48
49#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
50#[serde(deny_unknown_fields)]
51pub struct CaseMeasures {
52    pub case: String,
53    /// The record digest of the run record the case was measured from;
54    /// none for a case with no comparable run.
55    pub run: Option<ContentDigest>,
56    /// The outcome key of the case's `rustev.eval-detail/1` entry.
57    pub outcome: String,
58    /// The first `k` candidates of the declared ranking parameter, or none
59    /// when the case has no proposal carrying that parameter as a ranking
60    /// or shortlist (always none when no top-k is declared).
61    pub top_k: Option<Vec<String>>,
62    /// Per metric, the case's contribution; a metric whose denominator the
63    /// case is outside has no entry.
64    pub measures: BTreeMap<String, CaseMeasure>,
65}
66
67/// One case's contribution to one metric (3.2.2).
68#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
69#[serde(rename_all = "snake_case", deny_unknown_fields)]
70pub enum CaseMeasure {
71    Count {
72        num: u64,
73        den: u64,
74    },
75    Mean {
76        num: f64,
77        den: u64,
78    },
79    Sample {
80        value: f64,
81    },
82    /// Inside the denominator with no finite contribution, and why.
83    Unknown {
84        reason: String,
85    },
86}
87
88impl CaseMeasure {
89    pub fn contribution(&self) -> Contribution {
90        match self {
91            CaseMeasure::Count { num, den } => Contribution::Count {
92                num: *num,
93                den: *den,
94            },
95            CaseMeasure::Mean { num, den } => Contribution::Mean {
96                num: *num,
97                den: *den,
98            },
99            CaseMeasure::Sample { value } => Contribution::Sample(*value),
100            CaseMeasure::Unknown { .. } => Contribution::Unknown,
101        }
102    }
103}
104
105/// How a spec 004 metric decomposes into per-case measures (3.2.3), by its
106/// reported name; none for a total (cost units, unknown attempts,
107/// liability), for `brier` (see below), or a metric no spec gives a
108/// per-case measure.
109pub fn form_of(metric: &str) -> Option<Form> {
110    match metric {
111        "coverage"
112        | "acceptance_coverage"
113        | "labeled_proposal_coverage"
114        | "error_among_accepted"
115        | "candidate_outcome_change" => return Some(Form::CountRatio),
116        "mean_log_loss" | COST_PER_1000_OBSERVED | COST_PER_1000_ESTIMATED => {
117            return Some(Form::Mean);
118        }
119        // Spec 004 adds every class term of every case to one running sum,
120        // so per-case sums cannot reproduce its bits (3.2.6). Until the
121        // owner decides how 3.2.3 and 3.2.6 meet for it, brier has no
122        // per-case measure: its interval is unknown, never approximated.
123        "brier" => return None,
124        _ => {}
125    }
126    let parts: Vec<&str> = metric.split('/').collect();
127    match parts.as_slice() {
128        ["reliability", _, _, "confidence"] => Some(Form::Mean),
129        ["reliability", _, _, "accuracy"] => Some(Form::CountRatio),
130        // Spec 041, 3.11.1.
131        ["ranking", _, spec] => {
132            if ["hit@", "short@", "backfill@p"]
133                .iter()
134                .any(|p| spec.starts_with(p))
135            {
136                Some(Form::CountRatio)
137            } else {
138                Some(Form::Mean)
139            }
140        }
141        // Every origin rate is a count ratio (spec 041, 3.11.1).
142        ["origin", ..] => Some(Form::CountRatio),
143        // `qwk` is not case-decomposable (spec 041, 3.11.1).
144        ["agreement", _, "exact" | "adjacent"]
145        | ["model", "agreement", _, "exact" | "adjacent"] => Some(Form::CountRatio),
146        ["agreement", _, "mean_abs_diff"] | ["model", "agreement", _, "mean_abs_diff"] => {
147            Some(Form::Mean)
148        }
149        ["latency_ms" | "queue_ms", q] => {
150            let q = Decimal::parse(q.strip_prefix('p')?).ok()?;
151            Some(Form::Sample { quantile: q })
152        }
153        _ => None,
154    }
155}
156
157/// A metric's form, by name or, for a spec 041 constraint or rubric
158/// metric, whose name does not say whether its aggregate is a mean
159/// (`share_at_k`, `mean_level_index`, `mean`) or a count ratio
160/// (`all_at_k`, `share_at_least`), by the kind of its first case measure
161/// (3.11.1).
162pub fn form_in(metric: &str, runs: &[&EvalCases]) -> Option<Form> {
163    if let Some(form) = form_of(metric) {
164        return Some(form);
165    }
166    let by_measures = ["constraint/", "rubric/", "model/rubric/"];
167    if !by_measures.iter().any(|p| metric.starts_with(p)) {
168        return None;
169    }
170    runs.iter()
171        .flat_map(|r| &r.cases)
172        .find_map(|c| match c.measures.get(metric)? {
173            CaseMeasure::Count { .. } => Some(Form::CountRatio),
174            CaseMeasure::Mean { .. } => Some(Form::Mean),
175            CaseMeasure::Sample { .. } | CaseMeasure::Unknown { .. } => None,
176        })
177}
178
179/// One metric's contributions over a run set's case documents:
180/// `cases[case][run]`. Every document must list the same case ids in the
181/// same order (3.1.3); otherwise the set is refused (`mixed-runs`), never
182/// truncated or padded.
183pub fn metric_cases(metric: &str, form: Form, runs: &[&EvalCases]) -> Result<MetricCases, String> {
184    let first = runs.first().map_or(&[][..], |r| r.cases.as_slice());
185    for r in runs {
186        if r.cases.len() != first.len() || r.cases.iter().zip(first).any(|(a, b)| a.case != b.case)
187        {
188            return Err("mixed-runs: the runs do not list the same cases in the same order".into());
189        }
190    }
191    let cases = (0..first.len())
192        .map(|c| {
193            runs.iter()
194                .map(|r| {
195                    r.cases[c]
196                        .measures
197                        .get(metric)
198                        .map(CaseMeasure::contribution)
199                })
200                .collect()
201        })
202        .collect();
203    Ok(MetricCases {
204        name: metric.into(),
205        form,
206        cases,
207    })
208}
209
210/// Whether one run's case measures reproduce its report (3.2.6): the
211/// document binds the report's record digest; its cases are the detail's,
212/// in order, with the same outcome keys; every case-decomposable metric
213/// computed from the measures over the whole cohort equals the report's
214/// value bit for bit (or both are unknown); and the count-ratio sums equal
215/// the detail's counts.
216pub fn reconcile(run: &Evaluated, cases: &EvalCases) -> Result<(), String> {
217    let digest = run
218        .report
219        .record_digest()
220        .map_err(|_| "the report has no record digest".to_string())?;
221    if cases.schema != EVAL_CASES || cases.report != digest || run.detail.report != digest {
222        return Err("the case measures are not bound to this report".into());
223    }
224    if cases.cases.len() != run.detail.cases.len()
225        || cases
226            .cases
227            .iter()
228            .zip(&run.detail.cases)
229            .any(|(m, d)| m.case != d.case || m.outcome != d.outcome)
230    {
231        return Err("the case measures do not list the detail's cases and outcomes".into());
232    }
233    for c in &cases.cases {
234        if let Some(name) = c.measures.keys().find(|m| form_in(m, &[cases]).is_none()) {
235            return Err(format!(
236                "case {} has a measure for `{name}`, which has none",
237                c.case
238            ));
239        }
240    }
241    for m in &run.report.metrics {
242        let Some(form) = form_in(&m.name, &[cases]) else {
243            continue;
244        };
245        let metric = metric_cases(&m.name, form, &[cases])?;
246        let stats = run_stats(&metric, 1, false).map_err(|e| format!("`{}`: {e}", m.name))?;
247        let same = match (&stats.runs[0], &m.value) {
248            (Figure::Known(a), MetricValue::Measured { value, .. }) => {
249                a.to_bits() == value.to_bits()
250            }
251            (Figure::Unknown { .. }, MetricValue::Unknown { .. }) => true,
252            _ => false,
253        };
254        if !same {
255            return Err(format!(
256                "`{}` computed from the case measures is not the report's value",
257                m.name
258            ));
259        }
260        // A count ratio's sums are its count; a mean's count (spec 041)
261        // states measured cases, which no sum of means gives.
262        if let (Form::CountRatio, Some(count)) = (&metric.form, run.detail.count(&m.name)) {
263            let (mut num, mut den) = (0u128, 0u128);
264            for per_run in &metric.cases {
265                if let Some(Contribution::Count { num: n, den: d }) = per_run[0] {
266                    num += u128::from(n);
267                    den += u128::from(d);
268                }
269            }
270            if (num, den) != (u128::from(count.numerator), u128::from(count.denominator)) {
271                return Err(format!("`{}` sums are not the detail's counts", m.name));
272            }
273        }
274    }
275    // A measure for a metric the report does not state is not a measure of
276    // this run.
277    for c in &cases.cases {
278        if let Some(name) = c
279            .measures
280            .keys()
281            .find(|n| !derived(n) && !run.report.metrics.iter().any(|m| &m.name == *n))
282        {
283            return Err(format!(
284                "case {} has a measure for unreported `{name}`",
285                c.case
286            ));
287        }
288    }
289    Ok(())
290}
rustev_eval/eval_cases.rs

rustev_eval/
eval_cases.rs
