rustev_eval/
diff.rs

1//! Baseline differences and their deterministic Markdown rendering (spec 029, 3.10).
2
3use std::collections::{BTreeMap, BTreeSet};
4use std::fmt::Write;
5
6use rustev_contract::ids::ContentDigest;
7use rustev_contract::limits::REPLAY_V1;
8use rustev_contract::{Document, Identified};
9use serde::{Deserialize, Serialize};
10
11use crate::baseline::{BASELINE, Baseline};
12use crate::config::AdapterRules;
13use crate::eval_cases::{EVAL_CASES, EvalCases};
14use crate::repeat::{BandDoc, FigureDoc, IntervalDoc, REPEAT_REPORT, RepeatReport, RunRef};
15use crate::repeat_config::RepeatConfig;
16
17pub const EVAL_DIFF: &str = "rustev.eval-diff/1";
18pub const MAX_CHANGED_CASES: usize = 10_000;
19
20#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
21#[serde(deny_unknown_fields)]
22pub struct EvalDiff {
23    pub schema: String,
24    pub baseline: ContentDigest,
25    pub run: ContentDigest,
26    pub result: DiffResult,
27}
28
29crate::doc::eval_document!(EvalDiff, EVAL_DIFF, REPLAY_V1);
30
31#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
32#[serde(rename_all = "snake_case", deny_unknown_fields)]
33pub enum DiffResult {
34    Unknown {
35        reason: String,
36    },
37    Compared {
38        cohorts: Vec<DiffCohort>,
39        changed_cases: Vec<ChangedCase>,
40        total_changed_cases: u64,
41        truncated: bool,
42    },
43}
44
45#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
46#[serde(deny_unknown_fields)]
47pub struct DiffCohort {
48    pub name: String,
49    pub metrics: Vec<DiffMetric>,
50}
51
52#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
53#[serde(deny_unknown_fields)]
54pub struct DiffMetric {
55    pub name: String,
56    pub baseline: FigureDoc,
57    pub run: FigureDoc,
58    pub delta: FigureDoc,
59    pub baseline_interval: IntervalDoc,
60    pub run_interval: IntervalDoc,
61    pub overlap: DiffBool,
62    pub within_noise: DiffBool,
63}
64
65#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
66#[serde(rename_all = "snake_case", deny_unknown_fields)]
67pub enum DiffBool {
68    Value(bool),
69    Unknown(String),
70}
71
72#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
73#[serde(deny_unknown_fields)]
74pub struct ChangedCase {
75    pub case: String,
76    pub baseline: CaseDistribution,
77    pub run: CaseDistribution,
78}
79
80#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
81#[serde(deny_unknown_fields)]
82pub struct CaseDistribution {
83    /// Outcome keys in byte order, with their multiplicities.
84    pub outcomes: BTreeMap<String, u64>,
85    /// None when not declared. Lists retain ranking order; entries are
86    /// sorted lexicographically, with the absent list first.
87    pub top_k: Option<Vec<TopKCount>>,
88    /// In run order, retaining null for a case without a run record.
89    pub runs: Vec<Option<ContentDigest>>,
90}
91
92#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
93#[serde(deny_unknown_fields)]
94pub struct TopKCount {
95    pub list: Option<Vec<String>>,
96    pub count: u64,
97}
98
99/// Compare already assembled reports. Case documents must match each side's
100/// recorded digests and list identical case ids in manifest order. Invalid
101/// bindings are refused; incompatible configurations yield an unknown diff.
102/// The configuration supplies the declared slice order and top-k declaration.
103pub fn diff(
104    baseline: &Baseline,
105    run: &RepeatReport,
106    config: &RepeatConfig,
107    baseline_cases: &[&EvalCases],
108    run_cases: &[&EvalCases],
109) -> Result<EvalDiff, String> {
110    if baseline.schema != BASELINE || run.schema != REPEAT_REPORT {
111        return Err("wrong baseline or repeat report schema".into());
112    }
113    let mut doc = EvalDiff {
114        schema: EVAL_DIFF.into(),
115        baseline: baseline.record_digest().map_err(|e| e.to_string())?,
116        run: run.record_digest().map_err(|e| e.to_string())?,
117        result: DiffResult::Unknown {
118            reason: String::new(),
119        },
120    };
121    let mismatch = if baseline.dataset != run.dataset {
122        Some("dataset mismatch")
123    } else if baseline.split != run.split {
124        Some("split mismatch")
125    } else if baseline.evaluator_config != run.evaluator_config {
126        Some("evaluator configuration mismatch")
127    } else if baseline.adapter_rules != run.adapter_rules {
128        Some("adapter rules mismatch")
129    } else if !matches!(baseline.adapter_rules, Some(AdapterRules::Bound(_))) {
130        Some("adapter rules are not bound")
131    } else if baseline.repeat_config != run.repeat_config {
132        Some("repeat configuration mismatch")
133    } else {
134        None
135    };
136    if let Some(reason) = mismatch {
137        doc.result = DiffResult::Unknown {
138            reason: reason.into(),
139        };
140        return Ok(doc);
141    }
142    config.check().map_err(|e| e.to_string())?;
143    if config.id().map_err(|e| e.to_string())? != baseline.repeat_config {
144        return Err("supplied repeat configuration does not bind the reports".into());
145    }
146    check_cases(&baseline.runs, baseline_cases, config)?;
147    check_cases(&run.runs, run_cases, config)?;
148    let manifest = &baseline_cases[0].cases;
149    if run_cases[0]
150        .cases
151        .iter()
152        .map(|c| &c.case)
153        .ne(manifest.iter().map(|c| &c.case))
154    {
155        return Err("mixed-runs: case order differs between sides".into());
156    }
157    let names: Vec<&str> = std::iter::once("all")
158        .chain(config.slices.iter().map(|s| s.name.as_str()))
159        .collect();
160    if baseline
161        .cohorts
162        .iter()
163        .map(|c| c.name.as_str())
164        .ne(names.iter().copied())
165        || run
166            .cohorts
167            .iter()
168            .map(|c| c.name.as_str())
169            .ne(names.iter().copied())
170    {
171        return Err("cohorts do not match the declared order".into());
172    }
173    let mut cohorts = Vec::new();
174    for (b, r) in baseline.cohorts.iter().zip(&run.cohorts) {
175        if b.metrics
176            .iter()
177            .map(|m| &m.name)
178            .ne(r.metrics.iter().map(|m| &m.name))
179        {
180            return Err("metric lists differ".into());
181        }
182        let metrics = b
183            .metrics
184            .iter()
185            .zip(&r.metrics)
186            .map(|(b, r)| {
187                let delta = match (&b.pooled, &r.pooled) {
188                    (FigureDoc::Unknown(s), _) => FigureDoc::Unknown(format!("baseline: {s}")),
189                    (_, FigureDoc::Unknown(s)) => FigureDoc::Unknown(format!("run: {s}")),
190                    (FigureDoc::Value(b), FigureDoc::Value(r)) => {
191                        let d = r - b;
192                        if d.is_finite() {
193                            FigureDoc::Value(d)
194                        } else {
195                            FigureDoc::Unknown("non-finite delta".into())
196                        }
197                    }
198                };
199                let overlap = match (&b.interval, &r.interval) {
200                    (IntervalDoc::Unknown { reason, .. }, _) => {
201                        DiffBool::Unknown(format!("baseline interval: {reason}"))
202                    }
203                    (_, IntervalDoc::Unknown { reason, .. }) => {
204                        DiffBool::Unknown(format!("run interval: {reason}"))
205                    }
206                    (
207                        IntervalDoc::Bounds {
208                            lower: bl,
209                            upper: bu,
210                        },
211                        IntervalDoc::Bounds {
212                            lower: rl,
213                            upper: ru,
214                        },
215                    ) => DiffBool::Value(bl <= ru && rl <= bu),
216                };
217                let within_noise = match (&b.band, &r.band, &delta) {
218                    (BandDoc::Unknown(s), _, _) => DiffBool::Unknown(format!("baseline band: {s}")),
219                    (_, BandDoc::Unknown(s), _) => DiffBool::Unknown(format!("run band: {s}")),
220                    (_, _, FigureDoc::Unknown(s)) => DiffBool::Unknown(s.clone()),
221                    (
222                        BandDoc::Value { value: b, .. },
223                        BandDoc::Value { value: r, .. },
224                        FigureDoc::Value(d),
225                    ) => DiffBool::Value(d.abs() <= b.max(*r)),
226                };
227                DiffMetric {
228                    name: b.name.clone(),
229                    baseline: b.pooled.clone(),
230                    run: r.pooled.clone(),
231                    delta,
232                    baseline_interval: b.interval.clone(),
233                    run_interval: r.interval.clone(),
234                    overlap,
235                    within_noise,
236                }
237            })
238            .collect();
239        cohorts.push(DiffCohort {
240            name: b.name.clone(),
241            metrics,
242        });
243    }
244    let mut changed_cases = Vec::new();
245    let mut total_changed_cases = 0;
246    for (i, case) in manifest.iter().enumerate() {
247        let b = distribution(baseline_cases, i, config.top_k.is_some());
248        let r = distribution(run_cases, i, config.top_k.is_some());
249        if b.outcomes != r.outcomes || b.top_k != r.top_k {
250            total_changed_cases += 1;
251            if changed_cases.len() < MAX_CHANGED_CASES {
252                changed_cases.push(ChangedCase {
253                    case: case.case.clone(),
254                    baseline: b,
255                    run: r,
256                });
257            }
258        }
259    }
260    doc.result = DiffResult::Compared {
261        cohorts,
262        truncated: total_changed_cases > MAX_CHANGED_CASES as u64,
263        changed_cases,
264        total_changed_cases,
265    };
266    Ok(doc)
267}
268
269fn check_cases(refs: &[RunRef], cases: &[&EvalCases], config: &RepeatConfig) -> Result<(), String> {
270    if refs.is_empty() || refs.len() > 64 || refs.len() != cases.len() {
271        return Err("case documents must bind 1 to 64 runs".into());
272    }
273    let first = &cases[0].cases;
274    let mut seen = BTreeSet::new();
275    if first.iter().any(|c| !seen.insert(&c.case)) {
276        return Err("duplicate case id".into());
277    }
278    for (r, c) in refs.iter().zip(cases) {
279        if c.schema != EVAL_CASES
280            || c.report != r.report
281            || c.record_digest().map_err(|e| e.to_string())? != r.cases
282        {
283            return Err("case document does not bind its run".into());
284        }
285        if c.top_k != config.top_k
286            || c.cases
287                .iter()
288                .map(|c| &c.case)
289                .ne(first.iter().map(|c| &c.case))
290        {
291            return Err("mixed-runs: case order or top-k declaration differs".into());
292        }
293    }
294    Ok(())
295}
296
297fn distribution(cases: &[&EvalCases], i: usize, top_k: bool) -> CaseDistribution {
298    let mut outcomes = BTreeMap::new();
299    let mut lists = BTreeMap::new();
300    for c in cases {
301        *outcomes.entry(c.cases[i].outcome.clone()).or_insert(0) += 1;
302        if top_k {
303            *lists.entry(c.cases[i].top_k.clone()).or_insert(0) += 1;
304        }
305    }
306    CaseDistribution {
307        outcomes,
308        top_k: top_k.then(|| {
309            lists
310                .into_iter()
311                .map(|(list, count)| TopKCount { list, count })
312                .collect()
313        }),
314        runs: cases.iter().map(|c| c.cases[i].run.clone()).collect(),
315    }
316}
317
318/// Render only the diff document, with fixed ordering and a final newline.
319pub fn render_markdown(doc: &EvalDiff) -> String {
320    let mut out = String::from("# Evaluation diff\n\n");
321    let DiffResult::Compared {
322        cohorts,
323        changed_cases,
324        total_changed_cases,
325        truncated,
326    } = &doc.result
327    else {
328        if let DiffResult::Unknown { reason } = &doc.result {
329            writeln!(out, "unknown: {}", escape(reason)).unwrap();
330        }
331        return out;
332    };
333    for c in cohorts {
334        writeln!(out, "## {}\n", escape(&c.name)).unwrap();
335        out.push_str("| Metric | Baseline | Run | Delta | Baseline interval | Run interval | Overlap | Within noise |\n| --- | --- | --- | --- | --- | --- | --- | --- |\n");
336        for m in &c.metrics {
337            writeln!(
338                out,
339                "| {} | {} | {} | {} | {} | {} | {} | {} |",
340                escape(&m.name),
341                figure(&m.baseline),
342                figure(&m.run),
343                figure(&m.delta),
344                interval(&m.baseline_interval),
345                interval(&m.run_interval),
346                boolean(&m.overlap),
347                boolean(&m.within_noise)
348            )
349            .unwrap();
350        }
351        out.push('\n');
352    }
353    writeln!(
354        out,
355        "## Changed cases\n\nTotal: {total_changed_cases}; truncated: {truncated}\n"
356    )
357    .unwrap();
358    for c in changed_cases {
359        writeln!(out, "### {}\n", escape(&c.case)).unwrap();
360        for (side, d) in [("Baseline", &c.baseline), ("Run", &c.run)] {
361            writeln!(out, "{side}:\n").unwrap();
362            for (key, count) in &d.outcomes {
363                writeln!(out, "- Outcome {}: {count}", escape(key)).unwrap();
364            }
365            if let Some(lists) = &d.top_k {
366                for t in lists {
367                    writeln!(
368                        out,
369                        "- Top-k {}: {}",
370                        escape(&serde_json::to_string(&t.list).unwrap()),
371                        t.count
372                    )
373                    .unwrap();
374                }
375            }
376            writeln!(
377                out,
378                "- Run records: {}\n",
379                escape(&serde_json::to_string(&d.runs).unwrap())
380            )
381            .unwrap();
382        }
383    }
384    out
385}
386
387fn escape(s: &str) -> String {
388    s.chars()
389        .map(|c| match c {
390            '&' => "&amp;".into(),
391            '<' => "&lt;".into(),
392            '>' => "&gt;".into(),
393            '|' => "&#124;".into(),
394            '\n' => "&#10;".into(),
395            '\r' => "&#13;".into(),
396            '\\' | '`' | '*' | '_' | '[' | ']' | '#' => format!("\\{c}"),
397            _ => c.to_string(),
398        })
399        .collect()
400}
401fn number(v: f64) -> String {
402    let mut bytes = Vec::new();
403    rustev_contract::canonical::write_binary64(v, &mut bytes);
404    String::from_utf8(bytes).expect("binary64 is ASCII")
405}
406fn figure(v: &FigureDoc) -> String {
407    match v {
408        FigureDoc::Value(v) => number(*v),
409        FigureDoc::Unknown(s) => format!("unknown: {}", escape(s)),
410    }
411}
412fn interval(v: &IntervalDoc) -> String {
413    match v {
414        IntervalDoc::Bounds { lower, upper } => format!("[{}, {}]", number(*lower), number(*upper)),
415        IntervalDoc::Unknown { reason, .. } => format!("unknown: {}", escape(reason)),
416    }
417}
418fn boolean(v: &DiffBool) -> String {
419    match v {
420        DiffBool::Value(v) => v.to_string(),
421        DiffBool::Unknown(s) => format!("unknown: {}", escape(s)),
422    }
423}
rustev_eval/diff.rs

rustev_eval/
diff.rs
