lean_ctx/core/eval_ab/
report.rs

1//! Paired report + non-regression gate (#237).
2//!
3//! Every task is scored under *both* conditions, giving a paired sample of per-task deltas
4//! (`lean_ctx − baseline`). From those we compute the mean delta, a **deterministic bootstrap**
5//! 95% confidence interval (fixed seed → byte-identical CI on every machine), win/tie/loss
6//! counts and pass-rate deltas, then collapse it all into a single [`Verdict`] that drives the
7//! CI quality gate.
8
9use serde::{Deserialize, Serialize};
10
11use super::model::ModelFingerprint;
12
13/// Report schema discriminator + version (also guards artifact parsing).
14pub const REPORT_KIND: &str = "lean-ctx.eval-ab-report";
15pub const REPORT_SCHEMA_VERSION: u32 = 1;
16
17/// Equality tolerance when classifying a task as win/tie/loss.
18const EPS: f64 = 1e-9;
19
20/// One task scored under both conditions, with the audit digests for each window + answer.
21#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
22pub struct PairRecord {
23    pub task_id: String,
24    pub domain: String,
25    pub baseline_value: f64,
26    pub lean_ctx_value: f64,
27    pub baseline_passed: bool,
28    pub lean_ctx_passed: bool,
29    pub baseline_tokens: usize,
30    pub lean_ctx_tokens: usize,
31    pub baseline_context_digest: String,
32    pub lean_ctx_context_digest: String,
33    pub baseline_answer_digest: String,
34    pub lean_ctx_answer_digest: String,
35}
36
37impl PairRecord {
38    fn delta(&self) -> f64 {
39        self.lean_ctx_value - self.baseline_value
40    }
41}
42
43/// Aggregate statistics over all paired records.
44#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
45pub struct AbStats {
46    pub n: usize,
47    pub baseline_mean: f64,
48    pub lean_ctx_mean: f64,
49    pub mean_delta: f64,
50    pub ci_low: f64,
51    pub ci_high: f64,
52    pub wins: usize,
53    pub ties: usize,
54    pub losses: usize,
55    pub baseline_pass_rate: f64,
56    pub lean_ctx_pass_rate: f64,
57    pub bootstrap_iters: usize,
58    pub bootstrap_seed: u64,
59    pub noninferiority_margin: f64,
60}
61
62/// The headline conclusion of a run.
63#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
64#[serde(rename_all = "snake_case")]
65pub enum Verdict {
66    /// CI lower bound is strictly positive — lean-ctx improves quality.
67    Improved,
68    /// CI lower bound ≥ −margin — no regression within the tolerated margin.
69    NonInferior,
70    /// CI lower bound below −margin — a regression the gate must block.
71    Regressed,
72}
73
74impl Verdict {
75    pub fn label(self) -> &'static str {
76        match self {
77            Verdict::Improved => "IMPROVED",
78            Verdict::NonInferior => "NO REGRESSION",
79            Verdict::Regressed => "REGRESSED",
80        }
81    }
82
83    /// Whether the CI quality gate should pass.
84    pub fn gate_passes(self) -> bool {
85        !matches!(self, Verdict::Regressed)
86    }
87}
88
89/// Knobs for the statistics + gate. Defaults are deterministic and strict.
90#[derive(Debug, Clone, Copy)]
91pub struct ReportConfig {
92    pub bootstrap_iters: usize,
93    pub bootstrap_seed: u64,
94    /// How far the CI lower bound may sit below zero and still count as "no regression".
95    pub noninferiority_margin: f64,
96}
97
98impl Default for ReportConfig {
99    fn default() -> Self {
100        Self {
101            bootstrap_iters: 2000,
102            bootstrap_seed: 0x5EED_5EED_5EED_5EED,
103            noninferiority_margin: 0.0,
104        }
105    }
106}
107
108/// The full A/B report: provenance, per-task records, aggregate stats and the verdict.
109#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
110pub struct AbReport {
111    pub schema_version: u32,
112    pub kind: String,
113    pub created_at: String,
114    pub lean_ctx_version: String,
115    pub suite: String,
116    pub budget_tokens: usize,
117    pub model: ModelFingerprint,
118    pub records: Vec<PairRecord>,
119    pub stats: AbStats,
120    pub verdict: Verdict,
121}
122
123impl AbReport {
124    /// Computes stats + verdict over the records and assembles the report.
125    pub fn build(
126        suite: impl Into<String>,
127        budget_tokens: usize,
128        model: ModelFingerprint,
129        records: Vec<PairRecord>,
130        cfg: ReportConfig,
131    ) -> Self {
132        let stats = compute_stats(&records, cfg);
133        let verdict = verdict_for(&stats, cfg);
134        Self {
135            schema_version: REPORT_SCHEMA_VERSION,
136            kind: REPORT_KIND.to_string(),
137            created_at: chrono::Utc::now().to_rfc3339(),
138            lean_ctx_version: env!("CARGO_PKG_VERSION").to_string(),
139            suite: suite.into(),
140            budget_tokens,
141            model,
142            records,
143            stats,
144            verdict,
145        }
146    }
147
148    /// Pretty JSON for machine consumption / artifact embedding.
149    pub fn to_json(&self) -> String {
150        serde_json::to_string_pretty(self).unwrap_or_default()
151    }
152
153    /// Compact human summary for the terminal.
154    pub fn render(&self) -> String {
155        let s = &self.stats;
156        let mut out = String::new();
157        out.push_str(&format!("Suite:   {}\n", self.suite));
158        out.push_str(&format!(
159            "Model:   {} ({}, temp={}, seed={})\n",
160            self.model.params.model,
161            self.model.provider,
162            self.model.params.temperature,
163            self.model.params.seed
164        ));
165        out.push_str(&format!(
166            "Budget:  {} tokens / condition\n",
167            self.budget_tokens
168        ));
169        out.push_str(&format!("Tasks:   {}\n\n", s.n));
170        out.push_str(&format!(
171            "Mean score   baseline={:.3}  lean-ctx={:.3}  Δ={:+.3}\n",
172            s.baseline_mean, s.lean_ctx_mean, s.mean_delta
173        ));
174        out.push_str(&format!(
175            "Pass rate    baseline={:.0}%   lean-ctx={:.0}%\n",
176            s.baseline_pass_rate * 100.0,
177            s.lean_ctx_pass_rate * 100.0
178        ));
179        out.push_str(&format!(
180            "Δ 95% CI     [{:+.3}, {:+.3}]  ({} bootstrap, seed {:#x})\n",
181            s.ci_low, s.ci_high, s.bootstrap_iters, s.bootstrap_seed
182        ));
183        out.push_str(&format!(
184            "Win/Tie/Loss {} / {} / {}\n\n",
185            s.wins, s.ties, s.losses
186        ));
187        out.push_str(&format!("VERDICT: {}\n", self.verdict.label()));
188        out
189    }
190}
191
192fn mean(values: impl Iterator<Item = f64>) -> f64 {
193    let mut sum = 0.0;
194    let mut count = 0usize;
195    for v in values {
196        sum += v;
197        count += 1;
198    }
199    if count == 0 {
200        0.0
201    } else {
202        sum / count as f64
203    }
204}
205
206fn compute_stats(records: &[PairRecord], cfg: ReportConfig) -> AbStats {
207    let n = records.len();
208    let baseline_mean = mean(records.iter().map(|r| r.baseline_value));
209    let lean_ctx_mean = mean(records.iter().map(|r| r.lean_ctx_value));
210    let diffs: Vec<f64> = records.iter().map(PairRecord::delta).collect();
211    let mean_delta = mean(diffs.iter().copied());
212
213    let (mut wins, mut ties, mut losses) = (0usize, 0usize, 0usize);
214    for d in &diffs {
215        if *d > EPS {
216            wins += 1;
217        } else if *d < -EPS {
218            losses += 1;
219        } else {
220            ties += 1;
221        }
222    }
223
224    let (ci_low, ci_high) = bootstrap_ci(&diffs, cfg.bootstrap_iters, cfg.bootstrap_seed);
225
226    AbStats {
227        n,
228        baseline_mean,
229        lean_ctx_mean,
230        mean_delta,
231        ci_low,
232        ci_high,
233        wins,
234        ties,
235        losses,
236        baseline_pass_rate: mean(
237            records
238                .iter()
239                .map(|r| f64::from(u8::from(r.baseline_passed))),
240        ),
241        lean_ctx_pass_rate: mean(
242            records
243                .iter()
244                .map(|r| f64::from(u8::from(r.lean_ctx_passed))),
245        ),
246        bootstrap_iters: cfg.bootstrap_iters,
247        bootstrap_seed: cfg.bootstrap_seed,
248        noninferiority_margin: cfg.noninferiority_margin,
249    }
250}
251
252fn verdict_for(stats: &AbStats, cfg: ReportConfig) -> Verdict {
253    if stats.n == 0 {
254        return Verdict::NonInferior;
255    }
256    if stats.ci_low > EPS {
257        Verdict::Improved
258    } else if stats.ci_low >= -cfg.noninferiority_margin - EPS {
259        Verdict::NonInferior
260    } else {
261        Verdict::Regressed
262    }
263}
264
265/// Deterministic percentile bootstrap of the mean of `diffs` (paired deltas).
266fn bootstrap_ci(diffs: &[f64], iters: usize, seed: u64) -> (f64, f64) {
267    let n = diffs.len();
268    if n == 0 || iters == 0 {
269        return (0.0, 0.0);
270    }
271    let mut rng = SplitMix64::new(seed);
272    let mut means: Vec<f64> = Vec::with_capacity(iters);
273    for _ in 0..iters {
274        let mut sum = 0.0;
275        for _ in 0..n {
276            sum += diffs[rng.below(n)];
277        }
278        means.push(sum / n as f64);
279    }
280    means.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
281    (percentile(&means, 2.5), percentile(&means, 97.5))
282}
283
284/// Nearest-rank percentile of a pre-sorted slice.
285fn percentile(sorted: &[f64], p: f64) -> f64 {
286    if sorted.is_empty() {
287        return 0.0;
288    }
289    let rank = (p / 100.0 * (sorted.len() as f64 - 1.0)).round() as usize;
290    sorted[rank.min(sorted.len() - 1)]
291}
292
293/// Tiny seedable PRNG (SplitMix64) — keeps the bootstrap CI reproducible without a dependency.
294struct SplitMix64(u64);
295
296impl SplitMix64 {
297    fn new(seed: u64) -> Self {
298        Self(seed)
299    }
300
301    fn next_u64(&mut self) -> u64 {
302        self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
303        let mut z = self.0;
304        z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
305        z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
306        z ^ (z >> 31)
307    }
308
309    fn below(&mut self, n: usize) -> usize {
310        (self.next_u64() % n as u64) as usize
311    }
312}
313
314#[cfg(test)]
315mod tests {
316    use super::*;
317    use crate::core::eval_ab::model::ModelParams;
318
319    fn rec(id: &str, base: f64, lean: f64) -> PairRecord {
320        PairRecord {
321            task_id: id.into(),
322            domain: "qa".into(),
323            baseline_value: base,
324            lean_ctx_value: lean,
325            baseline_passed: base >= 0.5,
326            lean_ctx_passed: lean >= 0.5,
327            baseline_tokens: 100,
328            lean_ctx_tokens: 100,
329            baseline_context_digest: "a".into(),
330            lean_ctx_context_digest: "b".into(),
331            baseline_answer_digest: "c".into(),
332            lean_ctx_answer_digest: "d".into(),
333        }
334    }
335
336    fn fp() -> ModelFingerprint {
337        ModelFingerprint {
338            provider: "recorded".into(),
339            endpoint: "rec".into(),
340            params: ModelParams::default(),
341        }
342    }
343
344    #[test]
345    fn clear_improvement_is_verdict_improved() {
346        let records = vec![
347            rec("1", 0.0, 1.0),
348            rec("2", 0.0, 1.0),
349            rec("3", 0.2, 0.9),
350            rec("4", 0.1, 1.0),
351            rec("5", 0.0, 0.8),
352        ];
353        let report = AbReport::build("s", 4000, fp(), records, ReportConfig::default());
354        assert_eq!(report.verdict, Verdict::Improved, "{:?}", report.stats);
355        assert!(report.verdict.gate_passes());
356        assert_eq!(report.stats.wins, 5);
357    }
358
359    #[test]
360    fn clear_regression_is_blocked() {
361        let records = vec![
362            rec("1", 1.0, 0.0),
363            rec("2", 1.0, 0.0),
364            rec("3", 0.9, 0.1),
365            rec("4", 1.0, 0.2),
366        ];
367        let report = AbReport::build("s", 4000, fp(), records, ReportConfig::default());
368        assert_eq!(report.verdict, Verdict::Regressed);
369        assert!(!report.verdict.gate_passes());
370    }
371
372    #[test]
373    fn identical_scores_are_non_inferior() {
374        let records = vec![rec("1", 0.7, 0.7), rec("2", 0.4, 0.4)];
375        let report = AbReport::build("s", 4000, fp(), records, ReportConfig::default());
376        assert_eq!(report.verdict, Verdict::NonInferior);
377        assert_eq!(report.stats.ties, 2);
378        assert!(report.verdict.gate_passes());
379    }
380
381    #[test]
382    fn bootstrap_ci_is_deterministic() {
383        let diffs = vec![0.1, 0.3, -0.2, 0.5, 0.0, 0.4];
384        let a = bootstrap_ci(&diffs, 1000, 42);
385        let b = bootstrap_ci(&diffs, 1000, 42);
386        assert_eq!(a, b);
387    }
388}
lean_ctx/core/eval_ab/report.rs

lean_ctx/core/eval_ab/
report.rs
