lean_ctx/core/eval_ab/
report.rs

1//! Paired report + non-regression gate (#237).
2//!
3//! Every task is scored under *both* conditions, giving a paired sample of per-task deltas
4//! (`lean_ctx − baseline`). From those we compute the mean delta, a **deterministic bootstrap**
5//! 95% confidence interval (fixed seed → byte-identical CI on every machine), win/tie/loss
6//! counts and pass-rate deltas, then collapse it all into a single [`Verdict`] that drives the
7//! CI quality gate.
8
9use serde::{Deserialize, Serialize};
10
11use super::model::ModelFingerprint;
12
13/// Report schema discriminator + version (also guards artifact parsing).
14pub const REPORT_KIND: &str = "lean-ctx.eval-ab-report";
15pub const REPORT_SCHEMA_VERSION: u32 = 1;
16
17/// Equality tolerance when classifying a task as win/tie/loss.
18const EPS: f64 = 1e-9;
19
20/// One task scored under both conditions, with the audit digests for each window + answer.
21#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
22pub struct PairRecord {
23    pub task_id: String,
24    pub domain: String,
25    pub baseline_value: f64,
26    pub lean_ctx_value: f64,
27    pub baseline_passed: bool,
28    pub lean_ctx_passed: bool,
29    pub baseline_tokens: usize,
30    pub lean_ctx_tokens: usize,
31    pub baseline_context_digest: String,
32    pub lean_ctx_context_digest: String,
33    pub baseline_answer_digest: String,
34    pub lean_ctx_answer_digest: String,
35}
36
37impl PairRecord {
38    fn delta(&self) -> f64 {
39        self.lean_ctx_value - self.baseline_value
40    }
41}
42
43/// Aggregate statistics over all paired records.
44#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
45pub struct AbStats {
46    pub n: usize,
47    pub baseline_mean: f64,
48    pub lean_ctx_mean: f64,
49    pub mean_delta: f64,
50    pub ci_low: f64,
51    pub ci_high: f64,
52    pub wins: usize,
53    pub ties: usize,
54    pub losses: usize,
55    pub baseline_pass_rate: f64,
56    pub lean_ctx_pass_rate: f64,
57    pub bootstrap_iters: usize,
58    pub bootstrap_seed: u64,
59    pub noninferiority_margin: f64,
60}
61
62/// The headline conclusion of a run.
63#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
64#[serde(rename_all = "snake_case")]
65pub enum Verdict {
66    /// CI lower bound is strictly positive — lean-ctx improves quality.
67    Improved,
68    /// CI lower bound ≥ −margin — no regression within the tolerated margin.
69    NonInferior,
70    /// CI lower bound below −margin — a regression the gate must block.
71    Regressed,
72}
73
74impl Verdict {
75    pub fn label(self) -> &'static str {
76        match self {
77            Verdict::Improved => "IMPROVED",
78            Verdict::NonInferior => "NO REGRESSION",
79            Verdict::Regressed => "REGRESSED",
80        }
81    }
82
83    /// Whether the CI quality gate should pass.
84    pub fn gate_passes(self) -> bool {
85        !matches!(self, Verdict::Regressed)
86    }
87}
88
89/// Knobs for the statistics + gate. Defaults are deterministic and strict.
90#[derive(Debug, Clone, Copy)]
91pub struct ReportConfig {
92    pub bootstrap_iters: usize,
93    pub bootstrap_seed: u64,
94    /// How far the CI lower bound may sit below zero and still count as "no regression".
95    pub noninferiority_margin: f64,
96}
97
98impl Default for ReportConfig {
99    fn default() -> Self {
100        Self {
101            bootstrap_iters: 2000,
102            bootstrap_seed: 0x5EED_5EED_5EED_5EED,
103            noninferiority_margin: 0.0,
104        }
105    }
106}
107
108/// The full A/B report: provenance, per-task records, aggregate stats and the verdict.
109#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
110pub struct AbReport {
111    pub schema_version: u32,
112    pub kind: String,
113    pub created_at: String,
114    pub lean_ctx_version: String,
115    pub suite: String,
116    pub budget_tokens: usize,
117    pub model: ModelFingerprint,
118    pub records: Vec<PairRecord>,
119    pub stats: AbStats,
120    pub verdict: Verdict,
121}
122
123impl AbReport {
124    /// Computes stats + verdict over the records and assembles the report.
125    pub fn build(
126        suite: impl Into<String>,
127        budget_tokens: usize,
128        model: ModelFingerprint,
129        records: Vec<PairRecord>,
130        cfg: ReportConfig,
131    ) -> Self {
132        let stats = compute_stats(&records, cfg);
133        let verdict = verdict_for(&stats, cfg);
134        Self {
135            schema_version: REPORT_SCHEMA_VERSION,
136            kind: REPORT_KIND.to_string(),
137            created_at: chrono::Utc::now().to_rfc3339(),
138            lean_ctx_version: env!("CARGO_PKG_VERSION").to_string(),
139            suite: suite.into(),
140            budget_tokens,
141            model,
142            records,
143            stats,
144            verdict,
145        }
146    }
147
148    /// Pretty JSON for machine consumption / artifact embedding.
149    pub fn to_json(&self) -> String {
150        serde_json::to_string_pretty(self).unwrap_or_default()
151    }
152
153    /// Compact human summary for the terminal.
154    pub fn render(&self) -> String {
155        let s = &self.stats;
156        let mut out = String::new();
157        out.push_str(&format!("Suite:   {}\n", self.suite));
158        out.push_str(&format!(
159            "Model:   {} ({}, temp={}, seed={})\n",
160            self.model.params.model,
161            self.model.provider,
162            self.model.params.temperature,
163            self.model.params.seed
164        ));
165        out.push_str(&format!(
166            "Budget:  {} tokens / condition\n",
167            self.budget_tokens
168        ));
169        out.push_str(&format!("Tasks:   {}\n\n", s.n));
170        out.push_str(&format!(
171            "Mean score   baseline={:.3}  lean-ctx={:.3}  Δ={:+.3}\n",
172            s.baseline_mean, s.lean_ctx_mean, s.mean_delta
173        ));
174        out.push_str(&format!(
175            "Pass rate    baseline={:.0}%   lean-ctx={:.0}%\n",
176            s.baseline_pass_rate * 100.0,
177            s.lean_ctx_pass_rate * 100.0
178        ));
179        out.push_str(&format!(
180            "Δ 95% CI     [{:+.3}, {:+.3}]  ({} bootstrap, seed {:#x})\n",
181            s.ci_low, s.ci_high, s.bootstrap_iters, s.bootstrap_seed
182        ));
183        out.push_str(&format!(
184            "Win/Tie/Loss {} / {} / {}\n\n",
185            s.wins, s.ties, s.losses
186        ));
187        out.push_str(&format!("VERDICT: {}\n", self.verdict.label()));
188        out
189    }
190}
191
192fn mean(values: impl Iterator<Item = f64>) -> f64 {
193    let mut sum = 0.0;
194    let mut count = 0usize;
195    for v in values {
196        sum += v;
197        count += 1;
198    }
199    if count == 0 { 0.0 } else { sum / count as f64 }
200}
201
202fn compute_stats(records: &[PairRecord], cfg: ReportConfig) -> AbStats {
203    let n = records.len();
204    let baseline_mean = mean(records.iter().map(|r| r.baseline_value));
205    let lean_ctx_mean = mean(records.iter().map(|r| r.lean_ctx_value));
206    let diffs: Vec<f64> = records.iter().map(PairRecord::delta).collect();
207    let mean_delta = mean(diffs.iter().copied());
208
209    let (mut wins, mut ties, mut losses) = (0usize, 0usize, 0usize);
210    for d in &diffs {
211        if *d > EPS {
212            wins += 1;
213        } else if *d < -EPS {
214            losses += 1;
215        } else {
216            ties += 1;
217        }
218    }
219
220    let (ci_low, ci_high) = bootstrap_ci(&diffs, cfg.bootstrap_iters, cfg.bootstrap_seed);
221
222    AbStats {
223        n,
224        baseline_mean,
225        lean_ctx_mean,
226        mean_delta,
227        ci_low,
228        ci_high,
229        wins,
230        ties,
231        losses,
232        baseline_pass_rate: mean(
233            records
234                .iter()
235                .map(|r| f64::from(u8::from(r.baseline_passed))),
236        ),
237        lean_ctx_pass_rate: mean(
238            records
239                .iter()
240                .map(|r| f64::from(u8::from(r.lean_ctx_passed))),
241        ),
242        bootstrap_iters: cfg.bootstrap_iters,
243        bootstrap_seed: cfg.bootstrap_seed,
244        noninferiority_margin: cfg.noninferiority_margin,
245    }
246}
247
248fn verdict_for(stats: &AbStats, cfg: ReportConfig) -> Verdict {
249    if stats.n == 0 {
250        return Verdict::NonInferior;
251    }
252    if stats.ci_low > EPS {
253        Verdict::Improved
254    } else if stats.ci_low >= -cfg.noninferiority_margin - EPS {
255        Verdict::NonInferior
256    } else {
257        Verdict::Regressed
258    }
259}
260
261/// Deterministic percentile bootstrap of the mean of `diffs` (paired deltas).
262fn bootstrap_ci(diffs: &[f64], iters: usize, seed: u64) -> (f64, f64) {
263    let n = diffs.len();
264    if n == 0 || iters == 0 {
265        return (0.0, 0.0);
266    }
267    let mut rng = SplitMix64::new(seed);
268    let mut means: Vec<f64> = Vec::with_capacity(iters);
269    for _ in 0..iters {
270        let mut sum = 0.0;
271        for _ in 0..n {
272            sum += diffs[rng.below(n)];
273        }
274        means.push(sum / n as f64);
275    }
276    means.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
277    (percentile(&means, 2.5), percentile(&means, 97.5))
278}
279
280/// Nearest-rank percentile of a pre-sorted slice.
281fn percentile(sorted: &[f64], p: f64) -> f64 {
282    if sorted.is_empty() {
283        return 0.0;
284    }
285    let rank = (p / 100.0 * (sorted.len() as f64 - 1.0)).round() as usize;
286    sorted[rank.min(sorted.len() - 1)]
287}
288
289/// Tiny seedable PRNG (SplitMix64) — keeps the bootstrap CI reproducible without a dependency.
290struct SplitMix64(u64);
291
292impl SplitMix64 {
293    fn new(seed: u64) -> Self {
294        Self(seed)
295    }
296
297    fn next_u64(&mut self) -> u64 {
298        self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
299        let mut z = self.0;
300        z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
301        z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
302        z ^ (z >> 31)
303    }
304
305    fn below(&mut self, n: usize) -> usize {
306        (self.next_u64() % n as u64) as usize
307    }
308}
309
310#[cfg(test)]
311mod tests {
312    use super::*;
313    use crate::core::eval_ab::model::ModelParams;
314
315    fn rec(id: &str, base: f64, lean: f64) -> PairRecord {
316        PairRecord {
317            task_id: id.into(),
318            domain: "qa".into(),
319            baseline_value: base,
320            lean_ctx_value: lean,
321            baseline_passed: base >= 0.5,
322            lean_ctx_passed: lean >= 0.5,
323            baseline_tokens: 100,
324            lean_ctx_tokens: 100,
325            baseline_context_digest: "a".into(),
326            lean_ctx_context_digest: "b".into(),
327            baseline_answer_digest: "c".into(),
328            lean_ctx_answer_digest: "d".into(),
329        }
330    }
331
332    fn fp() -> ModelFingerprint {
333        ModelFingerprint {
334            provider: "recorded".into(),
335            endpoint: "rec".into(),
336            params: ModelParams::default(),
337        }
338    }
339
340    #[test]
341    fn clear_improvement_is_verdict_improved() {
342        let records = vec![
343            rec("1", 0.0, 1.0),
344            rec("2", 0.0, 1.0),
345            rec("3", 0.2, 0.9),
346            rec("4", 0.1, 1.0),
347            rec("5", 0.0, 0.8),
348        ];
349        let report = AbReport::build("s", 4000, fp(), records, ReportConfig::default());
350        assert_eq!(report.verdict, Verdict::Improved, "{:?}", report.stats);
351        assert!(report.verdict.gate_passes());
352        assert_eq!(report.stats.wins, 5);
353    }
354
355    #[test]
356    fn clear_regression_is_blocked() {
357        let records = vec![
358            rec("1", 1.0, 0.0),
359            rec("2", 1.0, 0.0),
360            rec("3", 0.9, 0.1),
361            rec("4", 1.0, 0.2),
362        ];
363        let report = AbReport::build("s", 4000, fp(), records, ReportConfig::default());
364        assert_eq!(report.verdict, Verdict::Regressed);
365        assert!(!report.verdict.gate_passes());
366    }
367
368    #[test]
369    fn identical_scores_are_non_inferior() {
370        let records = vec![rec("1", 0.7, 0.7), rec("2", 0.4, 0.4)];
371        let report = AbReport::build("s", 4000, fp(), records, ReportConfig::default());
372        assert_eq!(report.verdict, Verdict::NonInferior);
373        assert_eq!(report.stats.ties, 2);
374        assert!(report.verdict.gate_passes());
375    }
376
377    #[test]
378    fn bootstrap_ci_is_deterministic() {
379        let diffs = vec![0.1, 0.3, -0.2, 0.5, 0.0, 0.4];
380        let a = bootstrap_ci(&diffs, 1000, 42);
381        let b = bootstrap_ci(&diffs, 1000, 42);
382        assert_eq!(a, b);
383    }
384}
lean_ctx/core/eval_ab/report.rs

lean_ctx/core/eval_ab/
report.rs
