aver/bench/
compare.rs

1//! Baseline comparison — diff a current `BenchReport` against a stored
2//! one and decide whether the run regressed past the configured tolerance.
3//!
4//! Two metrics gated in 0.15.1: `p50_ms` and `p95_ms`. `response_bytes`
5//! and `passes_applied` mismatches are reported but not gated yet —
6//! once `response_bytes` capture lands in 0.15.2, exact-match becomes
7//! a hard regression. `passes_applied` mismatch is interesting (a pass
8//! stopped firing) but happens legitimately when a pipeline-level
9//! refactor changes which stages run, so it's reported at info level
10//! for now.
11
12use crate::bench::manifest::Tolerance;
13use crate::bench::report::BenchReport;
14
15#[derive(Debug)]
16pub struct DiffReport {
17    pub scenario: String,
18    pub p50: MetricDiff,
19    pub p95: MetricDiff,
20    /// `true` if any gated metric exceeded its tolerance.
21    pub regressed: bool,
22    /// Non-gated observations worth printing — e.g. response_bytes
23    /// changed, passes_applied set differs.
24    pub notes: Vec<String>,
25}
26
27#[derive(Debug, Clone, Copy)]
28pub struct MetricDiff {
29    pub baseline: f64,
30    pub current: f64,
31    pub delta_pct: f64,
32    pub tolerance_pct: f64,
33    pub regressed: bool,
34}
35
36impl MetricDiff {
37    fn new(baseline: f64, current: f64, tolerance_pct: f64) -> Self {
38        let delta_pct = if baseline > 0.0 {
39            ((current - baseline) / baseline) * 100.0
40        } else {
41            0.0
42        };
43        // Negative delta = faster than baseline = never a regression.
44        let regressed = delta_pct > tolerance_pct;
45        Self {
46            baseline,
47            current,
48            delta_pct,
49            tolerance_pct,
50            regressed,
51        }
52    }
53}
54
55pub fn diff(current: &BenchReport, baseline: &BenchReport, tolerance: Tolerance) -> DiffReport {
56    let p50 = MetricDiff::new(
57        baseline.iterations.p50_ms,
58        current.iterations.p50_ms,
59        tolerance.wall_time_p50_pct,
60    );
61    let p95 = MetricDiff::new(
62        baseline.iterations.p95_ms,
63        current.iterations.p95_ms,
64        tolerance.wall_time_p95_pct,
65    );
66
67    let mut notes = Vec::new();
68    if current.scenario.target != baseline.scenario.target {
69        notes.push(format!(
70            "target changed: baseline={} current={} — different units, do not trust the diff",
71            baseline.scenario.target, current.scenario.target
72        ));
73    }
74    if current.passes_applied != baseline.passes_applied {
75        notes.push(format!(
76            "passes_applied changed: baseline={:?} current={:?}",
77            baseline.passes_applied, current.passes_applied
78        ));
79    }
80    match (current.response_bytes, baseline.response_bytes) {
81        (Some(c), Some(b)) if c != b => {
82            notes.push(format!("response_bytes: baseline={} current={}", b, c));
83        }
84        _ => {}
85    }
86
87    DiffReport {
88        scenario: current.scenario.name.clone(),
89        p50,
90        p95,
91        regressed: p50.regressed || p95.regressed,
92        notes,
93    }
94}
95
96/// Render a diff in the same compact column shape as `format_human` for
97/// `BenchReport` — single block per scenario, colour-free (callers add
98/// terminal colour at the print site if desired).
99pub fn format_diff(diff: &DiffReport) -> String {
100    use std::fmt::Write;
101
102    fn fmt_ms(ms: f64) -> String {
103        if ms >= 1.0 {
104            format!("{:.2}ms", ms)
105        } else {
106            format!("{:.0}µs", ms * 1000.0)
107        }
108    }
109    fn fmt_metric(label: &str, m: &MetricDiff) -> String {
110        let sign = if m.delta_pct >= 0.0 { "+" } else { "" };
111        let verdict = if m.regressed {
112            format!("REGRESSION (limit +{:.0}%)", m.tolerance_pct)
113        } else {
114            "ok".to_string()
115        };
116        format!(
117            "  {:<6} {} (baseline {}, {}{:.1}%)  {}",
118            label,
119            fmt_ms(m.current),
120            fmt_ms(m.baseline),
121            sign,
122            m.delta_pct,
123            verdict,
124        )
125    }
126
127    let mut out = String::new();
128    writeln!(
129        out,
130        "{}: {}",
131        diff.scenario,
132        if diff.regressed { "REGRESSION" } else { "ok" }
133    )
134    .ok();
135    writeln!(out, "{}", fmt_metric("p50", &diff.p50)).ok();
136    writeln!(out, "{}", fmt_metric("p95", &diff.p95)).ok();
137    for note in &diff.notes {
138        writeln!(out, "  note: {}", note).ok();
139    }
140    out
141}
aver/bench/compare.rs

aver/bench/
compare.rs
