Skip to main content

rusty_eval/
compare.rs

1//! Baseline-vs-candidate comparison of experiment reports.
2//!
3//! [`fn@compare`] takes two [`ExperimentReport`]s — typically a stored baseline
4//! and a fresh candidate — and answers the release question: did anything
5//! get worse beyond an acceptable threshold? It reports per-assertion pass
6//! rate deltas, per-case regressions and improvements, latency and cost
7//! movement, and a list of [`Regression`] flags. The verdict is deliberately
8//! simple and explainable: a threshold breach is a regression, everything
9//! else is noise or improvement.
10
11use serde::{Deserialize, Serialize};
12
13use crate::experiment::ExperimentReport;
14
15/// How far the candidate may fall behind the baseline before it counts as a
16/// regression.
17#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
18pub struct CompareThresholds {
19    /// Maximum tolerable absolute pass-rate drop, per assertion and per case
20    /// (e.g. `0.05`: a case dropping from 1.0 to 0.9 is within tolerance,
21    /// 1.0 to 0.94 flags).
22    pub max_pass_rate_drop: f64,
23
24    /// Maximum tolerable p95 latency ratio `candidate / baseline` (e.g.
25    /// `1.25`: p95 may grow by 25%).
26    pub max_latency_p95_ratio: f64,
27}
28
29impl Default for CompareThresholds {
30    /// 5-point pass-rate drop tolerance, 25% p95 latency tolerance.
31    fn default() -> Self {
32        Self {
33            max_pass_rate_drop: 0.05,
34            max_latency_p95_ratio: 1.25,
35        }
36    }
37}
38
39/// How a case moved between baseline and candidate.
40#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
41#[serde(rename_all = "snake_case")]
42pub enum CaseChange {
43    /// Candidate pass rate is higher.
44    Improved,
45    /// Candidate pass rate is lower (threshold decides whether it flags).
46    Regressed,
47    /// Same pass rate on both sides.
48    Unchanged,
49    /// Present only in the candidate.
50    Added,
51    /// Present only in the baseline.
52    Removed,
53}
54
55/// Pass-rate movement for one assertion key.
56#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
57pub struct AssertionDelta {
58    /// The assertion's report key.
59    pub assertion: String,
60    /// Baseline pass rate.
61    pub baseline_rate: f64,
62    /// Candidate pass rate.
63    pub candidate_rate: f64,
64    /// `candidate_rate - baseline_rate` (negative is worse).
65    pub delta: f64,
66}
67
68/// Pass-rate movement for one case. Cases present on only one side carry
69/// `None` for the missing side.
70#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
71pub struct CaseDelta {
72    /// The case id.
73    pub case_id: String,
74    /// Baseline pass rate, when the case exists there.
75    pub baseline_pass_rate: Option<f64>,
76    /// Candidate pass rate, when the case exists there.
77    pub candidate_pass_rate: Option<f64>,
78    /// The classification.
79    pub change: CaseChange,
80}
81
82/// Latency movement between the two summaries.
83#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
84pub struct LatencyDelta {
85    /// Baseline median (ms).
86    pub baseline_p50: u64,
87    /// Candidate median (ms).
88    pub candidate_p50: u64,
89    /// `candidate_p50 / baseline_p50` (`null` when the baseline is zero).
90    pub p50_ratio: Option<f64>,
91    /// Baseline p95 (ms).
92    pub baseline_p95: u64,
93    /// Candidate p95 (ms).
94    pub candidate_p95: u64,
95    /// `candidate_p95 / baseline_p95` (`null` when the baseline is zero).
96    pub p95_ratio: Option<f64>,
97}
98
99/// A threshold breach: one concrete, explainable regression.
100#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
101#[serde(tag = "regression", rename_all = "snake_case")]
102pub enum Regression {
103    /// An assertion's pass rate dropped beyond the threshold.
104    AssertionPassRate {
105        /// The assertion's report key.
106        assertion: String,
107        /// Baseline pass rate.
108        baseline: f64,
109        /// Candidate pass rate.
110        candidate: f64,
111    },
112    /// A case's pass rate dropped beyond the threshold.
113    CasePassRate {
114        /// The case id.
115        case_id: String,
116        /// Baseline pass rate.
117        baseline: f64,
118        /// Candidate pass rate.
119        candidate: f64,
120    },
121    /// p95 latency grew beyond the threshold ratio.
122    LatencyP95 {
123        /// Baseline p95 (ms).
124        baseline_ms: u64,
125        /// Candidate p95 (ms).
126        candidate_ms: u64,
127        /// `candidate / baseline` (`inf` is serialized as `1e999`-style JSON
128        /// only by extension; in practice this is a finite ratio or the
129        /// zero-baseline case, recorded as `-1.0` — see [`fn@compare`]).
130        ratio: f64,
131    },
132}
133
134/// The full comparison of a candidate against a baseline.
135#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
136pub struct ComparisonReport {
137    /// Baseline experiment identity.
138    pub baseline: String,
139    /// Candidate experiment identity.
140    pub candidate: String,
141    /// The thresholds applied.
142    pub thresholds: CompareThresholds,
143    /// Per-assertion pass-rate deltas (keys present in both reports).
144    pub assertion_deltas: Vec<AssertionDelta>,
145    /// Per-case movement (union of case ids, sorted).
146    pub case_deltas: Vec<CaseDelta>,
147    /// Latency movement.
148    pub latency: LatencyDelta,
149    /// Baseline total cost (USD).
150    pub baseline_cost_usd: f64,
151    /// Candidate total cost (USD).
152    pub candidate_cost_usd: f64,
153    /// Every threshold breach.
154    pub regressions: Vec<Regression>,
155    /// `true` when any regression flagged — the release-gate bit.
156    pub regressed: bool,
157}
158
159/// Compare `candidate` against `baseline` under `thresholds`.
160///
161/// Matching is by key: assertions by report key, cases by case id. Keys
162/// present on only one side are reported (`Added` / `Removed` for cases;
163/// one-sided assertions are listed in neither delta nor flags — comparing
164/// rates against nothing would invent a baseline).
165///
166/// Zero-baseline latency: when the baseline p95 is 0 ms a ratio is
167/// meaningless, so any positive candidate p95 is treated as breaching and
168/// recorded with ratio `-1.0` (a sentinel, since `inf` is not JSON).
169pub fn compare(
170    baseline: &ExperimentReport,
171    candidate: &ExperimentReport,
172    thresholds: &CompareThresholds,
173) -> ComparisonReport {
174    let mut regressions = Vec::new();
175
176    // ---- per-assertion deltas (shared keys only) ----
177    let mut assertion_deltas = Vec::new();
178    for base in &baseline.summary.assertions {
179        if let Some(cand) = candidate
180            .summary
181            .assertions
182            .iter()
183            .find(|cand| cand.assertion == base.assertion)
184        {
185            let delta = cand.rate - base.rate;
186            if -delta > thresholds.max_pass_rate_drop {
187                regressions.push(Regression::AssertionPassRate {
188                    assertion: base.assertion.clone(),
189                    baseline: base.rate,
190                    candidate: cand.rate,
191                });
192            }
193            assertion_deltas.push(AssertionDelta {
194                assertion: base.assertion.clone(),
195                baseline_rate: base.rate,
196                candidate_rate: cand.rate,
197                delta,
198            });
199        }
200    }
201
202    // ---- per-case deltas (union, sorted by case id) ----
203    let mut case_deltas = Vec::new();
204    let mut ids: Vec<&str> = baseline
205        .cases
206        .iter()
207        .map(|case| case.case_id.as_str())
208        .chain(candidate.cases.iter().map(|case| case.case_id.as_str()))
209        .collect();
210    ids.sort_unstable();
211    ids.dedup();
212    for id in ids {
213        let base = baseline.cases.iter().find(|case| case.case_id == id);
214        let cand = candidate.cases.iter().find(|case| case.case_id == id);
215        let (baseline_pass_rate, candidate_pass_rate, change) = match (base, cand) {
216            (Some(base), Some(cand)) => {
217                let change = if cand.pass_rate < base.pass_rate {
218                    CaseChange::Regressed
219                } else if cand.pass_rate > base.pass_rate {
220                    CaseChange::Improved
221                } else {
222                    CaseChange::Unchanged
223                };
224                if base.pass_rate - cand.pass_rate > thresholds.max_pass_rate_drop {
225                    regressions.push(Regression::CasePassRate {
226                        case_id: id.to_owned(),
227                        baseline: base.pass_rate,
228                        candidate: cand.pass_rate,
229                    });
230                }
231                (Some(base.pass_rate), Some(cand.pass_rate), change)
232            }
233            (Some(base), None) => (Some(base.pass_rate), None, CaseChange::Removed),
234            (None, Some(cand)) => (None, Some(cand.pass_rate), CaseChange::Added),
235            (None, None) => unreachable!("case id came from one of the two reports"),
236        };
237        case_deltas.push(CaseDelta {
238            case_id: id.to_owned(),
239            baseline_pass_rate,
240            candidate_pass_rate,
241            change,
242        });
243    }
244
245    // ---- latency ----
246    let ratio =
247        |base: u64, cand: u64| -> Option<f64> { (base > 0).then(|| cand as f64 / base as f64) };
248    let baseline_p95 = baseline.summary.latency_ms.p95;
249    let candidate_p95 = candidate.summary.latency_ms.p95;
250    let p95_breach = match ratio(baseline_p95, candidate_p95) {
251        Some(r) => r > thresholds.max_latency_p95_ratio,
252        // Baseline measured 0 ms: any positive candidate p95 breaches.
253        None => candidate_p95 > 0,
254    };
255    if p95_breach {
256        regressions.push(Regression::LatencyP95 {
257            baseline_ms: baseline_p95,
258            candidate_ms: candidate_p95,
259            ratio: ratio(baseline_p95, candidate_p95).unwrap_or(-1.0),
260        });
261    }
262    let latency = LatencyDelta {
263        baseline_p50: baseline.summary.latency_ms.p50,
264        candidate_p50: candidate.summary.latency_ms.p50,
265        p50_ratio: ratio(
266            baseline.summary.latency_ms.p50,
267            candidate.summary.latency_ms.p50,
268        ),
269        baseline_p95,
270        candidate_p95,
271        p95_ratio: ratio(baseline_p95, candidate_p95),
272    };
273
274    let regressed = !regressions.is_empty();
275    ComparisonReport {
276        baseline: baseline.name.clone(),
277        candidate: candidate.name.clone(),
278        thresholds: *thresholds,
279        assertion_deltas,
280        case_deltas,
281        latency,
282        baseline_cost_usd: baseline.summary.total_cost_usd,
283        candidate_cost_usd: candidate.summary.total_cost_usd,
284        regressions,
285        regressed,
286    }
287}