1use serde::{Deserialize, Serialize};
12
13use crate::experiment::ExperimentReport;
14
15#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
18pub struct CompareThresholds {
19 pub max_pass_rate_drop: f64,
23
24 pub max_latency_p95_ratio: f64,
27}
28
29impl Default for CompareThresholds {
30 fn default() -> Self {
32 Self {
33 max_pass_rate_drop: 0.05,
34 max_latency_p95_ratio: 1.25,
35 }
36 }
37}
38
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
41#[serde(rename_all = "snake_case")]
42pub enum CaseChange {
43 Improved,
45 Regressed,
47 Unchanged,
49 Added,
51 Removed,
53}
54
55#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
57pub struct AssertionDelta {
58 pub assertion: String,
60 pub baseline_rate: f64,
62 pub candidate_rate: f64,
64 pub delta: f64,
66}
67
68#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
71pub struct CaseDelta {
72 pub case_id: String,
74 pub baseline_pass_rate: Option<f64>,
76 pub candidate_pass_rate: Option<f64>,
78 pub change: CaseChange,
80}
81
82#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
84pub struct LatencyDelta {
85 pub baseline_p50: u64,
87 pub candidate_p50: u64,
89 pub p50_ratio: Option<f64>,
91 pub baseline_p95: u64,
93 pub candidate_p95: u64,
95 pub p95_ratio: Option<f64>,
97}
98
99#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
101#[serde(tag = "regression", rename_all = "snake_case")]
102pub enum Regression {
103 AssertionPassRate {
105 assertion: String,
107 baseline: f64,
109 candidate: f64,
111 },
112 CasePassRate {
114 case_id: String,
116 baseline: f64,
118 candidate: f64,
120 },
121 LatencyP95 {
123 baseline_ms: u64,
125 candidate_ms: u64,
127 ratio: f64,
131 },
132}
133
134#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
136pub struct ComparisonReport {
137 pub baseline: String,
139 pub candidate: String,
141 pub thresholds: CompareThresholds,
143 pub assertion_deltas: Vec<AssertionDelta>,
145 pub case_deltas: Vec<CaseDelta>,
147 pub latency: LatencyDelta,
149 pub baseline_cost_usd: f64,
151 pub candidate_cost_usd: f64,
153 pub regressions: Vec<Regression>,
155 pub regressed: bool,
157}
158
159pub fn compare(
170 baseline: &ExperimentReport,
171 candidate: &ExperimentReport,
172 thresholds: &CompareThresholds,
173) -> ComparisonReport {
174 let mut regressions = Vec::new();
175
176 let mut assertion_deltas = Vec::new();
178 for base in &baseline.summary.assertions {
179 if let Some(cand) = candidate
180 .summary
181 .assertions
182 .iter()
183 .find(|cand| cand.assertion == base.assertion)
184 {
185 let delta = cand.rate - base.rate;
186 if -delta > thresholds.max_pass_rate_drop {
187 regressions.push(Regression::AssertionPassRate {
188 assertion: base.assertion.clone(),
189 baseline: base.rate,
190 candidate: cand.rate,
191 });
192 }
193 assertion_deltas.push(AssertionDelta {
194 assertion: base.assertion.clone(),
195 baseline_rate: base.rate,
196 candidate_rate: cand.rate,
197 delta,
198 });
199 }
200 }
201
202 let mut case_deltas = Vec::new();
204 let mut ids: Vec<&str> = baseline
205 .cases
206 .iter()
207 .map(|case| case.case_id.as_str())
208 .chain(candidate.cases.iter().map(|case| case.case_id.as_str()))
209 .collect();
210 ids.sort_unstable();
211 ids.dedup();
212 for id in ids {
213 let base = baseline.cases.iter().find(|case| case.case_id == id);
214 let cand = candidate.cases.iter().find(|case| case.case_id == id);
215 let (baseline_pass_rate, candidate_pass_rate, change) = match (base, cand) {
216 (Some(base), Some(cand)) => {
217 let change = if cand.pass_rate < base.pass_rate {
218 CaseChange::Regressed
219 } else if cand.pass_rate > base.pass_rate {
220 CaseChange::Improved
221 } else {
222 CaseChange::Unchanged
223 };
224 if base.pass_rate - cand.pass_rate > thresholds.max_pass_rate_drop {
225 regressions.push(Regression::CasePassRate {
226 case_id: id.to_owned(),
227 baseline: base.pass_rate,
228 candidate: cand.pass_rate,
229 });
230 }
231 (Some(base.pass_rate), Some(cand.pass_rate), change)
232 }
233 (Some(base), None) => (Some(base.pass_rate), None, CaseChange::Removed),
234 (None, Some(cand)) => (None, Some(cand.pass_rate), CaseChange::Added),
235 (None, None) => unreachable!("case id came from one of the two reports"),
236 };
237 case_deltas.push(CaseDelta {
238 case_id: id.to_owned(),
239 baseline_pass_rate,
240 candidate_pass_rate,
241 change,
242 });
243 }
244
245 let ratio =
247 |base: u64, cand: u64| -> Option<f64> { (base > 0).then(|| cand as f64 / base as f64) };
248 let baseline_p95 = baseline.summary.latency_ms.p95;
249 let candidate_p95 = candidate.summary.latency_ms.p95;
250 let p95_breach = match ratio(baseline_p95, candidate_p95) {
251 Some(r) => r > thresholds.max_latency_p95_ratio,
252 None => candidate_p95 > 0,
254 };
255 if p95_breach {
256 regressions.push(Regression::LatencyP95 {
257 baseline_ms: baseline_p95,
258 candidate_ms: candidate_p95,
259 ratio: ratio(baseline_p95, candidate_p95).unwrap_or(-1.0),
260 });
261 }
262 let latency = LatencyDelta {
263 baseline_p50: baseline.summary.latency_ms.p50,
264 candidate_p50: candidate.summary.latency_ms.p50,
265 p50_ratio: ratio(
266 baseline.summary.latency_ms.p50,
267 candidate.summary.latency_ms.p50,
268 ),
269 baseline_p95,
270 candidate_p95,
271 p95_ratio: ratio(baseline_p95, candidate_p95),
272 };
273
274 let regressed = !regressions.is_empty();
275 ComparisonReport {
276 baseline: baseline.name.clone(),
277 candidate: candidate.name.clone(),
278 thresholds: *thresholds,
279 assertion_deltas,
280 case_deltas,
281 latency,
282 baseline_cost_usd: baseline.summary.total_cost_usd,
283 candidate_cost_usd: candidate.summary.total_cost_usd,
284 regressions,
285 regressed,
286 }
287}