use serde::{Deserialize, Serialize};
use crate::experiment::ExperimentReport;
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub struct CompareThresholds {
pub max_pass_rate_drop: f64,
pub max_latency_p95_ratio: f64,
}
impl Default for CompareThresholds {
fn default() -> Self {
Self {
max_pass_rate_drop: 0.05,
max_latency_p95_ratio: 1.25,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CaseChange {
Improved,
Regressed,
Unchanged,
Added,
Removed,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct AssertionDelta {
pub assertion: String,
pub baseline_rate: f64,
pub candidate_rate: f64,
pub delta: f64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CaseDelta {
pub case_id: String,
pub baseline_pass_rate: Option<f64>,
pub candidate_pass_rate: Option<f64>,
pub change: CaseChange,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct LatencyDelta {
pub baseline_p50: u64,
pub candidate_p50: u64,
pub p50_ratio: Option<f64>,
pub baseline_p95: u64,
pub candidate_p95: u64,
pub p95_ratio: Option<f64>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "regression", rename_all = "snake_case")]
pub enum Regression {
AssertionPassRate {
assertion: String,
baseline: f64,
candidate: f64,
},
CasePassRate {
case_id: String,
baseline: f64,
candidate: f64,
},
LatencyP95 {
baseline_ms: u64,
candidate_ms: u64,
ratio: f64,
},
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ComparisonReport {
pub baseline: String,
pub candidate: String,
pub thresholds: CompareThresholds,
pub assertion_deltas: Vec<AssertionDelta>,
pub case_deltas: Vec<CaseDelta>,
pub latency: LatencyDelta,
pub baseline_cost_usd: f64,
pub candidate_cost_usd: f64,
pub regressions: Vec<Regression>,
pub regressed: bool,
}
pub fn compare(
baseline: &ExperimentReport,
candidate: &ExperimentReport,
thresholds: &CompareThresholds,
) -> ComparisonReport {
let mut regressions = Vec::new();
let mut assertion_deltas = Vec::new();
for base in &baseline.summary.assertions {
if let Some(cand) = candidate
.summary
.assertions
.iter()
.find(|cand| cand.assertion == base.assertion)
{
let delta = cand.rate - base.rate;
if -delta > thresholds.max_pass_rate_drop {
regressions.push(Regression::AssertionPassRate {
assertion: base.assertion.clone(),
baseline: base.rate,
candidate: cand.rate,
});
}
assertion_deltas.push(AssertionDelta {
assertion: base.assertion.clone(),
baseline_rate: base.rate,
candidate_rate: cand.rate,
delta,
});
}
}
let mut case_deltas = Vec::new();
let mut ids: Vec<&str> = baseline
.cases
.iter()
.map(|case| case.case_id.as_str())
.chain(candidate.cases.iter().map(|case| case.case_id.as_str()))
.collect();
ids.sort_unstable();
ids.dedup();
for id in ids {
let base = baseline.cases.iter().find(|case| case.case_id == id);
let cand = candidate.cases.iter().find(|case| case.case_id == id);
let (baseline_pass_rate, candidate_pass_rate, change) = match (base, cand) {
(Some(base), Some(cand)) => {
let change = if cand.pass_rate < base.pass_rate {
CaseChange::Regressed
} else if cand.pass_rate > base.pass_rate {
CaseChange::Improved
} else {
CaseChange::Unchanged
};
if base.pass_rate - cand.pass_rate > thresholds.max_pass_rate_drop {
regressions.push(Regression::CasePassRate {
case_id: id.to_owned(),
baseline: base.pass_rate,
candidate: cand.pass_rate,
});
}
(Some(base.pass_rate), Some(cand.pass_rate), change)
}
(Some(base), None) => (Some(base.pass_rate), None, CaseChange::Removed),
(None, Some(cand)) => (None, Some(cand.pass_rate), CaseChange::Added),
(None, None) => unreachable!("case id came from one of the two reports"),
};
case_deltas.push(CaseDelta {
case_id: id.to_owned(),
baseline_pass_rate,
candidate_pass_rate,
change,
});
}
let ratio =
|base: u64, cand: u64| -> Option<f64> { (base > 0).then(|| cand as f64 / base as f64) };
let baseline_p95 = baseline.summary.latency_ms.p95;
let candidate_p95 = candidate.summary.latency_ms.p95;
let p95_breach = match ratio(baseline_p95, candidate_p95) {
Some(r) => r > thresholds.max_latency_p95_ratio,
None => candidate_p95 > 0,
};
if p95_breach {
regressions.push(Regression::LatencyP95 {
baseline_ms: baseline_p95,
candidate_ms: candidate_p95,
ratio: ratio(baseline_p95, candidate_p95).unwrap_or(-1.0),
});
}
let latency = LatencyDelta {
baseline_p50: baseline.summary.latency_ms.p50,
candidate_p50: candidate.summary.latency_ms.p50,
p50_ratio: ratio(
baseline.summary.latency_ms.p50,
candidate.summary.latency_ms.p50,
),
baseline_p95,
candidate_p95,
p95_ratio: ratio(baseline_p95, candidate_p95),
};
let regressed = !regressions.is_empty();
ComparisonReport {
baseline: baseline.name.clone(),
candidate: candidate.name.clone(),
thresholds: *thresholds,
assertion_deltas,
case_deltas,
latency,
baseline_cost_usd: baseline.summary.total_cost_usd,
candidate_cost_usd: candidate.summary.total_cost_usd,
regressions,
regressed,
}
}