use crate::error::VeridictError;
use crate::metrics;
use crate::report::{MultiReport, Report};
use crate::stats::elo;
use crate::stats::{exact, jeffreys, wilson};
use crate::{CiMethod, MetricConfig, MetricKind, Promotion, Verdict};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Correction {
None,
Bonferroni,
Holm,
}
impl Correction {
fn label(self) -> &'static str {
match self {
Self::None => "none",
Self::Bonferroni => "bonferroni",
Self::Holm => "holm",
}
}
}
const ACHIEVED_ALPHA_FLOOR: f64 = 1e-12;
const BISECTION_ITERATIONS: u32 = 60;
fn achieved_alpha(
metric: MetricKind,
ci_method: CiMethod,
successes: u64,
paired_count: u64,
effect: f64,
pass_above: f64,
confidence: f64,
) -> Option<f64> {
if metrics::lacks_closed_form_ci(metric) {
return None;
}
let p_hat = if metric == MetricKind::Elo {
1.0 / (1.0 + 10f64.powf(-effect / 400.0))
} else {
effect + 0.5
};
let ci_lower_effect_at = |gamma: f64| -> Option<f64> {
let candidate_confidence = 1.0 - gamma;
if metric == MetricKind::Elo {
let (lo, _) =
wilson::wilson_ci_from_proportion(p_hat, paired_count as f64, candidate_confidence)
.ok()?;
Some(elo::elo_from_score(lo))
} else {
let (lo, _) = match ci_method {
CiMethod::Wilson => {
wilson::wilson_ci(successes, paired_count, candidate_confidence).ok()?
}
CiMethod::Jeffreys => {
jeffreys::jeffreys_ci(successes, paired_count, candidate_confidence).ok()?
}
CiMethod::Exact => {
exact::clopper_pearson_ci(successes, paired_count, candidate_confidence).ok()?
}
};
Some(lo - 0.5)
}
};
let gamma_orig = 1.0 - confidence;
let mut lo = ACHIEVED_ALPHA_FLOOR;
let mut hi = gamma_orig;
for _ in 0..BISECTION_ITERATIONS {
let mid = lo + (hi - lo) / 2.0;
match ci_lower_effect_at(mid) {
Some(lower) if lower >= pass_above => hi = mid,
_ => lo = mid,
}
}
Some(hi)
}
fn holm_reject(
achieved: &[Option<f64>],
alpha: f64,
family_size: usize,
) -> (Vec<Option<bool>>, Vec<Option<f64>>) {
let mut order: Vec<usize> = (0..achieved.len())
.filter(|&i| achieved[i].is_some())
.collect();
order.sort_by(|&a, &b| achieved[a].unwrap().total_cmp(&achieved[b].unwrap()));
let mut rejected = vec![None; achieved.len()];
let mut thresholds = vec![None; achieved.len()];
let mut still_rejecting = true;
for (rank, &idx) in order.iter().enumerate() {
let k = rank + 1;
let threshold = alpha / (family_size - k + 1) as f64;
thresholds[idx] = Some(threshold);
still_rejecting = still_rejecting && achieved[idx].unwrap() <= threshold;
rejected[idx] = Some(still_rejecting);
}
(rejected, thresholds)
}
pub fn apply_correction(
reports: &mut [Report],
configs: &[MetricConfig],
correction: Correction,
confidence: f64,
) -> Result<(), VeridictError> {
if correction == Correction::None {
return Ok(());
}
if reports.iter().any(|r| r.cluster_count.is_some()) {
return Err(VeridictError::CorrectionConflictsWithClusterById);
}
if let Some(report) = reports
.iter()
.find(|r| metrics::lacks_closed_form_ci(r.metric))
{
return Err(VeridictError::CorrectionRequiresClosedFormCi {
metric: metrics::metric_label(report.metric),
});
}
let family_size = reports.len();
let alpha = 1.0 - confidence;
let achieved: Vec<Option<f64>> = reports
.iter()
.zip(configs)
.map(|(report, config)| {
if report.verdict != Verdict::Pass {
return None;
}
achieved_alpha(
report.metric,
config.ci_method(),
report.candidate_count,
report.paired_count,
report.effect,
report.pass_above,
confidence,
)
})
.collect();
let (rejected, thresholds): (Vec<Option<bool>>, Vec<Option<f64>>) = match correction {
Correction::None => unreachable!("returned above"),
Correction::Bonferroni => {
let uniform = alpha / family_size as f64;
(
achieved.iter().map(|a| a.map(|v| v <= uniform)).collect(),
achieved.iter().map(|a| a.map(|_| uniform)).collect(),
)
}
Correction::Holm => holm_reject(&achieved, alpha, family_size),
};
for (i, report) in reports.iter_mut().enumerate() {
let family_adjusted_verdict = match rejected[i] {
Some(true) => Verdict::Pass,
Some(false) => Verdict::Inconclusive,
None => report.verdict,
};
report.family_adjusted_verdict = Some(family_adjusted_verdict);
report.family_adjusted_promotion =
Some(Promotion::decide(report.validity, family_adjusted_verdict));
report.correction_method = Some(correction.label());
report.family_size = Some(family_size);
report.achieved_alpha = achieved[i];
report.adjusted_alpha_threshold = thresholds[i];
report.unadjusted_verdict = Some(report.verdict);
}
Ok(())
}
pub fn apply_correction_to_multi(
multi: &mut MultiReport,
configs: &[MetricConfig],
correction: Correction,
confidence: f64,
) -> Result<(), VeridictError> {
apply_correction(&mut multi.reports, configs, correction, confidence)?;
if correction != Correction::None {
let promoted = multi
.reports
.iter()
.all(|r| r.family_adjusted_promotion == Some(Promotion::Promoted));
multi.simultaneous_claims_promotion = Some(if promoted {
Promotion::Promoted
} else {
Promotion::NotPromoted
});
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::report::{DataQuality, REPORT_SCHEMA_VERSION};
use crate::stats::wilson;
use crate::{Promotion, Validity};
#[allow(clippy::too_many_arguments)]
fn report_with(
metric: MetricKind,
verdict: Verdict,
candidate_count: u64,
paired_count: u64,
effect: f64,
ci_low: f64,
ci_high: f64,
pass_above: f64,
fail_below: f64,
) -> Report {
Report {
schema_version: REPORT_SCHEMA_VERSION,
verdict,
validity: Validity::Valid,
promotion: Promotion::decide(Validity::Valid, verdict),
metric,
baseline_count: paired_count - candidate_count,
candidate_count,
paired_count,
effect,
confidence: 0.95,
ci_low,
ci_high,
pass_above,
fail_below,
timeouts: 0,
crashes: 0,
invalid: 0,
failure_breakdown: crate::metrics::FailureBreakdown::default(),
reason: "test fixture".to_string(),
estimated_additional_trials: None,
inconclusive_kind: None,
warnings: Vec::new(),
data_quality: DataQuality::default(),
quantile: None,
cluster_count: None,
max_cluster_size: None,
effective_sample_size: None,
design_effect: None,
scale_diagnostics: None,
tied_count: None,
pair_score_counts: None,
correction_method: None,
family_size: None,
achieved_alpha: None,
adjusted_alpha_threshold: None,
unadjusted_verdict: None,
family_adjusted_verdict: None,
family_adjusted_promotion: None,
}
}
fn winrate_pass(candidate_wins: u64, n: u64, pass_above: f64) -> Report {
let (lo, hi) = wilson::wilson_ci(candidate_wins, n, 0.95).unwrap();
let p_hat = candidate_wins as f64 / n as f64;
report_with(
MetricKind::WinRate,
Verdict::Pass,
candidate_wins,
n,
p_hat - 0.5,
lo - 0.5,
hi - 0.5,
pass_above,
-pass_above,
)
}
fn winrate_config() -> MetricConfig {
MetricConfig::WinRate {
ci_method: CiMethod::Wilson,
failure_policy: crate::FailurePolicy::ReportOnly,
}
}
#[test]
fn stronger_effect_has_smaller_achieved_alpha() {
let weak = achieved_alpha(
MetricKind::WinRate,
CiMethod::Wilson,
58,
100,
0.08,
0.02,
0.95,
)
.unwrap();
let strong = achieved_alpha(
MetricKind::WinRate,
CiMethod::Wilson,
80,
100,
0.30,
0.02,
0.95,
)
.unwrap();
assert!(
strong < weak,
"a larger, cleaner effect should need a smaller achieved alpha to still pass: \
strong={strong}, weak={weak}"
);
}
#[test]
fn achieved_alpha_self_consistency_across_all_ci_methods() {
let successes = 64u64;
let n = 100u64;
let effect = successes as f64 / n as f64 - 0.5;
let pass_above = 0.02;
for ci_method in [CiMethod::Wilson, CiMethod::Exact, CiMethod::Jeffreys] {
let gamma = achieved_alpha(
MetricKind::WinRate,
ci_method,
successes,
n,
effect,
pass_above,
0.95,
)
.unwrap();
let confidence = 1.0 - gamma;
let (lo, _hi) = match ci_method {
CiMethod::Wilson => wilson::wilson_ci(successes, n, confidence).unwrap(),
CiMethod::Exact => exact::clopper_pearson_ci(successes, n, confidence).unwrap(),
CiMethod::Jeffreys => jeffreys::jeffreys_ci(successes, n, confidence).unwrap(),
};
let ci_low_effect = lo - 0.5;
assert!(
(ci_low_effect - pass_above).abs() < 1e-6,
"{ci_method:?}: recomputed ci_low {ci_low_effect} should land on pass_above \
{pass_above}, gamma={gamma}"
);
}
}
#[test]
fn mean_diff_has_no_achieved_alpha() {
assert_eq!(
achieved_alpha(
MetricKind::MeanDiff,
CiMethod::Wilson,
0,
100,
0.05,
0.02,
0.95
),
None
);
}
#[test]
fn none_leaves_reports_completely_untouched() {
let mut reports = vec![winrate_pass(80, 100, 0.02)];
let configs = vec![winrate_config()];
apply_correction(&mut reports, &configs, Correction::None, 0.95).unwrap();
assert_eq!(reports[0].verdict, Verdict::Pass);
assert_eq!(reports[0].correction_method, None);
assert_eq!(reports[0].family_size, None);
assert_eq!(reports[0].achieved_alpha, None);
assert_eq!(reports[0].unadjusted_verdict, None);
assert_eq!(reports[0].family_adjusted_verdict, None);
assert_eq!(reports[0].family_adjusted_promotion, None);
}
#[test]
fn single_metric_family_degenerates_to_a_no_op() {
let mut reports = vec![winrate_pass(80, 100, 0.02)];
let configs = vec![winrate_config()];
apply_correction(&mut reports, &configs, Correction::Bonferroni, 0.95).unwrap();
assert_eq!(reports[0].verdict, Verdict::Pass);
assert_eq!(reports[0].family_adjusted_verdict, Some(Verdict::Pass));
assert_eq!(
reports[0].family_adjusted_promotion,
Some(Promotion::Promoted)
);
apply_correction(&mut reports, &configs, Correction::Holm, 0.95).unwrap();
assert_eq!(reports[0].verdict, Verdict::Pass);
assert_eq!(reports[0].family_adjusted_verdict, Some(Verdict::Pass));
}
#[test]
fn bonferroni_downgrades_a_marginal_claim_without_touching_verdict() {
let mut reports = vec![winrate_pass(62, 100, 0.02)];
let configs = vec![winrate_config()];
apply_correction(&mut reports, &configs, Correction::None, 0.95).unwrap();
assert_eq!(reports[0].verdict, Verdict::Pass);
let single = winrate_pass(62, 100, 0.02);
let mut family = vec![
clone_report(&single),
clone_report(&single),
clone_report(&single),
];
let configs = vec![winrate_config(), winrate_config(), winrate_config()];
apply_correction(&mut family, &configs, Correction::Bonferroni, 0.95).unwrap();
assert_eq!(family[0].verdict, Verdict::Pass);
assert_eq!(family[0].promotion, Promotion::Promoted);
assert_eq!(family[0].unadjusted_verdict, Some(Verdict::Pass));
assert_eq!(
family[0].family_adjusted_verdict,
Some(Verdict::Inconclusive)
);
assert_eq!(
family[0].family_adjusted_promotion,
Some(Promotion::NotPromoted)
);
assert_eq!(family[0].family_size, Some(3));
}
fn clone_report(r: &Report) -> Report {
Report {
schema_version: r.schema_version,
verdict: r.verdict,
validity: r.validity,
promotion: r.promotion,
metric: r.metric,
baseline_count: r.baseline_count,
candidate_count: r.candidate_count,
paired_count: r.paired_count,
effect: r.effect,
confidence: r.confidence,
ci_low: r.ci_low,
ci_high: r.ci_high,
pass_above: r.pass_above,
fail_below: r.fail_below,
timeouts: r.timeouts,
crashes: r.crashes,
invalid: r.invalid,
failure_breakdown: r.failure_breakdown,
reason: r.reason.clone(),
estimated_additional_trials: r.estimated_additional_trials,
inconclusive_kind: r.inconclusive_kind,
warnings: r.warnings.clone(),
data_quality: r.data_quality,
quantile: r.quantile,
cluster_count: r.cluster_count,
max_cluster_size: r.max_cluster_size,
effective_sample_size: r.effective_sample_size,
design_effect: r.design_effect,
scale_diagnostics: r.scale_diagnostics,
tied_count: r.tied_count,
pair_score_counts: r.pair_score_counts,
correction_method: r.correction_method,
family_size: r.family_size,
achieved_alpha: r.achieved_alpha,
adjusted_alpha_threshold: r.adjusted_alpha_threshold,
unadjusted_verdict: r.unadjusted_verdict,
family_adjusted_verdict: r.family_adjusted_verdict,
family_adjusted_promotion: r.family_adjusted_promotion,
}
}
#[test]
fn holm_sequential_stop_holds_back_a_report_that_would_pass_its_own_threshold() {
let alpha = 0.05;
let achieved = vec![
Some(alpha / 3.0 + 0.001), Some(alpha / 2.0 - 0.001), Some(alpha - 0.001), ];
let (rejected, _) = holm_reject(&achieved, alpha, 3);
assert_eq!(rejected, vec![Some(false), Some(false), Some(false)]);
}
#[test]
fn holm_rejects_every_report_when_the_whole_ordered_prefix_holds() {
let alpha = 0.05;
let achieved = vec![
Some(alpha / 3.0 - 1e-6),
Some(alpha / 2.0 - 1e-6),
Some(alpha - 1e-6),
];
let (rejected, _) = holm_reject(&achieved, alpha, 3);
assert_eq!(rejected, vec![Some(true), Some(true), Some(true)]);
}
#[test]
fn correction_rejects_a_mean_diff_report() {
let mean_diff = report_with(
MetricKind::MeanDiff,
Verdict::Pass,
0,
100,
0.05,
0.01,
0.09,
0.0,
0.0,
);
let winrate = winrate_pass(62, 100, 0.02);
let mut reports = vec![mean_diff, winrate];
let configs = vec![
MetricConfig::MeanDiff {
bootstrap_method: crate::BootstrapMethod::Percentile,
},
winrate_config(),
];
let err =
apply_correction(&mut reports, &configs, Correction::Bonferroni, 0.95).unwrap_err();
assert!(matches!(
err,
crate::VeridictError::CorrectionRequiresClosedFormCi {
metric: "metric mean-diff"
}
));
assert_eq!(reports[0].correction_method, None);
}
#[test]
fn correction_rejects_a_quantile_diff_report() {
let quantile_diff = report_with(
MetricKind::QuantileDiff,
Verdict::Pass,
0,
100,
0.05,
0.01,
0.09,
0.0,
0.0,
);
let mut reports = vec![quantile_diff];
let configs = vec![MetricConfig::QuantileDiff {
quantile: 0.95,
bootstrap_method: crate::BootstrapMethod::Percentile,
}];
let err =
apply_correction(&mut reports, &configs, Correction::Bonferroni, 0.95).unwrap_err();
assert!(matches!(
err,
crate::VeridictError::CorrectionRequiresClosedFormCi {
metric: "metric quantile-diff"
}
));
}
#[test]
fn correction_rejects_a_cluster_by_id_report() {
let mut report = winrate_pass(80, 100, 0.02);
report.cluster_count = Some(20);
let mut reports = vec![report];
let configs = vec![winrate_config()];
let err =
apply_correction(&mut reports, &configs, Correction::Bonferroni, 0.95).unwrap_err();
assert!(matches!(
err,
crate::VeridictError::CorrectionConflictsWithClusterById
));
assert_eq!(reports[0].verdict, Verdict::Pass);
assert_eq!(reports[0].correction_method, None);
}
fn multi_report_with(reports: Vec<Report>) -> MultiReport {
let verdict = crate::verdict::aggregate(reports.iter().map(|r| r.verdict));
MultiReport {
schema_version: REPORT_SCHEMA_VERSION,
verdict,
validity: Validity::Valid,
promotion: Promotion::decide(Validity::Valid, verdict),
simultaneous_claims_promotion: None,
reports,
}
}
#[test]
fn simultaneous_claims_promotion_is_promoted_only_when_every_claim_survives() {
let mut multi = multi_report_with(vec![
winrate_pass(90, 100, 0.02),
winrate_pass(90, 100, 0.02),
]);
let configs = vec![winrate_config(), winrate_config()];
apply_correction_to_multi(&mut multi, &configs, Correction::Bonferroni, 0.95).unwrap();
assert_eq!(
multi.simultaneous_claims_promotion,
Some(Promotion::Promoted)
);
assert_eq!(multi.verdict, Verdict::Pass);
assert_eq!(multi.promotion, Promotion::Promoted);
}
#[test]
fn simultaneous_claims_promotion_is_not_promoted_if_any_claim_is_downgraded() {
let single = winrate_pass(62, 100, 0.02);
let mut multi = multi_report_with(vec![
clone_report(&single),
clone_report(&single),
clone_report(&single),
]);
let configs = vec![winrate_config(), winrate_config(), winrate_config()];
apply_correction_to_multi(&mut multi, &configs, Correction::Bonferroni, 0.95).unwrap();
assert_eq!(
multi.simultaneous_claims_promotion,
Some(Promotion::NotPromoted)
);
assert_eq!(multi.verdict, Verdict::Pass);
assert_eq!(multi.promotion, Promotion::Promoted);
}
#[test]
fn simultaneous_claims_promotion_stays_none_when_correction_is_none() {
let mut multi = multi_report_with(vec![winrate_pass(90, 100, 0.02)]);
let configs = vec![winrate_config()];
apply_correction_to_multi(&mut multi, &configs, Correction::None, 0.95).unwrap();
assert_eq!(multi.simultaneous_claims_promotion, None);
}
}