use crate::error::VeridictError;
use crate::metrics::FailureBreakdown;
use crate::report::{append_markdown_warnings, serde_str};
use crate::stats::sprt as math;
use crate::{FailurePolicy, Promotion, Validity, Verdict};
use serde::{Deserialize, Serialize};
pub const SPRT_REPORT_SCHEMA_VERSION: u32 = 2;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SprtVariant {
Wald,
Trinomial,
Pentanomial,
}
impl SprtVariant {
pub(super) fn label(self) -> &'static str {
match self {
SprtVariant::Wald => "wald",
SprtVariant::Trinomial => "trinomial",
SprtVariant::Pentanomial => "pentanomial",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct SprtRunOptions {
pub variant: SprtVariant,
pub paired_by_id: bool,
pub failure_policy: FailurePolicy,
pub require_complete_pairs: bool,
pub min_paired_ids: Option<u64>,
pub max_paired_ids: Option<u64>,
}
impl SprtRunOptions {
pub const fn new(variant: SprtVariant) -> Self {
Self {
variant,
paired_by_id: false,
failure_policy: FailurePolicy::ReportOnly,
require_complete_pairs: false,
min_paired_ids: None,
max_paired_ids: None,
}
}
}
impl Default for SprtRunOptions {
fn default() -> Self {
Self::new(SprtVariant::Wald)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub struct SprtConfig {
pub elo0: f64,
pub elo1: f64,
pub alpha: f64,
pub beta: f64,
}
impl SprtConfig {
pub fn new(elo0: f64, elo1: f64, alpha: f64, beta: f64) -> Result<Self, VeridictError> {
if !elo0.is_finite() || !elo1.is_finite() {
return Err(VeridictError::InvalidThreshold(
"elo0/elo1 must be finite".to_string(),
));
}
if elo0 >= elo1 {
return Err(VeridictError::InvalidThreshold(format!(
"elo0 ({elo0}) must be less than elo1 ({elo1})"
)));
}
for (name, v) in [("alpha", alpha), ("beta", beta)] {
if !v.is_finite() || v <= 0.0 || v >= 1.0 {
return Err(VeridictError::InvalidThreshold(format!(
"{name} must be finite and in (0, 1), got {v}"
)));
}
}
if alpha + beta >= 1.0 {
return Err(VeridictError::InvalidThreshold(format!(
"alpha + beta must be less than 1, got {}",
alpha + beta
)));
}
let bounds = math::bounds(alpha, beta);
if !bounds.lower.is_finite() || !bounds.upper.is_finite() {
return Err(VeridictError::InvalidThreshold(
"alpha/beta are too close to zero to produce finite SPRT bounds".to_string(),
));
}
for (name, elo) in [("elo0", elo0), ("elo1", elo1)] {
let score = math::score_from_elo(elo);
if !score.is_finite() || score <= 0.0 || score >= 1.0 {
return Err(VeridictError::InvalidThreshold(format!(
"{name} ({elo}) is too extreme to produce a representable probability"
)));
}
}
Ok(Self {
elo0,
elo1,
alpha,
beta,
})
}
}
#[derive(Debug, Serialize)]
pub struct PentanomialCounts {
pub score_0_0: u64,
pub score_0_5: u64,
pub score_1_0: u64,
pub score_1_5: u64,
pub score_2_0: u64,
}
impl PentanomialCounts {
pub(super) fn from_buckets(buckets: [u64; 5]) -> Self {
Self {
score_0_0: buckets[0],
score_0_5: buckets[1],
score_1_0: buckets[2],
score_1_5: buckets[3],
score_2_0: buckets[4],
}
}
}
#[derive(Debug, Serialize)]
pub struct SprtReport {
pub schema_version: u32,
pub verdict: Verdict,
pub validity: Validity,
pub promotion: Promotion,
pub llr: f64,
pub decision_llr: f64,
pub lower_bound: f64,
pub upper_bound: f64,
pub elo0: f64,
pub elo1: f64,
pub alpha: f64,
pub beta: f64,
pub candidate_wins: u64,
pub baseline_wins: u64,
pub draws: u64,
pub decision_candidate_wins: u64,
pub decision_baseline_wins: u64,
pub decision_draws: u64,
pub timeouts: u64,
pub crashes: u64,
pub invalid: u64,
pub failure_breakdown: FailureBreakdown,
pub reason: String,
pub warnings: Vec<String>,
pub data_quality: SprtDataQuality,
pub drawelo: Option<f64>,
pub decision_drawelo: Option<f64>,
pub sprt_variant: &'static str,
pub pentanomial_counts: Option<PentanomialCounts>,
pub raw_trial_count: Option<u64>,
pub paired_count: Option<u64>,
pub available_paired_count: Option<u64>,
pub stopping_pair_count: Option<u64>,
pub stopping_reason: Option<&'static str>,
pub ignored_pairs_after_stop: Option<u64>,
pub analysis_unit: &'static str,
pub available_observation_count: u64,
pub analyzed_observation_count: u64,
pub stopping_observation_count: Option<u64>,
pub stopping_observation_reason: &'static str,
pub ignored_observations_after_stop: u64,
pub available_record_count: u64,
pub analyzed_record_count: u64,
pub ignored_records_after_stop: u64,
pub min_paired_ids: Option<u64>,
pub max_paired_ids: Option<u64>,
pub require_complete_pairs: bool,
}
#[derive(Debug, Serialize, Default, Clone, Copy, PartialEq, Eq)]
pub struct SprtDataQuality {
pub low_id_diversity: bool,
}
impl SprtReport {
pub fn to_json_pretty(&self) -> String {
serde_json::to_string_pretty(self)
.expect("SprtReport contains only finite fields and strings; serialization cannot fail")
}
pub fn to_markdown(&self) -> String {
let b = &self.failure_breakdown.baseline;
let c = &self.failure_breakdown.candidate;
let unit = if self.sprt_variant == "trinomial" {
"belo"
} else {
"elo"
};
let mut markdown = format!(
"# Veridict SPRT Report\n\n\
Verdict: {verdict}\n\
Validity: {validity}\n\
Promotion: {promotion}\n\n\
H0: {unit} <= {elo0:+.1} / H1: {unit} >= {elo1:+.1} (alpha={alpha}, beta={beta})\n\
Decision LLR: {decision_llr:.4} (bounds: {lower:.4} to {upper:.4})\n\
Full-input LLR: {llr:.4}\n\
{drawelo_line}\
{reason}\n\n\
Schedule: {analyzed_observations} of {available_observations} {analysis_unit}(s) analyzed; stop={stopping_reason}; ignored={ignored_observations} {analysis_unit}(s) / {ignored_records} raw record(s)\n\
Decision outcomes: candidate_wins={decision_candidate_wins}, baseline_wins={decision_baseline_wins}, draws={decision_draws}\n\
Full-input outcomes: candidate_wins={candidate_wins}, baseline_wins={baseline_wins}, draws={draws}\n\
{pentanomial_line}\n\
Status counts:\n\
- timeout: {timeouts} (baseline={b_timeout}, candidate={c_timeout})\n\
- crash: {crashes} (baseline={b_crash}, candidate={c_crash})\n\
- invalid: {invalid} (baseline={b_invalid}, candidate={c_invalid})\n",
verdict = serde_str(&self.verdict),
validity = serde_str(&self.validity),
promotion = serde_str(&self.promotion),
elo0 = self.elo0,
elo1 = self.elo1,
alpha = self.alpha,
beta = self.beta,
decision_llr = self.decision_llr,
llr = self.llr,
lower = self.lower_bound,
upper = self.upper_bound,
drawelo_line = match (self.decision_drawelo, self.drawelo) {
(Some(decision), Some(full)) => format!(
"Decision-prefix drawelo: {decision:+.1}\nFull-input drawelo: {full:+.1}\n"
),
_ => String::new(),
},
reason = self.reason,
analyzed_observations = self.analyzed_observation_count,
available_observations = self.available_observation_count,
analysis_unit = self.analysis_unit,
stopping_reason = self.stopping_observation_reason,
ignored_observations = self.ignored_observations_after_stop,
ignored_records = self.ignored_records_after_stop,
decision_candidate_wins = self.decision_candidate_wins,
decision_baseline_wins = self.decision_baseline_wins,
decision_draws = self.decision_draws,
candidate_wins = self.candidate_wins,
baseline_wins = self.baseline_wins,
draws = self.draws,
pentanomial_line = match &self.pentanomial_counts {
Some(p) => {
let ignored = self.ignored_pairs_after_stop.unwrap_or(0);
let stopped_early =
format!(" ({ignored} pair(s) after the stopping point ignored)");
format!(
"\nPentanomial pairs ({} analyzed of {} available, {} raw trials): \
0-0={} 0.5-0={} 1-1={} 1.5-0.5={} 2-0={}{}\n",
self.paired_count.unwrap_or(0),
self.available_paired_count.unwrap_or(0),
self.raw_trial_count.unwrap_or(0),
p.score_0_0,
p.score_0_5,
p.score_1_0,
p.score_1_5,
p.score_2_0,
if ignored > 0 {
stopped_early
} else {
String::new()
},
)
}
None => String::new(),
},
timeouts = self.timeouts,
crashes = self.crashes,
invalid = self.invalid,
b_timeout = b.timeout,
c_timeout = c.timeout,
b_crash = b.crash,
c_crash = c.crash,
b_invalid = b.invalid,
c_invalid = c.invalid,
);
append_markdown_warnings(&mut markdown, &self.warnings);
markdown
}
}