use serde::Serialize;
use crate::metrics::FailureBreakdown;
use crate::{InconclusiveKind, MetricKind, Promotion, Validity, Verdict};
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
pub struct PairScoreCounts {
pub score_0_0: u64,
pub score_0_5: u64,
pub score_1_0: u64,
pub score_1_5: u64,
pub score_2_0: u64,
}
impl PairScoreCounts {
pub(crate) fn from_buckets(buckets: [u64; 5]) -> Self {
Self {
score_0_0: buckets[0],
score_0_5: buckets[1],
score_1_0: buckets[2],
score_1_5: buckets[3],
score_2_0: buckets[4],
}
}
}
pub const REPORT_SCHEMA_VERSION: u32 = 1;
pub const WIDE_BASELINE_SCALE_ORDERS: f64 = 1.0;
pub const TIE_DILUTION_FRACTION: f64 = 0.5;
pub(crate) fn id_diversity_advisory(
records_with_id: u64,
max_id_count: u64,
) -> (bool, Option<String>) {
let low_id_diversity = records_with_id >= 10 && max_id_count >= 3;
let warning = low_id_diversity.then(|| {
format!(
"low id diversity: one id repeated {max_id_count} times among {records_with_id} \
id-tagged trial(s) - looks like the same test case was logged multiple times, not \
independent samples"
)
});
(low_id_diversity, warning)
}
pub(crate) fn append_markdown_warnings(body: &mut String, warnings: &[String]) {
if warnings.is_empty() {
return;
}
body.push_str("\nWarnings:\n");
for warning in warnings {
body.push_str(&format!("- {warning}\n"));
}
}
#[derive(Debug, Serialize)]
pub struct Report {
pub schema_version: u32,
pub verdict: Verdict,
pub validity: Validity,
pub promotion: Promotion,
pub metric: MetricKind,
pub baseline_count: u64,
pub candidate_count: u64,
pub paired_count: u64,
pub effect: f64,
pub confidence: f64,
pub ci_low: f64,
pub ci_high: f64,
pub pass_above: f64,
pub fail_below: f64,
pub timeouts: u64,
pub crashes: u64,
pub invalid: u64,
pub failure_breakdown: FailureBreakdown,
pub reason: String,
pub estimated_additional_trials: Option<u64>,
pub inconclusive_kind: Option<InconclusiveKind>,
pub warnings: Vec<String>,
pub data_quality: DataQuality,
#[serde(skip_serializing_if = "Option::is_none")]
pub quantile: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cluster_count: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub max_cluster_size: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub effective_sample_size: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub design_effect: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub scale_diagnostics: Option<ScaleDiagnostics>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tied_count: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub pair_score_counts: Option<PairScoreCounts>,
#[serde(skip_serializing_if = "Option::is_none")]
pub correction_method: Option<&'static str>,
#[serde(skip_serializing_if = "Option::is_none")]
pub family_size: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub achieved_alpha: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub adjusted_alpha_threshold: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub unadjusted_verdict: Option<crate::Verdict>,
#[serde(skip_serializing_if = "Option::is_none")]
pub family_adjusted_verdict: Option<crate::Verdict>,
#[serde(skip_serializing_if = "Option::is_none")]
pub family_adjusted_promotion: Option<crate::Promotion>,
}
#[derive(Debug, Serialize, Default, Clone, Copy, PartialEq, Eq)]
pub struct DataQuality {
pub tiny_sample: bool,
pub high_failure_rate: bool,
pub draw_heavy: bool,
pub effect_within_noise_floor: bool,
pub low_id_diversity: bool,
pub thin_quantile_tail: bool,
pub wide_baseline_scale: bool,
pub diluted_by_ties: bool,
}
#[derive(Debug, Clone, Copy, Serialize)]
pub struct ScaleDiagnostics {
pub positive_baseline_count: u64,
pub non_positive_baseline_count: u64,
pub min_positive_baseline: f64,
pub max_positive_baseline: f64,
pub raw_orders_of_magnitude: f64,
pub robust_orders_of_magnitude: Option<f64>,
}
fn fmt_effect(metric: MetricKind, value: f64) -> String {
match metric {
MetricKind::WinRate | MetricKind::SignTest => format!("{:+.1} pp", value * 100.0),
MetricKind::Elo | MetricKind::PairedElo => format!("{value:+.1} elo"),
MetricKind::MeanDiff | MetricKind::QuantileDiff => format!("{value:+.4}"),
MetricKind::RelativeDiff => format!("{:+.1}%", value * 100.0),
}
}
pub(crate) fn serde_str<T: Serialize>(value: &T) -> String {
serde_json::to_value(value)
.ok()
.and_then(|v| v.as_str().map(str::to_string))
.unwrap_or_default()
}
impl Report {
pub fn to_json_pretty(&self) -> String {
serde_json::to_string_pretty(self)
.expect("Report contains only finite fields and strings; serialization cannot fail")
}
pub fn to_markdown(&self) -> String {
format!("# Veridict Report\n\n{}", self.to_markdown_body())
}
fn to_markdown_body(&self) -> String {
let b = &self.failure_breakdown.baseline;
let c = &self.failure_breakdown.candidate;
let mut body = format!(
"Verdict: {verdict}\n\
Validity: {validity}\n\
Promotion: {promotion}\n\n\
Metric: {metric}\n\
Effect: {effect}\n\
{confidence_pct}% CI: {ci_low} to {ci_high}\n\
Threshold for pass: {pass_above} / Threshold for fail: {fail_below}\n\n\
{reason}\n\
{estimate_line}\n\
Samples: baseline={baseline_count}, candidate={candidate_count}, paired={paired_count}\n\n\
Status counts:\n\
- timeout: {timeouts} (baseline={b_timeout}, candidate={c_timeout})\n\
- crash: {crashes} (baseline={b_crash}, candidate={c_crash})\n\
- invalid: {invalid} (baseline={b_invalid}, candidate={c_invalid})\n",
verdict = match self.inconclusive_kind {
Some(kind) => format!("{} ({})", serde_str(&self.verdict), serde_str(&kind)),
None => serde_str(&self.verdict),
},
validity = serde_str(&self.validity),
promotion = serde_str(&self.promotion),
metric = match self.quantile {
Some(q) => format!("{} (q={q:.2})", serde_str(&self.metric)),
None => serde_str(&self.metric),
},
effect = fmt_effect(self.metric, self.effect),
confidence_pct = self.confidence * 100.0,
ci_low = fmt_effect(self.metric, self.ci_low),
ci_high = fmt_effect(self.metric, self.ci_high),
pass_above = fmt_effect(self.metric, self.pass_above),
fail_below = fmt_effect(self.metric, self.fail_below),
reason = self.reason,
estimate_line = match self.estimated_additional_trials {
Some(n) =>
format!("\nEstimated additional trials to reach a decisive verdict: ~{n}\n"),
None => String::new(),
},
baseline_count = self.baseline_count,
candidate_count = self.candidate_count,
paired_count = self.paired_count,
timeouts = self.timeouts,
crashes = self.crashes,
invalid = self.invalid,
b_timeout = b.timeout,
c_timeout = c.timeout,
b_crash = b.crash,
c_crash = c.crash,
b_invalid = b.invalid,
c_invalid = c.invalid,
);
if let Some(cluster_count) = self.cluster_count {
body.push_str(&format!(
"\nClusters: {cluster_count} (largest: {max_cluster_size}), effective_sample_size={effective_sample_size:.1}, design_effect={design_effect:.2}\n",
max_cluster_size = self.max_cluster_size.unwrap_or(0),
effective_sample_size = self.effective_sample_size.unwrap_or(0.0),
design_effect = self.design_effect.unwrap_or(1.0),
));
}
if let Some(counts) = self.pair_score_counts {
body.push_str(&format!(
"\nPair-score counts: 0={score_0_0}, 0.5={score_0_5}, 1={score_1_0}, \
1.5={score_1_5}, 2={score_2_0}\n",
score_0_0 = counts.score_0_0,
score_0_5 = counts.score_0_5,
score_1_0 = counts.score_1_0,
score_1_5 = counts.score_1_5,
score_2_0 = counts.score_2_0,
));
}
if let Some(family_adjusted_verdict) = self.family_adjusted_verdict {
body.push_str(&format!(
"\n{method} correction (family_size={family_size}): family-adjusted verdict \
{verdict}, family-adjusted promotion {promotion}\n",
method = self.correction_method.unwrap_or("?"),
family_size = self.family_size.unwrap_or(0),
verdict = serde_str(&family_adjusted_verdict),
promotion = self
.family_adjusted_promotion
.map(|p| serde_str(&p))
.unwrap_or_default(),
));
}
append_markdown_warnings(&mut body, &self.warnings);
body
}
}
#[derive(Debug, Serialize)]
pub struct MultiReport {
pub schema_version: u32,
pub verdict: Verdict,
pub validity: Validity,
pub promotion: Promotion,
#[serde(skip_serializing_if = "Option::is_none")]
pub simultaneous_claims_promotion: Option<Promotion>,
pub reports: Vec<Report>,
}
impl MultiReport {
pub fn to_json_pretty(&self) -> String {
serde_json::to_string_pretty(self).expect(
"MultiReport contains only finite fields and strings; serialization cannot fail",
)
}
pub fn to_markdown(&self) -> String {
let mut out = format!(
"# Veridict Report\n\nOverall verdict: {}\nOverall validity: {}\nOverall promotion: {}\n",
serde_str(&self.verdict),
serde_str(&self.validity),
serde_str(&self.promotion),
);
if let Some(simultaneous_claims_promotion) = self.simultaneous_claims_promotion {
out.push_str(&format!(
"Simultaneous claims promotion: {}\n",
serde_str(&simultaneous_claims_promotion)
));
}
for report in &self.reports {
out.push_str("\n---\n\n");
out.push_str(&report.to_markdown_body());
}
out
}
}
#[cfg(test)]
mod tests {
use super::*;
fn sample_report() -> Report {
Report {
schema_version: REPORT_SCHEMA_VERSION,
verdict: Verdict::Pass,
validity: Validity::Valid,
promotion: Promotion::Promoted,
metric: MetricKind::WinRate,
baseline_count: 20,
candidate_count: 80,
paired_count: 100,
effect: 0.06,
confidence: 0.95,
ci_low: 0.02,
ci_high: 0.10,
pass_above: 0.02,
fail_below: -0.02,
timeouts: 1,
crashes: 0,
invalid: 0,
failure_breakdown: FailureBreakdown::default(),
reason: "ok".to_string(),
estimated_additional_trials: None,
inconclusive_kind: None,
warnings: Vec::new(),
data_quality: DataQuality::default(),
quantile: None,
cluster_count: None,
max_cluster_size: None,
effective_sample_size: None,
design_effect: None,
scale_diagnostics: None,
tied_count: None,
pair_score_counts: None,
correction_method: None,
family_size: None,
achieved_alpha: None,
adjusted_alpha_threshold: None,
unadjusted_verdict: None,
family_adjusted_verdict: None,
family_adjusted_promotion: None,
}
}
#[test]
fn serializes_metric_and_verdict_as_lowercase() {
let json = sample_report().to_json_pretty();
assert!(json.contains("\"verdict\": \"pass\""));
assert!(json.contains("\"metric\": \"winrate\""));
}
#[test]
fn report_and_multi_report_both_carry_schema_version() {
let report_json = sample_report().to_json_pretty();
assert!(report_json.contains("\"schema_version\": 1"));
let multi = MultiReport {
schema_version: REPORT_SCHEMA_VERSION,
verdict: Verdict::Pass,
validity: Validity::Valid,
promotion: Promotion::Promoted,
simultaneous_claims_promotion: None,
reports: vec![sample_report()],
};
assert!(multi.to_json_pretty().contains("\"schema_version\": 1"));
}
#[test]
fn markdown_reports_effect_in_percentage_points_for_winrate() {
let md = sample_report().to_markdown();
assert!(md.contains("Verdict: pass"));
assert!(md.contains("+6.0 pp"));
}
#[test]
fn markdown_reports_effect_in_raw_units_for_mean_diff() {
let mut report = sample_report();
report.metric = MetricKind::MeanDiff;
report.effect = 0.032;
let md = report.to_markdown();
assert!(md.contains("+0.0320"));
}
#[test]
fn markdown_reports_effect_in_elo_units() {
let mut report = sample_report();
report.metric = MetricKind::Elo;
report.effect = 12.3;
let md = report.to_markdown();
assert!(md.contains("+12.3 elo"));
}
#[test]
fn estimated_additional_trials_null_is_omitted_from_markdown_but_present_in_json() {
let report = sample_report();
assert!(!report.to_markdown().contains("Estimated additional trials"));
assert!(
report
.to_json_pretty()
.contains("\"estimated_additional_trials\": null")
);
}
#[test]
fn estimated_additional_trials_some_appears_in_markdown_and_json() {
let mut report = sample_report();
report.estimated_additional_trials = Some(750);
assert!(
report
.to_markdown()
.contains("Estimated additional trials to reach a decisive verdict: ~750")
);
assert!(
report
.to_json_pretty()
.contains("\"estimated_additional_trials\": 750")
);
}
#[test]
fn inconclusive_kind_none_leaves_the_verdict_line_bare() {
let report = sample_report();
assert!(report.to_markdown().contains("Verdict: pass\n"));
}
#[test]
fn inconclusive_kind_directional_appears_on_the_verdict_line() {
let mut report = sample_report();
report.verdict = Verdict::Inconclusive;
report.inconclusive_kind = Some(crate::InconclusiveKind::Directional);
assert!(
report
.to_markdown()
.contains("Verdict: inconclusive (directional)\n")
);
}
#[test]
fn inconclusive_kind_noise_appears_on_the_verdict_line() {
let mut report = sample_report();
report.verdict = Verdict::Inconclusive;
report.inconclusive_kind = Some(crate::InconclusiveKind::Noise);
assert!(
report
.to_markdown()
.contains("Verdict: inconclusive (noise)\n")
);
}
#[test]
fn empty_warnings_produce_no_markdown_section_but_serialize_as_empty_array() {
let report = sample_report();
assert!(!report.to_markdown().contains("Warnings:"));
assert!(report.to_json_pretty().contains("\"warnings\": []"));
}
#[test]
fn non_empty_warnings_are_bulleted_in_markdown_and_present_in_json() {
let mut report = sample_report();
report.warnings = vec![
"small sample size".to_string(),
"high failure rate".to_string(),
];
let md = report.to_markdown();
assert!(md.contains("Warnings:\n- small sample size\n- high failure rate\n"));
assert!(report.to_json_pretty().contains("\"small sample size\""));
}
#[test]
fn multi_report_wraps_each_metric_section() {
let multi = MultiReport {
schema_version: REPORT_SCHEMA_VERSION,
verdict: Verdict::Fail,
validity: Validity::Valid,
promotion: Promotion::NotPromoted,
simultaneous_claims_promotion: None,
reports: vec![sample_report()],
};
let md = multi.to_markdown();
assert!(md.contains("Overall verdict: fail"));
assert!(md.contains("Verdict: pass"));
let json = multi.to_json_pretty();
assert!(json.contains("\"reports\""));
}
}