use std::collections::{HashMap, HashSet};
use serde::Serialize;
use crate::error::VeridictError;
use crate::input::{MatchRecord, Record};
use crate::stats::bradley_terry::{self, PairRecord};
use crate::stats::{elo as elo_math, wilson};
use crate::{BootstrapMethod, FailurePolicy, MatchOutcome, MetricConfig, metrics};
const BASELINE: &str = "baseline";
#[derive(Debug, Serialize)]
pub struct CandidateSummary {
pub name: String,
pub elo: f64,
pub ci_low: Option<f64>,
pub ci_high: Option<f64>,
pub baseline_count: u64,
pub candidate_count: u64,
pub paired_count: u64,
pub timeouts: u64,
pub crashes: u64,
pub invalid: u64,
pub component: u32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum CellStatus {
Direct,
Inferred,
Disconnected,
}
#[derive(Debug, Serialize)]
pub struct MatrixEntry {
pub row: String,
pub col: String,
pub elo_diff: Option<f64>,
pub ci_low: Option<f64>,
pub ci_high: Option<f64>,
pub direct: bool,
pub status: CellStatus,
}
#[derive(Debug, Serialize)]
pub struct ComparisonMatrix {
pub schema_version: u32,
pub confidence: f64,
pub candidates: Vec<CandidateSummary>,
pub matrix: Vec<MatrixEntry>,
}
fn add_wins(
edges: &mut HashMap<(String, String), (u64, u64, u64)>,
x: &str,
x_wins: u64,
y: &str,
y_wins: u64,
draws: u64,
) {
let (lo, hi, lo_wins, hi_wins) = if x <= y {
(x, y, x_wins, y_wins)
} else {
(y, x, y_wins, x_wins)
};
let entry = edges
.entry((lo.to_string(), hi.to_string()))
.or_insert((0, 0, 0));
entry.0 += lo_wins;
entry.1 += hi_wins;
entry.2 += draws;
}
fn canonical_pair(a: &str, b: &str) -> (String, String) {
if a <= b {
(a.to_string(), b.to_string())
} else {
(b.to_string(), a.to_string())
}
}
fn tally_matches<J, K>(
matches: J,
paired_by_id: bool,
edges: &mut HashMap<(String, String), (u64, u64, u64)>,
match_order: &mut Vec<String>,
seen: &mut HashSet<String>,
) -> Result<(), VeridictError>
where
J: IntoIterator<Item = Result<K, VeridictError>>,
K: IntoIterator<Item = Result<(usize, MatchRecord), VeridictError>>,
{
let mut groups: HashMap<(String, String, String), Vec<(usize, f64)>> = HashMap::new();
for file in matches {
for item in file? {
let (line, rec) = item?;
if rec.a.is_empty() || rec.b.is_empty() {
return Err(VeridictError::SchemaMismatch {
line,
context: "matrix graph",
detail: "a and b must both be non-empty competitor names".to_string(),
});
}
if rec.a == rec.b {
return Err(VeridictError::SchemaMismatch {
line,
context: "matrix graph",
detail: format!(
"a and b are both '{}'; a competitor can't play itself",
rec.a
),
});
}
for name in [&rec.a, &rec.b] {
if name != BASELINE && seen.insert(name.clone()) {
match_order.push(name.clone());
}
}
let outcome = MatchOutcome::parse(&rec.result).ok_or_else(|| {
VeridictError::UnrecognizedOutcome {
line,
value: rec.result.clone(),
expected: "a_win|b_win|draw",
}
})?;
let (lo, hi, points_to_hi) = canonicalize_outcome(&rec.a, &rec.b, outcome);
match (paired_by_id, rec.id.as_deref()) {
(true, Some(id)) => {
groups
.entry((lo, hi, id.to_string()))
.or_default()
.push((line, points_to_hi));
}
_ => tally_points(edges, lo, hi, points_to_hi),
}
}
}
for ((lo, hi, id), group) in groups {
match group.as_slice() {
[(_, p)] => tally_points(edges, lo, hi, *p),
[(_, a), (_, b)] => {
let total = a + b;
#[allow(clippy::float_cmp)]
let net = if total > 1.0 {
1.0
} else if total < 1.0 {
0.0
} else {
0.5
};
tally_points(edges, lo, hi, net);
}
more => {
return Err(VeridictError::SchemaMismatch {
line: more[0].0,
context: "paired-by-id",
detail: format!(
"id '{id}' appears {} times for pair ({lo}, {hi}); paired mode expects at most 2 records per id per pair",
more.len()
),
});
}
}
}
Ok(())
}
fn canonicalize_outcome(a: &str, b: &str, outcome: MatchOutcome) -> (String, String, f64) {
let points_a = match outcome {
MatchOutcome::AWin => 1.0,
MatchOutcome::Draw => 0.5,
MatchOutcome::BWin => 0.0,
};
if a <= b {
(a.to_string(), b.to_string(), 1.0 - points_a)
} else {
(b.to_string(), a.to_string(), points_a)
}
}
fn tally_points(
edges: &mut HashMap<(String, String), (u64, u64, u64)>,
lo: String,
hi: String,
points_to_hi: f64,
) {
let entry = edges.entry((lo, hi)).or_insert((0, 0, 0));
if points_to_hi > 0.5 {
entry.1 += 1;
} else if points_to_hi < 0.5 {
entry.0 += 1;
} else {
entry.2 += 1;
}
}
fn node_totals(name: &str, edges: &HashMap<(String, String), (u64, u64, u64)>) -> (u64, u64, u64) {
let mut wins = 0u64;
let mut draws = 0u64;
let mut n = 0u64;
for ((lo, hi), &(lo_wins, hi_wins, pair_draws)) in edges {
if lo == name {
wins += lo_wins;
draws += pair_draws;
n += lo_wins + hi_wins + pair_draws;
} else if hi == name {
wins += hi_wins;
draws += pair_draws;
n += lo_wins + hi_wins + pair_draws;
}
}
(wins, draws, n)
}
fn elo_ci_from_counts(
wins: u64,
draws: u64,
n: u64,
confidence: f64,
) -> Result<(f64, f64, f64), VeridictError> {
if n == 0 {
return Ok((0.0, 0.0, 0.0));
}
let score = (wins as f64 + 0.5 * draws as f64) / n as f64;
let (lo, hi) = wilson::wilson_ci_from_proportion(score, n as f64, confidence)?;
Ok((
elo_math::elo_from_score(score),
elo_math::elo_from_score(lo),
elo_math::elo_from_score(hi),
))
}
#[allow(clippy::too_many_arguments)]
pub fn run<I, R, J, K>(
named_records: I,
matches: J,
confidence: f64,
paired_by_id: bool,
resamples: usize,
seed: u64,
bootstrap_method: BootstrapMethod,
) -> Result<ComparisonMatrix, VeridictError>
where
I: IntoIterator<Item = Result<(String, R), VeridictError>>,
R: IntoIterator<Item = Result<(usize, Record), VeridictError>>,
J: IntoIterator<Item = Result<K, VeridictError>>,
K: IntoIterator<Item = Result<(usize, MatchRecord), VeridictError>>,
{
if !confidence.is_finite() || confidence <= 0.0 || confidence >= 1.0 {
return Err(VeridictError::InvalidConfidence(confidence));
}
let mut legacy_order = Vec::new();
let mut legacy_summaries: HashMap<String, CandidateSummary> = HashMap::new();
let mut edges: HashMap<(String, String), (u64, u64, u64)> = HashMap::new();
for item in named_records {
let (name, records) = item?;
let out = metrics::compute(
records,
MetricConfig::Elo {
failure_policy: FailurePolicy::ReportOnly,
},
confidence,
1,
0,
paired_by_id,
false,
)?;
let draws = out
.paired_count
.saturating_sub(out.baseline_count)
.saturating_sub(out.candidate_count);
add_wins(
&mut edges,
BASELINE,
out.baseline_count,
&name,
out.candidate_count,
draws,
);
legacy_summaries.insert(
name.clone(),
CandidateSummary {
name: name.clone(),
elo: out.effect,
ci_low: Some(out.ci_low),
ci_high: Some(out.ci_high),
baseline_count: out.baseline_count,
candidate_count: out.candidate_count,
paired_count: out.paired_count,
timeouts: out.timeouts,
crashes: out.crashes,
invalid: out.invalid,
component: 0,
},
);
legacy_order.push(name);
}
let mut match_order: Vec<String> = Vec::new();
let mut seen: HashSet<String> = legacy_order.iter().cloned().collect();
seen.insert(BASELINE.to_string());
tally_matches(
matches,
paired_by_id,
&mut edges,
&mut match_order,
&mut seen,
)?;
let candidate_order: Vec<String> = legacy_order.into_iter().chain(match_order).collect();
if candidate_order.is_empty() {
return Err(VeridictError::EmptyInput);
}
let has_baseline = edges.keys().any(|(a, b)| a == BASELINE || b == BASELINE);
let star_shaped = edges.keys().all(|(a, b)| a == BASELINE || b == BASELINE);
if !star_shaped && resamples == 0 {
return Err(VeridictError::InvalidResamples(resamples));
}
let (candidates, matrix) = if star_shaped {
build_star_graph(&candidate_order, &legacy_summaries, &edges, confidence)?
} else {
build_general_graph(
&candidate_order,
has_baseline,
&edges,
resamples,
seed,
confidence,
bootstrap_method,
)?
};
Ok(ComparisonMatrix {
schema_version: crate::report::REPORT_SCHEMA_VERSION,
confidence,
candidates,
matrix,
})
}
fn build_star_graph(
candidate_order: &[String],
legacy_summaries: &HashMap<String, CandidateSummary>,
edges: &HashMap<(String, String), (u64, u64, u64)>,
confidence: f64,
) -> Result<(Vec<CandidateSummary>, Vec<MatrixEntry>), VeridictError> {
let mut candidates = Vec::with_capacity(candidate_order.len());
for name in candidate_order {
if let Some(summary) = legacy_summaries.get(name) {
candidates.push(CandidateSummary {
name: summary.name.clone(),
elo: summary.elo,
ci_low: summary.ci_low,
ci_high: summary.ci_high,
baseline_count: summary.baseline_count,
candidate_count: summary.candidate_count,
paired_count: summary.paired_count,
timeouts: summary.timeouts,
crashes: summary.crashes,
invalid: summary.invalid,
component: 0,
});
} else {
let key = canonical_pair(BASELINE, name);
let (lo_wins, hi_wins, draws) = edges[&key];
let (baseline_wins, candidate_wins) = if key.0 == BASELINE {
(lo_wins, hi_wins)
} else {
(hi_wins, lo_wins)
};
let n = baseline_wins + candidate_wins + draws;
let (elo, ci_low, ci_high) = elo_ci_from_counts(candidate_wins, draws, n, confidence)?;
candidates.push(CandidateSummary {
name: name.clone(),
elo,
ci_low: Some(ci_low),
ci_high: Some(ci_high),
baseline_count: baseline_wins,
candidate_count: candidate_wins,
paired_count: n,
timeouts: 0,
crashes: 0,
invalid: 0,
component: 0,
});
}
}
let mut matrix = Vec::new();
for c in &candidates {
let ci_low = c.ci_low.expect("star-graph candidates always have a CI");
let ci_high = c.ci_high.expect("star-graph candidates always have a CI");
matrix.push(MatrixEntry {
row: BASELINE.to_string(),
col: c.name.clone(),
elo_diff: Some(-c.elo),
ci_low: Some(-ci_high),
ci_high: Some(-ci_low),
direct: true,
status: CellStatus::Direct,
});
}
for i in 0..candidates.len() {
for j in (i + 1)..candidates.len() {
let (a, b) = (&candidates[i], &candidates[j]);
let margin_a = (a.ci_high.unwrap() - a.ci_low.unwrap()) / 2.0;
let margin_b = (b.ci_high.unwrap() - b.ci_low.unwrap()) / 2.0;
let margin = margin_a.hypot(margin_b);
let diff = a.elo - b.elo;
matrix.push(MatrixEntry {
row: a.name.clone(),
col: b.name.clone(),
elo_diff: Some(diff),
ci_low: Some(diff - margin),
ci_high: Some(diff + margin),
direct: false,
status: CellStatus::Inferred,
});
}
}
Ok((candidates, matrix))
}
#[allow(clippy::too_many_arguments)]
fn build_general_graph(
candidate_order: &[String],
has_baseline: bool,
edges: &HashMap<(String, String), (u64, u64, u64)>,
resamples: usize,
seed: u64,
confidence: f64,
bootstrap_method: BootstrapMethod,
) -> Result<(Vec<CandidateSummary>, Vec<MatrixEntry>), VeridictError> {
let mut all_nodes: Vec<String> = Vec::new();
if has_baseline {
all_nodes.push(BASELINE.to_string());
}
all_nodes.extend(candidate_order.iter().cloned());
let index_of: HashMap<&str, usize> = all_nodes
.iter()
.enumerate()
.map(|(i, n)| (n.as_str(), i))
.collect();
let mut pairs: Vec<PairRecord> = edges
.iter()
.map(|((lo, hi), &(lo_wins, hi_wins, draws))| PairRecord {
a: index_of[lo.as_str()],
b: index_of[hi.as_str()],
a_wins: lo_wins,
b_wins: hi_wins,
draws,
})
.collect();
pairs.sort_by_key(|p| (p.a, p.b));
let fits = bradley_terry::fit_graph(all_nodes.len(), &pairs)?;
let cis = bradley_terry::bootstrap_pairwise_elo_diff_cis(
all_nodes.len(),
&pairs,
&fits,
resamples,
seed,
confidence,
bootstrap_method,
);
let mut candidates = Vec::with_capacity(candidate_order.len());
for name in candidate_order {
let idx = index_of[name.as_str()];
let fit = fits[idx];
let (wins, draws, n) = node_totals(name, edges);
candidates.push(CandidateSummary {
name: name.clone(),
elo: fit.elo,
ci_low: None,
ci_high: None,
baseline_count: n.saturating_sub(wins).saturating_sub(draws),
candidate_count: wins,
paired_count: n,
timeouts: 0,
crashes: 0,
invalid: 0,
component: fit.component as u32,
});
}
let mut matrix = Vec::new();
for i in 0..all_nodes.len() {
for j in (i + 1)..all_nodes.len() {
let (row, col) = (&all_nodes[i], &all_nodes[j]);
let same_component = fits[i].component == fits[j].component;
let direct_edge = edges.contains_key(&canonical_pair(row, col));
let status = if !same_component {
CellStatus::Disconnected
} else if direct_edge {
CellStatus::Direct
} else {
CellStatus::Inferred
};
let elo_diff = same_component.then(|| fits[i].elo - fits[j].elo);
let ci = cis.get(&(i, j)).copied();
matrix.push(MatrixEntry {
row: row.clone(),
col: col.clone(),
elo_diff,
ci_low: ci.map(|(lo, _)| lo),
ci_high: ci.map(|(_, hi)| hi),
direct: status == CellStatus::Direct,
status,
});
}
}
Ok((candidates, matrix))
}
impl ComparisonMatrix {
pub fn to_json_pretty(&self) -> String {
serde_json::to_string_pretty(self).expect(
"ComparisonMatrix contains only finite fields and strings; serialization cannot fail",
)
}
pub fn to_markdown(&self) -> String {
let mut names = Vec::new();
if self
.matrix
.iter()
.any(|e| e.row == BASELINE || e.col == BASELINE)
{
names.push(BASELINE.to_string());
}
for c in &self.candidates {
if !names.contains(&c.name) {
names.push(c.name.clone());
}
}
let lookup = |row: &str, col: &str| -> Option<(Option<f64>, CellStatus)> {
self.matrix.iter().find_map(|e| {
if e.row == row && e.col == col {
Some((e.elo_diff, e.status))
} else if e.row == col && e.col == row {
Some((e.elo_diff.map(|v| -v), e.status))
} else {
None
}
})
};
let num_components: HashSet<u32> = self.candidates.iter().map(|c| c.component).collect();
let mut out = String::from("# Veridict Comparison Matrix\n\n");
out.push_str(
"Cell = row's Elo advantage over column. `*` marks an inferred cell (no direct \
head-to-head data, estimated transitively through the graph). `n/a` marks \
disconnected candidates (no path between them; not comparable).\n\n",
);
if num_components.len() > 1 {
out.push_str(&format!(
"Note: candidates span {} disconnected components; only cells within the same \
component have a rating comparison.\n\n",
num_components.len()
));
}
out.push_str("| vs |");
for n in &names {
out.push_str(&format!(" {n} |"));
}
out.push('\n');
out.push_str("|---|");
for _ in &names {
out.push_str("---|");
}
out.push('\n');
for row in &names {
out.push_str(&format!("| {row} |"));
for col in &names {
if row == col {
out.push_str(" - |");
continue;
}
match lookup(row, col) {
Some((Some(v), CellStatus::Direct)) => out.push_str(&format!(" {v:+.1} |")),
Some((Some(v), CellStatus::Inferred)) => out.push_str(&format!(" {v:+.1}* |")),
Some((None, CellStatus::Disconnected)) => out.push_str(" n/a |"),
_ => out.push_str(" ? |"),
}
}
out.push('\n');
}
out
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::stats::bootstrap::DEFAULT_SEED as TEST_SEED;
const TEST_RESAMPLES: usize = 200;
fn rec(result: &str) -> Record {
Record {
id: None,
baseline: None,
candidate: None,
result: Some(result.to_string()),
baseline_status: None,
candidate_status: None,
}
}
fn stream(result: &str, n: usize) -> Vec<(usize, Record)> {
(0..n).map(|i| (i + 1, rec(result))).collect()
}
type NamedRecords =
Result<(String, Vec<Result<(usize, Record), VeridictError>>), VeridictError>;
fn ok_named(data: Vec<(String, Vec<(usize, Record)>)>) -> Vec<NamedRecords> {
data.into_iter()
.map(|(name, records)| Ok((name, records.into_iter().map(Ok).collect())))
.collect()
}
type MatchFile = Vec<Result<(usize, MatchRecord), VeridictError>>;
fn no_matches() -> Vec<Result<MatchFile, VeridictError>> {
Vec::new()
}
fn match_rec(id: &str, a: &str, b: &str, result: &str) -> MatchRecord {
MatchRecord {
id: Some(id.to_string()),
a: a.to_string(),
b: b.to_string(),
result: result.to_string(),
}
}
fn ok_matches(files: Vec<Vec<MatchRecord>>) -> Vec<Result<MatchFile, VeridictError>> {
files
.into_iter()
.map(|records| {
Ok(records
.into_iter()
.enumerate()
.map(|(i, r)| Ok((i + 1, r)))
.collect())
})
.collect()
}
#[test]
fn baseline_vs_candidate_is_direct() {
let data = vec![("a".to_string(), stream("candidate_win", 20))];
let m = run(
ok_named(data),
no_matches(),
0.95,
false,
TEST_RESAMPLES,
TEST_SEED,
BootstrapMethod::Percentile,
)
.unwrap();
assert_eq!(m.matrix.len(), 1);
assert!(m.matrix[0].direct);
assert_eq!(m.matrix[0].row, "baseline");
assert_eq!(m.matrix[0].col, "a");
assert!(m.matrix[0].elo_diff.unwrap() < 0.0); }
#[test]
fn candidate_vs_candidate_is_extrapolated() {
let data = vec![
("a".to_string(), stream("candidate_win", 20)),
("b".to_string(), stream("baseline_win", 20)),
];
let m = run(
ok_named(data),
no_matches(),
0.95,
false,
TEST_RESAMPLES,
TEST_SEED,
BootstrapMethod::Percentile,
)
.unwrap();
let cross = m
.matrix
.iter()
.find(|e| e.row == "a" && e.col == "b")
.unwrap();
assert!(!cross.direct);
assert_eq!(cross.status, CellStatus::Inferred);
assert!(cross.elo_diff.unwrap() > 0.0); }
#[test]
fn markdown_mirrors_into_a_full_grid() {
let data = vec![("a".to_string(), stream("candidate_win", 20))];
let md = run(
ok_named(data),
no_matches(),
0.95,
false,
TEST_RESAMPLES,
TEST_SEED,
BootstrapMethod::Percentile,
)
.unwrap()
.to_markdown();
assert!(md.contains("| baseline |"));
assert!(md.contains("| a |"));
assert!(md.contains(" - |"));
}
#[test]
fn empty_input_is_an_error() {
assert!(matches!(
run(
ok_named(vec![]),
no_matches(),
0.95,
false,
TEST_RESAMPLES,
TEST_SEED,
BootstrapMethod::Percentile,
),
Err(VeridictError::EmptyInput)
));
}
fn general_graph_matches() -> Vec<Result<MatchFile, VeridictError>> {
ok_matches(vec![vec![
match_rec("ab", "a", "b", "draw"),
match_rec("bc", "b", "c", "draw"),
match_rec("ca", "c", "a", "draw"),
]])
}
#[test]
fn general_graph_rejects_zero_resamples_before_bootstrap() {
assert!(matches!(
run(
ok_named(vec![]),
general_graph_matches(),
0.95,
false,
0,
TEST_SEED,
BootstrapMethod::Percentile,
),
Err(VeridictError::InvalidResamples(0))
));
}
#[test]
fn matches_only_rejects_invalid_confidence() {
for confidence in [f64::NAN, 0.0, 1.0, 2.0] {
assert!(matches!(
run(
ok_named(vec![]),
general_graph_matches(),
confidence,
false,
TEST_RESAMPLES,
TEST_SEED,
BootstrapMethod::Percentile,
),
Err(VeridictError::InvalidConfidence(_))
));
}
}
fn stream_mixed(
candidate_wins: usize,
draws: usize,
baseline_wins: usize,
) -> Vec<(usize, Record)> {
let mut line = 0;
let mut records = Vec::new();
for result in ["candidate_win"]
.repeat(candidate_wins)
.into_iter()
.chain(["draw"].repeat(draws))
.chain(["baseline_win"].repeat(baseline_wins))
{
line += 1;
records.push((line, rec(result)));
}
records
}
#[test]
fn star_graph_elo_matches_bradley_terry_mle_reduction() {
let (candidate_wins, draws, baseline_wins) = (12, 3, 5);
let w_candidate = candidate_wins as f64 + 0.5 * draws as f64;
let w_baseline = baseline_wins as f64 + 0.5 * draws as f64;
let n = (candidate_wins + draws + baseline_wins) as f64;
let (mut pi_baseline, mut pi_candidate) = (1.0_f64, 1.0_f64);
for _ in 0..1000 {
let denom = n / (pi_baseline + pi_candidate);
let new_baseline = w_baseline / denom;
let new_candidate = w_candidate / denom;
pi_candidate = new_candidate / new_baseline;
pi_baseline = new_baseline / new_baseline;
}
let mle_elo = 400.0 * (pi_candidate / pi_baseline).log10();
let data = vec![(
"candidate".to_string(),
stream_mixed(candidate_wins, draws, baseline_wins),
)];
let m = run(
ok_named(data),
no_matches(),
0.95,
false,
TEST_RESAMPLES,
TEST_SEED,
BootstrapMethod::Percentile,
)
.unwrap();
let reported_elo = m.candidates[0].elo;
assert!(
(mle_elo - reported_elo).abs() < 1e-6,
"general MM fixed point {mle_elo} vs matrix::run's reported {reported_elo}"
);
}
#[test]
fn shutout_via_matches_still_uses_the_closed_form_not_disconnected() {
let records: Vec<MatchRecord> = (0..5)
.map(|_| match_rec("g", "candidate", "baseline", "a_win"))
.collect();
let matches = ok_matches(vec![records]);
let m = run(
ok_named(vec![]),
matches,
0.95,
false,
TEST_RESAMPLES,
TEST_SEED,
BootstrapMethod::Percentile,
)
.unwrap();
assert_eq!(m.candidates.len(), 1);
let candidate = &m.candidates[0];
assert_eq!(candidate.component, 0);
assert!(candidate.elo.is_finite() && candidate.elo > 0.0);
let cell = &m.matrix[0];
assert_eq!(cell.row, "baseline");
assert_eq!(cell.col, "candidate");
assert_eq!(cell.status, CellStatus::Direct);
assert!(cell.direct);
let diff = cell.elo_diff.unwrap();
assert!(diff.is_finite() && diff < 0.0);
}
#[test]
fn matches_only_pure_head_to_head_ranks_without_a_baseline() {
let mut records = Vec::new();
for _ in 0..4 {
records.push(match_rec("x", "alpha", "beta", "a_win"));
}
records.push(match_rec("x", "alpha", "beta", "b_win"));
for _ in 0..3 {
records.push(match_rec("x", "beta", "gamma", "a_win"));
}
for _ in 0..2 {
records.push(match_rec("x", "beta", "gamma", "b_win"));
}
records.push(match_rec("x", "gamma", "alpha", "a_win"));
for _ in 0..4 {
records.push(match_rec("x", "gamma", "alpha", "b_win"));
}
let matches = ok_matches(vec![records]);
let m = run(
ok_named(vec![]),
matches,
0.95,
false,
TEST_RESAMPLES,
TEST_SEED,
BootstrapMethod::Percentile,
)
.unwrap();
assert_eq!(m.candidates.len(), 3);
assert!(!m.candidates.iter().any(|c| c.name == "baseline"));
let elo = |name: &str| m.candidates.iter().find(|c| c.name == name).unwrap().elo;
assert!(elo("alpha") > elo("beta"));
assert!(elo("beta") > elo("gamma"));
assert!(
(elo("beta") - elo("alpha") - (-215.5356238480202)).abs() < 1e-6,
"expected the same converged ratio as stats::bradley_terry's cycle test"
);
}
#[test]
fn mixed_legacy_and_matches_connects_candidates_directly() {
let named = ok_named(vec![
("a".to_string(), stream_mixed(6, 2, 2)),
("b".to_string(), stream_mixed(3, 2, 5)),
]);
let matches = ok_matches(vec![vec![match_rec("h2h", "a", "b", "a_win")]]);
let m = run(
named,
matches,
0.95,
false,
TEST_RESAMPLES,
TEST_SEED,
BootstrapMethod::Percentile,
)
.unwrap();
let cross = m
.matrix
.iter()
.find(|e| (e.row == "a" && e.col == "b") || (e.row == "b" && e.col == "a"))
.unwrap();
assert_eq!(cross.status, CellStatus::Direct);
assert!(cross.direct);
assert!(cross.elo_diff.is_some());
}
#[test]
fn genuinely_disconnected_candidates_report_no_comparison() {
let matches = ok_matches(vec![vec![
match_rec("m1", "a", "b", "a_win"),
match_rec("m2", "c", "d", "a_win"),
]]);
let m = run(
ok_named(vec![]),
matches,
0.95,
false,
TEST_RESAMPLES,
TEST_SEED,
BootstrapMethod::Percentile,
)
.unwrap();
let a_component = m
.candidates
.iter()
.find(|c| c.name == "a")
.unwrap()
.component;
let c_component = m
.candidates
.iter()
.find(|c| c.name == "c")
.unwrap()
.component;
assert_ne!(a_component, c_component);
let cross = m
.matrix
.iter()
.find(|e| (e.row == "a" && e.col == "c") || (e.row == "c" && e.col == "a"))
.unwrap();
assert_eq!(cross.status, CellStatus::Disconnected);
assert!(cross.elo_diff.is_none());
assert!(cross.ci_low.is_none());
assert!(cross.ci_high.is_none());
}
#[test]
fn paired_by_id_nets_opposite_order_records_to_a_draw() {
let matches = ok_matches(vec![vec![
match_rec("p1", "x", "y", "a_win"),
match_rec("p1", "y", "x", "a_win"),
]]);
let m = run(
ok_named(vec![]),
matches,
0.95,
true,
TEST_RESAMPLES,
TEST_SEED,
BootstrapMethod::Percentile,
)
.unwrap();
let x = m.candidates.iter().find(|c| c.name == "x").unwrap();
assert_eq!(
x.paired_count, 1,
"the two records should net to one game, not two"
);
assert_eq!(x.candidate_count, 0);
assert_eq!(x.baseline_count, 0);
let cross = m
.matrix
.iter()
.find(|e| (e.row == "x" && e.col == "y") || (e.row == "y" && e.col == "x"))
.unwrap();
assert!(
(cross.elo_diff.unwrap()).abs() < 1e-9,
"a net draw is an even match"
);
}
#[test]
fn paired_by_id_does_not_net_across_different_pairs_sharing_an_id() {
let matches = ok_matches(vec![vec![
match_rec("shared", "x", "y", "a_win"),
match_rec("shared", "z", "w", "a_win"),
]]);
let m = run(
ok_named(vec![]),
matches,
0.95,
true,
TEST_RESAMPLES,
TEST_SEED,
BootstrapMethod::Percentile,
)
.unwrap();
let x = m.candidates.iter().find(|c| c.name == "x").unwrap();
assert_eq!(x.paired_count, 1);
assert_eq!(x.candidate_count, 1, "x won its single game");
let z = m.candidates.iter().find(|c| c.name == "z").unwrap();
assert_eq!(z.paired_count, 1);
assert_eq!(z.candidate_count, 1, "z won its single game");
let x_component = x.component;
let z_component = z.component;
assert_ne!(x_component, z_component);
}
#[test]
fn general_graph_direct_cells_get_real_cis_but_candidate_summary_stays_none() {
let mut records = Vec::new();
for _ in 0..15 {
records.push(match_rec("ab", "alpha", "beta", "a_win"));
}
for _ in 0..5 {
records.push(match_rec("ab", "alpha", "beta", "b_win"));
}
for _ in 0..15 {
records.push(match_rec("bc", "beta", "gamma", "a_win"));
}
for _ in 0..5 {
records.push(match_rec("bc", "beta", "gamma", "b_win"));
}
for _ in 0..15 {
records.push(match_rec("ca", "gamma", "alpha", "a_win"));
}
for _ in 0..5 {
records.push(match_rec("ca", "gamma", "alpha", "b_win"));
}
let matches = ok_matches(vec![records]);
let m = run(
ok_named(vec![]),
matches,
0.95,
false,
300,
TEST_SEED,
BootstrapMethod::Percentile,
)
.unwrap();
assert_eq!(m.matrix.len(), 3);
for cell in &m.matrix {
assert_eq!(cell.status, CellStatus::Direct);
assert!(cell.elo_diff.is_some());
assert!(
cell.ci_low.is_some() && cell.ci_high.is_some(),
"{}-{} should get a real bootstrap CI",
cell.row,
cell.col
);
}
for c in &m.candidates {
assert!(c.ci_low.is_none());
assert!(c.ci_high.is_none());
}
}
#[test]
fn fragile_connection_reports_elo_diff_but_no_ci() {
let mut records = vec![
match_rec("bridge1", "beta", "gamma", "a_win"),
match_rec("bridge2", "beta", "gamma", "b_win"),
];
for _ in 0..20 {
records.push(match_rec("ab_a", "alpha", "beta", "a_win"));
}
for _ in 0..20 {
records.push(match_rec("ab_b", "alpha", "beta", "b_win"));
}
for _ in 0..20 {
records.push(match_rec("cd_a", "gamma", "delta", "a_win"));
}
for _ in 0..20 {
records.push(match_rec("cd_b", "gamma", "delta", "b_win"));
}
let matches = ok_matches(vec![records]);
let m = run(
ok_named(vec![]),
matches,
0.95,
false,
2000,
TEST_SEED,
BootstrapMethod::Percentile,
)
.unwrap();
let cell = |x: &str, y: &str| {
m.matrix
.iter()
.find(|e| (e.row == x && e.col == y) || (e.row == y && e.col == x))
.unwrap()
};
for (x, y) in [
("alpha", "gamma"),
("alpha", "delta"),
("beta", "gamma"),
("beta", "delta"),
] {
let c = cell(x, y);
assert_ne!(
c.status,
CellStatus::Disconnected,
"{x}-{y} should still be in the same original component"
);
assert!(c.elo_diff.is_some(), "{x}-{y} elo_diff should be Some");
assert!(
c.ci_low.is_none() && c.ci_high.is_none(),
"{x}-{y} should have no reliable CI (fragile bridge)"
);
}
for (x, y) in [("alpha", "beta"), ("gamma", "delta")] {
let c = cell(x, y);
assert!(
c.ci_low.is_some() && c.ci_high.is_some(),
"{x}-{y} should keep a real CI"
);
}
}
#[test]
fn empty_competitor_name_is_rejected() {
let matches = ok_matches(vec![vec![match_rec("m1", "", "b", "a_win")]]);
assert!(matches!(
run(
ok_named(vec![]),
matches,
0.95,
false,
TEST_RESAMPLES,
TEST_SEED,
BootstrapMethod::Percentile,
),
Err(VeridictError::SchemaMismatch { .. })
));
}
#[test]
fn general_graph_component_ids_are_deterministic_across_runs() {
let build = || {
let matches = ok_matches(vec![vec![
match_rec("m1", "a", "b", "a_win"),
match_rec("m2", "c", "d", "a_win"),
]]);
run(
ok_named(vec![]),
matches,
0.95,
false,
TEST_RESAMPLES,
TEST_SEED,
BootstrapMethod::Percentile,
)
.unwrap()
};
let first = build();
let second = build();
for name in ["a", "b", "c", "d"] {
let comp = |m: &ComparisonMatrix| {
m.candidates
.iter()
.find(|c| c.name == name)
.unwrap()
.component
};
assert_eq!(
comp(&first),
comp(&second),
"component id for '{name}' differed between runs"
);
}
}
}