veridict 0.19.7

Statistical regression gate for A/B and hypothesis testing: compare candidate vs baseline results with win-rate, Elo, SPRT, and bootstrap confidence intervals.
Documentation
//! Trinomial (draw-aware) generalized log-likelihood-ratio SPRT, in the
//! BayesElo parameterization used historically by chess-engine testing
//! tools (Fishtest's `LLRlegacy`). Unlike `stats::sprt`'s classic two-
//! outcome Wald test (draws excluded entirely), this estimates the draw
//! rate as a nuisance parameter (`drawelo`) from the pooled win/draw/loss
//! counts, then evaluates both hypotheses at that shared estimate - this is
//! what lets it converge faster than the plain Wald test on draw-heavy
//! data, without needing the caller to supply a separate draw-rate
//! assumption.
//!
//! **Units are BayesElo, not the plain Wald variant's logistic Elo**
//! (`stats::sprt::score_from_elo`'s scale) - the two only coincide when
//! `drawelo == 0`. Verified concretely: at `drawelo = 200`, a BayesElo gap
//! of 10 corresponds to a logistic-Elo gap of only about 7.3 (see
//! `matches_hand_worked_conversion_example` below). This is why the CLI
//! exposes this variant through separate `--belo0`/`--belo1` flags instead
//! of reinterpreting `--elo0`/`--elo1`.
//!
//! Reduction check: at zero draws, `draw_elo_calc` estimates `drawelo = 0`
//! (not just in a limit - the algebra collapses exactly, up to ordinary
//! floating-point rounding), `bayeselo_to_proba` collapses exactly to
//! `score_from_elo`/`1 - score_from_elo`, and the LLR sum's draw term is
//! skipped (zero draws contribute nothing, same "decisive games only"
//! convention `stats::sprt` already documents) - so this reduces to
//! `stats::sprt`'s existing two-outcome LLR to within float precision (see
//! `zero_draws_reduces_to_the_plain_wald_llr` below).

const P_EPSILON: f64 = 1e-4;

/// `(p_win, p_draw, p_loss)` implied by a BayesElo rating gap `elo` and a
/// draw-rate parameter `drawelo`.
fn bayeselo_to_proba(elo: f64, drawelo: f64) -> (f64, f64, f64) {
    let p_win = 1.0 / (1.0 + 10f64.powf((-elo + drawelo) / 400.0));
    let p_loss = 1.0 / (1.0 + 10f64.powf((elo + drawelo) / 400.0));
    let p_draw = 1.0 - p_win - p_loss;
    (p_win, p_draw, p_loss)
}

/// Inverse of `bayeselo_to_proba`, restricted to the `(p_win, p_loss)` pair
/// (implies `p_draw` and is what pooled win/loss counts give directly).
fn proba_to_bayeselo(p_win: f64, p_loss: f64) -> (f64, f64) {
    let elo = 200.0 * ((p_win / p_loss) * (1.0 - p_loss) / (1.0 - p_win)).log10();
    let drawelo = 200.0 * ((1.0 - p_loss) / p_loss * (1.0 - p_win) / p_win).log10();
    (elo, drawelo)
}

/// Estimates `drawelo` from pooled win/draw/loss counts. Empirical
/// proportions are clamped to `[P_EPSILON, 1-P_EPSILON]` first (same
/// convention as `stats::elo::elo_from_score`'s shutout handling) so an
/// all-wins or all-losses sample never divides by zero or takes `log10` of
/// a non-positive value.
fn draw_elo_calc(win: u64, draw: u64, loss: u64) -> f64 {
    let n = (win + draw + loss) as f64;
    let p_win = (win as f64 / n).clamp(P_EPSILON, 1.0 - P_EPSILON);
    let p_loss = (loss as f64 / n).clamp(P_EPSILON, 1.0 - P_EPSILON);
    let (_, drawelo) = proba_to_bayeselo(p_win, p_loss);
    drawelo
}

/// Trinomial LLR for `win`/`draw`/`loss` counts against BayesElo hypotheses
/// `belo0`/`belo1`. Returns `(llr, drawelo)` - `drawelo` is exposed too
/// since it's estimated from the same data being judged, worth reporting
/// for transparency rather than left as an invisible intermediate. Each
/// outcome's term is skipped entirely when that outcome's count is zero,
/// not computed and multiplied by zero: at zero draws, `drawelo` is (up to
/// float rounding) exactly `0`, which forces `p_draw` to exactly `0` under
/// both hypotheses - `0 * ln(0/0)` is `NaN`, not `0`, so the zero-count
/// guard is required, not just a performance nicety.
pub(crate) fn llr(belo0: f64, belo1: f64, win: u64, draw: u64, loss: u64) -> (f64, f64) {
    let drawelo = draw_elo_calc(win, draw, loss);
    let (p0w, p0d, p0l) = bayeselo_to_proba(belo0, drawelo);
    let (p1w, p1d, p1l) = bayeselo_to_proba(belo1, drawelo);

    let mut total = 0.0;
    if win > 0 {
        total += win as f64 * (p1w / p0w).ln();
    }
    if draw > 0 {
        total += draw as f64 * (p1d / p0d).ln();
    }
    if loss > 0 {
        total += loss as f64 * (p1l / p0l).ln();
    }
    (total, drawelo)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::stats::elo::elo_from_score;
    use crate::stats::sprt::{llr_delta, score_from_elo};

    fn assert_close(actual: f64, expected: f64, tol: f64) {
        assert!(
            (actual - expected).abs() < tol,
            "expected {expected}, got {actual}"
        );
    }

    #[test]
    fn bayeselo_proba_round_trips_through_inverse() {
        for (belo, drawelo) in [(10.0, 200.0), (50.0, 100.0), (-30.0, 50.0), (0.0, 0.0)] {
            let (p_win, _p_draw, p_loss) = bayeselo_to_proba(belo, drawelo);
            let (belo2, drawelo2) = proba_to_bayeselo(p_win, p_loss);
            assert_close(belo2, belo, 1e-6);
            assert_close(drawelo2, drawelo, 1e-6);
        }
    }

    // Hand-worked example verifying the units claim in the module doc:
    // a BayesElo gap of 10 at drawelo=200 is a *smaller* logistic-Elo gap
    // (~7.3), not the same number reinterpreted - confirmed independently
    // in Python before writing this Rust version.
    #[test]
    fn matches_hand_worked_conversion_example() {
        let (p_win, p_draw, _p_loss) = bayeselo_to_proba(10.0, 200.0);
        let score = p_win + 0.5 * p_draw;
        assert_close(elo_from_score(score), 7.302, 1e-2);
    }

    #[test]
    fn zero_draws_gives_zero_drawelo() {
        assert_close(draw_elo_calc(700, 0, 300), 0.0, 1e-9);
    }

    // Strong structural check, not just a plausibility test: at zero draws
    // this must reduce to `stats::sprt`'s existing closed-form two-outcome
    // LLR to within ordinary float rounding, since the algebra collapses
    // exactly (verified independently in Python: difference ~1.2e-13).
    #[test]
    fn zero_draws_reduces_to_the_plain_wald_llr() {
        let (belo0, belo1) = (0.0, 10.0);
        let (win, draw, loss) = (700, 0, 300);

        let (trinomial_llr, _drawelo) = llr(belo0, belo1, win, draw, loss);

        let p0 = score_from_elo(belo0);
        let p1 = score_from_elo(belo1);
        let wald_llr =
            win as f64 * llr_delta(true, p0, p1) + loss as f64 * llr_delta(false, p0, p1);

        assert_close(trinomial_llr, wald_llr, 1e-9);
    }

    #[test]
    fn draw_heavy_sample_produces_a_finite_llr() {
        let (llr_value, drawelo) = llr(0.0, 10.0, 500, 300, 200);
        assert!(llr_value.is_finite());
        assert!(drawelo.is_finite());
        assert!(
            drawelo > 0.0,
            "500/300/200 has a real draw rate, drawelo should be positive"
        );
    }

    #[test]
    fn all_wins_does_not_panic_or_produce_nan() {
        let (llr_value, drawelo) = llr(0.0, 10.0, 100, 0, 0);
        assert!(llr_value.is_finite());
        assert!(drawelo.is_finite());
    }

    #[test]
    fn all_losses_does_not_panic_or_produce_nan() {
        let (llr_value, drawelo) = llr(0.0, 10.0, 0, 0, 100);
        assert!(llr_value.is_finite());
        assert!(drawelo.is_finite());
    }

    #[test]
    fn all_draws_does_not_panic_or_produce_nan() {
        let (llr_value, drawelo) = llr(0.0, 10.0, 0, 100, 0);
        assert!(llr_value.is_finite());
        assert!(drawelo.is_finite());
    }

    #[test]
    fn single_trial_of_each_kind_does_not_panic() {
        for (w, d, l) in [(1, 0, 0), (0, 1, 0), (0, 0, 1)] {
            let (llr_value, drawelo) = llr(0.0, 10.0, w, d, l);
            assert!(llr_value.is_finite());
            assert!(drawelo.is_finite());
        }
    }

    // Monte Carlo operating-characteristic check: this is what the Fishtest
    // team itself uses to validate this class of test - it checks the
    // property that actually matters (does alpha bound the false-accept-H1
    // rate), not just formula algebra. Simulates trials one at a time from
    // a *true* model sitting exactly at H0, running the SPRT sequentially
    // until a bound is crossed (a genuine SPRT guarantee assumes exactly
    // this "optional stopping" usage, not evaluating one fixed-size batch).
    // Calibrated in an independent Python port before writing this test
    // (200 sims, same true model: false-accept-H1 rate came out to 4.0%,
    // tracking alpha=5% closely) - the margin here (3x alpha) is
    // deliberately generous since Rust's `StdRng` draws a different
    // pseudorandom stream than Python's, so the exact realized rate under
    // this fixed seed will differ from that calibration run, not just
    // reproduce it.
    #[test]
    fn monte_carlo_false_accept_rate_tracks_alpha_under_h0() {
        use rand::rngs::StdRng;
        use rand::{RngExt, SeedableRng};

        let (belo0, belo1) = (0.0, 20.0);
        let (alpha, beta) = (0.05, 0.05);
        let bounds = crate::stats::sprt::bounds(alpha, beta);

        // True model sits exactly at H0, with a real (nonzero) draw rate -
        // the scenario this variant exists for.
        let (true_p_win, true_p_draw, _true_p_loss) = bayeselo_to_proba(belo0, 100.0);

        let mut rng = StdRng::seed_from_u64(0x5EED);
        let simulations = 200;
        let max_trials = 3000;
        let mut false_accept_h1 = 0;

        for _ in 0..simulations {
            let (mut win, mut draw, mut loss) = (0u64, 0u64, 0u64);
            for _ in 0..max_trials {
                let r: f64 = rng.random_range(0.0..1.0);
                if r < true_p_win {
                    win += 1;
                } else if r < true_p_win + true_p_draw {
                    draw += 1;
                } else {
                    loss += 1;
                }
                let n = win + draw + loss;
                if n < 10 {
                    continue; // avoid noisy LLR swings on a near-empty sample
                }
                let (llr_value, _drawelo) = llr(belo0, belo1, win, draw, loss);
                if llr_value >= bounds.upper {
                    false_accept_h1 += 1;
                    break;
                } else if llr_value <= bounds.lower {
                    break;
                }
            }
        }

        let false_accept_rate = false_accept_h1 as f64 / simulations as f64;
        assert!(
            false_accept_rate < alpha * 3.0,
            "false_accept_rate={false_accept_rate} too high for alpha={alpha} - SPRT should bound this near alpha, not wildly exceed it"
        );
    }
}