cp_validator/

rating.rs

//! Bayesian rating implementation using skillratings' Weng-Lin model.
//!
//! Wraps the `skillratings` crate's Weng-Lin implementation for pairwise
//! skill estimation. Each player (contributor or peer) has a mean skill
//! estimate (mu) and an uncertainty (sigma). Ratings update through pairwise
//! comparisons.
//!
//! Per CP-015 section 8: the core algorithm follows Weng and Lin (2011),
//! adapted from the `TrueSkill` framework.

use serde::{Deserialize, Serialize};
use skillratings::weng_lin::{weng_lin, WengLinConfig, WengLinRating};
use skillratings::Outcomes;

/// Default mean skill for a new player.
pub const DEFAULT_MU: f64 = 25.0;

/// Default uncertainty for a new player (mu / 3).
pub const DEFAULT_SIGMA: f64 = 25.0 / 3.0; // 8.333...

const WENG_LIN_CONFIG: WengLinConfig = WengLinConfig {
    beta: 25.0 / 6.0,
    uncertainty_tolerance: 0.0001,
};

/// A Bayesian rating with mean skill estimate and uncertainty.
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub struct Rating {
    /// Mean skill estimate.
    pub mu: f64,
    /// Uncertainty (standard deviation of the skill belief).
    pub sigma: f64,
}

impl Rating {
    /// Create a new rating with the given mu and sigma.
    pub fn new(mu: f64, sigma: f64) -> Self {
        Self { mu, sigma }
    }

    /// Conservative skill estimate: mu - 2*sigma.
    ///
    /// This is the lower bound of a ~95% confidence interval for the
    /// player's true skill. It penalizes uncertainty, meaning players
    /// with few observations are ranked lower than confidently-good players.
    pub fn conservative_score(&self) -> f64 {
        self.mu - 2.0 * self.sigma
    }
}

impl Default for Rating {
    fn default() -> Self {
        Self {
            mu: DEFAULT_MU,
            sigma: DEFAULT_SIGMA,
        }
    }
}

/// Perform a pairwise rating update after a comparison where the winner
/// demonstrated higher quality than the loser.
///
/// Returns the updated (`winner_rating`, `loser_rating`).
///
/// Per CP-015 section 8: follows the Weng-Lin model. The update
/// magnitude depends on the surprise of the outcome. If the winner was already
/// rated much higher than the loser, the update is small. If the loser was
/// rated higher (an upset), the update is large.
pub fn pairwise_update(winner: &Rating, loser: &Rating) -> (Rating, Rating) {
    let w = WengLinRating {
        rating: winner.mu,
        uncertainty: winner.sigma,
    };
    let l = WengLinRating {
        rating: loser.mu,
        uncertainty: loser.sigma,
    };

    let (new_w, new_l) = weng_lin(&w, &l, &Outcomes::WIN, &WENG_LIN_CONFIG);

    (
        Rating::new(new_w.rating, new_w.uncertainty),
        Rating::new(new_l.rating, new_l.uncertainty),
    )
}

#[cfg(test)]
mod tests {
    use super::*;

    const EPSILON: f64 = 1e-6;

    #[test]
    fn test_default_rating() {
        let r = Rating::default();
        assert!((r.mu - 25.0).abs() < EPSILON);
        assert!((r.sigma - 8.333333333333334).abs() < EPSILON);
    }

    #[test]
    fn test_conservative_score_default() {
        let r = Rating::default();
        // mu - 2*sigma = 25.0 - 2*8.333 = 8.333
        let score = r.conservative_score();
        assert!((score - (25.0 - 2.0 * DEFAULT_SIGMA)).abs() < EPSILON);
        assert!(score > 8.0 && score < 9.0);
    }

    #[test]
    fn test_conservative_score_high_certainty() {
        let r = Rating::new(30.0, 1.0);
        // mu - 2*sigma = 30 - 2 = 28
        assert!((r.conservative_score() - 28.0).abs() < EPSILON);
    }

    #[test]
    fn test_conservative_score_high_uncertainty() {
        let r = Rating::new(30.0, 10.0);
        // mu - 2*sigma = 30 - 20 = 10
        assert!((r.conservative_score() - 10.0).abs() < EPSILON);
    }

    #[test]
    fn test_pairwise_equal_ratings() {
        let a = Rating::default();
        let b = Rating::default();

        let (winner, loser) = pairwise_update(&a, &b);

        // Winner should gain skill, loser should lose skill
        assert!(winner.mu > a.mu);
        assert!(loser.mu < b.mu);

        // Both should have reduced uncertainty
        assert!(winner.sigma < a.sigma);
        assert!(loser.sigma < b.sigma);

        // Updates should be symmetric for equal starting ratings
        let gain = winner.mu - a.mu;
        let loss = b.mu - loser.mu;
        assert!((gain - loss).abs() < 1e-10);
    }

    #[test]
    fn test_pairwise_expected_outcome() {
        // When the higher-rated player wins, the update should be small
        let strong = Rating::new(30.0, 5.0);
        let weak = Rating::new(20.0, 5.0);

        let (new_strong, new_weak) = pairwise_update(&strong, &weak);

        // Strong player still wins, weak still loses
        assert!(new_strong.mu > strong.mu);
        assert!(new_weak.mu < weak.mu);

        // But the magnitude should be small because this was expected
        let gain = new_strong.mu - strong.mu;
        assert!(
            gain < 2.0,
            "Expected small gain for expected outcome, got {gain}"
        );
    }

    #[test]
    fn test_pairwise_upset() {
        // When the lower-rated player wins, the update should be large
        let strong = Rating::new(30.0, 5.0);
        let weak = Rating::new(20.0, 5.0);

        // Weak player wins (upset)
        let (new_weak, _new_strong) = pairwise_update(&weak, &strong);

        // The weak player gains more than in the expected case
        let gain = new_weak.mu - weak.mu;
        assert!(gain > 1.0, "Expected larger gain for upset, got {gain}");
    }

    #[test]
    fn test_pairwise_sigma_never_negative() {
        // Even after many updates, sigma should remain positive
        let mut a = Rating::default();
        let mut b = Rating::default();

        for _ in 0..100 {
            let (new_a, new_b) = pairwise_update(&a, &b);
            a = new_a;
            b = new_b;
        }

        assert!(a.sigma > 0.0);
        assert!(b.sigma > 0.0);
    }

    #[test]
    fn test_pairwise_sigma_decreases() {
        let a = Rating::default();
        let b = Rating::default();

        let (winner, loser) = pairwise_update(&a, &b);

        // Sigma should decrease for both players after an observation
        assert!(winner.sigma < a.sigma);
        assert!(loser.sigma < b.sigma);
    }

    #[test]
    fn test_convergence_consistent_winner() {
        // A player that always wins should converge to a high rating
        let mut winner = Rating::default();
        let mut loser = Rating::default();

        for _ in 0..50 {
            let (new_w, new_l) = pairwise_update(&winner, &loser);
            winner = new_w;
            loser = new_l;
        }

        // Winner should be well above the starting mu
        assert!(
            winner.mu > 30.0,
            "Winner mu should be high, got {}",
            winner.mu
        );
        // Loser should be well below
        assert!(loser.mu < 20.0, "Loser mu should be low, got {}", loser.mu);

        // Conservative scores should reflect the separation
        assert!(winner.conservative_score() > loser.conservative_score());
    }

    #[test]
    fn test_transitivity() {
        // If A beats B and B beats C, A's rating should be above C's
        let mut a = Rating::default();
        let mut b = Rating::default();
        let mut c = Rating::default();

        // A beats B five times
        for _ in 0..5 {
            let (new_a, new_b) = pairwise_update(&a, &b);
            a = new_a;
            b = new_b;
        }

        // B beats C five times
        for _ in 0..5 {
            let (new_b, new_c) = pairwise_update(&b, &c);
            b = new_b;
            c = new_c;
        }

        // A > B > C in conservative score
        assert!(a.conservative_score() > b.conservative_score());
        assert!(b.conservative_score() > c.conservative_score());
        assert!(a.conservative_score() > c.conservative_score());
    }

    #[test]
    fn test_rating_serialization() {
        let r = Rating::new(28.5, 6.1);
        let json = serde_json::to_string(&r).unwrap();
        let deserialized: Rating = serde_json::from_str(&json).unwrap();
        assert!((deserialized.mu - r.mu).abs() < EPSILON);
        assert!((deserialized.sigma - r.sigma).abs() < EPSILON);
    }

    #[test]
    fn test_pairwise_with_different_sigmas() {
        // A certain player vs an uncertain player
        let certain = Rating::new(25.0, 2.0);
        let uncertain = Rating::new(25.0, 8.0);

        let (new_certain, new_uncertain) = pairwise_update(&certain, &uncertain);

        // Both gain/lose, but the uncertain player's sigma should shrink more
        assert!(new_certain.mu > certain.mu);
        assert!(new_uncertain.mu < uncertain.mu);

        // The uncertain player's rating should change more
        let certain_change = (new_certain.mu - certain.mu).abs();
        let uncertain_change = (new_uncertain.mu - uncertain.mu).abs();
        assert!(
            uncertain_change > certain_change,
            "Uncertain player should change more: {uncertain_change} vs {certain_change}"
        );
    }

    #[test]
    fn test_conservative_score_ordering() {
        // A player with high mu but high sigma should rank lower than
        // a player with moderate mu but low sigma
        let uncertain_good = Rating::new(35.0, 10.0);
        let certain_moderate = Rating::new(28.0, 2.0);

        // 35 - 20 = 15 vs 28 - 4 = 24
        assert!(
            certain_moderate.conservative_score() > uncertain_good.conservative_score(),
            "Certain moderate ({}) should outrank uncertain good ({})",
            certain_moderate.conservative_score(),
            uncertain_good.conservative_score()
        );
    }
}