gold_cmp/
lib.rs

1//! # gold-cmp
2//!
3//! Pairwise comparison runner for gold-set LLM evals.
4//!
5//! Pattern: for each gold case, ask a judge (LLM or human) which of two
6//! candidate outputs is better, or call it a tie. Aggregate into a
7//! win-rate with a 95% CI and a yes/no answer on "is A meaningfully
8//! better than B" using a two-sided binomial test (normal
9//! approximation; OK for n ≥ 30).
10//!
11//! ## Example
12//!
13//! ```
14//! use gold_cmp::{summarize, Outcome};
15//! let outcomes = vec![
16//!     Outcome::AWins, Outcome::AWins, Outcome::BWins,
17//!     Outcome::Tie,   Outcome::AWins, Outcome::AWins,
18//!     Outcome::BWins, Outcome::AWins,
19//! ];
20//! let s = summarize(&outcomes);
21//! assert_eq!(s.a_wins, 5);
22//! assert!(s.a_win_rate_excl_ties > 0.5);
23//! ```
24
25#![deny(missing_docs)]
26
27/// Outcome of one pairwise judgment.
28#[derive(Debug, Clone, Copy, PartialEq, Eq)]
29pub enum Outcome {
30    /// A is better.
31    AWins,
32    /// B is better.
33    BWins,
34    /// The judge called it equivalent.
35    Tie,
36}
37
38/// Aggregated summary.
39#[derive(Debug, Clone, Copy, PartialEq)]
40pub struct Summary {
41    /// Total judgments.
42    pub n: u64,
43    /// Wins for A.
44    pub a_wins: u64,
45    /// Wins for B.
46    pub b_wins: u64,
47    /// Ties.
48    pub ties: u64,
49    /// A's win rate excluding ties.
50    pub a_win_rate_excl_ties: f64,
51    /// Normal-approximation 95% CI half-width on the excl-ties win rate.
52    pub ci95_half_width: f64,
53    /// True when A is meaningfully ahead (CI excludes 0.5).
54    pub a_meaningfully_ahead: bool,
55}
56
57/// Summarize a list of outcomes.
58pub fn summarize(outcomes: &[Outcome]) -> Summary {
59    let n = outcomes.len() as u64;
60    let a_wins = outcomes.iter().filter(|o| **o == Outcome::AWins).count() as u64;
61    let b_wins = outcomes.iter().filter(|o| **o == Outcome::BWins).count() as u64;
62    let ties = outcomes.iter().filter(|o| **o == Outcome::Tie).count() as u64;
63    let decided = a_wins + b_wins;
64    let p = if decided == 0 {
65        0.5
66    } else {
67        a_wins as f64 / decided as f64
68    };
69    let ci = if decided == 0 {
70        0.0
71    } else {
72        1.96 * (p * (1.0 - p) / decided as f64).sqrt()
73    };
74    let ahead = decided > 0 && (p - ci) > 0.5;
75    Summary {
76        n,
77        a_wins,
78        b_wins,
79        ties,
80        a_win_rate_excl_ties: p,
81        ci95_half_width: ci,
82        a_meaningfully_ahead: ahead,
83    }
84}
gold_cmp/lib.rs

gold_cmp/
lib.rs
