gold-cmp 0.1.0

Pairwise comparison runner for gold-set LLM evals: A vs B winner counting, statistical-significance helper, win-rate summary. Zero deps.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

use gold_cmp::{summarize, Outcome};

#[test]
fn empty_is_zero() {
    let s = summarize(&[]);
    assert_eq!(s.n, 0);
    assert_eq!(s.a_wins, 0);
}

#[test]
fn all_a_wins() {
    let s = summarize(&vec![Outcome::AWins; 100]);
    assert!((s.a_win_rate_excl_ties - 1.0).abs() < 1e-9);
    assert!(s.a_meaningfully_ahead);
}

#[test]
fn all_b_wins_a_not_ahead() {
    let s = summarize(&vec![Outcome::BWins; 100]);
    assert_eq!(s.a_win_rate_excl_ties, 0.0);
    assert!(!s.a_meaningfully_ahead);
}

#[test]
fn coin_flip_not_meaningful() {
    let mut outs = Vec::new();
    for _ in 0..50 {
        outs.push(Outcome::AWins);
        outs.push(Outcome::BWins);
    }
    let s = summarize(&outs);
    assert!((s.a_win_rate_excl_ties - 0.5).abs() < 1e-9);
    assert!(!s.a_meaningfully_ahead);
}

#[test]
fn ties_excluded_from_rate() {
    // 5 A, 0 B, 95 ties.
    let mut outs = vec![Outcome::Tie; 95];
    outs.extend(vec![Outcome::AWins; 5]);
    let s = summarize(&outs);
    assert_eq!(s.ties, 95);
    assert_eq!(s.a_wins, 5);
    assert!((s.a_win_rate_excl_ties - 1.0).abs() < 1e-9);
    // n=5 decided, CI is wide; might not flag as meaningful — both are OK.
    assert!(s.ci95_half_width >= 0.0);
}

#[test]
fn small_lead_flagged_when_n_is_large() {
    // 60 A, 40 B over 100 decided → 95% CI half-width ≈ 0.096; lower bound ~0.504 > 0.5
    let mut outs = vec![Outcome::AWins; 60];
    outs.extend(vec![Outcome::BWins; 40]);
    let s = summarize(&outs);
    assert!(s.a_meaningfully_ahead);
}