loadmaster_probe/
compare.rs

1//! `probe compare`: two eval runs side by side, with significance for the proportions.
2
3use std::fmt::Write;
4
5use crate::db::{RunMeta, Summary};
6
7/// Two-sided p-value of a two-proportion z-test; 1.0 when undefined (no variance).
8pub fn two_proportion_p(x1: u64, n1: u64, x2: u64, n2: u64) -> f64 {
9    if n1 == 0 || n2 == 0 {
10        return 1.0;
11    }
12    let (p1, p2) = (x1 as f64 / n1 as f64, x2 as f64 / n2 as f64);
13    let p = (x1 + x2) as f64 / (n1 + n2) as f64;
14    let se = (p * (1.0 - p) * (1.0 / n1 as f64 + 1.0 / n2 as f64)).sqrt();
15    if se == 0.0 {
16        return 1.0;
17    }
18    let z = ((p1 - p2) / se).abs();
19    2.0 * (1.0 - normal_cdf(z))
20}
21
22/// Standard normal CDF via the Abramowitz-Stegun 7.1.26 erf approximation (error < 1.5e-7).
23fn normal_cdf(z: f64) -> f64 {
24    let x = z / std::f64::consts::SQRT_2;
25    let t = 1.0 / (1.0 + 0.327_591_1 * x.abs());
26    let poly = t
27        * (0.254_829_592
28            + t * (-0.284_496_736
29                + t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429))));
30    let erf = 1.0 - poly * (-x * x).exp();
31    let erf = if x >= 0.0 { erf } else { -erf };
32    0.5 * (1.0 + erf)
33}
34
35fn pct(x: u64, n: u64) -> String {
36    if n == 0 {
37        "-".into()
38    } else {
39        format!("{:.1}%", 100.0 * x as f64 / n as f64)
40    }
41}
42
43fn proportion_row(out: &mut String, label: &str, (x1, n1): (u64, u64), (x2, n2): (u64, u64)) {
44    if n1 == 0 && n2 == 0 {
45        return;
46    }
47    let delta = if n1 > 0 && n2 > 0 {
48        format!(
49            "{:+.1}pp",
50            100.0 * (x2 as f64 / n2 as f64 - x1 as f64 / n1 as f64)
51        )
52    } else {
53        "-".into()
54    };
55    let p = two_proportion_p(x1, n1, x2, n2);
56    let sig = if p < 0.05 {
57        format!("p={p:.3}")
58    } else {
59        format!("p={p:.2} n.s.")
60    };
61    writeln!(
62        out,
63        "{label:<16} {:>10} {:>10} {delta:>9}  {sig}",
64        pct(x1, n1),
65        pct(x2, n2)
66    )
67    .unwrap();
68}
69
70/// At most 20 lines.
71pub fn render(
72    a: &RunMeta,
73    sa: &Summary,
74    b: &RunMeta,
75    sb: &Summary,
76    changed_outputs: usize,
77) -> String {
78    let mut o = String::new();
79    let who = |m: &RunMeta| {
80        format!(
81            "{} @ {}{}",
82            m.model,
83            m.runtime,
84            m.runtime_version
85                .as_ref()
86                .map_or(String::new(), |v| format!(" {v}"))
87        )
88    };
89    writeln!(o, "run {}: {}  ({})", a.id, who(a), a.created_at).unwrap();
90    writeln!(o, "run {}: {}  ({})", b.id, who(b), b.created_at).unwrap();
91    if a.prompts_blake3 != b.prompts_blake3 {
92        writeln!(o, "warning: the runs used different prompt files").unwrap();
93    }
94    writeln!(
95        o,
96        "{:<16} {:>10} {:>10} {:>9}",
97        "",
98        format!("run {}", a.id),
99        format!("run {}", b.id),
100        "delta"
101    )
102    .unwrap();
103    proportion_row(&mut o, "accuracy", (sa.ok, sa.n), (sb.ok, sb.n));
104    proportion_row(
105        &mut o,
106        "schema valid",
107        (sa.schema_ok, sa.schema_n),
108        (sb.schema_ok, sb.schema_n),
109    );
110    let num = |v: Option<f64>| v.map_or("-".into(), |x| format!("{x:.1}"));
111    let delta = match (sa.tokens_per_s_mean, sb.tokens_per_s_mean) {
112        (Some(x), Some(y)) => format!("{:+.1}", y - x),
113        _ => "-".into(),
114    };
115    writeln!(
116        o,
117        "{:<16} {:>10} {:>10} {delta:>9}",
118        "tokens/s (mean)",
119        num(sa.tokens_per_s_mean),
120        num(sb.tokens_per_s_mean)
121    )
122    .unwrap();
123    for (label, x, y) in [
124        ("latency p50", sa.latency_p50_ms, sb.latency_p50_ms),
125        ("latency p95", sa.latency_p95_ms, sb.latency_p95_ms),
126    ] {
127        writeln!(
128            o,
129            "{label:<16} {:>8}ms {:>8}ms {:>+7}ms",
130            x,
131            y,
132            y as i64 - x as i64
133        )
134        .unwrap();
135    }
136    writeln!(
137        o,
138        "{changed_outputs} of {} responses differ between the runs",
139        sa.n.min(sb.n)
140    )
141    .unwrap();
142    writeln!(
143        o,
144        "n.s. = not significant (two-proportion z-test, p >= 0.05); timings are not tested"
145    )
146    .unwrap();
147    o
148}
149
150#[cfg(test)]
151mod tests {
152    use super::*;
153
154    #[test]
155    fn z_test_matches_reference_values() {
156        // 45/50 vs 30/50: z = 3.464, p = 0.000532 (checked with Python math.erf).
157        let p = two_proportion_p(45, 50, 30, 50);
158        assert!((p - 0.000_532).abs() < 0.000_001, "{p}");
159        // 41/50 vs 43/50: clearly not significant.
160        assert!(two_proportion_p(41, 50, 43, 50) > 0.5);
161        assert_eq!(two_proportion_p(10, 10, 10, 10), 1.0);
162    }
163}
loadmaster_probe/compare.rs

loadmaster_probe/
compare.rs
