subms_stats/
compare.rs

1//! Distribution comparison: "is candidate slower than baseline?"
2//! Behind the `compare` Cargo feature (on by default).
3
4use crate::percentiles::{mean, stddev};
5
6/// Two-sample Kolmogorov-Smirnov D statistic: the maximum vertical
7/// gap between the two empirical CDFs. Range `[0.0, 1.0]`; larger
8/// means more different. `None` if either side is empty.
9///
10/// Use to detect "did the distribution shift between baseline and
11/// candidate?" - more sensitive than comparing p99 alone, since a
12/// distribution can shift its body without moving p99 (or vice
13/// versa).
14pub fn ks_statistic(baseline: &[u64], candidate: &[u64]) -> Option<f64> {
15    if baseline.is_empty() || candidate.is_empty() {
16        return None;
17    }
18    let mut a = baseline.to_vec();
19    let mut b = candidate.to_vec();
20    a.sort_unstable();
21    b.sort_unstable();
22    let mut i = 0usize;
23    let mut j = 0usize;
24    let na = a.len() as f64;
25    let nb = b.len() as f64;
26    let mut max_d = 0.0f64;
27    while i < a.len() && j < b.len() {
28        if a[i] <= b[j] {
29            i += 1;
30        } else {
31            j += 1;
32        }
33        let cdf_a = i as f64 / na;
34        let cdf_b = j as f64 / nb;
35        let d = (cdf_a - cdf_b).abs();
36        if d > max_d {
37            max_d = d;
38        }
39    }
40    Some(max_d)
41}
42
43/// Cohen's d effect size: standardised difference between two means.
44/// Magnitude `< 0.2` is small, `~ 0.5` medium, `> 0.8` large. Use
45/// alongside [`ks_statistic`] to decide whether a p99 regression is
46/// "real" or noise.
47pub fn cohens_d(baseline: &[u64], candidate: &[u64]) -> Option<f64> {
48    if baseline.is_empty() || candidate.is_empty() {
49        return None;
50    }
51    let na = baseline.len();
52    let nb = candidate.len();
53    let ma = mean(baseline) as f64;
54    let mb = mean(candidate) as f64;
55    let sa = stddev(baseline) as f64;
56    let sb = stddev(candidate) as f64;
57    let pooled =
58        (((na - 1) as f64 * sa * sa + (nb - 1) as f64 * sb * sb) / ((na + nb - 2) as f64)).sqrt();
59    if pooled <= 0.0 {
60        return Some(0.0);
61    }
62    Some((mb - ma) / pooled)
63}
64
65#[cfg(test)]
66mod tests {
67    use super::*;
68
69    #[test]
70    fn ks_same_distribution_near_zero() {
71        let a: Vec<u64> = (0..1000).collect();
72        let b: Vec<u64> = (0..1000).collect();
73        assert!(ks_statistic(&a, &b).unwrap() < 0.01);
74    }
75
76    #[test]
77    fn ks_shifted_distribution_large() {
78        let a: Vec<u64> = (0..1000).collect();
79        let b: Vec<u64> = (500..1500).collect();
80        assert!(ks_statistic(&a, &b).unwrap() > 0.4);
81    }
82
83    #[test]
84    fn ks_empty_returns_none() {
85        assert!(ks_statistic(&[], &[1, 2]).is_none());
86        assert!(ks_statistic(&[1, 2], &[]).is_none());
87    }
88
89    #[test]
90    fn cohens_d_zero_for_identical() {
91        let a: Vec<u64> = (0..100).collect();
92        let b: Vec<u64> = (0..100).collect();
93        assert!(cohens_d(&a, &b).unwrap().abs() < 0.01);
94    }
95
96    #[test]
97    fn cohens_d_positive_when_candidate_slower() {
98        let a: Vec<u64> = (100..200).collect();
99        let b: Vec<u64> = (200..300).collect();
100        assert!(cohens_d(&a, &b).unwrap() > 0.5);
101    }
102}
subms_stats/compare.rs

subms_stats/
compare.rs
