launchbound_bench/
stats.rs

1//! Interval statistics (docs/BENCHMARKING.md): a benchmark that reports a mean
2//! and no interval is not evidence. Median with a distribution-free 95% CI
3//! (order statistics), Tukey-fence outlier rejection, and an overlap test —
4//! configurations whose intervals overlap are indistinguishable, never
5//! ranked.
6
7use serde::{Deserialize, Serialize};
8
9#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
10pub struct Summary {
11    /// Samples kept after outlier rejection.
12    pub n: usize,
13    pub outliers_rejected: usize,
14    pub median_ms: f64,
15    /// Distribution-free 95% CI on the median (order statistics).
16    pub ci95_lo_ms: f64,
17    pub ci95_hi_ms: f64,
18    pub min_ms: f64,
19    pub max_ms: f64,
20    pub mean_ms: f64,
21}
22
23/// Summarize raw timings. The Tukey fences (1.5 IQR) run first; the median
24/// CI uses the normal approximation to the binomial order-statistic
25/// interval, clamped to the sample range.
26pub fn summarize(samples_ms: &[f64]) -> Option<Summary> {
27    if samples_ms.is_empty() {
28        return None;
29    }
30    let mut sorted: Vec<f64> = samples_ms.to_vec();
31    sorted.sort_by(|a, b| a.partial_cmp(b).expect("no NaN timings"));
32
33    let q1 = quantile(&sorted, 0.25);
34    let q3 = quantile(&sorted, 0.75);
35    let iqr = q3 - q1;
36    let (lo_fence, hi_fence) = (q1 - 1.5 * iqr, q3 + 1.5 * iqr);
37    let kept: Vec<f64> = sorted
38        .iter()
39        .copied()
40        .filter(|&x| x >= lo_fence && x <= hi_fence)
41        .collect();
42    let outliers_rejected = sorted.len() - kept.len();
43    let n = kept.len();
44    if n == 0 {
45        return None;
46    }
47
48    let median = quantile(&kept, 0.5);
49    // Order-statistic 95% CI for the median: ranks n/2 ± 1.96*sqrt(n)/2.
50    let half_width = 1.96 * (n as f64).sqrt() / 2.0;
51    let lo_rank = ((n as f64) / 2.0 - half_width).floor().max(0.0) as usize;
52    let hi_rank = (((n as f64) / 2.0 + half_width).ceil() as usize).min(n - 1);
53    let mean = kept.iter().sum::<f64>() / n as f64;
54
55    Some(Summary {
56        n,
57        outliers_rejected,
58        median_ms: median,
59        ci95_lo_ms: kept[lo_rank],
60        ci95_hi_ms: kept[hi_rank],
61        min_ms: kept[0],
62        max_ms: kept[n - 1],
63        mean_ms: mean,
64    })
65}
66
67/// Linear-interpolated quantile of a sorted slice.
68fn quantile(sorted: &[f64], q: f64) -> f64 {
69    if sorted.len() == 1 {
70        return sorted[0];
71    }
72    let pos = q * (sorted.len() - 1) as f64;
73    let base = pos.floor() as usize;
74    let frac = pos - base as f64;
75    if base + 1 < sorted.len() {
76        sorted[base] * (1.0 - frac) + sorted[base + 1] * frac
77    } else {
78        sorted[base]
79    }
80}
81
82/// Two summaries whose 95% CIs overlap are indistinguishable (docs/BENCHMARKING.md).
83pub fn indistinguishable(a: &Summary, b: &Summary) -> bool {
84    a.ci95_lo_ms <= b.ci95_hi_ms && b.ci95_lo_ms <= a.ci95_hi_ms
85}
86
87#[cfg(test)]
88mod tests {
89    use super::*;
90
91    #[test]
92    fn summarizes_and_rejects_outliers() {
93        let mut samples: Vec<f64> = (0..100).map(|i| 1.0 + (i % 7) as f64 * 0.001).collect();
94        samples.push(50.0); // gross outlier
95        let s = summarize(&samples).unwrap();
96        assert_eq!(s.outliers_rejected, 1);
97        assert!(s.median_ms > 0.99 && s.median_ms < 1.01);
98        assert!(s.ci95_lo_ms <= s.median_ms && s.median_ms <= s.ci95_hi_ms);
99    }
100
101    #[test]
102    fn overlap_means_indistinguishable() {
103        let a = summarize(&[1.0, 1.01, 1.02, 0.99, 1.0]).unwrap();
104        let b = summarize(&[1.01, 1.02, 1.03, 1.0, 1.01]).unwrap();
105        assert!(indistinguishable(&a, &b));
106        let c = summarize(&[2.0, 2.01, 2.02, 1.99, 2.0]).unwrap();
107        assert!(!indistinguishable(&a, &c));
108    }
109
110    #[test]
111    fn empty_input_is_none_not_zero() {
112        assert!(summarize(&[]).is_none());
113    }
114}
launchbound_bench/stats.rs

launchbound_bench/
stats.rs
