asupersync_conformance/bench/
stats.rs

1//! Statistics and comparison helpers for benchmarks.
2
3use serde::{Deserialize, Serialize};
4use std::fmt;
5use std::time::Duration;
6
7/// Errors that can occur while computing statistics.
8#[derive(Debug, Clone)]
9pub enum StatsError {
10    /// No samples were provided.
11    EmptySamples,
12}
13
14impl fmt::Display for StatsError {
15    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
16        match self {
17            StatsError::EmptySamples => write!(f, "no samples provided"),
18        }
19    }
20}
21
22impl std::error::Error for StatsError {}
23
24/// Statistical summary of benchmark results.
25#[derive(Debug, Clone, Serialize, Deserialize)]
26pub struct Stats {
27    pub min: Duration,
28    pub max: Duration,
29    pub mean: Duration,
30    pub median: Duration,
31    pub std_dev: Duration,
32    pub p50: Duration,
33    pub p75: Duration,
34    pub p90: Duration,
35    pub p95: Duration,
36    pub p99: Duration,
37    pub p999: Duration,
38    pub sample_count: usize,
39}
40
41impl Stats {
42    /// Compute statistics from samples.
43    pub fn from_samples(samples: &[Duration]) -> Result<Self, StatsError> {
44        if samples.is_empty() {
45            return Err(StatsError::EmptySamples);
46        }
47
48        let mut sorted: Vec<u128> = samples.iter().map(|d| d.as_nanos()).collect();
49        sorted.sort_unstable();
50
51        let n = sorted.len();
52        let sum_nanos: u128 = sorted.iter().copied().sum();
53        let mean_nanos = sum_nanos / n as u128;
54
55        let mean_f64 = mean_nanos as f64;
56        let variance = sorted
57            .iter()
58            .map(|value| {
59                let diff = *value as f64 - mean_f64;
60                diff * diff
61            })
62            .sum::<f64>()
63            / n as f64;
64        let std_dev_nanos = variance.sqrt();
65
66        Ok(Self {
67            min: nanos_to_duration(*sorted.first().unwrap()),
68            max: nanos_to_duration(*sorted.last().unwrap()),
69            mean: nanos_to_duration(mean_nanos),
70            median: nanos_to_duration(percentile(&sorted, 1, 2)),
71            std_dev: Duration::from_nanos(f64_to_u64_saturating(std_dev_nanos)),
72            p50: nanos_to_duration(percentile(&sorted, 50, 100)),
73            p75: nanos_to_duration(percentile(&sorted, 75, 100)),
74            p90: nanos_to_duration(percentile(&sorted, 90, 100)),
75            p95: nanos_to_duration(percentile(&sorted, 95, 100)),
76            p99: nanos_to_duration(percentile(&sorted, 99, 100)),
77            p999: nanos_to_duration(percentile(&sorted, 999, 1000)),
78            sample_count: n,
79        })
80    }
81
82    /// Coefficient of variation (std_dev / mean).
83    pub fn cv(&self) -> f64 {
84        let mean = self.mean.as_nanos() as f64;
85        if mean == 0.0 {
86            return 0.0;
87        }
88        self.std_dev.as_nanos() as f64 / mean
89    }
90}
91
92/// Comparison between two implementations.
93#[derive(Debug, Clone, Serialize, Deserialize)]
94pub struct Comparison {
95    pub a: Stats,
96    pub b: Stats,
97    pub speedup: f64,
98    pub confidence: ComparisonConfidence,
99}
100
101#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
102pub enum ComparisonConfidence {
103    /// Clear winner, low variance.
104    High,
105    /// Likely winner, some variance.
106    Medium,
107    /// Too close to call.
108    Low,
109    /// High variance, unreliable.
110    Uncertain,
111}
112
113impl Comparison {
114    /// Compute a comparison summary between two stats.
115    pub fn compute(a: &Stats, b: &Stats) -> Self {
116        let a_mean = a.mean.as_nanos() as f64;
117        let b_mean = b.mean.as_nanos() as f64;
118        let speedup = if a_mean == 0.0 {
119            f64::INFINITY
120        } else {
121            b_mean / a_mean
122        };
123
124        let avg_cv = (a.cv() + b.cv()) / 2.0;
125        let diff_pct = (speedup - 1.0).abs();
126
127        let confidence = if avg_cv > 0.5 {
128            ComparisonConfidence::Uncertain
129        } else if diff_pct < 0.05 {
130            ComparisonConfidence::Low
131        } else if avg_cv > 0.2 {
132            ComparisonConfidence::Medium
133        } else {
134            ComparisonConfidence::High
135        };
136
137        Self {
138            a: a.clone(),
139            b: b.clone(),
140            speedup,
141            confidence,
142        }
143    }
144}
145
146fn percentile(sorted: &[u128], numerator: usize, denominator: usize) -> u128 {
147    let n = sorted.len();
148    let idx = (n.saturating_sub(1) * numerator) / denominator;
149    sorted[idx]
150}
151
152fn nanos_to_duration(nanos: u128) -> Duration {
153    Duration::from_nanos(u128_to_u64_saturating(nanos))
154}
155
156fn u128_to_u64_saturating(value: u128) -> u64 {
157    u64::try_from(value).unwrap_or(u64::MAX)
158}
159
160fn f64_to_u64_saturating(value: f64) -> u64 {
161    if !value.is_finite() || value <= 0.0 {
162        return 0;
163    }
164    if value >= u64::MAX as f64 {
165        return u64::MAX;
166    }
167    value.round() as u64
168}
169
170#[cfg(test)]
171mod tests {
172    use super::*;
173
174    #[test]
175    fn stats_basic() {
176        let samples = [
177            Duration::from_micros(10),
178            Duration::from_micros(20),
179            Duration::from_micros(30),
180            Duration::from_micros(40),
181        ];
182        let stats = Stats::from_samples(&samples).expect("stats computed");
183
184        assert_eq!(stats.sample_count, 4);
185        assert_eq!(stats.min, Duration::from_micros(10));
186        assert_eq!(stats.max, Duration::from_micros(40));
187        assert_eq!(stats.p50, Duration::from_micros(20));
188    }
189}
asupersync_conformance/bench/stats.rs

asupersync_conformance/bench/
stats.rs
