silicera/
measure.rs

1//! Measurement engine with statistical summaries and instability flags.
2//!
3//! Correctness and statistical skepticism take priority over chasing speedups.
4//! Results include median, mean, variance, percentiles, outlier counts, and
5//! explicit stability classification.
6
7use std::time::{Duration, Instant};
8
9use serde::{Deserialize, Serialize};
10
11use crate::error::{Result, SiliceraError};
12
13/// Configuration for a measurement campaign.
14#[derive(Debug, Clone, Serialize, Deserialize)]
15pub struct MeasurementConfig {
16    /// Warmup iterations (discarded).
17    pub warmup: usize,
18    /// Timed iterations.
19    pub iterations: usize,
20    /// Maximum relative IQR / median before marking unstable (e.g. 0.25 = 25%).
21    pub instability_iqr_ratio: f64,
22    /// Maximum coefficient of variation (stddev/mean) before unstable.
23    pub instability_cv: f64,
24    /// Outlier fence in IQR multiples (Tukey; typically 1.5).
25    pub outlier_fence: f64,
26}
27
28impl Default for MeasurementConfig {
29    fn default() -> Self {
30        Self {
31            warmup: 5,
32            iterations: 30,
33            instability_iqr_ratio: 0.35,
34            instability_cv: 0.20,
35            outlier_fence: 1.5,
36        }
37    }
38}
39
40/// Stability classification for a sample set.
41#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
42pub enum Stability {
43    /// Tight distribution; safe for ranking.
44    Stable,
45    /// Elevated variance; report but do not claim strong wins.
46    Noisy,
47    /// Unusable for specialization decisions.
48    Unstable,
49}
50
51impl Stability {
52    /// Display label.
53    pub fn label(self) -> &'static str {
54        match self {
55            Stability::Stable => "STABLE",
56            Stability::Noisy => "NOISY",
57            Stability::Unstable => "UNSTABLE",
58        }
59    }
60}
61
62/// Statistical summary of timed samples (nanoseconds).
63#[derive(Debug, Clone, Serialize, Deserialize)]
64pub struct MeasurementSummary {
65    /// Sample count (after warmup).
66    pub n: usize,
67    /// Minimum (ns).
68    pub min_ns: f64,
69    /// Maximum (ns).
70    pub max_ns: f64,
71    /// Arithmetic mean (ns).
72    pub mean_ns: f64,
73    /// Median (ns).
74    pub median_ns: f64,
75    /// Sample variance (ns²).
76    pub variance_ns2: f64,
77    /// Sample standard deviation (ns).
78    pub stddev_ns: f64,
79    /// 25th percentile (ns).
80    pub p25_ns: f64,
81    /// 75th percentile (ns).
82    pub p75_ns: f64,
83    /// 95th percentile (ns).
84    pub p95_ns: f64,
85    /// 99th percentile (ns).
86    pub p99_ns: f64,
87    /// Tukey outlier count.
88    pub outlier_count: usize,
89    /// Stability classification.
90    pub stability: Stability,
91    /// Human-readable flags (e.g. high CV).
92    pub flags: Vec<String>,
93}
94
95impl MeasurementSummary {
96    /// Compute summary from raw nanosecond samples.
97    pub fn from_samples(samples: &[f64], cfg: &MeasurementConfig) -> Result<Self> {
98        if samples.is_empty() {
99            return Err(SiliceraError::MeasurementUnstable(
100                "no samples collected".into(),
101            ));
102        }
103        let mut sorted = samples.to_vec();
104        sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
105        let n = sorted.len();
106        let min_ns = sorted[0];
107        let max_ns = sorted[n - 1];
108        let mean_ns = sorted.iter().sum::<f64>() / n as f64;
109        let median_ns = percentile_sorted(&sorted, 0.50);
110        let p25_ns = percentile_sorted(&sorted, 0.25);
111        let p75_ns = percentile_sorted(&sorted, 0.75);
112        let p95_ns = percentile_sorted(&sorted, 0.95);
113        let p99_ns = percentile_sorted(&sorted, 0.99);
114        let variance_ns2 = if n > 1 {
115            sorted.iter().map(|x| {
116                let d = x - mean_ns;
117                d * d
118            }).sum::<f64>() / (n - 1) as f64
119        } else {
120            0.0
121        };
122        let stddev_ns = variance_ns2.sqrt();
123        let iqr = p75_ns - p25_ns;
124        let lo = p25_ns - cfg.outlier_fence * iqr;
125        let hi = p75_ns + cfg.outlier_fence * iqr;
126        let outlier_count = sorted.iter().filter(|&&x| x < lo || x > hi).count();
127
128        let mut flags = Vec::new();
129        let cv = if mean_ns > 0.0 {
130            stddev_ns / mean_ns
131        } else {
132            0.0
133        };
134        let iqr_ratio = if median_ns > 0.0 {
135            iqr / median_ns
136        } else {
137            0.0
138        };
139
140        if cv > cfg.instability_cv {
141            flags.push(format!("cv={cv:.3} > {}", cfg.instability_cv));
142        }
143        if iqr_ratio > cfg.instability_iqr_ratio {
144            flags.push(format!(
145                "iqr/median={iqr_ratio:.3} > {}",
146                cfg.instability_iqr_ratio
147            ));
148        }
149        if outlier_count * 5 > n {
150            flags.push(format!("outliers={outlier_count}/{n}"));
151        }
152
153        let stability = if cv > cfg.instability_cv * 1.5
154            || iqr_ratio > cfg.instability_iqr_ratio * 1.5
155        {
156            Stability::Unstable
157        } else if !flags.is_empty() {
158            Stability::Noisy
159        } else {
160            Stability::Stable
161        };
162
163        Ok(Self {
164            n,
165            min_ns,
166            max_ns,
167            mean_ns,
168            median_ns,
169            variance_ns2,
170            stddev_ns,
171            p25_ns,
172            p75_ns,
173            p95_ns,
174            p99_ns,
175            outlier_count,
176            stability,
177            flags,
178        })
179    }
180
181    /// Primary ranking key (median nanoseconds; lower is better).
182    pub fn score_ns(&self) -> f64 {
183        self.median_ns
184    }
185}
186
187/// Runs timed closures with warmup and statistical aggregation.
188#[derive(Debug, Clone)]
189pub struct MeasurementEngine {
190    /// Configuration.
191    pub config: MeasurementConfig,
192}
193
194impl Default for MeasurementEngine {
195    fn default() -> Self {
196        Self {
197            config: MeasurementConfig::default(),
198        }
199    }
200}
201
202impl MeasurementEngine {
203    /// Create with config.
204    pub fn new(config: MeasurementConfig) -> Self {
205        Self { config }
206    }
207
208    /// Measure a closure that returns a unit value (timing only).
209    pub fn measure<F>(&self, mut f: F) -> Result<MeasurementSummary>
210    where
211        F: FnMut(),
212    {
213        for _ in 0..self.config.warmup {
214            f();
215            // Prevent the optimizer from considering the loop dead in trivial cases.
216            std::hint::black_box(());
217        }
218        let mut samples = Vec::with_capacity(self.config.iterations);
219        for _ in 0..self.config.iterations {
220            let start = Instant::now();
221            f();
222            let elapsed = start.elapsed();
223            samples.push(duration_ns(elapsed));
224            std::hint::black_box(());
225        }
226        MeasurementSummary::from_samples(&samples, &self.config)
227    }
228
229    /// Measure a closure that also produces an output for correctness checking.
230    pub fn measure_with_output<F, T>(&self, mut f: F) -> Result<(MeasurementSummary, T)>
231    where
232        F: FnMut() -> T,
233        T: Clone,
234    {
235        let mut last = None;
236        for _ in 0..self.config.warmup {
237            last = Some(f());
238            std::hint::black_box(());
239        }
240        let mut samples = Vec::with_capacity(self.config.iterations);
241        for _ in 0..self.config.iterations {
242            let start = Instant::now();
243            let out = f();
244            let elapsed = start.elapsed();
245            samples.push(duration_ns(elapsed));
246            last = Some(out);
247            std::hint::black_box(());
248        }
249        let summary = MeasurementSummary::from_samples(&samples, &self.config)?;
250        let out = last.ok_or_else(|| {
251            SiliceraError::Internal("measure_with_output produced no output".into())
252        })?;
253        Ok((summary, out))
254    }
255}
256
257fn duration_ns(d: Duration) -> f64 {
258    d.as_secs_f64() * 1_000_000_000.0
259}
260
261fn percentile_sorted(sorted: &[f64], p: f64) -> f64 {
262    if sorted.is_empty() {
263        return 0.0;
264    }
265    if sorted.len() == 1 {
266        return sorted[0];
267    }
268    let idx = p * (sorted.len() - 1) as f64;
269    let lo = idx.floor() as usize;
270    let hi = idx.ceil() as usize;
271    if lo == hi {
272        sorted[lo]
273    } else {
274        let w = idx - lo as f64;
275        sorted[lo] * (1.0 - w) + sorted[hi] * w
276    }
277}
278
279#[cfg(test)]
280mod tests {
281    use super::*;
282
283    #[test]
284    fn summary_stable_constant() {
285        let samples: Vec<f64> = (0..40).map(|_| 1000.0).collect();
286        let s = MeasurementSummary::from_samples(&samples, &MeasurementConfig::default()).unwrap();
287        assert_eq!(s.stability, Stability::Stable);
288        assert_eq!(s.median_ns, 1000.0);
289    }
290
291    #[test]
292    fn engine_runs() {
293        let eng = MeasurementEngine::new(MeasurementConfig {
294            warmup: 1,
295            iterations: 5,
296            ..Default::default()
297        });
298        let mut x = 0u64;
299        let s = eng
300            .measure(|| {
301                x = x.wrapping_add(1);
302                std::hint::black_box(x);
303            })
304            .unwrap();
305        assert_eq!(s.n, 5);
306    }
307}
silicera/measure.rs

silicera/
measure.rs
