Skip to main content

kevy_bench/
lib.rs

1//! kevy-bench — a pure-Rust micro-benchmark harness. Zero dependencies.
2//!
3//! kevy is 0-dependency by policy, so we cannot pull `criterion`. This is the
4//! self-hosted equivalent, used by two callers:
5//!
6//! * per-crate exploration benches (`examples/bench_*.rs`) — compare two
7//!   implementations head-to-head and print a speedup ratio. This ratio is the
8//!   evidence the dep/std self-host evaluation is gated on ("if the candidate
9//!   isn't ≥ N× the incumbent, don't self-host it").
10//! * per-crate regression gates (`tests/perf_gate.rs`) — assert a hot path stays
11//!   under a documented budget (see [`time_median`]).
12//!
13//! # Why the ratio survives a loaded host
14//!
15//! The campaign that birthed this crate could not get a clean full-system
16//! throughput number because the dev host is permanently busy. The escape is to
17//! measure *components*, not the whole server: run the variants **back-to-back
18//! in one process**. Both pay the same contention in the same window, so the
19//! absolute ns drifts with load but the **ratio between them holds**. Treat
20//! `median_ns` as relative, not absolute, unless the host is known idle.
21//!
22//! # Method
23//!
24//! Each *sample* times `inner` iterations of the closure and divides, so the
25//! per-op figure isn't swamped by `Instant::now()` overhead (~tens of ns) for
26//! sub-microsecond work. We report the **median** sample (robust to the
27//! occasional scheduler hiccup) plus p95 and min.
28//!
29//! ```
30//! use kevy_bench::{bench, black_box};
31//!
32//! let s = bench(50, 1000, || { black_box(1u64 + black_box(2)); });
33//! assert!(s.median_ns < 1_000); // trivial add is nanoseconds
34//! ```
35#![forbid(unsafe_code)]
36#![warn(missing_docs)]
37
38use std::time::{Duration, Instant};
39
40/// Re-exported so benches can fence the optimizer without importing `std::hint`.
41pub use std::hint::black_box;
42
43/// Per-operation timing summary, in nanoseconds. See [`bench()`].
44/// # Examples
45///
46/// ```
47/// let s = kevy_bench::bench(5, 100, || { std::hint::black_box(1u64 + 1); });
48/// assert_eq!((s.samples, s.inner), (5, 100));
49/// // The order the summary always holds, whatever the machine did.
50/// assert!(s.min_ns <= s.median_ns && s.median_ns <= s.p95_ns);
51/// ```
52#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)]
53#[non_exhaustive]
54pub struct Stats {
55    /// Number of samples collected.
56    ///
57    /// # Examples
58    ///
59    /// ```
60    /// let s = kevy_bench::bench(7, 10, || { std::hint::black_box(1u8); });
61    /// assert_eq!(s.samples, 7); // one timed sample per outer repetition
62    /// ```
63    pub samples: usize,
64    /// Iterations timed per sample (the divisor applied to each sample).
65    ///
66    /// # Examples
67    ///
68    /// ```
69    /// let s = kevy_bench::bench(3, 250, || { std::hint::black_box(1u8); });
70    /// // each sample's elapsed time was divided by this
71    /// assert_eq!(s.inner, 250);
72    /// ```
73    pub inner: usize,
74    /// Fastest sample — the least-disturbed run, closest to the true cost.
75    ///
76    /// # Examples
77    ///
78    /// ```
79    /// let s = kevy_bench::bench(20, 100, || { std::hint::black_box(1u8); });
80    /// // no sample was faster, so the median and the mean sit at or above it
81    /// assert!(s.min_ns <= s.median_ns && s.min_ns <= s.mean_ns);
82    /// ```
83    pub min_ns: u64,
84    /// Median sample — the headline figure, robust to occasional hiccups.
85    ///
86    /// # Examples
87    ///
88    /// ```
89    /// use std::time::Duration;
90    /// let s = kevy_bench::bench(3, 1, || std::thread::sleep(Duration::from_micros(200)));
91    /// // a 200 µs sleep never measures shorter than that
92    /// assert!(s.median_ns >= 200_000);
93    /// ```
94    pub median_ns: u64,
95    /// 95th-percentile sample — tail behaviour under scheduler noise.
96    ///
97    /// # Examples
98    ///
99    /// ```
100    /// let s = kevy_bench::bench(40, 100, || { std::hint::black_box(1u8); });
101    /// assert!(s.p95_ns >= s.median_ns); // the tail sits at or above the median
102    /// ```
103    pub p95_ns: u64,
104    /// Mean across all samples.
105    ///
106    /// # Examples
107    ///
108    /// ```
109    /// use std::time::Duration;
110    /// let s = kevy_bench::bench(4, 1, || std::thread::sleep(Duration::from_micros(100)));
111    /// // every sample took at least the sleep, so their average did too
112    /// assert!(s.mean_ns >= 100_000 && s.mean_ns >= s.min_ns);
113    /// ```
114    pub mean_ns: u64,
115    /// Sample standard deviation across samples (Bessel-corrected; 0 when
116    /// `samples == 1`). Reported alongside the median in baseline tables so
117    /// a future delta can be judged against the run-to-run noise band.
118    ///
119    /// # Examples
120    ///
121    /// ```
122    /// let one = kevy_bench::bench(1, 100, || { std::hint::black_box(1u8); });
123    /// assert_eq!(one.stdev_ns, 0); // a single sample has no spread
124    /// ```
125    pub stdev_ns: u64,
126}
127
128/// Run `op` and return per-operation timing stats.
129///
130/// `samples` outer repetitions, each timing `inner` calls of `op` and dividing.
131/// One untimed warm-up sample primes caches/branch predictors first. Pick
132/// `inner` large enough that one sample is comfortably above `Instant`
133/// resolution (≥ a few µs); for ns-scale ops use `inner` in the thousands.
134///
135/// # Panics
136///
137/// When `samples` or `inner` is zero.
138///
139/// # Examples
140///
141/// ```
142/// // `inner` is the divisor: each sample times `inner` iterations and the
143/// // reported figure is per-iteration, so a cheap op is still measurable.
144/// let s = kevy_bench::bench(3, 1_000, || { std::hint::black_box(0u8); });
145/// assert_eq!(s.samples, 3);
146/// assert_eq!(s.inner, 1_000);
147/// ```
148pub fn bench<F: FnMut()>(samples: usize, inner: usize, mut op: F) -> Stats {
149    assert!(samples > 0 && inner > 0, "samples and inner must be non-zero");
150
151    // Warm-up: not recorded.
152    for _ in 0..inner {
153        op();
154    }
155
156    let mut per_op: Vec<u64> = Vec::with_capacity(samples);
157    for _ in 0..samples {
158        let start = Instant::now();
159        for _ in 0..inner {
160            op();
161        }
162        let elapsed = start.elapsed();
163        per_op.push(elapsed.as_nanos() as u64 / inner as u64);
164    }
165    per_op.sort_unstable();
166
167    let sum: u64 = per_op.iter().sum();
168    let mean = sum / samples as u64;
169    let stdev_ns = if samples > 1 {
170        let var: f64 = per_op
171            .iter()
172            .map(|&x| {
173                let d = x as f64 - mean as f64;
174                d * d
175            })
176            .sum::<f64>()
177            / (samples - 1) as f64;
178        var.sqrt() as u64
179    } else {
180        0
181    };
182    Stats {
183        samples,
184        inner,
185        min_ns: per_op[0],
186        median_ns: per_op[samples / 2],
187        p95_ns: per_op[((samples * 95) / 100).min(samples - 1)],
188        mean_ns: mean,
189        stdev_ns,
190    }
191}
192
193/// Median wall-clock time of a single `op` call over `iters` runs.
194///
195/// The shape `tests/perf_gate.rs` wants: time each call individually, take the
196/// median, assert it under a budget. Budgets should carry generous headroom —
197/// dev (unoptimised) is typically 5–25× slower than release, and a loaded host
198/// adds more — so size the budget off the *release* number times a safety factor
199/// and document the observed dev figure alongside it.
200///
201/// # Panics
202///
203/// When `iters` is zero.
204///
205/// # Examples
206///
207/// ```
208/// use std::time::Duration;
209/// // The median of `iters` whole-op timings — for one expensive thing,
210/// // where `bench`'s inner loop would be the wrong shape.
211/// let d = kevy_bench::time_median(3, || std::thread::sleep(Duration::from_millis(1)));
212/// assert!(d >= Duration::from_millis(1));
213/// ```
214pub fn time_median<F: FnMut()>(iters: usize, mut op: F) -> Duration {
215    assert!(iters > 0, "iters must be non-zero");
216    let mut samples: Vec<Duration> = Vec::with_capacity(iters);
217    for _ in 0..iters {
218        let start = Instant::now();
219        op();
220        samples.push(start.elapsed());
221    }
222    samples.sort_unstable();
223    samples[iters / 2]
224}
225
226impl Stats {
227    /// Print one labelled timing line.
228    ///
229    /// # Examples
230    ///
231    /// ```
232    /// let s = kevy_bench::bench(3, 100, || { std::hint::black_box(1u32); });
233    /// s.report("noop"); // median / p95 / min, one line
234    /// ```
235    pub fn report(&self, label: &str) {
236        println!(
237            "  {label:<30} median {:>8} ns   p95 {:>8} ns   min {:>8} ns",
238            self.median_ns, self.p95_ns, self.min_ns
239        );
240    }
241}
242
243/// Print both timings and the candidate's speedup over the baseline (by median).
244///
245/// `ratio = baseline.median / candidate.median`; > 1 means the candidate is
246/// faster. This is the number the self-host decision is gated on.
247/// # Examples
248///
249/// ```
250/// let a = kevy_bench::bench(3, 100, || { std::hint::black_box(1u32); });
251/// let b = kevy_bench::bench(3, 100, || { std::hint::black_box(1u32); });
252/// let ratio = kevy_bench::compare("base", a, "cand", b);
253/// // > 1 means the candidate is faster; two runs of the same work sit
254/// // near 1, and how near is the noise band that any claim must clear.
255/// assert!(ratio > 0.0 && ratio.is_finite());
256/// ```
257pub fn compare(base_label: &str, base: Stats, cand_label: &str, cand: Stats) -> f64 {
258    base.report(base_label);
259    cand.report(cand_label);
260    let ratio = base.median_ns as f64 / (cand.median_ns.max(1)) as f64;
261    let verdict = if ratio >= 1.0 { "faster" } else { "slower" };
262    println!("  → {cand_label} is {ratio:.2}× {verdict} than {base_label} (median)\n");
263    ratio
264}
265
266// Send and Sync are part of the public contract: a change that loses
267// either fails to compile here rather than in a caller.
268const _: () = {
269    const fn send_sync<T: Send + Sync>() {}
270    send_sync::<Stats>();
271};
272
273#[cfg(test)]
274mod tests {
275    use super::*;
276
277    #[test]
278    fn bench_runs_and_orders_stats() {
279        let s = bench(20, 500, || {
280            black_box(black_box(2u64).wrapping_mul(black_box(3)));
281        });
282        assert_eq!(s.samples, 20);
283        assert_eq!(s.inner, 500);
284        assert!(s.min_ns <= s.median_ns);
285        assert!(s.median_ns <= s.p95_ns);
286    }
287
288    #[test]
289    fn time_median_nonzero() {
290        let d = time_median(50, || {
291            black_box((0..100).sum::<u64>());
292        });
293        assert!(d.as_nanos() < 1_000_000);
294    }
295
296    #[test]
297    fn compare_returns_ratio() {
298        let fast = Stats {
299            samples: 1,
300            inner: 1,
301            min_ns: 10,
302            median_ns: 10,
303            p95_ns: 10,
304            mean_ns: 10,
305            stdev_ns: 0,
306        };
307        let slow = Stats { median_ns: 40, ..fast };
308        // candidate `fast` vs baseline `slow` → 4× faster.
309        let r = compare("slow", slow, "fast", fast);
310        assert!((r - 4.0).abs() < 0.01);
311    }
312}