kevy_bench/lib.rs
1//! kevy-bench — a pure-Rust micro-benchmark harness. Zero dependencies.
2//!
3//! kevy is 0-dependency by policy, so we cannot pull `criterion`. This is the
4//! self-hosted equivalent, used by two callers:
5//!
6//! * per-crate exploration benches (`examples/bench_*.rs`) — compare two
7//! implementations head-to-head and print a speedup ratio. This ratio is the
8//! evidence the dep/std self-host evaluation is gated on ("if the candidate
9//! isn't ≥ N× the incumbent, don't self-host it").
10//! * per-crate regression gates (`tests/perf_gate.rs`) — assert a hot path stays
11//! under a documented budget (see [`time_median`]).
12//!
13//! # Why the ratio survives a loaded host
14//!
15//! The campaign that birthed this crate could not get a clean full-system
16//! throughput number because the dev host is permanently busy. The escape is to
17//! measure *components*, not the whole server: run the variants **back-to-back
18//! in one process**. Both pay the same contention in the same window, so the
19//! absolute ns drifts with load but the **ratio between them holds**. Treat
20//! `median_ns` as relative, not absolute, unless the host is known idle.
21//!
22//! # Method
23//!
24//! Each *sample* times `inner` iterations of the closure and divides, so the
25//! per-op figure isn't swamped by `Instant::now()` overhead (~tens of ns) for
26//! sub-microsecond work. We report the **median** sample (robust to the
27//! occasional scheduler hiccup) plus p95 and min.
28//!
29//! ```
30//! use kevy_bench::{bench, black_box};
31//!
32//! let s = bench(50, 1000, || { black_box(1u64 + black_box(2)); });
33//! assert!(s.median_ns < 1_000); // trivial add is nanoseconds
34//! ```
35#![forbid(unsafe_code)]
36#![warn(missing_docs)]
37
38use std::time::{Duration, Instant};
39
40/// Re-exported so benches can fence the optimizer without importing `std::hint`.
41pub use std::hint::black_box;
42
43/// Per-operation timing summary, in nanoseconds. See [`bench()`].
44/// # Examples
45///
46/// ```
47/// let s = kevy_bench::bench(5, 100, || { std::hint::black_box(1u64 + 1); });
48/// assert_eq!((s.samples, s.inner), (5, 100));
49/// // The order the summary always holds, whatever the machine did.
50/// assert!(s.min_ns <= s.median_ns && s.median_ns <= s.p95_ns);
51/// ```
52#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)]
53#[non_exhaustive]
54pub struct Stats {
55 /// Number of samples collected.
56 ///
57 /// # Examples
58 ///
59 /// ```
60 /// let s = kevy_bench::bench(7, 10, || { std::hint::black_box(1u8); });
61 /// assert_eq!(s.samples, 7); // one timed sample per outer repetition
62 /// ```
63 pub samples: usize,
64 /// Iterations timed per sample (the divisor applied to each sample).
65 ///
66 /// # Examples
67 ///
68 /// ```
69 /// let s = kevy_bench::bench(3, 250, || { std::hint::black_box(1u8); });
70 /// // each sample's elapsed time was divided by this
71 /// assert_eq!(s.inner, 250);
72 /// ```
73 pub inner: usize,
74 /// Fastest sample — the least-disturbed run, closest to the true cost.
75 ///
76 /// # Examples
77 ///
78 /// ```
79 /// let s = kevy_bench::bench(20, 100, || { std::hint::black_box(1u8); });
80 /// // no sample was faster, so the median and the mean sit at or above it
81 /// assert!(s.min_ns <= s.median_ns && s.min_ns <= s.mean_ns);
82 /// ```
83 pub min_ns: u64,
84 /// Median sample — the headline figure, robust to occasional hiccups.
85 ///
86 /// # Examples
87 ///
88 /// ```
89 /// use std::time::Duration;
90 /// let s = kevy_bench::bench(3, 1, || std::thread::sleep(Duration::from_micros(200)));
91 /// // a 200 µs sleep never measures shorter than that
92 /// assert!(s.median_ns >= 200_000);
93 /// ```
94 pub median_ns: u64,
95 /// 95th-percentile sample — tail behaviour under scheduler noise.
96 ///
97 /// # Examples
98 ///
99 /// ```
100 /// let s = kevy_bench::bench(40, 100, || { std::hint::black_box(1u8); });
101 /// assert!(s.p95_ns >= s.median_ns); // the tail sits at or above the median
102 /// ```
103 pub p95_ns: u64,
104 /// Mean across all samples.
105 ///
106 /// # Examples
107 ///
108 /// ```
109 /// use std::time::Duration;
110 /// let s = kevy_bench::bench(4, 1, || std::thread::sleep(Duration::from_micros(100)));
111 /// // every sample took at least the sleep, so their average did too
112 /// assert!(s.mean_ns >= 100_000 && s.mean_ns >= s.min_ns);
113 /// ```
114 pub mean_ns: u64,
115 /// Sample standard deviation across samples (Bessel-corrected; 0 when
116 /// `samples == 1`). Reported alongside the median in baseline tables so
117 /// a future delta can be judged against the run-to-run noise band.
118 ///
119 /// # Examples
120 ///
121 /// ```
122 /// let one = kevy_bench::bench(1, 100, || { std::hint::black_box(1u8); });
123 /// assert_eq!(one.stdev_ns, 0); // a single sample has no spread
124 /// ```
125 pub stdev_ns: u64,
126}
127
128/// Run `op` and return per-operation timing stats.
129///
130/// `samples` outer repetitions, each timing `inner` calls of `op` and dividing.
131/// One untimed warm-up sample primes caches/branch predictors first. Pick
132/// `inner` large enough that one sample is comfortably above `Instant`
133/// resolution (≥ a few µs); for ns-scale ops use `inner` in the thousands.
134///
135/// # Panics
136///
137/// When `samples` or `inner` is zero.
138///
139/// # Examples
140///
141/// ```
142/// // `inner` is the divisor: each sample times `inner` iterations and the
143/// // reported figure is per-iteration, so a cheap op is still measurable.
144/// let s = kevy_bench::bench(3, 1_000, || { std::hint::black_box(0u8); });
145/// assert_eq!(s.samples, 3);
146/// assert_eq!(s.inner, 1_000);
147/// ```
148pub fn bench<F: FnMut()>(samples: usize, inner: usize, mut op: F) -> Stats {
149 assert!(samples > 0 && inner > 0, "samples and inner must be non-zero");
150
151 // Warm-up: not recorded.
152 for _ in 0..inner {
153 op();
154 }
155
156 let mut per_op: Vec<u64> = Vec::with_capacity(samples);
157 for _ in 0..samples {
158 let start = Instant::now();
159 for _ in 0..inner {
160 op();
161 }
162 let elapsed = start.elapsed();
163 per_op.push(elapsed.as_nanos() as u64 / inner as u64);
164 }
165 per_op.sort_unstable();
166
167 let sum: u64 = per_op.iter().sum();
168 let mean = sum / samples as u64;
169 let stdev_ns = if samples > 1 {
170 let var: f64 = per_op
171 .iter()
172 .map(|&x| {
173 let d = x as f64 - mean as f64;
174 d * d
175 })
176 .sum::<f64>()
177 / (samples - 1) as f64;
178 var.sqrt() as u64
179 } else {
180 0
181 };
182 Stats {
183 samples,
184 inner,
185 min_ns: per_op[0],
186 median_ns: per_op[samples / 2],
187 p95_ns: per_op[((samples * 95) / 100).min(samples - 1)],
188 mean_ns: mean,
189 stdev_ns,
190 }
191}
192
193/// Median wall-clock time of a single `op` call over `iters` runs.
194///
195/// The shape `tests/perf_gate.rs` wants: time each call individually, take the
196/// median, assert it under a budget. Budgets should carry generous headroom —
197/// dev (unoptimised) is typically 5–25× slower than release, and a loaded host
198/// adds more — so size the budget off the *release* number times a safety factor
199/// and document the observed dev figure alongside it.
200///
201/// # Panics
202///
203/// When `iters` is zero.
204///
205/// # Examples
206///
207/// ```
208/// use std::time::Duration;
209/// // The median of `iters` whole-op timings — for one expensive thing,
210/// // where `bench`'s inner loop would be the wrong shape.
211/// let d = kevy_bench::time_median(3, || std::thread::sleep(Duration::from_millis(1)));
212/// assert!(d >= Duration::from_millis(1));
213/// ```
214pub fn time_median<F: FnMut()>(iters: usize, mut op: F) -> Duration {
215 assert!(iters > 0, "iters must be non-zero");
216 let mut samples: Vec<Duration> = Vec::with_capacity(iters);
217 for _ in 0..iters {
218 let start = Instant::now();
219 op();
220 samples.push(start.elapsed());
221 }
222 samples.sort_unstable();
223 samples[iters / 2]
224}
225
226impl Stats {
227 /// Print one labelled timing line.
228 ///
229 /// # Examples
230 ///
231 /// ```
232 /// let s = kevy_bench::bench(3, 100, || { std::hint::black_box(1u32); });
233 /// s.report("noop"); // median / p95 / min, one line
234 /// ```
235 pub fn report(&self, label: &str) {
236 println!(
237 " {label:<30} median {:>8} ns p95 {:>8} ns min {:>8} ns",
238 self.median_ns, self.p95_ns, self.min_ns
239 );
240 }
241}
242
243/// Print both timings and the candidate's speedup over the baseline (by median).
244///
245/// `ratio = baseline.median / candidate.median`; > 1 means the candidate is
246/// faster. This is the number the self-host decision is gated on.
247/// # Examples
248///
249/// ```
250/// let a = kevy_bench::bench(3, 100, || { std::hint::black_box(1u32); });
251/// let b = kevy_bench::bench(3, 100, || { std::hint::black_box(1u32); });
252/// let ratio = kevy_bench::compare("base", a, "cand", b);
253/// // > 1 means the candidate is faster; two runs of the same work sit
254/// // near 1, and how near is the noise band that any claim must clear.
255/// assert!(ratio > 0.0 && ratio.is_finite());
256/// ```
257pub fn compare(base_label: &str, base: Stats, cand_label: &str, cand: Stats) -> f64 {
258 base.report(base_label);
259 cand.report(cand_label);
260 let ratio = base.median_ns as f64 / (cand.median_ns.max(1)) as f64;
261 let verdict = if ratio >= 1.0 { "faster" } else { "slower" };
262 println!(" → {cand_label} is {ratio:.2}× {verdict} than {base_label} (median)\n");
263 ratio
264}
265
266// Send and Sync are part of the public contract: a change that loses
267// either fails to compile here rather than in a caller.
268const _: () = {
269 const fn send_sync<T: Send + Sync>() {}
270 send_sync::<Stats>();
271};
272
273#[cfg(test)]
274mod tests {
275 use super::*;
276
277 #[test]
278 fn bench_runs_and_orders_stats() {
279 let s = bench(20, 500, || {
280 black_box(black_box(2u64).wrapping_mul(black_box(3)));
281 });
282 assert_eq!(s.samples, 20);
283 assert_eq!(s.inner, 500);
284 assert!(s.min_ns <= s.median_ns);
285 assert!(s.median_ns <= s.p95_ns);
286 }
287
288 #[test]
289 fn time_median_nonzero() {
290 let d = time_median(50, || {
291 black_box((0..100).sum::<u64>());
292 });
293 assert!(d.as_nanos() < 1_000_000);
294 }
295
296 #[test]
297 fn compare_returns_ratio() {
298 let fast = Stats {
299 samples: 1,
300 inner: 1,
301 min_ns: 10,
302 median_ns: 10,
303 p95_ns: 10,
304 mean_ns: 10,
305 stdev_ns: 0,
306 };
307 let slow = Stats { median_ns: 40, ..fast };
308 // candidate `fast` vs baseline `slow` → 4× faster.
309 let r = compare("slow", slow, "fast", fast);
310 assert!((r - 4.0).abs() < 0.01);
311 }
312}