Skip to main content

sys_lib/
bench.rs

1//! The benchmark environment: what a result needs printed next to it so it can
2//! be trusted and repeated.
3//!
4//! Whether the CPU frequency could wander (the governor, turbo), whether the
5//! machine was busy, on battery, throttled, or a VM, and whether the timer is
6//! stable. Benchmark harnesses print it at the start of a run. Read-only, like
7//! the [`readiness`](crate::readiness) report.
8//!
9//! ```
10//! use sys_lib::bench;
11//!
12//! let env = bench::environment();
13//! println!("{env}");
14//! if !env.is_reproducible() {
15//!     eprintln!("results may be noisy: {:?}", env.warnings);
16//! }
17//! ```
18
19use core::{fmt, time::Duration};
20
21use crate::cpu::Load;
22
23/// The machine, as a benchmark sees it.
24#[derive(Clone, Debug, PartialEq)]
25#[non_exhaustive]
26pub struct Environment {
27    /// The CPU frequency governor (`"performance"`, `"powersave"`,
28    /// `"schedutil"`), or the Windows power plan's name.
29    pub governor: Option<String>,
30    /// Turbo or boost is enabled.
31    pub turbo: Option<bool>,
32    /// Simultaneous multithreading (hyper-threading) is active.
33    pub smt: Option<bool>,
34    /// The time-stamp counter runs at a constant rate (x86 only).
35    pub invariant_tsc: Option<bool>,
36    /// The kernel's clock source (Linux): `"tsc"`, `"hpet"`.
37    pub clock_source: Option<String>,
38    /// The CPU was throttled for heat or power: since boot on Linux (the
39    /// kernel's counters), right now on Windows.
40    pub throttled: Option<bool>,
41    /// The CPUs kept away from the scheduler (Linux `isolcpus`, `nohz_full`),
42    /// as the kernel lists them: `"2-3"`.
43    pub isolated_cpus: Option<String>,
44    /// Address-space randomisation is on: run-to-run layout noise.
45    pub aslr: Option<bool>,
46    /// On mains power.
47    pub on_ac: Option<bool>,
48    /// The load average, where the OS keeps one.
49    pub load: Option<Load>,
50    /// Running in a virtual machine.
51    pub virtualized: bool,
52    /// What makes results noisy, in the order of the variants.
53    pub warnings: Vec<Warning>,
54}
55
56/// Something that makes benchmark results noisy.
57#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
58#[non_exhaustive]
59pub enum Warning {
60    /// The governor or power plan lowers the frequency when idle.
61    PowerSaving,
62    /// Turbo is on: the frequency depends on temperature and on other cores.
63    TurboEnabled,
64    /// The CPU has been throttled.
65    Throttled,
66    /// On battery, where the OS saves power.
67    OnBattery,
68    /// Other work is running: the load is over a quarter of the CPUs.
69    Busy,
70    /// A VM: other guests share the hardware.
71    Virtualized,
72    /// The clock is slow to read or drifts with the frequency.
73    UnstableClock,
74}
75
76impl Warning {
77    /// A short lowercase name: `"power-saving"`.
78    #[must_use]
79    pub const fn name(self) -> &'static str {
80        match self {
81            Self::PowerSaving => "power-saving",
82            Self::TurboEnabled => "turbo",
83            Self::Throttled => "throttled",
84            Self::OnBattery => "on-battery",
85            Self::Busy => "busy",
86            Self::Virtualized => "virtualized",
87            Self::UnstableClock => "unstable-clock",
88        }
89    }
90}
91
92/// What a platform reports for [`environment`]; the rest comes from the
93/// public modules.
94#[derive(Clone, Debug, Default, PartialEq, Eq)]
95pub(crate) struct BenchFacts {
96    pub(crate) governor: Option<String>,
97    /// The governor or plan lets the frequency drop (anything but Linux
98    /// `performance` or a Windows high-performance plan).
99    pub(crate) power_saving: Option<bool>,
100    pub(crate) turbo: Option<bool>,
101    pub(crate) smt: Option<bool>,
102    pub(crate) clock_source: Option<String>,
103    pub(crate) throttled: Option<bool>,
104    pub(crate) isolated_cpus: Option<String>,
105    pub(crate) aslr: Option<bool>,
106}
107
108/// The load per CPU above which the machine counts as busy.
109const BUSY: f32 = 0.25;
110/// How long [`environment`] samples the CPU where there is no load average.
111const BUSY_WINDOW: Duration = Duration::from_millis(100);
112
113/// Clock sources that are slow to read or not constant-rate.
114pub(crate) fn slow_clock(source: &str) -> bool {
115    matches!(source, "hpet" | "acpi_pm" | "jiffies" | "pit" | "xen")
116}
117
118#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
119fn invariant_tsc() -> Option<bool> {
120    use std::sync::OnceLock;
121    static TSC: OnceLock<Option<bool>> = OnceLock::new();
122    *TSC.get_or_init(crate::sys::x86_invariant_tsc)
123}
124
125#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
126fn invariant_tsc() -> Option<bool> {
127    None
128}
129
130/// The benchmark environment, read now. Never fails: each value the OS does
131/// not answer is `None`. Where the OS keeps no load average (Windows), the CPU
132/// is sampled for 100 ms to tell whether it is busy, so the call takes that
133/// long; elsewhere it costs 0.1 to 1 ms [est.].
134#[must_use]
135pub fn environment() -> Environment {
136    let facts = crate::sys::bench_facts();
137    let load = crate::cpu::load().ok().flatten();
138    let busy = match load {
139        Some(l) => Some(l.per_cpu().one > BUSY),
140        None => crate::cpu::usage_over(BUSY_WINDOW)
141            .ok()
142            .map(|u| u.used().value() > BUSY * 100.0),
143    };
144    let on_ac = crate::power::on_ac().ok().flatten();
145    assess(
146        facts,
147        invariant_tsc(),
148        load,
149        busy,
150        on_ac,
151        crate::virt::is_vm(),
152    )
153}
154
155fn assess(
156    facts: BenchFacts,
157    invariant_tsc: Option<bool>,
158    load: Option<Load>,
159    busy: Option<bool>,
160    on_ac: Option<bool>,
161    virtualized: bool,
162) -> Environment {
163    let mut warnings = Vec::new();
164    if facts.power_saving == Some(true) {
165        warnings.push(Warning::PowerSaving);
166    }
167    if facts.turbo == Some(true) {
168        warnings.push(Warning::TurboEnabled);
169    }
170    if facts.throttled == Some(true) {
171        warnings.push(Warning::Throttled);
172    }
173    if on_ac == Some(false) {
174        warnings.push(Warning::OnBattery);
175    }
176    if busy == Some(true) {
177        warnings.push(Warning::Busy);
178    }
179    if virtualized {
180        warnings.push(Warning::Virtualized);
181    }
182    if invariant_tsc == Some(false) || facts.clock_source.as_deref().is_some_and(slow_clock) {
183        warnings.push(Warning::UnstableClock);
184    }
185    Environment {
186        governor: facts.governor,
187        turbo: facts.turbo,
188        smt: facts.smt,
189        invariant_tsc,
190        clock_source: facts.clock_source,
191        throttled: facts.throttled,
192        isolated_cpus: facts.isolated_cpus,
193        aslr: facts.aslr,
194        on_ac,
195        load,
196        virtualized,
197        warnings,
198    }
199}
200
201impl Environment {
202    /// No warning that makes results noisy.
203    #[must_use]
204    pub fn is_reproducible(&self) -> bool {
205        self.warnings.is_empty()
206    }
207
208    fn pairs<W: fmt::Write>(&self, out: &mut W) -> fmt::Result {
209        fn flag(v: Option<bool>) -> &'static str {
210            match v {
211                Some(true) => "yes",
212                Some(false) => "no",
213                None => "unknown",
214            }
215        }
216        let load = self
217            .load
218            .map_or_else(|| "unknown".to_owned(), |l| format!("{:.2}", l.one));
219        let warnings = if self.warnings.is_empty() {
220            "none".to_owned()
221        } else {
222            let names: Vec<&str> = self.warnings.iter().map(|w| w.name()).collect();
223            names.join(",")
224        };
225        let items = [
226            ("governor", self.governor.as_deref().unwrap_or("unknown")),
227            ("turbo", flag(self.turbo)),
228            ("smt", flag(self.smt)),
229            ("invariant_tsc", flag(self.invariant_tsc)),
230            (
231                "clock_source",
232                self.clock_source.as_deref().unwrap_or("unknown"),
233            ),
234            ("throttled", flag(self.throttled)),
235            (
236                "isolated_cpus",
237                self.isolated_cpus.as_deref().unwrap_or("none"),
238            ),
239            ("aslr", flag(self.aslr)),
240            ("on_ac", flag(self.on_ac)),
241            ("load", &load),
242            ("virtualized", flag(Some(self.virtualized))),
243            ("warnings", &warnings),
244            ("reproducible", flag(Some(self.is_reproducible()))),
245        ];
246        for (i, (key, value)) in items.iter().enumerate() {
247            if i > 0 {
248                out.write_char(' ')?;
249            }
250            crate::fmt::kv(out, key, value)?;
251        }
252        Ok(())
253    }
254
255    /// One line of `key=value` pairs; values with spaces are quoted.
256    ///
257    /// # Errors
258    ///
259    /// Whatever `out` returns.
260    pub fn write_kv(&self, out: &mut impl fmt::Write) -> fmt::Result {
261        self.pairs(out)
262    }
263
264    /// One JSON object; unknown values are `null`.
265    ///
266    /// # Errors
267    ///
268    /// Whatever `out` returns.
269    pub fn write_json(&self, out: &mut impl fmt::Write) -> fmt::Result {
270        fn text(out: &mut impl fmt::Write, v: Option<&str>) -> fmt::Result {
271            match v {
272                Some(s) => crate::fmt::json_str(out, s),
273                None => out.write_str("null"),
274            }
275        }
276        fn flag(out: &mut impl fmt::Write, v: Option<bool>) -> fmt::Result {
277            match v {
278                Some(b) => write!(out, "{b}"),
279                None => out.write_str("null"),
280            }
281        }
282        out.write_str("{\"governor\":")?;
283        text(out, self.governor.as_deref())?;
284        out.write_str(",\"turbo\":")?;
285        flag(out, self.turbo)?;
286        out.write_str(",\"smt\":")?;
287        flag(out, self.smt)?;
288        out.write_str(",\"invariant_tsc\":")?;
289        flag(out, self.invariant_tsc)?;
290        out.write_str(",\"clock_source\":")?;
291        text(out, self.clock_source.as_deref())?;
292        out.write_str(",\"throttled\":")?;
293        flag(out, self.throttled)?;
294        out.write_str(",\"isolated_cpus\":")?;
295        text(out, self.isolated_cpus.as_deref())?;
296        out.write_str(",\"aslr\":")?;
297        flag(out, self.aslr)?;
298        out.write_str(",\"on_ac\":")?;
299        flag(out, self.on_ac)?;
300        out.write_str(",\"load\":")?;
301        match self.load {
302            Some(l) => write!(out, "[{},{},{}]", l.one, l.five, l.fifteen)?,
303            None => out.write_str("null")?,
304        }
305        write!(out, ",\"virtualized\":{},\"warnings\":[", self.virtualized)?;
306        for (i, w) in self.warnings.iter().enumerate() {
307            if i > 0 {
308                out.write_char(',')?;
309            }
310            crate::fmt::json_str(out, w.name())?;
311        }
312        write!(out, "],\"reproducible\":{}}}", self.is_reproducible())
313    }
314}
315
316impl fmt::Display for Environment {
317    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
318        self.pairs(f)
319    }
320}
321
322#[cfg(test)]
323mod tests {
324    use super::*;
325
326    fn quiet() -> BenchFacts {
327        BenchFacts {
328            governor: Some("performance".into()),
329            power_saving: Some(false),
330            turbo: Some(false),
331            smt: Some(true),
332            clock_source: Some("tsc".into()),
333            throttled: Some(false),
334            isolated_cpus: None,
335            aslr: Some(true),
336        }
337    }
338
339    #[test]
340    fn test_bench_quiet_machine_is_reproducible() {
341        let env = assess(quiet(), Some(true), None, Some(false), Some(true), false);
342        assert!(env.is_reproducible(), "{:?}", env.warnings);
343    }
344
345    #[test]
346    fn test_bench_every_warning() {
347        let facts = BenchFacts {
348            governor: Some("powersave".into()),
349            power_saving: Some(true),
350            turbo: Some(true),
351            throttled: Some(true),
352            clock_source: Some("hpet".into()),
353            ..BenchFacts::default()
354        };
355        let env = assess(facts, Some(true), None, Some(true), Some(false), true);
356        assert_eq!(
357            env.warnings,
358            vec![
359                Warning::PowerSaving,
360                Warning::TurboEnabled,
361                Warning::Throttled,
362                Warning::OnBattery,
363                Warning::Busy,
364                Warning::Virtualized,
365                Warning::UnstableClock
366            ]
367        );
368        assert!(!env.is_reproducible());
369        let env = assess(quiet(), Some(false), None, None, None, false);
370        assert_eq!(env.warnings, vec![Warning::UnstableClock]);
371    }
372
373    #[test]
374    fn test_bench_unknown_is_not_a_warning() {
375        let env = assess(BenchFacts::default(), None, None, None, None, false);
376        assert!(env.is_reproducible());
377        let mut j = String::new();
378        env.write_json(&mut j).unwrap();
379        assert!(j.contains("\"governor\":null") && j.ends_with("\"reproducible\":true}"));
380        assert!(env.to_string().contains("warnings=none"));
381    }
382
383    #[test]
384    fn test_bench_live_outputs() {
385        let env = environment();
386        let mut kv = String::new();
387        env.write_kv(&mut kv).unwrap();
388        assert_eq!(kv, env.to_string());
389        assert!(kv.starts_with("governor=") && kv.contains(" reproducible="));
390        let mut j = String::new();
391        env.write_json(&mut j).unwrap();
392        assert!(j.starts_with("{\"governor\":") && j.ends_with('}'));
393    }
394}