megafine 0.1.0

A multithreaded command-line benchmarking tool
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

use std::process::ExitStatus;

use crate::stats;

/// The complete outcome of one command execution: its exit status, wall-clock
/// time, and the resource usage reported by `wait4`. Only successful runs are
/// kept in a `BenchmarkResult` (a non-zero exit is surfaced as an error instead).
pub struct Execution {
    /// Exit status of the process.
    pub status: ExitStatus,
    /// Elapsed wall clock time, in seconds.
    pub wall_clock: f64,
    /// Time spent in user mode, in seconds.
    pub time_user: f64,
    /// Time spent in kernel mode, in seconds.
    pub time_system: f64,
    /// Peak resident set size, in bytes.
    pub max_rss: u64,
}

/// All measurements collected for one benchmarked command.
pub struct BenchmarkResult {
    pub label: String,
    pub measurements: Vec<Execution>,
}

impl BenchmarkResult {
    pub fn times(&self, field: impl Fn(&Execution) -> f64) -> Vec<f64> {
        self.measurements.iter().map(field).collect()
    }
}

pub struct NormBenchmark<'a> {
    pub result: &'a BenchmarkResult,
    pub mean: f64,
    /// `mean / reference_mean` (the first command's mean).
    pub ratio: f64,
    /// Propagated uncertainty on `ratio`, if both stddevs are known.
    pub stddev: Option<f64>,
    pub is_reference: bool,
}

/// Compare every result against the first one (the reference). Returns `None` if
/// the reference mean time is zero (relative speed would be meaningless).
pub fn compute(results: &[BenchmarkResult]) -> Option<Vec<NormBenchmark<'_>>> {
    // Each command's wall-clock (mean, stddev), computed once.
    let summaries: Vec<(f64, Option<f64>)> = results
        .iter()
        .map(|r| stats::mean_stddev(&r.times(|x| x.wall_clock)))
        .collect();

    let (ref_mean, ref_stddev) = *summaries.first()?;
    if ref_mean == 0.0 {
        return None;
    }

    let relative = results
        .iter()
        .zip(&summaries)
        .enumerate()
        .map(|(idx, (result, &(mean, stddev)))| {
            let ratio = mean / ref_mean;
            let stddev = match (stddev, ref_stddev) {
                (Some(stddev), Some(ref_stddev)) => {
                    // same formula than hyperfine:
                    // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulae
                    // for f=A/B
                    // with σ_{AB} assumed to be = 0

                    Some(ratio * ((stddev / mean).powi(2) + (ref_stddev / ref_mean).powi(2)).sqrt())
                }
                _ => None,
            };
            NormBenchmark {
                result,
                mean,
                ratio,
                stddev,
                is_reference: idx == 0,
            }
        })
        .collect();

    Some(relative)
}