hyperfine 2.0.0-alpha.2

A command-line benchmarking tool
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156

use super::benchmark_result::BenchmarkResult;
use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
use super::Benchmark;

use crate::command::Commands;
use crate::export::ExportManager;
use crate::options::{ExecutorKind, Options};

use anyhow::Result;

pub struct Scheduler<'a> {
    commands: &'a Commands<'a>,
    options: &'a Options,
    export_manager: &'a ExportManager,
    results: Vec<BenchmarkResult>,
}

impl<'a> Scheduler<'a> {
    pub fn new(
        commands: &'a Commands,
        options: &'a Options,
        export_manager: &'a ExportManager,
    ) -> Self {
        Self {
            commands,
            options,
            export_manager,
            results: vec![],
        }
    }

    pub fn run_benchmarks(&mut self) -> Result<()> {
        let mut executor: Box<dyn Executor> = match self.options.executor_kind {
            ExecutorKind::Raw => Box::new(RawExecutor::new(self.options)),
            ExecutorKind::Mock(ref shell) => Box::new(MockExecutor::new(shell.clone())),
            ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)),
        };

        executor.calibrate()?;

        for (number, cmd) in self.commands.iter().enumerate() {
            self.results.push(
                Benchmark::new(number, cmd, self.options, &*executor).run(self.results.first())?,
            );

            // We export results after each individual benchmark, because
            // we would risk losing them if a later benchmark fails.
            self.export_manager.write_results(&self.results, true)?;
        }

        Ok(())
    }

    pub fn final_export(&self) -> Result<()> {
        self.export_manager.write_results(&self.results, false)
    }
}

#[cfg(test)]
fn generate_results(args: &[&'static str]) -> Result<Vec<BenchmarkResult>> {
    use crate::cli::get_cli_arguments;

    let args = ["hyperfine", "--debug-mode", "--style=none"]
        .iter()
        .chain(args);
    let cli_arguments = get_cli_arguments(args);
    let mut options = Options::from_cli_arguments(&cli_arguments)?;

    assert_eq!(options.executor_kind, ExecutorKind::Mock(None));

    let commands = Commands::from_cli_arguments(&cli_arguments)?;
    let export_manager = ExportManager::from_cli_arguments(&cli_arguments, options.metrics[0])?;

    options.validate_against_command_list(&commands)?;

    let mut scheduler = Scheduler::new(&commands, &options, &export_manager);

    scheduler.run_benchmarks()?;
    Ok(scheduler.results)
}

#[test]
fn scheduler_basic() -> Result<()> {
    insta::assert_yaml_snapshot!(generate_results(&["--runs=2", "sleep 0.123", "sleep 0.456"])?, @"
    - command: sleep 0.123
      measurements:
        - time_wall_clock:
            value: 0.123
            unit: second
          time_cpu:
            value: 0
            unit: second
          time_user:
            value: 0
            unit: second
          time_system:
            value: 0
            unit: second
          memory_peak_resident:
            value: 0
            unit: byte
          exit_code: 0
        - time_wall_clock:
            value: 0.123
            unit: second
          time_cpu:
            value: 0
            unit: second
          time_user:
            value: 0
            unit: second
          time_system:
            value: 0
            unit: second
          memory_peak_resident:
            value: 0
            unit: byte
          exit_code: 0
    - command: sleep 0.456
      measurements:
        - time_wall_clock:
            value: 0.456
            unit: second
          time_cpu:
            value: 0
            unit: second
          time_user:
            value: 0
            unit: second
          time_system:
            value: 0
            unit: second
          memory_peak_resident:
            value: 0
            unit: byte
          exit_code: 0
        - time_wall_clock:
            value: 0.456
            unit: second
          time_cpu:
            value: 0
            unit: second
          time_user:
            value: 0
            unit: second
          time_system:
            value: 0
            unit: second
          memory_peak_resident:
            value: 0
            unit: byte
          exit_code: 0
    ");

    Ok(())
}