hyperfine 2.0.0

A command-line benchmarking tool
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

#!/usr/bin/env python
# /// script
# requires-python = ">=3.10"
# dependencies = [
#     "matplotlib",
#     "pyqt6",
#     "numpy",
# ]
# ///

"""
This script shows `hyperfine` benchmark results as a bar plot grouped by command.
Note all the input files must contain results for all commands.
"""

import argparse
import pathlib

import matplotlib.pyplot as plt
import numpy as np

from plot_utils import METRICS, add_metric_argument, load_results


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    add_metric_argument(parser)
    parser.add_argument(
        "files", nargs="+", type=pathlib.Path, help="JSON files with benchmark results"
    )
    parser.add_argument("--title", help="Plot Title")
    parser.add_argument(
        "--benchmark-names", nargs="+", help="Names of the benchmark groups"
    )
    parser.add_argument("-o", "--output", help="Save image to the given filename")

    args = parser.parse_args()
    metric, datasets = load_results(parser, args.files, args.metric)
    metric_info = METRICS[metric]

    commands = None
    data = []
    inputs = []

    if args.benchmark_names:
        assert len(args.files) == len(args.benchmark_names), (
            "Number of benchmark names must match the number of input files."
        )

    for i, (filename, results) in enumerate(zip(args.files, datasets)):
        benchmark_commands = [b.get("name", b["command"]) for b in results]
        if commands is None:
            commands = benchmark_commands
        else:
            assert commands == benchmark_commands, (
                f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}"
            )
        data.append([b["summary"][metric]["mean"] / metric_info.scale for b in results])
        if args.benchmark_names:
            inputs.append(args.benchmark_names[i])
        else:
            inputs.append(filename.stem)

    data = np.transpose(data)
    x = np.arange(len(inputs))  # the label locations
    width = 0.25  # the width of the bars

    fig, ax = plt.subplots(layout="constrained")
    fig.set_figheight(5)
    fig.set_figwidth(10)
    for i, command in enumerate(commands):
        offset = width * (i + 1)
        ax.bar(x + offset, data[i], width, label=command)

    ax.set_xticks(x + 0.5, inputs)
    ax.grid(visible=True, axis="y")

    if args.title:
        plt.title(args.title)
    plt.xlabel("Benchmark")
    plt.ylabel(metric_info.axis_label)
    plt.legend(title="Command")

    if args.output:
        plt.savefig(args.output)
    else:
        plt.show()


if __name__ == "__main__":
    main()