hyperfine 2.0.0

A command-line benchmarking tool
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

#!/usr/bin/env python
# /// script
# requires-python = ">=3.10"
# dependencies = [
#     "pyqt6",
#     "matplotlib",
#     "numpy",
# ]
# ///

"""This program shows `hyperfine` benchmark results in a sequential way
in order to debug possible background interference, caching effects,
thermal throttling and similar effects.
"""

import argparse

import matplotlib.pyplot as plt
import numpy as np

from plot_utils import METRICS, add_metric_argument, load_results


def moving_average(values, num_runs):
    values_padded = np.pad(
        values, (num_runs // 2, num_runs - 1 - num_runs // 2), mode="edge"
    )
    kernel = np.ones(num_runs) / num_runs
    return np.convolve(values_padded, kernel, mode="valid")


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    add_metric_argument(parser)
    parser.add_argument("file", help="JSON file with benchmark results")
    parser.add_argument("--title", help="Plot Title")
    parser.add_argument("-o", "--output", help="Save image to the given filename.")
    parser.add_argument(
        "-w",
        "--moving-average-width",
        type=int,
        metavar="num_runs",
        help="Width of the moving-average window (default: N/5)",
    )
    parser.add_argument(
        "--no-moving-average",
        action="store_true",
        help="Do not show moving average curve",
    )

    args = parser.parse_args()
    metric, [results] = load_results(parser, [args.file], args.metric)
    metric_info = METRICS[metric]

    for result in results:
        values = [
            m[metric]["value"] / metric_info.scale for m in result["measurements"]
        ]
        num = len(values)
        nums = range(num)

        plt.scatter(x=nums, y=values, marker=".")
        plt.ylim([0, None])
        plt.xlim([-1, num])

        if not args.no_moving_average:
            moving_average_width = (
                max(1, num // 5)
                if args.moving_average_width is None
                else args.moving_average_width
            )

            average = moving_average(values, moving_average_width)
            plt.plot(nums, average, "-")

    if args.title:
        plt.title(args.title)

    legend = []
    for result in results:
        legend.append(result.get("name", result["command"]))
        if not args.no_moving_average:
            legend.append("moving average")
    plt.legend(legend)

    plt.ylabel(metric_info.axis_label)

    if args.output:
        plt.savefig(args.output)
    else:
        plt.show()


if __name__ == "__main__":
    main()