1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
name: Bench
# The performance record (dev/PLAN.md section 18): on every pull request, the
# 5% regression gate; on release pull requests and on demand, the full report
# on every OS, published in docs/PERFORMANCE.md.
on:
pull_request:
branches:
workflow_dispatch:
env:
CARGO_TERM_COLOR: always
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
jobs:
gate:
name: Regression gate (5%, 3 rounds)
if: github.event_name == 'pull_request'
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@v5
with:
path: change
- uses: actions/checkout@v5
with:
ref: ${{ github.base_ref }}
path: base
- uses: dtolnay/rust-toolchain@stable
- name: Gate
shell: bash
run: |
if [ ! -d base/bench ]; then
echo "The base branch has no benchmarks yet: nothing to compare." | tee -a "$GITHUB_STEP_SUMMARY"
exit 0
fi
bash change/bench/gate.sh base change 3 | tee -a "$GITHUB_STEP_SUMMARY"
report:
name: Report (${{ matrix.os }})
# On demand, and on release pull requests (titled "vX.Y.Z - ..."), so
# every version's numbers are recorded.
if: github.event_name == 'workflow_dispatch' || startsWith(github.event.pull_request.title, 'v')
runs-on: ${{ matrix.os }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
os:
defaults:
run:
shell: bash
working-directory: bench
steps:
- uses: actions/checkout@v5
- uses: dtolnay/rust-toolchain@stable
- name: Machine
run: |
{
echo "### ${{ matrix.os }}"
echo
echo '```'
rustc -V
case "$RUNNER_OS" in
Linux) nproc; grep -m1 'model name' /proc/cpuinfo || true; free -h | head -2 ;;
macOS) sysctl -n machdep.cpu.brand_string hw.ncpu hw.memsize ;;
Windows) powershell -NoProfile -Command "(Get-CimInstance Win32_Processor).Name; [Environment]::ProcessorCount" ;;
esac
echo '```'
} | tee -a "$GITHUB_STEP_SUMMARY"
- name: Cold (fresh process per sample, 200 samples)
run: |
cargo build -q --release --bin cold
{ echo; echo '```'; target/release/cold 200; echo '```'; } | tee -a "$GITHUB_STEP_SUMMARY"
env:
CARGO_TARGET_DIR: target
- name: Warm (criterion)
env:
CARGO_TARGET_DIR: target
CRITERION_HOME: ${{ runner.temp }}/criterion
run: |
cargo bench -q --bench readings -- --noplot > /dev/null
{ echo; python3 report.py table "$CRITERION_HOME"; } | tee -a "$GITHUB_STEP_SUMMARY"
- name: Footprint
env:
CARGO_TARGET_DIR: target
run: |
cargo build -q --release --bin baseline --bin footprint
exe=""; [ "$RUNNER_OS" = Windows ] && exe=.exe
size() { wc -c < "target/release/$1$exe"; }
b=$(size baseline); f=$(size footprint)
echo "Every module adds $(( (f - b) / 1024 )) KiB to a stripped binary ($b to $f bytes)." | tee -a "$GITHUB_STEP_SUMMARY"
container:
name: Report (Docker, cgroup v2 limits)
# On demand, and on release pull requests (titled "vX.Y.Z - ..."), so
# every version's numbers are recorded.
if: github.event_name == 'workflow_dispatch' || startsWith(github.event.pull_request.title, 'v')
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@v5
- uses: dtolnay/rust-toolchain@stable
# Built on the runner, run inside a container with real limits: the
# numbers a containerised service sees, with sys-lib reading the
# container's memory and CPUs.
- name: Build
working-directory: bench
env:
CARGO_TARGET_DIR: target
run: |
cargo build -q --release --bin cold
cargo bench -q --bench readings --no-run --message-format=json 2>/dev/null \
| jq -r 'select(.executable != null and .target.name == "readings") | .executable' > readings-path
- name: Run under --memory=1g --cpus=2
shell: bash
run: |
r=$(cat bench/readings-path)
{
echo "### Docker, --memory=1g --cpus=2"
echo; echo '```'
docker run --rm --memory=1g --cpus=2 -v "$PWD/bench/target/release/cold:/cold:ro" ubuntu:24.04 /cold 200
echo '```'; echo
} | tee -a "$GITHUB_STEP_SUMMARY"
mkdir -p "$RUNNER_TEMP/out"
docker run --rm --memory=1g --cpus=2 -e CRITERION_HOME=/out -v "$r:/readings:ro" -v "$RUNNER_TEMP/out:/out" \
ubuntu:24.04 /readings --bench --noplot > /dev/null
python3 bench/report.py table "$RUNNER_TEMP/out" | tee -a "$GITHUB_STEP_SUMMARY"