1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
name: Bench
# Clone benchmark — builds kache from THIS ref and runs the cross-clone
# cold/warm scenarios (Firefox, Substrate) against one shared cache, then
# uploads the reports. This is the heavy, hours-long, tens-of-GB-disk job the
# bench engine's doc-comment says is "intentionally NOT wired into [PR] CI": it
# lives in its own workflow so it never blocks a push/PR.
#
# Triggers:
# - schedule : nightly, runs BOTH scenarios.
# - dispatch : manual, pick a scenario (default all) and pass extra `just
# bench` flags (e.g. `--skip-clone`, `--retry`).
#
# Runs on the kunobi self-hosted Linux ARC runner: the `kunobi-runners`
# gha-runner-scale-set in tenant-int-dev/zur1-builder1. A gha-runner-scale-set
# is targeted by its INSTALLATION NAME as a single `runs-on` label
# (`runs-on: kunobi-runners`) — NOT by a `[self-hosted, Linux, X64, …]` label
# set (that scheme only matches the legacy multi-label runners, e.g.
# kunobi-windows). These are EPHEMERAL pods (fresh per job,
# `_work` is an emptyDir): there is no stale tool state to isolate, but there is
# also no persistent clone-ref cache, so every run re-clones the upstream repo,
# and Firefox's `./mach bootstrap` re-provisions ~/.mozbuild each time. The image
# is minimal, so the base build toolchain is installed below.
on:
schedule:
# ~04:37 UTC nightly. Off the top-of-hour to dodge GitHub's cron congestion.
- cron: "37 4 * * *"
workflow_dispatch:
inputs:
scenario:
description: "Which scenario(s) to run"
type: choice
options:
default: all
extra_args:
description: "Extra args appended to `just bench <profile>` (e.g. --skip-clone, --retry)"
type: string
default: ""
concurrency:
# One bench run per scenario+ref at a time; don't cancel a multi-hour run.
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
permissions:
contents: read
env:
CARGO_TERM_COLOR: always
jobs:
# Build the bench matrix from the trigger. Selection must happen HERE, not in
# a job-level `if` on `bench`: the `matrix` context is NOT available in a
# job-level `if` (it is evaluated before the matrix is expanded, so it reads
# empty) — a dispatch picking a single scenario would otherwise run zero arms.
# This tiny job runs on a GitHub-hosted runner so a skipped scenario never
# claims a self-hosted kunobi runner. schedule + `all` -> both; otherwise the
# one picked scenario. Per-arm timeout/min_free_gb travel with each entry.
plan:
name: Plan scenarios
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.pick.outputs.matrix }}
steps:
- id: pick
shell: bash
env:
# schedule has no inputs -> treat as `all`; dispatch honours the input.
SCENARIO: ${{ github.event_name == 'schedule' && 'all' || github.event.inputs.scenario }}
run: |
# Each arm carries: name (display + artifact label), cmd (the `just`
# recipe + profile to run), dir (scratch subdir under tmp/bench to
# upload), timeout (minutes), min_free_gb (precheck floor — fail the
# arm if free disk is below this before we start, since disk-full
# mid-build is the worst failure mode; emptyDir is node-backed, so the
# precheck also catches a too-small node).
#
# Smaller than Firefox but still tens of minutes cold.
substrate='{"name":"substrate","cmd":"bench substrate","dir":"bench-substrate","timeout":180,"min_free_gb":60}'
# Full cold Firefox build is the long pole — hours.
firefox='{"name":"firefox","cmd":"bench firefox","dir":"bench-firefox","timeout":360,"min_free_gb":120}'
# Same Firefox shape through sccache for a side-by-side comparison.
firefox_sccache='{"name":"firefox-sccache","cmd":"bench-sccache firefox","dir":"bench-firefox-sccache","timeout":360,"min_free_gb":120}'
# Almost-pure C/C++ CMake build (X86-only Release) — lighter than Firefox.
llvm='{"name":"llvm","cmd":"bench llvm","dir":"bench-llvm","timeout":240,"min_free_gb":100}'
case "$SCENARIO" in
substrate) inc="[$substrate]" ;;
firefox) inc="[$firefox]" ;;
firefox-sccache) inc="[$firefox_sccache]" ;;
llvm) inc="[$llvm]" ;;
*) inc="[$substrate,$firefox,$firefox_sccache,$llvm]" ;;
esac
echo "matrix={\"include\":$inc}" >> "$GITHUB_OUTPUT"
bench:
name: Bench (${{ matrix.name }})
needs: plan
# gha-runner-scale-set: match by the scale-set's installation name (single
# label), not a `[self-hosted, …]` set. See header comment.
runs-on: kunobi-runners
timeout-minutes: ${{ matrix.timeout }}
strategy:
# The arms run concurrently when the scale set has spare runners
# (maxRunners >= 2); otherwise they serialize. Neither aborts the other.
# Matrix is generated by `plan` (see above) — selection can't live in a
# job-level `if` because `matrix` isn't available there.
fail-fast: false
matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v6
timeout-minutes: 10
# The ARC runner image is minimal. Install the base toolchain the bench
# needs around the build: substrate links rocksdb/secp256k1 and needs
# protoc + clang + cmake + openssl headers; Firefox's `./mach bootstrap`
# pulls its own clang/rust toolchain but still needs python3 + common
# archive/file utilities present first; the LLVM scenario drives CMake
# with the Ninja generator, so it needs ninja-build. git/curl are needed by
# the clone and by mise. (Firefox also installs further packages itself via
# bootstrap, which uses passwordless sudo on this image.)
- name: Install build prerequisites (apt)
run: |
export DEBIAN_FRONTEND=noninteractive
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential pkg-config \
protobuf-compiler clang libclang-dev cmake ninja-build \
libssl-dev \
git curl wget ca-certificates file unzip bzip2 xz-utils \
python3 python3-dev python3-pip
# mise installs rust (per rust-toolchain.toml) + just (the bench recipe is
# `just bench`) + sccache (the firefox-sccache arm wires sccache as the
# compiler cache; the version is pinned by mise.toml). On the ephemeral
# Linux pod there's no stale state and no competing system Rust, so
# mise-action puts the tools on PATH directly — none of the macOS
# isolation/PATH workarounds from ci.yml are needed.
- name: Install tools via mise
uses: jdx/mise-action@v4
with:
# Pin mise CLI — see ci.yml (v2026.6.10 regression).
version: 2026.6.9
install_args: --force rust github:casey/just github:mozilla/sccache
cache: false
reshim: false
# Fail fast on a full disk — disk-full mid-build is the worst failure mode
# on the emptyDir-backed work volume.
- name: Free-disk precheck
run: |
avail_gb=$(df -BG --output=avail . | awk 'NR==2 {gsub(/G/,"",$1); print $1}')
echo "Free disk: ${avail_gb} GB (need >= ${{ matrix.min_free_gb }} GB)"
if [ "${avail_gb:-0}" -lt "${{ matrix.min_free_gb }}" ]; then
echo "::error::Insufficient free disk (${avail_gb} GB < ${{ matrix.min_free_gb }} GB) for the ${{ matrix.name }} bench"
exit 1
fi
- name: Run bench (${{ matrix.name }})
# `just bench` builds release kache + the scenario engine, then runs cold
# then warm against one cache and writes reports under
# tmp/bench/bench-<scenario>/. The engine self-diagnoses: a non-zero exit
# means the run did not validly exercise kache, so we surface it as a
# failure rather than uploading a misleading green report.
run: |
rustc --version && cargo --version && just --version
# EXTRA_ARGS is passed via env (not interpolated into the command line)
# so a dispatch input can't inject shell. Unquoted on purpose: the
# bench flags (e.g. `--skip-clone --retry`) must word-split. matrix.cmd
# is the full recipe + profile, e.g. "bench firefox" or
# "bench-sccache firefox".
# shellcheck disable=SC2086
just ${{ matrix.cmd }} $EXTRA_ARGS
env:
# Empty on schedule; the dispatch input otherwise (`--skip-clone`, …).
EXTRA_ARGS: ${{ github.event.inputs.extra_args }}
# The bench measures kache itself; it must not run under a wrapper.
RUSTC_WRAPPER: ""
# Unset the toolchain pin mise exports. The cloned projects pin their
# OWN Rust via rust-toolchain.toml (polkadot-sdk -> 1.93.0); a forced
# RUSTUP_TOOLCHAIN overrides that and breaks substrate's wasm-runtime
# build. With it cleared, rustup honours each tree's rust-toolchain.toml
# (kache builds with its pin, substrate with 1.93.0).
RUSTUP_TOOLCHAIN: ""
- name: Upload bench reports
# always(): a failed/leaky run's reports + build/wrapper logs are the most
# useful artifacts to inspect. NO hashFiles() gate: hashFiles globs the
# WHOLE scratch tree (multi-GB clone worktrees, .git, the engine's live
# unix socket) and THROWS on the socket/special files — which errored the
# upload even on a valid run and discarded build-cold.log. `if-no-files-found:
# warn` already makes an empty match (e.g. after a precheck failure) a no-op.
if: always()
uses: actions/upload-artifact@v7
with:
name: bench-${{ matrix.name }}
# Reports, per-phase JSON/MD, key-diff, ALL top-level logs
# (build-cold/build-warm/wrapper-* — the build logs hold the actual
# compile error on a failed run), and sccache's *.sccache-adv.txt stats
# for the sccache arm. These are flat globs in the scratch dir, so they
# exclude the multi-GB clone worktrees / objdirs / cache.
path: |
tmp/bench/${{ matrix.dir }}/*.json
tmp/bench/${{ matrix.dir }}/*.md
tmp/bench/${{ matrix.dir }}/report-*.json
tmp/bench/${{ matrix.dir }}/report-*.md
tmp/bench/${{ matrix.dir }}/key-diff.*
tmp/bench/${{ matrix.dir }}/*.log
tmp/bench/${{ matrix.dir }}/*.txt
retention-days: 30
if-no-files-found: warn