1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
name: Bench
# Two-tier regression philosophy:
#
# `water bench` enforces deterministic counter budgets in-process (rebuild
# ratio, scene/clip/gpu-surface layer counts, cache misses, and similar
# environment-independent numbers). A budget violation is a hard failure —
# `water bench` exits nonzero and this job fails — because those counters
# don't depend on runner hardware or how busy the machine happens to be.
#
# Wall-clock timings are the opposite: on a shared GitHub-hosted runner
# they're noisy from one run to the next, so a hard pass/fail gate on wall
# time would be flaky by construction. Wall-clock trends are instead tracked
# over time on the `gh-pages` branch (via benchmark-action/github-action-
# benchmark below) and surfaced as alert comments when a trend crosses a
# threshold — informational, not a merge blocker.
#
# The trend series is sampled once a day rather than once per push. A daily
# point is enough to see a trend in numbers this noisy, and three bench
# matrix legs per push were paying for runner time and cache churn that the
# per-push counter gate on pull requests already covers.
on:
pull_request:
branches:
paths-ignore:
- "**.md"
- "docs/**"
# Off-peak UTC, staggered against the other scheduled workflows.
schedule:
- cron: "41 5 * * *"
workflow_dispatch:
concurrency:
group: bench-${{ github.ref }}
cancel-in-progress: true
# Read-only by default; only the scheduled `bench-history` job elevates to
# `contents: write`, which it needs to commit benchmark history to `gh-pages`.
permissions:
contents: read
env:
CARGO_TERM_COLOR: always
# rust-cache is the only cache layer (see the note in ci.yml).
CARGO_INCREMENTAL: 0
jobs:
bench:
name: Bench (${{ matrix.os }})
runs-on: ${{ matrix.os }}
timeout-minutes: 120
strategy:
fail-fast: false
# macOS runners have a real (paravirtualized Metal) GPU and run the full
# default measurement shape. The Linux and Windows runners rasterize in
# software (llvmpipe / WARP), where a heavy scene runs at seconds per
# frame — the full shape blew straight through nextest's terminate
# ceiling there. A reduced shape keeps those legs meaningful (the
# counter budgets are shape-independent, and the wall-clock series stays
# self-consistent per OS over time) while fitting the job budget.
matrix:
include:
- os: ubuntu-latest
bench-shape: --warmups 2 --samples 24 --repetitions 3
- os: macos-latest
bench-shape: ""
- os: windows-latest
bench-shape: --warmups 2 --samples 24 --repetitions 3
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- uses: dtolnay/rust-toolchain@stable
# GPU offscreen benches run here, same as test.yml's GPU snapshot tests,
# so the Linux leg needs the Vulkan ICD loader for llvmpipe/lavapipe.
- name: Install native dependencies
if: runner.os == 'Linux'
uses: ./.github/actions/setup-linux-deps
with:
gpu: "true"
- name: Install DirectX Shader Compiler
if: runner.os == 'Windows'
uses: tracel-ai/github-actions/install-dxc@295f2fdbae5271f4f8ad56d634e4ff6a95daafc8
- name: Verify DirectX Shader Compiler
if: runner.os == 'Windows'
run: dxc --version
- uses: Swatinem/rust-cache@v2
with:
shared-key: bench-${{ matrix.os }}
cache-on-failure: true
- uses: taiki-e/install-action@nextest
- name: Build water CLI
run: cargo build -p waterui-cli
- name: Resolve water binary path
shell: bash
run: |
if [ "$RUNNER_OS" = "Windows" ]; then
echo "WATER_BIN=target/debug/water.exe" >> "$GITHUB_ENV"
else
echo "WATER_BIN=target/debug/water" >> "$GITHUB_ENV"
fi
- name: Bench examples/stress
shell: bash
run: |
"$WATER_BIN" bench --path examples/stress ${{ matrix.bench-shape }} \
--report-dir bench-reports/stress \
--gha bench-gha-stress.json
- name: Bench examples/list
shell: bash
run: |
"$WATER_BIN" bench --path examples/list ${{ matrix.bench-shape }} \
--report-dir bench-reports/list \
--gha bench-gha-list.json
# `water bench --gha` writes one customSmallerIsBetter JSON array per
# suite; github-action-benchmark below wants a single array, so flatten
# the two suite files into one before handing it off.
- name: Merge per-suite benchmark JSON
shell: bash
run: jq -s 'add' bench-gha-stress.json bench-gha-list.json > bench-gha.json
- name: Upload bench reports
if: always()
uses: actions/upload-artifact@v4
with:
name: bench-reports-${{ matrix.os }}
path: bench-reports/**
if-no-files-found: ignore
retention-days: 14
- name: Upload benchmark JSON
uses: actions/upload-artifact@v4
with:
name: bench-gha-${{ matrix.os }}
path: bench-gha.json
retention-days: 14
# Store and compare wall-clock history only on the schedule. A pull_request
# from a fork has no write access to push `gh-pages` anyway, and the hard
# gate for pull requests is already the deterministic budgets `water bench`
# itself enforced in the matrix above (a violation already failed that job).
# A `workflow_dispatch` run is excluded too: it can be started from any
# branch, and one branch's numbers landing in the trend series would be
# indistinguishable from a regression on the default branch.
#
# This runs as a single job AFTER the matrix — not as a per-leg step —
# because github-action-benchmark commits to `gh-pages` and three matrix
# legs pushing concurrently race each other (non-fast-forward push
# failures). One job storing the three per-OS series sequentially cannot
# race, and keeps the expensive bench legs fully parallel.
bench-history:
name: Bench history
needs: bench
if: github.event_name == 'schedule'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: write
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
pattern: bench-gha-*
path: bench-gha
- name: Track benchmark history (ubuntu)
# A leg that failed uploads no JSON. Recording the platforms that did
# produce results is the point; erroring on the ones that did not
# would let one platform's breakage discard everyone else's history.
if: hashFiles('bench-gha/bench-gha-ubuntu-latest/bench-gha.json') != ''
uses: benchmark-action/github-action-benchmark@v1
with:
name: WaterUI Bench (ubuntu-latest)
tool: customSmallerIsBetter
output-file-path: bench-gha/bench-gha-ubuntu-latest/bench-gha.json
benchmark-data-dir-path: dev/bench
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: true
comment-on-alert: true
alert-threshold: "150%"
# Wall-clock on a shared runner is noisy; the hard gate is the
# deterministic counter budgets water bench enforces in-process
# (see the file header comment), which already failed the bench job
# on violation. This is a trend alert, not a merge gate.
fail-on-alert: false
- name: Track benchmark history (macos)
# A leg that failed uploads no JSON. Recording the platforms that did
# produce results is the point; erroring on the ones that did not
# would let one platform's breakage discard everyone else's history.
if: hashFiles('bench-gha/bench-gha-macos-latest/bench-gha.json') != ''
uses: benchmark-action/github-action-benchmark@v1
with:
name: WaterUI Bench (macos-latest)
tool: customSmallerIsBetter
output-file-path: bench-gha/bench-gha-macos-latest/bench-gha.json
benchmark-data-dir-path: dev/bench
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: true
comment-on-alert: true
alert-threshold: "150%"
fail-on-alert: false
- name: Track benchmark history (windows)
# A leg that failed uploads no JSON. Recording the platforms that did
# produce results is the point; erroring on the ones that did not
# would let one platform's breakage discard everyone else's history.
if: hashFiles('bench-gha/bench-gha-windows-latest/bench-gha.json') != ''
uses: benchmark-action/github-action-benchmark@v1
with:
name: WaterUI Bench (windows-latest)
tool: customSmallerIsBetter
output-file-path: bench-gha/bench-gha-windows-latest/bench-gha.json
benchmark-data-dir-path: dev/bench
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: true
comment-on-alert: true
alert-threshold: "150%"
fail-on-alert: false