name: Benchmarks
on:
push:
branches: [main]
pull_request:
branches: [main]
permissions:
contents: write
pull-requests: write
env:
CARGO_TERM_COLOR: always
jobs:
benchmark:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo
uses: Swatinem/rust-cache@v2
- name: Cache critcmp binary
id: cache-critcmp
uses: actions/cache@v4
with:
path: ~/.cargo/bin/critcmp
key: critcmp-${{ runner.os }}
- name: Install critcmp
if: steps.cache-critcmp.outputs.cache-hit != 'true'
run: cargo install critcmp
- name: Build benchmarks
run: cargo bench --all-features --no-run
- name: Run benchmarks
run: cargo bench --all-features -- --save-baseline current --output-format bencher 2>&1 | tee bench-output.txt
- name: Ensure _benchmarks branch exists
if: github.event_name == 'push'
run: git fetch origin _benchmarks:_benchmarks 2>/dev/null || git push origin HEAD:refs/heads/_benchmarks
- name: Update benchmark timeline
if: github.event_name == 'push'
uses: benchmark-action/github-action-benchmark@v1
with:
tool: cargo
output-file-path: bench-output.txt
gh-pages-branch: _benchmarks
benchmark-data-dir-path: timeline
auto-push: true
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Cache baseline for PR comparison
if: github.event_name == 'push'
run: |
critcmp --export current > baseline.json
# Rename the group inside the JSON so critcmp can diff two distinct columns.
sed -i 's/"current"/"main"/' baseline.json
- name: Save baseline to cache
if: github.event_name == 'push'
uses: actions/cache/save@v4
with:
path: baseline.json
key: benchmark-baseline-${{ github.sha }}
- name: Restore baseline
if: github.event_name == 'pull_request'
uses: actions/cache/restore@v4
with:
path: baseline.json
key: benchmark-baseline-${{ github.event.pull_request.base.sha }}
restore-keys: benchmark-baseline-
- name: Compare benchmarks
if: github.event_name == 'pull_request'
id: compare
run: |
if [ ! -f baseline.json ]; then
echo "::warning::No benchmark baseline found. Merge to main to generate one."
exit 0
fi
critcmp --export current > pr-raw.json
# Rename so critcmp sees "pr" vs "main" as two distinct columns.
sed -i 's/"current"/"pr"/' pr-raw.json
mv pr-raw.json pr.json
echo 'result<<EOF' >> $GITHUB_OUTPUT
critcmp baseline.json pr.json >> $GITHUB_OUTPUT
echo 'EOF' >> $GITHUB_OUTPUT
- name: Find existing comment
if: github.event_name == 'pull_request' && steps.compare.outputs.result
uses: peter-evans/find-comment@v3
id: find_comment
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: github-actions[bot]
body-includes: "## Benchmark Comparison"
- name: Post or update comment
if: github.event_name == 'pull_request' && steps.compare.outputs.result
uses: peter-evans/create-or-update-comment@v4
with:
issue-number: ${{ github.event.pull_request.number }}
comment-id: ${{ steps.find_comment.outputs.comment-id }}
edit-mode: replace
body: |
## Benchmark Comparison
<details>
<summary>Click to expand</summary>
```
${{ steps.compare.outputs.result }}
```
</details>