name: Bench
on:
pull_request:
branches: [main]
merge_group:
types: [checks_requested]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
jobs:
bench-hyperfine:
name: Hyperfine
runs-on: ubuntu-24.04
env:
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
MLIR_SYS_190_PREFIX: /usr/lib/llvm-19/
TABLEGEN_190_PREFIX: /usr/lib/llvm-19/
LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/
steps:
- uses: actions/checkout@v4
- name: check and free hdd space left
run: |
echo "Listing 20 largest packages"
dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 20
df -h
sudo apt-get update
sudo apt-get remove -y '^llvm-.*'
sudo apt-get remove -y 'php.*'
sudo apt-get remove -y '^dotnet-.*'
sudo apt-get remove -y '^temurin-.*'
sudo apt-get remove -y azure-cli microsoft-edge-stable google-chrome-stable firefox mono-devel
sudo apt-get autoremove -y
sudo apt-get clean
df -h
echo "Removing large directories"
# deleting 15GB
sudo rm -rf /usr/share/dotnet/
sudo rm -rf /usr/local/lib/android
df -h
- uses: dtolnay/rust-toolchain@1.83.0
with:
components: clippy
- uses: Swatinem/rust-cache@v2
- name: add llvm deb repository
uses: myci-actions/add-deb-repo@11
with:
repo: deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-19 main
repo-name: llvm-repo
keys-asc: https://apt.llvm.org/llvm-snapshot.gpg.key
- name: Install LLVM
run: sudo apt-get install llvm-19 llvm-19-dev llvm-19-runtime clang-19 clang-tools-19 lld-19 libpolly-19-dev libmlir-19-dev mlir-19-tools
- name: Install hyperfine
uses: taiki-e/install-action@v2
with:
tool: hyperfine@1.18
- name: Install deps
run: make deps
- name: Run benchmarks
run: make bench
- name: Create markdown file
run: bash .github/scripts/merge-benches.sh
- name: Find Bench Comment
continue-on-error: true
uses: peter-evans/find-comment@v3
id: fc
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
body-includes: Benchmarking
- name: Create or update bench comment
continue-on-error: true
uses: peter-evans/create-or-update-comment@v4
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body-path: bench-hyperfine.md
edit-mode: replace
build-binaries:
env:
MLIR_SYS_190_PREFIX: /usr/lib/llvm-19/
LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/
TABLEGEN_190_PREFIX: /usr/lib/llvm-19/
RUST_LOG: cairo_native=debug,cairo_native_test=debug
strategy:
matrix:
branch: [ base, head ]
name: Build Binary for ${{ matrix.branch }}
runs-on: ubuntu-24.04
steps:
- name: Cache binary
uses: actions/cache@v3
id: cache-binary
with:
path: bin/cairo-native-run-${{ matrix.branch }}
key: binary-${{ github.event.pull_request[matrix.branch].sha }}
- name: Cache runtime-library
uses: actions/cache@v3
id: cache-library
with:
path: lib/libcairo-native-runtime-${{ matrix.branch }}.a
key: library-${{ github.event.pull_request[matrix.branch].sha }}
- name: check and free hdd space left
if: ${{ steps.cache-binary.outputs.cache-hit != 'true' }}
run: |
echo "Listing 20 largest packages"
dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 20
df -h
sudo apt-get update
sudo apt-get remove -y '^llvm-.*'
sudo apt-get remove -y 'php.*'
sudo apt-get remove -y '^dotnet-.*'
sudo apt-get remove -y '^temurin-.*'
sudo apt-get remove -y azure-cli microsoft-edge-stable google-chrome-stable firefox mono-devel
sudo apt-get autoremove -y
sudo apt-get clean
df -h
echo "Removing large directories"
# deleting 15GB
sudo rm -rf /usr/share/dotnet/
sudo rm -rf /usr/local/lib/android
df -h
- name: Install Rust
if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
uses: dtolnay/rust-toolchain@1.83.0
- name: add llvm deb repository
if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
uses: myci-actions/add-deb-repo@11
with:
repo: deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-19 main
repo-name: llvm-repo
keys-asc: https://apt.llvm.org/llvm-snapshot.gpg.key
- name: Install LLVM
if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
run: sudo apt-get update && sudo apt-get install llvm-19 llvm-19-dev llvm-19-runtime clang-19 clang-tools-19 lld-19 libpolly-19-dev libmlir-19-dev mlir-19-tools
- name: Checkout base commit
if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request[matrix.branch].sha }}
- name: Fetch Rust cache
if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
uses: Swatinem/rust-cache@v2
- name: Build binary
if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
run: |
make build
mkdir lib bin
cp target/release/cairo-native-run bin/cairo-native-run-${{ matrix.branch }}
cp target/release/libcairo_native_runtime.a lib/libcairo-native-runtime-${{ matrix.branch }}.a
run-bench:
env:
MLIR_SYS_190_PREFIX: /usr/lib/llvm-19/
LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/
TABLEGEN_190_PREFIX: /usr/lib/llvm-19/
RUST_LOG: cairo_native=debug,cairo_native_test=debug
strategy:
matrix:
branch: [ base, head ]
name: Run benches for ${{ matrix.branch }}
runs-on: ubuntu-24.04
needs: [ build-binaries ]
steps:
- name: Checkout base commit
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request[matrix.branch].sha }}
- name: Install Hyperfine
uses: taiki-e/install-action@v2
with:
tool: hyperfine@1.16
- name: Fetch ${{ matrix.branch }} binary
uses: actions/cache/restore@v3
with:
path: bin/cairo-native-run-${{ matrix.branch }}
key: binary-${{ github.event.pull_request[matrix.branch].sha }}
- name: Fetch ${{ matrix.branch }} runtime-library
uses: actions/cache/restore@v3
with:
path: lib/libcairo-native-runtime-${{ matrix.branch }}.a
key: library-${{ github.event.pull_request[matrix.branch].sha }}
- name: Run Program
run: |
make deps
mkdir bench-results-${{ matrix.branch }}
cp programs/benches/*.cairo bench-results-${{ matrix.branch }}
for f in bench-results-${{ matrix.branch }}/*.cairo; do
hyperfine \
--show-output \
--export-markdown "${f%.*}.md" \
--export-json "${f%.*}.json" \
--warmup 3 \
-n "${{matrix.branch}} $(basename $f) (JIT)" "./bin/cairo-native-run-${{ matrix.branch }} --run-mode=jit -s $f --opt-level 3 --available-gas 18446744073709551615" \
-n "${{matrix.branch}} $(basename $f) (AOT)" "CAIRO_NATIVE_RUNTIME_LIBRARY=lib/libcairo-native-runtime-${{ matrix.branch }}.a ./bin/cairo-native-run-${{ matrix.branch }} --run-mode=aot -s $f --opt-level 3 --available-gas 18446744073709551615" \
>> /dev/stderr
done
- name: Upload benches ${{matrix.branch}}
uses: actions/upload-artifact@v4
with:
name: results-${{matrix.branch}}
path: bench-results-${{matrix.branch}}
show-outputs:
name: Print Bench Results
runs-on: ubuntu-24.04
needs: [ run-bench ]
steps:
- name: Fetch Base Result
uses: actions/download-artifact@v4
with:
name: results-base
path: bench-results-base
- name: Fetch Head Result
uses: actions/download-artifact@v4
with:
name: results-head
path: bench-results-head
- name: Print tables
run: |
{
echo "${bold}Benchmark results Main vs HEAD.${normal}"
for f in bench-results-base/*.md; do
echo
cat $f
cat bench-results-head/$(basename $f)
done
} | tee -a comment_body.md
- name: Find Bench Comment
continue-on-error: true
uses: peter-evans/find-comment@v3
id: fc
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
body-includes: Benchmark results Main vs HEAD
- name: Create or update bench comment
continue-on-error: true
uses: peter-evans/create-or-update-comment@v4
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body-path: comment_body.md
edit-mode: replace