name: Benchmark
on:
workflow_dispatch:
inputs:
duration:
description: Seconds per scenario
default: "20"
profile:
description: Record a perf profile of rws (lowers its throughput)
type: boolean
default: false
extended:
description: Also gzip, access-log, HTTPS, HTTP/2 and large-file-over-TLS scenarios
type: boolean
default: true
rws_workers:
description: tokio worker threads for rws (TOKIO_WORKER_THREADS; empty = one per CPU)
default: ""
base_workers:
description: tokio worker threads for rws-base (with no base_ref, the same build again)
default: ""
base_ref:
description: Also bench rws built from this ref as "rws-base", in the same run (an A/B on one machine)
default: ""
base_runtime:
description: rws-base runtime (RWS_CONFIG_RUNTIME; with no base_ref, the same build again)
default: ""
rws_runtime:
description: rws runtime (RWS_CONFIG_RUNTIME)
type: choice
options: [per-core, shared]
default: per-core
permissions:
contents: read
jobs:
bench:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- run: cargo build --release --bin rws --features "acme auth mimalloc"
env:
RUSTFLAGS: ${{ inputs.profile && '-C force-frame-pointers=yes' || '' }}
- name: Build the base ref
if: inputs.base_ref != ''
env:
RUSTFLAGS: ${{ inputs.profile && '-C force-frame-pointers=yes' || '' }}
BASE_REF: ${{ inputs.base_ref }}
run: |
git fetch --no-tags origin "$BASE_REF" && git worktree add "$RUNNER_TEMP/base" FETCH_HEAD
# Same features as the head build; refs from before the mimalloc feature build without it.
base="--release --bin rws --manifest-path $RUNNER_TEMP/base/Cargo.toml --target-dir $RUNNER_TEMP/base-target"
cargo build $base --features "acme auth mimalloc" || cargo build $base --features "acme auth"
- name: Install perf
if: inputs.profile
run: |
sudo apt-get update
sudo apt-get install -y "linux-tools-$(uname -r)" || sudo apt-get install -y linux-tools-azure linux-tools-generic
sudo sysctl -w kernel.perf_event_paranoid=-1 kernel.kptr_restrict=0
perf --version
- name: Install nginx, Caddy, oha
env:
GH_TOKEN: ${{ github.token }}
run: |
sudo apt-get update && sudo apt-get install -y nginx
sudo systemctl stop nginx || true
gh release download v2.11.7 --repo caddyserver/caddy --pattern 'caddy_2.11.7_linux_amd64.tar.gz' --dir "$RUNNER_TEMP"
tar xzf "$RUNNER_TEMP/caddy_2.11.7_linux_amd64.tar.gz" -C "$RUNNER_TEMP" caddy
gh release download v1.16.0 --repo hatoo/oha --pattern 'oha-linux-amd64' --dir "$RUNNER_TEMP"
chmod +x "$RUNNER_TEMP/oha-linux-amd64"
nginx -v; "$RUNNER_TEMP/caddy" version; "$RUNNER_TEMP/oha-linux-amd64" --version
- name: Run
run: |
RWS=target/release/rws OHA="$RUNNER_TEMP/oha-linux-amd64" CADDY="$RUNNER_TEMP/caddy" \
NGINX="$(command -v nginx)" DURATION="${{ inputs.duration }}s" \
EXTENDED="${{ inputs.extended && '1' || '' }}" RWS_WORKERS="${{ inputs.rws_workers }}" \
RWS_BASE="${{ inputs.base_ref != '' && format('{0}/base-target/release/rws', runner.temp) || '' }}" \
RWS_BASE_WORKERS="${{ inputs.base_workers }}" \
RWS_RUNTIME="${{ inputs.rws_runtime }}" \
RWS_BASE_RUNTIME="${{ inputs.base_runtime }}" \
PERF_DIR="${{ inputs.profile && 'perf' || '' }}" bench/run.sh
{
echo "## rws vs Caddy vs nginx"
echo
echo "rws $(grep -m1 '^version' Cargo.toml | cut -d'"' -f2) @ ${GITHUB_SHA::7} · Caddy 2.11.7 · $(nginx -v 2>&1)"
echo
cat bench-results.md
} >> "$GITHUB_STEP_SUMMARY"
- name: Profile reports
if: inputs.profile
run: |
for data in perf/*.data; do
scenario="$(basename "$data" .data)"
# Flat: where the CPU time is spent (self time per symbol / per binary).
perf report -i "$data" --stdio --no-children --sort dso,symbol --percent-limit 0.5 -g none > "perf/$scenario.flat.txt" 2>/dev/null
perf report -i "$data" --stdio --no-children --sort dso -g none > "perf/$scenario.dso.txt" 2>/dev/null
# Inclusive, with callers: what each subsystem costs including callees.
perf report -i "$data" --stdio --children --sort symbol --percent-limit 3 -g caller,0.5,callee,function,percent > "perf/$scenario.tree.txt" 2>/dev/null
# For flame graphs: inferno-collapse-perf < x.script | inferno-flamegraph > x.svg
perf script -i "$data" 2>/dev/null | gzip > "perf/$scenario.script.gz"
{
echo "### perf: $scenario (self time, ≥ 0.5%)"
echo '```'
grep -v '^#' "perf/$scenario.flat.txt" | grep -v '^$' | head -60
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
done
rm -f perf/*.data
- uses: actions/upload-artifact@v4
if: inputs.profile
with:
name: bench-profile
path: perf/
- uses: actions/upload-artifact@v4
with:
name: bench-results
path: |
bench-results.md
bench-results.json