name: Benchmark
on:
push:
branches: [main, dev]
schedule:
- cron: '0 4 * * *'
workflow_dispatch:
env:
CARGO_TERM_COLOR: always
jobs:
benchmark-build:
name: Build Benchmark Binary
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8
- name: Cache cargo build
uses: Swatinem/rust-cache@v2
with:
shared-key: release-full
- name: Build bench binary
run: |
set -o pipefail
cargo bench --bench performance_bench --features full --no-run --message-format=json | tee build-output.json
BIN_PATH=$(jq -r 'select(.reason == "compiler-artifact" and (.target.kind // [] | index("bench")) and .executable != null) | .executable' build-output.json | tail -n1)
if [ -z "$BIN_PATH" ]; then
echo "Could not locate performance_bench executable in cargo build output" >&2
exit 1
fi
cp "$BIN_PATH" ./performance_bench_bin
- name: Upload bench binary
uses: actions/upload-artifact@v4
with:
name: performance-bench-bin
path: performance_bench_bin
retention-days: 1
benchmark-streams:
name: Benchmark (streams)
needs: benchmark-build
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Download bench binary
uses: actions/download-artifact@v4
with:
name: performance-bench-bin
- name: Pre-pull docker images
run: |
for f in aws kafka amqp nats mqtt_performance; do
docker compose -f "tests/integration/docker-compose/${f}.yml" pull -q
done
- name: Run benchmarks
shell: bash
env:
MQB_TEST_BACKEND: "aws,kafka,amqp,nats,mqtt,ibm-mq"
run: |
set -o pipefail
chmod +x ./performance_bench_bin
export CARGO_TERM_COLOR=never
./performance_bench_bin --bench --output-format bencher | tee raw_output_streams.txt
./scripts/ci/format_bench_output.sh raw_output_streams.txt output_streams.txt
- name: Upload shard output
uses: actions/upload-artifact@v4
with:
name: bench-output-streams
path: |
raw_output_streams.txt
output_streams.txt
retention-days: 1
benchmark-database:
name: Benchmark (database)
needs: benchmark-build
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Download bench binary
uses: actions/download-artifact@v4
with:
name: performance-bench-bin
- name: Pre-pull docker images
run: |
for f in mongodb postgres; do
docker compose -f "tests/integration/docker-compose/${f}.yml" pull -q
done
- name: Run benchmarks
shell: bash
env:
MQB_TEST_BACKEND: "mongodb,postgres"
run: |
set -o pipefail
chmod +x ./performance_bench_bin
export CARGO_TERM_COLOR=never
./performance_bench_bin --bench --output-format bencher | tee raw_output_database.txt
./scripts/ci/format_bench_output.sh raw_output_database.txt output_database.txt
- name: Upload shard output
uses: actions/upload-artifact@v4
with:
name: bench-output-database
path: |
raw_output_database.txt
output_database.txt
retention-days: 1
benchmark-dockerless:
name: Benchmark (dockerless)
needs: benchmark-build
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Download bench binary
uses: actions/download-artifact@v4
with:
name: performance-bench-bin
- name: Run benchmarks
shell: bash
env:
MQB_TEST_BACKEND: "zeromq,memory,file,http,websocket,grpc"
run: |
set -o pipefail
chmod +x ./performance_bench_bin
export CARGO_TERM_COLOR=never
./performance_bench_bin --bench --output-format bencher | tee raw_output_dockerless.txt
./scripts/ci/format_bench_output.sh raw_output_dockerless.txt output_dockerless.txt
- name: Upload shard output
uses: actions/upload-artifact@v4
with:
name: bench-output-dockerless
path: |
raw_output_dockerless.txt
output_dockerless.txt
retention-days: 1
benchmark-collect:
name: Collect Benchmark Results
needs: [benchmark-streams, benchmark-database, benchmark-dockerless]
runs-on: ubuntu-latest
permissions:
contents: write
deployments: write
steps:
- uses: actions/checkout@v4
- name: Download shard outputs
uses: actions/download-artifact@v4
with:
pattern: bench-output-*
merge-multiple: true
path: bench-outputs
- name: Merge shard outputs
run: |
cat bench-outputs/raw_output_*.txt > raw_output.txt
cat bench-outputs/output_*.txt > output.txt
- name: Generate Throughput Summary
run: |
python3 -c '
import re
import os
data = {}
try:
with open("raw_output.txt", "r") as f:
for line in f:
# Match lines like: "aws single_write: 1 iters, total time 1.48s, 673.75 msgs/sec"
m = re.search(r"^\s*(.*): .* ([\d\.]+) msgs/sec", line)
if m:
name = m.group(1).strip()
rate = float(m.group(2))
if name not in data:
data[name] = []
data[name].append(rate)
if data:
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
f.write("### Benchmark Throughput Results\n")
f.write("| Test Case | Average msgs/sec | Samples |\n")
f.write("|-----------|------------------|---------|\n")
for name in sorted(data.keys()):
rates = data[name]
avg = sum(rates) / len(rates)
f.write(f"| {name} | {avg:,.2f} | {len(rates)} |\n")
except Exception as e:
print(f"Error generating summary: {e}")
'
- name: Create gh-pages branch if missing
run: |
git fetch origin gh-pages || {
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --global user.name "github-actions[bot]"
git checkout --orphan gh-pages
git rm -rf .
git commit --allow-empty -m "Initialize gh-pages branch"
git push origin gh-pages
git checkout ${{ github.sha }}
}
- name: Store benchmark result
uses: benchmark-action/github-action-benchmark@v1
with:
name: Rust Benchmark
tool: 'cargo'
output-file-path: output.txt
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: true
alert-threshold: '40000%'
comment-on-alert: true
fail-on-alert: false