name: Benchmarks
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
workflow_dispatch:
inputs:
baseline:
description: 'Baseline branch/commit to compare against'
required: false
default: 'main'
type: string
env:
CARGO_TERM_COLOR: always
jobs:
benchmark:
name: Run Benchmarks
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Install Rust
uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
- name: Cache dependencies
uses: Swatinem/rust-cache@v2
- name: Install benchmark tools
run: cargo install cargo-criterion
- name: Run arithmetic benchmarks
run: cargo criterion --bench arithmetic --message-format=json > arithmetic-bench.json
- name: Run Montgomery benchmarks
run: cargo criterion --bench montgomery --message-format=json > montgomery-bench.json
- name: Run modular exponentiation benchmarks
run: cargo criterion --bench modexp --message-format=json > modexp-bench.json
- name: Generate benchmark report
run: |
cat > benchmark_report.md << 'EOF'
# Benchmark Report
This report shows the performance of clock-bigint operations.
## Performance Targets
- **256-bit addition**: ~8 cycles
- **256-bit multiplication**: ~40 cycles
- **2048-bit modular exponentiation**: within 10% of GMP performance
## Current Performance
### Arithmetic Operations (256-bit)
EOF
# Extract and format arithmetic benchmark results
if [ -f arithmetic-bench.json ]; then
echo '```' >> benchmark_report.md
jq -r '.benchmarks[] | select(.name | contains("u256")) | "\(.name): \(.mean.estimate) ns ± \(.mean.stderr) ns"' arithmetic-bench.json >> benchmark_report.md 2>/dev/null || echo "Failed to parse arithmetic benchmarks" >> benchmark_report.md
echo '```' >> benchmark_report.md
fi
cat >> benchmark_report.md << 'EOF'
### Montgomery Operations
EOF
# Extract and format Montgomery benchmark results
if [ -f montgomery-bench.json ]; then
echo '```' >> benchmark_report.md
jq -r '.benchmarks[] | select(.name | contains("u256") or contains("u2048")) | "\(.name): \(.mean.estimate) ns ± \(.mean.stderr) ns"' montgomery-bench.json >> benchmark_report.md 2>/dev/null || echo "Failed to parse Montgomery benchmarks" >> benchmark_report.md
echo '```' >> benchmark_report.md
fi
cat >> benchmark_report.md << 'EOF'
### Modular Exponentiation
EOF
# Extract and format modular exponentiation benchmark results
if [ -f modexp-bench.json ]; then
echo '```' >> benchmark_report.md
jq -r '.benchmarks[] | select(.name | contains("u2048") or contains("u256")) | "\(.name): \(.mean.estimate) ns ± \(.mean.stderr) ns"' modexp-bench.json >> benchmark_report.md 2>/dev/null || echo "Failed to parse modexp benchmarks" >> benchmark_report.md
echo '```' >> benchmark_report.md
fi
echo "" >> benchmark_report.md
echo "*Report generated on $(date)*" >> benchmark_report.md
- name: Upload benchmark results
uses: actions/upload-artifact@v4
with:
name: benchmark-results
path: |
*-bench.json
benchmark_report.md
retention-days: 30
- name: Comment PR with benchmark results
if: github.event_name == 'pull_request'
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const report = fs.readFileSync('benchmark_report.md', 'utf8');
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: report
});
performance-regression:
name: Performance Regression Check
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Install Rust
uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
- name: Cache dependencies
uses: Swatinem/rust-cache@v2
- name: Install benchmark tools
run: cargo install cargo-criterion
- name: Checkout baseline
run: |
BASELINE="${{ github.event.inputs.baseline || 'origin/main' }}"
git checkout $BASELINE
- name: Run baseline benchmarks
run: |
cargo criterion --bench arithmetic --message-format=json > baseline-arithmetic.json
cargo criterion --bench montgomery --message-format=json > baseline-montgomery.json
cargo criterion --bench modexp --message-format=json > baseline-modexp.json
- name: Checkout PR branch
run: |
git checkout ${{ github.event.pull_request.head.sha }}
- name: Run PR benchmarks
run: |
cargo criterion --bench arithmetic --message-format=json > pr-arithmetic.json
cargo criterion --bench montgomery --message-format=json > pr-montgomery.json
cargo criterion --bench modexp --message-format=json > pr-modexp.json
- name: Compare performance
run: |
cat > performance_comparison.md << 'EOF'
# Performance Comparison
Comparing current branch against baseline (${{ github.event.inputs.baseline || 'main' }}).
## Key Operations
| Operation | Baseline | Current | Change |
|-----------|----------|---------|--------|
EOF
# Function to extract benchmark time
extract_time() {
local file=$1
local bench_name=$2
jq -r ".benchmarks[] | select(.name == \"$bench_name\") | .mean.estimate" "$file" 2>/dev/null || echo "N/A"
}
# Compare key benchmarks
for bench in "u256_add" "u256_mul" "mont_mul_u256" "modexp_u2048_small_exp"; do
baseline_time=$(extract_time "baseline-*.json" "$bench")
pr_time=$(extract_time "pr-*.json" "$bench")
if [ "$baseline_time" != "N/A" ] && [ "$pr_time" != "N/A" ]; then
# Calculate percentage change
if (( $(echo "$baseline_time > 0" | bc -l) )); then
change=$(echo "scale=2; (($pr_time - $baseline_time) / $baseline_time) * 100" | bc -l)
change_str="${change}%"
else
change_str="N/A"
fi
echo "| $bench | ${baseline_time} ns | ${pr_time} ns | ${change_str} |" >> performance_comparison.md
fi
done
echo "" >> performance_comparison.md
echo "*Comparison generated on $(date)*" >> performance_comparison.md
- name: Upload performance comparison
uses: actions/upload-artifact@v4
with:
name: performance-comparison
path: performance_comparison.md
retention-days: 30
- name: Check for significant regressions
run: |
# Simple check for major performance regressions (>10% slowdown)
if grep -q "+[0-9][0-9]\.[0-9]*%" performance_comparison.md; then
echo "⚠️ Significant performance regression detected!"
echo "Please review the performance comparison report."
exit 1
else
echo "✅ No significant performance regressions detected."
fi