name: Mutation Testing
on:
push:
branches: [main]
pull_request:
branches: [main]
types: [opened, synchronize, reopened, labeled]
workflow_dispatch:
schedule:
- cron: '0 3 * * 0'
env:
CARGO_TERM_COLOR: always
jobs:
mutation-pr:
runs-on: ubuntu-latest
if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-mutation')
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- uses: dtolnay/rust-toolchain@stable
- name: Install protobuf
run: |
sudo apt-get update
sudo apt-get install -y protobuf-compiler
- name: Cache cargo
uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cargo-mutants
run: cargo install cargo-mutants
- name: Run mutation testing on changed files
continue-on-error: true
run: |
# Ensure base branch ref exists locally for reliable diffing
git fetch origin "${{ github.base_ref }}":"refs/remotes/origin/${{ github.base_ref }}"
# Collect changed Rust source files (exclude test-only and workflow files)
changed_rs=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- '*.rs' | grep -v '_test\.rs$' || true)
if [ -z "$changed_rs" ]; then
echo "No Rust source files changed in this PR"
exit 0
fi
# Build --file flags for each changed file
file_args=""
for f in $changed_rs; do
file_args="$file_args --file $f"
done
echo "Running mutation testing for changed files:"
echo "$changed_rs"
cargo mutants $file_args
- name: Upload mutation report
uses: actions/upload-artifact@v4
if: always()
with:
name: mutation-report-pr
path: mutants.out/
retention-days: 30
- name: Comment PR with mutation results
if: always() && github.event_name == 'pull_request'
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const path = require('path');
// Try the expected path first
let outcomesPath = path.join('mutants.out', 'outcomes.json');
// If not found, search under the workspace so we can handle any
// unexpected layout changes while still using the results.
if (!fs.existsSync(outcomesPath)) {
const workspace = process.env.GITHUB_WORKSPACE || process.cwd();
/** @param {string} start */
function findOutcomes(start) {
const entries = fs.readdirSync(start, { withFileTypes: true });
for (const entry of entries) {
const full = path.join(start, entry.name);
if (entry.isDirectory()) {
const found = findOutcomes(full);
if (found) return found;
} else if (entry.isFile() && entry.name === 'outcomes.json') {
return full;
}
}
return null;
}
const found = fs.existsSync(workspace) ? findOutcomes(workspace) : null;
if (found) {
console.log(`Found outcomes.json at ${found}`);
outcomesPath = found;
} else {
console.log('No mutation outcomes file found (no outcomes.json anywhere under workspace)');
await github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: '## 🧬 Mutation Testing Results\n\nNo mutation outcomes file was found for this run. This usually means no mutants were generated for the changed files or mutation testing did not complete.'
});
return;
}
}
const data = JSON.parse(fs.readFileSync(outcomesPath, 'utf8'));
const outcomes = data.outcomes || [];
const mutants = outcomes.filter(o => o.scenario !== 'Baseline');
const total = mutants.length;
const killed = mutants.filter(o => o.summary === 'CaughtMutant').length;
const survived = mutants.filter(o => o.summary === 'MissedMutant').length;
const timeout = mutants.filter(o => o.summary === 'Timeout').length;
const score = total > 0 ? ((killed / total) * 100).toFixed(1) : 0;
const survivorList = mutants
.filter(o => o.summary === 'MissedMutant')
.map(o => {
const name = o.scenario?.Mutant?.name || 'unknown';
return `- \`${name}\``;
})
.join('\n') || 'None';
const body = [
'## 🧬 Mutation Testing Results',
'',
'| Metric | Count |',
'|--------|-------|',
`| Total Mutants | ${total} |`,
`| Killed | ${killed} |`,
`| Survived | ${survived} |`,
`| Timeout | ${timeout} |`,
`| **Score** | **${score}%** |`,
'',
survived > 0 ? '**Note:** Some mutants survived. Consider improving tests for the changed code.' : 'All mutants killed!',
'',
'<details>',
'<summary>Surviving Mutants</summary>',
'',
survivorList,
'',
'</details>'
].join('\n');
await github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: body
});
mutation-baseline:
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/main' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@stable
- name: Install protobuf
run: |
sudo apt-get update
sudo apt-get install -y protobuf-compiler
- name: Cache cargo
uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cargo-mutants
run: cargo install cargo-mutants
- name: Run full mutation testing
run: cargo mutants
continue-on-error: true
- name: Upload mutation report
uses: actions/upload-artifact@v4
if: always()
with:
name: mutation-report-baseline
path: |
mutants.out/
retention-days: 90
- name: Upload HTML report to GitHub Pages
if: github.ref == 'refs/heads/main' && always()
uses: peaceiris/actions-gh-pages@v4
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./mutants.out/html
destination_dir: mutation-testing
- name: Generate mutation summary
if: always()
run: |
echo "## Mutation Testing Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
if [ -f mutants.out/outcomes.json ]; then
total=$(jq '[.outcomes[] | select(.scenario != "Baseline")] | length' mutants.out/outcomes.json)
killed=$(jq '[.outcomes[] | select(.scenario != "Baseline") | select(.summary == "CaughtMutant")] | length' mutants.out/outcomes.json)
survived=$(jq '[.outcomes[] | select(.scenario != "Baseline") | select(.summary == "MissedMutant")] | length' mutants.out/outcomes.json)
timeout=$(jq '[.outcomes[] | select(.scenario != "Baseline") | select(.summary == "Timeout")] | length' mutants.out/outcomes.json)
unviable=$(jq '[.outcomes[] | select(.scenario != "Baseline") | select(.summary == "Unviable")] | length' mutants.out/outcomes.json)
if [ "$total" -gt 0 ]; then
score=$(echo "scale=1; ($killed / $total) * 100" | bc)
echo "| Metric | Count |" >> $GITHUB_STEP_SUMMARY
echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY
echo "| Total Mutants | $total |" >> $GITHUB_STEP_SUMMARY
echo "| ✅ Killed | $killed |" >> $GITHUB_STEP_SUMMARY
echo "| ❌ Survived | $survived |" >> $GITHUB_STEP_SUMMARY
echo "| ⏱️ Timeout | $timeout |" >> $GITHUB_STEP_SUMMARY
echo "| ⚪ Unviable | $unviable |" >> $GITHUB_STEP_SUMMARY
echo "| **Mutation Score** | **$score%** |" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
if (( $(echo "$score >= 80" | bc -l) )); then
echo "✅ **Excellent test quality!**" >> $GITHUB_STEP_SUMMARY
elif (( $(echo "$score >= 50" | bc -l) )); then
echo "⚠️ **Acceptable test quality, room for improvement.**" >> $GITHUB_STEP_SUMMARY
else
echo "🔴 **Poor test quality. Consider improving tests.**" >> $GITHUB_STEP_SUMMARY
fi
fi
fi
echo "" >> $GITHUB_STEP_SUMMARY
echo "📊 [View Full Report](https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/mutation-testing/)" >> $GITHUB_STEP_SUMMARY