name: Mutation Testing
on:
pull_request:
branches:
- main
paths:
- 'src/**'
- 'tests/**'
- 'Cargo.toml'
- 'Cargo.lock'
- '.cargo/mutants.toml'
- '.github/workflows/mutation.yml'
workflow_dispatch:
inputs:
full:
description: 'Run the full mutation test suite instead of incremental'
type: boolean
default: false
permissions:
contents: read
pull-requests: write
concurrency:
group: ${{ github.ref }}-mutation
cancel-in-progress: true
jobs:
incremental:
if: ${{ github.event_name == 'pull_request' || !inputs.full }}
name: Incremental mutation testing
runs-on: ubuntu-latest
timeout-minutes: 90
steps:
- name: Checkout
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: stable
cache: false
- uses: Swatinem/rust-cache@v2
- name: Install cargo-mutants (from upstream main, post-#613)
run: cargo install --locked --git https://github.com/sourcefrog/cargo-mutants --rev cbdfe8a574566e01cef9ffaa7475dfaf69c88440 cargo-mutants
- name: Compute diff vs base
id: diff
env:
BASE_REF: ${{ github.base_ref || 'main' }}
run: |
set -euo pipefail
git fetch --no-tags --depth=1 origin "$BASE_REF"
MERGE_BASE=$(git merge-base HEAD "origin/$BASE_REF")
git diff "$MERGE_BASE" HEAD -- src tests Cargo.toml Cargo.lock > mutation.diff
# Track which Rust files changed so we can fall back to --file
# when --in-diff finds no mutable lines (e.g. a PR that only edits
# `#[cfg(test)] mod tests`). cargo-mutants does not mutate test
# code, so an in-diff filter on a test-only PR would yield no
# mutants and silently pass.
git diff --name-only --diff-filter=AM "$MERGE_BASE" HEAD -- 'src/**' 'tests/**' \
| grep '\.rs$' > changed-rs.txt || true
if [ -s mutation.diff ]; then
echo "has_diff=true" >> "$GITHUB_OUTPUT"
echo "Diff lines: $(wc -l < mutation.diff)"
if [ -s changed-rs.txt ]; then
echo "Changed Rust files:"
cat changed-rs.txt
fi
else
echo "has_diff=false" >> "$GITHUB_OUTPUT"
echo "No relevant changes; mutation testing will be skipped."
fi
- name: Run incremental mutation testing
if: steps.diff.outputs.has_diff == 'true'
run: |
set -euo pipefail
# Fast path: only mutate the production lines that appear in the
# PR diff. cargo-mutants prints "No mutants to filter" and exits 0
# when the diff covers only test code, in which case we fall back
# to mutating every changed Rust file so the new tests get a
# chance to kill the mutants they were written to target.
LOG=$(mktemp)
cargo mutants --no-shuffle --in-diff mutation.diff 2>&1 | tee "$LOG"
if grep -q "No mutants to filter" "$LOG"; then
if [ ! -s changed-rs.txt ]; then
echo "::notice::No Rust source changes; nothing to mutate."
exit 0
fi
echo "::notice::--in-diff produced no mutants (test-only diff); re-running scoped to changed files."
FILE_ARGS=()
while IFS= read -r f; do
FILE_ARGS+=("--file" "$f")
done < changed-rs.txt
cargo mutants --no-shuffle "${FILE_ARGS[@]}"
fi
- name: Upload mutation report
if: always() && steps.diff.outputs.has_diff == 'true'
uses: actions/upload-artifact@v7
with:
name: mutants-report-incremental
path: mutants.out
if-no-files-found: ignore
full:
if: ${{ github.event_name == 'workflow_dispatch' && inputs.full }}
name: Full mutation testing
runs-on: ubuntu-latest
timeout-minutes: 360
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Install Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: stable
cache: false
- uses: Swatinem/rust-cache@v2
- name: Install cargo-mutants (from upstream main, post-#613)
run: cargo install --locked --git https://github.com/sourcefrog/cargo-mutants --rev cbdfe8a574566e01cef9ffaa7475dfaf69c88440 cargo-mutants
- name: Run full mutation testing
run: cargo mutants --no-shuffle
- name: Upload mutation report
if: always()
uses: actions/upload-artifact@v7
with:
name: mutants-report-full
path: mutants.out
if-no-files-found: ignore
- name: Compute mutation score
id: score
if: always()
run: |
set -euo pipefail
if [ ! -f mutants.out/outcomes.json ]; then
echo "::notice::No mutants.out/outcomes.json -- skipping badge update."
echo "skip=true" >> "$GITHUB_OUTPUT"
exit 0
fi
CAUGHT=$(jq '.caught // 0' mutants.out/outcomes.json)
MISSED=$(jq '.missed // 0' mutants.out/outcomes.json)
TIMEOUT=$(jq '.timeout // 0' mutants.out/outcomes.json)
UNVIABLE=$(jq '.unviable // 0' mutants.out/outcomes.json)
DENOM=$(( CAUGHT + MISSED + TIMEOUT ))
KILLED=$(( CAUGHT + TIMEOUT ))
if [ "$DENOM" -eq 0 ]; then
PCT_RAW="0.00"
MESSAGE="N/A"
COLOR="lightgrey"
else
PCT_RAW=$(awk -v k="$KILLED" -v d="$DENOM" 'BEGIN { printf "%.2f", k/d*100 }')
# Trim trailing ".00" so a perfect score reads "100%" not "100.00%".
PCT_TRIMMED="${PCT_RAW%.00}"
MESSAGE="${PCT_TRIMMED}%"
INT=${PCT_RAW%.*}
if [ "$INT" -ge 100 ]; then COLOR=brightgreen
elif [ "$INT" -ge 95 ]; then COLOR=green
elif [ "$INT" -ge 90 ]; then COLOR=yellowgreen
elif [ "$INT" -ge 80 ]; then COLOR=yellow
elif [ "$INT" -ge 70 ]; then COLOR=orange
else COLOR=red
fi
fi
{
echo "## Mutation score"
echo ""
echo "**${MESSAGE}** (${KILLED} killed / ${DENOM} viable; ${UNVIABLE} unviable)"
echo ""
echo "| Outcome | Count |"
echo "| --- | ---: |"
echo "| Caught | ${CAUGHT} |"
echo "| Timeout (counted as caught) | ${TIMEOUT} |"
echo "| **Missed** | **${MISSED}** |"
echo "| Unviable (excluded from score) | ${UNVIABLE} |"
} >> "$GITHUB_STEP_SUMMARY"
echo "message=$MESSAGE" >> "$GITHUB_OUTPUT"
echo "color=$COLOR" >> "$GITHUB_OUTPUT"
echo "Mutation score: $MESSAGE ($COLOR) -- killed=$KILLED denom=$DENOM"
- name: Publish mutation score badge
if: |
always()
&& steps.score.outputs.skip != 'true'
&& github.ref == 'refs/heads/main'
&& vars.MUTANTS_BADGE_GIST_ID != ''
uses: schneegans/dynamic-badges-action@v1.8.0
with:
auth: ${{ secrets.RELEASE_PLZ_TOKEN }}
gistID: ${{ vars.MUTANTS_BADGE_GIST_ID }}
filename: mutants-badge.json
label: mutants
message: ${{ steps.score.outputs.message }}
color: ${{ steps.score.outputs.color }}