name: Mutants
on:
pull_request:
paths-ignore: ["**.md", "docs/**"]
push:
branches: [main]
paths-ignore: ["**.md", "docs/**"]
permissions:
contents: read
env:
CARGO_TERM_COLOR: always
JJ_VERSION: v0.45.1
jobs:
in-diff:
name: Mutants in changed code
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 0
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- uses: taiki-e/install-action@v2
with:
tool: cargo-mutants
- name: Install jj
run: |
set -euo pipefail
url="https://github.com/jj-vcs/jj/releases/download/${JJ_VERSION}/jj-${JJ_VERSION}-x86_64-unknown-linux-musl.tar.gz"
tmp="$(mktemp -d)"
curl -fsSL "$url" | tar xz -C "$tmp"
mkdir -p "$HOME/.local/bin"
install -m755 "$tmp/jj" "$HOME/.local/bin/jj"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
"$HOME/.local/bin/jj" --version
- name: Compute diff
run: |
set -euo pipefail
if [ "${{ github.event_name }}" = "pull_request" ]; then
base="origin/${{ github.base_ref }}"
else
before="${{ github.event.before }}"
if [ -z "$before" ] || [ "$before" = "0000000000000000000000000000000000000000" ]; then
base="HEAD~1"
else
base="$before"
fi
fi
echo "Diffing against $base"
# Force a/ b/ prefixes. With `diff.mnemonicPrefix` git writes i/ and w/,
# which cargo-mutants does not recognise: it matches no files and exits
# 0, a silent pass.
git -c diff.mnemonicPrefix=false -c diff.noprefix=false \
diff "$base".. > mutants.diff
wc -l mutants.diff
if [ -s mutants.diff ] && ! grep -q '^diff --git a/' mutants.diff; then
echo "::error::diff lacks a/ b/ prefixes; cargo-mutants would silently match nothing"
head -3 mutants.diff
exit 1
fi
- name: Mutants
id: mutants
run: |
set -euo pipefail
echo "ran" > mutants-status.txt
if [ ! -s mutants.diff ]; then
echo "empty" > mutants-status.txt
echo "::notice::empty diff, nothing to mutate"
exit 0
fi
selected=$(cargo mutants --list --in-diff mutants.diff | wc -l)
echo "$selected" > mutants-selected.txt
echo "Selected $selected mutants in changed regions"
if [ "$selected" -eq 0 ]; then
echo "none" > mutants-status.txt
echo "::notice::no mutants overlap the changed regions (test- or doc-only change)"
exit 0
fi
# A broad diff (a reformat, a mass rename) selects much of the tree and
# cannot finish inside the job timeout. Skip it loudly rather than
# report a partial run as a pass. The cap is sized from the measured
# rate in AGENTS.md so a run under it fits in the timeout.
MAX_SELECTED=25
if [ "$selected" -gt "$MAX_SELECTED" ]; then
echo "skipped" > mutants-status.txt
echo "::warning::Mutants skipped: $selected selected, over the limit of $MAX_SELECTED. Nothing was mutation-tested for this change."
exit 0
fi
# Exit codes: 2 a mutant was MISSED (the finding); 3 a mutant TIMED
# OUT (usually an injected infinite loop, a real result, so counted as
# caught and passed); 4 the unmutated baseline failed; 5 and 6 the diff
# does not match the tree or is malformed, which are bugs in "Compute
# diff", not in workon.
rc=0
cargo mutants --no-shuffle -vV --jobs 2 --in-diff mutants.diff || rc=$?
if [ "$rc" -eq 3 ]; then
echo "::notice::timeouts counted as caught; see mutants.out/timeout.txt"
rc=0
fi
exit "$rc"
- name: Summarise
if: always()
run: |
set -euo pipefail
status="$(cat mutants-status.txt 2>/dev/null || echo unknown)"
selected="$(cat mutants-selected.txt 2>/dev/null || echo '?')"
{
echo '## Mutants in changed code'
echo
echo "_step outcome: ${{ steps.mutants.outcome }}_"
echo
case "$status" in
empty)
echo 'No diff to analyse. Nothing was mutation-tested.'
;;
none)
echo 'No mutants overlap the changed regions (a test- or docs-only'
echo 'change). Nothing was mutation-tested; cargo-mutants does not'
echo 'mutate test code.'
;;
skipped)
echo "**Skipped: $selected mutants selected, over the limit.**"
echo
echo 'A diff this broad touches many functions without necessarily'
echo 'changing behaviour. **Nothing was mutation-tested.** To cover it'
echo 'deliberately, shard a manual run:'
echo '`cargo mutants --in-diff <diff> --shard 0/4`'
;;
unknown)
echo 'The mutants step did not run: an earlier step failed (the jj'
echo 'install or "Compute diff"). **Nothing was mutation-tested.**'
;;
*)
if [ ! -d mutants.out ]; then
echo 'cargo-mutants started but produced no results directory; see'
echo 'the step log (exit 4, 5 and 6 are harness failures).'
echo '**Treat this as no coverage, not as a pass.**'
exit 0
fi
for kind in missed caught unviable timeout; do
f="mutants.out/$kind.txt"
n=0
[ -f "$f" ] && n=$(wc -l < "$f" | tr -d ' ')
echo "- **$kind**: $n"
done
if [ -s mutants.out/missed.txt ]; then
echo
echo '### Missed: a bug was inserted here and no test noticed'
echo
echo '```'
cat mutants.out/missed.txt
echo '```'
echo
echo 'Write a test that fails on it, or exclude it in'
echo '`.cargo/mutants.toml` with the reason no test can.'
fi
;;
esac
} >> "$GITHUB_STEP_SUMMARY"
- name: Archive mutants.out
if: always()
uses: actions/upload-artifact@v4
with:
name: mutants-in-diff
path: mutants.out
if-no-files-found: ignore
slice:
name: Mutants, rotating slice
if: github.event_name == 'push'
runs-on: ubuntu-latest
timeout-minutes: 20
env:
SLICES: 24
steps:
- uses: actions/checkout@v5
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- uses: taiki-e/install-action@v2
with:
tool: cargo-mutants
- name: Install jj
run: |
set -euo pipefail
url="https://github.com/jj-vcs/jj/releases/download/${JJ_VERSION}/jj-${JJ_VERSION}-x86_64-unknown-linux-musl.tar.gz"
tmp="$(mktemp -d)"
curl -fsSL "$url" | tar xz -C "$tmp"
mkdir -p "$HOME/.local/bin"
install -m755 "$tmp/jj" "$HOME/.local/bin/jj"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
"$HOME/.local/bin/jj" --version
- name: Mutants
run: |
set -euo pipefail
k=$(( ${{ github.run_number }} % SLICES ))
echo "$k" > mutants-slice.txt
echo "Slice $k of $SLICES"
# Recorded however the run ends (exit 2 on a miss included): the
# slice count was chosen from a local measurement, and this is the
# figure to re-choose it from.
trap 'echo "$SECONDS" > mutants-seconds.txt' EXIT
rc=0
cargo mutants --no-shuffle -vV --jobs 2 --shard "$k/$SLICES" || rc=$?
if [ "$rc" -eq 3 ]; then
echo "::notice::timeouts counted as caught; see mutants.out/timeout.txt"
rc=0
fi
exit "$rc"
- name: Summarise
if: always()
run: |
set -euo pipefail
k="$(cat mutants-slice.txt 2>/dev/null || echo '?')"
secs="$(cat mutants-seconds.txt 2>/dev/null || echo '?')"
{
echo "## Mutants, slice $k of $SLICES"
echo
echo "_cargo mutants ran for ${secs}s_"
echo
if [ ! -d mutants.out ]; then
echo 'No results directory: the run failed before reporting.'
echo '**Treat this as no coverage, not as a pass.**'
exit 0
fi
for kind in missed caught unviable timeout; do
f="mutants.out/$kind.txt"
n=0
[ -f "$f" ] && n=$(wc -l < "$f" | tr -d ' ')
echo "- **$kind**: $n"
done
if [ -s mutants.out/missed.txt ]; then
echo
echo '### Missed in this slice'
echo
echo '```'
cat mutants.out/missed.txt
echo '```'
echo
echo 'Old code, not this push: give each a test or an exclusion'
echo 'with its reason, as for any other miss.'
fi
} >> "$GITHUB_STEP_SUMMARY"
- name: Archive mutants.out
if: always()
uses: actions/upload-artifact@v4
with:
name: mutants-slice
path: mutants.out
if-no-files-found: ignore