name: Mutants
on:
pull_request:
paths-ignore: ["**.md", "docs/**"]
push:
branches: [main]
paths-ignore: ["**.md", "docs/**"]
workflow_dispatch:
inputs:
slice:
description: "Slice k to run (0 to SLICES-1)"
required: true
type: number
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
permissions:
contents: read
env:
CARGO_TERM_COLOR: always
JJ_VERSION: v0.45.1
jobs:
in-diff:
name: Mutants in changed code
if: github.event_name != 'workflow_dispatch'
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 0
- name: Rust toolchain (rust-toolchain.toml)
run: rustup toolchain install
- uses: Swatinem/rust-cache@v2
- uses: taiki-e/install-action@v2
with:
tool: cargo-mutants
- name: Install jj
run: |
set -euo pipefail
url="https://github.com/jj-vcs/jj/releases/download/${JJ_VERSION}/jj-${JJ_VERSION}-x86_64-unknown-linux-musl.tar.gz"
tmp="$(mktemp -d)"
curl -fsSL "$url" | tar xz -C "$tmp"
mkdir -p "$HOME/.local/bin"
install -m755 "$tmp/jj" "$HOME/.local/bin/jj"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
"$HOME/.local/bin/jj" --version
- name: Compute diff
run: |
set -euo pipefail
if [ "${{ github.event_name }}" = "pull_request" ]; then
base="origin/${{ github.base_ref }}"
else
before="${{ github.event.before }}"
if [ -z "$before" ] || [ "$before" = "0000000000000000000000000000000000000000" ]; then
base="HEAD~1"
else
base="$before"
fi
fi
echo "Diffing against $base"
# Force a/ b/ prefixes. With `diff.mnemonicPrefix` git writes i/ and w/,
# which cargo-mutants does not recognise: it matches no files and exits
# 0, a silent pass.
git -c diff.mnemonicPrefix=false -c diff.noprefix=false \
diff "$base".. > mutants.diff
wc -l mutants.diff
if [ -s mutants.diff ] && ! grep -q '^diff --git a/' mutants.diff; then
echo "::error::diff lacks a/ b/ prefixes; cargo-mutants would silently match nothing"
head -3 mutants.diff
exit 1
fi
- name: Mutants
id: mutants
run: |
set -euo pipefail
echo "ran" > mutants-status.txt
if [ ! -s mutants.diff ]; then
echo "empty" > mutants-status.txt
echo "::notice::empty diff, nothing to mutate"
exit 0
fi
selected=$(cargo mutants --list --in-diff mutants.diff | wc -l)
echo "$selected" > mutants-selected.txt
echo "Selected $selected mutants in changed regions"
if [ "$selected" -eq 0 ]; then
echo "none" > mutants-status.txt
echo "::notice::no mutants overlap the changed regions (test- or doc-only change)"
exit 0
fi
# A broad diff (a reformat, a mass rename) selects much of the tree and
# cannot finish inside the job timeout. Skip it loudly rather than
# report a partial run as a pass. The cap is sized from the measured
# rate in AGENTS.md so a run under it fits in the timeout.
MAX_SELECTED=25
if [ "$selected" -gt "$MAX_SELECTED" ]; then
echo "skipped" > mutants-status.txt
echo "::warning::Mutants skipped: $selected selected, over the limit of $MAX_SELECTED. Nothing was mutation-tested for this change."
exit 0
fi
# scripts/mutants-run.sh decides the verdict from mutants.out/missed.txt, so exit 3 passes only when nothing was missed, and stops cargo-mutants at a time budget.
# Exit codes: 2 a mutant was MISSED (the finding); 3 a mutant TIMED
# OUT (usually an injected infinite loop, a real result, so counted as
# caught and passed); 4 the unmutated baseline failed; 5 and 6 the diff
# does not match the tree or is malformed, which are bugs in "Compute
# diff", not in workon.
rc=0
scripts/mutants-run.sh --no-shuffle -vV --jobs 2 --in-diff mutants.diff || rc=$?
exit "$rc"
- name: Summarise
if: always()
run: |
set -euo pipefail
status="$(cat mutants-status.txt 2>/dev/null || echo unknown)"
selected="$(cat mutants-selected.txt 2>/dev/null || echo '?')"
{
echo '## Mutants in changed code'
echo
echo "_step outcome: ${{ steps.mutants.outcome }}_"
echo
case "$status" in
empty)
echo 'No diff to analyse. Nothing was mutation-tested.'
;;
none)
echo 'No mutants overlap the changed regions (a test- or docs-only'
echo 'change). Nothing was mutation-tested; cargo-mutants does not'
echo 'mutate test code.'
;;
skipped)
echo "**Skipped: $selected mutants selected, over the limit.**"
echo
echo 'A diff this broad touches many functions without necessarily'
echo 'changing behaviour. **Nothing was mutation-tested.** To cover it'
echo 'deliberately, shard a manual run:'
echo '`cargo mutants --in-diff <diff> --shard 0/4`'
;;
unknown)
echo 'The mutants step did not run: an earlier step failed (the jj'
echo 'install or "Compute diff"). **Nothing was mutation-tested.**'
;;
*)
if [ ! -d mutants.out ]; then
echo 'cargo-mutants started but produced no results directory; see'
echo 'the step log (exit 4, 5 and 6 are harness failures).'
echo '**Treat this as no coverage, not as a pass.**'
exit 0
fi
for kind in missed caught unviable timeout; do
f="mutants.out/$kind.txt"
n=0
[ -f "$f" ] && n=$(wc -l < "$f" | tr -d ' ')
echo "- **$kind**: $n"
done
if [ -s mutants.out/missed.txt ]; then
echo
echo '### Missed: a bug was inserted here and no test noticed'
echo
echo '```'
cat mutants.out/missed.txt
echo '```'
echo
echo 'Write a test that fails on it, or exclude it in'
echo '`.cargo/mutants.toml` with the reason no test can.'
fi
;;
esac
} >> "$GITHUB_STEP_SUMMARY"
- name: Archive mutants.out
if: always()
uses: actions/upload-artifact@v7
with:
name: mutants-in-diff
path: mutants.out
if-no-files-found: ignore
slice:
name: Mutants, rotating slice
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
timeout-minutes: 25
env:
SLICES: 24
MUTANTS_WALL: 15m
MUTANTS_BUDGET: 900
steps:
- uses: actions/checkout@v7
- name: Rust toolchain (rust-toolchain.toml)
run: rustup toolchain install
- uses: Swatinem/rust-cache@v2
- uses: taiki-e/install-action@v2
with:
tool: cargo-mutants
- name: Install jj
run: |
set -euo pipefail
url="https://github.com/jj-vcs/jj/releases/download/${JJ_VERSION}/jj-${JJ_VERSION}-x86_64-unknown-linux-musl.tar.gz"
tmp="$(mktemp -d)"
curl -fsSL "$url" | tar xz -C "$tmp"
mkdir -p "$HOME/.local/bin"
install -m755 "$tmp/jj" "$HOME/.local/bin/jj"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
"$HOME/.local/bin/jj" --version
- name: Mutants
run: |
set -euo pipefail
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
k="${{ inputs.slice }}"
else
k=$(( ${{ github.run_number }} % SLICES ))
fi
echo "$k" > mutants-slice.txt
echo "Slice $k of $SLICES"
# Recorded however the run ends (exit 2 on a miss included): the
# slice count was chosen from a local measurement, and this is the
# figure to re-choose it from.
trap 'echo "$SECONDS" > mutants-seconds.txt' EXIT
rc=0
# stdin closed: a mutant that reaches a prompt must fail, not wait.
# scripts/mutants-run.sh stops cargo-mutants at MUTANTS_BUDGET and exits 124.
scripts/mutants-run.sh --no-shuffle -vV --jobs 2 --shard "$k/$SLICES" < /dev/null || rc=$?
if [ "$rc" -eq 124 ]; then
echo "stopped" > mutants-stopped.txt
echo "::error::cargo-mutants still running after $MUTANTS_WALL; the newest logs in mutants.out/log are the mutants that hung"
ls -t mutants.out/log 2>/dev/null | head -5 || true
exit 1
fi
exit "$rc"
- name: Summarise
if: always()
run: |
set -euo pipefail
k="$(cat mutants-slice.txt 2>/dev/null || echo '?')"
secs="$(cat mutants-seconds.txt 2>/dev/null || echo '?')"
{
echo "## Mutants, slice $k of $SLICES"
echo
echo "_cargo mutants ran for ${secs}s_"
echo
if [ -f mutants-stopped.txt ]; then
echo "**Stopped after $MUTANTS_WALL: the counts below are partial.**"
echo 'The most recently written logs, which include the mutants that hung:'
echo
echo '```'
ls -t mutants.out/log 2>/dev/null | head -5 || true
echo '```'
echo
fi
if [ ! -d mutants.out ]; then
echo 'No results directory: the run failed before reporting.'
echo '**Treat this as no coverage, not as a pass.**'
exit 0
fi
for kind in missed caught unviable timeout; do
f="mutants.out/$kind.txt"
n=0
[ -f "$f" ] && n=$(wc -l < "$f" | tr -d ' ')
echo "- **$kind**: $n"
done
if [ -s mutants.out/missed.txt ]; then
echo
echo '### Missed in this slice'
echo
echo '```'
cat mutants.out/missed.txt
echo '```'
echo
echo 'Old code, not this push: give each a test or an exclusion'
echo 'with its reason, as for any other miss.'
fi
} >> "$GITHUB_STEP_SUMMARY"
- name: Archive mutants.out
if: always()
uses: actions/upload-artifact@v7
with:
name: mutants-slice
path: mutants.out
if-no-files-found: ignore