testing_conventions/
mutation.rs

1//! Mutation testing for Rust (`unit mutation --language rust`, #201) — the rung
2//! above coverage. A test that *runs* a line still passes if you delete its
3//! assertions; a surviving mutant proves it. This module wraps
4//! [cargo-mutants](https://github.com/sourcefrog/cargo-mutants): it runs the engine,
5//! reads its `outcomes.json`, and reports the **surviving** mutants the suite failed
6//! to catch.
7//!
8//! The gate is **binary, not a percentage** (equivalent mutants make a fixed score
9//! unreachable, and a score isn't comparable across engines) and on by default: any
10//! *un-exempted* surviving mutant is a finding. This module stays a pure measurement —
11//! [`measure_rust`] returns the survivors and [`unexplained_survivors`] is the pure
12//! core over a parsed report; the CLI layer turns a non-empty result into the failure.
13//!
14//! Diff-scoping (`--base`) is delegated to cargo-mutants' own `--in-diff`: the
15//! `<base>...HEAD` diff is written out and passed through, so only mutants on changed
16//! lines are tested ("no unexplained surviving mutant on the lines you touched").
17
18use std::path::{Path, PathBuf};
19use std::process::Command;
20use std::sync::atomic::{AtomicU64, Ordering};
21
22use anyhow::{bail, Context, Result};
23use serde::Deserialize;
24
25/// A surviving mutant — a mutation the unit suite ran but failed to catch.
26#[derive(Debug, Clone, PartialEq, Eq)]
27pub struct Survivor {
28    /// The mutated file, as cargo-mutants reports it (crate-root-relative, `/`-separated).
29    pub file: String,
30    /// The 1-based line the mutation starts on.
31    pub line: u32,
32    /// cargo-mutants' human description (e.g. `replace > with == in is_positive`).
33    pub description: String,
34}
35
36/// A cargo-mutants `outcomes.json` export, pared to what the rule reads. Unmodeled
37/// fields (`total_mutants`, `caught`, timings, …) are ignored.
38#[derive(Debug, Clone, Deserialize)]
39pub struct MutantsReport {
40    pub outcomes: Vec<MutantOutcome>,
41}
42
43/// One scenario's outcome. `summary` is cargo-mutants' result word — `Success` for the
44/// unmutated baseline, `CaughtMutant` / `MissedMutant` (and `Timeout` / `Unviable`)
45/// for each mutant.
46#[derive(Debug, Clone, Deserialize)]
47pub struct MutantOutcome {
48    pub summary: String,
49    pub scenario: Scenario,
50}
51
52/// The scenario a result came from: the unmutated baseline, or one mutant. Matches
53/// cargo-mutants' externally-tagged JSON (`"Baseline"` vs `{"Mutant": {…}}`).
54#[derive(Debug, Clone, Deserialize)]
55pub enum Scenario {
56    Baseline,
57    Mutant(MutantInfo),
58}
59
60/// The mutant a scenario describes, pared to the location + description the report
61/// needs. cargo-mutants also carries `function`, `genre`, `package`, `replacement`;
62/// those are ignored.
63#[derive(Debug, Clone, Deserialize)]
64pub struct MutantInfo {
65    pub file: String,
66    pub span: Span,
67    pub name: String,
68}
69
70/// A source span; only the start line is read.
71#[derive(Debug, Clone, Deserialize)]
72pub struct Span {
73    pub start: LineCol,
74}
75
76/// A line/column position; only the line is read.
77#[derive(Debug, Clone, Deserialize)]
78pub struct LineCol {
79    pub line: u32,
80}
81
82/// Parse a cargo-mutants `outcomes.json` export.
83pub fn parse_mutants_report(json: &str) -> Result<MutantsReport> {
84    serde_json::from_str(json).context("parsing cargo-mutants outcomes.json")
85}
86
87/// The surviving mutants not lifted by a `mutation` exemption — the rule's findings.
88///
89/// A survivor is a `MissedMutant` outcome (the suite ran the mutated code but no test
90/// failed). `exempt` is the resolved set of `mutation`-rule exempt paths (crate-root
91/// relative); a survivor in an exempt file is dropped (an equivalent or deliberately
92/// defensive mutation, lifted with a reason). `Timeout` / `Unviable` are *not*
93/// survivors — a timeout is inconclusive, not a pass, and an unviable mutant never
94/// compiled.
95pub fn unexplained_survivors(report: &MutantsReport, exempt: &[String]) -> Vec<Survivor> {
96    report
97        .outcomes
98        .iter()
99        .filter_map(|outcome| {
100            if outcome.summary != "MissedMutant" {
101                return None;
102            }
103            let Scenario::Mutant(mutant) = &outcome.scenario else {
104                return None;
105            };
106            if exempt.iter().any(|path| path == &mutant.file) {
107                return None;
108            }
109            Some(Survivor {
110                file: mutant.file.clone(),
111                line: mutant.span.start.line,
112                description: mutant.name.clone(),
113            })
114        })
115        .collect()
116}
117
118/// Run cargo-mutants over the crate at `root` and return its un-exempted survivors.
119///
120/// With `base` set, only mutants on the `<base>...HEAD` changed lines are tested (via
121/// cargo-mutants' `--in-diff`); without it, the whole crate. `exempt` is the resolved
122/// `mutation`-rule exempt paths. `cargo-mutants` must be installed.
123pub fn measure_rust(root: &Path, exempt: &[String], base: Option<&str>) -> Result<Vec<Survivor>> {
124    let out = MutantsOut::new();
125    let diff = match base {
126        Some(base) => Some(write_base_diff(root, base, &out)?),
127        None => None,
128    };
129    run_cargo_mutants(root, &out.0, diff.as_deref())?;
130    let outcomes = out.0.join("mutants.out").join("outcomes.json");
131    let json = std::fs::read_to_string(&outcomes).with_context(|| {
132        format!(
133            "reading cargo-mutants outcomes at `{}` — the run wrote none",
134            outcomes.display()
135        )
136    })?;
137    let report = parse_mutants_report(&json)?;
138    Ok(unexplained_survivors(&report, exempt))
139}
140
141/// A unique temp dir for one cargo-mutants run's `--output`, removed on drop so the
142/// scanned crate stays pristine and parallel runs don't collide.
143struct MutantsOut(PathBuf);
144
145impl MutantsOut {
146    fn new() -> Self {
147        static COUNTER: AtomicU64 = AtomicU64::new(0);
148        let name = format!(
149            "testing-conventions-mutants-{}-{}",
150            std::process::id(),
151            COUNTER.fetch_add(1, Ordering::Relaxed),
152        );
153        MutantsOut(std::env::temp_dir().join(name))
154    }
155}
156
157impl Drop for MutantsOut {
158    fn drop(&mut self) {
159        let _ = std::fs::remove_dir_all(&self.0);
160    }
161}
162
163/// Write the `<base>...HEAD` diff cargo-mutants' `--in-diff` scopes to, returning its path.
164fn write_base_diff(root: &Path, base: &str, out: &MutantsOut) -> Result<PathBuf> {
165    let range = format!("{base}...HEAD");
166    let output = Command::new("git")
167        .current_dir(root)
168        .args(["diff", &range])
169        .output()
170        .context("running `git diff` for `--base` (is git installed?)")?;
171    if !output.status.success() {
172        bail!(
173            "git diff {range} failed: {}",
174            String::from_utf8_lossy(&output.stderr)
175        );
176    }
177    std::fs::create_dir_all(&out.0).context("creating the mutants output dir")?;
178    let path = out.0.join("base.diff");
179    std::fs::write(&path, &output.stdout).context("writing the base diff")?;
180    Ok(path)
181}
182
183/// Run `cargo mutants --output <out> [--in-diff <diff>]` in `root`.
184///
185/// cargo-mutants exits `0` when every mutant is caught and `2` when some survive (or
186/// time out / are unviable) — both are normal here, since survivors are the rule's
187/// *output*, not an error. Any other code (usage error, or a baseline that didn't
188/// build/pass) is fatal. As with the coverage run, the outer instrumentation env is
189/// stripped so a nested run (this rule's own tests under `cargo llvm-cov`) doesn't
190/// re-enter the rustc wrapper and hang.
191fn run_cargo_mutants(root: &Path, out: &Path, in_diff: Option<&Path>) -> Result<()> {
192    let mut command = Command::new("cargo");
193    command
194        .current_dir(root)
195        .arg("mutants")
196        .arg("--output")
197        .arg(out);
198    if let Some(diff) = in_diff {
199        command.arg("--in-diff").arg(diff);
200    }
201    for var in [
202        "RUSTFLAGS",
203        "CARGO_ENCODED_RUSTFLAGS",
204        "RUSTDOCFLAGS",
205        "CARGO_ENCODED_RUSTDOCFLAGS",
206        "LLVM_PROFILE_FILE",
207        "CARGO_LLVM_COV",
208        "CARGO_LLVM_COV_SHOW_ENV",
209        "CARGO_LLVM_COV_TARGET_DIR",
210        "CARGO_LLVM_COV_BUILD_DIR",
211        "RUSTC_WRAPPER",
212        "RUSTC_WORKSPACE_WRAPPER",
213        "__CARGO_LLVM_COV_RUSTC_WRAPPER",
214        "__CARGO_LLVM_COV_RUSTC_WRAPPER_RUSTFLAGS",
215        "__CARGO_LLVM_COV_RUSTC_WRAPPER_CRATE_NAMES",
216    ] {
217        command.env_remove(var);
218    }
219    let output = command
220        .output()
221        .context("running `cargo mutants` (is cargo-mutants installed?)")?;
222    match output.status.code() {
223        // 0 = all caught, 2 = some survived/timed out: both produce a report to read.
224        Some(0) | Some(2) => Ok(()),
225        _ => bail!(
226            "cargo-mutants did not run cleanly in `{}` (baseline build/test failure?):\n{}{}",
227            root.display(),
228            String::from_utf8_lossy(&output.stdout),
229            String::from_utf8_lossy(&output.stderr),
230        ),
231    }
232}
233
234#[cfg(test)]
235mod tests {
236    use super::*;
237
238    // A pared `outcomes.json`: the baseline, one missed mutant, and one caught — the
239    // real shape (externally-tagged `scenario`, extra fields the rule ignores).
240    const SAMPLE: &str = r#"{
241        "outcomes": [
242            {"scenario": "Baseline", "summary": "Success",
243             "phase_results": []},
244            {"scenario": {"Mutant": {"file": "src/lib.rs", "package": "p", "genre": "FnValue",
245                "replacement": "true", "name": "src/lib.rs:7:7: replace > with == in is_positive",
246                "function": {"function_name": "is_positive"},
247                "span": {"start": {"line": 7, "column": 7}, "end": {"line": 7, "column": 8}}}},
248             "summary": "MissedMutant"},
249            {"scenario": {"Mutant": {"file": "src/other.rs", "package": "p", "genre": "FnValue",
250                "replacement": "0", "name": "src/other.rs:3:5: replace add -> i32 with 0",
251                "span": {"start": {"line": 3, "column": 5}, "end": {"line": 3, "column": 9}}}},
252             "summary": "CaughtMutant"}
253        ],
254        "total_mutants": 2
255    }"#;
256
257    #[test]
258    fn parses_the_outcomes_export() {
259        let report = parse_mutants_report(SAMPLE).expect("valid outcomes.json");
260        assert_eq!(report.outcomes.len(), 3);
261        assert!(matches!(report.outcomes[0].scenario, Scenario::Baseline));
262    }
263
264    #[test]
265    fn collects_only_missed_mutants_as_survivors() {
266        let report = parse_mutants_report(SAMPLE).unwrap();
267        let survivors = unexplained_survivors(&report, &[]);
268        // Only the MissedMutant — the baseline and the CaughtMutant are not survivors.
269        assert_eq!(survivors.len(), 1);
270        assert_eq!(survivors[0].file, "src/lib.rs");
271        assert_eq!(survivors[0].line, 7);
272        assert!(survivors[0].description.contains("replace > with =="));
273    }
274
275    #[test]
276    fn an_exemption_drops_a_survivor_in_that_file() {
277        let report = parse_mutants_report(SAMPLE).unwrap();
278        let exempt = vec!["src/lib.rs".to_string()];
279        assert!(unexplained_survivors(&report, &exempt).is_empty());
280    }
281
282    #[test]
283    fn an_exemption_on_another_file_leaves_the_survivor() {
284        let report = parse_mutants_report(SAMPLE).unwrap();
285        let exempt = vec!["src/elsewhere.rs".to_string()];
286        assert_eq!(unexplained_survivors(&report, &exempt).len(), 1);
287    }
288}
testing_conventions/mutation.rs

testing_conventions/
mutation.rs
