speccoder_runtime/
mutation.rs

1//! Mutation testing as evidence: after a passing verdict, the
2//! attempt's changed code is mutated and the tests run against each mutant.
3//! A mutant the tests do not catch ("survivor") is code the tests do not
4//! really check — the shape of a test written to pass rather than to verify.
5//!
6//! `[mutation] tool = "cargo-mutants"` runs `cargo mutants --in-diff` on
7//! the attempt's patch. `tool = "command"` runs any tool that writes the
8//! standard mutation-testing JSON report (Stryker, Infection, PIT with its
9//! plugin, …): `argv` may use `{diff}`, `{out}`, `{files}` (one argument per
10//! changed source file) and `{files_csv}`, and `report` names the JSON file.
11//!
12//! `mode = "report"` (default) records the result as evidence; `"require"`
13//! fails the verdict when mutants survive, and the retry digest names them.
14//!
15//! The same tool measures the acceptance set's strength: the
16//! mutants run against the acceptance command only ([`run_against`]).
17//! `suite = false` keeps the tool for that alone; `acceptance_argv` is the
18//! command tool's invocation then, with `{tests}` for the acceptance
19//! command.
20
21use std::path::{Path, PathBuf};
22use std::time::Duration;
23
24use serde::Deserialize;
25use serde_json::{Value, json};
26
27use crate::verify::process::{CommandRunner, Invocation, OUTPUT_CAP, scrubbed_env};
28
29/// Which tool.
30#[derive(Debug, Clone, PartialEq, Eq)]
31pub enum Tool {
32    /// `cargo mutants --in-diff`.
33    CargoMutants,
34    /// Any command writing the mutation-testing JSON report at `report`.
35    Command {
36        /// With placeholders.
37        argv: Vec<String>,
38        /// Relative to the project (or `{out}`).
39        report: String,
40    },
41}
42
43/// What survivors mean.
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
45pub enum Mode {
46    /// Evidence only.
47    Report,
48    /// Survivors fail the verdict (and are retried with the digest).
49    Require,
50}
51
52/// `[mutation]`.
53#[derive(Debug, Clone, PartialEq, Eq)]
54pub struct Config {
55    /// Which tool.
56    pub tool: Tool,
57    /// What survivors mean.
58    pub mode: Mode,
59    /// The whole run's limit.
60    pub timeout: Duration,
61    /// Run against the whole suite after a passing verdict; `false`:
62    /// the tool serves acceptance strength only.
63    pub suite: bool,
64    /// The command tool's argv against the acceptance tests (`{tests}`).
65    pub acceptance_argv: Option<Vec<String>>,
66}
67
68#[derive(Debug, Deserialize)]
69#[serde(deny_unknown_fields)]
70struct Raw {
71    tool: String,
72    #[serde(default)]
73    argv: Vec<String>,
74    report: Option<String>,
75    mode: Option<String>,
76    timeout_secs: Option<u64>,
77    suite: Option<bool>,
78    acceptance_argv: Option<Vec<String>>,
79}
80
81/// Reads `[mutation]`; `None` when absent.
82pub fn parse(text: &str) -> Result<Option<Config>, String> {
83    let table: toml::Table = toml::from_str(text).map_err(|e| e.to_string())?;
84    let Some(v) = table.get("mutation") else {
85        return Ok(None);
86    };
87    let raw: Raw = v
88        .clone()
89        .try_into()
90        .map_err(|e| format!("[mutation]: {e}"))?;
91    let tool = match raw.tool.as_str() {
92        "cargo-mutants" => Tool::CargoMutants,
93        "command" => match (raw.argv.is_empty(), raw.report) {
94            (false, Some(report)) => Tool::Command {
95                argv: raw.argv,
96                report,
97            },
98            _ => return Err("[mutation]: tool = \"command\" needs `argv` and `report`".to_owned()),
99        },
100        o => {
101            return Err(format!(
102                "[mutation] tool = \"{o}\" (cargo-mutants or command)"
103            ));
104        }
105    };
106    let mode = match raw.mode.as_deref() {
107        None | Some("report") => Mode::Report,
108        Some("require") => Mode::Require,
109        Some(o) => return Err(format!("[mutation] mode = \"{o}\" (report or require)")),
110    };
111    if raw.acceptance_argv.is_some() && tool == Tool::CargoMutants {
112        return Err(
113            "[mutation]: acceptance_argv is for tool = \"command\"; cargo-mutants runs the acceptance command itself".to_owned(),
114        );
115    }
116    Ok(Some(Config {
117        tool,
118        mode,
119        timeout: Duration::from_secs(raw.timeout_secs.unwrap_or(900)),
120        suite: raw.suite.unwrap_or(true),
121        acceptance_argv: raw.acceptance_argv.filter(|a| !a.is_empty()),
122    }))
123}
124
125/// Source files a patch changes (`+++ b/<path>`), tests excluded.
126#[must_use]
127pub fn changed_sources(patch: &str) -> Vec<String> {
128    let mut out: Vec<String> = patch
129        .lines()
130        .filter_map(|l| l.strip_prefix("+++ b/"))
131        .map(str::to_owned)
132        .filter(|p| !crate::tools::paths::is_test_path(p))
133        .collect();
134    out.dedup();
135    out
136}
137
138/// `patch` with only the sections of changed source files: test files —
139/// the acceptance tests among them — removed.
140#[must_use]
141pub fn source_patch(patch: &str) -> String {
142    let mut out = String::new();
143    let mut keep = false;
144    for line in patch.split_inclusive('\n') {
145        if let Some(rest) = line.strip_prefix("diff --git ") {
146            let path = rest.trim_end().rsplit(" b/").next().unwrap_or_default();
147            keep = !crate::tools::paths::is_test_path(path);
148        }
149        if keep {
150            out.push_str(line);
151        }
152    }
153    out
154}
155
156/// The result of `cargo mutants` (`mutants.out/outcomes.json`).
157#[must_use]
158pub fn from_cargo_mutants(outcomes: &Value) -> Value {
159    let n = |k: &str| outcomes[k].as_u64().unwrap_or(0);
160    let survivors: Vec<String> = outcomes["outcomes"]
161        .as_array()
162        .into_iter()
163        .flatten()
164        .filter(|o| o["summary"] == "MissedMutant")
165        .filter_map(|o| o["scenario"]["Mutant"]["name"].as_str().map(str::to_owned))
166        .collect();
167    json!({
168        "tool": "cargo-mutants",
169        "caught": n("caught"),
170        "missed": n("missed"),
171        "timeout": n("timeout"),
172        "unviable": n("unviable"),
173        "survivors": survivors,
174    })
175}
176
177/// The result in the mutation-testing report schema (Stryker and others):
178/// `files.<path>.mutants[].status` — `Killed`, `Survived`, `NoCoverage`,
179/// `Timeout`, `CompileError`, `RuntimeError`, `Ignored`.
180#[must_use]
181pub fn from_report_schema(report: &Value, tool: &str) -> Value {
182    let (mut caught, mut missed, mut timeout, mut unviable) = (0u64, 0u64, 0u64, 0u64);
183    let mut survivors = Vec::new();
184    for (file, f) in report["files"].as_object().into_iter().flatten() {
185        for m in f["mutants"].as_array().into_iter().flatten() {
186            match m["status"].as_str().unwrap_or_default() {
187                "Killed" => caught += 1,
188                "Survived" | "NoCoverage" => {
189                    missed += 1;
190                    survivors.push(format!(
191                        "{file}:{}: {} → {}",
192                        m["location"]["start"]["line"],
193                        m["mutatorName"].as_str().unwrap_or("mutant"),
194                        m["replacement"].as_str().unwrap_or("?")
195                    ));
196                }
197                "Timeout" => timeout += 1,
198                "CompileError" | "RuntimeError" => unviable += 1,
199                _ => {}
200            }
201        }
202    }
203    json!({ "tool": tool, "caught": caught, "missed": missed, "timeout": timeout, "unviable": unviable, "survivors": survivors })
204}
205
206fn expand(argv: &[String], diff: &Path, out: &Path, files: &[String], tests: &str) -> Vec<String> {
207    let mut res = Vec::new();
208    for a in argv {
209        match a.as_str() {
210            "{files}" => res.extend(files.iter().cloned()),
211            _ => res.push(
212                a.replace("{diff}", &diff.to_string_lossy())
213                    .replace("{out}", &out.to_string_lossy())
214                    .replace("{files_csv}", &files.join(","))
215                    .replace("{tests}", tests),
216            ),
217        }
218    }
219    res
220}
221
222/// What `cargo mutants` passes to `cargo test` to run only `acceptance`
223/// (`cargo test --test accept` → `--test accept`); `None` when the
224/// acceptance command is not a `cargo test`.
225#[must_use]
226pub fn cargo_test_args(acceptance: &[String]) -> Option<Vec<String>> {
227    match acceptance {
228        [cargo, test, rest @ ..] if cargo == "cargo" && test == "test" => Some(rest.to_vec()),
229        _ => None,
230    }
231}
232
233/// The argv that mutates the changed code and runs `acceptance` against
234/// each mutant; `Err`: why it cannot be measured.
235pub fn acceptance_invocation(
236    config: &Config,
237    acceptance: &[String],
238    diff: &Path,
239    out: &Path,
240    files: &[String],
241) -> Result<Vec<String>, String> {
242    match &config.tool {
243        Tool::CargoMutants => {
244            let args = cargo_test_args(acceptance).ok_or_else(|| {
245                format!(
246                    "the acceptance command `{}` is not `cargo test …`, which cargo-mutants needs",
247                    acceptance.join(" ")
248                )
249            })?;
250            let mut argv = cargo_mutants_argv(diff, out);
251            if !args.is_empty() {
252                argv.push("--".to_owned());
253                argv.extend(args);
254            }
255            Ok(argv)
256        }
257        Tool::Command { .. } => config
258            .acceptance_argv
259            .as_ref()
260            .map(|a| {
261                expand(
262                    a,
263                    diff,
264                    out,
265                    files,
266                    &crate::approvals::present::shell_quote(acceptance),
267                )
268            })
269            .ok_or_else(|| "[mutation] acceptance_argv is not set for the command tool".to_owned()),
270    }
271}
272
273fn cargo_mutants_argv(diff: &Path, out: &Path) -> Vec<String> {
274    vec![
275        "cargo".to_owned(),
276        "mutants".to_owned(),
277        "--in-diff".to_owned(),
278        diff.to_string_lossy().into_owned(),
279        "--output".to_owned(),
280        out.to_string_lossy().into_owned(),
281        "--no-shuffle".to_owned(),
282    ]
283}
284
285/// Runs `config`'s tool on `patch` in `project`; `None` when the patch
286/// changes no source file. Scratch files go to `tmp`, outside the worktree.
287pub fn run(
288    config: &Config,
289    project: &Path,
290    patch: &str,
291    tmp: &Path,
292    runner: &dyn CommandRunner,
293) -> Option<Value> {
294    run_with(config, project, patch, tmp, runner, None)
295}
296
297/// [`run`] with the mutants judged by `acceptance` alone: the
298/// result as [`run`]'s, or `{"error"}` when it cannot be measured.
299pub fn run_against(
300    config: &Config,
301    project: &Path,
302    patch: &str,
303    tmp: &Path,
304    runner: &dyn CommandRunner,
305    acceptance: &[String],
306) -> Option<Value> {
307    run_with(config, project, patch, tmp, runner, Some(acceptance))
308}
309
310fn run_with(
311    config: &Config,
312    project: &Path,
313    patch: &str,
314    tmp: &Path,
315    runner: &dyn CommandRunner,
316    acceptance: Option<&[String]>,
317) -> Option<Value> {
318    let files = changed_sources(patch);
319    if files.is_empty() {
320        return None;
321    }
322    let dir: PathBuf = tmp.join(format!(
323        "mutation-{}",
324        std::time::SystemTime::now()
325            .duration_since(std::time::UNIX_EPOCH)
326            .map_or(0, |d| d.as_nanos())
327    ));
328    let error = |e: String| Some(json!({ "error": e }));
329    if let Err(e) = std::fs::create_dir_all(&dir) {
330        return error(e.to_string());
331    }
332    let diff = dir.join("attempt.patch");
333    if let Err(e) = std::fs::write(&diff, patch) {
334        return error(e.to_string());
335    }
336    let out = dir.join("out");
337    let argv = match (acceptance, &config.tool) {
338        (Some(a), _) => match acceptance_invocation(config, a, &diff, &out, &files) {
339            Ok(argv) => argv,
340            Err(e) => {
341                let _ = std::fs::remove_dir_all(&dir);
342                return error(e);
343            }
344        },
345        (None, Tool::CargoMutants) => cargo_mutants_argv(&diff, &out),
346        (None, Tool::Command { argv, .. }) => expand(argv, &diff, &out, &files, ""),
347    };
348    let ran = runner.run(&Invocation {
349        argv: argv.clone(),
350        cwd: project.to_path_buf(),
351        env: scrubbed_env(std::env::vars()),
352        timeout: config.timeout,
353        output_cap: OUTPUT_CAP,
354    });
355    if let Some(e) = ran.spawn_error {
356        return error(e);
357    }
358    if ran.timed_out {
359        return error(format!(
360            "`{}` did not finish within {} s",
361            argv.join(" "),
362            config.timeout.as_secs()
363        ));
364    }
365    let report_path = match &config.tool {
366        Tool::CargoMutants => out.join("mutants.out/outcomes.json"),
367        Tool::Command { report, .. } => {
368            let r = report.replace("{out}", &out.to_string_lossy());
369            let p = PathBuf::from(&r);
370            if p.is_absolute() { p } else { project.join(r) }
371        }
372    };
373    let parsed = std::fs::read_to_string(&report_path)
374        .ok()
375        .and_then(|t| serde_json::from_str::<Value>(&t).ok());
376    let _ = std::fs::remove_dir_all(&dir);
377    let Some(report) = parsed else {
378        let tail = String::from_utf8_lossy(&ran.stderr.kept);
379        let tail: String = tail
380            .lines()
381            .rev()
382            .take(5)
383            .collect::<Vec<_>>()
384            .into_iter()
385            .rev()
386            .collect::<Vec<_>>()
387            .join(" | ");
388        return error(format!(
389            "`{}` exited {:?} and wrote no report: {tail}",
390            argv.join(" "),
391            ran.exit_code
392        ));
393    };
394    let mut result = match &config.tool {
395        Tool::CargoMutants => from_cargo_mutants(&report),
396        Tool::Command { argv, .. } => from_report_schema(&report, &argv[0]),
397    };
398    result["files"] = json!(files);
399    Some(result)
400}
401
402/// The verdict reason for survivors (`mode = "require"`), if any; a tool
403/// that could not run is a failure too (no evidence is not good evidence).
404#[must_use]
405pub fn failure(result: &Value) -> Option<String> {
406    if let Some(e) = result["error"].as_str() {
407        return Some(format!("mutation testing could not run: {e}"));
408    }
409    let missed = result["missed"].as_u64().unwrap_or(0);
410    if missed == 0 {
411        return None;
412    }
413    let total = missed + result["caught"].as_u64().unwrap_or(0);
414    let names: Vec<&str> = result["survivors"]
415        .as_array()
416        .into_iter()
417        .flatten()
418        .filter_map(Value::as_str)
419        .take(8)
420        .collect();
421    Some(format!(
422        "mutation: {missed} of {total} mutants of the changed code survived the tests — strengthen the tests so they fail for: {}",
423        names.join("; ")
424    ))
425}
426
427/// The commit trailer value.
428#[must_use]
429pub fn trailer(result: &Value) -> Option<String> {
430    result["caught"].as_u64().map(|c| {
431        format!(
432            "Mutation: caught={c} missed={} timeout={} unviable={}\n",
433            result["missed"].as_u64().unwrap_or(0),
434            result["timeout"].as_u64().unwrap_or(0),
435            result["unviable"].as_u64().unwrap_or(0)
436        )
437    })
438}
439
440#[cfg(test)]
441mod tests {
442    use super::*;
443
444    #[test]
445    fn config_and_reports_parse() {
446        assert!(parse("").unwrap_or_default().is_none());
447        let c = parse("[mutation]\ntool = \"cargo-mutants\"\nmode = \"require\"\n")
448            .unwrap_or_default()
449            .unwrap_or_else(|| panic!("config"));
450        assert_eq!((c.tool, c.mode), (Tool::CargoMutants, Mode::Require));
451        assert!(parse("[mutation]\ntool = \"command\"\n").is_err());
452
453        let cm = json!({ "caught": 6, "missed": 1, "timeout": 0, "unviable": 2, "outcomes": [
454            { "summary": "Success", "scenario": "Baseline" },
455            { "summary": "MissedMutant", "scenario": { "Mutant": { "name": "src/lib.rs:2:10: replace < with <= in clamp" } } } ] });
456        let r = from_cargo_mutants(&cm);
457        assert_eq!(
458            r["survivors"],
459            json!(["src/lib.rs:2:10: replace < with <= in clamp"])
460        );
461        assert!(failure(&r).unwrap_or_default().contains("1 of 7 mutants"));
462        assert_eq!(
463            trailer(&r).unwrap_or_default(),
464            "Mutation: caught=6 missed=1 timeout=0 unviable=2\n"
465        );
466
467        let st = json!({ "files": { "src/a.ts": { "mutants": [
468            { "status": "Killed" },
469            { "status": "Survived", "mutatorName": "EqualityOperator", "replacement": "a <= b", "location": { "start": { "line": 3 } } } ] } } });
470        let r = from_report_schema(&st, "npx");
471        assert_eq!(
472            (r["caught"].as_u64(), r["missed"].as_u64()),
473            (Some(1), Some(1))
474        );
475        assert_eq!(r["survivors"][0], "src/a.ts:3: EqualityOperator → a <= b");
476        assert_eq!(
477            changed_sources("+++ b/src/lib.rs\n+++ b/tests/t.rs\n"),
478            ["src/lib.rs"]
479        );
480        let p = "diff --git a/src/lib.rs b/src/lib.rs\n+++ b/src/lib.rs\n+x\ndiff --git a/tests/acceptance_t5.rs b/tests/acceptance_t5.rs\n+++ b/tests/acceptance_t5.rs\n+secret\n";
481        assert_eq!(
482            source_patch(p),
483            "diff --git a/src/lib.rs b/src/lib.rs\n+++ b/src/lib.rs\n+x\n"
484        );
485    }
486
487    #[test]
488    fn mutants_run_against_the_acceptance_command_alone() {
489        let v = |a: &[&str]| a.iter().map(|s| (*s).to_owned()).collect::<Vec<_>>();
490        let c = parse("[mutation]\ntool = \"cargo-mutants\"\nsuite = false\n")
491            .unwrap_or_default()
492            .unwrap_or_else(|| panic!("config"));
493        assert!(!c.suite);
494        let argv = acceptance_invocation(
495            &c,
496            &v(&["cargo", "test", "--test", "accept_t005"]),
497            Path::new("/d"),
498            Path::new("/o"),
499            &[],
500        )
501        .unwrap_or_default();
502        assert_eq!(argv[..2], v(&["cargo", "mutants"]));
503        assert_eq!(argv[argv.len() - 3..], v(&["--", "--test", "accept_t005"]));
504        assert!(
505            acceptance_invocation(
506                &c,
507                &v(&["pytest", "t.py"]),
508                Path::new("/d"),
509                Path::new("/o"),
510                &[]
511            )
512            .unwrap_err()
513            .contains("not `cargo test")
514        );
515        let c = parse("[mutation]\ntool = \"command\"\nargv = [\"stryker\"]\nreport = \"r.json\"\nacceptance_argv = [\"stryker\", \"--mutate\", \"{files_csv}\", \"--command\", \"{tests}\"]\n")
516            .unwrap_or_default()
517            .unwrap_or_else(|| panic!("config"));
518        let argv = acceptance_invocation(
519            &c,
520            &v(&["npx", "vitest", "run", "tests/accept t5.test.ts"]),
521            Path::new("/d"),
522            Path::new("/o"),
523            &v(&["src/a.ts", "src/b.ts"]),
524        )
525        .unwrap_or_default();
526        assert_eq!(
527            argv,
528            v(&[
529                "stryker",
530                "--mutate",
531                "src/a.ts,src/b.ts",
532                "--command",
533                "npx vitest run 'tests/accept t5.test.ts'"
534            ])
535        );
536        assert!(
537            parse("[mutation]\ntool = \"cargo-mutants\"\nacceptance_argv = [\"x\"]\n").is_err()
538        );
539    }
540}
speccoder_runtime/mutation.rs

speccoder_runtime/
mutation.rs
