dev_mutate/
lib.rs

1//! # dev-mutate
2//!
3//! Mutation testing for Rust. Wraps [`cargo-mutants`][cargo-mutants]
4//! and emits results as [`dev_report::Report`].
5//!
6//! Mutation testing makes small deliberate changes to your code —
7//! flipping `<` to `>`, changing `+` to `-`, removing a `return`. It
8//! then runs your tests against each mutation. A test suite that
9//! catches the mutations *kills* them; surviving mutations are
10//! evidence the suite isn't actually testing that behavior.
11//!
12//! ## Kill rate
13//!
14//! ```text
15//! kill_pct = killed / (killed + survived) * 100
16//! ```
17//!
18//! Timeouts MUST NOT count toward either numerator or denominator —
19//! they don't reflect test quality (REPS § 3).
20//!
21//! ## Quick example
22//!
23//! ```no_run
24//! use dev_mutate::{MutateRun, MutateThreshold};
25//!
26//! let run = MutateRun::new("my-crate", "0.1.0");
27//! let result = run.execute().unwrap();
28//!
29//! let threshold = MutateThreshold::min_kill_pct(70.0);
30//! let check = result.into_check_result(threshold);
31//! ```
32//!
33//! ## Requirements
34//!
35//! ```text
36//! cargo install cargo-mutants
37//! ```
38//!
39//! The crate detects absence and surfaces
40//! [`MutateError::ToolNotInstalled`] without panicking.
41//!
42//! [cargo-mutants]: https://crates.io/crates/cargo-mutants
43
44#![cfg_attr(docsrs, feature(doc_cfg))]
45#![warn(missing_docs)]
46#![warn(rust_2018_idioms)]
47
48use std::collections::BTreeMap;
49use std::path::PathBuf;
50use std::time::Duration;
51
52use dev_report::{CheckResult, Evidence, Severity};
53use serde::{Deserialize, Serialize};
54
55mod producer;
56mod runner;
57
58pub use producer::MutateProducer;
59
60// ---------------------------------------------------------------------------
61// MutateRun
62// ---------------------------------------------------------------------------
63
64/// Configuration for a mutation testing run.
65///
66/// # Example
67///
68/// ```no_run
69/// use dev_mutate::MutateRun;
70/// use std::time::Duration;
71///
72/// let run = MutateRun::new("my-crate", "0.1.0")
73///     .workspace()
74///     .jobs(4)
75///     .timeout(Duration::from_secs(120))
76///     .exclude_re(r"^src/generated/");
77///
78/// let _result = run.execute().unwrap();
79/// ```
80#[derive(Debug, Clone)]
81pub struct MutateRun {
82    name: String,
83    version: String,
84    workdir: Option<PathBuf>,
85    workspace: bool,
86    jobs: Option<u32>,
87    timeout: Option<Duration>,
88    exclude_re: Vec<String>,
89    file_filters: Vec<String>,
90    allow_list: Vec<String>,
91}
92
93impl MutateRun {
94    /// Begin a new mutation testing run.
95    ///
96    /// `name` and `version` are descriptive — they identify the
97    /// subject in the produced `Report`.
98    pub fn new(name: impl Into<String>, version: impl Into<String>) -> Self {
99        Self {
100            name: name.into(),
101            version: version.into(),
102            workdir: None,
103            workspace: false,
104            jobs: None,
105            timeout: None,
106            exclude_re: Vec::new(),
107            file_filters: Vec::new(),
108            allow_list: Vec::new(),
109        }
110    }
111
112    /// Run `cargo mutants` from `dir` instead of the current directory.
113    pub fn in_dir(mut self, dir: impl Into<PathBuf>) -> Self {
114        self.workdir = Some(dir.into());
115        self
116    }
117
118    /// Pass `--workspace` so every workspace member is mutated.
119    pub fn workspace(mut self) -> Self {
120        self.workspace = true;
121        self
122    }
123
124    /// Pass `--jobs <N>` (parallel mutation runs). Maps to cargo-mutants'
125    /// own job limit.
126    pub fn jobs(mut self, n: u32) -> Self {
127        self.jobs = Some(n);
128        self
129    }
130
131    /// Per-mutant timeout (passed through as `--timeout <secs>`).
132    pub fn timeout(mut self, d: Duration) -> Self {
133        self.timeout = Some(d);
134        self
135    }
136
137    /// Skip files matching the given regex. Repeatable; maps to
138    /// `--exclude-re <pattern>` per invocation.
139    pub fn exclude_re(mut self, pattern: impl Into<String>) -> Self {
140        self.exclude_re.push(pattern.into());
141        self
142    }
143
144    /// Restrict to files matching the given glob / path. Repeatable;
145    /// maps to `--file <pattern>` per invocation.
146    pub fn file(mut self, pattern: impl Into<String>) -> Self {
147        self.file_filters.push(pattern.into());
148        self
149    }
150
151    /// Suppress a known-survivor by descriptor. The match is on the
152    /// `description` field of `SurvivingMutant` (e.g. `"replace + with -"`).
153    pub fn allow(mut self, description: impl Into<String>) -> Self {
154        self.allow_list.push(description.into());
155        self
156    }
157
158    /// Bulk version of [`allow`](Self::allow).
159    pub fn allow_all<I, S>(mut self, descriptions: I) -> Self
160    where
161        I: IntoIterator<Item = S>,
162        S: Into<String>,
163    {
164        self.allow_list
165            .extend(descriptions.into_iter().map(Into::into));
166        self
167    }
168
169    /// Subject name passed in via [`new`](Self::new).
170    pub fn subject(&self) -> &str {
171        &self.name
172    }
173
174    /// Subject version passed in via [`new`](Self::new).
175    pub fn subject_version(&self) -> &str {
176        &self.version
177    }
178
179    /// Execute the run.
180    ///
181    /// Spawns `cargo mutants --json` with the configured flags. Tool
182    /// absence, subprocess failure, and parse failure surface as
183    /// typed [`MutateError`] variants. No panics.
184    pub fn execute(&self) -> Result<MutateResult, MutateError> {
185        runner::run(self)
186    }
187
188    pub(crate) fn workdir_path(&self) -> Option<&std::path::Path> {
189        self.workdir.as_deref()
190    }
191
192    pub(crate) fn workspace_flag(&self) -> bool {
193        self.workspace
194    }
195
196    pub(crate) fn jobs_value(&self) -> Option<u32> {
197        self.jobs
198    }
199
200    pub(crate) fn timeout_value(&self) -> Option<Duration> {
201        self.timeout
202    }
203
204    pub(crate) fn exclude_re_view(&self) -> &[String] {
205        &self.exclude_re
206    }
207
208    pub(crate) fn file_filters_view(&self) -> &[String] {
209        &self.file_filters
210    }
211
212    pub(crate) fn allow_list_view(&self) -> &[String] {
213        &self.allow_list
214    }
215}
216
217// ---------------------------------------------------------------------------
218// SurvivingMutant
219// ---------------------------------------------------------------------------
220
221/// A surviving mutant — one the test suite did NOT catch.
222#[derive(Debug, Clone, Serialize, Deserialize)]
223pub struct SurvivingMutant {
224    /// Source file the mutation was applied to.
225    pub file: String,
226    /// Line number (1-indexed).
227    pub line: u32,
228    /// Description of the mutation (e.g. "replace `+` with `-`").
229    pub description: String,
230    /// Function the mutation lived in, when the underlying tool exposed it.
231    #[serde(default, skip_serializing_if = "Option::is_none")]
232    pub function: Option<String>,
233}
234
235// ---------------------------------------------------------------------------
236// FileBreakdown
237// ---------------------------------------------------------------------------
238
239/// Per-file mutation outcome counts and derived kill rate.
240#[derive(Debug, Clone, Serialize, Deserialize)]
241pub struct FileBreakdown {
242    /// Source file path.
243    pub file: String,
244    /// Mutants caught by the test suite (for this file).
245    pub killed: u64,
246    /// Mutants the test suite missed (for this file).
247    pub survived: u64,
248    /// Mutants that timed out (excluded from `kill_pct`).
249    pub timeout: u64,
250}
251
252impl FileBreakdown {
253    /// Kill rate for this file in `[0.0, 100.0]`. Timeouts excluded.
254    pub fn kill_pct(&self) -> f64 {
255        let counted = self.killed + self.survived;
256        if counted == 0 {
257            return 0.0;
258        }
259        (self.killed as f64 / counted as f64) * 100.0
260    }
261}
262
263// ---------------------------------------------------------------------------
264// MutateResult
265// ---------------------------------------------------------------------------
266
267/// Result of a mutation testing run.
268#[derive(Debug, Clone, Serialize, Deserialize)]
269pub struct MutateResult {
270    /// Subject name.
271    pub name: String,
272    /// Subject version.
273    pub version: String,
274    /// Total mutants generated.
275    pub mutants_total: u64,
276    /// Mutants caught by the test suite.
277    pub mutants_killed: u64,
278    /// Mutants the test suite missed.
279    pub mutants_survived: u64,
280    /// Mutants that timed out (excluded from `kill_pct`).
281    pub mutants_timeout: u64,
282    /// Per-survivor detail (sorted by `(file, line)` for determinism).
283    pub survivors: Vec<SurvivingMutant>,
284    /// Per-file breakdown (sorted by `file`).
285    #[serde(default, skip_serializing_if = "Vec::is_empty")]
286    pub files: Vec<FileBreakdown>,
287}
288
289impl MutateResult {
290    /// Kill rate as a percent in `[0.0, 100.0]`.
291    ///
292    /// `killed / (killed + survived) * 100`. Timeouts excluded.
293    pub fn kill_pct(&self) -> f64 {
294        let counted = self.mutants_killed + self.mutants_survived;
295        if counted == 0 {
296            return 0.0;
297        }
298        (self.mutants_killed as f64 / counted as f64) * 100.0
299    }
300
301    /// `true` when the kill rate meets or exceeds the given threshold.
302    ///
303    /// Exists so callers can avoid recomputing `kill_pct` and the
304    /// comparison logic locally.
305    pub fn meets(&self, threshold: MutateThreshold) -> bool {
306        match threshold {
307            MutateThreshold::MinKillPct(target) => self.kill_pct() >= target,
308        }
309    }
310
311    /// Files with the lowest kill rate, ascending. Useful for emitting
312    /// evidence about test-quality hotspots.
313    pub fn weakest_files(&self, n: usize) -> Vec<&FileBreakdown> {
314        let mut refs: Vec<&FileBreakdown> = self.files.iter().collect();
315        refs.sort_by(|a, b| {
316            a.kill_pct()
317                .partial_cmp(&b.kill_pct())
318                .unwrap_or(std::cmp::Ordering::Equal)
319        });
320        refs.into_iter().take(n).collect()
321    }
322
323    /// Convert this result into a [`CheckResult`] against `threshold`.
324    ///
325    /// Pass when `kill_pct >= threshold`, otherwise fail with
326    /// `Severity::Warning` (per REPS § 4 — mutation testing is
327    /// advisory by default). The verdict carries numeric evidence for
328    /// both the measured and target percentages plus the raw counts.
329    /// When survivors are present, the first one is attached as
330    /// `Evidence::FileRef` pointing at the affected `(file, line)`.
331    pub fn into_check_result(self, threshold: MutateThreshold) -> CheckResult {
332        let name = format!("mutate::{}", self.name);
333        let kill_pct = self.kill_pct();
334        let MutateThreshold::MinKillPct(target) = threshold;
335        let detail = format!(
336            "kill rate {:.2}% ({}/{}; {} timeouts; {} survivors)",
337            kill_pct,
338            self.mutants_killed,
339            self.mutants_killed + self.mutants_survived,
340            self.mutants_timeout,
341            self.mutants_survived
342        );
343        let mut check = if kill_pct < target {
344            CheckResult::fail(name, Severity::Warning).with_detail(detail)
345        } else {
346            CheckResult::pass(name).with_detail(detail)
347        };
348        check = check
349            .with_tag("mutate")
350            .with_evidence(Evidence::numeric("kill_pct", kill_pct))
351            .with_evidence(Evidence::numeric("kill_pct_threshold", target))
352            .with_evidence(Evidence::numeric_int(
353                "mutants_killed",
354                self.mutants_killed as i64,
355            ))
356            .with_evidence(Evidence::numeric_int(
357                "mutants_survived",
358                self.mutants_survived as i64,
359            ))
360            .with_evidence(Evidence::numeric_int(
361                "mutants_timeout",
362                self.mutants_timeout as i64,
363            ));
364        if let Some(first) = self.survivors.first() {
365            check = check.with_evidence(Evidence::file_ref_lines(
366                "first_survivor",
367                first.file.clone(),
368                first.line.max(1),
369                first.line.max(1),
370            ));
371        }
372        check
373    }
374}
375
376// ---------------------------------------------------------------------------
377// MutateThreshold
378// ---------------------------------------------------------------------------
379
380/// Threshold defining the minimum acceptable kill rate.
381#[derive(Debug, Clone, Copy)]
382pub enum MutateThreshold {
383    /// Fail when `kill_pct < pct`.
384    MinKillPct(f64),
385}
386
387impl MutateThreshold {
388    /// Build a kill-rate threshold.
389    pub fn min_kill_pct(pct: f64) -> Self {
390        Self::MinKillPct(pct)
391    }
392}
393
394// ---------------------------------------------------------------------------
395// MutateError
396// ---------------------------------------------------------------------------
397
398/// Errors that can arise during a mutation testing run.
399#[derive(Debug)]
400pub enum MutateError {
401    /// `cargo-mutants` is not installed.
402    ToolNotInstalled,
403    /// Subprocess returned a fatal error and no parseable JSON.
404    SubprocessFailed(String),
405    /// JSON output could not be parsed.
406    ParseError(String),
407}
408
409impl std::fmt::Display for MutateError {
410    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
411        match self {
412            Self::ToolNotInstalled => write!(
413                f,
414                "cargo-mutants is not installed; run `cargo install cargo-mutants`"
415            ),
416            Self::SubprocessFailed(s) => write!(f, "cargo mutants failed: {s}"),
417            Self::ParseError(s) => write!(f, "could not parse cargo mutants output: {s}"),
418        }
419    }
420}
421
422impl std::error::Error for MutateError {}
423
424// ---------------------------------------------------------------------------
425// Helpers for the runner
426// ---------------------------------------------------------------------------
427
428pub(crate) fn aggregate_breakdown(
429    by_file_killed: &BTreeMap<String, u64>,
430    by_file_survived: &BTreeMap<String, u64>,
431    by_file_timeout: &BTreeMap<String, u64>,
432) -> Vec<FileBreakdown> {
433    let mut all_files: BTreeMap<String, FileBreakdown> = BTreeMap::new();
434    for (file, count) in by_file_killed {
435        all_files
436            .entry(file.clone())
437            .or_insert(FileBreakdown {
438                file: file.clone(),
439                killed: 0,
440                survived: 0,
441                timeout: 0,
442            })
443            .killed = *count;
444    }
445    for (file, count) in by_file_survived {
446        all_files
447            .entry(file.clone())
448            .or_insert(FileBreakdown {
449                file: file.clone(),
450                killed: 0,
451                survived: 0,
452                timeout: 0,
453            })
454            .survived = *count;
455    }
456    for (file, count) in by_file_timeout {
457        all_files
458            .entry(file.clone())
459            .or_insert(FileBreakdown {
460                file: file.clone(),
461                killed: 0,
462                survived: 0,
463                timeout: 0,
464            })
465            .timeout = *count;
466    }
467    all_files.into_values().collect()
468}
469
470#[cfg(test)]
471mod tests {
472    use super::*;
473    use dev_report::Verdict;
474
475    fn sample(killed: u64, survived: u64, timeout: u64) -> MutateResult {
476        MutateResult {
477            name: "x".into(),
478            version: "0.1.0".into(),
479            mutants_total: killed + survived + timeout,
480            mutants_killed: killed,
481            mutants_survived: survived,
482            mutants_timeout: timeout,
483            survivors: Vec::new(),
484            files: Vec::new(),
485        }
486    }
487
488    #[test]
489    fn kill_pct_excludes_timeouts() {
490        // 80 / (80 + 20) = 80%, with 10 timeouts not counted.
491        let r = sample(80, 20, 10);
492        assert!((r.kill_pct() - 80.0).abs() < 1e-9);
493    }
494
495    #[test]
496    fn kill_pct_zero_when_nothing_counted() {
497        let r = sample(0, 0, 5);
498        assert_eq!(r.kill_pct(), 0.0);
499    }
500
501    #[test]
502    fn threshold_pass_when_above() {
503        let r = sample(85, 15, 0);
504        let c = r.into_check_result(MutateThreshold::min_kill_pct(80.0));
505        assert_eq!(c.verdict, Verdict::Pass);
506        assert!(c.has_tag("mutate"));
507        assert!(c.evidence.iter().any(|e| e.label == "kill_pct"));
508        assert!(c.evidence.iter().any(|e| e.label == "kill_pct_threshold"));
509    }
510
511    #[test]
512    fn threshold_fail_uses_warning_severity() {
513        let r = sample(50, 50, 0);
514        let c = r.into_check_result(MutateThreshold::min_kill_pct(80.0));
515        assert_eq!(c.verdict, Verdict::Fail);
516        assert_eq!(c.severity, Some(Severity::Warning));
517    }
518
519    #[test]
520    fn meets_helper_matches_into_check_result() {
521        let r = sample(85, 15, 0);
522        assert!(r.meets(MutateThreshold::min_kill_pct(80.0)));
523        assert!(!r.meets(MutateThreshold::min_kill_pct(95.0)));
524    }
525
526    #[test]
527    fn first_survivor_attached_as_evidence() {
528        let mut r = sample(80, 20, 0);
529        r.survivors = vec![SurvivingMutant {
530            file: "src/lib.rs".into(),
531            line: 42,
532            description: "replace + with -".into(),
533            function: Some("add".into()),
534        }];
535        let c = r.into_check_result(MutateThreshold::min_kill_pct(85.0));
536        // 80% < 85%, fails. Verify first_survivor evidence present.
537        assert!(c.evidence.iter().any(|e| e.label == "first_survivor"));
538    }
539
540    #[test]
541    fn weakest_files_sorts_ascending_by_kill_pct() {
542        let r = MutateResult {
543            name: "x".into(),
544            version: "0.1.0".into(),
545            mutants_total: 0,
546            mutants_killed: 0,
547            mutants_survived: 0,
548            mutants_timeout: 0,
549            survivors: Vec::new(),
550            files: vec![
551                FileBreakdown {
552                    file: "a.rs".into(),
553                    killed: 9,
554                    survived: 1,
555                    timeout: 0,
556                },
557                FileBreakdown {
558                    file: "b.rs".into(),
559                    killed: 5,
560                    survived: 5,
561                    timeout: 0,
562                },
563                FileBreakdown {
564                    file: "c.rs".into(),
565                    killed: 7,
566                    survived: 3,
567                    timeout: 0,
568                },
569            ],
570        };
571        let weakest = r.weakest_files(2);
572        assert_eq!(weakest.len(), 2);
573        assert_eq!(weakest[0].file, "b.rs"); // 50%
574        assert_eq!(weakest[1].file, "c.rs"); // 70%
575    }
576
577    #[test]
578    fn file_breakdown_kill_pct() {
579        let f = FileBreakdown {
580            file: "x".into(),
581            killed: 7,
582            survived: 3,
583            timeout: 0,
584        };
585        assert!((f.kill_pct() - 70.0).abs() < 1e-9);
586    }
587
588    #[test]
589    fn result_round_trips_through_json() {
590        let mut r = sample(80, 20, 0);
591        r.survivors.push(SurvivingMutant {
592            file: "src/lib.rs".into(),
593            line: 1,
594            description: "x".into(),
595            function: None,
596        });
597        let s = serde_json::to_string(&r).unwrap();
598        let back: MutateResult = serde_json::from_str(&s).unwrap();
599        assert_eq!(back.survivors.len(), 1);
600    }
601
602    #[test]
603    fn builder_chain_compiles() {
604        let r = MutateRun::new("x", "0.1.0")
605            .workspace()
606            .jobs(4)
607            .timeout(Duration::from_secs(120))
608            .exclude_re(r"^src/generated/")
609            .file("src/lib.rs")
610            .allow("replace + with -")
611            .allow_all(["a", "b"]);
612        assert_eq!(r.subject(), "x");
613        assert_eq!(r.subject_version(), "0.1.0");
614        assert!(r.workspace_flag());
615        assert_eq!(r.jobs_value(), Some(4));
616        assert_eq!(r.exclude_re_view().len(), 1);
617        assert_eq!(r.file_filters_view().len(), 1);
618        assert_eq!(r.allow_list_view().len(), 3);
619    }
620}
dev_mutate/lib.rs

dev_mutate/
lib.rs
